diff --git a/lbrynet/core/looping_call_manager.py b/lbrynet/core/looping_call_manager.py index 7dbc9e022..fa7b9a924 100644 --- a/lbrynet/core/looping_call_manager.py +++ b/lbrynet/core/looping_call_manager.py @@ -15,6 +15,6 @@ class LoopingCallManager(object): self.calls[name].stop() def shutdown(self): - for lcall in self.calls.itervalues(): + for lcall in self.calls.values(): if lcall.running: lcall.stop() diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index cd48e6538..0d3f6ebfb 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -178,7 +178,9 @@ class WalletIsLocked(RequiredCondition): @staticmethod def evaluate(component): - return component.check_locked() + d = component.check_locked() + d.addCallback(lambda r: not r) + return d class Daemon(AuthJSONRPCServer): @@ -245,7 +247,7 @@ class Daemon(AuthJSONRPCServer): def _stop_streams(self): """stop pending GetStream downloads""" - for sd_hash, stream in self.streams.iteritems(): + for sd_hash, stream in self.streams.items(): stream.cancel(reason="daemon shutdown") def _shutdown(self): @@ -360,10 +362,10 @@ class Daemon(AuthJSONRPCServer): defer.returnValue(result) @defer.inlineCallbacks - def _publish_stream(self, name, bid, claim_dict, file_path=None, certificate_id=None, + def _publish_stream(self, name, bid, claim_dict, file_path=None, certificate=None, claim_address=None, change_address=None): publisher = Publisher( - self.blob_manager, self.payment_rate_manager, self.storage, self.file_manager, self.wallet, certificate_id + self.blob_manager, self.payment_rate_manager, self.storage, self.file_manager, self.wallet, certificate ) parse_lbry_uri(name) if not file_path: @@ -1723,23 +1725,23 @@ class Daemon(AuthJSONRPCServer): if bid <= 0.0: raise ValueError("Bid value must be greater than 0.0") - for address in [claim_address, change_address]: - if address is not None: - # raises an error if the address is invalid - decode_address(address) + bid = int(bid * COIN) - yield self.wallet.update_balance() - if bid >= self.wallet.get_balance(): - balance = yield self.wallet.get_max_usable_balance_for_claim(name) - max_bid_amount = balance - MAX_UPDATE_FEE_ESTIMATE - if balance <= MAX_UPDATE_FEE_ESTIMATE: - raise InsufficientFundsError( - "Insufficient funds, please deposit additional LBC. Minimum additional LBC needed {}" - .format(MAX_UPDATE_FEE_ESTIMATE - balance)) - elif bid > max_bid_amount: - raise InsufficientFundsError( - "Please lower the bid value, the maximum amount you can specify for this claim is {}." - .format(max_bid_amount)) + available = yield self.wallet.default_account.get_balance() + if bid >= available: + # TODO: add check for existing claim balance + #balance = yield self.wallet.get_max_usable_balance_for_claim(name) + #max_bid_amount = balance - MAX_UPDATE_FEE_ESTIMATE + #if balance <= MAX_UPDATE_FEE_ESTIMATE: + raise InsufficientFundsError( + "Insufficient funds, please deposit additional LBC. Minimum additional LBC needed {}" + .format(round((bid - available)/COIN + 0.01, 2)) + ) + # .format(MAX_UPDATE_FEE_ESTIMATE - balance)) + #elif bid > max_bid_amount: + # raise InsufficientFundsError( + # "Please lower the bid value, the maximum amount you can specify for this claim is {}." + # .format(max_bid_amount)) metadata = metadata or {} if fee is not None: @@ -1777,7 +1779,7 @@ class Daemon(AuthJSONRPCServer): log.warning("Stripping empty fee from published metadata") del metadata['fee'] elif 'address' not in metadata['fee']: - address = yield self.wallet.get_least_used_address() + address = yield self.wallet.default_account.receiving.get_or_create_usable_address() metadata['fee']['address'] = address if 'fee' in metadata and 'version' not in metadata['fee']: metadata['fee']['version'] = '_0_0_1' @@ -1830,20 +1832,15 @@ class Daemon(AuthJSONRPCServer): }) if channel_id: - certificate_id = channel_id + certificate = self.wallet.default_account.get_certificate(by_claim_id=channel_id) elif channel_name: - certificate_id = None - my_certificates = yield self.wallet.channel_list() - for certificate in my_certificates: - if channel_name == certificate['name']: - certificate_id = certificate['claim_id'] - break - if not certificate_id: + certificate = self.wallet.default_account.get_certificate(by_name=channel_name) + if not certificate: raise Exception("Cannot publish using channel %s" % channel_name) else: - certificate_id = None + certificate = None - result = yield self._publish_stream(name, bid, claim_dict, file_path, certificate_id, + result = yield self._publish_stream(name, bid, claim_dict, file_path, certificate, claim_address, change_address) response = yield self._render_response(result) defer.returnValue(response) @@ -2756,7 +2753,7 @@ class Daemon(AuthJSONRPCServer): if sd_hash in self.blob_manager.blobs: blobs = [self.blob_manager.blobs[sd_hash]] + blobs else: - blobs = self.blob_manager.blobs.itervalues() + blobs = self.blob_manager.blobs.values() if needed: blobs = [blob for blob in blobs if not blob.get_is_verified()] diff --git a/lbrynet/daemon/Publisher.py b/lbrynet/daemon/Publisher.py index b64adebfe..69aa6011e 100644 --- a/lbrynet/daemon/Publisher.py +++ b/lbrynet/daemon/Publisher.py @@ -11,13 +11,13 @@ log = logging.getLogger(__name__) class Publisher(object): - def __init__(self, blob_manager, payment_rate_manager, storage, lbry_file_manager, wallet, certificate_id): + def __init__(self, blob_manager, payment_rate_manager, storage, lbry_file_manager, wallet, certificate): self.blob_manager = blob_manager self.payment_rate_manager = payment_rate_manager self.storage = storage self.lbry_file_manager = lbry_file_manager self.wallet = wallet - self.certificate_id = certificate_id + self.certificate = certificate self.lbry_file = None @defer.inlineCallbacks @@ -74,7 +74,7 @@ class Publisher(object): @defer.inlineCallbacks def make_claim(self, name, bid, claim_dict, claim_address=None, change_address=None): claim_out = yield self.wallet.claim_name(name, bid, claim_dict, - certificate_id=self.certificate_id, + certificate=self.certificate, claim_address=claim_address, change_address=change_address) defer.returnValue(claim_out) diff --git a/lbrynet/wallet/account.py b/lbrynet/wallet/account.py index 0cb1beffd..c56ceb604 100644 --- a/lbrynet/wallet/account.py +++ b/lbrynet/wallet/account.py @@ -1,7 +1,12 @@ +from binascii import hexlify +from twisted.internet import defer + +from torba.baseaccount import BaseAccount + from lbryschema.claim import ClaimDict from lbryschema.signer import SECP256k1, get_signer -from torba.baseaccount import BaseAccount +from .transaction import Transaction def generate_certificate(): @@ -9,19 +14,32 @@ def generate_certificate(): return ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1), secp256k1_private_key +def get_certificate_lookup(tx_or_hash, nout): + if isinstance(tx_or_hash, Transaction): + return '{}:{}'.format(tx_or_hash.hex_id.decode(), nout) + else: + return '{}:{}'.format(hexlify(tx_or_hash[::-1]).decode(), nout) + + class Account(BaseAccount): def __init__(self, *args, **kwargs): super(Account, self).__init__(*args, **kwargs) self.certificates = {} - def add_certificate(self, claim_id, key): - assert claim_id not in self.certificates, 'Trying to add a duplicate certificate.' - self.certificates[claim_id] = key + def add_certificate(self, tx, nout, private_key): + lookup_key = '{}:{}'.format(tx.hex_id.decode(), nout) + assert lookup_key not in self.certificates, 'Trying to add a duplicate certificate.' + self.certificates[lookup_key] = private_key - def get_certificate(self, claim_id): - return self.certificates[claim_id] + def get_certificate_private_key(self, tx_or_hash, nout): + return self.certificates.get(get_certificate_lookup(tx_or_hash, nout)) + @defer.inlineCallbacks + def maybe_migrate_certificates(self): + for lookup_key in self.certificates.keys(): + if ':' not in lookup_key: + claim = self.ledger. def get_balance(self, include_claims=False): if include_claims: return super(Account, self).get_balance() diff --git a/lbrynet/wallet/database.py b/lbrynet/wallet/database.py index c402c0c03..4c94e34c2 100644 --- a/lbrynet/wallet/database.py +++ b/lbrynet/wallet/database.py @@ -1,4 +1,7 @@ +import sqlite3 +from twisted.internet import defer from torba.basedatabase import BaseDatabase +from .certificate import Certificate class WalletDatabase(BaseDatabase): @@ -12,7 +15,8 @@ class WalletDatabase(BaseDatabase): amount integer not null, script blob not null, is_reserved boolean not null default 0, - + + claim_id blob, claim_name text, is_claim boolean not null default 0, is_update boolean not null default 0, @@ -37,3 +41,43 @@ class WalletDatabase(BaseDatabase): if txo.script.is_claim_involved: row['claim_name'] = txo.script.values['claim_name'] return row + + @defer.inlineCallbacks + def get_certificates(self, name, private_key_accounts=None, exclude_without_key=False): + txos = yield self.db.runQuery( + """ + SELECT tx.hash, txo.position, txo.claim_id + FROM txo JOIN tx ON tx.txhash=txo.txhash + WHERE claim_name=:claim AND (is_claim=1 OR is_update=1) + ORDER BY tx.height DESC + GROUP BY txo.claim_id + """, {'name': name} + ) + + certificates = [ + Certificate( + values[0], + values[1], + values[2], + name, + None + ) for values in txos + ] + + # Lookup private keys for each certificate. + if private_key_accounts is not None: + for cert in certificates: + for account in private_key_accounts: + private_key = account.get_certificate_private_key( + cert.txhash, cert.nout + ) + if private_key is not None: + cert.private_key = private_key + break + + if exclude_without_key: + defer.returnValue([ + c for c in certificates if c.private_key is not None + ]) + + defer.returnValue(certificates) diff --git a/lbrynet/wallet/ledger.py b/lbrynet/wallet/ledger.py index e45e87ce5..7110eec2f 100644 --- a/lbrynet/wallet/ledger.py +++ b/lbrynet/wallet/ledger.py @@ -2,6 +2,8 @@ import struct from six import int2byte from binascii import unhexlify +from twisted.internet import defer + from torba.baseledger import BaseLedger from torba.baseheader import BaseHeaders, _ArithUint256 from torba.util import int_to_hex, rev_hex, hash_encode @@ -132,6 +134,13 @@ class MainNetLedger(BaseLedger): self.headers.hash(), *uris ) + @defer.inlineCallbacks + def start(self): + yield super(MainNetLedger, self).start() + yield defer.DeferredList([ + a.maybe_migrate_certificates() for a in self.accounts + ]) + class TestNetLedger(MainNetLedger): network_name = 'testnet' diff --git a/lbrynet/wallet/manager.py b/lbrynet/wallet/manager.py index 50cac63df..793b448db 100644 --- a/lbrynet/wallet/manager.py +++ b/lbrynet/wallet/manager.py @@ -1,7 +1,9 @@ import os +import json from twisted.internet import defer from torba.manager import WalletManager as BaseWalletManager +from torba.wallet import WalletStorage from lbryschema.uri import parse_lbry_uri from lbryschema.error import URIParseError @@ -45,7 +47,7 @@ class LbryWalletManager(BaseWalletManager): return defer.succeed(False) @classmethod - def from_old_config(cls, settings): + def from_lbrynet_config(cls, settings, db): ledger_id = { 'lbrycrd_main': 'lbc_mainnet', @@ -57,12 +59,45 @@ class LbryWalletManager(BaseWalletManager): 'auto_connect': True, 'default_servers': settings['lbryum_servers'], 'data_path': settings['lbryum_wallet_dir'], - 'use_keyring': settings['use_keyring'] + 'use_keyring': settings['use_keyring'], + 'db': db } + wallet_file_path = os.path.join(settings['lbryum_wallet_dir'], 'default_wallet') + if os.path.exists(wallet_file_path): + with open(wallet_file_path, 'r') as f: + json_data = f.read() + json_dict = json.loads(json_data) + # TODO: After several public releases of new torba based wallet, we can delete + # this lbryum->torba conversion code and require that users who still + # have old structured wallets install one of the earlier releases that + # still has the below conversion code. + if 'master_public_keys' in json_dict: + json_data = json.dumps({ + 'version': 1, + 'name': 'My Wallet', + 'accounts': [{ + 'version': 1, + 'name': 'Main Account', + 'ledger': 'lbc_mainnet', + 'encrypted': json_dict['use_encryption'], + 'seed': json_dict['seed'], + 'seed_version': json_dict['seed_version'], + 'private_key': json_dict['master_private_keys']['x/'], + 'public_key': json_dict['master_public_keys']['x/'], + 'certificates': json_dict['claim_certificates'], + 'receiving_gap': 20, + 'change_gap': 6, + 'receiving_maximum_use_per_address': 2, + 'change_maximum_use_per_address': 2 + }] + }, indent=4, sort_keys=True) + with open(wallet_file_path, 'w') as f: + f.write(json_data) + return cls.from_config({ 'ledgers': {ledger_id: ledger_config}, - 'wallets': [os.path.join(settings['lbryum_wallet_dir'], 'default_wallet')] + 'wallets': [wallet_file_path] }) def get_best_blockhash(self): @@ -101,8 +136,19 @@ class LbryWalletManager(BaseWalletManager): def get_history(self): return defer.succeed([]) - def claim_name(self, name, amount, claim): - pass + @defer.inlineCallbacks + def claim_name(self, name, amount, claim, certificate=None, claim_address=None): + account = self.default_account + if not claim_address: + claim_address = yield account.receiving.get_or_create_usable_address() + if certificate: + claim = claim.sign( + certificate['private_key'], claim_address, certificate['claim_id'] + ) + tx = yield Transaction.claim(name.encode(), claim, amount, claim_address, [account], account) + yield account.ledger.broadcast(tx) + # TODO: release reserved tx outputs in case anything fails by this point + defer.returnValue(tx) @defer.inlineCallbacks def claim_new_channel(self, channel_name, amount): @@ -121,7 +167,7 @@ class LbryWalletManager(BaseWalletManager): cert, key = generate_certificate() tx = yield Transaction.claim(channel_name.encode(), cert, amount, address, [account], account) yield account.ledger.broadcast(tx) - account.add_certificate(tx.get_claim_id(0), key) + account.add_certificate(tx, 0, tx.get_claim_id(0), channel_name, key) # TODO: release reserved tx outputs in case anything fails by this point defer.returnValue(tx) diff --git a/tests/integration/wallet/test_commands.py b/tests/integration/wallet/test_commands.py index 13a1d629e..65f934603 100644 --- a/tests/integration/wallet/test_commands.py +++ b/tests/integration/wallet/test_commands.py @@ -10,13 +10,21 @@ lbryschema.BLOCKCHAIN_NAME = 'lbrycrd_regtest' from lbrynet import conf as lbry_conf from lbrynet.daemon.Daemon import Daemon from lbrynet.wallet.manager import LbryWalletManager -from lbrynet.daemon.Components import WalletComponent, FileManager +from lbrynet.daemon.Components import WalletComponent, FileManager, SessionComponent +from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager class FakeAnalytics: def send_new_channel(self): pass + def shutdown(self): + pass + + +class FakeSession: + storage = None + class CommandTestCase(IntegrationTestCase): @@ -48,11 +56,19 @@ class CommandTestCase(IntegrationTestCase): wallet_component.wallet = self.manager wallet_component._running = True self.daemon.component_manager.components.add(wallet_component) + session_component = SessionComponent(self.daemon.component_manager) + session_component.session = FakeSession() + session_component._running = True + self.daemon.component_manager.components.add(session_component) + file_manager = FileManager(self.daemon.component_manager) + file_manager.file_manager = EncryptedFileManager(session_component.session, True) + file_manager._running = True + self.daemon.component_manager.components.add(file_manager) class ChannelNewCommandTests(CommandTestCase): - VERBOSE = False + VERBOSE = True @defer.inlineCallbacks def test_new_channel(self):