wip

lbrynet/core/looping_call_manager.py

@@ -15,6 +15,6 @@ class LoopingCallManager(object):
         self.calls[name].stop()
 
     def shutdown(self):
-        for lcall in self.calls.itervalues():
+        for lcall in self.calls.values():
             if lcall.running:
                 lcall.stop()

lbrynet/daemon/Daemon.py

@@ -178,7 +178,9 @@ class WalletIsLocked(RequiredCondition):
 
     @staticmethod
     def evaluate(component):
-        return component.check_locked()
+        d = component.check_locked()
+        d.addCallback(lambda r: not r)
+        return d
 
 
 class Daemon(AuthJSONRPCServer):
@@ -245,7 +247,7 @@ class Daemon(AuthJSONRPCServer):
 
     def _stop_streams(self):
         """stop pending GetStream downloads"""
-        for sd_hash, stream in self.streams.iteritems():
+        for sd_hash, stream in self.streams.items():
             stream.cancel(reason="daemon shutdown")
 
     def _shutdown(self):
@@ -360,10 +362,10 @@ class Daemon(AuthJSONRPCServer):
             defer.returnValue(result)
 
     @defer.inlineCallbacks
-    def _publish_stream(self, name, bid, claim_dict, file_path=None, certificate_id=None,
+    def _publish_stream(self, name, bid, claim_dict, file_path=None, certificate=None,
                         claim_address=None, change_address=None):
         publisher = Publisher(
-            self.blob_manager, self.payment_rate_manager, self.storage, self.file_manager, self.wallet, certificate_id
+            self.blob_manager, self.payment_rate_manager, self.storage, self.file_manager, self.wallet, certificate
         )
         parse_lbry_uri(name)
         if not file_path:
@@ -1723,23 +1725,23 @@ class Daemon(AuthJSONRPCServer):
         if bid <= 0.0:
             raise ValueError("Bid value must be greater than 0.0")
 
-        for address in [claim_address, change_address]:
+        bid = int(bid * COIN)
-            if address is not None:
-                # raises an error if the address is invalid
-                decode_address(address)
 
-        yield self.wallet.update_balance()
+        available = yield self.wallet.default_account.get_balance()
-        if bid >= self.wallet.get_balance():
+        if bid >= available:
-            balance = yield self.wallet.get_max_usable_balance_for_claim(name)
+            # TODO: add check for existing claim balance
-            max_bid_amount = balance - MAX_UPDATE_FEE_ESTIMATE
+            #balance = yield self.wallet.get_max_usable_balance_for_claim(name)
-            if balance <= MAX_UPDATE_FEE_ESTIMATE:
+            #max_bid_amount = balance - MAX_UPDATE_FEE_ESTIMATE
-                raise InsufficientFundsError(
+            #if balance <= MAX_UPDATE_FEE_ESTIMATE:
-                    "Insufficient funds, please deposit additional LBC. Minimum additional LBC needed {}"
+            raise InsufficientFundsError(
-                        .format(MAX_UPDATE_FEE_ESTIMATE - balance))
+                "Insufficient funds, please deposit additional LBC. Minimum additional LBC needed {}"
-            elif bid > max_bid_amount:
+                .format(round((bid - available)/COIN + 0.01, 2))
-                raise InsufficientFundsError(
+            )
-                    "Please lower the bid value, the maximum amount you can specify for this claim is {}."
+            #       .format(MAX_UPDATE_FEE_ESTIMATE - balance))
-                        .format(max_bid_amount))
+            #elif bid > max_bid_amount:
+            #    raise InsufficientFundsError(
+            #        "Please lower the bid value, the maximum amount you can specify for this claim is {}."
+            #            .format(max_bid_amount))
 
         metadata = metadata or {}
         if fee is not None:
@@ -1777,7 +1779,7 @@ class Daemon(AuthJSONRPCServer):
                     log.warning("Stripping empty fee from published metadata")
                     del metadata['fee']
                 elif 'address' not in metadata['fee']:
-                    address = yield self.wallet.get_least_used_address()
+                    address = yield self.wallet.default_account.receiving.get_or_create_usable_address()
                     metadata['fee']['address'] = address
             if 'fee' in metadata and 'version' not in metadata['fee']:
                 metadata['fee']['version'] = '_0_0_1'
@@ -1830,20 +1832,15 @@ class Daemon(AuthJSONRPCServer):
         })
 
         if channel_id:
-            certificate_id = channel_id
+            certificate = self.wallet.default_account.get_certificate(by_claim_id=channel_id)
         elif channel_name:
-            certificate_id = None
+            certificate = self.wallet.default_account.get_certificate(by_name=channel_name)
-            my_certificates = yield self.wallet.channel_list()
+            if not certificate:
-            for certificate in my_certificates:
-                if channel_name == certificate['name']:
-                    certificate_id = certificate['claim_id']
-                    break
-            if not certificate_id:
                 raise Exception("Cannot publish using channel %s" % channel_name)
         else:
-            certificate_id = None
+            certificate = None
 
-        result = yield self._publish_stream(name, bid, claim_dict, file_path, certificate_id,
+        result = yield self._publish_stream(name, bid, claim_dict, file_path, certificate,
                                             claim_address, change_address)
         response = yield self._render_response(result)
         defer.returnValue(response)
@@ -2756,7 +2753,7 @@ class Daemon(AuthJSONRPCServer):
             if sd_hash in self.blob_manager.blobs:
                 blobs = [self.blob_manager.blobs[sd_hash]] + blobs
         else:
-            blobs = self.blob_manager.blobs.itervalues()
+            blobs = self.blob_manager.blobs.values()
 
         if needed:
             blobs = [blob for blob in blobs if not blob.get_is_verified()]

lbrynet/daemon/Publisher.py

@@ -11,13 +11,13 @@ log = logging.getLogger(__name__)
 
 
 class Publisher(object):
-    def __init__(self, blob_manager, payment_rate_manager, storage, lbry_file_manager, wallet, certificate_id):
+    def __init__(self, blob_manager, payment_rate_manager, storage, lbry_file_manager, wallet, certificate):
         self.blob_manager = blob_manager
         self.payment_rate_manager = payment_rate_manager
         self.storage = storage
         self.lbry_file_manager = lbry_file_manager
         self.wallet = wallet
-        self.certificate_id = certificate_id
+        self.certificate = certificate
         self.lbry_file = None
 
     @defer.inlineCallbacks
@@ -74,7 +74,7 @@ class Publisher(object):
     @defer.inlineCallbacks
     def make_claim(self, name, bid, claim_dict, claim_address=None, change_address=None):
         claim_out = yield self.wallet.claim_name(name, bid, claim_dict,
-                                                 certificate_id=self.certificate_id,
+                                                 certificate=self.certificate,
                                                  claim_address=claim_address,
                                                  change_address=change_address)
         defer.returnValue(claim_out)

lbrynet/wallet/account.py

@@ -1,7 +1,12 @@
+from binascii import hexlify
+from twisted.internet import defer
+
+from torba.baseaccount import BaseAccount
+
 from lbryschema.claim import ClaimDict
 from lbryschema.signer import SECP256k1, get_signer
 
-from torba.baseaccount import BaseAccount
+from .transaction import Transaction
 
 
 def generate_certificate():
@@ -9,19 +14,32 @@ def generate_certificate():
     return ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1), secp256k1_private_key
 
 
+def get_certificate_lookup(tx_or_hash, nout):
+    if isinstance(tx_or_hash, Transaction):
+        return '{}:{}'.format(tx_or_hash.hex_id.decode(), nout)
+    else:
+        return '{}:{}'.format(hexlify(tx_or_hash[::-1]).decode(), nout)
+
+
 class Account(BaseAccount):
 
     def __init__(self, *args, **kwargs):
         super(Account, self).__init__(*args, **kwargs)
         self.certificates = {}
 
-    def add_certificate(self, claim_id, key):
+    def add_certificate(self, tx, nout, private_key):
-        assert claim_id not in self.certificates, 'Trying to add a duplicate certificate.'
+        lookup_key = '{}:{}'.format(tx.hex_id.decode(), nout)
-        self.certificates[claim_id] = key
+        assert lookup_key not in self.certificates, 'Trying to add a duplicate certificate.'
+        self.certificates[lookup_key] = private_key
 
-    def get_certificate(self, claim_id):
+    def get_certificate_private_key(self, tx_or_hash, nout):
-        return self.certificates[claim_id]
+        return self.certificates.get(get_certificate_lookup(tx_or_hash, nout))
 
+    @defer.inlineCallbacks
+    def maybe_migrate_certificates(self):
+        for lookup_key in self.certificates.keys():
+            if ':' not in lookup_key:
+                claim = self.ledger.
     def get_balance(self, include_claims=False):
         if include_claims:
             return super(Account, self).get_balance()

lbrynet/wallet/database.py

@@ -1,4 +1,7 @@
+import sqlite3
+from twisted.internet import defer
 from torba.basedatabase import BaseDatabase
+from .certificate import Certificate
 
 
 class WalletDatabase(BaseDatabase):
@@ -12,7 +15,8 @@ class WalletDatabase(BaseDatabase):
             amount integer not null,
             script blob not null,
             is_reserved boolean not null default 0,
-            
+
+            claim_id blob,
             claim_name text,
             is_claim boolean not null default 0,
             is_update boolean not null default 0,
@@ -37,3 +41,43 @@ class WalletDatabase(BaseDatabase):
         if txo.script.is_claim_involved:
             row['claim_name'] = txo.script.values['claim_name']
         return row
+
+    @defer.inlineCallbacks
+    def get_certificates(self, name, private_key_accounts=None, exclude_without_key=False):
+        txos = yield self.db.runQuery(
+            """
+            SELECT tx.hash, txo.position, txo.claim_id
+            FROM txo JOIN tx ON tx.txhash=txo.txhash
+            WHERE claim_name=:claim AND (is_claim=1 OR is_update=1)
+            ORDER BY tx.height DESC
+            GROUP BY txo.claim_id
+            """, {'name': name}
+        )
+
+        certificates = [
+            Certificate(
+                values[0],
+                values[1],
+                values[2],
+                name,
+                None
+            ) for values in txos
+        ]
+
+        # Lookup private keys for each certificate.
+        if private_key_accounts is not None:
+            for cert in certificates:
+                for account in private_key_accounts:
+                    private_key = account.get_certificate_private_key(
+                        cert.txhash, cert.nout
+                    )
+                    if private_key is not None:
+                        cert.private_key = private_key
+                        break
+
+        if exclude_without_key:
+            defer.returnValue([
+                c for c in certificates if c.private_key is not None
+            ])
+
+        defer.returnValue(certificates)

lbrynet/wallet/ledger.py

@@ -2,6 +2,8 @@ import struct
 from six import int2byte
 from binascii import unhexlify
 
+from twisted.internet import defer
+
 from torba.baseledger import BaseLedger
 from torba.baseheader import BaseHeaders, _ArithUint256
 from torba.util import int_to_hex, rev_hex, hash_encode
@@ -132,6 +134,13 @@ class MainNetLedger(BaseLedger):
             self.headers.hash(), *uris
         )
 
+    @defer.inlineCallbacks
+    def start(self):
+        yield super(MainNetLedger, self).start()
+        yield defer.DeferredList([
+            a.maybe_migrate_certificates() for a in self.accounts
+        ])
+
 
 class TestNetLedger(MainNetLedger):
     network_name = 'testnet'

lbrynet/wallet/manager.py

@@ -1,7 +1,9 @@
 import os
+import json
 from twisted.internet import defer
 
 from torba.manager import WalletManager as BaseWalletManager
+from torba.wallet import WalletStorage
 
 from lbryschema.uri import parse_lbry_uri
 from lbryschema.error import URIParseError
@@ -45,7 +47,7 @@ class LbryWalletManager(BaseWalletManager):
         return defer.succeed(False)
 
     @classmethod
-    def from_old_config(cls, settings):
+    def from_lbrynet_config(cls, settings, db):
 
         ledger_id = {
             'lbrycrd_main':    'lbc_mainnet',
@@ -57,12 +59,45 @@ class LbryWalletManager(BaseWalletManager):
             'auto_connect': True,
             'default_servers': settings['lbryum_servers'],
             'data_path': settings['lbryum_wallet_dir'],
-            'use_keyring': settings['use_keyring']
+            'use_keyring': settings['use_keyring'],
+            'db': db
         }
 
+        wallet_file_path = os.path.join(settings['lbryum_wallet_dir'], 'default_wallet')
+        if os.path.exists(wallet_file_path):
+            with open(wallet_file_path, 'r') as f:
+                json_data = f.read()
+                json_dict = json.loads(json_data)
+            # TODO: After several public releases of new torba based wallet, we can delete
+            #       this lbryum->torba conversion code and require that users who still
+            #       have old structured wallets install one of the earlier releases that
+            #       still has the below conversion code.
+            if 'master_public_keys' in json_dict:
+                json_data = json.dumps({
+                    'version': 1,
+                    'name': 'My Wallet',
+                    'accounts': [{
+                        'version': 1,
+                        'name': 'Main Account',
+                        'ledger': 'lbc_mainnet',
+                        'encrypted': json_dict['use_encryption'],
+                        'seed': json_dict['seed'],
+                        'seed_version': json_dict['seed_version'],
+                        'private_key': json_dict['master_private_keys']['x/'],
+                        'public_key': json_dict['master_public_keys']['x/'],
+                        'certificates': json_dict['claim_certificates'],
+                        'receiving_gap': 20,
+                        'change_gap': 6,
+                        'receiving_maximum_use_per_address': 2,
+                        'change_maximum_use_per_address': 2
+                    }]
+                }, indent=4, sort_keys=True)
+                with open(wallet_file_path, 'w') as f:
+                    f.write(json_data)
+
         return cls.from_config({
             'ledgers': {ledger_id: ledger_config},
-            'wallets': [os.path.join(settings['lbryum_wallet_dir'], 'default_wallet')]
+            'wallets': [wallet_file_path]
         })
 
     def get_best_blockhash(self):
@@ -101,8 +136,19 @@ class LbryWalletManager(BaseWalletManager):
     def get_history(self):
         return defer.succeed([])
 
-    def claim_name(self, name, amount, claim):
+    @defer.inlineCallbacks
-        pass
+    def claim_name(self, name, amount, claim, certificate=None, claim_address=None):
+        account = self.default_account
+        if not claim_address:
+            claim_address = yield account.receiving.get_or_create_usable_address()
+        if certificate:
+            claim = claim.sign(
+                certificate['private_key'], claim_address, certificate['claim_id']
+            )
+        tx = yield Transaction.claim(name.encode(), claim, amount, claim_address, [account], account)
+        yield account.ledger.broadcast(tx)
+        # TODO: release reserved tx outputs in case anything fails by this point
+        defer.returnValue(tx)
 
     @defer.inlineCallbacks
     def claim_new_channel(self, channel_name, amount):
@@ -121,7 +167,7 @@ class LbryWalletManager(BaseWalletManager):
         cert, key = generate_certificate()
         tx = yield Transaction.claim(channel_name.encode(), cert, amount, address, [account], account)
         yield account.ledger.broadcast(tx)
-        account.add_certificate(tx.get_claim_id(0), key)
+        account.add_certificate(tx, 0, tx.get_claim_id(0), channel_name, key)
         # TODO: release reserved tx outputs in case anything fails by this point
         defer.returnValue(tx)
 

tests/integration/wallet/test_commands.py

@@ -10,13 +10,21 @@ lbryschema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
 from lbrynet import conf as lbry_conf
 from lbrynet.daemon.Daemon import Daemon
 from lbrynet.wallet.manager import LbryWalletManager
-from lbrynet.daemon.Components import WalletComponent, FileManager
+from lbrynet.daemon.Components import WalletComponent, FileManager, SessionComponent
+from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
 
 
 class FakeAnalytics:
     def send_new_channel(self):
         pass
 
+    def shutdown(self):
+        pass
+
+
+class FakeSession:
+    storage = None
+
 
 class CommandTestCase(IntegrationTestCase):
 
@@ -48,11 +56,19 @@ class CommandTestCase(IntegrationTestCase):
         wallet_component.wallet = self.manager
         wallet_component._running = True
         self.daemon.component_manager.components.add(wallet_component)
+        session_component = SessionComponent(self.daemon.component_manager)
+        session_component.session = FakeSession()
+        session_component._running = True
+        self.daemon.component_manager.components.add(session_component)
+        file_manager = FileManager(self.daemon.component_manager)
+        file_manager.file_manager = EncryptedFileManager(session_component.session, True)
+        file_manager._running = True
+        self.daemon.component_manager.components.add(file_manager)
 
 
 class ChannelNewCommandTests(CommandTestCase):
 
-    VERBOSE = False
+    VERBOSE = True
 
     @defer.inlineCallbacks
     def test_new_channel(self):