txo.claim_hash and other fixes related to proto3 refactor

lbrynet/extras/daemon/json_response_encoder.py

@@ -72,12 +72,13 @@ class JSONResponseEncoder(JSONEncoder):
             if txo.script.is_claim_name or txo.script.is_update_claim:
                 claim = txo.claim
                 output['value'] = claim.to_dict()
+                if claim.is_stream:
+                    output['value']['stream']['hash'] = claim.stream.hash
                 if claim.is_signed:
                     output['valid_signature'] = None
                     if txo.channel is not None:
                         output['channel_name'] = txo.channel.claim_name
                         try:
-                            raise ValueError()
                             output['valid_signature'] = txo.is_signed_by(txo.channel, self.ledger)
                         except BadSignatureError:
                             output['valid_signature'] = False

lbrynet/schema/base.py

@@ -1,4 +1,4 @@
-from binascii import hexlify
+from binascii import hexlify, unhexlify
 
 from google.protobuf.message import DecodeError
 from google.protobuf.json_format import MessageToDict
@@ -8,7 +8,7 @@ class Signable:
 
     __slots__ = (
         'message', 'version', 'signature',
-        'signature_type', 'unsigned_payload', 'signing_channel_id'
+        'signature_type', 'unsigned_payload', 'signing_channel_hash'
     )
 
     message_class = None
@@ -19,15 +19,19 @@ class Signable:
         self.signature = None
         self.signature_type = 'SECP256k1'
         self.unsigned_payload = None
-        self.signing_channel_id = None
+        self.signing_channel_hash = None
 
     @property
     def is_undetermined(self):
         return self.message.WhichOneof('type') is None
 
     @property
-    def signing_channel_hash(self):
+    def signing_channel_id(self):
-        return hexlify(self.signing_channel_id[::-1]).decode() if self.signing_channel_id else None
+        return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None
+
+    @signing_channel_id.setter
+    def signing_channel_id(self, channel_id: str):
+        self.signing_channel_hash = unhexlify(channel_id)[::-1]
 
     @property
     def is_signed(self):
@@ -43,7 +47,7 @@ class Signable:
         pieces = bytearray()
         if self.is_signed:
             pieces.append(1)
-            pieces.extend(self.signing_channel_id)
+            pieces.extend(self.signing_channel_hash)
             pieces.extend(self.signature)
         else:
             pieces.append(0)
@@ -56,7 +60,7 @@ class Signable:
         if data[0] == 0:
             signable.message.ParseFromString(data[1:])
         elif data[0] == 1:
-            signable.signing_channel_id = data[1:21]
+            signable.signing_channel_hash = data[1:21]
             signable.signature = data[21:85]
             signable.message.ParseFromString(data[85:])
         else:

lbrynet/schema/claim.py

@@ -2,6 +2,7 @@ from typing import List, Tuple
 from decimal import Decimal
 from binascii import hexlify, unhexlify
 
+from google.protobuf.json_format import MessageToDict
 from google.protobuf.message import DecodeError
 
 from torba.client.hash import Base58
@@ -215,6 +216,9 @@ class Channel:
         self._claim = claim or Claim()
         self._channel = self._claim.channel_message
 
+    def to_dict(self):
+        return MessageToDict(self._channel)
+
     @property
     def claim(self) -> Claim:
         return self._claim
@@ -304,6 +308,9 @@ class Stream:
         self._claim = claim or Claim()
         self._stream = self._claim.stream_message
 
+    def to_dict(self):
+        return MessageToDict(self._stream)
+
     @property
     def claim(self) -> Claim:
         return self._claim

lbrynet/schema/compat.py

@@ -64,7 +64,7 @@ def from_types_v1(claim, payload: bytes):
             sig = old.publisherSignature
             claim.signature = sig.signature
             claim.signature_type = KeyType.Name(sig.signatureType)
-            claim.signing_channel_id = sig.certificateId
+            claim.signing_channel_hash = sig.certificateId
             old.ClearField("publisherSignature")
             claim.unsigned_payload = old.SerializeToString()
     elif old.claimType == 2:

lbrynet/stream/managed_stream.py

@@ -96,7 +96,7 @@ class ManagedStream:
 
     @property
     def metadata(self) ->typing.Optional[typing.Dict]:
-        return None if not self.stream_claim_info else self.stream_claim_info.claim.claim_dict['stream']['metadata']
+        return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
 
     @property
     def blobs_completed(self) -> int:

lbrynet/wallet/database.py

@@ -1,5 +1,3 @@
-from binascii import hexlify
-
 from torba.client.basedatabase import BaseDatabase
 
 
@@ -59,7 +57,7 @@ class WalletDatabase(BaseDatabase):
         for txo in txos:
             if txo.script.is_claim_name or txo.script.is_update_claim:
                 if txo.claim.is_signed:
-                    channel_ids.add(hexlify(txo.claim.signing_channel_id[::-1]).decode())
+                    channel_ids.add(txo.claim.signing_channel_id)
                 if txo.claim_name.startswith('@') and my_account is not None:
                     txo.private_key = my_account.get_certificate_private_key(txo.ref)
 
@@ -73,7 +71,7 @@ class WalletDatabase(BaseDatabase):
             }
             for txo in txos:
                 if txo.script.is_claim_name or txo.script.is_update_claim:
-                    txo.channel = channels.get(txo.claim.signing_channel_hash, None)
+                    txo.channel = channels.get(txo.claim.signing_channel_id, None)
 
         return txos
 

lbrynet/wallet/manager.py

@@ -416,7 +416,6 @@ class LbryWalletManager(BaseWalletManager):
 
         if certificate:
             claim_output.sign(certificate, first_input_id=b'placeholder')
-            claim_output.script.generate()
 
         tx = await Transaction.create(inputs, [claim_output], [account], account)
 

lbrynet/wallet/resolve.py

@@ -156,7 +156,6 @@ class Resolver:
         certificate = None
         certificate_id = Claim.from_bytes(unhexlify(claim_result['value'])).signing_channel_id
         if certificate_id:
-            certificate_id = hexlify(certificate_id[::-1]).decode()
             certificate = await self.network.get_claims_by_ids(certificate_id)
             certificate = certificate.pop(certificate_id) if certificate else None
         return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)

lbrynet/wallet/server/block_processor.py

@@ -149,7 +149,7 @@ class LBRYBlockProcessor(BlockProcessor):
         try:
             parse_lbry_uri(name.decode())  # skip invalid names
             claim_dict = Claim.from_bytes(value)
-            cert_id = claim_dict.signing_channel_id
+            cert_id = claim_dict.signing_channel_hash
             if not self.should_validate_signatures:
                 return cert_id
             if cert_id:

lbrynet/wallet/transaction.py

@@ -51,14 +51,17 @@ class Output(BaseOutput):
         return self.script.is_claim_name or self.script.is_update_claim
 
     @property
-    def claim_id(self) -> str:
+    def claim_hash(self) -> bytes:
         if self.script.is_claim_name:
-            claim_id = hash160(self.tx_ref.hash + struct.pack('>I', self.position))
+            return hash160(self.tx_ref.hash + struct.pack('>I', self.position))
         elif self.script.is_update_claim or self.script.is_support_claim:
-            claim_id = self.script.values['claim_id']
+            return self.script.values['claim_id']
         else:
             raise ValueError('No claim_id associated.')
-        return hexlify(claim_id[::-1]).decode()
+
+    @property
+    def claim_id(self) -> str:
+        return hexlify(self.claim_hash[::-1]).decode()
 
     @property
     def claim_name(self) -> str:
@@ -95,12 +98,12 @@ class Output(BaseOutput):
             pieces = [
                 Base58.decode(self.get_address(ledger)),
                 self.claim.unsigned_payload,
-                self.claim.signing_channel_id
+                self.claim.signing_channel_hash
             ]
         else:
             pieces = [
                 self.tx_ref.tx.inputs[0].txo_ref.id.encode(),
-                self.claim.signing_channel_id,
+                self.claim.signing_channel_hash,
                 self.claim.to_message_bytes()
             ]
         digest = sha256(b''.join(pieces))
@@ -114,10 +117,10 @@ class Output(BaseOutput):
         return True
 
     def sign(self, channel: 'Output', first_input_id=None):
-        self.claim.signing_channel_id = unhexlify(channel.claim_id)[::-1]
+        self.claim.signing_channel_hash = channel.claim_hash
         digest = sha256(b''.join([
             first_input_id or self.tx_ref.tx.inputs[0].txo_ref.id.encode(),
-            self.claim.signing_channel_id,
+            self.claim.signing_channel_hash,
             self.claim.to_message_bytes()
         ]))
         private_key = ecdsa.SigningKey.from_pem(channel.private_key, hashfunc=hashlib.sha256)

tests/integration/test_claim_commands.py

@@ -142,11 +142,11 @@ class ClaimCommands(CommandTestCase):
 
         # update the same claim
         await self.make_claim(amount='9.0')
-        await self.assertBalance(self.account, '0.9796205')
+        await self.assertBalance(self.account, '0.979637')
 
         # update the claim a second time but use even more funds
         await self.make_claim(amount='9.97')
-        await self.assertBalance(self.account, '0.009348')
+        await self.assertBalance(self.account, '0.009381')
 
         # fails when specifying more than available
         with tempfile.NamedTemporaryFile() as file:
@@ -155,7 +155,7 @@ class ClaimCommands(CommandTestCase):
             with self.assertRaisesRegex(
                 InsufficientFundsError,
                 "Please lower the bid value, the maximum amount"
-                " you can specify for this claim is 9.979274."
+                " you can specify for this claim is 9.979307."
             ):
                 await self.out(self.daemon.jsonrpc_publish(
                     'hovercraft', '9.98', file_path=file.name

tests/integration/test_file_commands.py

@@ -29,7 +29,7 @@ class FileCommands(CommandTestCase):
 
     async def test_download_different_timeouts(self):
         claim = await self.make_claim('foo', '0.01')
-        sd_hash = claim['output']['value']['stream']['source']['source']
+        sd_hash = claim['output']['value']['stream']['hash']
         await self.daemon.jsonrpc_file_delete(claim_name='foo')
         all_except_sd = [
             blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
@@ -71,7 +71,7 @@ class FileCommands(CommandTestCase):
 
     async def test_incomplete_downloads_erases_output_file_on_stop(self):
         claim = await self.make_claim('foo', '0.01')
-        sd_hash = claim['output']['value']['stream']['source']['source']
+        sd_hash = claim['output']['value']['stream']['hash']
         file_info = self.daemon.jsonrpc_file_list()[0]
         await self.daemon.jsonrpc_file_delete(claim_name='foo')
         blobs = await self.server_storage.get_blobs_for_stream(
@@ -90,7 +90,7 @@ class FileCommands(CommandTestCase):
 
     async def test_incomplete_downloads_retry(self):
         claim = await self.make_claim('foo', '0.01')
-        sd_hash = claim['output']['value']['stream']['source']['source']
+        sd_hash = claim['output']['value']['stream']['hash']
         await self.daemon.jsonrpc_file_delete(claim_name='foo')
         blobs = await self.server_storage.get_blobs_for_stream(
             await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
@@ -130,7 +130,7 @@ class FileCommands(CommandTestCase):
     async def test_unban_recovers_stream(self):
         BlobDownloader.BAN_TIME = .5  # fixme: temporary field, will move to connection manager or a conf
         claim = await self.make_claim('foo', '0.01', data=bytes([0]*(1<<23)))
-        sd_hash = claim['output']['value']['stream']['source']['source']
+        sd_hash = claim['output']['value']['stream']['hash']
         missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))[-2]
         await self.daemon.jsonrpc_file_delete(claim_name='foo')
         # backup blob
@@ -156,7 +156,7 @@ class FileCommands(CommandTestCase):
             fee={'currency': 'LBC', 'amount': 11.0, 'address': target_address})
         await self.daemon.jsonrpc_file_delete(claim_name='expensive')
         response = await self.daemon.jsonrpc_get('lbry://expensive')
-        self.assertEqual(response['error'], 'fee of 11.0 exceeds max available balance')
+        self.assertEqual(response['error'], 'fee of 11.00000 exceeds max available balance')
         self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
 
         # FAIL: beyond maximum key fee
@@ -166,7 +166,7 @@ class FileCommands(CommandTestCase):
         await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
         response = await self.daemon.jsonrpc_get('lbry://maxkey')
         self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
-        self.assertEqual(response['error'], 'fee of 111.0 exceeds max configured to allow of 50.0')
+        self.assertEqual(response['error'], 'fee of 111.00000 exceeds max configured to allow of 50.00000')
 
         # PASS: purchase is successful
         await self.make_claim(

tests/integration/test_internal_transaction_api.py

@@ -1,52 +1,16 @@
-import binascii
-import logging
 import asyncio
 
-from lbrynet.schema.address import decode_address
-from lbrynet.schema.schema import SECP256k1
-from lbrynet.schema.signature import Signature, NAMED_SECP256K1
-from lbrynet.schema.signer import get_signer
 from torba.testcase import IntegrationTestCase
-from lbrynet.schema.claim import ClaimDict
+
 import lbrynet.extras.wallet
-from lbrynet.extras.wallet.transaction import Transaction
+from lbrynet.schema.claim import Claim
-from lbrynet.extras.wallet.account import generate_certificate
+from lbrynet.wallet.transaction import Transaction, Output
-from lbrynet.extras.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d
+from lbrynet.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d
-
-import lbrynet.schema
-lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
-
-
-example_claim_dict = {
-    "version": "_0_0_1",
-    "claimType": "streamType",
-    "stream": {
-        "source": {
-            "source": "d5169241150022f996fa7cd6a9a1c421937276a3275eb912790bd07ba7aec1fac5fd45431d226b8fb402691e79aeb24b",
-            "version": "_0_0_1",
-            "contentType": "video/mp4",
-            "sourceType": "lbry_sd_hash"
-        },
-        "version": "_0_0_1",
-        "metadata": {
-            "license": "LBRY Inc",
-            "description": "What is LBRY? An introduction with Alex Tabarrok",
-            "language": "en",
-            "title": "What is LBRY?",
-            "author": "Samuel Bryan",
-            "version": "_0_1_0",
-            "nsfw": False,
-            "licenseUrl": "",
-            "preview": "",
-            "thumbnail": "https://s3.amazonaws.com/files.lbry.io/logo.png"
-        }
-    }
-}
 
 
 class BasicTransactionTest(IntegrationTestCase):
 
-    LEDGER = lbrynet.extras.wallet
+    LEDGER = lbrynet.wallet
 
     async def test_creating_updating_and_abandoning_claim_with_channel(self):
 
@@ -63,32 +27,43 @@ class BasicTransactionTest(IntegrationTestCase):
 
         self.assertEqual(d2l(await self.account.get_balance()), '10.0')
 
-        cert, key = generate_certificate()
+        channel = Claim()
-        cert_tx = await Transaction.claim('@bar', cert, l2d('1.0'), address1, [self.account], self.account)
+        channel_txo = Output.pay_claim_name_pubkey_hash(
-        claim = ClaimDict.load_dict(example_claim_dict)
+            l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
-        claim = claim.sign(key, address1, cert_tx.outputs[0].claim_id, name='foo')
+        )
-        claim_tx = await Transaction.claim('foo', claim, l2d('1.0'), address1, [self.account], self.account)
+        channel_txo.generate_channel_private_key()
+        channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
 
-        await self.broadcast(cert_tx)
+        stream = Claim()
-        await self.broadcast(claim_tx)
+        stream.stream.media_type = "video/mp4"
+        stream_txo = Output.pay_claim_name_pubkey_hash(
+            l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1)
+        )
+        stream_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
+        stream_tx._reset()
+        stream_txo.sign(channel_txo, b'placeholder')
+        await stream_tx.sign([self.account])
+
+        await self.broadcast(channel_tx)
+        await self.broadcast(stream_tx)
         await asyncio.wait([  # mempool
-            self.ledger.wait(claim_tx),
+            self.ledger.wait(channel_tx),
-            self.ledger.wait(cert_tx)
+            self.ledger.wait(stream_tx)
         ])
         await self.blockchain.generate(1)
         await asyncio.wait([  # confirmed
-            self.ledger.wait(claim_tx),
+            self.ledger.wait(channel_tx),
-            self.ledger.wait(cert_tx)
+            self.ledger.wait(stream_tx)
         ])
 
-        self.assertEqual(d2l(await self.account.get_balance()), '7.985786')
+        self.assertEqual(d2l(await self.account.get_balance()), '7.983786')
-        self.assertEqual(d2l(await self.account.get_balance(include_claims=True)), '9.985786')
+        self.assertEqual(d2l(await self.account.get_balance(include_claims=True)), '9.983786')
 
         response = await self.ledger.resolve(0, 10, 'lbry://@bar/foo')
         self.assertIn('lbry://@bar/foo', response)
         self.assertIn('claim', response['lbry://@bar/foo'])
 
-        abandon_tx = await Transaction.abandon([claim_tx.outputs[0]], [self.account], self.account)
+        abandon_tx = await Transaction.abandon([stream_tx.outputs[0]], [self.account], self.account)
         await self.broadcast(abandon_tx)
         await self.ledger.wait(abandon_tx)
         await self.blockchain.generate(1)
@@ -101,73 +76,3 @@ class BasicTransactionTest(IntegrationTestCase):
         response = await self.ledger.resolve(0, 10, 'lbry://404', 'lbry://@404')
         self.assertEqual('URI lbry://404 cannot be resolved', response['lbry://404']['error'])
         self.assertEqual('URI lbry://@404 cannot be resolved', response['lbry://@404']['error'])
-
-    async def test_new_signature_model(self):
-        address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
-        sendtxid1 = await self.blockchain.send_to_address(address1, 5)
-        sendtxid2 = await self.blockchain.send_to_address(address2, 5)
-        await self.blockchain.generate(1)
-        await asyncio.wait([
-            self.on_transaction_id(sendtxid1),
-            self.on_transaction_id(sendtxid2)
-        ])
-
-        self.assertEqual(d2l(await self.account.get_balance()), '10.0')
-
-        cert, key = generate_certificate()
-        cert_tx = await Transaction.claim('@bar', cert, l2d('1.0'), address1, [self.account], self.account)
-        claim = ClaimDict.load_dict(example_claim_dict)
-        claim = claim.sign(key, address1, cert_tx.outputs[0].claim_id, name='foo', curve=SECP256k1, force_detached=True)
-        claim_tx = await Transaction.claim('foo', claim, l2d('1.0'), address1, [self.account], self.account)
-
-        await self.broadcast(cert_tx)
-        await self.broadcast(claim_tx)
-        await self.ledger.wait(claim_tx)
-        await self.blockchain.generate(1)
-        await self.ledger.wait(claim_tx)
-
-        response = await self.ledger.resolve(0, 10, 'lbry://@bar/foo')
-        self.assertIn('lbry://@bar/foo', response)
-        self.assertIn('claim', response['lbry://@bar/foo'])
-
-    async def test_new_signature_model_from_unserializable_claim(self):
-        address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
-        sendtxid1 = await self.blockchain.send_to_address(address1, 5)
-        sendtxid2 = await self.blockchain.send_to_address(address2, 5)
-        await self.blockchain.generate(1)
-        await asyncio.wait([
-            self.on_transaction_id(sendtxid1),
-            self.on_transaction_id(sendtxid2)
-        ])
-
-        self.assertEqual(d2l(await self.account.get_balance()), '10.0')
-
-        cert, key = generate_certificate()
-        cert_tx = await Transaction.claim('@bar', cert, l2d('1.0'), address1, [self.account], self.account)
-        original = ClaimDict.load_dict(example_claim_dict).serialized
-        altered = original + b'\x00\x01\x02\x30\x50\x80\x99'  # pretend this extra trash is from some unknown protobuf
-
-        # manually sign
-        signer = get_signer(SECP256k1).load_pem(key)
-        signature = signer.sign(
-            b'foo',
-            decode_address(address1),
-            altered,
-            binascii.unhexlify(cert_tx.outputs[0].claim_id),
-        )
-        detached_sig = Signature(NAMED_SECP256K1(
-            signature,
-            binascii.unhexlify(cert_tx.outputs[0].claim_id),
-            altered
-        ))
-        claim_tx = await Transaction.claim('foo', detached_sig, l2d('1.0'), address1, [self.account], self.account)
-
-        await self.broadcast(cert_tx)
-        await self.broadcast(claim_tx)
-        await self.ledger.wait(claim_tx)
-        await self.blockchain.generate(1)
-        await self.ledger.wait(claim_tx)
-
-        response = await self.ledger.resolve(0, 10, 'lbry://@bar/foo')
-        self.assertIn('lbry://@bar/foo', response)
-        self.assertIn('claim', response['lbry://@bar/foo'])