update to an even newer protobufs

lbrynet/extras/daemon/Daemon.py

@@ -2213,7 +2213,7 @@ class Daemon(metaclass=JSONRPCServerType):
 
         if not preview:
             file_stream = await self.stream_manager.create_stream(file_path)
-            claim.stream.sd_hash = file_stream.sd_hash
+            claim.stream.source.sd_hash = file_stream.sd_hash
             new_txo.script.generate()
             if channel:
                 new_txo.sign(channel)
@@ -2222,7 +2222,7 @@ class Daemon(metaclass=JSONRPCServerType):
             await self.storage.save_claims([self._old_get_temp_claim_info(
                 tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
             )])
-            stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim.stream.sd_hash)
+            stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim.stream.source.sd_hash)
             if stream_hash:
                 await self.storage.save_content_claim(stream_hash, new_txo.id)
             await self.analytics_manager.send_claim_action('publish')
@@ -2376,7 +2376,7 @@ class Daemon(metaclass=JSONRPCServerType):
         if not preview:
             if file_path is not None:
                 file_stream = await self.stream_manager.create_stream(file_path)
-                new_txo.claim.stream.sd_hash = file_stream.sd_hash
+                new_txo.claim.stream.source.sd_hash = file_stream.sd_hash
                 new_txo.script.generate()
             if channel:
                 new_txo.sign(channel)
@@ -2385,7 +2385,7 @@ class Daemon(metaclass=JSONRPCServerType):
             await self.storage.save_claims([self._old_get_temp_claim_info(
                 tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
             )])
-            stream_hash = await self.storage.get_stream_hash_for_sd_hash(new_txo.claim.stream.sd_hash)
+            stream_hash = await self.storage.get_stream_hash_for_sd_hash(new_txo.claim.stream.source.sd_hash)
             if stream_hash:
                 await self.storage.save_content_claim(stream_hash, new_txo.id)
             await self.analytics_manager.send_claim_action('publish')

lbrynet/extras/daemon/json_response_encoder.py

@@ -17,11 +17,22 @@ log = logging.getLogger(__name__)
 def encode_txo_doc():
     return {
         'txid': "hash of transaction in hex",
-        'height': "block where transaction was recorded",
         'nout': "position in the transaction",
+        'height': "block where transaction was recorded",
         'amount': "value of the txo as a decimal",
         'address': "address of who can spend the txo",
-        'confirmations': "number of confirmed blocks"
+        'confirmations': "number of confirmed blocks",
+        'is_change': "payment to change address, only available when it can be determined",
+        'is_mine': "payment to one of your accounts, only available when it can be determined",
+        'type': "one of 'claim', 'support' or 'payment'",
+        'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
+        'name': "when type is 'claim' or 'support', this is the claim name",
+        'claim_id': "when type is 'claim' or 'support', this is the claim id",
+        'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
+        'value': "when type is 'claim', this is the claim metadata",
+        'sub_type': "when type is 'claim', this determines if it is 'channel' or 'stream' claim",
+        'signing_channel': "for signed claims only, metadata of signing channel",
+        'is_channel_signature_valid': "for signed claims only, whether signature is valid",
     }
 
 
@@ -101,21 +112,7 @@ class JSONResponseEncoder(JSONEncoder):
         if isinstance(obj, Output):
             return self.encode_output(obj)
         if isinstance(obj, Claim):
-            claim_dict = obj.to_dict()
+            return self.encode_claim(obj)
-            if obj.is_stream:
-                claim_dict['stream']['sd_hash'] = obj.stream.sd_hash
-                fee = claim_dict['stream'].get('fee', {})
-                if 'address' in fee:
-                    fee['address'] = obj.stream.fee.address
-                if 'amount' in fee:
-                    fee['amount'] = obj.stream.fee.amount
-                if 'languages' in claim_dict['stream']:
-                    claim_dict['stream']['languages'] = obj.stream.langtags
-            elif obj.is_channel:
-                claim_dict['channel']['public_key'] = obj.channel.public_key
-                if 'languages' in claim_dict['channel']:
-                    claim_dict['channel']['languages'] = obj.channel.langtags
-            return claim_dict
         if isinstance(obj, datetime):
             return obj.strftime("%Y%m%dT%H:%M:%S")
         if isinstance(obj, Decimal):
@@ -142,9 +139,9 @@ class JSONResponseEncoder(JSONEncoder):
         output = {
             'txid': txo.tx_ref.id,
             'nout': txo.position,
+            'height': tx_height,
             'amount': dewies_to_lbc(txo.amount),
             'address': txo.get_address(self.ledger),
-            'height': tx_height,
             'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
         }
         if txo.is_change is not None:
@@ -152,40 +149,46 @@ class JSONResponseEncoder(JSONEncoder):
         if txo.is_my_account is not None:
             output['is_mine'] = txo.is_my_account
 
+        if txo.script.is_claim_name:
+            output['type'] = 'claim'
+            output['claim_op'] = 'create'
+        elif txo.script.is_update_claim:
+            output['type'] = 'claim'
+            output['claim_op'] = 'update'
+        elif txo.script.is_support_claim:
+            output['type'] = 'support'
+        else:
+            output['type'] = 'payment'
+
         if txo.script.is_claim_involved:
             output.update({
                 'name': txo.claim_name,
                 'claim_id': txo.claim_id,
                 'permanent_url': txo.permanent_url,
             })
-
             if txo.script.is_claim_name or txo.script.is_update_claim:
-                claim = txo.claim
+                output['value'] = txo.claim
-                output['value'] = claim
+                if txo.claim.is_channel:
-                if claim.is_signed:
+                    output['sub_type'] = 'channel'
-                    output['valid_signature'] = None
+                elif txo.claim.is_stream:
-                    if check_signature and txo.channel is not None:
+                    output['sub_type'] = 'stream'
-                        output['channel_name'] = txo.channel.claim_name
+                if txo.channel is not None:
+                    output['signing_channel'] = {
+                        'name': txo.channel.claim_name,
+                        'claim_id': txo.channel.claim_id,
+                        'value': txo.channel.claim
+                    }
+                    if check_signature and txo.claim.is_signed:
+                        output['is_channel_signature_valid'] = False
                         try:
-                            output['valid_signature'] = txo.is_signed_by(txo.channel, self.ledger)
+                            output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
                         except BadSignatureError:
-                            output['valid_signature'] = False
+                            pass
                         except ValueError:
                             log.exception(
                                 'txo.id: %s, txo.channel.id:%s, output: %s',
                                 txo.id, txo.channel.id, output
                             )
-                            output['valid_signature'] = False
-
-            if txo.script.is_claim_name:
-                output['type'] = 'claim'
-            elif txo.script.is_update_claim:
-                output['type'] = 'update'
-            elif txo.script.is_support_claim:
-                output['type'] = 'support'
-            else:
-                output['type'] = 'basic'
-
         return output
 
     def encode_input(self, txi):
@@ -204,3 +207,11 @@ class JSONResponseEncoder(JSONEncoder):
     @staticmethod
     def encode_file(managed_stream):
         return managed_stream.as_dict()
+
+    @staticmethod
+    def encode_claim(claim):
+        if claim.is_stream:
+            return claim.stream.to_dict()
+        elif claim.is_channel:
+            return claim.channel.to_dict()
+        return claim.to_dict()

lbrynet/extras/daemon/storage.py

@@ -593,7 +593,7 @@ class SQLiteStorage(SQLiteMixin):
                 sequence = claim_info['claim_sequence']
                 certificate_id = claim_info['value'].signing_channel_id
                 try:
-                    source_hash = claim_info['value'].stream.sd_hash
+                    source_hash = claim_info['value'].stream.source.sd_hash
                 except (AttributeError, ValueError):
                     source_hash = None
                 serialized = binascii.hexlify(claim_info['value'].to_bytes())
@@ -670,7 +670,7 @@ class SQLiteStorage(SQLiteMixin):
         if not known_sd_hash:
             raise Exception("stream not found")
         # check the claim contains the same sd hash
-        if known_sd_hash[0] != claim.stream.sd_hash:
+        if known_sd_hash[0] != claim.stream.source.sd_hash:
             raise Exception("stream mismatch")
 
         # if there is a current claim associated to the file, check that the new claim is an update to it

lbrynet/schema/claim.py

@@ -358,6 +358,43 @@ class Fee:
         self._fee.currency = FeeMessage.USD
 
 
+class ClaimReference:
+
+    __slots__ = 'message',
+
+    def __init__(self, message):
+        self.message = message
+
+    @property
+    def claim_id(self) -> str:
+        return hexlify(self.claim_hash[::-1]).decode()
+
+    @claim_id.setter
+    def claim_id(self, claim_id: str):
+        self.claim_hash = unhexlify(claim_id)[::-1]
+
+    @property
+    def claim_hash(self) -> bytes:
+        return self.message.claim_hash
+
+    @claim_hash.setter
+    def claim_hash(self, claim_hash: bytes):
+        self.message.claim_hash = claim_hash
+
+
+class ClaimList(BaseMessageList[ClaimReference]):
+
+    __slots__ = ()
+    item_class = ClaimReference
+
+    def append(self, value):
+        self.add().claim_id = value
+
+    @property
+    def claim_ids(self) -> List[str]:
+        return [c.claim_id for c in self]
+
+
 class Language:
 
     __slots__ = 'message',
@@ -532,9 +569,42 @@ class BaseClaimSubType:
 
     __slots__ = 'claim', 'message'
 
+    object_fields = 'thumbnail',
+    repeat_fields = 'tags', 'languages', 'locations'
+
     def __init__(self, claim: Claim):
         self.claim = claim or Claim()
 
+    def to_dict(self):
+        claim = self.claim.to_dict()
+        if 'languages' in claim:
+            claim['languages'] = self.langtags
+        return claim
+
+    def update(self, **kwargs):
+        for key in list(kwargs):
+            for field in self.object_fields:
+                if key.startswith(f'{field}_'):
+                    attr = getattr(self, field)
+                    setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
+                    continue
+
+        for l in self.repeat_fields:
+            field = getattr(self, l)
+            if kwargs.pop(f'clear_{l}', False):
+                del field[:]
+            items = kwargs.pop(l, None)
+            if items is not None:
+                if isinstance(items, str):
+                    field.append(items)
+                elif isinstance(items, list):
+                    field.extend(items)
+                else:
+                    raise ValueError(f"Unknown {l} value: {items}")
+
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
     @property
     def title(self) -> str:
         return self.claim.message.title
@@ -571,34 +641,24 @@ class BaseClaimSubType:
     def locations(self) -> LocationList:
         return LocationList(self.claim.message.locations)
 
-    def to_dict(self):
-        return MessageToDict(self.message, preserving_proto_field_name=True)
-
-    def update(self, **kwargs):
-        for l in ('tags', 'languages', 'locations'):
-            if kwargs.pop(f'clear_{l}', False):
-                self.message.ClearField('tags')
-            items = kwargs.pop(l, None)
-            if items is not None:
-                if isinstance(items, str):
-                    getattr(self, l).append(items)
-                elif isinstance(items, list):
-                    getattr(self, l).extend(items)
-                else:
-                    raise ValueError(f"Unknown {l} value: {items}")
-
-        for key, value in kwargs.items():
-            setattr(self, key, value)
-
 
 class Channel(BaseClaimSubType):
 
     __slots__ = ()
 
+    object_fields = BaseClaimSubType.object_fields + ('cover',)
+    repeat_fields = BaseClaimSubType.repeat_fields + ('featured',)
+
     def __init__(self, claim: Claim = None):
         super().__init__(claim)
         self.message = self.claim.channel_message
 
+    def to_dict(self):
+        claim = super().to_dict()
+        claim.update(claim.pop('channel'))
+        claim['public_key'] = self.public_key
+        return claim
+
     @property
     def public_key(self) -> str:
         return hexlify(self.message.public_key).decode()
@@ -616,34 +676,50 @@ class Channel(BaseClaimSubType):
         self.message.public_key = public_key
 
     @property
-    def contact_email(self) -> str:
+    def email(self) -> str:
-        return self.message.contact_email
+        return self.message.email
 
-    @contact_email.setter
+    @email.setter
-    def contact_email(self, contact_email: str):
+    def email(self, email: str):
-        self.message.contact_email = contact_email
+        self.message.email = email
 
     @property
-    def homepage_url(self) -> str:
+    def website_url(self) -> str:
-        return self.message.homepage_url
+        return self.message.website_url
 
-    @homepage_url.setter
+    @website_url.setter
-    def homepage_url(self, homepage_url: str):
+    def website_url(self, website_url: str):
-        self.message.homepage_url = homepage_url
+        self.message.website_url = website_url
 
     @property
     def cover(self) -> Source:
         return Source(self.message.cover)
 
+    @property
+    def featured(self) -> ClaimList:
+        return ClaimList(self.message.featured)
+
 
 class Stream(BaseClaimSubType):
 
     __slots__ = ()
 
+    object_fields = BaseClaimSubType.object_fields + ('source',)
+
     def __init__(self, claim: Claim = None):
         super().__init__(claim)
         self.message = self.claim.stream_message
 
+    def to_dict(self):
+        claim = super().to_dict()
+        claim.update(claim.pop('stream'))
+        fee = claim.get('fee', {})
+        if 'address' in fee:
+            fee['address'] = self.fee.address
+        if 'amount' in fee:
+            fee['amount'] = self.fee.amount
+        return claim
+
     def update(
             self, file_path=None, stream_type=None,
             fee_currency=None, fee_amount=None, fee_address=None,
@@ -672,14 +748,17 @@ class Stream(BaseClaimSubType):
             if duration_was_not_set and file_path and isinstance(sub_obj, Playable):
                 sub_obj.set_duration_from_path(file_path)
 
+        if 'sd_hash' in kwargs:
+            self.source.sd_hash = kwargs.pop('sd_hash')
+
         super().update(**kwargs)
 
         if file_path is not None:
-            self.media_type = guess_media_type(file_path)
+            self.source.media_type = guess_media_type(file_path)
             if not os.path.isfile(file_path):
                 raise Exception(f"File does not exist: {file_path}")
-            self.file.size = os.path.getsize(file_path)
+            self.source.size = os.path.getsize(file_path)
-            if self.file.size == 0:
+            if self.source.size == 0:
                 raise Exception(f"Cannot publish empty file: {file_path}")
 
         if fee_amount and fee_currency:

lbrynet/stream/stream_manager.py

@@ -353,7 +353,7 @@ class StreamManager:
         if existing:
             await self.start_stream(existing[0])
             return existing[0], None
-        existing = self.get_filtered_streams(sd_hash=claim.stream.sd_hash)
+        existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
         if existing and existing[0].claim_id != claim_id:
             raise ResolveError(f"stream for {existing[0].claim_id} collides with existing "
                                f"download {claim_id}")
@@ -437,7 +437,7 @@ class StreamManager:
 
         # download the stream
         download_id = binascii.hexlify(generate_id()).decode()
-        downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.stream.sd_hash,
+        downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash,
                                       self.config.download_dir, file_name)
 
         stream = None
@@ -484,7 +484,7 @@ class StreamManager:
                     None if not stream else len(stream.downloader.blob_downloader.scores),
                     False if not downloader else downloader.added_fixed_peers,
                     self.config.fixed_peer_delay if not downloader else downloader.fixed_peers_delay,
-                    claim.stream.sd_hash, time_to_descriptor,
+                    claim.stream.source.sd_hash, time_to_descriptor,
                     None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
                     None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
                     time_to_first_bytes, None if not error else error.__class__.__name__

lbrynet/utils.py

@@ -116,7 +116,7 @@ def get_sd_hash(stream_info):
     if not stream_info:
         return None
     if isinstance(stream_info, Claim):
-        return stream_info.stream.sd_hash
+        return stream_info.stream.source.sd_hash
     result = stream_info.get('claim', {}).\
         get('value', {}).\
         get('stream', {}).\

tests/integration/test_claim_commands.py

@@ -75,37 +75,39 @@ class ChannelCommands(CommandTestCase):
             'thumbnail_url': "https://co.ol/thumbnail.png",
             'languages': ["en-US"],
             'locations': ['US::Manchester'],
-            'contact_email': "human@email.com",
+            'email': "human@email.com",
-            'homepage_url': "https://co.ol",
+            'website_url': "https://co.ol",
             'cover_url': "https://co.ol/cover.png",
         }
         fixed_values = values.copy()
         fixed_values['languages'] = ['en-US']
         fixed_values['locations'] = [{'country': 'US', 'city': 'Manchester'}]
+        fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
+        fixed_values['cover'] = {'url': fixed_values.pop('cover_url')}
 
         # create new channel with all fields set
         tx = await self.out(self.channel_create('@bigchannel', **values))
         txo = tx['outputs'][0]
         self.assertEqual(
-            txo['value']['channel'],
+            txo['value'],
-            {'public_key': txo['value']['channel']['public_key'], **fixed_values}
+            {'public_key': txo['value']['public_key'], **fixed_values}
         )
 
         # create channel with nothing set
         tx = await self.out(self.channel_create('@lightchannel'))
         txo = tx['outputs'][0]
         self.assertEqual(
-            txo['value']['channel'],
+            txo['value'],
-            {'public_key': txo['value']['channel']['public_key']}
+            {'public_key': txo['value']['public_key']}
         )
 
         # create channel with just some tags
         tx = await self.out(self.channel_create('@updatedchannel', tags='blah'))
         txo = tx['outputs'][0]
         claim_id = txo['claim_id']
-        public_key = txo['value']['channel']['public_key']
+        public_key = txo['value']['public_key']
         self.assertEqual(
-            txo['value']['channel'],
+            txo['value'],
             {'public_key': public_key, 'tags': ['blah']}
         )
 
@@ -115,7 +117,7 @@ class ChannelCommands(CommandTestCase):
         fixed_values['public_key'] = public_key
         fixed_values['tags'].insert(0, 'blah')  # existing tag
         self.assertEqual(
-            txo['value']['channel'],
+            txo['value'],
             fixed_values
         )
 
@@ -124,7 +126,7 @@ class ChannelCommands(CommandTestCase):
         txo = tx['outputs'][0]
         fixed_values['tags'] = ['single']
         self.assertEqual(
-            txo['value']['channel'],
+            txo['value'],
             fixed_values
         )
 
@@ -132,7 +134,7 @@ class ChannelCommands(CommandTestCase):
         tx = await self.out(self.channel_update(claim_id, new_signing_key=True))
         txo = tx['outputs'][0]
         self.assertNotEqual(
-            txo['value']['channel']['public_key'],
+            txo['value']['public_key'],
             fixed_values['public_key']
         )
 
@@ -284,15 +286,12 @@ class StreamCommands(CommandTestCase):
         fixed_values = values.copy()
         fixed_values['languages'] = ['en']
         fixed_values['locations'] = [{'country': 'UA'}]
-
+        fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
-        # create new channel with all fields set
-        tx = await self.out(self.stream_create('big', **values))
-        txo = tx['outputs'][0]
-        stream = txo['value']['stream']
-        fixed_values['sd_hash'] = stream['sd_hash']
-        fixed_values['file'] = stream['file']
-        fixed_values['media_type'] = 'application/octet-stream'
         fixed_values['release_time'] = str(values['release_time'])
+        fixed_values['source'] = {
+            'media_type': 'application/octet-stream',
+            'size': '3'
+        }
         fixed_values['fee'] = {
             'address': fixed_values.pop('fee_address'),
             'amount': float(fixed_values.pop('fee_amount')),
@@ -302,16 +301,24 @@ class StreamCommands(CommandTestCase):
             'height': fixed_values.pop('video_height'),
             'width': fixed_values.pop('video_width')
         }
+
+        # create new channel with all fields set
+        tx = await self.out(self.stream_create('big', **values))
+        txo = tx['outputs'][0]
+        stream = txo['value']
+        fixed_values['source']['sd_hash'] = stream['source']['sd_hash']
         self.assertEqual(stream, fixed_values)
 
         # create channel with nothing set
         tx = await self.out(self.stream_create('light'))
         txo = tx['outputs'][0]
         self.assertEqual(
-            txo['value']['stream'], {
+            txo['value'], {
-                'file': {'size': '3'},
+                'source': {
-                'media_type': 'application/octet-stream',
+                    'size': '3',
-                'sd_hash': txo['value']['stream']['sd_hash']
+                    'media_type': 'application/octet-stream',
+                    'sd_hash': txo['value']['source']['sd_hash']
+                },
             }
         )
 
@@ -319,12 +326,14 @@ class StreamCommands(CommandTestCase):
         tx = await self.out(self.stream_create('updated', tags='blah'))
         txo = tx['outputs'][0]
         claim_id = txo['claim_id']
-        fixed_values['sd_hash'] = txo['value']['stream']['sd_hash']
+        fixed_values['source']['sd_hash'] = txo['value']['source']['sd_hash']
         self.assertEqual(
-            txo['value']['stream'], {
+            txo['value'], {
-                'file': {'size': '3'},
+                'source': {
-                'media_type': 'application/octet-stream',
+                    'size': '3',
-                'sd_hash': fixed_values['sd_hash'],
+                    'media_type': 'application/octet-stream',
+                    'sd_hash': fixed_values['source']['sd_hash'],
+                },
                 'tags': ['blah']
             }
         )
@@ -333,13 +342,13 @@ class StreamCommands(CommandTestCase):
         tx = await self.out(self.stream_update(claim_id, **values))
         txo = tx['outputs'][0]
         fixed_values['tags'].insert(0, 'blah')  # existing tag
-        self.assertEqual(txo['value']['stream'], fixed_values)
+        self.assertEqual(txo['value'], fixed_values)
 
         # clearing and settings tags
         tx = await self.out(self.stream_update(claim_id, tags='single', clear_tags=True))
         txo = tx['outputs'][0]
         fixed_values['tags'] = ['single']
-        self.assertEqual(txo['value']['stream'], fixed_values)
+        self.assertEqual(txo['value'], fixed_values)
 
         # send claim to someone else
         new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
@@ -441,7 +450,7 @@ class StreamCommands(CommandTestCase):
         self.assertEqual(claim['txid'], tx4['outputs'][0]['txid'])
         self.assertEqual(claim['channel_name'], '@abc')
         self.assertEqual(claim['signature_is_valid'], True)
-        self.assertEqual(claim['value']['stream']['languages'], ['uk-UA'])
+        self.assertEqual(claim['value']['languages'], ['uk-UA'])
 
     async def test_claim_search(self):
         # search for channel claim