claim_type/stream_type/media_type claim search integration test

lbrynet/extras/daemon/Daemon.py

@@ -1705,6 +1705,7 @@ class Daemon(metaclass=JSONRPCServerType):
                          [--support_amount=<support_amount>] [--trending_group=<trending_group>]
                          [--trending_mixed=<trending_mixed>] [--trending_local=<trending_local>]
                          [--trending_global=<trending_global]
+                         [--claim_type=<claim_type>] [--stream_types=<stream_types>...] [--media_types=<media_types>...]
                          [--any_tags=<any_tags>...] [--all_tags=<all_tags>...] [--not_tags=<not_tags>...]
                          [--any_languages=<any_languages>...] [--all_languages=<all_languages>...]
                          [--not_languages=<not_languages>...]
@@ -1772,6 +1773,9 @@ class Daemon(metaclass=JSONRPCServerType):
             --trending_global=<trending_global>: (int) trending value calculated relative to all
                                                     trending content globally (supports
                                                     equality constraints)
+            --claim_type=<claim_type>       : (str) filter by 'channel', 'stream' or 'unknown'
+            --stream_types=<stream_types>   : (list) filter by 'video', 'image', 'document', etc
+            --media_types=<media_types>     : (list) filter by 'video/mp4', 'image/png', etc
             --any_tags=<any_tags>           : (list) find claims containing any of the tags
             --all_tags=<all_tags>           : (list) find claims containing every tag
             --not_tags=<not_tags>           : (list) find claims not containing any of these tags

lbrynet/wallet/server/db.py

@@ -10,6 +10,7 @@ from torba.server.util import class_logger
 from torba.client.basedatabase import query, constraints_to_sql
 
 from lbrynet.schema.url import URL, normalize_name
+from lbrynet.schema.mime_types import guess_stream_type
 from lbrynet.wallet.ledger import MainNetLedger, RegTestLedger
 from lbrynet.wallet.transaction import Transaction, Output
 from lbrynet.wallet.server.canonical import register_canonical_functions
@@ -19,6 +20,18 @@ from lbrynet.wallet.server.trending import (
 
 
 ATTRIBUTE_ARRAY_MAX_LENGTH = 100
+CLAIM_TYPES = {
+    'stream': 1,
+    'channel': 2,
+}
+STREAM_TYPES = {
+    'video': 1,
+    'audio': 2,
+    'image': 3,
+    'document': 4,
+    'binary': 5,
+    'model': 6
+}
 
 
 def _apply_constraints_for_array_attributes(constraints, attr):
@@ -87,8 +100,13 @@ class SQLDB:
             short_url text not null, -- normalized#shortest-unique-claim_id
             canonical_url text, -- channel's-short_url/normalized#shortest-unique-claim_id-within-channel
 
+            claim_type integer,
+
+            -- streams
+            stream_type text,
+            media_type text,
+
             -- claims which are channels
-            is_channel bool not null,
             public_key_bytes bytes,
             public_key_hash bytes,
             claims_in_channel integer,
@@ -119,6 +137,10 @@ class SQLDB:
         create index if not exists claim_activation_height_idx on claim (activation_height);
         create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
 
+        create index if not exists claim_claim_type_idx on claim (claim_type);
+        create index if not exists claim_stream_type_idx on claim (stream_type);
+        create index if not exists claim_media_type_idx on claim (media_type);
+
         create index if not exists claim_effective_amount_idx on claim (effective_amount);
         create index if not exists claim_trending_group_idx on claim (trending_group);
         create index if not exists claim_trending_mixed_idx on claim (trending_mixed);
@@ -242,9 +264,11 @@ class SQLDB:
                 'txo_hash': sqlite3.Binary(txo.ref.hash),
                 'tx_position': tx.position,
                 'amount': txo.amount,
-                'is_channel': False,
                 'timestamp': header['timestamp'],
                 'height': tx.height,
+                'claim_type': None,
+                'stream_type': None,
+                'media_type': None,
                 'release_time': None,
             }
             claims.append(claim_record)
@@ -256,10 +280,13 @@ class SQLDB:
                 continue
 
             if claim.is_stream:
+                claim_record['claim_type'] = CLAIM_TYPES['stream']
+                claim_record['media_type'] = claim.stream.source.media_type
+                claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(claim_record['media_type'])]
                 if claim.stream.release_time:
                     claim_record['release_time'] = claim.stream.release_time
             elif claim.is_channel:
-                claim_record['is_channel'] = True
+                claim_record['claim_type'] = CLAIM_TYPES['channel']
 
             for tag in claim.message.tags:
                 tags.append((tag, claim_hash, tx.height))
@@ -280,11 +307,11 @@ class SQLDB:
             self.db.executemany("""
                 INSERT INTO claim (
                     claim_hash, claim_id, claim_name, normalized, txo_hash, tx_position, amount,
-                    is_channel, timestamp, creation_timestamp, height, creation_height,
+                    claim_type, media_type, stream_type, timestamp, creation_timestamp, height,
-                    release_time, activation_height, expiration_height, short_url)
+                    creation_height, release_time, activation_height, expiration_height, short_url)
                 VALUES (
                     :claim_hash, :claim_id, :claim_name, :normalized, :txo_hash, :tx_position, :amount,
-                    :is_channel, :timestamp, :timestamp, :height, :height,
+                    :claim_type, :media_type, :stream_type, :timestamp, :timestamp, :height, :height,
                     CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE :timestamp END,
                     CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END,
                     CASE WHEN :height >= 262974 THEN :height+2102400 ELSE :height+262974 END,
@@ -299,7 +326,9 @@ class SQLDB:
         if claims:
             self.db.executemany("""
                 UPDATE claim SET
-                    txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height, timestamp=:timestamp,
+                    txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height,
+                    claim_type=:claim_type, media_type=:media_type, stream_type=:stream_type,
+                    timestamp=:timestamp,
                     release_time=CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE release_time END
                 WHERE claim_hash=:claim_hash;
                 """, claims)
@@ -720,6 +749,15 @@ class SQLDB:
                 tx_hash + struct.pack('<I', nout)
             )
 
+        if 'claim_type' in constraints:
+            constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
+        if 'stream_types' in constraints:
+            constraints['claim.stream_type__in'] = [
+                STREAM_TYPES[stream_type] for stream_type in constraints.pop('stream_types')
+            ]
+        if 'media_types' in constraints:
+            constraints['claim.media_type__in'] = constraints.pop('media_types')
+
         _apply_constraints_for_array_attributes(constraints, 'tag')
         _apply_constraints_for_array_attributes(constraints, 'language')
         _apply_constraints_for_array_attributes(constraints, 'location')
@@ -750,7 +788,7 @@ class SQLDB:
             """
             claimtrie.claim_hash as is_controlling,
             claim.claim_hash, claim.txo_hash,
-            claim.is_channel, claim.claims_in_channel,
+            claim.claims_in_channel,
             claim.height, claim.creation_height,
             claim.activation_height, claim.expiration_height,
             claim.effective_amount, claim.support_amount,
@@ -773,6 +811,7 @@ class SQLDB:
 
     SEARCH_PARAMS = {
         'name', 'claim_id', 'txid', 'nout', 'channel', 'channel_ids', 'public_key_id',
+        'claim_type', 'stream_types', 'media_types',
         'any_tags', 'all_tags', 'not_tags',
         'any_locations', 'all_locations', 'not_locations',
         'any_languages', 'all_languages', 'not_languages',

tests/integration/test_claim_commands.py

@@ -8,12 +8,61 @@ from urllib.request import urlopen
 from torba.client.errors import InsufficientFundsError
 
 from lbrynet.testcase import CommandTestCase
+from lbrynet.wallet.transaction import Transaction
 
 
 log = logging.getLogger(__name__)
 
 
-class ClaimSearchCommand(CommandTestCase):
+class ClaimTestCase(CommandTestCase):
+
+    files_directory = os.path.join(os.path.dirname(__file__), 'files')
+    video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
+    video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
+
+    def setUp(self):
+        if not os.path.exists(self.video_file_name):
+            if not os.path.exists(self.files_directory):
+                os.mkdir(self.files_directory)
+            log.info(f'downloading test video from {self.video_file_name}')
+            with urlopen(self.video_file_url) as response, \
+                    open(self.video_file_name, 'wb') as video_file:
+                video_file.write(response.read())
+
+    async def image_stream_create(self, name='blank-image', bid='1.0', confirm=True):
+        with tempfile.NamedTemporaryFile(suffix='.png') as file:
+            file.write(unhexlify(
+                b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
+                b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
+                b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
+                b'9454e44ae426082'
+            ))
+            file.flush()
+            tx = await self.out(
+                self.daemon.jsonrpc_stream_create(
+                    name, bid, file_path=file.name
+                )
+            )
+        if confirm:
+            await self.on_transaction_dict(tx)
+            await self.generate(1)
+            await self.on_transaction_dict(tx)
+        return tx
+
+    async def video_stream_create(self, name='chrome', bid='1.0', confirm=True):
+        tx = await self.out(
+            self.daemon.jsonrpc_stream_create(
+                name, bid, file_path=self.video_file_name
+            )
+        )
+        if confirm:
+            await self.on_transaction_dict(tx)
+            await self.generate(1)
+            await self.on_transaction_dict(tx)
+        return tx
+
+
+class ClaimSearchCommand(ClaimTestCase):
 
     async def create_channel(self):
         self.channel = await self.channel_create('@abc', '1.0')
@@ -170,6 +219,37 @@ class ClaimSearchCommand(CommandTestCase):
 
         await self.assertFindsClaims(claims, order_by=["^name"])
 
+    async def test_claim_type_and_media_type_search(self):
+        # create an invalid/unknown claim
+        address = await self.account.receiving.get_or_create_usable_address()
+        tx = await Transaction.claim_create(
+            'unknown', b'{"sources":{"lbry_sd_hash":""}}', 1, address, [self.account], self.account)
+        await tx.sign([self.account])
+        await self.broadcast(tx)
+        await self.confirm_tx(tx.id)
+
+        octet = await self.stream_create()
+        video = await self.video_stream_create()
+        image = await self.image_stream_create()
+        channel = await self.channel_create()
+        unknown = self.sout(tx)
+
+        # claim_type
+        await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
+        await self.assertFindsClaims([channel], claim_type='channel')
+
+        # stream_type
+        await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
+        await self.assertFindsClaims([video], stream_types=['video'])
+        await self.assertFindsClaims([image], stream_types=['image'])
+        await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
+
+        # stream_type
+        await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
+        await self.assertFindsClaims([video], media_types=['video/mp4'])
+        await self.assertFindsClaims([image], media_types=['image/png'])
+        await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
+
 
 class ChannelCommands(CommandTestCase):
 
@@ -370,20 +450,7 @@ class ChannelCommands(CommandTestCase):
         self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
 
 
-class StreamCommands(CommandTestCase):
+class StreamCommands(ClaimTestCase):
-
-    files_directory = os.path.join(os.path.dirname(__file__), 'files')
-    video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
-    video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
-
-    def setUp(self):
-        if not os.path.exists(self.video_file_name):
-            if not os.path.exists(self.files_directory):
-                os.mkdir(self.files_directory)
-            log.info(f'downloading test video from {self.video_file_name}')
-            with urlopen(self.video_file_url) as response,\
-                    open(self.video_file_name, 'wb') as video_file:
-                video_file.write(response.read())
 
     async def test_create_stream_names(self):
         # claim new name
@@ -662,44 +729,26 @@ class StreamCommands(CommandTestCase):
         self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 1)
 
     async def test_automatic_type_and_metadata_detection_for_image(self):
-        with tempfile.NamedTemporaryFile(suffix='.png') as file:
+        txo = (await self.image_stream_create())['outputs'][0]
-            file.write(unhexlify(
+        self.assertEqual(
-                b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
+            txo['value'], {
-                b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
+                'source': {
-                b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
+                    'size': '99',
-                b'9454e44ae426082'
+                    'name': txo['value']['source']['name'],
-            ))
+                    'media_type': 'image/png',
-            file.flush()
+                    'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
-            tx = await self.out(
+                    'sd_hash': txo['value']['source']['sd_hash'],
-                self.daemon.jsonrpc_stream_create(
+                },
-                    'blank-image', '1.0', file_path=file.name
+                'stream_type': 'image',
-                )
+                'image': {
-            )
+                    'width': 5,
-            txo = tx['outputs'][0]
+                    'height': 7
-            self.assertEqual(
-                txo['value'], {
-                    'source': {
-                        'size': '99',
-                        'name': os.path.basename(file.name),
-                        'media_type': 'image/png',
-                        'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
-                        'sd_hash': txo['value']['source']['sd_hash'],
-                    },
-                    'stream_type': 'image',
-                    'image': {
-                        'width': 5,
-                        'height': 7
-                    }
                 }
-            )
+            }
+        )
 
     async def test_automatic_type_and_metadata_detection_for_video(self):
-        tx = await self.out(
+        txo = (await self.video_stream_create())['outputs'][0]
-            self.daemon.jsonrpc_stream_create(
-                'chrome', '1.0', file_path=self.video_file_name
-            )
-        )
-        txo = tx['outputs'][0]
         self.assertEqual(
             txo['value'], {
                 'source': {