claim_type/stream_type/media_type claim search integration test
This commit is contained in:
parent
9dbf47916b
commit
8d2c9e5785
3 changed files with 150 additions and 58 deletions
|
@ -1705,6 +1705,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
|||
[--support_amount=<support_amount>] [--trending_group=<trending_group>]
|
||||
[--trending_mixed=<trending_mixed>] [--trending_local=<trending_local>]
|
||||
[--trending_global=<trending_global]
|
||||
[--claim_type=<claim_type>] [--stream_types=<stream_types>...] [--media_types=<media_types>...]
|
||||
[--any_tags=<any_tags>...] [--all_tags=<all_tags>...] [--not_tags=<not_tags>...]
|
||||
[--any_languages=<any_languages>...] [--all_languages=<all_languages>...]
|
||||
[--not_languages=<not_languages>...]
|
||||
|
@ -1772,6 +1773,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
|||
--trending_global=<trending_global>: (int) trending value calculated relative to all
|
||||
trending content globally (supports
|
||||
equality constraints)
|
||||
--claim_type=<claim_type> : (str) filter by 'channel', 'stream' or 'unknown'
|
||||
--stream_types=<stream_types> : (list) filter by 'video', 'image', 'document', etc
|
||||
--media_types=<media_types> : (list) filter by 'video/mp4', 'image/png', etc
|
||||
--any_tags=<any_tags> : (list) find claims containing any of the tags
|
||||
--all_tags=<all_tags> : (list) find claims containing every tag
|
||||
--not_tags=<not_tags> : (list) find claims not containing any of these tags
|
||||
|
|
|
@ -10,6 +10,7 @@ from torba.server.util import class_logger
|
|||
from torba.client.basedatabase import query, constraints_to_sql
|
||||
|
||||
from lbrynet.schema.url import URL, normalize_name
|
||||
from lbrynet.schema.mime_types import guess_stream_type
|
||||
from lbrynet.wallet.ledger import MainNetLedger, RegTestLedger
|
||||
from lbrynet.wallet.transaction import Transaction, Output
|
||||
from lbrynet.wallet.server.canonical import register_canonical_functions
|
||||
|
@ -19,6 +20,18 @@ from lbrynet.wallet.server.trending import (
|
|||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
CLAIM_TYPES = {
|
||||
'stream': 1,
|
||||
'channel': 2,
|
||||
}
|
||||
STREAM_TYPES = {
|
||||
'video': 1,
|
||||
'audio': 2,
|
||||
'image': 3,
|
||||
'document': 4,
|
||||
'binary': 5,
|
||||
'model': 6
|
||||
}
|
||||
|
||||
|
||||
def _apply_constraints_for_array_attributes(constraints, attr):
|
||||
|
@ -87,8 +100,13 @@ class SQLDB:
|
|||
short_url text not null, -- normalized#shortest-unique-claim_id
|
||||
canonical_url text, -- channel's-short_url/normalized#shortest-unique-claim_id-within-channel
|
||||
|
||||
claim_type integer,
|
||||
|
||||
-- streams
|
||||
stream_type text,
|
||||
media_type text,
|
||||
|
||||
-- claims which are channels
|
||||
is_channel bool not null,
|
||||
public_key_bytes bytes,
|
||||
public_key_hash bytes,
|
||||
claims_in_channel integer,
|
||||
|
@ -119,6 +137,10 @@ class SQLDB:
|
|||
create index if not exists claim_activation_height_idx on claim (activation_height);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_claim_type_idx on claim (claim_type);
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
|
||||
create index if not exists claim_effective_amount_idx on claim (effective_amount);
|
||||
create index if not exists claim_trending_group_idx on claim (trending_group);
|
||||
create index if not exists claim_trending_mixed_idx on claim (trending_mixed);
|
||||
|
@ -242,9 +264,11 @@ class SQLDB:
|
|||
'txo_hash': sqlite3.Binary(txo.ref.hash),
|
||||
'tx_position': tx.position,
|
||||
'amount': txo.amount,
|
||||
'is_channel': False,
|
||||
'timestamp': header['timestamp'],
|
||||
'height': tx.height,
|
||||
'claim_type': None,
|
||||
'stream_type': None,
|
||||
'media_type': None,
|
||||
'release_time': None,
|
||||
}
|
||||
claims.append(claim_record)
|
||||
|
@ -256,10 +280,13 @@ class SQLDB:
|
|||
continue
|
||||
|
||||
if claim.is_stream:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['stream']
|
||||
claim_record['media_type'] = claim.stream.source.media_type
|
||||
claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(claim_record['media_type'])]
|
||||
if claim.stream.release_time:
|
||||
claim_record['release_time'] = claim.stream.release_time
|
||||
elif claim.is_channel:
|
||||
claim_record['is_channel'] = True
|
||||
claim_record['claim_type'] = CLAIM_TYPES['channel']
|
||||
|
||||
for tag in claim.message.tags:
|
||||
tags.append((tag, claim_hash, tx.height))
|
||||
|
@ -280,11 +307,11 @@ class SQLDB:
|
|||
self.db.executemany("""
|
||||
INSERT INTO claim (
|
||||
claim_hash, claim_id, claim_name, normalized, txo_hash, tx_position, amount,
|
||||
is_channel, timestamp, creation_timestamp, height, creation_height,
|
||||
release_time, activation_height, expiration_height, short_url)
|
||||
claim_type, media_type, stream_type, timestamp, creation_timestamp, height,
|
||||
creation_height, release_time, activation_height, expiration_height, short_url)
|
||||
VALUES (
|
||||
:claim_hash, :claim_id, :claim_name, :normalized, :txo_hash, :tx_position, :amount,
|
||||
:is_channel, :timestamp, :timestamp, :height, :height,
|
||||
:claim_type, :media_type, :stream_type, :timestamp, :timestamp, :height, :height,
|
||||
CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE :timestamp END,
|
||||
CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END,
|
||||
CASE WHEN :height >= 262974 THEN :height+2102400 ELSE :height+262974 END,
|
||||
|
@ -299,7 +326,9 @@ class SQLDB:
|
|||
if claims:
|
||||
self.db.executemany("""
|
||||
UPDATE claim SET
|
||||
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height, timestamp=:timestamp,
|
||||
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height,
|
||||
claim_type=:claim_type, media_type=:media_type, stream_type=:stream_type,
|
||||
timestamp=:timestamp,
|
||||
release_time=CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE release_time END
|
||||
WHERE claim_hash=:claim_hash;
|
||||
""", claims)
|
||||
|
@ -720,6 +749,15 @@ class SQLDB:
|
|||
tx_hash + struct.pack('<I', nout)
|
||||
)
|
||||
|
||||
if 'claim_type' in constraints:
|
||||
constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
|
||||
if 'stream_types' in constraints:
|
||||
constraints['claim.stream_type__in'] = [
|
||||
STREAM_TYPES[stream_type] for stream_type in constraints.pop('stream_types')
|
||||
]
|
||||
if 'media_types' in constraints:
|
||||
constraints['claim.media_type__in'] = constraints.pop('media_types')
|
||||
|
||||
_apply_constraints_for_array_attributes(constraints, 'tag')
|
||||
_apply_constraints_for_array_attributes(constraints, 'language')
|
||||
_apply_constraints_for_array_attributes(constraints, 'location')
|
||||
|
@ -750,7 +788,7 @@ class SQLDB:
|
|||
"""
|
||||
claimtrie.claim_hash as is_controlling,
|
||||
claim.claim_hash, claim.txo_hash,
|
||||
claim.is_channel, claim.claims_in_channel,
|
||||
claim.claims_in_channel,
|
||||
claim.height, claim.creation_height,
|
||||
claim.activation_height, claim.expiration_height,
|
||||
claim.effective_amount, claim.support_amount,
|
||||
|
@ -773,6 +811,7 @@ class SQLDB:
|
|||
|
||||
SEARCH_PARAMS = {
|
||||
'name', 'claim_id', 'txid', 'nout', 'channel', 'channel_ids', 'public_key_id',
|
||||
'claim_type', 'stream_types', 'media_types',
|
||||
'any_tags', 'all_tags', 'not_tags',
|
||||
'any_locations', 'all_locations', 'not_locations',
|
||||
'any_languages', 'all_languages', 'not_languages',
|
||||
|
|
|
@ -8,12 +8,61 @@ from urllib.request import urlopen
|
|||
from torba.client.errors import InsufficientFundsError
|
||||
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClaimSearchCommand(CommandTestCase):
|
||||
class ClaimTestCase(CommandTestCase):
|
||||
|
||||
files_directory = os.path.join(os.path.dirname(__file__), 'files')
|
||||
video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
|
||||
video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
|
||||
|
||||
def setUp(self):
|
||||
if not os.path.exists(self.video_file_name):
|
||||
if not os.path.exists(self.files_directory):
|
||||
os.mkdir(self.files_directory)
|
||||
log.info(f'downloading test video from {self.video_file_name}')
|
||||
with urlopen(self.video_file_url) as response, \
|
||||
open(self.video_file_name, 'wb') as video_file:
|
||||
video_file.write(response.read())
|
||||
|
||||
async def image_stream_create(self, name='blank-image', bid='1.0', confirm=True):
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as file:
|
||||
file.write(unhexlify(
|
||||
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
|
||||
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
|
||||
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
|
||||
b'9454e44ae426082'
|
||||
))
|
||||
file.flush()
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
name, bid, file_path=file.name
|
||||
)
|
||||
)
|
||||
if confirm:
|
||||
await self.on_transaction_dict(tx)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(tx)
|
||||
return tx
|
||||
|
||||
async def video_stream_create(self, name='chrome', bid='1.0', confirm=True):
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
name, bid, file_path=self.video_file_name
|
||||
)
|
||||
)
|
||||
if confirm:
|
||||
await self.on_transaction_dict(tx)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(tx)
|
||||
return tx
|
||||
|
||||
|
||||
class ClaimSearchCommand(ClaimTestCase):
|
||||
|
||||
async def create_channel(self):
|
||||
self.channel = await self.channel_create('@abc', '1.0')
|
||||
|
@ -170,6 +219,37 @@ class ClaimSearchCommand(CommandTestCase):
|
|||
|
||||
await self.assertFindsClaims(claims, order_by=["^name"])
|
||||
|
||||
async def test_claim_type_and_media_type_search(self):
|
||||
# create an invalid/unknown claim
|
||||
address = await self.account.receiving.get_or_create_usable_address()
|
||||
tx = await Transaction.claim_create(
|
||||
'unknown', b'{"sources":{"lbry_sd_hash":""}}', 1, address, [self.account], self.account)
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
octet = await self.stream_create()
|
||||
video = await self.video_stream_create()
|
||||
image = await self.image_stream_create()
|
||||
channel = await self.channel_create()
|
||||
unknown = self.sout(tx)
|
||||
|
||||
# claim_type
|
||||
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
|
||||
await self.assertFindsClaims([channel], claim_type='channel')
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
|
||||
await self.assertFindsClaims([video], stream_types=['video'])
|
||||
await self.assertFindsClaims([image], stream_types=['image'])
|
||||
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
|
||||
await self.assertFindsClaims([video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image], media_types=['image/png'])
|
||||
await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
|
||||
|
||||
|
||||
class ChannelCommands(CommandTestCase):
|
||||
|
||||
|
@ -370,20 +450,7 @@ class ChannelCommands(CommandTestCase):
|
|||
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
|
||||
|
||||
|
||||
class StreamCommands(CommandTestCase):
|
||||
|
||||
files_directory = os.path.join(os.path.dirname(__file__), 'files')
|
||||
video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
|
||||
video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
|
||||
|
||||
def setUp(self):
|
||||
if not os.path.exists(self.video_file_name):
|
||||
if not os.path.exists(self.files_directory):
|
||||
os.mkdir(self.files_directory)
|
||||
log.info(f'downloading test video from {self.video_file_name}')
|
||||
with urlopen(self.video_file_url) as response,\
|
||||
open(self.video_file_name, 'wb') as video_file:
|
||||
video_file.write(response.read())
|
||||
class StreamCommands(ClaimTestCase):
|
||||
|
||||
async def test_create_stream_names(self):
|
||||
# claim new name
|
||||
|
@ -662,44 +729,26 @@ class StreamCommands(CommandTestCase):
|
|||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 1)
|
||||
|
||||
async def test_automatic_type_and_metadata_detection_for_image(self):
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as file:
|
||||
file.write(unhexlify(
|
||||
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
|
||||
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
|
||||
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
|
||||
b'9454e44ae426082'
|
||||
))
|
||||
file.flush()
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
'blank-image', '1.0', file_path=file.name
|
||||
)
|
||||
)
|
||||
txo = tx['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value'], {
|
||||
'source': {
|
||||
'size': '99',
|
||||
'name': os.path.basename(file.name),
|
||||
'media_type': 'image/png',
|
||||
'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
|
||||
'sd_hash': txo['value']['source']['sd_hash'],
|
||||
},
|
||||
'stream_type': 'image',
|
||||
'image': {
|
||||
'width': 5,
|
||||
'height': 7
|
||||
}
|
||||
txo = (await self.image_stream_create())['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value'], {
|
||||
'source': {
|
||||
'size': '99',
|
||||
'name': txo['value']['source']['name'],
|
||||
'media_type': 'image/png',
|
||||
'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
|
||||
'sd_hash': txo['value']['source']['sd_hash'],
|
||||
},
|
||||
'stream_type': 'image',
|
||||
'image': {
|
||||
'width': 5,
|
||||
'height': 7
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
async def test_automatic_type_and_metadata_detection_for_video(self):
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
'chrome', '1.0', file_path=self.video_file_name
|
||||
)
|
||||
)
|
||||
txo = tx['outputs'][0]
|
||||
txo = (await self.video_stream_create())['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value'], {
|
||||
'source': {
|
||||
|
|
Loading…
Reference in a new issue