Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

claim_type/stream_type/media_type claim search integration test

Browse source
This commit is contained in:
Lex Berezhny 2019-06-04 00:10:59 -04:00
parent 9dbf47916b
commit 8d2c9e5785
3 changed files with 150 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
lbrynet/extras/daemon/Daemon.py
View file

@ -1705,6 +1705,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--support_amount=<support_amount>] [--trending_group=<trending_group>]
[--trending_mixed=<trending_mixed>] [--trending_local=<trending_local>]
[--trending_global=<trending_global]
[--claim_type=<claim_type>] [--stream_types=<stream_types>...] [--media_types=<media_types>...]
[--any_tags=<any_tags>...] [--all_tags=<all_tags>...] [--not_tags=<not_tags>...]
[--any_languages=<any_languages>...] [--all_languages=<all_languages>...]
[--not_languages=<not_languages>...]
@ -1772,6 +1773,9 @@ class Daemon(metaclass=JSONRPCServerType):
--trending_global=<trending_global>: (int) trending value calculated relative to all
trending content globally (supports
equality constraints)
--claim_type=<claim_type> : (str) filter by 'channel', 'stream' or 'unknown'
--stream_types=<stream_types> : (list) filter by 'video', 'image', 'document', etc
--media_types=<media_types> : (list) filter by 'video/mp4', 'image/png', etc
--any_tags=<any_tags> : (list) find claims containing any of the tags
--all_tags=<all_tags> : (list) find claims containing every tag
--not_tags=<not_tags> : (list) find claims not containing any of these tags

55
lbrynet/wallet/server/db.py
View file

@ -10,6 +10,7 @@ from torba.server.util import class_logger
from torba.client.basedatabase import query, constraints_to_sql
from lbrynet.schema.url import URL, normalize_name
from lbrynet.schema.mime_types import guess_stream_type
from lbrynet.wallet.ledger import MainNetLedger, RegTestLedger
from lbrynet.wallet.transaction import Transaction, Output
from lbrynet.wallet.server.canonical import register_canonical_functions
@ -19,6 +20,18 @@ from lbrynet.wallet.server.trending import (
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
CLAIM_TYPES = {
'stream': 1,
'channel': 2,
}
STREAM_TYPES = {
'video': 1,
'audio': 2,
'image': 3,
'document': 4,
'binary': 5,
'model': 6
}
def _apply_constraints_for_array_attributes(constraints, attr):
@ -87,8 +100,13 @@ class SQLDB:
short_url text not null, -- normalized#shortest-unique-claim_id
canonical_url text, -- channel's-short_url/normalized#shortest-unique-claim_id-within-channel
claim_type integer,
-- streams
stream_type text,
media_type text,
-- claims which are channels
is_channel bool not null,
public_key_bytes bytes,
public_key_hash bytes,
claims_in_channel integer,
@ -119,6 +137,10 @@ class SQLDB:
create index if not exists claim_activation_height_idx on claim (activation_height);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_effective_amount_idx on claim (effective_amount);
create index if not exists claim_trending_group_idx on claim (trending_group);
create index if not exists claim_trending_mixed_idx on claim (trending_mixed);
@ -242,9 +264,11 @@ class SQLDB:
'txo_hash': sqlite3.Binary(txo.ref.hash),
'tx_position': tx.position,
'amount': txo.amount,
'is_channel': False,
'timestamp': header['timestamp'],
'height': tx.height,
'claim_type': None,
'stream_type': None,
'media_type': None,
'release_time': None,
}
claims.append(claim_record)
@ -256,10 +280,13 @@ class SQLDB:
continue
if claim.is_stream:
claim_record['claim_type'] = CLAIM_TYPES['stream']
claim_record['media_type'] = claim.stream.source.media_type
claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(claim_record['media_type'])]
if claim.stream.release_time:
claim_record['release_time'] = claim.stream.release_time
elif claim.is_channel:
claim_record['is_channel'] = True
claim_record['claim_type'] = CLAIM_TYPES['channel']
for tag in claim.message.tags:
tags.append((tag, claim_hash, tx.height))
@ -280,11 +307,11 @@ class SQLDB:
self.db.executemany("""
INSERT INTO claim (
claim_hash, claim_id, claim_name, normalized, txo_hash, tx_position, amount,
is_channel, timestamp, creation_timestamp, height, creation_height,
release_time, activation_height, expiration_height, short_url)
claim_type, media_type, stream_type, timestamp, creation_timestamp, height,
creation_height, release_time, activation_height, expiration_height, short_url)
VALUES (
:claim_hash, :claim_id, :claim_name, :normalized, :txo_hash, :tx_position, :amount,
:is_channel, :timestamp, :timestamp, :height, :height,
:claim_type, :media_type, :stream_type, :timestamp, :timestamp, :height, :height,
CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE :timestamp END,
CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END,
CASE WHEN :height >= 262974 THEN :height+2102400 ELSE :height+262974 END,
@ -299,7 +326,9 @@ class SQLDB:
if claims:
self.db.executemany("""
UPDATE claim SET
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height, timestamp=:timestamp,
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height,
claim_type=:claim_type, media_type=:media_type, stream_type=:stream_type,
timestamp=:timestamp,
release_time=CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE release_time END
WHERE claim_hash=:claim_hash;
""", claims)
@ -720,6 +749,15 @@ class SQLDB:
tx_hash + struct.pack('<I', nout)
)
if 'claim_type' in constraints:
constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
if 'stream_types' in constraints:
constraints['claim.stream_type__in'] = [
STREAM_TYPES[stream_type] for stream_type in constraints.pop('stream_types')
]
if 'media_types' in constraints:
constraints['claim.media_type__in'] = constraints.pop('media_types')
_apply_constraints_for_array_attributes(constraints, 'tag')
_apply_constraints_for_array_attributes(constraints, 'language')
_apply_constraints_for_array_attributes(constraints, 'location')
@ -750,7 +788,7 @@ class SQLDB:
"""
claimtrie.claim_hash as is_controlling,
claim.claim_hash, claim.txo_hash,
claim.is_channel, claim.claims_in_channel,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
@ -773,6 +811,7 @@ class SQLDB:
SEARCH_PARAMS = {
'name', 'claim_id', 'txid', 'nout', 'channel', 'channel_ids', 'public_key_id',
'claim_type', 'stream_types', 'media_types',
'any_tags', 'all_tags', 'not_tags',
'any_locations', 'all_locations', 'not_locations',
'any_languages', 'all_languages', 'not_languages',

149
tests/integration/test_claim_commands.py
View file

@ -8,12 +8,61 @@ from urllib.request import urlopen
from torba.client.errors import InsufficientFundsError
from lbrynet.testcase import CommandTestCase
from lbrynet.wallet.transaction import Transaction
log = logging.getLogger(__name__)
class ClaimSearchCommand(CommandTestCase):
class ClaimTestCase(CommandTestCase):
files_directory = os.path.join(os.path.dirname(__file__), 'files')
video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
def setUp(self):
if not os.path.exists(self.video_file_name):
if not os.path.exists(self.files_directory):
os.mkdir(self.files_directory)
log.info(f'downloading test video from {self.video_file_name}')
with urlopen(self.video_file_url) as response, \
open(self.video_file_name, 'wb') as video_file:
video_file.write(response.read())
async def image_stream_create(self, name='blank-image', bid='1.0', confirm=True):
with tempfile.NamedTemporaryFile(suffix='.png') as file:
file.write(unhexlify(
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
b'9454e44ae426082'
))
file.flush()
tx = await self.out(
self.daemon.jsonrpc_stream_create(
name, bid, file_path=file.name
)
)
if confirm:
await self.on_transaction_dict(tx)
await self.generate(1)
await self.on_transaction_dict(tx)
return tx
async def video_stream_create(self, name='chrome', bid='1.0', confirm=True):
tx = await self.out(
self.daemon.jsonrpc_stream_create(
name, bid, file_path=self.video_file_name
)
)
if confirm:
await self.on_transaction_dict(tx)
await self.generate(1)
await self.on_transaction_dict(tx)
return tx
class ClaimSearchCommand(ClaimTestCase):
async def create_channel(self):
self.channel = await self.channel_create('@abc', '1.0')
@ -170,6 +219,37 @@ class ClaimSearchCommand(CommandTestCase):
await self.assertFindsClaims(claims, order_by=["^name"])
async def test_claim_type_and_media_type_search(self):
# create an invalid/unknown claim
address = await self.account.receiving.get_or_create_usable_address()
tx = await Transaction.claim_create(
'unknown', b'{"sources":{"lbry_sd_hash":""}}', 1, address, [self.account], self.account)
await tx.sign([self.account])
await self.broadcast(tx)
await self.confirm_tx(tx.id)
octet = await self.stream_create()
video = await self.video_stream_create()
image = await self.image_stream_create()
channel = await self.channel_create()
unknown = self.sout(tx)
# claim_type
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
await self.assertFindsClaims([channel], claim_type='channel')
# stream_type
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
await self.assertFindsClaims([video], stream_types=['video'])
await self.assertFindsClaims([image], stream_types=['image'])
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
# stream_type
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
await self.assertFindsClaims([video], media_types=['video/mp4'])
await self.assertFindsClaims([image], media_types=['image/png'])
await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
class ChannelCommands(CommandTestCase):
@ -370,20 +450,7 @@ class ChannelCommands(CommandTestCase):
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
class StreamCommands(CommandTestCase):
files_directory = os.path.join(os.path.dirname(__file__), 'files')
video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
def setUp(self):
if not os.path.exists(self.video_file_name):
if not os.path.exists(self.files_directory):
os.mkdir(self.files_directory)
log.info(f'downloading test video from {self.video_file_name}')
with urlopen(self.video_file_url) as response,\
open(self.video_file_name, 'wb') as video_file:
video_file.write(response.read())
class StreamCommands(ClaimTestCase):
async def test_create_stream_names(self):
# claim new name
@ -662,44 +729,26 @@ class StreamCommands(CommandTestCase):
self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 1)
async def test_automatic_type_and_metadata_detection_for_image(self):
with tempfile.NamedTemporaryFile(suffix='.png') as file:
file.write(unhexlify(
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
b'9454e44ae426082'
))
file.flush()
tx = await self.out(
self.daemon.jsonrpc_stream_create(
'blank-image', '1.0', file_path=file.name
)
)
txo = tx['outputs'][0]
self.assertEqual(
txo['value'], {
'source': {
'size': '99',
'name': os.path.basename(file.name),
'media_type': 'image/png',
'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
'sd_hash': txo['value']['source']['sd_hash'],
},
'stream_type': 'image',
'image': {
'width': 5,
'height': 7
}
txo = (await self.image_stream_create())['outputs'][0]
self.assertEqual(
txo['value'], {
'source': {
'size': '99',
'name': txo['value']['source']['name'],
'media_type': 'image/png',
'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
'sd_hash': txo['value']['source']['sd_hash'],
},
'stream_type': 'image',
'image': {
'width': 5,
'height': 7
}
)
}
)
async def test_automatic_type_and_metadata_detection_for_video(self):
tx = await self.out(
self.daemon.jsonrpc_stream_create(
'chrome', '1.0', file_path=self.video_file_name
)
)
txo = tx['outputs'][0]
txo = (await self.video_stream_create())['outputs'][0]
self.assertEqual(
txo['value'], {
'source': {