LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

bug fixes

Browse source
This commit is contained in:
Lex Berezhny 2019-05-27 22:20:21 -04:00
parent 1848462ad9
commit 9004f240e6
7 changed files with 185 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
lbrynet/extras/daemon/Daemon.py
View file

@ -114,6 +114,7 @@ def encode_pagination_doc(items):
"page": "Page number of the current items.", "page": "Page number of the current items.",
"page_size": "Number of items to show on a page.", "page_size": "Number of items to show on a page.",
"total_pages": "Total number of pages.", "total_pages": "Total number of pages.",
"total_items": "Total number of items.",
"items": [items], "items": [items],
} }
@ -125,9 +126,11 @@ async def maybe_paginate(get_records: Callable, get_record_count: Callable,
"offset": page_size * (page-1), "offset": page_size * (page-1),
"limit": page_size "limit": page_size
}) })
total_items = await get_record_count(**constraints)
return { return {
"items": await get_records(**constraints), "items": await get_records(**constraints),
"total_pages": int(((await get_record_count(**constraints)) + (page_size-1)) / page_size), "total_pages": int((total_items + (page_size-1)) / page_size),
"total_items": total_items,
"page": page, "page_size": page_size "page": page, "page_size": page_size
} }
return await get_records(**constraints) return await get_records(**constraints)
@ -1684,47 +1687,53 @@ class Daemon(metaclass=JSONRPCServerType):
""" """
Search for stream and channel claims on the blockchain. Search for stream and channel claims on the blockchain.
Use --channel_id=<channel_id> to list all stream claims in a channel.
Arguments marked with "supports equality constraints" allow prepending the Arguments marked with "supports equality constraints" allow prepending the
value with an equality constraint such as '>', '>=', '<' and '<=' value with an equality constraint such as '>', '>=', '<' and '<='
eg. --height=">400000" would limit results to only claims above 400k block height. eg. --height=">400000" would limit results to only claims above 400k block height.
Usage: Usage:
claim_search [<name> | --name=<name>] [--claim_id=<claim_id>] [--txid=<txid> --nout=<nout>] claim_search [<name> | --name=<name>] [--claim_id=<claim_id>] [--txid=<txid>] [--nout=<nout>]
[--channel_id=<channel_id>] [--channel_name=<channel_name>] [--is_controlling] [--channel=<channel> | --channel_ids=<channel_ids>...] [--is_channel_signature_valid]
[--order_by=<order_by>...] [--is_controlling] [--release_time=<release_time>]
[--height=<height>] [--publish_time=<publish_time>] [--release_time=<release_time>] [--timestamp=<timestamp>] [--creation_timestamp=<creation_timestamp>]
[--height=<height>] [--creation_height=<creation_height>]
[--activation_height=<activation_height>] [--expiration_height=<expiration_height>]
[--amount=<amount>] [--effective_amount=<effective_amount>] [--amount=<amount>] [--effective_amount=<effective_amount>]
[--support_amount=<support_amount>] [--trending_group=<trending_group>] [--support_amount=<support_amount>] [--trending_group=<trending_group>]
[--trending_mixed=<trending_mixed>] [--trending_local=<trending_local>] [--trending_mixed=<trending_mixed>] [--trending_local=<trending_local>]
[--trending_global=<trending_global] [--activation_height=<activation_height>] [--trending_global=<trending_global]
[--any_tags=<any_tags>...] [--all_tags=<all_tags>...] [--not_tags=<not_tags>...] [--any_tags=<any_tags>...] [--all_tags=<all_tags>...] [--not_tags=<not_tags>...]
[--any_languages=<any_languages>...] [--all_languages=<all_languages>...] [--any_languages=<any_languages>...] [--all_languages=<all_languages>...]
[--not_languages=<not_languages>...] [--not_languages=<not_languages>...]
[--any_locations=<any_locations>...] [--all_locations=<all_locations>...] [--any_locations=<any_locations>...] [--all_locations=<all_locations>...]
[--not_locations=<not_locations>...] [--not_locations=<not_locations>...]
[--page=<page>] [--page_size=<page_size>] [--order_by=<order_by>...] [--page=<page>] [--page_size=<page_size>]
Options: Options:
--name=<name> : (str) find claims with this name --name=<name> : (str) claim name (normalized)
--claim_id=<claim_id> : (str) find a claim with this claim_id --claim_id=<claim_id> : (str) full or partial claim id
--txid=<txid> : (str) find a claim with this txid:nout --txid=<txid> : (str) transaction id
--nout=<nout> : (str) find a claim with this txid:nout --nout=<nout> : (str) position in the transaction
--channel_id=<channel_id> : (str) limit search to specific channel claim id (returns stream claims) --channel=<channel> : (str) claims signed by this channel (argument is
--channel_name=<channel_name> : (str) limit search to specific channel name (returns stream claims) a URL which automatically gets resolved),
--is_controlling : (bool) limit to controlling claims for their respective name see --channel_ids if you need to filter by
--order_by=<order_by> : (str) field to order by, default is descending order, to do an multiple channels at the same time,
ascending order prepend ^ to the field name, eg. '^amount' includes claims with invalid signatures,
available fields: 'name', 'height', 'release_time', use in conjunction with --is_channel_signature_valid
'publish_time', 'amount', 'effective_amount', --channel_ids=<channel_ids> : (str) claims signed by any of these channels
'support_amount', 'trending_group', 'trending_mixed', (arguments must be claim ids of the channels),
'trending_local', 'trending_global', 'activation_height' includes claims with invalid signatures,
--height=<height> : (int) limit by block height (supports equality constraints) use in conjunction with --is_channel_signature_valid
--activation_height=<activation_height>: (int) height at which claim starts competing for name --is_channel_signature_valid : (bool) only return claims with valid channel signatures
(supports equality constraints) --is_controlling : (bool) only return winning claims of their respective name
--publish_time=<publish_time> : (int) limit by UTC timestamp of containing block (supports --height=<height> : (int) last updated block height (supports equality constraints)
equality constraints) --timestamp=<timestamp> : (int) last updated timestamp (supports equality constraints)
--creation_height=<creation_height> : (int) created at block height (supports equality constraints)
--creation_timestamp=<creation_timestamp>: (int) created at timestamp (supports equality constraints)
--activation_height=<activation_height> : (int) height at which claim starts competing for name
(supports equality constraints)
--expiration_height=<expiration_height> : (int) height at which claim will expire
(supports equality constraints)
--release_time=<release_time> : (int) limit to claims self-described as having been --release_time=<release_time> : (int) limit to claims self-described as having been
released to the public on or after this UTC released to the public on or after this UTC
timestamp, when claim does not provide timestamp, when claim does not provide
@ -1767,6 +1776,12 @@ class Daemon(metaclass=JSONRPCServerType):
--not_locations=<not_locations> : (list) find claims not containing any of these locations --not_locations=<not_locations> : (list) find claims not containing any of these locations
--page=<page> : (int) page to return during paginating --page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination --page_size=<page_size> : (int) number of items on page during pagination
--order_by=<order_by> : (str) field to order by, default is descending order, to do an
ascending order prepend ^ to the field name, eg. '^amount'
available fields: 'name', 'height', 'release_time',
'publish_time', 'amount', 'effective_amount',
'support_amount', 'trending_group', 'trending_mixed',
'trending_local', 'trending_global', 'activation_height'
Returns: {Paginated[Output]} Returns: {Paginated[Output]}
""" """
@ -1775,7 +1790,8 @@ class Daemon(metaclass=JSONRPCServerType):
txos, offset, total = await self.ledger.claim_search(**kwargs) txos, offset, total = await self.ledger.claim_search(**kwargs)
return { return {
"items": txos, "page": page_num, "page_size": page_size, "items": txos, "page": page_num, "page_size": page_size,
"total_pages": int((total + (page_size-1)) / page_size) "total_pages": int((total + (page_size-1)) / page_size),
"total_items": total
} }
CHANNEL_DOC = """ CHANNEL_DOC = """

4
lbrynet/extras/daemon/json_response_encoder.py
View file

@ -169,7 +169,7 @@ class JSONResponseEncoder(JSONEncoder):
if txo.script.is_claim_involved: if txo.script.is_claim_involved:
output.update({ output.update({
'name': txo.claim_name, 'name': txo.claim_name,
'normalized': txo.normalized_name, 'normalized_name': txo.normalized_name,
'claim_id': txo.claim_id, 'claim_id': txo.claim_id,
'permanent_url': txo.permanent_url, 'permanent_url': txo.permanent_url,
'meta': self.encode_claim_meta(txo.meta) 'meta': self.encode_claim_meta(txo.meta)
@ -199,6 +199,8 @@ class JSONResponseEncoder(JSONEncoder):
if key.endswith('_amount'): if key.endswith('_amount'):
if isinstance(value, int): if isinstance(value, int):
meta[key] = dewies_to_lbc(value) meta[key] = dewies_to_lbc(value)
if meta.get('creation_height', 0) > 0:
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
return meta return meta
def encode_input(self, txi): def encode_input(self, txi):

11
lbrynet/schema/result.py
View file

@ -4,8 +4,6 @@ from typing import List
from binascii import hexlify from binascii import hexlify
from itertools import chain from itertools import chain
from google.protobuf.message import DecodeError
from lbrynet.schema.types.v2.result_pb2 import Outputs as OutputsMessage from lbrynet.schema.types.v2.result_pb2 import Outputs as OutputsMessage
@ -36,6 +34,7 @@ class Outputs:
'short_url': f'lbry://{claim.short_url}', 'short_url': f'lbry://{claim.short_url}',
'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}', 'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}',
'is_controlling': claim.is_controlling, 'is_controlling': claim.is_controlling,
'creation_height': claim.creation_height,
'activation_height': claim.activation_height, 'activation_height': claim.activation_height,
'expiration_height': claim.expiration_height, 'expiration_height': claim.expiration_height,
'effective_amount': claim.effective_amount, 'effective_amount': claim.effective_amount,
@ -47,8 +46,11 @@ class Outputs:
} }
if claim.HasField('channel'): if claim.HasField('channel'):
txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout] txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
if claim.claims_in_channel is not None: try:
txo.meta['claims_in_channel'] = claim.claims_in_channel if txo.claim.is_channel:
txo.meta['claims_in_channel'] = claim.claims_in_channel
except:
pass
return txo return txo
@classmethod @classmethod
@ -97,6 +99,7 @@ class Outputs:
if txo['canonical_url'] is not None: if txo['canonical_url'] is not None:
txo_message.claim.canonical_url = txo['canonical_url'] txo_message.claim.canonical_url = txo['canonical_url']
txo_message.claim.is_controlling = bool(txo['is_controlling']) txo_message.claim.is_controlling = bool(txo['is_controlling'])
txo_message.claim.creation_height = txo['creation_height']
txo_message.claim.activation_height = txo['activation_height'] txo_message.claim.activation_height = txo['activation_height']
txo_message.claim.expiration_height = txo['expiration_height'] txo_message.claim.expiration_height = txo['expiration_height']
if txo['claims_in_channel'] is not None: if txo['claims_in_channel'] is not None:

37
lbrynet/schema/types/v2/result_pb2.py
View file

@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
package='pb', package='pb',
syntax='proto3', syntax='proto3',
serialized_options=None, serialized_options=None,
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"b\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xce\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x02 \x01(\t\x12\x15\n\rcanonical_url\x18\x03 \x01(\t\x12\x16\n\x0eis_controlling\x18\x04 \x01(\x08\x12\x19\n\x11\x61\x63tivation_height\x18\x05 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\x06 \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\x07 \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\n \x01(\x04\x12\x16\n\x0esupport_amount\x18\x0b \x01(\x04\x12\x16\n\x0etrending_group\x18\x0c \x01(\r\x12\x16\n\x0etrending_mixed\x18\r \x01(\x02\x12\x16\n\x0etrending_local\x18\x0e \x01(\x02\x12\x17\n\x0ftrending_global\x18\x0f \x01(\x02\"i\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\"4\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x62\x06proto3') serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"b\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xe7\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x02 \x01(\t\x12\x15\n\rcanonical_url\x18\x03 \x01(\t\x12\x16\n\x0eis_controlling\x18\x04 \x01(\x08\x12\x17\n\x0f\x63reation_height\x18\x05 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x06 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\x07 \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\x08 \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\n \x01(\x04\x12\x16\n\x0esupport_amount\x18\x0b \x01(\x04\x12\x16\n\x0etrending_group\x18\x0c \x01(\r\x12\x16\n\x0etrending_mixed\x18\r \x01(\x02\x12\x16\n\x0etrending_local\x18\x0e \x01(\x02\x12\x17\n\x0ftrending_global\x18\x0f \x01(\x02\"i\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\"4\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x62\x06proto3')
) )
@ -45,8 +45,8 @@ _ERROR_CODE = _descriptor.EnumDescriptor(
], ],
containing_type=None, containing_type=None,
serialized_options=None, serialized_options=None,
serialized_start=635, serialized_start=660,
serialized_end=687, serialized_end=712,
) )
_sym_db.RegisterEnumDescriptor(_ERROR_CODE) _sym_db.RegisterEnumDescriptor(_ERROR_CODE)
@ -201,63 +201,70 @@ _CLAIMMETA = _descriptor.Descriptor(
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR), serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='activation_height', full_name='pb.ClaimMeta.activation_height', index=4, name='creation_height', full_name='pb.ClaimMeta.creation_height', index=4,
number=5, type=13, cpp_type=3, label=1, number=5, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR), serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=5, name='activation_height', full_name='pb.ClaimMeta.activation_height', index=5,
number=6, type=13, cpp_type=3, label=1, number=6, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR), serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=6, name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=6,
number=7, type=13, cpp_type=3, label=1, number=7, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR), serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=7, name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=7,
number=8, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=8,
number=10, type=4, cpp_type=4, label=1, number=10, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR), serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='support_amount', full_name='pb.ClaimMeta.support_amount', index=8, name='support_amount', full_name='pb.ClaimMeta.support_amount', index=9,
number=11, type=4, cpp_type=4, label=1, number=11, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR), serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='trending_group', full_name='pb.ClaimMeta.trending_group', index=9, name='trending_group', full_name='pb.ClaimMeta.trending_group', index=10,
number=12, type=13, cpp_type=3, label=1, number=12, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR), serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='trending_mixed', full_name='pb.ClaimMeta.trending_mixed', index=10, name='trending_mixed', full_name='pb.ClaimMeta.trending_mixed', index=11,
number=13, type=2, cpp_type=6, label=1, number=13, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0), has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR), serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='trending_local', full_name='pb.ClaimMeta.trending_local', index=11, name='trending_local', full_name='pb.ClaimMeta.trending_local', index=12,
number=14, type=2, cpp_type=6, label=1, number=14, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0), has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR), serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='trending_global', full_name='pb.ClaimMeta.trending_global', index=12, name='trending_global', full_name='pb.ClaimMeta.trending_global', index=13,
number=15, type=2, cpp_type=6, label=1, number=15, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0), has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
@ -276,7 +283,7 @@ _CLAIMMETA = _descriptor.Descriptor(
oneofs=[ oneofs=[
], ],
serialized_start=246, serialized_start=246,
serialized_end=580, serialized_end=605,
) )
@ -314,8 +321,8 @@ _ERROR = _descriptor.Descriptor(
extension_ranges=[], extension_ranges=[],
oneofs=[ oneofs=[
], ],
serialized_start=582, serialized_start=607,
serialized_end=687, serialized_end=712,
) )
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT _OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT

153
lbrynet/wallet/server/db.py
View file

@ -109,7 +109,7 @@ class SQLDB:
create index if not exists claim_id_idx on claim (claim_id); create index if not exists claim_id_idx on claim (claim_id);
create index if not exists claim_normalized_idx on claim (normalized); create index if not exists claim_normalized_idx on claim (normalized);
create index if not exists claim_search_idx on claim (normalized, claim_id); create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_txo_hash_idx on claim (txo_hash); create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_channel_hash_idx on claim (channel_hash); create index if not exists claim_channel_hash_idx on claim (channel_hash);
create index if not exists claim_release_time_idx on claim (release_time); create index if not exists claim_release_time_idx on claim (release_time);
@ -352,8 +352,8 @@ class SQLDB:
'support', {'txo_hash__in': [sqlite3.Binary(txo_hash) for txo_hash in txo_hashes]} 'support', {'txo_hash__in': [sqlite3.Binary(txo_hash) for txo_hash in txo_hashes]}
)) ))
def validate_channel_signatures(self, height, new_claims, updated_claims): def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims):
if not new_claims and not updated_claims: if not new_claims and not updated_claims and not spent_claims:
return return
channels, new_channel_keys, signables = {}, {}, {} channels, new_channel_keys, signables = {}, {}, {}
@ -456,6 +456,14 @@ class SQLDB:
WHERE claim_hash=:claim_hash; WHERE claim_hash=:claim_hash;
""", claim_updates) """, claim_updates)
if spent_claims:
self.execute(
f"""
UPDATE claim SET is_channel_signature_valid=0, channel_join=NULL, canonical_url=NULL
WHERE channel_hash IN ({','.join('?' for _ in spent_claims)})
""", [sqlite3.Binary(cid) for cid in spent_claims]
)
if channels: if channels:
self.db.executemany( self.db.executemany(
"UPDATE claim SET public_key_bytes=:public_key_bytes WHERE claim_hash=:claim_hash", [{ "UPDATE claim SET public_key_bytes=:public_key_bytes WHERE claim_hash=:claim_hash", [{
@ -620,7 +628,7 @@ class SQLDB:
r(self.delete_supports, delete_support_txo_hashes) r(self.delete_supports, delete_support_txo_hashes)
r(self.insert_claims, insert_claims, header) r(self.insert_claims, insert_claims, header)
r(self.update_claims, update_claims, header) r(self.update_claims, update_claims, header)
r(self.validate_channel_signatures, height, insert_claims, update_claims) r(self.validate_channel_signatures, height, insert_claims, update_claims, delete_claim_hashes)
r(self.insert_supports, insert_supports) r(self.insert_supports, insert_supports)
r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True) r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True)
r(calculate_trending, self.db, height, self.main.first_sync, daemon_height) r(calculate_trending, self.db, height, self.main.first_sync, daemon_height)
@ -670,21 +678,25 @@ class SQLDB:
constraints['claim.claim_id'] = claim_id constraints['claim.claim_id'] = claim_id
else: else:
constraints['claim.claim_id__like'] = f'{claim_id[:40]}%' constraints['claim.claim_id__like'] = f'{claim_id[:40]}%'
if 'name' in constraints: if 'name' in constraints:
constraints['claim.normalized'] = normalize_name(constraints.pop('name')) constraints['claim.normalized'] = normalize_name(constraints.pop('name'))
if 'channel' in constraints: if 'channel' in constraints:
url = URL.parse(constraints.pop('channel')) channel_url = constraints.pop('channel')
if url.channel.claim_id: match = self._resolve_one(channel_url)
constraints['channel_id'] = url.channel.claim_id if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else: else:
constraints['channel_name'] = url.channel.name raise LookupError(f'Could not resolve channel "{channel_url}".')
if 'channel_id' in constraints:
constraints['channel_hash'] = unhexlify(constraints.pop('channel_id'))[::-1]
if 'channel_hash' in constraints: if 'channel_hash' in constraints:
constraints['channel.claim_hash'] = sqlite3.Binary(constraints.pop('channel_hash')) constraints['claim.channel_hash'] = sqlite3.Binary(constraints.pop('channel_hash'))
if 'channel_name' in constraints: if 'channel_ids' in constraints:
constraints['channel.normalized'] = normalize_name(constraints.pop('channel_name')) channel_ids = constraints.pop('channel_ids')
if channel_ids:
constraints['claim.channel_hash__in'] = [
sqlite3.Binary(unhexlify(cid)[::-1]) for cid in channel_ids
]
if 'txid' in constraints: if 'txid' in constraints:
tx_hash = unhexlify(constraints.pop('txid'))[::-1] tx_hash = unhexlify(constraints.pop('txid'))[::-1]
@ -697,23 +709,17 @@ class SQLDB:
_apply_constraints_for_array_attributes(constraints, 'language') _apply_constraints_for_array_attributes(constraints, 'language')
_apply_constraints_for_array_attributes(constraints, 'location') _apply_constraints_for_array_attributes(constraints, 'location')
try: sql, values = query(
return self.db.execute(*query( f"""
f"""
SELECT {cols} FROM claim
LEFT JOIN claimtrie USING (claim_hash)
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
""", **constraints
)).fetchall()
except:
self.logger.exception('Failed to execute claim search query:')
print(query(
f"""
SELECT {cols} FROM claim SELECT {cols} FROM claim
LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claimtrie USING (claim_hash)
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash) LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
""", **constraints """, **constraints
)) )
try:
return self.db.execute(sql, values).fetchall()
except:
self.logger.exception(f'Failed to execute claim search query: {sql}')
raise raise
def get_claims_count(self, **constraints): def get_claims_count(self, **constraints):
@ -727,8 +733,9 @@ class SQLDB:
return self.get_claims( return self.get_claims(
""" """
claimtrie.claim_hash as is_controlling, claimtrie.claim_hash as is_controlling,
claim.claim_hash, claim.txo_hash, claim.height, claim.claim_hash, claim.txo_hash,
claim.is_channel, claim.claims_in_channel, claim.is_channel, claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height, claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount, claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed, claim.trending_group, claim.trending_mixed,
@ -740,16 +747,16 @@ class SQLDB:
) )
INTEGER_PARAMS = { INTEGER_PARAMS = {
'height', 'creation_height', 'activation_height', 'tx_position', 'height', 'creation_height', 'activation_height', 'expiration_height',
'release_time', 'timestamp', 'is_channel_signature_valid', 'channel_join', 'timestamp', 'creation_timestamp', 'release_time',
'tx_position', 'channel_join', 'is_channel_signature_valid',
'amount', 'effective_amount', 'support_amount', 'amount', 'effective_amount', 'support_amount',
'trending_group', 'trending_mixed', 'trending_group', 'trending_mixed',
'trending_local', 'trending_global', 'trending_local', 'trending_global',
} }
SEARCH_PARAMS = { SEARCH_PARAMS = {
'name', 'claim_id', 'txid', 'nout', 'name', 'claim_id', 'txid', 'nout', 'channel', 'channel_ids',
'channel', 'channel_id', 'channel_name',
'any_tags', 'all_tags', 'not_tags', 'any_tags', 'all_tags', 'not_tags',
'any_locations', 'all_locations', 'not_locations', 'any_locations', 'all_locations', 'not_locations',
'any_languages', 'all_languages', 'not_languages', 'any_languages', 'all_languages', 'not_languages',
@ -775,50 +782,54 @@ class SQLDB:
extra_txo_rows = self._search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}) extra_txo_rows = self._search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]})
return txo_rows, extra_txo_rows, constraints['offset'], total return txo_rows, extra_txo_rows, constraints['offset'], total
def _resolve_one(self, raw_url):
try:
url = URL.parse(raw_url)
except ValueError as e:
return e
channel = None
if url.has_channel:
query = url.channel.to_dict()
if set(query) == {'name'}:
query['is_controlling'] = True
else:
query['order_by'] = ['^height']
matches = self._search(**query, limit=1)
if matches:
channel = matches[0]
else:
return LookupError(f'Could not find channel in "{raw_url}".')
if url.has_stream:
query = url.stream.to_dict()
if channel is not None:
if set(query) == {'name'}:
# temporarily emulate is_controlling for claims in channel
query['order_by'] = ['effective_amount']
else:
query['order_by'] = ['^channel_join']
query['channel_hash'] = channel['claim_hash']
query['is_channel_signature_valid'] = 1
elif set(query) == {'name'}:
query['is_controlling'] = 1
matches = self._search(**query, limit=1)
if matches:
return matches[0]
else:
return LookupError(f'Could not find stream in "{raw_url}".')
return channel
def resolve(self, urls) -> Tuple[List, List]: def resolve(self, urls) -> Tuple[List, List]:
result = [] result = []
channel_hashes = set() channel_hashes = set()
for raw_url in urls: for raw_url in urls:
try: match = self._resolve_one(raw_url)
url = URL.parse(raw_url) result.append(match)
except ValueError as e: if isinstance(match, sqlite3.Row) and match['channel_hash']:
result.append(e) channel_hashes.add(match['channel_hash'])
continue
channel = None
if url.has_channel:
query = url.channel.to_dict()
if set(query) == {'name'}:
query['is_controlling'] = True
else:
query['order_by'] = ['^height']
matches = self._search(**query, limit=1)
if matches:
channel = matches[0]
else:
result.append(LookupError(f'Could not find channel in "{raw_url}".'))
continue
if url.has_stream:
query = url.stream.to_dict()
if channel is not None:
if set(query) == {'name'}:
# temporarily emulate is_controlling for claims in channel
query['order_by'] = ['effective_amount']
else:
query['order_by'] = ['^channel_join']
query['channel_hash'] = channel['claim_hash']
query['is_channel_signature_valid'] = 1
elif set(query) == {'name'}:
query['is_controlling'] = True
matches = self._search(**query, limit=1)
if matches:
result.append(matches[0])
if matches[0]['channel_hash']:
channel_hashes.add(matches[0]['channel_hash'])
else:
result.append(LookupError(f'Could not find stream in "{raw_url}".'))
continue
else:
result.append(channel)
extra_txo_rows = [] extra_txo_rows = []
if channel_hashes: if channel_hashes:
extra_txo_rows = self._search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}) extra_txo_rows = self._search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]})

28
tests/integration/test_claim_commands.py
View file

@ -68,8 +68,9 @@ class ClaimSearchCommand(CommandTestCase):
# finding claims with and without a channel # finding claims with and without a channel
await self.assertFindsClaims([signed2, signed], name='on-channel-claim') await self.assertFindsClaims([signed2, signed], name='on-channel-claim')
await self.assertFindsClaim(signed, name='on-channel-claim', channel_id=self.channel_id) await self.assertFindsClaims([signed2, signed], channel_ids=[self.channel_id, channel_id2])
await self.assertFindsClaim(signed2, name='on-channel-claim', channel_id=channel_id2) await self.assertFindsClaim(signed, name='on-channel-claim', channel_ids=[self.channel_id])
await self.assertFindsClaim(signed2, name='on-channel-claim', channel_ids=[channel_id2])
await self.assertFindsClaim(unsigned, name='unsigned') await self.assertFindsClaim(unsigned, name='unsigned')
await self.assertFindsClaim(unsigned, txid=unsigned['txid'], nout=0) await self.assertFindsClaim(unsigned, txid=unsigned['txid'], nout=0)
await self.assertFindsClaim(unsigned, claim_id=unsigned['outputs'][0]['claim_id']) await self.assertFindsClaim(unsigned, claim_id=unsigned['outputs'][0]['claim_id'])
@ -79,37 +80,36 @@ class ClaimSearchCommand(CommandTestCase):
# three streams in channel, zero streams in abandoned channel # three streams in channel, zero streams in abandoned channel
claims = [three, two, signed] claims = [three, two, signed]
await self.assertFindsClaims(claims, channel_id=self.channel_id) await self.assertFindsClaims(claims, channel_ids=[self.channel_id])
await self.assertFindsClaims([three, two, signed2, signed], channel_name="@abc")
await self.assertFindsClaims(claims, channel_name="@abc", channel_id=self.channel_id)
await self.assertFindsClaims(claims, channel=f"@abc#{self.channel_id}") await self.assertFindsClaims(claims, channel=f"@abc#{self.channel_id}")
await self.assertFindsClaims([three, two, signed2, signed], channel_ids=[channel_id2, self.channel_id])
await self.channel_abandon(claim_id=self.channel_id) await self.channel_abandon(claim_id=self.channel_id)
await self.assertFindsClaims([], channel_id=self.channel_id) await self.assertFindsClaims([], channel_ids=[self.channel_id], is_channel_signature_valid=True)
await self.assertFindsClaims([signed2], channel_name="@abc") await self.assertFindsClaims([signed2], channel_ids=[channel_id2], is_channel_signature_valid=True)
await self.assertFindsClaims([], channel_name="@abc", channel_id=self.channel_id) await self.assertFindsClaims([signed2], channel_ids=[channel_id2, self.channel_id],
await self.assertFindsClaims([], channel=f"@abc#{self.channel_id}") is_channel_signature_valid=True)
# abandoned stream won't show up for streams in channel search # abandoned stream won't show up for streams in channel search
await self.stream_abandon(txid=signed2['txid'], nout=0) await self.stream_abandon(txid=signed2['txid'], nout=0)
await self.assertFindsClaims([], channel_name="@abc") await self.assertFindsClaims([], channel_ids=[channel_id2])
async def test_pagination(self): async def test_pagination(self):
await self.create_channel() await self.create_channel()
await self.create_lots_of_streams() await self.create_lots_of_streams()
page = await self.claim_search(page_size=20, channel_id=self.channel_id) page = await self.claim_search(page_size=20, channel='@abc')
page_claim_ids = [item['name'] for item in page] page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams) self.assertEqual(page_claim_ids, self.streams)
page = await self.claim_search(page_size=6, channel_id=self.channel_id) page = await self.claim_search(page_size=6, channel='@abc')
page_claim_ids = [item['name'] for item in page] page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[:6]) self.assertEqual(page_claim_ids, self.streams[:6])
page = await self.claim_search(page=2, page_size=6, channel_id=self.channel_id) page = await self.claim_search(page=2, page_size=6, channel='@abc')
page_claim_ids = [item['name'] for item in page] page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[6:]) self.assertEqual(page_claim_ids, self.streams[6:])
out_of_bounds = await self.claim_search(page=2, page_size=20, channel_id=self.channel_id) out_of_bounds = await self.claim_search(page=2, page_size=20, channel='@abc')
self.assertEqual(out_of_bounds, []) self.assertEqual(out_of_bounds, [])
async def test_tag_search(self): async def test_tag_search(self):

17
tests/unit/wallet/server/test_sqldb.py
View file

@ -94,7 +94,7 @@ class TestSQLDB(unittest.TestCase):
Input.spend(claim) Input.spend(claim)
) )
def get_stream_abandon(self, tx): def get_abandon(self, tx):
claim = Transaction(tx[0].serialize()).outputs[0] claim = Transaction(tx[0].serialize()).outputs[0]
return self._make_tx( return self._make_tx(
Output.pay_pubkey_hash(claim.amount, b'abc'), Output.pay_pubkey_hash(claim.amount, b'abc'),
@ -264,7 +264,7 @@ class TestSQLDB(unittest.TestCase):
active=[('Claim A', 10*COIN, 10*COIN, 13)], active=[('Claim A', 10*COIN, 10*COIN, 13)],
accepted=[] accepted=[]
) )
advance(14, [self.get_stream_abandon(stream2)]) advance(14, [self.get_abandon(stream2)])
state( state(
controlling=('Claim A', 10*COIN, 10*COIN, 13), controlling=('Claim A', 10*COIN, 10*COIN, 13),
active=[], active=[],
@ -281,7 +281,7 @@ class TestSQLDB(unittest.TestCase):
active=[('Claim A', 10*COIN, 10*COIN, 13)], active=[('Claim A', 10*COIN, 10*COIN, 13)],
accepted=[] accepted=[]
) )
advance(15, [self.get_stream_abandon(stream2), self.get_stream('Claim C', 12*COIN)]) advance(15, [self.get_abandon(stream2), self.get_stream('Claim C', 12*COIN)])
state( state(
controlling=('Claim C', 12*COIN, 12*COIN, 15), controlling=('Claim C', 12*COIN, 12*COIN, 15),
active=[('Claim A', 10*COIN, 10*COIN, 13)], active=[('Claim A', 10*COIN, 10*COIN, 13)],
@ -380,7 +380,8 @@ class TestSQLDB(unittest.TestCase):
self.assertEqual(0, self.sql._search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel']) self.assertEqual(0, self.sql._search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
# reinstate previous channel public key (previous stream claim signatures become valid again) # reinstate previous channel public key (previous stream claim signatures become valid again)
advance(9, [self.get_channel_update(txo_chan_a, COIN, key=b'c')]) channel_update = self.get_channel_update(txo_chan_a, COIN, key=b'c')
advance(9, [channel_update])
r_ab2, r_a2 = self.sql._search(order_by=['creation_height'], limit=2) r_ab2, r_a2 = self.sql._search(order_by=['creation_height'], limit=2)
self.assertEqual(f"foo#{a2_claim_id[:2]}", r_a2['short_url']) self.assertEqual(f"foo#{a2_claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo#{ab2_claim_id[:4]}", r_ab2['short_url']) self.assertEqual(f"foo#{ab2_claim_id[:4]}", r_ab2['short_url'])
@ -388,6 +389,14 @@ class TestSQLDB(unittest.TestCase):
self.assertEqual("@foo#a/foo#ab", r_ab2['canonical_url']) self.assertEqual("@foo#a/foo#ab", r_ab2['canonical_url'])
self.assertEqual(2, self.sql._search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel']) self.assertEqual(2, self.sql._search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
# delete channel, invaliding stream claim signatures
advance(10, [self.get_abandon(channel_update)])
r_ab2, r_a2 = self.sql._search(order_by=['creation_height'], limit=2)
self.assertEqual(f"foo#{a2_claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo#{ab2_claim_id[:4]}", r_ab2['short_url'])
self.assertIsNone(r_a2['canonical_url'])
self.assertIsNone(r_ab2['canonical_url'])
def test_canonical_find_shortest_id(self): def test_canonical_find_shortest_id(self):
new_hash = 'abcdef0123456789beef' new_hash = 'abcdef0123456789beef'
other0 = '1bcdef0123456789beef' other0 = '1bcdef0123456789beef'