LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

update to an even newer protobufs

Browse source
This commit is contained in:
Lex Berezhny 2019-04-20 01:12:43 -04:00
parent 90bef98bc3
commit fa9a4a75c6
8 changed files with 247 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
lbrynet/extras/daemon/Daemon.py
View file

@ -2213,7 +2213,7 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
file_stream = await self.stream_manager.create_stream(file_path)
claim.stream.sd_hash = file_stream.sd_hash
claim.stream.source.sd_hash = file_stream.sd_hash
new_txo.script.generate()
if channel:
new_txo.sign(channel)
@ -2222,7 +2222,7 @@ class Daemon(metaclass=JSONRPCServerType):
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
)])
stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim.stream.sd_hash)
stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim.stream.source.sd_hash)
if stream_hash:
await self.storage.save_content_claim(stream_hash, new_txo.id)
await self.analytics_manager.send_claim_action('publish')
@ -2376,7 +2376,7 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
if file_path is not None:
file_stream = await self.stream_manager.create_stream(file_path)
new_txo.claim.stream.sd_hash = file_stream.sd_hash
new_txo.claim.stream.source.sd_hash = file_stream.sd_hash
new_txo.script.generate()
if channel:
new_txo.sign(channel)
@ -2385,7 +2385,7 @@ class Daemon(metaclass=JSONRPCServerType):
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
)])
stream_hash = await self.storage.get_stream_hash_for_sd_hash(new_txo.claim.stream.sd_hash)
stream_hash = await self.storage.get_stream_hash_for_sd_hash(new_txo.claim.stream.source.sd_hash)
if stream_hash:
await self.storage.save_content_claim(stream_hash, new_txo.id)
await self.analytics_manager.send_claim_action('publish')

87
lbrynet/extras/daemon/json_response_encoder.py
View file

@ -17,11 +17,22 @@ log = logging.getLogger(__name__)
def encode_txo_doc():
return {
'txid': "hash of transaction in hex",
'height': "block where transaction was recorded",
'nout': "position in the transaction",
'height': "block where transaction was recorded",
'amount': "value of the txo as a decimal",
'address': "address of who can spend the txo",
'confirmations': "number of confirmed blocks"
'confirmations': "number of confirmed blocks",
'is_change': "payment to change address, only available when it can be determined",
'is_mine': "payment to one of your accounts, only available when it can be determined",
'type': "one of 'claim', 'support' or 'payment'",
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
'name': "when type is 'claim' or 'support', this is the claim name",
'claim_id': "when type is 'claim' or 'support', this is the claim id",
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
'value': "when type is 'claim', this is the claim metadata",
'sub_type': "when type is 'claim', this determines if it is 'channel' or 'stream' claim",
'signing_channel': "for signed claims only, metadata of signing channel",
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
}
@ -101,21 +112,7 @@ class JSONResponseEncoder(JSONEncoder):
if isinstance(obj, Output):
return self.encode_output(obj)
if isinstance(obj, Claim):
claim_dict = obj.to_dict()
if obj.is_stream:
claim_dict['stream']['sd_hash'] = obj.stream.sd_hash
fee = claim_dict['stream'].get('fee', {})
if 'address' in fee:
fee['address'] = obj.stream.fee.address
if 'amount' in fee:
fee['amount'] = obj.stream.fee.amount
if 'languages' in claim_dict['stream']:
claim_dict['stream']['languages'] = obj.stream.langtags
elif obj.is_channel:
claim_dict['channel']['public_key'] = obj.channel.public_key
if 'languages' in claim_dict['channel']:
claim_dict['channel']['languages'] = obj.channel.langtags
return claim_dict
return self.encode_claim(obj)
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
@ -142,9 +139,9 @@ class JSONResponseEncoder(JSONEncoder):
output = {
'txid': txo.tx_ref.id,
'nout': txo.position,
'height': tx_height,
'amount': dewies_to_lbc(txo.amount),
'address': txo.get_address(self.ledger),
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
}
if txo.is_change is not None:
@ -152,40 +149,46 @@ class JSONResponseEncoder(JSONEncoder):
if txo.is_my_account is not None:
output['is_mine'] = txo.is_my_account
if txo.script.is_claim_name:
output['type'] = 'claim'
output['claim_op'] = 'create'
elif txo.script.is_update_claim:
output['type'] = 'claim'
output['claim_op'] = 'update'
elif txo.script.is_support_claim:
output['type'] = 'support'
else:
output['type'] = 'payment'
if txo.script.is_claim_involved:
output.update({
'name': txo.claim_name,
'claim_id': txo.claim_id,
'permanent_url': txo.permanent_url,
})
if txo.script.is_claim_name or txo.script.is_update_claim:
claim = txo.claim
output['value'] = claim
if claim.is_signed:
output['valid_signature'] = None
if check_signature and txo.channel is not None:
output['channel_name'] = txo.channel.claim_name
output['value'] = txo.claim
if txo.claim.is_channel:
output['sub_type'] = 'channel'
elif txo.claim.is_stream:
output['sub_type'] = 'stream'
if txo.channel is not None:
output['signing_channel'] = {
'name': txo.channel.claim_name,
'claim_id': txo.channel.claim_id,
'value': txo.channel.claim
}
if check_signature and txo.claim.is_signed:
output['is_channel_signature_valid'] = False
try:
output['valid_signature'] = txo.is_signed_by(txo.channel, self.ledger)
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
except BadSignatureError:
output['valid_signature'] = False
pass
except ValueError:
log.exception(
'txo.id: %s, txo.channel.id:%s, output: %s',
txo.id, txo.channel.id, output
)
output['valid_signature'] = False
if txo.script.is_claim_name:
output['type'] = 'claim'
elif txo.script.is_update_claim:
output['type'] = 'update'
elif txo.script.is_support_claim:
output['type'] = 'support'
else:
output['type'] = 'basic'
return output
def encode_input(self, txi):
@ -204,3 +207,11 @@ class JSONResponseEncoder(JSONEncoder):
@staticmethod
def encode_file(managed_stream):
return managed_stream.as_dict()
@staticmethod
def encode_claim(claim):
if claim.is_stream:
return claim.stream.to_dict()
elif claim.is_channel:
return claim.channel.to_dict()
return claim.to_dict()

4
lbrynet/extras/daemon/storage.py
View file

@ -593,7 +593,7 @@ class SQLiteStorage(SQLiteMixin):
sequence = claim_info['claim_sequence']
certificate_id = claim_info['value'].signing_channel_id
try:
source_hash = claim_info['value'].stream.sd_hash
source_hash = claim_info['value'].stream.source.sd_hash
except (AttributeError, ValueError):
source_hash = None
serialized = binascii.hexlify(claim_info['value'].to_bytes())
@ -670,7 +670,7 @@ class SQLiteStorage(SQLiteMixin):
if not known_sd_hash:
raise Exception("stream not found")
# check the claim contains the same sd hash
if known_sd_hash[0] != claim.stream.sd_hash:
if known_sd_hash[0] != claim.stream.source.sd_hash:
raise Exception("stream mismatch")
# if there is a current claim associated to the file, check that the new claim is an update to it

143
lbrynet/schema/claim.py
View file

@ -358,6 +358,43 @@ class Fee:
self._fee.currency = FeeMessage.USD
class ClaimReference:
__slots__ = 'message',
def __init__(self, message):
self.message = message
@property
def claim_id(self) -> str:
return hexlify(self.claim_hash[::-1]).decode()
@claim_id.setter
def claim_id(self, claim_id: str):
self.claim_hash = unhexlify(claim_id)[::-1]
@property
def claim_hash(self) -> bytes:
return self.message.claim_hash
@claim_hash.setter
def claim_hash(self, claim_hash: bytes):
self.message.claim_hash = claim_hash
class ClaimList(BaseMessageList[ClaimReference]):
__slots__ = ()
item_class = ClaimReference
def append(self, value):
self.add().claim_id = value
@property
def claim_ids(self) -> List[str]:
return [c.claim_id for c in self]
class Language:
__slots__ = 'message',
@ -532,9 +569,42 @@ class BaseClaimSubType:
__slots__ = 'claim', 'message'
object_fields = 'thumbnail',
repeat_fields = 'tags', 'languages', 'locations'
def __init__(self, claim: Claim):
self.claim = claim or Claim()
def to_dict(self):
claim = self.claim.to_dict()
if 'languages' in claim:
claim['languages'] = self.langtags
return claim
def update(self, **kwargs):
for key in list(kwargs):
for field in self.object_fields:
if key.startswith(f'{field}_'):
attr = getattr(self, field)
setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
continue
for l in self.repeat_fields:
field = getattr(self, l)
if kwargs.pop(f'clear_{l}', False):
del field[:]
items = kwargs.pop(l, None)
if items is not None:
if isinstance(items, str):
field.append(items)
elif isinstance(items, list):
field.extend(items)
else:
raise ValueError(f"Unknown {l} value: {items}")
for key, value in kwargs.items():
setattr(self, key, value)
@property
def title(self) -> str:
return self.claim.message.title
@ -571,34 +641,24 @@ class BaseClaimSubType:
def locations(self) -> LocationList:
return LocationList(self.claim.message.locations)
def to_dict(self):
return MessageToDict(self.message, preserving_proto_field_name=True)
def update(self, **kwargs):
for l in ('tags', 'languages', 'locations'):
if kwargs.pop(f'clear_{l}', False):
self.message.ClearField('tags')
items = kwargs.pop(l, None)
if items is not None:
if isinstance(items, str):
getattr(self, l).append(items)
elif isinstance(items, list):
getattr(self, l).extend(items)
else:
raise ValueError(f"Unknown {l} value: {items}")
for key, value in kwargs.items():
setattr(self, key, value)
class Channel(BaseClaimSubType):
__slots__ = ()
object_fields = BaseClaimSubType.object_fields + ('cover',)
repeat_fields = BaseClaimSubType.repeat_fields + ('featured',)
def __init__(self, claim: Claim = None):
super().__init__(claim)
self.message = self.claim.channel_message
def to_dict(self):
claim = super().to_dict()
claim.update(claim.pop('channel'))
claim['public_key'] = self.public_key
return claim
@property
def public_key(self) -> str:
return hexlify(self.message.public_key).decode()
@ -616,34 +676,50 @@ class Channel(BaseClaimSubType):
self.message.public_key = public_key
@property
def contact_email(self) -> str:
return self.message.contact_email
def email(self) -> str:
return self.message.email
@contact_email.setter
def contact_email(self, contact_email: str):
self.message.contact_email = contact_email
@email.setter
def email(self, email: str):
self.message.email = email
@property
def homepage_url(self) -> str:
return self.message.homepage_url
def website_url(self) -> str:
return self.message.website_url
@homepage_url.setter
def homepage_url(self, homepage_url: str):
self.message.homepage_url = homepage_url
@website_url.setter
def website_url(self, website_url: str):
self.message.website_url = website_url
@property
def cover(self) -> Source:
return Source(self.message.cover)
@property
def featured(self) -> ClaimList:
return ClaimList(self.message.featured)
class Stream(BaseClaimSubType):
__slots__ = ()
object_fields = BaseClaimSubType.object_fields + ('source',)
def __init__(self, claim: Claim = None):
super().__init__(claim)
self.message = self.claim.stream_message
def to_dict(self):
claim = super().to_dict()
claim.update(claim.pop('stream'))
fee = claim.get('fee', {})
if 'address' in fee:
fee['address'] = self.fee.address
if 'amount' in fee:
fee['amount'] = self.fee.amount
return claim
def update(
self, file_path=None, stream_type=None,
fee_currency=None, fee_amount=None, fee_address=None,
@ -672,14 +748,17 @@ class Stream(BaseClaimSubType):
if duration_was_not_set and file_path and isinstance(sub_obj, Playable):
sub_obj.set_duration_from_path(file_path)
if 'sd_hash' in kwargs:
self.source.sd_hash = kwargs.pop('sd_hash')
super().update(**kwargs)
if file_path is not None:
self.media_type = guess_media_type(file_path)
self.source.media_type = guess_media_type(file_path)
if not os.path.isfile(file_path):
raise Exception(f"File does not exist: {file_path}")
self.file.size = os.path.getsize(file_path)
if self.file.size == 0:
self.source.size = os.path.getsize(file_path)
if self.source.size == 0:
raise Exception(f"Cannot publish empty file: {file_path}")
if fee_amount and fee_currency:

74
lbrynet/schema/types/v2/claim_pb2.py
View file

File diff suppressed because one or more lines are too long

6
lbrynet/stream/stream_manager.py
View file

@ -353,7 +353,7 @@ class StreamManager:
if existing:
await self.start_stream(existing[0])
return existing[0], None
existing = self.get_filtered_streams(sd_hash=claim.stream.sd_hash)
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
if existing and existing[0].claim_id != claim_id:
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing "
f"download {claim_id}")
@ -437,7 +437,7 @@ class StreamManager:
# download the stream
download_id = binascii.hexlify(generate_id()).decode()
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.stream.sd_hash,
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash,
self.config.download_dir, file_name)
stream = None
@ -484,7 +484,7 @@ class StreamManager:
None if not stream else len(stream.downloader.blob_downloader.scores),
False if not downloader else downloader.added_fixed_peers,
self.config.fixed_peer_delay if not downloader else downloader.fixed_peers_delay,
claim.stream.sd_hash, time_to_descriptor,
claim.stream.source.sd_hash, time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
time_to_first_bytes, None if not error else error.__class__.__name__

2
lbrynet/utils.py
View file

@ -116,7 +116,7 @@ def get_sd_hash(stream_info):
if not stream_info:
return None
if isinstance(stream_info, Claim):
return stream_info.stream.sd_hash
return stream_info.stream.source.sd_hash
result = stream_info.get('claim', {}).\
get('value', {}).\
get('stream', {}).\

71
tests/integration/test_claim_commands.py
View file

@ -75,37 +75,39 @@ class ChannelCommands(CommandTestCase):
'thumbnail_url': "https://co.ol/thumbnail.png",
'languages': ["en-US"],
'locations': ['US::Manchester'],
'contact_email': "human@email.com",
'homepage_url': "https://co.ol",
'email': "human@email.com",
'website_url': "https://co.ol",
'cover_url': "https://co.ol/cover.png",
}
fixed_values = values.copy()
fixed_values['languages'] = ['en-US']
fixed_values['locations'] = [{'country': 'US', 'city': 'Manchester'}]
fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
fixed_values['cover'] = {'url': fixed_values.pop('cover_url')}
# create new channel with all fields set
tx = await self.out(self.channel_create('@bigchannel', **values))
txo = tx['outputs'][0]
self.assertEqual(
txo['value']['channel'],
{'public_key': txo['value']['channel']['public_key'], **fixed_values}
txo['value'],
{'public_key': txo['value']['public_key'], **fixed_values}
)
# create channel with nothing set
tx = await self.out(self.channel_create('@lightchannel'))
txo = tx['outputs'][0]
self.assertEqual(
txo['value']['channel'],
{'public_key': txo['value']['channel']['public_key']}
txo['value'],
{'public_key': txo['value']['public_key']}
)
# create channel with just some tags
tx = await self.out(self.channel_create('@updatedchannel', tags='blah'))
txo = tx['outputs'][0]
claim_id = txo['claim_id']
public_key = txo['value']['channel']['public_key']
public_key = txo['value']['public_key']
self.assertEqual(
txo['value']['channel'],
txo['value'],
{'public_key': public_key, 'tags': ['blah']}
)
@ -115,7 +117,7 @@ class ChannelCommands(CommandTestCase):
fixed_values['public_key'] = public_key
fixed_values['tags'].insert(0, 'blah') # existing tag
self.assertEqual(
txo['value']['channel'],
txo['value'],
fixed_values
)
@ -124,7 +126,7 @@ class ChannelCommands(CommandTestCase):
txo = tx['outputs'][0]
fixed_values['tags'] = ['single']
self.assertEqual(
txo['value']['channel'],
txo['value'],
fixed_values
)
@ -132,7 +134,7 @@ class ChannelCommands(CommandTestCase):
tx = await self.out(self.channel_update(claim_id, new_signing_key=True))
txo = tx['outputs'][0]
self.assertNotEqual(
txo['value']['channel']['public_key'],
txo['value']['public_key'],
fixed_values['public_key']
)
@ -284,15 +286,12 @@ class StreamCommands(CommandTestCase):
fixed_values = values.copy()
fixed_values['languages'] = ['en']
fixed_values['locations'] = [{'country': 'UA'}]
# create new channel with all fields set
tx = await self.out(self.stream_create('big', **values))
txo = tx['outputs'][0]
stream = txo['value']['stream']
fixed_values['sd_hash'] = stream['sd_hash']
fixed_values['file'] = stream['file']
fixed_values['media_type'] = 'application/octet-stream'
fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
fixed_values['release_time'] = str(values['release_time'])
fixed_values['source'] = {
'media_type': 'application/octet-stream',
'size': '3'
}
fixed_values['fee'] = {
'address': fixed_values.pop('fee_address'),
'amount': float(fixed_values.pop('fee_amount')),
@ -302,16 +301,24 @@ class StreamCommands(CommandTestCase):
'height': fixed_values.pop('video_height'),
'width': fixed_values.pop('video_width')
}
# create new channel with all fields set
tx = await self.out(self.stream_create('big', **values))
txo = tx['outputs'][0]
stream = txo['value']
fixed_values['source']['sd_hash'] = stream['source']['sd_hash']
self.assertEqual(stream, fixed_values)
# create channel with nothing set
tx = await self.out(self.stream_create('light'))
txo = tx['outputs'][0]
self.assertEqual(
txo['value']['stream'], {
'file': {'size': '3'},
'media_type': 'application/octet-stream',
'sd_hash': txo['value']['stream']['sd_hash']
txo['value'], {
'source': {
'size': '3',
'media_type': 'application/octet-stream',
'sd_hash': txo['value']['source']['sd_hash']
},
}
)
@ -319,12 +326,14 @@ class StreamCommands(CommandTestCase):
tx = await self.out(self.stream_create('updated', tags='blah'))
txo = tx['outputs'][0]
claim_id = txo['claim_id']
fixed_values['sd_hash'] = txo['value']['stream']['sd_hash']
fixed_values['source']['sd_hash'] = txo['value']['source']['sd_hash']
self.assertEqual(
txo['value']['stream'], {
'file': {'size': '3'},
'media_type': 'application/octet-stream',
'sd_hash': fixed_values['sd_hash'],
txo['value'], {
'source': {
'size': '3',
'media_type': 'application/octet-stream',
'sd_hash': fixed_values['source']['sd_hash'],
},
'tags': ['blah']
}
)
@ -333,13 +342,13 @@ class StreamCommands(CommandTestCase):
tx = await self.out(self.stream_update(claim_id, **values))
txo = tx['outputs'][0]
fixed_values['tags'].insert(0, 'blah') # existing tag
self.assertEqual(txo['value']['stream'], fixed_values)
self.assertEqual(txo['value'], fixed_values)
# clearing and settings tags
tx = await self.out(self.stream_update(claim_id, tags='single', clear_tags=True))
txo = tx['outputs'][0]
fixed_values['tags'] = ['single']
self.assertEqual(txo['value']['stream'], fixed_values)
self.assertEqual(txo['value'], fixed_values)
# send claim to someone else
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
@ -441,7 +450,7 @@ class StreamCommands(CommandTestCase):
self.assertEqual(claim['txid'], tx4['outputs'][0]['txid'])
self.assertEqual(claim['channel_name'], '@abc')
self.assertEqual(claim['signature_is_valid'], True)
self.assertEqual(claim['value']['stream']['languages'], ['uk-UA'])
self.assertEqual(claim['value']['languages'], ['uk-UA'])
async def test_claim_search(self):
# search for channel claim