From 2336015f9a27496ace0a73bdef25e4099762d231 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 1 Feb 2019 11:37:13 -0500 Subject: [PATCH] fix publish fixes https://github.com/lbryio/lbry/issues/1826 - remove `sources` argument from `publish` - only add/update files we have - delete existing stream if updating with a new one --- lbrynet/extras/daemon/Daemon.py | 97 +++++++++++++++++--------------- lbrynet/stream/stream_manager.py | 5 ++ 2 files changed, 56 insertions(+), 46 deletions(-) diff --git a/lbrynet/extras/daemon/Daemon.py b/lbrynet/extras/daemon/Daemon.py index 76732566b..6635b4e50 100644 --- a/lbrynet/extras/daemon/Daemon.py +++ b/lbrynet/extras/daemon/Daemon.py @@ -10,7 +10,6 @@ import random from urllib.parse import urlencode, quote from typing import Callable, Optional, List from binascii import hexlify, unhexlify -from copy import deepcopy from traceback import format_exc from aiohttp import web from functools import wraps @@ -30,6 +29,7 @@ from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UP from lbrynet.extras.daemon.ComponentManager import RequiredCondition from lbrynet.extras.daemon.ComponentManager import ComponentManager from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder +from lbrynet.extras.daemon.mime_types import guess_media_type from lbrynet.extras.daemon.undecorated import undecorated from lbrynet.extras.wallet.account import Account as LBCAccount from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies @@ -1828,7 +1828,7 @@ class Daemon(metaclass=JSONRPCServerType): async def jsonrpc_publish( self, name, bid, metadata=None, file_path=None, fee=None, title=None, description=None, author=None, language=None, license=None, - license_url=None, thumbnail=None, preview=None, nsfw=None, sources=None, + license_url=None, thumbnail=None, preview=None, nsfw=None, channel_name=None, channel_id=None, channel_account_id=None, account_id=None, claim_address=None, change_address=None): """ @@ -1852,7 +1852,7 @@ class Daemon(metaclass=JSONRPCServerType): [--file_path=] [--fee=] [--title=] [--description=<description>] [--author=<author>] [--language=<language>] [--license=<license>] [--license_url=<license_url>] [--thumbnail=<thumbnail>] - [--preview=<preview>] [--nsfw=<nsfw>] [--sources=<sources>] + [--preview=<preview>] [--nsfw=<nsfw>] [--channel_name=<channel_name>] [--channel_id=<channel_id>] [--channel_account_id=<channel_account_id>...] [--account_id=<account_id>] [--claim_address=<claim_address>] [--change_address=<change_address>] @@ -1888,7 +1888,6 @@ class Daemon(metaclass=JSONRPCServerType): --thumbnail=<thumbnail> : (str) thumbnail url --preview=<preview> : (str) preview url --nsfw=<nsfw> : (bool) whether the content is nsfw - --sources=<sources> : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file --channel_name=<channel_name> : (str) name of the publisher channel name in the wallet --channel_id=<channel_id> : (str) claim id of the publisher channel, does not check for channel claim being in the wallet. This allows @@ -1912,7 +1911,9 @@ class Daemon(metaclass=JSONRPCServerType): """ try: - parse_lbry_uri(name) + parsed = parse_lbry_uri(name) + if parsed.name != name: + raise Exception("Name given to publish has invalid characters") except (TypeError, URIParseError): raise Exception("Invalid name given to publish") @@ -1928,9 +1929,10 @@ class Daemon(metaclass=JSONRPCServerType): account = self.get_account_or_default(account_id) available = await account.get_balance() - existing_claims = [] + existing_claims = await account.get_claims(claim_name=name) + if len(existing_claims) > 1: + log.warning("found %i claims for the name %s", len(existing_claims), name) if amount >= available: - existing_claims = await account.get_claims(claim_name=name) if len(existing_claims) == 1: available += existing_claims[0].get_estimator(self.ledger).effective_amount if amount >= available: @@ -1989,39 +1991,64 @@ class Daemon(metaclass=JSONRPCServerType): } } - # this will be used to verify the format with lbrynet.schema - claim_copy = deepcopy(claim_dict) - if sources is not None: - claim_dict['stream']['source'] = sources - claim_copy['stream']['source'] = sources - elif file_path is not None: + sd_to_delete = None + + if file_path: if not os.path.isfile(file_path): raise Exception("invalid file path to publish") + if os.path.getsize(file_path) == 0: + raise Exception(f"Cannot publish empty file {file_path}") + if existing_claims: + sd_to_delete = existing_claims[-1].claim_dict['stream']['source']['source'] + # since the file hasn't yet been made into a stream, we don't have # a valid Source for the claim when validating the format, we'll use a fake one - claim_copy['stream']['source'] = { + claim_dict['stream']['source'] = { 'version': '_0_0_1', 'sourceType': 'lbry_sd_hash', 'source': '0' * 96, 'contentType': '' } + elif not existing_claims: + raise Exception("no previous stream to update") else: - # there is no existing source to use, and a file was not provided to make a new one - raise Exception("no source provided to publish") + claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source'] try: - ClaimDict.load_dict(claim_copy) + claim_dict = ClaimDict.load_dict(claim_dict).claim_dict # the metadata to use in the claim can be serialized by lbrynet.schema except DecodeError as err: # there was a problem with a metadata field, raise an error here rather than # waiting to find out when we go to publish the claim (after having made the stream) raise Exception(f"invalid publish metadata: {err}") - certificate = None if channel_id or channel_name: certificate = await self.get_channel_or_error( self.get_accounts_or_all(channel_account_id), channel_id, channel_name ) + if file_path: + stream = await self.stream_manager.create_stream(file_path) + await self.storage.save_published_file(stream.stream_hash, os.path.basename(file_path), + os.path.dirname(file_path), 0) + claim_dict['stream']['source']['source'] = stream.sd_hash + claim_dict['stream']['source']['contentType'] = guess_media_type(file_path) + + if sd_to_delete: + stream_hash_to_delete = await self.storage.get_stream_hash_for_sd_hash(sd_to_delete) + stream_to_delete = self.stream_manager.get_stream_by_stream_hash(stream_hash_to_delete) + if stream_to_delete: + await self.stream_manager.delete_stream(stream_to_delete, delete_file=False) + log.info("updating claim to stream generated from %s, deleted previous stream %s", + sd_to_delete[:8], file_path) + else: + log.info("previous stream %s from claim was not saved locally, nothing to delete", + sd_to_delete[:8]) + else: + log.info("generated stream from %s for claim", file_path) + else: + log.info("updating claim with stream %s from previous", claim_dict['stream']['source']['source'][:8]) + + sd_hash = claim_dict['stream']['source']['source'] log.info("Publish: %s", { 'name': name, 'file_path': file_path, @@ -2032,36 +2059,14 @@ class Daemon(metaclass=JSONRPCServerType): 'channel_id': channel_id, 'channel_name': channel_name }) - - from lbrynet.extras.daemon.mime_types import guess_media_type - - if file_path: - if not os.path.isfile(file_path): - raise Exception(f"File {file_path} not found") - if os.path.getsize(file_path) == 0: - raise Exception(f"Cannot publish empty file {file_path}") - claim_dict['stream']['source'] = {} - - stream = await self.stream_manager.create_stream(file_path) - stream_hash = stream.stream_hash - await self.storage.save_published_file(stream_hash, os.path.basename(file_path), - os.path.dirname(file_path), 0) - claim_dict['stream']['source']['source'] = stream.sd_hash - claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash' - claim_dict['stream']['source']['contentType'] = guess_media_type(file_path) - claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here - else: - if 'source' not in claim_dict['stream'] and not existing_claims: - raise Exception("no previous stream to update") - claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source'] - stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim_dict['stream']['source']['source']) tx = await self.wallet_manager.claim_name( account, name, amount, claim_dict, certificate, claim_address ) - await self.storage.save_content_claim( - stream_hash, tx.outputs[0].id - ) - + stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash) + if stream_hash: + await self.storage.save_content_claim( + stream_hash, tx.outputs[0].id + ) await self.analytics_manager.send_claim_action('publish') nout = 0 txo = tx.outputs[nout] @@ -2289,7 +2294,7 @@ class Daemon(metaclass=JSONRPCServerType): } """ claims = await self.wallet_manager.get_claims_for_name(name) # type: dict - sort_claim_results(claims['claims']) + claims['claims'] = sort_claim_results(claims['claims']) return claims @requires(WALLET_COMPONENT) diff --git a/lbrynet/stream/stream_manager.py b/lbrynet/stream/stream_manager.py index b142b9c7b..66e9cdf06 100644 --- a/lbrynet/stream/stream_manager.py +++ b/lbrynet/stream/stream_manager.py @@ -235,6 +235,11 @@ class StreamManager: del self.starting_streams[sd_hash] log.info("returned from get lbry://%s#%s", claim_info['name'], claim_info['claim_id']) + def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]: + streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams)) + if streams: + return streams[0] + def get_filtered_streams(self, sort_by: typing.Optional[str] = None, reverse: typing.Optional[bool] = False, comparison: typing.Optional[str] = None, **search_by) -> typing.List[ManagedStream]: