LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

fix files

Browse source 
-don’t automatically remove files that can’t be found from
LBRYFileManager

-add timeout to GetStream
This commit is contained in:
Jack 2016-03-17 00:44:04 -04:00
parent 4e34f861c9
commit 36e299b4cf
5 changed files with 127 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
lbrynet/core/LBRYcrdWallet.py
View file

@ -297,8 +297,8 @@ class LBRYWallet(object):
except (ValueError, TypeError): except (ValueError, TypeError):
return Failure(InvalidStreamInfoError(name)) return Failure(InvalidStreamInfoError(name))
known_fields = ['stream_hash', 'name', 'description', 'key_fee', 'key_fee_address', 'thumbnail', known_fields = ['stream_hash', 'name', 'description', 'key_fee', 'key_fee_address', 'thumbnail',
'content_license', 'sources', 'fee'] 'content_license', 'sources', 'fee', 'author']
known_sources = ['lbry_sd_hash'] known_sources = ['lbry_sd_hash', 'btih', 'url']
known_fee_types = {'LBC': ['amount', 'address']} known_fee_types = {'LBC': ['amount', 'address']}
for field in known_fields: for field in known_fields:
if field in value_dict: if field in value_dict:
@ -334,7 +334,7 @@ class LBRYWallet(object):
return Failure(UnknownNameError(name)) return Failure(UnknownNameError(name))
def claim_name(self, name, sd_hash, amount, description=None, key_fee=None, def claim_name(self, name, sd_hash, amount, description=None, key_fee=None,
key_fee_address=None, thumbnail=None, content_license=None): key_fee_address=None, thumbnail=None, content_license=None, author=None, sources=None):
value = {"sources": {'lbry_sd_hash': sd_hash}} value = {"sources": {'lbry_sd_hash': sd_hash}}
if description is not None: if description is not None:
value['description'] = description value['description'] = description
@ -344,6 +344,10 @@ class LBRYWallet(object):
value['thumbnail'] = thumbnail value['thumbnail'] = thumbnail
if content_license is not None: if content_license is not None:
value['content_license'] = content_license value['content_license'] = content_license
if author is not None:
value['author'] = author
if sources is not None:
value['sources'] = sources
d = self._send_name_claim(name, json.dumps(value), amount) d = self._send_name_claim(name, json.dumps(value), amount)

50
lbrynet/lbryfilemanager/LBRYFileManager.py
View file

@ -34,8 +34,8 @@ class LBRYFileManager(object):
self.sd_identifier = sd_identifier self.sd_identifier = sd_identifier
self.lbry_files = [] self.lbry_files = []
self.sql_db = None self.sql_db = None
self.delete_data = delete_data # self.delete_data = delete_data
self.check_exists_loop = LoopingCall(self.check_files_exist) # self.check_exists_loop = LoopingCall(self.check_files_exist)
if sys.platform.startswith("darwin"): if sys.platform.startswith("darwin"):
self.download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') self.download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
else: else:
@ -43,34 +43,34 @@ class LBRYFileManager(object):
log.debug("Download directory for LBRYFileManager: %s", str(self.download_directory)) log.debug("Download directory for LBRYFileManager: %s", str(self.download_directory))
def setup(self): def setup(self):
self.check_exists_loop.start(10) # self.check_exists_loop.start(10)
d = self._open_db() d = self._open_db()
d.addCallback(lambda _: self._add_to_sd_identifier()) d.addCallback(lambda _: self._add_to_sd_identifier())
d.addCallback(lambda _: self._start_lbry_files()) d.addCallback(lambda _: self._start_lbry_files())
return d return d
def check_files_exist(self): # def check_files_exist(self):
def _disp(deleted_files): # def _disp(deleted_files):
if deleted_files[0][0]: # if deleted_files[0][0]:
for file in bad_files: # for file in bad_files:
log.info("[" + str(datetime.now()) + "] Detected " + file.file_name + " was deleted, removing from file manager") # log.info("[" + str(datetime.now()) + "] Detected " + file.file_name + " was deleted, removing from file manager")
#
def _delete_stream_data(lbry_file): # def _delete_stream_data(lbry_file):
s_h = lbry_file.stream_hash # s_h = lbry_file.stream_hash
d = self.get_count_for_stream_hash(s_h) # d = self.get_count_for_stream_hash(s_h)
# TODO: could possibly be a timing issue here # # TODO: could possibly be a timing issue here
d.addCallback(lambda c: self.stream_info_manager.delete_stream(s_h) if c == 0 else True) # d.addCallback(lambda c: self.stream_info_manager.delete_stream(s_h) if c == 0 else True)
return d # return d
#
bad_files = [lbry_file for lbry_file in self.lbry_files # bad_files = [lbry_file for lbry_file in self.lbry_files
if lbry_file.completed == True and # if lbry_file.completed == True and
os.path.isfile(os.path.join(self.download_directory, lbry_file.file_name)) == False] # os.path.isfile(os.path.join(self.download_directory, lbry_file.file_name)) == False]
d = defer.DeferredList([self.delete_lbry_file(lbry_file) for lbry_file in bad_files], consumeErrors=True) # d = defer.DeferredList([self.delete_lbry_file(lbry_file) for lbry_file in bad_files], consumeErrors=True)
d.addCallback(lambda files: _disp(files) if len(files) else defer.succeed(None)) # d.addCallback(lambda files: _disp(files) if len(files) else defer.succeed(None))
#
if self.delete_data: # if self.delete_data:
d2 = defer.DeferredList([_delete_stream_data(lbry_file) for lbry_file in bad_files], consumeErrors=True) # d2 = defer.DeferredList([_delete_stream_data(lbry_file) for lbry_file in bad_files], consumeErrors=True)
def get_lbry_file_status(self, lbry_file): def get_lbry_file_status(self, lbry_file):
return self._get_lbry_file_status(lbry_file.rowid) return self._get_lbry_file_status(lbry_file.rowid)
@ -180,7 +180,7 @@ class LBRYFileManager(object):
return defer.fail(Failure(ValueError("Could not find that LBRY file"))) return defer.fail(Failure(ValueError("Could not find that LBRY file")))
def stop(self): def stop(self):
self.check_exists_loop.stop() # self.check_exists_loop.stop()
ds = [] ds = []

80
lbrynet/lbrynet_daemon/LBRYDaemon.py
View file

@ -6,10 +6,14 @@ import binascii
import subprocess import subprocess
import logging import logging
import requests import requests
# import rumps
# import httplib2
from twisted.web import server, resource, static from twisted.web import server, resource, static
from twisted.internet import defer, threads, error, reactor from twisted.internet import defer, threads, error, reactor
from txjsonrpc.web import jsonrpc from txjsonrpc.web import jsonrpc
from jsonrpc.proxy import JSONRPCProxy
from datetime import datetime from datetime import datetime
from decimal import Decimal from decimal import Decimal
from StringIO import StringIO from StringIO import StringIO
@ -537,10 +541,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return defer.succeed(True) return defer.succeed(True)
def _download_name(self, name): def _download_name(self, name):
def _disp_file(file): def _disp_file(f):
log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(file.stream_hash)) file_path = os.path.join(self.download_directory, f.file_name)
d = self._path_from_lbry_file(file) log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.stream_hash) + " --> " + file_path)
return d return defer.succeed(f)
def _get_stream(name): def _get_stream(name):
def _disp(stream): def _disp(stream):
@ -558,12 +562,9 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = self._check_history(name) d = self._check_history(name)
d.addCallback(lambda lbry_file: _get_stream(name) if not lbry_file else _disp_file(lbry_file)) d.addCallback(lambda lbry_file: _get_stream(name) if not lbry_file else _disp_file(lbry_file))
d.addCallback(lambda _: self._check_history(name)) d.addCallback(lambda _: self._path_from_name(name))
d.addCallback(lambda lbry_file: ({'stream_hash': lbry_file.stream_hash,
'path': os.path.join(self.download_directory,
lbry_file.file_name)})
if lbry_file else defer.fail(NOT_FOUND))
d.addErrback(lambda err: defer.fail(NOT_FOUND)) d.addErrback(lambda err: defer.fail(NOT_FOUND))
return d return d
def _resolve_name(self, name): def _resolve_name(self, name):
@ -586,12 +587,19 @@ class LBRYDaemon(jsonrpc.JSONRPC):
f = open(path, 'r') f = open(path, 'r')
l = json.loads(f.read()) l = json.loads(f.read())
f.close() f.close()
file_name = l['stream_name'].decode('hex') file_name = l['stream_name'].decode('hex')
lbry_file = [file for file in self.lbry_file_manager.lbry_files if file.stream_name == file_name] for lbry_file in self.lbry_file_manager.lbry_files:
if lbry_file: if lbry_file.stream_name == file_name:
return lbry_file[0] if sys.platform == "darwin":
if os.path.isfile(os.path.join(self.download_directory, lbry_file.stream_name)):
return lbry_file
else:
return False
else:
return lbry_file
else: else:
return None return False
def _check(info): def _check(info):
stream_hash = info['stream_hash'] stream_hash = info['stream_hash']
@ -604,7 +612,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return defer.succeed(False) return defer.succeed(False)
d = self._resolve_name(name) d = self._resolve_name(name)
d.addCallbacks(_check, lambda _: False) d.addCallback(_check)
d.callback(None) d.callback(None)
return d return d
@ -633,7 +641,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = self._check_history(name) d = self._check_history(name)
d.addCallback(lambda lbry_file: {'stream_hash': lbry_file.stream_hash, d.addCallback(lambda lbry_file: {'stream_hash': lbry_file.stream_hash,
'path': os.path.join(self.download_directory, lbry_file.file_name)} 'path': os.path.join(self.download_directory, lbry_file.file_name)}
if lbry_file else defer.fail(UnknownNameError)) if lbry_file else defer.fail(UnknownNameError))
return d return d
def _path_from_lbry_file(self, lbry_file): def _path_from_lbry_file(self, lbry_file):
@ -673,6 +681,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
def _render_response(self, result, code): def _render_response(self, result, code):
return json.dumps({'result': result, 'code': code}) return json.dumps({'result': result, 'code': code})
# def _log_to_slack(self, msg):
# URL = "https://hooks.slack.com/services/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"
# h = httplib2.Http()
# h.request(URL, 'POST', json.dumps({"text": msg}), headers={'Content-Type': 'application/json'})
def jsonrpc_is_running(self): def jsonrpc_is_running(self):
""" """
Returns a startup message when the daemon starts, after which it will return True Returns a startup message when the daemon starts, after which it will return True
@ -722,6 +735,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
self.fetcher.start() self.fetcher.start()
log.info('[' + str(datetime.now()) + '] Start autofetcher') log.info('[' + str(datetime.now()) + '] Start autofetcher')
# self._log_to_slack('[' + str(datetime.now()) + '] Start autofetcher')
return self._render_response("Started autofetching claims", OK_CODE) return self._render_response("Started autofetching claims", OK_CODE)
def jsonrpc_stop_fetcher(self): def jsonrpc_stop_fetcher(self):
@ -829,6 +843,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
lambda err: self._render_response('error', NOT_FOUND)) lambda err: self._render_response('error', NOT_FOUND))
else: else:
d = self._render_response('error', BAD_REQUEST) d = self._render_response('error', BAD_REQUEST)
return d return d
def jsonrpc_stop_lbry_file(self, p): def jsonrpc_stop_lbry_file(self, p):
@ -933,27 +948,40 @@ class LBRYDaemon(jsonrpc.JSONRPC):
@param: @param:
@return: @return:
""" """
params = Bunch(p) params = Bunch(p)
metadata_fields = {"name": str, "file_path": str, "bid": float, "author": str, "title": str, metadata_fields = {"name": unicode, "file_path": unicode, "bid": float, "author": unicode, "title": unicode,
"description": str, "thumbnail": str, "key_fee": float, "key_fee_address": str, "description": unicode, "thumbnail": unicode, "key_fee": float, "key_fee_address": unicode,
"content_license": str} "content_license": unicode, "sources": dict}
log.info(params)
log.info(params.__dict__)
for k in metadata_fields.keys(): for k in metadata_fields.keys():
if k in params.__dict__.keys(): if k in params.__dict__.keys():
assert isinstance(params.__dict__[k], metadata_fields[k]) if isinstance(params.__dict__[k], metadata_fields[k]):
metadata_fields[k] = params.__dict__[k] if type(params.__dict__[k]) == unicode:
metadata_fields[k] = str(params.__dict__[k])
else:
metadata_fields[k] = params.__dict__[k]
else:
metadata_fields[k] = None
else: else:
metadata_fields[k] = None metadata_fields[k] = None
log.info("[" + str(datetime.now()) + "] Publish: ", metadata_fields) log.info("[" + str(datetime.now()) + "] Publish: ", metadata_fields)
p = Publisher(self.session, self.lbry_file_manager, self.session.wallet) p = Publisher(self.session, self.lbry_file_manager, self.session.wallet)
d = p.start(metadata_fields['name'], metadata_fields['file_path'], metadata_fields['bid'], d = p.start(name=metadata_fields['name'],
metadata_fields['title'], metadata_fields['description'], metadata_fields['thumbnail'], file_path=metadata_fields['file_path'],
metadata_fields['key_fee'], metadata_fields['key_fee_address'], metadata_fields['content_license']) bid=metadata_fields['bid'],
title=metadata_fields['title'],
description=metadata_fields['description'],
thumbnail=metadata_fields['thumbnail'],
key_fee=metadata_fields['key_fee'],
key_fee_address=metadata_fields['key_fee_address'],
content_license=metadata_fields['content_license'],
author=metadata_fields['author'],
sources=metadata_fields['sources'])
d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE), d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE),
lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) lambda err: self._render_response(err.getTraceback(), BAD_REQUEST))
@ -1175,4 +1203,4 @@ class LBRYDaemonWeb(resource.Resource):
d.addCallbacks(lambda results: self._delayed_render(request, results), d.addCallbacks(lambda results: self._delayed_render(request, results),
lambda err: self._delayed_render(request, json.dumps({'message': err.getTraceback(), 'code': BAD_REQUEST}))) lambda err: self._delayed_render(request, json.dumps({'message': err.getTraceback(), 'code': BAD_REQUEST})))
return server.NOT_DONE_YET return server.NOT_DONE_YET

47
lbrynet/lbrynet_daemon/LBRYDownloader.py
View file

@ -13,7 +13,8 @@ log = logging.getLogger(__name__)
class GetStream(object): class GetStream(object):
def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee, pay_key=True, data_rate=0.5): def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee, pay_key=True, data_rate=0.5,
timeout=30):
self.wallet = wallet self.wallet = wallet
self.resolved_name = None self.resolved_name = None
self.description = None self.description = None
@ -30,8 +31,25 @@ class GetStream(object):
self.max_key_fee = max_key_fee self.max_key_fee = max_key_fee
self.stream_info = None self.stream_info = None
self.stream_info_manager = None self.stream_info_manager = None
self.d = defer.Deferred(None)
self.timeout = timeout
self.timeout_counter = 0
self.download_path = None
self.checker = LoopingCall(self.check_status)
def check_status(self):
self.timeout_counter += 1
if self.download_path and os.path.isfile(self.download_path):
self.checker.stop()
return defer.succeed(True)
elif self.timeout_counter >= self.timeout:
log.info("Timeout downloading " + str(self.stream_info))
self.checker.stop()
self.d.cancel()
def start(self, stream_info): def start(self, stream_info):
self.stream_info = stream_info self.stream_info = stream_info
if 'stream_hash' in self.stream_info.keys(): if 'stream_hash' in self.stream_info.keys():
@ -60,17 +78,18 @@ class GetStream(object):
else: else:
pass pass
d = defer.Deferred(None) self.checker.start(1)
d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
d.addCallback(lambda metadata: (next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)), metadata))
d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata, [self.data_rate, True], self.payment_rate_manager))
d.addErrback(lambda err: err.trap(defer.CancelledError))
d.addErrback(lambda err: log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback()))
d.addCallback(self._start_download)
d.callback(None)
return d self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self.d.addCallback(lambda metadata: (next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)), metadata))
self.d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata, [self.data_rate, True], self.payment_rate_manager))
self.d.addErrback(lambda err: err.trap(defer.CancelledError))
self.d.addErrback(lambda err: log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback()))
self.d.addCallback(self._start_download)
self.d.callback(None)
return self.d
def _start_download(self, downloader): def _start_download(self, downloader):
def _pay_key_fee(): def _pay_key_fee():
@ -87,9 +106,9 @@ class GetStream(object):
else: else:
d = defer.Deferred() d = defer.Deferred()
downloader.start() self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
d.addCallback(lambda _: downloader.start())
log.info("Downloading", self.stream_hash, "-->", os.path.join(downloader.download_directory, downloader.file_name)) d.addCallback(lambda _: log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
return d return d

11
lbrynet/lbrynet_daemon/LBRYPublisher.py
View file

@ -33,9 +33,11 @@ class Publisher(object):
self.sd_hash = None self.sd_hash = None
self.tx_hash = None self.tx_hash = None
self.content_license = None self.content_license = None
self.author = None
self.sources = None
def start(self, name, file_path, bid, title=None, description=None, thumbnail=None, def start(self, name, file_path, bid, title=None, description=None, thumbnail=None,
key_fee=None, key_fee_address=None, content_license=None): key_fee=None, key_fee_address=None, content_license=None, author=None, sources=None):
def _show_result(): def _show_result():
message = "[" + str(datetime.now()) + "] Published " + self.file_name + " --> lbry://" + \ message = "[" + str(datetime.now()) + "] Published " + self.file_name + " --> lbry://" + \
@ -52,6 +54,8 @@ class Publisher(object):
self.key_fee = key_fee self.key_fee = key_fee
self.key_fee_address = key_fee_address self.key_fee_address = key_fee_address
self.content_license = content_license self.content_license = content_license
self.author = author
self.sources = sources
d = self._check_file_path(self.file_path) d = self._check_file_path(self.file_path)
d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager, d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager,
@ -104,10 +108,11 @@ class Publisher(object):
return d return d
def _claim_name(self): def _claim_name(self):
d = self.wallet.claim_name(self.publish_name, self.sd_hash, self.bid_amount, d = self.wallet.claim_name(self.publish_name, {'sd_hash': self.sd_hash}, self.bid_amount,
description=self.description, key_fee=self.key_fee, description=self.description, key_fee=self.key_fee,
key_fee_address=self.key_fee_address, thumbnail=self.thumbnail, key_fee_address=self.key_fee_address, thumbnail=self.thumbnail,
content_license=self.content_license) content_license=self.content_license, author=self.author,
sources=self.sources)
def set_tx_hash(tx_hash): def set_tx_hash(tx_hash):
self.tx_hash = tx_hash self.tx_hash = tx_hash