LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

Merge branch 'master' into electron

Browse source
This commit is contained in:
jobevers 2017-02-21 10:47:28 -06:00
commit 1d30ae447a
13 changed files with 111 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.bumpversion.cfg
View file

@ -19,6 +19,6 @@ values =
[bumpversion:file:CHANGELOG.md]
search = [Unreleased]
replace = [Unreleased]
\#\# [{new_version}] - {now:%Y-%m-%d}
\#\# [{new_version}] - {now:%Y-%m-%d}

1
.gitattributes vendored Normal file
View file

@ -0,0 +1 @@
/CHANGELOG.md merge=union

24
CHANGELOG.md
View file

@ -8,12 +8,36 @@ can and probably will change functionality and break backwards compatability
at anytime.
## [Unreleased]
\#\# [0.8.6] - 2017-02-19
\#\# [0.8.6rc0] - 2017-02-19
### Changed
* Add `file_get` by stream hash
* Add utils.call_later to replace reactor.callLater
###
* Fix unhandled error in `get`
* Fix sd blob timeout handling in `get_availability`, return 0.0
## [0.8.5] - 2017-02-18
## [0.8.5rc0] - 2017-02-18
### Fixed
* Fix result expected by ui from file_get for missing files
## [0.8.4] - 2017-02-17
## [0.8.4rc0] - 2017-02-17
### Changed
* Remove unused upload_allowed option
* Remove code related to packaging as that step is now done in the electron client
* Remove lbryum version check; use lbry-electron as version source
* Include download url in version check
### Fixed
* add misssing traceback to logging
## [0.8.3] - 2017-02-15
### Fixed
* Get lbry files with pending claims

1
lbrynet/analytics/events.py
View file

@ -67,6 +67,7 @@ class Events(object):
'module': log_record.module,
'lineno': log_record.lineno,
'name': log_record.name,
'traceback': log_record.exc_text,
}
return self._event('Error', properties)

3
lbrynet/analytics/logging_handler.py
View file

@ -8,4 +8,7 @@ class Handler(logging.Handler):
logging.Handler.__init__(self, level)
def emit(self, record):
# We need to call format to ensure that record.message and
# record.exc_text attributes are populated
self.format(record)
self.manager.send_error(record)

3
lbrynet/core/HashAnnouncer.py
View file

@ -11,5 +11,8 @@ class DummyHashAnnouncer(object):
def add_supplier(self, *args):
pass
def hash_queue_size(self):
return 0
def immediate_announce(self, *args):
pass

7
lbrynet/core/client/ConnectionManager.py
View file

@ -5,7 +5,7 @@ from lbrynet import interfaces
from lbrynet import conf
from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
from lbrynet.core.Error import InsufficientFundsError
from lbrynet.core import utils
log = logging.getLogger(__name__)
@ -19,7 +19,6 @@ class PeerConnectionHandler(object):
class ConnectionManager(object):
implements(interfaces.IConnectionManager)
callLater = reactor.callLater
MANAGE_CALL_INTERVAL_SEC = 1
def __init__(self, downloader, rate_limiter,
@ -54,7 +53,7 @@ class ConnectionManager(object):
def start(self):
log.debug("%s starting", self._get_log_name())
self._start()
self._next_manage_call = self.callLater(0, self.manage)
self._next_manage_call = utils.call_later(0, self.manage)
return defer.succeed(True)
@ -156,7 +155,7 @@ class ConnectionManager(object):
self._manage_deferred.callback(None)
self._manage_deferred = None
if not self.stopped and schedule_next_call:
self._next_manage_call = self.callLater(self.MANAGE_CALL_INTERVAL_SEC, self.manage)
self._next_manage_call = utils.call_later(self.MANAGE_CALL_INTERVAL_SEC, self.manage)
def _rank_request_creator_connections(self):
"""Returns an ordered list of our request creators, ranked according

2
lbrynet/core/client/DHTPeerFinder.py
View file

@ -40,7 +40,7 @@ class DHTPeerFinder(object):
def find_peers_for_blob(self, blob_hash, timeout=None):
def _trigger_timeout():
if not finished_deferred.called:
log.warning("Peer search for %s timed out", short_hash(blob_hash))
log.debug("Peer search for %s timed out", short_hash(blob_hash))
finished_deferred.cancel()
bin_hash = binascii.unhexlify(blob_hash)

9
lbrynet/core/server/DHTHashAnnouncer.py
View file

@ -3,14 +3,13 @@ import collections
import logging
import time
from twisted.internet import defer, reactor
from twisted.internet import defer
from lbrynet.core import utils
log = logging.getLogger(__name__)
class DHTHashAnnouncer(object):
callLater = reactor.callLater
ANNOUNCE_CHECK_INTERVAL = 60
CONCURRENT_ANNOUNCERS = 5
@ -26,7 +25,7 @@ class DHTHashAnnouncer(object):
def run_manage_loop(self):
if self.peer_port is not None:
self._announce_available_hashes()
self.next_manage_call = self.callLater(self.ANNOUNCE_CHECK_INTERVAL, self.run_manage_loop)
self.next_manage_call = utils.call_later(self.ANNOUNCE_CHECK_INTERVAL, self.run_manage_loop)
def stop(self):
log.info("Stopping %s", self)
@ -79,7 +78,7 @@ class DHTHashAnnouncer(object):
log.debug('Announcing blob %s to dht', h)
d = self.dht_node.announceHaveBlob(binascii.unhexlify(h), self.peer_port)
d.chainDeferred(announce_deferred)
d.addBoth(lambda _: self.callLater(0, announce))
d.addBoth(lambda _: utils.call_later(0, announce))
else:
self._concurrent_announcers -= 1

7
lbrynet/core/utils.py
View file

@ -43,6 +43,13 @@ def datetime_obj(*args, **kwargs):
return datetime.datetime(*args, **kwargs)
def call_later(delay, func, *args, **kwargs):
# Import here to ensure that it gets called after installing a reator
# see: http://twistedmatrix.com/documents/current/core/howto/choosing-reactor.html
from twisted.internet import reactor
return reactor.callLater(delay, func, *args, **kwargs)
def generate_id(num=None):
h = get_lbry_hash_obj()
if num is not None:

80
lbrynet/lbrynet_daemon/Daemon.py
View file

@ -105,6 +105,7 @@ class FileID:
NAME = 'name'
SD_HASH = 'sd_hash'
FILE_NAME = 'file_name'
STREAM_HASH = 'stream_hash'
# TODO add login credentials in a conf file
@ -819,17 +820,13 @@ class Daemon(AuthJSONRPCServer):
"""
timeout = timeout if timeout is not None else conf.settings['download_timeout']
try:
helper = _DownloadNameHelper(self, name, timeout, download_directory, file_name,
wait_for_write)
except Exception as err:
log.exception(err)
raise err
helper = _DownloadNameHelper(self, name, timeout, download_directory, file_name,
wait_for_write)
if not stream_info:
self.waiting_on[name] = True
stream_info = yield self._resolve_name(name)
del self.waiting_on[name]
lbry_file = yield helper.setup_stream(stream_info)
sd_hash, file_path = yield helper.wait_or_get_stream(stream_info, lbry_file)
defer.returnValue((sd_hash, file_path))
@ -1029,6 +1026,12 @@ class Daemon(AuthJSONRPCServer):
return lbry_file
raise Exception("File %s not found" % file_name)
def _find_lbry_file_by_stream_hash(self, stream_hash):
for lbry_file in self.lbry_file_manager.lbry_files:
if lbry_file.stream_hash == stream_hash:
return lbry_file
raise NoSuchStreamHash(stream_hash)
@defer.inlineCallbacks
def _get_lbry_file_by_uri(self, name):
try:
@ -1049,6 +1052,11 @@ class Daemon(AuthJSONRPCServer):
lbry_file = yield self._get_lbry_file_by_file_name(file_name)
defer.returnValue(lbry_file)
@defer.inlineCallbacks
def _get_lbry_file_by_stream_hash(self, stream_hash):
lbry_file = yield self._find_lbry_file_by_stream_hash(stream_hash)
defer.returnValue(lbry_file)
@defer.inlineCallbacks
def _get_lbry_file(self, search_by, val, return_json=True):
helper = _GetFileHelper(self, search_by, val, return_json)
@ -1057,7 +1065,7 @@ class Daemon(AuthJSONRPCServer):
defer.returnValue(lbry_file)
except Exception as err:
# TODO: do something with the error, don't return None when a file isn't found
defer.returnValue(None)
defer.returnValue(False)
def _get_lbry_files(self):
def safe_get(sd_hash):
@ -1376,7 +1384,10 @@ class Daemon(AuthJSONRPCServer):
Returns:
list
"""
return self._render_response(sorted(self.callable_methods.keys()))
return self._render_response(sorted(
[command for command in self.callable_methods.keys()
if 'DEPRECATED' not in getattr(self, "jsonrpc_" + command).__doc__]
))
def jsonrpc_get_balance(self):
"""
@ -1453,22 +1464,35 @@ class Daemon(AuthJSONRPCServer):
def jsonrpc_file_get(self, **kwargs):
"""
Get a file
Get a file, if no matching file exists returns False
Args:
'name': get file by lbry uri,
'sd_hash': get file by the hash in the name claim,
'file_name': get file by its name in the downloads folder,
'stream_hash': get file by its stream hash
Returns:
'completed': bool
'file_name': string
'key': hex string
'points_paid': float
'stopped': bool
'stream_hash': base 58 string
'stream_name': string
'suggested_file_name': string
'sd_hash': string
'completed': bool,
'file_name': str,
'download_directory': str,
'points_paid': float,
'stopped': bool,
'stream_hash': str (hex),
'stream_name': str,
'suggested_file_name': str,
'sd_hash': str (hex),
'lbry_uri': str,
'txid': str (b58),
'claim_id': str (b58),
'download_path': str,
'mime_type': str,
'key': str (hex),
'total_bytes': int,
'written_bytes': int,
'code': str,
'message': str
'metadata': Metadata dict if claim is valid, otherwise status str
}
"""
d = self._get_deferred_for_lbry_file(kwargs)
d.addCallback(lambda r: self._render_response(r))
@ -1557,6 +1581,7 @@ class Daemon(AuthJSONRPCServer):
'stream_hash': hex string
'path': path of download
"""
timeout = timeout if timeout is not None else self.download_timeout
download_directory = download_directory or self.download_directory
sd_hash = get_sd_hash(stream_info)
@ -2451,7 +2476,11 @@ class Daemon(AuthJSONRPCServer):
blob_hashes = [blob.blob_hash for blob in blobs]
if need_sd_blob:
# we don't want to use self._download_descriptor here because it would create a stream
sd_blob = yield self._download_blob(sd_hash, timeout=sd_timeout)
try:
sd_blob = yield self._download_blob(sd_hash, timeout=sd_timeout)
except Exception as err:
response = yield self._render_response(0.0)
defer.returnValue(response)
decoded = read_sd_blob(sd_blob)
blob_hashes = [blob.get("blob_hash") for blob in decoded['blobs']
if blob.get("blob_hash")]
@ -2488,10 +2517,7 @@ class Daemon(AuthJSONRPCServer):
def get_sd_hash(stream_info):
if not stream_info:
return None
try:
return stream_info['sources']['lbry_sd_hash']
except KeyError:
return stream_info.get('stream_hash')
return stream_info['sources']['lbry_sd_hash']
class _DownloadNameHelper(object):
@ -2510,7 +2536,7 @@ class _DownloadNameHelper(object):
@defer.inlineCallbacks
def setup_stream(self, stream_info):
sd_hash = get_sd_hash(stream_info)
lbry_file = yield self.daemon._get_lbry_file_by_sd_hash(sd_hash)
lbry_file = yield self.daemon._get_lbry_file(FileID.SD_HASH, sd_hash)
if self._does_lbry_file_exists(lbry_file):
defer.returnValue(lbry_file)
else:
@ -2673,6 +2699,8 @@ class _GetFileHelper(object):
return self.daemon._get_lbry_file_by_sd_hash(self.val)
elif self.search_by == FileID.FILE_NAME:
return self.daemon._get_lbry_file_by_file_name(self.val)
elif self.search_by == FileID.STREAM_HASH:
return self.daemon._get_lbry_file_by_stream_hash(self.val)
raise Exception('{} is not a valid search operation'.format(self.search_by))
def _get_json(self, lbry_file):
@ -2806,7 +2834,7 @@ def report_bug_to_slack(message, installation_id, platform_name, app_version):
def get_lbry_file_search_value(search_fields):
for searchtype in (FileID.SD_HASH, FileID.NAME, FileID.FILE_NAME):
for searchtype in (FileID.SD_HASH, FileID.NAME, FileID.FILE_NAME, FileID.STREAM_HASH):
value = search_fields.get(searchtype)
if value:
return searchtype, value

8
tests/unit/core/client/test_ConnectionManager.py
View file

@ -3,7 +3,7 @@ import time
import logging
from lbrynet.core import log_support
from lbrynet.core.client.ConnectionManager import ConnectionManager
#from lbrynet.core.client.ConnectionManager import ConnectionManager
from lbrynet.core.client.ClientRequest import ClientRequest
from lbrynet.core.server.ServerProtocol import ServerProtocol
from lbrynet.core.RateLimiter import RateLimiter
@ -16,6 +16,7 @@ from twisted.internet import defer, reactor, task
from twisted.internet.task import deferLater
from twisted.internet.protocol import Protocol, ServerFactory
from lbrynet import conf
from lbrynet.core import utils
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler, IRequestCreator
from zope.interface import implements
@ -122,11 +123,12 @@ class TestIntegrationConnectionManager(unittest.TestCase):
self.downloader = MocDownloader()
self.rate_limiter = RateLimiter()
self.primary_request_creator = MocRequestCreator([self.TEST_PEER])
self.clock = task.Clock()
utils.call_later = self.clock.callLater
from lbrynet.core.client.ConnectionManager import ConnectionManager
self.connection_manager = ConnectionManager(self.downloader, self.rate_limiter,
[self.primary_request_creator], [])
self.clock = task.Clock()
self.connection_manager.callLater = self.clock.callLater
self.connection_manager._start()
self.server_port = None

7
tests/unit/core/server/test_DHTHashAnnouncer.py
View file

@ -2,9 +2,7 @@ import os
import binascii
from twisted.trial import unittest
from twisted.internet import defer,task
from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer,DHTHashSupplier
from lbrynet.core.utils import random_string
from lbrynet.core import log_support
from lbrynet.core import log_support, utils
class MocDHTNode(object):
@ -35,8 +33,9 @@ class DHTHashAnnouncerTest(unittest.TestCase):
self.blobs_to_announce.append(binascii.b2a_hex(os.urandom(32)))
self.clock = task.Clock()
self.dht_node = MocDHTNode()
utils.call_later = self.clock.callLater
from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer,DHTHashSupplier
self.announcer = DHTHashAnnouncer(self.dht_node, peer_port=3333)
self.announcer.callLater = self.clock.callLater
self.supplier = MocSupplier(self.blobs_to_announce)
self.announcer.add_supplier(self.supplier)