Merge pull request #29 from lbryio/development

0.2.3
This commit is contained in:
Jack Robison 2016-05-06 18:06:46 -04:00
commit 7d5c618246
14 changed files with 1321 additions and 761 deletions

View file

@ -4,5 +4,5 @@ import logging
logging.getLogger(__name__).addHandler(logging.NullHandler())
version = (0, 2, 2)
version = (0, 2, 3)
__version__ = ".".join([str(x) for x in version])

View file

@ -18,8 +18,6 @@ MIN_BLOB_INFO_PAYMENT_RATE = .02 # points/1000 infos
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05 # points/1000 infos
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .05 # points/1000 infos
MAX_CONNECTIONS_PER_STREAM = 5
DEFAULT_MAX_SEARCH_RESULTS = 25
DEFAULT_MAX_KEY_FEE = 100.0
KNOWN_DHT_NODES = [('104.236.42.182', 4000)]
@ -33,11 +31,13 @@ API_ADDRESS = "lbryapi"
API_PORT = 5279
ICON_PATH = "app.icns"
APP_NAME = "LBRY"
DEFAULT_WALLET = "lbryum"
API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRESS)
UI_ADDRESS = "http://" + API_INTERFACE + ":" + str(API_PORT)
PROTOCOL_PREFIX = "lbry"
DEFAULT_WALLET = "lbryum"
DEFAULT_TIMEOUT = 30
DEFAULT_MAX_SEARCH_RESULTS = 25
DEFAULT_MAX_KEY_FEE = 100.0
DEFAULT_SEARCH_TIMEOUT = 3.0
DEFAULT_CACHE_TIME = 3600

View file

@ -1014,7 +1014,7 @@ class LBRYumWallet(LBRYWallet):
self.max_behind = self.blocks_behind_alert
self.catchup_progress = int(100 * (self.blocks_behind_alert / (5 + self.max_behind)))
if self._caught_up_counter == 0:
alert.info('Catching up to the blockchain...showing blocks left...')
alert.info('Catching up with the blockchain...showing blocks left...')
if self._caught_up_counter % 30 == 0:
alert.info('%d...', (remote_height - local_height))
@ -1136,6 +1136,39 @@ class LBRYumWallet(LBRYWallet):
func = getattr(self.cmd_runner, cmd.name)
return threads.deferToThread(func)
def get_history(self):
cmd = known_commands['history']
func = getattr(self.cmd_runner, cmd.name)
return threads.deferToThread(func)
def get_tx_json(self, txid):
def _decode(raw_tx):
tx = Transaction(raw_tx).deserialize()
decoded_tx = {}
for txkey in tx.keys():
if isinstance(tx[txkey], list):
decoded_tx[txkey] = []
for i in tx[txkey]:
tmp = {}
for k in i.keys():
if isinstance(i[k], Decimal):
tmp[k] = float(i[k] / 1e8)
else:
tmp[k] = i[k]
decoded_tx[txkey].append(tmp)
else:
decoded_tx[txkey] = tx[txkey]
return decoded_tx
d = self._get_raw_tx(txid)
d.addCallback(_decode)
return d
def get_pub_keys(self, wallet):
cmd = known_commands['getpubkeys']
func = getattr(self.cmd_runner, cmd.name)
return threads.deferToThread(func, wallet)
def _save_wallet(self, val):
d = threads.deferToThread(self.wallet.storage.write)
d.addCallback(lambda _: val)

View file

@ -211,10 +211,20 @@ class LBRYFileSaver(LBRYFileDownloader):
file_name = "_"
if os.path.exists(os.path.join(self.download_directory, file_name)):
ext_num = 1
def _get_file_name(ext):
if len(file_name.split(".")):
fn = ''.join(file_name.split(".")[:-1])
file_ext = ''.join(file_name.split(".")[-1])
return fn + "-" + str(ext) + "." + file_ext
else:
return file_name + "_" + str(ext)
while os.path.exists(os.path.join(self.download_directory,
file_name + "_" + str(ext_num))):
_get_file_name(ext_num))):
ext_num += 1
file_name = file_name + "_" + str(ext_num)
file_name = _get_file_name(ext_num)
try:
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
self.file_written_to = os.path.join(self.download_directory, file_name)

View file

@ -131,7 +131,8 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
def make_stream_desc_file(stream_hash):
log.debug("creating the stream descriptor file")
descriptor_writer = PlainStreamDescriptorWriter(file_name + conf.CRYPTSD_FILE_EXTENSION)
descriptor_file_path = os.path.join(session.db_dir, file_name + conf.CRYPTSD_FILE_EXTENSION)
descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path)
d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True)

View file

@ -24,12 +24,22 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
LBRYFileSaver.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
stream_info_manager, payment_rate_manager, wallet, download_directory,
upload_allowed, file_name)
self.sd_hash = None
self.rowid = rowid
self.lbry_file_manager = lbry_file_manager
self.saving_status = False
def restore(self):
d = self.lbry_file_manager.get_lbry_file_status(self)
d = self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash)
def _save_sd_hash(sd_hash):
if len(sd_hash):
self.sd_hash = sd_hash[0]
return defer.succeed(None)
d.addCallback(_save_sd_hash)
d.addCallback(lambda _: self.lbry_file_manager.get_lbry_file_status(self))
def restore_status(status):
if status == ManagedLBRYFileDownloader.STATUS_RUNNING:
@ -87,6 +97,14 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
d = LBRYFileSaver._start(self)
d.addCallback(lambda _: self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash))
def _save_sd_hash(sd_hash):
self.sd_hash = sd_hash[0]
return defer.succeed(None)
d.addCallback(_save_sd_hash)
d.addCallback(lambda _: self._save_status())
return d
@ -119,7 +137,7 @@ class ManagedLBRYFileDownloaderFactory(object):
def can_download(self, sd_validator):
return True
def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None):
def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None, file_name=None):
data_rate = options[0]
upload_allowed = options[1]
@ -138,7 +156,8 @@ class ManagedLBRYFileDownloaderFactory(object):
payment_rate_manager,
data_rate,
upload_allowed,
download_directory=download_directory))
download_directory=download_directory,
file_name=file_name))
return d
@staticmethod

View file

@ -4,10 +4,7 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
import logging
import os
import sys
from datetime import datetime
from twisted.internet.task import LoopingCall
from twisted.enterprise import adbapi
from twisted.internet import defer, task, reactor
from twisted.python.failure import Failure
@ -28,14 +25,12 @@ class LBRYFileManager(object):
Keeps track of currently opened LBRY Files, their options, and their LBRY File specific metadata.
"""
def __init__(self, session, stream_info_manager, sd_identifier, delete_data=False, download_directory=None):
def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
self.session = session
self.stream_info_manager = stream_info_manager
self.sd_identifier = sd_identifier
self.lbry_files = []
self.sql_db = None
# self.delete_data = delete_data
# self.check_exists_loop = LoopingCall(self.check_files_exist)
if download_directory:
self.download_directory = download_directory
else:
@ -43,35 +38,11 @@ class LBRYFileManager(object):
log.debug("Download directory for LBRYFileManager: %s", str(self.download_directory))
def setup(self):
# self.check_exists_loop.start(10)
d = self._open_db()
d.addCallback(lambda _: self._add_to_sd_identifier())
d.addCallback(lambda _: self._start_lbry_files())
return d
# def check_files_exist(self):
# def _disp(deleted_files):
# if deleted_files[0][0]:
# for file in bad_files:
# log.info("[" + str(datetime.now()) + "] Detected " + file.file_name + " was deleted, removing from file manager")
#
# def _delete_stream_data(lbry_file):
# s_h = lbry_file.stream_hash
# d = self.get_count_for_stream_hash(s_h)
# # TODO: could possibly be a timing issue here
# d.addCallback(lambda c: self.stream_info_manager.delete_stream(s_h) if c == 0 else True)
# return d
#
# bad_files = [lbry_file for lbry_file in self.lbry_files
# if lbry_file.completed == True and
# os.path.isfile(os.path.join(self.download_directory, lbry_file.file_name)) == False]
# d = defer.DeferredList([self.delete_lbry_file(lbry_file) for lbry_file in bad_files], consumeErrors=True)
# d.addCallback(lambda files: _disp(files) if len(files) else defer.succeed(None))
#
# if self.delete_data:
# d2 = defer.DeferredList([_delete_stream_data(lbry_file) for lbry_file in bad_files], consumeErrors=True)
def get_lbry_file_status(self, lbry_file):
return self._get_lbry_file_status(lbry_file.rowid)
@ -123,7 +94,7 @@ class LBRYFileManager(object):
return d
def start_lbry_file(self, rowid, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
download_directory=None):
download_directory=None, file_name=None):
if not download_directory:
download_directory = self.download_directory
payment_rate_manager.min_blob_data_payment_rate = blob_data_rate
@ -134,16 +105,18 @@ class LBRYFileManager(object):
self.stream_info_manager, self,
payment_rate_manager, self.session.wallet,
download_directory,
upload_allowed)
upload_allowed,
file_name=file_name)
self.lbry_files.append(lbry_file_downloader)
d = lbry_file_downloader.set_stream_info()
d.addCallback(lambda _: lbry_file_downloader)
return d
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True, download_directory=None):
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
download_directory=None, file_name=None):
d = self._save_lbry_file(stream_hash, blob_data_rate)
d.addCallback(lambda rowid: self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
blob_data_rate, upload_allowed, download_directory))
blob_data_rate, upload_allowed, download_directory, file_name))
return d
def delete_lbry_file(self, lbry_file):
@ -183,7 +156,6 @@ class LBRYFileManager(object):
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
def stop(self):
# self.check_exists_loop.stop()
ds = []

View file

@ -11,73 +11,39 @@ API_CONNECTION_STRING = "http://localhost:5279/lbryapi"
UI_ADDRESS = "http://localhost:5279"
class Timeout(Exception):
def __init__(self, value):
self.parameter = value
def __str__(self):
return repr(self.parameter)
class LBRYURIHandler(object):
def __init__(self):
self.started_daemon = False
self.start_timeout = 0
self.daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING)
def check_status(self):
status = None
try:
status = self.daemon.is_running()
if self.start_timeout < 30 and not status:
sleep(1)
self.start_timeout += 1
self.check_status()
elif status:
return True
else:
raise Timeout("LBRY daemon is running, but connection timed out")
except:
if self.start_timeout < 30:
sleep(1)
self.start_timeout += 1
self.check_status()
else:
raise Timeout("Timed out trying to start LBRY daemon")
def handle_osx(self, lbry_name):
lbry_process = [d for d in subprocess.Popen(['ps','aux'], stdout=subprocess.PIPE).stdout.readlines()
if 'LBRY.app' in d and 'LBRYURIHandler' not in d]
try:
status = self.daemon.is_running()
except:
status = None
if lbry_process or status:
self.check_status()
started = False
else:
os.system("open /Applications/LBRY.app")
self.check_status()
started = True
sleep(3)
if lbry_name == "lbry" or lbry_name == "" and not started:
if lbry_name == "lbry" or lbry_name == "":
webbrowser.open(UI_ADDRESS)
else:
webbrowser.open(UI_ADDRESS + "/view?name=" + lbry_name)
webbrowser.open(UI_ADDRESS + "/?watch=" + lbry_name)
def handle_linux(self, lbry_name):
try:
is_running = self.daemon.is_running()
if not is_running:
sys.exit(0)
status = self.daemon.is_running()
except:
sys.exit(0)
cmd = r'DIR = "$( cd "$(dirname "${BASH_SOURCE[0]}" )" && pwd )"' \
r'if [-z "$(pgrep lbrynet-daemon)"]; then' \
r'echo "running lbrynet-daemon..."' \
r'$DIR / lbrynet - daemon &' \
r'sleep 3 # let the daemon load before connecting' \
r'fi'
subprocess.Popen(cmd, shell=True)
if lbry_name == "lbry":
if lbry_name == "lbry" or lbry_name == "":
webbrowser.open(UI_ADDRESS)
else:
webbrowser.open(UI_ADDRESS + "/view?name=" + lbry_name)
webbrowser.open(UI_ADDRESS + "/?watch=" + lbry_name)
def main(args):

File diff suppressed because it is too large Load diff

View file

@ -1,25 +1,21 @@
import argparse
import logging
import logging.handlers
import subprocess
import os
import shutil
import webbrowser
import sys
import socket
import platform
from StringIO import StringIO
from zipfile import ZipFile
from urllib import urlopen
from datetime import datetime
from appdirs import user_data_dir
from twisted.web import server, static
from twisted.web import server
from twisted.internet import reactor, defer
from jsonrpc.proxy import JSONRPCProxy
from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon, LBRYindex, LBRYFileRender
from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer
from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, DEFAULT_WALLET, UI_ADDRESS
if sys.platform != "darwin":
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
@ -30,10 +26,11 @@ if not os.path.isdir(log_dir):
LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log')
log = logging.getLogger(__name__)
handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=262144, backupCount=5)
handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, backupCount=5)
log.addHandler(handler)
log.setLevel(logging.INFO)
REMOTE_SERVER = "www.google.com"
@ -48,9 +45,11 @@ def test_internet_connection():
def stop():
def _disp_shutdown():
print "Shutting down lbrynet-daemon from command line"
log.info("Shutting down lbrynet-daemon from command line")
def _disp_not_running():
print "Attempt to shut down lbrynet-daemon from command line when daemon isn't running"
log.info("Attempt to shut down lbrynet-daemon from command line when daemon isn't running")
d = defer.Deferred(None)
@ -67,18 +66,26 @@ def start():
default=DEFAULT_WALLET)
parser.add_argument("--ui",
help="path to custom UI folder",
default="")
default=None)
parser.add_argument("--branch",
help="Branch of lbry-web-ui repo to use, defaults on HEAD",
default="HEAD")
help="Branch of lbry-web-ui repo to use, defaults on master",
default="master")
parser.add_argument('--no-launch', dest='launchui', action="store_false")
parser.set_defaults(launchui=True)
parser.add_argument('--log-to-console', dest='logtoconsole', action="store_true")
parser.add_argument('--quiet', dest='quiet', action="store_true")
parser.set_defaults(launchui=True, logtoconsole=False, quiet=False)
args = parser.parse_args()
if args.logtoconsole:
logging.basicConfig(level=logging.INFO)
args = parser.parse_args()
try:
JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()
log.info("lbrynet-daemon is already running")
if not args.logtoconsole:
print "lbrynet-daemon is already running"
if args.launchui:
webbrowser.open(UI_ADDRESS)
return
@ -86,95 +93,31 @@ def start():
pass
log.info("Starting lbrynet-daemon from command line")
if not args.logtoconsole and not args.quiet:
print "Starting lbrynet-daemon from command line"
print "To view activity, view the log file here: " + LOG_FILENAME
print "Web UI is available at http://%s:%i" % (API_INTERFACE, API_PORT)
print "JSONRPC API is available at " + API_CONNECTION_STRING
print "To quit press ctrl-c or call 'stop' via the API"
if args.branch == "HEAD":
GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep %s | cut -f 1" % args.branch
DIST_URL = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip"
else:
log.info("Using UI branch: " + args.branch)
GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep refs/heads/%s | cut -f 1" % args.branch
DIST_URL = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % args.branch
def getui(ui_dir=None):
if ui_dir:
if os.path.isdir(ui_dir):
log.info("Using user specified UI directory: " + str(ui_dir))
ui_version_info = "user-specified"
return defer.succeed([ui_dir, ui_version_info])
else:
log.info("User specified UI directory doesn't exist: " + str(ui_dir))
def download_ui(dest_dir, ui_version):
url = urlopen(DIST_URL)
z = ZipFile(StringIO(url.read()))
names = [i for i in z.namelist() if '.DS_Store' not in i and '__MACOSX' not in i]
z.extractall(dest_dir, members=names)
return defer.succeed([dest_dir, ui_version])
data_dir = user_data_dir("LBRY")
version_dir = os.path.join(data_dir, "ui_version_history")
git_version = subprocess.check_output(GIT_CMD_STRING, shell=True)
if not git_version:
log.info("You should have been notified to install xcode command line tools, once it's installed you can start LBRY")
print "You should have been notified to install xcode command line tools, once it's installed you can start LBRY"
sys.exit(0)
ui_version_info = git_version
if not os.path.isdir(data_dir):
os.mkdir(data_dir)
if not os.path.isdir(os.path.join(data_dir, "ui_version_history")):
os.mkdir(version_dir)
if not os.path.isfile(os.path.join(version_dir, git_version)):
f = open(os.path.join(version_dir, git_version), "w")
version_message = "[" + str(datetime.now()) + "] Updating UI --> " + git_version
f.write(version_message)
f.close()
log.info(version_message)
if os.path.isdir(os.path.join(data_dir, "lbry-web-ui")):
shutil.rmtree(os.path.join(data_dir, "lbry-web-ui"))
else:
version_message = "[" + str(datetime.now()) + "] UI version " + git_version + " up to date"
log.info(version_message)
if os.path.isdir(os.path.join(data_dir, "lbry-web-ui")):
return defer.succeed([os.path.join(data_dir, "lbry-web-ui"), ui_version_info])
else:
return download_ui(os.path.join(data_dir, "lbry-web-ui"), ui_version_info)
def setupserver(ui_dir, ui_version):
root = LBRYindex(ui_dir)
root.putChild("css", static.File(os.path.join(ui_dir, "css")))
root.putChild("font", static.File(os.path.join(ui_dir, "font")))
root.putChild("img", static.File(os.path.join(ui_dir, "img")))
root.putChild("js", static.File(os.path.join(ui_dir, "js")))
root.putChild("view", LBRYFileRender())
return defer.succeed([root, ui_version])
def setupapi(root, wallet, ui_version):
daemon = LBRYDaemon(ui_version, wallet_type=wallet)
root.putChild(API_ADDRESS, daemon)
reactor.listenTCP(API_PORT, server.Site(root), interface=API_INTERFACE)
return daemon.setup()
if test_internet_connection():
d = getui(args.ui)
d.addCallback(lambda r: setupserver(r[0], r[1]))
d.addCallback(lambda r: setupapi(r[0], args.wallet, r[1]))
lbry = LBRYDaemonServer()
d = lbry.start(branch=args.branch, user_specified=args.ui)
if args.launchui:
d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
reactor.listenTCP(API_PORT, server.Site(lbry.root), interface=API_INTERFACE)
reactor.run()
if not args.logtoconsole and not args.quiet:
print "\nClosing lbrynet-daemon"
else:
log.info("Not connected to internet, unable to start")
if not args.logtoconsole:
print "Not connected to internet, unable to start"
return
if __name__ == "__main__":
start()

View file

@ -0,0 +1,353 @@
import logging
import os
import shutil
import json
import sys
import mimetypes
from StringIO import StringIO
from zipfile import ZipFile
from urllib import urlopen
from datetime import datetime
from appdirs import user_data_dir
from twisted.web import server, static, resource
from twisted.internet import defer, interfaces, error, reactor, task, threads
from twisted.python.failure import Failure
from txjsonrpc.web import jsonrpc
from zope.interface import implements
from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon
from lbrynet.conf import API_CONNECTION_STRING, API_ADDRESS, DEFAULT_WALLET, UI_ADDRESS
if sys.platform != "darwin":
data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
data_dir = user_data_dir("LBRY")
if not os.path.isdir(data_dir):
os.mkdir(data_dir)
version_dir = os.path.join(data_dir, "ui_version_history")
if not os.path.isdir(version_dir):
os.mkdir(version_dir)
version_log = logging.getLogger("lbry_version")
version_log.addHandler(logging.FileHandler(os.path.join(version_dir, "lbry_version.log")))
version_log.setLevel(logging.INFO)
log = logging.getLogger(__name__)
log.addHandler(logging.FileHandler(os.path.join(data_dir, 'lbrynet-daemon.log')))
log.setLevel(logging.INFO)
class LBRYindex(resource.Resource):
def __init__(self, ui_dir):
resource.Resource.__init__(self)
self.ui_dir = ui_dir
isLeaf = False
def _delayed_render(self, request, results):
request.write(str(results))
request.finish()
def getChild(self, name, request):
if name == '':
return self
return resource.Resource.getChild(self, name, request)
def render_GET(self, request):
return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request)
class LBRYFileStreamer(object):
"""
Writes downloaded LBRY file to request as the download comes in, pausing and resuming as requested
used for Chrome
"""
implements(interfaces.IPushProducer)
def __init__(self, request, path, start, stop, size):
self._request = request
self._fileObject = file(path)
self._content_type = mimetypes.guess_type(path)[0]
self._stop_pos = size - 1 if stop == '' else int(stop) #chrome and firefox send range requests for "0-"
self._cursor = self._start_pos = int(start)
self._file_size = size
self._depth = 0
self._paused = self._sent_bytes = self._stopped = False
self._delay = 0.25
self._deferred = defer.succeed(None)
self._request.setResponseCode(206)
self._request.setHeader('accept-ranges', 'bytes')
self._request.setHeader('content-type', self._content_type)
self.resumeProducing()
def pauseProducing(self):
self._paused = True
log.info("[" + str(datetime.now()) + "] Pausing producer")
return defer.succeed(None)
def resumeProducing(self):
def _check_for_new_data():
self._depth += 1
self._fileObject.seek(self._start_pos, os.SEEK_END)
readable_bytes = self._fileObject.tell()
self._fileObject.seek(self._cursor)
self._sent_bytes = False
if (readable_bytes > self._cursor) and not (self._stopped or self._paused):
read_length = min(readable_bytes, self._stop_pos) - self._cursor + 1
self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._cursor, self._cursor + read_length - 1, self._file_size))
self._request.setHeader('content-length', str(read_length))
start_cur = self._cursor
for i in range(read_length):
if self._paused or self._stopped:
break
else:
data = self._fileObject.read(1)
self._request.write(data)
self._cursor += 1
log.info("[" + str(datetime.now()) + "] Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" %
(start_cur, self._cursor, self._file_size, self._cursor - start_cur, readable_bytes, self._depth))
self._sent_bytes = True
if self._cursor == self._stop_pos + 1:
self.stopProducing()
return defer.succeed(None)
elif self._paused or self._stopped:
return defer.succeed(None)
else:
self._deferred.addCallback(lambda _: threads.deferToThread(reactor.callLater, self._delay, _check_for_new_data))
return defer.succeed(None)
log.info("[" + str(datetime.now()) + "] Resuming producer")
self._paused = False
self._deferred.addCallback(lambda _: _check_for_new_data())
def stopProducing(self):
log.info("[" + str(datetime.now()) + "] Stopping producer")
self._stopped = True
# self._fileObject.close()
self._deferred.addErrback(lambda err: err.trap(defer.CancelledError))
self._deferred.addErrback(lambda err: err.trap(error.ConnectionDone))
self._deferred.cancel()
# self._request.finish()
self._request.unregisterProducer()
return defer.succeed(None)
class HostedLBRYFile(resource.Resource):
def __init__(self, api):
self._api = api
self._producer = None
resource.Resource.__init__(self)
def makeProducer(self, request, stream):
def _save_producer(producer):
self._producer = producer
return defer.succeed(None)
range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-')
start, stop = int(range_header[0]), range_header[1]
log.info("[" + str(datetime.now()) + "] GET range %s-%s" % (start, stop))
path = os.path.join(self._api.download_directory, stream.file_name)
d = stream.get_total_bytes()
d.addCallback(lambda size: _save_producer(LBRYFileStreamer(request, path, start, stop, size)))
d.addCallback(lambda _: request.registerProducer(self._producer, streaming=True))
# request.notifyFinish().addCallback(lambda _: self._producer.stopProducing())
request.notifyFinish().addErrback(self._responseFailed, d)
return d
def render_GET(self, request):
if 'name' in request.args.keys():
if request.args['name'][0] != 'lbry' and request.args['name'][0] not in self._api.waiting_on.keys():
d = self._api._download_name(request.args['name'][0])
# d.addCallback(lambda stream: self.makeProducer(request, stream))
d.addCallback(lambda stream: static.File(os.path.join(self._api.download_directory,
stream.file_name)).render_GET(request))
elif request.args['name'][0] in self._api.waiting_on.keys():
request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0])
request.finish()
else:
request.redirect(UI_ADDRESS)
request.finish()
return server.NOT_DONE_YET
def _responseFailed(self, err, call):
call.addErrback(lambda err: err.trap(error.ConnectionDone))
call.addErrback(lambda err: err.trap(defer.CancelledError))
call.addErrback(lambda err: log.info("Error: " + str(err)))
call.cancel()
class MyLBRYFiles(resource.Resource):
isLeaf = False
def __init__(self):
resource.Resource.__init__(self)
self.files_table = None
def delayed_render(self, request, result):
request.write(result.encode('utf-8'))
request.finish()
def render_GET(self, request):
self.files_table = None
api = jsonrpc.Proxy(API_CONNECTION_STRING)
d = api.callRemote("get_lbry_files", {})
d.addCallback(self._get_table)
d.addCallback(lambda results: self.delayed_render(request, results))
return server.NOT_DONE_YET
def _get_table(self, files):
if not self.files_table:
self.files_table = r'<html><head><title>My LBRY files</title></head><body><table border="1">'
self.files_table += r'<tr>'
self.files_table += r'<td>Stream name</td>'
self.files_table += r'<td>Completed</td>'
self.files_table += r'<td>Toggle</td>'
self.files_table += r'<td>Remove</td>'
self.files_table += r'</tr>'
return self._get_table(files)
if not len(files):
self.files_table += r'</table></body></html>'
return self.files_table
else:
f = files.pop()
self.files_table += r'<tr>'
self.files_table += r'<td>%s</td>' % (f['stream_name'])
self.files_table += r'<td>%s</td>' % (f['completed'])
self.files_table += r'<td>Start</td>' if f['stopped'] else r'<td>Stop</td>'
self.files_table += r'<td>Delete</td>'
self.files_table += r'</tr>'
return self._get_table(files)
class LBRYDaemonServer(object):
def __init__(self):
self.data_dir = user_data_dir("LBRY")
if not os.path.isdir(self.data_dir):
os.mkdir(self.data_dir)
self.version_dir = os.path.join(self.data_dir, "ui_version_history")
if not os.path.isdir(self.version_dir):
os.mkdir(self.version_dir)
self.config = os.path.join(self.version_dir, "active.json")
self.ui_dir = os.path.join(self.data_dir, "lbry-web-ui")
self.git_version = None
self._api = None
self.root = None
if not os.path.isfile(os.path.join(self.config)):
self.loaded_git_version = None
else:
try:
f = open(self.config, "r")
loaded_ui = json.loads(f.read())
f.close()
self.loaded_git_version = loaded_ui['commit']
self.loaded_branch = loaded_ui['branch']
version_log.info("[" + str(datetime.now()) + "] Last used " + self.loaded_branch + " commit " + str(self.loaded_git_version).replace("\n", ""))
except:
self.loaded_git_version = None
self.loaded_branch = None
def setup(self, branch="master", user_specified=None):
self.branch = branch
if user_specified:
if os.path.isdir(user_specified):
log.info("Using user specified UI directory: " + str(user_specified))
self.branch = "user-specified"
self.loaded_git_version = "user-specified"
self.ui_dir = user_specified
return defer.succeed("user-specified")
else:
log.info("User specified UI directory doesn't exist, using " + branch)
else:
log.info("Using UI branch: " + branch)
self._git_url = "https://api.github.com/repos/lbryio/lbry-web-ui/git/refs/heads/%s" % branch
self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % branch
d = self._up_to_date()
d.addCallback(lambda r: self._download_ui() if not r else self.branch)
return d
def _up_to_date(self):
def _get_git_info():
response = urlopen(self._git_url)
data = json.loads(response.read())
return defer.succeed(data['object']['sha'])
def _set_git(version):
self.git_version = version
version_log.info("[" + str(datetime.now()) + "] UI branch " + self.branch + " has a most recent commit of: " + str(self.git_version).replace("\n", ""))
if self.git_version == self.loaded_git_version and os.path.isdir(self.ui_dir):
version_log.info("[" + str(datetime.now()) + "] local copy of " + self.branch + " is up to date")
return defer.succeed(True)
else:
if self.git_version == self.loaded_git_version:
version_log.info("[" + str(datetime.now()) + "] Can't find ui files, downloading them again")
else:
version_log.info("[" + str(datetime.now()) + "] local copy of " + self.branch + " branch is out of date, updating")
f = open(self.config, "w")
f.write(json.dumps({'commit': self.git_version,
'time': str(datetime.now()),
'branch': self.branch}))
f.close()
return defer.succeed(False)
d = _get_git_info()
d.addCallback(_set_git)
return d
def _download_ui(self):
def _delete_ui_dir():
if os.path.isdir(self.ui_dir):
if self.loaded_git_version:
version_log.info("[" + str(datetime.now()) + "] Removed ui files for commit " + str(self.loaded_git_version).replace("\n", ""))
log.info("Removing out of date ui files")
shutil.rmtree(self.ui_dir)
return defer.succeed(None)
def _dl_ui():
url = urlopen(self._dist_url)
z = ZipFile(StringIO(url.read()))
names = [i for i in z.namelist() if '.DS_exStore' not in i and '__MACOSX' not in i]
z.extractall(self.ui_dir, members=names)
version_log.info("[" + str(datetime.now()) + "] Updated branch " + self.branch + ": " + str(self.loaded_git_version).replace("\n", "") + " --> " + self.git_version.replace("\n", ""))
log.info("Downloaded files for UI commit " + str(self.git_version).replace("\n", ""))
self.loaded_git_version = self.git_version
return self.branch
d = _delete_ui_dir()
d.addCallback(lambda _: _dl_ui())
return d
def _setup_server(self, ui_ver, wallet):
self._api = LBRYDaemon(ui_ver, wallet_type=wallet)
self.root = LBRYindex(self.ui_dir)
self.root.putChild("css", static.File(os.path.join(self.ui_dir, "css")))
self.root.putChild("font", static.File(os.path.join(self.ui_dir, "font")))
self.root.putChild("img", static.File(os.path.join(self.ui_dir, "img")))
self.root.putChild("js", static.File(os.path.join(self.ui_dir, "js")))
self.root.putChild("view", HostedLBRYFile(self._api))
self.root.putChild("files", MyLBRYFiles())
self.root.putChild(API_ADDRESS, self._api)
return defer.succeed(True)
def start(self, branch="master", user_specified=False, wallet=DEFAULT_WALLET):
d = self.setup(branch=branch, user_specified=user_specified)
d.addCallback(lambda v: self._setup_server(v, wallet))
d.addCallback(lambda _: self._api.setup())
return d

View file

@ -1,7 +1,9 @@
import json
import logging
import os
import sys
from appdirs import user_data_dir
from datetime import datetime
from twisted.internet import defer
from twisted.internet.task import LoopingCall
@ -12,12 +14,37 @@ from lbrynet.core.StreamDescriptor import download_sd_blob
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
from lbrynet.conf import DEFAULT_TIMEOUT
INITIALIZING_CODE = 'initializing'
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
DOWNLOAD_TIMEOUT_CODE = 'timeout'
DOWNLOAD_RUNNING_CODE = 'running'
DOWNLOAD_STOPPED_CODE = 'stopped'
STREAM_STAGES = [
(INITIALIZING_CODE, 'Initializing...'),
(DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
(DOWNLOAD_RUNNING_CODE, 'Started stream'),
(DOWNLOAD_STOPPED_CODE, 'Paused stream'),
(DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
]
if sys.platform != "darwin":
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
log_dir = user_data_dir("LBRY")
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log')
log = logging.getLogger(__name__)
handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, backupCount=5)
log.addHandler(handler)
log.setLevel(logging.INFO)
class GetStream(object):
def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee, pay_key=True, data_rate=0.5,
timeout=DEFAULT_TIMEOUT, download_directory=None):
timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None):
self.wallet = wallet
self.resolved_name = None
self.description = None
@ -26,6 +53,7 @@ class GetStream(object):
self.data_rate = data_rate
self.pay_key = pay_key
self.name = None
self.file_name = file_name
self.session = session
self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
self.lbry_file_manager = lbry_file_manager
@ -39,24 +67,35 @@ class GetStream(object):
self.timeout_counter = 0
self.download_directory = download_directory
self.download_path = None
self.downloader = None
self.finished = defer.Deferred()
self.checker = LoopingCall(self.check_status)
self.code = STREAM_STAGES[0]
def check_status(self):
self.timeout_counter += 1
if self.download_path and os.path.isfile(self.download_path):
if self.download_path:
self.checker.stop()
return defer.succeed(True)
self.finished.callback((self.stream_hash, self.download_path))
elif self.timeout_counter >= self.timeout:
log.info("Timeout downloading " + str(self.stream_info))
log.info("Timeout downloading lbry://" + self.resolved_name + ", " + str(self.stream_info))
self.checker.stop()
self.d.cancel()
self.code = STREAM_STAGES[4]
self.finished.callback(False)
def start(self, stream_info):
def start(self, stream_info, name):
self.resolved_name = name
self.stream_info = stream_info
if 'stream_hash' in self.stream_info.keys():
self.stream_hash = self.stream_info['stream_hash']
elif 'sources' in self.stream_info.keys():
self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
else:
raise InvalidStreamInfoError(self.stream_info)
if 'description' in self.stream_info.keys():
self.description = self.stream_info['description']
if 'key_fee' in self.stream_info.keys():
self.key_fee = float(self.stream_info['key_fee'])
@ -67,15 +106,6 @@ class GetStream(object):
else:
self.key_fee = None
self.key_fee_address = None
self.stream_hash = self.stream_info['stream_hash']
if isinstance(self.stream_hash, dict):
self.stream_hash = self.stream_hash['sd_hash']
else:
log.error("InvalidStreamInfoError in autofetcher: ", stream_info)
raise InvalidStreamInfoError(self.stream_info)
if self.key_fee > self.max_key_fee:
if self.pay_key:
log.info("Key fee (" + str(self.key_fee) + ") above limit of " + str(
@ -84,21 +114,31 @@ class GetStream(object):
else:
pass
def _cause_timeout():
self.timeout_counter = self.timeout * 2
def _set_status(x, status):
self.code = next(s for s in STREAM_STAGES if s[0] == status)
return x
self.checker.start(1)
self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self.d.addCallback(lambda metadata: (next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)), metadata))
self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
self.d.addCallback(lambda metadata: (
next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)),
metadata))
self.d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata,
[self.data_rate, True],
self.payment_rate_manager,
download_directory=self.download_directory))
self.d.addErrback(lambda err: err.trap(defer.CancelledError))
self.d.addErrback(lambda err: log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback()))
self.d.addCallback(self._start_download)
download_directory=self.download_directory,
file_name=self.file_name))
self.d.addCallbacks(self._start_download, lambda _: _cause_timeout())
self.d.callback(None)
return self.d
return self.finished
def _start_download(self, downloader):
def _pay_key_fee():
@ -114,126 +154,7 @@ class GetStream(object):
d = _pay_key_fee()
else:
d = defer.Deferred()
downloader.start()
self.downloader = downloader
self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
d.addCallback(lambda _: log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
return d
class FetcherDaemon(object):
def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf,
verbose=False):
self.autofetcher_conf = autofetcher_conf
self.max_key_fee = 0.0
self.sd_identifier = sd_identifier
self.wallet = wallet
self.session = session
self.lbry_file_manager = lbry_file_manager
self.lbry_metadata_manager = lbry_file_metadata_manager
self.seen = []
self.lastbestblock = None
self.search = None
self.first_run = True
self.is_running = False
self.verbose = verbose
self._get_autofetcher_conf()
def start(self):
if not self.is_running:
self.is_running = True
self.search = LoopingCall(self._looped_search)
self.search.start(1)
log.info("Starting autofetcher")
else:
log.info("Autofetcher is already running")
def stop(self):
if self.is_running:
self.search.stop()
self.is_running = False
else:
log.info("Autofetcher isn't running, there's nothing to stop")
def check_if_running(self):
if self.is_running:
msg = "Autofetcher is running\n"
msg += "Last block hash: " + str(self.lastbestblock)
else:
msg = "Autofetcher is not running"
return msg
def _get_names(self):
d = self.wallet.get_best_blockhash()
d.addCallback(lambda blockhash: get_new_streams(blockhash) if blockhash != self.lastbestblock else [])
def get_new_streams(blockhash):
self.lastbestblock = blockhash
d = self.wallet.get_block(blockhash)
d.addCallback(lambda block: get_new_streams_in_txes(block['tx'], blockhash))
return d
def get_new_streams_in_txes(txids, blockhash):
ds = []
for t in txids:
d = self.wallet.get_claims_from_tx(t)
d.addCallback(get_new_streams_in_tx, t, blockhash)
ds.append(d)
d = defer.DeferredList(ds, consumeErrors=True)
d.addCallback(lambda result: [r[1] for r in result if r[0]])
d.addCallback(lambda stream_lists: [stream for streams in stream_lists for stream in streams])
return d
def get_new_streams_in_tx(claims, t, blockhash):
rtn = []
if claims:
for claim in claims:
if claim not in self.seen:
msg = "[" + str(datetime.now()) + "] New claim | lbry://" + str(claim['name']) + \
" | stream hash: " + str(json.loads(claim['value'])['stream_hash'])
log.info(msg)
if self.verbose:
print msg
rtn.append((claim['name'], t))
self.seen.append(claim)
else:
if self.verbose:
print "[" + str(datetime.now()) + "] No claims in block", blockhash
return rtn
d.addCallback(lambda streams: defer.DeferredList(
[self.wallet.get_stream_info_from_txid(name, t) for name, t in streams]))
return d
def _download_claims(self, claims):
if claims:
for claim in claims:
stream = GetStream(self.sd_identifier, self.session, self.wallet, self.lbry_file_manager,
self.max_key_fee, pay_key=False)
stream.start(claim[1])
return defer.succeed(None)
def _looped_search(self):
d = self._get_names()
d.addCallback(self._download_claims)
return d
def _get_autofetcher_conf(self):
settings = {"maxkey": "0.0"}
if os.path.exists(self.autofetcher_conf):
conf = open(self.autofetcher_conf)
for l in conf:
if l.startswith("maxkey="):
settings["maxkey"] = float(l[7:].rstrip('\n'))
conf.close()
else:
conf = open(self.autofetcher_conf, "w")
conf.write("maxkey=10.0")
conf.close()
settings["maxkey"] = 10.0
log.info("No autofetcher conf file found, making one with max key fee of 10.0")
self.max_key_fee = settings["maxkey"]
d.addCallback(lambda _: log.info("[" + str(datetime.now()) + "] Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
d.addCallback(lambda _: self.downloader.start())

View file

@ -1,15 +1,30 @@
import logging
import os
import sys
from appdirs import user_data_dir
from datetime import datetime
from lbrynet.core.Error import InsufficientFundsError
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
from twisted.internet import threads, defer
import os
import logging
from datetime import datetime
if sys.platform != "darwin":
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
log_dir = user_data_dir("LBRY")
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log')
log = logging.getLogger(__name__)
handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, backupCount=5)
log.addHandler(handler)
log.setLevel(logging.INFO)
class Publisher(object):

View file

@ -28,7 +28,12 @@ ARG=${1:-}
if [ -z "$ARG" ]; then
URL=""
else
URL="view?name=$(urlencode "$(echo "$ARG" | cut -c 8-)")"
NAME=$(echo "$ARG" | cut -c 8-)
if [ -z "$NAME" -o "$NAME" == "lbry" ]; then
URL=""
else
URL="/?watch=$(urlencode "$NAME")"
fi
fi
/usr/bin/xdg-open "http://localhost:5279/$URL"
/usr/bin/xdg-open "http://localhost:5279$URL"