Merge pull request #5 from lbryio/autodownloader

lbrynet-daemon
This commit is contained in:
jackrobison 2015-12-14 13:19:01 -05:00
commit 28ce23350d
12 changed files with 1085 additions and 2 deletions

14
.gitignore vendored
View file

@ -1 +1,15 @@
*.pyc
*.xml
.idea/.name
*.iml
*.log
*.txt
lbrynet.egg-info/PKG-INFO
*.egg

View file

@ -348,6 +348,9 @@ class LBRYcrdWallet(object):
def _get_rpc_conn(self):
return AuthServiceProxy(self.rpc_conn_string)
def get_rpc_conn_x(self):
return AuthServiceProxy(self.rpc_conn_string)
def _start_daemon(self):
tries = 0

View file

@ -0,0 +1,452 @@
import json
import logging
import os
from datetime import datetime
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.core.StreamDescriptor import download_sd_blob
log = logging.getLogger(__name__)
class AutoAddStream(object):
def __init__(self, console, sd_identifier, session, wallet, lbry_file_manager, max_key_fee):
self.finished_deferred = defer.Deferred(None)
self.console = console
self.wallet = wallet
self.resolved_name = None
self.description = None
self.key_fee = None
self.key_fee_address = None
self.name = None
self.session = session
self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
self.loading_metadata_deferred = defer.Deferred()
self.lbry_file_manager = lbry_file_manager
self.sd_identifier = sd_identifier
self.metadata = None
self.loading_failed = False
self.resolved_name = None
self.description = None
self.key_fee = None
self.key_fee_address = None
self.stream_hash = None
self.max_key_fee = max_key_fee
self.stream_info = None
def start(self, stream_info):
self.stream_info = stream_info
if 'stream_hash' not in json.loads(self.stream_info['value']):
print 'InvalidStreamInfoError'
raise InvalidStreamInfoError(self.stream_info)
self.resolved_name = self.stream_info.get('name', None)
self.description = json.loads(self.stream_info['value']).get('description', None)
try:
if 'key_fee' in json.loads(self.stream_info['value']):
self.key_fee = float(json.loads(self.stream_info['value'])['key_fee'])
except ValueError:
self.key_fee = None
self.key_fee_address = json.loads(self.stream_info['value']).get('key_fee_address', None)
self.stream_hash = json.loads(self.stream_info['value'])['stream_hash']
if self.key_fee > self.max_key_fee:
pass
# self.console.sendLine("Key fee (" + str(self.key_fee) + ") above limit of " + str(
# self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name))
# return self.finished_deferred.callback(None)
self.loading_metadata_deferred = defer.Deferred(None)
self.loading_metadata_deferred.addCallback(
lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
self.loading_metadata_deferred.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self.loading_metadata_deferred.addCallback(self._handle_metadata)
self.loading_metadata_deferred.addErrback(self._handle_load_canceled)
self.loading_metadata_deferred.addErrback(self._handle_load_failed)
self.finished_deferred.addCallback(lambda _: self.loading_metadata_deferred.callback(None))
return self.finished_deferred.callback(None)
def _start_download(self):
#d = self._pay_key_fee()
d = defer.Deferred(None)
d.addCallback(lambda _: self._make_downloader())
d.addCallback(lambda stream_downloader: stream_downloader.start())
d.addErrback(self._handle_download_error)
return d
def _pay_key_fee(self):
if self.key_fee is not None and self.key_fee_address is not None:
reserved_points = self.wallet.reserve_points(self.key_fee_address, self.key_fee)
if reserved_points is None:
return defer.fail(InsufficientFundsError())
return self.wallet.send_points_to_address(reserved_points, self.key_fee)
self.console.sendLine("Sent key fee" + str(self.key_fee_address) + " | " + str(self.key_fee))
return defer.succeed(None)
def _handle_load_canceled(self, err):
err.trap(defer.CancelledError)
self.finished_deferred.callback(None)
def _handle_load_failed(self, err):
self.loading_failed = True
self.console.sendLine("handle load failed: " + str(err.getTraceback()))
log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback())
self.finished_deferred.callback(None)
def _handle_metadata(self, metadata):
self.metadata = metadata
self.factory = self.metadata.factories[0]
self.finished_deferred.addCallback(lambda _: self._start_download())
def _handle_download_error(self, err):
if err.check(InsufficientFundsError):
self.console.sendLine("Download stopped due to insufficient funds.")
else:
self.console.sendLine(
"Autoaddstream: An unexpected error has caused the download to stop: %s" % err.getTraceback())
def _make_downloader(self):
self.downloader = self.factory.make_downloader(self.metadata, [0.5, True], self.payment_rate_manager)
return self.downloader
class GetStream(object):
def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee):
self.finished_deferred = defer.Deferred(None)
self.wallet = wallet
self.resolved_name = None
self.description = None
self.key_fee = None
self.key_fee_address = None
self.name = None
self.session = session
self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
self.loading_metadata_deferred = defer.Deferred()
self.lbry_file_manager = lbry_file_manager
self.sd_identifier = sd_identifier
self.metadata = None
self.loading_failed = False
self.resolved_name = None
self.description = None
self.key_fee = None
self.key_fee_address = None
self.stream_hash = None
self.max_key_fee = max_key_fee
self.stream_info = None
self.stream_info_manager = None
self.downloader = None
def start(self, stream_info):
self.stream_info = stream_info
if 'stream_hash' in self.stream_info.keys():
self.description = self.stream_info['description']
if 'key_fee' in self.stream_info.keys():
self.key_fee = float(self.stream_info['key_fee'])
if 'key_fee_address' in self.stream_info.keys():
self.key_fee_address = self.stream_info['key_fee_address']
else:
self.key_fee_address = None
else:
self.key_fee = None
self.key_fee_address = None
self.stream_hash = self.stream_info['stream_hash']
elif 'stream_hash' in json.loads(self.stream_info['value']):
self.resolved_name = self.stream_info.get('name', None)
self.description = json.loads(self.stream_info['value']).get('description', None)
try:
if 'key_fee' in json.loads(self.stream_info['value']):
self.key_fee = float(json.loads(self.stream_info['value'])['key_fee'])
except ValueError:
self.key_fee = None
self.key_fee_address = json.loads(self.stream_info['value']).get('key_fee_address', None)
self.stream_hash = json.loads(self.stream_info['value'])['stream_hash']
else:
print 'InvalidStreamInfoError'
raise InvalidStreamInfoError(self.stream_info)
if self.key_fee > self.max_key_fee:
print "Key fee (" + str(self.key_fee) + ") above limit of " + str(
self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name)
return self.finished_deferred.callback(None)
else:
pass
def _get_downloader_for_return():
return defer.succeed(self.downloader)
self.loading_metadata_deferred = defer.Deferred(None)
self.loading_metadata_deferred.addCallback(
lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
self.loading_metadata_deferred.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self.loading_metadata_deferred.addCallback(self._handle_metadata)
self.loading_metadata_deferred.addErrback(self._handle_load_canceled)
self.loading_metadata_deferred.addErrback(self._handle_load_failed)
self.loading_metadata_deferred.addCallback(lambda _: self._pay_key_fee())
self.loading_metadata_deferred.addCallback(lambda _: self._make_downloader())
self.loading_metadata_deferred.addCallback(lambda _: self.downloader.start())
self.loading_metadata_deferred.addErrback(self._handle_download_error)
self.loading_metadata_deferred.addCallback(lambda _: _get_downloader_for_return())
self.loading_metadata_deferred.callback(None)
return defer.succeed(None)
def _pay_key_fee(self):
if self.key_fee is not None and self.key_fee_address is not None:
reserved_points = self.wallet.reserve_points(self.key_fee_address, self.key_fee)
if reserved_points is None:
return defer.fail(InsufficientFundsError())
print 'Key fee: ' + str(self.key_fee) + ' | ' + str(self.key_fee_address)
return self.wallet.send_points_to_address(reserved_points, self.key_fee)
return defer.succeed(None)
def _handle_load_canceled(self, err):
err.trap(defer.CancelledError)
self.finished_deferred.callback(None)
def _handle_load_failed(self, err):
self.loading_failed = True
log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback())
print 'Load Failed: ', err.getTraceback()
self.finished_deferred.callback(None)
def _handle_metadata(self, metadata):
self.metadata = metadata
self.factory = self.metadata.factories[0]
return defer.succeed(None)
def _handle_download_error(self, err):
if err.check(InsufficientFundsError):
print "Download stopped due to insufficient funds."
else:
print "Autoaddstream: An unexpected error has caused the download to stop: ", err.getTraceback()
def _make_downloader(self):
def _set_downloader(downloader):
self.downloader = downloader
print "Downloading", self.stream_hash, " -->", os.path.join(self.downloader.download_directory, self.downloader.file_name)
return self.downloader
self.downloader = self.factory.make_downloader(self.metadata, [0.5, True], self.payment_rate_manager)
self.downloader.addCallback(_set_downloader)
return defer.succeed(self.downloader)
class AutoFetcher(object):
def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf):
self.autofetcher_conf = autofetcher_conf
self.max_key_fee = 0.0
self.console = None
self.sd_identifier = sd_identifier
self.wallet = wallet
self.session = session
self.lbry_file_manager = lbry_file_manager
self.lbry_metadata_manager = lbry_file_metadata_manager
self.seen = []
self.lastbestblock = None
self.rpc_conn = self.wallet.get_rpc_conn_x()
self.search = None
self.first_run = True
self.is_running = False
self._get_autofetcher_conf()
def start(self, console):
# TODO first search through the nametrie before monitoring live updates
# TODO load previously downloaded streams
self.console = console
if not self.is_running:
self.is_running = True
self.search = LoopingCall(self._looped_search)
self.search.start(1)
else:
self.console.sendLine("Autofetcher is already running")
def stop(self, console):
self.console = console
if self.is_running:
self.search.stop()
self.is_running = False
else:
self.console.sendLine("Autofetcher isn't running, there's nothing to stop")
def check_if_running(self, console):
self.console = console
if self.is_running:
self.console.sendLine("Autofetcher is running")
self.console.sendLine("Last block hash: " + str(self.lastbestblock['bestblockhash']))
else:
self.console.sendLine("Autofetcher is not running")
def _get_names(self):
c = self.rpc_conn.getblockchaininfo()
rtn = []
if self.lastbestblock != c:
block = self.rpc_conn.getblock(c['bestblockhash'])
txids = block['tx']
transactions = [self.rpc_conn.decoderawtransaction(self.rpc_conn.getrawtransaction(t)) for t in txids]
for t in transactions:
claims = self.rpc_conn.getclaimsfortx(t['txid'])
# if self.first_run:
# #claims = self.rpc_conn.getclaimsfortx("96aca2c60efded5806b7336430c5987b9092ffbea9c6ed444e3bf8e008993e11")
# claims = self.rpc_conn.getclaimsfortx("cc9c7f5225ecb38877e6ca7574d110b23214ac3556b9d65784065ad3a85b4f74")
# self.first_run = False
if claims:
for claim in claims:
if claim not in self.seen:
self.console.sendLine("lbry://" + str(claim['name']) + " | stream hash: " +
str(json.loads(claim['value'])['stream_hash']))
rtn.append(claim)
self.seen.append(claim)
else:
# self.console.sendLine("No new claims in block #" + str(block['height']))
pass
self.lastbestblock = c
if len(rtn):
return defer.succeed(rtn)
def _download_claims(self, claims):
if claims:
for claim in claims:
download = defer.Deferred()
stream = AutoAddStream(self.console, self.sd_identifier, self.session,
self.wallet, self.lbry_file_manager, self.max_key_fee)
download.addCallback(lambda _: stream.start(claim))
download.callback(None)
return defer.succeed(None)
def _looped_search(self):
d = defer.Deferred(None)
d.addCallback(lambda _: self._get_names())
d.addCallback(self._download_claims)
d.callback(None)
def _get_autofetcher_conf(self):
settings = {"maxkey": "0.0"}
if os.path.exists(self.autofetcher_conf):
conf = open(self.autofetcher_conf)
for l in conf:
if l.startswith("maxkey="):
settings["maxkey"] = float(l[7:].rstrip('\n'))
else:
print "Autofetcher using default max key price of 0.0"
print "To change this create the file:"
print str(self.autofetcher_conf)
print "Example contents of conf file:"
print "maxkey=1.0"
self.max_key_fee = settings["maxkey"]
class FetcherDaemon(object):
def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf):
self.autofetcher_conf = autofetcher_conf
self.max_key_fee = 0.0
self.sd_identifier = sd_identifier
self.wallet = wallet
self.session = session
self.lbry_file_manager = lbry_file_manager
self.lbry_metadata_manager = lbry_file_metadata_manager
self.seen = []
self.lastbestblock = None
self.rpc_conn = self.wallet.get_rpc_conn_x()
self.search = None
self.first_run = True
self.is_running = False
self._get_autofetcher_conf()
def start(self):
if not self.is_running:
self.is_running = True
self.search = LoopingCall(self._looped_search)
self.search.start(1)
else:
print "Autofetcher is already running"
def stop(self):
if self.is_running:
self.search.stop()
self.is_running = False
else:
print "Autofetcher isn't running, there's nothing to stop"
def check_if_running(self):
if self.is_running:
msg = "Autofetcher is running\n"
msg += "Last block hash: " + str(self.lastbestblock['bestblockhash'])
else:
msg = "Autofetcher is not running"
return msg
def _get_names(self):
c = self.rpc_conn.getblockchaininfo()
rtn = []
if self.lastbestblock != c:
block = self.rpc_conn.getblock(c['bestblockhash'])
txids = block['tx']
transactions = [self.rpc_conn.decoderawtransaction(self.rpc_conn.getrawtransaction(t)) for t in txids]
for t in transactions:
claims = self.rpc_conn.getclaimsfortx(t['txid'])
# if self.first_run:
# # claims = self.rpc_conn.getclaimsfortx("96aca2c60efded5806b7336430c5987b9092ffbea9c6ed444e3bf8e008993e11")
# # claims = self.rpc_conn.getclaimsfortx("cc9c7f5225ecb38877e6ca7574d110b23214ac3556b9d65784065ad3a85b4f74")
# self.first_run = False
if claims:
for claim in claims:
if claim not in self.seen:
msg = "[" + str(datetime.now()) + "] New claim | lbry://" + str(claim['name']) + \
" | stream hash: " + str(json.loads(claim['value'])['stream_hash'])
print msg
log.debug(msg)
rtn.append(claim)
self.seen.append(claim)
else:
# self.console.sendLine("No new claims in block #" + str(block['height']))
pass
self.lastbestblock = c
if len(rtn):
return defer.succeed(rtn)
def _download_claims(self, claims):
if claims:
for claim in claims:
download = defer.Deferred()
stream = GetStream(self.sd_identifier, self.session, self.wallet, self.lbry_file_manager, self.max_key_fee)
download.addCallback(lambda _: stream.start(claim))
download.callback(None)
return defer.succeed(None)
def _looped_search(self):
d = defer.Deferred(None)
d.addCallback(lambda _: self._get_names())
d.addCallback(self._download_claims)
d.callback(None)
def _get_autofetcher_conf(self):
settings = {"maxkey": "0.0"}
if os.path.exists(self.autofetcher_conf):
conf = open(self.autofetcher_conf)
for l in conf:
if l.startswith("maxkey="):
settings["maxkey"] = float(l[7:].rstrip('\n'))
else:
print "Autofetcher using default max key price of 0.0"
print "To change this create the file:"
print str(self.autofetcher_conf)
print "Example contents of conf file:"
print "maxkey=1.0"
self.max_key_fee = settings["maxkey"]

View file

@ -31,7 +31,10 @@ class LBRYFileManager(object):
self.sd_identifier = sd_identifier
self.lbry_files = []
self.sql_db = None
self.download_directory = os.getcwd()
if os.name == 'posix':
self.download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
else:
self.download_directory = os.getcwd()
def setup(self):
d = self._open_db()

View file

@ -1,4 +1,9 @@
import json
import logging
from time import sleep
from bitcoinrpc.authproxy import AuthServiceProxy
from twisted.internet.task import LoopingCall
from zope.interface import implements
#from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter, BlobStreamDescriptorWriter
from lbrynet.core.PaymentRateManager import PaymentRateManager
@ -2028,6 +2033,10 @@ class ImmediateAnnounceAllBlobs(CommandHandler):
class ImmediateAnnounceAllBlobsFactory(CommandHandlerFactory):
control_handler_class = ImmediateAnnounceAllBlobs
command = "announce-blobs"
short_help = "Announce all blobs to the dht"
full_help = "Immediately re-broadcast all hashes associated with the server to " \
"the distributed hash table."
class ModifyServerSettings(RecursiveCommandHandler):
@ -2353,6 +2362,54 @@ class StatusFactory(CommandHandlerFactory):
"to remove the file."
# class AutoFetcherStart(CommandHandler):
# def __init__(self, console, autofetcher):
# CommandHandler.__init__(self, console)
# self.autofetcher = autofetcher
#
# def start(self):
# self.autofetcher.start(self.console)
# self.finished_deferred.callback(None)
#
#
# class AutoFetcherStop(CommandHandler):
# def __init__(self, console, autofetcher):
# CommandHandler.__init__(self, console)
# self.autofetcher = autofetcher
#
# def start(self):
# self.autofetcher.stop(self.console)
# self.finished_deferred.callback(None)
#
#
# class AutoFetcherStatus(CommandHandler):
# def __init__(self, console, autofetcher):
# CommandHandler.__init__(self, console)
# self.autofetcher = autofetcher
#
# def start(self):
# self.autofetcher.check_if_running(self.console)
# self.finished_deferred.callback(None)
# class AutoFetcherStartFactory(CommandHandlerFactory):
# control_handler_class = AutoFetcherStart
# command = "start-autofetcher"
# short_help = "Start downloading all lbry files as they are published"
#
#
# class AutoFetcherStopFactory(CommandHandlerFactory):
# control_handler_class = AutoFetcherStop
# command = "stop-autofetcher"
# short_help = "Stop downloading all lbry files as they are published"
#
#
# class AutoFetcherStatusFactory(CommandHandlerFactory):
# control_handler_class = AutoFetcherStatus
# command = "autofetcher-status"
# short_help = "Check autofetcher status"
class BlockchainStatus(CommandHandler):
def __init__(self, console, wallet=None):
CommandHandler.__init__(self, console)

View file

@ -6,6 +6,8 @@ import requests
import locale
from yapsy.PluginManager import PluginManager
from twisted.internet import defer, threads, stdio, task, error
from lbrynet.core.client.AutoDownloader import AutoFetcher
from lbrynet.lbrynet_console.ConsoleControl import ConsoleControl
from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager
@ -23,6 +25,8 @@ from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager
#from lbrynet.lbrylive.PaymentRateManager import LiveStreamPaymentRateManager
from lbrynet.lbrynet_console.ControlHandlers import ApplicationStatusFactory, GetWalletBalancesFactory, ShutDownFactory
from lbrynet.lbrynet_console.ControlHandlers import AutoFetcherStartFactory, AutoFetcherStopFactory
from lbrynet.lbrynet_console.ControlHandlers import ImmediateAnnounceAllBlobsFactory, AutoFetcherStatusFactory
from lbrynet.lbrynet_console.ControlHandlers import LBRYFileStatusFactory, DeleteLBRYFileChooserFactory
from lbrynet.lbrynet_console.ControlHandlers import ToggleLBRYFileRunningChooserFactory
from lbrynet.lbrynet_console.ControlHandlers import ModifyApplicationDefaultsFactory
@ -65,6 +69,7 @@ class LBRYConsole():
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd")
if not self.lbrycrd_conf:
self.lbrycrd_conf = os.path.join(self.lbrycrd_dir, "lbrycrd.conf")
self.autofetcher_conf = os.path.join(self.lbrycrd_dir, "autofetcher.conf")
self.lbrycrdd_path = lbrycrdd_path
self.default_lbrycrdd_path = "./lbrycrdd"
self.start_lbrycrdd = start_lbrycrdd
@ -91,6 +96,7 @@ class LBRYConsole():
self.sd_identifier = StreamDescriptorIdentifier()
self.plugin_objects = []
self.db_migration_revisions = None
self.autofetcher = None
def start(self):
"""Initialize the session and restore everything to its saved state"""
@ -102,6 +108,7 @@ class LBRYConsole():
d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier))
d.addCallback(lambda _: self._setup_lbry_file_manager())
d.addCallback(lambda _: self._setup_lbry_file_opener())
d.addCallback(lambda _: self._get_autofetcher())
d.addCallback(lambda _: self._setup_control_handlers())
d.addCallback(lambda _: self._setup_query_handlers())
d.addCallback(lambda _: self._load_plugins())
@ -110,6 +117,10 @@ class LBRYConsole():
d.addErrback(self._show_start_error)
return d
def _get_autofetcher(self):
self.autofetcher = AutoFetcher(self.session, self.lbry_file_manager, self.lbry_file_metadata_manager,
self.session.wallet, self.sd_identifier, self.autofetcher_conf)
def _show_start_error(self, error):
print error.getErrorMessage()
log.error("An error occurred during start up: %s", error.getTraceback())
@ -358,7 +369,11 @@ class LBRYConsole():
ModifyLBRYFileOptionsChooserFactory(self.lbry_file_manager),
AddStreamFromHashFactory(self.sd_identifier, self.session),
StatusFactory(self, self.session.rate_limiter, self.lbry_file_manager,
self.session.blob_manager, self.session.wallet if self.wallet_type == 'lbrycrd' else None)
self.session.blob_manager, self.session.wallet if self.wallet_type == 'lbrycrd' else None),
# AutoFetcherStartFactory(self.autofetcher),
# AutoFetcherStopFactory(self.autofetcher),
# AutoFetcherStatusFactory(self.autofetcher),
ImmediateAnnounceAllBlobsFactory(self.session.blob_manager)
]
self.add_control_handlers(handlers)
if self.wallet_type == 'lbrycrd':
@ -585,3 +600,6 @@ def launch_lbry_console():
reactor.addSystemEventTrigger('before', 'shutdown', console.shut_down)
reactor.run()
if __name__ == "__main__":
launch_lbry_console()

View file

@ -0,0 +1,379 @@
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory
from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.core.client.AutoDownloader import GetStream, FetcherDaemon
from lbrynet.core.utils import generate_id
from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
from lbrynet.core.Session import LBRYSession
from lbrynet.core.PTCWallet import PTCWallet
from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet
from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager
from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager
from twisted.web import xmlrpc, server
from twisted.internet import defer, threads, reactor
from datetime import datetime
import logging
import os
log = logging.getLogger(__name__)
class LBRYDaemon(xmlrpc.XMLRPC):
"""
LBRYnet daemon
"""
def setup(self):
def _set_vars():
self.fetcher = None
self.current_db_revision = 1
self.run_server = True
self.session = None
self.known_dht_nodes = [('104.236.42.182', 4000)]
self.db_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
self.blobfile_dir = os.path.join(self.db_dir, "blobfiles")
self.peer_port = 3333
self.dht_node_port = 4444
self.first_run = False
self.current_db_revision = 1
if os.name == "nt":
from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
self.download_directory = get_path(FOLDERID.Downloads, UserHandle.current)
self.wallet_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrycrd")
else:
self.download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
self.wallet_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd")
self.wallet_conf = os.path.join(self.wallet_dir, "lbrycrd.conf")
self.wallet_user = None
self.wallet_password = None
self.sd_identifier = StreamDescriptorIdentifier()
self.stream_info_manager = TempLBRYFileMetadataManager()
self.wallet_rpc_port = 8332
self.downloads = []
self.stream_frames = []
self.default_blob_data_payment_rate = MIN_BLOB_DATA_PAYMENT_RATE
self.use_upnp = True
self.start_lbrycrdd = True
if os.name == "nt":
self.lbrycrdd_path = "lbrycrdd.exe"
else:
self.lbrycrdd_path = "./lbrycrdd"
self.delete_blobs_on_remove = True
self.blob_request_payment_rate_manager = None
self.lbry_file_metadata_manager = None
self.lbry_file_manager = None
self.settings = LBRYSettings(self.db_dir)
self.wallet_type = "lbrycrd"
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd")
self.lbrycrd_conf = os.path.join(self.lbrycrd_dir, "lbrycrd.conf")
self.autofetcher_conf = os.path.join(self.lbrycrd_dir, "autofetcher.conf")
self.rpc_conn = None
self.files = []
self.created_data_dir = False
if not os.path.exists(self.db_dir):
os.mkdir(self.db_dir)
self.created_data_dir = True
return defer.succeed(None)
d = defer.Deferred()
d.addCallback(lambda _: _set_vars())
d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory))
d.addCallback(lambda _: self._check_db_migration())
d.addCallback(lambda _: self._get_settings())
d.addCallback(lambda _: self.get_lbrycrdd_path())
d.addCallback(lambda _: self._get_session())
d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier))
d.addCallback(lambda _: self._setup_stream_identifier())
d.addCallback(lambda _: self._setup_lbry_file_manager())
d.addCallback(lambda _: self._setup_lbry_file_opener())
d.addCallback(lambda _: self._setup_fetcher())
d.callback(None)
return defer.succeed(None)
def _setup_fetcher(self):
self.fetcher = FetcherDaemon(self.session, self.lbry_file_manager, self.lbry_file_metadata_manager,
self.session.wallet, self.sd_identifier, self.autofetcher_conf)
return defer.succeed(None)
def _setup_data_directory(self):
print "Loading databases..."
if self.created_data_dir:
db_revision = open(os.path.join(self.db_dir, "db_revision"), mode='w')
db_revision.write(str(self.current_db_revision))
db_revision.close()
log.debug("Created the db revision file: %s", str(os.path.join(self.db_dir, "db_revision")))
if not os.path.exists(self.blobfile_dir):
os.mkdir(self.blobfile_dir)
log.debug("Created the blobfile directory: %s", str(self.blobfile_dir))
def _check_db_migration(self):
old_revision = 0
db_revision_file = os.path.join(self.db_dir, "db_revision")
if os.path.exists(db_revision_file):
old_revision = int(open(db_revision_file).read().strip())
if old_revision < self.current_db_revision:
from lbrynet.db_migrator import dbmigrator
print "Upgrading your databases..."
d = threads.deferToThread(dbmigrator.migrate_db, self.db_dir, old_revision, self.current_db_revision)
def print_success(old_dirs):
success_string = "Finished upgrading the databases. It is now safe to delete the"
success_string += " following directories, if you feel like it. It won't make any"
success_string += " difference.\nAnyway here they are: "
for i, old_dir in enumerate(old_dirs):
success_string += old_dir
if i + 1 < len(old_dir):
success_string += ", "
print success_string
d.addCallback(print_success)
return d
return defer.succeed(True)
def _get_settings(self):
d = self.settings.start()
d.addCallback(lambda _: self.settings.get_lbryid())
d.addCallback(self.set_lbryid)
d.addCallback(lambda _: self.get_lbrycrdd_path())
return d
def set_lbryid(self, lbryid):
if lbryid is None:
return self._make_lbryid()
else:
self.lbryid = lbryid
def _make_lbryid(self):
self.lbryid = generate_id()
d = self.settings.save_lbryid(self.lbryid)
return d
def _setup_lbry_file_manager(self):
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir)
d = self.lbry_file_metadata_manager.setup()
def set_lbry_file_manager():
self.lbry_file_manager = LBRYFileManager(self.session, self.lbry_file_metadata_manager, self.sd_identifier)
return self.lbry_file_manager.setup()
d.addCallback(lambda _: set_lbry_file_manager())
return d
def _get_session(self):
def get_default_data_rate():
d = self.settings.get_default_data_payment_rate()
d.addCallback(lambda rate: {"default_data_payment_rate":
rate if rate is not None else MIN_BLOB_DATA_PAYMENT_RATE})
return d
def get_wallet():
if self.wallet_type == "lbrycrd":
lbrycrdd_path = None
if self.start_lbrycrdd is True:
lbrycrdd_path = self.lbrycrdd_path
if not lbrycrdd_path:
lbrycrdd_path = self.default_lbrycrdd_path
d = defer.succeed(LBRYcrdWallet(self.db_dir, wallet_dir=self.lbrycrd_dir, wallet_conf=self.lbrycrd_conf,
lbrycrdd_path=lbrycrdd_path))
else:
d = defer.succeed(PTCWallet(self.db_dir))
d.addCallback(lambda wallet: {"wallet": wallet})
return d
d1 = get_default_data_rate()
d2 = get_wallet()
def combine_results(results):
r = {}
for success, result in results:
if success is True:
r.update(result)
return r
def create_session(results):
self.session = LBRYSession(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid,
blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port,
known_dht_nodes=self.known_dht_nodes, peer_port=self.peer_port,
use_upnp=self.use_upnp, wallet=results['wallet'])
self.rpc_conn = self.session.wallet.get_rpc_conn_x()
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
dl.addCallback(combine_results)
dl.addCallback(create_session)
dl.addCallback(lambda _: self.session.setup())
return dl
def get_lbrycrdd_path(self):
def get_lbrycrdd_path_conf_file():
lbrycrdd_path_conf_path = os.path.join(os.path.expanduser("~"), ".lbrycrddpath.conf")
if not os.path.exists(lbrycrdd_path_conf_path):
return ""
lbrycrdd_path_conf = open(lbrycrdd_path_conf_path)
lines = lbrycrdd_path_conf.readlines()
return lines
d = threads.deferToThread(get_lbrycrdd_path_conf_file)
def load_lbrycrdd_path(conf):
for line in conf:
if len(line.strip()) and line.strip()[0] != "#":
self.lbrycrdd_path = line.strip()
print self.lbrycrdd_path
d.addCallback(load_lbrycrdd_path)
return d
def _setup_stream_identifier(self):
file_saver_factory = LBRYFileSaverFactory(self.session.peer_finder, self.session.rate_limiter,
self.session.blob_manager, self.stream_info_manager,
self.session.wallet, self.download_directory)
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, file_saver_factory)
file_opener_factory = LBRYFileOpenerFactory(self.session.peer_finder, self.session.rate_limiter,
self.session.blob_manager, self.stream_info_manager,
self.session.wallet)
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, file_opener_factory)
return defer.succeed(None)
def _setup_lbry_file_manager(self):
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir)
d = self.lbry_file_metadata_manager.setup()
def set_lbry_file_manager():
self.lbry_file_manager = LBRYFileManager(self.session, self.lbry_file_metadata_manager, self.sd_identifier)
return self.lbry_file_manager.setup()
d.addCallback(lambda _: set_lbry_file_manager())
return d
def _setup_lbry_file_opener(self):
downloader_factory = LBRYFileOpenerFactory(self.session.peer_finder, self.session.rate_limiter,
self.session.blob_manager, self.stream_info_manager,
self.session.wallet)
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory)
return defer.succeed(True)
def xmlrpc_start_fetcher(self):
"""
Start autofetcher
"""
self.fetcher.start()
return str('Started autofetching')
def xmlrpc_stop_fetcher(self):
"""
Start autofetcher
"""
self.fetcher.stop()
return str('Started autofetching')
def xmlrpc_fetcher_status(self):
"""
Start autofetcher
"""
return str(self.fetcher.check_if_running())
def xmlrpc_get_balance(self):
"""
Get LBC balance
"""
return str(self.session.wallet.wallet_balance)
def xmlrpc_stop(self):
"""
Stop the reactor
"""
reactor.stop()
return defer.succeed('Stopping')
def xmlrpc_resolve_name(self, name):
"""
Resolve stream info from a LBRY uri
"""
def _disp(info):
log.debug('[' + str(datetime.now()) + ']' + ' Resolved info: ' + str(info))
print '[' + str(datetime.now()) + ']' + ' Resolved info: ' + str(info)
return info
d = defer.Deferred()
d.addCallback(lambda _: self.session.wallet.get_stream_info_for_name(name))
d.addErrback(lambda _: 'UnknownNameError')
d.addCallback(_disp)
d.callback(None)
return d
def xmlrpc_get_downloads(self):
"""
Get downloads
"""
downloads = []
for stream in self.downloads:
try:
downloads.append({'stream_hash': stream.stream_hash,
'path': os.path.join(stream.downloader.download_directory, stream.downloader.file_name)})
except:
pass
return downloads
def xmlrpc_download_name(self, name):
"""
Download stream from a LBRY uri
"""
def _disp():
try:
stream = self.downloads[-1]
log.debug('[' + str(datetime.now()) + ']' + ' Downloading: ' + str(stream.stream_hash))
print '[' + str(datetime.now()) + ']' + ' Downloading: ' + str(stream.stream_hash)
return defer.succeed(None)
except:
pass
stream = GetStream(self.sd_identifier, self.session, self.session.wallet, self.lbry_file_manager, 25.0)
self.downloads.append(stream)
d = self.session.wallet.get_stream_info_for_name(name)
d.addCallback(lambda stream_info: stream.start(stream_info))
d.addCallback(lambda _: _disp())
# d.addCallback(lambda _: self.files.append({'name': name, 'stream_hash': stream.stream_hash,
# 'path': os.path.join(stream.downloader.download_directory, stream.downloader.file_name)}))
d.addCallback(lambda _: {'ts': datetime.now(),'name': name})
d.addErrback(lambda err: str(err.getTraceback()))
return d
def xmlrpc_path_from_name(self, name):
d = self.session.wallet.get_stream_info_for_name(name)
d.addCallback(lambda stream_info: stream_info['stream_hash'])
d.addCallback(lambda stream_hash: [{'stream_hash': stream.stream_hash,
'path': os.path.join(stream.downloader.download_directory,
stream.downloader.file_name)}
for stream in self.downloads if stream.stream_hash == stream_hash])
d.addErrback(lambda _: 'UnknownNameError')
return d
def main():
daemon = LBRYDaemon()
daemon.setup()
reactor.listenTCP(7080, server.Site(daemon))
reactor.run()
if __name__ == '__main__':
main()

View file

@ -0,0 +1,20 @@
import webbrowser
import xmlrpclib, sys
def main(args):
if len(args) == 0:
args.append('lbry://wonderfullife')
daemon = xmlrpclib.ServerProxy('http://localhost:7080/')
if len(args) > 1:
print 'Too many args', args
else:
daemon.download_name(str(args[0])[7:])
path = daemon.path_from_name(args[0][7:])[0]['path']
webbrowser.open('file://' + path)
if __name__ == "__main__":
main(sys.argv[1:])

View file

@ -0,0 +1,117 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>English</string>
<key>CFBundleDisplayName</key>
<string>LBRYURIHandler</string>
<key>CFBundleDocumentTypes</key>
<array>
<dict>
<key>CFBundleTypeOSTypes</key>
<array>
<string>****</string>
<string>fold</string>
<string>disk</string>
</array>
<key>CFBundleTypeRole</key>
<string>Viewer</string>
</dict>
</array>
<key>CFBundleExecutable</key>
<string>LBRYURIHandler</string>
<key>CFBundleURLTypes</key>
<array>
<dict>
<key>CFBundleURLName</key>
<string>LBRYURIHandler</string>
<key>CFBundleURLSchemes</key>
<array>
<string>lbry</string>
</array>
</dict>
</array>
<key>NSUIElement</key>
<true/>
<key>CFBundleIconFile</key>
<string>PythonApplet.icns</string>
<key>CFBundleIdentifier</key>
<string>org.pythonmac.unspecified.LBRYURIHandler</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>LBRYURIHandler</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>0.0.0</string>
<key>CFBundleSignature</key>
<string>????</string>
<key>CFBundleVersion</key>
<string>0.0.0</string>
<key>LSHasLocalizedDisplayName</key>
<false/>
<key>NSAppleScriptEnabled</key>
<false/>
<key>NSHumanReadableCopyright</key>
<string>Copyright not specified</string>
<key>NSMainNibFile</key>
<string>MainMenu</string>
<key>NSPrincipalClass</key>
<string>NSApplication</string>
<key>PyMainFileNames</key>
<array>
<string>__boot__</string>
</array>
<key>PyOptions</key>
<dict>
<key>alias</key>
<false/>
<key>argv_emulation</key>
<true/>
<key>emulate_shell_environment</key>
<false/>
<key>no_chdir</key>
<false/>
<key>prefer_ppc</key>
<false/>
<key>site_packages</key>
<false/>
<key>use_faulthandler</key>
<false/>
<key>use_pythonpath</key>
<false/>
<key>verbose</key>
<false/>
</dict>
<key>PyResourcePackages</key>
<array>
</array>
<key>PyRuntimeLocations</key>
<array>
<string>@executable_path/../Frameworks/Python.framework/Versions/2.7/Python</string>
</array>
<key>PythonInfoDict</key>
<dict>
<key>PythonExecutable</key>
<string>/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python</string>
<key>PythonLongVersion</key>
<string>2.7.10 (v2.7.10:15c95b7d81dc, May 23 2015, 09:33:12)
[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]</string>
<key>PythonShortVersion</key>
<string>2.7</string>
<key>py2app</key>
<dict>
<key>alias</key>
<false/>
<key>template</key>
<string>app</string>
<key>version</key>
<string>0.9</string>
</dict>
</dict>
</dict>
</plist>

View file

@ -0,0 +1,19 @@
"""
This is a setup.py script generated by py2applet
Usage:
python setup.py py2app
"""
from setuptools import setup
APP = ['LBRYURIHandler.py']
DATA_FILES = []
OPTIONS = {'argv_emulation': True}
setup(
app=APP,
data_files=DATA_FILES,
options={'py2app': OPTIONS},
setup_requires=['py2app'],
)

View file

View file

@ -21,6 +21,7 @@ setup(name='lbrynet',
'lbrynet-gui = lbrynet.lbrynet_gui.gui:start_gui',
'lbrynet-lookup-hosts-for-hash = lbrynet.dht_scripts:get_hosts_for_hash_in_dht',
'lbrynet-announce_hash_to_dht = lbrynet.dht_scripts:announce_hash_to_dht',
'lbrynet-daemon = lbrynet.lbrynet_daemon.LBRYDaemon:main',
]
},
data_files=[