diff --git a/.gitignore b/.gitignore index 0d20b6487..63cca906f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,15 @@ *.pyc + +*.xml + +.idea/.name + +*.iml + +*.log + +*.txt + +lbrynet.egg-info/PKG-INFO + +*.egg diff --git a/lbrynet/core/LBRYcrdWallet.py b/lbrynet/core/LBRYcrdWallet.py index 5327cb6f0..982eebf10 100644 --- a/lbrynet/core/LBRYcrdWallet.py +++ b/lbrynet/core/LBRYcrdWallet.py @@ -348,6 +348,9 @@ class LBRYcrdWallet(object): def _get_rpc_conn(self): return AuthServiceProxy(self.rpc_conn_string) + def get_rpc_conn_x(self): + return AuthServiceProxy(self.rpc_conn_string) + def _start_daemon(self): tries = 0 diff --git a/lbrynet/core/client/AutoDownloader.py b/lbrynet/core/client/AutoDownloader.py new file mode 100644 index 000000000..939e543bb --- /dev/null +++ b/lbrynet/core/client/AutoDownloader.py @@ -0,0 +1,452 @@ +import json +import logging +import os +from datetime import datetime +from twisted.internet import defer +from twisted.internet.task import LoopingCall +from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError +from lbrynet.core.PaymentRateManager import PaymentRateManager +from lbrynet.core.StreamDescriptor import download_sd_blob + +log = logging.getLogger(__name__) + + +class AutoAddStream(object): + def __init__(self, console, sd_identifier, session, wallet, lbry_file_manager, max_key_fee): + self.finished_deferred = defer.Deferred(None) + self.console = console + self.wallet = wallet + self.resolved_name = None + self.description = None + self.key_fee = None + self.key_fee_address = None + self.name = None + self.session = session + self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager) + self.loading_metadata_deferred = defer.Deferred() + self.lbry_file_manager = lbry_file_manager + self.sd_identifier = sd_identifier + self.metadata = None + self.loading_failed = False + self.resolved_name = None + self.description = None + self.key_fee = None + self.key_fee_address = None + self.stream_hash = None + self.max_key_fee = max_key_fee + self.stream_info = None + + def start(self, stream_info): + self.stream_info = stream_info + if 'stream_hash' not in json.loads(self.stream_info['value']): + print 'InvalidStreamInfoError' + raise InvalidStreamInfoError(self.stream_info) + self.resolved_name = self.stream_info.get('name', None) + self.description = json.loads(self.stream_info['value']).get('description', None) + try: + if 'key_fee' in json.loads(self.stream_info['value']): + self.key_fee = float(json.loads(self.stream_info['value'])['key_fee']) + except ValueError: + self.key_fee = None + self.key_fee_address = json.loads(self.stream_info['value']).get('key_fee_address', None) + self.stream_hash = json.loads(self.stream_info['value'])['stream_hash'] + + if self.key_fee > self.max_key_fee: + pass + # self.console.sendLine("Key fee (" + str(self.key_fee) + ") above limit of " + str( + # self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name)) + # return self.finished_deferred.callback(None) + + self.loading_metadata_deferred = defer.Deferred(None) + self.loading_metadata_deferred.addCallback( + lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager)) + self.loading_metadata_deferred.addCallback(self.sd_identifier.get_metadata_for_sd_blob) + self.loading_metadata_deferred.addCallback(self._handle_metadata) + self.loading_metadata_deferred.addErrback(self._handle_load_canceled) + self.loading_metadata_deferred.addErrback(self._handle_load_failed) + + self.finished_deferred.addCallback(lambda _: self.loading_metadata_deferred.callback(None)) + + return self.finished_deferred.callback(None) + + def _start_download(self): + #d = self._pay_key_fee() + d = defer.Deferred(None) + d.addCallback(lambda _: self._make_downloader()) + d.addCallback(lambda stream_downloader: stream_downloader.start()) + d.addErrback(self._handle_download_error) + return d + + def _pay_key_fee(self): + if self.key_fee is not None and self.key_fee_address is not None: + reserved_points = self.wallet.reserve_points(self.key_fee_address, self.key_fee) + if reserved_points is None: + return defer.fail(InsufficientFundsError()) + return self.wallet.send_points_to_address(reserved_points, self.key_fee) + self.console.sendLine("Sent key fee" + str(self.key_fee_address) + " | " + str(self.key_fee)) + return defer.succeed(None) + + def _handle_load_canceled(self, err): + err.trap(defer.CancelledError) + self.finished_deferred.callback(None) + + def _handle_load_failed(self, err): + self.loading_failed = True + self.console.sendLine("handle load failed: " + str(err.getTraceback())) + log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback()) + self.finished_deferred.callback(None) + + def _handle_metadata(self, metadata): + self.metadata = metadata + self.factory = self.metadata.factories[0] + self.finished_deferred.addCallback(lambda _: self._start_download()) + + def _handle_download_error(self, err): + if err.check(InsufficientFundsError): + self.console.sendLine("Download stopped due to insufficient funds.") + else: + self.console.sendLine( + "Autoaddstream: An unexpected error has caused the download to stop: %s" % err.getTraceback()) + + def _make_downloader(self): + self.downloader = self.factory.make_downloader(self.metadata, [0.5, True], self.payment_rate_manager) + return self.downloader + + +class GetStream(object): + def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee): + self.finished_deferred = defer.Deferred(None) + self.wallet = wallet + self.resolved_name = None + self.description = None + self.key_fee = None + self.key_fee_address = None + self.name = None + self.session = session + self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager) + self.loading_metadata_deferred = defer.Deferred() + self.lbry_file_manager = lbry_file_manager + self.sd_identifier = sd_identifier + self.metadata = None + self.loading_failed = False + self.resolved_name = None + self.description = None + self.key_fee = None + self.key_fee_address = None + self.stream_hash = None + self.max_key_fee = max_key_fee + self.stream_info = None + self.stream_info_manager = None + self.downloader = None + + def start(self, stream_info): + self.stream_info = stream_info + if 'stream_hash' in self.stream_info.keys(): + self.description = self.stream_info['description'] + if 'key_fee' in self.stream_info.keys(): + self.key_fee = float(self.stream_info['key_fee']) + if 'key_fee_address' in self.stream_info.keys(): + self.key_fee_address = self.stream_info['key_fee_address'] + else: + self.key_fee_address = None + else: + self.key_fee = None + self.key_fee_address = None + self.stream_hash = self.stream_info['stream_hash'] + elif 'stream_hash' in json.loads(self.stream_info['value']): + self.resolved_name = self.stream_info.get('name', None) + self.description = json.loads(self.stream_info['value']).get('description', None) + + try: + if 'key_fee' in json.loads(self.stream_info['value']): + self.key_fee = float(json.loads(self.stream_info['value'])['key_fee']) + except ValueError: + self.key_fee = None + self.key_fee_address = json.loads(self.stream_info['value']).get('key_fee_address', None) + self.stream_hash = json.loads(self.stream_info['value'])['stream_hash'] + else: + print 'InvalidStreamInfoError' + raise InvalidStreamInfoError(self.stream_info) + + if self.key_fee > self.max_key_fee: + print "Key fee (" + str(self.key_fee) + ") above limit of " + str( + self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name) + return self.finished_deferred.callback(None) + else: + pass + + def _get_downloader_for_return(): + return defer.succeed(self.downloader) + + self.loading_metadata_deferred = defer.Deferred(None) + self.loading_metadata_deferred.addCallback( + lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager)) + self.loading_metadata_deferred.addCallback(self.sd_identifier.get_metadata_for_sd_blob) + self.loading_metadata_deferred.addCallback(self._handle_metadata) + self.loading_metadata_deferred.addErrback(self._handle_load_canceled) + self.loading_metadata_deferred.addErrback(self._handle_load_failed) + self.loading_metadata_deferred.addCallback(lambda _: self._pay_key_fee()) + self.loading_metadata_deferred.addCallback(lambda _: self._make_downloader()) + self.loading_metadata_deferred.addCallback(lambda _: self.downloader.start()) + self.loading_metadata_deferred.addErrback(self._handle_download_error) + self.loading_metadata_deferred.addCallback(lambda _: _get_downloader_for_return()) + self.loading_metadata_deferred.callback(None) + + return defer.succeed(None) + + def _pay_key_fee(self): + if self.key_fee is not None and self.key_fee_address is not None: + reserved_points = self.wallet.reserve_points(self.key_fee_address, self.key_fee) + if reserved_points is None: + return defer.fail(InsufficientFundsError()) + print 'Key fee: ' + str(self.key_fee) + ' | ' + str(self.key_fee_address) + return self.wallet.send_points_to_address(reserved_points, self.key_fee) + return defer.succeed(None) + + def _handle_load_canceled(self, err): + err.trap(defer.CancelledError) + self.finished_deferred.callback(None) + + def _handle_load_failed(self, err): + self.loading_failed = True + log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback()) + print 'Load Failed: ', err.getTraceback() + self.finished_deferred.callback(None) + + def _handle_metadata(self, metadata): + self.metadata = metadata + self.factory = self.metadata.factories[0] + return defer.succeed(None) + + def _handle_download_error(self, err): + if err.check(InsufficientFundsError): + print "Download stopped due to insufficient funds." + else: + print "Autoaddstream: An unexpected error has caused the download to stop: ", err.getTraceback() + + def _make_downloader(self): + + def _set_downloader(downloader): + self.downloader = downloader + print "Downloading", self.stream_hash, " -->", os.path.join(self.downloader.download_directory, self.downloader.file_name) + return self.downloader + + self.downloader = self.factory.make_downloader(self.metadata, [0.5, True], self.payment_rate_manager) + self.downloader.addCallback(_set_downloader) + return defer.succeed(self.downloader) + + +class AutoFetcher(object): + def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf): + self.autofetcher_conf = autofetcher_conf + self.max_key_fee = 0.0 + self.console = None + self.sd_identifier = sd_identifier + self.wallet = wallet + self.session = session + self.lbry_file_manager = lbry_file_manager + self.lbry_metadata_manager = lbry_file_metadata_manager + self.seen = [] + self.lastbestblock = None + self.rpc_conn = self.wallet.get_rpc_conn_x() + self.search = None + self.first_run = True + self.is_running = False + self._get_autofetcher_conf() + + def start(self, console): + # TODO first search through the nametrie before monitoring live updates + # TODO load previously downloaded streams + + self.console = console + + if not self.is_running: + self.is_running = True + self.search = LoopingCall(self._looped_search) + self.search.start(1) + else: + self.console.sendLine("Autofetcher is already running") + + def stop(self, console): + self.console = console + + if self.is_running: + self.search.stop() + self.is_running = False + else: + self.console.sendLine("Autofetcher isn't running, there's nothing to stop") + + def check_if_running(self, console): + self.console = console + + if self.is_running: + self.console.sendLine("Autofetcher is running") + self.console.sendLine("Last block hash: " + str(self.lastbestblock['bestblockhash'])) + else: + self.console.sendLine("Autofetcher is not running") + + def _get_names(self): + c = self.rpc_conn.getblockchaininfo() + rtn = [] + if self.lastbestblock != c: + block = self.rpc_conn.getblock(c['bestblockhash']) + txids = block['tx'] + transactions = [self.rpc_conn.decoderawtransaction(self.rpc_conn.getrawtransaction(t)) for t in txids] + for t in transactions: + claims = self.rpc_conn.getclaimsfortx(t['txid']) + # if self.first_run: + # #claims = self.rpc_conn.getclaimsfortx("96aca2c60efded5806b7336430c5987b9092ffbea9c6ed444e3bf8e008993e11") + # claims = self.rpc_conn.getclaimsfortx("cc9c7f5225ecb38877e6ca7574d110b23214ac3556b9d65784065ad3a85b4f74") + # self.first_run = False + if claims: + for claim in claims: + if claim not in self.seen: + self.console.sendLine("lbry://" + str(claim['name']) + " | stream hash: " + + str(json.loads(claim['value'])['stream_hash'])) + rtn.append(claim) + self.seen.append(claim) + else: + # self.console.sendLine("No new claims in block #" + str(block['height'])) + pass + + self.lastbestblock = c + + if len(rtn): + return defer.succeed(rtn) + + def _download_claims(self, claims): + if claims: + for claim in claims: + download = defer.Deferred() + stream = AutoAddStream(self.console, self.sd_identifier, self.session, + self.wallet, self.lbry_file_manager, self.max_key_fee) + download.addCallback(lambda _: stream.start(claim)) + download.callback(None) + + return defer.succeed(None) + + def _looped_search(self): + d = defer.Deferred(None) + d.addCallback(lambda _: self._get_names()) + d.addCallback(self._download_claims) + d.callback(None) + + def _get_autofetcher_conf(self): + settings = {"maxkey": "0.0"} + if os.path.exists(self.autofetcher_conf): + conf = open(self.autofetcher_conf) + for l in conf: + if l.startswith("maxkey="): + settings["maxkey"] = float(l[7:].rstrip('\n')) + else: + print "Autofetcher using default max key price of 0.0" + print "To change this create the file:" + print str(self.autofetcher_conf) + print "Example contents of conf file:" + print "maxkey=1.0" + + self.max_key_fee = settings["maxkey"] + + +class FetcherDaemon(object): + def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf): + self.autofetcher_conf = autofetcher_conf + self.max_key_fee = 0.0 + self.sd_identifier = sd_identifier + self.wallet = wallet + self.session = session + self.lbry_file_manager = lbry_file_manager + self.lbry_metadata_manager = lbry_file_metadata_manager + self.seen = [] + self.lastbestblock = None + self.rpc_conn = self.wallet.get_rpc_conn_x() + self.search = None + self.first_run = True + self.is_running = False + self._get_autofetcher_conf() + + def start(self): + if not self.is_running: + self.is_running = True + self.search = LoopingCall(self._looped_search) + self.search.start(1) + else: + print "Autofetcher is already running" + + def stop(self): + if self.is_running: + self.search.stop() + self.is_running = False + else: + print "Autofetcher isn't running, there's nothing to stop" + + def check_if_running(self): + if self.is_running: + msg = "Autofetcher is running\n" + msg += "Last block hash: " + str(self.lastbestblock['bestblockhash']) + else: + msg = "Autofetcher is not running" + return msg + + def _get_names(self): + c = self.rpc_conn.getblockchaininfo() + rtn = [] + if self.lastbestblock != c: + block = self.rpc_conn.getblock(c['bestblockhash']) + txids = block['tx'] + transactions = [self.rpc_conn.decoderawtransaction(self.rpc_conn.getrawtransaction(t)) for t in txids] + for t in transactions: + claims = self.rpc_conn.getclaimsfortx(t['txid']) + # if self.first_run: + # # claims = self.rpc_conn.getclaimsfortx("96aca2c60efded5806b7336430c5987b9092ffbea9c6ed444e3bf8e008993e11") + # # claims = self.rpc_conn.getclaimsfortx("cc9c7f5225ecb38877e6ca7574d110b23214ac3556b9d65784065ad3a85b4f74") + # self.first_run = False + if claims: + for claim in claims: + if claim not in self.seen: + msg = "[" + str(datetime.now()) + "] New claim | lbry://" + str(claim['name']) + \ + " | stream hash: " + str(json.loads(claim['value'])['stream_hash']) + print msg + log.debug(msg) + rtn.append(claim) + self.seen.append(claim) + else: + # self.console.sendLine("No new claims in block #" + str(block['height'])) + pass + + self.lastbestblock = c + + if len(rtn): + return defer.succeed(rtn) + + def _download_claims(self, claims): + if claims: + for claim in claims: + download = defer.Deferred() + stream = GetStream(self.sd_identifier, self.session, self.wallet, self.lbry_file_manager, self.max_key_fee) + download.addCallback(lambda _: stream.start(claim)) + download.callback(None) + + return defer.succeed(None) + + def _looped_search(self): + d = defer.Deferred(None) + d.addCallback(lambda _: self._get_names()) + d.addCallback(self._download_claims) + d.callback(None) + + def _get_autofetcher_conf(self): + settings = {"maxkey": "0.0"} + if os.path.exists(self.autofetcher_conf): + conf = open(self.autofetcher_conf) + for l in conf: + if l.startswith("maxkey="): + settings["maxkey"] = float(l[7:].rstrip('\n')) + else: + print "Autofetcher using default max key price of 0.0" + print "To change this create the file:" + print str(self.autofetcher_conf) + print "Example contents of conf file:" + print "maxkey=1.0" + + self.max_key_fee = settings["maxkey"] \ No newline at end of file diff --git a/lbrynet/lbryfilemanager/LBRYFileManager.py b/lbrynet/lbryfilemanager/LBRYFileManager.py index f27f6052e..c2e426bfa 100644 --- a/lbrynet/lbryfilemanager/LBRYFileManager.py +++ b/lbrynet/lbryfilemanager/LBRYFileManager.py @@ -31,7 +31,10 @@ class LBRYFileManager(object): self.sd_identifier = sd_identifier self.lbry_files = [] self.sql_db = None - self.download_directory = os.getcwd() + if os.name == 'posix': + self.download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') + else: + self.download_directory = os.getcwd() def setup(self): d = self._open_db() diff --git a/lbrynet/lbrynet_console/ControlHandlers.py b/lbrynet/lbrynet_console/ControlHandlers.py index dc560ba16..77a9562c6 100644 --- a/lbrynet/lbrynet_console/ControlHandlers.py +++ b/lbrynet/lbrynet_console/ControlHandlers.py @@ -1,4 +1,9 @@ +import json import logging +from time import sleep + +from bitcoinrpc.authproxy import AuthServiceProxy +from twisted.internet.task import LoopingCall from zope.interface import implements #from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter, BlobStreamDescriptorWriter from lbrynet.core.PaymentRateManager import PaymentRateManager @@ -2028,6 +2033,10 @@ class ImmediateAnnounceAllBlobs(CommandHandler): class ImmediateAnnounceAllBlobsFactory(CommandHandlerFactory): control_handler_class = ImmediateAnnounceAllBlobs + command = "announce-blobs" + short_help = "Announce all blobs to the dht" + full_help = "Immediately re-broadcast all hashes associated with the server to " \ + "the distributed hash table." class ModifyServerSettings(RecursiveCommandHandler): @@ -2353,6 +2362,54 @@ class StatusFactory(CommandHandlerFactory): "to remove the file." +# class AutoFetcherStart(CommandHandler): +# def __init__(self, console, autofetcher): +# CommandHandler.__init__(self, console) +# self.autofetcher = autofetcher +# +# def start(self): +# self.autofetcher.start(self.console) +# self.finished_deferred.callback(None) +# +# +# class AutoFetcherStop(CommandHandler): +# def __init__(self, console, autofetcher): +# CommandHandler.__init__(self, console) +# self.autofetcher = autofetcher +# +# def start(self): +# self.autofetcher.stop(self.console) +# self.finished_deferred.callback(None) +# +# +# class AutoFetcherStatus(CommandHandler): +# def __init__(self, console, autofetcher): +# CommandHandler.__init__(self, console) +# self.autofetcher = autofetcher +# +# def start(self): +# self.autofetcher.check_if_running(self.console) +# self.finished_deferred.callback(None) + + +# class AutoFetcherStartFactory(CommandHandlerFactory): +# control_handler_class = AutoFetcherStart +# command = "start-autofetcher" +# short_help = "Start downloading all lbry files as they are published" +# +# +# class AutoFetcherStopFactory(CommandHandlerFactory): +# control_handler_class = AutoFetcherStop +# command = "stop-autofetcher" +# short_help = "Stop downloading all lbry files as they are published" +# +# +# class AutoFetcherStatusFactory(CommandHandlerFactory): +# control_handler_class = AutoFetcherStatus +# command = "autofetcher-status" +# short_help = "Check autofetcher status" + + class BlockchainStatus(CommandHandler): def __init__(self, console, wallet=None): CommandHandler.__init__(self, console) diff --git a/lbrynet/lbrynet_console/LBRYConsole.py b/lbrynet/lbrynet_console/LBRYConsole.py index 738d20541..2d473bcb7 100644 --- a/lbrynet/lbrynet_console/LBRYConsole.py +++ b/lbrynet/lbrynet_console/LBRYConsole.py @@ -6,6 +6,8 @@ import requests import locale from yapsy.PluginManager import PluginManager from twisted.internet import defer, threads, stdio, task, error + +from lbrynet.core.client.AutoDownloader import AutoFetcher from lbrynet.lbrynet_console.ConsoleControl import ConsoleControl from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager @@ -23,6 +25,8 @@ from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager #from lbrynet.lbrylive.PaymentRateManager import LiveStreamPaymentRateManager from lbrynet.lbrynet_console.ControlHandlers import ApplicationStatusFactory, GetWalletBalancesFactory, ShutDownFactory +from lbrynet.lbrynet_console.ControlHandlers import AutoFetcherStartFactory, AutoFetcherStopFactory +from lbrynet.lbrynet_console.ControlHandlers import ImmediateAnnounceAllBlobsFactory, AutoFetcherStatusFactory from lbrynet.lbrynet_console.ControlHandlers import LBRYFileStatusFactory, DeleteLBRYFileChooserFactory from lbrynet.lbrynet_console.ControlHandlers import ToggleLBRYFileRunningChooserFactory from lbrynet.lbrynet_console.ControlHandlers import ModifyApplicationDefaultsFactory @@ -65,6 +69,7 @@ class LBRYConsole(): self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd") if not self.lbrycrd_conf: self.lbrycrd_conf = os.path.join(self.lbrycrd_dir, "lbrycrd.conf") + self.autofetcher_conf = os.path.join(self.lbrycrd_dir, "autofetcher.conf") self.lbrycrdd_path = lbrycrdd_path self.default_lbrycrdd_path = "./lbrycrdd" self.start_lbrycrdd = start_lbrycrdd @@ -91,6 +96,7 @@ class LBRYConsole(): self.sd_identifier = StreamDescriptorIdentifier() self.plugin_objects = [] self.db_migration_revisions = None + self.autofetcher = None def start(self): """Initialize the session and restore everything to its saved state""" @@ -102,6 +108,7 @@ class LBRYConsole(): d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier)) d.addCallback(lambda _: self._setup_lbry_file_manager()) d.addCallback(lambda _: self._setup_lbry_file_opener()) + d.addCallback(lambda _: self._get_autofetcher()) d.addCallback(lambda _: self._setup_control_handlers()) d.addCallback(lambda _: self._setup_query_handlers()) d.addCallback(lambda _: self._load_plugins()) @@ -110,6 +117,10 @@ class LBRYConsole(): d.addErrback(self._show_start_error) return d + def _get_autofetcher(self): + self.autofetcher = AutoFetcher(self.session, self.lbry_file_manager, self.lbry_file_metadata_manager, + self.session.wallet, self.sd_identifier, self.autofetcher_conf) + def _show_start_error(self, error): print error.getErrorMessage() log.error("An error occurred during start up: %s", error.getTraceback()) @@ -358,7 +369,11 @@ class LBRYConsole(): ModifyLBRYFileOptionsChooserFactory(self.lbry_file_manager), AddStreamFromHashFactory(self.sd_identifier, self.session), StatusFactory(self, self.session.rate_limiter, self.lbry_file_manager, - self.session.blob_manager, self.session.wallet if self.wallet_type == 'lbrycrd' else None) + self.session.blob_manager, self.session.wallet if self.wallet_type == 'lbrycrd' else None), + # AutoFetcherStartFactory(self.autofetcher), + # AutoFetcherStopFactory(self.autofetcher), + # AutoFetcherStatusFactory(self.autofetcher), + ImmediateAnnounceAllBlobsFactory(self.session.blob_manager) ] self.add_control_handlers(handlers) if self.wallet_type == 'lbrycrd': @@ -585,3 +600,6 @@ def launch_lbry_console(): reactor.addSystemEventTrigger('before', 'shutdown', console.shut_down) reactor.run() + +if __name__ == "__main__": + launch_lbry_console() \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py new file mode 100644 index 000000000..543aac02b --- /dev/null +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -0,0 +1,379 @@ +from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType +from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory +from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier +from lbrynet.core.client.AutoDownloader import GetStream, FetcherDaemon +from lbrynet.core.utils import generate_id +from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings +from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE +from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier +from lbrynet.core.Session import LBRYSession +from lbrynet.core.PTCWallet import PTCWallet +from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet +from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager +from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager +from twisted.web import xmlrpc, server +from twisted.internet import defer, threads, reactor +from datetime import datetime +import logging +import os + +log = logging.getLogger(__name__) + + +class LBRYDaemon(xmlrpc.XMLRPC): + """ + LBRYnet daemon + """ + + def setup(self): + def _set_vars(): + self.fetcher = None + self.current_db_revision = 1 + self.run_server = True + self.session = None + self.known_dht_nodes = [('104.236.42.182', 4000)] + self.db_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") + self.blobfile_dir = os.path.join(self.db_dir, "blobfiles") + self.peer_port = 3333 + self.dht_node_port = 4444 + self.first_run = False + self.current_db_revision = 1 + if os.name == "nt": + from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle + self.download_directory = get_path(FOLDERID.Downloads, UserHandle.current) + self.wallet_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrycrd") + else: + self.download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') + self.wallet_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd") + self.wallet_conf = os.path.join(self.wallet_dir, "lbrycrd.conf") + self.wallet_user = None + self.wallet_password = None + self.sd_identifier = StreamDescriptorIdentifier() + self.stream_info_manager = TempLBRYFileMetadataManager() + self.wallet_rpc_port = 8332 + self.downloads = [] + self.stream_frames = [] + self.default_blob_data_payment_rate = MIN_BLOB_DATA_PAYMENT_RATE + self.use_upnp = True + self.start_lbrycrdd = True + if os.name == "nt": + self.lbrycrdd_path = "lbrycrdd.exe" + else: + self.lbrycrdd_path = "./lbrycrdd" + self.delete_blobs_on_remove = True + self.blob_request_payment_rate_manager = None + self.lbry_file_metadata_manager = None + self.lbry_file_manager = None + self.settings = LBRYSettings(self.db_dir) + self.wallet_type = "lbrycrd" + self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd") + self.lbrycrd_conf = os.path.join(self.lbrycrd_dir, "lbrycrd.conf") + self.autofetcher_conf = os.path.join(self.lbrycrd_dir, "autofetcher.conf") + self.rpc_conn = None + self.files = [] + self.created_data_dir = False + if not os.path.exists(self.db_dir): + os.mkdir(self.db_dir) + self.created_data_dir = True + return defer.succeed(None) + + d = defer.Deferred() + d.addCallback(lambda _: _set_vars()) + d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory)) + d.addCallback(lambda _: self._check_db_migration()) + d.addCallback(lambda _: self._get_settings()) + d.addCallback(lambda _: self.get_lbrycrdd_path()) + d.addCallback(lambda _: self._get_session()) + d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier)) + d.addCallback(lambda _: self._setup_stream_identifier()) + d.addCallback(lambda _: self._setup_lbry_file_manager()) + d.addCallback(lambda _: self._setup_lbry_file_opener()) + d.addCallback(lambda _: self._setup_fetcher()) + d.callback(None) + + return defer.succeed(None) + + def _setup_fetcher(self): + self.fetcher = FetcherDaemon(self.session, self.lbry_file_manager, self.lbry_file_metadata_manager, + self.session.wallet, self.sd_identifier, self.autofetcher_conf) + return defer.succeed(None) + + def _setup_data_directory(self): + print "Loading databases..." + if self.created_data_dir: + db_revision = open(os.path.join(self.db_dir, "db_revision"), mode='w') + db_revision.write(str(self.current_db_revision)) + db_revision.close() + log.debug("Created the db revision file: %s", str(os.path.join(self.db_dir, "db_revision"))) + if not os.path.exists(self.blobfile_dir): + os.mkdir(self.blobfile_dir) + log.debug("Created the blobfile directory: %s", str(self.blobfile_dir)) + + def _check_db_migration(self): + old_revision = 0 + db_revision_file = os.path.join(self.db_dir, "db_revision") + if os.path.exists(db_revision_file): + old_revision = int(open(db_revision_file).read().strip()) + if old_revision < self.current_db_revision: + from lbrynet.db_migrator import dbmigrator + print "Upgrading your databases..." + d = threads.deferToThread(dbmigrator.migrate_db, self.db_dir, old_revision, self.current_db_revision) + + def print_success(old_dirs): + success_string = "Finished upgrading the databases. It is now safe to delete the" + success_string += " following directories, if you feel like it. It won't make any" + success_string += " difference.\nAnyway here they are: " + for i, old_dir in enumerate(old_dirs): + success_string += old_dir + if i + 1 < len(old_dir): + success_string += ", " + print success_string + + d.addCallback(print_success) + return d + return defer.succeed(True) + + def _get_settings(self): + d = self.settings.start() + d.addCallback(lambda _: self.settings.get_lbryid()) + d.addCallback(self.set_lbryid) + d.addCallback(lambda _: self.get_lbrycrdd_path()) + return d + + def set_lbryid(self, lbryid): + if lbryid is None: + return self._make_lbryid() + else: + self.lbryid = lbryid + + def _make_lbryid(self): + self.lbryid = generate_id() + d = self.settings.save_lbryid(self.lbryid) + return d + + def _setup_lbry_file_manager(self): + self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir) + d = self.lbry_file_metadata_manager.setup() + + def set_lbry_file_manager(): + self.lbry_file_manager = LBRYFileManager(self.session, self.lbry_file_metadata_manager, self.sd_identifier) + return self.lbry_file_manager.setup() + + d.addCallback(lambda _: set_lbry_file_manager()) + + return d + + def _get_session(self): + def get_default_data_rate(): + d = self.settings.get_default_data_payment_rate() + d.addCallback(lambda rate: {"default_data_payment_rate": + rate if rate is not None else MIN_BLOB_DATA_PAYMENT_RATE}) + return d + + def get_wallet(): + if self.wallet_type == "lbrycrd": + lbrycrdd_path = None + if self.start_lbrycrdd is True: + lbrycrdd_path = self.lbrycrdd_path + if not lbrycrdd_path: + lbrycrdd_path = self.default_lbrycrdd_path + d = defer.succeed(LBRYcrdWallet(self.db_dir, wallet_dir=self.lbrycrd_dir, wallet_conf=self.lbrycrd_conf, + lbrycrdd_path=lbrycrdd_path)) + else: + d = defer.succeed(PTCWallet(self.db_dir)) + d.addCallback(lambda wallet: {"wallet": wallet}) + return d + + d1 = get_default_data_rate() + d2 = get_wallet() + + def combine_results(results): + r = {} + for success, result in results: + if success is True: + r.update(result) + return r + + def create_session(results): + self.session = LBRYSession(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid, + blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port, + known_dht_nodes=self.known_dht_nodes, peer_port=self.peer_port, + use_upnp=self.use_upnp, wallet=results['wallet']) + self.rpc_conn = self.session.wallet.get_rpc_conn_x() + + dl = defer.DeferredList([d1, d2], fireOnOneErrback=True) + dl.addCallback(combine_results) + dl.addCallback(create_session) + dl.addCallback(lambda _: self.session.setup()) + return dl + + def get_lbrycrdd_path(self): + def get_lbrycrdd_path_conf_file(): + lbrycrdd_path_conf_path = os.path.join(os.path.expanduser("~"), ".lbrycrddpath.conf") + if not os.path.exists(lbrycrdd_path_conf_path): + return "" + lbrycrdd_path_conf = open(lbrycrdd_path_conf_path) + lines = lbrycrdd_path_conf.readlines() + return lines + + d = threads.deferToThread(get_lbrycrdd_path_conf_file) + + def load_lbrycrdd_path(conf): + for line in conf: + if len(line.strip()) and line.strip()[0] != "#": + self.lbrycrdd_path = line.strip() + print self.lbrycrdd_path + + d.addCallback(load_lbrycrdd_path) + return d + + def _setup_stream_identifier(self): + file_saver_factory = LBRYFileSaverFactory(self.session.peer_finder, self.session.rate_limiter, + self.session.blob_manager, self.stream_info_manager, + self.session.wallet, self.download_directory) + self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, file_saver_factory) + file_opener_factory = LBRYFileOpenerFactory(self.session.peer_finder, self.session.rate_limiter, + self.session.blob_manager, self.stream_info_manager, + self.session.wallet) + self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, file_opener_factory) + return defer.succeed(None) + + def _setup_lbry_file_manager(self): + self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir) + d = self.lbry_file_metadata_manager.setup() + + def set_lbry_file_manager(): + self.lbry_file_manager = LBRYFileManager(self.session, self.lbry_file_metadata_manager, self.sd_identifier) + return self.lbry_file_manager.setup() + + d.addCallback(lambda _: set_lbry_file_manager()) + + return d + + def _setup_lbry_file_opener(self): + + downloader_factory = LBRYFileOpenerFactory(self.session.peer_finder, self.session.rate_limiter, + self.session.blob_manager, self.stream_info_manager, + self.session.wallet) + self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory) + return defer.succeed(True) + + def xmlrpc_start_fetcher(self): + """ + Start autofetcher + """ + + self.fetcher.start() + + return str('Started autofetching') + + def xmlrpc_stop_fetcher(self): + """ + Start autofetcher + """ + + self.fetcher.stop() + + return str('Started autofetching') + + def xmlrpc_fetcher_status(self): + """ + Start autofetcher + """ + + return str(self.fetcher.check_if_running()) + + def xmlrpc_get_balance(self): + """ + Get LBC balance + """ + + return str(self.session.wallet.wallet_balance) + + def xmlrpc_stop(self): + """ + Stop the reactor + """ + + reactor.stop() + return defer.succeed('Stopping') + + def xmlrpc_resolve_name(self, name): + """ + Resolve stream info from a LBRY uri + """ + + def _disp(info): + log.debug('[' + str(datetime.now()) + ']' + ' Resolved info: ' + str(info)) + print '[' + str(datetime.now()) + ']' + ' Resolved info: ' + str(info) + return info + + d = defer.Deferred() + d.addCallback(lambda _: self.session.wallet.get_stream_info_for_name(name)) + d.addErrback(lambda _: 'UnknownNameError') + d.addCallback(_disp) + d.callback(None) + return d + + def xmlrpc_get_downloads(self): + """ + Get downloads + """ + + downloads = [] + + for stream in self.downloads: + try: + downloads.append({'stream_hash': stream.stream_hash, + 'path': os.path.join(stream.downloader.download_directory, stream.downloader.file_name)}) + except: + pass + + return downloads + + def xmlrpc_download_name(self, name): + """ + Download stream from a LBRY uri + """ + + def _disp(): + try: + stream = self.downloads[-1] + log.debug('[' + str(datetime.now()) + ']' + ' Downloading: ' + str(stream.stream_hash)) + print '[' + str(datetime.now()) + ']' + ' Downloading: ' + str(stream.stream_hash) + return defer.succeed(None) + except: + pass + + stream = GetStream(self.sd_identifier, self.session, self.session.wallet, self.lbry_file_manager, 25.0) + self.downloads.append(stream) + + d = self.session.wallet.get_stream_info_for_name(name) + d.addCallback(lambda stream_info: stream.start(stream_info)) + d.addCallback(lambda _: _disp()) + # d.addCallback(lambda _: self.files.append({'name': name, 'stream_hash': stream.stream_hash, + # 'path': os.path.join(stream.downloader.download_directory, stream.downloader.file_name)})) + d.addCallback(lambda _: {'ts': datetime.now(),'name': name}) + d.addErrback(lambda err: str(err.getTraceback())) + + return d + + def xmlrpc_path_from_name(self, name): + d = self.session.wallet.get_stream_info_for_name(name) + d.addCallback(lambda stream_info: stream_info['stream_hash']) + d.addCallback(lambda stream_hash: [{'stream_hash': stream.stream_hash, + 'path': os.path.join(stream.downloader.download_directory, + stream.downloader.file_name)} + for stream in self.downloads if stream.stream_hash == stream_hash]) + d.addErrback(lambda _: 'UnknownNameError') + return d + + +def main(): + daemon = LBRYDaemon() + daemon.setup() + reactor.listenTCP(7080, server.Site(daemon)) + reactor.run() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYURIHandler/LBRYURIHandler.py b/lbrynet/lbrynet_daemon/LBRYURIHandler/LBRYURIHandler.py new file mode 100644 index 000000000..81bc0fb83 --- /dev/null +++ b/lbrynet/lbrynet_daemon/LBRYURIHandler/LBRYURIHandler.py @@ -0,0 +1,20 @@ +import webbrowser +import xmlrpclib, sys + + +def main(args): + if len(args) == 0: + args.append('lbry://wonderfullife') + + daemon = xmlrpclib.ServerProxy('http://localhost:7080/') + + if len(args) > 1: + print 'Too many args', args + + else: + daemon.download_name(str(args[0])[7:]) + path = daemon.path_from_name(args[0][7:])[0]['path'] + webbrowser.open('file://' + path) + +if __name__ == "__main__": + main(sys.argv[1:]) \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYURIHandler/dist/LBRYURIHandler.app/Contents/Info.plist b/lbrynet/lbrynet_daemon/LBRYURIHandler/dist/LBRYURIHandler.app/Contents/Info.plist new file mode 100644 index 000000000..4f5993ff1 --- /dev/null +++ b/lbrynet/lbrynet_daemon/LBRYURIHandler/dist/LBRYURIHandler.app/Contents/Info.plist @@ -0,0 +1,117 @@ + + + + + CFBundleDevelopmentRegion + English + CFBundleDisplayName + LBRYURIHandler + CFBundleDocumentTypes + + + CFBundleTypeOSTypes + + **** + fold + disk + + CFBundleTypeRole + Viewer + + + CFBundleExecutable + LBRYURIHandler + +CFBundleURLTypes + + + CFBundleURLName + LBRYURIHandler + CFBundleURLSchemes + + lbry + + + +NSUIElement + + + CFBundleIconFile + PythonApplet.icns + CFBundleIdentifier + org.pythonmac.unspecified.LBRYURIHandler + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + LBRYURIHandler + CFBundlePackageType + APPL + CFBundleShortVersionString + 0.0.0 + CFBundleSignature + ???? + CFBundleVersion + 0.0.0 + LSHasLocalizedDisplayName + + NSAppleScriptEnabled + + NSHumanReadableCopyright + Copyright not specified + NSMainNibFile + MainMenu + NSPrincipalClass + NSApplication + PyMainFileNames + + __boot__ + + PyOptions + + alias + + argv_emulation + + emulate_shell_environment + + no_chdir + + prefer_ppc + + site_packages + + use_faulthandler + + use_pythonpath + + verbose + + + PyResourcePackages + + + PyRuntimeLocations + + @executable_path/../Frameworks/Python.framework/Versions/2.7/Python + + PythonInfoDict + + PythonExecutable + /Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python + PythonLongVersion + 2.7.10 (v2.7.10:15c95b7d81dc, May 23 2015, 09:33:12) +[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] + PythonShortVersion + 2.7 + py2app + + alias + + template + app + version + 0.9 + + + + diff --git a/lbrynet/lbrynet_daemon/LBRYURIHandler/setup.py b/lbrynet/lbrynet_daemon/LBRYURIHandler/setup.py new file mode 100644 index 000000000..fbb83f5dc --- /dev/null +++ b/lbrynet/lbrynet_daemon/LBRYURIHandler/setup.py @@ -0,0 +1,19 @@ +""" +This is a setup.py script generated by py2applet + +Usage: + python setup.py py2app +""" + +from setuptools import setup + +APP = ['LBRYURIHandler.py'] +DATA_FILES = [] +OPTIONS = {'argv_emulation': True} + +setup( + app=APP, + data_files=DATA_FILES, + options={'py2app': OPTIONS}, + setup_requires=['py2app'], +) diff --git a/lbrynet/lbrynet_daemon/__init__.py b/lbrynet/lbrynet_daemon/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index c4fea7d50..80e5ac735 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ setup(name='lbrynet', 'lbrynet-gui = lbrynet.lbrynet_gui.gui:start_gui', 'lbrynet-lookup-hosts-for-hash = lbrynet.dht_scripts:get_hosts_for_hash_in_dht', 'lbrynet-announce_hash_to_dht = lbrynet.dht_scripts:announce_hash_to_dht', + 'lbrynet-daemon = lbrynet.lbrynet_daemon.LBRYDaemon:main', ] }, data_files=[