diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cb7f8ec7f..715cdfe80 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.6 +current_version = 0.3.10 commit = True tag = True message = Bump version: {current_version} -> {new_version} diff --git a/.pylintrc b/.pylintrc index 5e5aab35c..e49732469 100644 --- a/.pylintrc +++ b/.pylintrc @@ -298,7 +298,7 @@ ignored-classes=twisted.internet,RequestMessage # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters [IMPORTS] diff --git a/.travis.yml b/.travis.yml index cead1a849..ca2557860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ cache: directories: - $HOME/.cache/pip - $HOME/Library/Caches/pip + - $TRAVIS_BUILD_DIR/cache/wheel before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/travis/setup_osx.sh; fi diff --git a/FAQ.md b/FAQ.md index 6336272da..091acd711 100644 --- a/FAQ.md +++ b/FAQ.md @@ -12,7 +12,7 @@ You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio On Ubuntu or Mint you can install the prerequisites and lbrynet by running - sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip + sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git git clone https://github.com/lbryio/lbry.git cd lbry sudo python setup.py install diff --git a/INSTALL.md b/INSTALL.md index 23491e60c..8bf9f8917 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -23,7 +23,7 @@ You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio On Ubuntu or Mint you can install the prerequisites and lbrynet by running ``` - sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip + sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git git clone https://github.com/lbryio/lbry.git cd lbry sudo python setup.py install diff --git a/RUNNING.md b/RUNNING.md index b9f60acc9..d2f1842b4 100644 --- a/RUNNING.md +++ b/RUNNING.md @@ -43,9 +43,9 @@ To stop lbrynet-console, enter the command 'exit'. Note: this process takes upwards of an hour and is not necessary to use lbrynet. ``` +sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev libboost-all-dev libdb-dev libdb++-dev libqt4-dev libprotobuf-dev protobuf-compiler git git clone --depth=1 -b alpha https://github.com/lbryio/lbrycrd.git cd lbrycrd -sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev libboost-all-dev libdb-dev libdb++-dev libqt4-dev libprotobuf-dev protobuf-compiler ./autogen.sh ./configure --with-incompatible-bdb --without-gui diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index f106a1889..969401685 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -4,5 +4,5 @@ log = logging.getLogger(__name__) logging.getLogger(__name__).addHandler(logging.NullHandler()) log.setLevel(logging.ERROR) -__version__ = "0.3.6" +__version__ = "0.3.10" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 59150a085..736381c4a 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -48,8 +48,7 @@ DEFAULT_CACHE_TIME = 3600 DEFAULT_UI_BRANCH = "master" SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] - CURRENCIES = [ {'BTC': {'type': 'crypto'}}, {'LBC': {'type': 'crypto'}}, -] \ No newline at end of file +] diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 0e9e3a360..e5f7f1889 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -917,7 +917,7 @@ class LBRYumWallet(LBRYWallet): network_start_d = defer.Deferred() def setup_network(): - self.config = SimpleConfig() + self.config = SimpleConfig({'auto_connect': True}) self.network = Network(self.config) alert.info("Loading the wallet...") return defer.succeed(self.network.start()) @@ -989,7 +989,7 @@ class LBRYumWallet(LBRYWallet): blockchain_caught_d = defer.Deferred() def check_caught_up(): - local_height = self.network.get_local_height() + local_height = self.network.get_catchup_progress() remote_height = self.network.get_server_height() if remote_height != 0 and remote_height - local_height <= 5: @@ -1115,11 +1115,9 @@ class LBRYumWallet(LBRYWallet): def _do_send_many(self, payments_to_send): log.warning("Doing send many. payments to send: %s", str(payments_to_send)) - outputs = [(TYPE_ADDRESS, address, int(amount*COIN)) for address, amount in payments_to_send.iteritems()] - d = threads.deferToThread(self.wallet.mktx, outputs, None, self.config) - d.addCallback(lambda tx: threads.deferToThread(self.wallet.sendtx, tx)) - d.addCallback(self._save_wallet) - return d + cmd = known_commands['paytomanyandsend'] + func = getattr(self.cmd_runner, cmd.name) + return threads.deferToThread(func, payments_to_send.iteritems()) def _get_value_for_name(self, name): cmd = known_commands['getvalueforname'] diff --git a/lbrynet/core/server/BlobRequestHandler.py b/lbrynet/core/server/BlobRequestHandler.py index 2ce3688b4..36f67d797 100644 --- a/lbrynet/core/server/BlobRequestHandler.py +++ b/lbrynet/core/server/BlobRequestHandler.py @@ -140,6 +140,7 @@ class BlobRequestHandler(object): def set_expected_payment(): log.info("Setting expected payment") if self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None: + # TODO: explain why 2**20 self.wallet.add_expected_payment(self.peer, self.currently_uploading.length * 1.0 * self.blob_data_payment_rate / 2**20) @@ -156,4 +157,4 @@ class BlobRequestHandler(object): if reason is not None and isinstance(reason, Failure): log.info("Upload has failed. Reason: %s", reason.getErrorMessage()) - return _send_file() \ No newline at end of file + return _send_file() diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 75aaa0096..09f8f0db4 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -67,6 +67,11 @@ if not os.path.isdir(log_dir): lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) log = logging.getLogger(__name__) + +# TODO: configuring a logger on module import drastically reduces the +# amount of control the caller of this code has over logging +# +# Better would be to configure all logging at runtime. handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, backupCount=5) log.addHandler(handler) log.setLevel(logging.INFO) @@ -160,6 +165,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbryum_version = None self.ui_version = None self.ip = None + # TODO: this is confusing to set here, and then to be reset below. self.wallet_type = wallet_type self.first_run = None self.log_file = lbrynet_log @@ -171,6 +177,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.known_dht_nodes = KNOWN_DHT_NODES self.first_run_after_update = False self.last_traded_rate = None + self.uploaded_temp_files = [] if os.name == "nt": from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle @@ -268,13 +275,30 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.search_timeout = self.session_settings['search_timeout'] self.download_timeout = self.session_settings['download_timeout'] self.max_search_results = self.session_settings['max_search_results'] - if self.session_settings['wallet_type'] in WALLET_TYPES and not wallet_type: - self.wallet_type = self.session_settings['wallet_type'] - log.info("Using wallet type %s from config" % self.wallet_type) - else: + #### + # + # Ignore the saved wallet type. Some users will have their wallet type + # saved as lbrycrd and we want wallets to be lbryum unless explicitly + # set on the command line to be lbrycrd. + # + # if self.session_settings['wallet_type'] in WALLET_TYPES and not wallet_type: + # self.wallet_type = self.session_settings['wallet_type'] + # log.info("Using wallet type %s from config" % self.wallet_type) + # else: + # self.wallet_type = wallet_type + # self.session_settings['wallet_type'] = wallet_type + # log.info("Using wallet type %s specified from command line" % self.wallet_type) + # + # Instead, if wallet is not set on the command line, default to the default wallet + # + if wallet_type: + log.info("Using wallet type %s specified from command line", wallet_type) self.wallet_type = wallet_type - self.session_settings['wallet_type'] = wallet_type - log.info("Using wallet type %s specified from command line" % self.wallet_type) + else: + log.info("Using the default wallet type %s", DEFAULT_WALLET) + self.wallet_type = DEFAULT_WALLET + # + #### self.delete_blobs_on_remove = self.session_settings['delete_blobs_on_remove'] self.peer_port = self.session_settings['peer_port'] self.dht_node_port = self.session_settings['dht_node_port'] @@ -448,7 +472,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _announce(): self.announced_startup = True self.startup_status = STARTUP_STAGES[5] - log.info("[" + str(datetime.now()) + "] Started lbrynet-daemon") + log.info("Started lbrynet-daemon") if len(self.startup_scripts): log.info("Scheduling scripts") reactor.callLater(3, self._run_scripts) @@ -470,7 +494,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: _announce()) return d - log.info("[" + str(datetime.now()) + "] Starting lbrynet-daemon") + log.info("Starting lbrynet-daemon") self.internet_connection_checker.start(3600) self.version_checker.start(3600 * 12) @@ -536,14 +560,14 @@ class LBRYDaemon(jsonrpc.JSONRPC): s = socket.create_connection((host, 80), 2) self.connected_to_internet = True except: - log.info("[" + str(datetime.now()) + "] Internet connection not working") + log.info("Internet connection not working") self.connected_to_internet = False def _check_lbrynet_connection(self): def _log_success(): - log.info("[" + str(datetime.now()) + "] lbrynet connectivity test passed") + log.info("lbrynet connectivity test passed") def _log_failure(): - log.info("[" + str(datetime.now()) + "] lbrynet connectivity test failed") + log.info("lbrynet connectivity test failed") wonderfullife_sh = "6f3af0fa3924be98a54766aa2715d22c6c1509c3f7fa32566df4899a41f3530a9f97b2ecb817fa1dcbf1b30553aefaa7" d = download_sd_blob(self.session, wonderfullife_sh, self.session.base_payment_rate_manager) @@ -561,7 +585,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbryum_version = version return defer.succeed(None) except: - log.info("[" + str(datetime.now()) + "] Failed to get lbryum version from git") + log.info("Failed to get lbryum version from git") self.git_lbryum_version = None return defer.fail(None) @@ -576,7 +600,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbrynet_version = vr return defer.succeed(None) except: - log.info("[" + str(datetime.now()) + "] Failed to get lbrynet version from git") + log.info("Failed to get lbrynet version from git") self.git_lbrynet_version = None return defer.fail(None) @@ -703,6 +727,13 @@ class LBRYDaemon(jsonrpc.JSONRPC): else: return defer.succeed(None) + def _clean_up_temp_files(self): + for path in self.uploaded_temp_files: + try: + os.remove(path) + except OSError: + pass + def _shutdown(self): log.info("Closing lbrynet session") log.info("Status at time of shutdown: " + self.startup_status[0]) @@ -717,6 +748,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): if self.price_checker.running: self.price_checker.stop() + self._clean_up_temp_files() + d = self._upload_log(log_type="close", exclude_previous=False if self.first_run else True) d.addCallback(lambda _: self._stop_server()) d.addErrback(lambda err: True) @@ -892,7 +925,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d def get_wallet(): - if self.wallet_type == "lbrycrd": #force lbrycrd wallet no matter what while lbryum is down + if self.wallet_type == "lbrycrd": log.info("Using lbrycrd wallet") d = defer.succeed(LBRYcrdWallet(self.db_dir, wallet_dir=self.wallet_dir, wallet_conf=self.lbrycrd_conf, lbrycrdd_path=self.lbrycrdd_path)) @@ -903,7 +936,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): log.info("Using PTC wallet") d = defer.succeed(PTCWallet(self.db_dir)) else: - log.info("Requested unknown wallet '%s', using default lbryum" % self.wallet_type) + # TODO: should fail here. Can't switch to lbrycrd because the wallet_dir, conf and path won't be set + log.info("Requested unknown wallet '%s', using default lbryum", self.wallet_type) d = defer.succeed(LBRYumWallet(self.db_dir)) d.addCallback(lambda wallet: {"wallet": wallet}) @@ -1164,15 +1198,15 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d d.addCallback(lambda _: finish_deletion(lbry_file)) - d.addCallback(lambda _: log.info("[" + str(datetime.now()) + "] Delete lbry file")) + d.addCallback(lambda _: log.info("Delete lbry file")) return d def _get_est_cost(self, name): def _check_est(d, name): if isinstance(d.result, float): - log.info("[" + str(datetime.now()) + "] Cost est for lbry://" + name + ": " + str(d.result) + "LBC") + log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC") else: - log.info("[" + str(datetime.now()) + "] Timeout estimating cost for lbry://" + name + ", using key fee") + log.info("Timeout estimating cost for lbry://" + name + ", using key fee") d.cancel() return defer.succeed(None) @@ -1367,7 +1401,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): Returns: true if daemon completed startup, otherwise false """ - log.info("[" + str(datetime.now()) + "] is_running: " + str(self.announced_startup)) + log.info("is_running: " + str(self.announced_startup)) if self.announced_startup: return self._render_response(True, OK_CODE) @@ -1405,7 +1439,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): else: r['message'] = "Catching up with the blockchain" r['progress'] = 0 - log.info("[" + str(datetime.now()) + "] daemon status: " + str(r)) + log.info("daemon status: " + str(r)) return self._render_response(r, OK_CODE) def jsonrpc_is_first_run(self): @@ -1418,7 +1452,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): True if first run, otherwise False """ - log.info("[" + str(datetime.now()) + "] Check if is first run") + log.info("Check if is first run") try: d = self.session.wallet.is_first_run() except: @@ -1438,7 +1472,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): Startup message, such as first run notification """ - log.info("[" + str(datetime.now()) + "] Get startup notice") + log.info("Get startup notice") if self.first_run and not self.session.wallet.wallet_balance: return self._render_response(self.startup_message, OK_CODE) @@ -1478,7 +1512,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'lbryum_update_available': lbryum_version < self.git_lbryum_version } - log.info("[" + str(datetime.now()) + "] Get version info: " + json.dumps(msg)) + log.info("Get version info: " + json.dumps(msg)) return self._render_response(msg, OK_CODE) def jsonrpc_get_settings(self): @@ -1506,7 +1540,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'start_lbrycrdd': bool, """ - log.info("[" + str(datetime.now()) + "] Get daemon settings") + log.info("Get daemon settings") return self._render_response(self.session_settings, OK_CODE) def jsonrpc_set_settings(self, p): @@ -1527,7 +1561,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): """ def _log_settings_change(): - log.info("[" + str(datetime.now()) + "] Set daemon settings to " + json.dumps(self.session_settings)) + log.info("Set daemon settings to " + json.dumps(self.session_settings)) d = self._update_settings(p) d.addErrback(lambda err: log.info(err.getTraceback())) @@ -1570,7 +1604,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): balance, float """ - log.info("[" + str(datetime.now()) + "] Get balance") + log.info("Get balance") return self._render_response(float(self.session.wallet.wallet_balance), OK_CODE) def jsonrpc_stop(self): @@ -1811,7 +1845,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.DeferredList(ds) def _disp(results): - log.info('[' + str(datetime.now()) + '] Found ' + str(len(results)) + ' search results') + log.info('Found ' + str(len(results)) + ' search results') consolidated_results = [] for r in results: t = {} @@ -1825,7 +1859,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return consolidated_results - log.info('[' + str(datetime.now()) + '] Search nametrie: ' + search) + log.info('Search nametrie: ' + search) d = self.session.wallet.get_nametrie() d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) @@ -1913,7 +1947,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return server.failure def _disp(x): - log.info("[" + str(datetime.now()) + "] Abandoned name claim tx " + str(x)) + log.info("Abandoned name claim tx " + str(x)) return self._render_response(x, OK_CODE) d = defer.Deferred() @@ -2024,7 +2058,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): """ def _disp(address): - log.info("[" + str(datetime.now()) + "] Got new wallet address: " + address) + log.info("Got new wallet address: " + address) return defer.succeed(address) d = self.session.wallet.get_new_address() @@ -2216,7 +2250,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): exclude_previous = True if 'message' in p.keys(): - log.info("[" + str(datetime.now()) + "] Upload log message: " + str(p['message'])) + log.info("Upload log message: " + str(p['message'])) if 'force' in p.keys(): force = p['force'] diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py index 2b34d4301..0b5d0ba0c 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py @@ -35,7 +35,7 @@ def main(): if len(args) > 1: if isinstance(args[1], dict): params = args[1] - elif isinstance(args[1], str, unicode): + elif isinstance(args[1], basestring): params = json.loads(args[1]) else: params = None @@ -56,4 +56,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 309570873..b25f73a95 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -12,7 +12,7 @@ from twisted.web import server from twisted.internet import reactor, defer from jsonrpc.proxy import JSONRPCProxy -from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer +from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, \ DEFAULT_WALLET, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME @@ -25,8 +25,13 @@ if not os.path.isdir(log_dir): os.mkdir(log_dir) lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) + +DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s" +DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT) + log = logging.getLogger(__name__) handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, backupCount=5) +handler.setFormatter(DEFAULT_FORMATTER) log.addHandler(handler) log.setLevel(logging.INFO) @@ -57,6 +62,13 @@ def stop(): d.callback(None) +def configureConsoleLogger(): + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(DEFAULT_FORMATTER) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel(level=logging.INFO) + + def start(): parser = argparse.ArgumentParser(description="Launch lbrynet-daemon") parser.add_argument("--wallet", @@ -75,7 +87,7 @@ def start(): args = parser.parse_args() if args.logtoconsole: - logging.basicConfig(level=logging.INFO) + configureConsoleLogger() args = parser.parse_args() @@ -109,7 +121,9 @@ def start(): if args.launchui: d.addCallback(lambda _: webbrowser.open(UI_ADDRESS)) - reactor.listenTCP(API_PORT, server.Site(lbry.root), interface=API_INTERFACE) + lbrynet_server = server.Site(lbry.root) + lbrynet_server.requestFactory = LBRYDaemonRequest + reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE) reactor.run() if not args.logtoconsole and not args.quiet: @@ -121,4 +135,4 @@ def start(): return if __name__ == "__main__": - start() \ No newline at end of file + start() diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py index 74906da0b..6a1fc7c9b 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py @@ -4,6 +4,10 @@ import shutil import json import sys import mimetypes +import mimetools +import tempfile +import time +import cgi from datetime import datetime from appdirs import user_data_dir @@ -29,6 +33,184 @@ handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, ba log.addHandler(handler) log.setLevel(logging.INFO) +class LBRYDaemonRequest(server.Request): + """ + For LBRY specific request functionality. Currently just provides + handling for large multipart POST requests, taken from here: + http://sammitch.ca/2013/07/handling-large-requests-in-twisted/ + + For multipart POST requests, this populates self.args with temp + file objects instead of strings. Note that these files don't auto-delete + on close because we want to be able to move and rename them. + + """ + + # max amount of memory to allow any ~single~ request argument [ie: POSTed file] + # note: this value seems to be taken with a grain of salt, memory usage may spike + # FAR above this value in some cases. + # eg: set the memory limit to 5 MB, write 2 blocks of 4MB, mem usage will + # have spiked to 8MB before the data is rolled to disk after the + # second write completes. + memorylimit = 1024*1024*100 + + # enable/disable debug logging + do_log = False + + # re-defined only for debug/logging purposes + def gotLength(self, length): + if self.do_log: + print '%f Headers received, Content-Length: %d' % (time.time(), length) + server.Request.gotLength(self, length) + + # re-definition of twisted.web.server.Request.requestreceived, the only difference + # is that self.parse_multipart() is used rather than cgi.parse_multipart() + def requestReceived(self, command, path, version): + from twisted.web.http import parse_qs + if self.do_log: + print '%f Request Received' % time.time() + print self.content + + self.content.seek(0,0) + self.args = {} + self.stack = [] + + self.method, self.uri = command, path + self.clientproto = version + x = self.uri.split(b'?', 1) + + if len(x) == 1: + self.path = self.uri + else: + self.path, argstring = x + self.args = parse_qs(argstring, 1) + + # cache the client and server information, we'll need this later to be + # serialized and sent with the request so CGIs will work remotely + self.client = self.channel.transport.getPeer() + self.host = self.channel.transport.getHost() + + # Argument processing + args = self.args + ctype = self.requestHeaders.getRawHeaders(b'content-type') + if ctype is not None: + ctype = ctype[0] + + if self.method == b"POST" and ctype: + mfd = b'multipart/form-data' + key, pdict = cgi.parse_header(ctype) + if key == b'application/x-www-form-urlencoded': + args.update(parse_qs(self.content.read(), 1)) + elif key == mfd: + try: + self.content.seek(0,0) + args.update(self.parse_multipart(self.content, pdict)) + #args.update(cgi.parse_multipart(self.content, pdict)) + + except KeyError as e: + if e.args[0] == b'content-disposition': + # Parse_multipart can't cope with missing + # content-dispostion headers in multipart/form-data + # parts, so we catch the exception and tell the client + # it was a bad request. + self.channel.transport.write( + b"HTTP/1.1 400 Bad Request\r\n\r\n") + self.channel.transport.loseConnection() + return + raise + + self.content.seek(0, 0) + + self.process() + + # re-definition of cgi.parse_multipart that uses a single temporary file to store + # data rather than storing 2 to 3 copies in various lists. + def parse_multipart(self, fp, pdict): + if self.do_log: + print '%f Parsing Multipart data: ' % time.time() + rewind = fp.tell() #save cursor + fp.seek(0,0) #reset cursor + + boundary = "" + if 'boundary' in pdict: + boundary = pdict['boundary'] + if not cgi.valid_boundary(boundary): + raise ValueError, ('Invalid boundary in multipart form: %r' + % (boundary,)) + + nextpart = "--" + boundary + lastpart = "--" + boundary + "--" + partdict = {} + terminator = "" + + while terminator != lastpart: + c_bytes = -1 + + data = tempfile.NamedTemporaryFile(delete=False) + if terminator: + # At start of next part. Read headers first. + headers = mimetools.Message(fp) + clength = headers.getheader('content-length') + if clength: + try: + c_bytes = int(clength) + except ValueError: + pass + if c_bytes > 0: + data.write(fp.read(c_bytes)) + # Read lines until end of part. + while 1: + line = fp.readline() + if not line: + terminator = lastpart # End outer loop + break + if line[:2] == "--": + terminator = line.strip() + if terminator in (nextpart, lastpart): + break + data.write(line) + # Done with part. + if data.tell() == 0: + continue + if c_bytes < 0: + # if a Content-Length header was not supplied with the MIME part + # then the trailing line break must be removed. + # we have data, read the last 2 bytes + rewind = min(2, data.tell()) + data.seek(-rewind, os.SEEK_END) + line = data.read(2) + if line[-2:] == "\r\n": + data.seek(-2, os.SEEK_END) + data.truncate() + elif line[-1:] == "\n": + data.seek(-1, os.SEEK_END) + data.truncate() + + line = headers['content-disposition'] + if not line: + continue + key, params = cgi.parse_header(line) + if key != 'form-data': + continue + if 'name' in params: + name = params['name'] + # kludge in the filename + if 'filename' in params: + fname_index = name + '_filename' + if fname_index in partdict: + partdict[fname_index].append(params['filename']) + else: + partdict[fname_index] = [params['filename']] + else: + # Unnamed parts are not returned at all. + continue + data.seek(0,0) + if name in partdict: + partdict[name].append(data) + else: + partdict[name] = [data] + + fp.seek(rewind) # Restore cursor + return partdict class LBRYindex(resource.Resource): def __init__(self, ui_dir): @@ -79,7 +261,7 @@ class LBRYFileStreamer(object): def pauseProducing(self): self._paused = True - log.info("[" + str(datetime.now()) + "] Pausing producer") + log.info("Pausing producer") return defer.succeed(None) def resumeProducing(self): @@ -104,7 +286,7 @@ class LBRYFileStreamer(object): self._request.write(data) self._cursor += 1 - log.info("[" + str(datetime.now()) + "] Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" % + log.info("Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" % (start_cur, self._cursor, self._file_size, self._cursor - start_cur, readable_bytes, self._depth)) self._sent_bytes = True @@ -117,12 +299,12 @@ class LBRYFileStreamer(object): self._deferred.addCallback(lambda _: threads.deferToThread(reactor.callLater, self._delay, _check_for_new_data)) return defer.succeed(None) - log.info("[" + str(datetime.now()) + "] Resuming producer") + log.info("Resuming producer") self._paused = False self._deferred.addCallback(lambda _: _check_for_new_data()) def stopProducing(self): - log.info("[" + str(datetime.now()) + "] Stopping producer") + log.info("Stopping producer") self._stopped = True # self._fileObject.close() self._deferred.addErrback(lambda err: err.trap(defer.CancelledError)) @@ -147,7 +329,7 @@ class HostedLBRYFile(resource.Resource): # # range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-') # start, stop = int(range_header[0]), range_header[1] - # log.info("[" + str(datetime.now()) + "] GET range %s-%s" % (start, stop)) + # log.info("GET range %s-%s" % (start, stop)) # path = os.path.join(self._api.download_directory, stream.file_name) # # d = stream.get_total_bytes() @@ -179,12 +361,35 @@ class HostedLBRYFile(resource.Resource): # call.addErrback(lambda err: log.info("Error: " + str(err))) # call.cancel() +class LBRYFileUpload(resource.Resource): + """ + Accepts a file sent via the file upload widget in the web UI, saves + it into a temporary dir, and responds with a JSON string containing + the path of the newly created file. + """ + + def __init__(self, api): + self._api = api + + def render_POST(self, request): + origfilename = request.args['file_filename'][0] + uploaded_file = request.args['file'][0] # Temp file created by request + + # Move to a new temporary dir and restore the original file name + newdirpath = tempfile.mkdtemp() + newpath = os.path.join(newdirpath, origfilename) + shutil.move(uploaded_file.name, newpath) + self._api.uploaded_temp_files.append(newpath) + + return json.dumps(newpath) + class LBRYDaemonServer(object): def _setup_server(self, wallet): self.root = LBRYindex(os.path.join(os.path.join(data_dir, "lbry-ui"), "active")) self._api = LBRYDaemon(self.root, wallet_type=wallet) self.root.putChild("view", HostedLBRYFile(self._api)) + self.root.putChild("upload", LBRYFileUpload(self._api)) self.root.putChild(API_ADDRESS, self._api) return defer.succeed(True) diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 9cfbf07c1..eddffa0b5 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -143,6 +143,6 @@ class GetStream(object): d = _pay_key_fee() self.downloader = downloader self.download_path = os.path.join(downloader.download_directory, downloader.file_name) - d.addCallback(lambda _: log.info("[%s] Downloading %s --> %s" % (datetime.now(), self.stream_hash, self.downloader.file_name))) + d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name)) d.addCallback(lambda _: self.downloader.start()) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index c0658bfec..bd5c50e4b 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -48,9 +48,7 @@ class Publisher(object): def start(self, name, file_path, bid, metadata, fee=None, sources={}): def _show_result(): - - message = "[%s] Published %s --> lbry://%s txid: %s" % (datetime.now(), self.file_name, self.publish_name, self.txid) - log.info(message) + log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) return defer.succeed(self.txid) self.publish_name = name diff --git a/packaging/osx/lbry-osx-app/setup_app.sh b/packaging/osx/lbry-osx-app/setup_app.sh index 5630a1e3f..ad3ca8449 100755 --- a/packaging/osx/lbry-osx-app/setup_app.sh +++ b/packaging/osx/lbry-osx-app/setup_app.sh @@ -5,9 +5,11 @@ set -o xtrace DEST=`pwd` tmp="${DEST}/build" +ON_TRAVIS=false rm -rf build dist LBRY.app +pip install wheel # the default py2app (v0.9) has a bug that is fixed in the head of /metachris/py2app pip install git+https://github.com/metachris/py2app pip install jsonrpc @@ -23,6 +25,7 @@ if [ -z ${TRAVIS_BUILD_DIR+x} ]; then LBRY="${tmp}/lbry" else # building on travis + ON_TRAVIS=true cd ${TRAVIS_BUILD_DIR} LBRY=${TRAVIS_BUILD_DIR} fi @@ -45,7 +48,25 @@ codesign -s "${LBRY_DEVELOPER_ID}" -f "${DEST}/dist/LBRYURIHandler.app/Contents/ codesign --deep -s "${LBRY_DEVELOPER_ID}" -f "${DEST}/dist/LBRYURIHandler.app/Contents/MacOS/LBRYURIHandler" codesign -vvvv "${DEST}/dist/LBRYURIHandler.app" -pip install certifi pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CFNetwork +pip install certifi +MODULES="pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CFNetwork" +if [ ${ON_TRAVIS} = true ]; then + WHEEL_DIR="${TRAVIS_BUILD_DIR}/cache/wheel" + mkdir -p "${WHEEL_DIR}" + # mapping from the package name to the + # actual built wheel file is surprisingly + # hard so instead of checking for the existance + # of each wheel, we mark with a file when they've all been + # built and skip when that file exists + if [ ! -f "${WHEEL_DIR}"/finished ]; then + pip wheel -w "${WHEEL_DIR}" ${MODULES} + touch "${WHEEL_DIR}"/finished + fi + pip install "${WHEEL_DIR}"/*.whl +else + pip install $MODULES +fi + # add lbrycrdd as a resource. Following # http://stackoverflow.com/questions/11370012/can-executables-made-with-py2app-include-other-terminal-scripts-and-run-them diff --git a/packaging/travis/install_dependencies_and_run_tests.sh b/packaging/travis/install_dependencies_and_run_tests.sh index f55ab609b..6014087e1 100755 --- a/packaging/travis/install_dependencies_and_run_tests.sh +++ b/packaging/travis/install_dependencies_and_run_tests.sh @@ -38,8 +38,8 @@ rm get-pip.py pip install -r requirements.txt -pip install nose coverage coveralls pylint -nosetests --with-coverage --cover-package=lbrynet -v -I functional_tests.py tests/ +pip install mock pylint +trial tests # TODO: submit coverage report to coveralls # TODO: as code quality improves, make pylint be more strict diff --git a/packaging/ubuntu/lbry.desktop b/packaging/ubuntu/lbry.desktop index 53408274c..7adaf1da7 100644 --- a/packaging/ubuntu/lbry.desktop +++ b/packaging/ubuntu/lbry.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Version=0.3.6 +Version=0.3.10 Name=LBRY Comment=The world's first user-owned content marketplace Icon=lbry diff --git a/requirements.txt b/requirements.txt index d4423b5db..780abee9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum leveldb==0.193 miniupnpc==1.9 pbkdf2==1.3 -protobuf==3.0.0b2 +protobuf==3.0.0b3 pycrypto==2.6.1 python-bitcoinrpc==0.1 qrcode==5.2.2 diff --git a/scripts/migrate_lbryum_to_lbrycrd.py b/scripts/migrate_lbryum_to_lbrycrd.py new file mode 100644 index 000000000..fdafacd6e --- /dev/null +++ b/scripts/migrate_lbryum_to_lbrycrd.py @@ -0,0 +1,110 @@ +import argparse +import hashlib +import json +import subprocess +import sys + +import base58 + +from lbryum import SimpleConfig, Network +from lbryum.wallet import WalletStorage, Wallet +from lbryum.commands import known_commands, Commands +from lbryum import lbrycrd + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--wallet', help='path to lbryum wallet') + args = parser.parse_args() + + ensureCliIsOnPathAndServerIsRunning() + + wallet = getWallet(args.wallet) + addresses = wallet.addresses(True) + for addr in addresses[:-1]: + printBalance(wallet, addr) + saveAddr(wallet, addr) + # on the last one, rescan. Don't rescan early for sake of efficiency + addr = addresses[-1] + printBalance(wallet, addr) + saveAddr(wallet, addr, "true") + + +def ensureCliIsOnPathAndServerIsRunning(): + try: + output = subprocess.check_output(['lbrycrd-cli', 'getinfo']) + except OSError: + print 'Failed to run: lbrycrd-cli needs to be on the PATH' + sys.exit(1) + except subprocess.CalledProcessError: + print 'Failed to run: could not connect to the lbrycrd server.' + print 'Make sure it is running and able to be connected to.' + print 'One way to do this is to run:' + print ' lbrycrdd -server -printtoconsole' + sys.exit(1) + + +def validateAddress(addr): + raw_output = subprocess.check_output( + ['lbrycrd-cli', 'validateaddress', addr]) + output = json.loads(raw_output) + if not output['isvalid']: + raise Exception('Address {} is not valid'.format(addr)) + if not output['ismine']: + raise Exception('Address {} is not yours'.format(addr)) + + +def printBalance(wallet, addr): + balance = getBalance(wallet, addr) + print 'Importing private key for %s with balance %s' % (addr, balance) + + +def getBalance(wallet, addr): + return sum(wallet.get_addr_balance(addr)) + + +def getWallet(path=None): + if not path: + config = SimpleConfig() + path = config.get_wallet_path() + storage = WalletStorage(path) + if not storage.file_exists: + print "Failed to run: No wallet to migrate" + sys.exit(1) + return Wallet(storage) + + +def saveAddr(wallet, addr, rescan="false"): + keys = wallet.get_private_key(addr, None) + assert len(keys) == 1, 'Address {} has {} keys. Expected 1'.format(addr, len(keys)) + key = keys[0] + # copied from lbrycrd.regenerate_key + b = lbrycrd.ASecretToSecret(key) + pkey = b[0:32] + is_compressed = lbrycrd.is_compressed(key) + wif = pkeyToWif(pkey, is_compressed) + subprocess.check_call( + ['lbrycrd-cli', 'importprivkey', wif, "", rescan]) + validateAddress(addr) + + +def pkeyToWif(pkey, compressed): + # Follow https://en.bitcoin.it/wiki/Wallet_import_format + # to convert from a private key to the wallet import format + prefix = '\x1c' + wif = prefix + pkey + if compressed: + wif += '\x01' + intermediate_checksum = hashlib.sha256(wif).digest() + checksum = hashlib.sha256(intermediate_checksum).digest() + wif = wif + checksum[:4] + return base58.b58encode(wif) + + +def wifToPkey(wif): + pkey = base58.b58decode(wif) + return pkey[1:-4] + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/setup.py b/setup.py index 01346fb88..60f800df2 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,7 @@ from setuptools import setup, find_packages base_dir = os.path.abspath(os.path.dirname(__file__)) -console_scripts = ['lbrynet-console = lbrynet.lbrynet_console.LBRYConsole:launch_lbry_console', - 'lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUploader:launch_stdin_uploader', +console_scripts = ['lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUploader:launch_stdin_uploader', 'lbrynet-stdout-downloader = lbrynet.lbrynet_console.LBRYStdoutDownloader:launch_stdout_downloader', 'lbrynet-create-network = lbrynet.create_network:main', 'lbrynet-launch-node = lbrynet.dht.node:main', diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/__init__.py b/tests/lbrynet/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/core/__init__.py b/tests/lbrynet/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/core/server/__init__.py b/tests/lbrynet/core/server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/core/server/test_BlobRequestHandler.py b/tests/lbrynet/core/server/test_BlobRequestHandler.py new file mode 100644 index 000000000..5c55af574 --- /dev/null +++ b/tests/lbrynet/core/server/test_BlobRequestHandler.py @@ -0,0 +1,127 @@ +import StringIO + +import mock +from twisted.internet import defer, protocol +from twisted.test import proto_helpers +from twisted.trial import unittest + +from lbrynet.core import Peer +from lbrynet.core.server import BlobRequestHandler + + +class TestBlobRequestHandlerQueries(unittest.TestCase): + def setUp(self): + self.blob_manager = mock.Mock() + self.payment_rate_manager = mock.Mock() + self.handler = BlobRequestHandler.BlobRequestHandler( + self.blob_manager, None, self.payment_rate_manager) + + def test_empty_response_when_empty_query(self): + self.assertEqual( + {}, self.successResultOf(self.handler.handle_queries({}))) + + def test_error_set_when_rate_is_missing(self): + query = {'requested_blob': 'blob'} + deferred = self.handler.handle_queries(query) + response = {'incoming_blob': {'error': 'RATE_UNSET'}} + self.assertEqual(response, self.successResultOf(deferred)) + + def test_error_set_when_rate_too_low(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = False + query = { + 'blob_data_payment_rate': 'way_too_low', + 'requested_blob': 'blob' + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_TOO_LOW', + 'incoming_blob': {'error': 'RATE_UNSET'} + } + self.assertEqual(response, self.successResultOf(deferred)) + + def test_response_when_rate_too_low(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = False + query = { + 'blob_data_payment_rate': 'way_too_low', + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_TOO_LOW', + } + self.assertEqual(response, self.successResultOf(deferred)) + + def test_blob_unavailable_when_blob_not_validated(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = True + blob = mock.Mock() + blob.is_validated.return_value = False + self.blob_manager.get_blob.return_value = defer.succeed(blob) + query = { + 'blob_data_payment_rate': 'rate', + 'requested_blob': 'blob' + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_ACCEPTED', + 'incoming_blob': {'error': 'BLOB_UNAVAILABLE'} + } + self.assertEqual(response, self.successResultOf(deferred)) + + def test_blob_unavailable_when_blob_cannot_be_opened(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = True + blob = mock.Mock() + blob.is_validated.return_value = True + blob.open_for_reading.return_value = None + self.blob_manager.get_blob.return_value = defer.succeed(blob) + query = { + 'blob_data_payment_rate': 'rate', + 'requested_blob': 'blob' + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_ACCEPTED', + 'incoming_blob': {'error': 'BLOB_UNAVAILABLE'} + } + self.assertEqual(response, self.successResultOf(deferred)) + + def test_blob_details_are_set_when_all_conditions_are_met(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = True + blob = mock.Mock() + blob.is_validated.return_value = True + blob.open_for_reading.return_value = True + blob.blob_hash = 'DEADBEEF' + blob.length = 42 + self.blob_manager.get_blob.return_value = defer.succeed(blob) + query = { + 'blob_data_payment_rate': 'rate', + 'requested_blob': 'blob' + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_ACCEPTED', + 'incoming_blob': { + 'blob_hash': 'DEADBEEF', + 'length': 42 + } + } + self.assertEqual(response, self.successResultOf(deferred)) + + +class TestBlobRequestHandlerSender(unittest.TestCase): + def test_nothing_happens_if_not_currently_uploading(self): + handler = BlobRequestHandler.BlobRequestHandler(None, None, None) + handler.currently_uploading = None + deferred = handler.send_blob_if_requested(None) + self.assertEqual(True, self.successResultOf(deferred)) + + def test_file_is_sent_to_consumer(self): + # TODO: also check that the expected payment values are set + consumer = proto_helpers.StringTransport() + test_file = StringIO.StringIO('test') + handler = BlobRequestHandler.BlobRequestHandler(None, None, None) + handler.peer = mock.create_autospec(Peer.Peer) + handler.currently_uploading = mock.Mock() + handler.read_handle = test_file + handler.send_blob_if_requested(consumer) + while consumer.producer: + consumer.producer.resumeProducing() + self.assertEqual(consumer.value(), 'test')