From 130f9cfc4d8dd239e6c5abce0359497702604ae0 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 20 Sep 2016 16:58:30 -0400 Subject: [PATCH 01/14] api sessions -user starts a httpauthsession with an api key and name -user initializes jsonrpc hmac secret to sha256 of session id -server sends new random hmac secret after each api call -a user without an authenticated session will get a authorization error --- lbrynet/conf.py | 5 + lbrynet/core/Error.py | 6 + lbrynet/lbrynet_daemon/LBRYDaemon.py | 164 ++++++----------- lbrynet/lbrynet_daemon/LBRYDaemonCLI.py | 9 +- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 17 +- lbrynet/lbrynet_daemon/auth/__init__.py | 0 lbrynet/lbrynet_daemon/auth/auth.py | 50 +++++ lbrynet/lbrynet_daemon/auth/client.py | 168 +++++++++++++++++ lbrynet/lbrynet_daemon/auth/server.py | 193 ++++++++++++++++++++ lbrynet/lbrynet_daemon/auth/util.py | 82 +++++++++ 10 files changed, 577 insertions(+), 117 deletions(-) create mode 100644 lbrynet/lbrynet_daemon/auth/__init__.py create mode 100644 lbrynet/lbrynet_daemon/auth/auth.py create mode 100644 lbrynet/lbrynet_daemon/auth/client.py create mode 100644 lbrynet/lbrynet_daemon/auth/server.py create mode 100644 lbrynet/lbrynet_daemon/auth/util.py diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 2005d7784..05f735352 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -71,6 +71,11 @@ CURRENCIES = { 'USD': {'type': 'fiat'}, } +ALLOWED_DURING_STARTUP = ['is_running', 'is_first_run', + 'get_time_behind_blockchain', 'stop', + 'daemon_status', 'get_start_notice', + 'version', 'get_search_servers'] + LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv' ANALYTICS_ENDPOINT = 'https://api.segment.io/v1' diff --git a/lbrynet/core/Error.py b/lbrynet/core/Error.py index 8146dc169..363dddb3d 100644 --- a/lbrynet/core/Error.py +++ b/lbrynet/core/Error.py @@ -87,4 +87,10 @@ class NoSuchStreamHashError(Exception): class InvalidBlobHashError(Exception): + pass + +class InvalidHeaderError(Exception): + pass + +class InvalidAuthenticationToken(Exception): pass \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index b258d914d..f72cef38a 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -40,6 +40,7 @@ from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient +from lbrynet.lbrynet_daemon.auth.server import LBRYJSONRPCServer, auth_required, authorizer from lbrynet.metadata.LBRYMetadata import Metadata, verify_name_characters from lbrynet.core import log_support from lbrynet.core import utils @@ -48,7 +49,7 @@ from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, \ KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, DEFAULT_WALLET, \ DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, \ - LOG_POST_URL, LOG_FILE_NAME, REFLECTOR_SERVERS, SEARCH_SERVERS + LOG_POST_URL, LOG_FILE_NAME, REFLECTOR_SERVERS, SEARCH_SERVERS, ALLOWED_DURING_STARTUP from lbrynet.conf import DEFAULT_SD_DOWNLOAD_TIMEOUT from lbrynet.conf import DEFAULT_TIMEOUT from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader @@ -117,11 +118,6 @@ CONNECTION_PROBLEM_CODES = [ (CONNECT_CODE_WALLET, "Synchronization with the blockchain is lagging... if this continues try restarting LBRY") ] -ALLOWED_DURING_STARTUP = ['is_running', 'is_first_run', - 'get_time_behind_blockchain', 'stop', - 'daemon_status', 'get_start_notice', - 'version', 'get_search_servers'] - BAD_REQUEST = 400 NOT_FOUND = 404 OK_CODE = 200 @@ -138,15 +134,14 @@ class Parameters(object): self.__dict__.update(kwargs) -class LBRYDaemon(jsonrpc.JSONRPC): +@authorizer +class LBRYDaemon(LBRYJSONRPCServer): """ LBRYnet daemon, a jsonrpc interface to lbry functions """ - isLeaf = True - def __init__(self, root, wallet_type=None): - jsonrpc.JSONRPC.__init__(self) + LBRYJSONRPCServer.__init__(self) reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown) self.startup_status = STARTUP_STAGES[0] @@ -397,99 +392,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): f.write("rpcpassword=" + password) log.info("Done writing lbrycrd.conf") - def _responseFailed(self, err, call): - log.debug(err.getTraceback()) - - def render(self, request): - origin = request.getHeader("Origin") - referer = request.getHeader("Referer") - - if origin not in [None, 'http://localhost:5279']: - log.warning("Attempted api call from %s", origin) - return server.failure - - if referer is not None and not referer.startswith('http://localhost:5279/'): - log.warning("Attempted api call from %s", referer) - return server.failure - - request.content.seek(0, 0) - # Unmarshal the JSON-RPC data. - content = request.content.read() - parsed = jsonrpclib.loads(content) - functionPath = parsed.get("method") - args = parsed.get('params') - - #TODO convert args to correct types if possible - - id = parsed.get('id') - version = parsed.get('jsonrpc') - if version: - version = int(float(version)) - elif id and not version: - version = jsonrpclib.VERSION_1 - else: - version = jsonrpclib.VERSION_PRE1 - # XXX this all needs to be re-worked to support logic for multiple - # versions... - - if not self.announced_startup: - if functionPath not in ALLOWED_DURING_STARTUP: - return server.failure - - if self.wallet_type == "lbryum" and functionPath in ['set_miner', 'get_miner_status']: - return server.failure - - try: - function = self._getFunction(functionPath) - except jsonrpclib.Fault, f: - self._cbRender(f, request, id, version) - else: - request.setHeader("Access-Control-Allow-Origin", "localhost") - request.setHeader("content-type", "text/json") - if args == [{}]: - d = defer.maybeDeferred(function) - else: - d = defer.maybeDeferred(function, *args) - - # cancel the response if the connection is broken - notify_finish = request.notifyFinish() - notify_finish.addErrback(self._responseFailed, d) - d.addErrback(self._ebRender, id) - d.addCallback(self._cbRender, request, id, version) - d.addErrback(notify_finish.errback) - return server.NOT_DONE_YET - - def _cbRender(self, result, request, id, version): - def default_decimal(obj): - if isinstance(obj, Decimal): - return float(obj) - - if isinstance(result, Handler): - result = result.result - - if isinstance(result, dict): - result = result['result'] - - if version == jsonrpclib.VERSION_PRE1: - if not isinstance(result, jsonrpclib.Fault): - result = (result,) - # Convert the result (python) to JSON-RPC - try: - s = jsonrpclib.dumps(result, version=version, default=default_decimal) - except: - f = jsonrpclib.Fault(self.FAILURE, "can't serialize output") - s = jsonrpclib.dumps(f, version=version) - - request.setHeader("content-length", str(len(s))) - request.write(s) - request.finish() - - def _ebRender(self, failure, id): - if isinstance(failure.value, jsonrpclib.Fault): - return failure.value - log.error(failure) - return jsonrpclib.Fault(self.FAILURE, "error") - def setup(self, branch=DEFAULT_UI_BRANCH, user_specified=False, branch_specified=False, host_ui=True): def _log_starting_vals(): log.info("Starting balance: " + str(self.session.wallet.wallet_balance)) @@ -1435,9 +1337,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _search(self, search): return self.lighthouse_client.search(search) - def _render_response(self, result, code): - return defer.succeed({'result': result, 'code': code}) - + @auth_required def jsonrpc_is_running(self): """ Check if lbrynet daemon is running @@ -1454,6 +1354,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): else: return self._render_response(False, OK_CODE) + @auth_required def jsonrpc_daemon_status(self): """ Get lbrynet daemon status information @@ -1488,6 +1389,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): log.info("daemon status: " + str(r)) return self._render_response(r, OK_CODE) + @auth_required def jsonrpc_is_first_run(self): """ Check if this is the first time lbrynet daemon has been run @@ -1508,6 +1410,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d + @auth_required def jsonrpc_get_start_notice(self): """ Get special message to be displayed at startup @@ -1527,6 +1430,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): else: self._render_response(self.startup_message, OK_CODE) + @auth_required def jsonrpc_version(self): """ Get lbry version information @@ -1561,6 +1465,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): log.info("Get version info: " + json.dumps(msg)) return self._render_response(msg, OK_CODE) + @auth_required def jsonrpc_get_settings(self): """ Get lbrynet daemon settings @@ -1589,6 +1494,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): log.info("Get daemon settings") return self._render_response(self.session_settings, OK_CODE) + @auth_required def jsonrpc_set_settings(self, p): """ Set lbrynet daemon settings @@ -1616,6 +1522,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d + @auth_required def jsonrpc_help(self, p=None): """ Function to retrieve docstring for API function @@ -1640,6 +1547,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): else: return self._render_response(self.jsonrpc_help.__doc__, OK_CODE) + @auth_required def jsonrpc_get_balance(self): """ Get balance @@ -1653,6 +1561,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): log.info("Get balance") return self._render_response(float(self.session.wallet.wallet_balance), OK_CODE) + @auth_required def jsonrpc_stop(self): """ Stop lbrynet-daemon @@ -1672,6 +1581,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return self._render_response("Shutting down", OK_CODE) + @auth_required def jsonrpc_get_lbry_files(self): """ Get LBRY files @@ -1698,6 +1608,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d + @auth_required def jsonrpc_get_lbry_file(self, p): """ Get lbry file @@ -1727,6 +1638,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_resolve_name(self, p): """ Resolve stream info from a LBRY uri @@ -1748,6 +1660,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) return d + @auth_required def jsonrpc_get_claim_info(self, p): """ Resolve claim info from a LBRY uri @@ -1772,6 +1685,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def _process_get_parameters(self, p): """Extract info from input parameters and fill in default values for `get` call.""" # TODO: this process can be abstracted s.t. each method @@ -1793,6 +1707,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): name=name ) + @auth_required def jsonrpc_get(self, p): """Download stream from a LBRY uri. @@ -1823,6 +1738,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda message: self._render_response(message, OK_CODE)) return d + @auth_required def jsonrpc_stop_lbry_file(self, p): """ Stop lbry file @@ -1848,6 +1764,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_start_lbry_file(self, p): """ Stop lbry file @@ -1872,6 +1789,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_get_est_cost(self, p): """ Get estimated cost for a lbry uri @@ -1893,6 +1811,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_search_nametrie(self, p): """ Search the nametrie for claims @@ -1929,6 +1848,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d + @auth_required def jsonrpc_delete_lbry_file(self, p): """ Delete a lbry file @@ -1958,6 +1878,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_publish(self, p): """ Make a new name claim and publish associated data to lbrynet @@ -2034,6 +1955,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d + @auth_required def jsonrpc_abandon_claim(self, p): """ Abandon a name and reclaim credits from the claim @@ -2060,6 +1982,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d + @auth_required def jsonrpc_abandon_name(self, p): """ DEPRECIATED, use abandon_claim @@ -2072,7 +1995,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return self.jsonrpc_abandon_claim(p) - + @auth_required def jsonrpc_support_claim(self, p): """ Support a name claim @@ -2092,6 +2015,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_get_name_claims(self): """ Get my name claims @@ -2115,6 +2039,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d + @auth_required def jsonrpc_get_claims_for_name(self, p): """ Get claims for a name @@ -2130,6 +2055,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_get_transaction_history(self): """ Get transaction history @@ -2144,6 +2070,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_get_transaction(self, p): """ Get a decoded transaction from a txid @@ -2160,6 +2087,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_address_is_mine(self, p): """ Checks if an address is associated with the current wallet. @@ -2177,7 +2105,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d - + @auth_required def jsonrpc_get_public_key_from_wallet(self, p): """ Get public key from wallet address @@ -2192,6 +2120,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d = self.session.wallet.get_pub_keys(wallet) d.addCallback(lambda r: self._render_response(r, OK_CODE)) + @auth_required def jsonrpc_get_time_behind_blockchain(self): """ Get number of blocks behind the blockchain @@ -2215,6 +2144,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d + @auth_required def jsonrpc_get_new_address(self): """ Generate a new wallet address @@ -2234,6 +2164,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda address: self._render_response(address, OK_CODE)) return d + @auth_required def jsonrpc_send_amount_to_address(self, p): """ Send credits to an address @@ -2258,6 +2189,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d + @auth_required def jsonrpc_get_best_blockhash(self): """ Get hash of most recent block @@ -2272,6 +2204,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_get_block(self, p): """ Get contents of a block @@ -2294,6 +2227,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_get_claims_for_tx(self, p): """ Get claims for tx @@ -2313,6 +2247,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_download_descriptor(self, p): """ Download and return a sd blob @@ -2329,6 +2264,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallbacks(lambda r: self._render_response(r, OK_CODE), lambda _: self._render_response(False, OK_CODE)) return d + @auth_required def jsonrpc_get_nametrie(self): """ Get the nametrie @@ -2344,6 +2280,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_set_miner(self, p): """ Start of stop the miner, function only available when lbrycrd is set as the wallet @@ -2363,6 +2300,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_get_miner_status(self): """ Get status of miner @@ -2377,6 +2315,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_log(self, p): """ Log message @@ -2391,6 +2330,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): log.info("API client log request: %s" % message) return self._render_response(True, OK_CODE) + @auth_required def jsonrpc_upload_log(self, p=None): """ Upload log @@ -2432,6 +2372,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d + @auth_required def jsonrpc_configure_ui(self, p): """ Configure the UI being hosted @@ -2456,6 +2397,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d + @auth_required def jsonrpc_reveal(self, p): """ Reveal a file or directory in file browser @@ -2475,6 +2417,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d + @auth_required def jsonrpc_get_peers_for_hash(self, p): """ Get peers for blob hash @@ -2492,6 +2435,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_announce_all_blobs_to_dht(self): """ Announce all blobs to the dht @@ -2506,6 +2450,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: self._render_response("Announced", OK_CODE)) return d + @auth_required def jsonrpc_reflect(self, p): """ Reflect a stream @@ -2522,6 +2467,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallbacks(lambda _: self._render_response(True, OK_CODE), lambda err: self._render_response(err.getTraceback(), OK_CODE)) return d + @auth_required def jsonrpc_get_blob_hashes(self): """ Returns all blob hashes @@ -2536,6 +2482,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_reflect_all_blobs(self): """ Reflects all saved blobs @@ -2551,6 +2498,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + @auth_required def jsonrpc_get_search_servers(self): """ Get list of lighthouse servers diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py index ea4f2234d..b63e1af25 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py @@ -2,8 +2,7 @@ import sys import json import argparse -from lbrynet.conf import API_CONNECTION_STRING -from jsonrpc.proxy import JSONRPCProxy +from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient help_msg = "Usage: lbrynet-cli method json-args\n" \ + "Examples: " \ @@ -36,7 +35,7 @@ def get_params_from_kwargs(params): def main(): - api = JSONRPCProxy.from_url(API_CONNECTION_STRING) + api = LBRYAPIClient() try: s = api.is_running() @@ -72,9 +71,9 @@ def main(): if meth in api.help(): try: if params: - r = api.call(meth, params) + r = LBRYAPIClient(service=meth)(params) else: - r = api.call(meth) + r = LBRYAPIClient(service=meth)() print json.dumps(r, sort_keys=True) except: print "Something went wrong, here's the usage for %s:" % meth diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 10b5b5d0b..e6464a8e5 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -7,12 +7,15 @@ import sys import socket from appdirs import user_data_dir -from twisted.web import server -from twisted.internet import reactor, defer +from twisted.web import server, guard +from twisted.internet import defer, reactor +from twisted.cred import portal + from jsonrpc.proxy import JSONRPCProxy from lbrynet.core import log_support from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest +from lbrynet.lbrynet_daemon.auth.auth import PasswordChecker, HttpPasswordRealm from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_PORT, \ UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME @@ -113,8 +116,14 @@ def start(): if args.launchui: d.addCallback(lambda _: webbrowser.open(UI_ADDRESS)) - lbrynet_server = server.Site(lbry.root) + checker = PasswordChecker() + realm = HttpPasswordRealm(lbry.root) + p = portal.Portal(realm, [checker, ]) + factory = guard.BasicCredentialFactory('Login to lbrynet api') + protected_resource = guard.HTTPAuthSessionWrapper(p, [factory, ]) + lbrynet_server = server.Site(protected_resource) lbrynet_server.requestFactory = LBRYDaemonRequest + reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE) reactor.run() @@ -127,4 +136,4 @@ def start(): return if __name__ == "__main__": - start() + start() \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/auth/__init__.py b/lbrynet/lbrynet_daemon/auth/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lbrynet/lbrynet_daemon/auth/auth.py b/lbrynet/lbrynet_daemon/auth/auth.py new file mode 100644 index 000000000..57762e2b3 --- /dev/null +++ b/lbrynet/lbrynet_daemon/auth/auth.py @@ -0,0 +1,50 @@ +import logging +import os +from zope.interface import implements, implementer +from twisted.cred import portal, checkers, credentials, error as cred_error +from twisted.internet import defer +from twisted.web import resource +from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, save_api_keys +from lbrynet.lbrynet_daemon.LBRYDaemon import log_dir as DATA_DIR + +log = logging.getLogger(__name__) + + +# initialize api key if none exist +if not os.path.isfile(os.path.join(DATA_DIR, ".api_keys")): + keys = {} + api_key = APIKey.new() + api_key.rename(API_KEY_NAME) + keys.update(api_key) + save_api_keys(keys, os.path.join(DATA_DIR, ".api_keys")) + + +@implementer(portal.IRealm) +class HttpPasswordRealm: + def __init__(self, resource): + self.resource = resource + + def requestAvatar(self, avatarId, mind, *interfaces): + log.info("Processing request for %s", avatarId) + if resource.IResource in interfaces: + return (resource.IResource, self.resource, lambda: None) + raise NotImplementedError() + + +class PasswordChecker: + implements(checkers.ICredentialsChecker) + credentialInterfaces = (credentials.IUsernamePassword,) + + def __init__(self): + keys = load_api_keys(os.path.join(DATA_DIR, ".api_keys")) + self.passwords = {key: keys[key]['token'] for key in keys} + + def requestAvatarId(self, creds): + if creds.username in self.passwords: + pw = self.passwords.get(creds.username) + pw_match = creds.checkPassword(pw) + if pw_match is True: + return defer.succeed(creds.username) + log.warning('Incorrect username or password') + return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password')) + diff --git a/lbrynet/lbrynet_daemon/auth/client.py b/lbrynet/lbrynet_daemon/auth/client.py new file mode 100644 index 000000000..a4370e3b5 --- /dev/null +++ b/lbrynet/lbrynet_daemon/auth/client.py @@ -0,0 +1,168 @@ +try: + import http.client as httplib +except ImportError: + import httplib +try: + import urllib.parse as urlparse +except ImportError: + import urlparse + +import logging +import requests +import os +import base64 +import json + +from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME +from lbrynet.conf import API_INTERFACE, API_ADDRESS, API_PORT +from lbrynet.lbrynet_daemon.LBRYDaemon import log_dir as DATA_DIR + +log = logging.getLogger(__name__) +USER_AGENT = "AuthServiceProxy/0.1" +HTTP_TIMEOUT = 30 + + +class JSONRPCException(Exception): + def __init__(self, rpc_error): + Exception.__init__(self) + self.error = rpc_error + + +class LBRYAPIClient(object): + __api_token = None + + def __init__(self, key_name=None, key=None, pw_path=None, timeout=HTTP_TIMEOUT, connection=None, count=0, + service=None, cookies=None, auth=None, url=None, login_url=None): + self.__api_key_name = API_KEY_NAME if not key_name else key_name + self.__api_token = key + self.__pw_path = os.path.join(DATA_DIR, ".api_keys") if not pw_path else pw_path + self.__service_name = service + + if not key: + keys = load_api_keys(self.__pw_path) + api_key = keys.get(self.__api_key_name, False) + self.__api_token = api_key['token'] + self.__api_key_obj = api_key + else: + self.__api_key_obj = APIKey({'token': key}) + + if login_url is None: + self.__service_url = "http://%s:%s@%s:%i/%s" % (self.__api_key_name, self.__api_token, + API_INTERFACE, API_PORT, API_ADDRESS) + else: + self.__service_url = login_url + + self.__id_count = count + + if auth is None and connection is None and cookies is None and url is None: + self.__url = urlparse.urlparse(self.__service_url) + (user, passwd) = (self.__url.username, self.__url.password) + try: + user = user.encode('utf8') + except AttributeError: + pass + try: + passwd = passwd.encode('utf8') + except AttributeError: + pass + authpair = user + b':' + passwd + self.__auth_header = b'Basic ' + base64.b64encode(authpair) + + self.__conn = requests.Session() + self.__conn.auth = (user, passwd) + + req = requests.Request(method='POST', + url=self.__service_url, + auth=self.__conn.auth, + headers={'Host': self.__url.hostname, + 'User-Agent': USER_AGENT, + 'Authorization': self.__auth_header, + 'Content-type': 'application/json'},) + r = req.prepare() + http_response = self.__conn.send(r) + cookies = http_response.cookies + self.__cookies = cookies + # print "Logged in" + + uid = cookies.get('TWISTED_SESSION') + api_key = APIKey.new(seed=uid) + # print "Created temporary api key" + + self.__api_token = api_key.token() + self.__api_key_obj = api_key + else: + self.__auth_header = auth + self.__conn = connection + self.__cookies = cookies + self.__url = url + + if cookies.get("secret", False): + self.__api_token = cookies.get("secret") + self.__api_key_obj = APIKey({'name': self.__api_key_name, 'token': self.__api_token}) + + + def __getattr__(self, name): + if name.startswith('__') and name.endswith('__'): + # Python internal stuff + raise AttributeError + if self.__service_name is not None: + name = "%s.%s" % (self.__service_name, name) + return LBRYAPIClient(key_name=self.__api_key_name, + key=self.__api_token, + connection=self.__conn, + service=name, + count=self.__id_count, + cookies=self.__cookies, + auth=self.__auth_header, + url=self.__url, + login_url=self.__service_url) + + def __call__(self, *args): + self.__id_count += 1 + pre_auth_postdata = {'version': '1.1', + 'method': self.__service_name, + 'params': args, + 'id': self.__id_count} + to_auth = str(pre_auth_postdata['method']).encode('hex') + str(pre_auth_postdata['id']).encode('hex') + token = self.__api_key_obj.get_hmac(to_auth.decode('hex')) + pre_auth_postdata.update({'hmac': token}) + postdata = json.dumps(pre_auth_postdata) + service_url = self.__service_url + auth_header = self.__auth_header + cookies = self.__cookies + host = self.__url.hostname + + req = requests.Request(method='POST', + url=service_url, + data=postdata, + headers={'Host': host, + 'User-Agent': USER_AGENT, + 'Authorization': auth_header, + 'Content-type': 'application/json'}, + cookies=cookies) + r = req.prepare() + http_response = self.__conn.send(r) + self.__cookies = http_response.cookies + headers = http_response.headers + next_secret = headers.get('Next-Secret', False) + + if next_secret: + cookies.update({'secret': next_secret}) + + # print "Postdata: %s" % postdata + if http_response is None: + raise JSONRPCException({ + 'code': -342, 'message': 'missing HTTP response from server'}) + + # print "-----\n%s\n------" % http_response.text + http_response.raise_for_status() + + response = http_response.json() + + if response['error'] is not None: + raise JSONRPCException(response['error']) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC result'}) + else: + return response['result'] \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py new file mode 100644 index 000000000..b2ad51402 --- /dev/null +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -0,0 +1,193 @@ +import logging + +from decimal import Decimal +from twisted.web import server +from twisted.internet import defer +from txjsonrpc import jsonrpclib +from txjsonrpc.web import jsonrpc +from txjsonrpc.web.jsonrpc import Handler + +from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError +from lbrynet.lbrynet_daemon.auth.util import APIKey +from lbrynet.conf import ALLOWED_DURING_STARTUP + +log = logging.getLogger(__name__) + + +def default_decimal(obj): + if isinstance(obj, Decimal): + return float(obj) + + +def authorizer(cls): + cls.authorized_functions = [] + for methodname in dir(cls): + if methodname.startswith("jsonrpc_"): + method = getattr(cls, methodname) + if hasattr(method, '_auth_required'): + cls.authorized_functions.append(methodname.split("jsonrpc_")[1]) + return cls + + +def auth_required(f): + f._auth_required = True + return f + + +@authorizer +class LBRYJSONRPCServer(jsonrpc.JSONRPC): + + isLeaf = True + + def __init__(self): + jsonrpc.JSONRPC.__init__(self) + self.sessions = {} + + def _register_user_session(self, session_id): + token = APIKey.new() + self.sessions.update({session_id: token}) + return token + + def _responseFailed(self, err, call): + log.debug(err.getTraceback()) + + def _set_headers(self, request, data): + request.setHeader("Access-Control-Allow-Origin", "localhost") + request.setHeader("Content-Type", "text/json") + request.setHeader("Content-Length", str(len(data))) + + def _render_message(self, request, message): + request.write(message) + request.finish() + + def _check_headers(self, request): + origin = request.getHeader("Origin") + referer = request.getHeader("Referer") + + if origin not in [None, 'http://localhost:5279']: + log.warning("Attempted api call from %s", origin) + raise InvalidHeaderError + + if referer is not None and not referer.startswith('http://localhost:5279/'): + log.warning("Attempted api call from %s", referer) + raise InvalidHeaderError + + def _handle(self, request): + def _check_function_path(function_path): + if not self.announced_startup: + if function_path not in ALLOWED_DURING_STARTUP: + log.warning("Cannot call %s during startup", function_path) + raise Exception("Function not allowed") + + def _get_function(function_path): + function = self._getFunction(function_path) + return function + + def _verify_token(session_id, message, token): + request.setHeader("Next-Secret", "") + api_key = self.sessions.get(session_id, None) + assert api_key is not None, InvalidAuthenticationToken + r = api_key.compare_hmac(message, token) + assert r, InvalidAuthenticationToken + # log.info("Generating new token for next request") + self.sessions.update({session_id: APIKey.new()}) + request.setHeader("Next-Secret", self.sessions.get(session_id).token()) + + session = request.getSession() + session_id = session.uid + session_store = self.sessions.get(session_id, False) + + if not session_store: + token = APIKey.new(seed=session_id) + log.info("Initializing new api session") + self.sessions.update({session_id: token}) + # log.info("Generated token %s", str(self.sessions[session_id])) + + request.content.seek(0, 0) + content = request.content.read() + + parsed = jsonrpclib.loads(content) + + functionPath = parsed.get("method") + + _check_function_path(functionPath) + require_auth = functionPath in self.authorized_functions + if require_auth: + token = parsed.pop('hmac') + to_auth = functionPath.encode('hex') + str(parsed.get('id')).encode('hex') + _verify_token(session_id, to_auth.decode('hex'), token) + + args = parsed.get('params') + id = parsed.get('id') + version = parsed.get('jsonrpc') + + if version: + version = int(float(version)) + elif id and not version: + version = jsonrpclib.VERSION_1 + else: + version = jsonrpclib.VERSION_PRE1 + + if self.wallet_type == "lbryum" and functionPath in ['set_miner', 'get_miner_status']: + log.warning("Mining commands are not available in lbryum") + raise Exception("Command not available in lbryum") + + try: + function = _get_function(functionPath) + if args == [{}]: + d = defer.maybeDeferred(function) + else: + d = defer.maybeDeferred(function, *args) + except jsonrpclib.Fault as f: + d = self._cbRender(f, request, id, version) + finally: + # cancel the response if the connection is broken + notify_finish = request.notifyFinish() + notify_finish.addErrback(self._responseFailed, d) + d.addErrback(self._ebRender, id) + d.addCallback(self._cbRender, request, id, version) + d.addErrback(notify_finish.errback) + + def _cbRender(self, result, request, id, version): + if isinstance(result, Handler): + result = result.result + + if isinstance(result, dict): + result = result['result'] + + if version == jsonrpclib.VERSION_PRE1: + if not isinstance(result, jsonrpclib.Fault): + result = (result,) + # Convert the result (python) to JSON-RPC + try: + s = jsonrpclib.dumps(result, version=version, default=default_decimal) + self._render_message(request, s) + except: + f = jsonrpclib.Fault(self.FAILURE, "can't serialize output") + s = jsonrpclib.dumps(f, version=version) + self._set_headers(request, s) + self._render_message(request, s) + + def _ebRender(self, failure, id): + log.error(failure) + log.error(failure.value) + log.error(id) + if isinstance(failure.value, jsonrpclib.Fault): + return failure.value + return server.failure + + def render(self, request): + try: + self._check_headers(request) + except InvalidHeaderError: + return server.failure + + try: + self._handle(request) + except: + return server.failure + + return server.NOT_DONE_YET + + def _render_response(self, result, code): + return defer.succeed({'result': result, 'code': code}) diff --git a/lbrynet/lbrynet_daemon/auth/util.py b/lbrynet/lbrynet_daemon/auth/util.py new file mode 100644 index 000000000..69daeee56 --- /dev/null +++ b/lbrynet/lbrynet_daemon/auth/util.py @@ -0,0 +1,82 @@ +import base58 +import hmac +import hashlib +import yaml +import os +import logging + +log = logging.getLogger(__name__) + +API_KEY_NAME = "api" + + +def sha(x): + h = hashlib.sha256(x).digest() + return base58.b58encode(h) + + +def generate_key(x=None): + if x is None: + return sha(os.urandom(256)) + else: + return sha(x) + + +class APIKey(dict): + def __init__(self, key, name=None): + self.key = key if isinstance(key, str) else key['token'] + self.name = name if name else hashlib.sha256(self.key).hexdigest() + self.expiration = None if isinstance(key, str) else key.get('expiration', None) + self.update({self.name: {'token': self.key, 'expiration': self.expiration}}) + + @classmethod + def new(cls, expiration=None, seed=None, name=None): + key_val = generate_key(seed) + key = {'token': key_val, 'expiration': expiration} + return APIKey(key, name) + + def token(self): + return self[self.name]['token'] + + def _raw_key(self): + return base58.b58decode(self.token()) + + def get_hmac(self, message): + decoded_key = self._raw_key() + signature = hmac.new(decoded_key, message, hashlib.sha256) + return base58.b58encode(signature.digest()) + + def compare_hmac(self, message, token): + decoded_token = base58.b58decode(token) + target = base58.b58decode(self.get_hmac(message)) + try: + assert len(decoded_token) == len(target), "Length mismatch" + r = hmac.compare_digest(decoded_token, target) + except: + return False + return r + + def rename(self, name): + old = self.keys()[0] + t = self.pop(old) + self.update({name: t}) + + +def load_api_keys(path): + if not os.path.isfile(path): + raise Exception("Invalid api key path") + + f = open(path, "r") + data = yaml.load(f.read()) + f.close() + + keys = {key: APIKey(data[key], name=key)[key] for key in data} + + return keys + + +def save_api_keys(keys, path): + data = yaml.safe_dump(dict(keys)) + f = open(path, "w") + f.write(data) + f.close() From aecafbbebd35444a254f2e0f08897e525c884fa5 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 21 Sep 2016 03:49:52 -0400 Subject: [PATCH 02/14] clean up --- lbrynet/conf.py | 17 ++ lbrynet/lbrynet_daemon/LBRYDaemonCLI.py | 11 +- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 16 +- .../lbrynet_daemon/LBRYExchangeRateManager.py | 4 + lbrynet/lbrynet_daemon/auth/auth.py | 36 ++-- lbrynet/lbrynet_daemon/auth/client.py | 169 +++++++++--------- lbrynet/lbrynet_daemon/auth/server.py | 11 +- lbrynet/lbrynet_daemon/auth/util.py | 61 ++++--- 8 files changed, 173 insertions(+), 152 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 05f735352..26ef18e3b 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -2,7 +2,24 @@ Some network wide and also application specific parameters """ import os +import sys +from appdirs import user_data_dir +LINUX = 1 +DARWIN = 2 +WINDOWS = 3 + +if sys.platform.startswith("linux"): + platform = LINUX +elif sys.platform.startswith("darwin"): + platform = DARWIN +elif sys.platform.startswith("win"): + platform = WINDOWS + +if platform is LINUX: + DATA_DIR = os.path.join(os.path.expanduser("~"), ".lbrynet") +else: + DATA_DIR = user_data_dir("LBRY") IS_DEVELOPMENT_VERSION = False diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py index b63e1af25..07253cc2a 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py @@ -35,10 +35,11 @@ def get_params_from_kwargs(params): def main(): - api = LBRYAPIClient() + api = LBRYAPIClient.config() try: - s = api.is_running() + status = api.daemon_status() + assert status.get('code', False) == "started" except: print "lbrynet-daemon isn't running" sys.exit(1) @@ -71,10 +72,10 @@ def main(): if meth in api.help(): try: if params: - r = LBRYAPIClient(service=meth)(params) + result = LBRYAPIClient.config(service=meth)(params) else: - r = LBRYAPIClient(service=meth)() - print json.dumps(r, sort_keys=True) + result = LBRYAPIClient.config(service=meth)() + print json.dumps(result, sort_keys=True) except: print "Something went wrong, here's the usage for %s:" % meth print api.help({'function': meth}) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index e6464a8e5..8b1290903 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -16,14 +16,10 @@ from jsonrpc.proxy import JSONRPCProxy from lbrynet.core import log_support from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest from lbrynet.lbrynet_daemon.auth.auth import PasswordChecker, HttpPasswordRealm +from lbrynet.lbrynet_daemon.auth.util import initialize_api_key_file from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_PORT, \ UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME - -# TODO: stop it! -if sys.platform != "darwin": - log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") -else: - log_dir = user_data_dir("LBRY") +from lbrynet.conf import DATA_DIR as log_dir if not os.path.isdir(log_dir): os.mkdir(log_dir) @@ -116,11 +112,13 @@ def start(): if args.launchui: d.addCallback(lambda _: webbrowser.open(UI_ADDRESS)) - checker = PasswordChecker() + pw_path = os.path.join(log_dir, ".api_keys") + initialize_api_key_file(pw_path) + checker = PasswordChecker.load_file(pw_path) realm = HttpPasswordRealm(lbry.root) - p = portal.Portal(realm, [checker, ]) + portal_to_realm = portal.Portal(realm, [checker, ]) factory = guard.BasicCredentialFactory('Login to lbrynet api') - protected_resource = guard.HTTPAuthSessionWrapper(p, [factory, ]) + protected_resource = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ]) lbrynet_server = server.Site(protected_resource) lbrynet_server.requestFactory = LBRYDaemonRequest diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index e896c8b0f..60008d514 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -50,11 +50,15 @@ class MarketFeed(object): log.debug("Saving price update %f for %s" % (price, self.market)) self.rate = ExchangeRate(self.market, price, int(time.time())) + def _log_error(self): + log.warning("%s failed to update exchange rate information", self.name) + def _update_price(self): d = defer.succeed(self._make_request()) d.addCallback(self._handle_response) d.addCallback(self._subtract_fee) d.addCallback(self._save_price) + d.addErrback(lambda _: self._log_error()) def start(self): if not self._updater.running: diff --git a/lbrynet/lbrynet_daemon/auth/auth.py b/lbrynet/lbrynet_daemon/auth/auth.py index 57762e2b3..f68772fe2 100644 --- a/lbrynet/lbrynet_daemon/auth/auth.py +++ b/lbrynet/lbrynet_daemon/auth/auth.py @@ -1,26 +1,15 @@ import logging -import os -from zope.interface import implements, implementer +from zope.interface import implementer from twisted.cred import portal, checkers, credentials, error as cred_error from twisted.internet import defer from twisted.web import resource from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, save_api_keys -from lbrynet.lbrynet_daemon.LBRYDaemon import log_dir as DATA_DIR log = logging.getLogger(__name__) -# initialize api key if none exist -if not os.path.isfile(os.path.join(DATA_DIR, ".api_keys")): - keys = {} - api_key = APIKey.new() - api_key.rename(API_KEY_NAME) - keys.update(api_key) - save_api_keys(keys, os.path.join(DATA_DIR, ".api_keys")) - - @implementer(portal.IRealm) -class HttpPasswordRealm: +class HttpPasswordRealm(object): def __init__(self, resource): self.resource = resource @@ -31,19 +20,28 @@ class HttpPasswordRealm: raise NotImplementedError() -class PasswordChecker: - implements(checkers.ICredentialsChecker) +@implementer(checkers.ICredentialsChecker) +class PasswordChecker(object): credentialInterfaces = (credentials.IUsernamePassword,) - def __init__(self): - keys = load_api_keys(os.path.join(DATA_DIR, ".api_keys")) - self.passwords = {key: keys[key]['token'] for key in keys} + def __init__(self, passwords): + self.passwords = passwords + + @classmethod + def load_file(cls, key_path): + keys = load_api_keys(key_path) + return cls.load(keys) + + @classmethod + def load(cls, password_dict): + passwords = {key: password_dict[key].secret for key in password_dict} + return cls(passwords) def requestAvatarId(self, creds): if creds.username in self.passwords: pw = self.passwords.get(creds.username) pw_match = creds.checkPassword(pw) - if pw_match is True: + if pw_match: return defer.succeed(creds.username) log.warning('Incorrect username or password') return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password')) diff --git a/lbrynet/lbrynet_daemon/auth/client.py b/lbrynet/lbrynet_daemon/auth/client.py index a4370e3b5..31f62949c 100644 --- a/lbrynet/lbrynet_daemon/auth/client.py +++ b/lbrynet/lbrynet_daemon/auth/client.py @@ -1,12 +1,4 @@ -try: - import http.client as httplib -except ImportError: - import httplib -try: - import urllib.parse as urlparse -except ImportError: - import urlparse - +import urlparse import logging import requests import os @@ -15,10 +7,12 @@ import json from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME from lbrynet.conf import API_INTERFACE, API_ADDRESS, API_PORT -from lbrynet.lbrynet_daemon.LBRYDaemon import log_dir as DATA_DIR +from lbrynet.conf import DATA_DIR log = logging.getLogger(__name__) USER_AGENT = "AuthServiceProxy/0.1" +TWISTED_SESSION = "TWISTED_SESSION" +LBRY_SECRET = "LBRY_SECRET" HTTP_TIMEOUT = 30 @@ -29,77 +23,15 @@ class JSONRPCException(Exception): class LBRYAPIClient(object): - __api_token = None - - def __init__(self, key_name=None, key=None, pw_path=None, timeout=HTTP_TIMEOUT, connection=None, count=0, - service=None, cookies=None, auth=None, url=None, login_url=None): - self.__api_key_name = API_KEY_NAME if not key_name else key_name - self.__api_token = key - self.__pw_path = os.path.join(DATA_DIR, ".api_keys") if not pw_path else pw_path + def __init__(self, key, timeout, connection, count, service, cookies, auth, url, login_url): self.__service_name = service - - if not key: - keys = load_api_keys(self.__pw_path) - api_key = keys.get(self.__api_key_name, False) - self.__api_token = api_key['token'] - self.__api_key_obj = api_key - else: - self.__api_key_obj = APIKey({'token': key}) - - if login_url is None: - self.__service_url = "http://%s:%s@%s:%i/%s" % (self.__api_key_name, self.__api_token, - API_INTERFACE, API_PORT, API_ADDRESS) - else: - self.__service_url = login_url - + self.__api_key = key + self.__service_url = login_url self.__id_count = count - - if auth is None and connection is None and cookies is None and url is None: - self.__url = urlparse.urlparse(self.__service_url) - (user, passwd) = (self.__url.username, self.__url.password) - try: - user = user.encode('utf8') - except AttributeError: - pass - try: - passwd = passwd.encode('utf8') - except AttributeError: - pass - authpair = user + b':' + passwd - self.__auth_header = b'Basic ' + base64.b64encode(authpair) - - self.__conn = requests.Session() - self.__conn.auth = (user, passwd) - - req = requests.Request(method='POST', - url=self.__service_url, - auth=self.__conn.auth, - headers={'Host': self.__url.hostname, - 'User-Agent': USER_AGENT, - 'Authorization': self.__auth_header, - 'Content-type': 'application/json'},) - r = req.prepare() - http_response = self.__conn.send(r) - cookies = http_response.cookies - self.__cookies = cookies - # print "Logged in" - - uid = cookies.get('TWISTED_SESSION') - api_key = APIKey.new(seed=uid) - # print "Created temporary api key" - - self.__api_token = api_key.token() - self.__api_key_obj = api_key - else: - self.__auth_header = auth - self.__conn = connection - self.__cookies = cookies - self.__url = url - - if cookies.get("secret", False): - self.__api_token = cookies.get("secret") - self.__api_key_obj = APIKey({'name': self.__api_key_name, 'token': self.__api_token}) - + self.__url = url + self.__auth_header = auth + self.__conn = connection + self.__cookies = cookies def __getattr__(self, name): if name.startswith('__') and name.endswith('__'): @@ -107,11 +39,11 @@ class LBRYAPIClient(object): raise AttributeError if self.__service_name is not None: name = "%s.%s" % (self.__service_name, name) - return LBRYAPIClient(key_name=self.__api_key_name, - key=self.__api_token, + return LBRYAPIClient(key=self.__api_key, + timeout=HTTP_TIMEOUT, connection=self.__conn, - service=name, count=self.__id_count, + service=name, cookies=self.__cookies, auth=self.__auth_header, url=self.__url, @@ -124,7 +56,7 @@ class LBRYAPIClient(object): 'params': args, 'id': self.__id_count} to_auth = str(pre_auth_postdata['method']).encode('hex') + str(pre_auth_postdata['id']).encode('hex') - token = self.__api_key_obj.get_hmac(to_auth.decode('hex')) + token = self.__api_key.get_hmac(to_auth.decode('hex')) pre_auth_postdata.update({'hmac': token}) postdata = json.dumps(pre_auth_postdata) service_url = self.__service_url @@ -142,12 +74,14 @@ class LBRYAPIClient(object): cookies=cookies) r = req.prepare() http_response = self.__conn.send(r) - self.__cookies = http_response.cookies + cookies = http_response.cookies headers = http_response.headers - next_secret = headers.get('Next-Secret', False) + next_secret = headers.get(LBRY_SECRET, False) if next_secret: - cookies.update({'secret': next_secret}) + # print "Next secret: %s" % next_secret + self.__api_key.secret = next_secret + self.__cookies = cookies # print "Postdata: %s" % postdata if http_response is None: @@ -165,4 +99,65 @@ class LBRYAPIClient(object): raise JSONRPCException({ 'code': -343, 'message': 'missing JSON-RPC result'}) else: - return response['result'] \ No newline at end of file + return response['result'] + + @classmethod + def config(cls, key_name=None, key=None, pw_path=None, timeout=HTTP_TIMEOUT, connection=None, count=0, + service=None, cookies=None, auth=None, url=None, login_url=None): + api_key_name = API_KEY_NAME if not key_name else key_name + pw_path = os.path.join(DATA_DIR, ".api_keys") if not pw_path else pw_path + + if not key: + keys = load_api_keys(pw_path) + api_key = keys.get(api_key_name, False) + else: + api_key = APIKey(name=api_key_name, secret=key) + + if login_url is None: + service_url = "http://%s:%s@%s:%i/%s" % (api_key_name, api_key.secret, API_INTERFACE, API_PORT, API_ADDRESS) + else: + service_url = login_url + + id_count = count + + if auth is None and connection is None and cookies is None and url is None: + # This is a new client instance, initialize the auth header and start a session + url = urlparse.urlparse(service_url) + (user, passwd) = (url.username, url.password) + try: + user = user.encode('utf8') + except AttributeError: + pass + try: + passwd = passwd.encode('utf8') + except AttributeError: + pass + authpair = user + b':' + passwd + auth_header = b'Basic ' + base64.b64encode(authpair) + + conn = requests.Session() + conn.auth = (user, passwd) + + req = requests.Request(method='POST', + url=service_url, + auth=conn.auth, + headers={'Host': url.hostname, + 'User-Agent': USER_AGENT, + 'Authorization': auth_header, + 'Content-type': 'application/json'},) + r = req.prepare() + http_response = conn.send(r) + cookies = http_response.cookies + # print "Logged in" + + uid = cookies.get(TWISTED_SESSION) + api_key = APIKey.new(seed=uid) + # print "Created temporary api key" + else: + # This is a client that already has a session, use it + auth_header = auth + conn = connection + assert cookies.get(LBRY_SECRET, False), "Missing cookie" + secret = cookies.get(LBRY_SECRET) + api_key = APIKey(secret, api_key_name) + return cls(api_key, timeout, conn, id_count, service, cookies, auth_header, url, service_url) \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index b2ad51402..10b6cf65b 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -9,6 +9,7 @@ from txjsonrpc.web.jsonrpc import Handler from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError from lbrynet.lbrynet_daemon.auth.util import APIKey +from lbrynet.lbrynet_daemon.auth.client import LBRY_SECRET from lbrynet.conf import ALLOWED_DURING_STARTUP log = logging.getLogger(__name__) @@ -84,21 +85,21 @@ class LBRYJSONRPCServer(jsonrpc.JSONRPC): return function def _verify_token(session_id, message, token): - request.setHeader("Next-Secret", "") + request.setHeader(LBRY_SECRET, "") api_key = self.sessions.get(session_id, None) assert api_key is not None, InvalidAuthenticationToken r = api_key.compare_hmac(message, token) assert r, InvalidAuthenticationToken # log.info("Generating new token for next request") - self.sessions.update({session_id: APIKey.new()}) - request.setHeader("Next-Secret", self.sessions.get(session_id).token()) + self.sessions.update({session_id: APIKey.new(name=session_id)}) + request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret) session = request.getSession() session_id = session.uid session_store = self.sessions.get(session_id, False) if not session_store: - token = APIKey.new(seed=session_id) + token = APIKey.new(seed=session_id, name=session_id) log.info("Initializing new api session") self.sessions.update({session_id: token}) # log.info("Generated token %s", str(self.sessions[session_id])) @@ -191,3 +192,5 @@ class LBRYJSONRPCServer(jsonrpc.JSONRPC): def _render_response(self, result, code): return defer.succeed({'result': result, 'code': code}) + + diff --git a/lbrynet/lbrynet_daemon/auth/util.py b/lbrynet/lbrynet_daemon/auth/util.py index 69daeee56..1ed625569 100644 --- a/lbrynet/lbrynet_daemon/auth/util.py +++ b/lbrynet/lbrynet_daemon/auth/util.py @@ -22,24 +22,20 @@ def generate_key(x=None): return sha(x) -class APIKey(dict): - def __init__(self, key, name=None): - self.key = key if isinstance(key, str) else key['token'] - self.name = name if name else hashlib.sha256(self.key).hexdigest() - self.expiration = None if isinstance(key, str) else key.get('expiration', None) - self.update({self.name: {'token': self.key, 'expiration': self.expiration}}) +class APIKey(object): + def __init__(self, secret, name, expiration=None): + self.secret = secret + self.name = name + self.expiration = expiration @classmethod - def new(cls, expiration=None, seed=None, name=None): - key_val = generate_key(seed) - key = {'token': key_val, 'expiration': expiration} - return APIKey(key, name) - - def token(self): - return self[self.name]['token'] + def new(cls, seed=None, name=None, expiration=None): + secret = generate_key(seed) + key_name = name if name else sha(secret) + return APIKey(secret, key_name, expiration) def _raw_key(self): - return base58.b58decode(self.token()) + return base58.b58decode(self.secret) def get_hmac(self, message): decoded_key = self._raw_key() @@ -56,27 +52,36 @@ class APIKey(dict): return False return r - def rename(self, name): - old = self.keys()[0] - t = self.pop(old) - self.update({name: t}) - def load_api_keys(path): if not os.path.isfile(path): raise Exception("Invalid api key path") - f = open(path, "r") - data = yaml.load(f.read()) - f.close() + with open(path, "r") as f: + data = yaml.load(f.read()) - keys = {key: APIKey(data[key], name=key)[key] for key in data} + keys_for_return = {} + for key_name in data: + key = data[key_name] + secret = key['secret'] + expiration = key['expiration'] + keys_for_return.update({key_name: APIKey(secret, key_name, expiration)}) - return keys + return keys_for_return def save_api_keys(keys, path): - data = yaml.safe_dump(dict(keys)) - f = open(path, "w") - f.write(data) - f.close() + with open(path, "w") as f: + key_dict = {keys[key_name].name: {'secret': keys[key_name].secret, + 'expiration': keys[key_name].expiration} + for key_name in keys} + data = yaml.safe_dump(key_dict) + f.write(data) + + +def initialize_api_key_file(key_path): + if not os.path.isfile(key_path): + keys = {} + new_api_key = APIKey.new(name=API_KEY_NAME) + keys.update({new_api_key.name: new_api_key}) + save_api_keys(keys, key_path) \ No newline at end of file From 4d79a7828c7c8bcc77d5fb1462565861eb2ac5ea Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 21 Sep 2016 21:36:06 -0400 Subject: [PATCH 03/14] more cleaning up --- lbrynet/conf.py | 17 +- lbrynet/core/Error.py | 3 + lbrynet/lbrynet_daemon/LBRYDaemon.py | 130 +++++------ lbrynet/lbrynet_daemon/auth/auth.py | 2 +- lbrynet/lbrynet_daemon/auth/client.py | 19 +- lbrynet/lbrynet_daemon/auth/server.py | 319 ++++++++++++++++---------- lbrynet/lbrynet_daemon/auth/util.py | 8 +- 7 files changed, 276 insertions(+), 222 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 26ef18e3b..4989ddd77 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -45,12 +45,9 @@ KNOWN_DHT_NODES = [('104.236.42.182', 4000), POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424' #POINTTRADER_SERVER = 'http://127.0.0.1:2424' -if IS_DEVELOPMENT_VERSION: - SEARCH_SERVERS = ["http://107.170.207.64:50005"] -else: - SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005", - "http://lighthouse2.lbry.io:50005", - "http://lighthouse3.lbry.io:50005"] +SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005", + "http://lighthouse2.lbry.io:50005", + "http://lighthouse3.lbry.io:50005"] REFLECTOR_SERVERS = [("reflector.lbry.io", 5566)] @@ -67,6 +64,9 @@ if os.name == "nt": else: ICON_PATH = "app.icns" APP_NAME = "LBRY" + +ORIGIN = "http://%s:%i" % (API_INTERFACE, API_PORT) +REFERER = "http://%s:%i/" % (API_INTERFACE, API_PORT) API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRESS) UI_ADDRESS = "http://%s:%i" % (API_INTERFACE, API_PORT) PROTOCOL_PREFIX = "lbry" @@ -88,11 +88,6 @@ CURRENCIES = { 'USD': {'type': 'fiat'}, } -ALLOWED_DURING_STARTUP = ['is_running', 'is_first_run', - 'get_time_behind_blockchain', 'stop', - 'daemon_status', 'get_start_notice', - 'version', 'get_search_servers'] - LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv' ANALYTICS_ENDPOINT = 'https://api.segment.io/v1' diff --git a/lbrynet/core/Error.py b/lbrynet/core/Error.py index 363dddb3d..1f8355cea 100644 --- a/lbrynet/core/Error.py +++ b/lbrynet/core/Error.py @@ -93,4 +93,7 @@ class InvalidHeaderError(Exception): pass class InvalidAuthenticationToken(Exception): + pass + +class SubhandlerError(Exception): pass \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index f72cef38a..e5265264d 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -40,7 +40,8 @@ from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient -from lbrynet.lbrynet_daemon.auth.server import LBRYJSONRPCServer, auth_required, authorizer +from lbrynet.lbrynet_daemon.auth.server import AuthJSONRPCServer +from lbrynet.lbrynet_daemon.auth.util import get_auth_message from lbrynet.metadata.LBRYMetadata import Metadata, verify_name_characters from lbrynet.core import log_support from lbrynet.core import utils @@ -49,7 +50,7 @@ from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, \ KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, DEFAULT_WALLET, \ DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, \ - LOG_POST_URL, LOG_FILE_NAME, REFLECTOR_SERVERS, SEARCH_SERVERS, ALLOWED_DURING_STARTUP + LOG_POST_URL, LOG_FILE_NAME, REFLECTOR_SERVERS, SEARCH_SERVERS from lbrynet.conf import DEFAULT_SD_DOWNLOAD_TIMEOUT from lbrynet.conf import DEFAULT_TIMEOUT from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader @@ -122,10 +123,8 @@ BAD_REQUEST = 400 NOT_FOUND = 404 OK_CODE = 200 -# TODO add login credentials in a conf file # TODO alert if your copy of a lbry file is out of date with the name record - REMOTE_SERVER = "www.google.com" @@ -134,14 +133,17 @@ class Parameters(object): self.__dict__.update(kwargs) -@authorizer -class LBRYDaemon(LBRYJSONRPCServer): +class LBRYDaemon(AuthJSONRPCServer): """ LBRYnet daemon, a jsonrpc interface to lbry functions """ def __init__(self, root, wallet_type=None): - LBRYJSONRPCServer.__init__(self) + AuthJSONRPCServer.__init__(self) + self.allowed_during_startup = ['is_running', 'is_first_run', + 'get_time_behind_blockchain', 'stop', + 'daemon_status', 'get_start_notice', + 'version', 'get_search_servers'] reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown) self.startup_status = STARTUP_STAGES[0] @@ -392,6 +394,17 @@ class LBRYDaemon(LBRYJSONRPCServer): f.write("rpcpassword=" + password) log.info("Done writing lbrycrd.conf") + @AuthJSONRPCServer.subhandler + def _exclude_lbrycrd_only_commands_from_lbryum_session(self, request): + request.content.seek(0, 0) + content = request.content.read() + parsed = jsonrpclib.loads(content) + function_path = parsed.get("method") + if self.wallet_type == "lbryum" and function_path in ['set_miner', 'get_miner_status']: + log.warning("Mining commands are not available in lbryum") + raise Exception("Command not available in lbryum") + return True + def setup(self, branch=DEFAULT_UI_BRANCH, user_specified=False, branch_specified=False, host_ui=True): def _log_starting_vals(): log.info("Starting balance: " + str(self.session.wallet.wallet_balance)) @@ -1337,7 +1350,7 @@ class LBRYDaemon(LBRYJSONRPCServer): def _search(self, search): return self.lighthouse_client.search(search) - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_is_running(self): """ Check if lbrynet daemon is running @@ -1354,7 +1367,6 @@ class LBRYDaemon(LBRYJSONRPCServer): else: return self._render_response(False, OK_CODE) - @auth_required def jsonrpc_daemon_status(self): """ Get lbrynet daemon status information @@ -1386,10 +1398,8 @@ class LBRYDaemon(LBRYJSONRPCServer): else: r['message'] = "Catching up with the blockchain" r['progress'] = 0 - log.info("daemon status: " + str(r)) return self._render_response(r, OK_CODE) - @auth_required def jsonrpc_is_first_run(self): """ Check if this is the first time lbrynet daemon has been run @@ -1410,7 +1420,6 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required def jsonrpc_get_start_notice(self): """ Get special message to be displayed at startup @@ -1430,7 +1439,6 @@ class LBRYDaemon(LBRYJSONRPCServer): else: self._render_response(self.startup_message, OK_CODE) - @auth_required def jsonrpc_version(self): """ Get lbry version information @@ -1465,7 +1473,7 @@ class LBRYDaemon(LBRYJSONRPCServer): log.info("Get version info: " + json.dumps(msg)) return self._render_response(msg, OK_CODE) - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_settings(self): """ Get lbrynet daemon settings @@ -1494,7 +1502,7 @@ class LBRYDaemon(LBRYJSONRPCServer): log.info("Get daemon settings") return self._render_response(self.session_settings, OK_CODE) - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_set_settings(self, p): """ Set lbrynet daemon settings @@ -1522,7 +1530,6 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required def jsonrpc_help(self, p=None): """ Function to retrieve docstring for API function @@ -1537,17 +1544,17 @@ class LBRYDaemon(LBRYJSONRPCServer): """ if not p: - return self._render_response(self._listFunctions(), OK_CODE) + return self._render_response(self.callable_methods.keys(), OK_CODE) elif 'callable_during_start' in p.keys(): - return self._render_response(ALLOWED_DURING_STARTUP, OK_CODE) + return self._render_response(self.allowed_during_startup, OK_CODE) elif 'function' in p.keys(): func_path = p['function'] - function = self._getFunction(func_path) + function = self.callable_methods.get(func_path) return self._render_response(function.__doc__, OK_CODE) else: return self._render_response(self.jsonrpc_help.__doc__, OK_CODE) - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_balance(self): """ Get balance @@ -1561,7 +1568,6 @@ class LBRYDaemon(LBRYJSONRPCServer): log.info("Get balance") return self._render_response(float(self.session.wallet.wallet_balance), OK_CODE) - @auth_required def jsonrpc_stop(self): """ Stop lbrynet-daemon @@ -1581,7 +1587,7 @@ class LBRYDaemon(LBRYJSONRPCServer): return self._render_response("Shutting down", OK_CODE) - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_lbry_files(self): """ Get LBRY files @@ -1608,7 +1614,7 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_lbry_file(self, p): """ Get lbry file @@ -1638,7 +1644,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required def jsonrpc_resolve_name(self, p): """ Resolve stream info from a LBRY uri @@ -1660,7 +1665,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) return d - @auth_required def jsonrpc_get_claim_info(self, p): """ Resolve claim info from a LBRY uri @@ -1685,7 +1689,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def _process_get_parameters(self, p): """Extract info from input parameters and fill in default values for `get` call.""" # TODO: this process can be abstracted s.t. each method @@ -1707,7 +1711,7 @@ class LBRYDaemon(LBRYJSONRPCServer): name=name ) - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get(self, p): """Download stream from a LBRY uri. @@ -1738,7 +1742,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda message: self._render_response(message, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_stop_lbry_file(self, p): """ Stop lbry file @@ -1764,7 +1768,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_start_lbry_file(self, p): """ Stop lbry file @@ -1789,7 +1793,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required def jsonrpc_get_est_cost(self, p): """ Get estimated cost for a lbry uri @@ -1811,7 +1814,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_search_nametrie(self, p): """ Search the nametrie for claims @@ -1848,7 +1851,7 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_delete_lbry_file(self, p): """ Delete a lbry file @@ -1878,7 +1881,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_publish(self, p): """ Make a new name claim and publish associated data to lbrynet @@ -1955,7 +1958,7 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_abandon_claim(self, p): """ Abandon a name and reclaim credits from the claim @@ -1982,7 +1985,7 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_abandon_name(self, p): """ DEPRECIATED, use abandon_claim @@ -1995,7 +1998,7 @@ class LBRYDaemon(LBRYJSONRPCServer): return self.jsonrpc_abandon_claim(p) - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_support_claim(self, p): """ Support a name claim @@ -2015,7 +2018,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_name_claims(self): """ Get my name claims @@ -2039,7 +2042,6 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required def jsonrpc_get_claims_for_name(self, p): """ Get claims for a name @@ -2055,7 +2057,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_transaction_history(self): """ Get transaction history @@ -2070,7 +2072,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required def jsonrpc_get_transaction(self, p): """ Get a decoded transaction from a txid @@ -2087,7 +2088,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_address_is_mine(self, p): """ Checks if an address is associated with the current wallet. @@ -2105,7 +2106,7 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_public_key_from_wallet(self, p): """ Get public key from wallet address @@ -2120,7 +2121,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d = self.session.wallet.get_pub_keys(wallet) d.addCallback(lambda r: self._render_response(r, OK_CODE)) - @auth_required def jsonrpc_get_time_behind_blockchain(self): """ Get number of blocks behind the blockchain @@ -2144,7 +2144,7 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_new_address(self): """ Generate a new wallet address @@ -2164,7 +2164,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda address: self._render_response(address, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_send_amount_to_address(self, p): """ Send credits to an address @@ -2189,7 +2189,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d - @auth_required def jsonrpc_get_best_blockhash(self): """ Get hash of most recent block @@ -2204,7 +2203,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required def jsonrpc_get_block(self, p): """ Get contents of a block @@ -2227,7 +2225,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required def jsonrpc_get_claims_for_tx(self, p): """ Get claims for tx @@ -2247,7 +2244,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_download_descriptor(self, p): """ Download and return a sd blob @@ -2264,7 +2261,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallbacks(lambda r: self._render_response(r, OK_CODE), lambda _: self._render_response(False, OK_CODE)) return d - @auth_required def jsonrpc_get_nametrie(self): """ Get the nametrie @@ -2280,7 +2276,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_set_miner(self, p): """ Start of stop the miner, function only available when lbrycrd is set as the wallet @@ -2300,7 +2296,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_miner_status(self): """ Get status of miner @@ -2315,7 +2311,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required def jsonrpc_log(self, p): """ Log message @@ -2330,7 +2325,6 @@ class LBRYDaemon(LBRYJSONRPCServer): log.info("API client log request: %s" % message) return self._render_response(True, OK_CODE) - @auth_required def jsonrpc_upload_log(self, p=None): """ Upload log @@ -2372,7 +2366,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_configure_ui(self, p): """ Configure the UI being hosted @@ -2397,7 +2391,7 @@ class LBRYDaemon(LBRYJSONRPCServer): return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_reveal(self, p): """ Reveal a file or directory in file browser @@ -2417,7 +2411,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_peers_for_hash(self, p): """ Get peers for blob hash @@ -2435,7 +2429,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_announce_all_blobs_to_dht(self): """ Announce all blobs to the dht @@ -2450,7 +2444,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda _: self._render_response("Announced", OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_reflect(self, p): """ Reflect a stream @@ -2467,7 +2461,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallbacks(lambda _: self._render_response(True, OK_CODE), lambda err: self._render_response(err.getTraceback(), OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_blob_hashes(self): """ Returns all blob hashes @@ -2482,7 +2476,7 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_reflect_all_blobs(self): """ Reflects all saved blobs @@ -2498,20 +2492,6 @@ class LBRYDaemon(LBRYJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @auth_required - def jsonrpc_get_search_servers(self): - """ - Get list of lighthouse servers - - Args: - None - Returns: - List of address:port - """ - - d = self._render_response(SEARCH_SERVERS, OK_CODE) - return d - def get_lbrynet_version_from_github(): """Return the latest released version from github.""" diff --git a/lbrynet/lbrynet_daemon/auth/auth.py b/lbrynet/lbrynet_daemon/auth/auth.py index f68772fe2..bcda95829 100644 --- a/lbrynet/lbrynet_daemon/auth/auth.py +++ b/lbrynet/lbrynet_daemon/auth/auth.py @@ -14,7 +14,7 @@ class HttpPasswordRealm(object): self.resource = resource def requestAvatar(self, avatarId, mind, *interfaces): - log.info("Processing request for %s", avatarId) + log.debug("Processing request for %s", avatarId) if resource.IResource in interfaces: return (resource.IResource, self.resource, lambda: None) raise NotImplementedError() diff --git a/lbrynet/lbrynet_daemon/auth/client.py b/lbrynet/lbrynet_daemon/auth/client.py index 31f62949c..2e479726b 100644 --- a/lbrynet/lbrynet_daemon/auth/client.py +++ b/lbrynet/lbrynet_daemon/auth/client.py @@ -5,7 +5,7 @@ import os import base64 import json -from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME +from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, get_auth_message from lbrynet.conf import API_INTERFACE, API_ADDRESS, API_PORT from lbrynet.conf import DATA_DIR @@ -55,8 +55,8 @@ class LBRYAPIClient(object): 'method': self.__service_name, 'params': args, 'id': self.__id_count} - to_auth = str(pre_auth_postdata['method']).encode('hex') + str(pre_auth_postdata['id']).encode('hex') - token = self.__api_key.get_hmac(to_auth.decode('hex')) + to_auth = get_auth_message(pre_auth_postdata) + token = self.__api_key.get_hmac(to_auth) pre_auth_postdata.update({'hmac': token}) postdata = json.dumps(pre_auth_postdata) service_url = self.__service_url @@ -79,16 +79,13 @@ class LBRYAPIClient(object): next_secret = headers.get(LBRY_SECRET, False) if next_secret: - # print "Next secret: %s" % next_secret self.__api_key.secret = next_secret self.__cookies = cookies - # print "Postdata: %s" % postdata if http_response is None: raise JSONRPCException({ 'code': -342, 'message': 'missing HTTP response from server'}) - # print "-----\n%s\n------" % http_response.text http_response.raise_for_status() response = http_response.json() @@ -104,20 +101,18 @@ class LBRYAPIClient(object): @classmethod def config(cls, key_name=None, key=None, pw_path=None, timeout=HTTP_TIMEOUT, connection=None, count=0, service=None, cookies=None, auth=None, url=None, login_url=None): + api_key_name = API_KEY_NAME if not key_name else key_name pw_path = os.path.join(DATA_DIR, ".api_keys") if not pw_path else pw_path - if not key: keys = load_api_keys(pw_path) api_key = keys.get(api_key_name, False) else: api_key = APIKey(name=api_key_name, secret=key) - if login_url is None: service_url = "http://%s:%s@%s:%i/%s" % (api_key_name, api_key.secret, API_INTERFACE, API_PORT, API_ADDRESS) else: service_url = login_url - id_count = count if auth is None and connection is None and cookies is None and url is None: @@ -134,10 +129,8 @@ class LBRYAPIClient(object): pass authpair = user + b':' + passwd auth_header = b'Basic ' + base64.b64encode(authpair) - conn = requests.Session() conn.auth = (user, passwd) - req = requests.Request(method='POST', url=service_url, auth=conn.auth, @@ -148,11 +141,9 @@ class LBRYAPIClient(object): r = req.prepare() http_response = conn.send(r) cookies = http_response.cookies - # print "Logged in" - uid = cookies.get(TWISTED_SESSION) api_key = APIKey.new(seed=uid) - # print "Created temporary api key" + else: # This is a client that already has a session, use it auth_header = auth diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index 10b6cf65b..1c675b5cf 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -1,16 +1,15 @@ import logging from decimal import Decimal -from twisted.web import server +from zope.interface import implements +from twisted.web import server, resource from twisted.internet import defer from txjsonrpc import jsonrpclib -from txjsonrpc.web import jsonrpc -from txjsonrpc.web.jsonrpc import Handler -from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError -from lbrynet.lbrynet_daemon.auth.util import APIKey +from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError, SubhandlerError +from lbrynet.conf import API_INTERFACE, REFERER, ORIGIN +from lbrynet.lbrynet_daemon.auth.util import APIKey, get_auth_message from lbrynet.lbrynet_daemon.auth.client import LBRY_SECRET -from lbrynet.conf import ALLOWED_DURING_STARTUP log = logging.getLogger(__name__) @@ -20,42 +19,166 @@ def default_decimal(obj): return float(obj) -def authorizer(cls): - cls.authorized_functions = [] - for methodname in dir(cls): - if methodname.startswith("jsonrpc_"): - method = getattr(cls, methodname) - if hasattr(method, '_auth_required'): - cls.authorized_functions.append(methodname.split("jsonrpc_")[1]) - return cls +class AuthorizedBase(object): + def __init__(self): + self.authorized_functions = [] + self.subhandlers = [] + self.callable_methods = {} + + for methodname in dir(self): + if methodname.startswith("jsonrpc_"): + method = getattr(self, methodname) + self.callable_methods.update({methodname.split("jsonrpc_")[1]: method}) + if hasattr(method, '_auth_required'): + self.authorized_functions.append(methodname.split("jsonrpc_")[1]) + elif not methodname.startswith("__"): + method = getattr(self, methodname) + if hasattr(method, '_subhandler'): + self.subhandlers.append(method) + + @staticmethod + def auth_required(f): + f._auth_required = True + return f + + @staticmethod + def subhandler(f): + f._subhandler = True + return f -def auth_required(f): - f._auth_required = True - return f +class AuthJSONRPCServer(AuthorizedBase): + """ + Authorized JSONRPC server used as the base class for the LBRY API + API methods are named with a leading "jsonrpc_" -@authorizer -class LBRYJSONRPCServer(jsonrpc.JSONRPC): + Decorators: + @AuthJSONRPCServer.auth_required: this requires the client include a valid hmac authentication token in their + request + + @AuthJSONRPCServer.subhandler: include the tagged method in the processing of requests, to allow inheriting + classes to modify request handling. Tagged methods will be passed the request + object, and return True when finished to indicate success + + Attributes: + allowed_during_startup (list): list of api methods that are callable before the server has finished + startup + + sessions (dict): dictionary of active session_id: lbrynet.lbrynet_daemon.auth.util.APIKey values + + authorized_functions (list): list of api methods that require authentication + + subhandlers (list): list of subhandlers + + callable_methods (dict): dictionary of api_callable_name: method values + """ + implements(resource.IResource) isLeaf = True + OK = 200 + UNAUTHORIZED = 401 + NOT_FOUND = 8001 + FAILURE = 8002 def __init__(self): - jsonrpc.JSONRPC.__init__(self) + AuthorizedBase.__init__(self) + self.allowed_during_startup = [] self.sessions = {} + def setup(self): + return NotImplementedError() + + def render(self, request): + assert self._check_headers(request), InvalidHeaderError + + session = request.getSession() + session_id = session.uid + + # if this is a new session, send a new secret and set the expiration, otherwise, session.touch() + if self._initialize_session(session_id): + def expire_session(): + self._unregister_user_session(session_id) + session.startCheckingExpiration() + session.notifyOnExpire(expire_session) + message = "OK" + request.setResponseCode(self.OK) + self._set_headers(request, message, True) + self._render_message(request, message) + return server.NOT_DONE_YET + session.touch() + + request.content.seek(0, 0) + content = request.content.read() + try: + parsed = jsonrpclib.loads(content) + except ValueError: + return server.failure + + function_name = parsed.get('method') + args = parsed.get('params') + id = parsed.get('id') + token = parsed.pop('hmac', None) + version = self._get_jsonrpc_version(parsed.get('jsonrpc'), id) + + try: + self._run_subhandlers(request) + except SubhandlerError: + return server.failure + + reply_with_next_secret = False + if function_name in self.authorized_functions: + try: + self._verify_token(session_id, parsed, token) + except InvalidAuthenticationToken: + log.warning("API validation failed") + request.setResponseCode(self.UNAUTHORIZED) + request.finish() + return server.NOT_DONE_YET + self._update_session_secret(session_id) + reply_with_next_secret = True + + try: + function = self._get_jsonrpc_method(function_name) + except Exception: + log.warning("Unknown method: %s", function_name) + return server.failure + + d = defer.maybeDeferred(function) if args == [{}] else defer.maybeDeferred(function, *args) + # cancel the response if the connection is broken + notify_finish = request.notifyFinish() + notify_finish.addErrback(self._response_failed, d) + d.addErrback(self._errback_render, id) + d.addCallback(self._callback_render, request, id, version, reply_with_next_secret) + d.addErrback(notify_finish.errback) + + return server.NOT_DONE_YET + def _register_user_session(self, session_id): + """ + Add or update a HMAC secret for a session + + @param session_id: + @return: secret + """ token = APIKey.new() self.sessions.update({session_id: token}) return token - def _responseFailed(self, err, call): + def _unregister_user_session(self, session_id): + log.info("Unregister API session") + del self.sessions[session_id] + + def _response_failed(self, err, call): log.debug(err.getTraceback()) - def _set_headers(self, request, data): - request.setHeader("Access-Control-Allow-Origin", "localhost") + def _set_headers(self, request, data, update_secret=False): + request.setHeader("Access-Control-Allow-Origin", API_INTERFACE) request.setHeader("Content-Type", "text/json") request.setHeader("Content-Length", str(len(data))) + if update_secret: + session_id = request.getSession().uid + request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret) def _render_message(self, request, message): request.write(message) @@ -64,112 +187,81 @@ class LBRYJSONRPCServer(jsonrpc.JSONRPC): def _check_headers(self, request): origin = request.getHeader("Origin") referer = request.getHeader("Referer") - - if origin not in [None, 'http://localhost:5279']: + if origin not in [None, ORIGIN]: log.warning("Attempted api call from %s", origin) - raise InvalidHeaderError - - if referer is not None and not referer.startswith('http://localhost:5279/'): + return False + if referer is not None and not referer.startswith(REFERER): log.warning("Attempted api call from %s", referer) - raise InvalidHeaderError + return False + return True - def _handle(self, request): - def _check_function_path(function_path): - if not self.announced_startup: - if function_path not in ALLOWED_DURING_STARTUP: - log.warning("Cannot call %s during startup", function_path) - raise Exception("Function not allowed") + def _check_function_path(self, function_path): + if function_path not in self.callable_methods: + log.warning("Unknown method: %s", function_path) + return False + if not self.announced_startup: + if function_path not in self.allowed_during_startup: + log.warning("Cannot call %s during startup", function_path) + return False + return True - def _get_function(function_path): - function = self._getFunction(function_path) - return function + def _get_jsonrpc_method(self, function_path): + assert self._check_function_path(function_path) + return self.callable_methods.get(function_path) - def _verify_token(session_id, message, token): - request.setHeader(LBRY_SECRET, "") - api_key = self.sessions.get(session_id, None) - assert api_key is not None, InvalidAuthenticationToken - r = api_key.compare_hmac(message, token) - assert r, InvalidAuthenticationToken - # log.info("Generating new token for next request") - self.sessions.update({session_id: APIKey.new(name=session_id)}) - request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret) - - session = request.getSession() - session_id = session.uid - session_store = self.sessions.get(session_id, False) - - if not session_store: - token = APIKey.new(seed=session_id, name=session_id) + def _initialize_session(self, session_id): + if not self.sessions.get(session_id, False): log.info("Initializing new api session") - self.sessions.update({session_id: token}) - # log.info("Generated token %s", str(self.sessions[session_id])) + self.sessions.update({session_id: APIKey.new(seed=session_id, name=session_id)}) + return True + return False - request.content.seek(0, 0) - content = request.content.read() + def _verify_token(self, session_id, message, token): + to_auth = get_auth_message(message) + api_key = self.sessions.get(session_id) + assert api_key.compare_hmac(to_auth, token), InvalidAuthenticationToken - parsed = jsonrpclib.loads(content) - - functionPath = parsed.get("method") - - _check_function_path(functionPath) - require_auth = functionPath in self.authorized_functions - if require_auth: - token = parsed.pop('hmac') - to_auth = functionPath.encode('hex') + str(parsed.get('id')).encode('hex') - _verify_token(session_id, to_auth.decode('hex'), token) - - args = parsed.get('params') - id = parsed.get('id') - version = parsed.get('jsonrpc') + def _update_session_secret(self, session_id): + # log.info("Generating new token for next request") + self.sessions.update({session_id: APIKey.new(name=session_id)}) + def _get_jsonrpc_version(self, version=None, id=None): if version: - version = int(float(version)) + version_for_return = int(float(version)) elif id and not version: - version = jsonrpclib.VERSION_1 + version_for_return = jsonrpclib.VERSION_1 else: - version = jsonrpclib.VERSION_PRE1 + version_for_return = jsonrpclib.VERSION_PRE1 + return version_for_return - if self.wallet_type == "lbryum" and functionPath in ['set_miner', 'get_miner_status']: - log.warning("Mining commands are not available in lbryum") - raise Exception("Command not available in lbryum") + def _run_subhandlers(self, request): + for handler in self.subhandlers: + try: + assert handler(request) + except Exception as err: + log.error(err.message) + raise SubhandlerError - try: - function = _get_function(functionPath) - if args == [{}]: - d = defer.maybeDeferred(function) - else: - d = defer.maybeDeferred(function, *args) - except jsonrpclib.Fault as f: - d = self._cbRender(f, request, id, version) - finally: - # cancel the response if the connection is broken - notify_finish = request.notifyFinish() - notify_finish.addErrback(self._responseFailed, d) - d.addErrback(self._ebRender, id) - d.addCallback(self._cbRender, request, id, version) - d.addErrback(notify_finish.errback) - def _cbRender(self, result, request, id, version): - if isinstance(result, Handler): - result = result.result - - if isinstance(result, dict): - result = result['result'] + def _callback_render(self, result, request, id, version, auth_required=False): + result_for_return = result if not isinstance(result, dict) else result['result'] if version == jsonrpclib.VERSION_PRE1: if not isinstance(result, jsonrpclib.Fault): - result = (result,) + result_for_return = (result_for_return,) # Convert the result (python) to JSON-RPC try: - s = jsonrpclib.dumps(result, version=version, default=default_decimal) - self._render_message(request, s) + encoded_message = jsonrpclib.dumps(result_for_return, version=version, default=default_decimal) + self._set_headers(request, encoded_message, auth_required) + self._render_message(request, encoded_message) except: - f = jsonrpclib.Fault(self.FAILURE, "can't serialize output") - s = jsonrpclib.dumps(f, version=version) - self._set_headers(request, s) - self._render_message(request, s) + fault = jsonrpclib.Fault(self.FAILURE, "can't serialize output") + encoded_message = jsonrpclib.dumps(fault, version=version) + self._set_headers(request, encoded_message) + self._render_message(request, encoded_message) - def _ebRender(self, failure, id): + def _errback_render(self, failure, id): + log.error("Request failed:") log.error(failure) log.error(failure.value) log.error(id) @@ -177,19 +269,6 @@ class LBRYJSONRPCServer(jsonrpc.JSONRPC): return failure.value return server.failure - def render(self, request): - try: - self._check_headers(request) - except InvalidHeaderError: - return server.failure - - try: - self._handle(request) - except: - return server.failure - - return server.NOT_DONE_YET - def _render_response(self, result, code): return defer.succeed({'result': result, 'code': code}) diff --git a/lbrynet/lbrynet_daemon/auth/util.py b/lbrynet/lbrynet_daemon/auth/util.py index 1ed625569..8a1e078a4 100644 --- a/lbrynet/lbrynet_daemon/auth/util.py +++ b/lbrynet/lbrynet_daemon/auth/util.py @@ -84,4 +84,10 @@ def initialize_api_key_file(key_path): keys = {} new_api_key = APIKey.new(name=API_KEY_NAME) keys.update({new_api_key.name: new_api_key}) - save_api_keys(keys, key_path) \ No newline at end of file + save_api_keys(keys, key_path) + + +def get_auth_message(message_dict): + to_auth = message_dict.get('method').encode('hex') + to_auth += str(message_dict.get('id')).encode('hex') + return to_auth.decode('hex') \ No newline at end of file From dbeb6bc3bc53cbe8cdc2c6e2409430035afcd86f Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 21 Sep 2016 21:55:10 -0400 Subject: [PATCH 04/14] update uri handler --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 1 - packaging/uri_handler/LBRYURIHandler.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index e5265264d..8ac3bc1ce 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1350,7 +1350,6 @@ class LBRYDaemon(AuthJSONRPCServer): def _search(self, search): return self.lighthouse_client.search(search) - @AuthJSONRPCServer.auth_required def jsonrpc_is_running(self): """ Check if lbrynet daemon is running diff --git a/packaging/uri_handler/LBRYURIHandler.py b/packaging/uri_handler/LBRYURIHandler.py index dd1c5b13e..bb17a33d6 100644 --- a/packaging/uri_handler/LBRYURIHandler.py +++ b/packaging/uri_handler/LBRYURIHandler.py @@ -4,15 +4,14 @@ import subprocess import sys from time import sleep -from jsonrpc.proxy import JSONRPCProxy - -from lbrynet.conf import UI_ADDRESS, API_CONNECTION_STRING +from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient +from lbrynet.conf import UI_ADDRESS class LBRYURIHandler(object): def __init__(self): self.started_daemon = False - self.daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING) + self.daemon = LBRYAPIClient.config() def handle_osx(self, lbry_name): self.check_daemon() From 50e5a594201bfb9b2f43f92911ac1f0def2465a5 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 7 Oct 2016 17:29:20 -0400 Subject: [PATCH 05/14] remove unused imports --- lbrynet/lbrynet_daemon/Daemon.py | 2 -- lbrynet/lbrynet_daemon/auth/auth.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index 434c28e27..52d4b6ae4 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -21,7 +21,6 @@ from twisted.web import server from twisted.internet import defer, threads, error, reactor, task from twisted.internet.task import LoopingCall from txjsonrpc import jsonrpclib -from txjsonrpc.web import jsonrpc from lbrynet import __version__ as lbrynet_version from lbryum.version import LBRYUM_VERSION as lbryum_version @@ -40,7 +39,6 @@ from lbrynet.lbrynet_daemon.Publisher import Publisher from lbrynet.lbrynet_daemon.ExchangeRateManager import ExchangeRateManager from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient from lbrynet.lbrynet_daemon.auth.server import AuthJSONRPCServer -from lbrynet.lbrynet_daemon.auth.util import get_auth_message from lbrynet.metadata.Metadata import Metadata, verify_name_characters from lbrynet.core import log_support from lbrynet.core import utils diff --git a/lbrynet/lbrynet_daemon/auth/auth.py b/lbrynet/lbrynet_daemon/auth/auth.py index bcda95829..fc61929ea 100644 --- a/lbrynet/lbrynet_daemon/auth/auth.py +++ b/lbrynet/lbrynet_daemon/auth/auth.py @@ -3,7 +3,7 @@ from zope.interface import implementer from twisted.cred import portal, checkers, credentials, error as cred_error from twisted.internet import defer from twisted.web import resource -from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, save_api_keys +from lbrynet.lbrynet_daemon.auth.util import load_api_keys log = logging.getLogger(__name__) From 06a88599df6b1fbb5aaf6092a10ef9dc52e36d5e Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 19 Oct 2016 00:12:44 -0400 Subject: [PATCH 06/14] --http-auth flag to use authentication, refactor settings settings will prefer, in order: -defaults -settings in config file -settings given as environmental variables -settings given as command line args --- lbrynet/__init__.py | 3 +- lbrynet/analytics/api.py | 6 +- lbrynet/conf.py | 201 ++++--- lbrynet/core/HashBlob.py | 4 +- lbrynet/core/PaymentRateManager.py | 5 +- lbrynet/core/PriceModel.py | 4 +- lbrynet/core/Wallet.py | 14 +- lbrynet/core/client/ClientProtocol.py | 4 +- lbrynet/core/client/ClientRequest.py | 4 +- lbrynet/core/client/ConnectionManager.py | 4 +- lbrynet/core/log_support.py | 4 +- lbrynet/core/utils.py | 15 +- lbrynet/cryptstream/CryptBlob.py | 4 +- .../lbryfilemanager/EncryptedFileCreator.py | 4 +- lbrynet/lbrylive/LiveStreamCreator.py | 4 +- .../client/LiveStreamMetadataHandler.py | 4 +- lbrynet/lbrynet_console/Console.py | 8 +- lbrynet/lbrynet_daemon/Daemon.py | 510 ++++++------------ lbrynet/lbrynet_daemon/DaemonControl.py | 97 ++-- lbrynet/lbrynet_daemon/DaemonServer.py | 16 +- lbrynet/lbrynet_daemon/Downloader.py | 6 +- lbrynet/lbrynet_daemon/Lighthouse.py | 4 +- lbrynet/lbrynet_daemon/Publisher.py | 8 +- lbrynet/lbrynet_daemon/Resources.py | 6 +- lbrynet/lbrynet_daemon/UIManager.py | 30 +- lbrynet/lbrynet_daemon/auth/client.py | 13 +- lbrynet/lbrynet_daemon/auth/server.py | 67 ++- .../pointtraderclient/pointtraderclient.py | 4 +- packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py | 16 +- packaging/uri_handler/LBRYURIHandler.py | 6 +- .../windows/lbry-win32-app/LBRYWin32App.py | 20 +- tests/functional/test_misc.py | 25 +- tests/functional/test_reflector.py | 4 +- tests/mocks.py | 2 +- 34 files changed, 523 insertions(+), 603 deletions(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 87f99d259..47139be3a 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,6 +1,7 @@ import logging +from conf import Config __version__ = "0.6.4" version = tuple(__version__.split('.')) - +settings = Config() logging.getLogger(__name__).addHandler(logging.NullHandler()) \ No newline at end of file diff --git a/lbrynet/analytics/api.py b/lbrynet/analytics/api.py index 2b8a3344c..84d25a7bb 100644 --- a/lbrynet/analytics/api.py +++ b/lbrynet/analytics/api.py @@ -5,7 +5,7 @@ import logging from requests import auth from requests_futures import sessions -from lbrynet import conf +from lbrynet import settings from lbrynet.analytics import utils @@ -66,6 +66,6 @@ class AnalyticsApi(object): session = sessions.FuturesSession() return cls( session, - conf.ANALYTICS_ENDPOINT, - utils.deobfuscate(conf.ANALYTICS_TOKEN) + settings.ANALYTICS_ENDPOINT, + utils.deobfuscate(settings.ANALYTICS_TOKEN) ) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 20ac04926..f4a97af11 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -1,10 +1,13 @@ -""" -Some network wide and also application specific parameters -""" +import copy import os import sys +import logging from appdirs import user_data_dir +log = logging.getLogger(__name__) + +PRIORITIZE_ENV = True + LINUX = 1 DARWIN = 2 WINDOWS = 3 @@ -15,82 +18,154 @@ elif sys.platform.startswith("darwin"): platform = DARWIN elif sys.platform.startswith("win"): platform = WINDOWS +else: + platform = LINUX if platform is LINUX: - DATA_DIR = os.path.join(os.path.expanduser("~"), ".lbrynet") + default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') + default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") + default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum") +elif platform is DARWIN: + default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') + default_data_dir = user_data_dir("LBRY") + default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum") else: - DATA_DIR = user_data_dir("LBRY") + from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle + default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current) + default_data_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrynet") + default_lbryum_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum") -IS_DEVELOPMENT_VERSION = False +ADJUSTABLE_SETTINGS = { + 'run_on_startup': False, + 'download_directory': default_download_directory, + 'max_upload': 0.0, + 'max_download': 0.0, + 'upload_log': True, + 'delete_blobs_on_remove': True, + 'use_upnp': True, + 'start_lbrycrdd': True, + 'run_reflector_server': False, + 'startup_scripts': [], + 'last_version': {}, + 'peer_port': 3333, + 'dht_node_port': 4444, + 'reflector_port': 5566, + 'download_timeout': 30, + 'max_search_results': 25, + 'search_timeout': 3.0, + 'cache_time': 150, + 'host_ui': True, + 'check_ui_requirements': True, + 'local_ui_path': False, + 'API_PORT': 5279, + 'search_servers':['lighthouse1.lbry.io:50005'], + 'data_rate': .0001, # points/megabyte + 'MIN_BLOB_INFO_PAYMENT_RATE': .02, # points/1000 infos + 'MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE': .05, # points/1000 infos + 'MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE': .05, # points/1000 infos + 'max_connections_per_stream': 5, + 'known_dht_nodes': [('104.236.42.182', 4000), + ('lbrynet1.lbry.io', 4444), + ('lbrynet2.lbry.io', 4444), + ('lbrynet3.lbry.io', 4444)], + 'POINTTRADER_SERVER': 'http://127.0.0.1:2424', + 'REFLECTOR_SERVERS': [("reflector.lbry.io", 5566)], + 'WALLET': "lbryum", + 'UI_BRANCH': "master", + 'DEFAULT_UI_BRANCH': 'master', + 'DATA_DIR': default_data_dir, + 'LBRYUM_WALLET_DIR': default_lbryum_dir, + 'sd_download_timeout': 3, + 'max_key_fee': {'USD': {'amount': 25.0, 'address': ''}}, -MAX_HANDSHAKE_SIZE = 2**16 -MAX_REQUEST_SIZE = 2**16 -MAX_BLOB_REQUEST_SIZE = 2**16 -MAX_RESPONSE_INFO_SIZE = 2**16 -MAX_BLOB_INFOS_TO_REQUEST = 20 -BLOBFILES_DIR = ".blobfiles" -BLOB_SIZE = 2**21 +} -MIN_BLOB_DATA_PAYMENT_RATE = .0001 # points/megabyte -MIN_BLOB_INFO_PAYMENT_RATE = .02 # points/1000 infos -MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05 # points/1000 infos -MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .05 # points/1000 infos -MAX_CONNECTIONS_PER_STREAM = 5 -KNOWN_DHT_NODES = [('104.236.42.182', 4000), - ('lbrynet1.lbry.io', 4444), - ('lbrynet2.lbry.io', 4444), - ('lbrynet3.lbry.io', 4444)] +class ApplicationSettings(object): + USE_AUTH_HTTP = True + MAX_HANDSHAKE_SIZE = 2**16 + MAX_REQUEST_SIZE = 2**16 + MAX_BLOB_REQUEST_SIZE = 2**16 + MAX_RESPONSE_INFO_SIZE = 2**16 + MAX_BLOB_INFOS_TO_REQUEST = 20 + BLOBFILES_DIR = "blobfiles" + BLOB_SIZE = 2**21 + LOG_FILE_NAME = "lbrynet.log" + LOG_POST_URL = "https://lbry.io/log-upload" + CRYPTSD_FILE_EXTENSION = ".cryptsd" + API_INTERFACE = "localhost" + API_ADDRESS = "lbryapi" + ICON_PATH = "icons" if platform is WINDOWS else "app.icns" + APP_NAME = "LBRY" + PROTOCOL_PREFIX = "lbry" + WALLET_TYPES = ["lbryum", "lbrycrd"] + SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] + CURRENCIES = { + 'BTC': {'type': 'crypto'}, + 'LBC': {'type': 'crypto'}, + 'USD': {'type': 'fiat'}, + } + LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv' + ANALYTICS_ENDPOINT = 'https://api.segment.io/v1' + ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H=' -POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424' -#POINTTRADER_SERVER = 'http://127.0.0.1:2424' + @staticmethod + def get_dict(): + r = {k: v for k, v in ApplicationSettings.__dict__.iteritems() if not k.startswith('__')} + if PRIORITIZE_ENV: + log.info("Checking env settings") + r = add_env_settings(r) + return r -SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005", - "http://lighthouse2.lbry.io:50005", - "http://lighthouse3.lbry.io:50005"] -REFLECTOR_SERVERS = [("reflector.lbry.io", 5566)] +def add_env_settings(settings_dict): + with_env_settings = copy.deepcopy(settings_dict) + for setting, setting_val in settings_dict.iteritems(): + env_val = os.environ.get(setting, None) + if env_val != setting_val and env_val is not None: + log.info("Using env value for %s", setting) + with_env_settings.update({setting: env_val}) + return with_env_settings -LOG_FILE_NAME = "lbrynet.log" -LOG_POST_URL = "https://lbry.io/log-upload" -CRYPTSD_FILE_EXTENSION = ".cryptsd" +DEFAULT_CONFIG = ApplicationSettings.get_dict() +DEFAULT_CONFIG.update(add_env_settings(ADJUSTABLE_SETTINGS)) -API_INTERFACE = "localhost" -API_ADDRESS = "lbryapi" -API_PORT = 5279 -if os.name == "nt": - ICON_PATH = "icons" -else: - ICON_PATH = "app.icns" -APP_NAME = "LBRY" -ORIGIN = "http://%s:%i" % (API_INTERFACE, API_PORT) -REFERER = "http://%s:%i/" % (API_INTERFACE, API_PORT) -API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRESS) -UI_ADDRESS = "http://%s:%i" % (API_INTERFACE, API_PORT) -PROTOCOL_PREFIX = "lbry" +class Config(object): + __shared_state = copy.deepcopy(DEFAULT_CONFIG) -DEFAULT_WALLET = "lbryum" -WALLET_TYPES = ["lbryum", "lbrycrd"] -DEFAULT_TIMEOUT = 30 -DEFAULT_MAX_SEARCH_RESULTS = 25 -DEFAULT_MAX_KEY_FEE = {'USD': {'amount': 25.0, 'address': ''}} -DEFAULT_SEARCH_TIMEOUT = 3.0 -DEFAULT_SD_DOWNLOAD_TIMEOUT = 3 -DEFAULT_CACHE_TIME = 150 -DEFAULT_UI_BRANCH = "master" + def __init__(self): + self.__dict__ = self.__shared_state -SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] -CURRENCIES = { - 'BTC': {'type': 'crypto'}, - 'LBC': {'type': 'crypto'}, - 'USD': {'type': 'fiat'}, - } + def update(self, settings): + for k, v in settings.iteritems(): + if k in ADJUSTABLE_SETTINGS: + self.__dict__.update({k: v}) -LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv' + @property + def configurable_settings(self): + return {k: v for k, v in copy.deepcopy(self.__dict__).iteritems() if k in ADJUSTABLE_SETTINGS} -ANALYTICS_ENDPOINT = 'https://api.segment.io/v1' -ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H=' + @property + def ORIGIN(self): + return "http://%s:%i" % (ApplicationSettings.API_INTERFACE, self.API_PORT) -LBRYUM_WALLET_DIR = os.environ.get('LBRYUM_WALLET_DIR') + @property + def REFERER(self): + return "http://%s:%i/" % (ApplicationSettings.API_INTERFACE, self.API_PORT) + + @property + def API_CONNECTION_STRING(self): + return "http://%s:%i/%s" % (ApplicationSettings.API_INTERFACE, self.API_PORT, ApplicationSettings.API_ADDRESS) + + @property + def UI_ADDRESS(self): + return "http://%s:%i" % (ApplicationSettings.API_INTERFACE, self.API_PORT) + + @property + def LBRYUM_WALLET_DIR(self): + env_dir = os.environ.get('LBRYUM_WALLET_DIR') + if env_dir: + return env_dir + return self.__dict__.get('LBRYUM_WALLET_DIR') diff --git a/lbrynet/core/HashBlob.py b/lbrynet/core/HashBlob.py index 072d2c5a6..881ff79e9 100644 --- a/lbrynet/core/HashBlob.py +++ b/lbrynet/core/HashBlob.py @@ -8,7 +8,7 @@ from twisted.internet import interfaces, defer, threads from twisted.protocols.basic import FileSender from twisted.python.failure import Failure from zope.interface import implements -from lbrynet.conf import BLOB_SIZE +from lbrynet import settings from lbrynet.core.Error import DownloadCanceledError, InvalidDataError from lbrynet.core.cryptoutils import get_lbry_hash_obj @@ -87,7 +87,7 @@ class HashBlob(object): def set_length(self, length): if self.length is not None and length == self.length: return True - if self.length is None and 0 <= length <= BLOB_SIZE: + if self.length is None and 0 <= length <= settings.BLOB_SIZE: self.length = length return True log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length)) diff --git a/lbrynet/core/PaymentRateManager.py b/lbrynet/core/PaymentRateManager.py index 914e21947..eddc30405 100644 --- a/lbrynet/core/PaymentRateManager.py +++ b/lbrynet/core/PaymentRateManager.py @@ -1,8 +1,9 @@ from lbrynet.core.Strategy import get_default_strategy -from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, MIN_BLOB_INFO_PAYMENT_RATE +from lbrynet import settings + class BasePaymentRateManager(object): - def __init__(self, rate=MIN_BLOB_DATA_PAYMENT_RATE, info_rate=MIN_BLOB_INFO_PAYMENT_RATE): + def __init__(self, rate=settings.data_rate, info_rate=settings.MIN_BLOB_INFO_PAYMENT_RATE): self.min_blob_data_payment_rate = rate self.min_blob_info_payment_rate = info_rate diff --git a/lbrynet/core/PriceModel.py b/lbrynet/core/PriceModel.py index 299d50223..8c16b4ace 100644 --- a/lbrynet/core/PriceModel.py +++ b/lbrynet/core/PriceModel.py @@ -2,7 +2,7 @@ from zope.interface import implementer from decimal import Decimal from lbrynet.interfaces import IBlobPriceModel -from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE +from lbrynet import settings def get_default_price_model(blob_tracker, base_price, **kwargs): @@ -21,7 +21,7 @@ class MeanAvailabilityWeightedPrice(object): """ implementer(IBlobPriceModel) - def __init__(self, tracker, base_price=MIN_BLOB_DATA_PAYMENT_RATE, alpha=1.0): + def __init__(self, tracker, base_price=settings.data_rate, alpha=1.0): self.blob_tracker = tracker self.base_price = Decimal(base_price) self.alpha = Decimal(alpha) diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index 896d71de8..e3d008421 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -153,7 +153,12 @@ class Wallet(object): log.info("Got a new balance: %s", str(balance)) self.wallet_balance = balance - d.addCallback(set_wallet_balance) + def log_error(err): + if isinstance(err, AttributeError): + log.warning("Failed to get an updated balance") + log.warning("Last balance update: %s", str(self.wallet_balance)) + + d.addCallbacks(set_wallet_balance, log_error) return d d.addCallback(lambda should_run: do_manage() if should_run else None) @@ -1169,7 +1174,8 @@ class LBRYumWallet(Wallet): self._start_check = None if self._catch_up_check is not None: - self._catch_up_check.stop() + if self._catch_up_check.running: + self._catch_up_check.stop() self._catch_up_check = None d = defer.Deferred() @@ -1241,6 +1247,9 @@ class LBRYumWallet(Wallet): self._caught_up_counter += 1 + def log_error(err): + log.warning(err.getErrorMessage()) + return defer.fail(err) self._catch_up_check = task.LoopingCall(check_caught_up) @@ -1248,6 +1257,7 @@ class LBRYumWallet(Wallet): d.addCallback(self._save_wallet) d.addCallback(lambda _: self.wallet.start_threads(self.network)) d.addCallback(lambda _: self._catch_up_check.start(.1)) + d.addErrback(log_error) d.addCallback(lambda _: blockchain_caught_d) return d diff --git a/lbrynet/core/client/ClientProtocol.py b/lbrynet/core/client/ClientProtocol.py index 1989f8c96..99d55edfe 100644 --- a/lbrynet/core/client/ClientProtocol.py +++ b/lbrynet/core/client/ClientProtocol.py @@ -4,7 +4,7 @@ from decimal import Decimal from twisted.internet import error, defer from twisted.internet.protocol import Protocol, ClientFactory from twisted.python import failure -from lbrynet.conf import MAX_RESPONSE_INFO_SIZE as MAX_RESPONSE_SIZE +from lbrynet import settings from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError from lbrynet.core.Error import RequestCanceledError @@ -48,7 +48,7 @@ class ClientProtocol(Protocol): self._blob_download_request.write(data) else: self._response_buff += data - if len(self._response_buff) > MAX_RESPONSE_SIZE: + if len(self._response_buff) > settings.MAX_RESPONSE_INFO_SIZE: log.warning("Response is too large. Size %s", len(self._response_buff)) self.transport.loseConnection() response, extra_data = self._get_valid_response(self._response_buff) diff --git a/lbrynet/core/client/ClientRequest.py b/lbrynet/core/client/ClientRequest.py index b5630f729..55abe132c 100644 --- a/lbrynet/core/client/ClientRequest.py +++ b/lbrynet/core/client/ClientRequest.py @@ -1,4 +1,4 @@ -from lbrynet.conf import BLOB_SIZE +from lbrynet import settings class ClientRequest(object): @@ -17,7 +17,7 @@ class ClientBlobRequest(ClientPaidRequest): def __init__(self, request_dict, response_identifier, write_func, finished_deferred, cancel_func, blob): if blob.length is None: - max_pay_units = BLOB_SIZE + max_pay_units = settings.BLOB_SIZE else: max_pay_units = blob.length ClientPaidRequest.__init__(self, request_dict, response_identifier, max_pay_units) diff --git a/lbrynet/core/client/ConnectionManager.py b/lbrynet/core/client/ConnectionManager.py index 15e3ac686..8e443f2d3 100644 --- a/lbrynet/core/client/ConnectionManager.py +++ b/lbrynet/core/client/ConnectionManager.py @@ -2,7 +2,7 @@ import logging from twisted.internet import defer from zope.interface import implements from lbrynet import interfaces -from lbrynet.conf import MAX_CONNECTIONS_PER_STREAM +from lbrynet import settings from lbrynet.core.client.ClientProtocol import ClientProtocolFactory from lbrynet.core.Error import InsufficientFundsError @@ -183,7 +183,7 @@ class ConnectionManager(object): log.debug("Couldn't find a good peer to connect to") return None - if len(self._peer_connections) < MAX_CONNECTIONS_PER_STREAM: + if len(self._peer_connections) < settings.max_connections_per_stream: ordered_request_creators = self._rank_request_creator_connections() d = get_new_peers(ordered_request_creators) d.addCallback(pick_best_peer) diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index 2f2a5c75b..2f6bacbea 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -7,7 +7,7 @@ import traceback from requests_futures.sessions import FuturesSession import lbrynet -from lbrynet import conf +from lbrynet import settings from lbrynet.core import utils session = FuturesSession() @@ -104,7 +104,7 @@ def configure_file_handler(file_name, **kwargs): def get_loggly_url(token=None, version=None): - token = token or utils.deobfuscate(conf.LOGGLY_TOKEN) + token = token or utils.deobfuscate(settings.LOGGLY_TOKEN) version = version or lbrynet.__version__ return LOGGLY_URL.format(token=token, tag='lbrynet-' + version) diff --git a/lbrynet/core/utils.py b/lbrynet/core/utils.py index f9f114233..e76f450ab 100644 --- a/lbrynet/core/utils.py +++ b/lbrynet/core/utils.py @@ -6,6 +6,8 @@ import json import yaml import datetime +from lbrynet import settings +from lbrynet.conf import ADJUSTABLE_SETTINGS from lbrynet.core.cryptoutils import get_lbry_hash_obj blobhash_length = get_lbry_hash_obj().digest_size * 2 # digest_size is in bytes, and blob hashes are hex encoded @@ -63,21 +65,20 @@ settings_encoders = { def load_settings(path): ext = os.path.splitext(path)[1] - f = open(path, 'r') - data = f.read() - f.close() + with open(path, 'r') as settings_file: + data = settings_file.read() decoder = settings_decoders.get(ext, False) assert decoder is not False, "Unknown settings format .%s" % ext return decoder(data) -def save_settings(path, settings): +def save_settings(path): + to_save = {k: v for k, v in settings.__dict__.iteritems() if k in ADJUSTABLE_SETTINGS} ext = os.path.splitext(path)[1] encoder = settings_encoders.get(ext, False) assert encoder is not False, "Unknown settings format .%s" % ext - f = open(path, 'w') - f.write(encoder(settings)) - f.close() + with open(path, 'w') as settings_file: + settings_file.write(encoder(to_save)) def today(): diff --git a/lbrynet/cryptstream/CryptBlob.py b/lbrynet/cryptstream/CryptBlob.py index aba149d9d..0de296597 100644 --- a/lbrynet/cryptstream/CryptBlob.py +++ b/lbrynet/cryptstream/CryptBlob.py @@ -1,7 +1,7 @@ import binascii import logging from Crypto.Cipher import AES -from lbrynet.conf import BLOB_SIZE +from lbrynet import settings from lbrynet.core.BlobInfo import BlobInfo @@ -67,7 +67,7 @@ class CryptStreamBlobMaker(object): self.length = 0 def write(self, data): - max_bytes_to_write = BLOB_SIZE - self.length - 1 + max_bytes_to_write = settings.BLOB_SIZE - self.length - 1 done = False if max_bytes_to_write <= len(data): num_bytes_to_write = max_bytes_to_write diff --git a/lbrynet/lbryfilemanager/EncryptedFileCreator.py b/lbrynet/lbryfilemanager/EncryptedFileCreator.py index 0acdf32cb..cec47de13 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileCreator.py +++ b/lbrynet/lbryfilemanager/EncryptedFileCreator.py @@ -7,7 +7,7 @@ import logging import os from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator -from lbrynet import conf +from lbrynet import settings from lbrynet.lbryfile.StreamDescriptor import get_sd_info from lbrynet.core.cryptoutils import get_lbry_hash_obj from twisted.protocols.basic import FileSender @@ -130,7 +130,7 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non def make_stream_desc_file(stream_hash): log.debug("creating the stream descriptor file") - descriptor_file_path = os.path.join(session.db_dir, file_name + conf.CRYPTSD_FILE_EXTENSION) + descriptor_file_path = os.path.join(session.db_dir, file_name + settings.CRYPTSD_FILE_EXTENSION) descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path) d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True) diff --git a/lbrynet/lbrylive/LiveStreamCreator.py b/lbrynet/lbrylive/LiveStreamCreator.py index 0f1f0bb88..e37e6bf46 100644 --- a/lbrynet/lbrylive/LiveStreamCreator.py +++ b/lbrynet/lbrylive/LiveStreamCreator.py @@ -6,7 +6,7 @@ from lbrynet.core.cryptoutils import get_lbry_hash_obj, get_pub_key, sign_with_p from Crypto import Random import binascii import logging -from lbrynet.conf import CRYPTSD_FILE_EXTENSION +from lbrynet import settings from twisted.internet import interfaces, defer from twisted.protocols.basic import FileSender from zope.interface import implements @@ -23,7 +23,7 @@ class LiveStreamCreator(CryptStreamCreator): self.stream_info_manager = stream_info_manager self.delete_after_num = delete_after_num self.secret_pass_phrase = secret_pass_phrase - self.file_extension = CRYPTSD_FILE_EXTENSION + self.file_extension = settings.CRYPTSD_FILE_EXTENSION self.finished_blob_hashes = {} def _save_stream(self): diff --git a/lbrynet/lbrylive/client/LiveStreamMetadataHandler.py b/lbrynet/lbrylive/client/LiveStreamMetadataHandler.py index 33a3ad381..cfef9c405 100644 --- a/lbrynet/lbrylive/client/LiveStreamMetadataHandler.py +++ b/lbrynet/lbrylive/client/LiveStreamMetadataHandler.py @@ -3,7 +3,7 @@ import logging from zope.interface import implements from twisted.internet import defer from twisted.python.failure import Failure -from lbrynet.conf import MAX_BLOB_INFOS_TO_REQUEST +from lbrynet import settings from lbrynet.core.client.ClientRequest import ClientRequest, ClientPaidRequest from lbrynet.lbrylive.LiveBlob import LiveBlobInfo from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature @@ -136,7 +136,7 @@ class LiveStreamMetadataHandler(object): if count is not None: further_blobs_request['count'] = count else: - further_blobs_request['count'] = MAX_BLOB_INFOS_TO_REQUEST + further_blobs_request['count'] = settings.MAX_BLOB_INFOS_TO_REQUEST log.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer)) r_dict = {'further_blobs': further_blobs_request} response_identifier = 'further_blobs' diff --git a/lbrynet/lbrynet_console/Console.py b/lbrynet/lbrynet_console/Console.py index 3f700222e..44a37ee90 100644 --- a/lbrynet/lbrynet_console/Console.py +++ b/lbrynet/lbrynet_console/Console.py @@ -10,13 +10,13 @@ if sys.platform == "darwin": from appdirs import user_data_dir from yapsy.PluginManager import PluginManager from twisted.internet import defer, threads, stdio, task, error -from jsonrpc.proxy import JSONRPCProxy +from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient from lbrynet.core.Session import Session from lbrynet.lbrynet_console.ConsoleControl import ConsoleControl from lbrynet.lbrynet_console.Settings import Settings from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager -from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, API_CONNECTION_STRING # , MIN_BLOB_INFO_PAYMENT_RATE +from lbrynet import settings from lbrynet.core.utils import generate_id from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.core.PaymentRateManager import PaymentRateManager @@ -209,7 +209,7 @@ class Console(): def _get_session(self): def get_default_data_rate(): d = self.settings.get_default_data_payment_rate() - d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else MIN_BLOB_DATA_PAYMENT_RATE}) + d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else settings.data_rate}) return d def get_wallet(): @@ -537,7 +537,7 @@ def launch_lbry_console(): os.mkdir(data_dir) created_data_dir = True - daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING) + daemon = LBRYAPIClient.config() try: daemon.is_running() log.info("Attempt to start lbrynet-console while lbrynet-daemon is running") diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index 79f880dae..fec8f142b 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -23,40 +23,37 @@ from twisted.internet.task import LoopingCall from txjsonrpc import jsonrpclib from jsonschema import ValidationError -from lbrynet import __version__ as lbrynet_version from lbryum.version import LBRYUM_VERSION as lbryum_version + +from lbrynet import __version__ as lbrynet_version +from lbrynet import settings as lbrynet_settings from lbrynet import analytics +from lbrynet import reflector +from lbrynet.metadata.Metadata import Metadata, verify_name_characters +from lbrynet.metadata.Fee import FeeValidator +from lbrynet.core import log_support +from lbrynet.core import utils +from lbrynet.core.utils import generate_id +from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader +from lbrynet.core.Session import Session from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory from lbrynet.core.Error import UnknownNameError, InsufficientFundsError, InvalidNameError +from lbrynet.core.PTCWallet import PTCWallet +from lbrynet.core.Wallet import LBRYcrdWallet, LBRYumWallet +from lbrynet.lbrynet_console.Settings import Settings +from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamType from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileSaverFactory, EncryptedFileOpenerFactory from lbrynet.lbryfile.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier +from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager +from lbrynet.lbryfile.EncryptedFileMetadataManager import TempEncryptedFileMetadataManager from lbrynet.lbrynet_daemon.UIManager import UIManager from lbrynet.lbrynet_daemon.Downloader import GetStream from lbrynet.lbrynet_daemon.Publisher import Publisher from lbrynet.lbrynet_daemon.ExchangeRateManager import ExchangeRateManager from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient from lbrynet.lbrynet_daemon.auth.server import AuthJSONRPCServer -from lbrynet.metadata.Metadata import Metadata, verify_name_characters -from lbrynet.core import log_support -from lbrynet.core import utils -from lbrynet.core.utils import generate_id -from lbrynet.lbrynet_console.Settings import Settings -from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS -from lbrynet.conf import KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, DEFAULT_WALLET -from lbrynet.conf import DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME -from lbrynet.conf import LOG_POST_URL, LOG_FILE_NAME, REFLECTOR_SERVERS, SEARCH_SERVERS -from lbrynet.conf import DEFAULT_SD_DOWNLOAD_TIMEOUT, DEFAULT_UI_BRANCH -from lbrynet.conf import DEFAULT_TIMEOUT -from lbrynet import conf -from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader -from lbrynet.core.Session import Session -from lbrynet.core.PTCWallet import PTCWallet -from lbrynet.core.Wallet import LBRYcrdWallet, LBRYumWallet -from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager -from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager, TempEncryptedFileMetadataManager -from lbrynet import reflector # TODO: this code snippet is everywhere. Make it go away @@ -68,7 +65,7 @@ else: if not os.path.isdir(log_dir): os.mkdir(log_dir) -lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) +lbrynet_log = os.path.join(log_dir, lbrynet_settings.LOG_FILE_NAME) log = logging.getLogger(__name__) @@ -135,225 +132,81 @@ class Daemon(AuthJSONRPCServer): LBRYnet daemon, a jsonrpc interface to lbry functions """ - def __init__(self, root, wallet_type=None): - AuthJSONRPCServer.__init__(self) + def __init__(self, root, use_authentication=lbrynet_settings.USE_AUTH_HTTP): + AuthJSONRPCServer.__init__(self, use_authentication) + reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown) + self.allowed_during_startup = ['is_running', 'is_first_run', 'get_time_behind_blockchain', 'stop', 'daemon_status', 'get_start_notice', 'version', 'get_search_servers'] - reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown) + last_version = {'last_version': {'lbrynet': lbrynet_version, 'lbryum': lbryum_version}} + lbrynet_settings.update(last_version) + self.db_dir = lbrynet_settings.DATA_DIR + self.download_directory = lbrynet_settings.download_directory + self.created_data_dir = False + if not os.path.exists(self.db_dir): + os.mkdir(self.db_dir) + self.created_data_dir = True + if lbrynet_settings.BLOBFILES_DIR == "blobfiles": + self.blobfile_dir = os.path.join(self.db_dir, "blobfiles") + else: + log.info("Using non-default blobfiles directory: %s", lbrynet_settings.BLOBFILES_DIR) + self.blobfile_dir = lbrynet_settings.BLOBFILES_DIR + + self.run_on_startup = lbrynet_settings.run_on_startup + self.data_rate = lbrynet_settings.data_rate + self.max_key_fee = lbrynet_settings.max_key_fee + self.max_upload = lbrynet_settings.max_upload + self.max_download = lbrynet_settings.max_download + self.upload_log = lbrynet_settings.upload_log + self.search_timeout = lbrynet_settings.search_timeout + self.download_timeout = lbrynet_settings.download_timeout + self.max_search_results = lbrynet_settings.max_search_results + self.run_reflector_server = lbrynet_settings.run_reflector_server + self.wallet_type = lbrynet_settings.WALLET + self.delete_blobs_on_remove = lbrynet_settings.delete_blobs_on_remove + self.peer_port = lbrynet_settings.peer_port + self.reflector_port = lbrynet_settings.reflector_port + self.dht_node_port = lbrynet_settings.dht_node_port + self.use_upnp = lbrynet_settings.use_upnp + self.start_lbrycrdd = lbrynet_settings.start_lbrycrdd + self.cache_time = lbrynet_settings.cache_time + self.startup_scripts = lbrynet_settings.startup_scripts self.startup_status = STARTUP_STAGES[0] self.startup_message = None self.announced_startup = False self.connected_to_internet = True self.connection_problem = None - self.query_handlers = {} self.git_lbrynet_version = None self.git_lbryum_version = None self.ui_version = None self.ip = None - # TODO: this is confusing to set here, and then to be reset below. - self.wallet_type = wallet_type self.first_run = None self.log_file = lbrynet_log self.current_db_revision = 1 - self.run_server = True self.session = None - self.exchange_rate_manager = ExchangeRateManager() - self.lighthouse_client = LighthouseClient() - self.waiting_on = {} - self.streams = {} - self.pending_claims = {} - self.known_dht_nodes = KNOWN_DHT_NODES self.first_run_after_update = False self.uploaded_temp_files = [] self._session_id = base58.b58encode(generate_id()) + self.lbryid = None - if os.name == "nt": - from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle - default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current) - self.db_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrynet") - try: - os.makedirs(self.db_dir) - except OSError: - if not os.path.isdir(self.db_dir): - raise - elif sys.platform == "darwin": - default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') - self.db_dir = user_data_dir("LBRY") - else: - default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') - self.db_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") - try: - if not os.path.isdir(default_download_directory): - os.mkdir(default_download_directory) - except: - log.info("Couldn't make download directory, using home") - default_download_directory = os.path.expanduser("~") - - old_conf_path = os.path.join(self.db_dir, 'daemon_settings.json') - self.daemon_conf = os.path.join(self.db_dir, 'daemon_settings.yml') - - if os.path.isfile(old_conf_path): - log.info("Migrating .json config file to .yml") - tmp_settings = utils.load_settings(old_conf_path) - utils.save_settings(self.daemon_conf, tmp_settings) - try: - os.remove(old_conf_path) - log.info("Cleaned up old config file") - except: - log.warning("Failed to remove old config file") - - self.default_settings = { - 'run_on_startup': False, - 'data_rate': MIN_BLOB_DATA_PAYMENT_RATE, - 'max_key_fee': DEFAULT_MAX_KEY_FEE, - 'download_directory': default_download_directory, - 'max_upload': 0.0, - 'max_download': 0.0, - 'upload_log': True, - 'search_timeout': DEFAULT_SEARCH_TIMEOUT, - 'download_timeout': DEFAULT_TIMEOUT, - 'max_search_results': DEFAULT_MAX_SEARCH_RESULTS, - 'wallet_type': DEFAULT_WALLET, - 'delete_blobs_on_remove': True, - 'peer_port': 3333, - 'dht_node_port': 4444, - 'reflector_port': 5566, - 'use_upnp': True, - 'start_lbrycrdd': True, - 'requested_first_run_credits': False, - 'run_reflector_server': False, - 'cache_time': DEFAULT_CACHE_TIME, - 'startup_scripts': [], - 'last_version': {'lbrynet': lbrynet_version, 'lbryum': lbryum_version} - } - - if os.path.isfile(self.daemon_conf): - loaded_settings = utils.load_settings(self.daemon_conf) - missing_settings = {} - removed_settings = {} - for k in self.default_settings.keys(): - if k not in loaded_settings.keys(): - missing_settings[k] = self.default_settings[k] - for k in loaded_settings.keys(): - if not k in self.default_settings.keys(): - log.info("Removing unused setting: " + k + " with value: " + str(loaded_settings[k])) - removed_settings[k] = loaded_settings[k] - del loaded_settings[k] - for k in missing_settings.keys(): - log.info("Adding missing setting: " + k + " with default value: " + str(missing_settings[k])) - loaded_settings[k] = missing_settings[k] - if loaded_settings['wallet_type'] != self.wallet_type and self.wallet_type: - loaded_settings['wallet_type'] = self.wallet_type - - if missing_settings or removed_settings: - log.info("Updated and loaded lbrynet-daemon configuration") - else: - log.info("Loaded lbrynet-daemon configuration") - self.session_settings = loaded_settings - else: - missing_settings = self.default_settings - log.info("Writing default settings : " + json.dumps(self.default_settings) + " --> " + str(self.daemon_conf)) - self.session_settings = self.default_settings - - if 'last_version' in missing_settings.keys(): - self.session_settings['last_version'] = None - - if self.session_settings['last_version'] != self.default_settings['last_version']: - self.session_settings['last_version'] = self.default_settings['last_version'] - self.first_run_after_update = True - log.info("First run after update") - log.info("lbrynet %s --> %s" % (self.session_settings['last_version']['lbrynet'], self.default_settings['last_version']['lbrynet'])) - log.info("lbryum %s --> %s" % (self.session_settings['last_version']['lbryum'], self.default_settings['last_version']['lbryum'])) - if "0.4.5" == self.default_settings['last_version']['lbrynet']: - log.info("Lowering name cache time") - self.session_settings['cache_time'] = DEFAULT_CACHE_TIME - - utils.save_settings(self.daemon_conf, self.session_settings) - - self.run_on_startup = self.session_settings['run_on_startup'] - self.data_rate = self.session_settings['data_rate'] - self.max_key_fee = self.session_settings['max_key_fee'] - self.download_directory = self.session_settings['download_directory'] - self.max_upload = self.session_settings['max_upload'] - self.max_download = self.session_settings['max_download'] - self.upload_log = self.session_settings['upload_log'] - self.search_timeout = self.session_settings['search_timeout'] - self.download_timeout = self.session_settings['download_timeout'] - self.max_search_results = self.session_settings['max_search_results'] - self.run_reflector_server = self.session_settings['run_reflector_server'] - #### - # - # Ignore the saved wallet type. Some users will have their wallet type - # saved as lbrycrd and we want wallets to be lbryum unless explicitly - # set on the command line to be lbrycrd. - # - # if self.session_settings['wallet_type'] in WALLET_TYPES and not wallet_type: - # self.wallet_type = self.session_settings['wallet_type'] - # log.info("Using wallet type %s from config" % self.wallet_type) - # else: - # self.wallet_type = wallet_type - # self.session_settings['wallet_type'] = wallet_type - # log.info("Using wallet type %s specified from command line" % self.wallet_type) - # - # Instead, if wallet is not set on the command line, default to the default wallet - # - if wallet_type: - log.info("Using wallet type %s specified from command line", wallet_type) - self.wallet_type = wallet_type - else: - log.info("Using the default wallet type %s", DEFAULT_WALLET) - self.wallet_type = DEFAULT_WALLET - if self.wallet_type not in conf.WALLET_TYPES: - raise ValueError('Wallet Type {} is not valid'.format(wallet_type)) - # - #### - self.delete_blobs_on_remove = self.session_settings['delete_blobs_on_remove'] - self.peer_port = self.session_settings['peer_port'] - self.reflector_port = self.session_settings['reflector_port'] - self.dht_node_port = self.session_settings['dht_node_port'] - self.use_upnp = self.session_settings['use_upnp'] - self.start_lbrycrdd = self.session_settings['start_lbrycrdd'] - self.requested_first_run_credits = self.session_settings['requested_first_run_credits'] - self.cache_time = self.session_settings['cache_time'] - self.startup_scripts = self.session_settings['startup_scripts'] - - if os.path.isfile(os.path.join(self.db_dir, "stream_info_cache.json")): - f = open(os.path.join(self.db_dir, "stream_info_cache.json"), "r") - self.name_cache = json.loads(f.read()) - f.close() - log.info("Loaded claim info cache") - else: - self.name_cache = {} - - self.set_wallet_attributes() - - if os.name != 'nt': - # TODO: are we still using this? - lbrycrdd_path_conf = os.path.join(os.path.expanduser("~"), ".lbrycrddpath.conf") - if not os.path.isfile(lbrycrdd_path_conf): - f = open(lbrycrdd_path_conf, "w") - f.write(str(self.lbrycrdd_path)) - f.close() - - self.created_data_dir = False - if not os.path.exists(self.db_dir): - os.mkdir(self.db_dir) - self.created_data_dir = True - - self.blobfile_dir = os.path.join(self.db_dir, "blobfiles") self.wallet_user = None self.wallet_password = None - + self.query_handlers = {} + self.waiting_on = {} + self.streams = {} + self.pending_claims = {} + self.name_cache = {} + self.set_wallet_attributes() self.internet_connection_checker = LoopingCall(self._check_network_connection) self.version_checker = LoopingCall(self._check_remote_versions) self.connection_problem_checker = LoopingCall(self._check_connection_problems) self.pending_claim_checker = LoopingCall(self._check_pending_claims) self.send_heartbeat = LoopingCall(self._send_heartbeat) - # self.lbrynet_connection_checker = LoopingCall(self._check_lbrynet_connection) - + self.exchange_rate_manager = ExchangeRateManager() + self.lighthouse_client = LighthouseClient() self.sd_identifier = StreamDescriptorIdentifier() self.stream_info_manager = TempEncryptedFileMetadataManager() self.settings = Settings(self.db_dir) @@ -362,16 +215,6 @@ class Daemon(AuthJSONRPCServer): self.lbry_file_metadata_manager = None self.lbry_file_manager = None - if self.wallet_type == "lbrycrd": - if os.path.isfile(self.lbrycrd_conf): - log.info("Using lbrycrd.conf found at " + self.lbrycrd_conf) - else: - log.info("No lbrycrd.conf found at " + self.lbrycrd_conf + ". Generating now...") - password = "".join(random.SystemRandom().choice(string.ascii_letters + string.digits + "_") for i in range(20)) - with open(self.lbrycrd_conf, 'w') as f: - f.write("rpcuser=rpcuser\n") - f.write("rpcpassword=" + password) - log.info("Done writing lbrycrd.conf") @AuthJSONRPCServer.subhandler def _exclude_lbrycrd_only_commands_from_lbryum_session(self, request): @@ -402,7 +245,7 @@ class Daemon(AuthJSONRPCServer): self.lbrycrd_conf = os.path.join(self.wallet_dir, "lbrycrd.conf") self.wallet_conf = os.path.join(self.wallet_dir, "lbrycrd.conf") - def setup(self, branch=DEFAULT_UI_BRANCH, user_specified=False, branch_specified=False, host_ui=True): + def setup(self): def _log_starting_vals(): log.info("Starting balance: " + str(self.session.wallet.wallet_balance)) return defer.succeed(None) @@ -440,18 +283,16 @@ class Daemon(AuthJSONRPCServer): self.connection_problem_checker.start(1) self.exchange_rate_manager.start() - if host_ui: - self.lbry_ui_manager.update_checker.start(1800, now=False) - d = defer.Deferred() - if host_ui: - d.addCallback(lambda _: self.lbry_ui_manager.setup(branch=branch, - user_specified=user_specified, - branch_specified=branch_specified)) + + if lbrynet_settings.host_ui: + self.lbry_ui_manager.update_checker.start(1800, now=False) + d.addCallback(lambda _: self.lbry_ui_manager.setup()) d.addCallback(lambda _: self._initial_setup()) d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory)) d.addCallback(lambda _: self._check_db_migration()) d.addCallback(lambda _: self._get_settings()) + d.addCallback(lambda _: self._load_caches()) d.addCallback(lambda _: self._set_events()) d.addCallback(lambda _: self._get_session()) d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier)) @@ -507,6 +348,12 @@ class Daemon(AuthJSONRPCServer): d = _log_platform() return d + def _load_caches(self): + if os.path.isfile(os.path.join(self.db_dir, "stream_info_cache.json")): + with open(os.path.join(self.db_dir, "stream_info_cache.json"), "r") as stream_info_cache: + self.name_cache = json.loads(stream_info_cache.read()) + log.info("Loaded claim info cache") + def _set_events(self): context = analytics.make_context(self._get_platform(), self.wallet_type) self._events = analytics.Events(context, base58.b58encode(self.lbryid), self._session_id) @@ -736,7 +583,7 @@ class Daemon(AuthJSONRPCServer): 'type': "%s-%s" % (lm, log_type) if log_type else lm, 'log': log_contents } - requests.post(LOG_POST_URL, params) + requests.post(lbrynet_settings.LOG_POST_URL, params) return defer.succeed(None) else: @@ -779,87 +626,43 @@ class Daemon(AuthJSONRPCServer): return d def _update_settings(self, settings): - for k in settings.keys(): - if k == 'run_on_startup': - if type(settings['run_on_startup']) is bool: - self.session_settings['run_on_startup'] = settings['run_on_startup'] - else: - return defer.fail() - elif k == 'data_rate': - if type(settings['data_rate']) is float: - self.session_settings['data_rate'] = settings['data_rate'] - elif type(settings['data_rate']) is int: - self.session_settings['data_rate'] = float(settings['data_rate']) - else: - return defer.fail() - elif k == 'max_key_fee': - if type(settings['max_key_fee']) is float: - self.session_settings['max_key_fee'] = settings['max_key_fee'] - elif type(settings['max_key_fee']) is int: - self.session_settings['max_key_fee'] = float(settings['max_key_fee']) - else: - return defer.fail() - elif k == 'download_directory': - if type(settings['download_directory']) is unicode: - if os.path.isdir(settings['download_directory']): - self.session_settings['download_directory'] = settings['download_directory'] - else: - pass - else: - return defer.fail() - elif k == 'max_upload': - if type(settings['max_upload']) is float: - self.session_settings['max_upload'] = settings['max_upload'] - elif type(settings['max_upload']) is int: - self.session_settings['max_upload'] = float(settings['max_upload']) - else: - return defer.fail() - elif k == 'max_download': - if type(settings['max_download']) is float: - self.session_settings['max_download'] = settings['max_download'] - if type(settings['max_download']) is int: - self.session_settings['max_download'] = float(settings['max_download']) - else: - return defer.fail() - elif k == 'upload_log': - if type(settings['upload_log']) is bool: - self.session_settings['upload_log'] = settings['upload_log'] - else: - return defer.fail() - elif k == 'download_timeout': - if type(settings['download_timeout']) is int: - self.session_settings['download_timeout'] = settings['download_timeout'] - elif type(settings['download_timeout']) is float: - self.session_settings['download_timeout'] = int(settings['download_timeout']) - else: - return defer.fail() - elif k == 'search_timeout': - if type(settings['search_timeout']) is float: - self.session_settings['search_timeout'] = settings['search_timeout'] - elif type(settings['search_timeout']) is int: - self.session_settings['search_timeout'] = float(settings['search_timeout']) - else: - return defer.fail() - elif k == 'cache_time': - if type(settings['cache_time']) is int: - self.session_settings['cache_time'] = settings['cache_time'] - elif type(settings['cache_time']) is float: - self.session_settings['cache_time'] = int(settings['cache_time']) - else: - return defer.fail() + setting_types = { + 'run_on_startup': bool, + 'data_rate': float, + 'max_key_fee': float, + 'download_directory': str, + 'max_upload': float, + 'max_download': float, + 'upload_log': bool, + 'download_timeout': int, + 'search_timeout': float, + 'cache_time': int + } - self.run_on_startup = self.session_settings['run_on_startup'] - self.data_rate = self.session_settings['data_rate'] - self.max_key_fee = self.session_settings['max_key_fee'] - self.download_directory = self.session_settings['download_directory'] - self.max_upload = self.session_settings['max_upload'] - self.max_download = self.session_settings['max_download'] - self.upload_log = self.session_settings['upload_log'] - self.download_timeout = self.session_settings['download_timeout'] - self.search_timeout = self.session_settings['search_timeout'] - self.cache_time = self.session_settings['cache_time'] + for key, setting_type in setting_types.iteritems(): + if key in settings: + if isinstance(settings[key], setting_type): + lbrynet_settings.update({key: settings[key]}) + elif key == "max_key_fee" and isinstance(FeeValidator(settings[key]).amount, setting_type): + lbrynet_settings.update({key: settings[key]}) + else: + try: + converted = setting_type(settings[key]) + lbrynet_settings.update({key: converted}) + except Exception as err: + log.warning(err.message) + log.warning("error converting setting '%s' to type %s", key, setting_type) - utils.save_settings(self.daemon_conf, self.session_settings) + self.run_on_startup = lbrynet_settings.run_on_startup + self.data_rate = lbrynet_settings.data_rate + self.max_key_fee = lbrynet_settings.max_key_fee + self.download_directory = lbrynet_settings.download_directory + self.max_upload = lbrynet_settings.max_upload + self.max_download = lbrynet_settings.max_download + self.upload_log = lbrynet_settings.upload_log + self.download_timeout = lbrynet_settings.download_timeout + self.search_timeout = lbrynet_settings.search_timeout + self.cache_time = lbrynet_settings.cache_time return defer.succeed(True) @@ -946,7 +749,7 @@ class Daemon(AuthJSONRPCServer): def get_default_data_rate(): d = self.settings.get_default_data_payment_rate() d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else - MIN_BLOB_DATA_PAYMENT_RATE}) + lbrynet_settings.data_rate}) return d def get_wallet(): @@ -960,8 +763,8 @@ class Daemon(AuthJSONRPCServer): elif self.wallet_type == "lbryum": log.info("Using lbryum wallet") config = {'auto-connect': True} - if conf.LBRYUM_WALLET_DIR: - config['lbryum_path'] = conf.LBRYUM_WALLET_DIR + if lbrynet_settings.LBRYUM_WALLET_DIR: + config['lbryum_path'] = lbrynet_settings.LBRYUM_WALLET_DIR d = defer.succeed(LBRYumWallet(self.db_dir, config)) elif self.wallet_type == "ptc": log.info("Using PTC wallet") @@ -984,7 +787,7 @@ class Daemon(AuthJSONRPCServer): def create_session(results): self.session = Session(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid, blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port, - known_dht_nodes=self.known_dht_nodes, peer_port=self.peer_port, + known_dht_nodes=lbrynet_settings.known_dht_nodes, peer_port=self.peer_port, use_upnp=self.use_upnp, wallet=results['wallet']) self.startup_status = STARTUP_STAGES[2] @@ -1006,7 +809,7 @@ class Daemon(AuthJSONRPCServer): self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_opener_factory) return defer.succeed(None) - def _download_sd_blob(self, sd_hash, timeout=DEFAULT_SD_DOWNLOAD_TIMEOUT): + def _download_sd_blob(self, sd_hash, timeout=lbrynet_settings.sd_download_timeout): def cb(result): if not r.called: r.callback(result) @@ -1024,7 +827,7 @@ class Daemon(AuthJSONRPCServer): return r - def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None, + def _download_name(self, name, timeout=lbrynet_settings.download_timeout, download_directory=None, file_name=None, stream_info=None, wait_for_write=True): """ Add a lbry file to the file manager, start the download, and return the new lbry file. @@ -1275,7 +1078,7 @@ class Daemon(AuthJSONRPCServer): log.info("Reflecting stream: %s" % stream_hash) - reflector_server = random.choice(REFLECTOR_SERVERS) + reflector_server = random.choice(lbrynet_settings.REFLECTOR_SERVERS) reflector_address, reflector_port = reflector_server[0], reflector_server[1] log.info("Start reflector client") factory = reflector.ClientFactory( @@ -1294,7 +1097,7 @@ class Daemon(AuthJSONRPCServer): log.info("Reflecting %i blobs" % len(blob_hashes)) - reflector_server = random.choice(REFLECTOR_SERVERS) + reflector_server = random.choice(lbrynet_settings.REFLECTOR_SERVERS) reflector_address, reflector_port = reflector_server[0], reflector_server[1] log.info("Start reflector client") factory = reflector.BlobClientFactory( @@ -1317,9 +1120,9 @@ class Daemon(AuthJSONRPCServer): log.info("Removing one time startup scripts") remaining_scripts = [s for s in self.startup_scripts if 'run_once' not in s.keys()] startup_scripts = self.startup_scripts - self.startup_scripts = self.session_settings['startup_scripts'] = remaining_scripts + self.startup_scripts = lbrynet_settings['startup_scripts'] = remaining_scripts - utils.save_settings(self.daemon_conf, self.session_settings) + utils.save_settings(self.daemon_conf, lbrynet_settings) for script in startup_scripts: if script['script_name'] == 'migrateto025': @@ -1469,7 +1272,6 @@ class Daemon(AuthJSONRPCServer): log.info("Get version info: " + json.dumps(msg)) return self._render_response(msg, OK_CODE) - @AuthJSONRPCServer.auth_required def jsonrpc_get_settings(self): """ Get lbrynet daemon settings @@ -1496,9 +1298,9 @@ class Daemon(AuthJSONRPCServer): """ log.info("Get daemon settings") - return self._render_response(self.session_settings, OK_CODE) + return self._render_response(lbrynet_settings.configurable_settings, OK_CODE) - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_set_settings(self, p): """ Set lbrynet daemon settings @@ -1517,12 +1319,12 @@ class Daemon(AuthJSONRPCServer): """ def _log_settings_change(): - log.info("Set daemon settings to " + json.dumps(self.session_settings)) + log.info("Set daemon settings to " + json.dumps(lbrynet_settings.configurable_settings)) d = self._update_settings(p) d.addErrback(lambda err: log.info(err.getTraceback())) d.addCallback(lambda _: _log_settings_change()) - d.addCallback(lambda _: self._render_response(self.session_settings, OK_CODE)) + d.addCallback(lambda _: self._render_response(lbrynet_settings.configurable_settings, OK_CODE)) return d @@ -1550,7 +1352,6 @@ class Daemon(AuthJSONRPCServer): else: return self._render_response(self.jsonrpc_help.__doc__, OK_CODE) - @AuthJSONRPCServer.auth_required def jsonrpc_get_balance(self): """ Get balance @@ -1583,7 +1384,6 @@ class Daemon(AuthJSONRPCServer): return self._render_response("Shutting down", OK_CODE) - @AuthJSONRPCServer.auth_required def jsonrpc_get_lbry_files(self): """ Get LBRY files @@ -1610,7 +1410,6 @@ class Daemon(AuthJSONRPCServer): return d - @AuthJSONRPCServer.auth_required def jsonrpc_get_lbry_file(self, p): """ Get lbry file @@ -1661,6 +1460,7 @@ class Daemon(AuthJSONRPCServer): d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) return d + @AuthJSONRPCServer.auth_required def jsonrpc_get_my_claim(self, p): """ Return existing claim for a given name @@ -1700,7 +1500,6 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required def _process_get_parameters(self, p): """Extract info from input parameters and fill in default values for `get` call.""" # TODO: this process can be abstracted s.t. each method @@ -1722,7 +1521,7 @@ class Daemon(AuthJSONRPCServer): name=name ) - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get(self, p): """Download stream from a LBRY uri. @@ -1753,7 +1552,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda message: self._render_response(message, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_stop_lbry_file(self, p): """ Stop lbry file @@ -1779,7 +1578,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_start_lbry_file(self, p): """ Stop lbry file @@ -1820,7 +1619,6 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required def jsonrpc_search_nametrie(self, p): """ Search the nametrie for claims @@ -1857,7 +1655,7 @@ class Daemon(AuthJSONRPCServer): return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_delete_lbry_file(self, p): """ Delete a lbry file @@ -1887,7 +1685,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_publish(self, p): """ Make a new name claim and publish associated data to lbrynet @@ -1964,7 +1762,7 @@ class Daemon(AuthJSONRPCServer): return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_abandon_claim(self, p): """ Abandon a name and reclaim credits from the claim @@ -1991,7 +1789,7 @@ class Daemon(AuthJSONRPCServer): return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_abandon_name(self, p): """ DEPRECIATED, use abandon_claim @@ -2004,7 +1802,7 @@ class Daemon(AuthJSONRPCServer): return self.jsonrpc_abandon_claim(p) - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_support_claim(self, p): """ Support a name claim @@ -2024,7 +1822,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_name_claims(self): """ Get my name claims @@ -2063,7 +1861,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_transaction_history(self): """ Get transaction history @@ -2094,7 +1892,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_address_is_mine(self, p): """ Checks if an address is associated with the current wallet. @@ -2112,7 +1910,7 @@ class Daemon(AuthJSONRPCServer): return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_public_key_from_wallet(self, p): """ Get public key from wallet address @@ -2150,7 +1948,7 @@ class Daemon(AuthJSONRPCServer): return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_get_new_address(self): """ Generate a new wallet address @@ -2170,7 +1968,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda address: self._render_response(address, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_send_amount_to_address(self, p): """ Send credits to an address @@ -2250,7 +2048,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_download_descriptor(self, p): """ Download and return a sd blob @@ -2261,7 +2059,7 @@ class Daemon(AuthJSONRPCServer): sd blob, dict """ sd_hash = p['sd_hash'] - timeout = p.get('timeout', DEFAULT_SD_DOWNLOAD_TIMEOUT) + timeout = p.get('timeout', lbrynet_settings.sd_download_timeout) d = self._download_sd_blob(sd_hash, timeout) d.addCallbacks(lambda r: self._render_response(r, OK_CODE), lambda _: self._render_response(False, OK_CODE)) @@ -2282,7 +2080,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_set_miner(self, p): """ Start of stop the miner, function only available when lbrycrd is set as the wallet @@ -2302,7 +2100,6 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required def jsonrpc_get_miner_status(self): """ Get status of miner @@ -2372,7 +2169,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_configure_ui(self, p): """ Configure the UI being hosted @@ -2397,7 +2194,7 @@ class Daemon(AuthJSONRPCServer): return d - @AuthJSONRPCServer.auth_required + @AuthJSONRPCServer.auth_required def jsonrpc_reveal(self, p): """ Reveal a file or directory in file browser @@ -2417,7 +2214,6 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d - @AuthJSONRPCServer.auth_required def jsonrpc_get_peers_for_hash(self, p): """ Get peers for blob hash @@ -2435,7 +2231,6 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required def jsonrpc_announce_all_blobs_to_dht(self): """ Announce all blobs to the dht @@ -2450,7 +2245,6 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda _: self._render_response("Announced", OK_CODE)) return d - @AuthJSONRPCServer.auth_required def jsonrpc_reflect(self, p): """ Reflect a stream @@ -2467,7 +2261,6 @@ class Daemon(AuthJSONRPCServer): d.addCallbacks(lambda _: self._render_response(True, OK_CODE), lambda err: self._render_response(err.getTraceback(), OK_CODE)) return d - @AuthJSONRPCServer.auth_required def jsonrpc_get_blob_hashes(self): """ Returns all blob hashes @@ -2482,7 +2275,6 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - @AuthJSONRPCServer.auth_required def jsonrpc_reflect_all_blobs(self): """ Reflects all saved blobs @@ -2507,7 +2299,7 @@ class Daemon(AuthJSONRPCServer): List of address:port """ - d = self._render_response(SEARCH_SERVERS, OK_CODE) + d = self._render_response(lbrynet_settings.search_servers, OK_CODE) return d def jsonrpc_get_mean_availability(self): @@ -2557,6 +2349,12 @@ class Daemon(AuthJSONRPCServer): return d + @AuthJSONRPCServer.auth_required + def jsonrpc_test_api_authentication(self): + if self._use_authentication: + return self._render_response(True, OK_CODE) + return self._render_response("Not using authentication", OK_CODE) + def get_lbrynet_version_from_github(): """Return the latest released version from github.""" @@ -2613,7 +2411,7 @@ def get_darwin_lbrycrdd_path(): class _DownloadNameHelper(object): - def __init__(self, daemon, name, timeout=DEFAULT_TIMEOUT, download_directory=None, + def __init__(self, daemon, name, timeout=lbrynet_settings.download_timeout, download_directory=None, file_name=None, wait_for_write=True): self.daemon = daemon self.name = name diff --git a/lbrynet/lbrynet_daemon/DaemonControl.py b/lbrynet/lbrynet_daemon/DaemonControl.py index 42e0f03af..65bb54ceb 100644 --- a/lbrynet/lbrynet_daemon/DaemonControl.py +++ b/lbrynet/lbrynet_daemon/DaemonControl.py @@ -11,19 +11,19 @@ from twisted.cred import portal from jsonrpc.proxy import JSONRPCProxy -from lbrynet.core import log_support +from lbrynet.core import log_support, utils from lbrynet.lbrynet_daemon.auth.auth import PasswordChecker, HttpPasswordRealm from lbrynet.lbrynet_daemon.auth.util import initialize_api_key_file from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest -from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_PORT -from lbrynet.conf import UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME -from lbrynet.conf import DATA_DIR as log_dir +from lbrynet import settings + +log_dir = settings.DATA_DIR if not os.path.isdir(log_dir): os.mkdir(log_dir) -lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) +lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) @@ -52,7 +52,7 @@ def stop(): log.info("Attempt to shut down lbrynet-daemon from command line when daemon isn't running") d = defer.Deferred(None) - d.addCallback(lambda _: JSONRPCProxy.from_url(API_CONNECTION_STRING).stop()) + d.addCallback(lambda _: JSONRPCProxy.from_url(settings.API_CONNECTION_STRING).stop()) d.addCallbacks(lambda _: _disp_shutdown(), lambda _: _disp_not_running()) d.callback(None) @@ -62,18 +62,37 @@ def start(): parser.add_argument("--wallet", help="lbrycrd or lbryum, default lbryum", type=str, - default='') + default='lbryum') + parser.add_argument("--ui", help="path to custom UI folder", default=None) + parser.add_argument("--branch", - help="Branch of lbry-web-ui repo to use, defaults on master") - parser.add_argument('--no-launch', dest='launchui', action="store_false") - parser.add_argument('--log-to-console', dest='logtoconsole', action="store_true") - parser.add_argument('--quiet', dest='quiet', action="store_true") - parser.add_argument('--verbose', action='store_true', + help="Branch of lbry-web-ui repo to use, defaults on master", + default=settings.UI_BRANCH) + + parser.add_argument("--http-auth", + dest="useauth", + action="store_true") + + parser.add_argument('--no-launch', + dest='launchui', + action="store_false") + + parser.add_argument('--log-to-console', + dest='logtoconsole', + action="store_true") + + parser.add_argument('--quiet', + dest='quiet', + action="store_true") + + parser.add_argument('--verbose', + action='store_true', help='enable more debug output for the console') - parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False) + + parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False, useauth=False) args = parser.parse_args() log_support.configure_file_handler(lbrynet_log) @@ -84,13 +103,26 @@ def start(): if not args.verbose: log_support.disable_noisy_loggers() + to_pass = {} + settings_path = os.path.join(settings.DATA_DIR, "daemon_settings.yml") + if os.path.isfile(settings_path): + to_pass.update(utils.load_settings(settings_path)) + log.info("Loaded settings file") + if args.ui: + to_pass.update({'local_ui_path': args.ui}) + if args.branch: + to_pass.update({'UI_BRANCH': args.branch}) + to_pass.update({'USE_AUTH_HTTP': args.useauth}) + to_pass.update({'WALLET': args.wallet}) + settings.update(to_pass) + try: - JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running() + JSONRPCProxy.from_url(settings.API_CONNECTION_STRING).is_running() log.info("lbrynet-daemon is already running") if not args.logtoconsole: print "lbrynet-daemon is already running" if args.launchui: - webbrowser.open(UI_ADDRESS) + webbrowser.open(settings.UI_ADDRESS) return except: pass @@ -100,30 +132,33 @@ def start(): if not args.logtoconsole and not args.quiet: print "Starting lbrynet-daemon from command line" print "To view activity, view the log file here: " + lbrynet_log - print "Web UI is available at http://%s:%i" % (API_INTERFACE, API_PORT) - print "JSONRPC API is available at " + API_CONNECTION_STRING + print "Web UI is available at http://%s:%i" % (settings.API_INTERFACE, settings.API_PORT) + print "JSONRPC API is available at " + settings.API_CONNECTION_STRING print "To quit press ctrl-c or call 'stop' via the API" if test_internet_connection(): lbry = DaemonServer() - d = lbry.start(branch=args.branch if args.branch else DEFAULT_UI_BRANCH, - user_specified=args.ui, - wallet=args.wallet, - branch_specified=True if args.branch else False) + d = lbry.start(args.useauth) if args.launchui: - d.addCallback(lambda _: webbrowser.open(UI_ADDRESS)) + d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS)) - pw_path = os.path.join(log_dir, ".api_keys") - initialize_api_key_file(pw_path) - checker = PasswordChecker.load_file(pw_path) - realm = HttpPasswordRealm(lbry.root) - portal_to_realm = portal.Portal(realm, [checker, ]) - factory = guard.BasicCredentialFactory('Login to lbrynet api') - protected_resource = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ]) - lbrynet_server = server.Site(protected_resource) + if args.useauth: + log.info("Using authenticated API") + pw_path = os.path.join(settings.DATA_DIR, ".api_keys") + initialize_api_key_file(pw_path) + checker = PasswordChecker.load_file(pw_path) + realm = HttpPasswordRealm(lbry.root) + portal_to_realm = portal.Portal(realm, [checker, ]) + factory = guard.BasicCredentialFactory('Login to lbrynet api') + _lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ]) + else: + log.info("Using non-authenticated API") + _lbrynet_server = server.Site(lbry.root) + + lbrynet_server = server.Site(_lbrynet_server) lbrynet_server.requestFactory = DaemonRequest - reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE) + reactor.listenTCP(settings.API_PORT, lbrynet_server, interface=settings.API_INTERFACE) reactor.run() if not args.logtoconsole and not args.quiet: diff --git a/lbrynet/lbrynet_daemon/DaemonServer.py b/lbrynet/lbrynet_daemon/DaemonServer.py index 39268f5bd..94235f7fa 100644 --- a/lbrynet/lbrynet_daemon/DaemonServer.py +++ b/lbrynet/lbrynet_daemon/DaemonServer.py @@ -6,7 +6,7 @@ from appdirs import user_data_dir from twisted.internet import defer from lbrynet.lbrynet_daemon.Daemon import Daemon from lbrynet.lbrynet_daemon.Resources import LBRYindex, HostedEncryptedFile, EncryptedFileUpload -from lbrynet.conf import API_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME +from lbrynet import settings # TODO: omg, this code is essentially duplicated in Daemon @@ -17,20 +17,20 @@ else: if not os.path.isdir(data_dir): os.mkdir(data_dir) -lbrynet_log = os.path.join(data_dir, LOG_FILE_NAME) +lbrynet_log = os.path.join(data_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) class DaemonServer(object): - def _setup_server(self, wallet): + def _setup_server(self, use_authentication): self.root = LBRYindex(os.path.join(os.path.join(data_dir, "lbry-ui"), "active")) - self._api = Daemon(self.root, wallet_type=wallet) + self._api = Daemon(self.root, use_authentication=use_authentication) self.root.putChild("view", HostedEncryptedFile(self._api)) self.root.putChild("upload", EncryptedFileUpload(self._api)) - self.root.putChild(API_ADDRESS, self._api) + self.root.putChild(settings.API_ADDRESS, self._api) return defer.succeed(True) - def start(self, branch=DEFAULT_UI_BRANCH, user_specified=False, branch_specified=False, wallet=None): - d = self._setup_server(wallet) - d.addCallback(lambda _: self._api.setup(branch, user_specified, branch_specified)) + def start(self, use_authentication): + d = self._setup_server(use_authentication) + d.addCallback(lambda _: self._api.setup()) return d diff --git a/lbrynet/lbrynet_daemon/Downloader.py b/lbrynet/lbrynet_daemon/Downloader.py index f4056bc9e..9b34e4476 100644 --- a/lbrynet/lbrynet_daemon/Downloader.py +++ b/lbrynet/lbrynet_daemon/Downloader.py @@ -12,7 +12,7 @@ from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed from lbrynet.core.StreamDescriptor import download_sd_blob from lbrynet.metadata.Fee import FeeValidator from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory -from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME +from lbrynet import settings INITIALIZING_CODE = 'initializing' DOWNLOAD_METADATA_CODE = 'downloading_metadata' @@ -35,13 +35,13 @@ else: if not os.path.isdir(log_dir): os.mkdir(log_dir) -lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) +lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) class GetStream(object): def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager, - max_key_fee, data_rate=0.5, timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None): + max_key_fee, data_rate=0.5, timeout=settings.download_timeout, download_directory=None, file_name=None): self.wallet = wallet self.resolved_name = None self.description = None diff --git a/lbrynet/lbrynet_daemon/Lighthouse.py b/lbrynet/lbrynet_daemon/Lighthouse.py index 0c9dadb4c..3e5419e9e 100644 --- a/lbrynet/lbrynet_daemon/Lighthouse.py +++ b/lbrynet/lbrynet_daemon/Lighthouse.py @@ -1,14 +1,14 @@ import logging import random from txjsonrpc.web.jsonrpc import Proxy -from lbrynet.conf import SEARCH_SERVERS +from lbrynet import settings log = logging.getLogger(__name__) class LighthouseClient(object): def __init__(self, servers=None): - self.servers = servers or SEARCH_SERVERS + self.servers = servers or settings.search_servers def _get_random_server(self): return Proxy(random.choice(self.servers)) diff --git a/lbrynet/lbrynet_daemon/Publisher.py b/lbrynet/lbrynet_daemon/Publisher.py index 1836befbf..8362ea864 100644 --- a/lbrynet/lbrynet_daemon/Publisher.py +++ b/lbrynet/lbrynet_daemon/Publisher.py @@ -12,7 +12,7 @@ from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob from lbrynet.metadata.Metadata import Metadata from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloader from lbrynet import reflector -from lbrynet.conf import LOG_FILE_NAME, REFLECTOR_SERVERS +from lbrynet import settings from twisted.internet import threads, defer, reactor if sys.platform != "darwin": @@ -23,7 +23,7 @@ else: if not os.path.isdir(log_dir): os.mkdir(log_dir) -lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) +lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) @@ -41,7 +41,7 @@ class Publisher(object): self.lbry_file = None self.txid = None self.stream_hash = None - reflector_server = random.choice(REFLECTOR_SERVERS) + reflector_server = random.choice(settings.REFLECTOR_SERVERS) self.reflector_server, self.reflector_port = reflector_server[0], reflector_server[1] self.metadata = {} @@ -74,7 +74,7 @@ class Publisher(object): return d def start_reflector(self): - reflector_server = random.choice(REFLECTOR_SERVERS) + reflector_server = random.choice(settings.REFLECTOR_SERVERS) reflector_address, reflector_port = reflector_server[0], reflector_server[1] log.info("Reflecting new publication") factory = reflector.ClientFactory( diff --git a/lbrynet/lbrynet_daemon/Resources.py b/lbrynet/lbrynet_daemon/Resources.py index ef994efd5..02f7a4cbb 100644 --- a/lbrynet/lbrynet_daemon/Resources.py +++ b/lbrynet/lbrynet_daemon/Resources.py @@ -10,7 +10,7 @@ from appdirs import user_data_dir from twisted.web import server, static, resource from twisted.internet import defer, error -from lbrynet.conf import UI_ADDRESS +from lbrynet import settings from lbrynet.lbrynet_daemon.FileStreamer import EncryptedFileStreamer # TODO: omg, this code is essentially duplicated in Daemon @@ -80,10 +80,10 @@ class HostedEncryptedFile(resource.Resource): d = self._api._download_name(request.args['name'][0]) d.addCallback(lambda stream: self._make_stream_producer(request, stream)) elif request.args['name'][0] in self._api.waiting_on.keys(): - request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0]) + request.redirect(settings.UI_ADDRESS + "/?watch=" + request.args['name'][0]) request.finish() else: - request.redirect(UI_ADDRESS) + request.redirect(settings.UI_ADDRESS) request.finish() return server.NOT_DONE_YET diff --git a/lbrynet/lbrynet_daemon/UIManager.py b/lbrynet/lbrynet_daemon/UIManager.py index 09ab4755f..82d84ec9c 100644 --- a/lbrynet/lbrynet_daemon/UIManager.py +++ b/lbrynet/lbrynet_daemon/UIManager.py @@ -8,7 +8,7 @@ from urllib2 import urlopen from StringIO import StringIO from twisted.internet import defer from twisted.internet.task import LoopingCall -from lbrynet.conf import DEFAULT_UI_BRANCH, LOG_FILE_NAME +from lbrynet import settings from lbrynet.lbrynet_daemon.Resources import NoCacheStaticFile from lbrynet import __version__ as lbrynet_version from lbryum.version import LBRYUM_VERSION as lbryum_version @@ -23,7 +23,7 @@ else: if not os.path.isdir(log_dir): os.mkdir(log_dir) -lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) +lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) @@ -74,29 +74,29 @@ class UIManager(object): self.loaded_branch = None self.loaded_requirements = None - def setup(self, branch=DEFAULT_UI_BRANCH, user_specified=None, branch_specified=False, check_requirements=None): - if check_requirements is not None: - self.check_requirements = check_requirements - if self.branch is not None: - self.branch = branch - if user_specified: - if os.path.isdir(user_specified): - log.info("Checking user specified UI directory: " + str(user_specified)) + def setup(self, branch=None, check_requirements=None, user_specified=None): + local_ui_path = settings.local_ui_path or user_specified + self.branch = settings.UI_BRANCH or branch + self.check_requirements = settings.check_ui_requirements or check_requirements + + if local_ui_path: + if os.path.isdir(local_ui_path): + log.info("Checking user specified UI directory: " + str(local_ui_path)) self.branch = "user-specified" self.loaded_git_version = "user-specified" - d = self.migrate_ui(source=user_specified) + d = self.migrate_ui(source=local_ui_path) d.addCallback(lambda _: self._load_ui()) return d else: log.info("User specified UI directory doesn't exist, using " + self.branch) - elif self.loaded_branch == "user-specified" and not branch_specified: + elif self.loaded_branch == "user-specified": log.info("Loading user provided UI") d = self._load_ui() return d else: - log.info("Checking for updates for UI branch: " + branch) - self._git_url = "https://s3.amazonaws.com/lbry-ui/{}/data.json".format(branch) - self._dist_url = "https://s3.amazonaws.com/lbry-ui/{}/dist.zip".format(branch) + log.info("Checking for updates for UI branch: " + self.branch) + self._git_url = "https://s3.amazonaws.com/lbry-ui/{}/data.json".format(self.branch) + self._dist_url = "https://s3.amazonaws.com/lbry-ui/{}/dist.zip".format(self.branch) d = self._up_to_date() d.addCallback(lambda r: self._download_ui() if not r else self._load_ui()) diff --git a/lbrynet/lbrynet_daemon/auth/client.py b/lbrynet/lbrynet_daemon/auth/client.py index 2e479726b..02cf30111 100644 --- a/lbrynet/lbrynet_daemon/auth/client.py +++ b/lbrynet/lbrynet_daemon/auth/client.py @@ -6,8 +6,7 @@ import base64 import json from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, get_auth_message -from lbrynet.conf import API_INTERFACE, API_ADDRESS, API_PORT -from lbrynet.conf import DATA_DIR +from lbrynet import settings log = logging.getLogger(__name__) USER_AGENT = "AuthServiceProxy/0.1" @@ -77,7 +76,6 @@ class LBRYAPIClient(object): cookies = http_response.cookies headers = http_response.headers next_secret = headers.get(LBRY_SECRET, False) - if next_secret: self.__api_key.secret = next_secret self.__cookies = cookies @@ -103,14 +101,18 @@ class LBRYAPIClient(object): service=None, cookies=None, auth=None, url=None, login_url=None): api_key_name = API_KEY_NAME if not key_name else key_name - pw_path = os.path.join(DATA_DIR, ".api_keys") if not pw_path else pw_path + pw_path = os.path.join(settings.DATA_DIR, ".api_keys") if not pw_path else pw_path if not key: keys = load_api_keys(pw_path) api_key = keys.get(api_key_name, False) else: api_key = APIKey(name=api_key_name, secret=key) if login_url is None: - service_url = "http://%s:%s@%s:%i/%s" % (api_key_name, api_key.secret, API_INTERFACE, API_PORT, API_ADDRESS) + service_url = "http://%s:%s@%s:%i/%s" % (api_key_name, + api_key.secret, + settings.API_INTERFACE, + settings.API_PORT, + settings.API_ADDRESS) else: service_url = login_url id_count = count @@ -143,7 +145,6 @@ class LBRYAPIClient(object): cookies = http_response.cookies uid = cookies.get(TWISTED_SESSION) api_key = APIKey.new(seed=uid) - else: # This is a client that already has a session, use it auth_header = auth diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index 1c675b5cf..c8690d9d9 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -7,7 +7,7 @@ from twisted.internet import defer from txjsonrpc import jsonrpclib from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError, SubhandlerError -from lbrynet.conf import API_INTERFACE, REFERER, ORIGIN +from lbrynet import settings from lbrynet.lbrynet_daemon.auth.util import APIKey, get_auth_message from lbrynet.lbrynet_daemon.auth.client import LBRY_SECRET @@ -81,8 +81,9 @@ class AuthJSONRPCServer(AuthorizedBase): NOT_FOUND = 8001 FAILURE = 8002 - def __init__(self): + def __init__(self, use_authentication=True): AuthorizedBase.__init__(self) + self._use_authentication = use_authentication self.allowed_during_startup = [] self.sessions = {} @@ -95,18 +96,19 @@ class AuthJSONRPCServer(AuthorizedBase): session = request.getSession() session_id = session.uid - # if this is a new session, send a new secret and set the expiration, otherwise, session.touch() - if self._initialize_session(session_id): - def expire_session(): - self._unregister_user_session(session_id) - session.startCheckingExpiration() - session.notifyOnExpire(expire_session) - message = "OK" - request.setResponseCode(self.OK) - self._set_headers(request, message, True) - self._render_message(request, message) - return server.NOT_DONE_YET - session.touch() + if self._use_authentication: + # if this is a new session, send a new secret and set the expiration, otherwise, session.touch() + if self._initialize_session(session_id): + def expire_session(): + self._unregister_user_session(session_id) + session.startCheckingExpiration() + session.notifyOnExpire(expire_session) + message = "OK" + request.setResponseCode(self.OK) + self._set_headers(request, message, True) + self._render_message(request, message) + return server.NOT_DONE_YET + session.touch() request.content.seek(0, 0) content = request.content.read() @@ -127,16 +129,17 @@ class AuthJSONRPCServer(AuthorizedBase): return server.failure reply_with_next_secret = False - if function_name in self.authorized_functions: - try: - self._verify_token(session_id, parsed, token) - except InvalidAuthenticationToken: - log.warning("API validation failed") - request.setResponseCode(self.UNAUTHORIZED) - request.finish() - return server.NOT_DONE_YET - self._update_session_secret(session_id) - reply_with_next_secret = True + if self._use_authentication: + if function_name in self.authorized_functions: + try: + self._verify_token(session_id, parsed, token) + except InvalidAuthenticationToken: + log.warning("API validation failed") + request.setResponseCode(self.UNAUTHORIZED) + request.finish() + return server.NOT_DONE_YET + self._update_session_secret(session_id) + reply_with_next_secret = True try: function = self._get_jsonrpc_method(function_name) @@ -161,9 +164,9 @@ class AuthJSONRPCServer(AuthorizedBase): @param session_id: @return: secret """ - token = APIKey.new() + log.info("Register api session") + token = APIKey.new(seed=session_id) self.sessions.update({session_id: token}) - return token def _unregister_user_session(self, session_id): log.info("Unregister API session") @@ -173,7 +176,7 @@ class AuthJSONRPCServer(AuthorizedBase): log.debug(err.getTraceback()) def _set_headers(self, request, data, update_secret=False): - request.setHeader("Access-Control-Allow-Origin", API_INTERFACE) + request.setHeader("Access-Control-Allow-Origin", settings.API_INTERFACE) request.setHeader("Content-Type", "text/json") request.setHeader("Content-Length", str(len(data))) if update_secret: @@ -187,10 +190,10 @@ class AuthJSONRPCServer(AuthorizedBase): def _check_headers(self, request): origin = request.getHeader("Origin") referer = request.getHeader("Referer") - if origin not in [None, ORIGIN]: + if origin not in [None, settings.ORIGIN]: log.warning("Attempted api call from %s", origin) return False - if referer is not None and not referer.startswith(REFERER): + if referer is not None and not referer.startswith(settings.REFERER): log.warning("Attempted api call from %s", referer) return False return True @@ -211,8 +214,7 @@ class AuthJSONRPCServer(AuthorizedBase): def _initialize_session(self, session_id): if not self.sessions.get(session_id, False): - log.info("Initializing new api session") - self.sessions.update({session_id: APIKey.new(seed=session_id, name=session_id)}) + self._register_user_session(session_id) return True return False @@ -242,7 +244,6 @@ class AuthJSONRPCServer(AuthorizedBase): log.error(err.message) raise SubhandlerError - def _callback_render(self, result, request, id, version, auth_required=False): result_for_return = result if not isinstance(result, dict) else result['result'] @@ -271,5 +272,3 @@ class AuthJSONRPCServer(AuthorizedBase): def _render_response(self, result, code): return defer.succeed({'result': result, 'code': code}) - - diff --git a/lbrynet/pointtraderclient/pointtraderclient.py b/lbrynet/pointtraderclient/pointtraderclient.py index 0499a8b9e..c468e8c71 100644 --- a/lbrynet/pointtraderclient/pointtraderclient.py +++ b/lbrynet/pointtraderclient/pointtraderclient.py @@ -1,4 +1,4 @@ -from lbrynet.conf import POINTTRADER_SERVER +from lbrynet import settings from twisted.web.client import Agent, FileBodyProducer, Headers, ResponseDone from twisted.internet import threads, defer, protocol @@ -46,7 +46,7 @@ def get_body_from_request(path, data): jsondata = FileBodyProducer(StringIO(json.dumps(data))) agent = Agent(reactor) - d = agent.request('POST', POINTTRADER_SERVER + path, Headers({'Content-Type': ['application/json']}), jsondata) + d = agent.request('POST', settings.POINTTRADER_SERVER + path, Headers({'Content-Type': ['application/json']}), jsondata) d.addCallback(get_body) return d diff --git a/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py b/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py index 072eb9f00..d84515572 100644 --- a/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py +++ b/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py @@ -27,9 +27,9 @@ if not os.path.isfile(lbrycrdd_path_conf): f.write(lbrycrdd_path) f.close() -from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer, DaemonRequest -from lbrynet.conf import API_PORT, API_INTERFACE, ICON_PATH, APP_NAME -from lbrynet.conf import UI_ADDRESS +from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer +from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest +from lbrynet import settings if platform.mac_ver()[0] >= "10.10": from LBRYNotify import LBRYNotify @@ -53,7 +53,7 @@ class LBRYDaemonApp(AppKit.NSApplication): self.connection = False statusbar = AppKit.NSStatusBar.systemStatusBar() self.statusitem = statusbar.statusItemWithLength_(AppKit.NSVariableStatusItemLength) - self.icon = AppKit.NSImage.alloc().initByReferencingFile_(ICON_PATH) + self.icon = AppKit.NSImage.alloc().initByReferencingFile_(settings.ICON_PATH) self.icon.setScalesWhenResized_(True) self.icon.setSize_((20, 20)) self.statusitem.setImage_(self.icon) @@ -63,7 +63,7 @@ class LBRYDaemonApp(AppKit.NSApplication): self.quit = AppKit.NSMenuItem.alloc().initWithTitle_action_keyEquivalent_("Quit", "replyToApplicationShouldTerminate:", "") self.menubarMenu.addItem_(self.quit) self.statusitem.setMenu_(self.menubarMenu) - self.statusitem.setToolTip_(APP_NAME) + self.statusitem.setToolTip_(settings.APP_NAME) if test_internet_connection(): @@ -77,13 +77,13 @@ class LBRYDaemonApp(AppKit.NSApplication): lbry = DaemonServer() d = lbry.start() - d.addCallback(lambda _: webbrowser.open(UI_ADDRESS)) + d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS)) lbrynet_server = server.Site(lbry.root) lbrynet_server.requestFactory = DaemonRequest - reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE) + reactor.listenTCP(settings.API_PORT, lbrynet_server, interface=settings.API_INTERFACE) def openui_(self, sender): - webbrowser.open(UI_ADDRESS) + webbrowser.open(settings.UI_ADDRESS) def replyToApplicationShouldTerminate_(self, shouldTerminate): if platform.mac_ver()[0] >= "10.10": diff --git a/packaging/uri_handler/LBRYURIHandler.py b/packaging/uri_handler/LBRYURIHandler.py index bb17a33d6..a46eaecf3 100644 --- a/packaging/uri_handler/LBRYURIHandler.py +++ b/packaging/uri_handler/LBRYURIHandler.py @@ -5,7 +5,7 @@ import sys from time import sleep from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient -from lbrynet.conf import UI_ADDRESS +from lbrynet import settings class LBRYURIHandler(object): @@ -69,9 +69,9 @@ class LBRYURIHandler(object): @staticmethod def open_address(lbry_name): if lbry_name == "lbry" or lbry_name == "" or lbry_name is None: - webbrowser.open(UI_ADDRESS) + webbrowser.open(settings.UI_ADDRESS) else: - webbrowser.open(UI_ADDRESS + "/?show=" + lbry_name) + webbrowser.open(settings.UI_ADDRESS + "/?show=" + lbry_name) def main(args): diff --git a/packaging/windows/lbry-win32-app/LBRYWin32App.py b/packaging/windows/lbry-win32-app/LBRYWin32App.py index d197f8cfc..3fef3fd70 100644 --- a/packaging/windows/lbry-win32-app/LBRYWin32App.py +++ b/packaging/windows/lbry-win32-app/LBRYWin32App.py @@ -18,9 +18,9 @@ try: except ImportError: import win32gui -from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer, DaemonRequest -from lbrynet.conf import API_PORT, API_INTERFACE, ICON_PATH, APP_NAME -from lbrynet.conf import UI_ADDRESS, API_CONNECTION_STRING, LOG_FILE_NAME +from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer +from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest +from lbrynet import settings from packaging.uri_handler.LBRYURIHandler import LBRYURIHandler @@ -29,7 +29,7 @@ data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") if not os.path.isdir(data_dir): os.mkdir(data_dir) -lbrynet_log = os.path.join(data_dir, LOG_FILE_NAME) +lbrynet_log = os.path.join(data_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) if getattr(sys, 'frozen', False) and os.name == "nt": @@ -257,7 +257,7 @@ def main(lbry_name=None): return SysTrayIcon(icon, hover_text, menu_options, on_quit=stop) def openui_(sender): - webbrowser.open(UI_ADDRESS) + webbrowser.open(settings.UI_ADDRESS) def replyToApplicationShouldTerminate_(): try: @@ -269,11 +269,11 @@ def main(lbry_name=None): replyToApplicationShouldTerminate_() if getattr(sys, 'frozen', False) and os.name == "nt": - icon = os.path.join(os.path.dirname(sys.executable), ICON_PATH, 'lbry16.ico') + icon = os.path.join(os.path.dirname(sys.executable), settings.ICON_PATH, 'lbry16.ico') else: - icon = os.path.join(ICON_PATH, 'lbry16.ico') + icon = os.path.join(settings.ICON_PATH, 'lbry16.ico') - hover_text = APP_NAME + hover_text = settings.APP_NAME menu_options = (('Open', icon, openui_),) if not test_internet_connection(): @@ -290,14 +290,14 @@ def main(lbry_name=None): lbrynet_server = server.Site(lbry.root) lbrynet_server.requestFactory = DaemonRequest try: - reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE) + reactor.listenTCP(settings.API_PORT, lbrynet_server, interface=settings.API_INTERFACE) except error.CannotListenError: log.info('Daemon already running, exiting app') sys.exit(1) reactor.run() if __name__ == '__main__': - lbry_daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING) + lbry_daemon = JSONRPCProxy.from_url(settings.API_CONNECTION_STRING) try: daemon_running = lbry_daemon.is_running() diff --git a/tests/functional/test_misc.py b/tests/functional/test_misc.py index 9bfe49ffd..8ad75569a 100644 --- a/tests/functional/test_misc.py +++ b/tests/functional/test_misc.py @@ -10,8 +10,7 @@ import unittest from Crypto.PublicKey import RSA from Crypto import Random from Crypto.Hash import MD5 -from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE -from lbrynet.conf import MIN_BLOB_INFO_PAYMENT_RATE +from lbrynet import settings from lbrynet.lbrylive.LiveStreamCreator import FileLiveStreamCreator from lbrynet.lbrylive.LiveStreamMetadataManager import DBLiveStreamMetadataManager from lbrynet.lbrylive.LiveStreamMetadataManager import TempLiveStreamMetadataManager @@ -236,7 +235,7 @@ def start_lbry_uploader(sd_hash_queue, kill_event, dead_event, file_size, ul_rat db_dir = "server" os.mkdir(db_dir) - session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, dht_node_class=Node) @@ -351,7 +350,7 @@ def start_lbry_reuploader(sd_hash, kill_event, dead_event, ready_event, n, ul_ra os.mkdir(db_dir) os.mkdir(blob_dir) - session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd" + str(n), + session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd" + str(n), peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=None, peer_port=peer_port, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker) @@ -462,7 +461,7 @@ def start_live_server(sd_hash_queue, kill_event, dead_event): db_dir = "server" os.mkdir(db_dir) - session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker) @@ -596,7 +595,7 @@ def start_blob_uploader(blob_hash_queue, kill_event, dead_event, slow): os.mkdir(db_dir) os.mkdir(blob_dir) - session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="efgh", + session = Session(settings.data_rate, db_dir=db_dir, lbryid="efgh", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=peer_port, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker) @@ -768,7 +767,7 @@ class TestTransfer(TestCase): os.mkdir(db_dir) os.mkdir(blob_dir) - self.session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, @@ -854,7 +853,7 @@ class TestTransfer(TestCase): db_dir = "client" os.mkdir(db_dir) - self.session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=None, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, dht_node_class=Node) @@ -955,7 +954,7 @@ class TestTransfer(TestCase): os.mkdir(db_dir) os.mkdir(blob_dir) - self.session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker) @@ -1035,7 +1034,7 @@ class TestTransfer(TestCase): os.mkdir(db_dir) os.mkdir(blob_dir) - self.session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker) @@ -1150,7 +1149,7 @@ class TestTransfer(TestCase): os.mkdir(db_dir) os.mkdir(blob_dir) - self.session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=None, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker) @@ -1269,7 +1268,7 @@ class TestStreamify(TestCase): os.mkdir(db_dir) os.mkdir(blob_dir) - self.session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker) @@ -1322,7 +1321,7 @@ class TestStreamify(TestCase): os.mkdir(db_dir) os.mkdir(blob_dir) - self.session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker) diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py index 7dd0ad6e7..2199f6294 100644 --- a/tests/functional/test_reflector.py +++ b/tests/functional/test_reflector.py @@ -4,7 +4,7 @@ import shutil from twisted.internet import defer, threads, error from twisted.trial import unittest -from lbrynet import conf +from lbrynet import settings from lbrynet import lbryfile from lbrynet import reflector from lbrynet.core import BlobManager @@ -83,7 +83,7 @@ class TestReflector(unittest.TestCase): os.mkdir(db_dir) self.session = Session.Session( - conf.MIN_BLOB_DATA_PAYMENT_RATE, + settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, diff --git a/tests/mocks.py b/tests/mocks.py index 40e37dcca..7de8668ff 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -2,7 +2,7 @@ import io from Crypto.PublicKey import RSA from decimal import Decimal -from twisted.internet import defer, threads, task, error +from twisted.internet import defer from lbrynet.core import PTCWallet from lbrynet.core.BlobAvailability import BlobAvailabilityTracker From 33674f8cc35431954e787ebbe94cf142c77a8e30 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 19 Oct 2016 00:39:19 -0400 Subject: [PATCH 07/14] remove logging in conf, rearrange platform --- lbrynet/conf.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index f4a97af11..c60f14c93 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -1,39 +1,29 @@ import copy import os import sys -import logging from appdirs import user_data_dir -log = logging.getLogger(__name__) - PRIORITIZE_ENV = True - LINUX = 1 DARWIN = 2 WINDOWS = 3 -if sys.platform.startswith("linux"): - platform = LINUX -elif sys.platform.startswith("darwin"): +if sys.platform.startswith("darwin"): platform = DARWIN -elif sys.platform.startswith("win"): - platform = WINDOWS -else: - platform = LINUX - -if platform is LINUX: - default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') - default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") - default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum") -elif platform is DARWIN: default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') default_data_dir = user_data_dir("LBRY") default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum") -else: +elif sys.platform.startswith("win"): + platform = WINDOWS from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current) default_data_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrynet") default_lbryum_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum") +else: + platform = LINUX + default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') + default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") + default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum") ADJUSTABLE_SETTINGS = { 'run_on_startup': False, @@ -76,8 +66,7 @@ ADJUSTABLE_SETTINGS = { 'DATA_DIR': default_data_dir, 'LBRYUM_WALLET_DIR': default_lbryum_dir, 'sd_download_timeout': 3, - 'max_key_fee': {'USD': {'amount': 25.0, 'address': ''}}, - + 'max_key_fee': {'USD': {'amount': 25.0, 'address': ''}} } @@ -113,7 +102,6 @@ class ApplicationSettings(object): def get_dict(): r = {k: v for k, v in ApplicationSettings.__dict__.iteritems() if not k.startswith('__')} if PRIORITIZE_ENV: - log.info("Checking env settings") r = add_env_settings(r) return r @@ -123,7 +111,6 @@ def add_env_settings(settings_dict): for setting, setting_val in settings_dict.iteritems(): env_val = os.environ.get(setting, None) if env_val != setting_val and env_val is not None: - log.info("Using env value for %s", setting) with_env_settings.update({setting: env_val}) return with_env_settings From 8a95c8b7af1299c0b328f6873c83411d50aacb1b Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 19 Oct 2016 14:26:19 -0400 Subject: [PATCH 08/14] whoops --- lbrynet/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index c60f14c93..51d799818 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -22,7 +22,7 @@ elif sys.platform.startswith("win"): else: platform = LINUX default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') - default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") + default_data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum") ADJUSTABLE_SETTINGS = { From 22fc26797d6c37533f4377b7c5ea4f11bda6e872 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 19 Oct 2016 14:40:01 -0400 Subject: [PATCH 09/14] pylint fixes --- lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py | 6 +++--- lbrynet/lbrynet_daemon/Daemon.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py b/lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py index 01686de4c..84afefce8 100644 --- a/lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py +++ b/lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py @@ -1,6 +1,6 @@ from lbrynet.lbrynet_console import Plugin from twisted.internet import defer -from lbrynet.conf import MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE, MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE +from lbrynet import settings from BlindRepeater import BlindRepeater from BlindInfoManager import BlindInfoManager from BlindRepeaterSettings import BlindRepeaterSettings @@ -59,9 +59,9 @@ class BlindRepeaterPlugin(Plugin.Plugin): def get_payment_rate_manager(rates): data_rate = rates[0][1] if rates[0][0] is True else None info_rate = rates[1][1] if rates[1][0] is True else None - info_rate = info_rate if info_rate is not None else MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE + info_rate = info_rate if info_rate is not None else settings.MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE hash_rate = rates[2][1] if rates[2][0] is True else None - hash_rate = hash_rate if hash_rate is not None else MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE + hash_rate = hash_rate if hash_rate is not None else settings.MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE self.payment_rate_manager = BlindRepeaterPaymentRateManager(default_payment_rate_manager, info_rate, hash_rate, blob_data_rate=data_rate) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index fec8f142b..b6b71aad0 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -6,7 +6,6 @@ import platform import random import re import socket -import string import subprocess import sys import base58 @@ -1120,9 +1119,9 @@ class Daemon(AuthJSONRPCServer): log.info("Removing one time startup scripts") remaining_scripts = [s for s in self.startup_scripts if 'run_once' not in s.keys()] startup_scripts = self.startup_scripts - self.startup_scripts = lbrynet_settings['startup_scripts'] = remaining_scripts - - utils.save_settings(self.daemon_conf, lbrynet_settings) + self.startup_scripts = lbrynet_settings.startup_scripts = remaining_scripts + conf = os.path.join(lbrynet_settings.DATA_DIR, "daemon_settings.yml") + utils.save_settings(conf) for script in startup_scripts: if script['script_name'] == 'migrateto025': From 293933163f0dc13d9aab56bf37e1597cf5d83031 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 19 Oct 2016 14:59:10 -0400 Subject: [PATCH 10/14] move USE_AUTH_HTTP to adjustable settings --- lbrynet/conf.py | 2 +- lbrynet/lbrynet_daemon/DaemonControl.py | 4 ++-- lbrynet/lbrynet_daemon/auth/server.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 51d799818..2f387c572 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -65,13 +65,13 @@ ADJUSTABLE_SETTINGS = { 'DEFAULT_UI_BRANCH': 'master', 'DATA_DIR': default_data_dir, 'LBRYUM_WALLET_DIR': default_lbryum_dir, + 'USE_AUTH_HTTP': False, 'sd_download_timeout': 3, 'max_key_fee': {'USD': {'amount': 25.0, 'address': ''}} } class ApplicationSettings(object): - USE_AUTH_HTTP = True MAX_HANDSHAKE_SIZE = 2**16 MAX_REQUEST_SIZE = 2**16 MAX_BLOB_REQUEST_SIZE = 2**16 diff --git a/lbrynet/lbrynet_daemon/DaemonControl.py b/lbrynet/lbrynet_daemon/DaemonControl.py index 65bb54ceb..6a7250a31 100644 --- a/lbrynet/lbrynet_daemon/DaemonControl.py +++ b/lbrynet/lbrynet_daemon/DaemonControl.py @@ -92,7 +92,7 @@ def start(): action='store_true', help='enable more debug output for the console') - parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False, useauth=False) + parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False, useauth=settings.USE_AUTH_HTTP) args = parser.parse_args() log_support.configure_file_handler(lbrynet_log) @@ -143,7 +143,7 @@ def start(): if args.launchui: d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS)) - if args.useauth: + if settings.USE_AUTH_HTTP: log.info("Using authenticated API") pw_path = os.path.join(settings.DATA_DIR, ".api_keys") initialize_api_key_file(pw_path) diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index c8690d9d9..2f0b577d7 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -81,7 +81,7 @@ class AuthJSONRPCServer(AuthorizedBase): NOT_FOUND = 8001 FAILURE = 8002 - def __init__(self, use_authentication=True): + def __init__(self, use_authentication=settings.USE_AUTH_HTTP): AuthorizedBase.__init__(self) self._use_authentication = use_authentication self.allowed_during_startup = [] From 24320d64440548dd1c100d9f52f909bd738a43dc Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 26 Oct 2016 03:22:45 -0400 Subject: [PATCH 11/14] remove debug line --- lbrynet/lbrynet_daemon/Daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index 853dba2e1..473ced25c 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -723,7 +723,7 @@ class Daemon(AuthJSONRPCServer): return d def _set_lbryid(self, lbryid): - if lbryid is None or True: + if lbryid is None: return self._make_lbryid() else: log.info("LBRY ID: " + base58.b58encode(lbryid)) From 7076c0c870ebb7c780d4c8fdc307008ee43acc98 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Thu, 27 Oct 2016 10:49:28 -0500 Subject: [PATCH 12/14] Require env settings are in LBRY namespace --- lbrynet/__init__.py | 8 +++++- lbrynet/conf.py | 62 ++++++++++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index d5fa34df3..570c17757 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -3,5 +3,11 @@ from conf import Config __version__ = "0.6.9" version = tuple(__version__.split('.')) + +# TODO: don't load the configuration automatically. The configuration +# should be loaded at runtime, not at module import time. Module +# import should have no side-effects. This is also bad because +# it means that settings are read from the environment even for +# tests, which is rarely what you want to happen. settings = Config() -logging.getLogger(__name__).addHandler(logging.NullHandler()) \ No newline at end of file +logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index f3a1893d5..3f05c4636 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -16,8 +16,10 @@ elif sys.platform.startswith("win"): platform = WINDOWS from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current) - default_data_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrynet") - default_lbryum_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum") + default_data_dir = os.path.join( + get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrynet") + default_lbryum_dir = os.path.join( + get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum") else: platform = LINUX default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') @@ -25,42 +27,47 @@ else: default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum") -def convert_setting(setting, current_val): - new_type = setting.__class__ +def convert_setting(env_val, current_val): + new_type = env_val.__class__ current_type = current_val.__class__ if current_type is bool: if new_type is bool: - return setting - elif str(setting).lower() == "false": + return env_val + elif str(env_val).lower() == "false": return False - elif str(setting).lower() == "true": + elif str(env_val).lower() == "true": return True else: raise ValueError elif current_type is int: - return int(setting) + return int(env_val) elif current_type is float: - return float(setting) + return float(env_val) elif current_type is str: - return str(setting) + return str(env_val) elif current_type is dict: - return dict(setting) + return dict(env_val) elif current_type is list: - return list(setting) + return list(env_val) elif current_type is tuple: - return tuple(setting) + return tuple(env_val) else: - raise ValueError() + raise ValueError('Type {} cannot be converted'.format(current_type)) def convert_env_setting(setting, value): - env_val = os.environ.get(setting, value) - return convert_setting(env_val, value) + try: + env_val = os.environ[setting] + except KeyError: + return value + else: + return convert_setting(env_val, value) def get_env_settings(settings): - for setting in settings: - yield convert_env_setting(setting, settings[setting]) + for setting, value in settings.iteritems(): + setting = 'LBRY_' + setting.upper() + yield convert_env_setting(setting, value) def add_env_settings_to_dict(settings_dict): @@ -135,10 +142,12 @@ class AdjustableSettings(Setting): self.min_valuable_info_rate = .05 # points/1000 infos self.min_valuable_hash_rate = .05 # points/1000 infos self.max_connections_per_stream = 5 - self.known_dht_nodes = [('104.236.42.182', 4000), - ('lbrynet1.lbry.io', 4444), - ('lbrynet2.lbry.io', 4444), - ('lbrynet3.lbry.io', 4444)] + self.known_dht_nodes = [ + ('104.236.42.182', 4000), + ('lbrynet1.lbry.io', 4444), + ('lbrynet2.lbry.io', 4444), + ('lbrynet3.lbry.io', 4444) + ] self.pointtrader_server = 'http://127.0.0.1:2424' self.reflector_servers = [("reflector.lbry.io", 5566)] self.wallet = "lbryum" @@ -171,9 +180,9 @@ class ApplicationSettings(Setting): self.wallet_TYPES = ["lbryum", "lbrycrd"] self.SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] self.CURRENCIES = { - 'BTC': {'type': 'crypto'}, - 'LBC': {'type': 'crypto'}, - 'USD': {'type': 'fiat'}, + 'BTC': {'type': 'crypto'}, + 'LBC': {'type': 'crypto'}, + 'USD': {'type': 'fiat'}, } self.LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv' self.ANALYTICS_ENDPOINT = 'https://api.segment.io/v1' @@ -211,7 +220,8 @@ class Config(DefaultSettings): @property def API_CONNECTION_STRING(self): - return "http://%s:%i/%s" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port, DEFAULT_SETTINGS.API_ADDRESS) + return "http://%s:%i/%s" % ( + DEFAULT_SETTINGS.API_INTERFACE, self.api_port, DEFAULT_SETTINGS.API_ADDRESS) @property def UI_ADDRESS(self): From 862055a6de281abacbaa5fd9f76381692b3c4f70 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Thu, 27 Oct 2016 14:18:25 -0500 Subject: [PATCH 13/14] move settings into conf file --- lbrynet/__init__.py | 7 ------- lbrynet/analytics/api.py | 2 +- lbrynet/conf.py | 8 ++++++++ lbrynet/core/HashBlob.py | 2 +- lbrynet/core/PaymentRateManager.py | 2 +- lbrynet/core/PriceModel.py | 2 +- lbrynet/core/Strategy.py | 2 +- lbrynet/core/client/ClientProtocol.py | 2 +- lbrynet/core/client/ClientRequest.py | 2 +- lbrynet/core/client/ConnectionManager.py | 2 +- lbrynet/core/log_support.py | 2 +- lbrynet/core/utils.py | 2 +- lbrynet/cryptstream/CryptBlob.py | 2 +- lbrynet/lbryfilemanager/EncryptedFileCreator.py | 2 +- lbrynet/lbrylive/LiveStreamCreator.py | 2 +- lbrynet/lbrylive/client/LiveStreamMetadataHandler.py | 2 +- lbrynet/lbrynet_console/Console.py | 2 +- lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py | 2 +- lbrynet/lbrynet_daemon/Daemon.py | 2 +- lbrynet/lbrynet_daemon/DaemonCLI.py | 2 +- lbrynet/lbrynet_daemon/DaemonControl.py | 2 +- lbrynet/lbrynet_daemon/DaemonServer.py | 2 +- lbrynet/lbrynet_daemon/Downloader.py | 2 +- lbrynet/lbrynet_daemon/Lighthouse.py | 2 +- lbrynet/lbrynet_daemon/Publisher.py | 2 +- lbrynet/lbrynet_daemon/Resources.py | 2 +- lbrynet/lbrynet_daemon/UIManager.py | 2 +- lbrynet/lbrynet_daemon/auth/client.py | 2 +- lbrynet/lbrynet_daemon/auth/server.py | 2 +- lbrynet/pointtraderclient/pointtraderclient.py | 2 +- tests/functional/test_misc.py | 2 +- tests/functional/test_reflector.py | 2 +- tests/functional/test_streamify.py | 2 +- 33 files changed, 39 insertions(+), 38 deletions(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 570c17757..3e809bd2e 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,13 +1,6 @@ import logging -from conf import Config __version__ = "0.6.9" version = tuple(__version__.split('.')) -# TODO: don't load the configuration automatically. The configuration -# should be loaded at runtime, not at module import time. Module -# import should have no side-effects. This is also bad because -# it means that settings are read from the environment even for -# tests, which is rarely what you want to happen. -settings = Config() logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/lbrynet/analytics/api.py b/lbrynet/analytics/api.py index d5920daa3..50c20212d 100644 --- a/lbrynet/analytics/api.py +++ b/lbrynet/analytics/api.py @@ -5,7 +5,7 @@ import logging from requests import auth from requests_futures import sessions -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.analytics import utils diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 3f05c4636..9f46406e4 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -226,3 +226,11 @@ class Config(DefaultSettings): @property def UI_ADDRESS(self): return "http://%s:%i" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port) + + +# TODO: don't load the configuration automatically. The configuration +# should be loaded at runtime, not at module import time. Module +# import should have no side-effects. This is also bad because +# it means that settings are read from the environment even for +# tests, which is rarely what you want to happen. +settings = Config() diff --git a/lbrynet/core/HashBlob.py b/lbrynet/core/HashBlob.py index 881ff79e9..a0e95acfb 100644 --- a/lbrynet/core/HashBlob.py +++ b/lbrynet/core/HashBlob.py @@ -8,7 +8,7 @@ from twisted.internet import interfaces, defer, threads from twisted.protocols.basic import FileSender from twisted.python.failure import Failure from zope.interface import implements -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.core.Error import DownloadCanceledError, InvalidDataError from lbrynet.core.cryptoutils import get_lbry_hash_obj diff --git a/lbrynet/core/PaymentRateManager.py b/lbrynet/core/PaymentRateManager.py index 7c0cd2d7c..726f3ba19 100644 --- a/lbrynet/core/PaymentRateManager.py +++ b/lbrynet/core/PaymentRateManager.py @@ -1,5 +1,5 @@ from lbrynet.core.Strategy import get_default_strategy -from lbrynet import settings +from lbrynet.conf import settings from decimal import Decimal diff --git a/lbrynet/core/PriceModel.py b/lbrynet/core/PriceModel.py index 01c72eea9..70e57ebd0 100644 --- a/lbrynet/core/PriceModel.py +++ b/lbrynet/core/PriceModel.py @@ -2,7 +2,7 @@ from zope.interface import implementer from decimal import Decimal from lbrynet.interfaces import IBlobPriceModel -from lbrynet import settings +from lbrynet.conf import settings def get_default_price_model(blob_tracker, base_price, **kwargs): diff --git a/lbrynet/core/Strategy.py b/lbrynet/core/Strategy.py index 8582ec170..6a3fc41c9 100644 --- a/lbrynet/core/Strategy.py +++ b/lbrynet/core/Strategy.py @@ -1,6 +1,6 @@ from zope.interface import implementer from decimal import Decimal -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.interfaces import INegotiationStrategy from lbrynet.core.Offer import Offer from lbrynet.core.PriceModel import MeanAvailabilityWeightedPrice diff --git a/lbrynet/core/client/ClientProtocol.py b/lbrynet/core/client/ClientProtocol.py index 99d55edfe..09e7a32ca 100644 --- a/lbrynet/core/client/ClientProtocol.py +++ b/lbrynet/core/client/ClientProtocol.py @@ -4,7 +4,7 @@ from decimal import Decimal from twisted.internet import error, defer from twisted.internet.protocol import Protocol, ClientFactory from twisted.python import failure -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError from lbrynet.core.Error import RequestCanceledError diff --git a/lbrynet/core/client/ClientRequest.py b/lbrynet/core/client/ClientRequest.py index 55abe132c..106982ccc 100644 --- a/lbrynet/core/client/ClientRequest.py +++ b/lbrynet/core/client/ClientRequest.py @@ -1,4 +1,4 @@ -from lbrynet import settings +from lbrynet.conf import settings class ClientRequest(object): diff --git a/lbrynet/core/client/ConnectionManager.py b/lbrynet/core/client/ConnectionManager.py index 8e443f2d3..d68af47b0 100644 --- a/lbrynet/core/client/ConnectionManager.py +++ b/lbrynet/core/client/ConnectionManager.py @@ -2,7 +2,7 @@ import logging from twisted.internet import defer from zope.interface import implements from lbrynet import interfaces -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.core.client.ClientProtocol import ClientProtocolFactory from lbrynet.core.Error import InsufficientFundsError diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index 735fbba9a..46ce6e4e0 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -7,7 +7,7 @@ import traceback from requests_futures.sessions import FuturesSession import lbrynet -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.core import utils session = FuturesSession() diff --git a/lbrynet/core/utils.py b/lbrynet/core/utils.py index a8ef10e97..ad88d3fe4 100644 --- a/lbrynet/core/utils.py +++ b/lbrynet/core/utils.py @@ -8,7 +8,7 @@ import os import socket import yaml -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.conf import AdjustableSettings from lbrynet.core.cryptoutils import get_lbry_hash_obj diff --git a/lbrynet/cryptstream/CryptBlob.py b/lbrynet/cryptstream/CryptBlob.py index 0de296597..da94f631b 100644 --- a/lbrynet/cryptstream/CryptBlob.py +++ b/lbrynet/cryptstream/CryptBlob.py @@ -1,7 +1,7 @@ import binascii import logging from Crypto.Cipher import AES -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.core.BlobInfo import BlobInfo diff --git a/lbrynet/lbryfilemanager/EncryptedFileCreator.py b/lbrynet/lbryfilemanager/EncryptedFileCreator.py index cec47de13..c9feaccf3 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileCreator.py +++ b/lbrynet/lbryfilemanager/EncryptedFileCreator.py @@ -7,7 +7,7 @@ import logging import os from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.lbryfile.StreamDescriptor import get_sd_info from lbrynet.core.cryptoutils import get_lbry_hash_obj from twisted.protocols.basic import FileSender diff --git a/lbrynet/lbrylive/LiveStreamCreator.py b/lbrynet/lbrylive/LiveStreamCreator.py index e37e6bf46..c8fd6e937 100644 --- a/lbrynet/lbrylive/LiveStreamCreator.py +++ b/lbrynet/lbrylive/LiveStreamCreator.py @@ -6,7 +6,7 @@ from lbrynet.core.cryptoutils import get_lbry_hash_obj, get_pub_key, sign_with_p from Crypto import Random import binascii import logging -from lbrynet import settings +from lbrynet.conf import settings from twisted.internet import interfaces, defer from twisted.protocols.basic import FileSender from zope.interface import implements diff --git a/lbrynet/lbrylive/client/LiveStreamMetadataHandler.py b/lbrynet/lbrylive/client/LiveStreamMetadataHandler.py index cfef9c405..b98ed72c1 100644 --- a/lbrynet/lbrylive/client/LiveStreamMetadataHandler.py +++ b/lbrynet/lbrylive/client/LiveStreamMetadataHandler.py @@ -3,7 +3,7 @@ import logging from zope.interface import implements from twisted.internet import defer from twisted.python.failure import Failure -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.core.client.ClientRequest import ClientRequest, ClientPaidRequest from lbrynet.lbrylive.LiveBlob import LiveBlobInfo from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature diff --git a/lbrynet/lbrynet_console/Console.py b/lbrynet/lbrynet_console/Console.py index 8ece7b539..ab0841a8b 100644 --- a/lbrynet/lbrynet_console/Console.py +++ b/lbrynet/lbrynet_console/Console.py @@ -19,7 +19,7 @@ from lbrynet.core.Session import Session from lbrynet.lbrynet_console.ConsoleControl import ConsoleControl from lbrynet.lbrynet_console.Settings import Settings from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.core.utils import generate_id from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.core.PaymentRateManager import PaymentRateManager diff --git a/lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py b/lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py index b8b469a6e..44966fea5 100644 --- a/lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py +++ b/lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py @@ -1,6 +1,6 @@ from lbrynet.lbrynet_console import Plugin from twisted.internet import defer -from lbrynet import settings +from lbrynet.conf import settings from BlindRepeater import BlindRepeater from BlindInfoManager import BlindInfoManager from BlindRepeaterSettings import BlindRepeaterSettings diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index ce0bb4414..bb0f0a9cf 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -26,7 +26,7 @@ from lbrynet import __version__ as lbrynet_version from lbryum.version import LBRYUM_VERSION as lbryum_version from lbrynet import __version__ as lbrynet_version -from lbrynet import settings as lbrynet_settings +from lbrynet.conf import settings as lbrynet_settings from lbrynet import analytics from lbrynet import reflector from lbrynet.metadata.Metadata import Metadata, verify_name_characters diff --git a/lbrynet/lbrynet_daemon/DaemonCLI.py b/lbrynet/lbrynet_daemon/DaemonCLI.py index 3e3df37d1..2f9fb49ba 100644 --- a/lbrynet/lbrynet_daemon/DaemonCLI.py +++ b/lbrynet/lbrynet_daemon/DaemonCLI.py @@ -2,7 +2,7 @@ import sys import json import argparse -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient help_msg = "Usage: lbrynet-cli method json-args\n" \ diff --git a/lbrynet/lbrynet_daemon/DaemonControl.py b/lbrynet/lbrynet_daemon/DaemonControl.py index e4c3bd7e9..474110ab5 100644 --- a/lbrynet/lbrynet_daemon/DaemonControl.py +++ b/lbrynet/lbrynet_daemon/DaemonControl.py @@ -16,7 +16,7 @@ from lbrynet.core import log_support from lbrynet.core import utils from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest -from lbrynet import settings +from lbrynet.conf import settings log_dir = settings.data_dir diff --git a/lbrynet/lbrynet_daemon/DaemonServer.py b/lbrynet/lbrynet_daemon/DaemonServer.py index a9e320d25..0980ece2c 100644 --- a/lbrynet/lbrynet_daemon/DaemonServer.py +++ b/lbrynet/lbrynet_daemon/DaemonServer.py @@ -6,7 +6,7 @@ from appdirs import user_data_dir from twisted.internet import defer from lbrynet.lbrynet_daemon.Daemon import Daemon from lbrynet.lbrynet_daemon.Resources import LBRYindex, HostedEncryptedFile, EncryptedFileUpload -from lbrynet import settings +from lbrynet.conf import settings # TODO: omg, this code is essentially duplicated in Daemon diff --git a/lbrynet/lbrynet_daemon/Downloader.py b/lbrynet/lbrynet_daemon/Downloader.py index 9b34e4476..5d7c364d7 100644 --- a/lbrynet/lbrynet_daemon/Downloader.py +++ b/lbrynet/lbrynet_daemon/Downloader.py @@ -12,7 +12,7 @@ from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed from lbrynet.core.StreamDescriptor import download_sd_blob from lbrynet.metadata.Fee import FeeValidator from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory -from lbrynet import settings +from lbrynet.conf import settings INITIALIZING_CODE = 'initializing' DOWNLOAD_METADATA_CODE = 'downloading_metadata' diff --git a/lbrynet/lbrynet_daemon/Lighthouse.py b/lbrynet/lbrynet_daemon/Lighthouse.py index 3e5419e9e..fb574e55c 100644 --- a/lbrynet/lbrynet_daemon/Lighthouse.py +++ b/lbrynet/lbrynet_daemon/Lighthouse.py @@ -1,7 +1,7 @@ import logging import random from txjsonrpc.web.jsonrpc import Proxy -from lbrynet import settings +from lbrynet.conf import settings log = logging.getLogger(__name__) diff --git a/lbrynet/lbrynet_daemon/Publisher.py b/lbrynet/lbrynet_daemon/Publisher.py index b9390bba8..8cd956d42 100644 --- a/lbrynet/lbrynet_daemon/Publisher.py +++ b/lbrynet/lbrynet_daemon/Publisher.py @@ -12,7 +12,7 @@ from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob from lbrynet.metadata.Metadata import Metadata from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloader from lbrynet import reflector -from lbrynet import settings +from lbrynet.conf import settings from twisted.internet import threads, defer, reactor if sys.platform != "darwin": diff --git a/lbrynet/lbrynet_daemon/Resources.py b/lbrynet/lbrynet_daemon/Resources.py index 02f7a4cbb..b400e519b 100644 --- a/lbrynet/lbrynet_daemon/Resources.py +++ b/lbrynet/lbrynet_daemon/Resources.py @@ -10,7 +10,7 @@ from appdirs import user_data_dir from twisted.web import server, static, resource from twisted.internet import defer, error -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.lbrynet_daemon.FileStreamer import EncryptedFileStreamer # TODO: omg, this code is essentially duplicated in Daemon diff --git a/lbrynet/lbrynet_daemon/UIManager.py b/lbrynet/lbrynet_daemon/UIManager.py index 2770204b2..c0e524eab 100644 --- a/lbrynet/lbrynet_daemon/UIManager.py +++ b/lbrynet/lbrynet_daemon/UIManager.py @@ -8,7 +8,7 @@ from urllib2 import urlopen from StringIO import StringIO from twisted.internet import defer from twisted.internet.task import LoopingCall -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.lbrynet_daemon.Resources import NoCacheStaticFile from lbrynet import __version__ as lbrynet_version from lbryum.version import LBRYUM_VERSION as lbryum_version diff --git a/lbrynet/lbrynet_daemon/auth/client.py b/lbrynet/lbrynet_daemon/auth/client.py index 07fde2d90..e0032a8b6 100644 --- a/lbrynet/lbrynet_daemon/auth/client.py +++ b/lbrynet/lbrynet_daemon/auth/client.py @@ -6,7 +6,7 @@ import base64 import json from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, get_auth_message -from lbrynet import settings +from lbrynet.conf import settings from jsonrpc.proxy import JSONRPCProxy log = logging.getLogger(__name__) diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index 66b213aec..4a3637fe6 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -7,7 +7,7 @@ from twisted.internet import defer from txjsonrpc import jsonrpclib from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError, SubhandlerError -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.lbrynet_daemon.auth.util import APIKey, get_auth_message from lbrynet.lbrynet_daemon.auth.client import LBRY_SECRET diff --git a/lbrynet/pointtraderclient/pointtraderclient.py b/lbrynet/pointtraderclient/pointtraderclient.py index 05c4fdeef..1c533843c 100644 --- a/lbrynet/pointtraderclient/pointtraderclient.py +++ b/lbrynet/pointtraderclient/pointtraderclient.py @@ -1,4 +1,4 @@ -from lbrynet import settings +from lbrynet.conf import settings from twisted.web.client import Agent, FileBodyProducer, Headers, ResponseDone from twisted.internet import threads, defer, protocol diff --git a/tests/functional/test_misc.py b/tests/functional/test_misc.py index 8490507da..0729af819 100644 --- a/tests/functional/test_misc.py +++ b/tests/functional/test_misc.py @@ -11,7 +11,7 @@ import unittest from Crypto.PublicKey import RSA from Crypto import Random from Crypto.Hash import MD5 -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.lbrylive.LiveStreamCreator import FileLiveStreamCreator from lbrynet.lbrylive.LiveStreamMetadataManager import DBLiveStreamMetadataManager from lbrynet.lbrylive.LiveStreamMetadataManager import TempLiveStreamMetadataManager diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py index f71ddffbc..6603bc6d3 100644 --- a/tests/functional/test_reflector.py +++ b/tests/functional/test_reflector.py @@ -4,7 +4,7 @@ import shutil from twisted.internet import defer, threads, error from twisted.trial import unittest -from lbrynet import settings +from lbrynet.conf import settings from lbrynet import lbryfile from lbrynet import reflector from lbrynet.core import BlobManager diff --git a/tests/functional/test_streamify.py b/tests/functional/test_streamify.py index 01e290bc5..c6378956d 100644 --- a/tests/functional/test_streamify.py +++ b/tests/functional/test_streamify.py @@ -6,7 +6,7 @@ from Crypto.Hash import MD5 from twisted.trial.unittest import TestCase from twisted.internet import defer, threads -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.lbryfile.EncryptedFileMetadataManager import TempEncryptedFileMetadataManager from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager From e70ecc8ac9e351768530670d52a64c2c849f9802 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Thu, 27 Oct 2016 14:57:02 -0500 Subject: [PATCH 14/14] change settings object for packages too --- packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py | 2 +- packaging/osx/lbry-osx-app/setup_app.py | 6 +++--- packaging/osx/lbry-osx-app/setup_uri_handler.py | 6 +++--- packaging/uri_handler/LBRYURIHandler.py | 2 +- packaging/windows/lbry-win32-app/LBRYWin32App.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py b/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py index 796b9fbeb..ab43236f3 100644 --- a/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py +++ b/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py @@ -24,7 +24,7 @@ if not os.path.isfile(lbrycrdd_path_conf): from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest -from lbrynet import settings +from lbrynet.conf import settings from lbrynet.core import utils diff --git a/packaging/osx/lbry-osx-app/setup_app.py b/packaging/osx/lbry-osx-app/setup_app.py index 1f4cba717..29169a9da 100644 --- a/packaging/osx/lbry-osx-app/setup_app.py +++ b/packaging/osx/lbry-osx-app/setup_app.py @@ -2,14 +2,14 @@ import os from setuptools import setup -from lbrynet.conf import APP_NAME, ICON_PATH +from lbrynet.conf import settings APP = [os.path.join('lbrygui', 'main.py')] DATA_FILES = [] DATA_FILES.append('app.icns') OPTIONS = { - 'iconfile': ICON_PATH, + 'iconfile': settings.ICON_PATH, 'plist': { 'CFBundleIdentifier': 'io.lbry.LBRY', 'LSUIElement': True, @@ -22,7 +22,7 @@ OPTIONS = { setup( - name=APP_NAME, + name=settings.APP_NAME, app=APP, options={'py2app': OPTIONS}, data_files=DATA_FILES, diff --git a/packaging/osx/lbry-osx-app/setup_uri_handler.py b/packaging/osx/lbry-osx-app/setup_uri_handler.py index 21b2050a9..62d185e08 100644 --- a/packaging/osx/lbry-osx-app/setup_uri_handler.py +++ b/packaging/osx/lbry-osx-app/setup_uri_handler.py @@ -1,6 +1,6 @@ from setuptools import setup import os -from lbrynet.conf import PROTOCOL_PREFIX +from lbrynet.conf import settings APP = [os.path.join('lbry_uri_handler', 'LBRYURIHandler.py')] DATA_FILES = [] @@ -12,7 +12,7 @@ OPTIONS = {'argv_emulation': True, 'CFBundleURLTypes': [ { 'CFBundleURLTypes': 'LBRYURIHandler', - 'CFBundleURLSchemes': [PROTOCOL_PREFIX] + 'CFBundleURLSchemes': [settings.PROTOCOL_PREFIX] } ] } @@ -23,4 +23,4 @@ setup( data_files=DATA_FILES, options={'py2app': OPTIONS}, setup_requires=['py2app'], -) \ No newline at end of file +) diff --git a/packaging/uri_handler/LBRYURIHandler.py b/packaging/uri_handler/LBRYURIHandler.py index a46eaecf3..f8d9b081f 100644 --- a/packaging/uri_handler/LBRYURIHandler.py +++ b/packaging/uri_handler/LBRYURIHandler.py @@ -5,7 +5,7 @@ import sys from time import sleep from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient -from lbrynet import settings +from lbrynet.conf import settings class LBRYURIHandler(object): diff --git a/packaging/windows/lbry-win32-app/LBRYWin32App.py b/packaging/windows/lbry-win32-app/LBRYWin32App.py index a286bdc78..d3591dcab 100644 --- a/packaging/windows/lbry-win32-app/LBRYWin32App.py +++ b/packaging/windows/lbry-win32-app/LBRYWin32App.py @@ -21,7 +21,7 @@ except ImportError: from lbrynet.core import utils from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest -from lbrynet import settings +from lbrynet.conf import settings from packaging.uri_handler.LBRYURIHandler import LBRYURIHandler