Merge pull request #211 from lbryio/refactor-settings

Refactor settings

lbrynet/__init__.py

@@ -3,4 +3,4 @@ import logging
 __version__ = "0.6.9"
 version = tuple(__version__.split('.'))
 
-logging.getLogger(__name__).addHandler(logging.NullHandler())
+logging.getLogger(__name__).addHandler(logging.NullHandler())

lbrynet/analytics/api.py

@@ -5,7 +5,7 @@ import logging
 from requests import auth
 from requests_futures import sessions
 
-from lbrynet import conf
+from lbrynet.conf import settings
 from lbrynet.analytics import utils
 
 
@@ -66,6 +66,6 @@ class Api(object):
             session = sessions.FuturesSession()
         return cls(
             session,
-            conf.ANALYTICS_ENDPOINT,
+            settings.ANALYTICS_ENDPOINT,
-            utils.deobfuscate(conf.ANALYTICS_TOKEN)
+            utils.deobfuscate(settings.ANALYTICS_TOKEN)
         )

lbrynet/conf.py

@@ -1,78 +1,236 @@
-"""
+import copy
-Some network wide and also application specific parameters
-"""
 import os
+import sys
+from appdirs import user_data_dir
 
-is_generous_host = True
+LINUX = 1
-IS_DEVELOPMENT_VERSION = (os.environ.get('LBRY_DEV') is not None)
+DARWIN = 2
+WINDOWS = 3
 
-
+if sys.platform.startswith("darwin"):
-MAX_HANDSHAKE_SIZE = 2**16
+    platform = DARWIN
-MAX_REQUEST_SIZE = 2**16
+    default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
-MAX_BLOB_REQUEST_SIZE = 2**16
+    default_data_dir = user_data_dir("LBRY")
-MAX_RESPONSE_INFO_SIZE = 2**16
+    default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum")
-MAX_BLOB_INFOS_TO_REQUEST = 20
+elif sys.platform.startswith("win"):
-BLOBFILES_DIR = ".blobfiles"
+    platform = WINDOWS
-BLOB_SIZE = 2**21
+    from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
-
+    default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current)
-MIN_BLOB_DATA_PAYMENT_RATE = .0001  # points/megabyte
+    default_data_dir = os.path.join(
-MIN_BLOB_INFO_PAYMENT_RATE = .02  # points/1000 infos
+        get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrynet")
-MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05  # points/1000 infos
+    default_lbryum_dir = os.path.join(
-MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .05  # points/1000 infos
+        get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum")
-MAX_CONNECTIONS_PER_STREAM = 5
-
-KNOWN_DHT_NODES = [('104.236.42.182', 4000),
-                   ('lbrynet1.lbry.io', 4444),
-                   ('lbrynet2.lbry.io', 4444),
-                   ('lbrynet3.lbry.io', 4444)]
-
-POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424'
-#POINTTRADER_SERVER = 'http://127.0.0.1:2424'
-
-
-SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005",
-                    "http://lighthouse2.lbry.io:50005",
-                    "http://lighthouse3.lbry.io:50005"]
-
-REFLECTOR_SERVERS = [("reflector.lbry.io", 5566)]
-
-LOG_FILE_NAME = "lbrynet.log"
-LOG_POST_URL = "https://lbry.io/log-upload"
-
-CRYPTSD_FILE_EXTENSION = ".cryptsd"
-
-API_INTERFACE = "localhost"
-API_ADDRESS = "lbryapi"
-API_PORT = 5279
-if os.name == "nt":
-    ICON_PATH = "icons"
 else:
-    ICON_PATH = "app.icns"
+    platform = LINUX
-APP_NAME = "LBRY"
+    default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
-API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRESS)
+    default_data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
-UI_ADDRESS = "http://%s:%i" % (API_INTERFACE, API_PORT)
+    default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum")
-PROTOCOL_PREFIX = "lbry"
 
-DEFAULT_WALLET = "lbryum"
-WALLET_TYPES = ["lbryum", "lbrycrd"]
-DEFAULT_TIMEOUT = 30
-DEFAULT_MAX_SEARCH_RESULTS = 25
-DEFAULT_MAX_KEY_FEE = {'USD': {'amount': 25.0, 'address': ''}}
-DEFAULT_SEARCH_TIMEOUT = 3.0
-DEFAULT_SD_DOWNLOAD_TIMEOUT = 3
-DEFAULT_CACHE_TIME = 150
-DEFAULT_UI_BRANCH = "master"
 
-SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
+def convert_setting(env_val, current_val):
-CURRENCIES = {
+    new_type = env_val.__class__
-                'BTC': {'type': 'crypto'},
+    current_type = current_val.__class__
-                'LBC': {'type': 'crypto'},
+    if current_type is bool:
-                'USD': {'type': 'fiat'},
+        if new_type is bool:
-             }
+            return env_val
+        elif str(env_val).lower() == "false":
+            return False
+        elif str(env_val).lower() == "true":
+            return True
+        else:
+            raise ValueError
+    elif current_type is int:
+        return int(env_val)
+    elif current_type is float:
+        return float(env_val)
+    elif current_type is str:
+        return str(env_val)
+    elif current_type is dict:
+        return dict(env_val)
+    elif current_type is list:
+        return list(env_val)
+    elif current_type is tuple:
+        return tuple(env_val)
+    else:
+        raise ValueError('Type {} cannot be converted'.format(current_type))
 
-LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv'
 
-ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
+def convert_env_setting(setting, value):
-ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
+    try:
+        env_val = os.environ[setting]
+    except KeyError:
+        return value
+    else:
+        return convert_setting(env_val, value)
 
-LBRYUM_WALLET_DIR = os.environ.get('LBRYUM_WALLET_DIR')
+
+def get_env_settings(settings):
+    for setting, value in settings.iteritems():
+        setting = 'LBRY_' + setting.upper()
+        yield convert_env_setting(setting, value)
+
+
+def add_env_settings_to_dict(settings_dict):
+    for setting, env_setting in zip(settings_dict, get_env_settings(settings_dict)):
+        settings_dict.update({setting: env_setting})
+    return settings_dict
+
+
+class Setting(object):
+    __fixed = []
+    __excluded = ['get_dict', 'update']
+
+    def __iter__(self):
+        for k in self.__dict__.iterkeys():
+            if k.startswith('_') or k in self.__excluded:
+                continue
+            yield k
+
+    def __getitem__(self, item):
+        assert item in self, IndexError
+        return self.__dict__[item]
+
+    def __setitem__(self, key, value):
+        assert key in self and key not in self.__fixed, KeyError(key)
+        _value = convert_setting(value, self[key])
+        self.__dict__.update({key: _value})
+
+    def __contains__(self, item):
+        return item in iter(self)
+
+    def get_dict(self):
+        return {k: self[k] for k in self}
+
+    def update(self, other):
+        for k, v in other.iteritems():
+            try:
+                self.__setitem__(k, v)
+            except KeyError:
+                pass
+            except AssertionError:
+                pass
+
+
+class AdjustableSettings(Setting):
+    def __init__(self):
+        self.is_generous_host = True
+        self.run_on_startup = False
+        self.download_directory = default_download_directory
+        self.max_upload = 0.0
+        self.max_download = 0.0
+        self.upload_log = True
+        self.delete_blobs_on_remove = True
+        self.use_upnp = True
+        self.start_lbrycrdd = True
+        self.run_reflector_server = False
+        self.startup_scripts = []
+        self.last_version = {'lbrynet': '0.0.1', 'lbryum': '0.0.1'}
+        self.peer_port = 3333
+        self.dht_node_port = 4444
+        self.reflector_port = 5566
+        self.download_timeout = 30
+        self.max_search_results = 25
+        self.search_timeout = 3.0
+        self.cache_time = 150
+        self.host_ui = True
+        self.check_ui_requirements = True
+        self.local_ui_path = False
+        self.api_port = 5279
+        self.search_servers = ['lighthouse1.lbry.io:50005']
+        self.data_rate = .0001  # points/megabyte
+        self.min_info_rate = .02  # points/1000 infos
+        self.min_valuable_info_rate = .05  # points/1000 infos
+        self.min_valuable_hash_rate = .05  # points/1000 infos
+        self.max_connections_per_stream = 5
+        self.known_dht_nodes = [
+            ('104.236.42.182', 4000),
+            ('lbrynet1.lbry.io', 4444),
+            ('lbrynet2.lbry.io', 4444),
+            ('lbrynet3.lbry.io', 4444)
+        ]
+        self.pointtrader_server = 'http://127.0.0.1:2424'
+        self.reflector_servers = [("reflector.lbry.io", 5566)]
+        self.wallet = "lbryum"
+        self.ui_branch = "master"
+        self.default_ui_branch = 'master'
+        self.data_dir = default_data_dir
+        self.lbryum_wallet_dir = default_lbryum_dir
+        self.use_auth_http = False
+        self.sd_download_timeout = 3
+        self.max_key_fee = {'USD': {'amount': 25.0, 'address': ''}}
+
+
+class ApplicationSettings(Setting):
+    def __init__(self):
+        self.MAX_HANDSHAKE_SIZE = 2**16
+        self.MAX_REQUEST_SIZE = 2**16
+        self.MAX_BLOB_REQUEST_SIZE = 2**16
+        self.MAX_RESPONSE_INFO_SIZE = 2**16
+        self.MAX_BLOB_INFOS_TO_REQUEST = 20
+        self.BLOBFILES_DIR = "blobfiles"
+        self.BLOB_SIZE = 2**21
+        self.LOG_FILE_NAME = "lbrynet.log"
+        self.LOG_POST_URL = "https://lbry.io/log-upload"
+        self.CRYPTSD_FILE_EXTENSION = ".cryptsd"
+        self.API_INTERFACE = "localhost"
+        self.API_ADDRESS = "lbryapi"
+        self.ICON_PATH = "icons" if platform is WINDOWS else "app.icns"
+        self.APP_NAME = "LBRY"
+        self.PROTOCOL_PREFIX = "lbry"
+        self.wallet_TYPES = ["lbryum", "lbrycrd"]
+        self.SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
+        self.CURRENCIES = {
+            'BTC': {'type': 'crypto'},
+            'LBC': {'type': 'crypto'},
+            'USD': {'type': 'fiat'},
+        }
+        self.LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv'
+        self.ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
+        self.ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
+
+
+APPLICATION_SETTINGS = AdjustableSettings()
+ADJUSTABLE_SETTINGS = AdjustableSettings()
+
+
+class DefaultSettings(ApplicationSettings, AdjustableSettings):
+    __fixed = APPLICATION_SETTINGS.get_dict().keys()
+
+    def __init__(self):
+        ApplicationSettings.__init__(self)
+        AdjustableSettings.__init__(self)
+
+
+DEFAULT_SETTINGS = DefaultSettings()
+
+
+class Config(DefaultSettings):
+    __shared_state = copy.deepcopy(DEFAULT_SETTINGS.get_dict())
+
+    def __init__(self):
+        self.__dict__ = add_env_settings_to_dict(self.__shared_state)
+
+    @property
+    def ORIGIN(self):
+        return "http://%s:%i" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port)
+
+    @property
+    def REFERER(self):
+        return "http://%s:%i/" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port)
+
+    @property
+    def API_CONNECTION_STRING(self):
+        return "http://%s:%i/%s" % (
+            DEFAULT_SETTINGS.API_INTERFACE, self.api_port, DEFAULT_SETTINGS.API_ADDRESS)
+
+    @property
+    def UI_ADDRESS(self):
+        return "http://%s:%i" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port)
+
+
+# TODO: don't load the configuration automatically. The configuration
+#       should be loaded at runtime, not at module import time. Module
+#       import should have no side-effects. This is also bad because
+#       it means that settings are read from the environment even for
+#       tests, which is rarely what you want to happen.
+settings = Config()

lbrynet/core/Error.py

@@ -90,5 +90,17 @@ class InvalidBlobHashError(Exception):
     pass
 
 
+class InvalidHeaderError(Exception):
+    pass
+
+
+class InvalidAuthenticationToken(Exception):
+    pass
+
+
+class SubhandlerError(Exception):
+    pass
+
+
 class NegotiationError(Exception):
-    pass
+    pass

lbrynet/core/HashBlob.py

@@ -8,7 +8,7 @@ from twisted.internet import interfaces, defer, threads
 from twisted.protocols.basic import FileSender
 from twisted.python.failure import Failure
 from zope.interface import implements
-from lbrynet.conf import BLOB_SIZE
+from lbrynet.conf import settings
 from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
 from lbrynet.core.cryptoutils import get_lbry_hash_obj
 
@@ -87,7 +87,7 @@ class HashBlob(object):
     def set_length(self, length):
         if self.length is not None and length == self.length:
             return True
-        if self.length is None and 0 <= length <= BLOB_SIZE:
+        if self.length is None and 0 <= length <= settings.BLOB_SIZE:
             self.length = length
             return True
         log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))

lbrynet/core/PaymentRateManager.py

@@ -1,10 +1,10 @@
 from lbrynet.core.Strategy import get_default_strategy
-from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, MIN_BLOB_INFO_PAYMENT_RATE, is_generous_host
+from lbrynet.conf import settings
 from decimal import Decimal
 
 
 class BasePaymentRateManager(object):
-    def __init__(self, rate=MIN_BLOB_DATA_PAYMENT_RATE, info_rate=MIN_BLOB_INFO_PAYMENT_RATE):
+    def __init__(self, rate=settings.data_rate, info_rate=settings.min_info_rate):
         self.min_blob_data_payment_rate = rate
         self.min_blob_info_payment_rate = info_rate
 
@@ -36,7 +36,7 @@ class PaymentRateManager(object):
 
 
 class NegotiatedPaymentRateManager(object):
-    def __init__(self, base, availability_tracker, generous=is_generous_host):
+    def __init__(self, base, availability_tracker, generous=settings.is_generous_host):
         """
         @param base: a BasePaymentRateManager
         @param availability_tracker: a BlobAvailabilityTracker

lbrynet/core/PriceModel.py

@@ -2,7 +2,7 @@ from zope.interface import implementer
 from decimal import Decimal
 
 from lbrynet.interfaces import IBlobPriceModel
-from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE
+from lbrynet.conf import settings
 
 
 def get_default_price_model(blob_tracker, base_price, **kwargs):
@@ -21,7 +21,7 @@ class MeanAvailabilityWeightedPrice(object):
     """
     implementer(IBlobPriceModel)
 
-    def __init__(self, tracker, base_price=MIN_BLOB_DATA_PAYMENT_RATE, alpha=1.0):
+    def __init__(self, tracker, base_price=settings.data_rate, alpha=1.0):
         self.blob_tracker = tracker
         self.base_price = Decimal(base_price)
         self.alpha = Decimal(alpha)

lbrynet/core/Strategy.py

@@ -1,6 +1,6 @@
 from zope.interface import implementer
 from decimal import Decimal
-from lbrynet.conf import is_generous_host
+from lbrynet.conf import settings
 from lbrynet.interfaces import INegotiationStrategy
 from lbrynet.core.Offer import Offer
 from lbrynet.core.PriceModel import MeanAvailabilityWeightedPrice
@@ -16,7 +16,7 @@ class Strategy(object):
     """
     implementer(INegotiationStrategy)
 
-    def __init__(self, price_model, max_rate, min_rate, is_generous=is_generous_host):
+    def __init__(self, price_model, max_rate, min_rate, is_generous=settings.is_generous_host):
         self.price_model = price_model
         self.is_generous = is_generous
         self.accepted_offers = {}
@@ -101,7 +101,7 @@ class BasicAvailabilityWeightedStrategy(Strategy):
     implementer(INegotiationStrategy)
 
     def __init__(self, blob_tracker, acceleration=1.25, deceleration=0.9, max_rate=None, min_rate=0.0,
-                                    is_generous=is_generous_host, base_price=0.0001, alpha=1.0):
+                                    is_generous=settings.is_generous_host, base_price=0.0001, alpha=1.0):
         price_model = MeanAvailabilityWeightedPrice(blob_tracker, base_price=base_price, alpha=alpha)
         Strategy.__init__(self, price_model, max_rate, min_rate, is_generous)
         self._acceleration = Decimal(acceleration)  # rate of how quickly to ramp offer

lbrynet/core/Wallet.py

@@ -153,7 +153,12 @@ class Wallet(object):
                     log.info("Got a new balance: %s", str(balance))
                 self.wallet_balance = balance
 
-            d.addCallback(set_wallet_balance)
+            def log_error(err):
+                if isinstance(err, AttributeError):
+                    log.warning("Failed to get an updated balance")
+                    log.warning("Last balance update: %s", str(self.wallet_balance))
+
+            d.addCallbacks(set_wallet_balance, log_error)
             return d
 
         d.addCallback(lambda should_run: do_manage() if should_run else None)
@@ -1169,7 +1174,8 @@ class LBRYumWallet(Wallet):
             self._start_check = None
 
         if self._catch_up_check is not None:
-            self._catch_up_check.stop()
+            if self._catch_up_check.running:
+                self._catch_up_check.stop()
             self._catch_up_check = None
 
         d = defer.Deferred()
@@ -1241,6 +1247,9 @@ class LBRYumWallet(Wallet):
 
                 self._caught_up_counter += 1
 
+        def log_error(err):
+            log.warning(err.getErrorMessage())
+            return defer.fail(err)
 
         self._catch_up_check = task.LoopingCall(check_caught_up)
 
@@ -1248,6 +1257,7 @@ class LBRYumWallet(Wallet):
         d.addCallback(self._save_wallet)
         d.addCallback(lambda _: self.wallet.start_threads(self.network))
         d.addCallback(lambda _: self._catch_up_check.start(.1))
+        d.addErrback(log_error)
         d.addCallback(lambda _: blockchain_caught_d)
         return d
 

lbrynet/core/client/ClientProtocol.py

@@ -4,7 +4,7 @@ from decimal import Decimal
 from twisted.internet import error, defer
 from twisted.internet.protocol import Protocol, ClientFactory
 from twisted.python import failure
-from lbrynet.conf import MAX_RESPONSE_INFO_SIZE as MAX_RESPONSE_SIZE
+from lbrynet.conf import settings
 from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError
 from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError
 from lbrynet.core.Error import RequestCanceledError
@@ -48,7 +48,7 @@ class ClientProtocol(Protocol):
             self._blob_download_request.write(data)
         else:
             self._response_buff += data
-            if len(self._response_buff) > MAX_RESPONSE_SIZE:
+            if len(self._response_buff) > settings.MAX_RESPONSE_INFO_SIZE:
                 log.warning("Response is too large. Size %s", len(self._response_buff))
                 self.transport.loseConnection()
             response, extra_data = self._get_valid_response(self._response_buff)

lbrynet/core/client/ClientRequest.py

@@ -1,4 +1,4 @@
-from lbrynet.conf import BLOB_SIZE
+from lbrynet.conf import settings
 
 
 class ClientRequest(object):
@@ -17,7 +17,7 @@ class ClientBlobRequest(ClientPaidRequest):
     def __init__(self, request_dict, response_identifier, write_func, finished_deferred,
                  cancel_func, blob):
         if blob.length is None:
-            max_pay_units = BLOB_SIZE
+            max_pay_units = settings.BLOB_SIZE
         else:
             max_pay_units = blob.length
         ClientPaidRequest.__init__(self, request_dict, response_identifier, max_pay_units)

lbrynet/core/client/ConnectionManager.py

@@ -2,7 +2,7 @@ import logging
 from twisted.internet import defer
 from zope.interface import implements
 from lbrynet import interfaces
-from lbrynet.conf import MAX_CONNECTIONS_PER_STREAM
+from lbrynet.conf import settings
 from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
 from lbrynet.core.Error import InsufficientFundsError
 
@@ -183,7 +183,7 @@ class ConnectionManager(object):
             log.debug("Couldn't find a good peer to connect to")
             return None
 
-        if len(self._peer_connections) < MAX_CONNECTIONS_PER_STREAM:
+        if len(self._peer_connections) < settings.max_connections_per_stream:
             ordered_request_creators = self._rank_request_creator_connections()
             d = get_new_peers(ordered_request_creators)
             d.addCallback(pick_best_peer)

lbrynet/core/log_support.py

@@ -7,7 +7,7 @@ import traceback
 from requests_futures.sessions import FuturesSession
 
 import lbrynet
-from lbrynet import conf
+from lbrynet.conf import settings
 from lbrynet.core import utils
 
 session = FuturesSession()
@@ -104,7 +104,7 @@ def configure_file_handler(file_name, **kwargs):
 
 
 def get_loggly_url(token=None, version=None):
-    token = token or utils.deobfuscate(conf.LOGGLY_TOKEN)
+    token = token or utils.deobfuscate(settings.LOGGLY_TOKEN)
     version = version or lbrynet.__version__
     return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
 

lbrynet/core/utils.py

@@ -8,6 +8,8 @@ import os
 import socket
 import yaml
 
+from lbrynet.conf import settings
+from lbrynet.conf import AdjustableSettings
 from lbrynet.core.cryptoutils import get_lbry_hash_obj
 
 
@@ -85,24 +87,25 @@ settings_encoders = {
     '.yml': yaml.safe_dump
 }
 
+ADJUSTABLE_SETTINGS = AdjustableSettings().get_dict()
+
 
 def load_settings(path):
     ext = os.path.splitext(path)[1]
-    f = open(path, 'r')
+    with open(path, 'r') as settings_file:
-    data = f.read()
+        data = settings_file.read()
-    f.close()
     decoder = settings_decoders.get(ext, False)
     assert decoder is not False, "Unknown settings format .%s" % ext
     return decoder(data)
 
 
-def save_settings(path, settings):
+def save_settings(path):
+    to_save = {k: v for k, v in settings.__dict__.iteritems() if k in ADJUSTABLE_SETTINGS}
     ext = os.path.splitext(path)[1]
     encoder = settings_encoders.get(ext, False)
     assert encoder is not False, "Unknown settings format .%s" % ext
-    f = open(path, 'w')
+    with open(path, 'w') as settings_file:
-    f.write(encoder(settings))
+        settings_file.write(encoder(to_save))
-    f.close()
 
 
 def check_connection(server="www.lbry.io", port=80):

lbrynet/cryptstream/CryptBlob.py

@@ -1,7 +1,7 @@
 import binascii
 import logging
 from Crypto.Cipher import AES
-from lbrynet.conf import BLOB_SIZE
+from lbrynet.conf import settings
 from lbrynet.core.BlobInfo import BlobInfo
 
 
@@ -67,7 +67,7 @@ class CryptStreamBlobMaker(object):
         self.length = 0
 
     def write(self, data):
-        max_bytes_to_write = BLOB_SIZE - self.length - 1
+        max_bytes_to_write = settings.BLOB_SIZE - self.length - 1
         done = False
         if max_bytes_to_write <= len(data):
             num_bytes_to_write = max_bytes_to_write

lbrynet/lbryfilemanager/EncryptedFileCreator.py

@@ -7,7 +7,7 @@ import logging
 import os
 from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter
 from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
-from lbrynet import conf
+from lbrynet.conf import settings
 from lbrynet.lbryfile.StreamDescriptor import get_sd_info
 from lbrynet.core.cryptoutils import get_lbry_hash_obj
 from twisted.protocols.basic import FileSender
@@ -130,7 +130,7 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
 
     def make_stream_desc_file(stream_hash):
         log.debug("creating the stream descriptor file")
-        descriptor_file_path = os.path.join(session.db_dir, file_name + conf.CRYPTSD_FILE_EXTENSION)
+        descriptor_file_path = os.path.join(session.db_dir, file_name + settings.CRYPTSD_FILE_EXTENSION)
         descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path)
 
         d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True)

lbrynet/lbrylive/LiveStreamCreator.py

@@ -6,7 +6,7 @@ from lbrynet.core.cryptoutils import get_lbry_hash_obj, get_pub_key, sign_with_p
 from Crypto import Random
 import binascii
 import logging
-from lbrynet.conf import CRYPTSD_FILE_EXTENSION
+from lbrynet.conf import settings
 from twisted.internet import interfaces, defer
 from twisted.protocols.basic import FileSender
 from zope.interface import implements
@@ -23,7 +23,7 @@ class LiveStreamCreator(CryptStreamCreator):
         self.stream_info_manager = stream_info_manager
         self.delete_after_num = delete_after_num
         self.secret_pass_phrase = secret_pass_phrase
-        self.file_extension = CRYPTSD_FILE_EXTENSION
+        self.file_extension = settings.CRYPTSD_FILE_EXTENSION
         self.finished_blob_hashes = {}
 
     def _save_stream(self):

lbrynet/lbrylive/client/LiveStreamMetadataHandler.py

@@ -3,7 +3,7 @@ import logging
 from zope.interface import implements
 from twisted.internet import defer
 from twisted.python.failure import Failure
-from lbrynet.conf import MAX_BLOB_INFOS_TO_REQUEST
+from lbrynet.conf import settings
 from lbrynet.core.client.ClientRequest import ClientRequest, ClientPaidRequest
 from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
 from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
@@ -136,7 +136,7 @@ class LiveStreamMetadataHandler(object):
             if count is not None:
                 further_blobs_request['count'] = count
             else:
-                further_blobs_request['count'] = MAX_BLOB_INFOS_TO_REQUEST
+                further_blobs_request['count'] = settings.MAX_BLOB_INFOS_TO_REQUEST
             log.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer))
             r_dict = {'further_blobs': further_blobs_request}
             response_identifier = 'further_blobs'

lbrynet/lbrynet_console/Console.py

@@ -12,14 +12,14 @@ if sys.platform == "darwin":
     from appdirs import user_data_dir
 from yapsy.PluginManager import PluginManager
 from twisted.internet import defer, threads, stdio, task, error
-from jsonrpc.proxy import JSONRPCProxy
+from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient
 
 from lbrynet import analytics
 from lbrynet.core.Session import Session
 from lbrynet.lbrynet_console.ConsoleControl import ConsoleControl
 from lbrynet.lbrynet_console.Settings import Settings
 from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager
-from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, API_CONNECTION_STRING  # , MIN_BLOB_INFO_PAYMENT_RATE
+from lbrynet.conf import settings
 from lbrynet.core.utils import generate_id
 from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
 from lbrynet.core.PaymentRateManager import PaymentRateManager
@@ -212,7 +212,7 @@ class Console():
     def _get_session(self):
         def get_default_data_rate():
             d = self.settings.get_default_data_payment_rate()
-            d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else MIN_BLOB_DATA_PAYMENT_RATE})
+            d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else settings.data_rate})
             return d
 
         def get_wallet():
@@ -542,7 +542,7 @@ def launch_lbry_console():
         os.mkdir(data_dir)
         created_data_dir = True
 
-    daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING)
+    daemon = LBRYAPIClient.config()
     try:
         daemon.is_running()
         log.info("Attempt to start lbrynet-console while lbrynet-daemon is running")

lbrynet/lbrynet_console/plugins/BlindRepeater/__init__.py

@@ -1,6 +1,6 @@
 from lbrynet.lbrynet_console import Plugin
 from twisted.internet import defer
-from lbrynet.conf import MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE, MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE
+from lbrynet.conf import settings
 from BlindRepeater import BlindRepeater
 from BlindInfoManager import BlindInfoManager
 from BlindRepeaterSettings import BlindRepeaterSettings
@@ -59,9 +59,9 @@ class BlindRepeaterPlugin(Plugin.Plugin):
         def get_payment_rate_manager(rates):
             data_rate = rates[0][1] if rates[0][0] is True else None
             info_rate = rates[1][1] if rates[1][0] is True else None
-            info_rate = info_rate if info_rate is not None else MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE
+            info_rate = info_rate if info_rate is not None else settings.min_valuable_info_rate
             hash_rate = rates[2][1] if rates[2][0] is True else None
-            hash_rate = hash_rate if hash_rate is not None else MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE
+            hash_rate = hash_rate if hash_rate is not None else settings.min_valuable_hash_rate
             self.payment_rate_manager = BlindRepeaterPaymentRateManager(default_payment_rate_manager,
                                                                         info_rate, hash_rate,
                                                                         blob_data_rate=data_rate)

lbrynet/lbrynet_daemon/DaemonCLI.py

@@ -2,8 +2,8 @@ import sys
 import json
 import argparse
 
-from lbrynet.conf import API_CONNECTION_STRING
+from lbrynet.conf import settings
-from jsonrpc.proxy import JSONRPCProxy
+from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient
 
 help_msg = "Usage: lbrynet-cli method json-args\n" \
              + "Examples: " \
@@ -36,13 +36,20 @@ def get_params_from_kwargs(params):
 
 
 def main():
-    api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
+    api = LBRYAPIClient.config()
 
     try:
-        s = api.is_running()
+        status = api.daemon_status()
-    except:
+        assert status.get('code', False) == "started"
-        print "lbrynet-daemon isn't running"
+    except Exception:
-        sys.exit(1)
+        try:
+            settings.update({'use_auth_http': not settings.use_auth_http})
+            api = LBRYAPIClient.config()
+            status = api.daemon_status()
+            assert status.get('code', False) == "started"
+        except Exception:
+            print "lbrynet-daemon isn't running"
+            sys.exit(1)
 
     parser = argparse.ArgumentParser()
     parser.add_argument('method', nargs=1)
@@ -72,16 +79,17 @@ def main():
     if meth in api.help():
         try:
             if params:
-                resp = api.call(meth, params)
+                result = LBRYAPIClient.config(service=meth, params=params)
             else:
-                resp = api.call(meth)
+                result = LBRYAPIClient.config(service=meth, params=params)
-            print json.dumps(resp, sort_keys=True)
+            print json.dumps(result, sort_keys=True)
-        except Exception:
+        except:
             # TODO: The api should return proper error codes
             # and messages so that they can be passed along to the user
             # instead of this generic message.
             # https://app.asana.com/0/158602294500137/200173944358192
-            print "Something went wrong. Here's the usage for {}:".format(meth)
+
+            print "Something went wrong, here's the usage for %s:" % meth
             print api.help({'function': meth})
     else:
         print "Unknown function"

lbrynet/lbrynet_daemon/DaemonControl.py

@@ -1,32 +1,29 @@
 import argparse
-import logging
 import logging.handlers
 import os
 import webbrowser
 import sys
-from appdirs import user_data_dir
 
-from twisted.web import server
+from twisted.web import server, guard
-from twisted.internet import reactor, defer
+from twisted.internet import defer, reactor
+from twisted.cred import portal
+
 from jsonrpc.proxy import JSONRPCProxy
 
+from lbrynet.lbrynet_daemon.auth.auth import PasswordChecker, HttpPasswordRealm
+from lbrynet.lbrynet_daemon.auth.util import initialize_api_key_file
 from lbrynet.core import log_support
 from lbrynet.core import utils
 from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer
 from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest
-from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_PORT, \
+from lbrynet.conf import settings
-                         UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
 
-# TODO: stop it!
+log_dir = settings.data_dir
-if sys.platform != "darwin":
-    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
-else:
-    log_dir = user_data_dir("LBRY")
 
 if not os.path.isdir(log_dir):
     os.mkdir(log_dir)
 
-lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
+lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 
@@ -48,7 +45,7 @@ def stop():
         log.info("Attempt to shut down lbrynet-daemon from command line when daemon isn't running")
 
     d = defer.Deferred(None)
-    d.addCallback(lambda _: JSONRPCProxy.from_url(API_CONNECTION_STRING).stop())
+    d.addCallback(lambda _: JSONRPCProxy.from_url(settings.API_CONNECTION_STRING).stop())
     d.addCallbacks(lambda _: _disp_shutdown(), lambda _: _disp_not_running())
     d.callback(None)
 
@@ -58,18 +55,37 @@ def start():
     parser.add_argument("--wallet",
                         help="lbrycrd or lbryum, default lbryum",
                         type=str,
-                        default='')
+                        default='lbryum')
+
     parser.add_argument("--ui",
                         help="path to custom UI folder",
                         default=None)
+
     parser.add_argument("--branch",
-                        help="Branch of lbry-web-ui repo to use, defaults on master")
+                        help="Branch of lbry-web-ui repo to use, defaults on master",
-    parser.add_argument('--no-launch', dest='launchui', action="store_false")
+                        default=settings.ui_branch)
-    parser.add_argument('--log-to-console', dest='logtoconsole', action="store_true")
+
-    parser.add_argument('--quiet', dest='quiet', action="store_true")
+    parser.add_argument("--http-auth",
-    parser.add_argument('--verbose', action='store_true',
+                        dest="useauth",
+                        action="store_true")
+
+    parser.add_argument('--no-launch',
+                        dest='launchui',
+                        action="store_false")
+
+    parser.add_argument('--log-to-console',
+                        dest='logtoconsole',
+                        action="store_true")
+
+    parser.add_argument('--quiet',
+                        dest='quiet',
+                        action="store_true")
+
+    parser.add_argument('--verbose',
+                        action='store_true',
                         help='enable more debug output for the console')
-    parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False)
+
+    parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False, useauth=settings.use_auth_http)
     args = parser.parse_args()
 
     log_support.configure_file_handler(lbrynet_log)
@@ -80,13 +96,27 @@ def start():
     if not args.verbose:
         log_support.disable_noisy_loggers()
 
+    to_pass = {}
+    settings_path = os.path.join(settings.data_dir, "daemon_settings.yml")
+    if os.path.isfile(settings_path):
+        to_pass.update(utils.load_settings(settings_path))
+        log.info("Loaded settings file")
+    if args.ui:
+        to_pass.update({'local_ui_path': args.ui})
+    if args.branch:
+        to_pass.update({'ui_branch': args.branch})
+    to_pass.update({'use_auth_http': args.useauth})
+    to_pass.update({'wallet': args.wallet})
+    print to_pass
+    settings.update(to_pass)
+
     try:
-        JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()
+        JSONRPCProxy.from_url(settings.API_CONNECTION_STRING).is_running()
         log.info("lbrynet-daemon is already running")
         if not args.logtoconsole:
             print "lbrynet-daemon is already running"
         if args.launchui:
-            webbrowser.open(UI_ADDRESS)
+            webbrowser.open(settings.UI_ADDRESS)
         return
     except:
         pass
@@ -96,24 +126,34 @@ def start():
     if not args.logtoconsole and not args.quiet:
         print "Starting lbrynet-daemon from command line"
         print "To view activity, view the log file here: " + lbrynet_log
-        print "Web UI is available at http://%s:%i" % (API_INTERFACE, API_PORT)
+        print "Web UI is available at http://%s:%i" % (settings.API_INTERFACE, settings.api_port)
-        print "JSONRPC API is available at " + API_CONNECTION_STRING
+        print "JSONRPC API is available at " + settings.API_CONNECTION_STRING
         print "To quit press ctrl-c or call 'stop' via the API"
 
     if test_internet_connection():
         lbry = DaemonServer()
 
-        d = lbry.start(branch=args.branch if args.branch else DEFAULT_UI_BRANCH,
+        d = lbry.start()
-                       user_specified=args.ui,
-                       wallet=args.wallet,
-                       branch_specified=True if args.branch else False)
         if args.launchui:
-            d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
+            d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS))
         d.addErrback(log_and_kill)
 
-        lbrynet_server = server.Site(lbry.root)
+        if settings.use_auth_http:
+            log.info("Using authenticated API")
+            pw_path = os.path.join(settings.data_dir, ".api_keys")
+            initialize_api_key_file(pw_path)
+            checker = PasswordChecker.load_file(pw_path)
+            realm = HttpPasswordRealm(lbry.root)
+            portal_to_realm = portal.Portal(realm, [checker, ])
+            factory = guard.BasicCredentialFactory('Login to lbrynet api')
+            _lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ])
+        else:
+            log.info("Using non-authenticated API")
+            _lbrynet_server = server.Site(lbry.root)
+
+        lbrynet_server = server.Site(_lbrynet_server)
         lbrynet_server.requestFactory = DaemonRequest
-        reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE)
+        reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE)
         reactor.run()
 
         if not args.logtoconsole and not args.quiet:

lbrynet/lbrynet_daemon/DaemonServer.py

@@ -6,7 +6,7 @@ from appdirs import user_data_dir
 from twisted.internet import defer
 from lbrynet.lbrynet_daemon.Daemon import Daemon
 from lbrynet.lbrynet_daemon.Resources import LBRYindex, HostedEncryptedFile, EncryptedFileUpload
-from lbrynet.conf import API_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
+from lbrynet.conf import settings
 
 
 # TODO: omg, this code is essentially duplicated in Daemon
@@ -17,21 +17,20 @@ else:
 if not os.path.isdir(data_dir):
     os.mkdir(data_dir)
 
-lbrynet_log = os.path.join(data_dir, LOG_FILE_NAME)
+lbrynet_log = os.path.join(data_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 
 class DaemonServer(object):
-    def _setup_server(self, wallet):
+    def _setup_server(self):
         self.root = LBRYindex(os.path.join(os.path.join(data_dir, "lbry-ui"), "active"))
-        self._api = Daemon(self.root, wallet_type=wallet)
+        self._api = Daemon(self.root)
         self.root.putChild("view", HostedEncryptedFile(self._api))
         self.root.putChild("upload", EncryptedFileUpload(self._api))
-        self.root.putChild(API_ADDRESS, self._api)
+        self.root.putChild(settings.API_ADDRESS, self._api)
         return defer.succeed(True)
 
-    def start(self, branch=DEFAULT_UI_BRANCH, user_specified=False,
+    def start(self):
-              branch_specified=False, wallet=None):
+        d = self._setup_server()
-        d = self._setup_server(wallet)
+        d.addCallback(lambda _: self._api.setup())
-        d.addCallback(lambda _: self._api.setup(branch, user_specified, branch_specified))
         return d

lbrynet/lbrynet_daemon/Downloader.py

@@ -12,7 +12,7 @@ from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed
 from lbrynet.core.StreamDescriptor import download_sd_blob
 from lbrynet.metadata.Fee import FeeValidator
 from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
-from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME
+from lbrynet.conf import settings
 
 INITIALIZING_CODE = 'initializing'
 DOWNLOAD_METADATA_CODE = 'downloading_metadata'
@@ -35,13 +35,13 @@ else:
 if not os.path.isdir(log_dir):
     os.mkdir(log_dir)
 
-lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
+lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 
 class GetStream(object):
     def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager,
-                 max_key_fee, data_rate=0.5, timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None):
+                 max_key_fee, data_rate=0.5, timeout=settings.download_timeout, download_directory=None, file_name=None):
         self.wallet = wallet
         self.resolved_name = None
         self.description = None

lbrynet/lbrynet_daemon/Lighthouse.py

@@ -1,14 +1,14 @@
 import logging
 import random
 from txjsonrpc.web.jsonrpc import Proxy
-from lbrynet.conf import SEARCH_SERVERS
+from lbrynet.conf import settings
 
 log = logging.getLogger(__name__)
 
 
 class LighthouseClient(object):
     def __init__(self, servers=None):
-        self.servers = servers or SEARCH_SERVERS
+        self.servers = servers or settings.search_servers
 
     def _get_random_server(self):
         return Proxy(random.choice(self.servers))

lbrynet/lbrynet_daemon/Publisher.py

@@ -12,7 +12,7 @@ from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
 from lbrynet.metadata.Metadata import Metadata
 from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
 from lbrynet import reflector
-from lbrynet.conf import LOG_FILE_NAME, REFLECTOR_SERVERS
+from lbrynet.conf import settings
 from twisted.internet import threads, defer, reactor
 
 if sys.platform != "darwin":
@@ -23,7 +23,7 @@ else:
 if not os.path.isdir(log_dir):
     os.mkdir(log_dir)
 
-lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
+lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 
@@ -41,7 +41,7 @@ class Publisher(object):
         self.lbry_file = None
         self.txid = None
         self.stream_hash = None
-        reflector_server = random.choice(REFLECTOR_SERVERS)
+        reflector_server = random.choice(settings.reflector_servers)
         self.reflector_server, self.reflector_port = reflector_server[0], reflector_server[1]
         self.metadata = {}
 
@@ -74,7 +74,7 @@ class Publisher(object):
         return d
 
     def start_reflector(self):
-        reflector_server = random.choice(REFLECTOR_SERVERS)
+        reflector_server = random.choice(settings.reflector_servers)
         reflector_address, reflector_port = reflector_server[0], reflector_server[1]
         log.info("Reflecting new publication")
         factory = reflector.ClientFactory(

lbrynet/lbrynet_daemon/Resources.py

@@ -10,7 +10,7 @@ from appdirs import user_data_dir
 from twisted.web import server, static, resource
 from twisted.internet import defer, error
 
-from lbrynet.conf import UI_ADDRESS
+from lbrynet.conf import settings
 from lbrynet.lbrynet_daemon.FileStreamer import EncryptedFileStreamer
 
 # TODO: omg, this code is essentially duplicated in Daemon
@@ -80,10 +80,10 @@ class HostedEncryptedFile(resource.Resource):
                 d = self._api._download_name(request.args['name'][0])
                 d.addCallback(lambda stream: self._make_stream_producer(request, stream))
             elif request.args['name'][0] in self._api.waiting_on.keys():
-                request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0])
+                request.redirect(settings.UI_ADDRESS + "/?watch=" + request.args['name'][0])
                 request.finish()
             else:
-                request.redirect(UI_ADDRESS)
+                request.redirect(settings.UI_ADDRESS)
                 request.finish()
             return server.NOT_DONE_YET
 

lbrynet/lbrynet_daemon/UIManager.py

@@ -8,7 +8,7 @@ from urllib2 import urlopen
 from StringIO import StringIO
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
-from lbrynet.conf import DEFAULT_UI_BRANCH, LOG_FILE_NAME
+from lbrynet.conf import settings
 from lbrynet.lbrynet_daemon.Resources import NoCacheStaticFile
 from lbrynet import __version__ as lbrynet_version
 from lbryum.version import LBRYUM_VERSION as lbryum_version
@@ -23,7 +23,7 @@ else:
 if not os.path.isdir(log_dir):
     os.mkdir(log_dir)
 
-lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
+lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 
@@ -74,30 +74,29 @@ class UIManager(object):
                 self.loaded_branch = None
                 self.loaded_requirements = None
 
-    def setup(self, branch=DEFAULT_UI_BRANCH, user_specified=None,
+    def setup(self, branch=None, check_requirements=None, user_specified=None):
-              branch_specified=False, check_requirements=None):
+        local_ui_path = settings.local_ui_path or user_specified
-        if check_requirements is not None:
+        self.branch = settings.ui_branch or branch
-            self.check_requirements = check_requirements
+        self.check_requirements = settings.check_ui_requirements or check_requirements
-        if self.branch is not None:
+
-            self.branch = branch
+        if local_ui_path:
-        if user_specified:
+            if os.path.isdir(local_ui_path):
-            if os.path.isdir(user_specified):
+                log.info("Checking user specified UI directory: " + str(local_ui_path))
-                log.info("Checking user specified UI directory: " + str(user_specified))
                 self.branch = "user-specified"
                 self.loaded_git_version = "user-specified"
-                d = self.migrate_ui(source=user_specified)
+                d = self.migrate_ui(source=local_ui_path)
                 d.addCallback(lambda _: self._load_ui())
                 return d
             else:
                 log.info("User specified UI directory doesn't exist, using " + self.branch)
-        elif self.loaded_branch == "user-specified" and not branch_specified:
+        elif self.loaded_branch == "user-specified":
             log.info("Loading user provided UI")
             d = self._load_ui()
             return d
         else:
-            log.info("Checking for updates for UI branch: " + branch)
+            log.info("Checking for updates for UI branch: " + self.branch)
-            self._git_url = "https://s3.amazonaws.com/lbry-ui/{}/data.json".format(branch)
+            self._git_url = "https://s3.amazonaws.com/lbry-ui/{}/data.json".format(self.branch)
-            self._dist_url = "https://s3.amazonaws.com/lbry-ui/{}/dist.zip".format(branch)
+            self._dist_url = "https://s3.amazonaws.com/lbry-ui/{}/dist.zip".format(self.branch)
 
         d = self._up_to_date()
         d.addCallback(lambda r: self._download_ui() if not r else self._load_ui())

lbrynet/lbrynet_daemon/auth/auth.py

@@ -0,0 +1,48 @@
+import logging
+from zope.interface import implementer
+from twisted.cred import portal, checkers, credentials, error as cred_error
+from twisted.internet import defer
+from twisted.web import resource
+from lbrynet.lbrynet_daemon.auth.util import load_api_keys
+
+log = logging.getLogger(__name__)
+
+
+@implementer(portal.IRealm)
+class HttpPasswordRealm(object):
+    def __init__(self, resource):
+        self.resource = resource
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        log.debug("Processing request for %s", avatarId)
+        if resource.IResource in interfaces:
+            return (resource.IResource, self.resource, lambda: None)
+        raise NotImplementedError()
+
+
+@implementer(checkers.ICredentialsChecker)
+class PasswordChecker(object):
+    credentialInterfaces = (credentials.IUsernamePassword,)
+
+    def __init__(self, passwords):
+        self.passwords = passwords
+
+    @classmethod
+    def load_file(cls, key_path):
+        keys = load_api_keys(key_path)
+        return cls.load(keys)
+
+    @classmethod
+    def load(cls, password_dict):
+        passwords = {key: password_dict[key].secret for key in password_dict}
+        return cls(passwords)
+
+    def requestAvatarId(self, creds):
+        if creds.username in self.passwords:
+            pw = self.passwords.get(creds.username)
+            pw_match = creds.checkPassword(pw)
+            if pw_match:
+                return defer.succeed(creds.username)
+        log.warning('Incorrect username or password')
+        return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password'))
+

lbrynet/lbrynet_daemon/auth/client.py

@@ -0,0 +1,172 @@
+import urlparse
+import logging
+import requests
+import os
+import base64
+import json
+
+from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, get_auth_message
+from lbrynet.conf import settings
+from jsonrpc.proxy import JSONRPCProxy
+
+log = logging.getLogger(__name__)
+USER_AGENT = "AuthServiceProxy/0.1"
+TWISTED_SESSION = "TWISTED_SESSION"
+LBRY_SECRET = "LBRY_SECRET"
+HTTP_TIMEOUT = 30
+
+
+class JSONRPCException(Exception):
+    def __init__(self, rpc_error):
+        Exception.__init__(self)
+        self.error = rpc_error
+
+
+class AuthAPIClient(object):
+    def __init__(self, key, timeout, connection, count, service, cookies, auth, url, login_url):
+        self.__service_name = service
+        self.__api_key = key
+        self.__service_url = login_url
+        self.__id_count = count
+        self.__url = url
+        self.__auth_header = auth
+        self.__conn = connection
+        self.__cookies = cookies
+
+    def __getattr__(self, name):
+        if name.startswith('__') and name.endswith('__'):
+            # Python internal stuff
+            raise AttributeError
+        if self.__service_name is not None:
+            name = "%s.%s" % (self.__service_name, name)
+        return AuthAPIClient(key=self.__api_key,
+                             timeout=HTTP_TIMEOUT,
+                             connection=self.__conn,
+                             count=self.__id_count,
+                             service=name,
+                             cookies=self.__cookies,
+                             auth=self.__auth_header,
+                             url=self.__url,
+                             login_url=self.__service_url)
+
+    def __call__(self, *args):
+        self.__id_count += 1
+        pre_auth_postdata = {'version': '1.1',
+                             'method': self.__service_name,
+                             'params': args,
+                             'id': self.__id_count}
+        to_auth = get_auth_message(pre_auth_postdata)
+        token = self.__api_key.get_hmac(to_auth)
+        pre_auth_postdata.update({'hmac': token})
+        postdata = json.dumps(pre_auth_postdata)
+        service_url = self.__service_url
+        auth_header = self.__auth_header
+        cookies = self.__cookies
+        host = self.__url.hostname
+
+        req = requests.Request(method='POST',
+                               url=service_url,
+                               data=postdata,
+                               headers={'Host': host,
+                                        'User-Agent': USER_AGENT,
+                                        'Authorization': auth_header,
+                                        'Content-type': 'application/json'},
+                               cookies=cookies)
+        r = req.prepare()
+        http_response = self.__conn.send(r)
+        cookies = http_response.cookies
+        headers = http_response.headers
+        next_secret = headers.get(LBRY_SECRET, False)
+        if next_secret:
+            self.__api_key.secret = next_secret
+            self.__cookies = cookies
+
+        if http_response is None:
+            raise JSONRPCException({
+                'code': -342, 'message': 'missing HTTP response from server'})
+
+        http_response.raise_for_status()
+
+        response = http_response.json()
+
+        if response['error'] is not None:
+            raise JSONRPCException(response['error'])
+        elif 'result' not in response:
+            raise JSONRPCException({
+                'code': -343, 'message': 'missing JSON-RPC result'})
+        else:
+            return response['result']
+
+    @classmethod
+    def config(cls, key_name=None, key=None, pw_path=None, timeout=HTTP_TIMEOUT, connection=None, count=0,
+                                            service=None, cookies=None, auth=None, url=None, login_url=None):
+
+        api_key_name = API_KEY_NAME if not key_name else key_name
+        pw_path = os.path.join(settings.data_dir, ".api_keys") if not pw_path else pw_path
+        if not key:
+            keys = load_api_keys(pw_path)
+            api_key = keys.get(api_key_name, False)
+        else:
+            api_key = APIKey(name=api_key_name, secret=key)
+        if login_url is None:
+            service_url = "http://%s:%s@%s:%i/%s" % (api_key_name,
+                                                     api_key.secret,
+                                                     settings.API_INTERFACE,
+                                                     settings.api_port,
+                                                     settings.API_ADDRESS)
+        else:
+            service_url = login_url
+        id_count = count
+
+        if auth is None and connection is None and cookies is None and url is None:
+            # This is a new client instance, initialize the auth header and start a session
+            url = urlparse.urlparse(service_url)
+            (user, passwd) = (url.username, url.password)
+            try:
+                user = user.encode('utf8')
+            except AttributeError:
+                pass
+            try:
+                passwd = passwd.encode('utf8')
+            except AttributeError:
+                pass
+            authpair = user + b':' + passwd
+            auth_header = b'Basic ' + base64.b64encode(authpair)
+            conn = requests.Session()
+            conn.auth = (user, passwd)
+            req = requests.Request(method='POST',
+                                   url=service_url,
+                                   auth=conn.auth,
+                                   headers={'Host': url.hostname,
+                                            'User-Agent': USER_AGENT,
+                                            'Authorization': auth_header,
+                                            'Content-type': 'application/json'},)
+            r = req.prepare()
+            http_response = conn.send(r)
+            cookies = http_response.cookies
+            uid = cookies.get(TWISTED_SESSION)
+            api_key = APIKey.new(seed=uid)
+        else:
+            # This is a client that already has a session, use it
+            auth_header = auth
+            conn = connection
+            assert cookies.get(LBRY_SECRET, False), "Missing cookie"
+            secret = cookies.get(LBRY_SECRET)
+            api_key = APIKey(secret, api_key_name)
+        return cls(api_key, timeout, conn, id_count, service, cookies, auth_header, url, service_url)
+
+
+class LBRYAPIClient(object):
+    @staticmethod
+    def config(service=None, params=None):
+        if settings.use_auth_http:
+            if service is None:
+                return AuthAPIClient.config()
+            log.error("Try auth")
+            if params is not None:
+                return AuthAPIClient.config(service=service)(params)
+            return AuthAPIClient.config(service=service)()
+        url = settings.API_CONNECTION_STRING
+        if service is None:
+            return JSONRPCProxy.from_url(url)
+        return JSONRPCProxy.from_url(url).call(service, params)

lbrynet/lbrynet_daemon/auth/server.py

@@ -0,0 +1,274 @@
+import logging
+
+from decimal import Decimal
+from zope.interface import implements
+from twisted.web import server, resource
+from twisted.internet import defer
+from txjsonrpc import jsonrpclib
+
+from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError, SubhandlerError
+from lbrynet.conf import settings
+from lbrynet.lbrynet_daemon.auth.util import APIKey, get_auth_message
+from lbrynet.lbrynet_daemon.auth.client import LBRY_SECRET
+
+log = logging.getLogger(__name__)
+
+
+def default_decimal(obj):
+    if isinstance(obj, Decimal):
+        return float(obj)
+
+
+class AuthorizedBase(object):
+    def __init__(self):
+        self.authorized_functions = []
+        self.subhandlers = []
+        self.callable_methods = {}
+
+        for methodname in dir(self):
+            if methodname.startswith("jsonrpc_"):
+                method = getattr(self, methodname)
+                self.callable_methods.update({methodname.split("jsonrpc_")[1]: method})
+                if hasattr(method, '_auth_required'):
+                    self.authorized_functions.append(methodname.split("jsonrpc_")[1])
+            elif not methodname.startswith("__"):
+                method = getattr(self, methodname)
+                if hasattr(method, '_subhandler'):
+                    self.subhandlers.append(method)
+
+    @staticmethod
+    def auth_required(f):
+        f._auth_required = True
+        return f
+
+    @staticmethod
+    def subhandler(f):
+        f._subhandler = True
+        return f
+
+
+class AuthJSONRPCServer(AuthorizedBase):
+    """
+    Authorized JSONRPC server used as the base class for the LBRY API
+
+    API methods are named with a leading "jsonrpc_"
+
+    Decorators:
+        @AuthJSONRPCServer.auth_required: this requires the client include a valid hmac authentication token in their
+                                          request
+
+        @AuthJSONRPCServer.subhandler: include the tagged method in the processing of requests, to allow inheriting
+                                       classes to modify request handling. Tagged methods will be passed the request
+                                       object, and return True when finished to indicate success
+
+    Attributes:
+        allowed_during_startup (list): list of api methods that are callable before the server has finished
+                                       startup
+
+        sessions (dict): dictionary of active session_id: lbrynet.lbrynet_daemon.auth.util.APIKey values
+
+        authorized_functions (list): list of api methods that require authentication
+
+        subhandlers (list): list of subhandlers
+
+        callable_methods (dict): dictionary of api_callable_name: method values
+    """
+    implements(resource.IResource)
+
+    isLeaf = True
+    OK = 200
+    UNAUTHORIZED = 401
+    NOT_FOUND = 8001
+    FAILURE = 8002
+
+    def __init__(self, use_authentication=settings.use_auth_http):
+        AuthorizedBase.__init__(self)
+        self._use_authentication = use_authentication
+        self.allowed_during_startup = []
+        self.sessions = {}
+
+    def setup(self):
+        return NotImplementedError()
+
+    def render(self, request):
+        assert self._check_headers(request), InvalidHeaderError
+
+        session = request.getSession()
+        session_id = session.uid
+
+        if self._use_authentication:
+            # if this is a new session, send a new secret and set the expiration, otherwise, session.touch()
+            if self._initialize_session(session_id):
+                def expire_session():
+                    self._unregister_user_session(session_id)
+                session.startCheckingExpiration()
+                session.notifyOnExpire(expire_session)
+                message = "OK"
+                request.setResponseCode(self.OK)
+                self._set_headers(request, message, True)
+                self._render_message(request, message)
+                return server.NOT_DONE_YET
+            session.touch()
+
+        request.content.seek(0, 0)
+        content = request.content.read()
+        try:
+            parsed = jsonrpclib.loads(content)
+        except ValueError:
+            return server.failure
+
+        function_name = parsed.get('method')
+        args = parsed.get('params')
+        id = parsed.get('id')
+        token = parsed.pop('hmac', None)
+        version = self._get_jsonrpc_version(parsed.get('jsonrpc'), id)
+
+        try:
+            self._run_subhandlers(request)
+        except SubhandlerError:
+            return server.failure
+
+        reply_with_next_secret = False
+        if self._use_authentication:
+            if function_name in self.authorized_functions:
+                try:
+                    self._verify_token(session_id, parsed, token)
+                except InvalidAuthenticationToken:
+                    log.warning("API validation failed")
+                    request.setResponseCode(self.UNAUTHORIZED)
+                    request.finish()
+                    return server.NOT_DONE_YET
+                self._update_session_secret(session_id)
+                reply_with_next_secret = True
+
+        try:
+            function = self._get_jsonrpc_method(function_name)
+        except Exception:
+            log.warning("Unknown method: %s", function_name)
+            return server.failure
+
+        d = defer.maybeDeferred(function) if args == [{}] else defer.maybeDeferred(function, *args)
+        # cancel the response if the connection is broken
+        notify_finish = request.notifyFinish()
+        notify_finish.addErrback(self._response_failed, d)
+        d.addErrback(self._errback_render, id)
+        d.addCallback(self._callback_render, request, id, version, reply_with_next_secret)
+        d.addErrback(notify_finish.errback)
+
+        return server.NOT_DONE_YET
+
+    def _register_user_session(self, session_id):
+        """
+        Add or update a HMAC secret for a session
+
+        @param session_id:
+        @return: secret
+        """
+        log.info("Register api session")
+        token = APIKey.new(seed=session_id)
+        self.sessions.update({session_id: token})
+
+    def _unregister_user_session(self, session_id):
+        log.info("Unregister API session")
+        del self.sessions[session_id]
+
+    def _response_failed(self, err, call):
+        log.debug(err.getTraceback())
+
+    def _set_headers(self, request, data, update_secret=False):
+        request.setHeader("Access-Control-Allow-Origin", settings.API_INTERFACE)
+        request.setHeader("Content-Type", "text/json")
+        request.setHeader("Content-Length", str(len(data)))
+        if update_secret:
+            session_id = request.getSession().uid
+            request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret)
+
+    def _render_message(self, request, message):
+        request.write(message)
+        request.finish()
+
+    def _check_headers(self, request):
+        origin = request.getHeader("Origin")
+        referer = request.getHeader("Referer")
+        if origin not in [None, settings.ORIGIN]:
+            log.warning("Attempted api call from %s", origin)
+            return False
+        if referer is not None and not referer.startswith(settings.REFERER):
+            log.warning("Attempted api call from %s", referer)
+            return False
+        return True
+
+    def _check_function_path(self, function_path):
+        if function_path not in self.callable_methods:
+            log.warning("Unknown method: %s", function_path)
+            return False
+        if not self.announced_startup:
+            if function_path not in self.allowed_during_startup:
+                log.warning("Cannot call %s during startup", function_path)
+                return False
+        return True
+
+    def _get_jsonrpc_method(self, function_path):
+        assert self._check_function_path(function_path)
+        return self.callable_methods.get(function_path)
+
+    def _initialize_session(self, session_id):
+        if not self.sessions.get(session_id, False):
+            self._register_user_session(session_id)
+            return True
+        return False
+
+    def _verify_token(self, session_id, message, token):
+        to_auth = get_auth_message(message)
+        api_key = self.sessions.get(session_id)
+        assert api_key.compare_hmac(to_auth, token), InvalidAuthenticationToken
+
+    def _update_session_secret(self, session_id):
+        # log.info("Generating new token for next request")
+        self.sessions.update({session_id: APIKey.new(name=session_id)})
+
+    def _get_jsonrpc_version(self, version=None, id=None):
+        if version:
+            version_for_return = int(float(version))
+        elif id and not version:
+            version_for_return = jsonrpclib.VERSION_1
+        else:
+            version_for_return = jsonrpclib.VERSION_PRE1
+        return version_for_return
+
+    def _run_subhandlers(self, request):
+        for handler in self.subhandlers:
+            try:
+                assert handler(request)
+            except Exception as err:
+                log.error(err.message)
+                raise SubhandlerError
+
+    def _callback_render(self, result, request, id, version, auth_required=False):
+        result_for_return = result if not isinstance(result, dict) else result['result']
+
+        if version == jsonrpclib.VERSION_PRE1:
+            if not isinstance(result, jsonrpclib.Fault):
+                result_for_return = (result_for_return,)
+            # Convert the result (python) to JSON-RPC
+        try:
+            encoded_message = jsonrpclib.dumps(result_for_return, version=version, default=default_decimal)
+            self._set_headers(request, encoded_message, auth_required)
+            self._render_message(request, encoded_message)
+        except:
+            fault = jsonrpclib.Fault(self.FAILURE, "can't serialize output")
+            encoded_message = jsonrpclib.dumps(fault, version=version)
+            self._set_headers(request, encoded_message)
+            self._render_message(request, encoded_message)
+
+    def _errback_render(self, failure, id):
+        log.error("Request failed:")
+        log.error(failure)
+        log.error(failure.value)
+        log.error(id)
+        if isinstance(failure.value, jsonrpclib.Fault):
+            return failure.value
+        return server.failure
+
+    def _render_response(self, result, code):
+        return defer.succeed({'result': result, 'code': code})

lbrynet/lbrynet_daemon/auth/util.py

@@ -0,0 +1,93 @@
+import base58
+import hmac
+import hashlib
+import yaml
+import os
+import logging
+
+log = logging.getLogger(__name__)
+
+API_KEY_NAME = "api"
+
+
+def sha(x):
+    h = hashlib.sha256(x).digest()
+    return base58.b58encode(h)
+
+
+def generate_key(x=None):
+    if x is None:
+        return sha(os.urandom(256))
+    else:
+        return sha(x)
+
+
+class APIKey(object):
+    def __init__(self, secret, name, expiration=None):
+        self.secret = secret
+        self.name = name
+        self.expiration = expiration
+
+    @classmethod
+    def new(cls, seed=None, name=None, expiration=None):
+        secret = generate_key(seed)
+        key_name = name if name else sha(secret)
+        return APIKey(secret, key_name, expiration)
+
+    def _raw_key(self):
+        return base58.b58decode(self.secret)
+
+    def get_hmac(self, message):
+        decoded_key = self._raw_key()
+        signature = hmac.new(decoded_key, message, hashlib.sha256)
+        return base58.b58encode(signature.digest())
+
+    def compare_hmac(self, message, token):
+        decoded_token = base58.b58decode(token)
+        target = base58.b58decode(self.get_hmac(message))
+        try:
+            assert len(decoded_token) == len(target), "Length mismatch"
+            r = hmac.compare_digest(decoded_token, target)
+        except:
+            return False
+        return r
+
+
+def load_api_keys(path):
+    if not os.path.isfile(path):
+        raise Exception("Invalid api key path")
+
+    with open(path, "r") as f:
+        data = yaml.load(f.read())
+
+    keys_for_return = {}
+    for key_name in data:
+        key = data[key_name]
+        secret = key['secret']
+        expiration = key['expiration']
+        keys_for_return.update({key_name: APIKey(secret, key_name, expiration)})
+
+    return keys_for_return
+
+
+def save_api_keys(keys, path):
+    with open(path, "w") as f:
+        key_dict = {keys[key_name].name: {'secret': keys[key_name].secret,
+                                          'expiration': keys[key_name].expiration}
+                    for key_name in keys}
+        data = yaml.safe_dump(key_dict)
+        f.write(data)
+
+
+def initialize_api_key_file(key_path):
+    if not os.path.isfile(key_path):
+        keys = {}
+        new_api_key = APIKey.new(name=API_KEY_NAME)
+        keys.update({new_api_key.name: new_api_key})
+        save_api_keys(keys, key_path)
+
+
+def get_auth_message(message_dict):
+    to_auth = message_dict.get('method').encode('hex')
+    to_auth += str(message_dict.get('id')).encode('hex')
+    return to_auth.decode('hex')

lbrynet/pointtraderclient/pointtraderclient.py

@@ -1,4 +1,4 @@
-from lbrynet.conf import POINTTRADER_SERVER
+from lbrynet.conf import settings
 
 from twisted.web.client import Agent, FileBodyProducer, Headers, ResponseDone
 from twisted.internet import threads, defer, protocol
@@ -46,7 +46,7 @@ def get_body_from_request(path, data):
 
     jsondata = FileBodyProducer(StringIO(json.dumps(data)))
     agent = Agent(reactor)
-    d = agent.request('POST', POINTTRADER_SERVER + path, Headers({'Content-Type': ['application/json']}), jsondata)
+    d = agent.request('POST', settings.pointtrader_server + path, Headers({'Content-Type': ['application/json']}), jsondata)
     d.addCallback(get_body)
     return d
 

packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py

@@ -3,16 +3,11 @@ import webbrowser
 import sys
 import os
 import logging
-import socket
 import platform
 import shutil
 from appdirs import user_data_dir
-
-from PyObjCTools import AppHelper
-
 from twisted.internet import reactor
 from twisted.web import server
-
 import Foundation
 bundle = Foundation.NSBundle.mainBundle()
 lbrycrdd_path = bundle.pathForResource_ofType_('lbrycrdd', None)
@@ -29,8 +24,7 @@ if not os.path.isfile(lbrycrdd_path_conf):
 
 from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer
 from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest
-from lbrynet.conf import API_PORT, API_INTERFACE, ICON_PATH, APP_NAME
+from lbrynet.conf import settings
-from lbrynet.conf import UI_ADDRESS
 from lbrynet.core import utils
 
 
@@ -49,7 +43,7 @@ class LBRYDaemonApp(AppKit.NSApplication):
         self.connection = False
         statusbar = AppKit.NSStatusBar.systemStatusBar()
         self.statusitem = statusbar.statusItemWithLength_(AppKit.NSVariableStatusItemLength)
-        self.icon = AppKit.NSImage.alloc().initByReferencingFile_(ICON_PATH)
+        self.icon = AppKit.NSImage.alloc().initByReferencingFile_(settings.ICON_PATH)
         self.icon.setScalesWhenResized_(True)
         self.icon.setSize_((20, 20))
         self.statusitem.setImage_(self.icon)
@@ -59,7 +53,7 @@ class LBRYDaemonApp(AppKit.NSApplication):
         self.quit = AppKit.NSMenuItem.alloc().initWithTitle_action_keyEquivalent_("Quit", "replyToApplicationShouldTerminate:", "")
         self.menubarMenu.addItem_(self.quit)
         self.statusitem.setMenu_(self.menubarMenu)
-        self.statusitem.setToolTip_(APP_NAME)
+        self.statusitem.setToolTip_(settings.APP_NAME)
 
 
         if test_internet_connection():
@@ -70,16 +64,15 @@ class LBRYDaemonApp(AppKit.NSApplication):
                 LBRYNotify("LBRY needs an internet connection to start, try again when one is available")
             sys.exit(0)
 
-
         lbry = DaemonServer()
-        d = lbry.start()
+        d = lbry.start(use_authentication=False)
-        d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
+        d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS))
         lbrynet_server = server.Site(lbry.root)
         lbrynet_server.requestFactory = DaemonRequest
-        reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE)
+        reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE)
 
     def openui_(self, sender):
-        webbrowser.open(UI_ADDRESS)
+        webbrowser.open(settings.UI_ADDRESS)
 
     def replyToApplicationShouldTerminate_(self, shouldTerminate):
         if platform.mac_ver()[0] >= "10.10":

packaging/osx/lbry-osx-app/setup_app.py

@@ -2,14 +2,14 @@
 
 import os
 from setuptools import setup
-from lbrynet.conf import APP_NAME, ICON_PATH
+from lbrynet.conf import settings
 
 APP = [os.path.join('lbrygui', 'main.py')]
 DATA_FILES = []
 DATA_FILES.append('app.icns')
 
 OPTIONS = {
-    'iconfile': ICON_PATH,
+    'iconfile': settings.ICON_PATH,
     'plist': {
         'CFBundleIdentifier': 'io.lbry.LBRY',
         'LSUIElement': True,
@@ -22,7 +22,7 @@ OPTIONS = {
 
 
 setup(
-    name=APP_NAME,
+    name=settings.APP_NAME,
     app=APP,
     options={'py2app': OPTIONS},
     data_files=DATA_FILES,

packaging/osx/lbry-osx-app/setup_uri_handler.py

@@ -1,6 +1,6 @@
 from setuptools import setup
 import os
-from lbrynet.conf import PROTOCOL_PREFIX
+from lbrynet.conf import settings
 
 APP = [os.path.join('lbry_uri_handler', 'LBRYURIHandler.py')]
 DATA_FILES = []
@@ -12,7 +12,7 @@ OPTIONS = {'argv_emulation': True,
                'CFBundleURLTypes': [
                     {
                     'CFBundleURLTypes': 'LBRYURIHandler',
-                    'CFBundleURLSchemes': [PROTOCOL_PREFIX]
+                    'CFBundleURLSchemes': [settings.PROTOCOL_PREFIX]
                     }
                ]
            }
@@ -23,4 +23,4 @@ setup(
     data_files=DATA_FILES,
     options={'py2app': OPTIONS},
     setup_requires=['py2app'],
-)
+)

packaging/ubuntu/lbry

@@ -15,8 +15,6 @@ if [ ! -f "$LBRYCRDCONF" ]; then
   echo -e "rpcuser=lbryrpc\nrpcpassword=$(env LC_CTYPE=C LC_ALL=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 16 | xargs)" > "$LBRYCRDCONF"
 fi
 
-WEB_UI_BRANCH='master'
-
 urlencode() {
   local LANG=C
   local length="${#1}"
@@ -42,7 +40,7 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
 if [ -z "$(pgrep lbrynet-daemon)" ]; then
   echo "running lbrynet-daemon..."
-  $DIR/lbrynet-daemon --no-launch --branch="$WEB_UI_BRANCH" &
+  $DIR/lbrynet-daemon --no-launch &
   sleep 3 # let the daemon load before connecting
 fi
 

packaging/uri_handler/LBRYURIHandler.py

@@ -4,15 +4,14 @@ import subprocess
 import sys
 from time import sleep
 
-from jsonrpc.proxy import JSONRPCProxy
+from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient
-
+from lbrynet.conf import settings
-from lbrynet.conf import UI_ADDRESS, API_CONNECTION_STRING
 
 
 class LBRYURIHandler(object):
     def __init__(self):
         self.started_daemon = False
-        self.daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING)
+        self.daemon = LBRYAPIClient.config()
 
     def handle_osx(self, lbry_name):
         self.check_daemon()
@@ -70,9 +69,9 @@ class LBRYURIHandler(object):
     @staticmethod
     def open_address(lbry_name):
         if lbry_name == "lbry" or lbry_name == "" or lbry_name is None:
-            webbrowser.open(UI_ADDRESS)
+            webbrowser.open(settings.UI_ADDRESS)
         else:
-            webbrowser.open(UI_ADDRESS + "/?show=" + lbry_name)
+            webbrowser.open(settings.UI_ADDRESS + "/?show=" + lbry_name)
 
 
 def main(args):

packaging/windows/lbry-win32-app/LBRYWin32App.py

@@ -21,8 +21,7 @@ except ImportError:
 from lbrynet.core import utils
 from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer
 from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest
-from lbrynet.conf import API_PORT, API_INTERFACE, ICON_PATH, APP_NAME
+from lbrynet.conf import settings
-from lbrynet.conf import UI_ADDRESS, API_CONNECTION_STRING, LOG_FILE_NAME
 from packaging.uri_handler.LBRYURIHandler import LBRYURIHandler
 
 
@@ -31,7 +30,7 @@ data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
 if not os.path.isdir(data_dir):
     os.mkdir(data_dir)
 
-lbrynet_log = os.path.join(data_dir, LOG_FILE_NAME)
+lbrynet_log = os.path.join(data_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 if getattr(sys, 'frozen', False) and os.name == "nt":
@@ -252,7 +251,7 @@ def main(lbry_name=None):
         return SysTrayIcon(icon, hover_text, menu_options, on_quit=stop)
 
     def openui_(sender):
-        webbrowser.open(UI_ADDRESS)
+        webbrowser.open(settings.UI_ADDRESS)
 
     def replyToApplicationShouldTerminate_():
         try:
@@ -264,11 +263,11 @@ def main(lbry_name=None):
         replyToApplicationShouldTerminate_()
 
     if getattr(sys, 'frozen', False) and os.name == "nt":
-        icon = os.path.join(os.path.dirname(sys.executable), ICON_PATH, 'lbry16.ico')
+        icon = os.path.join(os.path.dirname(sys.executable), settings.ICON_PATH, 'lbry16.ico')
     else:
-        icon = os.path.join(ICON_PATH, 'lbry16.ico')
+        icon = os.path.join(settings.ICON_PATH, 'lbry16.ico')
 
-    hover_text = APP_NAME
+    hover_text = settings.APP_NAME
     menu_options = (('Open', icon, openui_),)
 
     if not test_internet_connection():
@@ -280,19 +279,19 @@ def main(lbry_name=None):
     systray_thread.start()
 
     lbry = DaemonServer()
-    d = lbry.start()
+    d = lbry.start(use_authentication=False)
     d.addCallback(lambda _: LBRYURIHandler.open_address(lbry_name))
     lbrynet_server = server.Site(lbry.root)
     lbrynet_server.requestFactory = DaemonRequest
     try:
-        reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE)
+        reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE)
     except error.CannotListenError:
         log.info('Daemon already running, exiting app')
         sys.exit(1)
     reactor.run()
 
 if __name__ == '__main__':
-    lbry_daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING)
+    lbry_daemon = JSONRPCProxy.from_url(settings.API_CONNECTION_STRING)
 
     try:
         daemon_running = lbry_daemon.is_running()

requirements.txt

@@ -29,4 +29,5 @@ wsgiref==0.1.2
 zope.interface==4.1.3
 base58==0.2.2
 googlefinance==0.7
-pyyaml==3.12
+pyyaml==3.12
+service_identity==16.0.0

tests/functional/test_misc.py

@@ -11,8 +11,13 @@ import unittest
 from Crypto.PublicKey import RSA
 from Crypto import Random
 from Crypto.Hash import MD5
+from lbrynet.conf import settings
+from lbrynet.lbrylive.LiveStreamCreator import FileLiveStreamCreator
+from lbrynet.lbrylive.LiveStreamMetadataManager import DBLiveStreamMetadataManager
+from lbrynet.lbrylive.LiveStreamMetadataManager import TempLiveStreamMetadataManager
+from lbrynet.lbryfile.EncryptedFileMetadataManager import TempEncryptedFileMetadataManager, \
+    DBEncryptedFileMetadataManager
 from lbrynet import analytics
-from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE
 from lbrynet.lbrylive.LiveStreamCreator import FileLiveStreamCreator
 from lbrynet.lbrylive.LiveStreamMetadataManager import DBLiveStreamMetadataManager
 from lbrynet.lbrylive.LiveStreamMetadataManager import TempLiveStreamMetadataManager
@@ -45,6 +50,7 @@ from lbrynet.lbrylive.client.LiveStreamDownloader import add_full_live_stream_do
 
 from tests import mocks
 
+
 FakeNode = mocks.Node
 FakeWallet = mocks.Wallet
 FakePeerFinder = mocks.PeerFinder
@@ -53,6 +59,7 @@ GenFile = mocks.GenFile
 test_create_stream_sd_file = mocks.create_stream_sd_file
 DummyBlobAvailabilityTracker = mocks.BlobAvailabilityTracker
 
+
 log_format = "%(funcName)s(): %(message)s"
 logging.basicConfig(level=logging.WARNING, format=log_format)
 
@@ -60,11 +67,13 @@ logging.basicConfig(level=logging.WARNING, format=log_format)
 def require_system(system):
     def wrapper(fn):
         return fn
+
     if platform.system() == system:
         return wrapper
     else:
         return unittest.skip("Skipping. Test can only be run on " + system)
 
+
 def use_epoll_on_linux():
     if sys.platform.startswith("linux"):
         sys.modules = sys.modules.copy()
@@ -108,7 +117,7 @@ class LbryUploader(object):
         db_dir = "server"
         os.mkdir(db_dir)
         self.session = Session(
-            MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd",
+            settings.data_rate, db_dir=db_dir, lbryid="abcd",
             peer_finder=peer_finder, hash_announcer=hash_announcer, peer_port=5553,
             use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
             blob_tracker_class=DummyBlobAvailabilityTracker,
@@ -213,11 +222,11 @@ def start_lbry_reuploader(sd_hash, kill_event, dead_event,
     os.mkdir(db_dir)
     os.mkdir(blob_dir)
 
-    session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd" + str(n),
+    session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd" + str(n),
-                          peer_finder=peer_finder, hash_announcer=hash_announcer,
+                      peer_finder=peer_finder, hash_announcer=hash_announcer,
-                          blob_dir=None, peer_port=peer_port,
+                      blob_dir=None, peer_port=peer_port,
-                          use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
+                      use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
-                          blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=is_generous)
+                      blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=settings.is_generous_host)
 
     stream_info_manager = TempEncryptedFileMetadataManager()
 
@@ -317,14 +326,13 @@ def start_live_server(sd_hash_queue, kill_event, dead_event):
     rate_limiter = DummyRateLimiter()
     sd_identifier = StreamDescriptorIdentifier()
 
-
     db_dir = "server"
     os.mkdir(db_dir)
 
-    session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd",
+    session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd",
                       peer_finder=peer_finder, hash_announcer=hash_announcer, peer_port=5553,
                       use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
-                      blob_tracker_class=DummyBlobAvailabilityTracker)
+                      blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=settings.is_generous_host)
     stream_info_manager = DBLiveStreamMetadataManager(session.db_dir, hash_announcer)
 
     logging.debug("Created the session")
@@ -449,14 +457,14 @@ def start_blob_uploader(blob_hash_queue, kill_event, dead_event, slow, is_genero
     os.mkdir(db_dir)
     os.mkdir(blob_dir)
 
-    session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="efgh",
+    session = Session(settings.data_rate, db_dir=db_dir, lbryid="efgh",
-                          peer_finder=peer_finder, hash_announcer=hash_announcer,
+                      peer_finder=peer_finder, hash_announcer=hash_announcer,
-                          blob_dir=blob_dir, peer_port=peer_port,
+                      blob_dir=blob_dir, peer_port=peer_port,
-                          use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
+                      use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
-                          blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=is_generous)
+                      blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=settings.is_generous_host)
 
     if slow is True:
-        session.rate_limiter.set_ul_limit(2**11)
+        session.rate_limiter.set_ul_limit(2 ** 11)
 
     def start_all():
         d = session.setup()
@@ -510,7 +518,7 @@ def start_blob_uploader(blob_hash_queue, kill_event, dead_event, slow, is_genero
 
     def create_single_blob():
         blob_creator = session.blob_manager.get_blob_creator()
-        blob_creator.write("0" * 2**21)
+        blob_creator.write("0" * 2 ** 21)
         return blob_creator.close()
 
     def put_blob_hash_on_queue(blob_hash):
@@ -621,14 +629,13 @@ class TestTransfer(TestCase):
         rate_limiter = DummyRateLimiter()
         sd_identifier = StreamDescriptorIdentifier()
 
-
         db_dir = "client"
         blob_dir = os.path.join(db_dir, "blobfiles")
         os.mkdir(db_dir)
         os.mkdir(blob_dir)
 
         self.session = Session(
-            MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd",
+            settings.data_rate, db_dir=db_dir, lbryid="abcd",
             peer_finder=peer_finder, hash_announcer=hash_announcer,
             blob_dir=blob_dir, peer_port=5553,
             use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
@@ -718,7 +725,7 @@ class TestTransfer(TestCase):
         os.mkdir(db_dir)
 
         self.session = Session(
-            MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd",
+            settings.data_rate, db_dir=db_dir, lbryid="abcd",
             peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=None,
             peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
             blob_tracker_class=DummyBlobAvailabilityTracker, dht_node_class=Node
@@ -814,18 +821,18 @@ class TestTransfer(TestCase):
         hash_announcer = FakeAnnouncer()
         rate_limiter = DummyRateLimiter()
 
-
         db_dir = "client"
         blob_dir = os.path.join(db_dir, "blobfiles")
         os.mkdir(db_dir)
         os.mkdir(blob_dir)
 
         self.session = Session(
-            MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd",
+            settings.data_rate, db_dir=db_dir, lbryid="abcd",
             peer_finder=peer_finder, hash_announcer=hash_announcer,
             blob_dir=blob_dir, peer_port=5553,
             use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
-            blob_tracker_class=DummyBlobAvailabilityTracker)
+            blob_tracker_class=DummyBlobAvailabilityTracker,
+            is_generous=settings.is_generous_host)
 
         d1 = self.wait_for_hash_from_queue(blob_hash_queue_1)
         d2 = self.wait_for_hash_from_queue(blob_hash_queue_2)
@@ -891,7 +898,6 @@ class TestTransfer(TestCase):
         rate_limiter = DummyRateLimiter()
         sd_identifier = StreamDescriptorIdentifier()
 
-
         downloaders = []
 
         db_dir = "client"
@@ -899,10 +905,11 @@ class TestTransfer(TestCase):
         os.mkdir(db_dir)
         os.mkdir(blob_dir)
 
-        self.session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd",
+        self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd",
-                                   peer_finder=peer_finder, hash_announcer=hash_announcer,
+                               peer_finder=peer_finder, hash_announcer=hash_announcer,
-                                   blob_dir=blob_dir, peer_port=5553, use_upnp=False,
+                               blob_dir=blob_dir, peer_port=5553, use_upnp=False,
-                                   rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker)
+                               rate_limiter=rate_limiter, wallet=wallet,
+                               blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=settings.is_generous_host)
 
         self.stream_info_manager = DBEncryptedFileMetadataManager(self.session.db_dir)
         self.lbry_file_manager = EncryptedFileManager(self.session, self.stream_info_manager, sd_identifier)
@@ -937,7 +944,8 @@ class TestTransfer(TestCase):
             logging.debug("deleting the file...")
             d = self.lbry_file_manager.delete_lbry_file(downloaders[0])
             d.addCallback(lambda _: self.lbry_file_manager.get_count_for_stream_hash(downloaders[0].stream_hash))
-            d.addCallback(lambda c: self.stream_info_manager.delete_stream(downloaders[1].stream_hash) if c == 0 else True)
+            d.addCallback(
+                lambda c: self.stream_info_manager.delete_stream(downloaders[1].stream_hash) if c == 0 else True)
             return d
 
         def check_lbry_file():
@@ -1009,25 +1017,27 @@ class TestTransfer(TestCase):
         rate_limiter = DummyRateLimiter()
         sd_identifier = StreamDescriptorIdentifier()
 
-
         db_dir = "client"
         blob_dir = os.path.join(db_dir, "blobfiles")
         os.mkdir(db_dir)
         os.mkdir(blob_dir)
 
-        self.session = Session(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd",
+        self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd",
-                                   peer_finder=peer_finder, hash_announcer=hash_announcer,
+                               peer_finder=peer_finder, hash_announcer=hash_announcer,
-                                   blob_dir=None, peer_port=5553,
+                               blob_dir=None, peer_port=5553,
-                                   use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker)
+                               use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
+                               blob_tracker_class=DummyBlobAvailabilityTracker,
+                               is_generous=settings.is_generous_host)
 
         self.stream_info_manager = TempEncryptedFileMetadataManager()
 
-        self.lbry_file_manager = EncryptedFileManager(self.session, self.stream_info_manager, sd_identifier)
+        self.lbry_file_manager = EncryptedFileManager(
+            self.session, self.stream_info_manager, sd_identifier)
 
         def start_additional_uploaders(sd_hash):
             for i in range(1, num_uploaders):
                 uploader = Process(target=start_lbry_reuploader,
-                                   args=(sd_hash, kill_event, dead_events[i], ready_events[i-1], i, 2**10))
+                                   args=(sd_hash, kill_event, dead_events[i], ready_events[i - 1], i, 2 ** 10))
                 uploader.start()
                 self.server_processes.append(uploader)
             return defer.succeed(True)

tests/functional/test_reflector.py

@@ -4,7 +4,7 @@ import shutil
 from twisted.internet import defer, threads, error
 from twisted.trial import unittest
 
-from lbrynet import conf
+from lbrynet.conf import settings
 from lbrynet import lbryfile
 from lbrynet import reflector
 from lbrynet.core import BlobManager
@@ -83,7 +83,7 @@ class TestReflector(unittest.TestCase):
         os.mkdir(db_dir)
 
         self.session = Session.Session(
-            conf.MIN_BLOB_DATA_PAYMENT_RATE,
+            settings.data_rate,
             db_dir=db_dir,
             lbryid="abcd",
             peer_finder=peer_finder,

tests/functional/test_streamify.py

@@ -6,7 +6,7 @@ from Crypto.Hash import MD5
 from twisted.trial.unittest import TestCase
 from twisted.internet import defer, threads
 
-from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE
+from lbrynet.conf import settings
 from lbrynet.lbryfile.EncryptedFileMetadataManager import TempEncryptedFileMetadataManager
 from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
 from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager
@@ -70,7 +70,7 @@ class TestStreamify(TestCase):
         os.mkdir(blob_dir)
 
         self.session = Session(
-            MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd",
+            settings.data_rate, db_dir=db_dir, lbryid="abcd",
             peer_finder=peer_finder, hash_announcer=hash_announcer,
             blob_dir=blob_dir, peer_port=5553,
             use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
@@ -126,7 +126,7 @@ class TestStreamify(TestCase):
         os.mkdir(blob_dir)
 
         self.session = Session(
-            MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd",
+            settings.data_rate, db_dir=db_dir, lbryid="abcd",
             peer_finder=peer_finder, hash_announcer=hash_announcer,
             blob_dir=blob_dir, peer_port=5553,
             use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,

tests/mocks.py

@@ -2,7 +2,7 @@ import io
 
 from Crypto.PublicKey import RSA
 from decimal import Decimal
-from twisted.internet import defer, threads, task, error
+from twisted.internet import defer
 
 from lbrynet.core import PTCWallet
 from lbrynet.core import BlobAvailability