From 4385b90cca09397e8b37d882dc7e9fe389ff8270 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Sun, 17 Apr 2016 23:23:20 -0400
Subject: [PATCH 01/38] add get transaction and history functions to daemon

---
 lbrynet/core/LBRYcrdWallet.py               | 35 +++++++++++++++-
 lbrynet/lbrynet_daemon/LBRYDaemon.py        | 44 +++++++++++++++++++++
 lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 23 +++++++----
 3 files changed, 93 insertions(+), 9 deletions(-)

diff --git a/lbrynet/core/LBRYcrdWallet.py b/lbrynet/core/LBRYcrdWallet.py
index 895dbfa49..3ff1008fb 100644
--- a/lbrynet/core/LBRYcrdWallet.py
+++ b/lbrynet/core/LBRYcrdWallet.py
@@ -1003,7 +1003,7 @@ class LBRYumWallet(LBRYWallet):
                     self.max_behind = self.blocks_behind_alert
                 self.catchup_progress = int(100 * (self.blocks_behind_alert / (5 + self.max_behind)))
                 if self._caught_up_counter == 0:
-                    alert.info('Catching up to the blockchain...showing blocks left...')
+                    alert.info('Catching up with the blockchain...showing blocks left...')
                 if self._caught_up_counter % 30 == 0:
                     alert.info('%d...', (remote_height - local_height))
                     alert.info("Catching up: " + str(self.catchup_progress) + "%")
@@ -1128,6 +1128,39 @@ class LBRYumWallet(LBRYWallet):
         func = getattr(self.cmd_runner, cmd.name)
         return threads.deferToThread(func)
 
+    def get_history(self):
+        cmd = known_commands['history']
+        func = getattr(self.cmd_runner, cmd.name)
+        return threads.deferToThread(func)
+
+    def get_tx_json(self, txid):
+        def _decode(raw_tx):
+            tx = Transaction(raw_tx).deserialize()
+            decoded_tx = {}
+            for txkey in tx.keys():
+                if isinstance(tx[txkey], list):
+                    decoded_tx[txkey] = []
+                    for i in tx[txkey]:
+                        tmp = {}
+                        for k in i.keys():
+                            if isinstance(i[k], Decimal):
+                                tmp[k] = float(i[k] / 1e8)
+                            else:
+                                tmp[k] = i[k]
+                        decoded_tx[txkey].append(tmp)
+                else:
+                    decoded_tx[txkey] = tx[txkey]
+            return decoded_tx
+
+        d = self._get_raw_tx(txid)
+        d.addCallback(_decode)
+        return d
+
+    def get_pub_keys(self, wallet):
+        cmd = known_commands['getpubkeys']
+        func = getattr(self.cmd_runner, cmd.name)
+        return threads.deferToThread(func, wallet)
+
     def _save_wallet(self, val):
         d = threads.deferToThread(self.wallet.storage.write)
         d.addCallback(lambda _: val)
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 95340be0d..3b658c880 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -1459,6 +1459,50 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         return d
 
+    def jsonrpc_get_transaction_history(self):
+        """
+        Get transaction history
+
+        Args:
+            None
+        Returns:
+            list of transactions
+        """
+
+        d = self.session.wallet.get_history()
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
+    def jsonrpc_get_transaction(self, p):
+        """
+        Get a decoded transaction from a txid
+
+        Args:
+            txid: txid hex string
+        Returns:
+            JSON formatted transaction
+        """
+
+
+        txid = p['txid']
+        d = self.session.wallet.get_tx_json(txid)
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
+    def jsonrpc_get_public_key_from_wallet(self, p):
+        """
+        Get public key from wallet address
+
+        Args:
+            wallet: wallet address, base58
+        Returns:
+            public key
+        """
+
+        wallet = p['wallet']
+        d = self.session.wallet.get_pub_keys(wallet)
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+
     def jsonrpc_get_time_behind_blockchain(self):
         """
         Get number of blocks behind the blockchain
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
index 5fc6f98ff..2f0f00154 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
@@ -72,7 +72,13 @@ def start():
                         help="Branch of lbry-web-ui repo to use, defaults on HEAD",
                         default="HEAD")
     parser.add_argument('--no-launch', dest='launchui', action="store_false")
-    parser.set_defaults(launchui=True)
+    parser.add_argument('--log-to-console', dest='logtoconsole', action="store_true")
+    parser.add_argument('--quiet', dest='quiet', action="store_true")
+    parser.set_defaults(launchui=True, logtoconsole=False, quiet=False)
+    args = parser.parse_args()
+
+    if args.logtoconsole:
+        logging.basicConfig(level=logging.INFO)
 
     try:
         JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()
@@ -82,13 +88,13 @@ def start():
         pass
 
     log.info("Starting lbrynet-daemon from command line")
-    print "Starting lbrynet-daemon from command line"
-    print "To view activity, view the log file here: " + LOG_FILENAME
-    print "Web UI is available at http://%s:%i" %(API_INTERFACE, API_PORT)
-    print "JSONRPC API is available at " + API_CONNECTION_STRING
-    print "To quit press ctrl-c or call 'stop' via the API"
 
-    args = parser.parse_args()
+    if not args.logtoconsole and not args.quiet:
+        print "Starting lbrynet-daemon from command line"
+        print "To view activity, view the log file here: " + LOG_FILENAME
+        print "Web UI is available at http://%s:%i" %(API_INTERFACE, API_PORT)
+        print "JSONRPC API is available at " + API_CONNECTION_STRING
+        print "To quit press ctrl-c or call 'stop' via the API"
 
     if args.branch == "HEAD":
         GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep %s | cut -f 1" % args.branch
@@ -171,7 +177,8 @@ def start():
         if args.launchui:
             d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
         reactor.run()
-        print "\nClosing lbrynet-daemon"
+        if not args.logtoconsole and not args.quiet:
+            print "\nClosing lbrynet-daemon"
     else:
         log.info("Not connected to internet, unable to start")
         print "Not connected to internet, unable to start"

From 24eed71959e6e9188dadd903190cf1f89d13fa72 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Mon, 18 Apr 2016 03:41:16 -0400
Subject: [PATCH 02/38] upload log function doc string

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 5547b1367..5e3296c4c 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -1643,6 +1643,16 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         return _check_version()
 
     def jsonrpc_upload_log(self, p=None):
+        """
+        Upload log
+
+        Args, optional:
+            'name_prefix': prefix to denote what is requesting the log upload
+             'exclude_previous': true/false, whether or not to exclude previous sessions from upload, defaults on true
+        Returns
+            True
+        """
+
         if p:
             if 'name_prefix' in p.keys():
                 prefix = p['name_prefix'] + '_api'

From 29f7307f9d5bea59d3c0df436a183b1de0ddb51a Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Mon, 18 Apr 2016 13:21:53 -0400
Subject: [PATCH 03/38] periodically check internet connection and for new
 versions

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py | 147 +++++++++++++++++++--------
 1 file changed, 105 insertions(+), 42 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 5e3296c4c..f185e1de2 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -11,9 +11,11 @@ import base64
 import base58
 import platform
 import json
+import socket
 
 from twisted.web import server, resource, static
 from twisted.internet import defer, threads, error, reactor
+from twisted.internet.task import LoopingCall
 from txjsonrpc import jsonrpclib
 from txjsonrpc.web import jsonrpc
 from txjsonrpc.web.jsonrpc import Handler
@@ -70,15 +72,29 @@ handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=262144, ba
 log.addHandler(handler)
 log.setLevel(logging.INFO)
 
+
+INITIALIZING_CODE = 'initializing'
+LOADING_DB_CODE = 'loading_db'
+LOADING_WALLET_CODE = 'loading_wallet'
+LOADING_FILE_MANAGER_CODE = 'loading_file_manager'
+LOADING_SERVER_CODE = 'loading_server'
+STARTED_CODE = 'started'
 STARTUP_STAGES = [
-                    ('initializing', 'Initializing...'),
-                    ('loading_db', 'Loading databases...'),
-                    ('loading_wallet', 'Catching up with the blockchain... %s'),
-                    ('loading_file_manager', 'Setting up file manager'),
-                    ('loading_server', 'Starting lbrynet'),
-                    ('started', 'Started lbrynet')
+                    (INITIALIZING_CODE, 'Initializing...'),
+                    (LOADING_DB_CODE, 'Loading databases...'),
+                    (LOADING_WALLET_CODE, 'Catching up with the blockchain... %s'),
+                    (LOADING_FILE_MANAGER_CODE, 'Setting up file manager'),
+                    (LOADING_SERVER_CODE, 'Starting lbrynet'),
+                    (STARTED_CODE, 'Started lbrynet')
                   ]
 
+CONNECT_CODE_VERSION_CHECK = 'version_check'
+CONNECT_CODE_NETWORK = 'network_connection'
+CONNECT_CODE_WALLET = 'wallet_catchup_lag'
+CONNECTION_PROBLEM_CODES = [(CONNECT_CODE_VERSION_CHECK, "There was a problem checking for updates on github"),
+                            (CONNECT_CODE_NETWORK, "Your internet connection appears to have been interrupted"),
+                            (CONNECT_CODE_WALLET, "Synchronization with the blockchain is lagging... if this continues try restarting LBRY")]
+
 ALLOWED_DURING_STARTUP = ['is_running', 'is_first_run',
                           'get_time_behind_blockchain', 'stop',
                           'daemon_status', 'get_start_notice',
@@ -91,6 +107,9 @@ OK_CODE = 200
 # TODO alert if your copy of a lbry file is out of date with the name record
 
 
+REMOTE_SERVER = "www.google.com"
+
+
 class LBRYDaemon(jsonrpc.JSONRPC):
     """
     LBRYnet daemon, a jsonrpc interface to lbry functions
@@ -105,6 +124,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.startup_status = STARTUP_STAGES[0]
         self.startup_message = None
         self.announced_startup = False
+        self.connected_to_internet = True
+        self.connection_problem = None
         self.query_handlers = {}
         self.ui_version = ui_version_info.replace('\n', '')
         self.git_lbrynet_version = None
@@ -172,6 +193,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.wallet_user = None
         self.wallet_password = None
 
+        self.internet_connection_checker = LoopingCall(self._check_network_connection)
+        self.version_checker = LoopingCall(self._check_remote_versions)
+        self.connection_problem_checker = LoopingCall(self._check_connection_problems)
+
         self.sd_identifier = StreamDescriptorIdentifier()
         self.stream_info_manager = TempLBRYFileMetadataManager()
         self.settings = LBRYSettings(self.db_dir)
@@ -267,32 +292,9 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
     def setup(self):
         def _log_starting_vals():
-            def _get_lbry_files_json():
-                r = self._get_lbry_files()
-                return json.dumps(r)
+            r = json.dumps(self._get_lbry_files())
 
-            def _get_lbryum_version():
-                r = urlopen("https://raw.githubusercontent.com/lbryio/lbryum/master/lib/version.py").read().split('\n')
-                version = next(line.split("=")[1].split("#")[0].replace(" ", "")
-                               for line in r if "ELECTRUM_VERSION" in line)
-                version = version.replace("'", "")
-                log.info("remote lbryum " + str(version) + " > local lbryum " + str(lbryum_version) + " = " + str(
-                    version > lbryum_version))
-                return version
-
-            def _get_lbrynet_version():
-                r = urlopen("https://raw.githubusercontent.com/lbryio/lbry/master/lbrynet/__init__.py").read().split('\n')
-                vs = next(i for i in r if 'version =' in i).split("=")[1].replace(" ", "")
-                vt = tuple(int(x) for x in vs[1:-1].split(','))
-                vr = ".".join([str(x) for x in vt])
-                log.info("remote lbrynet " + str(vr) + " > local lbrynet " + str(lbrynet_version) + " = " + str(
-                    vr > lbrynet_version))
-                return vr
-
-            self.git_lbrynet_version = _get_lbrynet_version()
-            self.git_lbryum_version = _get_lbryum_version()
-
-            log.info("LBRY Files: " + _get_lbry_files_json())
+            log.info("LBRY Files: " + r)
             log.info("Starting balance: " + str(self.session.wallet.wallet_balance))
 
             return defer.succeed(None)
@@ -317,6 +319,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         log.info("[" + str(datetime.now()) + "] Starting lbrynet-daemon")
 
+        self.internet_connection_checker.start(60)
+        self.version_checker.start(3600)
+        self.connection_problem_checker.start(1)
+
         d = defer.Deferred()
         d.addCallback(lambda _: self._initial_setup())
         d.addCallback(self._set_daemon_settings)
@@ -379,8 +385,63 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.session_settings = settings
         return defer.succeed(None)
 
-    def _start_server(self):
+    def _check_network_connection(self):
+        try:
+            host = socket.gethostbyname(REMOTE_SERVER)
+            s = socket.create_connection((host, 80), 2)
+            self.connected_to_internet = True
+        except:
+            log.info("[" + str(datetime.now()) + "] Internet connection not working")
+            self.connected_to_internet = False
 
+    def _check_remote_versions(self):
+        def _get_lbryum_version():
+            try:
+                r = urlopen("https://raw.githubusercontent.com/lbryio/lbryum/master/lib/version.py").read().split('\n')
+                version = next(line.split("=")[1].split("#")[0].replace(" ", "")
+                               for line in r if "ELECTRUM_VERSION" in line)
+                version = version.replace("'", "")
+                log.info("remote lbryum " + str(version) + " > local lbryum " + str(lbryum_version) + " = " + str(
+                    version > lbryum_version))
+                self.git_lbryum_version = version
+                return defer.succeed(None)
+            except:
+                log.info("[" + str(datetime.now()) + "] Failed to get lbryum version from git")
+                self.git_lbryum_version = None
+                return defer.fail(None)
+
+        def _get_lbrynet_version():
+            try:
+                r = urlopen("https://raw.githubusercontent.com/lbryio/lbry/master/lbrynet/__init__.py").read().split('\n')
+                vs = next(i for i in r if 'version =' in i).split("=")[1].replace(" ", "")
+                vt = tuple(int(x) for x in vs[1:-1].split(','))
+                vr = ".".join([str(x) for x in vt])
+                log.info("remote lbrynet " + str(vr) + " > local lbrynet " + str(lbrynet_version) + " = " + str(
+                    vr > lbrynet_version))
+                self.git_lbrynet_version = vr
+                return defer.succeed(None)
+            except:
+                log.info("[" + str(datetime.now()) + "] Failed to get lbrynet version from git")
+                self.git_lbrynet_version = None
+                return defer.fail(None)
+
+        d = _get_lbrynet_version()
+        d.addCallback(lambda _: _get_lbryum_version())
+
+    def _check_connection_problems(self):
+        if not self.git_lbrynet_version or not self.git_lbryum_version:
+            self.connection_problem = CONNECTION_PROBLEM_CODES[0]
+
+        elif self.startup_status[0] == 'loading_wallet':
+            if self.session.wallet.is_lagging:
+                self.connection_problem = CONNECTION_PROBLEM_CODES[2]
+        else:
+            self.connection_problem = None
+
+        if not self.connected_to_internet:
+            self.connection_problem = CONNECTION_PROBLEM_CODES[1]
+
+    def _start_server(self):
         if self.peer_port is not None:
 
             server_factory = ServerProtocolFactory(self.session.rate_limiter,
@@ -402,7 +463,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             return defer.succeed(True)
 
     def _setup_server(self):
-
         def restore_running_status(running):
             if running is True:
                 return self._start_server()
@@ -439,7 +499,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         return dl
 
     def _add_query_handlers(self, query_handlers):
-
         def _set_query_handlers(statuses):
             from future_builtins import zip
             for handler, (success, status) in zip(query_handlers, statuses):
@@ -482,6 +541,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         log.info("Closing lbrynet session")
         log.info("Status at time of shutdown: " + self.startup_status[0])
 
+        self.internet_connection_checker.stop()
+        self.version_checker.stop()
+        self.connection_problem_checker.stop()
+
         d = self._upload_log(name_prefix="close", exclude_previous=False if self.first_run else True)
         d.addCallback(lambda _: self._stop_server())
         d.addErrback(lambda err: log.info("Bad server shutdown: " + err.getTraceback()))
@@ -965,12 +1028,12 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         """
 
         r = {'code': self.startup_status[0], 'message': self.startup_status[1], 'progress': None, 'is_lagging': None}
-        if self.startup_status[0] == 'loading_wallet':
-            r['is_lagging'] = self.session.wallet.is_lagging
-            if r['is_lagging'] == True:
-                r['message'] = "Synchronization with the blockchain is lagging... if this continues try restarting LBRY"
-            else:
-                r['message'] = r['message'] % (str(self.session.wallet.blocks_behind_alert) + " blocks behind")
+
+        if self.connection_problem:
+            r['message'] = self.connection_problem[1]
+            r['is_lagging'] = True
+        elif self.startup_status[0] == LOADING_WALLET_CODE:
+            r['message'] = r['message'] % (str(self.session.wallet.blocks_behind_alert) + " blocks behind")
             r['progress'] = self.session.wallet.catchup_progress
 
         log.info("[" + str(datetime.now()) + "] daemon status: " + str(r))
@@ -1647,8 +1710,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         Upload log
 
         Args, optional:
-            'name_prefix': prefix to denote what is requesting the log upload
-             'exclude_previous': true/false, whether or not to exclude previous sessions from upload, defaults on true
+            'name_prefix': prefix to indicate what is requesting the log upload
+            'exclude_previous': true/false, whether or not to exclude previous sessions from upload, defaults on true
         Returns
             True
         """

From a82f85f0586ed484da6734d8f55611f37fc6c5ef Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 20 Apr 2016 02:56:29 -0400
Subject: [PATCH 04/38] bump version, increase log size, fix shutdown problem

---
 lbrynet/__init__.py                         |  2 +-
 lbrynet/lbrynet_daemon/LBRYDaemon.py        | 15 ++++++---------
 lbrynet/lbrynet_daemon/LBRYDaemonControl.py |  2 +-
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py
index 8b1a36331..8f010329d 100644
--- a/lbrynet/__init__.py
+++ b/lbrynet/__init__.py
@@ -4,5 +4,5 @@ import logging
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
 
-version = (0, 2, 2)
+version = (0, 2, 3)
 __version__ = ".".join([str(x) for x in version])
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index f185e1de2..3afff6eaf 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -68,7 +68,7 @@ else:
     PREVIOUS_LOG = 0
 
 log = logging.getLogger(__name__)
-handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=262144, backupCount=5)
+handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, backupCount=5)
 log.addHandler(handler)
 log.setLevel(logging.INFO)
 
@@ -541,10 +541,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         log.info("Closing lbrynet session")
         log.info("Status at time of shutdown: " + self.startup_status[0])
 
-        self.internet_connection_checker.stop()
-        self.version_checker.stop()
-        self.connection_problem_checker.stop()
-
         d = self._upload_log(name_prefix="close", exclude_previous=False if self.first_run else True)
         d.addCallback(lambda _: self._stop_server())
         d.addErrback(lambda err: log.info("Bad server shutdown: " + err.getTraceback()))
@@ -1022,9 +1018,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         Args:
             None
         Returns:
-            'status_message': startup status message
-            'status_code': status_code
-            if status_code is 'loading_wallet', also contains key 'progress': blockchain catchup progress
+            'message': startup status message
+            'code': status_code
+            'progress': progress, only used in loading_wallet
+            'is_lagging': flag set to indicate lag, if set message will contain relevant message
         """
 
         r = {'code': self.startup_status[0], 'message': self.startup_status[1], 'progress': None, 'is_lagging': None}
@@ -1263,7 +1260,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         d = self._shutdown()
         d.addCallback(lambda _: _disp_shutdown())
-        d.addCallback(lambda _: reactor.callLater(1.0, reactor.stop))
+        d.addCallback(lambda _: reactor.callLater(0.0, reactor.stop))
 
         return self._render_response("Shutting down", OK_CODE)
 
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
index 3ecdab03c..13772a386 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
@@ -30,7 +30,7 @@ if not os.path.isdir(log_dir):
 
 LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log')
 log = logging.getLogger(__name__)
-handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=262144, backupCount=5)
+handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, backupCount=5)
 log.addHandler(handler)
 log.setLevel(logging.INFO)
 

From c4620a0b0cc689b48a0e4aee8b1fc94fdb93e55d Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 20 Apr 2016 04:24:32 -0400
Subject: [PATCH 05/38] add problem_code to daemon_status

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 3afff6eaf..05a089afd 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -1024,9 +1024,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             'is_lagging': flag set to indicate lag, if set message will contain relevant message
         """
 
-        r = {'code': self.startup_status[0], 'message': self.startup_status[1], 'progress': None, 'is_lagging': None}
+        r = {'code': self.startup_status[0], 'message': self.startup_status[1],
+             'progress': None, 'is_lagging': None, 'problem_code': None}
 
         if self.connection_problem:
+            r['problem_code'] = self.connection_problem[0]
             r['message'] = self.connection_problem[1]
             r['is_lagging'] = True
         elif self.startup_status[0] == LOADING_WALLET_CODE:

From c5d653a51aaf2d616edc577d4c6cefd2e7086a06 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 20 Apr 2016 14:44:26 -0400
Subject: [PATCH 06/38] add bug report page

adds a page to submit a message and upload log at /report
---
 lbrynet/lbrynet_daemon/LBRYDaemon.py        | 42 ++++++++++++++++++---
 lbrynet/lbrynet_daemon/LBRYDaemonControl.py |  3 +-
 2 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 05a089afd..7d0645404 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -512,13 +512,13 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         dl.addCallback(_set_query_handlers)
         return dl
 
-    def _upload_log(self, name_prefix=None, exclude_previous=False):
+    def _upload_log(self, name_prefix=None, exclude_previous=False, force=False):
         if name_prefix:
             name_prefix = name_prefix + "-" + platform.system()
         else:
             name_prefix = platform.system()
 
-        if self.session_settings['upload_log']:
+        if self.session_settings['upload_log'] or force:
             LOG_URL = "https://lbry.io/log-upload"
             if exclude_previous:
                 f = open(self.log_file, "r")
@@ -534,8 +534,9 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             params = {'name': log_name, 'log': log_contents}
 
             requests.post(LOG_URL, params)
-
-        return defer.succeed(None)
+            return defer.succeed(None)
+        else:
+            return defer.succeed(None)
 
     def _shutdown(self):
         log.info("Closing lbrynet session")
@@ -1720,15 +1721,21 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                 prefix = p['name_prefix'] + '_api'
             else:
                 prefix = None
-            if 'exclude_previous' in p.keys:
+
+            if 'exclude_previous' in p.keys():
                 exclude_previous = p['exclude_previous']
             else:
                 exclude_previous = True
+
+            if 'force' in p.keys():
+                force = p['force']
+            else:
+                force = False
         else:
             prefix = "api"
             exclude_previous = True
 
-        d = self._upload_log(name_prefix=prefix, exclude_previous=exclude_previous)
+        d = self._upload_log(name_prefix=prefix, exclude_previous=exclude_previous, force=force)
         d.addCallback(lambda _: self._render_response(True, OK_CODE))
         return d
 
@@ -1781,3 +1788,26 @@ class LBRYFileRender(resource.Resource):
             return server.NOT_DONE_YET
         else:
             return server.failure
+
+class LBRYBugReport(resource.Resource):
+    isLeaf = False
+
+    def _delayed_render(self, request, results):
+        request.write(results)
+        request.finish()
+
+    def render_GET(self, request):
+        return '<html><body><form method="POST">' \
+               '<br>Please describe the problem you experienced and any information you think might be useful to us. Links to screenshots are great!</br>' \
+               '<textarea cols="50" rows="10" name="message" type="text"></textarea>' \
+               '<button>Submit</button>' \
+               '</form></body></html>'
+
+    def render_POST(self, request):
+        msg = request.args["message"][0]
+        log.info("User submitted error report: " + str(msg))
+        api = jsonrpc.Proxy(API_CONNECTION_STRING)
+        d = api.callRemote("upload_log", {'name_prefix': 'report', 'exclude_previous': False, 'force': True})
+        d.addCallback(lambda _: self._delayed_render(request, "<html><body>Your bug report is greatly appreciated! <a href='lbry://lbry'>Click here to return to LBRY</a></body></html>"))
+
+        return server.NOT_DONE_YET
\ No newline at end of file
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
index 13772a386..3f7b7e411 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
@@ -17,7 +17,7 @@ from twisted.web import server, static
 from twisted.internet import reactor, defer
 from jsonrpc.proxy import JSONRPCProxy
 
-from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon, LBRYindex, LBRYFileRender
+from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon, LBRYindex, LBRYFileRender, LBRYBugReport
 from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, DEFAULT_WALLET, UI_ADDRESS
 
 if sys.platform != "darwin":
@@ -166,6 +166,7 @@ def start():
         root.putChild("img", static.File(os.path.join(ui_dir, "img")))
         root.putChild("js", static.File(os.path.join(ui_dir, "js")))
         root.putChild("view", LBRYFileRender())
+        root.putChild("report", LBRYBugReport())
         return defer.succeed([root, ui_version])
 
     def setupapi(root, wallet, ui_version):

From e5e25012ffc7ce2c86940b955208342a7e5a55ff Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 20 Apr 2016 22:02:52 -0400
Subject: [PATCH 07/38] move daemon server stuff into its own class

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py        |  87 +-------
 lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 125 ++++--------
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py  | 210 ++++++++++++++++++++
 3 files changed, 248 insertions(+), 174 deletions(-)
 create mode 100644 lbrynet/lbrynet_daemon/LBRYDaemonServer.py

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 7d0645404..c30b5b594 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -3,17 +3,13 @@ import os
 import sys
 import simplejson as json
 import binascii
-import subprocess
-import logging
 import logging.handlers
 import requests
-import base64
 import base58
 import platform
-import json
 import socket
 
-from twisted.web import server, resource, static
+from twisted.web import server
 from twisted.internet import defer, threads, error, reactor
 from twisted.internet.task import LoopingCall
 from txjsonrpc import jsonrpclib
@@ -31,7 +27,6 @@ from lbrynet.core.PaymentRateManager import PaymentRateManager
 from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
 from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
 from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
-from lbrynet.lbrynet_console.ControlHandlers import get_time_behind_blockchain
 from lbrynet.core.Error import UnknownNameError
 from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
 from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory
@@ -40,7 +35,8 @@ from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream, FetcherDaemon
 from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher
 from lbrynet.core.utils import generate_id
 from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
-from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE
+from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \
+    DEFAULT_WALLET, API_INTERFACE
 from lbrynet.conf import API_CONNECTION_STRING, API_PORT, API_ADDRESS, DEFAULT_TIMEOUT, UI_ADDRESS
 from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob
 from lbrynet.core.Session import LBRYSession
@@ -117,7 +113,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
     isLeaf = True
 
-    def __init__(self, ui_version_info, wallet_type="lbryum"):
+    def __init__(self, ui_version_info, wallet_type=DEFAULT_WALLET):
         jsonrpc.JSONRPC.__init__(self)
         reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
 
@@ -1693,8 +1689,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             true/false, true meaning that there is a new version available
         """
 
-
-
         def _check_version():
             if (lbrynet_version >= self.git_lbrynet_version) and (lbryum_version >= self.git_lbryum_version):
                 log.info("[" + str(datetime.now()) + "] Up to date")
@@ -1738,76 +1732,3 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d = self._upload_log(name_prefix=prefix, exclude_previous=exclude_previous, force=force)
         d.addCallback(lambda _: self._render_response(True, OK_CODE))
         return d
-
-
-class LBRYDaemonCommandHandler(object):
-    def __init__(self, command):
-        self._api = jsonrpc.Proxy(API_CONNECTION_STRING)
-        self.command = command
-
-    def run(self, params=None):
-        if params:
-            d = self._api.callRemote(self.command, params)
-        else:
-            d = self._api.callRemote(self.command)
-        return d
-
-
-class LBRYindex(resource.Resource):
-    def __init__(self, ui_dir):
-        resource.Resource.__init__(self)
-        self.ui_dir = ui_dir
-
-    isLeaf = False
-
-    def _delayed_render(self, request, results):
-        request.write(str(results))
-        request.finish()
-
-    def getChild(self, name, request):
-        if name == '':
-            return self
-        return resource.Resource.getChild(self, name, request)
-
-    def render_GET(self, request):
-        return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request)
-
-
-class LBRYFileRender(resource.Resource):
-    isLeaf = False
-
-    def render_GET(self, request):
-        if 'name' in request.args.keys():
-            api = jsonrpc.Proxy(API_CONNECTION_STRING)
-            if request.args['name'][0] != 'lbry':
-                d = api.callRemote("get", {'name': request.args['name'][0]})
-                d.addCallback(lambda results: static.File(results['path']).render_GET(request))
-            else:
-                request.redirect(UI_ADDRESS)
-                request.finish()
-            return server.NOT_DONE_YET
-        else:
-            return server.failure
-
-class LBRYBugReport(resource.Resource):
-    isLeaf = False
-
-    def _delayed_render(self, request, results):
-        request.write(results)
-        request.finish()
-
-    def render_GET(self, request):
-        return '<html><body><form method="POST">' \
-               '<br>Please describe the problem you experienced and any information you think might be useful to us. Links to screenshots are great!</br>' \
-               '<textarea cols="50" rows="10" name="message" type="text"></textarea>' \
-               '<button>Submit</button>' \
-               '</form></body></html>'
-
-    def render_POST(self, request):
-        msg = request.args["message"][0]
-        log.info("User submitted error report: " + str(msg))
-        api = jsonrpc.Proxy(API_CONNECTION_STRING)
-        d = api.callRemote("upload_log", {'name_prefix': 'report', 'exclude_previous': False, 'force': True})
-        d.addCallback(lambda _: self._delayed_render(request, "<html><body>Your bug report is greatly appreciated! <a href='lbry://lbry'>Click here to return to LBRY</a></body></html>"))
-
-        return server.NOT_DONE_YET
\ No newline at end of file
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
index 3f7b7e411..0321de4df 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
@@ -1,25 +1,22 @@
 import argparse
 import logging
 import logging.handlers
-import subprocess
 import os
-import shutil
 import webbrowser
 import sys
 import socket
+import subprocess
+import platform
 
-from StringIO import StringIO
-from zipfile import ZipFile
-from urllib import urlopen
-from datetime import datetime
 from appdirs import user_data_dir
-from twisted.web import server, static
+from twisted.web import server
 from twisted.internet import reactor, defer
 from jsonrpc.proxy import JSONRPCProxy
 
-from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon, LBRYindex, LBRYFileRender, LBRYBugReport
+from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer
 from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, DEFAULT_WALLET, UI_ADDRESS
 
+
 if sys.platform != "darwin":
     log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
 else:
@@ -34,6 +31,7 @@ handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, b
 log.addHandler(handler)
 log.setLevel(logging.INFO)
 
+
 REMOTE_SERVER = "www.google.com"
 
 
@@ -46,11 +44,24 @@ def test_internet_connection():
         return False
 
 
+def prompt_for_xcode_if_needed():
+    t = subprocess.check_output("git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep HEAD | cut -f 1", shell=True)
+    if not t:
+        if platform.system().lower() != "darwin":
+            print "Please install git"
+            sys.exit(0)
+        else:
+            print "You should have been alerted to install xcode command line tools, please do so and then start lbry"
+            sys.exit(0)
+
+
 def stop():
     def _disp_shutdown():
+        print "Shutting down lbrynet-daemon from command line"
         log.info("Shutting down lbrynet-daemon from command line")
 
     def _disp_not_running():
+        print "Attempt to shut down lbrynet-daemon from command line when daemon isn't running"
         log.info("Attempt to shut down lbrynet-daemon from command line when daemon isn't running")
 
     d = defer.Deferred(None)
@@ -67,7 +78,7 @@ def start():
                         default=DEFAULT_WALLET)
     parser.add_argument("--ui",
                         help="path to custom UI folder",
-                        default="")
+                        default=None)
     parser.add_argument("--branch",
                         help="Branch of lbry-web-ui repo to use, defaults on HEAD",
                         default="HEAD")
@@ -85,106 +96,38 @@ def start():
     try:
         JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()
         log.info("lbrynet-daemon is already running")
+        if not args.logtoconsole:
+            print "lbrynet-daemon is already running"
         if args.launchui:
             webbrowser.open(UI_ADDRESS)
         return
     except:
         pass
 
+    prompt_for_xcode_if_needed()
+
     log.info("Starting lbrynet-daemon from command line")
 
     if not args.logtoconsole and not args.quiet:
         print "Starting lbrynet-daemon from command line"
         print "To view activity, view the log file here: " + LOG_FILENAME
-        print "Web UI is available at http://%s:%i" %(API_INTERFACE, API_PORT)
+        print "Web UI is available at http://%s:%i" % (API_INTERFACE, API_PORT)
         print "JSONRPC API is available at " + API_CONNECTION_STRING
         print "To quit press ctrl-c or call 'stop' via the API"
 
-    if args.branch == "HEAD":
-        GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep %s | cut -f 1" % args.branch
-        DIST_URL = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip"
-    else:
-        log.info("Using UI branch: " + args.branch)
-        GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep refs/heads/%s | cut -f 1" % args.branch
-        DIST_URL = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % args.branch
-
-    def getui(ui_dir=None):
-        if ui_dir:
-            if os.path.isdir(ui_dir):
-                log.info("Using user specified UI directory: " + str(ui_dir))
-                ui_version_info = "user-specified"
-                return defer.succeed([ui_dir, ui_version_info])
-            else:
-                log.info("User specified UI directory doesn't exist: " + str(ui_dir))
-
-        def download_ui(dest_dir, ui_version):
-            url = urlopen(DIST_URL)
-            z = ZipFile(StringIO(url.read()))
-            names = [i for i in z.namelist() if '.DS_Store' not in i and '__MACOSX' not in i]
-            z.extractall(dest_dir, members=names)
-            return defer.succeed([dest_dir, ui_version])
-
-        data_dir = user_data_dir("LBRY")
-        version_dir = os.path.join(data_dir, "ui_version_history")
-
-        git_version = subprocess.check_output(GIT_CMD_STRING, shell=True)
-        if not git_version:
-            log.info("You should have been notified to install xcode command line tools, once it's installed you can start LBRY")
-            print "You should have been notified to install xcode command line tools, once it's installed you can start LBRY"
-            sys.exit(0)
-
-        ui_version_info = git_version
-
-        if not os.path.isdir(data_dir):
-            os.mkdir(data_dir)
-
-        if not os.path.isdir(os.path.join(data_dir, "ui_version_history")):
-            os.mkdir(version_dir)
-
-        if not os.path.isfile(os.path.join(version_dir, git_version)):
-            f = open(os.path.join(version_dir, git_version), "w")
-            version_message = "[" + str(datetime.now()) + "] Updating UI --> " + git_version
-            f.write(version_message)
-            f.close()
-            log.info(version_message)
-
-            if os.path.isdir(os.path.join(data_dir, "lbry-web-ui")):
-                shutil.rmtree(os.path.join(data_dir, "lbry-web-ui"))
-        else:
-            version_message = "[" + str(datetime.now()) + "] UI version " + git_version + " up to date"
-            log.info(version_message)
-
-        if os.path.isdir(os.path.join(data_dir, "lbry-web-ui")):
-            return defer.succeed([os.path.join(data_dir, "lbry-web-ui"), ui_version_info])
-        else:
-            return download_ui(os.path.join(data_dir, "lbry-web-ui"), ui_version_info)
-
-    def setupserver(ui_dir, ui_version):
-        root = LBRYindex(ui_dir)
-        root.putChild("css", static.File(os.path.join(ui_dir, "css")))
-        root.putChild("font", static.File(os.path.join(ui_dir, "font")))
-        root.putChild("img", static.File(os.path.join(ui_dir, "img")))
-        root.putChild("js", static.File(os.path.join(ui_dir, "js")))
-        root.putChild("view", LBRYFileRender())
-        root.putChild("report", LBRYBugReport())
-        return defer.succeed([root, ui_version])
-
-    def setupapi(root, wallet, ui_version):
-        daemon = LBRYDaemon(ui_version, wallet_type=wallet)
-        root.putChild(API_ADDRESS, daemon)
-        reactor.listenTCP(API_PORT, server.Site(root), interface=API_INTERFACE)
-        return daemon.setup()
-
     if test_internet_connection():
-        d = getui(args.ui)
-        d.addCallback(lambda r: setupserver(r[0], r[1]))
-        d.addCallback(lambda r: setupapi(r[0], args.wallet, r[1]))
-        if args.launchui:
-            d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
+        lbry = LBRYDaemonServer()
+
+        d = lbry.start(branch=args.branch, user_specified=args.ui)
+        d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
+
+        reactor.listenTCP(API_PORT, server.Site(lbry.root), interface=API_INTERFACE)
         reactor.run()
+
         if not args.logtoconsole and not args.quiet:
             print "\nClosing lbrynet-daemon"
     else:
         log.info("Not connected to internet, unable to start")
-        print "Not connected to internet, unable to start"
+        if not args.logtoconsole:
+            print "Not connected to internet, unable to start"
         return
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
new file mode 100644
index 000000000..deb23e51d
--- /dev/null
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -0,0 +1,210 @@
+import logging
+import subprocess
+import os
+import shutil
+import jsonrpc
+import json
+
+from StringIO import StringIO
+from zipfile import ZipFile
+from urllib import urlopen
+from datetime import datetime
+from appdirs import user_data_dir
+from twisted.web import server, static, resource
+from twisted.internet import defer
+
+from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon
+from lbrynet.conf import API_CONNECTION_STRING, API_ADDRESS, DEFAULT_WALLET, UI_ADDRESS
+
+log = logging.getLogger(__name__)
+
+data_dir = user_data_dir("LBRY")
+if not os.path.isdir(data_dir):
+    os.mkdir(data_dir)
+version_dir = os.path.join(data_dir, "ui_version_history")
+if not os.path.isdir(version_dir):
+    os.mkdir(version_dir)
+
+version_log = logging.getLogger("lbry_version")
+version_log.addHandler(logging.FileHandler(os.path.join(version_dir, "lbry_version.log")))
+version_log.setLevel(logging.INFO)
+
+
+class LBRYindex(resource.Resource):
+    def __init__(self, ui_dir):
+        resource.Resource.__init__(self)
+        self.ui_dir = ui_dir
+
+    isLeaf = False
+
+    def _delayed_render(self, request, results):
+        request.write(str(results))
+        request.finish()
+
+    def getChild(self, name, request):
+        if name == '':
+            return self
+        return resource.Resource.getChild(self, name, request)
+
+    def render_GET(self, request):
+        return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request)
+
+
+class LBRYFileRender(resource.Resource):
+    isLeaf = False
+
+    def render_GET(self, request):
+        if 'name' in request.args.keys():
+            api = jsonrpc.Proxy(API_CONNECTION_STRING)
+            if request.args['name'][0] != 'lbry':
+                d = api.callRemote("get", {'name': request.args['name'][0]})
+                d.addCallback(lambda results: static.File(results['path']).render_GET(request))
+            else:
+                request.redirect(UI_ADDRESS)
+                request.finish()
+            return server.NOT_DONE_YET
+        else:
+            return server.failure
+
+
+class LBRYBugReport(resource.Resource):
+    isLeaf = False
+
+    def _delayed_render(self, request, results):
+        request.write(results)
+        request.finish()
+
+    def render_GET(self, request):
+        return '<html><body><form method="POST">' \
+               '<br>Please describe the problem you experienced and any information you think might be useful to us. Links to screenshots are great!</br>' \
+               '<textarea cols="50" rows="10" name="message" type="text"></textarea>' \
+               '<button>Submit</button>' \
+               '</form></body></html>'
+
+    def render_POST(self, request):
+        msg = request.args["message"][0]
+        log.info("User submitted error report: " + str(msg))
+        api = jsonrpc.Proxy(API_CONNECTION_STRING)
+        d = api.callRemote("upload_log", {'name_prefix': 'report', 'exclude_previous': False, 'force': True})
+        d.addCallback(lambda _: self._delayed_render(request, "<html><body>Your bug report is greatly appreciated! <a href='lbry://lbry'>Click here to return to LBRY</a></body></html>"))
+
+        return server.NOT_DONE_YET
+
+
+class LBRYDaemonServer(object):
+    def __init__(self):
+        self.data_dir = user_data_dir("LBRY")
+        if not os.path.isdir(self.data_dir):
+            os.mkdir(self.data_dir)
+        self.version_dir = os.path.join(self.data_dir, "ui_version_history")
+        if not os.path.isdir(self.version_dir):
+            os.mkdir(self.version_dir)
+        self.config = os.path.join(self.version_dir, "active.json")
+        self.ui_dir = os.path.join(self.data_dir, "lbry-web-ui")
+        self.git_version = None
+        self._api = None
+        self.root = None
+
+        if not os.path.isfile(os.path.join(self.config)):
+            self.loaded_git_version = None
+        else:
+            try:
+                f = open(self.config, "r")
+                loaded_ui = json.loads(f.read())
+                f.close()
+                self.loaded_git_version = loaded_ui['commit']
+            except:
+                self.loaded_git_version = None
+
+    def setup(self, branch="HEAD", user_specified=None):
+        self.branch = branch
+        if user_specified:
+            if os.path.isdir(user_specified):
+                log.info("Using user specified UI directory: " + str(user_specified))
+                self.branch = "user-specified"
+                self.loaded_git_version = "user-specified"
+                self.ui_dir = user_specified
+                return defer.succeed("user-specified")
+            else:
+                log.info("User specified UI directory doesn't exist, using " + branch)
+        elif branch == "HEAD":
+            log.info("Using UI branch: " + branch)
+            self._gitcmd = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep %s | cut -f 1" % branch
+            self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip"
+        else:
+            log.info("Using UI branch: " + branch)
+            self._gitcmd = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep refs/heads/%s | cut -f 1" % branch
+            self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % branch
+
+        d = self._up_to_date()
+        d.addCallback(lambda r: self._download_ui() if not r else self.branch)
+        return d
+
+    def _up_to_date(self):
+        def _get_git_info():
+            r = subprocess.check_output(self._gitcmd, shell=True)
+            return defer.succeed(r)
+
+        def _set_git(version):
+            self.git_version = version
+            log.info("UI version from git: " + str(self.git_version).replace("\n", ""))
+            version_log.info("UI version from git: " + str(self.git_version).replace("\n", ""))
+
+            if self.git_version == self.loaded_git_version and os.path.isdir(self.ui_dir):
+                log.info("UI is up to date")
+                version_log.info("UI is up to date")
+                return defer.succeed(True)
+            else:
+                log.info("Downloading UI")
+                version_log.info("Downloading UI")
+                f = open(self.config, "w")
+                f.write(json.dumps({'commit': self.git_version,
+                                    'time': str(datetime.now())}))
+                f.close()
+                return defer.succeed(False)
+
+        d = _get_git_info()
+        d.addCallback(_set_git)
+        return d
+
+    def _download_ui(self):
+        def _delete_ui_dir():
+            if os.path.isdir(self.ui_dir):
+                if self.loaded_git_version:
+                    version_log.info("Removed ui files for commit " + str(self.loaded_git_version))
+                log.info("Removing out of date ui files")
+                shutil.rmtree(self.ui_dir)
+            return defer.succeed(None)
+
+        def _dl_ui():
+            url = urlopen(self._dist_url)
+            z = ZipFile(StringIO(url.read()))
+            names = [i for i in z.namelist() if '.DS_exStore' not in i and '__MACOSX' not in i]
+            z.extractall(self.ui_dir, members=names)
+            version_log.info("[" + str(datetime.now()) + "] Updated branch " + self.branch + " commit: " + str(self.loaded_git_version) + " to " + self.git_version)
+            log.info("Downloaded files for UI commit " + self.git_version)
+            self.loaded_git_version = self.git_version
+            return self.branch
+
+        d = _delete_ui_dir()
+        d.addCallback(lambda _: _dl_ui())
+        return d
+
+    def _setup_server(self, ui_ver):
+        self._api = LBRYDaemon(ui_ver, wallet_type=DEFAULT_WALLET)
+        self.root = LBRYindex(self.ui_dir)
+        self.root.putChild("css", static.File(os.path.join(self.ui_dir, "css")))
+        self.root.putChild("font", static.File(os.path.join(self.ui_dir, "font")))
+        self.root.putChild("img", static.File(os.path.join(self.ui_dir, "img")))
+        self.root.putChild("js", static.File(os.path.join(self.ui_dir, "js")))
+        self.root.putChild("view", LBRYFileRender())
+        self.root.putChild("report", LBRYBugReport())
+        self.root.putChild(API_ADDRESS, self._api)
+        return defer.succeed(True)
+
+    def start(self, branch="HEAD", user_specified=False):
+        d = self.setup(branch=branch, user_specified=user_specified)
+        d.addCallback(lambda v: self._setup_server(v))
+        d.addCallback(lambda _: self._api.setup())
+
+        return d

From 561cdba7c9c873402c96f478b9ea835c444d2592 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 20 Apr 2016 22:59:46 -0400
Subject: [PATCH 08/38] fix jsonrpc import

---
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index deb23e51d..5cb76072f 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -2,7 +2,7 @@ import logging
 import subprocess
 import os
 import shutil
-import jsonrpc
+from txjsonrpc.web import jsonrpc
 import json
 
 from StringIO import StringIO

From 2c1ba623a45928075c520f3d9eb56f2036d7a553 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 20 Apr 2016 23:50:13 -0400
Subject: [PATCH 09/38] add message key to upload_log

log the message which is then uploaded with a special file prefix, and
also send the message to slack
---
 lbrynet/lbrynet_daemon/LBRYDaemon.py       | 11 +++++++++++
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py |  3 +--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index c30b5b594..b9613c93e 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -989,6 +989,12 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             r.append(t)
         return r
 
+    def _log_to_slack(self, msg):
+        URL = "https://hooks.slack.com/services/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"
+        msg = platform.platform() + ": " + base58.b58encode(self.lbryid)[:20] + ", " + msg
+        requests.post(URL, json.dumps({"text": msg}))
+        return defer.succeed(None)
+
     def _render_response(self, result, code):
         return defer.succeed({'result': result, 'code': code})
 
@@ -1721,6 +1727,9 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             else:
                 exclude_previous = True
 
+            if 'message' in p.keys():
+                log.info("[" + str(datetime.now()) + "] Upload log message: " + str(p['message']))
+
             if 'force' in p.keys():
                 force = p['force']
             else:
@@ -1730,5 +1739,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             exclude_previous = True
 
         d = self._upload_log(name_prefix=prefix, exclude_previous=exclude_previous, force=force)
+        if 'message' in p.keys():
+            d.addCallback(lambda _: self._log_to_slack(p['message']))
         d.addCallback(lambda _: self._render_response(True, OK_CODE))
         return d
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 5cb76072f..14310dfcb 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -83,9 +83,8 @@ class LBRYBugReport(resource.Resource):
 
     def render_POST(self, request):
         msg = request.args["message"][0]
-        log.info("User submitted error report: " + str(msg))
         api = jsonrpc.Proxy(API_CONNECTION_STRING)
-        d = api.callRemote("upload_log", {'name_prefix': 'report', 'exclude_previous': False, 'force': True})
+        d = api.callRemote("upload_log", {'name_prefix': 'report', 'exclude_previous': False, 'force': True, 'message': str(msg)})
         d.addCallback(lambda _: self._delayed_render(request, "<html><body>Your bug report is greatly appreciated! <a href='lbry://lbry'>Click here to return to LBRY</a></body></html>"))
 
         return server.NOT_DONE_YET

From fecd29a9676e3993ad93b835d33a126d4a03eb7e Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 22 Apr 2016 02:45:05 -0400
Subject: [PATCH 10/38] only load /view page when file isn't empty

---
 lbrynet/conf.py                             |  12 +-
 lbrynet/lbrynet_daemon/LBRYDaemon.py        | 181 ++++++++++++--------
 lbrynet/lbrynet_daemon/LBRYDaemonControl.py |   3 +-
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py  |  28 +--
 lbrynet/lbrynet_daemon/LBRYDownloader.py    |   2 +-
 5 files changed, 131 insertions(+), 95 deletions(-)

diff --git a/lbrynet/conf.py b/lbrynet/conf.py
index f3903ae6a..a4a387991 100644
--- a/lbrynet/conf.py
+++ b/lbrynet/conf.py
@@ -18,8 +18,6 @@ MIN_BLOB_INFO_PAYMENT_RATE = .02  # points/1000 infos
 MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05  # points/1000 infos
 MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .05  # points/1000 infos
 MAX_CONNECTIONS_PER_STREAM = 5
-DEFAULT_MAX_SEARCH_RESULTS = 25
-DEFAULT_MAX_KEY_FEE = 100.0
 
 KNOWN_DHT_NODES = [('104.236.42.182', 4000)]
 
@@ -33,11 +31,13 @@ API_ADDRESS = "lbryapi"
 API_PORT = 5279
 ICON_PATH = "app.icns"
 APP_NAME = "LBRY"
-DEFAULT_WALLET = "lbryum"
-
 API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRESS)
 UI_ADDRESS = "http://" + API_INTERFACE + ":" + str(API_PORT)
-
 PROTOCOL_PREFIX = "lbry"
 
-DEFAULT_TIMEOUT = 30
\ No newline at end of file
+DEFAULT_WALLET = "lbryum"
+DEFAULT_TIMEOUT = 30
+DEFAULT_MAX_SEARCH_RESULTS = 25
+DEFAULT_MAX_KEY_FEE = 100.0
+DEFAULT_SEARCH_TIMEOUT = 3.0
+
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index b9613c93e..36a1546d9 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -36,7 +36,7 @@ from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher
 from lbrynet.core.utils import generate_id
 from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
 from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \
-    DEFAULT_WALLET, API_INTERFACE
+    DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT
 from lbrynet.conf import API_CONNECTION_STRING, API_PORT, API_ADDRESS, DEFAULT_TIMEOUT, UI_ADDRESS
 from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob
 from lbrynet.core.Session import LBRYSession
@@ -99,6 +99,7 @@ ALLOWED_DURING_STARTUP = ['is_running', 'is_first_run',
 BAD_REQUEST = 400
 NOT_FOUND = 404
 OK_CODE = 200
+
 # TODO add login credentials in a conf file
 # TODO alert if your copy of a lbry file is out of date with the name record
 
@@ -127,7 +128,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.git_lbrynet_version = None
         self.git_lbryum_version = None
         self.wallet_type = wallet_type
-        self.session_settings = None
         self.first_run = None
         self.log_file = LOG_FILENAME
         self.fetcher = None
@@ -152,7 +152,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         if os.name == "nt":
             from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
-            self.download_directory = get_path(FOLDERID.Downloads, UserHandle.current)
+            default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current)
             self.db_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrynet")
             self.lbrycrdd_path = "lbrycrdd.exe"
             if wallet_type == "lbrycrd":
@@ -160,7 +160,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             else:
                 self.wallet_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum")
         elif sys.platform == "darwin":
-            self.download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
+            default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
             self.db_dir = user_data_dir("LBRY")
             self.lbrycrdd_path = "./lbrycrdd"
             if wallet_type == "lbrycrd":
@@ -168,7 +168,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             else:
                 self.wallet_dir = user_data_dir("LBRY")
         else:
-            self.download_directory = os.getcwd()
+            default_download_directory = os.getcwd()
             self.db_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
             self.lbrycrdd_path = "./lbrycrdd"
             if wallet_type == "lbrycrd":
@@ -192,6 +192,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.internet_connection_checker = LoopingCall(self._check_network_connection)
         self.version_checker = LoopingCall(self._check_remote_versions)
         self.connection_problem_checker = LoopingCall(self._check_connection_problems)
+        # self.lbrynet_connection_checker = LoopingCall(self._check_lbrynet_connection)
 
         self.sd_identifier = StreamDescriptorIdentifier()
         self.stream_info_manager = TempLBRYFileMetadataManager()
@@ -200,16 +201,16 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.lbry_file_metadata_manager = None
         self.lbry_file_manager = None
 
-        #defaults for settings otherwise loaded from daemon_settings.json
         self.default_settings = {
             'run_on_startup': False,
             'data_rate': MIN_BLOB_DATA_PAYMENT_RATE,
             'max_key_fee': DEFAULT_MAX_KEY_FEE,
-            'default_download_directory': self.download_directory,
+            'download_directory': default_download_directory,
             'max_upload': 0.0,
             'max_download': 0.0,
             'upload_log': True,
-            'search_timeout': 3.0,
+            'search_timeout': DEFAULT_SEARCH_TIMEOUT,
+            'download_timeout': DEFAULT_TIMEOUT,
             'max_search_results': DEFAULT_MAX_SEARCH_RESULTS,
             'wallet_type': wallet_type,
             'delete_blobs_on_remove': True,
@@ -220,6 +221,58 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             'requested_first_run_credits': False
         }
 
+        if os.path.isfile(self.daemon_conf):
+            f = open(self.daemon_conf, "r")
+            loaded_settings = json.loads(f.read())
+            f.close()
+            missing_settings = {}
+            removed_settings = {}
+            for k in self.default_settings.keys():
+                if k not in loaded_settings.keys():
+                    missing_settings[k] = self.default_settings[k]
+            for k in loaded_settings.keys():
+                if not k in self.default_settings.keys():
+                    log.info("Removing unused setting: " + k + " with value: " + str(loaded_settings[k]))
+                    removed_settings[k] = loaded_settings[k]
+                    del loaded_settings[k]
+            for k in missing_settings.keys():
+                log.info("Adding missing setting: " + k + " with default value: " + str(missing_settings[k]))
+                loaded_settings[k] = missing_settings[k]
+            if missing_settings or removed_settings:
+                f = open(self.daemon_conf, "w")
+                f.write(json.dumps(loaded_settings))
+                f.close()
+            else:
+                log.info("Loaded lbrynet-daemon configuration")
+            settings_dict = loaded_settings
+        else:
+            log.info(
+                "Writing default settings : " + json.dumps(self.default_settings) + " --> " + str(self.daemon_conf))
+            f = open(self.daemon_conf, "w")
+            f.write(json.dumps(self.default_settings))
+            f.close()
+            settings_dict = self.default_settings
+
+        self.session_settings = settings_dict
+
+        self.run_on_startup = self.session_settings['run_on_startup']
+        self.data_rate = self.session_settings['data_rate']
+        self.max_key_fee = self.session_settings['max_key_fee']
+        self.download_directory = self.session_settings['download_directory']
+        self.max_upload = self.session_settings['max_upload']
+        self.max_download = self.session_settings['max_download']
+        self.upload_log = self.session_settings['upload_log']
+        self.search_timeout = self.session_settings['search_timeout']
+        self.download_timeout = self.session_settings['download_timeout']
+        self.max_search_results = self.session_settings['max_search_results']
+        self.wallet_type = self.session_settings['wallet_type']
+        self.delete_blobs_on_remove = self.session_settings['delete_blobs_on_remove']
+        self.peer_port = self.session_settings['peer_port']
+        self.dht_node_port = self.session_settings['dht_node_port']
+        self.use_upnp = self.session_settings['use_upnp']
+        self.start_lbrycrdd = self.session_settings['start_lbrycrdd']
+        self.requested_first_run_credits = self.session_settings['requested_first_run_credits']
+
     def render(self, request):
         request.content.seek(0, 0)
         # Unmarshal the JSON-RPC data.
@@ -300,6 +353,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                 self.announced_startup = True
                 self.startup_status = STARTUP_STAGES[5]
                 log.info("[" + str(datetime.now()) + "] Started lbrynet-daemon")
+                # self.lbrynet_connection_checker.start(3600)
 
             if self.first_run:
                 d = self._upload_log(name_prefix="fr")
@@ -321,7 +375,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         d = defer.Deferred()
         d.addCallback(lambda _: self._initial_setup())
-        d.addCallback(self._set_daemon_settings)
         d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory))
         d.addCallback(lambda _: self._check_db_migration())
         d.addCallback(lambda _: self._get_settings())
@@ -345,42 +398,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             log.info("Platform: " + json.dumps(self.platform_info))
             return defer.succeed(None)
 
-        def _load_daemon_conf():
-            if os.path.isfile(self.daemon_conf):
-                f = open(self.daemon_conf, "r")
-                loaded_settings = json.loads(f.read())
-                f.close()
-                missing_settings = {}
-                for k in self.default_settings.keys():
-                    if k not in loaded_settings.keys():
-                        missing_settings[k] = self.default_settings[k]
-                if missing_settings != {}:
-                    for k in missing_settings.keys():
-                        log.info("Adding missing setting: " + k + " with default value: " + str(missing_settings[k]))
-                        loaded_settings[k] = missing_settings[k]
-                    f = open(self.daemon_conf, "w")
-                    f.write(json.dumps(loaded_settings))
-                    f.close()
-                rsettings = loaded_settings
-            else:
-                log.info("Writing default settings : " + json.dumps(self.default_settings) + " --> " + str(self.daemon_conf))
-                f = open(self.daemon_conf, "w")
-                f.write(json.dumps(self.default_settings))
-                f.close()
-                rsettings = self.default_settings
-            for k in rsettings.keys():
-                self.__dict__[k] = rsettings[k]
-            return rsettings
-
         d = _log_platform()
-        d.addCallback(lambda _: _load_daemon_conf())
 
         return d
 
-    def _set_daemon_settings(self, settings):
-        self.session_settings = settings
-        return defer.succeed(None)
-
     def _check_network_connection(self):
         try:
             host = socket.gethostbyname(REMOTE_SERVER)
@@ -390,6 +411,16 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             log.info("[" + str(datetime.now()) + "] Internet connection not working")
             self.connected_to_internet = False
 
+    def _check_lbrynet_connection(self):
+        def _log_success():
+            log.info("[" + str(datetime.now()) + "] lbrynet connectivity test passed")
+        def _log_failure():
+            log.info("[" + str(datetime.now()) + "] lbrynet connectivity test failed")
+
+        wonderfullife_sh = "6f3af0fa3924be98a54766aa2715d22c6c1509c3f7fa32566df4899a41f3530a9f97b2ecb817fa1dcbf1b30553aefaa7"
+        d = download_sd_blob(self.session, wonderfullife_sh, self.session.base_payment_rate_manager)
+        d.addCallbacks(lambda _: _log_success, lambda _: _log_failure)
+
     def _check_remote_versions(self):
         def _get_lbryum_version():
             try:
@@ -514,7 +545,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         else:
             name_prefix = platform.system()
 
-        if self.session_settings['upload_log'] or force:
+        if self.upload_log or force:
             LOG_URL = "https://lbry.io/log-upload"
             if exclude_previous:
                 f = open(self.log_file, "r")
@@ -567,10 +598,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                     self.session_settings['max_key_fee'] = float(settings['max_key_fee'])
                 else:
                     return defer.fail()
-            elif k == 'default_download_directory':
-                if type(settings['default_download_directory']) is unicode:
-                    if os.path.isdir(settings['default_download_directory']):
-                        self.session_settings['default_download_directory'] = settings['default_download_directory']
+            elif k == 'download_directory':
+                if type(settings['download_directory']) is unicode:
+                    if os.path.isdir(settings['download_directory']):
+                        self.session_settings['download_directory'] = settings['download_directory']
                     else:
                         pass
                 else:
@@ -595,6 +626,14 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                 else:
                     return defer.fail()
 
+        self.run_on_startup = self.session_settings['run_on_startup']
+        self.data_rate = self.session_settings['data_rate']
+        self.max_key_fee = self.session_settings['max_key_fee']
+        self.download_directory = self.session_settings['download_directory']
+        self.max_upload = self.session_settings['max_upload']
+        self.max_download = self.session_settings['max_download']
+        self.upload_log = self.session_settings['upload_log']
+
         f = open(self.daemon_conf, "w")
         f.write(json.dumps(self.session_settings))
         f.close()
@@ -748,7 +787,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             r = requests.post(url, json=data)
             if r.status_code == 200:
                 self.requested_first_run_credits = True
-                self.session_settings['requested_first_run_credits'] = True
                 f = open(self.daemon_conf, "w")
                 f.write(json.dumps(self.session_settings))
                 f.close()
@@ -801,7 +839,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
     def _setup_stream_identifier(self):
         file_saver_factory = LBRYFileSaverFactory(self.session.peer_finder, self.session.rate_limiter,
                                                   self.session.blob_manager, self.stream_info_manager,
-                                                  self.session.wallet, self.session_settings['default_download_directory'])
+                                                  self.session.wallet, self.download_directory)
         self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, file_saver_factory)
         file_opener_factory = LBRYFileOpenerFactory(self.session.peer_finder, self.session.rate_limiter,
                                                     self.session.blob_manager, self.stream_info_manager,
@@ -819,12 +857,12 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
     def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None):
         if not download_directory:
-            download_directory = self.session_settings['default_download_directory']
+            download_directory = self.download_directory
         elif not os.path.isdir(download_directory):
-            download_directory = self.session_settings['default_download_directory']
+            download_directory = self.download_directory
 
         def _disp_file(f):
-            file_path = os.path.join(self.session_settings['default_download_directory'], f.file_name)
+            file_path = os.path.join(self.download_directory, f.file_name)
             log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.stream_hash) + " --> " + file_path)
             return defer.succeed(f)
 
@@ -850,7 +888,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d = self._check_history(name)
         d.addCallback(lambda lbry_file: _get_stream(name) if not lbry_file else _disp_file(lbry_file))
         d.addCallback(lambda _: self._path_from_name(name))
-        d.addErrback(lambda err: defer.fail(NOT_FOUND))
 
         return d
 
@@ -880,7 +917,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             for lbry_file in self.lbry_file_manager.lbry_files:
                 if lbry_file.stream_name == file_name:
                     if sys.platform == "darwin":
-                        if os.path.isfile(os.path.join(self.session_settings['default_download_directory'], lbry_file.stream_name)):
+                        if os.path.isfile(os.path.join(self.download_directory, lbry_file.stream_name)):
                             return lbry_file
                         else:
                             return False
@@ -921,8 +958,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             d = self.lbry_file_manager.get_count_for_stream_hash(s_h)
             # TODO: could possibly be a timing issue here
             d.addCallback(lambda c: self.stream_info_manager.delete_stream(s_h) if c == 0 else True)
-            d.addCallback(lambda _: os.remove(os.path.join(self.session_settings['default_download_directory'], lbry_file.file_name)) if
-                          os.path.isfile(os.path.join(self.session_settings['default_download_directory'], lbry_file.file_name)) else defer.succeed(None))
+            d.addCallback(lambda _: os.remove(os.path.join(self.download_directory, lbry_file.file_name)) if
+                          os.path.isfile(os.path.join(self.download_directory, lbry_file.file_name)) else defer.succeed(None))
             return d
 
         d.addCallback(lambda _: finish_deletion(lbry_file))
@@ -931,21 +968,21 @@ class LBRYDaemon(jsonrpc.JSONRPC):
     def _path_from_name(self, name):
         d = self._check_history(name)
         d.addCallback(lambda lbry_file: {'stream_hash': lbry_file.stream_hash,
-                                         'path': os.path.join(self.session_settings['default_download_directory'], lbry_file.file_name)}
+                                         'path': os.path.join(self.download_directory, lbry_file.file_name)}
                                         if lbry_file else defer.fail(UnknownNameError))
         return d
 
     def _path_from_lbry_file(self, lbry_file):
         if lbry_file:
             r = {'stream_hash': lbry_file.stream_hash,
-                 'path': os.path.join(self.session_settings['default_download_directory'], lbry_file.file_name)}
+                 'path': os.path.join(self.download_directory, lbry_file.file_name)}
             return defer.succeed(r)
         else:
             return defer.fail(UnknownNameError)
 
     def _get_est_cost(self, name):
         def _check_est(d, name):
-            if type(d.result) is float:
+            if isinstance(d.result, float):
                 log.info("[" + str(datetime.now()) + "] Cost est for lbry://" + name + ": " + str(d.result) + "LBC")
             else:
                 log.info("[" + str(datetime.now()) + "] Timeout estimating cost for lbry://" + name + ", using key fee")
@@ -1121,11 +1158,12 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             'run_on_startup': bool,
             'data_rate': float,
             'max_key_fee': float,
-            'default_download_directory': string,
+            'download_directory': string,
             'max_upload': float, 0.0 for unlimited
             'max_download': float, 0.0 for unlimited
             'upload_log': bool,
             'search_timeout': float,
+            'download_timeout': int
             'max_search_results': int,
             'wallet_type': string,
             'delete_blobs_on_remove': bool,
@@ -1146,27 +1184,19 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             'run_on_startup': bool,
             'data_rate': float,
             'max_key_fee': float,
-            'default_download_directory': string,
+            'download_directory': string,
             'max_upload': float, 0.0 for unlimited
             'max_download': float, 0.0 for unlimited
             'upload_log': bool,
-            'search_timeout': float,
-            'max_search_results': int,
-            'wallet_type': string,
-            'delete_blobs_on_remove': bool,
-            'peer_port': int,
-            'dht_node_port': int,
-            'use_upnp': bool,
-            'start_lbrycrdd': bool,
         Returns:
             settings dict
         """
 
-        def _log_settings_change(params):
-            log.info("[" + str(datetime.now()) + "] Set daemon settings to " + str(params))
+        def _log_settings_change():
+            log.info("[" + str(datetime.now()) + "] Set daemon settings to " + json.dumps(self.session_settings))
 
         d = self._update_settings(p)
-        d.addCallback(lambda _: _log_settings_change(p))
+        d.addCallback(lambda _: _log_settings_change())
         d.addCallback(lambda _: self._render_response(self.session_settings, OK_CODE))
 
         return d
@@ -1334,22 +1364,21 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         """
 
         if 'timeout' not in p.keys():
-            timeout = DEFAULT_TIMEOUT
+            timeout = self.download_timeout
         else:
             timeout = p['timeout']
 
         if 'download_directory' not in p.keys():
-            download_directory = self.session_settings['default_download_directory']
+            download_directory = self.download_directory
         else:
             download_directory = p['download_directory']
 
         if 'name' in p.keys():
             name = p['name']
             d = self._download_name(name=name, timeout=timeout, download_directory=download_directory)
-            d.addCallbacks(lambda message: self._render_response(message, OK_CODE),
-                           lambda err: self._render_response('error', NOT_FOUND))
+            d.addCallback(lambda message: self._render_response(message, OK_CODE))
         else:
-            d = self._render_response('error', BAD_REQUEST)
+            d = server.failure
 
         return d
 
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
index 0321de4df..9cf3f4143 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
@@ -119,7 +119,8 @@ def start():
         lbry = LBRYDaemonServer()
 
         d = lbry.start(branch=args.branch, user_specified=args.ui)
-        d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
+        if args.launchui:
+            d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
 
         reactor.listenTCP(API_PORT, server.Site(lbry.root), interface=API_INTERFACE)
         reactor.run()
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 14310dfcb..12dc9d0b2 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -58,7 +58,9 @@ class LBRYFileRender(resource.Resource):
             api = jsonrpc.Proxy(API_CONNECTION_STRING)
             if request.args['name'][0] != 'lbry':
                 d = api.callRemote("get", {'name': request.args['name'][0]})
-                d.addCallback(lambda results: static.File(results['path']).render_GET(request))
+                d.addCallback(lambda results: static.File(results['path']))
+                d.addCallback(lambda static_file: static_file.render_GET(request) if static_file.getFileSize() > 0
+                              else server.failure)
             else:
                 request.redirect(UI_ADDRESS)
                 request.finish()
@@ -112,8 +114,11 @@ class LBRYDaemonServer(object):
                 loaded_ui = json.loads(f.read())
                 f.close()
                 self.loaded_git_version = loaded_ui['commit']
+                self.loaded_branch = loaded_ui['branch']
+                version_log.info("[" + str(datetime.now()) + "] Last used " + self.loaded_branch + " commit " + str(self.loaded_git_version).replace("\n", ""))
             except:
                 self.loaded_git_version = None
+                self.loaded_branch = None
 
     def setup(self, branch="HEAD", user_specified=None):
         self.branch = branch
@@ -146,19 +151,20 @@ class LBRYDaemonServer(object):
 
         def _set_git(version):
             self.git_version = version
-            log.info("UI version from git: " + str(self.git_version).replace("\n", ""))
-            version_log.info("UI version from git: " + str(self.git_version).replace("\n", ""))
+            version_log.info("[" + str(datetime.now()) + "] UI branch " + self.branch + " has a most recent commit of: " + str(self.git_version).replace("\n", ""))
 
             if self.git_version == self.loaded_git_version and os.path.isdir(self.ui_dir):
-                log.info("UI is up to date")
-                version_log.info("UI is up to date")
+                version_log.info("[" + str(datetime.now()) + "] local copy of " + self.branch + " is up to date")
                 return defer.succeed(True)
             else:
-                log.info("Downloading UI")
-                version_log.info("Downloading UI")
+                if self.git_version == self.loaded_git_version:
+                    version_log.info("[" + str(datetime.now()) + "] Can't find ui files, downloading them again")
+                else:
+                    version_log.info("[" + str(datetime.now()) + "] local copy of " + self.branch + " branch is out of date, updating")
                 f = open(self.config, "w")
                 f.write(json.dumps({'commit': self.git_version,
-                                    'time': str(datetime.now())}))
+                                    'time': str(datetime.now()),
+                                    'branch': self.branch}))
                 f.close()
                 return defer.succeed(False)
 
@@ -170,7 +176,7 @@ class LBRYDaemonServer(object):
         def _delete_ui_dir():
             if os.path.isdir(self.ui_dir):
                 if self.loaded_git_version:
-                    version_log.info("Removed ui files for commit " + str(self.loaded_git_version))
+                    version_log.info("[" + str(datetime.now()) + "] Removed ui files for commit " + str(self.loaded_git_version).replace("\n", ""))
                 log.info("Removing out of date ui files")
                 shutil.rmtree(self.ui_dir)
             return defer.succeed(None)
@@ -180,8 +186,8 @@ class LBRYDaemonServer(object):
             z = ZipFile(StringIO(url.read()))
             names = [i for i in z.namelist() if '.DS_exStore' not in i and '__MACOSX' not in i]
             z.extractall(self.ui_dir, members=names)
-            version_log.info("[" + str(datetime.now()) + "] Updated branch " + self.branch + " commit: " + str(self.loaded_git_version) + " to " + self.git_version)
-            log.info("Downloaded files for UI commit " + self.git_version)
+            version_log.info("[" + str(datetime.now()) + "] Updated branch " + self.branch + ": " + str(self.loaded_git_version).replace("\n", "") + " --> " + self.git_version.replace("\n", ""))
+            log.info("Downloaded files for UI commit " + str(self.git_version).replace("\n", ""))
             self.loaded_git_version = self.git_version
             return self.branch
 
diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py
index 5ebc4b59e..f0c157e42 100644
--- a/lbrynet/lbrynet_daemon/LBRYDownloader.py
+++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py
@@ -118,7 +118,7 @@ class GetStream(object):
         downloader.start()
 
         self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
-        d.addCallback(lambda _: log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
+        log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path))
 
         return d
 

From bac7f097a99379769a1622e4d3f8926504992feb Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 22 Apr 2016 02:45:41 -0400
Subject: [PATCH 11/38] add -x to filename before extension for duplicate
 downloads

---
 lbrynet/lbryfile/client/LBRYFileDownloader.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/lbrynet/lbryfile/client/LBRYFileDownloader.py b/lbrynet/lbryfile/client/LBRYFileDownloader.py
index a65f49484..16ee9425d 100644
--- a/lbrynet/lbryfile/client/LBRYFileDownloader.py
+++ b/lbrynet/lbryfile/client/LBRYFileDownloader.py
@@ -211,10 +211,20 @@ class LBRYFileSaver(LBRYFileDownloader):
                     file_name = "_"
                 if os.path.exists(os.path.join(self.download_directory, file_name)):
                     ext_num = 1
+
+                    def _get_file_name(ext):
+                        if len(file_name.split(".")):
+                            fn = ''.join(file_name.split(".")[:-1])
+                            file_ext = ''.join(file_name.split(".")[-1])
+                            return fn + "-" + str(ext) + "." + file_ext
+                        else:
+                            return file_name + "_" + str(ext)
+
                     while os.path.exists(os.path.join(self.download_directory,
-                                                      file_name + "_" + str(ext_num))):
+                                                      _get_file_name(ext_num))):
                         ext_num += 1
-                    file_name = file_name + "_" + str(ext_num)
+
+                    file_name = _get_file_name(ext_num)
                 try:
                     self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
                     self.file_written_to = os.path.join(self.download_directory, file_name)

From 28f66e30dd19f769d93b2de5cc5d3520f350a94f Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 22 Apr 2016 18:18:17 -0400
Subject: [PATCH 12/38] improve timeouts

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py       | 17 +++++++---
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py |  7 ++++-
 lbrynet/lbrynet_daemon/LBRYDownloader.py   | 36 ++++++++++++----------
 3 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 36a1546d9..a44607505 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -625,6 +625,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                     self.session_settings['upload_log'] = settings['upload_log']
                 else:
                     return defer.fail()
+            elif k == 'download_timeout':
+                if type(settings['download_timeout']) is int:
+                    self.session_settings['download_timeout'] = settings['download_timeout']
+                else:
+                    return defer.fail()
 
         self.run_on_startup = self.session_settings['run_on_startup']
         self.data_rate = self.session_settings['data_rate']
@@ -633,6 +638,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.max_upload = self.session_settings['max_upload']
         self.max_download = self.session_settings['max_download']
         self.upload_log = self.session_settings['upload_log']
+        self.download_timeout = self.session_settings['download_timeout']
 
         f = open(self.daemon_conf, "w")
         f.write(json.dumps(self.session_settings))
@@ -864,7 +870,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         def _disp_file(f):
             file_path = os.path.join(self.download_directory, f.file_name)
             log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.stream_hash) + " --> " + file_path)
-            return defer.succeed(f)
+            return {'stream_hash': f.stream_hash, 'path': file_path}
 
         def _get_stream(name):
             def _disp(stream):
@@ -880,14 +886,13 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                                max_key_fee=self.max_key_fee, data_rate=self.data_rate, timeout=timeout,
                                download_directory=download_directory)
             d.addCallback(_disp)
-            d.addCallback(lambda stream_info: stream.start(stream_info))
-            d.addCallback(lambda _: self._path_from_name(name))
+            d.addCallback(lambda stream_info: stream.start(stream_info, name))
+            d.addCallback(lambda r: {'stream_hash': r[0], 'path': r[1]} if r else server.failure)
 
             return d
 
         d = self._check_history(name)
         d.addCallback(lambda lbry_file: _get_stream(name) if not lbry_file else _disp_file(lbry_file))
-        d.addCallback(lambda _: self._path_from_name(name))
 
         return d
 
@@ -933,7 +938,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
             path = os.path.join(self.blobfile_dir, stream_hash)
             if os.path.isfile(path):
-                log.info("[" + str(datetime.now()) + "] Search for lbry_file, returning: " + stream_hash)
+                file_size = os.stat(path).st_size
+                log.info("[" + str(datetime.now()) + "] Search for lbry_file, found " + str(file_size) + " bytes written from stream hash: " + stream_hash)
                 return defer.succeed(_get_lbry_file(path))
             else:
                 log.info("[" + str(datetime.now()) + "] Search for lbry_file didn't return anything")
@@ -1188,6 +1194,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             'max_upload': float, 0.0 for unlimited
             'max_download': float, 0.0 for unlimited
             'upload_log': bool,
+            'download_timeout': int
         Returns:
             settings dict
         """
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 12dc9d0b2..04ad5f561 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -50,6 +50,11 @@ class LBRYindex(resource.Resource):
         return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request)
 
 
+class HostedLBRYFile(static.File):
+    def __init__(self, path):
+        static.File.__init__(self, path=path)
+
+
 class LBRYFileRender(resource.Resource):
     isLeaf = False
 
@@ -58,7 +63,7 @@ class LBRYFileRender(resource.Resource):
             api = jsonrpc.Proxy(API_CONNECTION_STRING)
             if request.args['name'][0] != 'lbry':
                 d = api.callRemote("get", {'name': request.args['name'][0]})
-                d.addCallback(lambda results: static.File(results['path']))
+                d.addCallback(lambda results: HostedLBRYFile(results['path']))
                 d.addCallback(lambda static_file: static_file.render_GET(request) if static_file.getFileSize() > 0
                               else server.failure)
             else:
diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py
index f0c157e42..32622534c 100644
--- a/lbrynet/lbrynet_daemon/LBRYDownloader.py
+++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py
@@ -13,6 +13,7 @@ from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
 from lbrynet.conf import DEFAULT_TIMEOUT
 
 log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
 
 
 class GetStream(object):
@@ -39,22 +40,24 @@ class GetStream(object):
         self.timeout_counter = 0
         self.download_directory = download_directory
         self.download_path = None
+        self.finished = defer.Deferred()
         self.checker = LoopingCall(self.check_status)
 
-
     def check_status(self):
         self.timeout_counter += 1
 
-        if self.download_path and os.path.isfile(self.download_path):
+        if self.download_path:
             self.checker.stop()
-            return defer.succeed(True)
+            self.finished.callback((self.stream_hash, self.download_path))
 
         elif self.timeout_counter >= self.timeout:
-            log.info("Timeout downloading " + str(self.stream_info))
+            log.info("Timeout downloading lbry://" + self.resolved_name + ", " + str(self.stream_info))
             self.checker.stop()
             self.d.cancel()
+            self.finished.callback(False)
 
-    def start(self, stream_info):
+    def start(self, stream_info, name):
+        self.resolved_name = name
         self.stream_info = stream_info
         if 'stream_hash' in self.stream_info.keys():
             self.description = self.stream_info['description']
@@ -84,21 +87,24 @@ class GetStream(object):
         else:
             pass
 
+        def _cause_timeout():
+            self.timeout_counter = self.timeout * 2
+
         self.checker.start(1)
 
         self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
         self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
-        self.d.addCallback(lambda metadata: (next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)), metadata))
+        self.d.addCallback(lambda metadata: (
+        next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)),
+        metadata))
         self.d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata,
                                                                                [self.data_rate, True],
                                                                                self.payment_rate_manager,
                                                                                download_directory=self.download_directory))
-        self.d.addErrback(lambda err: err.trap(defer.CancelledError))
-        self.d.addErrback(lambda err: log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback()))
-        self.d.addCallback(self._start_download)
+        self.d.addCallbacks(self._start_download, lambda _: _cause_timeout())
         self.d.callback(None)
 
-        return self.d
+        return self.finished
 
     def _start_download(self, downloader):
         def _pay_key_fee():
@@ -114,14 +120,10 @@ class GetStream(object):
             d = _pay_key_fee()
         else:
             d = defer.Deferred()
-
-        downloader.start()
-
         self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
-        log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path))
-
-        return d
-
+        d.addCallback(lambda _: log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
+        d.addCallback(lambda _: downloader.start())
+        d.callback()
 
 class FetcherDaemon(object):
     def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf,

From 3e7c09bb44d81c6291609fd1812661574dc169c5 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 22 Apr 2016 18:48:44 -0400
Subject: [PATCH 13/38] fix duplicate/buffering problem

fix problem where get request takes time to respond, in this time it
can receive more get requests that each manage to start downloading
---
 lbrynet/lbrynet_daemon/LBRYDaemon.py     | 14 ++++++++++++--
 lbrynet/lbrynet_daemon/LBRYDownloader.py |  1 -
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index a44607505..555e1deff 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -134,6 +134,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.current_db_revision = 1
         self.run_server = True
         self.session = None
+        self.waiting_on = {}
         self.known_dht_nodes = KNOWN_DHT_NODES
         self.platform_info = {
             "processor": platform.processor(),
@@ -867,6 +868,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         elif not os.path.isdir(download_directory):
             download_directory = self.download_directory
 
+        def _remove_from_wait(r):
+            del self.waiting_on[name]
+            return r
+
         def _disp_file(f):
             file_path = os.path.join(self.download_directory, f.file_name)
             log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.stream_hash) + " --> " + file_path)
@@ -891,8 +896,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
             return d
 
+        self.waiting_on[name] = True
         d = self._check_history(name)
         d.addCallback(lambda lbry_file: _get_stream(name) if not lbry_file else _disp_file(lbry_file))
+        d.addCallback(_remove_from_wait)
 
         return d
 
@@ -1382,8 +1389,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         if 'name' in p.keys():
             name = p['name']
-            d = self._download_name(name=name, timeout=timeout, download_directory=download_directory)
-            d.addCallback(lambda message: self._render_response(message, OK_CODE))
+            if p['name'] not in self.waiting_on.keys():
+                d = self._download_name(name=name, timeout=timeout, download_directory=download_directory)
+                d.addCallback(lambda message: self._render_response(message, OK_CODE))
+            else:
+                d = server.failure
         else:
             d = server.failure
 
diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py
index 32622534c..c8cdc7fbb 100644
--- a/lbrynet/lbrynet_daemon/LBRYDownloader.py
+++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py
@@ -123,7 +123,6 @@ class GetStream(object):
         self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
         d.addCallback(lambda _: log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
         d.addCallback(lambda _: downloader.start())
-        d.callback()
 
 class FetcherDaemon(object):
     def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf,

From c2ec066c851ecb8e13070f3a3483cc4b81c04039 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Sun, 24 Apr 2016 04:42:42 -0400
Subject: [PATCH 14/38] add LBRYFileProducer

-Add LBRYFileProducer, to host the contents of a download without
having to keep re-opening it as it is added to

-included sd hash in ManagedLBRYFileDownloader, to make comparing the
contents of the file manager against name claims easier

-add get_lbry_file function, which returns information about a LBRY
file found by sd_hash, file name, or lbry uri
---
 lbrynet/lbryfilemanager/LBRYFileDownloader.py |  19 +++-
 lbrynet/lbrynet_daemon/LBRYDaemon.py          |  88 +++++++++++++--
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py    | 106 +++++++++++++-----
 lbrynet/lbrynet_daemon/LBRYDownloader.py      |   5 +-
 4 files changed, 180 insertions(+), 38 deletions(-)

diff --git a/lbrynet/lbryfilemanager/LBRYFileDownloader.py b/lbrynet/lbryfilemanager/LBRYFileDownloader.py
index 2afc17588..8bb1ada42 100644
--- a/lbrynet/lbryfilemanager/LBRYFileDownloader.py
+++ b/lbrynet/lbryfilemanager/LBRYFileDownloader.py
@@ -24,12 +24,21 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
         LBRYFileSaver.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
                                stream_info_manager, payment_rate_manager, wallet, download_directory,
                                upload_allowed, file_name)
+        self.sd_hash = None
         self.rowid = rowid
         self.lbry_file_manager = lbry_file_manager
         self.saving_status = False
 
     def restore(self):
-        d = self.lbry_file_manager.get_lbry_file_status(self)
+        d = self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash)
+
+        def _save_sd_hash(sd_hash):
+            self.sd_hash = sd_hash[0]
+            return defer.succeed(None)
+
+        d.addCallback(_save_sd_hash)
+
+        d.addCallback(lambda _: self.lbry_file_manager.get_lbry_file_status(self))
 
         def restore_status(status):
             if status == ManagedLBRYFileDownloader.STATUS_RUNNING:
@@ -87,6 +96,14 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
 
         d = LBRYFileSaver._start(self)
 
+        d.addCallback(lambda _: self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash))
+
+        def _save_sd_hash(sd_hash):
+            self.sd_hash = sd_hash[0]
+            return defer.succeed(None)
+
+        d.addCallback(_save_sd_hash)
+
         d.addCallback(lambda _: self._save_status())
 
         return d
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 555e1deff..c0c487b42 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -68,7 +68,6 @@ handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, b
 log.addHandler(handler)
 log.setLevel(logging.INFO)
 
-
 INITIALIZING_CODE = 'initializing'
 LOADING_DB_CODE = 'loading_db'
 LOADING_WALLET_CODE = 'loading_wallet'
@@ -875,7 +874,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         def _disp_file(f):
             file_path = os.path.join(self.download_directory, f.file_name)
             log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.stream_hash) + " --> " + file_path)
-            return {'stream_hash': f.stream_hash, 'path': file_path}
+            return f
 
         def _get_stream(name):
             def _disp(stream):
@@ -886,13 +885,14 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                 log.info("[" + str(datetime.now()) + "] Start stream: " + stream_hash)
                 return stream
 
-            d = self.session.wallet.get_stream_info_for_name(name)
-            stream = GetStream(self.sd_identifier, self.session, self.session.wallet, self.lbry_file_manager,
+            stream = GetStream(self.sd_identifier, self.session, self.session.wallet,
+                               self.lbry_file_manager,
                                max_key_fee=self.max_key_fee, data_rate=self.data_rate, timeout=timeout,
                                download_directory=download_directory)
+            d = self.session.wallet.get_stream_info_for_name(name)
             d.addCallback(_disp)
             d.addCallback(lambda stream_info: stream.start(stream_info, name))
-            d.addCallback(lambda r: {'stream_hash': r[0], 'path': r[1]} if r else server.failure)
+            d.addCallback(lambda _: stream.downloader)
 
             return d
 
@@ -1028,17 +1028,60 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                 t = {'completed': f.completed, 'file_name': f.file_name, 'key': binascii.b2a_hex(f.key),
                      'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
                      'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
-                     'upload_allowed': f.upload_allowed}
+                     'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash}
 
             else:
                 t = {'completed': f.completed, 'file_name': f.file_name, 'key': None,
                      'points_paid': f.points_paid,
                      'stopped': f.stopped, 'stream_hash': f.stream_hash, 'stream_name': f.stream_name,
-                     'suggested_file_name': f.suggested_file_name, 'upload_allowed': f.upload_allowed}
+                     'suggested_file_name': f.suggested_file_name, 'upload_allowed': f.upload_allowed,
+                     'sd_hash': f.sd_hash}
 
             r.append(t)
         return r
 
+    def _get_lbry_file_by_uri(self, name):
+        d = self.session.wallet.get_stream_info_for_name(name)
+        d.addCallback(lambda info: info['stream_hash'])
+        d.addCallback(lambda sd_hash: next(l for l in self.lbry_file_manager.lbry_files if l.sd_hash == sd_hash))
+        return d
+
+    def _get_lbry_file_by_sd_hash(self, sd_hash):
+        r = next(l for l in self.lbry_file_manager.lbry_files if l.sd_hash == sd_hash)
+        return defer.succeed(r)
+
+    def _get_lbry_file_by_file_name(self, file_name):
+        r = next(l for l in self.lbry_file_manager.lbry_files if l.file_name == file_name)
+        return defer.succeed(r)
+
+    def _get_lbry_file(self, search_by, val):
+        def _show_file(f):
+            if f:
+                if f.key:
+                    t = {'completed': f.completed, 'file_name': f.file_name, 'key': binascii.b2a_hex(f.key),
+                         'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
+                         'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
+                         'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash}
+
+                else:
+                    t = {'completed': f.completed, 'file_name': f.file_name, 'key': None,
+                         'points_paid': f.points_paid,
+                         'stopped': f.stopped, 'stream_hash': f.stream_hash, 'stream_name': f.stream_name,
+                         'suggested_file_name': f.suggested_file_name, 'upload_allowed': f.upload_allowed,
+                         'sd_hash': f.sd_hash}
+                return t
+            else:
+                return False
+
+        if search_by == "name":
+            d = self._get_lbry_file_by_uri(val)
+        elif search_by == "sd_hash":
+            d = self._get_lbry_file_by_sd_hash(val)
+        elif search_by == "file_name":
+            d = self._get_lbry_file_by_file_name(val)
+        d.addCallback(_show_file)
+        return d
+
     def _log_to_slack(self, msg):
         URL = "https://hooks.slack.com/services/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"
         msg = platform.platform() + ": " + base58.b58encode(self.lbryid)[:20] + ", " + msg
@@ -1330,12 +1373,42 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             'stream_name': string
             'suggested_file_name': string
             'upload_allowed': bool
+            'sd_hash': string
         """
 
         r = self._get_lbry_files()
         log.info("[" + str(datetime.now()) + "] Get LBRY files")
         return self._render_response(r, OK_CODE)
 
+    def jsonrpc_get_lbry_file(self, p):
+        """
+        Get lbry file
+
+        Args:
+            'name': get file by lbry uri,
+            'sd_hash': get file by the hash in the name claim,
+            'file_name': get file by its name in the downloads folder,
+        Returns:
+            'completed': bool
+            'file_name': string
+            'key': hex string
+            'points_paid': float
+            'stopped': bool
+            'stream_hash': base 58 string
+            'stream_name': string
+            'suggested_file_name': string
+            'upload_allowed': bool
+            'sd_hash': string
+        """
+
+        if p.keys()[0] in ['name', 'sd_hash', 'file_name']:
+            search_type = p.keys()[0]
+            d = self._get_lbry_file(search_type, p[search_type])
+        else:
+            d = defer.fail()
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
     def jsonrpc_resolve_name(self, p):
         """
         Resolve stream info from a LBRY uri
@@ -1391,6 +1464,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             name = p['name']
             if p['name'] not in self.waiting_on.keys():
                 d = self._download_name(name=name, timeout=timeout, download_directory=download_directory)
+                d.addCallback(lambda l: {'stream_hash': l.sd_hash, 'path': os.path.join(self.download_directory, l.file_name)})
                 d.addCallback(lambda message: self._render_response(message, OK_CODE))
             else:
                 d = server.failure
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 04ad5f561..e68479bbe 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -2,6 +2,8 @@ import logging
 import subprocess
 import os
 import shutil
+
+from twisted.internet.task import LoopingCall
 from txjsonrpc.web import jsonrpc
 import json
 
@@ -12,6 +14,7 @@ from datetime import datetime
 from appdirs import user_data_dir
 from twisted.web import server, static, resource
 from twisted.internet import defer
+from twisted.web.static import StaticProducer
 
 from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon
 from lbrynet.conf import API_CONNECTION_STRING, API_ADDRESS, DEFAULT_WALLET, UI_ADDRESS
@@ -50,9 +53,77 @@ class LBRYindex(resource.Resource):
         return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request)
 
 
-class HostedLBRYFile(static.File):
-    def __init__(self, path):
-        static.File.__init__(self, path=path)
+class LBRYFileProducer(StaticProducer):
+    def __init__(self, request, lbry_stream):
+        self.stream = lbry_stream
+        self.updater = LoopingCall(self._check_for_data)
+        StaticProducer.__init__(self, request, fileObject=file(lbry_stream.file_written_to))
+
+    def start(self):
+        d = self._set_size()
+        self.updater.start(5)
+
+    def _set_size(self):
+        def _set(size):
+            self.request.setHeader('content-length', str(size))
+            self.request.setHeader('content-type', ' application/octet-stream')
+            return defer.succeed(None)
+
+        d = self.stream.get_total_bytes()
+        d.addCallback(_set)
+        return d
+
+    def _check_for_data(self):
+        self.fileObject.seek(self.fileObject.tell())
+        data = self.fileObject.read()
+        if data:
+            self.request.write(data)
+
+        def _check_status(stream_status):
+            if stream_status.running_status == "completed":
+                self.stopProducing()
+            return defer.succeed(None)
+
+        d = self.stream.status()
+        d.addCallback(_check_status)
+
+    def resumeProducing(self):
+        self.updater.start(1)
+
+    def stopProducing(self):
+        self.updater.stop()
+        self.fileObject.close()
+        self.stream.stop()
+        self.request.finish()
+
+
+class HostedLBRYFile(resource.Resource):
+    def __init__(self, api):
+        self._api = api
+        self.stream = None
+        self.streaming_file = None
+        resource.Resource.__init__(self)
+
+    def _set_stream(self, stream):
+        self.stream = stream
+
+    def makeProducer(self, request, stream):
+        return LBRYFileProducer(request, stream)
+
+    def render_GET(self, request):
+        if 'name' in request.args.keys():
+            if request.args['name'][0] != 'lbry':
+                if request.args['name'][0] != self.streaming_file:
+                    self.streaming_file = request.args['name'][0]
+                    d = self._api._download_name(request.args['name'][0])
+                    d.addCallback(self._set_stream)
+                else:
+                    d = defer.succeed(None)
+                d.addCallback(lambda _: self.makeProducer(request, self.stream).start())
+            else:
+                request.redirect(UI_ADDRESS)
+                request.finish()
+            return server.NOT_DONE_YET
 
 
 class LBRYFileRender(resource.Resource):
@@ -63,7 +134,7 @@ class LBRYFileRender(resource.Resource):
             api = jsonrpc.Proxy(API_CONNECTION_STRING)
             if request.args['name'][0] != 'lbry':
                 d = api.callRemote("get", {'name': request.args['name'][0]})
-                d.addCallback(lambda results: HostedLBRYFile(results['path']))
+                d.addCallback(lambda results: static.File(results['path'], defaultType='video/octet-stream'))
                 d.addCallback(lambda static_file: static_file.render_GET(request) if static_file.getFileSize() > 0
                               else server.failure)
             else:
@@ -74,29 +145,6 @@ class LBRYFileRender(resource.Resource):
             return server.failure
 
 
-class LBRYBugReport(resource.Resource):
-    isLeaf = False
-
-    def _delayed_render(self, request, results):
-        request.write(results)
-        request.finish()
-
-    def render_GET(self, request):
-        return '<html><body><form method="POST">' \
-               '<br>Please describe the problem you experienced and any information you think might be useful to us. Links to screenshots are great!</br>' \
-               '<textarea cols="50" rows="10" name="message" type="text"></textarea>' \
-               '<button>Submit</button>' \
-               '</form></body></html>'
-
-    def render_POST(self, request):
-        msg = request.args["message"][0]
-        api = jsonrpc.Proxy(API_CONNECTION_STRING)
-        d = api.callRemote("upload_log", {'name_prefix': 'report', 'exclude_previous': False, 'force': True, 'message': str(msg)})
-        d.addCallback(lambda _: self._delayed_render(request, "<html><body>Your bug report is greatly appreciated! <a href='lbry://lbry'>Click here to return to LBRY</a></body></html>"))
-
-        return server.NOT_DONE_YET
-
-
 class LBRYDaemonServer(object):
     def __init__(self):
         self.data_dir = user_data_dir("LBRY")
@@ -207,8 +255,8 @@ class LBRYDaemonServer(object):
         self.root.putChild("font", static.File(os.path.join(self.ui_dir, "font")))
         self.root.putChild("img", static.File(os.path.join(self.ui_dir, "img")))
         self.root.putChild("js", static.File(os.path.join(self.ui_dir, "js")))
-        self.root.putChild("view", LBRYFileRender())
-        self.root.putChild("report", LBRYBugReport())
+        # self.root.putChild("view", LBRYFileRender())
+        self.root.putChild("view", HostedLBRYFile(self._api))
         self.root.putChild(API_ADDRESS, self._api)
         return defer.succeed(True)
 
diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py
index c8cdc7fbb..b52c8c92a 100644
--- a/lbrynet/lbrynet_daemon/LBRYDownloader.py
+++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py
@@ -40,6 +40,7 @@ class GetStream(object):
         self.timeout_counter = 0
         self.download_directory = download_directory
         self.download_path = None
+        self.downloader = None
         self.finished = defer.Deferred()
         self.checker = LoopingCall(self.check_status)
 
@@ -120,9 +121,11 @@ class GetStream(object):
             d = _pay_key_fee()
         else:
             d = defer.Deferred()
+        self.downloader = downloader
         self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
         d.addCallback(lambda _: log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
-        d.addCallback(lambda _: downloader.start())
+        d.addCallback(lambda _: self.downloader.start())
+
 
 class FetcherDaemon(object):
     def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf,

From dfaf51a432e6293600849915ac47754541f69c21 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Sun, 24 Apr 2016 17:51:24 -0400
Subject: [PATCH 15/38] lbry file improvements
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

-fix daemon functions to start/stop lbry files

-remove unused stuff in LBRYFileManager

-improve and use new get_lbry_file function instead of _check_history,
which didn’t use the lbry file manager

-use said function to let delete_lbry_file use the same search keys
(sd_hash, name, and file_name)

-logging in LBRYDownloader
---
 lbrynet/lbryfilemanager/LBRYFileManager.py |  32 +--
 lbrynet/lbrynet_daemon/LBRYDaemon.py       | 234 ++++++++++-----------
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py |  48 +++--
 lbrynet/lbrynet_daemon/LBRYDownloader.py   |  15 +-
 4 files changed, 159 insertions(+), 170 deletions(-)

diff --git a/lbrynet/lbryfilemanager/LBRYFileManager.py b/lbrynet/lbryfilemanager/LBRYFileManager.py
index d0dbca1ae..144425c6f 100644
--- a/lbrynet/lbryfilemanager/LBRYFileManager.py
+++ b/lbrynet/lbryfilemanager/LBRYFileManager.py
@@ -4,10 +4,7 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
 
 import logging
 import os
-import sys
-from datetime import datetime
 
-from twisted.internet.task import LoopingCall
 from twisted.enterprise import adbapi
 from twisted.internet import defer, task, reactor
 from twisted.python.failure import Failure
@@ -28,14 +25,12 @@ class LBRYFileManager(object):
     Keeps track of currently opened LBRY Files, their options, and their LBRY File specific metadata.
     """
 
-    def __init__(self, session, stream_info_manager, sd_identifier, delete_data=False, download_directory=None):
+    def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
         self.session = session
         self.stream_info_manager = stream_info_manager
         self.sd_identifier = sd_identifier
         self.lbry_files = []
         self.sql_db = None
-        # self.delete_data = delete_data
-        # self.check_exists_loop = LoopingCall(self.check_files_exist)
         if download_directory:
             self.download_directory = download_directory
         else:
@@ -43,35 +38,11 @@ class LBRYFileManager(object):
         log.debug("Download directory for LBRYFileManager: %s", str(self.download_directory))
 
     def setup(self):
-        # self.check_exists_loop.start(10)
-
         d = self._open_db()
         d.addCallback(lambda _: self._add_to_sd_identifier())
         d.addCallback(lambda _: self._start_lbry_files())
         return d
 
-    # def check_files_exist(self):
-    #     def _disp(deleted_files):
-    #         if deleted_files[0][0]:
-    #             for file in bad_files:
-    #                 log.info("[" + str(datetime.now()) + "] Detected " + file.file_name + " was deleted, removing from file manager")
-    #
-    #     def _delete_stream_data(lbry_file):
-    #         s_h = lbry_file.stream_hash
-    #         d = self.get_count_for_stream_hash(s_h)
-    #         # TODO: could possibly be a timing issue here
-    #         d.addCallback(lambda c: self.stream_info_manager.delete_stream(s_h) if c == 0 else True)
-    #         return d
-    #
-    #     bad_files = [lbry_file for lbry_file in self.lbry_files
-    #                                       if lbry_file.completed == True and
-    #                                       os.path.isfile(os.path.join(self.download_directory, lbry_file.file_name)) == False]
-    #     d = defer.DeferredList([self.delete_lbry_file(lbry_file) for lbry_file in bad_files], consumeErrors=True)
-    #     d.addCallback(lambda files: _disp(files) if len(files) else defer.succeed(None))
-    #
-    #     if self.delete_data:
-    #         d2 = defer.DeferredList([_delete_stream_data(lbry_file) for lbry_file in bad_files], consumeErrors=True)
-
     def get_lbry_file_status(self, lbry_file):
         return self._get_lbry_file_status(lbry_file.rowid)
 
@@ -183,7 +154,6 @@ class LBRYFileManager(object):
             return defer.fail(Failure(ValueError("Could not find that LBRY file")))
 
     def stop(self):
-        # self.check_exists_loop.stop()
 
         ds = []
 
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index c0c487b42..961d6bdca 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -716,7 +716,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             self.lbry_file_manager = LBRYFileManager(self.session,
                                                      self.lbry_file_metadata_manager,
                                                      self.sd_identifier,
-                                                     delete_data=True)
+                                                     download_directory=self.download_directory)
             return self.lbry_file_manager.setup()
 
         d.addCallback(lambda _: set_lbry_file_manager())
@@ -862,6 +862,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         return defer.succeed(True)
 
     def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None):
+        """
+        Add a lbry file to the file manager, start the download, and return the new lbry file
+        if it already exists in the file manager, return the existing lbry file
+        """
         if not download_directory:
             download_directory = self.download_directory
         elif not os.path.isdir(download_directory):
@@ -871,34 +875,36 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             del self.waiting_on[name]
             return r
 
+        def _setup_stream(stream_info):
+            stream_hash = stream_info['stream_hash']
+            if isinstance(stream_hash, dict):
+                stream_hash = stream_hash['sd_hash']
+            log.info("[" + str(datetime.now()) + "] Resolved lbry://" + name + " to sd hash: " + stream_hash)
+            d = self._get_lbry_file_by_sd_hash(stream_hash)
+            def _add_results(l):
+                return defer.succeed((stream_info, l))
+            d.addCallback(_add_results)
+            return d
+
         def _disp_file(f):
             file_path = os.path.join(self.download_directory, f.file_name)
-            log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.stream_hash) + " --> " + file_path)
+            log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.sd_hash) + " --> " + file_path)
             return f
 
-        def _get_stream(name):
-            def _disp(stream):
-                stream_hash = stream['stream_hash']
-                if isinstance(stream_hash, dict):
-                    stream_hash = stream_hash['sd_hash']
-
-                log.info("[" + str(datetime.now()) + "] Start stream: " + stream_hash)
-                return stream
-
+        def _get_stream(stream_info):
             stream = GetStream(self.sd_identifier, self.session, self.session.wallet,
                                self.lbry_file_manager,
                                max_key_fee=self.max_key_fee, data_rate=self.data_rate, timeout=timeout,
                                download_directory=download_directory)
-            d = self.session.wallet.get_stream_info_for_name(name)
-            d.addCallback(_disp)
-            d.addCallback(lambda stream_info: stream.start(stream_info, name))
+            d = stream.start(stream_info, name)
             d.addCallback(lambda _: stream.downloader)
 
             return d
 
         self.waiting_on[name] = True
-        d = self._check_history(name)
-        d.addCallback(lambda lbry_file: _get_stream(name) if not lbry_file else _disp_file(lbry_file))
+        d = self.session.wallet.get_stream_info_for_name(name)
+        d.addCallback(_setup_stream)
+        d.addCallback(lambda (stream_info, lbry_file): _get_stream(stream_info) if not lbry_file else _disp_file(lbry_file))
         d.addCallback(_remove_from_wait)
 
         return d
@@ -918,46 +924,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         return d
 
-    def _check_history(self, name):
-        def _get_lbry_file(path):
-            f = open(path, 'r')
-            l = json.loads(f.read())
-            f.close()
-
-            file_name = l['stream_name'].decode('hex')
-
-            for lbry_file in self.lbry_file_manager.lbry_files:
-                if lbry_file.stream_name == file_name:
-                    if sys.platform == "darwin":
-                        if os.path.isfile(os.path.join(self.download_directory, lbry_file.stream_name)):
-                            return lbry_file
-                        else:
-                            return False
-                    else:
-                        return lbry_file
-            else:
-                return False
-
-        def _check(info):
-            stream_hash = info['stream_hash']
-            if isinstance(stream_hash, dict):
-                stream_hash = stream_hash['sd_hash']
-
-            path = os.path.join(self.blobfile_dir, stream_hash)
-            if os.path.isfile(path):
-                file_size = os.stat(path).st_size
-                log.info("[" + str(datetime.now()) + "] Search for lbry_file, found " + str(file_size) + " bytes written from stream hash: " + stream_hash)
-                return defer.succeed(_get_lbry_file(path))
-            else:
-                log.info("[" + str(datetime.now()) + "] Search for lbry_file didn't return anything")
-                return defer.succeed(False)
-
-        d = self._resolve_name(name)
-        d.addCallback(_check)
-        d.callback(None)
-
-        return d
-
     def _delete_lbry_file(self, lbry_file):
         d = self.lbry_file_manager.delete_lbry_file(lbry_file)
 
@@ -978,21 +944,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d.addCallback(lambda _: finish_deletion(lbry_file))
         return d
 
-    def _path_from_name(self, name):
-        d = self._check_history(name)
-        d.addCallback(lambda lbry_file: {'stream_hash': lbry_file.stream_hash,
-                                         'path': os.path.join(self.download_directory, lbry_file.file_name)}
-                                        if lbry_file else defer.fail(UnknownNameError))
-        return d
-
-    def _path_from_lbry_file(self, lbry_file):
-        if lbry_file:
-            r = {'stream_hash': lbry_file.stream_hash,
-                 'path': os.path.join(self.download_directory, lbry_file.file_name)}
-            return defer.succeed(r)
-        else:
-            return defer.fail(UnknownNameError)
-
     def _get_est_cost(self, name):
         def _check_est(d, name):
             if isinstance(d.result, float):
@@ -1041,21 +992,43 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         return r
 
     def _get_lbry_file_by_uri(self, name):
+        def _get_file(stream_info):
+            if isinstance(stream_info['stream_hash'], str) or isinstance(stream_info['stream_hash'], unicode):
+                sd = stream_info['stream_hash']
+            elif isinstance(stream_info['stream_hash'], dict):
+                sd = stream_info['stream_hash']['sd_hash']
+
+            for l in self.lbry_file_manager.lbry_files:
+                if l.sd_hash == sd:
+                    return defer.succeed(l)
+            return defer.succeed(None)
+
         d = self.session.wallet.get_stream_info_for_name(name)
-        d.addCallback(lambda info: info['stream_hash'])
-        d.addCallback(lambda sd_hash: next(l for l in self.lbry_file_manager.lbry_files if l.sd_hash == sd_hash))
+        d.addCallback(_get_file)
+
         return d
 
     def _get_lbry_file_by_sd_hash(self, sd_hash):
-        r = next(l for l in self.lbry_file_manager.lbry_files if l.sd_hash == sd_hash)
-        return defer.succeed(r)
+        for l in self.lbry_file_manager.lbry_files:
+            if l.sd_hash == sd_hash:
+                return defer.succeed(l)
+        return defer.succeed(None)
 
     def _get_lbry_file_by_file_name(self, file_name):
-        r = next(l for l in self.lbry_file_manager.lbry_files if l.file_name == file_name)
-        return defer.succeed(r)
+        for l in self.lbry_file_manager.lbry_files:
+            if l.file_name == file_name:
+                return defer.succeed(l)
+        return defer.succeed(None)
 
-    def _get_lbry_file(self, search_by, val):
-        def _show_file(f):
+    def _get_lbry_file(self, search_by, val, return_json=True):
+        def _log_get_lbry_file(f):
+            if f:
+                log.info("Found LBRY file for " + search_by + ": " + val)
+            else:
+                log.info("Did not find LBRY file for " + search_by + ": " + val)
+            return f
+
+        def _get_json_for_return(f):
             if f:
                 if f.key:
                     t = {'completed': f.completed, 'file_name': f.file_name, 'key': binascii.b2a_hex(f.key),
@@ -1079,7 +1052,9 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             d = self._get_lbry_file_by_sd_hash(val)
         elif search_by == "file_name":
             d = self._get_lbry_file_by_file_name(val)
-        d.addCallback(_show_file)
+        d.addCallback(_log_get_lbry_file)
+        if return_json:
+            d.addCallback(_get_json_for_return)
         return d
 
     def _log_to_slack(self, msg):
@@ -1473,36 +1448,54 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         return d
 
-    # def jsonrpc_stop_lbry_file(self, p):
-    #     params = Bunch(p)
-    #
-    #     try:
-    #         lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == params.stream_hash][0]
-    #     except IndexError:
-    #         return defer.fail(UnknownNameError)
-    #
-    #     if not lbry_file.stopped:
-    #         d = self.lbry_file_manager.toggle_lbry_file_running(lbry_file)
-    #         d.addCallback(lambda _: self._render_response("Stream has been stopped", OK_CODE))
-    #         d.addErrback(lambda err: self._render_response(err.getTraceback(), ))
-    #         return d
-    #     else:
-    #         return json.dumps({'result': 'Stream was already stopped'})
-    #
-    # def jsonrpc_start_lbry_file(self, p):
-    #     params = Bunch(p)
-    #
-    #     try:
-    #         lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == params.stream_hash][0]
-    #     except IndexError:
-    #         return defer.fail(UnknownNameError)
-    #
-    #     if lbry_file.stopped:
-    #         d = self.lbry_file_manager.toggle_lbry_file_running(lbry_file)
-    #         d.callback(None)
-    #         return json.dumps({'result': 'Stream started'})
-    #     else:
-    #         return json.dumps({'result': 'Stream was already running'})
+    def jsonrpc_stop_lbry_file(self, p):
+        """
+        Stop lbry file
+
+        Args:
+            'name': stop file by lbry uri,
+            'sd_hash': stop file by the hash in the name claim,
+            'file_name': stop file by its name in the downloads folder,
+        Returns:
+            confirmation message
+        """
+
+        def _stop_file(f):
+            d =  self.lbry_file_manager.toggle_lbry_file_running(f)
+            d.addCallback(lambda _: "Stopped LBRY file")
+            return d
+
+        if p.keys()[0] in ['name', 'sd_hash', 'file_name']:
+            search_type = p.keys()[0]
+            d = self._get_lbry_file(search_type, p[search_type], return_json=False)
+            d.addCallback(lambda l: _stop_file(l) if not l.stopped else "LBRY file wasn't running")
+
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
+    def jsonrpc_start_lbry_file(self, p):
+        """
+        Stop lbry file
+
+        Args:
+            'name': stop file by lbry uri,
+            'sd_hash': stop file by the hash in the name claim,
+            'file_name': stop file by its name in the downloads folder,
+        Returns:
+            confirmation message
+        """
+
+        def _start_file(f):
+            d = self.lbry_file_manager.toggle_lbry_file_running(f)
+            return defer.succeed("Started LBRY file")
+
+        if p.keys()[0] in ['name', 'sd_hash', 'file_name']:
+            search_type = p.keys()[0]
+            d = self._get_lbry_file(search_type, p[search_type], return_json=False)
+            d.addCallback(lambda l: _start_file(l) if l.stopped else "LBRY file was already running")
+
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
 
     def jsonrpc_search_nametrie(self, p):
         """
@@ -1575,17 +1568,20 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             confirmation message
         """
 
-        def _disp(file_name):
-            log.info("[" + str(datetime.now()) + "] Deleted: " + file_name)
-            return self._render_response("Deleted: " + file_name, OK_CODE)
-
-        if "file_name" in p.keys():
-            lbry_files = [self._delete_lbry_file(f) for f in self.lbry_file_manager.lbry_files
-                                                    if p['file_name'] == f.file_name]
-            d = defer.DeferredList(lbry_files)
-            d.addCallback(lambda _: _disp(p['file_name']))
+        def _delete_file(f):
+            file_name = f.file_name
+            d = self._delete_lbry_file(f)
+            d.addCallback(lambda _: "Deleted LBRY file" + file_name)
             return d
 
+        if p.keys()[0] in ['name', 'sd_hash', 'file_name']:
+            search_type = p.keys()[0]
+            d = self._get_lbry_file(search_type, p[search_type], return_json=False)
+            d.addCallback(lambda l: _delete_file(l) if l else "Couldn't find LBRY file to delete")
+
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
     def jsonrpc_publish(self, p):
         """
         Make a new name claim
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index e68479bbe..167c11e92 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -54,13 +54,20 @@ class LBRYindex(resource.Resource):
 
 
 class LBRYFileProducer(StaticProducer):
-    def __init__(self, request, lbry_stream):
+    def __init__(self, request, lbry_stream, api):
+        self._api = api
         self.stream = lbry_stream
         self.updater = LoopingCall(self._check_for_data)
-        StaticProducer.__init__(self, request, fileObject=file(lbry_stream.file_written_to))
+        self.total_bytes = 0
+        if lbry_stream.file_written_to:
+            file_name = lbry_stream.file_written_to
+        else:
+            file_name = os.path.join(self._api.download_directory, lbry_stream.file_name)
+        StaticProducer.__init__(self, request, fileObject=file(file_name))
 
     def start(self):
         d = self._set_size()
+        self.fileObject.seek(0)
         self.updater.start(5)
 
     def _set_size(self):
@@ -74,17 +81,22 @@ class LBRYFileProducer(StaticProducer):
         return d
 
     def _check_for_data(self):
-        self.fileObject.seek(self.fileObject.tell())
-        data = self.fileObject.read()
-        if data:
-            self.request.write(data)
+        def _write_new_data_to_request():
+            self.fileObject.seek(self.fileObject.tell())
+            data = self.fileObject.read()
+            self.total_bytes += len(data)
+
+            if data:
+                self.request.write(data)
+            return defer.succeed(None)
 
         def _check_status(stream_status):
             if stream_status.running_status == "completed":
                 self.stopProducing()
             return defer.succeed(None)
 
-        d = self.stream.status()
+        d = _write_new_data_to_request()
+        d.addCallback(lambda _: self.stream.status())
         d.addCallback(_check_status)
 
     def resumeProducing(self):
@@ -102,24 +114,22 @@ class HostedLBRYFile(resource.Resource):
         self._api = api
         self.stream = None
         self.streaming_file = None
+        self.producer = None
         resource.Resource.__init__(self)
 
-    def _set_stream(self, stream):
-        self.stream = stream
-
     def makeProducer(self, request, stream):
-        return LBRYFileProducer(request, stream)
+        self.producer = LBRYFileProducer(request, stream, self._api)
+        return self.producer
 
     def render_GET(self, request):
         if 'name' in request.args.keys():
-            if request.args['name'][0] != 'lbry':
-                if request.args['name'][0] != self.streaming_file:
-                    self.streaming_file = request.args['name'][0]
-                    d = self._api._download_name(request.args['name'][0])
-                    d.addCallback(self._set_stream)
-                else:
-                    d = defer.succeed(None)
-                d.addCallback(lambda _: self.makeProducer(request, self.stream).start())
+            if request.args['name'][0] != 'lbry' and request.args['name'][0] not in self._api.waiting_on.keys():
+                d = self._api._download_name(request.args['name'][0])
+                d.addCallback(lambda stream: self.makeProducer(request, stream))
+                d.addCallback(lambda producer: producer.start())
+            elif request.args['name'][0] in self._api.waiting_on.keys():
+                request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0])
+                request.finish()
             else:
                 request.redirect(UI_ADDRESS)
                 request.finish()
diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py
index b52c8c92a..048eb3e5b 100644
--- a/lbrynet/lbrynet_daemon/LBRYDownloader.py
+++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py
@@ -1,7 +1,9 @@
 import json
 import logging
 import os
+import sys
 
+from appdirs import user_data_dir
 from datetime import datetime
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
@@ -12,7 +14,18 @@ from lbrynet.core.StreamDescriptor import download_sd_blob
 from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
 from lbrynet.conf import DEFAULT_TIMEOUT
 
+if sys.platform != "darwin":
+    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
+else:
+    log_dir = user_data_dir("LBRY")
+
+if not os.path.isdir(log_dir):
+    os.mkdir(log_dir)
+
+LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log')
 log = logging.getLogger(__name__)
+handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, backupCount=5)
+log.addHandler(handler)
 log.setLevel(logging.INFO)
 
 
@@ -123,7 +136,7 @@ class GetStream(object):
             d = defer.Deferred()
         self.downloader = downloader
         self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
-        d.addCallback(lambda _: log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
+        d.addCallback(lambda _: log.info("[" + str(datetime.now()) + "] Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
         d.addCallback(lambda _: self.downloader.start())
 
 

From 591634f175a8338c24fb4df022a12370b07135bc Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Mon, 25 Apr 2016 22:35:21 -0400
Subject: [PATCH 16/38] cache name claim info
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

-cache name claim info for an hour rather than looking it up each time
it’s required

-add default thumbnail to search results
---
 lbrynet/conf.py                            |   2 +-
 lbrynet/lbrynet_daemon/LBRYDaemon.py       | 170 ++++++++++++---------
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py |   6 +-
 3 files changed, 102 insertions(+), 76 deletions(-)

diff --git a/lbrynet/conf.py b/lbrynet/conf.py
index a4a387991..dfae3d274 100644
--- a/lbrynet/conf.py
+++ b/lbrynet/conf.py
@@ -40,4 +40,4 @@ DEFAULT_TIMEOUT = 30
 DEFAULT_MAX_SEARCH_RESULTS = 25
 DEFAULT_MAX_KEY_FEE = 100.0
 DEFAULT_SEARCH_TIMEOUT = 3.0
-
+DEFAULT_CACHE_TIME = 3600
\ No newline at end of file
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 961d6bdca..3fd60fe1f 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -36,7 +36,7 @@ from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher
 from lbrynet.core.utils import generate_id
 from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
 from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \
-    DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT
+    DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME
 from lbrynet.conf import API_CONNECTION_STRING, API_PORT, API_ADDRESS, DEFAULT_TIMEOUT, UI_ADDRESS
 from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob
 from lbrynet.core.Session import LBRYSession
@@ -83,12 +83,26 @@ STARTUP_STAGES = [
                     (STARTED_CODE, 'Started lbrynet')
                   ]
 
+DOWNLOAD_METADATA_CODE = 'downloading_metadata'
+DOWNLOAD_TIMEOUT_CODE = 'timeout'
+DOWNLOAD_RUNNING_CODE = 'running'
+DOWNLOAD_STOPPED_CODE = 'stopped'
+STREAM_STAGES = [
+                    (INITIALIZING_CODE, 'Initializing...'),
+                    (DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
+                    (DOWNLOAD_RUNNING_CODE, 'Started stream'),
+                    (DOWNLOAD_STOPPED_CODE, 'Paused stream'),
+                    (DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
+                ]
+
 CONNECT_CODE_VERSION_CHECK = 'version_check'
 CONNECT_CODE_NETWORK = 'network_connection'
 CONNECT_CODE_WALLET = 'wallet_catchup_lag'
-CONNECTION_PROBLEM_CODES = [(CONNECT_CODE_VERSION_CHECK, "There was a problem checking for updates on github"),
-                            (CONNECT_CODE_NETWORK, "Your internet connection appears to have been interrupted"),
-                            (CONNECT_CODE_WALLET, "Synchronization with the blockchain is lagging... if this continues try restarting LBRY")]
+CONNECTION_PROBLEM_CODES = [
+        (CONNECT_CODE_VERSION_CHECK, "There was a problem checking for updates on github"),
+        (CONNECT_CODE_NETWORK, "Your internet connection appears to have been interrupted"),
+        (CONNECT_CODE_WALLET, "Synchronization with the blockchain is lagging... if this continues try restarting LBRY")
+        ]
 
 ALLOWED_DURING_STARTUP = ['is_running', 'is_first_run',
                           'get_time_behind_blockchain', 'stop',
@@ -218,7 +232,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             'dht_node_port': 4444,
             'use_upnp': True,
             'start_lbrycrdd': True,
-            'requested_first_run_credits': False
+            'requested_first_run_credits': False,
+            'cache_time': DEFAULT_CACHE_TIME
         }
 
         if os.path.isfile(self.daemon_conf):
@@ -272,6 +287,15 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.use_upnp = self.session_settings['use_upnp']
         self.start_lbrycrdd = self.session_settings['start_lbrycrdd']
         self.requested_first_run_credits = self.session_settings['requested_first_run_credits']
+        self.cache_time = self.session_settings['cache_time']
+
+        if os.path.isfile(os.path.join(self.db_dir, "stream_info_cache.json")):
+            f = open(os.path.join(self.db_dir, "stream_info_cache.json"), "r")
+            self.name_cache = json.loads(f.read())
+            f.close()
+            log.info("Loaded claim info cache")
+        else:
+            self.name_cache = {}
 
     def render(self, request):
         request.content.seek(0, 0)
@@ -341,12 +365,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
     def setup(self):
         def _log_starting_vals():
-            r = json.dumps(self._get_lbry_files())
-
-            log.info("LBRY Files: " + r)
-            log.info("Starting balance: " + str(self.session.wallet.wallet_balance))
-
-            return defer.succeed(None)
+            d = self._get_lbry_files()
+            d.addCallback(lambda r: json.dumps([d[1] for d in r]))
+            d.addCallback(lambda r: log.info("LBRY Files: " + r))
+            d.addCallback(lambda _: log.info("Starting balance: " + str(self.session.wallet.wallet_balance)))
+            return d
 
         def _announce_startup():
             def _announce():
@@ -630,7 +653,16 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                     self.session_settings['download_timeout'] = settings['download_timeout']
                 else:
                     return defer.fail()
-
+            elif k == 'search_timeout':
+                if type(settings['search_timeout']) is float:
+                    self.session_settings['search_timeout'] = settings['search_timeout']
+                else:
+                    return defer.fail()
+            elif k == 'cache_time':
+                if type(settings['cache_time']) is int:
+                    self.session_settings['cache_time'] = settings['cache_time']
+                else:
+                    return defer.fail()
         self.run_on_startup = self.session_settings['run_on_startup']
         self.data_rate = self.session_settings['data_rate']
         self.max_key_fee = self.session_settings['max_key_fee']
@@ -639,6 +671,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.max_download = self.session_settings['max_download']
         self.upload_log = self.session_settings['upload_log']
         self.download_timeout = self.session_settings['download_timeout']
+        self.search_timeout = self.session_settings['search_timeout']
+        self.cache_time = self.session_settings['cache_time']
 
         f = open(self.daemon_conf, "w")
         f.write(json.dumps(self.session_settings))
@@ -879,7 +913,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             stream_hash = stream_info['stream_hash']
             if isinstance(stream_hash, dict):
                 stream_hash = stream_hash['sd_hash']
-            log.info("[" + str(datetime.now()) + "] Resolved lbry://" + name + " to sd hash: " + stream_hash)
             d = self._get_lbry_file_by_sd_hash(stream_hash)
             def _add_results(l):
                 return defer.succeed((stream_info, l))
@@ -902,25 +935,41 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             return d
 
         self.waiting_on[name] = True
-        d = self.session.wallet.get_stream_info_for_name(name)
+        d = self._resolve_name(name)
         d.addCallback(_setup_stream)
         d.addCallback(lambda (stream_info, lbry_file): _get_stream(stream_info) if not lbry_file else _disp_file(lbry_file))
         d.addCallback(_remove_from_wait)
 
         return d
 
+    def _get_long_count_timestamp(self):
+        return int((datetime.utcnow() - (datetime(year=2012, month=12, day=21))).total_seconds())
+
+    def _update_claim_cache(self):
+        f = open(os.path.join(self.db_dir, "stream_info_cache.json"), "w")
+        f.write(json.dumps(self.name_cache))
+        f.close()
+        return defer.succeed(True)
+
     def _resolve_name(self, name):
-        d = defer.Deferred()
-        d.addCallback(lambda _: self.session.wallet.get_stream_info_for_name(name))
-        d.addErrback(lambda _: defer.fail(UnknownNameError))
+        def _cache_stream_info(stream_info):
+            self.name_cache[name] = {'claim_metadata': stream_info, 'timestamp': self._get_long_count_timestamp()}
+            d = self._update_claim_cache()
+            d.addCallback(lambda _: self.name_cache[name]['claim_metadata'])
+            return d
 
-        return d
-
-    def _resolve_name_wc(self, name):
-        d = defer.Deferred()
-        d.addCallback(lambda _: self.session.wallet.get_stream_info_for_name(name))
-        d.addErrback(lambda _: defer.fail(UnknownNameError))
-        d.callback(None)
+        if name in self.name_cache.keys():
+            if (self._get_long_count_timestamp() - self.name_cache[name]['timestamp']) < self.cache_time:
+                log.info("[" + str(datetime.now()) + "] Returning cached stream info for lbry://" + name)
+                d = defer.succeed(self.name_cache[name]['claim_metadata'])
+            else:
+                log.info("[" + str(datetime.now()) + "] Refreshing stream info for lbry://" + name)
+                d = self.session.wallet.get_stream_info_for_name(name)
+                d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError))
+        else:
+            log.info("[" + str(datetime.now()) + "] Resolving stream info for lbry://" + name)
+            d = self.session.wallet.get_stream_info_for_name(name)
+            d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError))
 
         return d
 
@@ -954,11 +1003,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             return defer.succeed(None)
 
         def _add_key_fee(data_cost):
-            d = self.session.wallet.get_stream_info_for_name(name)
+            d = self._resolve_name(name)
             d.addCallback(lambda info: data_cost + info['key_fee'] if 'key_fee' in info.keys() else data_cost)
             return d
 
-        d = self.session.wallet.get_stream_info_for_name(name)
+        d = self._resolve_name(name)
         d.addCallback(lambda info: info['stream_hash'] if isinstance(info['stream_hash'], str)
                                     else info['stream_hash']['sd_hash'])
         d.addCallback(lambda sd_hash: download_sd_blob(self.session, sd_hash,
@@ -972,25 +1021,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         return d
 
-    def _get_lbry_files(self):
-        r = []
-        for f in self.lbry_file_manager.lbry_files:
-            if f.key:
-                t = {'completed': f.completed, 'file_name': f.file_name, 'key': binascii.b2a_hex(f.key),
-                     'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
-                     'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
-                     'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash}
-
-            else:
-                t = {'completed': f.completed, 'file_name': f.file_name, 'key': None,
-                     'points_paid': f.points_paid,
-                     'stopped': f.stopped, 'stream_hash': f.stream_hash, 'stream_name': f.stream_name,
-                     'suggested_file_name': f.suggested_file_name, 'upload_allowed': f.upload_allowed,
-                     'sd_hash': f.sd_hash}
-
-            r.append(t)
-        return r
-
     def _get_lbry_file_by_uri(self, name):
         def _get_file(stream_info):
             if isinstance(stream_info['stream_hash'], str) or isinstance(stream_info['stream_hash'], unicode):
@@ -1003,7 +1033,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                     return defer.succeed(l)
             return defer.succeed(None)
 
-        d = self.session.wallet.get_stream_info_for_name(name)
+        d = self._resolve_name(name)
         d.addCallback(_get_file)
 
         return d
@@ -1022,26 +1052,22 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
     def _get_lbry_file(self, search_by, val, return_json=True):
         def _log_get_lbry_file(f):
-            if f:
+            if f and val:
                 log.info("Found LBRY file for " + search_by + ": " + val)
-            else:
+            elif val:
                 log.info("Did not find LBRY file for " + search_by + ": " + val)
             return f
 
         def _get_json_for_return(f):
             if f:
                 if f.key:
-                    t = {'completed': f.completed, 'file_name': f.file_name, 'key': binascii.b2a_hex(f.key),
-                         'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
-                         'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
-                         'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash}
-
+                    key = binascii.b2a_hex(f.key)
                 else:
-                    t = {'completed': f.completed, 'file_name': f.file_name, 'key': None,
-                         'points_paid': f.points_paid,
-                         'stopped': f.stopped, 'stream_hash': f.stream_hash, 'stream_name': f.stream_name,
-                         'suggested_file_name': f.suggested_file_name, 'upload_allowed': f.upload_allowed,
-                         'sd_hash': f.sd_hash}
+                    key = None
+                t = {'completed': f.completed, 'file_name': f.file_name, 'key': key,
+                     'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
+                     'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
+                     'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash}
                 return t
             else:
                 return False
@@ -1057,6 +1083,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             d.addCallback(_get_json_for_return)
         return d
 
+    def _get_lbry_files(self):
+        d = defer.DeferredList([self._get_lbry_file('sd_hash', l.sd_hash) for l in self.lbry_file_manager.lbry_files])
+        return d
+
     def _log_to_slack(self, msg):
         URL = "https://hooks.slack.com/services/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"
         msg = platform.platform() + ": " + base58.b58encode(self.lbryid)[:20] + ", " + msg
@@ -1351,9 +1381,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             'sd_hash': string
         """
 
-        r = self._get_lbry_files()
-        log.info("[" + str(datetime.now()) + "] Get LBRY files")
-        return self._render_response(r, OK_CODE)
+        d = self._get_lbry_files()
+        d.addCallback(lambda r: [d[1] for d in r])
+        d.addCallback(lambda r: self._render_response(r, OK_CODE) if len(r) else self._render_response(False, OK_CODE))
+
+        return d
 
     def jsonrpc_get_lbry_file(self, p):
         """
@@ -1399,18 +1431,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         else:
             return self._render_response(None, BAD_REQUEST)
 
-        def _disp(info):
-            stream_hash = info['stream_hash']
-            if isinstance(stream_hash, dict):
-                stream_hash = stream_hash['sd_hash']
-
-            log.info("[" + str(datetime.now()) + "] Resolved info: " + stream_hash)
-
-            return self._render_response(info, OK_CODE)
-
         d = self._resolve_name(name)
-        d.addCallbacks(_disp, lambda _: server.failure)
-        d.callback(None)
+        d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure)
         return d
 
     def jsonrpc_get(self, p):
@@ -1525,7 +1547,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             ds = []
             for claim in claims:
                 d1 = defer.succeed(claim)
-                d2 = self._resolve_name_wc(claim['name'])
+                d2 = self._resolve_name(claim['name'])
                 d3 = self._get_est_cost(claim['name'])
                 dl = defer.DeferredList([d1, d2, d3], consumeErrors=True)
                 ds.append(dl)
@@ -1542,6 +1564,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                     del r[1]['name']
                 t.update(r[1])
                 t['cost_est'] = r[2]
+                if not 'thumbnail' in t.keys():
+                    t['thumbnail'] = "img/Free-speech-flag.svg"
                 consolidated_results.append(t)
                 # log.info(str(t))
             return consolidated_results
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 167c11e92..740a1bd42 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -68,12 +68,12 @@ class LBRYFileProducer(StaticProducer):
     def start(self):
         d = self._set_size()
         self.fileObject.seek(0)
-        self.updater.start(5)
+        self.updater.start(1)
 
     def _set_size(self):
         def _set(size):
             self.request.setHeader('content-length', str(size))
-            self.request.setHeader('content-type', ' application/octet-stream')
+            self.request.setHeader('content-type', 'application/octet-stream')
             return defer.succeed(None)
 
         d = self.stream.get_total_bytes()
@@ -85,6 +85,7 @@ class LBRYFileProducer(StaticProducer):
             self.fileObject.seek(self.fileObject.tell())
             data = self.fileObject.read()
             self.total_bytes += len(data)
+            log.info(str(self.total_bytes))
 
             if data:
                 self.request.write(data)
@@ -93,6 +94,7 @@ class LBRYFileProducer(StaticProducer):
         def _check_status(stream_status):
             if stream_status.running_status == "completed":
                 self.stopProducing()
+
             return defer.succeed(None)
 
         d = _write_new_data_to_request()

From caf5d2f7880fdfcc0e54111c98a91a534bf8150f Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Tue, 26 Apr 2016 19:48:43 -0400
Subject: [PATCH 17/38] update uri handler to use /watch

---
 lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py | 4 ++--
 lbrynet/lbrynet_daemon/LBRYDaemon.py          | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py b/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py
index c748a1015..ed63c6f03 100644
--- a/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py
+++ b/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py
@@ -64,7 +64,7 @@ class LBRYURIHandler(object):
         if lbry_name == "lbry" or lbry_name == "" and not started:
             webbrowser.open(UI_ADDRESS)
         else:
-            webbrowser.open(UI_ADDRESS + "/view?name=" + lbry_name)
+            webbrowser.open(UI_ADDRESS + "/?watch=" + lbry_name)
 
     def handle_linux(self, lbry_name):
         try:
@@ -77,7 +77,7 @@ class LBRYURIHandler(object):
         if lbry_name == "lbry":
             webbrowser.open(UI_ADDRESS)
         else:
-            webbrowser.open(UI_ADDRESS + "/view?name=" + lbry_name)
+            webbrowser.open(UI_ADDRESS + "/?watch=" + lbry_name)
 
 
 def main(args):
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 3fd60fe1f..12f17ca18 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -392,8 +392,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         log.info("[" + str(datetime.now()) + "] Starting lbrynet-daemon")
 
-        self.internet_connection_checker.start(60)
-        self.version_checker.start(3600)
+        self.internet_connection_checker.start(3600)
+        self.version_checker.start(3600 * 12)
         self.connection_problem_checker.start(1)
 
         d = defer.Deferred()

From 226e9084c934fee83c18c5fd3bbc284d77b6710d Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Sat, 30 Apr 2016 23:43:02 -0400
Subject: [PATCH 18/38] return false from get_lbry_file if the file doesnt
 exist

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 12f17ca18..bbc1f1384 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -1601,7 +1601,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         if p.keys()[0] in ['name', 'sd_hash', 'file_name']:
             search_type = p.keys()[0]
             d = self._get_lbry_file(search_type, p[search_type], return_json=False)
-            d.addCallback(lambda l: _delete_file(l) if l else "Couldn't find LBRY file to delete")
+            d.addCallback(lambda l: _delete_file(l) if l else False)
 
         d.addCallback(lambda r: self._render_response(r, OK_CODE))
         return d

From 46368f52f5fd2f4496f71cdf0675321b5a9be79a Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Sun, 1 May 2016 05:17:59 -0400
Subject: [PATCH 19/38] streaming files

still needs work, still has problems in safari and is otherwise slow
---
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 205 +++++++++++++--------
 1 file changed, 132 insertions(+), 73 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 740a1bd42..b6d4942cf 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -2,10 +2,8 @@ import logging
 import subprocess
 import os
 import shutil
-
-from twisted.internet.task import LoopingCall
-from txjsonrpc.web import jsonrpc
 import json
+import sys
 
 from StringIO import StringIO
 from zipfile import ZipFile
@@ -13,15 +11,21 @@ from urllib import urlopen
 from datetime import datetime
 from appdirs import user_data_dir
 from twisted.web import server, static, resource
-from twisted.internet import defer
-from twisted.web.static import StaticProducer
+from twisted.internet import defer, interfaces, error, reactor, task
+from twisted.python.failure import Failure
+from txjsonrpc.web import jsonrpc
+
+from zope.interface import implements
 
 from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon
 from lbrynet.conf import API_CONNECTION_STRING, API_ADDRESS, DEFAULT_WALLET, UI_ADDRESS
 
-log = logging.getLogger(__name__)
 
-data_dir = user_data_dir("LBRY")
+if sys.platform != "darwin":
+    data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
+else:
+    data_dir = user_data_dir("LBRY")
+
 if not os.path.isdir(data_dir):
     os.mkdir(data_dir)
 version_dir = os.path.join(data_dir, "ui_version_history")
@@ -31,6 +35,9 @@ if not os.path.isdir(version_dir):
 version_log = logging.getLogger("lbry_version")
 version_log.addHandler(logging.FileHandler(os.path.join(version_dir, "lbry_version.log")))
 version_log.setLevel(logging.INFO)
+log = logging.getLogger(__name__)
+log.addHandler(logging.FileHandler(os.path.join(data_dir, 'lbrynet-daemon.log')))
+log.setLevel(logging.INFO)
 
 
 class LBRYindex(resource.Resource):
@@ -53,82 +60,103 @@ class LBRYindex(resource.Resource):
         return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request)
 
 
-class LBRYFileProducer(StaticProducer):
-    def __init__(self, request, lbry_stream, api):
-        self._api = api
-        self.stream = lbry_stream
-        self.updater = LoopingCall(self._check_for_data)
-        self.total_bytes = 0
-        if lbry_stream.file_written_to:
-            file_name = lbry_stream.file_written_to
-        else:
-            file_name = os.path.join(self._api.download_directory, lbry_stream.file_name)
-        StaticProducer.__init__(self, request, fileObject=file(file_name))
+class LBRYFileStreamer(object):
+    """
+    Writes downloaded LBRY file to request as the download comes in, pausing and resuming as requested
+    used for Chrome
+    """
 
-    def start(self):
-        d = self._set_size()
-        self.fileObject.seek(0)
-        self.updater.start(1)
+    implements(interfaces.IPushProducer)
 
-    def _set_size(self):
-        def _set(size):
-            self.request.setHeader('content-length', str(size))
-            self.request.setHeader('content-type', 'application/octet-stream')
-            return defer.succeed(None)
+    def __init__(self, request, path, start, stop, size):
+        self._request = request
+        self._fileObject = file(path)
+        self._stop_pos = size if stop == '' else int(stop)  #chrome and firefox send range requests for "0-"
+        self._cursor = self._start_pos = int(start)
+        self._file_size = size
 
-        d = self.stream.get_total_bytes()
-        d.addCallback(_set)
-        return d
+        self._paused = self._sent_bytes = False
+        self._delay = 0.1
 
-    def _check_for_data(self):
-        def _write_new_data_to_request():
-            self.fileObject.seek(self.fileObject.tell())
-            data = self.fileObject.read()
-            self.total_bytes += len(data)
-            log.info(str(self.total_bytes))
+        self._request.setResponseCode(206)
+        self._request.setHeader('accept-ranges', 'bytes')
+        self._request.setHeader('content-type', 'application/octet-stream')
+        self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._start_pos, self._stop_pos, self._file_size))
 
-            if data:
-                self.request.write(data)
-            return defer.succeed(None)
+        self.resumeProducing()
 
-        def _check_status(stream_status):
-            if stream_status.running_status == "completed":
-                self.stopProducing()
-
-            return defer.succeed(None)
-
-        d = _write_new_data_to_request()
-        d.addCallback(lambda _: self.stream.status())
-        d.addCallback(_check_status)
+    def pauseProducing(self):
+        self._paused = True
+        log.info("[" + str(datetime.now()) + "] Pausing producer")
+        return defer.succeed(None)
 
     def resumeProducing(self):
-        self.updater.start(1)
+        def _check_for_new_data():
+            self._fileObject.seek(self._start_pos, os.SEEK_END)
+            readable_bytes = self._fileObject.tell()
+            self._fileObject.seek(self._cursor)
+
+            self._sent_bytes = False
+
+            if readable_bytes > self._cursor:
+                read_length = min(readable_bytes, self._stop_pos) - self._cursor
+                log.info('Writing range %s-%s/%s' % (self._cursor, self._cursor + read_length, self._file_size))
+                self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._cursor, self._cursor + read_length, self._file_size))
+                for i in range(read_length + 1):
+                    if self._paused:
+                        break
+                    else:
+                        data = self._fileObject.read(1)
+                        self._request.write(data)
+                        self._cursor += 1
+                self._sent_bytes = True
+            return defer.succeed(None)
+
+        def _write_reply():
+            if self._cursor == self._stop_pos + 1:
+                self.stopProducing()
+                return defer.succeed(None)
+            elif self._paused:
+                return defer.succeed(None)
+            else:
+                d = task.deferLater(reactor, self._delay, _check_for_new_data)
+                d.addCallback(lambda _: _write_reply())
+                return d
+
+        log.info("[" + str(datetime.now()) + "] Resuming producer")
+
+        self._paused = False
+        _write_reply()
+        return defer.succeed(None)
 
     def stopProducing(self):
-        self.updater.stop()
-        self.fileObject.close()
-        self.stream.stop()
-        self.request.finish()
+        log.info("Stopping producer")
+        # self._fileObject.close()
+        self._request.finish()
+        self._request.unregisterProducer()
 
 
 class HostedLBRYFile(resource.Resource):
     def __init__(self, api):
         self._api = api
-        self.stream = None
-        self.streaming_file = None
-        self.producer = None
         resource.Resource.__init__(self)
 
     def makeProducer(self, request, stream):
-        self.producer = LBRYFileProducer(request, stream, self._api)
-        return self.producer
+        range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-')
+        start, stop = int(range_header[0]), range_header[1]
+        path = os.path.join(self._api.download_directory, stream.file_name)
+
+        d = stream.get_total_bytes()
+        d.addCallback(lambda size: request.registerProducer(LBRYFileStreamer(request, path, start, stop, size), streaming=True))
+
+        return d
 
     def render_GET(self, request):
         if 'name' in request.args.keys():
             if request.args['name'][0] != 'lbry' and request.args['name'][0] not in self._api.waiting_on.keys():
                 d = self._api._download_name(request.args['name'][0])
                 d.addCallback(lambda stream: self.makeProducer(request, stream))
-                d.addCallback(lambda producer: producer.start())
+                request.notifyFinish().addErrback(self._responseFailed, d)
             elif request.args['name'][0] in self._api.waiting_on.keys():
                 request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0])
                 request.finish()
@@ -137,24 +165,55 @@ class HostedLBRYFile(resource.Resource):
                 request.finish()
             return server.NOT_DONE_YET
 
+    def _responseFailed(self, err, call):
+        call.addErrback(lambda err: err.trap(error.ConnectionDone))
+        call.addErrback(lambda err: err.trap(defer.CancelledError))
+        call.addErrback(lambda err: log.info("Error: " + str(err)))
+        call.cancel()
 
-class LBRYFileRender(resource.Resource):
+
+class MyLBRYFiles(resource.Resource):
     isLeaf = False
 
+    def __init__(self):
+        resource.Resource.__init__(self)
+        self.files_table = None
+
+    def delayed_render(self, request, result):
+        request.write(result.encode('utf-8'))
+        request.finish()
+
     def render_GET(self, request):
-        if 'name' in request.args.keys():
-            api = jsonrpc.Proxy(API_CONNECTION_STRING)
-            if request.args['name'][0] != 'lbry':
-                d = api.callRemote("get", {'name': request.args['name'][0]})
-                d.addCallback(lambda results: static.File(results['path'], defaultType='video/octet-stream'))
-                d.addCallback(lambda static_file: static_file.render_GET(request) if static_file.getFileSize() > 0
-                              else server.failure)
-            else:
-                request.redirect(UI_ADDRESS)
-                request.finish()
-            return server.NOT_DONE_YET
+        self.files_table = None
+        api = jsonrpc.Proxy(API_CONNECTION_STRING)
+        d = api.callRemote("get_lbry_files", {})
+        d.addCallback(self._get_table)
+        d.addCallback(lambda results: self.delayed_render(request, results))
+
+        return server.NOT_DONE_YET
+
+    def _get_table(self, files):
+        if not self.files_table:
+            self.files_table = r'<html><head><title>My LBRY files</title></head><body><table border="1">'
+            self.files_table += r'<tr>'
+            self.files_table += r'<td>Stream name</td>'
+            self.files_table += r'<td>Completed</td>'
+            self.files_table += r'<td>Toggle</td>'
+            self.files_table += r'<td>Remove</td>'
+            self.files_table += r'</tr>'
+            return self._get_table(files)
+        if not len(files):
+            self.files_table += r'</table></body></html>'
+            return self.files_table
         else:
-            return server.failure
+            f = files.pop()
+            self.files_table += r'<tr>'
+            self.files_table += r'<td>%s</td>' % (f['stream_name'])
+            self.files_table += r'<td>%s</td>' % (f['completed'])
+            self.files_table += r'<td>Start</td>' if f['stopped'] else r'<td>Stop</td>'
+            self.files_table += r'<td>Delete</td>'
+            self.files_table += r'</tr>'
+            return self._get_table(files)
 
 
 class LBRYDaemonServer(object):
@@ -267,8 +326,8 @@ class LBRYDaemonServer(object):
         self.root.putChild("font", static.File(os.path.join(self.ui_dir, "font")))
         self.root.putChild("img", static.File(os.path.join(self.ui_dir, "img")))
         self.root.putChild("js", static.File(os.path.join(self.ui_dir, "js")))
-        # self.root.putChild("view", LBRYFileRender())
         self.root.putChild("view", HostedLBRYFile(self._api))
+        self.root.putChild("files", MyLBRYFiles())
         self.root.putChild(API_ADDRESS, self._api)
         return defer.succeed(True)
 

From 04ee9894c99d8d3bc689a58da28044a73a37249c Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Mon, 2 May 2016 04:10:50 -0400
Subject: [PATCH 20/38] speed up streaming, off by one errors

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py        |  1 +
 lbrynet/lbrynet_daemon/LBRYDaemonControl.py |  3 +
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py  | 63 +++++++++++++--------
 3 files changed, 43 insertions(+), 24 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index bbc1f1384..17be5449e 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -991,6 +991,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             return d
 
         d.addCallback(lambda _: finish_deletion(lbry_file))
+        d.addCallback(lambda _: log.info("[" + str(datetime.now()) + "] Delete lbry file"))
         return d
 
     def _get_est_cost(self, name):
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
index 9cf3f4143..506dc23e1 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
@@ -132,3 +132,6 @@ def start():
         if not args.logtoconsole:
             print "Not connected to internet, unable to start"
         return
+
+if __name__ == "__main__":
+    start()
\ No newline at end of file
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index b6d4942cf..8a9aa0a05 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -4,6 +4,7 @@ import os
 import shutil
 import json
 import sys
+import mimetypes
 
 from StringIO import StringIO
 from zipfile import ZipFile
@@ -71,18 +72,21 @@ class LBRYFileStreamer(object):
     def __init__(self, request, path, start, stop, size):
         self._request = request
         self._fileObject = file(path)
-        self._stop_pos = size if stop == '' else int(stop)  #chrome and firefox send range requests for "0-"
+        self._content_type = mimetypes.guess_type(path)[0]
+        self._stop_pos = size - 1 if stop == '' else int(stop)  #chrome and firefox send range requests for "0-"
         self._cursor = self._start_pos = int(start)
         self._file_size = size
 
-        self._paused = self._sent_bytes = False
+        self._paused = self._sent_bytes = self._stopped = False
         self._delay = 0.1
+        self._deferred = defer.succeed(None)
 
         self._request.setResponseCode(206)
         self._request.setHeader('accept-ranges', 'bytes')
-        self._request.setHeader('content-type', 'application/octet-stream')
-        self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._start_pos, self._stop_pos, self._file_size))
+        # self._request.setHeader('content-type', 'application/octet-stream')
+        self._request.setHeader('content-type', self._content_type)
 
+        # self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._start_pos, self._stop_pos, self._file_size))
         self.resumeProducing()
 
     def pauseProducing(self):
@@ -98,57 +102,68 @@ class LBRYFileStreamer(object):
 
             self._sent_bytes = False
 
-            if readable_bytes > self._cursor:
-                read_length = min(readable_bytes, self._stop_pos) - self._cursor
-                log.info('Writing range %s-%s/%s' % (self._cursor, self._cursor + read_length, self._file_size))
-                self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._cursor, self._cursor + read_length, self._file_size))
-                for i in range(read_length + 1):
-                    if self._paused:
+            if (readable_bytes > self._cursor) and not (self._stopped or self._paused):
+                read_length = min(readable_bytes, self._stop_pos) - self._cursor + 1
+                self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._cursor, self._cursor + read_length - 1, self._file_size))
+                self._request.setHeader('content-length', str(read_length))
+                start_cur = self._cursor
+                for i in range(read_length):
+                    if self._paused or self._stopped:
                         break
                     else:
                         data = self._fileObject.read(1)
                         self._request.write(data)
                         self._cursor += 1
-                self._sent_bytes = True
-            return defer.succeed(None)
 
-        def _write_reply():
+                log.info("[" + str(datetime.now()) + "] Wrote range %s-%s/%s, length: %s" % (start_cur, self._cursor - 1, self._file_size, self._cursor - start_cur))
+                self._sent_bytes = True
+
             if self._cursor == self._stop_pos + 1:
                 self.stopProducing()
                 return defer.succeed(None)
-            elif self._paused:
+            elif self._paused or self._stopped:
                 return defer.succeed(None)
             else:
-                d = task.deferLater(reactor, self._delay, _check_for_new_data)
-                d.addCallback(lambda _: _write_reply())
-                return d
+                self._deferred.addCallback(lambda _: task.deferLater(reactor, self._delay, _check_for_new_data))
+                return defer.succeed(None)
 
         log.info("[" + str(datetime.now()) + "] Resuming producer")
-
         self._paused = False
-        _write_reply()
+        self._deferred.addCallback(lambda _: _check_for_new_data())
         return defer.succeed(None)
 
     def stopProducing(self):
-        log.info("Stopping producer")
+        log.info("[" + str(datetime.now()) + "] Stopping producer")
+        self._stopped = True
         # self._fileObject.close()
-        self._request.finish()
+        self._deferred.addErrback(lambda err: err.trap(defer.CancelledError))
+        self._deferred.addErrback(lambda err: err.trap(error.ConnectionDone))
+        self._deferred.cancel()
+        # self._request.finish()
         self._request.unregisterProducer()
 
 
 class HostedLBRYFile(resource.Resource):
     def __init__(self, api):
         self._api = api
+        self._producer = None
         resource.Resource.__init__(self)
 
     def makeProducer(self, request, stream):
+        def _save_producer(producer):
+            self._producer = producer
+            return defer.succeed(None)
+
         range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-')
         start, stop = int(range_header[0]), range_header[1]
+        log.info("[" + str(datetime.now()) + "] GET range %s-%s" % (start, stop))
         path = os.path.join(self._api.download_directory, stream.file_name)
 
         d = stream.get_total_bytes()
-        d.addCallback(lambda size: request.registerProducer(LBRYFileStreamer(request, path, start, stop, size), streaming=True))
-
+        d.addCallback(lambda size: _save_producer(LBRYFileStreamer(request, path, start, stop, size)))
+        d.addCallback(lambda _: request.registerProducer(self._producer, streaming=True))
+        # request.notifyFinish().addCallback(lambda _: self._producer.stopProducing())
+        request.notifyFinish().addErrback(self._responseFailed, d)
         return d
 
     def render_GET(self, request):
@@ -156,7 +171,7 @@ class HostedLBRYFile(resource.Resource):
             if request.args['name'][0] != 'lbry' and request.args['name'][0] not in self._api.waiting_on.keys():
                 d = self._api._download_name(request.args['name'][0])
                 d.addCallback(lambda stream: self.makeProducer(request, stream))
-                request.notifyFinish().addErrback(self._responseFailed, d)
+
             elif request.args['name'][0] in self._api.waiting_on.keys():
                 request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0])
                 request.finish()

From 30f88f99b7d825b08b455446f4a3e788f9573e44 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Mon, 2 May 2016 15:58:40 -0400
Subject: [PATCH 21/38] use static.File

---
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 8a9aa0a05..152e9c353 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -12,7 +12,7 @@ from urllib import urlopen
 from datetime import datetime
 from appdirs import user_data_dir
 from twisted.web import server, static, resource
-from twisted.internet import defer, interfaces, error, reactor, task
+from twisted.internet import defer, interfaces, error, reactor, task, threads
 from twisted.python.failure import Failure
 from txjsonrpc.web import jsonrpc
 
@@ -76,17 +76,16 @@ class LBRYFileStreamer(object):
         self._stop_pos = size - 1 if stop == '' else int(stop)  #chrome and firefox send range requests for "0-"
         self._cursor = self._start_pos = int(start)
         self._file_size = size
+        self._depth = 0
 
         self._paused = self._sent_bytes = self._stopped = False
-        self._delay = 0.1
+        self._delay = 0.25
         self._deferred = defer.succeed(None)
 
         self._request.setResponseCode(206)
         self._request.setHeader('accept-ranges', 'bytes')
-        # self._request.setHeader('content-type', 'application/octet-stream')
         self._request.setHeader('content-type', self._content_type)
 
-        # self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._start_pos, self._stop_pos, self._file_size))
         self.resumeProducing()
 
     def pauseProducing(self):
@@ -96,6 +95,7 @@ class LBRYFileStreamer(object):
 
     def resumeProducing(self):
         def _check_for_new_data():
+            self._depth += 1
             self._fileObject.seek(self._start_pos, os.SEEK_END)
             readable_bytes = self._fileObject.tell()
             self._fileObject.seek(self._cursor)
@@ -115,7 +115,8 @@ class LBRYFileStreamer(object):
                         self._request.write(data)
                         self._cursor += 1
 
-                log.info("[" + str(datetime.now()) + "] Wrote range %s-%s/%s, length: %s" % (start_cur, self._cursor - 1, self._file_size, self._cursor - start_cur))
+                log.info("[" + str(datetime.now()) + "] Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s"  %
+                         (start_cur, self._cursor, self._file_size, self._cursor - start_cur, readable_bytes, self._depth))
                 self._sent_bytes = True
 
             if self._cursor == self._stop_pos + 1:
@@ -124,13 +125,12 @@ class LBRYFileStreamer(object):
             elif self._paused or self._stopped:
                 return defer.succeed(None)
             else:
-                self._deferred.addCallback(lambda _: task.deferLater(reactor, self._delay, _check_for_new_data))
+                self._deferred.addCallback(lambda _: threads.deferToThread(reactor.callLater, self._delay, _check_for_new_data))
                 return defer.succeed(None)
 
         log.info("[" + str(datetime.now()) + "] Resuming producer")
         self._paused = False
         self._deferred.addCallback(lambda _: _check_for_new_data())
-        return defer.succeed(None)
 
     def stopProducing(self):
         log.info("[" + str(datetime.now()) + "] Stopping producer")
@@ -141,6 +141,7 @@ class LBRYFileStreamer(object):
         self._deferred.cancel()
         # self._request.finish()
         self._request.unregisterProducer()
+        return defer.succeed(None)
 
 
 class HostedLBRYFile(resource.Resource):
@@ -170,7 +171,9 @@ class HostedLBRYFile(resource.Resource):
         if 'name' in request.args.keys():
             if request.args['name'][0] != 'lbry' and request.args['name'][0] not in self._api.waiting_on.keys():
                 d = self._api._download_name(request.args['name'][0])
-                d.addCallback(lambda stream: self.makeProducer(request, stream))
+                # d.addCallback(lambda stream: self.makeProducer(request, stream))
+                d.addCallback(lambda stream: static.File(os.path.join(self._api.download_directory,
+                                                                          stream.file_name)).render_GET(request))
 
             elif request.args['name'][0] in self._api.waiting_on.keys():
                 request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0])

From 019de08c64dcef6a8cc20f423200869f4612aa16 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Tue, 3 May 2016 23:13:31 -0400
Subject: [PATCH 22/38] add more transaction functions to daemon
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

also fix —wallet command line option
---
 lbrynet/lbrynet_daemon/LBRYDaemon.py       | 94 +++++++++++++++++++++-
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py |  8 +-
 2 files changed, 97 insertions(+), 5 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 17be5449e..0731dcb71 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -27,7 +27,7 @@ from lbrynet.core.PaymentRateManager import PaymentRateManager
 from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
 from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
 from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
-from lbrynet.core.Error import UnknownNameError
+from lbrynet.core.Error import UnknownNameError, InsufficientFundsError
 from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
 from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory
 from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier
@@ -1789,6 +1789,98 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d.addCallback(lambda address: self._render_response(address, OK_CODE))
         return d
 
+    def jsonrpc_send_amount_to_address(self, p):
+        """
+            Send credits to an address
+
+            Args:
+                amount: the amount to send
+                address: the address of the recipient
+            Returns:
+                True if payment successfully scheduled
+        """
+
+        if 'amount' in p.keys() and 'address' in p.keys():
+            amount = p['amount']
+            address = p['address']
+        else:
+            return server.failure
+
+        reserved_points = self.session.wallet.reserve_points(address, amount)
+        if reserved_points is None:
+            return defer.fail(InsufficientFundsError())
+        d = self.session.wallet.send_points_to_address(reserved_points, amount)
+        d.addCallback(lambda _: self._render_response(True, OK_CODE))
+        return d
+
+    def jsonrpc_get_best_blockhash(self):
+        """
+            Get hash of most recent block
+
+            Args:
+                None
+            Returns:
+                Hash of most recent block
+        """
+
+        d = self.session.wallet.get_best_blockhash()
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
+    def jsonrpc_get_block(self, p):
+        """
+            Get contents of a block
+
+            Args:
+                blockhash: hash of the block to look up
+            Returns:
+                requested block
+        """
+
+        if 'blockhash' in p.keys():
+            blockhash = p['blockhash']
+        else:
+            return server.failure
+
+        d = self.session.wallet.get_block(blockhash)
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
+    def jsonrpc_get_claims_for_tx(self, p):
+        """
+            Get claims for tx
+
+            Args:
+                txid: txid of a name claim transaction
+            Returns:
+                any claims contained in the requested tx
+        """
+
+        if 'txid' in p.keys():
+            txid = p['txid']
+        else:
+            return server.failure
+
+        d = self.session.wallet.get_claims_from_tx(txid)
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
+
+    def jsonrpc_get_nametrie(self):
+        """
+            Get the nametrie
+
+            Args:
+                None
+            Returns:
+                Name claim trie
+        """
+
+        d = self.session.wallet.get_nametrie()
+        d.addCallback(lambda r: [i for i in r if 'txid' in i.keys()])
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
     # def jsonrpc_update_name(self, metadata):
     #     def _disp(x):
     #         print x
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 152e9c353..86c28c557 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -337,8 +337,8 @@ class LBRYDaemonServer(object):
         d.addCallback(lambda _: _dl_ui())
         return d
 
-    def _setup_server(self, ui_ver):
-        self._api = LBRYDaemon(ui_ver, wallet_type=DEFAULT_WALLET)
+    def _setup_server(self, ui_ver, wallet):
+        self._api = LBRYDaemon(ui_ver, wallet_type=wallet)
         self.root = LBRYindex(self.ui_dir)
         self.root.putChild("css", static.File(os.path.join(self.ui_dir, "css")))
         self.root.putChild("font", static.File(os.path.join(self.ui_dir, "font")))
@@ -349,9 +349,9 @@ class LBRYDaemonServer(object):
         self.root.putChild(API_ADDRESS, self._api)
         return defer.succeed(True)
 
-    def start(self, branch="HEAD", user_specified=False):
+    def start(self, branch="HEAD", user_specified=False, wallet=DEFAULT_WALLET):
         d = self.setup(branch=branch, user_specified=user_specified)
-        d.addCallback(lambda v: self._setup_server(v))
+        d.addCallback(lambda v: self._setup_server(v, wallet))
         d.addCallback(lambda _: self._api.setup())
 
         return d

From c1d0f9cf1baed82ea13c6496c1a369d3b27328c9 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 4 May 2016 04:12:11 -0400
Subject: [PATCH 23/38] get rid of autofetcher in daemon

the purpose of the autofetcher was to automatically back up and host
published content, it is simpler to do this in a separate script that
uses existing daemon functions than to have it be built in
---
 lbrynet/lbrynet_daemon/LBRYDaemon.py     |  66 -------------
 lbrynet/lbrynet_daemon/LBRYDownloader.py | 116 -----------------------
 2 files changed, 182 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index 0731dcb71..ba58d333c 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -143,7 +143,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.wallet_type = wallet_type
         self.first_run = None
         self.log_file = LOG_FILENAME
-        self.fetcher = None
         self.current_db_revision = 1
         self.run_server = True
         self.session = None
@@ -680,11 +679,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         return defer.succeed(True)
 
-    def _setup_fetcher(self):
-        self.fetcher = FetcherDaemon(self.session, self.lbry_file_manager, self.lbry_file_metadata_manager,
-                                     self.session.wallet, self.sd_identifier, self.autofetcher_conf)
-        return defer.succeed(None)
-
     def _setup_data_directory(self):
         self.startup_status = STARTUP_STAGES[1]
         log.info("Loading databases...")
@@ -1288,48 +1282,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         else:
             return self._render_response(self.jsonrpc_help.__doc__, OK_CODE)
 
-    def jsonrpc_start_fetcher(self):
-        """
-        Start automatically downloading new name claims as they occur (off by default)
-
-        Args:
-            None
-        Returns:
-            confirmation message
-        """
-
-        self.fetcher.start()
-        log.info('[' + str(datetime.now()) + '] Start autofetcher')
-        # self._log_to_slack('[' + str(datetime.now()) + '] Start autofetcher')
-        return self._render_response("Started autofetching claims", OK_CODE)
-
-    def jsonrpc_stop_fetcher(self):
-        """
-        Stop automatically downloading new name claims as they occur
-
-        Args:
-            None
-        Returns:
-            confirmation message
-        """
-
-        self.fetcher.stop()
-        log.info('[' + str(datetime.now()) + '] Stop autofetcher')
-        return self._render_response("Stopped autofetching claims", OK_CODE)
-
-    def jsonrpc_fetcher_status(self):
-        """
-        Get fetcher status
-
-        Args:
-            None
-        Returns:
-            True/False
-        """
-
-        log.info("[" + str(datetime.now()) + "] Get fetcher status")
-        return self._render_response(self.fetcher.check_if_running(), OK_CODE)
-
     def jsonrpc_get_balance(self):
         """
         Get balance
@@ -1865,7 +1817,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d.addCallback(lambda r: self._render_response(r, OK_CODE))
         return d
 
-
     def jsonrpc_get_nametrie(self):
         """
             Get the nametrie
@@ -1901,23 +1852,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
     #
     #     return d
 
-    def jsonrpc_toggle_fetcher_verbose(self):
-        """
-        Toggle fetcher verbose mode
-
-        Args:
-            None
-        Returns:
-            Fetcher verbose status, bool
-        """
-
-        if self.fetcher.verbose:
-            self.fetcher.verbose = False
-        else:
-            self.fetcher.verbose = True
-
-        return self._render_response(self.fetcher.verbose, OK_CODE)
-
     def jsonrpc_check_for_new_version(self):
         """
         Checks local version against versions in __init__.py and version.py in the lbrynet and lbryum repos
diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py
index 048eb3e5b..8504c4202 100644
--- a/lbrynet/lbrynet_daemon/LBRYDownloader.py
+++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py
@@ -138,119 +138,3 @@ class GetStream(object):
         self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
         d.addCallback(lambda _: log.info("[" + str(datetime.now()) + "] Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
         d.addCallback(lambda _: self.downloader.start())
-
-
-class FetcherDaemon(object):
-    def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf,
-                 verbose=False):
-        self.autofetcher_conf = autofetcher_conf
-        self.max_key_fee = 0.0
-        self.sd_identifier = sd_identifier
-        self.wallet = wallet
-        self.session = session
-        self.lbry_file_manager = lbry_file_manager
-        self.lbry_metadata_manager = lbry_file_metadata_manager
-        self.seen = []
-        self.lastbestblock = None
-        self.search = None
-        self.first_run = True
-        self.is_running = False
-        self.verbose = verbose
-        self._get_autofetcher_conf()
-
-    def start(self):
-        if not self.is_running:
-            self.is_running = True
-            self.search = LoopingCall(self._looped_search)
-            self.search.start(1)
-            log.info("Starting autofetcher")
-        else:
-            log.info("Autofetcher is already running")
-
-    def stop(self):
-        if self.is_running:
-            self.search.stop()
-            self.is_running = False
-        else:
-            log.info("Autofetcher isn't running, there's nothing to stop")
-
-    def check_if_running(self):
-        if self.is_running:
-            msg = "Autofetcher is running\n"
-            msg += "Last block hash: " + str(self.lastbestblock)
-        else:
-            msg = "Autofetcher is not running"
-        return msg
-
-    def _get_names(self):
-        d = self.wallet.get_best_blockhash()
-        d.addCallback(lambda blockhash: get_new_streams(blockhash) if blockhash != self.lastbestblock else [])
-
-        def get_new_streams(blockhash):
-            self.lastbestblock = blockhash
-            d = self.wallet.get_block(blockhash)
-            d.addCallback(lambda block: get_new_streams_in_txes(block['tx'], blockhash))
-            return d
-
-        def get_new_streams_in_txes(txids, blockhash):
-            ds = []
-            for t in txids:
-                d = self.wallet.get_claims_from_tx(t)
-                d.addCallback(get_new_streams_in_tx, t, blockhash)
-                ds.append(d)
-            d = defer.DeferredList(ds, consumeErrors=True)
-            d.addCallback(lambda result: [r[1] for r in result if r[0]])
-            d.addCallback(lambda stream_lists: [stream for streams in stream_lists for stream in streams])
-            return d
-
-        def get_new_streams_in_tx(claims, t, blockhash):
-            rtn = []
-            if claims:
-                for claim in claims:
-                    if claim not in self.seen:
-                        msg = "[" + str(datetime.now()) + "] New claim | lbry://" + str(claim['name']) + \
-                              " | stream hash: " + str(json.loads(claim['value'])['stream_hash'])
-                        log.info(msg)
-                        if self.verbose:
-                            print msg
-                        rtn.append((claim['name'], t))
-                        self.seen.append(claim)
-            else:
-                if self.verbose:
-                    print "[" + str(datetime.now()) + "] No claims in block", blockhash
-            return rtn
-
-        d.addCallback(lambda streams: defer.DeferredList(
-            [self.wallet.get_stream_info_from_txid(name, t) for name, t in streams]))
-        return d
-
-    def _download_claims(self, claims):
-        if claims:
-            for claim in claims:
-                stream = GetStream(self.sd_identifier, self.session, self.wallet, self.lbry_file_manager,
-                                   self.max_key_fee, pay_key=False)
-                stream.start(claim[1])
-
-        return defer.succeed(None)
-
-    def _looped_search(self):
-        d = self._get_names()
-        d.addCallback(self._download_claims)
-        return d
-
-    def _get_autofetcher_conf(self):
-        settings = {"maxkey": "0.0"}
-        if os.path.exists(self.autofetcher_conf):
-            conf = open(self.autofetcher_conf)
-            for l in conf:
-                if l.startswith("maxkey="):
-                    settings["maxkey"] = float(l[7:].rstrip('\n'))
-            conf.close()
-        else:
-            conf = open(self.autofetcher_conf, "w")
-            conf.write("maxkey=10.0")
-            conf.close()
-            settings["maxkey"] = 10.0
-            log.info("No autofetcher conf file found, making one with max key fee of 10.0")
-
-        self.max_key_fee = settings["maxkey"]

From 461c2f9055e0551547d7a4cb9a09acfeb1892511 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 4 May 2016 05:20:38 -0400
Subject: [PATCH 24/38] add file download statuses for get_lbry_file

also clean up remaining fetcher stuff
---
 lbrynet/lbrynet_daemon/LBRYDaemon.py     | 52 +++++++++++++++++-------
 lbrynet/lbrynet_daemon/LBRYDownloader.py | 21 ++++++++++
 2 files changed, 59 insertions(+), 14 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index ba58d333c..d87f6282a 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -31,7 +31,7 @@ from lbrynet.core.Error import UnknownNameError, InsufficientFundsError
 from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
 from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory
 from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier
-from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream, FetcherDaemon
+from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream
 from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher
 from lbrynet.core.utils import generate_id
 from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
@@ -147,6 +147,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.run_server = True
         self.session = None
         self.waiting_on = {}
+        self.streams = {}
         self.known_dht_nodes = KNOWN_DHT_NODES
         self.platform_info = {
             "processor": platform.processor(),
@@ -408,7 +409,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d.addCallback(lambda _: self._setup_lbry_file_opener())
         d.addCallback(lambda _: self._setup_query_handlers())
         d.addCallback(lambda _: self._setup_server())
-        d.addCallback(lambda _: self._setup_fetcher())
         d.addCallback(lambda _: _log_starting_vals())
         d.addCallback(lambda _: _announce_startup())
         d.callback(None)
@@ -891,9 +891,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
     def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None):
         """
-        Add a lbry file to the file manager, start the download, and return the new lbry file
-        if it already exists in the file manager, return the existing lbry file
+        Add a lbry file to the file manager, start the download, and return the new lbry file.
+        If it already exists in the file manager, return the existing lbry file
         """
+
         if not download_directory:
             download_directory = self.download_directory
         elif not os.path.isdir(download_directory):
@@ -919,12 +920,12 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             return f
 
         def _get_stream(stream_info):
-            stream = GetStream(self.sd_identifier, self.session, self.session.wallet,
-                               self.lbry_file_manager,
-                               max_key_fee=self.max_key_fee, data_rate=self.data_rate, timeout=timeout,
-                               download_directory=download_directory)
-            d = stream.start(stream_info, name)
-            d.addCallback(lambda _: stream.downloader)
+            self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet,
+                                           self.lbry_file_manager, max_key_fee=self.max_key_fee,
+                                           data_rate=self.data_rate, timeout=timeout,
+                                           download_directory=download_directory)
+            d = self.streams[name].start(stream_info, name)
+            d.addCallback(lambda _: self.streams[name].downloader)
 
             return d
 
@@ -1054,16 +1055,39 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             return f
 
         def _get_json_for_return(f):
-            if f:
+            def _generate_reply(size):
                 if f.key:
                     key = binascii.b2a_hex(f.key)
                 else:
                     key = None
+
+                if os.path.isfile(os.path.join(self.download_directory, f.file_name)):
+                    written_file = file(os.path.join(self.download_directory, f.file_name))
+                    written_file.seek(0, os.SEEK_END)
+                    written_bytes = written_file.tell()
+                    written_file.close()
+                else:
+                    written_bytes = False
+
+                if search_by == "name":
+                    if val in self.streams.keys():
+                        status = self.streams[val].code
+                    else:
+                        status = [False, False]
+                else:
+                    status = [False, False]
+
                 t = {'completed': f.completed, 'file_name': f.file_name, 'key': key,
-                     'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
-                     'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
-                     'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash}
+                        'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
+                        'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
+                        'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash, 'total_bytes': size,
+                        'written_bytes': written_bytes, 'code': status[0], 'message': status[1]}
                 return t
+
+            if f:
+                d = f.get_total_bytes()
+                d.addCallback(_generate_reply)
+                return d
             else:
                 return False
 
diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py
index 8504c4202..9ace4c9be 100644
--- a/lbrynet/lbrynet_daemon/LBRYDownloader.py
+++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py
@@ -14,6 +14,19 @@ from lbrynet.core.StreamDescriptor import download_sd_blob
 from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
 from lbrynet.conf import DEFAULT_TIMEOUT
 
+INITIALIZING_CODE = 'initializing'
+DOWNLOAD_METADATA_CODE = 'downloading_metadata'
+DOWNLOAD_TIMEOUT_CODE = 'timeout'
+DOWNLOAD_RUNNING_CODE = 'running'
+DOWNLOAD_STOPPED_CODE = 'stopped'
+STREAM_STAGES = [
+                    (INITIALIZING_CODE, 'Initializing...'),
+                    (DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
+                    (DOWNLOAD_RUNNING_CODE, 'Started stream'),
+                    (DOWNLOAD_STOPPED_CODE, 'Paused stream'),
+                    (DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
+                ]
+
 if sys.platform != "darwin":
     log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
 else:
@@ -56,6 +69,7 @@ class GetStream(object):
         self.downloader = None
         self.finished = defer.Deferred()
         self.checker = LoopingCall(self.check_status)
+        self.code = STREAM_STAGES[0]
 
     def check_status(self):
         self.timeout_counter += 1
@@ -68,6 +82,7 @@ class GetStream(object):
             log.info("Timeout downloading lbry://" + self.resolved_name + ", " + str(self.stream_info))
             self.checker.stop()
             self.d.cancel()
+            self.code = STREAM_STAGES[4]
             self.finished.callback(False)
 
     def start(self, stream_info, name):
@@ -104,10 +119,16 @@ class GetStream(object):
         def _cause_timeout():
             self.timeout_counter = self.timeout * 2
 
+        def _set_status(x, status):
+            self.code = next(s for s in STREAM_STAGES if s[0] == status)
+            return x
+
         self.checker.start(1)
 
+        self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
         self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
         self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
+        self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
         self.d.addCallback(lambda metadata: (
         next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)),
         metadata))

From 509b8f3a29f38d6c7e8e143a395f6535b46443cf Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 4 May 2016 21:25:46 -0400
Subject: [PATCH 25/38] add file_name and stream_info fields to get()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

-file_name is the name of the file in the downloads folder

-stream_info is a dict of the metadata in a name claim, it can be used
to download streams where the claim hasn’t yet been added to the
nametrie
---
 lbrynet/lbryfilemanager/LBRYFileDownloader.py |  5 +--
 lbrynet/lbryfilemanager/LBRYFileManager.py    | 10 +++---
 lbrynet/lbrynet_daemon/LBRYDaemon.py          | 31 ++++++++++++++-----
 lbrynet/lbrynet_daemon/LBRYDownloader.py      |  6 ++--
 4 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/lbrynet/lbryfilemanager/LBRYFileDownloader.py b/lbrynet/lbryfilemanager/LBRYFileDownloader.py
index 8bb1ada42..39f1f4899 100644
--- a/lbrynet/lbryfilemanager/LBRYFileDownloader.py
+++ b/lbrynet/lbryfilemanager/LBRYFileDownloader.py
@@ -136,7 +136,7 @@ class ManagedLBRYFileDownloaderFactory(object):
     def can_download(self, sd_validator):
         return True
 
-    def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None):
+    def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None, file_name=None):
         data_rate = options[0]
         upload_allowed = options[1]
 
@@ -155,7 +155,8 @@ class ManagedLBRYFileDownloaderFactory(object):
                                                                                payment_rate_manager,
                                                                                data_rate,
                                                                                upload_allowed,
-                                                                               download_directory=download_directory))
+                                                                               download_directory=download_directory,
+                                                                               file_name=file_name))
         return d
 
     @staticmethod
diff --git a/lbrynet/lbryfilemanager/LBRYFileManager.py b/lbrynet/lbryfilemanager/LBRYFileManager.py
index 144425c6f..d805fb38d 100644
--- a/lbrynet/lbryfilemanager/LBRYFileManager.py
+++ b/lbrynet/lbryfilemanager/LBRYFileManager.py
@@ -94,7 +94,7 @@ class LBRYFileManager(object):
         return d
 
     def start_lbry_file(self, rowid, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
-                                                                                        download_directory=None):
+                                                                        download_directory=None, file_name=None):
         if not download_directory:
             download_directory = self.download_directory
         payment_rate_manager.min_blob_data_payment_rate = blob_data_rate
@@ -105,16 +105,18 @@ class LBRYFileManager(object):
                                                          self.stream_info_manager, self,
                                                          payment_rate_manager, self.session.wallet,
                                                          download_directory,
-                                                         upload_allowed)
+                                                         upload_allowed,
+                                                         file_name=file_name)
         self.lbry_files.append(lbry_file_downloader)
         d = lbry_file_downloader.set_stream_info()
         d.addCallback(lambda _: lbry_file_downloader)
         return d
 
-    def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True, download_directory=None):
+    def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
+                                                                download_directory=None, file_name=None):
         d = self._save_lbry_file(stream_hash, blob_data_rate)
         d.addCallback(lambda rowid: self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
-                                                         blob_data_rate, upload_allowed, download_directory))
+                                                         blob_data_rate, upload_allowed, download_directory, file_name))
         return d
 
     def delete_lbry_file(self, lbry_file):
diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index d87f6282a..c30a1a539 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -889,7 +889,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory)
         return defer.succeed(True)
 
-    def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None):
+    def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None, stream_info=None):
         """
         Add a lbry file to the file manager, start the download, and return the new lbry file.
         If it already exists in the file manager, return the existing lbry file
@@ -923,17 +923,21 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet,
                                            self.lbry_file_manager, max_key_fee=self.max_key_fee,
                                            data_rate=self.data_rate, timeout=timeout,
-                                           download_directory=download_directory)
+                                           download_directory=download_directory, file_name=file_name)
             d = self.streams[name].start(stream_info, name)
             d.addCallback(lambda _: self.streams[name].downloader)
 
             return d
 
-        self.waiting_on[name] = True
-        d = self._resolve_name(name)
+        if not stream_info:
+            self.waiting_on[name] = True
+            d = self._resolve_name(name)
+        else:
+            d = defer.succeed(stream_info)
         d.addCallback(_setup_stream)
         d.addCallback(lambda (stream_info, lbry_file): _get_stream(stream_info) if not lbry_file else _disp_file(lbry_file))
-        d.addCallback(_remove_from_wait)
+        if not stream_info:
+            d.addCallback(_remove_from_wait)
 
         return d
 
@@ -1418,7 +1422,9 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         Args:
             'name': name to download, string
-            optional 'download_directory': path to directory where file will be saved, string
+            'download_directory': optional, path to directory where file will be saved, string
+            'file_name': optional, a user specified name for the downloaded file
+            'stream_info': optional, specified stream info overrides name
         Returns:
             'stream_hash': hex string
             'path': path of download
@@ -1434,10 +1440,21 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         else:
             download_directory = p['download_directory']
 
+        if 'file_name' in p.keys():
+            file_name = p['file_name']
+        else:
+            file_name = None
+
+        if 'stream_info' in p.keys():
+            stream_info = p['stream_info']
+        else:
+            stream_info = None
+
         if 'name' in p.keys():
             name = p['name']
             if p['name'] not in self.waiting_on.keys():
-                d = self._download_name(name=name, timeout=timeout, download_directory=download_directory)
+                d = self._download_name(name=name, timeout=timeout, download_directory=download_directory,
+                                                                stream_info=stream_info, file_name=file_name)
                 d.addCallback(lambda l: {'stream_hash': l.sd_hash, 'path': os.path.join(self.download_directory, l.file_name)})
                 d.addCallback(lambda message: self._render_response(message, OK_CODE))
             else:
diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py
index 9ace4c9be..002bb179e 100644
--- a/lbrynet/lbrynet_daemon/LBRYDownloader.py
+++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py
@@ -44,7 +44,7 @@ log.setLevel(logging.INFO)
 
 class GetStream(object):
     def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee, pay_key=True, data_rate=0.5,
-                                                                    timeout=DEFAULT_TIMEOUT, download_directory=None):
+                                                    timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None):
         self.wallet = wallet
         self.resolved_name = None
         self.description = None
@@ -53,6 +53,7 @@ class GetStream(object):
         self.data_rate = data_rate
         self.pay_key = pay_key
         self.name = None
+        self.file_name = file_name
         self.session = session
         self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
         self.lbry_file_manager = lbry_file_manager
@@ -135,7 +136,8 @@ class GetStream(object):
         self.d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata,
                                                                                [self.data_rate, True],
                                                                                self.payment_rate_manager,
-                                                                               download_directory=self.download_directory))
+                                                                               download_directory=self.download_directory,
+                                                                               file_name=self.file_name))
         self.d.addCallbacks(self._start_download, lambda _: _cause_timeout())
         self.d.callback(None)
 

From 4cab6726b07401cc182d5a32a1b0a0148f295691 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 4 May 2016 23:27:40 -0400
Subject: [PATCH 26/38] log to file from publisher

also fix sd_hash exception that could happen
---
 lbrynet/lbryfilemanager/LBRYFileDownloader.py |  3 ++-
 lbrynet/lbrynet_daemon/LBRYPublisher.py       | 21 ++++++++++++++++---
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/lbrynet/lbryfilemanager/LBRYFileDownloader.py b/lbrynet/lbryfilemanager/LBRYFileDownloader.py
index 39f1f4899..fa3ce530e 100644
--- a/lbrynet/lbryfilemanager/LBRYFileDownloader.py
+++ b/lbrynet/lbryfilemanager/LBRYFileDownloader.py
@@ -33,7 +33,8 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
         d = self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash)
 
         def _save_sd_hash(sd_hash):
-            self.sd_hash = sd_hash[0]
+            if len(sd_hash):
+                self.sd_hash = sd_hash[0]
             return defer.succeed(None)
 
         d.addCallback(_save_sd_hash)
diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py
index eeb8bbed3..e68a06498 100644
--- a/lbrynet/lbrynet_daemon/LBRYPublisher.py
+++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py
@@ -1,15 +1,30 @@
+import logging
+import os
+import sys
+
+from appdirs import user_data_dir
+from datetime import datetime
+
 from lbrynet.core.Error import InsufficientFundsError
 from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
 from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
 from lbrynet.core.PaymentRateManager import PaymentRateManager
 from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
 from twisted.internet import threads, defer
-import os
-import logging
-from datetime import datetime
 
+if sys.platform != "darwin":
+    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
+else:
+    log_dir = user_data_dir("LBRY")
 
+if not os.path.isdir(log_dir):
+    os.mkdir(log_dir)
+
+LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log')
 log = logging.getLogger(__name__)
+handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, backupCount=5)
+log.addHandler(handler)
+log.setLevel(logging.INFO)
 
 
 class Publisher(object):

From d6983a25b01d8bb43c725de56f25314532381541 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Wed, 4 May 2016 23:40:05 -0400
Subject: [PATCH 27/38] fix metadata problem in downloader and publisher

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py     |  8 ++++---
 lbrynet/lbrynet_daemon/LBRYDownloader.py | 29 +++++++++++-------------
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index c30a1a539..a66a1ee25 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -905,9 +905,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             return r
 
         def _setup_stream(stream_info):
-            stream_hash = stream_info['stream_hash']
-            if isinstance(stream_hash, dict):
-                stream_hash = stream_hash['sd_hash']
+            if 'sources' in stream_info.keys():
+                stream_hash = stream_info['sources']['lbry_sd_hash']
+            else:
+                stream_hash = stream_info['stream_hash']
+
             d = self._get_lbry_file_by_sd_hash(stream_hash)
             def _add_results(l):
                 return defer.succeed((stream_info, l))
diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py
index 002bb179e..d62bdc3b1 100644
--- a/lbrynet/lbrynet_daemon/LBRYDownloader.py
+++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py
@@ -90,25 +90,22 @@ class GetStream(object):
         self.resolved_name = name
         self.stream_info = stream_info
         if 'stream_hash' in self.stream_info.keys():
-            self.description = self.stream_info['description']
-            if 'key_fee' in self.stream_info.keys():
-                self.key_fee = float(self.stream_info['key_fee'])
-                if 'key_fee_address' in self.stream_info.keys():
-                    self.key_fee_address = self.stream_info['key_fee_address']
-                else:
-                    self.key_fee_address = None
-            else:
-                self.key_fee = None
-                self.key_fee_address = None
-
             self.stream_hash = self.stream_info['stream_hash']
-            if isinstance(self.stream_hash, dict):
-                self.stream_hash = self.stream_hash['sd_hash']
-
+        elif 'sources' in self.stream_info.keys():
+            self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
         else:
-            log.error("InvalidStreamInfoError in autofetcher: ", stream_info)
             raise InvalidStreamInfoError(self.stream_info)
-
+        if 'description' in self.stream_info.keys():
+            self.description = self.stream_info['description']
+        if 'key_fee' in self.stream_info.keys():
+            self.key_fee = float(self.stream_info['key_fee'])
+            if 'key_fee_address' in self.stream_info.keys():
+                self.key_fee_address = self.stream_info['key_fee_address']
+            else:
+                self.key_fee_address = None
+        else:
+            self.key_fee = None
+            self.key_fee_address = None
         if self.key_fee > self.max_key_fee:
             if self.pay_key:
                 log.info("Key fee (" + str(self.key_fee) + ") above limit of " + str(

From 29d6d824fc38c3ed686c73de1a9e7b37df3f5f7c Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Thu, 5 May 2016 19:16:36 -0400
Subject: [PATCH 28/38] write cryptsd files to data dir instead of cwd

---
 lbrynet/lbryfilemanager/LBRYFileCreator.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lbrynet/lbryfilemanager/LBRYFileCreator.py b/lbrynet/lbryfilemanager/LBRYFileCreator.py
index 1b0fcb5b0..770d1ba6b 100644
--- a/lbrynet/lbryfilemanager/LBRYFileCreator.py
+++ b/lbrynet/lbryfilemanager/LBRYFileCreator.py
@@ -131,7 +131,8 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
 
     def make_stream_desc_file(stream_hash):
         log.debug("creating the stream descriptor file")
-        descriptor_writer = PlainStreamDescriptorWriter(file_name + conf.CRYPTSD_FILE_EXTENSION)
+        descriptor_file_path = os.path.join(session.db_dir, file_name + conf.CRYPTSD_FILE_EXTENSION)
+        descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path)
 
         d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True)
 

From b017b6374521307b3f734a1a968b97f997e8d451 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Thu, 5 May 2016 21:45:25 -0400
Subject: [PATCH 29/38] status code for previously downloaded file

also fix return for files downloaded manually with stream_info
---
 lbrynet/lbrynet_daemon/LBRYDaemon.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index a66a1ee25..aacc4a70c 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -1078,6 +1078,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                 if search_by == "name":
                     if val in self.streams.keys():
                         status = self.streams[val].code
+                    elif f in self.lbry_file_manager.lbry_files:
+                        # if f.stopped:
+                        #     status = STREAM_STAGES[3]
+                        # else:
+                        status = STREAM_STAGES[2]
                     else:
                         status = [False, False]
                 else:
@@ -1449,6 +1454,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         if 'stream_info' in p.keys():
             stream_info = p['stream_info']
+            if 'sources' in stream_info.keys():
+                sd_hash = stream_info['sources']['lbry_sd_hash']
+            else:
+                sd_hash = stream_info['stream_hash']
         else:
             stream_info = None
 
@@ -1457,7 +1466,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             if p['name'] not in self.waiting_on.keys():
                 d = self._download_name(name=name, timeout=timeout, download_directory=download_directory,
                                                                 stream_info=stream_info, file_name=file_name)
-                d.addCallback(lambda l: {'stream_hash': l.sd_hash, 'path': os.path.join(self.download_directory, l.file_name)})
+                d.addCallback(lambda l: {'stream_hash': sd_hash,
+                                         'path': os.path.join(self.download_directory, l.file_name)}
+                                         if stream_info else
+                                         {'stream_hash': l.sd_hash,
+                                         'path': os.path.join(self.download_directory, l.file_name)})
                 d.addCallback(lambda message: self._render_response(message, OK_CODE))
             else:
                 d = server.failure

From e32853744a6b35a97cde822ec917e047554752a5 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Thu, 5 May 2016 22:26:25 -0400
Subject: [PATCH 30/38] better loading message

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py | 32 ++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index aacc4a70c..caa1a6873 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -90,7 +90,7 @@ DOWNLOAD_STOPPED_CODE = 'stopped'
 STREAM_STAGES = [
                     (INITIALIZING_CODE, 'Initializing...'),
                     (DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
-                    (DOWNLOAD_RUNNING_CODE, 'Started stream'),
+                    (DOWNLOAD_RUNNING_CODE, 'Started %s, got %s/%s blobs, stream status: %s'),
                     (DOWNLOAD_STOPPED_CODE, 'Paused stream'),
                     (DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
                 ]
@@ -1061,6 +1061,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             return f
 
         def _get_json_for_return(f):
+            def _get_file_status(file_status):
+                message = STREAM_STAGES[2][1] % (file_status.name, file_status.num_completed, file_status.num_known, file_status.running_status)
+                return defer.succeed(message)
+
             def _generate_reply(size):
                 if f.key:
                     key = binascii.b2a_hex(f.key)
@@ -1088,12 +1092,26 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                 else:
                     status = [False, False]
 
-                t = {'completed': f.completed, 'file_name': f.file_name, 'key': key,
-                        'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
-                        'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
-                        'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash, 'total_bytes': size,
-                        'written_bytes': written_bytes, 'code': status[0], 'message': status[1]}
-                return t
+                if status[0] == DOWNLOAD_RUNNING_CODE:
+                    d = f.status()
+                    d.addCallback(_get_file_status)
+                    d.addCallback(lambda message: {'completed': f.completed, 'file_name': f.file_name, 'key': key,
+                                                   'points_paid': f.points_paid, 'stopped': f.stopped,
+                                                   'stream_hash': f.stream_hash,
+                                                   'stream_name': f.stream_name,
+                                                   'suggested_file_name': f.suggested_file_name,
+                                                   'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash,
+                                                   'total_bytes': size,
+                                                   'written_bytes': written_bytes, 'code': status[0],
+                                                   'message': message})
+                else:
+                    d = defer.succeed({'completed': f.completed, 'file_name': f.file_name, 'key': key,
+                                       'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
+                                       'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
+                                       'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash, 'total_bytes': size,
+                                       'written_bytes': written_bytes, 'code': status[0], 'message': status[1]})
+
+                return d
 
             if f:
                 d = f.get_total_bytes()

From f3d3a0e57f0e98bb1ec08752a58bc1732fd907c2 Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 6 May 2016 14:31:44 -0400
Subject: [PATCH 31/38] Don't show 0 while downloading headers from amazon, fix
 uri handler

---
 lbrynet/lbrynet_daemon/LBRYDaemon.py | 9 ++++++---
 packaging/ubuntu/lbry                | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py
index caa1a6873..f16f5562a 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemon.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py
@@ -1181,9 +1181,12 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             r['message'] = self.connection_problem[1]
             r['is_lagging'] = True
         elif self.startup_status[0] == LOADING_WALLET_CODE:
-            r['message'] = r['message'] % (str(self.session.wallet.blocks_behind_alert) + " blocks behind")
-            r['progress'] = self.session.wallet.catchup_progress
-
+            if self.session.wallet.blocks_behind_alert != 0:
+                r['message'] = r['message'] % (str(self.session.wallet.blocks_behind_alert) + " blocks behind")
+                r['progress'] = self.session.wallet.catchup_progress
+            else:
+                r['message'] = "Catching up with the blockchain"
+                r['progress'] = 0
         log.info("[" + str(datetime.now()) + "] daemon status: " + str(r))
         return self._render_response(r, OK_CODE)
 
diff --git a/packaging/ubuntu/lbry b/packaging/ubuntu/lbry
index a4aed0ff8..37859de4d 100755
--- a/packaging/ubuntu/lbry
+++ b/packaging/ubuntu/lbry
@@ -26,7 +26,7 @@ ARG=${1:-}
 if [ -z "$ARG" ]; then
   URL=""
 else
-  URL="view?name=$(urlencode "$(echo "$ARG" | cut -c 8-)")"
+  URL="?watch=$(urlencode "$(echo "$ARG" | cut -c 8-)")"
 fi
 
 /usr/bin/xdg-open "http://localhost:5279/$URL"

From da315204658947c46e2f2436957e8ce9779d06e8 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <alex@grin.io>
Date: Fri, 6 May 2016 15:34:41 -0400
Subject: [PATCH 32/38] use github api instead of git

---
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 86c28c557..92161aaa9 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -273,13 +273,11 @@ class LBRYDaemonServer(object):
                 return defer.succeed("user-specified")
             else:
                 log.info("User specified UI directory doesn't exist, using " + branch)
-        elif branch == "HEAD":
-            log.info("Using UI branch: " + branch)
-            self._gitcmd = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep %s | cut -f 1" % branch
-            self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip"
         else:
             log.info("Using UI branch: " + branch)
-            self._gitcmd = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep refs/heads/%s | cut -f 1" % branch
+            if branch == "HEAD":
+              branch = "master"
+            self._git_url = "https://api.github.com/repos/lbryio/lbry.io/git/refs/heads/%s" % branch
             self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % branch
 
         d = self._up_to_date()
@@ -288,8 +286,9 @@ class LBRYDaemonServer(object):
 
     def _up_to_date(self):
         def _get_git_info():
-            r = subprocess.check_output(self._gitcmd, shell=True)
-            return defer.succeed(r)
+            response = urlopen(self._git_url)
+            data = json.loads(response.read())
+            return defer.succeed(data['object']['sha'])
 
         def _set_git(version):
             self.git_version = version

From da9b1b3f5873e30b8934b3aeaa08da80c305fead Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 6 May 2016 15:42:21 -0400
Subject: [PATCH 33/38] merge conflicts

---
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 92161aaa9..086786d83 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -1,5 +1,4 @@
 import logging
-import subprocess
 import os
 import shutil
 import json
@@ -273,11 +272,13 @@ class LBRYDaemonServer(object):
                 return defer.succeed("user-specified")
             else:
                 log.info("User specified UI directory doesn't exist, using " + branch)
+        elif branch == "HEAD":
+            log.info("Using UI branch: " + branch)
+            self._git_url = "https://api.github.com/repos/lbryio/lbry-web-ui/git/refs/heads/master"
+            self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip"
         else:
             log.info("Using UI branch: " + branch)
-            if branch == "HEAD":
-              branch = "master"
-            self._git_url = "https://api.github.com/repos/lbryio/lbry.io/git/refs/heads/%s" % branch
+            self._git_url = "https://api.github.com/repos/lbryio/lbry-web-ui/git/refs/heads/%s" % branch
             self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % branch
 
         d = self._up_to_date()

From b4500a29149e8a66242c3d183f5aeb9ac4c214da Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 6 May 2016 15:51:59 -0400
Subject: [PATCH 34/38] get rid of git HEAD stuff

---
 lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
index 086786d83..85494d21c 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py
@@ -261,7 +261,7 @@ class LBRYDaemonServer(object):
                 self.loaded_git_version = None
                 self.loaded_branch = None
 
-    def setup(self, branch="HEAD", user_specified=None):
+    def setup(self, branch="master", user_specified=None):
         self.branch = branch
         if user_specified:
             if os.path.isdir(user_specified):
@@ -272,10 +272,6 @@ class LBRYDaemonServer(object):
                 return defer.succeed("user-specified")
             else:
                 log.info("User specified UI directory doesn't exist, using " + branch)
-        elif branch == "HEAD":
-            log.info("Using UI branch: " + branch)
-            self._git_url = "https://api.github.com/repos/lbryio/lbry-web-ui/git/refs/heads/master"
-            self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip"
         else:
             log.info("Using UI branch: " + branch)
             self._git_url = "https://api.github.com/repos/lbryio/lbry-web-ui/git/refs/heads/%s" % branch
@@ -349,7 +345,7 @@ class LBRYDaemonServer(object):
         self.root.putChild(API_ADDRESS, self._api)
         return defer.succeed(True)
 
-    def start(self, branch="HEAD", user_specified=False, wallet=DEFAULT_WALLET):
+    def start(self, branch="master", user_specified=False, wallet=DEFAULT_WALLET):
         d = self.setup(branch=branch, user_specified=user_specified)
         d.addCallback(lambda v: self._setup_server(v, wallet))
         d.addCallback(lambda _: self._api.setup())

From 10b016312dae7915ee87fb3b6b1834533e534c0d Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 6 May 2016 16:16:09 -0400
Subject: [PATCH 35/38] get rid of Xcode tools and git requirement

---
 lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
index 506dc23e1..c9e9acf75 100644
--- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
+++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py
@@ -5,7 +5,6 @@ import os
 import webbrowser
 import sys
 import socket
-import subprocess
 import platform
 
 from appdirs import user_data_dir
@@ -44,17 +43,6 @@ def test_internet_connection():
         return False
 
 
-def prompt_for_xcode_if_needed():
-    t = subprocess.check_output("git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep HEAD | cut -f 1", shell=True)
-    if not t:
-        if platform.system().lower() != "darwin":
-            print "Please install git"
-            sys.exit(0)
-        else:
-            print "You should have been alerted to install xcode command line tools, please do so and then start lbry"
-            sys.exit(0)
-
-
 def stop():
     def _disp_shutdown():
         print "Shutting down lbrynet-daemon from command line"
@@ -80,8 +68,8 @@ def start():
                         help="path to custom UI folder",
                         default=None)
     parser.add_argument("--branch",
-                        help="Branch of lbry-web-ui repo to use, defaults on HEAD",
-                        default="HEAD")
+                        help="Branch of lbry-web-ui repo to use, defaults on master",
+                        default="master")
     parser.add_argument('--no-launch', dest='launchui', action="store_false")
     parser.add_argument('--log-to-console', dest='logtoconsole', action="store_true")
     parser.add_argument('--quiet', dest='quiet', action="store_true")
@@ -104,8 +92,6 @@ def start():
     except:
         pass
 
-    prompt_for_xcode_if_needed()
-
     log.info("Starting lbrynet-daemon from command line")
 
     if not args.logtoconsole and not args.quiet:

From bb2fb92b0eb8cc31baa2fde20042e3c9690c1ddb Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 6 May 2016 16:59:23 -0400
Subject: [PATCH 36/38] have uri handler fail silently to prevent misleading
 error

otherwise first run produced an error while it was downloading the
headers if the app was started by going to lbry://lbry
---
 lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py b/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py
index ed63c6f03..9bdc20ec8 100644
--- a/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py
+++ b/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py
@@ -36,14 +36,16 @@ class LBRYURIHandler(object):
             elif status:
                 return True
             else:
-                raise Timeout("LBRY daemon is running, but connection timed out")
+                sys.exit(0)
+                # raise Timeout("LBRY daemon is running, but connection timed out")
         except:
             if self.start_timeout < 30:
                 sleep(1)
                 self.start_timeout += 1
                 self.check_status()
             else:
-                raise Timeout("Timed out trying to start LBRY daemon")
+                sys.exit(0)
+                # raise Timeout("Timed out trying to start LBRY daemon")
 
     def handle_osx(self, lbry_name):
         lbry_process = [d for d in subprocess.Popen(['ps','aux'], stdout=subprocess.PIPE).stdout.readlines()
@@ -72,6 +74,7 @@ class LBRYURIHandler(object):
             if not is_running:
                 sys.exit(0)
         except:
+            #start lbrynet-daemon
             sys.exit(0)
 
         if lbry_name == "lbry":

From 8bd6fb5b4e30f61def0b51aeb2d88e8cfa3df3ee Mon Sep 17 00:00:00 2001
From: Jack <jack@robisonservice.com>
Date: Fri, 6 May 2016 17:20:03 -0400
Subject: [PATCH 37/38] clean up uri handler
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

clean up work arounds from when ui didn’t do loading nicely
---
 lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py | 59 ++++---------------
 1 file changed, 11 insertions(+), 48 deletions(-)

diff --git a/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py b/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py
index 9bdc20ec8..f6990cfea 100644
--- a/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py
+++ b/lbrynet/lbrynet_daemon/Apps/LBRYURIHandler.py
@@ -11,73 +11,36 @@ API_CONNECTION_STRING = "http://localhost:5279/lbryapi"
 UI_ADDRESS = "http://localhost:5279"
 
 
-class Timeout(Exception):
-    def __init__(self, value):
-        self.parameter = value
-
-    def __str__(self):
-        return repr(self.parameter)
-
-
 class LBRYURIHandler(object):
     def __init__(self):
         self.started_daemon = False
-        self.start_timeout = 0
         self.daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING)
 
-    def check_status(self):
-        status = None
-        try:
-            status = self.daemon.is_running()
-            if self.start_timeout < 30 and not status:
-                sleep(1)
-                self.start_timeout += 1
-                self.check_status()
-            elif status:
-                return True
-            else:
-                sys.exit(0)
-                # raise Timeout("LBRY daemon is running, but connection timed out")
-        except:
-            if self.start_timeout < 30:
-                sleep(1)
-                self.start_timeout += 1
-                self.check_status()
-            else:
-                sys.exit(0)
-                # raise Timeout("Timed out trying to start LBRY daemon")
-
     def handle_osx(self, lbry_name):
-        lbry_process = [d for d in subprocess.Popen(['ps','aux'], stdout=subprocess.PIPE).stdout.readlines()
-                            if 'LBRY.app' in d and 'LBRYURIHandler' not in d]
         try:
             status = self.daemon.is_running()
         except:
-            status = None
-
-        if lbry_process or status:
-            self.check_status()
-            started = False
-        else:
             os.system("open /Applications/LBRY.app")
-            self.check_status()
-            started = True
+            sleep(3)
 
-        if lbry_name == "lbry" or lbry_name == "" and not started:
+        if lbry_name == "lbry" or lbry_name == "":
             webbrowser.open(UI_ADDRESS)
         else:
             webbrowser.open(UI_ADDRESS + "/?watch=" + lbry_name)
 
     def handle_linux(self, lbry_name):
         try:
-            is_running = self.daemon.is_running()
-            if not is_running:
-                sys.exit(0)
+            status = self.daemon.is_running()
         except:
-            #start lbrynet-daemon
-            sys.exit(0)
+            cmd = r'DIR = "$( cd "$(dirname "${BASH_SOURCE[0]}" )" && pwd )"' \
+                  r'if [-z "$(pgrep lbrynet-daemon)"]; then' \
+                    r'echo "running lbrynet-daemon..."' \
+                    r'$DIR / lbrynet - daemon &' \
+                    r'sleep 3  # let the daemon load before connecting' \
+                  r'fi'
+            subprocess.Popen(cmd, shell=True)
 
-        if lbry_name == "lbry":
+        if lbry_name == "lbry" or lbry_name == "":
             webbrowser.open(UI_ADDRESS)
         else:
             webbrowser.open(UI_ADDRESS + "/?watch=" + lbry_name)

From 74250982d141156d3d80994a57fa5740cd4ce144 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <alex@grin.io>
Date: Fri, 6 May 2016 17:38:57 -0400
Subject: [PATCH 38/38] update linux uri handler

---
 packaging/ubuntu/lbry | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/packaging/ubuntu/lbry b/packaging/ubuntu/lbry
index 37859de4d..3735a253a 100755
--- a/packaging/ubuntu/lbry
+++ b/packaging/ubuntu/lbry
@@ -26,7 +26,12 @@ ARG=${1:-}
 if [ -z "$ARG" ]; then
   URL=""
 else
-  URL="?watch=$(urlencode "$(echo "$ARG" | cut -c 8-)")"
+  NAME=$(echo "$ARG" | cut -c 8-)
+  if [ -z "$NAME" -o "$NAME" == "lbry" ]; then
+    URL=""
+  else
+    URL="/?watch=$(urlencode "$NAME")"
+  fi
 fi
 
-/usr/bin/xdg-open "http://localhost:5279/$URL"
+/usr/bin/xdg-open "http://localhost:5279$URL"