Merge remote-tracking branch 'upstream/master'

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.14.1rc1
+current_version = 0.14.2rc1
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+)(?P<candidate>\d+))?

CHANGELOG.md

@@ -10,13 +10,18 @@ at anytime.
 ## [Unreleased]
 ### Added
   * Missing docstring for `blob_list`
-  *
+  * Added convenient import for setting up a daemon client, `from lbrynet.daemon import get_client`
 
 ### Changed
   * Change `max_key_fee` setting to be a dictionary with values for `currency` and `amount`
   * Linux default downloads folder changed from `~/Downloads` to `XDG_DOWNLOAD_DIR`
   * Linux folders moved from the home directory to `~/.local/share/lbry`
   * Windows folders moved from `%APPDATA%/Roaming` to `%APPDATA%/Local/lbry`
+  * Renamed `lbrynet.lbryfile` to `lbrynet.lbry_file`
+  * Renamed `lbrynet.lbryfilemanager` to `lbrynet.file_manager`
+  * Renamed `lbrynet.lbrynet_daemon` to `lbrynet.daemon`
+  * Initialize lbrynet settings when configuring an api client if they are not set yet
+  * Updated lbryum imports
   *
 
 ### Fixed
@@ -25,6 +30,8 @@ at anytime.
   * Fixed claim_new_support docstrings
   * Fix default directories to comply to XDG
   *
+  * Fixed BlobManager causing functional tests to fail, removed its unneeded manage() loop
+  * Increased max_key_fee
 
 ### Deprecated
   *

build/cli.py

@@ -1,4 +1,4 @@
-from lbrynet.lbrynet_daemon import DaemonCLI
+from lbrynet.daemon import DaemonCLI
 import logging
 
 logging.basicConfig()

build/daemon.py

@@ -1,4 +1,4 @@
-from lbrynet.lbrynet_daemon import DaemonControl
+from lbrynet.daemon import DaemonControl
 
 if __name__ == '__main__':
     DaemonControl.start()

lbrynet/__init__.py

@@ -1,6 +1,6 @@
 import logging
 
-__version__ = "0.14.1rc1"
+__version__ = "0.14.2rc1"
 version = tuple(__version__.split('.'))
 
 logging.getLogger(__name__).addHandler(logging.NullHandler())

lbrynet/conf.py

@@ -191,7 +191,7 @@ ADJUSTABLE_SETTINGS = {
     'max_connections_per_stream': (int, 5),
     # TODO: writing json on the cmd line is a pain, come up with a nicer
     # parser for this data structure. maybe 'USD:25'
-    'max_key_fee': (json.loads, {'currency': 'USD', 'amount': 25.0}),
+    'max_key_fee': (json.loads, {'currency': 'USD', 'amount': 50.0}),
     'min_info_rate': (float, .02),  # points/1000 infos
     'min_valuable_hash_rate': (float, .05),  # points/1000 infos
     'min_valuable_info_rate': (float, .05),  # points/1000 infos

lbrynet/core/BlobManager.py

@@ -11,7 +11,6 @@ from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
 from lbrynet.core.Error import NoSuchBlobError
 from lbrynet.core.sqlite_helpers import rerun_if_locked
 
-
 log = logging.getLogger(__name__)
 
 
@@ -90,20 +89,16 @@ class DiskBlobManager(BlobManager):
         #       be thousands of blobs loaded up, many stale
         self.blobs = {}
         self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
-        self._next_manage_call = None
 
+
+    @defer.inlineCallbacks
     def setup(self):
         log.info("Setting up the DiskBlobManager. blob_dir: %s, db_file: %s", str(self.blob_dir),
                  str(self.db_file))
-        d = self._open_db()
+        yield self._open_db()
-        d.addCallback(lambda _: self._manage())
-        return d
 
     def stop(self):
         log.info("Stopping the DiskBlobManager")
-        if self._next_manage_call is not None and self._next_manage_call.active():
-            self._next_manage_call.cancel()
-            self._next_manage_call = None
         self.db_conn.close()
         return defer.succeed(True)
 
@@ -150,11 +145,6 @@ class DiskBlobManager(BlobManager):
         d = self.blob_completed(new_blob, next_announce_time)
         return d
 
-    def delete_blobs(self, blob_hashes):
-        for blob_hash in blob_hashes:
-            if not blob_hash in self.blob_hashes_to_delete:
-                self.blob_hashes_to_delete[blob_hash] = False
-
     def immediate_announce_all_blobs(self):
         d = self._get_all_verified_blob_hashes()
         d.addCallback(self._immediate_announce)
@@ -173,58 +163,17 @@ class DiskBlobManager(BlobManager):
         d = self._add_blob_to_upload_history(blob_hash, host, rate)
         return d
 
-    def _manage(self):
+    @defer.inlineCallbacks
-        from twisted.internet import reactor
+    def delete_blobs(self, blob_hashes):
-
+        bh_to_delete_from_db = []
-        d = self._delete_blobs_marked_for_deletion()
+        for blob_hash in blob_hashes:
-
+            try:
-        def set_next_manage_call():
+                blob = yield self.get_blob(blob_hash)
-            self._next_manage_call = reactor.callLater(1, self._manage)
+                yield blob.delete()
-
+                bh_to_delete_from_db.append(blob_hash)
-        d.addCallback(lambda _: set_next_manage_call())
+            except Exception as e:
-
+                log.warning("Failed to delete blob file. Reason: %s", e)
-    def _delete_blobs_marked_for_deletion(self):
+        yield self._delete_blobs_from_db(bh_to_delete_from_db)
-
-        def remove_from_list(b_h):
-            del self.blob_hashes_to_delete[b_h]
-            return b_h
-
-        def set_not_deleting(err, b_h):
-            log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
-            self.blob_hashes_to_delete[b_h] = False
-            return err
-
-        def delete_from_db(result):
-            b_hs = [r[1] for r in result if r[0] is True]
-            if b_hs:
-                d = self._delete_blobs_from_db(b_hs)
-            else:
-                d = defer.succeed(True)
-
-            def log_error(err):
-                log.warning(
-                    "Failed to delete completed blobs from the db: %s", err.getErrorMessage())
-
-            d.addErrback(log_error)
-            return d
-
-        def delete(blob, b_h):
-            d = blob.delete()
-            d.addCallbacks(lambda _: remove_from_list(b_h), set_not_deleting, errbackArgs=(b_h,))
-            return d
-
-        ds = []
-        for blob_hash, being_deleted in self.blob_hashes_to_delete.items():
-            if being_deleted is False:
-                self.blob_hashes_to_delete[blob_hash] = True
-                d = self.get_blob(blob_hash)
-                d.addCallbacks(
-                    delete, set_not_deleting,
-                    callbackArgs=(blob_hash,), errbackArgs=(blob_hash,))
-                ds.append(d)
-        dl = defer.DeferredList(ds, consumeErrors=True)
-        dl.addCallback(delete_from_db)
-        return defer.DeferredList(ds)
 
     ######### database calls #########
 
@@ -308,8 +257,13 @@ class DiskBlobManager(BlobManager):
         return self.db_conn.runInteraction(delete_blobs)
 
     @rerun_if_locked
-    def _get_all_verified_blob_hashes(self):
+    def _get_all_blob_hashes(self):
         d = self.db_conn.runQuery("select blob_hash from blobs")
+        return d
+
+    @rerun_if_locked
+    def _get_all_verified_blob_hashes(self):
+        d = self._get_all_blob_hashes()
 
         def get_verified_blobs(blobs):
             verified_blobs = []

lbrynet/core/Wallet.py

@@ -12,9 +12,10 @@ from collections import defaultdict, deque
 from zope.interface import implements
 from decimal import Decimal
 
-from lbryum import SimpleConfig, Network
-from lbryum.lbrycrd import COIN
 import lbryum.wallet
+from lbryum.network import Network
+from lbryum.simple_config import SimpleConfig
+from lbryum.constants import COIN
 from lbryum.commands import known_commands, Commands
 
 from lbryschema.uri import parse_lbry_uri

lbrynet/core/system_info.py

@@ -4,11 +4,10 @@ import subprocess
 import os
 
 from urllib2 import urlopen
-
+from lbryschema import __version__ as lbryschema_version
+from lbryum import __version__ as LBRYUM_VERSION
 from lbrynet import build_type, __version__ as lbrynet_version
 from lbrynet.conf import ROOT_DIR
-from lbryum.version import LBRYUM_VERSION
-from lbryschema import __version__ as lbryschema_version
 
 
 def get_lbrynet_version():

lbrynet/daemon/Daemon.py

@@ -25,16 +25,16 @@ from lbrynet.conf import LBRYCRD_WALLET, LBRYUM_WALLET, PTC_WALLET
 from lbrynet.reflector import reupload
 from lbrynet.reflector import ServerFactory as reflector_server_factory
 
-from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileSaverFactory
+from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaverFactory
-from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileOpenerFactory
+from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileOpenerFactory
-from lbrynet.lbryfile.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
+from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
-from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
+from lbrynet.lbry_file.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
-from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamType
+from lbrynet.lbry_file.StreamDescriptor import EncryptedFileStreamType
-from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager
+from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
-from lbrynet.lbrynet_daemon.Downloader import GetStream
+from lbrynet.daemon.Downloader import GetStream
-from lbrynet.lbrynet_daemon.Publisher import Publisher
+from lbrynet.daemon.Publisher import Publisher
-from lbrynet.lbrynet_daemon.ExchangeRateManager import ExchangeRateManager
+from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager
-from lbrynet.lbrynet_daemon.auth.server import AuthJSONRPCServer
+from lbrynet.daemon.auth.server import AuthJSONRPCServer
 from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager
 from lbrynet.core import log_support, utils, system_info
 from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob

lbrynet/daemon/DaemonCLI.py

@@ -6,8 +6,8 @@ from docopt import docopt
 from collections import OrderedDict
 from lbrynet import conf
 from lbrynet.core import utils
-from lbrynet.lbrynet_daemon.auth.client import JSONRPCException, LBRYAPIClient
+from lbrynet.daemon.auth.client import JSONRPCException, LBRYAPIClient
-from lbrynet.lbrynet_daemon.Daemon import LOADING_WALLET_CODE, Daemon
+from lbrynet.daemon.Daemon import LOADING_WALLET_CODE, Daemon
 from lbrynet.core.system_info import get_platform
 from jsonrpc.common import RPCError
 from urllib2 import URLError, HTTPError

lbrynet/daemon/DaemonControl.py

@@ -9,8 +9,8 @@ from jsonrpc.proxy import JSONRPCProxy
 from lbrynet import analytics
 from lbrynet import conf
 from lbrynet.core import utils, system_info
-from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient
+from lbrynet.daemon.auth.client import LBRYAPIClient
-from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer
+from lbrynet.daemon.DaemonServer import DaemonServer
 
 log = logging.getLogger(__name__)
 

lbrynet/daemon/DaemonServer.py

@@ -6,11 +6,11 @@ from twisted.internet import defer, reactor, error
 from twisted.cred import portal
 
 from lbrynet import conf
-from lbrynet.lbrynet_daemon.Daemon import Daemon
+from lbrynet.daemon.Daemon import Daemon
-from lbrynet.lbrynet_daemon.Resources import LBRYindex, HostedEncryptedFile, EncryptedFileUpload
+from lbrynet.daemon.Resources import LBRYindex, HostedEncryptedFile, EncryptedFileUpload
-from lbrynet.lbrynet_daemon.auth.auth import PasswordChecker, HttpPasswordRealm
+from lbrynet.daemon.auth.auth import PasswordChecker, HttpPasswordRealm
-from lbrynet.lbrynet_daemon.auth.util import initialize_api_key_file
+from lbrynet.daemon.auth.util import initialize_api_key_file
-from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest
+from lbrynet.daemon.DaemonRequest import DaemonRequest
 
 
 log = logging.getLogger(__name__)

lbrynet/daemon/Downloader.py

@@ -7,8 +7,8 @@ from lbryschema.fee import Fee
 
 from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed
 from lbrynet.core.StreamDescriptor import download_sd_blob
-from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
+from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
-from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
+from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
 from lbrynet import conf
 
 INITIALIZING_CODE = 'initializing'

lbrynet/daemon/Publisher.py

@@ -5,8 +5,8 @@ import os
 from twisted.internet import defer
 
 from lbrynet.core import file_utils
-from lbrynet.lbryfilemanager.EncryptedFileCreator import create_lbry_file
+from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file
-from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
+from lbrynet.lbry_file.StreamDescriptor import publish_sd_blob
 
 
 log = logging.getLogger(__name__)

lbrynet/daemon/Resources.py

@@ -9,7 +9,7 @@ from twisted.web import server, static, resource
 from twisted.internet import defer, error
 
 from lbrynet import conf
-from lbrynet.lbrynet_daemon.FileStreamer import EncryptedFileStreamer
+from lbrynet.daemon.FileStreamer import EncryptedFileStreamer
 
 
 log = logging.getLogger(__name__)

lbrynet/daemon/__init__.py

@@ -0,0 +1,3 @@
+from lbrynet.daemon.auth.client import LBRYAPIClient
+
+get_client = LBRYAPIClient.get_client

lbrynet/daemon/auth/auth.py

@@ -3,7 +3,7 @@ from zope.interface import implementer
 from twisted.cred import portal, checkers, credentials, error as cred_error
 from twisted.internet import defer
 from twisted.web import resource
-from lbrynet.lbrynet_daemon.auth.util import load_api_keys
+from lbrynet.daemon.auth.util import load_api_keys
 
 log = logging.getLogger(__name__)
 

lbrynet/daemon/auth/client.py

@@ -5,7 +5,7 @@ import os
 import base64
 import json
 
-from lbrynet.lbrynet_daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, get_auth_message
+from lbrynet.daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, get_auth_message
 from lbrynet import conf
 from jsonrpc.proxy import JSONRPCProxy
 
@@ -158,5 +158,7 @@ class AuthAPIClient(object):
 class LBRYAPIClient(object):
     @staticmethod
     def get_client():
+        if not conf.settings:
+            conf.initialize_settings()
         return AuthAPIClient.config() if conf.settings['use_auth_http'] else \
             JSONRPCProxy.from_url(conf.settings.get_api_connection_string())

lbrynet/daemon/auth/server.py

@@ -15,8 +15,8 @@ from traceback import format_exc
 from lbrynet import conf
 from lbrynet.core.Error import InvalidAuthenticationToken
 from lbrynet.core import utils
-from lbrynet.lbrynet_daemon.auth.util import APIKey, get_auth_message
+from lbrynet.daemon.auth.util import APIKey, get_auth_message
-from lbrynet.lbrynet_daemon.auth.client import LBRY_SECRET
+from lbrynet.daemon.auth.client import LBRY_SECRET
 from lbrynet.undecorated import undecorated
 
 log = logging.getLogger(__name__)

lbrynet/file_manager/EncryptedFileCreator.py

@@ -8,7 +8,7 @@ import os
 from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter
 from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
 from lbrynet import conf
-from lbrynet.lbryfile.StreamDescriptor import get_sd_info
+from lbrynet.lbry_file.StreamDescriptor import get_sd_info
 from lbrynet.core.cryptoutils import get_lbry_hash_obj
 from twisted.protocols.basic import FileSender
 

lbrynet/file_manager/EncryptedFileDownloader.py

@@ -10,11 +10,11 @@ from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
 from lbrynet.core.Error import NoSuchSDHash, NoSuchStreamHash
 from lbrynet.core.utils import short_hash
 from lbrynet.core.StreamDescriptor import StreamMetadata
-from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileSaver
+from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaver
-from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileDownloader
+from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileDownloader
-from lbrynet.lbryfilemanager.EncryptedFileStatusReport import EncryptedFileStatusReport
+from lbrynet.file_manager.EncryptedFileStatusReport import EncryptedFileStatusReport
 from lbrynet.interfaces import IStreamDownloaderFactory
-from lbrynet.lbryfile.StreamDescriptor import save_sd_info
+from lbrynet.lbry_file.StreamDescriptor import save_sd_info
 from lbrynet.core.Wallet import ClaimOutpoint
 
 log = logging.getLogger(__name__)

lbrynet/file_manager/EncryptedFileManager.py

@@ -11,9 +11,9 @@ from twisted.python.failure import Failure
 
 from lbrynet.reflector.reupload import reflect_stream
 from lbrynet.core.PaymentRateManager import NegotiatedPaymentRateManager
-from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
+from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
-from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
+from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
-from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamType
+from lbrynet.lbry_file.StreamDescriptor import EncryptedFileStreamType
 from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError
 from lbrynet.cryptstream.client.CryptStreamDownloader import CurrentlyStoppingError
 from lbrynet.core.sqlite_helpers import rerun_if_locked

lbrynet/lbry_file/__init__.py

@@ -0,0 +1,2 @@
+from lbrynet.lbry_file.StreamDescriptor import get_sd_info
+from lbrynet.lbry_file.StreamDescriptor import publish_sd_blob

lbrynet/lbry_file/client/EncryptedFileDownloader.py

@@ -3,12 +3,12 @@ import binascii
 
 from zope.interface import implements
 
-from lbrynet.lbryfile.StreamDescriptor import save_sd_info
+from lbrynet.lbry_file.StreamDescriptor import save_sd_info
 from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
 from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
 from lbrynet.core.StreamDescriptor import StreamMetadata
 from lbrynet.interfaces import IStreamDownloaderFactory
-from lbrynet.lbryfile.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
+from lbrynet.lbry_file.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
 import os
 from twisted.internet import defer, threads, reactor
 from twisted.python.procutils import which

lbrynet/lbry_file/client/EncryptedFileOptions.py

@@ -1,5 +1,5 @@
-from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamType
+from lbrynet.lbry_file.StreamDescriptor import EncryptedFileStreamType
-from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamDescriptorValidator
+from lbrynet.lbry_file.StreamDescriptor import EncryptedFileStreamDescriptorValidator
 from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
 
 

lbrynet/lbryfile/__init__.py

@@ -1,2 +0,0 @@
-from lbrynet.lbryfile.StreamDescriptor import get_sd_info
-from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob

requirements.txt

@@ -14,7 +14,7 @@ jsonrpc==1.2
 jsonrpclib==0.1.7
 jsonschema==2.5.1
 git+https://github.com/lbryio/lbryschema.git@v0.0.7#egg=lbryschema
-git+https://github.com/lbryio/lbryum.git@v3.0.1#egg=lbryum
+git+https://github.com/lbryio/lbryum.git@v3.1.0#egg=lbryum
 miniupnpc==1.9
 pbkdf2==1.3
 pycrypto==2.6.1

scripts/gen_api_docs.py

@@ -9,7 +9,7 @@ import os.path as op
 import re
 import sys
 
-from lbrynet.lbrynet_daemon.Daemon import Daemon
+from lbrynet.daemon.Daemon import Daemon
 
 
 def _name(obj):

setup.py

@@ -21,7 +21,7 @@ requires = [
     'envparse',
     'jsonrpc',
     'jsonschema',
-    'lbryum==3.0.1',
+    'lbryum==3.1.0',
     'lbryschema==0.0.7',
     'miniupnpc',
     'pycrypto',
@@ -35,8 +35,8 @@ requires = [
 ]
 
 console_scripts = [
-    'lbrynet-daemon = lbrynet.lbrynet_daemon.DaemonControl:start',
+    'lbrynet-daemon = lbrynet.daemon.DaemonControl:start',
-    'lbrynet-cli = lbrynet.lbrynet_daemon.DaemonCLI:main'
+    'lbrynet-cli = lbrynet.daemon.DaemonCLI:main'
 ]
 
 
@@ -62,7 +62,6 @@ setup(
     long_description=long_description,
     keywords="lbry protocol media",
     license='MIT',
-
     packages=find_packages(base_dir, exclude=['tests']),
     install_requires=requires,
     entry_points={'console_scripts': console_scripts},

tests/functional/test_misc.py

@@ -10,18 +10,18 @@ import unittest
 from Crypto import Random
 from Crypto.Hash import MD5
 from lbrynet import conf
-from lbrynet.lbryfile.EncryptedFileMetadataManager import TempEncryptedFileMetadataManager
+from lbrynet.lbry_file.EncryptedFileMetadataManager import TempEncryptedFileMetadataManager
-from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
+from lbrynet.lbry_file.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
-from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager
+from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
 from lbrynet.core.Session import Session
 from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
 from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
 from lbrynet.core.StreamDescriptor import BlobStreamDescriptorWriter
 from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
 from lbrynet.core.StreamDescriptor import download_sd_blob
-from lbrynet.lbryfilemanager.EncryptedFileCreator import create_lbry_file
+from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file
-from lbrynet.lbryfile.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
+from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
-from lbrynet.lbryfile.StreamDescriptor import get_sd_info
+from lbrynet.lbry_file.StreamDescriptor import get_sd_info
 from twisted.internet import defer, threads, task
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
@@ -746,8 +746,9 @@ class TestTransfer(TestCase):
             yield download_file(sd_hash)
             yield check_md5_sum()
             yield download_file(sd_hash)
-            yield delete_lbry_file()
+
             yield check_lbry_file()
+            yield delete_lbry_file()
 
         def stop(arg):
             if isinstance(arg, Failure):

tests/functional/test_reflector.py

@@ -5,7 +5,7 @@ from twisted.internet import defer, threads, error
 from twisted.trial import unittest
 
 from lbrynet import conf
-from lbrynet import lbryfile
+from lbrynet import lbry_file
 from lbrynet import reflector
 from lbrynet.core import BlobManager
 from lbrynet.core import PeerManager
@@ -13,10 +13,10 @@ from lbrynet.core import RateLimiter
 from lbrynet.core import Session
 from lbrynet.core import StreamDescriptor
 from lbrynet.dht.node import Node
-from lbrynet.lbryfile import EncryptedFileMetadataManager
+from lbrynet.lbry_file import EncryptedFileMetadataManager
-from lbrynet.lbryfile.client import EncryptedFileOptions
+from lbrynet.lbry_file.client import EncryptedFileOptions
-from lbrynet.lbryfilemanager import EncryptedFileCreator
+from lbrynet.file_manager import EncryptedFileCreator
-from lbrynet.lbryfilemanager import EncryptedFileManager
+from lbrynet.file_manager import EncryptedFileManager
 
 from tests import mocks
 
@@ -95,10 +95,10 @@ class TestReflector(unittest.TestCase):
 
         def verify_stream_descriptor_file(stream_hash):
             self.stream_hash = stream_hash
-            d = lbryfile.get_sd_info(self.lbry_file_manager.stream_info_manager, stream_hash, True)
+            d = lbry_file.get_sd_info(self.lbry_file_manager.stream_info_manager, stream_hash, True)
             d.addCallback(verify_equal)
             d.addCallback(
-                lambda _: lbryfile.publish_sd_blob(
+                lambda _: lbry_file.publish_sd_blob(
                     self.lbry_file_manager.stream_info_manager,
                     self.session.blob_manager, stream_hash
                 )

tests/functional/test_streamify.py

@@ -7,15 +7,15 @@ from twisted.trial.unittest import TestCase
 from twisted.internet import defer, threads
 
 from lbrynet import conf
-from lbrynet.lbryfile.EncryptedFileMetadataManager import TempEncryptedFileMetadataManager
+from lbrynet.lbry_file.EncryptedFileMetadataManager import TempEncryptedFileMetadataManager
-from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
+from lbrynet.lbry_file.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
-from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager
+from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
 from lbrynet.core.Session import Session
 from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
-from lbrynet.lbryfile import publish_sd_blob
+from lbrynet.lbry_file import publish_sd_blob
-from lbrynet.lbryfilemanager.EncryptedFileCreator import create_lbry_file
+from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file
-from lbrynet.lbryfile.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
+from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
-from lbrynet.lbryfile.StreamDescriptor import get_sd_info
+from lbrynet.lbry_file.StreamDescriptor import get_sd_info
 from lbrynet.core.PeerManager import PeerManager
 from lbrynet.core.RateLimiter import DummyRateLimiter, RateLimiter
 

tests/mocks.py

@@ -6,7 +6,7 @@ from twisted.internet import defer
 
 from lbrynet.core import PTCWallet
 from lbrynet.core import BlobAvailability
-from lbrynet.lbrynet_daemon import ExchangeRateManager as ERM
+from lbrynet.daemon import ExchangeRateManager as ERM
 from lbrynet import conf
 
 KB = 2**10

tests/unit/core/test_BlobManager.py

@@ -0,0 +1,112 @@
+import tempfile
+import shutil
+import mock
+import os
+import random
+import string
+
+from tests.util import random_lbry_hash
+from lbrynet.core.BlobManager import DiskBlobManager
+from lbrynet.core.HashAnnouncer import DummyHashAnnouncer
+from lbrynet.core.Peer import Peer
+from lbrynet.core.cryptoutils import get_lbry_hash_obj
+from twisted.trial import unittest
+
+from twisted.internet import defer
+
+class BlobManagerTest(unittest.TestCase):
+    def setUp(self):
+        self.blob_dir = tempfile.mkdtemp()
+        self.db_dir = tempfile.mkdtemp()
+        hash_announcer  = DummyHashAnnouncer()
+        self.bm = DiskBlobManager(hash_announcer, self.blob_dir, self.db_dir)
+        self.peer = Peer('somehost',22)
+
+    def tearDown(self):
+        self.bm.stop()
+        shutil.rmtree(self.blob_dir)
+        shutil.rmtree(self.db_dir)
+
+    @defer.inlineCallbacks
+    def _create_and_add_blob(self):
+        # create and add blob to blob manager
+        data_len = random.randint(1,1000)
+        data = ''.join(random.choice(string.lowercase) for data_len in range(data_len))
+
+        hashobj = get_lbry_hash_obj()
+        hashobj.update(data)
+        out=hashobj.hexdigest()
+        blob_hash=out
+
+        # create new blob
+        yield self.bm.setup()
+        blob = yield self.bm.get_blob(blob_hash,len(data))
+
+        finished_d, write, cancel =yield blob.open_for_writing(self.peer)
+        yield write(data)
+        yield self.bm.blob_completed(blob)
+        yield self.bm.add_blob_to_upload_history(blob_hash,'test',len(data))
+
+        # check to see if blob is there
+        self.assertTrue(os.path.isfile(os.path.join(self.blob_dir,blob_hash)))
+        blobs = yield self.bm.get_all_verified_blobs()
+        self.assertTrue(blob_hash in blobs)
+        defer.returnValue(blob_hash)
+
+    @defer.inlineCallbacks
+    def test_create_blob(self):
+        blob_hashes = []
+
+        # create a bunch of blobs
+        for i in range(0,10):
+            blob_hash = yield self._create_and_add_blob()
+            blob_hashes.append(blob_hash)
+        blobs = yield self.bm.get_all_verified_blobs()
+        self.assertEqual(10,len(blobs))
+
+
+    @defer.inlineCallbacks
+    def test_delete_blob(self):
+        # create blob
+        blob_hash  = yield self._create_and_add_blob()
+        blobs = yield self.bm.get_all_verified_blobs()
+        self.assertEqual(len(blobs),1)
+
+        # delete blob 
+        yield self.bm.delete_blobs([blob_hash])
+        self.assertFalse(os.path.isfile(os.path.join(self.blob_dir,blob_hash)))
+        blobs = yield self.bm.get_all_verified_blobs()
+        self.assertEqual(len(blobs),0)
+        blobs = yield self.bm._get_all_blob_hashes() 
+        self.assertEqual(len(blobs),0)
+
+        # delete blob that does not exist, nothing will
+        # happen
+        blob_hash= random_lbry_hash()
+        out = yield self.bm.delete_blobs([blob_hash])
+
+
+    @defer.inlineCallbacks
+    def test_delete_open_blob(self):
+        # Test that a blob that is opened for writing will not be deleted
+
+        # create blobs
+        blob_hashes =[]
+        for i in range(0,10):
+            blob_hash  = yield self._create_and_add_blob()
+            blob_hashes.append(blob_hash)
+        blobs = yield self.bm.get_all_verified_blobs()
+        self.assertEqual(len(blobs),10)
+
+        # open the last blob
+        blob = yield self.bm.get_blob(blob_hashes[-1])
+        yield blob.open_for_writing(self.peer)
+        
+        # delete the last blob and check if it still exists
+        out = yield self.bm.delete_blobs([blob_hash])
+        blobs = yield self.bm.get_all_verified_blobs()
+        self.assertEqual(len(blobs),10)
+        self.assertTrue(blob_hashes[-1] in blobs)
+        self.assertTrue(os.path.isfile(os.path.join(self.blob_dir,blob_hashes[-1])))
+
+

tests/unit/lbryfile/client/test_EncryptedFileDownloader.py

@@ -1,7 +1,7 @@
 import os.path
 from twisted.trial import unittest
 from twisted.internet import defer
-from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileSaver
+from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaver
 
 
 

tests/unit/lbryfile/test_EncryptedFileMetadataManager.py

@@ -2,7 +2,7 @@ import tempfile
 import shutil
 from twisted.trial import unittest
 from twisted.internet import defer
-from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
+from lbrynet.lbry_file.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
 from lbrynet.core import utils
 from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
 from lbrynet.core.Error import NoSuchStreamHash

tests/unit/lbryfilemanager/test_EncryptedFileCreator.py

@@ -9,8 +9,8 @@ from twisted.trial import unittest
 from lbrynet.core import BlobManager
 from lbrynet.core import Session
 from lbrynet.core.server import DHTHashAnnouncer
-from lbrynet.lbryfilemanager import EncryptedFileCreator
+from lbrynet.file_manager import EncryptedFileCreator
-from lbrynet.lbryfilemanager import EncryptedFileManager
+from lbrynet.file_manager import EncryptedFileManager
 
 from tests import mocks
 

tests/unit/lbryfilemanager/test_EncryptedFileManager.py

@@ -1,7 +1,7 @@
 from twisted.internet import defer
 from twisted.trial import unittest
-from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
+from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
-from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager
+from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
 from tests.util import random_lbry_hash
 
 class TestEncryptedFileManager(unittest.TestCase):

tests/unit/lbrynet_daemon/auth/test_server.py

@@ -2,7 +2,7 @@ import mock
 from twisted.trial import unittest
 
 from tests.mocks import mock_conf_settings
-from lbrynet.lbrynet_daemon.auth import server
+from lbrynet.daemon.auth import server
 
 
 class AuthJSONRPCServerTest(unittest.TestCase):

tests/unit/lbrynet_daemon/test_Daemon.py

@@ -7,8 +7,8 @@ from twisted.trial import unittest
 from lbryschema.decode import smart_decode
 from lbrynet import conf
 from lbrynet.core import Session, PaymentRateManager, Wallet
-from lbrynet.lbrynet_daemon.Daemon import Daemon as LBRYDaemon
+from lbrynet.daemon.Daemon import Daemon as LBRYDaemon
-from lbrynet.lbrynet_daemon import ExchangeRateManager
+from lbrynet.daemon import ExchangeRateManager
 
 from tests import util
 from tests.mocks import mock_conf_settings, FakeNetwork

tests/unit/lbrynet_daemon/test_DaemonCLI.py

@@ -1,5 +1,5 @@
 from twisted.trial import unittest
-from lbrynet.lbrynet_daemon import DaemonCLI
+from lbrynet.daemon import DaemonCLI
 
 
 class DaemonCLITests(unittest.TestCase):

tests/unit/lbrynet_daemon/test_Downloader.py

@@ -7,13 +7,13 @@ from twisted.internet import defer
 from lbryschema.claim import ClaimDict
 
 from lbrynet.core import Session, PaymentRateManager, Wallet
-from lbrynet.lbrynet_daemon import Downloader
+from lbrynet.daemon import Downloader
 from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier,StreamMetadata
-from lbrynet.lbryfile.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
+from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
 from lbrynet.core.HashBlob import TempBlob
 from lbrynet.core.BlobManager import TempBlobManager
-from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
+from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
-from lbrynet.lbrynet_daemon.ExchangeRateManager import ExchangeRateManager
+from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager
 
 from tests.mocks import BlobAvailabilityTracker as DummyBlobAvailabilityTracker
 from tests.mocks import ExchangeRateManager as DummyExchangeRateManager

tests/unit/lbrynet_daemon/test_ExchangeRateManager.py

@@ -1,5 +1,5 @@
 from lbryschema.fee import Fee
-from lbrynet.lbrynet_daemon import ExchangeRateManager
+from lbrynet.daemon import ExchangeRateManager
 from lbrynet.core.Error import InvalidExchangeRateResponse
 
 from twisted.trial import unittest