update unit tests, add test_SQLiteStorage

lbrynet/tests/mocks.py

@@ -173,6 +173,7 @@ class GenFile(io.RawIOBase):
         self.read_so_far = 0
         self.buff = b''
         self.last_offset = 0
+        self.name = "."
 
     def readable(self):
         return True

lbrynet/tests/unit/core/test_BlobManager.py

@@ -7,6 +7,7 @@ import string
 from lbrynet.tests.util import random_lbry_hash
 from lbrynet.core.BlobManager import DiskBlobManager
 from lbrynet.core.HashAnnouncer import DummyHashAnnouncer
+from lbrynet.database.storage import SQLiteStorage
 from lbrynet.core.Peer import Peer
 from lbrynet import conf
 from lbrynet.core.cryptoutils import get_lbry_hash_obj
@@ -14,13 +15,14 @@ from twisted.trial import unittest
 
 from twisted.internet import defer
 
+
 class BlobManagerTest(unittest.TestCase):
     def setUp(self):
         conf.initialize_settings()
         self.blob_dir = tempfile.mkdtemp()
         self.db_dir = tempfile.mkdtemp()
         hash_announcer = DummyHashAnnouncer()
-        self.bm = DiskBlobManager(hash_announcer, self.blob_dir, self.db_dir)
+        self.bm = DiskBlobManager(hash_announcer, self.blob_dir, SQLiteStorage(self.db_dir))
         self.peer = Peer('somehost', 22)
 
     def tearDown(self):
@@ -43,13 +45,13 @@ class BlobManagerTest(unittest.TestCase):
         blob_hash = out
 
         # create new blob
+        yield self.bm.storage.setup()
         yield self.bm.setup()
         blob = yield self.bm.get_blob(blob_hash, len(data))
 
         writer, finished_d = yield blob.open_for_writing(self.peer)
         yield writer.write(data)
         yield self.bm.blob_completed(blob, should_announce)
-        yield self.bm.add_blob_to_upload_history(blob_hash, 'test', len(data))
 
         # check to see if blob is there
         self.assertTrue(os.path.isfile(os.path.join(self.blob_dir, blob_hash)))
@@ -81,7 +83,7 @@ class BlobManagerTest(unittest.TestCase):
         self.assertFalse(os.path.isfile(os.path.join(self.blob_dir, blob_hash)))
         blobs = yield self.bm.get_all_verified_blobs()
         self.assertEqual(len(blobs), 0)
-        blobs = yield self.bm._get_all_blob_hashes()
+        blobs = yield self.bm.storage.get_all_blob_hashes()
         self.assertEqual(len(blobs), 0)
         self.assertFalse(blob_hash in self.bm.blobs)
 

lbrynet/tests/unit/core/test_Wallet.py

@@ -7,11 +7,13 @@ from decimal import Decimal
 from collections import defaultdict
 from twisted.trial import unittest
 from twisted.internet import threads, defer
-
+from lbrynet.database.storage import SQLiteStorage
+from lbrynet.tests.mocks import FakeNetwork
 from lbrynet.core.Error import InsufficientFundsError
-from lbrynet.core.Wallet import Wallet, LBRYumWallet, ReservedPoints, InMemoryStorage
+from lbrynet.core.Wallet import LBRYumWallet, ReservedPoints
 from lbryum.commands import Commands
-
+from lbryum.simple_config import SimpleConfig
+from lbryschema.claim import ClaimDict
 
 test_metadata = {
 'license': 'NASA',
@@ -34,12 +36,22 @@ test_claim_dict = {
 }}
 
 
-class MocLbryumWallet(Wallet):
+class MocLbryumWallet(LBRYumWallet):
     def __init__(self):
+        # LBRYumWallet.__init__(self)
+        self.config = SimpleConfig()
         self.wallet_balance = Decimal(10.0)
         self.total_reserved_points = Decimal(0.0)
         self.queued_payments = defaultdict(Decimal)
-        self._storage = InMemoryStorage()
+        self.network = FakeNetwork()
+        self.db_dir = tempfile.mkdtemp()
+        self.storage = SQLiteStorage(self.db_dir)
+
+    def __del__(self):
+        shutil.rmtree(self.db_dir)
+
+    def setup(self):
+        return self.storage.setup()
 
     def get_least_used_address(self, account=None, for_change=False, max_count=100):
         return defer.succeed(None)
@@ -51,21 +63,15 @@ class MocLbryumWallet(Wallet):
         return defer.succeed(True)
 
 
-class MocEncryptedWallet(LBRYumWallet):
-    def __init__(self):
-        LBRYumWallet.__init__(self, InMemoryStorage())
-        self.wallet_balance = Decimal(10.0)
-        self.total_reserved_points = Decimal(0.0)
-        self.queued_payments = defaultdict(Decimal)
-
 class WalletTest(unittest.TestCase):
 
+    @defer.inlineCallbacks
     def setUp(self):
-        wallet = MocEncryptedWallet()
+        wallet = MocLbryumWallet()
+        yield wallet.setup()
         seed_text = "travel nowhere air position hill peace suffer parent beautiful rise " \
                     "blood power home crumble teach"
         password = "secret"
-
         user_dir = tempfile.mkdtemp()
         path = os.path.join(user_dir, "somewallet")
         storage = lbryum.wallet.WalletStorage(path)
@@ -88,7 +94,8 @@ class WalletTest(unittest.TestCase):
 
         MocLbryumWallet._send_name_claim = not_enough_funds_send_name_claim
         wallet = MocLbryumWallet()
-        d = wallet.claim_name('test', 1, test_claim_dict)
+        d = wallet.setup()
+        d.addCallback(lambda _: wallet.claim_name('test', 1, test_claim_dict))
         self.assertFailure(d, Exception)
         return d
 
@@ -98,7 +105,11 @@ class WalletTest(unittest.TestCase):
             "fee": "0.00012",
             "nout": 0,
             "success": True,
-            "txid": "6f8180002ef4d21f5b09ca7d9648a54d213c666daf8639dc283e2fd47450269e"
+            "txid": "6f8180002ef4d21f5b09ca7d9648a54d213c666daf8639dc283e2fd47450269e",
+            "value": ClaimDict.load_dict(test_claim_dict).serialized.encode('hex'),
+            "claim_address": "",
+            "channel_claim_id": "",
+            "channel_name": ""
         }
 
         def check_out(claim_out):
@@ -107,6 +118,7 @@ class WalletTest(unittest.TestCase):
             self.assertEqual(expected_claim_out['fee'], claim_out['fee'])
             self.assertEqual(expected_claim_out['nout'], claim_out['nout'])
             self.assertEqual(expected_claim_out['txid'], claim_out['txid'])
+            self.assertEqual(expected_claim_out['value'], claim_out['value'])
 
         def success_send_name_claim(self, name, val, amount, certificate_id=None,
                                     claim_address=None, change_address=None):
@@ -114,7 +126,8 @@ class WalletTest(unittest.TestCase):
 
         MocLbryumWallet._send_name_claim = success_send_name_claim
         wallet = MocLbryumWallet()
-        d = wallet.claim_name('test', 1, test_claim_dict)
+        d = wallet.setup()
+        d.addCallback(lambda _: wallet.claim_name('test', 1, test_claim_dict))
         d.addCallback(lambda claim_out: check_out(claim_out))
         return d
 
@@ -124,7 +137,8 @@ class WalletTest(unittest.TestCase):
             return threads.deferToThread(lambda: claim_out)
         MocLbryumWallet._support_claim = failed_support_claim
         wallet = MocLbryumWallet()
-        d = wallet.support_claim('test', "f43dc06256a69988bdbea09a58c80493ba15dcfa", 1)
+        d = wallet.setup()
+        d.addCallback(lambda _: wallet.support_claim('test', "f43dc06256a69988bdbea09a58c80493ba15dcfa", 1))
         self.assertFailure(d, Exception)
         return d
 
@@ -146,7 +160,8 @@ class WalletTest(unittest.TestCase):
             return threads.deferToThread(lambda: expected_support_out)
         MocLbryumWallet._support_claim = success_support_claim
         wallet = MocLbryumWallet()
-        d = wallet.support_claim('test', "f43dc06256a69988bdbea09a58c80493ba15dcfa", 1)
+        d = wallet.setup()
+        d.addCallback(lambda _: wallet.support_claim('test', "f43dc06256a69988bdbea09a58c80493ba15dcfa", 1))
         d.addCallback(lambda claim_out: check_out(claim_out))
         return d
 
@@ -156,7 +171,8 @@ class WalletTest(unittest.TestCase):
             return threads.deferToThread(lambda: claim_out)
         MocLbryumWallet._abandon_claim = failed_abandon_claim
         wallet = MocLbryumWallet()
-        d = wallet.abandon_claim("f43dc06256a69988bdbea09a58c80493ba15dcfa", None, None)
+        d = wallet.setup()
+        d.addCallback(lambda _: wallet.abandon_claim("f43dc06256a69988bdbea09a58c80493ba15dcfa", None, None))
         self.assertFailure(d, Exception)
         return d
 
@@ -177,7 +193,8 @@ class WalletTest(unittest.TestCase):
 
         MocLbryumWallet._abandon_claim = success_abandon_claim
         wallet = MocLbryumWallet()
-        d = wallet.abandon_claim("f43dc06256a69988bdbea09a58c80493ba15dcfa", None, None)
+        d = wallet.storage.setup()
+        d.addCallback(lambda _: wallet.abandon_claim("f43dc06256a69988bdbea09a58c80493ba15dcfa", None, None))
         d.addCallback(lambda claim_out: check_out(claim_out))
         return d
 
@@ -188,7 +205,8 @@ class WalletTest(unittest.TestCase):
             return defer.succeed(5)
         wallet = MocLbryumWallet()
         wallet._update_balance = update_balance
-        d = wallet.update_balance()
+        d = wallet.setup()
+        d.addCallback(lambda _: wallet.update_balance())
         # test point reservation
         d.addCallback(lambda _: self.assertEqual(5, wallet.get_balance()))
         d.addCallback(lambda _: wallet.reserve_points('testid', 2))
@@ -213,7 +231,8 @@ class WalletTest(unittest.TestCase):
             return defer.succeed(5)
         wallet = MocLbryumWallet()
         wallet._update_balance = update_balance
-        d = wallet.update_balance()
+        d = wallet.setup()
+        d.addCallback(lambda _: wallet.update_balance())
         d.addCallback(lambda _: self.assertEqual(5, wallet.get_balance()))
         d.addCallback(lambda _: wallet.reserve_points('testid', 2))
         d.addCallback(lambda _: wallet.claim_name('test', 4, test_claim_dict))
@@ -226,7 +245,8 @@ class WalletTest(unittest.TestCase):
             return defer.succeed(5)
         wallet = MocLbryumWallet()
         wallet._update_balance = update_balance
-        d = wallet.update_balance()
+        d = wallet.setup()
+        d.addCallback(lambda _: wallet.update_balance())
         d.addCallback(lambda _: self.assertEqual(5, wallet.get_balance()))
         d.addCallback(lambda _: wallet.reserve_points('testid', 2))
         d.addCallback(lambda _: wallet.support_claim(

lbrynet/tests/unit/database/test_SQLiteStorage.py

@@ -0,0 +1,332 @@
+import os
+import shutil
+import tempfile
+import logging
+from copy import deepcopy
+from twisted.internet import defer
+from twisted.trial import unittest
+from lbrynet import conf
+from lbrynet.database.storage import SQLiteStorage, open_file_for_writing
+from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
+from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
+from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
+from lbrynet.tests.util import random_lbry_hash
+
+log = logging.getLogger()
+
+
+def blob_info_dict(blob_info):
+    info = {
+        "length": blob_info.length,
+        "blob_num": blob_info.blob_num,
+        "iv": blob_info.iv
+    }
+    if blob_info.length:
+        info['blob_hash'] = blob_info.blob_hash
+    return info
+
+
+fake_claim_info = {
+    'name': "test",
+    'claim_id': 'deadbeef' * 5,
+    'address': "bT6wc54qiUUYt34HQF9wnW8b2o2yQTXf2S",
+    'claim_sequence': 1,
+    'value':  {
+        "version": "_0_0_1",
+        "claimType": "streamType",
+        "stream": {
+          "source": {
+            "source": 'deadbeef' * 12,
+            "version": "_0_0_1",
+            "contentType": "video/mp4",
+            "sourceType": "lbry_sd_hash"
+          },
+          "version": "_0_0_1",
+          "metadata": {
+            "license": "LBRY inc",
+            "description": "What is LBRY? An introduction with Alex Tabarrok",
+            "language": "en",
+            "title": "What is LBRY?",
+            "author": "Samuel Bryan",
+            "version": "_0_1_0",
+            "nsfw": False,
+            "licenseUrl": "",
+            "preview": "",
+            "thumbnail": "https://s3.amazonaws.com/files.lbry.io/logo.png"
+          }
+        }
+    },
+    'height': 10000,
+    'amount': 1.0,
+    'effective_amount': 1.0,
+    'nout': 0,
+    'txid': "deadbeef" * 8,
+    'supports': [],
+    'channel_claim_id': None,
+    'channel_name': None
+}
+
+
+class FakeAnnouncer(object):
+    def __init__(self):
+        self._queue_size = 0
+
+    def hash_queue_size(self):
+        return self._queue_size
+
+
+class MocSession(object):
+    def __init__(self, storage):
+        self.storage = storage
+
+
+class StorageTest(unittest.TestCase):
+    maxDiff = 5000
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        conf.initialize_settings()
+        self.db_dir = tempfile.mkdtemp()
+        self.storage = SQLiteStorage(self.db_dir)
+        yield self.storage.setup()
+
+    @defer.inlineCallbacks
+    def tearDown(self):
+        yield self.storage.stop()
+        shutil.rmtree(self.db_dir)
+
+    @defer.inlineCallbacks
+    def store_fake_blob(self, blob_hash, blob_length=100, next_announce=0, should_announce=0):
+        yield self.storage.add_completed_blob(blob_hash, blob_length, next_announce,
+                                              should_announce)
+        yield self.storage.set_blob_status(blob_hash, "finished")
+
+    @defer.inlineCallbacks
+    def store_fake_stream_blob(self, stream_hash, blob_hash, blob_num, length=100, iv="DEADBEEF"):
+        blob_info = {
+            'blob_hash': blob_hash, 'blob_num': blob_num, 'iv': iv
+        }
+        if length:
+            blob_info['length'] = length
+        yield self.storage.add_blobs_to_stream(stream_hash, [blob_info])
+
+    @defer.inlineCallbacks
+    def store_fake_stream(self, stream_hash, sd_hash, file_name="fake_file", key="DEADBEEF",
+                          blobs=[]):
+        yield self.storage.store_stream(stream_hash, sd_hash, file_name, key,
+                                           file_name, blobs)
+
+    @defer.inlineCallbacks
+    def make_and_store_fake_stream(self, blob_count=2, stream_hash=None, sd_hash=None):
+        stream_hash = stream_hash or random_lbry_hash()
+        sd_hash = sd_hash or random_lbry_hash()
+        blobs = {
+            i + 1: random_lbry_hash() for i in range(blob_count)
+        }
+
+        yield self.store_fake_blob(sd_hash)
+
+        for blob in blobs.itervalues():
+            yield self.store_fake_blob(blob)
+
+        yield self.store_fake_stream(stream_hash, sd_hash)
+
+        for pos, blob in sorted(blobs.iteritems(), key=lambda x: x[0]):
+            yield self.store_fake_stream_blob(stream_hash, blob, pos)
+
+
+class TestSetup(StorageTest):
+    @defer.inlineCallbacks
+    def test_setup(self):
+        files = yield self.storage.get_all_lbry_files()
+        self.assertEqual(len(files), 0)
+        blobs = yield self.storage.get_all_blob_hashes()
+        self.assertEqual(len(blobs), 0)
+
+
+class BlobStorageTests(StorageTest):
+    @defer.inlineCallbacks
+    def test_store_blob(self):
+        blob_hash = random_lbry_hash()
+        yield self.store_fake_blob(blob_hash)
+        blob_hashes = yield self.storage.get_all_blob_hashes()
+        self.assertEqual(blob_hashes, [blob_hash])
+
+    @defer.inlineCallbacks
+    def test_delete_blob(self):
+        blob_hash = random_lbry_hash()
+        yield self.store_fake_blob(blob_hash)
+        blob_hashes = yield self.storage.get_all_blob_hashes()
+        self.assertEqual(blob_hashes, [blob_hash])
+        yield self.storage.delete_blobs_from_db(blob_hashes)
+        blob_hashes = yield self.storage.get_all_blob_hashes()
+        self.assertEqual(blob_hashes, [])
+
+
+class StreamStorageTests(StorageTest):
+    @defer.inlineCallbacks
+    def test_store_stream(self, stream_hash=None):
+        stream_hash = stream_hash or random_lbry_hash()
+        sd_hash = random_lbry_hash()
+        blob1 = random_lbry_hash()
+        blob2 = random_lbry_hash()
+
+        yield self.store_fake_blob(sd_hash)
+        yield self.store_fake_blob(blob1)
+        yield self.store_fake_blob(blob2)
+
+        yield self.store_fake_stream(stream_hash, sd_hash)
+        yield self.store_fake_stream_blob(stream_hash, blob1, 1)
+        yield self.store_fake_stream_blob(stream_hash, blob2, 2)
+
+        stream_blobs = yield self.storage.get_blobs_for_stream(stream_hash)
+        stream_blob_hashes = [b.blob_hash for b in stream_blobs]
+        self.assertListEqual(stream_blob_hashes, [blob1, blob2])
+
+        blob_hashes = yield self.storage.get_all_blob_hashes()
+        self.assertSetEqual(set(blob_hashes), {sd_hash, blob1, blob2})
+
+        stream_blobs = yield self.storage.get_blobs_for_stream(stream_hash)
+        stream_blob_hashes = [b.blob_hash for b in stream_blobs]
+        self.assertListEqual(stream_blob_hashes, [blob1, blob2])
+
+        yield self.storage.set_should_announce(sd_hash, 1, 1)
+        yield self.storage.set_should_announce(blob1, 1, 1)
+
+        should_announce_count = yield self.storage.count_should_announce_blobs()
+        self.assertEqual(should_announce_count, 2)
+        should_announce_hashes = yield self.storage.get_blobs_to_announce(FakeAnnouncer())
+        self.assertSetEqual(set(should_announce_hashes), {sd_hash, blob1})
+
+        stream_hashes = yield self.storage.get_all_streams()
+        self.assertListEqual(stream_hashes, [stream_hash])
+
+    @defer.inlineCallbacks
+    def test_delete_stream(self):
+        stream_hash = random_lbry_hash()
+        yield self.test_store_stream(stream_hash)
+        yield self.storage.delete_stream(stream_hash)
+        stream_hashes = yield self.storage.get_all_streams()
+        self.assertListEqual(stream_hashes, [])
+
+        stream_blobs = yield self.storage.get_blobs_for_stream(stream_hash)
+        self.assertListEqual(stream_blobs, [])
+        blob_hashes = yield self.storage.get_all_blob_hashes()
+        self.assertListEqual(blob_hashes, [])
+
+
+class FileStorageTests(StorageTest):
+    @defer.inlineCallbacks
+    def test_setup_output(self):
+        file_name = 'encrypted_file_saver_test.tmp'
+        self.assertFalse(os.path.isfile(file_name))
+        written_to = yield open_file_for_writing(self.db_dir, file_name)
+        self.assertTrue(written_to == file_name)
+        self.assertTrue(os.path.isfile(os.path.join(self.db_dir, file_name)))
+
+    @defer.inlineCallbacks
+    def test_store_file(self):
+        session = MocSession(self.storage)
+        session.db_dir = self.db_dir
+        sd_identifier = StreamDescriptorIdentifier()
+        download_directory = self.db_dir
+        manager = EncryptedFileManager(session, sd_identifier)
+        out = yield manager.session.storage.get_all_lbry_files()
+        self.assertEqual(len(out), 0)
+
+        stream_hash = random_lbry_hash()
+        sd_hash = random_lbry_hash()
+        blob1 = random_lbry_hash()
+        blob2 = random_lbry_hash()
+
+        yield self.store_fake_blob(sd_hash)
+        yield self.store_fake_blob(blob1)
+        yield self.store_fake_blob(blob2)
+
+        yield self.store_fake_stream(stream_hash, sd_hash)
+        yield self.store_fake_stream_blob(stream_hash, blob1, 1)
+        yield self.store_fake_stream_blob(stream_hash, blob2, 2)
+
+        blob_data_rate = 0
+        file_name = "test file"
+        out = yield manager.session.storage.save_published_file(
+            stream_hash, file_name, download_directory, blob_data_rate
+        )
+        rowid = yield manager.session.storage.get_rowid_for_stream_hash(stream_hash)
+        self.assertEqual(out, rowid)
+
+        files = yield manager.session.storage.get_all_lbry_files()
+        self.assertEqual(1, len(files))
+
+        status = yield manager.session.storage.get_lbry_file_status(rowid)
+        self.assertEqual(status, ManagedEncryptedFileDownloader.STATUS_STOPPED)
+
+        running = ManagedEncryptedFileDownloader.STATUS_RUNNING
+        yield manager.session.storage.change_file_status(rowid, running)
+        status = yield manager.session.storage.get_lbry_file_status(rowid)
+        self.assertEqual(status, ManagedEncryptedFileDownloader.STATUS_RUNNING)
+
+
+class ContentClaimStorageTests(StorageTest):
+    @defer.inlineCallbacks
+    def test_store_content_claim(self):
+        session = MocSession(self.storage)
+        session.db_dir = self.db_dir
+        sd_identifier = StreamDescriptorIdentifier()
+        download_directory = self.db_dir
+        manager = EncryptedFileManager(session, sd_identifier)
+        out = yield manager.session.storage.get_all_lbry_files()
+        self.assertEqual(len(out), 0)
+
+        stream_hash = random_lbry_hash()
+        sd_hash = fake_claim_info['value']['stream']['source']['source']
+
+        # test that we can associate a content claim to a file
+        # use the generated sd hash in the fake claim
+        fake_outpoint = "%s:%i" % (fake_claim_info['txid'], fake_claim_info['nout'])
+
+        yield self.make_and_store_fake_stream(blob_count=2, stream_hash=stream_hash, sd_hash=sd_hash)
+        blob_data_rate = 0
+        file_name = "test file"
+        yield manager.session.storage.save_published_file(
+            stream_hash, file_name, download_directory, blob_data_rate
+        )
+        yield self.storage.save_claim(fake_claim_info)
+        yield self.storage.save_content_claim(stream_hash, fake_outpoint)
+        stored_content_claim = yield self.storage.get_content_claim(stream_hash)
+        self.assertDictEqual(stored_content_claim, fake_claim_info)
+
+        # test that we can't associate a claim update with a new stream to the file
+        second_stream_hash, second_sd_hash = random_lbry_hash(), random_lbry_hash()
+        yield self.make_and_store_fake_stream(blob_count=2, stream_hash=second_stream_hash, sd_hash=second_sd_hash)
+        try:
+            yield self.storage.save_content_claim(second_stream_hash, fake_outpoint)
+            raise Exception("test failed")
+        except Exception as err:
+            self.assertTrue(err.message == "stream mismatch")
+
+        # test that we can associate a new claim update containing the same stream to the file
+        update_info = deepcopy(fake_claim_info)
+        update_info['txid'] = "beef0000" * 12
+        update_info['nout'] = 0
+        second_outpoint = "%s:%i" % (update_info['txid'], update_info['nout'])
+        yield self.storage.save_claim(update_info)
+        yield self.storage.save_content_claim(stream_hash, second_outpoint)
+        update_info_result = yield self.storage.get_content_claim(stream_hash)
+        self.assertDictEqual(update_info_result, update_info)
+
+        # test that we can't associate an update with a mismatching claim id
+        invalid_update_info = deepcopy(fake_claim_info)
+        invalid_update_info['txid'] = "beef0001" * 12
+        invalid_update_info['nout'] = 0
+        invalid_update_info['claim_id'] = "beef0002" * 5
+        invalid_update_outpoint = "%s:%i" % (invalid_update_info['txid'], invalid_update_info['nout'])
+        yield self.storage.save_claim(invalid_update_info)
+        try:
+            yield self.storage.save_content_claim(stream_hash, invalid_update_outpoint)
+            raise Exception("test failed")
+        except Exception as err:
+            self.assertTrue(err.message == "invalid stream update")
+        current_claim_info = yield self.storage.get_content_claim(stream_hash)
+        # this should still be the previous update
+        self.assertDictEqual(current_claim_info, update_info)

lbrynet/tests/unit/lbryfile/client/test_EncryptedFileDownloader.py

@@ -1,34 +0,0 @@
-import os.path
-from twisted.trial import unittest
-from twisted.internet import defer
-from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaver
-
-
-
-class TestEncryptedFileSaver(unittest.TestCase):
-
-    @defer.inlineCallbacks
-    def test_setup_output(self):
-        file_name = 'encrypted_file_saver_test.tmp'
-        file_name_hex = file_name.encode('hex')
-        self.assertFalse(os.path.isfile(file_name))
-
-        # create file in the temporary trial folder
-        stream_hash = ''
-        peer_finder = None
-        rate_limiter = None
-        blob_manager = None
-        stream_info_manager = None
-        payment_rate_manager = None
-        wallet = None
-        download_directory = '.'
-        key = ''
-
-        saver = EncryptedFileSaver(stream_hash, peer_finder, rate_limiter, blob_manager,
-                                   stream_info_manager, payment_rate_manager, wallet,
-                                   download_directory, key,
-                                   file_name_hex, file_name_hex)
-
-        yield saver._setup_output()
-        self.assertTrue(os.path.isfile(file_name))
-        saver._close_output()

lbrynet/tests/unit/lbryfile/test_EncryptedFileMetadataManager.py

@@ -1,78 +0,0 @@
-import tempfile
-import shutil
-from twisted.trial import unittest
-from twisted.internet import defer
-from lbrynet.lbry_file.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
-from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
-from lbrynet.core.Error import NoSuchStreamHash
-from lbrynet.tests.util import random_lbry_hash
-
-
-class DBEncryptedFileMetadataManagerTest(unittest.TestCase):
-    def setUp(self):
-        self.db_dir = tempfile.mkdtemp()
-        self.manager = DBEncryptedFileMetadataManager(self.db_dir)
-
-    def tearDown(self):
-        self.manager.stop()
-        shutil.rmtree(self.db_dir)
-
-    @defer.inlineCallbacks
-    def test_basic(self):
-        yield self.manager.setup()
-        out = yield self.manager.get_all_streams()
-        self.assertEqual(len(out), 0)
-
-        stream_hash = random_lbry_hash()
-        file_name = 'file_name'
-        key = 'key'
-        suggested_file_name = 'sug_file_name'
-        blob1 = CryptBlobInfo(random_lbry_hash(), 0, 10, 1)
-        blob2 = CryptBlobInfo(random_lbry_hash(), 0, 10, 1)
-        blobs = [blob1, blob2]
-
-        # save stream
-        yield self.manager.save_stream(stream_hash, file_name, key, suggested_file_name, blobs)
-
-        out = yield self.manager.get_stream_info(stream_hash)
-        self.assertEqual(key, out[0])
-        self.assertEqual(file_name, out[1])
-        self.assertEqual(suggested_file_name, out[2])
-
-        out = yield self.manager.check_if_stream_exists(stream_hash)
-        self.assertTrue(out)
-
-        out = yield self.manager.get_blobs_for_stream(stream_hash)
-        self.assertEqual(2, len(out))
-
-        out = yield self.manager.get_all_streams()
-        self.assertEqual(1, len(out))
-
-        # add a blob to stream
-        blob3 = CryptBlobInfo(random_lbry_hash(), 0, 10, 1)
-        blobs = [blob3]
-        out = yield self.manager.add_blobs_to_stream(stream_hash, blobs)
-        out = yield self.manager.get_blobs_for_stream(stream_hash)
-        self.assertEqual(3, len(out))
-
-        out = yield self.manager.get_stream_of_blob(blob3.blob_hash)
-        self.assertEqual(stream_hash, out)
-
-        # check non existing stream
-        with self.assertRaises(NoSuchStreamHash):
-            out = yield self.manager.get_stream_info(random_lbry_hash())
-
-        # check save of sd blob hash
-        sd_blob_hash = random_lbry_hash()
-        yield self.manager.save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
-        out = yield self.manager.get_sd_blob_hashes_for_stream(stream_hash)
-        self.assertEqual(1, len(out))
-        self.assertEqual(sd_blob_hash, out[0])
-
-        out = yield self.manager.get_stream_hash_for_sd_hash(sd_blob_hash)
-        self.assertEqual(stream_hash, out)
-
-        # delete stream
-        yield self.manager.delete_stream(stream_hash)
-        out = yield self.manager.check_if_stream_exists(stream_hash)
-        self.assertFalse(out)

lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py

@@ -4,6 +4,7 @@ import mock
 from twisted.trial import unittest
 from twisted.internet import defer
 
+from lbrynet.database.storage import SQLiteStorage
 from lbrynet.core import BlobManager
 from lbrynet.core import Session
 from lbrynet.core.server import DHTHashAnnouncer
@@ -21,48 +22,60 @@ def iv_generator():
 
 class CreateEncryptedFileTest(unittest.TestCase):
     timeout = 5
+    @defer.inlineCallbacks
     def setUp(self):
         mocks.mock_conf_settings(self)
         self.tmp_db_dir, self.tmp_blob_dir = mk_db_and_blob_dir()
 
+        self.session = mock.Mock(spec=Session.Session)(None, None)
+        self.session.payment_rate_manager.min_blob_data_payment_rate = 0
+
+        hash_announcer = DHTHashAnnouncer.DHTHashAnnouncer(None, None)
+        self.blob_manager = BlobManager.DiskBlobManager(
+            hash_announcer, self.tmp_blob_dir, SQLiteStorage(self.tmp_db_dir))
+        self.session.blob_manager = self.blob_manager
+        self.session.storage = self.session.blob_manager.storage
+        self.file_manager = EncryptedFileManager.EncryptedFileManager(self.session, object())
+        yield self.session.blob_manager.storage.setup()
+        yield self.session.blob_manager.setup()
+
     @defer.inlineCallbacks
     def tearDown(self):
         yield self.blob_manager.stop()
+        yield self.session.storage.stop()
         rm_db_and_blob_dir(self.tmp_db_dir, self.tmp_blob_dir)
 
     @defer.inlineCallbacks
     def create_file(self, filename):
-        session = mock.Mock(spec=Session.Session)(None, None)
-        hash_announcer = DHTHashAnnouncer.DHTHashAnnouncer(None, None)
-        self.blob_manager = BlobManager.DiskBlobManager(
-            hash_announcer, self.tmp_blob_dir, self.tmp_db_dir)
-        session.blob_manager = self.blob_manager
-        yield session.blob_manager.setup()
-        session.db_dir = self.tmp_db_dir
-        manager = mock.Mock(spec=EncryptedFileManager.EncryptedFileManager)()
         handle = mocks.GenFile(3*MB, '1')
         key = '2'*AES.block_size
-        out = yield EncryptedFileCreator.create_lbry_file(
+        out = yield EncryptedFileCreator.create_lbry_file(self.session, self.file_manager, filename, handle,
-            session, manager, filename, handle, key, iv_generator())
+                                                          key, iv_generator())
         defer.returnValue(out)
 
     @defer.inlineCallbacks
     def test_can_create_file(self):
-        expected_stream_hash = ('41e6b247d923d191b154fb6f1b8529d6ddd6a73d65c357b1acb7'
+        expected_stream_hash = "41e6b247d923d191b154fb6f1b8529d6ddd6a73d65c35" \
-                                '42dd83151fb66393a7709e9f346260a4f4db6de10c25')
+                               "7b1acb742dd83151fb66393a7709e9f346260a4f4db6de10c25"
+        expected_sd_hash = "bc435ae0c4659635e6514e05bb1fcd0d365b234f6f0e78002" \
+                           "d2576ff84a0b8710a9847757a9aa8cbeda5a8e1aeafa48b"
         filename = 'test.file'
-        stream_hash = yield self.create_file(filename)
+        lbry_file = yield self.create_file(filename)
-        self.assertEqual(expected_stream_hash, stream_hash)
+        sd_hash = yield self.session.storage.get_sd_blob_hash_for_stream(lbry_file.stream_hash)
+
+        self.assertEqual(expected_stream_hash, lbry_file.stream_hash)
+        self.assertEqual(sd_hash, lbry_file.sd_hash)
+        self.assertEqual(sd_hash, expected_sd_hash)
 
         blobs = yield self.blob_manager.get_all_verified_blobs()
-        self.assertEqual(2, len(blobs))
+        self.assertEqual(3, len(blobs))
         num_should_announce_blobs = yield self.blob_manager.count_should_announce_blobs()
-        self.assertEqual(1, num_should_announce_blobs)
+        self.assertEqual(2, num_should_announce_blobs)
 
     @defer.inlineCallbacks
     def test_can_create_file_with_unicode_filename(self):
         expected_stream_hash = ('d1da4258f3ce12edb91d7e8e160d091d3ab1432c2e55a6352dce0'
                                 '2fd5adb86fe144e93e110075b5865fff8617776c6c0')
         filename = u'☃.file'
-        stream_hash = yield self.create_file(filename)
+        lbry_file = yield self.create_file(filename)
-        self.assertEqual(expected_stream_hash, stream_hash)
+        self.assertEqual(expected_stream_hash, lbry_file.stream_hash)

lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileManager.py

@@ -1,42 +0,0 @@
-from twisted.internet import defer
-from twisted.trial import unittest
-from lbrynet import conf
-from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
-from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
-from lbrynet.lbry_file.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager
-from lbrynet.tests.util import random_lbry_hash
-
-
-class TestEncryptedFileManager(unittest.TestCase):
-
-    def setUp(self):
-        conf.initialize_settings()
-
-    @defer.inlineCallbacks
-    def test_database_operations(self):
-        # test database read/write functions in EncrypteFileManager
-
-        class MocSession(object):
-            pass
-
-        session = MocSession()
-        session.db_dir = '.'
-        stream_info_manager = DBEncryptedFileMetadataManager('.')
-        sd_identifier = None
-        download_directory = '.'
-        manager = EncryptedFileManager(
-            session, stream_info_manager, sd_identifier, download_directory)
-        yield manager.stream_info_manager.setup()
-        out = yield manager._get_all_lbry_files()
-        self.assertEqual(len(out), 0)
-
-        stream_hash = random_lbry_hash()
-        blob_data_rate = 0
-        out = yield manager._save_lbry_file(stream_hash, blob_data_rate)
-        rowid = yield manager._get_rowid_for_stream_hash(stream_hash)
-        self.assertEqual(out, rowid)
-        files = yield manager._get_all_lbry_files()
-        self.assertEqual(1, len(files))
-        yield manager._change_file_status(rowid, ManagedEncryptedFileDownloader.STATUS_RUNNING)
-        out = yield manager._get_lbry_file_status(rowid)
-        self.assertEqual(out, ManagedEncryptedFileDownloader.STATUS_RUNNING)

lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py

@@ -6,8 +6,10 @@ from twisted.internet import defer
 from twisted import trial
 
 from lbryschema.decode import smart_decode
+from lbryum.wallet import NewWallet
 from lbrynet import conf
 from lbrynet.core import Session, PaymentRateManager, Wallet
+from lbrynet.database.storage import SQLiteStorage
 from lbrynet.daemon.Daemon import Daemon as LBRYDaemon
 
 from lbrynet.tests import util
@@ -33,6 +35,10 @@ def get_test_daemon(data_rate=None, generous=True, with_fee=False):
     daemon = LBRYDaemon(None)
     daemon.session = mock.Mock(spec=Session.Session)
     daemon.session.wallet = mock.Mock(spec=Wallet.LBRYumWallet)
+    daemon.session.wallet.wallet = mock.Mock(spec=NewWallet)
+    daemon.session.wallet.wallet.use_encryption = False
+    daemon.session.wallet.network = FakeNetwork()
+    daemon.session.storage = mock.Mock(spec=SQLiteStorage)
     market_feeds = [BTCLBCFeed(), USDBTCFeed()]
     daemon.exchange_rate_manager = DummyExchangeRateManager(market_feeds, rates)
     base_prm = PaymentRateManager.BasePaymentRateManager(rate=data_rate)
@@ -107,8 +113,7 @@ class TestJsonRpc(trial.unittest.TestCase):
         mock_conf_settings(self)
         util.resetTime(self)
         self.test_daemon = get_test_daemon()
-        self.test_daemon.session.wallet = Wallet.LBRYumWallet(storage=Wallet.InMemoryStorage())
+        self.test_daemon.session.wallet.is_first_run = False
-        self.test_daemon.session.wallet.network = FakeNetwork()
         self.test_daemon.session.wallet.get_best_blockhash = noop
 
     def test_status(self):

lbrynet/tests/unit/lbrynet_daemon/test_Downloader.py

@@ -39,22 +39,27 @@ class MocDownloader(object):
         self.stop_called = True
         self.finish_deferred.callback(True)
 
+
 def moc_initialize(self, stream_info):
     self.sd_hash = "d5169241150022f996fa7cd6a9a1c421937276a3275eb912" \
                    "790bd07ba7aec1fac5fd45431d226b8fb402691e79aeb24b"
     return None
 
+
 def moc_download_sd_blob(self):
     return None
 
-def moc_download(self, sd_blob, name, key_fee):
+
+def moc_download(self, sd_blob, name, txid, nout, key_fee, file_name):
     self.pay_key_fee(key_fee, name)
     self.downloader = MocDownloader()
     self.downloader.start()
 
+
 def moc_pay_key_fee(self, key_fee, name):
     self.pay_key_fee_called = True
 
+
 class GetStreamTests(unittest.TestCase):
 
     def init_getstream_with_mocs(self):
@@ -93,7 +98,7 @@ class GetStreamTests(unittest.TestCase):
         stream_info = None
 
         with self.assertRaises(AttributeError):
-            yield getstream.start(stream_info, name)
+            yield getstream.start(stream_info, name, "deadbeef" * 12, 0)
 
 
     @defer.inlineCallbacks
@@ -113,7 +118,7 @@ class GetStreamTests(unittest.TestCase):
         name = 'test'
         stream_info = None
         with self.assertRaises(DownloadSDTimeout):
-            yield getstream.start(stream_info, name)
+            yield getstream.start(stream_info, name, "deadbeef" * 12, 0)
         self.assertFalse(getstream.pay_key_fee_called)
 
     @defer.inlineCallbacks
@@ -129,7 +134,7 @@ class GetStreamTests(unittest.TestCase):
         getstream.pay_key_fee = types.MethodType(moc_pay_key_fee, getstream)
         name = 'test'
         stream_info = None
-        start = getstream.start(stream_info, name)
+        start = getstream.start(stream_info, name, "deadbeef" * 12, 0)
         self.clock.advance(1)
         self.clock.advance(1)
         self.clock.advance(1)
@@ -151,8 +156,7 @@ class GetStreamTests(unittest.TestCase):
         getstream.pay_key_fee = types.MethodType(moc_pay_key_fee, getstream)
         name = 'test'
         stream_info = None
-        start = getstream.start(stream_info, name)
+        start = getstream.start(stream_info, name, "deadbeef" * 12, 0)
-
         getstream.downloader.num_completed = 1
         self.clock.advance(1)