LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

pylint and tests

Browse source
This commit is contained in:
Jack Robison 2018-03-27 17:35:31 -04:00
parent 43d3f7c087
commit a41bbd5e27
No known key found for this signature in database
GPG key ID: DF25C68FE0239BB2
16 changed files with 66 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
lbrynet/core/BlobManager.py
View file

@ -59,7 +59,7 @@ class DiskBlobManager(object):
return defer.succeed(blob)
@defer.inlineCallbacks
def blob_completed(self, blob, next_announce_time=None, should_announce=True):
def blob_completed(self, blob, should_announce=False, next_announce_time=None):
yield self.storage.add_completed_blob(
blob.blob_hash, blob.length, next_announce_time, should_announce
)
@ -71,7 +71,8 @@ class DiskBlobManager(object):
return self.storage.count_should_announce_blobs()
def set_should_announce(self, blob_hash, should_announce):
return self.storage.set_should_announce(blob_hash, should_announce)
now = self.storage.clock.seconds()
return self.storage.set_should_announce(blob_hash, now, should_announce)
def get_should_announce(self, blob_hash):
return self.storage.should_announce(blob_hash)

2
lbrynet/daemon/Daemon.py
View file

@ -25,7 +25,7 @@ from lbryschema.decode import smart_decode
from lbrynet.core.system_info import get_lbrynet_version
from lbrynet.database.storage import SQLiteStorage
from lbrynet import conf
from lbrynet.conf import LBRYCRD_WALLET, LBRYUM_WALLET, PTC_WALLET
from lbrynet.conf import LBRYCRD_WALLET, LBRYUM_WALLET
from lbrynet.reflector import reupload
from lbrynet.reflector import ServerFactory as reflector_server_factory
from lbrynet.core.log_support import configure_loggly_handler

19
lbrynet/database/migrator/migrate6to7.py
View file

@ -1,24 +1,5 @@
import sqlite3
import os
import logging
from lbrynet.database.storage import SQLiteStorage
log = logging.getLogger(__name__)
def run_operation(db):
def _decorate(fn):
def _wrapper(*args):
cursor = db.cursor()
try:
result = fn(cursor, *args)
db.commit()
return result
except sqlite3.IntegrityError:
db.rollback()
raise
return _wrapper
return _decorate
def do_migration(db_dir):

1
lbrynet/database/storage.py
View file

@ -211,6 +211,7 @@ class SQLiteStorage(object):
)
def set_should_announce(self, blob_hash, next_announce_time, should_announce):
next_announce_time = next_announce_time or 0
should_announce = 1 if should_announce else 0
return self.db.runOperation(
"update blob set next_announce_time=?, should_announce=? where blob_hash=?",

2
lbrynet/dht/hashwatcher.py
View file

@ -1,6 +1,6 @@
from collections import Counter
import datetime
from twisted.internet import task, threads
from twisted.internet import task
class HashWatcher(object):

8
lbrynet/dht/node.py
View file

@ -23,7 +23,6 @@ import routingtable
import datastore
import protocol
from error import TimeoutError
from hashannouncer import DHTHashAnnouncer
from peerfinder import DHTPeerFinder
from contact import Contact
from hashwatcher import HashWatcher
@ -242,7 +241,12 @@ class Node(object):
return False
def announceHaveBlob(self, key):
return self.iterativeAnnounceHaveBlob(key, {'port': self.peerPort, 'lbryid': self.node_id})
return self.iterativeAnnounceHaveBlob(
key, {
'port': self.peerPort,
'lbryid': self.node_id,
}
)
@defer.inlineCallbacks
def getPeersForBlob(self, blob_hash):

15
lbrynet/tests/functional/test_dht.py
View file

@ -212,24 +212,11 @@ class TestKademliaBootstrapSixteenSeeds(TestKademliaBase):
@defer.inlineCallbacks
def tearDown(self):
yield TestKademliaBase.tearDown(self)
del self.seed_dns['lbrynet4.lbry.io']
del self.seed_dns['lbrynet5.lbry.io']
del self.seed_dns['lbrynet6.lbry.io']
del self.seed_dns['lbrynet7.lbry.io']
del self.seed_dns['lbrynet8.lbry.io']
del self.seed_dns['lbrynet9.lbry.io']
del self.seed_dns['lbrynet10.lbry.io']
del self.seed_dns['lbrynet11.lbry.io']
del self.seed_dns['lbrynet12.lbry.io']
del self.seed_dns['lbrynet13.lbry.io']
del self.seed_dns['lbrynet14.lbry.io']
del self.seed_dns['lbrynet15.lbry.io']
del self.seed_dns['lbrynet16.lbry.io']
def test_bootstrap_network(self):
pass
def test_all_nodes_are_pingable(self):
def _test_all_nodes_are_pingable(self):
return self.verify_all_nodes_are_pingable()

12
lbrynet/tests/functional/test_reflector.py
View file

@ -29,7 +29,6 @@ class TestReflector(unittest.TestCase):
wallet = mocks.Wallet()
peer_manager = PeerManager.PeerManager()
peer_finder = mocks.PeerFinder(5553, peer_manager, 2)
hash_announcer = mocks.Announcer()
sd_identifier = StreamDescriptor.StreamDescriptorIdentifier()
self.expected_blobs = [
@ -56,14 +55,14 @@ class TestReflector(unittest.TestCase):
db_dir=self.db_dir,
node_id="abcd",
peer_finder=peer_finder,
hash_announcer=hash_announcer,
blob_dir=self.blob_dir,
peer_port=5553,
use_upnp=False,
wallet=wallet,
blob_tracker_class=mocks.BlobAvailabilityTracker,
external_ip="127.0.0.1",
dht_node_class=Node
dht_node_class=Node,
hash_announcer=mocks.Announcer()
)
self.lbry_file_manager = EncryptedFileManager.EncryptedFileManager(self.session,
@ -76,18 +75,17 @@ class TestReflector(unittest.TestCase):
db_dir=self.server_db_dir,
node_id="abcd",
peer_finder=peer_finder,
hash_announcer=hash_announcer,
blob_dir=self.server_blob_dir,
peer_port=5553,
use_upnp=False,
wallet=wallet,
blob_tracker_class=mocks.BlobAvailabilityTracker,
external_ip="127.0.0.1",
dht_node_class=Node
dht_node_class=Node,
hash_announcer=mocks.Announcer()
)
self.server_blob_manager = BlobManager.DiskBlobManager(hash_announcer,
self.server_blob_dir,
self.server_blob_manager = BlobManager.DiskBlobManager(self.server_blob_dir,
self.server_session.storage)
self.server_lbry_file_manager = EncryptedFileManager.EncryptedFileManager(

12
lbrynet/tests/mocks.py
View file

@ -2,7 +2,7 @@ import struct
import io
from Crypto.PublicKey import RSA
from twisted.internet import defer, threads, error
from twisted.internet import defer, error
from twisted.python.failure import Failure
from lbrynet.core.client.ClientRequest import ClientRequest
@ -25,10 +25,10 @@ class FakeLBRYFile(object):
class Node(object):
def __init__(self, hash_announcer, peer_finder=None, peer_manager=None, **kwargs):
self.hash_announcer = hash_announcer
def __init__(self, peer_finder=None, peer_manager=None, **kwargs):
self.peer_finder = peer_finder
self.peer_manager = peer_manager
self.peerPort = 3333
def joinNetwork(self, *args):
return defer.succeed(True)
@ -77,7 +77,7 @@ class ExchangeRateManager(ERM.ExchangeRateManager):
feed.market, rates[feed.market]['spot'], rates[feed.market]['ts'])
class PointTraderKeyExchanger:
class PointTraderKeyExchanger(object):
def __init__(self, wallet):
self.wallet = wallet
@ -108,7 +108,7 @@ class PointTraderKeyExchanger:
return err
class PointTraderKeyQueryHandlerFactory:
class PointTraderKeyQueryHandlerFactory(object):
def __init__(self, wallet):
self.wallet = wallet
@ -125,7 +125,7 @@ class PointTraderKeyQueryHandlerFactory:
"point trader testing network")
class PointTraderKeyQueryHandler:
class PointTraderKeyQueryHandler(object):
def __init__(self, wallet):
self.wallet = wallet

52
lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py
View file

@ -1,34 +1,40 @@
import tempfile
import shutil
from twisted.trial import unittest
from twisted.internet import defer, reactor
from twisted.internet import defer, reactor, threads
from lbrynet.tests.util import random_lbry_hash
from lbrynet.dht.hashannouncer import DHTHashAnnouncer
from lbrynet.core.call_later_manager import CallLaterManager
from lbrynet.database.storage import SQLiteStorage
class MocDHTNode(object):
def __init__(self, announce_will_fail=False):
# if announce_will_fail is True,
# announceHaveBlob will return empty dict
self.can_store = True
self.call_later_manager = CallLaterManager
self.call_later_manager.setup(reactor.callLater)
self.blobs_announced = 0
self.announce_will_fail = announce_will_fail
@defer.inlineCallbacks
def announceHaveBlob(self, blob):
if self.announce_will_fail:
return_val = {}
else:
return_val = {blob:["ab"*48]}
return_val = {blob: ["ab"*48]}
self.blobs_announced += 1
d = defer.Deferred(None)
reactor.callLater(1, d.callback, return_val)
result = yield d
defer.returnValue(result)
d = defer.Deferred()
self.call_later_manager.call_later(1, d.callback, return_val)
return d
class DHTHashAnnouncerTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
from lbrynet.conf import initialize_settings
initialize_settings()
self.num_blobs = 10
self.blobs_to_announce = []
for i in range(0, self.num_blobs):
@ -36,31 +42,41 @@ class DHTHashAnnouncerTest(unittest.TestCase):
self.dht_node = MocDHTNode()
self.dht_node.peerPort = 3333
self.dht_node.clock = reactor
self.announcer = DHTHashAnnouncer(self.dht_node)
yield self.announcer.add_hashes_to_announce(self.blobs_to_announce)
self.db_dir = tempfile.mkdtemp()
self.storage = SQLiteStorage(self.db_dir)
yield self.storage.setup()
self.announcer = DHTHashAnnouncer(self.dht_node, self.storage, 10)
for blob_hash in self.blobs_to_announce:
yield self.storage.add_completed_blob(blob_hash, 100, 0, 1)
@defer.inlineCallbacks
def tearDown(self):
self.dht_node.call_later_manager.stop()
yield self.storage.stop()
yield threads.deferToThread(shutil.rmtree, self.db_dir)
@defer.inlineCallbacks
def test_announce_fail(self):
# test what happens when node.announceHaveBlob() returns empty dict
self.dht_node.announce_will_fail = True
d = yield self.announcer._announce_available_hashes()
d = yield self.announcer.manage()
yield d
@defer.inlineCallbacks
def test_basic(self):
d = self.announcer._announce_available_hashes()
self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS)
d = self.announcer.immediate_announce(self.blobs_to_announce)
self.assertEqual(len(self.announcer.hash_queue), self.num_blobs)
yield d
self.assertEqual(self.dht_node.blobs_announced, self.num_blobs)
self.assertEqual(self.announcer.hash_queue_size(), 0)
self.assertEqual(len(self.announcer.hash_queue), 0)
@defer.inlineCallbacks
def test_immediate_announce(self):
# Test that immediate announce puts a hash at the front of the queue
d = self.announcer._announce_available_hashes()
self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS)
d = self.announcer.immediate_announce(self.blobs_to_announce)
self.assertEqual(len(self.announcer.hash_queue), self.num_blobs)
blob_hash = random_lbry_hash()
self.announcer.immediate_announce([blob_hash])
self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS+1)
self.assertEqual(blob_hash, self.announcer.hash_queue[0][0])
self.assertEqual(len(self.announcer.hash_queue), self.num_blobs+1)
self.assertEqual(blob_hash, self.announcer.hash_queue[-1])
yield d

4
lbrynet/tests/unit/core/test_BlobManager.py
View file

@ -8,7 +8,6 @@ from twisted.internet import defer, threads
from lbrynet.tests.util import random_lbry_hash
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.dht.hashannouncer import DummyHashAnnouncer
from lbrynet.database.storage import SQLiteStorage
from lbrynet.core.Peer import Peer
from lbrynet import conf
@ -21,8 +20,7 @@ class BlobManagerTest(unittest.TestCase):
conf.initialize_settings()
self.blob_dir = tempfile.mkdtemp()
self.db_dir = tempfile.mkdtemp()
hash_announcer = DummyHashAnnouncer()
self.bm = DiskBlobManager(hash_announcer, self.blob_dir, SQLiteStorage(self.db_dir))
self.bm = DiskBlobManager(self.blob_dir, SQLiteStorage(self.db_dir))
self.peer = Peer('somehost', 22)
yield self.bm.storage.setup()

2
lbrynet/tests/unit/database/test_SQLiteStorage.py
View file

@ -195,7 +195,7 @@ class StreamStorageTests(StorageTest):
should_announce_count = yield self.storage.count_should_announce_blobs()
self.assertEqual(should_announce_count, 2)
should_announce_hashes = yield self.storage.get_blobs_to_announce(FakeAnnouncer())
should_announce_hashes = yield self.storage.get_blobs_to_announce()
self.assertSetEqual(set(should_announce_hashes), {sd_hash, blob1})
stream_hashes = yield self.storage.get_all_streams()

2
lbrynet/tests/unit/dht/test_node.py
View file

@ -198,7 +198,7 @@ class NodeLookupTest(unittest.TestCase):
h = hashlib.sha384()
h.update('node1')
node_id = str(h.digest())
self.node = lbrynet.dht.node.Node(None, node_id=node_id, udpPort=4000, networkProtocol=self._protocol)
self.node = lbrynet.dht.node.Node(node_id=node_id, udpPort=4000, networkProtocol=self._protocol)
self.updPort = 81173
self.contactsAmount = 80
# Reinitialise the routing table

2
lbrynet/tests/unit/dht/test_protocol.py
View file

@ -1,7 +1,7 @@
import time
import unittest
from twisted.internet.task import Clock
from twisted.internet import defer, threads
from twisted.internet import defer
import lbrynet.dht.protocol
import lbrynet.dht.contact
import lbrynet.dht.constants

8
lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py
View file

@ -2,18 +2,15 @@
from Crypto.Cipher import AES
import mock
from twisted.trial import unittest
from twisted.internet import defer, reactor
from twisted.internet import defer
from lbrynet.database.storage import SQLiteStorage
from lbrynet.core.StreamDescriptor import get_sd_info, BlobStreamDescriptorReader
from lbrynet.core import BlobManager
from lbrynet.core import Session
from lbrynet.dht import hashannouncer
from lbrynet.dht.node import Node
from lbrynet.file_manager import EncryptedFileCreator
from lbrynet.file_manager import EncryptedFileManager
from lbrynet.tests import mocks
from time import time
from lbrynet.tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir
MB = 2**20
@ -34,8 +31,7 @@ class CreateEncryptedFileTest(unittest.TestCase):
self.session = mock.Mock(spec=Session.Session)(None, None)
self.session.payment_rate_manager.min_blob_data_payment_rate = 0
self.blob_manager = BlobManager.DiskBlobManager(
hashannouncer.DummyHashAnnouncer(), self.tmp_blob_dir, SQLiteStorage(self.tmp_db_dir))
self.blob_manager = BlobManager.DiskBlobManager(self.tmp_blob_dir, SQLiteStorage(self.tmp_db_dir))
self.session.blob_manager = self.blob_manager
self.session.storage = self.session.blob_manager.storage
self.file_manager = EncryptedFileManager.EncryptedFileManager(self.session, object())

4
lbrynet/tests/util.py
View file

@ -67,8 +67,8 @@ def debug_kademlia_packet(data, source, destination, node):
log.debug("response %s <-- %s %s (node time %s)", destination[0], source[0], packet.response,
node.clock.seconds())
else:
log.debug("response %s <-- %s %i contacts (node time %s)", destination[0], source[0],
len(packet.response), node.clock.seconds())
log.debug("response %s <-- %s %i contacts (node time %s)", destination[0], source[0],
len(packet.response), node.clock.seconds())
elif isinstance(packet, ErrorMessage):
log.error("error %s <-- %s %s (node time %s)", destination[0], source[0], packet.exceptionType,
node.clock.seconds())