diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b89a203c..8db7731d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,14 +15,16 @@ at anytime. ### Fixed * fixed the inconsistencies in API and CLI docstrings * `blob_announce` error when announcing a single blob + * `blob_list` error when looking up blobs by stream or sd hash ### Deprecated * `report_bug` jsonrpc command * ### Changed - * - * + * reflector server to periodically check and set `should_announce` for sd and head blobs instead of during each request + * reflector server to use `SQLiteStorage` to find needed blob hashes for a stream + ### Added * scripts to autogenerate documentation * diff --git a/lbrynet/core/BlobManager.py b/lbrynet/core/BlobManager.py index a9142b369..520e6bd0f 100644 --- a/lbrynet/core/BlobManager.py +++ b/lbrynet/core/BlobManager.py @@ -1,7 +1,7 @@ import logging import os from sqlite3 import IntegrityError -from twisted.internet import threads, defer, reactor +from twisted.internet import threads, defer, reactor, task from lbrynet import conf from lbrynet.blob.blob_file import BlobFile from lbrynet.blob.creator import BlobFileCreator @@ -29,10 +29,18 @@ class DiskBlobManager(DHTHashSupplier): self.blobs = {} self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)} + self.check_should_announce_lc = None + if conf.settings['run_reflector_server']: + self.check_should_announce_lc = task.LoopingCall(self.storage.verify_will_announce_all_head_and_sd_blobs) + def setup(self): + if self.check_should_announce_lc and not self.check_should_announce_lc.running: + self.check_should_announce_lc.start(600) return defer.succeed(True) def stop(self): + if self.check_should_announce_lc and self.check_should_announce_lc.running: + self.check_should_announce_lc.stop() return defer.succeed(True) def get_blob(self, blob_hash, length=None): diff --git a/lbrynet/core/Session.py b/lbrynet/core/Session.py index a0d8eaa5d..2f8cf42b2 100644 --- a/lbrynet/core/Session.py +++ b/lbrynet/core/Session.py @@ -331,6 +331,7 @@ class Session(object): self.rate_limiter.start() d = self.storage.setup() + d.addCallback(lambda _: self.blob_manager.setup()) d.addCallback(lambda _: self.wallet.start()) d.addCallback(lambda _: self.blob_tracker.start()) return d diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 3e11b0354..516264633 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -3031,7 +3031,8 @@ class Daemon(AuthJSONRPCServer): stream_hash = yield self.session.storage.get_stream_hash_for_sd_hash(sd_hash) sd_hash = yield self.session.storage.get_sd_blob_hash_for_stream(stream_hash) if stream_hash: - blobs = yield self.session.storage.get_blobs_for_stream(stream_hash) + crypt_blobs = yield self.session.storage.get_blobs_for_stream(stream_hash) + blobs = [self.session.blob_manager.blobs[crypt_blob.blob_hash] for crypt_blob in crypt_blobs] else: blobs = [] # get_blobs_for_stream does not include the sd blob, so we'll add it manually diff --git a/lbrynet/database/storage.py b/lbrynet/database/storage.py index 59cf1df29..48d4a4079 100644 --- a/lbrynet/database/storage.py +++ b/lbrynet/database/storage.py @@ -261,12 +261,12 @@ class SQLiteStorage(object): def count_should_announce_blobs(self): return self.run_and_return_one_or_none( - "select count(*) from blob where should_announce=1 and status=?", "finished" + "select count(*) from blob where should_announce=1 and status='finished'" ) def get_all_should_announce_blobs(self): return self.run_and_return_list( - "select blob_hash from blob where should_announce=1 and status=?", "finished" + "select blob_hash from blob where should_announce=1 and status='finished'" ) def get_blobs_to_announce(self, hash_announcer): @@ -275,12 +275,13 @@ class SQLiteStorage(object): if conf.settings['announce_head_blobs_only']: r = transaction.execute( "select blob_hash from blob " - "where blob_hash is not null and should_announce=1 and next_announce_time