LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

resume download from S3 instead of starting from scratch

Browse source
This commit is contained in:
Victor Shyba 2018-05-05 02:05:02 -03:00
parent 3982e15091
commit 0bf65836c7
1 changed files with 21 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
lbrynet/core/Wallet.py
View file

@ -93,13 +93,23 @@ class Wallet(object):
@defer.inlineCallbacks @defer.inlineCallbacks
def fetch_headers_from_s3(self): def fetch_headers_from_s3(self):
response = yield treq.get(HEADERS_URL) local_header_size = self.local_header_file_size()
if not response.length % HEADER_SIZE: # should be divisible by the header size resume_header = {"Range": "bytes={}-".format(local_header_size)}
s3_height = (response.length / HEADER_SIZE) - 1 response = yield treq.get(HEADERS_URL, headers=resume_header)
got_406 = response.code == 406 # our file is bigger
final_size_after_download = response.length + local_header_size
# should have something to download and a final length divisible by the header size
if not got_406 and final_size_after_download and not final_size_after_download % HEADER_SIZE:
s3_height = (final_size_after_download / HEADER_SIZE) - 1
local_height = self.local_header_file_height() local_height = self.local_header_file_height()
if s3_height > local_height: if s3_height > local_height:
with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file: if local_header_size:
yield treq.collect(response, headers_file.write) log.info("Resuming download of %i bytes from s3", response.length)
with open(os.path.join(self.config.path, "blockchain_headers"), "a+b") as headers_file:
yield treq.collect(response, headers_file.write)
else:
with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file:
yield treq.collect(response, headers_file.write)
log.info("fetched headers from s3 (s3 height: %i)", s3_height) log.info("fetched headers from s3 (s3 height: %i)", s3_height)
else: else:
log.warning("s3 is more out of date than we are") log.warning("s3 is more out of date than we are")
@ -107,11 +117,15 @@ class Wallet(object):
log.error("invalid size for headers from s3") log.error("invalid size for headers from s3")
def local_header_file_height(self): def local_header_file_height(self):
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
def local_header_file_size(self):
headers_path = os.path.join(self.config.path, "blockchain_headers") headers_path = os.path.join(self.config.path, "blockchain_headers")
if os.path.isfile(headers_path): if os.path.isfile(headers_path):
return max((os.stat(headers_path).st_size / 112) - 1, 0) return os.stat(headers_path).st_size
return 0 return 0
@defer.inlineCallbacks @defer.inlineCallbacks
def get_remote_height(self, server, port): def get_remote_height(self, server, port):
connected = defer.Deferred() connected = defer.Deferred()
@ -136,7 +150,7 @@ class Wallet(object):
try: try:
remote_height = yield self.get_remote_height(server_url, port) remote_height = yield self.get_remote_height(server_url, port)
log.info("%s:%i height: %i, local height: %s", server_url, port, remote_height, local_height) log.info("%s:%i height: %i, local height: %s", server_url, port, remote_height, local_height)
if remote_height > local_height + s3_headers_depth: if remote_height > (local_height + s3_headers_depth):
defer.returnValue(True) defer.returnValue(True)
except Exception as err: except Exception as err:
log.warning("error requesting remote height from %s:%i - %s", server_url, port, err) log.warning("error requesting remote height from %s:%i - %s", server_url, port, err)