From d0607b6fec05c3dd648b2f02d715ea29c66e4b03 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor1984@riseup.net>
Date: Tue, 9 Jul 2019 15:00:29 -0300
Subject: [PATCH] make header component use torba headers
---
lbry/lbry/extras/daemon/Components.py | 70 ++++++++++-----------------
torba/torba/client/baseheader.py | 4 ++
2 files changed, 29 insertions(+), 45 deletions(-)
diff --git a/lbry/lbry/extras/daemon/Components.py b/lbry/lbry/extras/daemon/Components.py
index 55621aeda..c0b858865 100644
--- a/lbry/lbry/extras/daemon/Components.py
+++ b/lbry/lbry/extras/daemon/Components.py
@@ -4,7 +4,6 @@ import logging
import math
import binascii
import typing
-from hashlib import sha256
import base58
from aioupnp import __version__ as aioupnp_version
@@ -12,7 +11,6 @@ from aioupnp.upnp import UPnP
from aioupnp.fault import UPnPError
from lbry import utils
-from lbry.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbry.dht.node import Node
from lbry.dht.blob_announcer import BlobAnnouncer
from lbry.blob.blob_manager import BlobManager
@@ -22,7 +20,7 @@ from lbry.extras.daemon.Component import Component
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import LbryWalletManager
-
+from lbry.wallet.header import Headers
log = logging.getLogger(__name__)
@@ -110,6 +108,7 @@ class HeadersComponent(Component):
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
self.headers_file = os.path.join(self.headers_dir, 'headers')
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
+ self.headers = Headers(self.headers_file)
self._downloading_headers = None
self._headers_progress_percent = 0
@@ -137,7 +136,7 @@ class HeadersComponent(Component):
} if progress < 100 else {}
async def fetch_headers_from_s3(self):
- local_header_size = self.local_header_file_size()
+ local_header_size = self.headers.bytes_size
resume_header = {"Range": f"bytes={local_header_size}-"}
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
@@ -147,24 +146,21 @@ class HeadersComponent(Component):
log.warning("s3 appears to have corrupted header")
return
final_size_after_download = response.content_length + local_header_size
- write_mode = "wb"
if local_header_size > 0:
log.info("Resuming download of %i bytes from s3", response.content_length)
- write_mode = "a+b"
- with open(self.headers_file, write_mode) as fd:
- while True:
- chunk = await response.content.read(512)
- if not chunk:
- break
- fd.write(chunk)
- self._headers_progress_percent = math.ceil(
- float(fd.tell()) / float(final_size_after_download) * 100
- )
- log.info("fetched headers from s3, now verifying integrity after download.")
- self._check_header_file_integrity()
-
- def local_header_file_height(self):
- return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
+ buffer, header_size = b'', self.headers.header_size
+ async for chunk in response.content.iter_any():
+ chunk = buffer + chunk
+ remaining = len(chunk) % header_size
+ chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
+ if not chunk:
+ continue
+ if not await self.headers.connect(len(self.headers), chunk):
+ log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
+ return
+ self._headers_progress_percent = math.ceil(
+ float(self.headers.bytes_size) / float(final_size_after_download) * 100
+ )
def local_header_file_size(self):
if os.path.isfile(self.headers_file):
@@ -181,46 +177,29 @@ class HeadersComponent(Component):
async def should_download_headers_from_s3(self):
if self.conf.blockchain_name != "lbrycrd_main":
return False
- self._check_header_file_integrity()
s3_headers_depth = self.conf.s3_headers_depth
if not s3_headers_depth:
return False
- local_height = self.local_header_file_height()
- remote_height = await self.get_download_height()
+ local_height = self.headers.height
+ try:
+ remote_height = await self.get_download_height()
+ except OSError:
+ log.warning("Failed to download headers using https.")
+ return False
log.info("remote height: %i, local height: %i", remote_height, local_height)
if remote_height > (local_height + s3_headers_depth):
return True
return False
- def _check_header_file_integrity(self):
- # TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
- if self.conf.blockchain_name != "lbrycrd_main":
- return
- hashsum = sha256()
- checksum_height, checksum = HEADERS_FILE_SHA256_CHECKSUM
- checksum_length_in_bytes = checksum_height * HEADER_SIZE
- if self.local_header_file_size() < checksum_length_in_bytes:
- return
- with open(self.headers_file, "rb") as headers_file:
- hashsum.update(headers_file.read(checksum_length_in_bytes))
- current_checksum = hashsum.hexdigest()
- if current_checksum != checksum:
- msg = f"Expected checksum {checksum}, got {current_checksum}"
- log.warning("Headers file corrupted, checksum mismatch. " + msg)
- log.warning("Deleting header file so it can be downloaded again.")
- os.unlink(self.headers_file)
- elif (self.local_header_file_size() % HEADER_SIZE) != 0:
- log.warning("Header file is good up to checkpoint height, but incomplete. Truncating to checkpoint.")
- with open(self.headers_file, "rb+") as headers_file:
- headers_file.truncate(checksum_length_in_bytes)
-
async def start(self):
if not os.path.exists(self.headers_dir):
os.mkdir(self.headers_dir)
if os.path.exists(self.old_file):
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
os.rename(self.old_file, self.headers_file)
+ await self.headers.open()
+ self.headers.repair()
self._downloading_headers = await self.should_download_headers_from_s3()
if self._downloading_headers:
try:
@@ -229,6 +208,7 @@ class HeadersComponent(Component):
log.error("failed to fetch headers from s3: %s", err)
finally:
self._downloading_headers = False
+ await self.headers.close()
async def stop(self):
pass
diff --git a/torba/torba/client/baseheader.py b/torba/torba/client/baseheader.py
index c9b8e603c..59541d4ab 100644
--- a/torba/torba/client/baseheader.py
+++ b/torba/torba/client/baseheader.py
@@ -82,6 +82,10 @@ class BaseHeaders:
def height(self) -> int:
return len(self)-1
+ @property
+ def bytes_size(self):
+ return len(self) * self.header_size
+
def hash(self, height=None) -> bytes:
return self.hash_header(
self.get_raw_header(height if height is not None else self.height)