From 7ad34475987c9d61dfeb721b1e865cf5541609b5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 26 Mar 2020 14:25:25 -0300 Subject: [PATCH 1/3] repair tip on open --- lbry/wallet/header.py | 18 +++++++++++++----- tests/unit/wallet/test_headers.py | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 554782c00..0daf49cc1 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -59,7 +59,15 @@ class Headers: self.io = open(self.path, 'w+b') else: self.io = open(self.path, 'r+b') - self._size = self.io.seek(0, os.SEEK_END) // self.header_size + bytes_size = self.io.seek(0, os.SEEK_END) + self._size = bytes_size // self.header_size + max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000 + if bytes_size % self.header_size: + log.warning("Reader file size doesnt match header size. Repairing, might take a while.") + await self.repair() + else: + # try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok) + await self.repair(start_height=max_checkpointed_height) await self.ensure_checkpointed_size() await self.get_all_missing_headers() @@ -292,16 +300,16 @@ class Headers: height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}" ) - async def repair(self): + async def repair(self, start_height=0): previous_header_hash = fail = None batch_size = 36 - for start_height in range(0, self.height, batch_size): + for height in range(start_height, self.height, batch_size): headers = await asyncio.get_running_loop().run_in_executor( - self.executor, self._read, start_height, batch_size + self.executor, self._read, height, batch_size ) if len(headers) % self.header_size != 0: headers = headers[:(len(headers) // self.header_size) * self.header_size] - for header_hash, header in self._iterate_headers(start_height, headers): + for header_hash, header in self._iterate_headers(height, headers): height = header['block_height'] if height: if header['prev_block_hash'] != previous_header_hash: diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py index 52dfc2117..425d825e3 100644 --- a/tests/unit/wallet/test_headers.py +++ b/tests/unit/wallet/test_headers.py @@ -144,6 +144,28 @@ class TestHeaders(AsyncioTestCase): await headers.connect(len(headers), HEADERS[block_bytes(8):]) self.assertEqual(19, headers.height) + async def test_misalignment_triggers_repair_on_open(self): + headers = Headers(':memory:') + headers.io.seek(0) + headers.io.write(HEADERS) + with self.assertLogs(level='WARN') as cm: + await headers.open() + self.assertEqual(cm.output, []) + headers.io.seek(0) + headers.io.truncate() + headers.io.write(HEADERS[:block_bytes(10)]) + headers.io.write(b'ops') + headers.io.write(HEADERS[block_bytes(10):]) + await headers.open() + self.assertEqual( + cm.output, [ + 'WARNING:lbry.wallet.header:Reader file size doesnt match header size. ' + 'Repairing, might take a while.', + 'WARNING:lbry.wallet.header:Header file corrupted at height 9, truncating ' + 'it.' + ] + ) + async def test_concurrency(self): BLOCKS = 19 headers_temporary_file = tempfile.mktemp() From d2fb7a7151f497ce9366580a48522e413e3a0e63 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 26 Mar 2020 14:25:50 -0300 Subject: [PATCH 2/3] lock only when fetching, giving a chance for tip updates --- lbry/wallet/ledger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index e5eeac41e..eed6155f8 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -354,8 +354,8 @@ class Ledger(metaclass=LedgerRegistry): self.headers.chunk_getter = get_chunk async def doit(): - async with self._header_processing_lock: - for height in reversed(sorted(self.headers.known_missing_checkpointed_chunks)): + for height in reversed(sorted(self.headers.known_missing_checkpointed_chunks)): + async with self._header_processing_lock: await self.headers.ensure_chunk_at(height) self._other_tasks.add(doit()) await self.update_headers() From 1b83a1d09a12e0bf2925ef47ae734fe23be69779 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 27 Mar 2020 22:53:55 -0300 Subject: [PATCH 3/3] test and fix verifying from middle --- lbry/wallet/header.py | 7 +++++-- tests/unit/wallet/test_headers.py | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 0daf49cc1..2fa066cdf 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -311,12 +311,15 @@ class Headers: headers = headers[:(len(headers) // self.header_size) * self.header_size] for header_hash, header in self._iterate_headers(height, headers): height = header['block_height'] - if height: + if previous_header_hash: if header['prev_block_hash'] != previous_header_hash: fail = True - else: + elif height == 0: if header_hash != self.genesis_hash: fail = True + else: + # for sanity and clarity, since it is the only way we can end up here + assert start_height > 0 and height == start_height if fail: log.warning("Header file corrupted at height %s, truncating it.", height - 1) def __truncate(at_height): diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py index 425d825e3..5ebb333c3 100644 --- a/tests/unit/wallet/test_headers.py +++ b/tests/unit/wallet/test_headers.py @@ -143,6 +143,9 @@ class TestHeaders(AsyncioTestCase): self.assertEqual(7, headers.height) await headers.connect(len(headers), HEADERS[block_bytes(8):]) self.assertEqual(19, headers.height) + # verify from middle + await headers.repair(start_height=10) + self.assertEqual(19, headers.height) async def test_misalignment_triggers_repair_on_open(self): headers = Headers(':memory:')