From 7ad34475987c9d61dfeb721b1e865cf5541609b5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 26 Mar 2020 14:25:25 -0300 Subject: [PATCH] repair tip on open --- lbry/wallet/header.py | 18 +++++++++++++----- tests/unit/wallet/test_headers.py | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 554782c00..0daf49cc1 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -59,7 +59,15 @@ class Headers: self.io = open(self.path, 'w+b') else: self.io = open(self.path, 'r+b') - self._size = self.io.seek(0, os.SEEK_END) // self.header_size + bytes_size = self.io.seek(0, os.SEEK_END) + self._size = bytes_size // self.header_size + max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000 + if bytes_size % self.header_size: + log.warning("Reader file size doesnt match header size. Repairing, might take a while.") + await self.repair() + else: + # try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok) + await self.repair(start_height=max_checkpointed_height) await self.ensure_checkpointed_size() await self.get_all_missing_headers() @@ -292,16 +300,16 @@ class Headers: height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}" ) - async def repair(self): + async def repair(self, start_height=0): previous_header_hash = fail = None batch_size = 36 - for start_height in range(0, self.height, batch_size): + for height in range(start_height, self.height, batch_size): headers = await asyncio.get_running_loop().run_in_executor( - self.executor, self._read, start_height, batch_size + self.executor, self._read, height, batch_size ) if len(headers) % self.header_size != 0: headers = headers[:(len(headers) // self.header_size) * self.header_size] - for header_hash, header in self._iterate_headers(start_height, headers): + for header_hash, header in self._iterate_headers(height, headers): height = header['block_height'] if height: if header['prev_block_hash'] != previous_header_hash: diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py index 52dfc2117..425d825e3 100644 --- a/tests/unit/wallet/test_headers.py +++ b/tests/unit/wallet/test_headers.py @@ -144,6 +144,28 @@ class TestHeaders(AsyncioTestCase): await headers.connect(len(headers), HEADERS[block_bytes(8):]) self.assertEqual(19, headers.height) + async def test_misalignment_triggers_repair_on_open(self): + headers = Headers(':memory:') + headers.io.seek(0) + headers.io.write(HEADERS) + with self.assertLogs(level='WARN') as cm: + await headers.open() + self.assertEqual(cm.output, []) + headers.io.seek(0) + headers.io.truncate() + headers.io.write(HEADERS[:block_bytes(10)]) + headers.io.write(b'ops') + headers.io.write(HEADERS[block_bytes(10):]) + await headers.open() + self.assertEqual( + cm.output, [ + 'WARNING:lbry.wallet.header:Reader file size doesnt match header size. ' + 'Repairing, might take a while.', + 'WARNING:lbry.wallet.header:Header file corrupted at height 9, truncating ' + 'it.' + ] + ) + async def test_concurrency(self): BLOCKS = 19 headers_temporary_file = tempfile.mktemp()