From 7ad34475987c9d61dfeb721b1e865cf5541609b5 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor1984@riseup.net>
Date: Thu, 26 Mar 2020 14:25:25 -0300
Subject: [PATCH 1/3] repair tip on open

---
 lbry/wallet/header.py             | 18 +++++++++++++-----
 tests/unit/wallet/test_headers.py | 22 ++++++++++++++++++++++
 2 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py
index 554782c00..0daf49cc1 100644
--- a/lbry/wallet/header.py
+++ b/lbry/wallet/header.py
@@ -59,7 +59,15 @@ class Headers:
                 self.io = open(self.path, 'w+b')
             else:
                 self.io = open(self.path, 'r+b')
-        self._size = self.io.seek(0, os.SEEK_END) // self.header_size
+        bytes_size = self.io.seek(0, os.SEEK_END)
+        self._size = bytes_size // self.header_size
+        max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
+        if bytes_size % self.header_size:
+            log.warning("Reader file size doesnt match header size. Repairing, might take a while.")
+            await self.repair()
+        else:
+            # try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok)
+            await self.repair(start_height=max_checkpointed_height)
         await self.ensure_checkpointed_size()
         await self.get_all_missing_headers()
 
@@ -292,16 +300,16 @@ class Headers:
                     height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
                 )
 
-    async def repair(self):
+    async def repair(self, start_height=0):
         previous_header_hash = fail = None
         batch_size = 36
-        for start_height in range(0, self.height, batch_size):
+        for height in range(start_height, self.height, batch_size):
             headers = await asyncio.get_running_loop().run_in_executor(
-                self.executor, self._read, start_height, batch_size
+                self.executor, self._read, height, batch_size
             )
             if len(headers) % self.header_size != 0:
                 headers = headers[:(len(headers) // self.header_size) * self.header_size]
-            for header_hash, header in self._iterate_headers(start_height, headers):
+            for header_hash, header in self._iterate_headers(height, headers):
                 height = header['block_height']
                 if height:
                     if header['prev_block_hash'] != previous_header_hash:
diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py
index 52dfc2117..425d825e3 100644
--- a/tests/unit/wallet/test_headers.py
+++ b/tests/unit/wallet/test_headers.py
@@ -144,6 +144,28 @@ class TestHeaders(AsyncioTestCase):
         await headers.connect(len(headers), HEADERS[block_bytes(8):])
         self.assertEqual(19, headers.height)
 
+    async def test_misalignment_triggers_repair_on_open(self):
+        headers = Headers(':memory:')
+        headers.io.seek(0)
+        headers.io.write(HEADERS)
+        with self.assertLogs(level='WARN') as cm:
+            await headers.open()
+            self.assertEqual(cm.output, [])
+            headers.io.seek(0)
+            headers.io.truncate()
+            headers.io.write(HEADERS[:block_bytes(10)])
+            headers.io.write(b'ops')
+            headers.io.write(HEADERS[block_bytes(10):])
+            await headers.open()
+            self.assertEqual(
+                cm.output, [
+                    'WARNING:lbry.wallet.header:Reader file size doesnt match header size. '
+                    'Repairing, might take a while.',
+                    'WARNING:lbry.wallet.header:Header file corrupted at height 9, truncating '
+                    'it.'
+                ]
+            )
+
     async def test_concurrency(self):
         BLOCKS = 19
         headers_temporary_file = tempfile.mktemp()

From d2fb7a7151f497ce9366580a48522e413e3a0e63 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor1984@riseup.net>
Date: Thu, 26 Mar 2020 14:25:50 -0300
Subject: [PATCH 2/3] lock only when fetching, giving a chance for tip updates

---
 lbry/wallet/ledger.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py
index e5eeac41e..eed6155f8 100644
--- a/lbry/wallet/ledger.py
+++ b/lbry/wallet/ledger.py
@@ -354,8 +354,8 @@ class Ledger(metaclass=LedgerRegistry):
         self.headers.chunk_getter = get_chunk
 
         async def doit():
-            async with self._header_processing_lock:
-                for height in reversed(sorted(self.headers.known_missing_checkpointed_chunks)):
+            for height in reversed(sorted(self.headers.known_missing_checkpointed_chunks)):
+                async with self._header_processing_lock:
                     await self.headers.ensure_chunk_at(height)
         self._other_tasks.add(doit())
         await self.update_headers()

From 1b83a1d09a12e0bf2925ef47ae734fe23be69779 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor1984@riseup.net>
Date: Fri, 27 Mar 2020 22:53:55 -0300
Subject: [PATCH 3/3] test and fix verifying from middle

---
 lbry/wallet/header.py             | 7 +++++--
 tests/unit/wallet/test_headers.py | 3 +++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py
index 0daf49cc1..2fa066cdf 100644
--- a/lbry/wallet/header.py
+++ b/lbry/wallet/header.py
@@ -311,12 +311,15 @@ class Headers:
                 headers = headers[:(len(headers) // self.header_size) * self.header_size]
             for header_hash, header in self._iterate_headers(height, headers):
                 height = header['block_height']
-                if height:
+                if previous_header_hash:
                     if header['prev_block_hash'] != previous_header_hash:
                         fail = True
-                else:
+                elif height == 0:
                     if header_hash != self.genesis_hash:
                         fail = True
+                else:
+                    # for sanity and clarity, since it is the only way we can end up here
+                    assert start_height > 0 and height == start_height
                 if fail:
                     log.warning("Header file corrupted at height %s, truncating it.", height - 1)
                     def __truncate(at_height):
diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py
index 425d825e3..5ebb333c3 100644
--- a/tests/unit/wallet/test_headers.py
+++ b/tests/unit/wallet/test_headers.py
@@ -143,6 +143,9 @@ class TestHeaders(AsyncioTestCase):
         self.assertEqual(7, headers.height)
         await headers.connect(len(headers), HEADERS[block_bytes(8):])
         self.assertEqual(19, headers.height)
+        # verify from middle
+        await headers.repair(start_height=10)
+        self.assertEqual(19, headers.height)
 
     async def test_misalignment_triggers_repair_on_open(self):
         headers = Headers(':memory:')