add script that generates checkpoints

lbry/wallet/checkpoints.py

@@ -732,4 +732,6 @@ HASHES = {
     730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669',
     731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97',
     732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c',
+    733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630',
+    734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f',
 }

lbry/wallet/ledger.py

@@ -349,8 +349,9 @@ class Ledger(metaclass=LedgerRegistry):
         self.headers.chunk_getter = get_chunk
 
         async def doit():
-            for height in reversed(range(0, target, 1000)):
+            for height in reversed(range(0, target)):
-                await self.headers.ensure_chunk_at(height)
+                async with self._header_processing_lock:
+                    await self.headers.ensure_chunk_at(height)
         await self.headers.ensure_tip()
         self._update_tasks.add(doit())
         await self.update_headers()

scripts/checkpoints.py

@@ -0,0 +1,35 @@
+import asyncio
+import os
+
+from lbry.extras.cli import ensure_directory_exists
+from lbry.conf import Config
+from lbry.wallet.header import Headers
+import lbry.wallet.checkpoints
+
+
+async def main():
+    outpath = lbry.wallet.checkpoints.__file__
+    ledger_path = os.path.join(Config().wallet_dir, 'lbc_mainnet')
+    ensure_directory_exists(ledger_path)
+    headers_path = os.path.join(ledger_path, 'headers')
+    headers = Headers(headers_path)
+    await headers.open()
+    print(f"Working on headers at {outpath}")
+    print("Verifying integrity, might take a while.")
+    await headers.repair()
+    target = ((headers.height - 100) // 1000) * 1000
+    current_checkpoint_tip = max(lbry.wallet.checkpoints.HASHES.keys())
+    if target <= current_checkpoint_tip:
+        print(f"We have nothing to add: Local: {target}, checkpoint: {current_checkpoint_tip}")
+        return
+    print(f"Headers file at {headers.height}, checkpointing up to {target}."
+          f"Current checkpoint at {current_checkpoint_tip}.")
+    with open(outpath, 'w') as outfile:
+        print('HASHES = {', file=outfile)
+        for height in range(0, target, 1000):
+            print(f"    {height}: '{headers.chunk_hash(height, 1000)}',", file=outfile)
+        print('}', file=outfile)
+
+
+if __name__ == "__main__":
+    asyncio.get_event_loop().run_until_complete(main())