From 8a69ec062e779c3ddd9e374493410691bbee76dd Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor1984@riseup.net>
Date: Tue, 23 Jul 2019 15:05:39 -0300
Subject: [PATCH] download headers to disk and repair instead of connecting
 each

---
 lbry/lbry/extras/daemon/Components.py | 28 +++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/lbry/lbry/extras/daemon/Components.py b/lbry/lbry/extras/daemon/Components.py
index 24caf5aa5..bad617470 100644
--- a/lbry/lbry/extras/daemon/Components.py
+++ b/lbry/lbry/extras/daemon/Components.py
@@ -132,7 +132,7 @@ class HeadersComponent(Component):
         } if progress is not None and progress < 100 else {}
 
     async def fetch_headers_from_s3(self):
-        local_header_size = self.headers.bytes_size
+        local_header_size = self.local_header_file_size()
         resume_header = {"Range": f"bytes={local_header_size}-"}
         async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
             if response.status == 406 or response.content_length < self.headers.header_size:  # our file is bigger
@@ -142,17 +142,16 @@ class HeadersComponent(Component):
                 log.warning("s3 appears to have corrupted header")
                 return
             final_size_after_download = response.content_length + local_header_size
+            write_mode = "wb"
             if local_header_size > 0:
                 log.info("Resuming download of %i bytes from s3", response.content_length)
-            while not response.content.at_eof():
-                max_read = min(self.headers.header_size * 10000, final_size_after_download - self.headers.bytes_size)
-                chunk = await response.content.readexactly(max_read)
-                if not await self.headers.connect(len(self.headers), chunk):
-                    log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
-                    return
-                self._headers_progress_percent = self._round_progress(
-                    self.headers.bytes_size, final_size_after_download
-                )
+                write_mode = "a+b"
+            with open(self.headers_file, write_mode) as fd:
+                while not response.content.at_eof():
+                    local_header_size += fd.write(await response.content.readany())
+                    self._headers_progress_percent = self._round_progress(
+                        local_header_size, final_size_after_download
+                    )
 
     def local_header_file_size(self) -> int:
         if os.path.isfile(self.headers_file):
@@ -173,7 +172,7 @@ class HeadersComponent(Component):
         if not s3_headers_depth:
             return False
 
-        local_height = self.headers.height
+        local_height = self.local_header_file_size() // self.headers.header_size
         remote_height = await self.get_downloadable_header_height()
         if remote_height is not None:
             log.info("remote height: %i, local height: %i", remote_height, local_height)
@@ -187,9 +186,8 @@ class HeadersComponent(Component):
         if os.path.exists(self.old_file):
             log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
             os.rename(self.old_file, self.headers_file)
+
         try:
-            await self.headers.open()
-            await self.headers.repair()
             if await self.should_download_headers_from_s3():
                 self.is_downloading_headers = True
                 await self.fetch_headers_from_s3()
@@ -197,8 +195,10 @@ class HeadersComponent(Component):
             log.error("failed to fetch headers from s3: %s", err)
         finally:
             self.is_downloading_headers = False
+            # fixme: workaround, this should happen before download but happens after because headers.connect fail
+            await self.headers.open()
+            await self.headers.repair()
             await self.headers.close()
-
     async def stop(self):
         pass