LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

make header access synchronous for now

Browse source
This commit is contained in:
Victor Shyba 2019-06-25 23:01:22 -03:00 committed by Lex Berezhny
parent f09c18539b
commit f109aa1f73
1 changed files with 19 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
torba/torba/client/baseheader.py
View file

@ -1,5 +1,4 @@
import os import os
import asyncio
import logging import logging
from io import BytesIO from io import BytesIO
from typing import Optional, Iterator, Tuple from typing import Optional, Iterator, Tuple
@ -35,7 +34,6 @@ class BaseHeaders:
self.io = BytesIO() self.io = BytesIO()
self.path = path self.path = path
self._size: Optional[int] = None self._size: Optional[int] = None
self._header_connect_lock = asyncio.Lock()
async def open(self): async def open(self):
if self.path != ':memory:': if self.path != ':memory:':
@ -98,27 +96,25 @@ class BaseHeaders:
async def connect(self, start: int, headers: bytes) -> int: async def connect(self, start: int, headers: bytes) -> int:
added = 0 added = 0
bail = False bail = False
loop = asyncio.get_running_loop() for height, chunk in self._iterate_chunks(start, headers):
async with self._header_connect_lock: try:
for height, chunk in self._iterate_chunks(start, headers): # validate_chunk() is CPU bound and reads previous chunks from file system
try: self.validate_chunk(height, chunk)
# validate_chunk() is CPU bound and reads previous chunks from file system except InvalidHeader as e:
await loop.run_in_executor(None, self.validate_chunk, height, chunk) bail = True
except InvalidHeader as e: chunk = chunk[:(height-e.height)*self.header_size]
bail = True written = 0
chunk = chunk[:(height-e.height)*self.header_size] if chunk:
written = 0 self.io.seek(height * self.header_size, os.SEEK_SET)
if chunk: written = self.io.write(chunk) // self.header_size
self.io.seek(height * self.header_size, os.SEEK_SET) self.io.truncate()
written = self.io.write(chunk) // self.header_size # .seek()/.write()/.truncate() might also .flush() when needed
self.io.truncate() # the goal here is mainly to ensure we're definitely flush()'ing
# .seek()/.write()/.truncate() might also .flush() when needed self.io.flush()
# the goal here is mainly to ensure we're definitely flush()'ing self._size = None
await loop.run_in_executor(None, self.io.flush) added += written
self._size = None if bail:
added += written break
if bail:
break
return added return added
def validate_chunk(self, height, chunk): def validate_chunk(self, height, chunk):