improve claims_producer performance

This commit is contained in:
Jack Robison 2021-10-13 14:09:16 -04:00
parent ba1d0a12d1
commit 3dc3792478
No known key found for this signature in database
GPG key ID: DF25C68FE0239BB2

View file

@ -697,12 +697,13 @@ class LevelDB:
batch.clear()
async def claims_producer(self, claim_hashes: Set[bytes]):
loop = asyncio.get_event_loop()
def produce_claims(claims):
batch = []
results = []
loop = asyncio.get_event_loop()
def produce_claim(claim_hash):
for claim_hash in claims:
if claim_hash not in self.claim_to_txo:
self.logger.warning("can't sync non existent claim to ES: %s", claim_hash.hex())
return
@ -721,26 +722,20 @@ class LevelDB:
if claim:
batch.append(claim)
def get_metadata(claim):
batch.sort(key=lambda x: x.tx_hash)
for claim in batch:
meta = self._prepare_claim_metadata(claim.claim_hash, claim)
if meta:
results.append(meta)
return results
if claim_hashes:
await asyncio.wait(
[loop.run_in_executor(None, produce_claim, claim_hash) for claim_hash in claim_hashes]
)
batch.sort(key=lambda x: x.tx_hash)
results = await loop.run_in_executor(None, produce_claims, claim_hashes)
if batch:
await asyncio.wait(
[loop.run_in_executor(None, get_metadata, claim) for claim in batch]
)
for meta in results:
yield meta
batch.clear()
def get_activated_at_height(self, height: int) -> DefaultDict[PendingActivationValue, List[PendingActivationKey]]:
activated = defaultdict(list)
for k, v in self.prefix_db.pending_activation.iterate(prefix=(height,)):