Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

improve claims_producer performance

Browse source
This commit is contained in:
Jack Robison 2021-10-13 14:09:16 -04:00 committed by Victor Shyba
parent 15ac2ade59
commit e0086682b9
1 changed files with 32 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
lbry/wallet/server/leveldb.py
View file

@ -697,22 +697,23 @@ class LevelDB:
batch.clear() batch.clear()
async def claims_producer(self, claim_hashes: Set[bytes]): async def claims_producer(self, claim_hashes: Set[bytes]):
batch = []
results = []
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
def produce_claim(claim_hash): def produce_claims(claims):
batch = []
_results = []
for claim_hash in claims:
if claim_hash not in self.claim_to_txo: if claim_hash not in self.claim_to_txo:
self.logger.warning("can't sync non existent claim to ES: %s", claim_hash.hex()) self.logger.warning("can't sync non existent claim to ES: %s", claim_hash.hex())
return continue
name = self.claim_to_txo[claim_hash].normalized_name name = self.claim_to_txo[claim_hash].normalized_name
if not self.prefix_db.claim_takeover.get(name): if not self.prefix_db.claim_takeover.get(name):
self.logger.warning("can't sync non existent claim to ES: %s", claim_hash.hex()) self.logger.warning("can't sync non existent claim to ES: %s", claim_hash.hex())
return continue
claim_txo = self.claim_to_txo.get(claim_hash) claim_txo = self.claim_to_txo.get(claim_hash)
if not claim_txo: if not claim_txo:
return continue
activation = self.get_activation(claim_txo.tx_num, claim_txo.position) activation = self.get_activation(claim_txo.tx_num, claim_txo.position)
claim = self._prepare_resolve_result( claim = self._prepare_resolve_result(
claim_txo.tx_num, claim_txo.position, claim_hash, claim_txo.name, claim_txo.root_tx_num, claim_txo.tx_num, claim_txo.position, claim_hash, claim_txo.name, claim_txo.root_tx_num,
@ -721,26 +722,20 @@ class LevelDB:
if claim: if claim:
batch.append(claim) batch.append(claim)
def get_metadata(claim):
meta = self._prepare_claim_metadata(claim.claim_hash, claim)
if meta:
results.append(meta)
if claim_hashes:
await asyncio.wait(
[loop.run_in_executor(None, produce_claim, claim_hash) for claim_hash in claim_hashes]
)
batch.sort(key=lambda x: x.tx_hash) batch.sort(key=lambda x: x.tx_hash)
if batch: for claim in batch:
await asyncio.wait( _meta = self._prepare_claim_metadata(claim.claim_hash, claim)
[loop.run_in_executor(None, get_metadata, claim) for claim in batch] if _meta:
) _results.append(_meta)
return _results
if claim_hashes:
results = await loop.run_in_executor(None, produce_claims, claim_hashes)
for meta in results: for meta in results:
yield meta yield meta
batch.clear()
def get_activated_at_height(self, height: int) -> DefaultDict[PendingActivationValue, List[PendingActivationKey]]: def get_activated_at_height(self, height: int) -> DefaultDict[PendingActivationValue, List[PendingActivationKey]]:
activated = defaultdict(list) activated = defaultdict(list)
for k, v in self.prefix_db.pending_activation.iterate(prefix=(height,)): for k, v in self.prefix_db.pending_activation.iterate(prefix=(height,)):