import asyncio from random import randrange from typing import List from binascii import unhexlify from google.protobuf.message import DecodeError from lbry.schema.claim import Claim from lbry.blockchain import Ledger from lbry.service import FullNode from lbry.console import Advanced, Basic from lbry.conf import Config from lbry.db.utils import chunk def cause_protobuf_stderr(): try: Claim.from_bytes(unhexlify( '005a3c63779597cba4c0e6ee45c3074fc389bd564ccc5d4a90eb4baacb0b028f2f4930' '0db003d6a27f0cac8be8b45fdda597303208b81845534e4543494c07123e0a420a' )) except DecodeError: pass class Simulator: def __init__(self, console): self.console = console self.sync = console.service.sync self.progress = self.sync._on_progress_controller self.starting_height = 0 self.ending_height = 0 self.starting_file = 0 self.processes = console.service.db.processes self.steps = [] self.txs = 0 self.claims = 0 self.supports = 0 @property def blocks(self, ): if self.starting_height == 0: return self.ending_height-self.starting_height return (self.ending_height-self.starting_height)+1 async def advance(self, initial_sync: bool, ending_height: int, files: List[int], txs: int): self.ending_height = ending_height self.steps = self.sync.get_steps(initial_sync) self.txs = txs self.claims = int(txs/4) self.supports = int(txs/2) await self.progress.add({ "event": "blockchain.sync.start", "data": { "starting_height": self.starting_height, "ending_height": ending_height, "files": len(files), "blocks": self.blocks, "sync_steps": self.steps, "txs": self.txs, "claims": self.claims, "supports": self.supports, } }) blocks_synced = txs_synced = 0 for file_group in chunk(files, self.processes): tasks = [] for file in file_group: if file == files[-1]: cause_protobuf_stderr() tasks.append(self.sync_block_file(file, self.blocks-blocks_synced, self.txs-txs_synced)) cause_protobuf_stderr() else: blocks = int(self.blocks / len(files)) blocks_synced += blocks txs = int(self.txs / len(files)) txs_synced += txs tasks.append(self.sync_block_file(file, blocks, txs)) await asyncio.wait(tasks) for step in self.steps: if step in ("blockchain.sync.block.read", "blockchain.sync.block.save"): continue await getattr(self, step.replace('.', '_'))() await self.progress.add({ "event": "blockchain.sync.complete", "data": {"step": len(self.steps), "total": len(self.steps), "unit": "tasks"} }) self.ending_height = ending_height+1 self.starting_height = self.ending_height async def sync_block_file(self, block_file, blocks, txs): for i in range(0, blocks, 1000): await self.progress.add({ "event": "blockchain.sync.block.read", "data": {"step": i, "total": blocks, "unit": "blocks", "block_file": block_file} }) await asyncio.sleep(randrange(1, 10)/10) await self.progress.add({ "event": "blockchain.sync.block.read", "data": {"step": blocks, "total": blocks, "unit": "blocks", "block_file": block_file} }) await asyncio.sleep(0.5) for i in range(0, txs, 2000): await self.progress.add({ "event": "blockchain.sync.block.save", "data": {"step": i, "total": txs, "unit": "txs", "block_file": block_file} }) await asyncio.sleep(randrange(1, 10) / 10) await self.progress.add({ "event": "blockchain.sync.block.save", "data": {"step": txs, "total": txs, "unit": "txs", "block_file": block_file} }) async def generate_steps(self, event, steps, unit, delay=1.0, step=1): await self.progress.add({"event": event, "data": {"step": 0, "total": steps, "unit": unit}}) remaining = steps for i in range(1, steps+1, step): await asyncio.sleep(delay) await self.progress.add({"event": event, "data": {"step": i, "total": steps, "unit": unit}}) remaining -= i if remaining: await asyncio.sleep(delay) await self.progress.add({"event": event, "data": {"step": steps, "total": steps, "unit": unit}}) async def db_sync_input(self): await self.generate_steps("db.sync.input", 2, "txis") async def db_sync_claim_delete(self): await self.generate_steps("db.sync.claim.delete", 1, "claims") async def db_sync_claim_insert(self): await self.generate_steps("db.sync.claim.insert", 1, "claims") async def db_sync_claim_update(self): await self.generate_steps("db.sync.claim.update", 0, "claims") async def db_sync_support_delete(self): await self.generate_steps("db.sync.support.delete", 1, "supports") async def db_sync_support_insert(self): await self.generate_steps("db.sync.support.insert", 1, "supports") async def blockchain_sync_claim_trie(self): await self.generate_steps("blockchain.sync.claim.trie", 1, "claims") async def blockchain_sync_claim_meta(self): for i in range(0, self.claims, 1_000): await self.progress.add({ "event": "blockchain.sync.claim.meta", "data": {"step": i, "total": self.claims, "unit": "claims"} }) await asyncio.sleep(0.1) await self.progress.add({ "event": "blockchain.sync.claim.meta", "data": {"step": self.claims, "total": self.claims, "unit": "claims"} }) async def blockchain_sync_claim_signatures(self): await self.generate_steps("blockchain.sync.claim.signatures", self.claims, "claims", 0.5, 1000) async def blockchain_sync_support_signatures(self): await self.generate_steps("blockchain.sync.support.signatures", self.supports, "supports", 0.5, 1000) async def blockchain_sync_claim_stakes(self): await self.generate_steps("blockchain.sync.claim.stakes", 1, "claims", 0.5) async def blockchain_sync_claim_channels(self): await self.generate_steps("blockchain.sync.claim.channels", 0, "supports", 0.5) async def main(): console = Advanced(FullNode(Ledger(Config(processes=3, spv_address_filters=False)))) sim = Simulator(console) console.starting() await sim.advance(True, 100_000, [1, 2, 3, 4, 5], 100_000) await sim.advance(False, 100_001, [5], 100) console.stopping() if __name__ == "__main__": asyncio.run(main())