import os import sys import json import shutil import logging import tempfile import functools import asyncio import time from asyncio.runners import _cancel_all_tasks # type: ignore import unittest from unittest.case import _Outcome from typing import Optional from binascii import unhexlify from lbry.blockchain import ( RegTestLedger, Transaction, Input, Output, dewies_to_lbc ) from lbry.blockchain.lbrycrd import Lbrycrd from lbry.constants import CENT, NULL_HASH32 from lbry.service import Daemon, FullNode, jsonrpc_dumps_pretty from lbry.conf import Config from lbry.console import Console from lbry.wallet import Wallet, Account from lbry.service.exchange_rate_manager import ( ExchangeRateManager, ExchangeRate, LBRYFeed, LBRYBTCFeed ) def get_output(amount=CENT, pubkey_hash=NULL_HASH32, height=-2): return Transaction(height=height) \ .add_outputs([Output.pay_pubkey_hash(amount, pubkey_hash)]) \ .outputs[0] def get_input(amount=CENT, pubkey_hash=NULL_HASH32): return Input.spend(get_output(amount, pubkey_hash)) def get_transaction(txo=None): return Transaction() \ .add_inputs([get_input()]) \ .add_outputs([txo or Output.pay_pubkey_hash(CENT, NULL_HASH32)]) def get_claim_transaction(claim_name, claim=b''): return get_transaction( Output.pay_claim_name_pubkey_hash(CENT, claim_name, claim, NULL_HASH32) ) class ColorHandler(logging.StreamHandler): level_color = { logging.DEBUG: "black", logging.INFO: "light_gray", logging.WARNING: "yellow", logging.ERROR: "red" } color_code = dict( black=30, red=31, green=32, yellow=33, blue=34, magenta=35, cyan=36, white=37, light_gray='0;37', dark_gray='1;30' ) def emit(self, record): try: msg = self.format(record) color_name = self.level_color.get(record.levelno, "black") color_code = self.color_code[color_name] stream = self.stream stream.write(f'\x1b[{color_code}m{msg}\x1b[0m') stream.write(self.terminator) self.flush() except Exception: self.handleError(record) HANDLER = ColorHandler(sys.stdout) HANDLER.setFormatter( logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ) logging.getLogger().addHandler(HANDLER) class AsyncioTestCase(unittest.TestCase): # Implementation inspired by discussion: # https://bugs.python.org/issue32972 LOOP_SLOW_CALLBACK_DURATION = 0.1 maxDiff = None async def asyncSetUp(self): # pylint: disable=C0103 pass async def asyncTearDown(self): # pylint: disable=C0103 pass def run(self, result=None): # pylint: disable=R0915 orig_result = result if result is None: result = self.defaultTestResult() startTestRun = getattr(result, 'startTestRun', None) # pylint: disable=C0103 if startTestRun is not None: startTestRun() result.startTest(self) testMethod = getattr(self, self._testMethodName) # pylint: disable=C0103 if (getattr(self.__class__, "__unittest_skip__", False) or getattr(testMethod, "__unittest_skip__", False)): # If the class or method was skipped. try: skip_why = (getattr(self.__class__, '__unittest_skip_why__', '') or getattr(testMethod, '__unittest_skip_why__', '')) self._addSkip(result, self, skip_why) finally: result.stopTest(self) return expecting_failure_method = getattr(testMethod, "__unittest_expecting_failure__", False) expecting_failure_class = getattr(self, "__unittest_expecting_failure__", False) expecting_failure = expecting_failure_class or expecting_failure_method outcome = _Outcome(result) self.loop = asyncio.new_event_loop() # pylint: disable=W0201 asyncio.set_event_loop(self.loop) self.loop.set_debug(True) self.loop.slow_callback_duration = self.LOOP_SLOW_CALLBACK_DURATION try: self._outcome = outcome with outcome.testPartExecutor(self): self.setUp() self.loop.run_until_complete(self.asyncSetUp()) if outcome.success: outcome.expecting_failure = expecting_failure with outcome.testPartExecutor(self, isTest=True): maybe_coroutine = testMethod() if asyncio.iscoroutine(maybe_coroutine): self.loop.run_until_complete(maybe_coroutine) outcome.expecting_failure = False with outcome.testPartExecutor(self): self.loop.run_until_complete(self.asyncTearDown()) self.tearDown() self.doAsyncCleanups() try: _cancel_all_tasks(self.loop) self.loop.run_until_complete(self.loop.shutdown_asyncgens()) finally: asyncio.set_event_loop(None) self.loop.close() for test, reason in outcome.skipped: self._addSkip(result, test, reason) self._feedErrorsToResult(result, outcome.errors) if outcome.success: if expecting_failure: if outcome.expectedFailure: self._addExpectedFailure(result, outcome.expectedFailure) else: self._addUnexpectedSuccess(result) else: result.addSuccess(self) return result finally: result.stopTest(self) if orig_result is None: stopTestRun = getattr(result, 'stopTestRun', None) # pylint: disable=C0103 if stopTestRun is not None: stopTestRun() # pylint: disable=E1102 # explicitly break reference cycles: # outcome.errors -> frame -> outcome -> outcome.errors # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure outcome.errors.clear() outcome.expectedFailure = None # clear the outcome, no more needed self._outcome = None def doAsyncCleanups(self): # pylint: disable=C0103 outcome = self._outcome or _Outcome() while self._cleanups: function, args, kwargs = self._cleanups.pop() with outcome.testPartExecutor(self): maybe_coroutine = function(*args, **kwargs) if asyncio.iscoroutine(maybe_coroutine): self.loop.run_until_complete(maybe_coroutine) class AdvanceTimeTestCase(AsyncioTestCase): async def asyncSetUp(self): self._time = 0 # pylint: disable=W0201 self.loop.time = functools.wraps(self.loop.time)(lambda: self._time) await super().asyncSetUp() async def advance(self, seconds): while self.loop._ready: await asyncio.sleep(0) self._time += seconds await asyncio.sleep(0) while self.loop._ready: await asyncio.sleep(0) class IntegrationTestCase(AsyncioTestCase): SEED = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.ledger: Optional[RegTestLedger] = None self.chain: Optional[Lbrycrd] = None self.block_expected = 0 self.service = None self.api = None self.wallet: Optional[Wallet] = None self.account: Optional[Account] = None async def assertBalance(self, account, expected_balance: str): # pylint: disable=C0103 balance = await account.get_balance() self.assertEqual(dewies_to_lbc(balance), expected_balance) def broadcast(self, tx): return self.ledger.broadcast(tx) async def on_header(self, height): if self.ledger.headers.height < height: await self.ledger.on_header.where( lambda e: e.height == height ) return True def on_transaction_id(self, txid, ledger=None): return (ledger or self.ledger).on_transaction.where( lambda e: e.tx.id == txid ) def on_transaction_hash(self, tx_hash, ledger=None): return (ledger or self.ledger).on_transaction.where( lambda e: e.tx.hash == tx_hash ) def on_address_update(self, address): return self.ledger.on_transaction.where( lambda e: e.address == address ) def on_transaction_address(self, tx, address): return self.ledger.on_transaction.where( lambda e: e.tx.id == tx.id and e.address == address ) class FakeExchangeRateManager(ExchangeRateManager): def __init__(self, market_feeds, rates): # pylint: disable=super-init-not-called self.market_feeds = market_feeds for feed in self.market_feeds: feed.last_check = time.time() feed.rate = ExchangeRate(feed.market, rates[feed.market], time.time()) def start(self): pass def stop(self): pass def get_fake_exchange_rate_manager(rates=None): return FakeExchangeRateManager( [LBRYFeed(), LBRYBTCFeed()], rates or {'BTCLBC': 3.0, 'USDBTC': 2.0} ) class CommandTestCase(IntegrationTestCase): VERBOSITY = logging.WARN blob_lru_cache_size = 0 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.daemon_port = 5252 self.daemon = None self.daemons = [] self.server_config = None self.server_storage = None self.extra_wallet_nodes = [] self.extra_wallet_node_port = 5281 self.server_blob_manager = None self.server = None self.reflector = None async def asyncSetUp(self): self.chain = Lbrycrd.temp_regtest() await self.chain.ensure() self.addCleanup(self.chain.stop) await self.chain.start('-rpcworkqueue=128') await self.generate(200, wait=False) self.daemon = await self.add_daemon() self.service = self.daemon.service self.ledger = self.service.ledger self.api = self.daemon.api self.wallet = self.service.wallets.default self.account = self.wallet.accounts.default addresses = await self.account.ensure_address_gap() self.ledger.conf.upload_dir = os.path.join(self.ledger.conf.data_dir, 'uploads') os.mkdir(self.ledger.conf.upload_dir) await self.chain.send_to_address(addresses[0], '10.0') await self.generate(5) async def add_daemon(self): self.daemon_port += 1 path = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, path, True) ledger = RegTestLedger(Config.with_same_dir(path).set( api=f'localhost:{self.daemon_port}', lbrycrd_dir=self.chain.ledger.conf.lbrycrd_dir, spv_address_filters=False )) service = FullNode(ledger) console = Console(service) daemon = Daemon(service, console) self.addCleanup(daemon.stop) await daemon.start() return daemon async def asyncTearDown(self): await super().asyncTearDown() for wallet_node in self.extra_wallet_nodes: await wallet_node.stop(cleanup=True) for daemon in self.daemons: daemon.component_manager.get_component('wallet')._running = False await daemon.stop() async def confirm_tx(self, txid, ledger=None): """ Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """ await self.on_transaction_id(txid, ledger) await self.generate(1) await self.on_transaction_id(txid, ledger) return txid async def on_transaction_dict(self, tx): await self.ledger.wait(Transaction(unhexlify(tx['hex']))) @staticmethod def get_all_addresses(tx): addresses = set() for txi in tx['inputs']: addresses.add(txi['address']) for txo in tx['outputs']: addresses.add(txo['address']) return list(addresses) def is_expected_block(self, event): return self.block_expected == event.height async def generate(self, blocks, wait=True): """ Ask lbrycrd to generate some blocks and wait until ledger has them. """ await self.chain.generate(blocks) self.block_expected += blocks if wait: await self.service.sync.on_block.where(self.is_expected_block) async def out(self, awaitable): """ Serializes lbrynet API results to JSON then loads and returns it as dictionary. """ return json.loads(jsonrpc_dumps_pretty(await awaitable, service=self.service))['result'] def sout(self, value): """ Synchronous version of `out` method. """ return json.loads(jsonrpc_dumps_pretty(value, service=self.service))['result'] async def confirm_and_render(self, awaitable, confirm) -> Transaction: tx = await awaitable if confirm: await self.generate(1) await self.service.wait(tx) return self.sout(tx) def create_upload_file(self, data, prefix=None, suffix=None): file_path = tempfile.mktemp( prefix=prefix or "tmp", suffix=suffix or "", dir=self.ledger.conf.upload_dir ) with open(file_path, 'w+b') as file: file.write(data) file.flush() return file.name async def stream_create( self, name='hovercraft', bid='1.0', file_path=None, data=b'hi!', confirm=True, prefix=None, suffix=None, **kwargs): if file_path is None: file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix) return await self.confirm_and_render( self.api.stream_create(name, bid, file_path=file_path, **kwargs), confirm ) async def stream_update( self, claim_id, data=None, prefix=None, suffix=None, confirm=True, **kwargs): if data is not None: file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix) return await self.confirm_and_render( self.api.stream_update(claim_id, file_path=file_path, **kwargs), confirm ) return await self.confirm_and_render( self.api.stream_update(claim_id, **kwargs), confirm ) async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs): return await self.confirm_and_render( self.api.stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm ) async def stream_abandon(self, *args, confirm=True, **kwargs): if 'blocking' not in kwargs: kwargs['blocking'] = False return await self.confirm_and_render( self.api.stream_abandon(*args, **kwargs), confirm ) async def purchase_create(self, *args, confirm=True, **kwargs): return await self.confirm_and_render( self.api.purchase_create(*args, **kwargs), confirm ) async def publish(self, name, *args, confirm=True, **kwargs): return await self.confirm_and_render( self.api.publish(name, *args, **kwargs), confirm ) async def channel_create(self, name='@arena', bid='1.0', confirm=True, **kwargs): return await self.confirm_and_render( self.api.channel_create(name, bid, **kwargs), confirm ) async def channel_update(self, claim_id, confirm=True, **kwargs): return await self.confirm_and_render( self.api.channel_update(claim_id, **kwargs), confirm ) async def channel_abandon(self, *args, confirm=True, **kwargs): if 'blocking' not in kwargs: kwargs['blocking'] = False return await self.confirm_and_render( self.api.channel_abandon(*args, **kwargs), confirm ) async def collection_create( self, name='firstcollection', bid='1.0', confirm=True, **kwargs): return await self.confirm_and_render( self.api.collection_create(name, bid, **kwargs), confirm ) async def collection_update( self, claim_id, confirm=True, **kwargs): return await self.confirm_and_render( self.api.collection_update(claim_id, **kwargs), confirm ) async def collection_abandon(self, *args, confirm=True, **kwargs): if 'blocking' not in kwargs: kwargs['blocking'] = False return await self.confirm_and_render( self.api.stream_abandon(*args, **kwargs), confirm ) async def support_create(self, claim_id, bid='1.0', confirm=True, **kwargs): return await self.confirm_and_render( self.api.support_create(claim_id, bid, **kwargs), confirm ) async def support_abandon(self, *args, confirm=True, **kwargs): if 'blocking' not in kwargs: kwargs['blocking'] = False return await self.confirm_and_render( self.api.support_abandon(*args, **kwargs), confirm ) async def account_fund(self, *args, confirm=True, **kwargs): return await self.confirm_and_render( self.api.account_fund(*args, **kwargs), confirm ) async def account_send(self, *args, confirm=True, **kwargs): return await self.confirm_and_render( self.api.account_send(*args, **kwargs), confirm ) async def wallet_send(self, *args, confirm=True, **kwargs): return await self.confirm_and_render( self.api.wallet_send(*args, **kwargs), confirm ) async def txo_spend(self, *args, confirm=True, **kwargs): txs = await self.api.txo_spend(*args, **kwargs) if confirm: await asyncio.wait([self.ledger.wait(tx) for tx in txs]) await self.generate(1) await asyncio.wait([self.ledger.wait(tx, self.block_expected) for tx in txs]) return self.sout(txs) async def resolve(self, uri, **kwargs): return (await self.out(self.api.resolve(uri, **kwargs)))[uri] async def claim_search(self, **kwargs): return (await self.out(self.api.claim_search(**kwargs)))['items'] async def file_list(self, *args, **kwargs): return (await self.out(self.api.file_list(*args, **kwargs)))['items'] async def txo_list(self, *args, **kwargs): return (await self.out(self.api.txo_list(*args, **kwargs)))['items'] async def txo_sum(self, *args, **kwargs): return await self.out(self.api.txo_sum(*args, **kwargs)) async def txo_plot(self, *args, **kwargs): return await self.out(self.api.txo_plot(*args, **kwargs)) async def claim_list(self, *args, **kwargs): return (await self.out(self.api.claim_list(*args, **kwargs)))['items'] async def stream_list(self, *args, **kwargs): return (await self.out(self.api.stream_list(*args, **kwargs)))['items'] async def channel_list(self, *args, **kwargs): return (await self.out(self.api.channel_list(*args, **kwargs)))['items'] async def collection_list(self, *args, **kwargs): return (await self.out(self.api.collection_list(*args, **kwargs)))['items'] async def collection_resolve(self, *args, **kwargs): return (await self.out(self.api.collection_resolve(*args, **kwargs)))['items'] async def transaction_list(self, *args, **kwargs): return (await self.out(self.api.transaction_list(*args, **kwargs)))['items'] @staticmethod def get_claim_id(tx): return tx['outputs'][0]['claim_id'] @staticmethod def get_address(tx): return tx['outputs'][0]['address']