lbry-sdk/lbry/tests/unit/wallet/test_headers.py

import asyncio
import os
import tempfile
from binascii import hexlify

from torba.client.hash import sha256
from torba.testcase import AsyncioTestCase

from torba.coin.bitcoinsegwit import MainHeaders
from binascii import unhexlify

from torba.testcase import AsyncioTestCase
from torba.client.util import ArithUint256

from lbry.wallet.ledger import Headers


def block_bytes(blocks):
    return blocks * MainHeaders.header_size


class BitcoinHeadersTestCase(AsyncioTestCase):
    HEADER_FILE = 'bitcoin_headers'
    RETARGET_BLOCK = 32256  # difficulty: 1 -> 1.18

    def setUp(self):
        self.maxDiff = None
        self.header_file_name = os.path.join(os.path.dirname(__file__), self.HEADER_FILE)

    def get_bytes(self, upto: int = -1, after: int = 0) -> bytes:
        with open(self.header_file_name, 'rb') as headers:
            headers.seek(after, os.SEEK_SET)
            return headers.read(upto)

    async def get_headers(self, upto: int = -1):
        h = MainHeaders(':memory:')
        h.io.write(self.get_bytes(upto))
        return h


class BasicHeadersTests(BitcoinHeadersTestCase):

    async def test_serialization(self):
        h = await self.get_headers()
        self.assertDictEqual(h[0], {
            'bits': 486604799,
            'block_height': 0,
            'merkle_root': b'4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b',
            'nonce': 2083236893,
            'prev_block_hash': b'0000000000000000000000000000000000000000000000000000000000000000',
            'timestamp': 1231006505,
            'version': 1
        })
        self.assertDictEqual(h[self.RETARGET_BLOCK-1], {
            'bits': 486604799,
            'block_height': 32255,
            'merkle_root': b'89b4f223789e40b5b475af6483bb05bceda54059e17d2053334b358f6bb310ac',
            'nonce': 312762301,
            'prev_block_hash': b'000000006baebaa74cecde6c6787c26ee0a616a3c333261bff36653babdac149',
            'timestamp': 1262152739,
            'version': 1
        })
        self.assertDictEqual(h[self.RETARGET_BLOCK], {
            'bits': 486594666,
            'block_height': 32256,
            'merkle_root': b'64b5e5f5a262f47af443a0120609206a3305877693edfe03e994f20a024ab627',
            'nonce': 121087187,
            'prev_block_hash': b'00000000984f962134a7291e3693075ae03e521f0ee33378ec30a334d860034b',
            'timestamp': 1262153464,
            'version': 1
        })
        self.assertDictEqual(h[self.RETARGET_BLOCK+1], {
            'bits': 486594666,
            'block_height': 32257,
            'merkle_root': b'4d1488981f08b3037878193297dbac701a2054e0f803d4424fe6a4d763d62334',
            'nonce': 274675219,
            'prev_block_hash': b'000000004f2886a170adb7204cb0c7a824217dd24d11a74423d564c4e0904967',
            'timestamp': 1262154352,
            'version': 1
        })
        self.assertEqual(
            h.serialize(h[0]),
            h.get_raw_header(0)
        )
        self.assertEqual(
            h.serialize(h[self.RETARGET_BLOCK]),
            h.get_raw_header(self.RETARGET_BLOCK)
        )

    async def test_connect_from_genesis_to_3000_past_first_chunk_at_2016(self):
        headers = MainHeaders(':memory:')
        self.assertEqual(headers.height, -1)
        await headers.connect(0, self.get_bytes(block_bytes(3001)))
        self.assertEqual(headers.height, 3000)

    async def test_connect_9_blocks_passing_a_retarget_at_32256(self):
        retarget = block_bytes(self.RETARGET_BLOCK-5)
        headers = await self.get_headers(upto=retarget)
        remainder = self.get_bytes(after=retarget)
        self.assertEqual(headers.height, 32250)
        await headers.connect(len(headers), remainder)
        self.assertEqual(headers.height, 32259)

    async def test_bounds(self):
        headers = MainHeaders(':memory:')
        await headers.connect(0, self.get_bytes(block_bytes(3001)))
        self.assertEqual(headers.height, 3000)
        with self.assertRaises(IndexError):
            _ = headers[3001]
        with self.assertRaises(IndexError):
            _ = headers[-1]
        self.assertIsNotNone(headers[3000])
        self.assertIsNotNone(headers[0])

    async def test_repair(self):
        headers = MainHeaders(':memory:')
        await headers.connect(0, self.get_bytes(block_bytes(3001)))
        self.assertEqual(headers.height, 3000)
        await headers.repair()
        self.assertEqual(headers.height, 3000)
        # corrupt the middle of it
        headers.io.seek(block_bytes(1500))
        headers.io.write(b"wtf")
        await headers.repair()
        self.assertEqual(headers.height, 1499)
        self.assertEqual(len(headers), 1500)
        # corrupt by appending
        headers.io.seek(block_bytes(len(headers)))
        headers.io.write(b"appending")
        await headers.repair()
        self.assertEqual(headers.height, 1499)
        await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
        self.assertEqual(headers.height, 3000)

    async def test_checkpointed_writer(self):
        headers = MainHeaders(':memory:')
        headers.checkpoint = 100, hexlify(sha256(self.get_bytes(block_bytes(100))))
        genblocks = lambda start, end: self.get_bytes(block_bytes(end - start), block_bytes(start))
        async with headers.checkpointed_connector() as buff:
            buff.write(genblocks(0, 10))
        self.assertEqual(len(headers), 10)
        async with headers.checkpointed_connector() as buff:
            buff.write(genblocks(10, 100))
        self.assertEqual(len(headers), 100)
        headers = MainHeaders(':memory:')
        async with headers.checkpointed_connector() as buff:
            buff.write(genblocks(0, 300))
        self.assertEqual(len(headers), 300)

    async def test_concurrency(self):
        BLOCKS = 30
        headers_temporary_file = tempfile.mktemp()
        headers = MainHeaders(headers_temporary_file)
        await headers.open()
        self.addCleanup(os.remove, headers_temporary_file)
        async def writer():
            for block_index in range(BLOCKS):
                await headers.connect(block_index, self.get_bytes(block_bytes(block_index + 1), block_bytes(block_index)))
        async def reader():
            for block_index in range(BLOCKS):
                while len(headers) < block_index:
                    await asyncio.sleep(0.000001)
                assert headers[block_index]['block_height'] == block_index
        reader_task = asyncio.create_task(reader())
        await writer()
        await reader_task


class TestHeaders(AsyncioTestCase):

    def test_deserialize(self):
        self.maxDiff = None
        h = Headers(':memory:')
        h.io.write(HEADERS)
        self.assertEqual(h[0], {
            'bits': 520159231,
            'block_height': 0,
            'claim_trie_root': b'0000000000000000000000000000000000000000000000000000000000000001',
            'merkle_root': b'b8211c82c3d15bcd78bba57005b86fed515149a53a425eb592c07af99fe559cc',
            'nonce': 1287,
            'prev_block_hash': b'0000000000000000000000000000000000000000000000000000000000000000',
            'timestamp': 1446058291,
            'version': 1
        })
        self.assertEqual(h[10], {
            'bits': 509349720,
            'block_height': 10,
            'merkle_root': b'f4d8fded6a181d4a8a2817a0eb423cc0f414af29490004a620e66c35c498a554',
            'claim_trie_root': b'0000000000000000000000000000000000000000000000000000000000000001',
            'nonce': 75838,
            'prev_block_hash': b'fdab1b38bcf236bc85b6bcd52fe8ec19bcb0b6c7352e913de05fa5a4e5ae8d55',
            'timestamp': 1466646593,
            'version': 536870912
        })

    async def test_connect_from_genesis(self):
        headers = Headers(':memory:')
        self.assertEqual(headers.height, -1)
        await headers.connect(0, HEADERS)
        self.assertEqual(headers.height, 19)

    async def test_connect_from_middle(self):
        h = Headers(':memory:')
        h.io.write(HEADERS[:10*Headers.header_size])
        self.assertEqual(h.height, 9)
        await h.connect(len(h), HEADERS[10*Headers.header_size:20*Headers.header_size])
        self.assertEqual(h.height, 19)

    def test_target_calculation(self):
        # see: https://github.com/lbryio/lbrycrd/blob/master/src/test/lbry_tests.cpp
        # 1 test block 1 difficulty, should be a max retarget
        self.assertEqual(
            0x1f00e146,
            Headers(':memory').get_next_block_target(
                max_target=ArithUint256(Headers.max_target),
                previous={'timestamp': 1386475638},
                current={'timestamp': 1386475638, 'bits': 0x1f00ffff}
            ).compact
        )
        # test max retarget (difficulty increase)
        self.assertEqual(
            0x1f008ccc,
            Headers(':memory').get_next_block_target(
                max_target=ArithUint256(Headers.max_target),
                previous={'timestamp': 1386475638},
                current={'timestamp': 1386475638, 'bits': 0x1f00a000}
            ).compact
        )
        # test min retarget (difficulty decrease)
        self.assertEqual(
            0x1f00f000,
            Headers(':memory').get_next_block_target(
                max_target=ArithUint256(Headers.max_target),
                previous={'timestamp': 1386475638},
                current={'timestamp': 1386475638 + 60*20, 'bits': 0x1f00a000}
            ).compact
        )
        # test to see if pow limit is not exceeded
        self.assertEqual(
            0x1f00ffff,
            Headers(':memory').get_next_block_target(
                max_target=ArithUint256(Headers.max_target),
                previous={'timestamp': 1386475638},
                current={'timestamp': 1386475638 + 600, 'bits': 0x1f00ffff}
            ).compact
        )

    def test_get_proof_of_work_hash(self):
        # see: https://github.com/lbryio/lbrycrd/blob/master/src/test/lbry_tests.cpp
        self.assertEqual(
            Headers.header_hash_to_pow_hash(Headers.hash_header(b"test string")),
            b"485f3920d48a0448034b0852d1489cfa475341176838c7d36896765221be35ce"
        )
        self.assertEqual(
            Headers.header_hash_to_pow_hash(Headers.hash_header(b"a"*70)),
            b"eb44af2f41e7c6522fb8be4773661be5baa430b8b2c3a670247e9ab060608b75"
        )
        self.assertEqual(
            Headers.header_hash_to_pow_hash(Headers.hash_header(b"d"*140)),
            b"74044747b7c1ff867eb09a84d026b02d8dc539fb6adcec3536f3dfa9266495d9"
        )


HEADERS = unhexlify(
    b'010000000000000000000000000000000000000000000000000000000000000000000000cc59e59ff97ac092b55e4'
    b'23aa5495151ed6fb80570a5bb78cd5bd1c3821c21b801000000000000000000000000000000000000000000000000'
    b'0000000000000033193156ffff001f070500000000002063f4346a4db34fdfce29a70f5e8d11f065f6b91602b7036'
    b'c7f22f3a03b28899cba888e2f9c037f831046f8ad09f6d378f79c728d003b177a64d29621f481da5d010000000000'
    b'00000000000000000000000000000000000000000000000000003c406b5746e1001f5b4f000000000020246cb8584'
    b'3ac936d55388f2ff288b011add5b1b20cca9cfd19a403ca2c9ecbde09d8734d81b5f2eb1b653caf17491544ddfbc7'
    b'2f2f4c0c3f22a3362db5ba9d4701000000000000000000000000000000000000000000000000000000000000003d4'
    b'06b57ffff001f4ff20000000000200044e1258b865d262587c28ff98853bc52bb31266230c1c648cc9004047a5428'
    b'e285dbf24334585b9a924536a717160ee185a86d1eeb7b19684538685eca761a01000000000000000000000000000'
    b'000000000000000000000000000000000003d406b5746e1001fce9c010000000020bbf8980e3f7604896821203bf6'
    b'2f97f311124da1fbb95bf523fcfdb356ad19c9d83cf1408debbd631950b7a95b0c940772119cd8a615a3d44601568'
    b'713fec80c01000000000000000000000000000000000000000000000000000000000000003e406b573dc6001fec7b'
    b'0000000000201a650b9b7b9d132e257ff6b336ba7cd96b1796357c4fc8dd7d0bd1ff1de057d547638e54178dbdddf'
    b'2e81a3b7566860e5264df6066755f9760a893f5caecc5790100000000000000000000000000000000000000000000'
    b'0000000000000000003e406b5773ae001fcf770000000000206d694b93a2bb5ac23a13ed6749a789ca751cf73d598'
    b'2c459e0cd9d5d303da74cec91627e0dba856b933983425d7f72958e8f974682632a0fa2acee9cfd81940101000000'
    b'000000000000000000000000000000000000000000000000000000003e406b578399001f225c010000000020b5780'
    b'8c188b7315583cf120fe89de923583bc7a8ebff03189145b86bf859b21ba3c4a19948a1263722c45c5601fd10a7ae'
    b'a7cf73bfa45e060508f109155e80ab010000000000000000000000000000000000000000000000000000000000000'
    b'03f406b571787001f0816070000000020a6a5b330e816242d54c8586ba9b6d63c19d921171ef3d4525b8ffc635742'
    b'e83a0fc2da46cf0de0057c1b9fc93d997105ff6cf2c8c43269b446c1dbf5ac18be8c0100000000000000000000000'
    b'00000000000000000000000000000000000000040406b570ae1761edd8f030000000020b8447f415279dffe8a09af'
    b'e6f6d5e335a2f6911fce8e1d1866723d5e5e8a53067356a733f87e592ea133328792dd9d676ed83771c8ff0f51992'
    b'8ce752f159ba6010000000000000000000000000000000000000000000000000000000000000040406b57139d681e'
    b'd40d000000000020558daee5a4a55fe03d912e35c7b6b0bc19ece82fd5bcb685bc36f2bc381babfd54a598c4356ce'
    b'620a604004929af14f4c03c42eba017288a4a1d186aedfdd8f4010000000000000000000000000000000000000000'
    b'000000000000000000000041406b57580f5c1e3e280100000000200381bfc0b2f10c9a3c0fc2dc8ad06388aff8ea5'
    b'a9f7dba6a945073b021796197364b79f33ff3f3a7ccb676fc0a37b7d831bd5942a05eac314658c6a7e4c4b1a40100'
    b'00000000000000000000000000000000000000000000000000000000000041406b574303511ec0ae0100000000202'
    b'aae02063ae0f1025e6acecd5e8e2305956ecaefd185bb47a64ea2ae953233891df3d4c1fc547ab3bbca027c8bbba7'
    b'44c051add8615d289b567f97c64929dcf201000000000000000000000000000000000000000000000000000000000'
    b'0000042406b578c4a471e04ee00000000002016603ef45d5a7c02bfbb30f422016746872ff37f8b0b5824a0f70caa'
    b'668eea5415aad300e70f7d8755d93645d1fd21eda9c40c5d0ed797acd0e07ace34585aaf010000000000000000000'
    b'000000000000000000000000000000000000000000042406b577bbc3e1ea163000000000020cad8863b312914f2fd'
    b'2aad6e9420b64859039effd67ac4681a7cf60e42b09b7e7bafa1e8d5131f477785d8338294da0f998844a85b39d24'
    b'26e839b370e014e3b010000000000000000000000000000000000000000000000000000000000000042406b573935'
    b'371e20e900000000002053d5e608ce5a12eda5931f86ee81198fdd231fea64cf096e9aeae321cf2efbe241e888d5a'
    b'af495e4c2a9f11b932db979d7483aeb446f479179b0c0b8d24bfa0e01000000000000000000000000000000000000'
    b'0000000000000000000000000045406b573c95301e34af0a0000000020df0e494c02ff79e3929bc1f2491077ec4f6'
    b'a607d7a1a5e1be96536642c98f86e533febd715f8a234028fd52046708551c6b6ac415480a6568aaa35cb94dc7203'
    b'01000000000000000000000000000000000000000000000000000000000000004f406b57c4c02a1ec54d230000000'
    b'020341f7d8e7d242e5e46343c40840c44f07e7e7306eb2355521b51502e8070e569485ba7eec4efdff0fc755af6e7'
    b'3e38b381a88b0925a68193a25da19d0f616e9f0100000000000000000000000000000000000000000000000000000'
    b'00000000050406b575be8251e1f61010000000020cd399f8078166ca5f0bdd1080ab1bb22d3c271b9729b6000b44f'
    b'4592cc9fab08c00ebab1e7cd88677e3b77c1598c7ac58660567f49f3a30ec46a48a1ae7652fe01000000000000000'
    b'0000000000000000000000000000000000000000000000052406b57d55b211e6f53090000000020c6c14ed4a53bbb'
    b'4f181acf2bbfd8b74d13826732f2114140ca99ca371f7dd87c51d18a05a1a6ffa37c041877fa33c2229a45a0ab66b'
    b'5530f914200a8d6639a6f010000000000000000000000000000000000000000000000000000000000000055406b57'
    b'0d5b1d1eff1c0900'
)