299 lines
12 KiB
Python
299 lines
12 KiB
Python
from binascii import hexlify
|
|
from sqlalchemy.future import select
|
|
|
|
from lbry.crypto.hash import hash160
|
|
from lbry.crypto.bip32 import from_extended_key_string
|
|
from lbry.blockchain.block import create_address_filter
|
|
from lbry.db import queries as q
|
|
from lbry.db.tables import AccountAddress, TX
|
|
from lbry.db.query_context import context
|
|
from lbry.testcase import UnitDBTestCase
|
|
|
|
|
|
class TestMissingRequiredFiltersCalculation(UnitDBTestCase):
|
|
|
|
def test_get_missing_required_filters(self):
|
|
self.assertEqual(q.get_missing_required_filters(99), {(1, 0, 99)})
|
|
self.assertEqual(q.get_missing_required_filters(100), {(2, 0, 0)})
|
|
self.assertEqual(q.get_missing_required_filters(199), {(2, 0, 0), (1, 100, 199)})
|
|
self.assertEqual(q.get_missing_required_filters(201), {(2, 0, 100), (1, 200, 201)})
|
|
# all filters missing
|
|
self.assertEqual(q.get_missing_required_filters(134_567), {
|
|
(4, 0, 120_000),
|
|
(3, 130_000, 133_000),
|
|
(2, 134_000, 134_400),
|
|
(1, 134_500, 134_567)
|
|
})
|
|
|
|
q.insert_block_filter(110_000, 4, b'beef')
|
|
q.insert_block_filter(129_000, 3, b'beef')
|
|
q.insert_block_filter(133_900, 2, b'beef')
|
|
q.insert_block_filter(134_499, 1, b'beef')
|
|
# we we have some filters, but not recent enough (all except 10k are adjusted)
|
|
self.assertEqual(q.get_missing_required_filters(134_567), {
|
|
(4, 120_000, 120_000), # 0 -> 120_000
|
|
(3, 130_000, 133_000),
|
|
(2, 134_000, 134_400),
|
|
(1, 134_500, 134_567)
|
|
})
|
|
|
|
q.insert_block_filter(132_000, 3, b'beef')
|
|
q.insert_block_filter(134_300, 2, b'beef')
|
|
q.insert_block_filter(134_550, 1, b'beef')
|
|
# all filters get adjusted because we have recent of each
|
|
self.assertEqual(q.get_missing_required_filters(134_567), {
|
|
(4, 120_000, 120_000), # 0 -> 120_000
|
|
(3, 133_000, 133_000), # 130_000 -> 133_000
|
|
(2, 134_400, 134_400), # 134_000 -> 134_400
|
|
(1, 134_551, 134_567) # 134_500 -> 134_551
|
|
})
|
|
|
|
q.insert_block_filter(120_000, 4, b'beef')
|
|
q.insert_block_filter(133_000, 3, b'beef')
|
|
q.insert_block_filter(134_400, 2, b'beef')
|
|
q.insert_block_filter(134_566, 1, b'beef')
|
|
# we have latest filters for all except latest single block
|
|
self.assertEqual(q.get_missing_required_filters(134_567), {
|
|
(1, 134_567, 134_567) # 134_551 -> 134_567
|
|
})
|
|
|
|
q.insert_block_filter(134_567, 1, b'beef')
|
|
# we have all latest filters
|
|
self.assertEqual(q.get_missing_required_filters(134_567), set())
|
|
|
|
|
|
class TestAddressGenerationAndTXSync(UnitDBTestCase):
|
|
|
|
RECEIVING_KEY_N = 0
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.root_pubkey = from_extended_key_string(
|
|
self.ledger,
|
|
'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMmDgp66FxH'
|
|
'uDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9'
|
|
)
|
|
self.receiving_pubkey = self.root_pubkey.child(self.RECEIVING_KEY_N)
|
|
|
|
def generate(self, loops, is_new_starts=0):
|
|
with q.PersistingAddressIterator(
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
self.receiving_pubkey.pubkey_bytes,
|
|
self.receiving_pubkey.chain_code,
|
|
self.receiving_pubkey.depth
|
|
) as generator:
|
|
for address, n, is_new in generator:
|
|
if n >= is_new_starts:
|
|
self.assertTrue(is_new)
|
|
else:
|
|
self.assertFalse(is_new)
|
|
if n == loops:
|
|
break
|
|
|
|
@staticmethod
|
|
def get_ordered_addresses():
|
|
return [r["address"] for r in context().fetchall(
|
|
select(AccountAddress.c.address).order_by(AccountAddress.c.n)
|
|
)]
|
|
|
|
def test_generator_persisting(self):
|
|
expected = [self.receiving_pubkey.child(n).address for n in range(30)]
|
|
self.assertEqual([], self.get_ordered_addresses())
|
|
self.generate(5, 0)
|
|
self.assertEqual(expected[:6], self.get_ordered_addresses())
|
|
self.generate(7, 6)
|
|
self.assertEqual(expected[:8], self.get_ordered_addresses())
|
|
self.generate(12, 8)
|
|
self.assertEqual(expected[:13], self.get_ordered_addresses())
|
|
|
|
@staticmethod
|
|
def insert_sub_filters(granularity, addresses, height):
|
|
if granularity >= 2:
|
|
end = height + 10 ** granularity
|
|
step = 1 if granularity == 2 else 10 ** (granularity - 1)
|
|
for i, height in enumerate(range(height, end, step)):
|
|
if i == 3:
|
|
q.insert_block_filter(height, granularity - 1, create_address_filter(addresses))
|
|
else:
|
|
q.insert_block_filter(height, granularity - 1, create_address_filter([b'beef']))
|
|
elif granularity == 1:
|
|
q.insert_tx_filter(hexlify(f'tx{height}'.encode()), height, create_address_filter(addresses))
|
|
|
|
def test_generate_from_filters_and_download_txs(self):
|
|
# 15 addresses will get generated, 9 due to filters and 6 due to gap
|
|
pubkeys = [self.receiving_pubkey.child(n) for n in range(15)]
|
|
hashes = [hash160(key.pubkey_bytes) for key in pubkeys]
|
|
|
|
# create all required filters (include 9 of the addresses in the filters)
|
|
|
|
q.insert_block_filter(0, 4, create_address_filter(hashes[0:1]))
|
|
q.insert_block_filter(100_000, 4, create_address_filter(hashes[1:2]))
|
|
q.insert_block_filter(110_000, 4, create_address_filter([b'beef']))
|
|
q.insert_block_filter(120_000, 4, create_address_filter(hashes[2:3]))
|
|
|
|
q.insert_block_filter(130_000, 3, create_address_filter(hashes[3:4]))
|
|
q.insert_block_filter(131_000, 3, create_address_filter([b'beef']))
|
|
q.insert_block_filter(133_000, 3, create_address_filter(hashes[4:5]))
|
|
|
|
q.insert_block_filter(134_000, 2, create_address_filter(hashes[5:6]))
|
|
q.insert_block_filter(134_200, 2, create_address_filter([b'beef']))
|
|
q.insert_block_filter(134_400, 2, create_address_filter(hashes[6:7]))
|
|
|
|
q.insert_block_filter(134_500, 1, create_address_filter(hashes[7:8]))
|
|
q.insert_block_filter(134_566, 1, create_address_filter([b'beef']))
|
|
q.insert_block_filter(134_567, 1, create_address_filter(hashes[8:9]))
|
|
|
|
# check that all required filters did get created
|
|
self.assertEqual(q.get_missing_required_filters(134_567), set())
|
|
|
|
# no addresses
|
|
self.assertEqual([], self.get_ordered_addresses())
|
|
|
|
# generate addresses with 6 address gap, returns new sub filters needed
|
|
self.assertEqual(
|
|
q.generate_addresses_using_filters(134_567, 6, (
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
self.receiving_pubkey.pubkey_bytes,
|
|
self.receiving_pubkey.chain_code,
|
|
self.receiving_pubkey.depth
|
|
)), {
|
|
(0, 134500, 134500),
|
|
(0, 134567, 134567),
|
|
(1, 134000, 134099),
|
|
(1, 134400, 134499),
|
|
(2, 130000, 130900),
|
|
(2, 133000, 133900),
|
|
(3, 0, 9000),
|
|
(3, 100000, 109000),
|
|
(3, 120000, 129000)
|
|
}
|
|
)
|
|
|
|
# all addresses generated
|
|
self.assertEqual([key.address for key in pubkeys], self.get_ordered_addresses())
|
|
|
|
# "download" missing sub filters
|
|
self.insert_sub_filters(4, hashes[0:1], 0)
|
|
self.insert_sub_filters(4, hashes[1:2], 100_000)
|
|
self.insert_sub_filters(4, hashes[2:3], 120_000)
|
|
self.insert_sub_filters(3, hashes[3:4], 130_000)
|
|
self.insert_sub_filters(3, hashes[4:5], 133_000)
|
|
self.insert_sub_filters(2, hashes[5:6], 134_000)
|
|
self.insert_sub_filters(2, hashes[6:7], 134_400)
|
|
self.insert_sub_filters(1, hashes[7:8], 134_500)
|
|
self.insert_sub_filters(1, hashes[8:9], 134_567)
|
|
|
|
# no sub filters needed to be downloaded now when re-checking all addresses
|
|
self.assertEqual(
|
|
q.generate_addresses_using_filters(134_567, 6, (
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
self.receiving_pubkey.pubkey_bytes,
|
|
self.receiving_pubkey.chain_code,
|
|
self.receiving_pubkey.depth
|
|
)), set()
|
|
)
|
|
# no new addresses should have been generated either
|
|
self.assertEqual([key.address for key in pubkeys], self.get_ordered_addresses())
|
|
|
|
# check sub filters at 1,000
|
|
self.assertEqual(
|
|
q.get_missing_sub_filters_for_addresses(3, (
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
)), {
|
|
(2, 3000, 3900),
|
|
(2, 103000, 103900),
|
|
(2, 123000, 123900),
|
|
}
|
|
)
|
|
# "download" missing 1,000 sub filters
|
|
self.insert_sub_filters(3, hashes[0:1], 3000)
|
|
self.insert_sub_filters(3, hashes[1:2], 103_000)
|
|
self.insert_sub_filters(3, hashes[2:3], 123_000)
|
|
# no more missing sub filters at 1,000
|
|
self.assertEqual(
|
|
q.get_missing_sub_filters_for_addresses(3, (
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
)), set()
|
|
)
|
|
|
|
# check sub filters at 100
|
|
self.assertEqual(
|
|
q.get_missing_sub_filters_for_addresses(2, (
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
)), {
|
|
(1, 3300, 3399),
|
|
(1, 103300, 103399),
|
|
(1, 123300, 123399),
|
|
(1, 130300, 130399),
|
|
(1, 133300, 133399),
|
|
}
|
|
)
|
|
# "download" missing 100 sub filters
|
|
self.insert_sub_filters(2, hashes[0:1], 3300)
|
|
self.insert_sub_filters(2, hashes[1:2], 103_300)
|
|
self.insert_sub_filters(2, hashes[2:3], 123_300)
|
|
self.insert_sub_filters(2, hashes[3:4], 130_300)
|
|
self.insert_sub_filters(2, hashes[4:5], 133_300)
|
|
# no more missing sub filters at 100
|
|
self.assertEqual(
|
|
q.get_missing_sub_filters_for_addresses(2, (
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
)), set()
|
|
)
|
|
|
|
# check tx filters
|
|
self.assertEqual(
|
|
q.get_missing_sub_filters_for_addresses(1, (
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
)), {
|
|
(0, 3303, 3303),
|
|
(0, 103303, 103303),
|
|
(0, 123303, 123303),
|
|
(0, 130303, 130303),
|
|
(0, 133303, 133303),
|
|
(0, 134003, 134003),
|
|
(0, 134403, 134403),
|
|
}
|
|
)
|
|
# "download" missing tx filters
|
|
self.insert_sub_filters(1, hashes[0:1], 3303)
|
|
self.insert_sub_filters(1, hashes[1:2], 103_303)
|
|
self.insert_sub_filters(1, hashes[2:3], 123_303)
|
|
self.insert_sub_filters(1, hashes[3:4], 130_303)
|
|
self.insert_sub_filters(1, hashes[4:5], 133_303)
|
|
self.insert_sub_filters(1, hashes[5:6], 134_003)
|
|
self.insert_sub_filters(1, hashes[6:7], 134_403)
|
|
# no more missing tx filters
|
|
self.assertEqual(
|
|
q.get_missing_sub_filters_for_addresses(1, (
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
)), set()
|
|
)
|
|
|
|
# find TXs we need to download
|
|
missing_txs = {
|
|
b'7478313033333033',
|
|
b'7478313233333033',
|
|
b'7478313330333033',
|
|
b'7478313333333033',
|
|
b'7478313334303033',
|
|
b'7478313334343033',
|
|
b'7478313334353030',
|
|
b'7478313334353637',
|
|
b'747833333033'
|
|
}
|
|
self.assertEqual(
|
|
q.get_missing_tx_for_addresses((
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
)), missing_txs
|
|
)
|
|
# "download" missing TXs
|
|
ctx = context()
|
|
for tx_hash in missing_txs:
|
|
ctx.execute(TX.insert().values(tx_hash=tx_hash))
|
|
# check we have everything
|
|
self.assertEqual(
|
|
q.get_missing_tx_for_addresses((
|
|
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
|
)), set()
|
|
)
|