From 6986211c1e0db6bd2bf7199b082e84f7a89c0bf4 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 18 May 2020 08:22:23 -0400 Subject: [PATCH] lbry.blockchain fixes and cleanup --- lbry/blockchain/__init__.py | 4 ++++ lbry/blockchain/block.py | 9 ++++----- lbry/blockchain/database.py | 4 +++- lbry/blockchain/dewies.py | 18 +++++++++++++++--- lbry/blockchain/ledger.py | 8 ++++++++ lbry/blockchain/sync.py | 38 ++++++++++++++++++++++--------------- lbry/blockchain/util.py | 20 ------------------- 7 files changed, 57 insertions(+), 44 deletions(-) diff --git a/lbry/blockchain/__init__.py b/lbry/blockchain/__init__.py index e69de29bb..55b4e48bf 100644 --- a/lbry/blockchain/__init__.py +++ b/lbry/blockchain/__init__.py @@ -0,0 +1,4 @@ +from .ledger import Ledger, RegTestLedger, TestNetLedger +from .transaction import Transaction, Output, Input +from .bcd_data_stream import BCDataStream +from .dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc diff --git a/lbry/blockchain/block.py b/lbry/blockchain/block.py index 38e421307..203010833 100644 --- a/lbry/blockchain/block.py +++ b/lbry/blockchain/block.py @@ -1,5 +1,4 @@ import struct -from hashlib import sha256 from typing import Set from binascii import unhexlify from typing import NamedTuple, List @@ -14,12 +13,12 @@ from lbry.blockchain.bcd_data_stream import BCDataStream ZERO_BLOCK = bytes((0,)*32) -def create_block_filter(addresses: Set[str]) -> bytes: - return bytes(PyBIP158([bytearray(a.encode()) for a in addresses]).GetEncoded()) +def create_block_filter(address_hashes: Set[bytes]) -> bytes: + return bytes(PyBIP158([bytearray(a) for a in address_hashes]).GetEncoded()) -def get_block_filter(block_filter: str) -> PyBIP158: - return PyBIP158(bytearray(unhexlify(block_filter))) +def get_block_filter(block_filter: bytes) -> PyBIP158: + return PyBIP158(bytearray(block_filter)) class Block(NamedTuple): diff --git a/lbry/blockchain/database.py b/lbry/blockchain/database.py index 5041b2dce..490b2b544 100644 --- a/lbry/blockchain/database.py +++ b/lbry/blockchain/database.py @@ -75,7 +75,9 @@ class BlockchainDB: return self.sync_execute_fetchall( """ SELECT datapos as data_offset, height, hash as block_hash, txCount as txs - FROM block_info WHERE file = ? ORDER BY datapos ASC; + FROM block_info + WHERE file = ? and status&1 > 0 + ORDER BY datapos ASC; """, (block_file,) ) diff --git a/lbry/blockchain/dewies.py b/lbry/blockchain/dewies.py index 95d123fa7..272905711 100644 --- a/lbry/blockchain/dewies.py +++ b/lbry/blockchain/dewies.py @@ -1,11 +1,19 @@ +import re import textwrap from decimal import Decimal -from lbry.blockchain.util import coins_to_satoshis, satoshis_to_coins + +from lbry.constants import COIN def lbc_to_dewies(lbc: str) -> int: try: - return coins_to_satoshis(lbc) + if not isinstance(lbc, str): + raise ValueError("{coins} must be a string") + result = re.search(r'^(\d{1,10})\.(\d{1,8})$', lbc) + if result is not None: + whole, fractional = result.groups() + return int(whole + fractional.ljust(8, "0")) + raise ValueError(f"'{lbc}' is not a valid coin decimal") except ValueError: raise ValueError(textwrap.dedent( f""" @@ -31,7 +39,11 @@ def lbc_to_dewies(lbc: str) -> int: def dewies_to_lbc(dewies) -> str: - return satoshis_to_coins(dewies) + coins = '{:.8f}'.format(dewies / COIN).rstrip('0') + if coins.endswith('.'): + return coins+'0' + else: + return coins def dict_values_to_lbc(d): diff --git a/lbry/blockchain/ledger.py b/lbry/blockchain/ledger.py index aa372c8a1..04f938fbb 100644 --- a/lbry/blockchain/ledger.py +++ b/lbry/blockchain/ledger.py @@ -1,4 +1,5 @@ from binascii import unhexlify +from string import hexdigits from lbry.crypto.hash import hash160, double_sha256 from lbry.crypto.base58 import Base58 @@ -61,6 +62,13 @@ class Ledger: except: raise Exception(f"'{address}' is not a valid address") + @staticmethod + def valid_claim_id(claim_id: str): + if not len(claim_id) == 40: + raise Exception(f"Incorrect claimid length: {len(claim_id)}") + if set(claim_id).difference(hexdigits): + raise Exception("Claim id is not hex encoded") + @staticmethod def valid_channel_name_or_error(name: str): try: diff --git a/lbry/blockchain/sync.py b/lbry/blockchain/sync.py index 0f6efb274..e5f596363 100644 --- a/lbry/blockchain/sync.py +++ b/lbry/blockchain/sync.py @@ -11,12 +11,11 @@ from sqlalchemy.future import select from lbry.event import EventController, BroadcastSubscription from lbry.service.base import Service, Sync, BlockEvent -from lbry.db import ( - queries, TXO_TYPES, Claim, Claimtrie, TX, TXO, TXI, Block as BlockTable, -) +from lbry.db import queries, TXO_TYPES +from lbry.db.tables import Claim, Claimtrie, TX, TXO, TXI, Block as BlockTable from .lbrycrd import Lbrycrd -from .block import Block, create_block_filter +from .block import Block, create_block_filter, get_block_filter from .bcd_data_stream import BCDataStream from .ledger import Ledger @@ -106,23 +105,32 @@ def process_claimtrie(): def process_block_and_tx_filters(): - execute = queries.ctx().execute + context = queries.ctx() + execute = context.execute + ledger = context.ledger blocks = [] + all_filters = [] + all_addresses = [] for block in queries.get_blocks_without_filters(): - block_filter = create_block_filter( - {r['address'] for r in queries.get_block_tx_addresses(block_hash=block['block_hash'])} - ) + addresses = { + ledger.address_to_hash160(r['address']) + for r in queries.get_block_tx_addresses(block_hash=block['block_hash']) + } + all_addresses.extend(addresses) + block_filter = create_block_filter(addresses) + all_filters.append(block_filter) blocks.append({'pk': block['block_hash'], 'block_filter': block_filter}) + filters = [get_block_filter(f) for f in all_filters] execute(BlockTable.update().where(BlockTable.c.block_hash == bindparam('pk')), blocks) - txs = [] - for tx in queries.get_transactions_without_filters(): - tx_filter = create_block_filter( - {r['address'] for r in queries.get_block_tx_addresses(tx_hash=tx['tx_hash'])} - ) - txs.append({'pk': tx['tx_hash'], 'tx_filter': tx_filter}) - execute(TX.update().where(TX.c.tx_hash == bindparam('pk')), txs) +# txs = [] +# for tx in queries.get_transactions_without_filters(): +# tx_filter = create_block_filter( +# {r['address'] for r in queries.get_block_tx_addresses(tx_hash=tx['tx_hash'])} +# ) +# txs.append({'pk': tx['tx_hash'], 'tx_filter': tx_filter}) +# execute(TX.update().where(TX.c.tx_hash == bindparam('pk')), txs) class BlockchainSync(Sync): diff --git a/lbry/blockchain/util.py b/lbry/blockchain/util.py index 2ae6e4ed8..e1d1940eb 100644 --- a/lbry/blockchain/util.py +++ b/lbry/blockchain/util.py @@ -1,24 +1,4 @@ -import re from typing import TypeVar, Sequence, Optional -from lbry.constants import COIN - - -def coins_to_satoshis(coins): - if not isinstance(coins, str): - raise ValueError("{coins} must be a string") - result = re.search(r'^(\d{1,10})\.(\d{1,8})$', coins) - if result is not None: - whole, fractional = result.groups() - return int(whole+fractional.ljust(8, "0")) - raise ValueError("'{lbc}' is not a valid coin decimal") - - -def satoshis_to_coins(satoshis): - coins = '{:.8f}'.format(satoshis / COIN).rstrip('0') - if coins.endswith('.'): - return coins+'0' - else: - return coins T = TypeVar('T')