lbry.blockchain fixes and cleanup

lbry/blockchain/__init__.py

@@ -0,0 +1,4 @@
+from .ledger import Ledger, RegTestLedger, TestNetLedger
+from .transaction import Transaction, Output, Input
+from .bcd_data_stream import BCDataStream
+from .dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc

lbry/blockchain/block.py

@@ -1,5 +1,4 @@
 import struct
-from hashlib import sha256
 from typing import Set
 from binascii import unhexlify
 from typing import NamedTuple, List
@@ -14,12 +13,12 @@ from lbry.blockchain.bcd_data_stream import BCDataStream
 ZERO_BLOCK = bytes((0,)*32)
 
 
-def create_block_filter(addresses: Set[str]) -> bytes:
+def create_block_filter(address_hashes: Set[bytes]) -> bytes:
-    return bytes(PyBIP158([bytearray(a.encode()) for a in addresses]).GetEncoded())
+    return bytes(PyBIP158([bytearray(a) for a in address_hashes]).GetEncoded())
 
 
-def get_block_filter(block_filter: str) -> PyBIP158:
+def get_block_filter(block_filter: bytes) -> PyBIP158:
-    return PyBIP158(bytearray(unhexlify(block_filter)))
+    return PyBIP158(bytearray(block_filter))
 
 
 class Block(NamedTuple):

lbry/blockchain/database.py

@@ -75,7 +75,9 @@ class BlockchainDB:
         return self.sync_execute_fetchall(
             """
             SELECT datapos as data_offset, height, hash as block_hash, txCount as txs
-            FROM block_info WHERE file = ? ORDER BY datapos ASC;
+            FROM block_info
+            WHERE file = ? and status&1 > 0
+            ORDER BY datapos ASC;
             """, (block_file,)
         )
 

lbry/blockchain/dewies.py

@@ -1,11 +1,19 @@
+import re
 import textwrap
 from decimal import Decimal
-from lbry.blockchain.util import coins_to_satoshis, satoshis_to_coins
+
+from lbry.constants import COIN
 
 
 def lbc_to_dewies(lbc: str) -> int:
     try:
-        return coins_to_satoshis(lbc)
+        if not isinstance(lbc, str):
+            raise ValueError("{coins} must be a string")
+        result = re.search(r'^(\d{1,10})\.(\d{1,8})$', lbc)
+        if result is not None:
+            whole, fractional = result.groups()
+            return int(whole + fractional.ljust(8, "0"))
+        raise ValueError(f"'{lbc}' is not a valid coin decimal")
     except ValueError:
         raise ValueError(textwrap.dedent(
             f"""
@@ -31,7 +39,11 @@ def lbc_to_dewies(lbc: str) -> int:
 
 
 def dewies_to_lbc(dewies) -> str:
-    return satoshis_to_coins(dewies)
+    coins = '{:.8f}'.format(dewies / COIN).rstrip('0')
+    if coins.endswith('.'):
+        return coins+'0'
+    else:
+        return coins
 
 
 def dict_values_to_lbc(d):

lbry/blockchain/ledger.py

@@ -1,4 +1,5 @@
 from binascii import unhexlify
+from string import hexdigits
 
 from lbry.crypto.hash import hash160, double_sha256
 from lbry.crypto.base58 import Base58
@@ -61,6 +62,13 @@ class Ledger:
         except:
             raise Exception(f"'{address}' is not a valid address")
 
+    @staticmethod
+    def valid_claim_id(claim_id: str):
+        if not len(claim_id) == 40:
+            raise Exception(f"Incorrect claimid length: {len(claim_id)}")
+        if set(claim_id).difference(hexdigits):
+            raise Exception("Claim id is not hex encoded")
+
     @staticmethod
     def valid_channel_name_or_error(name: str):
         try:

lbry/blockchain/sync.py

@@ -11,12 +11,11 @@ from sqlalchemy.future import select
 
 from lbry.event import EventController, BroadcastSubscription
 from lbry.service.base import Service, Sync, BlockEvent
-from lbry.db import (
+from lbry.db import queries, TXO_TYPES
-    queries, TXO_TYPES, Claim, Claimtrie, TX, TXO, TXI, Block as BlockTable,
+from lbry.db.tables import Claim, Claimtrie, TX, TXO, TXI, Block as BlockTable
-)
 
 from .lbrycrd import Lbrycrd
-from .block import Block, create_block_filter
+from .block import Block, create_block_filter, get_block_filter
 from .bcd_data_stream import BCDataStream
 from .ledger import Ledger
 
@@ -106,23 +105,32 @@ def process_claimtrie():
 
 
 def process_block_and_tx_filters():
-    execute = queries.ctx().execute
+    context = queries.ctx()
+    execute = context.execute
+    ledger = context.ledger
 
     blocks = []
+    all_filters = []
+    all_addresses = []
     for block in queries.get_blocks_without_filters():
-        block_filter = create_block_filter(
+        addresses = {
-            {r['address'] for r in queries.get_block_tx_addresses(block_hash=block['block_hash'])}
+            ledger.address_to_hash160(r['address'])
-        )
+            for r in queries.get_block_tx_addresses(block_hash=block['block_hash'])
+        }
+        all_addresses.extend(addresses)
+        block_filter = create_block_filter(addresses)
+        all_filters.append(block_filter)
         blocks.append({'pk': block['block_hash'], 'block_filter': block_filter})
+    filters = [get_block_filter(f) for f in all_filters]
     execute(BlockTable.update().where(BlockTable.c.block_hash == bindparam('pk')), blocks)
 
-    txs = []
+#    txs = []
-    for tx in queries.get_transactions_without_filters():
+#    for tx in queries.get_transactions_without_filters():
-        tx_filter = create_block_filter(
+#        tx_filter = create_block_filter(
-            {r['address'] for r in queries.get_block_tx_addresses(tx_hash=tx['tx_hash'])}
+#            {r['address'] for r in queries.get_block_tx_addresses(tx_hash=tx['tx_hash'])}
-        )
+#        )
-        txs.append({'pk': tx['tx_hash'], 'tx_filter': tx_filter})
+#        txs.append({'pk': tx['tx_hash'], 'tx_filter': tx_filter})
-    execute(TX.update().where(TX.c.tx_hash == bindparam('pk')), txs)
+#    execute(TX.update().where(TX.c.tx_hash == bindparam('pk')), txs)
 
 
 class BlockchainSync(Sync):

lbry/blockchain/util.py

@@ -1,24 +1,4 @@
-import re
 from typing import TypeVar, Sequence, Optional
-from lbry.constants import COIN
-
-
-def coins_to_satoshis(coins):
-    if not isinstance(coins, str):
-        raise ValueError("{coins} must be a string")
-    result = re.search(r'^(\d{1,10})\.(\d{1,8})$', coins)
-    if result is not None:
-        whole, fractional = result.groups()
-        return int(whole+fractional.ljust(8, "0"))
-    raise ValueError("'{lbc}' is not a valid coin decimal")
-
-
-def satoshis_to_coins(satoshis):
-    coins = '{:.8f}'.format(satoshis / COIN).rstrip('0')
-    if coins.endswith('.'):
-        return coins+'0'
-    else:
-        return coins
 
 
 T = TypeVar('T')