From 066f797ad4ebe589cf47c1c039fa75a1be2ffeb7 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 9 Jun 2021 16:29:16 -0400 Subject: [PATCH] refactor ClaimToTXO prefix --- lbry/wallet/server/block_processor.py | 44 +++++++++++++-------------- lbry/wallet/server/db/prefixes.py | 43 +++++++++++++------------- lbry/wallet/server/leveldb.py | 31 ++++++++++--------- 3 files changed, 58 insertions(+), 60 deletions(-) diff --git a/lbry/wallet/server/block_processor.py b/lbry/wallet/server/block_processor.py index 3ddae77f6..506d2f300 100644 --- a/lbry/wallet/server/block_processor.py +++ b/lbry/wallet/server/block_processor.py @@ -296,7 +296,7 @@ class BlockProcessor: if not reposted_claim: continue reposted_metadata = get_claim_txo( - self.db.total_transactions[reposted_claim[0].tx_num], reposted_claim[0].position + self.db.total_transactions[reposted_claim.tx_num], reposted_claim.position ) if not reposted_metadata: continue @@ -305,14 +305,14 @@ class BlockProcessor: reposted_has_source = None reposted_claim_type = None if reposted_claim: - reposted_tx_hash = self.db.total_transactions[reposted_claim[0].tx_num] + reposted_tx_hash = self.db.total_transactions[reposted_claim.tx_num] raw_reposted_claim_tx = self.db.db.get( DB_PREFIXES.TX_PREFIX.value + reposted_tx_hash ) try: reposted_claim_txo: TxOutput = self.coin.transaction( raw_reposted_claim_tx - ).outputs[reposted_claim[0].position] + ).outputs[reposted_claim.position] reposted_script = OutputScript(reposted_claim_txo.pk_script) reposted_script.parse() except: @@ -635,7 +635,7 @@ class BlockProcessor: signing_channel = self.db.get_claim_txo(signing_channel_hash) if signing_channel: raw_channel_tx = self.db.db.get( - DB_PREFIXES.TX_PREFIX.value + self.db.total_transactions[signing_channel[0].tx_num] + DB_PREFIXES.TX_PREFIX.value + self.db.total_transactions[signing_channel.tx_num] ) channel_pub_key_bytes = None try: @@ -643,7 +643,7 @@ class BlockProcessor: if txo.signable.signing_channel_hash[::-1] in self.pending_channels: channel_pub_key_bytes = self.pending_channels[txo.signable.signing_channel_hash[::-1]] elif raw_channel_tx: - chan_output = self.coin.transaction(raw_channel_tx).outputs[signing_channel[0].position] + chan_output = self.coin.transaction(raw_channel_tx).outputs[signing_channel.position] chan_script = OutputScript(chan_output.pk_script) chan_script.parse() @@ -671,7 +671,7 @@ class BlockProcessor: previous_claim = self.pending_claims.pop((prev_tx_num, prev_idx)) root_tx_num, root_idx = previous_claim.root_claim_tx_num, previous_claim.root_claim_tx_position else: - k, v = self.db.get_claim_txo( + v = self.db.get_claim_txo( claim_hash ) root_tx_num, root_idx = v.root_tx_num, v.root_position @@ -750,7 +750,7 @@ class BlockProcessor: return [] claim_hash = spent_claim_hash_and_name.claim_hash signing_hash = self.db.get_channel_for_claim(claim_hash) - k, v = self.db.get_claim_txo(claim_hash) + v = self.db.get_claim_txo(claim_hash) reposted_claim_hash = self.db.get_repost(claim_hash) spent = StagedClaimtrieItem( v.name, claim_hash, v.amount, @@ -782,7 +782,7 @@ class BlockProcessor: expiration = self.coin.get_expiration_height(self.height) signature_is_valid = pending.channel_signature_is_valid else: - k, v = self.db.get_claim_txo( + v = self.db.get_claim_txo( claim_hash ) claim_root_tx_num, claim_root_idx, prev_amount = v.root_tx_num, v.root_position, v.amount @@ -838,7 +838,7 @@ class BlockProcessor: return self.pending_claims[claim_hash].name claim_info = self.db.get_claim_txo(claim_hash) if claim_info: - return claim_info[1].name + return claim_info.name def _get_pending_supported_amount(self, claim_hash: bytes, height: Optional[int] = None) -> int: amount = self.db._get_active_amount(claim_hash, ACTIVATED_SUPPORT_TXO_TYPE, height or (self.height + 1)) or 0 @@ -965,9 +965,9 @@ class BlockProcessor: # the supported claim doesn't exist continue else: - k, v = supported_claim_info + v = supported_claim_info name = v.name - staged_is_new_claim = (v.root_tx_num, v.root_position) == (k.tx_num, k.position) + staged_is_new_claim = (v.root_tx_num, v.root_position) == (v.tx_num, v.position) ops.extend(get_delayed_activate_ops( name, claim_hash, staged_is_new_claim, tx_num, nout, amount, is_support=True )) @@ -994,7 +994,7 @@ class BlockProcessor: amount = self.pending_claims[txo_tup].amount else: amount = self.db.get_claim_txo_amount( - activated.claim_hash, activated_txo.tx_num, activated_txo.position + activated.claim_hash ) self.staged_activated_claim[(activated.name, activated.claim_hash)] = amount else: @@ -1038,7 +1038,7 @@ class BlockProcessor: existing_activation, ACTIVATED_CLAIM_TXO_TYPE, tx_num, nout ) self.pending_activated[need_takeover][candidate_claim_hash].append(( - activate_key, self.db.get_claim_txo_amount(candidate_claim_hash, tx_num, nout) + activate_key, self.db.get_claim_txo_amount(candidate_claim_hash) )) need_reactivate_if_takes_over[(need_takeover, candidate_claim_hash)] = activate_key print(f"\tcandidate to takeover abandoned controlling claim for lbry://{need_takeover} - " @@ -1122,9 +1122,9 @@ class BlockProcessor: # handle a pending activated claim jumping the takeover delay when another name takes over if winning_including_future_activations not in self.pending_claim_txos: claim = self.db.get_claim_txo(winning_including_future_activations) - tx_num = claim[0].tx_num - position = claim[0].position - amount = claim[1].amount + tx_num = claim.tx_num + position = claim.position + amount = claim.amount activation = self.db.get_activation(tx_num, position) else: tx_num, position = self.pending_claim_txos[winning_including_future_activations] @@ -1173,7 +1173,7 @@ class BlockProcessor: if (name, winning_claim_hash) in need_reactivate_if_takes_over: previous_pending_activate = need_reactivate_if_takes_over[(name, winning_claim_hash)] amount = self.db.get_claim_txo_amount( - winning_claim_hash, previous_pending_activate.tx_num, previous_pending_activate.position + winning_claim_hash ) if winning_claim_hash in self.pending_claim_txos: tx_num, position = self.pending_claim_txos[winning_claim_hash] @@ -1232,8 +1232,7 @@ class BlockProcessor: removed_claim = self.db.get_claim_txo(removed) if not removed_claim: continue - k, v = removed_claim - name, tx_num, position = v.name, k.tx_num, k.position + name, tx_num, position = removed_claim.name, removed_claim.tx_num, removed_claim.position ops.extend(get_remove_effective_amount_ops( name, self.db.get_effective_amount(removed), tx_num, position, removed )) @@ -1243,14 +1242,13 @@ class BlockProcessor: name, tx_num, position = pending.name, pending.tx_num, pending.position claim_from_db = self.db.get_claim_txo(touched) if claim_from_db: - k, v = claim_from_db - prev_tx_num, prev_position = k.tx_num, k.position + prev_tx_num, prev_position = claim_from_db.tx_num, claim_from_db.position ops.extend(get_remove_effective_amount_ops( name, self.db.get_effective_amount(touched), prev_tx_num, prev_position, touched )) else: - k, v = self.db.get_claim_txo(touched) - name, tx_num, position = v.name, k.tx_num, k.position + v = self.db.get_claim_txo(touched) + name, tx_num, position = v.name, v.tx_num, v.position ops.extend(get_remove_effective_amount_ops( name, self.db.get_effective_amount(touched), tx_num, position, touched )) diff --git a/lbry/wallet/server/db/prefixes.py b/lbry/wallet/server/db/prefixes.py index d3236029c..32a2f5bbf 100644 --- a/lbry/wallet/server/db/prefixes.py +++ b/lbry/wallet/server/db/prefixes.py @@ -46,11 +46,11 @@ class PrefixRow: class ClaimToTXOKey(typing.NamedTuple): claim_hash: bytes - tx_num: int - position: int class ClaimToTXOValue(typing.NamedTuple): + tx_num: int + position: int root_tx_num: int root_position: int amount: int @@ -248,48 +248,47 @@ class ActiveAmountPrefixRow(PrefixRow): class ClaimToTXOPrefixRow(PrefixRow): prefix = DB_PREFIXES.claim_to_txo.value - key_struct = struct.Struct(b'>20sLH') - value_struct = struct.Struct(b'>LHQB') + key_struct = struct.Struct(b'>20s') + value_struct = struct.Struct(b'>LHLHQB') key_part_lambdas = [ lambda: b'', - struct.Struct(b'>20s').pack, - struct.Struct(b'>20sL').pack, - struct.Struct(b'>20sLH').pack + struct.Struct(b'>20s').pack ] @classmethod - def pack_key(cls, claim_hash: bytes, tx_num: int, position: int): + def pack_key(cls, claim_hash: bytes): return super().pack_key( - claim_hash, 0xffffffff - tx_num, 0xffff - position + claim_hash ) @classmethod def unpack_key(cls, key: bytes) -> ClaimToTXOKey: - assert key[:1] == cls.prefix - claim_hash, ones_comp_tx_num, ones_comp_position = cls.key_struct.unpack(key[1:]) - return ClaimToTXOKey( - claim_hash, 0xffffffff - ones_comp_tx_num, 0xffff - ones_comp_position - ) + assert key[:1] == cls.prefix and len(key) == 21 + return ClaimToTXOKey(key[1:]) @classmethod def unpack_value(cls, data: bytes) -> ClaimToTXOValue: - root_tx_num, root_position, amount, channel_signature_is_valid = cls.value_struct.unpack(data[:15]) - name_len = int.from_bytes(data[15:17], byteorder='big') - name = data[17:17 + name_len].decode() - return ClaimToTXOValue(root_tx_num, root_position, amount, bool(channel_signature_is_valid), name) + tx_num, position, root_tx_num, root_position, amount, channel_signature_is_valid = cls.value_struct.unpack( + data[:21] + ) + name_len = int.from_bytes(data[21:23], byteorder='big') + name = data[23:23 + name_len].decode() + return ClaimToTXOValue( + tx_num, position, root_tx_num, root_position, amount, bool(channel_signature_is_valid), name + ) @classmethod - def pack_value(cls, root_tx_num: int, root_position: int, amount: int, + def pack_value(cls, tx_num: int, position: int, root_tx_num: int, root_position: int, amount: int, channel_signature_is_valid: bool, name: str) -> bytes: return cls.value_struct.pack( - root_tx_num, root_position, amount, int(channel_signature_is_valid) + tx_num, position, root_tx_num, root_position, amount, int(channel_signature_is_valid) ) + length_encoded_name(name) @classmethod def pack_item(cls, claim_hash: bytes, tx_num: int, position: int, root_tx_num: int, root_position: int, amount: int, channel_signature_is_valid: bool, name: str): - return cls.pack_key(claim_hash, tx_num, position), \ - cls.pack_value(root_tx_num, root_position, amount, channel_signature_is_valid, name) + return cls.pack_key(claim_hash), \ + cls.pack_value(tx_num, position, root_tx_num, root_position, amount, channel_signature_is_valid, name) class TXOToClaimPrefixRow(PrefixRow): diff --git a/lbry/wallet/server/leveldb.py b/lbry/wallet/server/leveldb.py index 53b11c7a1..5c9504fda 100644 --- a/lbry/wallet/server/leveldb.py +++ b/lbry/wallet/server/leveldb.py @@ -214,7 +214,7 @@ class LevelDB: expiration_height = self.coin.get_expiration_height(height) support_amount = self.get_support_amount(claim_hash) - claim_amount = self.get_claim_txo_amount(claim_hash, tx_num, position) + claim_amount = self.get_claim_txo_amount(claim_hash) effective_amount = support_amount + claim_amount channel_hash = self.get_channel_for_claim(claim_hash) @@ -226,7 +226,7 @@ class LevelDB: if channel_hash: channel_vals = self.get_claim_txo(channel_hash) if channel_vals: - channel_name = channel_vals[1].name + channel_name = channel_vals.name canonical_url = f'{channel_name}#{channel_hash.hex()}/{name}#{claim_hash.hex()}' return ResolveResult( name, claim_hash, tx_num, position, tx_hash, height, claim_amount, short_url=short_url, @@ -279,8 +279,8 @@ class LevelDB: claim_txo = self.get_claim_txo(claim_val.claim_hash) activation = self.get_activation(key.tx_num, key.position) return self._prepare_resolve_result( - key.tx_num, key.position, claim_val.claim_hash, key.name, claim_txo[1].root_tx_num, - claim_txo[1].root_position, activation + key.tx_num, key.position, claim_val.claim_hash, key.name, claim_txo.root_tx_num, + claim_txo.root_position, activation ) return @@ -336,13 +336,13 @@ class LevelDB: return await asyncio.get_event_loop().run_in_executor(self.executor, self._fs_resolve, url) def _fs_get_claim_by_hash(self, claim_hash): - for k, v in self.db.iterator(prefix=Prefixes.claim_to_txo.pack_partial_key(claim_hash)): - unpacked_k = Prefixes.claim_to_txo.unpack_key(k) - unpacked_v = Prefixes.claim_to_txo.unpack_value(v) - activation_height = self.get_activation(unpacked_k.tx_num, unpacked_k.position) + claim = self.db.get(Prefixes.claim_to_txo.pack_key(claim_hash)) + if claim: + v = Prefixes.claim_to_txo.unpack_value(claim) + activation_height = self.get_activation(v.tx_num, v.position) return self._prepare_resolve_result( - unpacked_k.tx_num, unpacked_k.position, unpacked_k.claim_hash, unpacked_v.name, - unpacked_v.root_tx_num, unpacked_v.root_position, activation_height + v.tx_num, v.position, claim_hash, v.name, + v.root_tx_num, v.root_position, activation_height ) async def fs_getclaimbyid(self, claim_id): @@ -350,8 +350,8 @@ class LevelDB: self.executor, self._fs_get_claim_by_hash, bytes.fromhex(claim_id) ) - def get_claim_txo_amount(self, claim_hash: bytes, tx_num: int, position: int) -> Optional[int]: - v = self.db.get(Prefixes.claim_to_txo.pack_key(claim_hash, tx_num, position)) + def get_claim_txo_amount(self, claim_hash: bytes) -> Optional[int]: + v = self.db.get(Prefixes.claim_to_txo.pack_key(claim_hash)) if v: return Prefixes.claim_to_txo.unpack_value(v).amount @@ -360,10 +360,11 @@ class LevelDB: if v: return Prefixes.claim_to_support.unpack_value(v).amount - def get_claim_txo(self, claim_hash: bytes) -> Optional[Tuple[ClaimToTXOKey, ClaimToTXOValue]]: + def get_claim_txo(self, claim_hash: bytes) -> Optional[ClaimToTXOValue]: assert claim_hash - for k, v in self.db.iterator(prefix=Prefixes.claim_to_txo.pack_partial_key(claim_hash)): - return Prefixes.claim_to_txo.unpack_key(k), Prefixes.claim_to_txo.unpack_value(v) + v = self.db.get(Prefixes.claim_to_txo.pack_key(claim_hash)) + if v: + return Prefixes.claim_to_txo.unpack_value(v) def _get_active_amount(self, claim_hash: bytes, txo_type: int, height: int) -> int: return sum(