LBRYCommunity/herald.go
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases 14 Packages Wiki Activity

some cleanup

Browse source
This commit is contained in:
Jeffrey Picard 2022-03-24 18:47:58 -04:00
parent 7410f6a714
commit 65d98dfaa4
3 changed files with 20 additions and 393 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

402
db/prefixes/prefixes.go
View file

@ -1,5 +1,12 @@
package prefixes
// The prefixes package contains the key/value structures for the rocksdb prefix
// store and related functions.
// Each key/value prefix pair has a pack and unpack function for serializing / deserializing
// it into bytes for storage in rocksdb. They also have a pack partial function which allows
// for serialization of a set number of fields, this is used i.e. for keys with a hash, tx_num
// and height where there multiple be multiple with the same hash but different tx_num and height.
import (
"bytes"
"encoding/binary"
@ -59,6 +66,7 @@ const (
OnesCompTwiddle32 uint32 = 0xffffffff
)
// GetPrefixes returns an array of all the byte prefix constants.
func GetPrefixes() [][]byte {
return [][]byte{
{ClaimToSupport},
@ -94,6 +102,7 @@ func GetPrefixes() [][]byte {
}
}
// PrefixRowKV is a generic key/value pair for a prefix.
type PrefixRowKV struct {
Key interface{}
Value interface{}
@ -459,19 +468,6 @@ func HashXUTXOValueUnpack(value []byte) *HashXUTXOValue {
}
}
/*
class HashXHistoryKey(NamedTuple):
hashX: bytes
height: int
def __str__(self):
return f"{self.__class__.__name__}(hashX={self.hashX.hex()}, height={self.height})"
class HashXHistoryValue(NamedTuple):
hashXes: typing.List[int]
*/
type HashXHistoryKey struct {
Prefix []byte `json:"prefix"`
HashX []byte `json:"hashx"`
@ -586,18 +582,6 @@ func HashXHistoryValueUnpack(value []byte) *HashXHistoryValue {
}
}
/*
class BlockHashKey(NamedTuple):
height: int
class BlockHashValue(NamedTuple):
block_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(block_hash={self.block_hash.hex()})"
*/
type BlockHashKey struct {
Prefix []byte `json:"prefix"`
Height uint32 `json:"height"`
@ -689,15 +673,6 @@ func BlockHashValueUnpack(value []byte) *BlockHashValue {
}
}
/*
class BlockTxsKey(NamedTuple):
height: int
class BlockTxsValue(NamedTuple):
tx_hashes: typing.List[bytes]
*/
type BlockTxsKey struct {
Prefix []byte `json:"prefix"`
Height uint32 `json:"height"`
@ -798,15 +773,6 @@ func BlockTxsValueUnpack(value []byte) *BlockTxsValue {
}
}
/*
class TxCountKey(NamedTuple):
height: int
class TxCountValue(NamedTuple):
tx_count: int
*/
type TxCountKey struct {
Prefix []byte `json:"prefix"`
Height uint32 `json:"height"`
@ -900,18 +866,6 @@ func TxCountValueUnpack(value []byte) *TxCountValue {
}
}
/*
class TxHashKey(NamedTuple):
tx_num: int
class TxHashValue(NamedTuple):
tx_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(tx_hash={self.tx_hash.hex()})"
*/
type TxHashKey struct {
Prefix []byte `json:"prefix"`
TxNum uint32 `json:"tx_num"`
@ -1006,18 +960,6 @@ func TxHashValueUnpack(value []byte) *TxHashValue {
}
}
/*
class TxNumKey(NamedTuple):
tx_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(tx_hash={self.tx_hash.hex()})"
class TxNumValue(NamedTuple):
tx_num: int
*/
type TxNumKey struct {
Prefix []byte `json:"prefix"`
TxHash []byte `json:"tx_hash"`
@ -1104,21 +1046,6 @@ func TxNumValueUnpack(value []byte) *TxNumValue {
}
}
/*
class TxKey(NamedTuple):
tx_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(tx_hash={self.tx_hash.hex()})"
class TxValue(NamedTuple):
raw_tx: bytes
def __str__(self):
return f"{self.__class__.__name__}(raw_tx={base64.b64encode(self.raw_tx)})"
*/
type TxKey struct {
Prefix []byte `json:"prefix"`
TxHash []byte `json:"tx_hash"`
@ -1205,18 +1132,6 @@ func TxValueUnpack(value []byte) *TxValue {
}
}
/*
class BlockHeaderKey(NamedTuple):
height: int
class BlockHeaderValue(NamedTuple):
header: bytes
def __str__(self):
return f"{self.__class__.__name__}(header={base64.b64encode(self.header)})"
*/
type BlockHeaderKey struct {
Prefix []byte `json:"prefix"`
Height uint32 `json:"height"`
@ -1315,33 +1230,6 @@ func BlockHeaderValueUnpack(value []byte) *BlockHeaderValue {
}
}
/*
class ClaimToTXOKey(typing.NamedTuple):
claim_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})"
class ClaimToTXOValue(typing.NamedTuple):
tx_num: int
position: int
root_tx_num: int
root_position: int
amount: int
# activation: int
channel_signature_is_valid: bool
name: str
@property
def normalized_name(self) -> str:
try:
return normalize_name(self.name)
except UnicodeDecodeError:
return self.name
*/
type ClaimToTXOKey struct {
Prefix []byte `json:"prefix"`
ClaimHash []byte `json:"claim_hash"`
@ -1467,20 +1355,6 @@ func ClaimToTXOValueUnpack(value []byte) *ClaimToTXOValue {
}
}
/*
class TXOToClaimKey(typing.NamedTuple):
tx_num: int
position: int
class TXOToClaimValue(typing.NamedTuple):
claim_hash: bytes
name: str
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, name={self.name})"
*/
type TXOToClaimKey struct {
Prefix []byte `json:"prefix"`
TxNum uint32 `json:"tx_num"`
@ -1589,24 +1463,6 @@ func TXOToClaimValueUnpack(value []byte) *TXOToClaimValue {
}
}
/*
class ClaimShortIDKey(typing.NamedTuple):
normalized_name: str
partial_claim_id: str
root_tx_num: int
root_position: int
def __str__(self):
return f"{self.__class__.__name__}(normalized_name={self.normalized_name}, " \
f"partial_claim_id={self.partial_claim_id}, " \
f"root_tx_num={self.root_tx_num}, root_position={self.root_position})"
class ClaimShortIDValue(typing.NamedTuple):
tx_num: int
position: int
*/
type ClaimShortIDKey struct {
Prefix []byte `json:"prefix"`
NormalizedName string `json:"normalized_name"`
@ -1737,24 +1593,6 @@ func ClaimShortIDValueUnpack(value []byte) *ClaimShortIDValue {
}
}
/*
class ClaimToChannelKey(typing.NamedTuple):
claim_hash: bytes
tx_num: int
position: int
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, " \
f"tx_num={self.tx_num}, position={self.position})"
class ClaimToChannelValue(typing.NamedTuple):
signing_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(signing_hash={self.signing_hash.hex()})"
*/
type ClaimToChannelKey struct {
Prefix []byte `json:"prefix"`
ClaimHash []byte `json:"claim_hash"`
@ -1991,19 +1829,6 @@ func ChannelToClaimValueUnpack(value []byte) *ChannelToClaimValue {
}
}
/*
class ChannelCountKey(typing.NamedTuple):
channel_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(channel_hash={self.channel_hash.hex()})"
class ChannelCountValue(typing.NamedTuple):
count: int
*/
type ChannelCountKey struct {
Prefix []byte `json:"prefix"`
ChannelHash []byte `json:"channel_hash"`
@ -2097,18 +1922,6 @@ func ChannelCountValueUnpack(value []byte) *ChannelCountValue {
}
}
/*
class SupportAmountKey(typing.NamedTuple):
claim_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})"
class SupportAmountValue(typing.NamedTuple):
amount: int
*/
type SupportAmountKey struct {
Prefix []byte `json:"prefix"`
ClaimHash []byte `json:"claim_hash"`
@ -2202,22 +2015,6 @@ func SupportAmountValueUnpack(value []byte) *SupportAmountValue {
}
}
/*
class ClaimToSupportKey(typing.NamedTuple):
claim_hash: bytes
tx_num: int
position: int
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, tx_num={self.tx_num}, " \
f"position={self.position})"
class ClaimToSupportValue(typing.NamedTuple):
amount: int
*/
type ClaimToSupportKey struct {
Prefix []byte `json:"prefix"`
ClaimHash []byte `json:"claim_hash"`
@ -2319,19 +2116,6 @@ func ClaimToSupportValueUnpack(value []byte) *ClaimToSupportValue {
}
}
/*
class SupportToClaimKey(typing.NamedTuple):
tx_num: int
position: int
class SupportToClaimValue(typing.NamedTuple):
claim_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})"
*/
type SupportToClaimKey struct {
Prefix []byte `json:"prefix"`
TxNum uint32 `json:"tx_num"`
@ -2425,21 +2209,6 @@ func SupportToClaimValueUnpack(value []byte) *SupportToClaimValue {
}
}
/*
class ClaimExpirationKey(typing.NamedTuple):
expiration: int
tx_num: int
position: int
class ClaimExpirationValue(typing.NamedTuple):
claim_hash: bytes
normalized_name: str
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, normalized_name={self.normalized_name})"
*/
type ClaimExpirationKey struct {
Prefix []byte `json:"prefix"`
Expiration uint32 `json:"expiration"`
@ -2548,19 +2317,6 @@ func ClaimExpirationValueUnpack(value []byte) *ClaimExpirationValue {
}
}
/*
class ClaimTakeoverKey(typing.NamedTuple):
normalized_name: str
class ClaimTakeoverValue(typing.NamedTuple):
claim_hash: bytes
height: int
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, height={self.height})"
*/
type ClaimTakeoverKey struct {
Prefix []byte `json:"prefix"`
NormalizedName string `json:"normalized_name"`
@ -2671,31 +2427,6 @@ func ClaimTakeoverValueUnpack(value []byte) *ClaimTakeoverValue {
}
}
/*
class PendingActivationKey(typing.NamedTuple):
height: int
txo_type: int
tx_num: int
position: int
@property
def is_support(self) -> bool:
return self.txo_type == ACTIVATED_SUPPORT_TXO_TYPE
@property
def is_claim(self) -> bool:
return self.txo_type == ACTIVATED_CLAIM_TXO_TYPE
class PendingActivationValue(typing.NamedTuple):
claim_hash: bytes
normalized_name: str
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, normalized_name={self.normalized_name})"
*/
type PendingActivationKey struct {
Prefix []byte `json:"prefix"`
Height uint32 `json:"height"`
@ -2819,23 +2550,6 @@ func PendingActivationValueUnpack(value []byte) *PendingActivationValue {
}
}
/*
class ActivationKey(typing.NamedTuple):
txo_type: int
tx_num: int
position: int
class ActivationValue(typing.NamedTuple):
height: int
claim_hash: bytes
normalized_name: str
def __str__(self):
return f"{self.__class__.__name__}(height={self.height}, claim_hash={self.claim_hash.hex()}, " \
f"normalized_name={self.normalized_name})"
*/
type ActivationKey struct {
Prefix []byte `json:"prefix"`
TxoType uint8 `json:"txo_type"`
@ -2955,24 +2669,6 @@ func ActivationValueUnpack(value []byte) *ActivationValue {
}
}
/*
class ActiveAmountKey(typing.NamedTuple):
claim_hash: bytes
txo_type: int
activation_height: int
tx_num: int
position: int
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, txo_type={self.txo_type}, " \
f"activation_height={self.activation_height}, tx_num={self.tx_num}, position={self.position})"
class ActiveAmountValue(typing.NamedTuple):
amount: int
*/
type ActiveAmountKey struct {
Prefix []byte `json:"prefix"`
ClaimHash []byte `json:"claim_hash"`
@ -3097,25 +2793,6 @@ func ActiveAmountValueUnpack(value []byte) *ActiveAmountValue {
}
}
//
// EffectiveAmountKey / EffectiveAmountValue
//
/*
class EffectiveAmountKey(typing.NamedTuple):
normalized_name: str
effective_amount: int
tx_num: int
position: int
class EffectiveAmountValue(typing.NamedTuple):
claim_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})"
*/
type EffectiveAmountKey struct {
Prefix []byte `json:"prefix"`
NormalizedName string `json:"normalized_name"`
@ -3240,22 +2917,6 @@ func EffectiveAmountValueUnpack(value []byte) *EffectiveAmountValue {
}
}
/*
class RepostKey(typing.NamedTuple):
claim_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})"
class RepostValue(typing.NamedTuple):
reposted_claim_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(reposted_claim_hash={self.reposted_claim_hash.hex()})"
*/
type RepostKey struct {
Prefix []byte `json:"prefix"`
ClaimHash []byte `json:"claim_hash"`
@ -3351,25 +3012,6 @@ func RepostValueUnpack(value []byte) *RepostValue {
}
}
/*
class RepostedKey(typing.NamedTuple):
reposted_claim_hash: bytes
tx_num: int
position: int
def __str__(self):
return f"{self.__class__.__name__}(reposted_claim_hash={self.reposted_claim_hash.hex()}, " \
f"tx_num={self.tx_num}, position={self.position})"
class RepostedValue(typing.NamedTuple):
claim_hash: bytes
def __str__(self):
return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})"
*/
type RepostedKey struct {
Prefix []byte `json:"prefix"`
RepostedClaimHash []byte `json:"reposted_claim_hash"`
@ -3478,27 +3120,6 @@ func RepostedValueUnpack(value []byte) *RepostedValue {
}
}
//
// TouchedOrDeletedClaimKey / TouchedOrDeletedClaimValue
//
/*
class TouchedOrDeletedClaimKey(typing.NamedTuple):
height: int
class TouchedOrDeletedClaimValue(typing.NamedTuple):
touched_claims: typing.Set[bytes]
deleted_claims: typing.Set[bytes]
def __str__(self):
return f"{self.__class__.__name__}(" \
f"touched_claims={','.join(map(lambda x: x.hex(), self.touched_claims))}," \
f"deleted_claims={','.join(map(lambda x: x.hex(), self.deleted_claims))})"
*/
type TouchedOrDeletedClaimKey struct {
Prefix []byte `json:"prefix"`
Height int32 `json:"height"`
@ -3661,10 +3282,6 @@ func TouchedOrDeletedClaimValueUnpack(value []byte) *TouchedOrDeletedClaimValue
}
}
//
// UTXOKey / UTXOValue
//
func (k *UTXOKey) String() string {
return fmt.Sprintf(
"%s(hashX=%s, tx_num=%d, nout=%d)",
@ -3767,6 +3384,7 @@ func UTXOValueUnpack(value []byte) *UTXOValue {
}
}
// generic simulates a generic key packing / unpacking function for the prefixes
func generic(voidstar interface{}, firstByte byte, function byte, functionName string) (interface{}, error) {
var data []byte
if function < 2 {

3
main.go
View file

@ -9,6 +9,7 @@ import (
"github.com/lbryio/hub/db"
"github.com/lbryio/hub/db/prefixes"
"github.com/lbryio/hub/internal"
pb "github.com/lbryio/hub/protobuf/go"
"github.com/lbryio/hub/server"
"github.com/lbryio/lbry.go/v2/extras/util"
@ -112,7 +113,7 @@ func main() {
} else if args.CmdType == server.DBCmd3 {
// streamHash, _ := hex.DecodeString("9a0ed686ecdad9b6cb965c4d6681c02f0bbc66a6")
channelHash, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bd")
name := util.NormalizeName("@Styxhexenhammer666")
name := internal.NormalizeName("@Styxhexenhammer666")
// txNum := uint32(0x6284e3)
// position := uint16(0x0)
// For stream claim

8
readme.md
View file

@ -59,6 +59,14 @@ pip install grpcio grpcio-tools github3.py
Lastly the hub needs protobuf version 3.17.1, it may work with newer version but this is what it's built with, on ubuntu systems you'll have to install this from source see the GitHub actions in `.github/workflows` for an example of this.
Install rocksdb and setup your CGO flags, on ubuntu
```
sudo apt-get install libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev libzstd-dev liblz4-dev
export CGO_CFLAGS="-I/path/to/rocksdb/include"
export CGO_LDFLAGS="-L/path/to/rocksdb -lrocksdb -lstdc++ -lm -lz -lsnappy -llz4 -lzstd -lbz2"
```
```
https://github.com/protocolbuffers/protobuf/releases/download/v3.17.1/protobuf-all-3.17.1.tar.gz
```