db/prefixes/prefixes.go

@@ -54,6 +54,8 @@ const (
 
 	ACTIVATED_CLAIM_TXO_TYPE   = 1
 	ACTIVATED_SUPPORT_TXO_TYPE = 2
+
+	OnesCompTwiddle uint64 = 0xffffffffffffffff
 )
 
 type PrefixRowKV struct {
@@ -672,6 +674,9 @@ type ActiveAmountValue struct {
 	Amount int32 `json:"amount"`
 }
 
+//
+// EffectiveAmountKey / EffectiveAmountValue
+//
 /*
 
 class EffectiveAmountKey(typing.NamedTuple):
@@ -691,15 +696,114 @@ class EffectiveAmountValue(typing.NamedTuple):
 type EffectiveAmountKey struct {
 	Prefix          []byte `json:"prefix"`
 	NormalizedName  string `json:"normalized_name"`
-	EffectiveAmount int32  `json:"effective_amount"`
+	EffectiveAmount uint64 `json:"effective_amount"`
-	TxNum           int32  `json:"tx_num"`
+	TxNum           uint32 `json:"tx_num"`
-	Position        int32  `json:"position"`
+	Position        uint16 `json:"position"`
 }
 
 type EffectiveAmountValue struct {
 	ClaimHash []byte `json:"claim_hash"`
 }
 
+func (k *EffectiveAmountKey) PackKey() []byte {
+	prefixLen := 1
+	// 2 byte length field, plus number of bytes in name
+	nameLen := len(k.NormalizedName)
+	nameLenLen := 2 + nameLen
+	// b'>QLH'
+	n := prefixLen + nameLenLen + 8 + 4 + 2
+	key := make([]byte, n)
+	copy(key, k.Prefix)
+
+	binary.BigEndian.PutUint16(key[prefixLen:], uint16(nameLen))
+	copy(key[prefixLen+2:], []byte(k.NormalizedName))
+	binary.BigEndian.PutUint64(key[prefixLen+nameLenLen:], OnesCompTwiddle-k.EffectiveAmount)
+	binary.BigEndian.PutUint32(key[prefixLen+nameLenLen+8:], k.TxNum)
+	binary.BigEndian.PutUint16(key[prefixLen+nameLenLen+8+4:], k.Position)
+
+	return key
+}
+
+func (v *EffectiveAmountValue) PackValue() []byte {
+	// b'>20s'
+	value := make([]byte, 20)
+	copy(value, v.ClaimHash[:20])
+
+	return value
+}
+
+func EffectiveAmountKeyPackPartialNFields(nFields int) func(*EffectiveAmountKey) []byte {
+	return func(u *EffectiveAmountKey) []byte {
+		return EffectiveAmountKeyPackPartial(u, nFields)
+	}
+}
+
+func EffectiveAmountKeyPackPartial(k *EffectiveAmountKey, nFields int) []byte {
+	// Limit nFields between 0 and number of fields, we always at least need
+	// the prefix, and we never need to iterate past the number of fields.
+	nameLen := len(k.NormalizedName)
+	nameLenLen := 2 + nameLen
+	if nFields > 4 {
+		nFields = 4
+	}
+	if nFields < 0 {
+		nFields = 0
+	}
+
+	prefixLen := 1
+	var n = prefixLen
+	for i := 0; i <= nFields; i++ {
+		switch i {
+		case 1:
+			n += 2 + nameLen
+		case 2:
+			n += 8
+		case 3:
+			n += 4
+		case 4:
+			n += 2
+		}
+	}
+
+	key := make([]byte, n)
+
+	for i := 0; i <= nFields; i++ {
+		switch i {
+		case 0:
+			copy(key, k.Prefix)
+		case 1:
+			binary.BigEndian.PutUint16(key[prefixLen:], uint16(nameLen))
+			copy(key[prefixLen+2:], []byte(k.NormalizedName))
+		case 2:
+			binary.BigEndian.PutUint64(key[prefixLen+nameLenLen:], OnesCompTwiddle-k.EffectiveAmount)
+		case 3:
+			binary.BigEndian.PutUint32(key[prefixLen+nameLenLen+8:], k.TxNum)
+		case 4:
+			binary.BigEndian.PutUint16(key[prefixLen+nameLenLen+8+4:], k.Position)
+		}
+	}
+
+	return key
+}
+
+func EffectiveAmountKeyUnpack(key []byte) *EffectiveAmountKey {
+	prefixLen := 1
+	nameLen := binary.BigEndian.Uint16(key[prefixLen:])
+	return &EffectiveAmountKey{
+		Prefix:          key[:prefixLen],
+		NormalizedName:  string(key[prefixLen+2 : prefixLen+2+int(nameLen)]),
+		EffectiveAmount: OnesCompTwiddle - binary.BigEndian.Uint64(key[prefixLen+2+int(nameLen):]),
+		TxNum:           binary.BigEndian.Uint32(key[prefixLen+2+int(nameLen)+8:]),
+		Position:        binary.BigEndian.Uint16(key[prefixLen+2+int(nameLen)+8+4:]),
+	}
+}
+
+func EffectiveAmountValueUnpack(value []byte) *EffectiveAmountValue {
+	return &EffectiveAmountValue{
+		ClaimHash: value[:20],
+	}
+}
+
 /*
 
 class RepostKey(typing.NamedTuple):
@@ -1315,7 +1419,9 @@ func UnpackGenericKey(key []byte) (byte, interface{}, error) {
 	case ChannelToClaim:
 
 	case ClaimShortIdPrefix:
+		return 0x0, nil, errors.Base("key unpack function for %v not implemented", firstByte)
 	case EffectiveAmount:
+		return EffectiveAmount, EffectiveAmountKeyUnpack(key), nil
 	case ClaimExpiration:
 
 	case ClaimTakeover:
@@ -1374,14 +1480,16 @@ func UnpackGenericValue(key, value []byte) (byte, interface{}, error) {
 	case ChannelToClaim:
 
 	case ClaimShortIdPrefix:
+		return 0x0, nil, errors.Base("value unpack not implemented for key %v", key)
 	case EffectiveAmount:
+		return EffectiveAmount, EffectiveAmountValueUnpack(value), nil
 	case ClaimExpiration:
 
 	case ClaimTakeover:
 	case PendingActivation:
 	case ActivatedClaimAndSupport:
 	case ActiveAmount:
-		return 0x0, nil, nil
+		return 0x0, nil, errors.Base("value unpack not implemented for key %v", key)
 
 	case Repost:
 		return Repost, RepostValueUnpack(value), nil
@@ -1389,7 +1497,7 @@ func UnpackGenericValue(key, value []byte) (byte, interface{}, error) {
 		return RepostedClaim, RepostedValueUnpack(value), nil
 
 	case Undo:
-		return 0x0, nil, nil
+		return 0x0, nil, errors.Base("value unpack not implemented for key %v", key)
 	case ClaimDiff:
 		return ClaimDiff, TouchedOrDeletedClaimValueUnpack(value), nil
 
@@ -1399,7 +1507,7 @@ func UnpackGenericValue(key, value []byte) (byte, interface{}, error) {
 	case TxNum:
 	case TxCount:
 	case TxHash:
-		return 0x0, nil, nil
+		return 0x0, nil, errors.Base("value unpack not implemented for key %v", key)
 	case UTXO:
 		return UTXO, UTXOValueUnpack(value), nil
 	case HashXUTXO:
@@ -1410,5 +1518,5 @@ func UnpackGenericValue(key, value []byte) (byte, interface{}, error) {
 	case SupportAmount:
 	case BlockTXs:
 	}
-	return 0x0, nil, nil
+	return 0x0, nil, errors.Base("value unpack not implemented for key %v", key)
 }

db/prefixes/prefixes_test.go

@@ -44,6 +44,94 @@ func testInit(filePath string) (*grocksdb.DB, [][]string, func()) {
 	return db, records, toDefer
 }
 
+func TestEffectiveAmount(t *testing.T) {
+
+	filePath := "../../resources/effective_amount.csv"
+
+	wOpts := grocksdb.NewDefaultWriteOptions()
+	db, records, toDefer := testInit(filePath)
+	defer toDefer()
+	for _, record := range records {
+		key, err := hex.DecodeString(record[0])
+		if err != nil {
+			log.Println(err)
+		}
+		val, err := hex.DecodeString(record[1])
+		if err != nil {
+			log.Println(err)
+		}
+		db.Put(wOpts, key, val)
+	}
+	// test prefix
+	options := dbpkg.NewIterateOptions().WithPrefix([]byte{prefixes.EffectiveAmount}).WithIncludeValue(true)
+	ch := dbpkg.Iter(db, options)
+	var i = 0
+	for kv := range ch {
+		// log.Println(kv.Key)
+		gotKey := kv.Key.(*prefixes.EffectiveAmountKey).PackKey()
+
+		keyPartial1 := prefixes.EffectiveAmountKeyPackPartial(kv.Key.(*prefixes.EffectiveAmountKey), 1)
+		keyPartial2 := prefixes.EffectiveAmountKeyPackPartial(kv.Key.(*prefixes.EffectiveAmountKey), 2)
+		keyPartial3 := prefixes.EffectiveAmountKeyPackPartial(kv.Key.(*prefixes.EffectiveAmountKey), 3)
+		keyPartial4 := prefixes.EffectiveAmountKeyPackPartial(kv.Key.(*prefixes.EffectiveAmountKey), 4)
+
+		// Check pack partial for sanity
+		if !bytes.HasPrefix(gotKey, keyPartial1) {
+			t.Errorf("%+v should be prefix of %+v\n", keyPartial1, gotKey)
+		}
+		if !bytes.HasPrefix(gotKey, keyPartial2) {
+			t.Errorf("%+v should be prefix of %+v\n", keyPartial2, gotKey)
+		}
+		if !bytes.HasPrefix(gotKey, keyPartial3) {
+			t.Errorf("%+v should be prefix of %+v\n", keyPartial3, gotKey)
+		}
+		if !bytes.HasPrefix(gotKey, keyPartial4) {
+			t.Errorf("%+v should be prefix of %+v\n", keyPartial4, gotKey)
+		}
+
+		got := kv.Value.(*prefixes.EffectiveAmountValue).PackValue()
+		wantKey, err := hex.DecodeString(records[i][0])
+		if err != nil {
+			log.Println(err)
+		}
+		want, err := hex.DecodeString(records[i][1])
+		if err != nil {
+			log.Println(err)
+		}
+		if !bytes.Equal(gotKey, wantKey) {
+			t.Errorf("gotKey: %+v, wantKey: %+v\n", got, want)
+		}
+		if !bytes.Equal(got, want) {
+			t.Errorf("got: %+v, want: %+v\n", got, want)
+		}
+		i++
+	}
+
+	// Test start / stop
+	start, err := hex.DecodeString(records[0][0])
+	if err != nil {
+		log.Println(err)
+	}
+	stop, err := hex.DecodeString(records[9][0])
+	if err != nil {
+		log.Println(err)
+	}
+	options2 := dbpkg.NewIterateOptions().WithStart(start).WithStop(stop).WithIncludeValue(true)
+	ch2 := dbpkg.Iter(db, options2)
+	i = 0
+	for kv := range ch2 {
+		got := kv.Value.(*prefixes.EffectiveAmountValue).PackValue()
+		want, err := hex.DecodeString(records[i][1])
+		if err != nil {
+			log.Println(err)
+		}
+		if !bytes.Equal(got, want) {
+			t.Errorf("got: %+v, want: %+v\n", got, want)
+		}
+		i++
+	}
+}
+
 func TestRepost(t *testing.T) {
 
 	filePath := "../../resources/repost.csv"

main.go

@@ -38,7 +38,7 @@ func main() {
 
 		options := &db.IterOptions{
 			FillCache:    false,
-			Prefix:       []byte{prefixes.Repost},
+			Prefix:       []byte{prefixes.EffectiveAmount},
 			Start:        nil,
 			Stop:         nil,
 			IncludeStart: true,
@@ -49,7 +49,7 @@ func main() {
 			RawValue:     true,
 		}
 
-		db.ReadWriteRawN(dbVal, options, "./resources/repost.csv", 10)
+		db.ReadWriteRawN(dbVal, options, "./resources/effective_amount.csv", 10)
 
 		return
 	}

resources/effective_amount.csv

@@ -0,0 +1,10 @@
+44000100ffffffffffffd8ef002741130000,a51d5c567412654e6d741114fea6fb851dec7380
+44000101ffffffffffffd8ef002741140001,11158037afca9c2efabc3dff55e352bf1f5634c5
+44000102ffffffffffffd8ef002741150000,a4a575934de77d8ec8589595d8cd91857e3cf5ba
+44000103ffffffffffffd8ef002741160001,f595a21fb597bd030defefda3df9f8f4a3e0cb86
+44000104ffffffffffffd8ef002741170000,682ccb0518a6bd00c955949d9ef330d3ac18cb80
+44000105ffffffffffffd8ef002741180000,078a435851bf97c5cc36e8b03e3208a30d27679f
+44000106ffffffffffffd8ef002741190000,1a4a75246a766cf21a629f619bc5bcb531de7a5a
+44000107ffffffffffffd8ef0027411a0000,b144ad496b29b9c12c316f319773adcdd4c9bce2
+44000108ffffffffffffd8ef0027411b0001,af2a09232fc6bf664088d65da42fe0345b458960
+44000109ffffffffffffd8ef0027411c0001,7b2ab15758c519116fb9ba9331a3b9ee7530831f