db/prefixes/prefixes.go

@@ -725,6 +725,79 @@ type RepostValue struct {
 	RepostedClaimHash []byte `json:"reposted_claim_hash"`
 }
 
+func (k *RepostKey) PackKey() []byte {
+	prefixLen := 1
+	// b'>20s'
+	n := prefixLen + 20
+	key := make([]byte, n)
+	copy(key, k.Prefix)
+	copy(key[prefixLen:], k.ClaimHash)
+
+	return key
+}
+
+func (v *RepostValue) PackValue() []byte {
+	// FIXME: Is there a limit to this length?
+	n := len(v.RepostedClaimHash)
+	value := make([]byte, n)
+	copy(value, v.RepostedClaimHash)
+
+	return value
+}
+
+func RepostKeyPackPartialNFields(nFields int) func(*RepostKey) []byte {
+	return func(u *RepostKey) []byte {
+		return RepostKeyPackPartial(u, nFields)
+	}
+}
+
+func RepostKeyPackPartial(k *RepostKey, nFields int) []byte {
+	// Limit nFields between 0 and number of fields, we always at least need
+	// the prefix, and we never need to iterate past the number of fields.
+	if nFields > 1 {
+		nFields = 1
+	}
+	if nFields < 0 {
+		nFields = 0
+	}
+
+	prefixLen := 1
+	var n = prefixLen
+	for i := 0; i <= nFields; i++ {
+		switch i {
+		case 1:
+			n += 20
+		}
+	}
+
+	key := make([]byte, n)
+
+	for i := 0; i <= nFields; i++ {
+		switch i {
+		case 0:
+			copy(key, k.Prefix)
+		case 1:
+			copy(key[prefixLen:], k.ClaimHash)
+		}
+	}
+
+	return key
+}
+
+func RepostKeyUnpack(key []byte) *RepostKey {
+	prefixLen := 1
+	return &RepostKey{
+		Prefix:    key[:prefixLen],
+		ClaimHash: key[prefixLen : prefixLen+20],
+	}
+}
+
+func RepostValueUnpack(value []byte) *RepostValue {
+	return &RepostValue{
+		RepostedClaimHash: value[:],
+	}
+}
+
 /*
 
 class RepostedKey(typing.NamedTuple):
@@ -824,11 +897,12 @@ func RepostedKeyPackPartial(k *RepostedKey, nFields int) []byte {
 }
 
 func RepostedKeyUnpack(key []byte) *RepostedKey {
+	prefixLen := 1
 	return &RepostedKey{
-		Prefix:            key[:1],
-		RepostedClaimHash: key[1:21],
-		TxNum:             binary.BigEndian.Uint32(key[21:]),
-		Position:          binary.BigEndian.Uint16(key[25:]),
+		Prefix:            key[:prefixLen],
+		RepostedClaimHash: key[prefixLen : prefixLen+20],
+		TxNum:             binary.BigEndian.Uint32(key[prefixLen+20:]),
+		Position:          binary.BigEndian.Uint16(key[prefixLen+24:]),
 	}
 }
 
@@ -988,9 +1062,10 @@ func TouchedOrDeletedClaimKeyPackPartial(k *TouchedOrDeletedClaimKey, nFields in
 }
 
 func TouchedOrDeletedClaimKeyUnpack(key []byte) *TouchedOrDeletedClaimKey {
+	prefixLen := 1
 	return &TouchedOrDeletedClaimKey{
-		Prefix: key[:1],
-		Height: int32(binary.BigEndian.Uint32(key[1:])),
+		Prefix: key[:prefixLen],
+		Height: int32(binary.BigEndian.Uint32(key[prefixLen:])),
 	}
 }
 
@@ -1108,11 +1183,12 @@ func HashXUTXOKeyPackPartial(k *HashXUTXOKey, nFields int) []byte {
 }
 
 func HashXUTXOKeyUnpack(key []byte) *HashXUTXOKey {
+	prefixLen := 1
 	return &HashXUTXOKey{
-		Prefix:      key[:1],
-		ShortTXHash: key[1:5],
-		TxNum:       binary.BigEndian.Uint32(key[5:]),
-		Nout:        binary.BigEndian.Uint16(key[9:]),
+		Prefix:      key[:prefixLen],
+		ShortTXHash: key[prefixLen : prefixLen+4],
+		TxNum:       binary.BigEndian.Uint32(key[prefixLen+4:]),
+		Nout:        binary.BigEndian.Uint16(key[prefixLen+8:]),
 	}
 }
 
@@ -1208,11 +1284,12 @@ func UTXOKeyPackPartial(k *UTXOKey, nFields int) []byte {
 }
 
 func UTXOKeyUnpack(key []byte) *UTXOKey {
+	prefixLen := 1
 	return &UTXOKey{
-		Prefix: key[:1],
-		HashX:  key[1:12],
-		TxNum:  binary.BigEndian.Uint32(key[12:]),
-		Nout:   binary.BigEndian.Uint16(key[16:]),
+		Prefix: key[:prefixLen],
+		HashX:  key[prefixLen : prefixLen+11],
+		TxNum:  binary.BigEndian.Uint32(key[prefixLen+11:]),
+		Nout:   binary.BigEndian.Uint16(key[prefixLen+15:]),
 	}
 }
 
@@ -1245,9 +1322,10 @@ func UnpackGenericKey(key []byte) (byte, interface{}, error) {
 	case PendingActivation:
 	case ActivatedClaimAndSupport:
 	case ActiveAmount:
+		return 0x0, nil, errors.Base("key unpack function for %v not implemented", firstByte)
 
 	case Repost:
-		return 0x0, nil, errors.Base("key unpack function for %v not implemented", firstByte)
+		return Repost, RepostKeyUnpack(key), nil
 	case RepostedClaim:
 		return RepostedClaim, RepostedKeyUnpack(key), nil
 
@@ -1303,9 +1381,10 @@ func UnpackGenericValue(key, value []byte) (byte, interface{}, error) {
 	case PendingActivation:
 	case ActivatedClaimAndSupport:
 	case ActiveAmount:
+		return 0x0, nil, nil
 
 	case Repost:
-		return 0x0, nil, nil
+		return Repost, RepostValueUnpack(value), nil
 	case RepostedClaim:
 		return RepostedClaim, RepostedValueUnpack(value), nil
 

db/prefixes/prefixes_test.go

@@ -14,10 +14,7 @@ import (
 	"github.com/linxGnu/grocksdb"
 )
 
-func TestRepostedClaim(t *testing.T) {
-
-	filePath := "../../resources/reposted_claim.csv"
-
+func testInit(filePath string) (*grocksdb.DB, [][]string, func()) {
 	log.Println(filePath)
 	file, err := os.Open(filePath)
 	if err != nil {
@@ -29,20 +26,107 @@ func TestRepostedClaim(t *testing.T) {
 		log.Println(err)
 	}
 
-	wOpts := grocksdb.NewDefaultWriteOptions()
+	// wOpts := grocksdb.NewDefaultWriteOptions()
 	opts := grocksdb.NewDefaultOptions()
 	opts.SetCreateIfMissing(true)
 	db, err := grocksdb.OpenDb(opts, "tmp")
 	if err != nil {
 		log.Println(err)
 	}
-	defer func() {
+	toDefer := func() {
 		db.Close()
 		err = os.RemoveAll("./tmp")
 		if err != nil {
 			log.Println(err)
 		}
-	}()
+	}
+
+	return db, records, toDefer
+}
+
+func TestRepost(t *testing.T) {
+
+	filePath := "../../resources/repost.csv"
+
+	wOpts := grocksdb.NewDefaultWriteOptions()
+	db, records, toDefer := testInit(filePath)
+	defer toDefer()
+	for _, record := range records {
+		key, err := hex.DecodeString(record[0])
+		if err != nil {
+			log.Println(err)
+		}
+		val, err := hex.DecodeString(record[1])
+		if err != nil {
+			log.Println(err)
+		}
+		db.Put(wOpts, key, val)
+	}
+	// test prefix
+	options := dbpkg.NewIterateOptions().WithPrefix([]byte{prefixes.Repost}).WithIncludeValue(true)
+	ch := dbpkg.Iter(db, options)
+	var i = 0
+	for kv := range ch {
+		// log.Println(kv.Key)
+		gotKey := kv.Key.(*prefixes.RepostKey).PackKey()
+
+		keyPartial1 := prefixes.RepostKeyPackPartial(kv.Key.(*prefixes.RepostKey), 1)
+
+		// Check pack partial for sanity
+		if !bytes.HasPrefix(gotKey, keyPartial1) {
+			t.Errorf("%+v should be prefix of %+v\n", keyPartial1, gotKey)
+		}
+
+		got := kv.Value.(*prefixes.RepostValue).PackValue()
+		wantKey, err := hex.DecodeString(records[i][0])
+		if err != nil {
+			log.Println(err)
+		}
+		want, err := hex.DecodeString(records[i][1])
+		if err != nil {
+			log.Println(err)
+		}
+		if !bytes.Equal(gotKey, wantKey) {
+			t.Errorf("gotKey: %+v, wantKey: %+v\n", got, want)
+		}
+		if !bytes.Equal(got, want) {
+			t.Errorf("got: %+v, want: %+v\n", got, want)
+		}
+		i++
+	}
+
+	// Test start / stop
+	start, err := hex.DecodeString(records[0][0])
+	if err != nil {
+		log.Println(err)
+	}
+	stop, err := hex.DecodeString(records[9][0])
+	if err != nil {
+		log.Println(err)
+	}
+	options2 := dbpkg.NewIterateOptions().WithStart(start).WithStop(stop).WithIncludeValue(true)
+	ch2 := dbpkg.Iter(db, options2)
+	i = 0
+	for kv := range ch2 {
+		got := kv.Value.(*prefixes.RepostValue).PackValue()
+		want, err := hex.DecodeString(records[i][1])
+		if err != nil {
+			log.Println(err)
+		}
+		if !bytes.Equal(got, want) {
+			t.Errorf("got: %+v, want: %+v\n", got, want)
+		}
+		i++
+	}
+}
+
+func TestRepostedClaim(t *testing.T) {
+
+	filePath := "../../resources/reposted_claim.csv"
+
+	wOpts := grocksdb.NewDefaultWriteOptions()
+	db, records, toDefer := testInit(filePath)
+	defer toDefer()
 	for _, record := range records {
 		key, err := hex.DecodeString(record[0])
 		if err != nil {

main.go

@@ -31,7 +31,6 @@ func main() {
 
 		return
 	} else if args.CmdType == server.DBCmd {
-		//dbVal, err := db.GetDB("./resources/asdf.db")
 		dbVal, err := db.GetDB("/mnt/d/data/wallet/lbry-rocksdb/")
 		if err != nil {
 			log.Fatalln(err)
@@ -39,7 +38,7 @@ func main() {
 
 		options := &db.IterOptions{
 			FillCache:    false,
-			Prefix:       []byte{prefixes.ClaimDiff},
+			Prefix:       []byte{prefixes.Repost},
 			Start:        nil,
 			Stop:         nil,
 			IncludeStart: true,
@@ -50,37 +49,7 @@ func main() {
 			RawValue:     true,
 		}
 
-		db.ReadWriteRawN(dbVal, options, "./resources/claim_diff.csv", 10)
-
-		// b, err := hex.DecodeString("000013")
-		// if err != nil {
-		// 	log.Println(err)
-		// }
-		// stopKey := &prefixes.UTXOKey{
-		// 	Prefix: []byte{prefixes.UTXO},
-		// 	HashX:  b,
-		// 	TxNum:  0,
-		// 	Nout:   0,
-		// }
-		// stop := prefixes.UTXOKeyPackPartial(stopKey, 1)
-
-		// log.Println(stop)
-		// log.Print(hex.EncodeToString(stop))
-
-		// options := &db.IterOptions{
-		// 	FillCache:    false,
-		// 	Prefix:       []byte{prefixes.UTXO},
-		// 	Start:        nil,
-		// 	Stop:         stop,
-		// 	IncludeStart: true,
-		// 	IncludeStop:  false,
-		// 	IncludeKey:   true,
-		// 	IncludeValue: true,
-		// 	RawKey:       false,
-		// 	RawValue:     false,
-		// }
-
-		// db.OpenAndWriteDB(dbVal, options, "./resources/asdf2.db")
+		db.ReadWriteRawN(dbVal, options, "./resources/repost.csv", 10)
 
 		return
 	}

resources/repost.csv

@@ -0,0 +1,10 @@
+56000009ca6e0caaaef16872b4bd4f6f1b8c2363e2,dbdfb6cd5e83baf342eaab8b19662ed0c71aae9a
+56000011884f49bd8fcd16444dd44452a4e007f156,799c1f31da3021972570a9fafaa838fec36aecc4
+560000140cfb01f26c9177e99831ef05b61981852a,0f7f89b3fd788c928104fa7406b8c34c77edaba4
+56000014f799144c6cbadef2a3cc28552dc6cf8539,8fefe443580aee599c5450bfd422eed78091115c
+560000158cc5d5817bbe2dae2e97d569d2e812f310,a1a3fd82cfec76e34ab2467b5988c5e195c47de6
+5600005491516e90167b54f346ab2325d422445b60,865ff83242498653f8a088f1755a4fb116fbe39c
+56000080a0497721b900a1896c209b16ae9d74645d,d2fc2d13cce02143549e27fbb459380af3fa04a9
+560000843de01218f5bc2baa62f6e8891f2d5a31cd,47ec6edb8178816a6e0dca4db5a8c14be04b0869
+560000882f6d62f74883ec7fd594b2a723071709b8,a4bce36b4c341e046068c72f53bbf55914304ba1
+560000981099ab4955195aed48a5bf2aa6e80dbbff,4153e0b9d207cfc9a871c1679d41347cf55bdfab