diff --git a/db/db.go b/db/db.go index 14748c1..1df02df 100644 --- a/db/db.go +++ b/db/db.go @@ -21,10 +21,12 @@ type IterOptions struct { type PrefixRow struct { //KeyStruct interface{} //ValueStruct interface{} - Prefix []byte - KeyPackFunc interface{} - ValuePackFunc interface{} - DB *grocksdb.DB + Prefix []byte + KeyPackFunc interface{} + ValuePackFunc interface{} + KeyUnpackFunc interface{} + ValueUnpackFunc interface{} + DB *grocksdb.DB } type PrefixRowKV struct { @@ -85,7 +87,10 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV { it.Seek(pr.Prefix) if options.Start != nil { + log.Println("Seeking to start") it.Seek(options.Start) + } else { + log.Println("Not seeking to start") } /* @@ -118,6 +123,7 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV { var prevKey []byte = nil go func() { defer it.Close() + defer close(ch) for ; terminateFunc(prevKey); it.Next() { key := it.Key() prevKey = key.Data() @@ -131,7 +137,6 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV { key.Free() value.Free() } - close(ch) }() return ch @@ -150,6 +155,13 @@ func (k *UTXOKey) PackKey() []byte { return key } +// UTXOKeyPackPartialNFields creates a pack partial key function for n fields. +func UTXOKeyPackPartialNFields(nFields int) func(*UTXOKey) []byte { + return func(u *UTXOKey) []byte { + return UTXOKeyPackPartial(u, nFields) + } +} + // UTXOKeyPackPartial packs a variable number of fields for a UTXOKey into // a byte array. func UTXOKeyPackPartial(k *UTXOKey, nFields int) []byte { @@ -257,7 +269,7 @@ func ReadPrefixN(db *grocksdb.DB, prefix []byte, n int) []*PrefixRowKV { return res } -func OpenDB(name string) int { +func OpenDB(name string, start string) int { // Read db opts := grocksdb.NewDefaultOptions() db, err := grocksdb.OpenDb(opts, name) @@ -274,7 +286,8 @@ func OpenDB(name string) int { defer it.Close() var i = 0 - it.Seek([]byte("foo")) + it.Seek([]byte(start)) + // it.Seek([]byte{'u'}) for ; it.Valid(); it.Next() { key := it.Key() value := it.Value() @@ -292,46 +305,48 @@ func OpenDB(name string) int { return i } -func OpenAndWriteDB(in string, out string) { - // Read db - opts := grocksdb.NewDefaultOptions() - db, err := grocksdb.OpenDb(opts, in) - ro := grocksdb.NewDefaultReadOptions() - ro.SetFillCache(false) - if err != nil { - log.Println(err) - } +func OpenAndWriteDB(prIn *PrefixRow, options *IterOptions, out string) { // Write db + opts := grocksdb.NewDefaultOptions() opts.SetCreateIfMissing(true) - db2, err := grocksdb.OpenDb(opts, out) + db, err := grocksdb.OpenDb(opts, out) if err != nil { log.Println(err) } wo := grocksdb.NewDefaultWriteOptions() - defer db2.Close() + defer db.Close() - log.Println(db.Name()) - log.Println(db2.Name()) - - it := db.NewIterator(ro) - defer it.Close() + ch := prIn.Iter(options) var i = 0 - it.Seek([]byte("foo")) - for ; it.Valid() && i < 10; it.Next() { - key := it.Key() - value := it.Value() - fmt.Printf("Key: %v Value: %v\n", key.Data(), value.Data()) + var prevKey []byte = nil + for kv := range ch { + log.Println(kv) + key := kv.Key + value := kv.Value + unpackKeyFnValue := reflect.ValueOf(prIn.KeyUnpackFunc) + keyArgs := []reflect.Value{reflect.ValueOf(key)} + unpackKeyFnResult := unpackKeyFnValue.Call(keyArgs) + unpackedKey := unpackKeyFnResult[0].Interface() //.(reflect.TypeOf()) - if err := db2.Put(wo, key.Data(), value.Data()); err != nil { - log.Println(err) + unpackValueFnValue := reflect.ValueOf(prIn.ValueUnpackFunc) + valueArgs := []reflect.Value{reflect.ValueOf(value)} + unpackValueFnResult := unpackValueFnValue.Call(valueArgs) + unpackedValue := unpackValueFnResult[0].Interface() //.([]byte) + + log.Println(unpackedKey) + log.Println(unpackedValue) + + if bytes.Equal(prevKey, key) { + if err := db.Merge(wo, key, value); err != nil { + log.Println(err) + } + } else { + if err := db.Put(wo, key, value); err != nil { + log.Println(err) + } } - - key.Free() - value.Free() + prevKey = key i++ } - if err := it.Err(); err != nil { - log.Println(err) - } } diff --git a/db/db_test.go b/db/db_test.go index 28f9e9b..758ac1a 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -12,18 +12,20 @@ import ( func TestReadUTXO2(t *testing.T) { tests := []struct { - name string - want []uint64 + name string + want []uint64 + wantTotal int }{ { - name: "Read 10 UTXO Key Values", - want: []uint64{2174594, 200000000, 20000000, 100000, 603510, 75000000, 100000, 962984, 25000000, 50000000}, + name: "Read UTXO Key Values With Stop", + want: []uint64{2174594, 200000000, 20000000, 100000, 603510, 75000000, 100000, 962984, 25000000, 50000000}, + wantTotal: 7, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - db, err := GetDB("/mnt/d/data/wallet/lbry-rocksdb") + db, err := GetDB("../resources/asdf.db") if err != nil { t.Errorf("err not nil: %+v\n", err) } @@ -36,7 +38,7 @@ func TestReadUTXO2(t *testing.T) { ValuePackFunc: nil, DB: db, } - b, err := hex.DecodeString("000012b") + b, err := hex.DecodeString("000012") if err != nil { log.Println(err) } @@ -60,45 +62,13 @@ func TestReadUTXO2(t *testing.T) { log.Println(UTXOValueUnpack(kv.Value)) got := UTXOValueUnpack(kv.Value).Amount if got != tt.want[i] { - t.Errorf("got: %d, want: %d\n", got, tt.want) + t.Errorf("got: %d, want: %d\n", got, tt.want[i]) } i++ } - }) - } -} - -func TestReadUTXO(t *testing.T) { - - tests := []struct { - name string - want int - }{ - { - name: "Read 10 UTXO Key Values", - want: 10, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - db, err := GetDB("/mnt/d/data/wallet/lbry-rocksdb") - if err != nil { - t.Errorf("err not nil: %+v\n", err) - } - defer db.Close() - - data := ReadPrefixN(db, prefixes.UTXO, tt.want) - - got := len(data) - - for _, kv := range data { - log.Println(UTXOKeyUnpack(kv.Key)) - log.Println(UTXOValueUnpack(kv.Value)) - } - - if got != tt.want { + got := i + if got != tt.wantTotal { t.Errorf("got: %d, want: %d\n", got, tt.want) } }) @@ -106,6 +76,43 @@ func TestReadUTXO(t *testing.T) { } +// func TestReadUTXO(t *testing.T) { + +// tests := []struct { +// name string +// want int +// }{ +// { +// name: "Read UTXO Key Values", +// want: 12, +// }, +// } + +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// db, err := GetDB("../resources/asdf.db") +// if err != nil { +// t.Errorf("err not nil: %+v\n", err) +// } +// defer db.Close() + +// data := ReadPrefixN(db, prefixes.UTXO, tt.want) + +// got := len(data) + +// for _, kv := range data { +// log.Println(UTXOKeyUnpack(kv.Key)) +// log.Println(UTXOValueUnpack(kv.Value)) +// } + +// if got != tt.want { +// t.Errorf("got: %d, want: %d\n", got, tt.want) +// } +// }) +// } + +// } + // TestOpenDB test to see if we can open a db func TestOpenDB(t *testing.T) { @@ -121,7 +128,7 @@ func TestOpenDB(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vals := OpenDB("../resources/tmp.db") + vals := OpenDB("../resources/tmp.db", "foo") got := vals if got != tt.want { @@ -132,6 +139,30 @@ func TestOpenDB(t *testing.T) { } +func TestOpenDB2(t *testing.T) { + + tests := []struct { + name string + want int + }{ + { + name: "Open a rocksdb database", + want: 10, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + OpenDB("../resources/asdf.db", "u") + // got := vals + + // if got != tt.want { + // t.Errorf("got: %d, want: %d\n", got, tt.want) + // } + }) + } +} + func TestUTXOKey_String(t *testing.T) { tests := []struct { name string diff --git a/main.go b/main.go index 21664c6..a6d7e68 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,13 @@ package main import ( "context" + "encoding/hex" "fmt" "log" "time" "github.com/lbryio/hub/db" + "github.com/lbryio/hub/db/prefixes" pb "github.com/lbryio/hub/protobuf/go" "github.com/lbryio/hub/server" "github.com/lbryio/lbry.go/v2/extras/util" @@ -30,7 +32,35 @@ func main() { return } else if args.CmdType == server.DBCmd { - db.OpenDB("/mnt/d/data/wallet/lbry-rocksdb/") + dbVal, err := db.GetDB("/mnt/d/data/wallet/lbry-rocksdb/") + if err != nil { + log.Fatalln(err) + } + + pr := &db.PrefixRow{ + Prefix: prefixes.UTXO, + KeyPackFunc: nil, + ValuePackFunc: nil, + KeyUnpackFunc: db.UTXOKeyUnpack, + ValueUnpackFunc: db.UTXOValueUnpack, + DB: dbVal, + } + + b, err := hex.DecodeString("000013") + if err != nil { + log.Println(err) + } + stopKey := &db.UTXOKey{ + Prefix: prefixes.UTXO, + HashX: b, + TxNum: 0, + Nout: 0, + } + stop := db.UTXOKeyPackPartial(stopKey, 1) + + options := db.NewIterateOptions().WithFillCache(false).WithStop(stop) + + db.OpenAndWriteDB(pr, options, "./resources/asdf.db") return }