LBRYCommunity/herald.go
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases 14 Packages Wiki Activity

asdf

Browse source
This commit is contained in:
Jeffrey Picard 2021-12-12 12:40:51 -05:00
parent a20951ca7f
commit abbc86b1ac
3 changed files with 155 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
db/db.go
View file

@ -21,10 +21,12 @@ type IterOptions struct {
type PrefixRow struct {
//KeyStruct interface{}
//ValueStruct interface{}
Prefix []byte
KeyPackFunc interface{}
ValuePackFunc interface{}
DB *grocksdb.DB
Prefix []byte
KeyPackFunc interface{}
ValuePackFunc interface{}
KeyUnpackFunc interface{}
ValueUnpackFunc interface{}
DB *grocksdb.DB
}
type PrefixRowKV struct {
@ -85,7 +87,10 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV {
it.Seek(pr.Prefix)
if options.Start != nil {
log.Println("Seeking to start")
it.Seek(options.Start)
} else {
log.Println("Not seeking to start")
}
/*
@ -118,6 +123,7 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV {
var prevKey []byte = nil
go func() {
defer it.Close()
defer close(ch)
for ; terminateFunc(prevKey); it.Next() {
key := it.Key()
prevKey = key.Data()
@ -131,7 +137,6 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV {
key.Free()
value.Free()
}
close(ch)
}()
return ch
@ -150,6 +155,13 @@ func (k *UTXOKey) PackKey() []byte {
return key
}
// UTXOKeyPackPartialNFields creates a pack partial key function for n fields.
func UTXOKeyPackPartialNFields(nFields int) func(*UTXOKey) []byte {
return func(u *UTXOKey) []byte {
return UTXOKeyPackPartial(u, nFields)
}
}
// UTXOKeyPackPartial packs a variable number of fields for a UTXOKey into
// a byte array.
func UTXOKeyPackPartial(k *UTXOKey, nFields int) []byte {
@ -257,7 +269,7 @@ func ReadPrefixN(db *grocksdb.DB, prefix []byte, n int) []*PrefixRowKV {
return res
}
func OpenDB(name string) int {
func OpenDB(name string, start string) int {
// Read db
opts := grocksdb.NewDefaultOptions()
db, err := grocksdb.OpenDb(opts, name)
@ -274,7 +286,8 @@ func OpenDB(name string) int {
defer it.Close()
var i = 0
it.Seek([]byte("foo"))
it.Seek([]byte(start))
// it.Seek([]byte{'u'})
for ; it.Valid(); it.Next() {
key := it.Key()
value := it.Value()
@ -292,46 +305,48 @@ func OpenDB(name string) int {
return i
}
func OpenAndWriteDB(in string, out string) {
// Read db
opts := grocksdb.NewDefaultOptions()
db, err := grocksdb.OpenDb(opts, in)
ro := grocksdb.NewDefaultReadOptions()
ro.SetFillCache(false)
if err != nil {
log.Println(err)
}
func OpenAndWriteDB(prIn *PrefixRow, options *IterOptions, out string) {
// Write db
opts := grocksdb.NewDefaultOptions()
opts.SetCreateIfMissing(true)
db2, err := grocksdb.OpenDb(opts, out)
db, err := grocksdb.OpenDb(opts, out)
if err != nil {
log.Println(err)
}
wo := grocksdb.NewDefaultWriteOptions()
defer db2.Close()
defer db.Close()
log.Println(db.Name())
log.Println(db2.Name())
it := db.NewIterator(ro)
defer it.Close()
ch := prIn.Iter(options)
var i = 0
it.Seek([]byte("foo"))
for ; it.Valid() && i < 10; it.Next() {
key := it.Key()
value := it.Value()
fmt.Printf("Key: %v Value: %v\n", key.Data(), value.Data())
var prevKey []byte = nil
for kv := range ch {
log.Println(kv)
key := kv.Key
value := kv.Value
unpackKeyFnValue := reflect.ValueOf(prIn.KeyUnpackFunc)
keyArgs := []reflect.Value{reflect.ValueOf(key)}
unpackKeyFnResult := unpackKeyFnValue.Call(keyArgs)
unpackedKey := unpackKeyFnResult[0].Interface() //.(reflect.TypeOf())
if err := db2.Put(wo, key.Data(), value.Data()); err != nil {
log.Println(err)
unpackValueFnValue := reflect.ValueOf(prIn.ValueUnpackFunc)
valueArgs := []reflect.Value{reflect.ValueOf(value)}
unpackValueFnResult := unpackValueFnValue.Call(valueArgs)
unpackedValue := unpackValueFnResult[0].Interface() //.([]byte)
log.Println(unpackedKey)
log.Println(unpackedValue)
if bytes.Equal(prevKey, key) {
if err := db.Merge(wo, key, value); err != nil {
log.Println(err)
}
} else {
if err := db.Put(wo, key, value); err != nil {
log.Println(err)
}
}
key.Free()
value.Free()
prevKey = key
i++
}
if err := it.Err(); err != nil {
log.Println(err)
}
}

115
db/db_test.go
View file

@ -12,18 +12,20 @@ import (
func TestReadUTXO2(t *testing.T) {
tests := []struct {
name string
want []uint64
name string
want []uint64
wantTotal int
}{
{
name: "Read 10 UTXO Key Values",
want: []uint64{2174594, 200000000, 20000000, 100000, 603510, 75000000, 100000, 962984, 25000000, 50000000},
name: "Read UTXO Key Values With Stop",
want: []uint64{2174594, 200000000, 20000000, 100000, 603510, 75000000, 100000, 962984, 25000000, 50000000},
wantTotal: 7,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, err := GetDB("/mnt/d/data/wallet/lbry-rocksdb")
db, err := GetDB("../resources/asdf.db")
if err != nil {
t.Errorf("err not nil: %+v\n", err)
}
@ -36,7 +38,7 @@ func TestReadUTXO2(t *testing.T) {
ValuePackFunc: nil,
DB: db,
}
b, err := hex.DecodeString("000012b")
b, err := hex.DecodeString("000012")
if err != nil {
log.Println(err)
}
@ -60,45 +62,13 @@ func TestReadUTXO2(t *testing.T) {
log.Println(UTXOValueUnpack(kv.Value))
got := UTXOValueUnpack(kv.Value).Amount
if got != tt.want[i] {
t.Errorf("got: %d, want: %d\n", got, tt.want)
t.Errorf("got: %d, want: %d\n", got, tt.want[i])
}
i++
}
})
}
}
func TestReadUTXO(t *testing.T) {
tests := []struct {
name string
want int
}{
{
name: "Read 10 UTXO Key Values",
want: 10,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, err := GetDB("/mnt/d/data/wallet/lbry-rocksdb")
if err != nil {
t.Errorf("err not nil: %+v\n", err)
}
defer db.Close()
data := ReadPrefixN(db, prefixes.UTXO, tt.want)
got := len(data)
for _, kv := range data {
log.Println(UTXOKeyUnpack(kv.Key))
log.Println(UTXOValueUnpack(kv.Value))
}
if got != tt.want {
got := i
if got != tt.wantTotal {
t.Errorf("got: %d, want: %d\n", got, tt.want)
}
})
@ -106,6 +76,43 @@ func TestReadUTXO(t *testing.T) {
}
// func TestReadUTXO(t *testing.T) {
// tests := []struct {
// name string
// want int
// }{
// {
// name: "Read UTXO Key Values",
// want: 12,
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// db, err := GetDB("../resources/asdf.db")
// if err != nil {
// t.Errorf("err not nil: %+v\n", err)
// }
// defer db.Close()
// data := ReadPrefixN(db, prefixes.UTXO, tt.want)
// got := len(data)
// for _, kv := range data {
// log.Println(UTXOKeyUnpack(kv.Key))
// log.Println(UTXOValueUnpack(kv.Value))
// }
// if got != tt.want {
// t.Errorf("got: %d, want: %d\n", got, tt.want)
// }
// })
// }
// }
// TestOpenDB test to see if we can open a db
func TestOpenDB(t *testing.T) {
@ -121,7 +128,7 @@ func TestOpenDB(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
vals := OpenDB("../resources/tmp.db")
vals := OpenDB("../resources/tmp.db", "foo")
got := vals
if got != tt.want {
@ -132,6 +139,30 @@ func TestOpenDB(t *testing.T) {
}
func TestOpenDB2(t *testing.T) {
tests := []struct {
name string
want int
}{
{
name: "Open a rocksdb database",
want: 10,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
OpenDB("../resources/asdf.db", "u")
// got := vals
// if got != tt.want {
// t.Errorf("got: %d, want: %d\n", got, tt.want)
// }
})
}
}
func TestUTXOKey_String(t *testing.T) {
tests := []struct {
name string

32
main.go
View file

@ -2,11 +2,13 @@ package main
import (
"context"
"encoding/hex"
"fmt"
"log"
"time"
"github.com/lbryio/hub/db"
"github.com/lbryio/hub/db/prefixes"
pb "github.com/lbryio/hub/protobuf/go"
"github.com/lbryio/hub/server"
"github.com/lbryio/lbry.go/v2/extras/util"
@ -30,7 +32,35 @@ func main() {
return
} else if args.CmdType == server.DBCmd {
db.OpenDB("/mnt/d/data/wallet/lbry-rocksdb/")
dbVal, err := db.GetDB("/mnt/d/data/wallet/lbry-rocksdb/")
if err != nil {
log.Fatalln(err)
}
pr := &db.PrefixRow{
Prefix: prefixes.UTXO,
KeyPackFunc: nil,
ValuePackFunc: nil,
KeyUnpackFunc: db.UTXOKeyUnpack,
ValueUnpackFunc: db.UTXOValueUnpack,
DB: dbVal,
}
b, err := hex.DecodeString("000013")
if err != nil {
log.Println(err)
}
stopKey := &db.UTXOKey{
Prefix: prefixes.UTXO,
HashX: b,
TxNum: 0,
Nout: 0,
}
stop := db.UTXOKeyPackPartial(stopKey, 1)
options := db.NewIterateOptions().WithFillCache(false).WithStop(stop)
db.OpenAndWriteDB(pr, options, "./resources/asdf.db")
return
}