LBRYCommunity/herald.go
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases 14 Packages Wiki Activity

asdf

Browse source
This commit is contained in:
Jeffrey Picard 2021-12-12 12:40:51 -05:00
parent a20951ca7f
commit abbc86b1ac
3 changed files with 155 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
db/db.go
View file

@ -21,10 +21,12 @@ type IterOptions struct {
type PrefixRow struct { type PrefixRow struct {
//KeyStruct interface{} //KeyStruct interface{}
//ValueStruct interface{} //ValueStruct interface{}
Prefix []byte Prefix []byte
KeyPackFunc interface{} KeyPackFunc interface{}
ValuePackFunc interface{} ValuePackFunc interface{}
DB *grocksdb.DB KeyUnpackFunc interface{}
ValueUnpackFunc interface{}
DB *grocksdb.DB
} }
type PrefixRowKV struct { type PrefixRowKV struct {
@ -85,7 +87,10 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV {
it.Seek(pr.Prefix) it.Seek(pr.Prefix)
if options.Start != nil { if options.Start != nil {
log.Println("Seeking to start")
it.Seek(options.Start) it.Seek(options.Start)
} else {
log.Println("Not seeking to start")
} }
/* /*
@ -118,6 +123,7 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV {
var prevKey []byte = nil var prevKey []byte = nil
go func() { go func() {
defer it.Close() defer it.Close()
defer close(ch)
for ; terminateFunc(prevKey); it.Next() { for ; terminateFunc(prevKey); it.Next() {
key := it.Key() key := it.Key()
prevKey = key.Data() prevKey = key.Data()
@ -131,7 +137,6 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV {
key.Free() key.Free()
value.Free() value.Free()
} }
close(ch)
}() }()
return ch return ch
@ -150,6 +155,13 @@ func (k *UTXOKey) PackKey() []byte {
return key return key
} }
// UTXOKeyPackPartialNFields creates a pack partial key function for n fields.
func UTXOKeyPackPartialNFields(nFields int) func(*UTXOKey) []byte {
return func(u *UTXOKey) []byte {
return UTXOKeyPackPartial(u, nFields)
}
}
// UTXOKeyPackPartial packs a variable number of fields for a UTXOKey into // UTXOKeyPackPartial packs a variable number of fields for a UTXOKey into
// a byte array. // a byte array.
func UTXOKeyPackPartial(k *UTXOKey, nFields int) []byte { func UTXOKeyPackPartial(k *UTXOKey, nFields int) []byte {
@ -257,7 +269,7 @@ func ReadPrefixN(db *grocksdb.DB, prefix []byte, n int) []*PrefixRowKV {
return res return res
} }
func OpenDB(name string) int { func OpenDB(name string, start string) int {
// Read db // Read db
opts := grocksdb.NewDefaultOptions() opts := grocksdb.NewDefaultOptions()
db, err := grocksdb.OpenDb(opts, name) db, err := grocksdb.OpenDb(opts, name)
@ -274,7 +286,8 @@ func OpenDB(name string) int {
defer it.Close() defer it.Close()
var i = 0 var i = 0
it.Seek([]byte("foo")) it.Seek([]byte(start))
// it.Seek([]byte{'u'})
for ; it.Valid(); it.Next() { for ; it.Valid(); it.Next() {
key := it.Key() key := it.Key()
value := it.Value() value := it.Value()
@ -292,46 +305,48 @@ func OpenDB(name string) int {
return i return i
} }
func OpenAndWriteDB(in string, out string) { func OpenAndWriteDB(prIn *PrefixRow, options *IterOptions, out string) {
// Read db
opts := grocksdb.NewDefaultOptions()
db, err := grocksdb.OpenDb(opts, in)
ro := grocksdb.NewDefaultReadOptions()
ro.SetFillCache(false)
if err != nil {
log.Println(err)
}
// Write db // Write db
opts := grocksdb.NewDefaultOptions()
opts.SetCreateIfMissing(true) opts.SetCreateIfMissing(true)
db2, err := grocksdb.OpenDb(opts, out) db, err := grocksdb.OpenDb(opts, out)
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
wo := grocksdb.NewDefaultWriteOptions() wo := grocksdb.NewDefaultWriteOptions()
defer db2.Close() defer db.Close()
log.Println(db.Name()) ch := prIn.Iter(options)
log.Println(db2.Name())
it := db.NewIterator(ro)
defer it.Close()
var i = 0 var i = 0
it.Seek([]byte("foo")) var prevKey []byte = nil
for ; it.Valid() && i < 10; it.Next() { for kv := range ch {
key := it.Key() log.Println(kv)
value := it.Value() key := kv.Key
fmt.Printf("Key: %v Value: %v\n", key.Data(), value.Data()) value := kv.Value
unpackKeyFnValue := reflect.ValueOf(prIn.KeyUnpackFunc)
keyArgs := []reflect.Value{reflect.ValueOf(key)}
unpackKeyFnResult := unpackKeyFnValue.Call(keyArgs)
unpackedKey := unpackKeyFnResult[0].Interface() //.(reflect.TypeOf())
if err := db2.Put(wo, key.Data(), value.Data()); err != nil { unpackValueFnValue := reflect.ValueOf(prIn.ValueUnpackFunc)
log.Println(err) valueArgs := []reflect.Value{reflect.ValueOf(value)}
unpackValueFnResult := unpackValueFnValue.Call(valueArgs)
unpackedValue := unpackValueFnResult[0].Interface() //.([]byte)
log.Println(unpackedKey)
log.Println(unpackedValue)
if bytes.Equal(prevKey, key) {
if err := db.Merge(wo, key, value); err != nil {
log.Println(err)
}
} else {
if err := db.Put(wo, key, value); err != nil {
log.Println(err)
}
} }
prevKey = key
key.Free()
value.Free()
i++ i++
} }
if err := it.Err(); err != nil {
log.Println(err)
}
} }

115
db/db_test.go
View file

@ -12,18 +12,20 @@ import (
func TestReadUTXO2(t *testing.T) { func TestReadUTXO2(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
want []uint64 want []uint64
wantTotal int
}{ }{
{ {
name: "Read 10 UTXO Key Values", name: "Read UTXO Key Values With Stop",
want: []uint64{2174594, 200000000, 20000000, 100000, 603510, 75000000, 100000, 962984, 25000000, 50000000}, want: []uint64{2174594, 200000000, 20000000, 100000, 603510, 75000000, 100000, 962984, 25000000, 50000000},
wantTotal: 7,
}, },
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
db, err := GetDB("/mnt/d/data/wallet/lbry-rocksdb") db, err := GetDB("../resources/asdf.db")
if err != nil { if err != nil {
t.Errorf("err not nil: %+v\n", err) t.Errorf("err not nil: %+v\n", err)
} }
@ -36,7 +38,7 @@ func TestReadUTXO2(t *testing.T) {
ValuePackFunc: nil, ValuePackFunc: nil,
DB: db, DB: db,
} }
b, err := hex.DecodeString("000012b") b, err := hex.DecodeString("000012")
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
@ -60,45 +62,13 @@ func TestReadUTXO2(t *testing.T) {
log.Println(UTXOValueUnpack(kv.Value)) log.Println(UTXOValueUnpack(kv.Value))
got := UTXOValueUnpack(kv.Value).Amount got := UTXOValueUnpack(kv.Value).Amount
if got != tt.want[i] { if got != tt.want[i] {
t.Errorf("got: %d, want: %d\n", got, tt.want) t.Errorf("got: %d, want: %d\n", got, tt.want[i])
} }
i++ i++
} }
})
}
} got := i
if got != tt.wantTotal {
func TestReadUTXO(t *testing.T) {
tests := []struct {
name string
want int
}{
{
name: "Read 10 UTXO Key Values",
want: 10,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, err := GetDB("/mnt/d/data/wallet/lbry-rocksdb")
if err != nil {
t.Errorf("err not nil: %+v\n", err)
}
defer db.Close()
data := ReadPrefixN(db, prefixes.UTXO, tt.want)
got := len(data)
for _, kv := range data {
log.Println(UTXOKeyUnpack(kv.Key))
log.Println(UTXOValueUnpack(kv.Value))
}
if got != tt.want {
t.Errorf("got: %d, want: %d\n", got, tt.want) t.Errorf("got: %d, want: %d\n", got, tt.want)
} }
}) })
@ -106,6 +76,43 @@ func TestReadUTXO(t *testing.T) {
} }
// func TestReadUTXO(t *testing.T) {
// tests := []struct {
// name string
// want int
// }{
// {
// name: "Read UTXO Key Values",
// want: 12,
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// db, err := GetDB("../resources/asdf.db")
// if err != nil {
// t.Errorf("err not nil: %+v\n", err)
// }
// defer db.Close()
// data := ReadPrefixN(db, prefixes.UTXO, tt.want)
// got := len(data)
// for _, kv := range data {
// log.Println(UTXOKeyUnpack(kv.Key))
// log.Println(UTXOValueUnpack(kv.Value))
// }
// if got != tt.want {
// t.Errorf("got: %d, want: %d\n", got, tt.want)
// }
// })
// }
// }
// TestOpenDB test to see if we can open a db // TestOpenDB test to see if we can open a db
func TestOpenDB(t *testing.T) { func TestOpenDB(t *testing.T) {
@ -121,7 +128,7 @@ func TestOpenDB(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
vals := OpenDB("../resources/tmp.db") vals := OpenDB("../resources/tmp.db", "foo")
got := vals got := vals
if got != tt.want { if got != tt.want {
@ -132,6 +139,30 @@ func TestOpenDB(t *testing.T) {
} }
func TestOpenDB2(t *testing.T) {
tests := []struct {
name string
want int
}{
{
name: "Open a rocksdb database",
want: 10,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
OpenDB("../resources/asdf.db", "u")
// got := vals
// if got != tt.want {
// t.Errorf("got: %d, want: %d\n", got, tt.want)
// }
})
}
}
func TestUTXOKey_String(t *testing.T) { func TestUTXOKey_String(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

32
main.go
View file

@ -2,11 +2,13 @@ package main
import ( import (
"context" "context"
"encoding/hex"
"fmt" "fmt"
"log" "log"
"time" "time"
"github.com/lbryio/hub/db" "github.com/lbryio/hub/db"
"github.com/lbryio/hub/db/prefixes"
pb "github.com/lbryio/hub/protobuf/go" pb "github.com/lbryio/hub/protobuf/go"
"github.com/lbryio/hub/server" "github.com/lbryio/hub/server"
"github.com/lbryio/lbry.go/v2/extras/util" "github.com/lbryio/lbry.go/v2/extras/util"
@ -30,7 +32,35 @@ func main() {
return return
} else if args.CmdType == server.DBCmd { } else if args.CmdType == server.DBCmd {
db.OpenDB("/mnt/d/data/wallet/lbry-rocksdb/") dbVal, err := db.GetDB("/mnt/d/data/wallet/lbry-rocksdb/")
if err != nil {
log.Fatalln(err)
}
pr := &db.PrefixRow{
Prefix: prefixes.UTXO,
KeyPackFunc: nil,
ValuePackFunc: nil,
KeyUnpackFunc: db.UTXOKeyUnpack,
ValueUnpackFunc: db.UTXOValueUnpack,
DB: dbVal,
}
b, err := hex.DecodeString("000013")
if err != nil {
log.Println(err)
}
stopKey := &db.UTXOKey{
Prefix: prefixes.UTXO,
HashX: b,
TxNum: 0,
Nout: 0,
}
stop := db.UTXOKeyPackPartial(stopKey, 1)
options := db.NewIterateOptions().WithFillCache(false).WithStop(stop)
db.OpenAndWriteDB(pr, options, "./resources/asdf.db")
return return
} }