Compare commits

..

No commits in common. "master" and "blockchain_rpc1" have entirely different histories.

39 changed files with 688 additions and 3260 deletions

View file

@ -7,6 +7,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v2
- name: Build and Test
uses: ./

View file

@ -15,7 +15,6 @@ import (
"github.com/lbryio/herald.go/internal"
"github.com/lbryio/herald.go/internal/metrics"
pb "github.com/lbryio/herald.go/protobuf/go"
"github.com/lbryio/lbry.go/v3/extras/stop"
"github.com/linxGnu/grocksdb"
log "github.com/sirupsen/logrus"
@ -59,7 +58,8 @@ type ReadOnlyDBColumnFamily struct {
BlockedChannels map[string][]byte
FilteredStreams map[string][]byte
FilteredChannels map[string][]byte
Grp *stop.Group
ShutdownChan chan struct{}
DoneChan chan struct{}
Cleanup func()
}
@ -318,29 +318,9 @@ func intMin(a, b int) int {
return b
}
// FIXME: This was copied from the signal.go file, maybe move it to a more common place?
// interruptRequested returns true when the channel returned by
// interruptListener was closed. This simplifies early shutdown slightly since
// the caller can just use an if statement instead of a select.
func interruptRequested(interrupted <-chan struct{}) bool {
select {
case <-interrupted:
return true
default:
}
return false
}
func IterCF(db *grocksdb.DB, opts *IterOptions) <-chan *prefixes.PrefixRowKV {
ch := make(chan *prefixes.PrefixRowKV)
// Check if we've been told to shutdown in between getting created and getting here
if opts.Grp != nil && interruptRequested(opts.Grp.Ch()) {
opts.Grp.Done()
return ch
}
ro := grocksdb.NewDefaultReadOptions()
ro.SetFillCache(opts.FillCache)
it := db.NewIteratorCF(ro, opts.CfHandle)
@ -356,10 +336,6 @@ func IterCF(db *grocksdb.DB, opts *IterOptions) <-chan *prefixes.PrefixRowKV {
it.Close()
close(ch)
ro.Destroy()
if opts.Grp != nil {
// opts.Grp.DoneNamed(iterKey)
opts.Grp.Done()
}
}()
var prevKey []byte
@ -379,9 +355,6 @@ func IterCF(db *grocksdb.DB, opts *IterOptions) <-chan *prefixes.PrefixRowKV {
if kv = opts.ReadRow(&prevKey); kv != nil {
ch <- kv
}
if opts.Grp != nil && interruptRequested(opts.Grp.Ch()) {
return
}
}
}()
@ -439,10 +412,10 @@ func (db *ReadOnlyDBColumnFamily) selectFrom(prefix []byte, startKey, stopKey pr
return nil, err
}
// Prefix and handle
options := NewIterateOptions().WithDB(db).WithPrefix(prefix).WithCfHandle(handle)
options := NewIterateOptions().WithPrefix(prefix).WithCfHandle(handle)
// Start and stop bounds
options = options.WithStart(startKey.PackKey()).WithStop(stopKey.PackKey()).WithIncludeStop(false)
// Include the key and value
options = options.WithStart(startKey.PackKey()).WithStop(stopKey.PackKey()).WithIncludeStop(true)
// Don't include the key
options = options.WithIncludeKey(true).WithIncludeValue(true)
return []*IterOptions{options}, nil
}
@ -455,7 +428,7 @@ func iterate(db *grocksdb.DB, opts []*IterOptions) <-chan []*prefixes.PrefixRowK
for kv := range IterCF(db, o) {
row := make([]*prefixes.PrefixRowKV, 0, 1)
row = append(row, kv)
log.Debugf("iterate[%v][%v] %#v -> %#v", i, j, kv.Key, kv.Value)
log.Debugf("iterate[%v][%v] %#v", i, j, kv)
out <- row
j++
}
@ -481,7 +454,7 @@ func innerJoin(db *grocksdb.DB, in <-chan []*prefixes.PrefixRowKV, selectFn func
row = append(row, kvs...)
row = append(row, kv...)
for i, kv := range row {
log.Debugf("row[%v] %#v -> %#v", i, kv.Key, kv.Value)
log.Debugf("row[%v] %#v", i, kv)
}
out <- row
}
@ -531,7 +504,7 @@ func GetWriteDBCF(name string) (*grocksdb.DB, []*grocksdb.ColumnFamilyHandle, er
}
// GetProdDB returns a db that is used for production.
func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBColumnFamily, error) {
func GetProdDB(name string, secondaryPath string) (*ReadOnlyDBColumnFamily, func(), error) {
prefixNames := prefixes.GetPrefixes()
// additional prefixes that aren't in the code explicitly
cfNames := []string{"default", "e", "d", "c"}
@ -540,7 +513,7 @@ func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBC
cfNames = append(cfNames, cfName)
}
db, err := GetDBColumnFamilies(name, secondaryPath, cfNames, grp)
db, err := GetDBColumnFamilies(name, secondaryPath, cfNames)
cleanupFiles := func() {
err = os.RemoveAll(secondaryPath)
@ -550,8 +523,7 @@ func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBC
}
if err != nil {
cleanupFiles()
return nil, err
return nil, cleanupFiles, err
}
cleanupDB := func() {
@ -560,11 +532,11 @@ func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBC
}
db.Cleanup = cleanupDB
return db, nil
return db, cleanupDB, nil
}
// GetDBColumnFamilies gets a db with the specified column families and secondary path.
func GetDBColumnFamilies(name string, secondayPath string, cfNames []string, grp *stop.Group) (*ReadOnlyDBColumnFamily, error) {
func GetDBColumnFamilies(name string, secondayPath string, cfNames []string) (*ReadOnlyDBColumnFamily, error) {
opts := grocksdb.NewDefaultOptions()
roOpts := grocksdb.NewDefaultReadOptions()
cfOpt := grocksdb.NewDefaultOptions()
@ -579,7 +551,6 @@ func GetDBColumnFamilies(name string, secondayPath string, cfNames []string, grp
// db, handles, err := grocksdb.OpenDbColumnFamilies(opts, name, cfNames, cfOpts)
if err != nil {
log.Errorf("open db as secondary failed: %v", err)
return nil, err
}
@ -601,7 +572,8 @@ func GetDBColumnFamilies(name string, secondayPath string, cfNames []string, grp
LastState: nil,
Height: 0,
Headers: nil,
Grp: grp,
ShutdownChan: make(chan struct{}),
DoneChan: make(chan struct{}),
}
err = myDB.ReadDBState() //TODO: Figure out right place for this
@ -670,22 +642,21 @@ func (db *ReadOnlyDBColumnFamily) Unwind() {
// Shutdown shuts down the db.
func (db *ReadOnlyDBColumnFamily) Shutdown() {
log.Println("Calling cleanup...")
db.ShutdownChan <- struct{}{}
<-db.DoneChan
db.Cleanup()
log.Println("Leaving Shutdown...")
}
// RunDetectChanges Go routine the runs continuously while the hub is active
// to keep the db readonly view up to date and handle reorgs on the
// blockchain.
func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan<- interface{}) {
db.Grp.Add(1)
func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan *internal.HeightHash) {
go func() {
lastPrint := time.Now()
for {
// FIXME: Figure out best sleep interval
if time.Since(lastPrint) > time.Second {
log.Debugf("DetectChanges: %#v", db.LastState)
log.Debug("DetectChanges:", db.LastState)
lastPrint = time.Now()
}
err := db.detectChanges(notifCh)
@ -693,8 +664,8 @@ func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan<- interface{}) {
log.Infof("Error detecting changes: %#v", err)
}
select {
case <-db.Grp.Ch():
db.Grp.Done()
case <-db.ShutdownChan:
db.DoneChan <- struct{}{}
return
case <-time.After(time.Millisecond * 10):
}
@ -703,7 +674,7 @@ func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan<- interface{}) {
}
// DetectChanges keep the rocksdb db in sync and handle reorgs
func (db *ReadOnlyDBColumnFamily) detectChanges(notifCh chan<- interface{}) error {
func (db *ReadOnlyDBColumnFamily) detectChanges(notifCh chan *internal.HeightHash) error {
err := db.DB.TryCatchUpWithPrimary()
if err != nil {
return err
@ -775,12 +746,7 @@ func (db *ReadOnlyDBColumnFamily) detectChanges(notifCh chan<- interface{}) erro
log.Info("error getting block hash: ", err)
return err
}
header, err := db.GetHeader(height)
if err != nil {
log.Info("error getting block header: ", err)
return err
}
notifCh <- &internal.HeightHash{Height: uint64(height), BlockHash: hash, BlockHeader: header}
notifCh <- &internal.HeightHash{Height: uint64(height), BlockHash: hash}
}
//TODO: ClearCache
log.Warn("implement cache clearing")
@ -824,7 +790,7 @@ func (db *ReadOnlyDBColumnFamily) InitHeaders() error {
// endKey := prefixes.NewHeaderKey(db.LastState.Height)
startKeyRaw := startKey.PackKey()
// endKeyRaw := endKey.PackKey()
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
options := NewIterateOptions().WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
options = options.WithIncludeKey(false).WithIncludeValue(true) //.WithIncludeStop(true)
options = options.WithStart(startKeyRaw) //.WithStop(endKeyRaw)
@ -847,7 +813,7 @@ func (db *ReadOnlyDBColumnFamily) InitTxCounts() error {
db.TxCounts = stack.NewSliceBacked[uint32](InitialTxCountSize)
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.TxCount}).WithCfHandle(handle)
options := NewIterateOptions().WithPrefix([]byte{prefixes.TxCount}).WithCfHandle(handle)
options = options.WithIncludeKey(false).WithIncludeValue(true).WithIncludeStop(true)
ch := IterCF(db.DB, options)

View file

@ -3,19 +3,15 @@ package db
// db_get.go contains the basic access functions to the database.
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"log"
"math"
"github.com/lbryio/herald.go/db/prefixes"
"github.com/lbryio/herald.go/db/stack"
"github.com/lbryio/lbcd/chaincfg/chainhash"
"github.com/lbryio/lbcd/wire"
"github.com/linxGnu/grocksdb"
log "github.com/sirupsen/logrus"
)
// GetExpirationHeight returns the expiration height for the given height. Uses
@ -68,57 +64,6 @@ func (db *ReadOnlyDBColumnFamily) GetBlockHash(height uint32) ([]byte, error) {
return rawValue, nil
}
func (db *ReadOnlyDBColumnFamily) GetBlockTXs(height uint32) ([]*chainhash.Hash, error) {
handle, err := db.EnsureHandle(prefixes.BlockTXs)
if err != nil {
return nil, err
}
key := prefixes.BlockTxsKey{
Prefix: []byte{prefixes.BlockTXs},
Height: height,
}
slice, err := db.DB.GetCF(db.Opts, handle, key.PackKey())
defer slice.Free()
if err != nil {
return nil, err
}
if slice.Size() == 0 {
return nil, nil
}
rawValue := make([]byte, len(slice.Data()))
copy(rawValue, slice.Data())
value := prefixes.BlockTxsValueUnpack(rawValue)
return value.TxHashes, nil
}
func (db *ReadOnlyDBColumnFamily) GetTouchedHashXs(height uint32) ([][]byte, error) {
handle, err := db.EnsureHandle(prefixes.TouchedHashX)
if err != nil {
return nil, err
}
key := prefixes.TouchedHashXKey{
Prefix: []byte{prefixes.TouchedHashX},
Height: height,
}
slice, err := db.DB.GetCF(db.Opts, handle, key.PackKey())
defer slice.Free()
if err != nil {
return nil, err
}
if slice.Size() == 0 {
return nil, nil
}
rawValue := make([]byte, len(slice.Data()))
copy(rawValue, slice.Data())
value := prefixes.TouchedHashXValue{}
value.UnpackValue(rawValue)
return value.TouchedHashXs, nil
}
func (db *ReadOnlyDBColumnFamily) GetHeader(height uint32) ([]byte, error) {
handle, err := db.EnsureHandle(prefixes.Header)
if err != nil {
@ -148,7 +93,7 @@ func (db *ReadOnlyDBColumnFamily) GetHeaders(height uint32, count uint32) ([][11
startKeyRaw := prefixes.NewHeaderKey(height).PackKey()
endKeyRaw := prefixes.NewHeaderKey(height + count).PackKey()
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
options := NewIterateOptions().WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
options = options.WithIncludeKey(false).WithIncludeValue(true) //.WithIncludeStop(true)
options = options.WithStart(startKeyRaw).WithStop(endKeyRaw)
@ -184,7 +129,7 @@ func (db *ReadOnlyDBColumnFamily) GetBalance(hashX []byte) (uint64, uint64, erro
startKeyRaw := startKey.PackKey()
endKeyRaw := endKey.PackKey()
// Prefix and handle
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.UTXO}).WithCfHandle(handle)
options := NewIterateOptions().WithPrefix([]byte{prefixes.UTXO}).WithCfHandle(handle)
// Start and stop bounds
options = options.WithStart(startKeyRaw).WithStop(endKeyRaw).WithIncludeStop(true)
// Don't include the key
@ -321,79 +266,6 @@ func (db *ReadOnlyDBColumnFamily) GetHistory(hashX []byte) ([]TxInfo, error) {
return results, nil
}
func (db *ReadOnlyDBColumnFamily) GetStatus(hashX []byte) ([]byte, error) {
// Lookup in HashXMempoolStatus first.
status, err := db.getMempoolStatus(hashX)
if err == nil && status != nil {
log.Debugf("(mempool) status(%#v) -> %#v", hashX, status)
return status, err
}
// No indexed mempool status. Lookup in HashXStatus second.
handle, err := db.EnsureHandle(prefixes.HashXStatus)
if err != nil {
return nil, err
}
key := &prefixes.HashXStatusKey{
Prefix: []byte{prefixes.HashXStatus},
HashX: hashX,
}
rawKey := key.PackKey()
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
defer slice.Free()
if err == nil && slice.Size() > 0 {
rawValue := make([]byte, len(slice.Data()))
copy(rawValue, slice.Data())
value := prefixes.HashXStatusValue{}
value.UnpackValue(rawValue)
log.Debugf("status(%#v) -> %#v", hashX, value.Status)
return value.Status, nil
}
// No indexed status. Fall back to enumerating HashXHistory.
txs, err := db.GetHistory(hashX)
if err != nil {
return nil, err
}
if len(txs) == 0 {
return []byte{}, err
}
hash := sha256.New()
for _, tx := range txs {
hash.Write([]byte(fmt.Sprintf("%s:%d:", tx.TxHash.String(), tx.Height)))
}
// TODO: Mempool history
return hash.Sum(nil), err
}
func (db *ReadOnlyDBColumnFamily) getMempoolStatus(hashX []byte) ([]byte, error) {
handle, err := db.EnsureHandle(prefixes.HashXMempoolStatus)
if err != nil {
return nil, err
}
key := &prefixes.HashXMempoolStatusKey{
Prefix: []byte{prefixes.HashXMempoolStatus},
HashX: hashX,
}
rawKey := key.PackKey()
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
defer slice.Free()
if err != nil {
return nil, err
} else if slice.Size() == 0 {
return nil, nil
}
rawValue := make([]byte, len(slice.Data()))
copy(rawValue, slice.Data())
value := prefixes.HashXMempoolStatusValue{}
value.UnpackValue(rawValue)
return value.Status, nil
}
// GetStreamsAndChannelRepostedByChannelHashes returns a map of streams and channel hashes that are reposted by the given channel hashes.
func (db *ReadOnlyDBColumnFamily) GetStreamsAndChannelRepostedByChannelHashes(reposterChannelHashes [][]byte) (map[string][]byte, map[string][]byte, error) {
handle, err := db.EnsureHandle(prefixes.ChannelToClaim)
@ -407,7 +279,7 @@ func (db *ReadOnlyDBColumnFamily) GetStreamsAndChannelRepostedByChannelHashes(re
for _, reposterChannelHash := range reposterChannelHashes {
key := prefixes.NewChannelToClaimKeyWHash(reposterChannelHash)
rawKeyPrefix := key.PartialPack(1)
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options = options.WithIncludeKey(false).WithIncludeValue(true)
ch := IterCF(db.DB, options)
// for stream := range Iterate(db.DB, prefixes.ChannelToClaim, []byte{reposterChannelHash}, false) {
@ -481,7 +353,7 @@ func (db *ReadOnlyDBColumnFamily) GetShortClaimIdUrl(name string, normalizedName
log.Printf("partialKey: %#v\n", partialKey)
keyPrefix := partialKey.PartialPack(2)
// Prefix and handle
options := NewIterateOptions().WithDB(db).WithPrefix(prefix).WithCfHandle(handle)
options := NewIterateOptions().WithPrefix(prefix).WithCfHandle(handle)
// Start and stop bounds
options = options.WithStart(keyPrefix).WithStop(keyPrefix)
// Don't include the key
@ -574,12 +446,12 @@ func (db *ReadOnlyDBColumnFamily) GetActiveAmount(claimHash []byte, txoType uint
}
startKey := prefixes.NewActiveAmountKey(claimHash, txoType, 0)
endKey := prefixes.NewActiveAmountKey(claimHash, txoType, height+1)
endKey := prefixes.NewActiveAmountKey(claimHash, txoType, height)
startKeyRaw := startKey.PartialPack(3)
endKeyRaw := endKey.PartialPack(3)
// Prefix and handle
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.ActiveAmount}).WithCfHandle(handle)
options := NewIterateOptions().WithPrefix([]byte{prefixes.ActiveAmount}).WithCfHandle(handle)
// Start and stop bounds
options = options.WithStart(startKeyRaw).WithStop(endKeyRaw)
// Don't include the key
@ -595,6 +467,14 @@ func (db *ReadOnlyDBColumnFamily) GetActiveAmount(claimHash []byte, txoType uint
}
func (db *ReadOnlyDBColumnFamily) GetEffectiveAmount(claimHash []byte, supportOnly bool) (uint64, error) {
if supportOnly {
supportAmount, err := db.GetActiveAmount(claimHash, prefixes.ActivatedSupportTXOType, db.Height+1)
if err != nil {
return 0, err
}
return supportAmount, nil
}
handle, err := db.EnsureHandle(prefixes.EffectiveAmount)
if err != nil {
return 0, err
@ -612,13 +492,7 @@ func (db *ReadOnlyDBColumnFamily) GetEffectiveAmount(claimHash []byte, supportOn
value := prefixes.EffectiveAmountValue{}
value.UnpackValue(slice.Data())
var amount uint64
if supportOnly {
amount += value.ActivatedSupportSum
} else {
amount += value.ActivatedSum
}
return amount, nil
return value.EffectiveAmount, nil
}
func (db *ReadOnlyDBColumnFamily) GetSupportAmount(claimHash []byte) (uint64, error) {
@ -735,7 +609,7 @@ func (db *ReadOnlyDBColumnFamily) ControllingClaimIter() <-chan *prefixes.Prefix
key := prefixes.NewClaimTakeoverKey("")
var rawKeyPrefix []byte = nil
rawKeyPrefix = key.PartialPack(0)
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options = options.WithIncludeValue(true) //.WithIncludeStop(true)
ch := IterCF(db.DB, options)
return ch
@ -792,70 +666,6 @@ func (db *ReadOnlyDBColumnFamily) FsGetClaimByHash(claimHash []byte) (*ResolveRe
)
}
func (db *ReadOnlyDBColumnFamily) GetTx(txhash *chainhash.Hash) ([]byte, *wire.MsgTx, error) {
// Lookup in MempoolTx first.
raw, tx, err := db.getMempoolTx(txhash)
if err == nil && raw != nil && tx != nil {
return raw, tx, err
}
handle, err := db.EnsureHandle(prefixes.Tx)
if err != nil {
return nil, nil, err
}
key := prefixes.TxKey{Prefix: []byte{prefixes.Tx}, TxHash: txhash}
rawKey := key.PackKey()
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
defer slice.Free()
if err != nil {
return nil, nil, err
}
if slice.Size() == 0 {
return nil, nil, nil
}
rawValue := make([]byte, len(slice.Data()))
copy(rawValue, slice.Data())
value := prefixes.TxValue{}
value.UnpackValue(rawValue)
var msgTx wire.MsgTx
err = msgTx.Deserialize(bytes.NewReader(value.RawTx))
if err != nil {
return nil, nil, err
}
return value.RawTx, &msgTx, nil
}
func (db *ReadOnlyDBColumnFamily) getMempoolTx(txhash *chainhash.Hash) ([]byte, *wire.MsgTx, error) {
handle, err := db.EnsureHandle(prefixes.MempoolTx)
if err != nil {
return nil, nil, err
}
key := prefixes.MempoolTxKey{Prefix: []byte{prefixes.Tx}, TxHash: txhash}
rawKey := key.PackKey()
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
defer slice.Free()
if err != nil {
return nil, nil, err
}
if slice.Size() == 0 {
return nil, nil, nil
}
rawValue := make([]byte, len(slice.Data()))
copy(rawValue, slice.Data())
value := prefixes.MempoolTxValue{}
value.UnpackValue(rawValue)
var msgTx wire.MsgTx
err = msgTx.Deserialize(bytes.NewReader(value.RawTx))
if err != nil {
return nil, nil, err
}
return value.RawTx, &msgTx, nil
}
func (db *ReadOnlyDBColumnFamily) GetTxCount(height uint32) (*prefixes.TxCountValue, error) {
handle, err := db.EnsureHandle(prefixes.TxCount)
if err != nil {
@ -879,162 +689,6 @@ func (db *ReadOnlyDBColumnFamily) GetTxCount(height uint32) (*prefixes.TxCountVa
return value, nil
}
func (db *ReadOnlyDBColumnFamily) GetTxHeight(txhash *chainhash.Hash) (uint32, error) {
handle, err := db.EnsureHandle(prefixes.TxNum)
if err != nil {
return 0, err
}
key := prefixes.TxNumKey{Prefix: []byte{prefixes.TxNum}, TxHash: txhash}
rawKey := key.PackKey()
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
defer slice.Free()
if err != nil {
return 0, err
}
if slice.Size() == 0 {
return 0, nil
}
// No slice copy needed. Value will be abandoned.
value := prefixes.TxNumValueUnpack(slice.Data())
height := stack.BisectRight(db.TxCounts, []uint32{value.TxNum})[0]
return height, nil
}
type TxMerkle struct {
TxHash *chainhash.Hash
RawTx []byte
Height int
Pos uint32
Merkle []*chainhash.Hash
}
// merklePath selects specific transactions by position within blockTxs.
// The resulting merkle path (aka merkle branch, or merkle) is a list of TX hashes
// which are in sibling relationship with TX nodes on the path to the root.
func merklePath(pos uint32, blockTxs, partial []*chainhash.Hash) []*chainhash.Hash {
parent := func(p uint32) uint32 {
return p >> 1
}
sibling := func(p uint32) uint32 {
if p%2 == 0 {
return p + 1
} else {
return p - 1
}
}
p := parent(pos)
if p == 0 {
// No parent, path is complete.
return partial
}
// Add sibling to partial path and proceed to parent TX.
return merklePath(p, blockTxs, append(partial, blockTxs[sibling(pos)]))
}
func (db *ReadOnlyDBColumnFamily) GetTxMerkle(tx_hashes []chainhash.Hash) ([]TxMerkle, error) {
selectedTxNum := make([]*IterOptions, 0, len(tx_hashes))
for _, txhash := range tx_hashes {
key := prefixes.TxNumKey{Prefix: []byte{prefixes.TxNum}, TxHash: &txhash}
log.Debugf("%v", key)
opt, err := db.selectFrom(key.Prefix, &key, &key)
if err != nil {
return nil, err
}
selectedTxNum = append(selectedTxNum, opt...)
}
selectTxByTxNum := func(in []*prefixes.PrefixRowKV) ([]*IterOptions, error) {
txNumKey := in[0].Key.(*prefixes.TxNumKey)
log.Debugf("%v", txNumKey.TxHash.String())
out := make([]*IterOptions, 0, 100)
startKey := &prefixes.TxKey{
Prefix: []byte{prefixes.Tx},
TxHash: txNumKey.TxHash,
}
endKey := &prefixes.TxKey{
Prefix: []byte{prefixes.Tx},
TxHash: txNumKey.TxHash,
}
selectedTx, err := db.selectFrom([]byte{prefixes.Tx}, startKey, endKey)
if err != nil {
return nil, err
}
out = append(out, selectedTx...)
return out, nil
}
blockTxsCache := make(map[uint32][]*chainhash.Hash)
results := make([]TxMerkle, 0, 500)
for kvs := range innerJoin(db.DB, iterate(db.DB, selectedTxNum), selectTxByTxNum) {
if err := checkForError(kvs); err != nil {
return results, err
}
txNumKey, txNumVal := kvs[0].Key.(*prefixes.TxNumKey), kvs[0].Value.(*prefixes.TxNumValue)
_, txVal := kvs[1].Key.(*prefixes.TxKey), kvs[1].Value.(*prefixes.TxValue)
txHeight := stack.BisectRight(db.TxCounts, []uint32{txNumVal.TxNum})[0]
txPos := txNumVal.TxNum - db.TxCounts.Get(txHeight-1)
// We need all the TX hashes in order to select out the relevant ones.
if _, ok := blockTxsCache[txHeight]; !ok {
txs, err := db.GetBlockTXs(txHeight)
if err != nil {
return results, err
}
blockTxsCache[txHeight] = txs
}
blockTxs := blockTxsCache[txHeight]
results = append(results, TxMerkle{
TxHash: txNumKey.TxHash,
RawTx: txVal.RawTx,
Height: int(txHeight),
Pos: txPos,
Merkle: merklePath(txPos, blockTxs, []*chainhash.Hash{}),
})
}
return results, nil
}
func (db *ReadOnlyDBColumnFamily) GetClaimByID(claimID string) ([]*ExpandedResolveResult, []*ExpandedResolveResult, error) {
rows := make([]*ExpandedResolveResult, 0)
extras := make([]*ExpandedResolveResult, 0)
claimHash, err := hex.DecodeString(claimID)
if err != nil {
return nil, nil, err
}
stream, err := db.FsGetClaimByHash(claimHash)
if err != nil {
return nil, nil, err
}
var res = NewExpandedResolveResult()
res.Stream = &optionalResolveResultOrError{res: stream}
rows = append(rows, res)
if stream != nil && stream.ChannelHash != nil {
channel, err := db.FsGetClaimByHash(stream.ChannelHash)
if err != nil {
return nil, nil, err
}
var res = NewExpandedResolveResult()
res.Channel = &optionalResolveResultOrError{res: channel}
extras = append(extras, res)
}
if stream != nil && stream.RepostedClaimHash != nil {
repost, err := db.FsGetClaimByHash(stream.RepostedClaimHash)
if err != nil {
return nil, nil, err
}
var res = NewExpandedResolveResult()
res.Repost = &optionalResolveResultOrError{res: repost}
extras = append(extras, res)
}
return rows, extras, nil
}
func (db *ReadOnlyDBColumnFamily) GetDBState() (*prefixes.DBStateValue, error) {
handle, err := db.EnsureHandle(prefixes.DBState)
if err != nil {
@ -1066,7 +720,7 @@ func (db *ReadOnlyDBColumnFamily) BidOrderNameIter(normalizedName string) <-chan
key := prefixes.NewBidOrderKey(normalizedName)
var rawKeyPrefix []byte = nil
rawKeyPrefix = key.PartialPack(1)
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options = options.WithIncludeValue(true) //.WithIncludeStop(true)
ch := IterCF(db.DB, options)
return ch
@ -1084,7 +738,7 @@ func (db *ReadOnlyDBColumnFamily) ClaimShortIdIter(normalizedName string, claimI
} else {
rawKeyPrefix = key.PartialPack(1)
}
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options = options.WithIncludeValue(true) //.WithIncludeStop(true)
ch := IterCF(db.DB, options)
return ch

View file

@ -326,7 +326,7 @@ func (db *ReadOnlyDBColumnFamily) ResolveClaimInChannel(channelHash []byte, norm
key := prefixes.NewChannelToClaimKey(channelHash, normalizedName)
rawKeyPrefix := key.PartialPack(2)
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
options = options.WithIncludeValue(true) //.WithIncludeStop(true)
ch := IterCF(db.DB, options)
// TODO: what's a good default size for this?

View file

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/csv"
"encoding/hex"
"log"
"os"
"strings"
"testing"
@ -11,9 +12,7 @@ import (
dbpkg "github.com/lbryio/herald.go/db"
"github.com/lbryio/herald.go/db/prefixes"
"github.com/lbryio/herald.go/internal"
"github.com/lbryio/lbry.go/v3/extras/stop"
"github.com/linxGnu/grocksdb"
log "github.com/sirupsen/logrus"
)
////////////////////////////////////////////////////////////////////////////////
@ -21,7 +20,7 @@ import (
////////////////////////////////////////////////////////////////////////////////
// OpenAndFillTmpDBColumnFamlies opens a db and fills it with data from a csv file using the given column family names
func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFamily, [][]string, error) {
func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFamily, [][]string, func(), error) {
log.Println(filePath)
file, err := os.Open(filePath)
@ -31,7 +30,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
reader := csv.NewReader(file)
records, err := reader.ReadAll()
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
wOpts := grocksdb.NewDefaultWriteOptions()
@ -39,7 +38,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
opts.SetCreateIfMissing(true)
db, err := grocksdb.OpenDb(opts, "tmp")
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
var handleMap map[string]*grocksdb.ColumnFamilyHandle = make(map[string]*grocksdb.ColumnFamilyHandle)
@ -54,7 +53,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
log.Println(cfName)
handle, err := db.CreateColumnFamily(opts, cfName)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
handleMap[cfName] = handle
}
@ -68,16 +67,16 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
for _, record := range records[1:] {
cf := record[0]
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
handle := handleMap[string(cf)]
key, err := hex.DecodeString(record[1])
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
val, err := hex.DecodeString(record[2])
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
db.PutCF(wOpts, handle, key, val)
}
@ -94,8 +93,6 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
LastState: nil,
Height: 0,
Headers: nil,
Grp: stop.New(),
Cleanup: toDefer,
}
// err = dbpkg.ReadDBState(myDB) //TODO: Figure out right place for this
@ -105,7 +102,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
err = myDB.InitTxCounts()
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
// err = dbpkg.InitHeaders(myDB)
@ -113,7 +110,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
// return nil, nil, nil, err
// }
return myDB, records, nil
return myDB, records, toDefer, nil
}
// OpenAndFillTmpDBCF opens a db and fills it with data from a csv file
@ -244,18 +241,17 @@ func CatCSV(filePath string) {
func TestCatFullDB(t *testing.T) {
t.Skip("Skipping full db test")
grp := stop.New()
// url := "lbry://@lothrop#2/lothrop-livestream-games-and-code#c"
// "lbry://@lbry", "lbry://@lbry#3", "lbry://@lbry3f", "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a", "lbry://@lbry:1", "lbry://@lbry$1"
// url := "lbry://@Styxhexenhammer666#2/legacy-media-baron-les-moonves-(cbs#9"
// url := "lbry://@lbry"
// url := "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a"
dbPath := "/mnt/sda1/wallet_server/_data/lbry-rocksdb/"
dbPath := "/mnt/sda/wallet_server/_data/lbry-rocksdb/"
// dbPath := "/mnt/d/data/snapshot_1072108/lbry-rocksdb/"
secondaryPath := "asdf"
db, err := dbpkg.GetProdDB(dbPath, secondaryPath, grp)
defer db.Shutdown()
db, toDefer, err := dbpkg.GetProdDB(dbPath, secondaryPath)
defer toDefer()
if err != nil {
t.Error(err)
return
@ -275,7 +271,6 @@ func TestCatFullDB(t *testing.T) {
// TestOpenFullDB Tests running a resolve on a full db.
func TestOpenFullDB(t *testing.T) {
t.Skip("Skipping full db test")
grp := stop.New()
// url := "lbry://@lothrop#2/lothrop-livestream-games-and-code#c"
// "lbry://@lbry", "lbry://@lbry#3", "lbry://@lbry3f", "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a", "lbry://@lbry:1", "lbry://@lbry$1"
// url := "lbry://@Styxhexenhammer666#2/legacy-media-baron-les-moonves-(cbs#9"
@ -283,11 +278,11 @@ func TestOpenFullDB(t *testing.T) {
// url := "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a"
// url := "lbry://@lbry$1"
url := "https://lbry.tv/@lothrop:2/lothrop-livestream-games-and-code:c"
dbPath := "/mnt/sda1/wallet_server/_data/lbry-rocksdb/"
dbPath := "/mnt/sda/wallet_server/_data/lbry-rocksdb/"
// dbPath := "/mnt/d/data/snapshot_1072108/lbry-rocksdb/"
secondaryPath := "asdf"
db, err := dbpkg.GetProdDB(dbPath, secondaryPath, grp)
defer db.Shutdown()
db, toDefer, err := dbpkg.GetProdDB(dbPath, secondaryPath)
defer toDefer()
if err != nil {
t.Error(err)
return
@ -301,13 +296,12 @@ func TestOpenFullDB(t *testing.T) {
func TestResolve(t *testing.T) {
url := "lbry://@Styxhexenhammer666#2/legacy-media-baron-les-moonves-(cbs#9"
filePath := "../testdata/FULL_resolve.csv"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
return
}
defer toDefer()
expandedResolveResult := db.Resolve(url)
log.Printf("%#v\n", expandedResolveResult)
if expandedResolveResult != nil && expandedResolveResult.Channel != nil {
@ -321,11 +315,11 @@ func TestResolve(t *testing.T) {
func TestGetDBState(t *testing.T) {
filePath := "../testdata/s_resolve.csv"
want := uint32(1072108)
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
state, err := db.GetDBState()
if err != nil {
t.Error(err)
@ -343,11 +337,11 @@ func TestGetRepostedClaim(t *testing.T) {
// Should be non-existent
channelHash2, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bf")
filePath := "../testdata/W_resolve.csv"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
count, err := db.GetRepostedCount(channelHash)
if err != nil {
@ -376,11 +370,11 @@ func TestGetRepostedCount(t *testing.T) {
// Should be non-existent
channelHash2, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bf")
filePath := "../testdata/j_resolve.csv"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
count, err := db.GetRepostedCount(channelHash)
if err != nil {
@ -413,11 +407,11 @@ func TestGetRepost(t *testing.T) {
channelHash2, _ := hex.DecodeString("000009ca6e0caaaef16872b4bd4f6f1b8c2363e2")
filePath := "../testdata/V_resolve.csv"
// want := uint32(3670)
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
res, err := db.GetRepost(channelHash)
if err != nil {
@ -447,12 +441,11 @@ func TestGetClaimsInChannelCount(t *testing.T) {
channelHash, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bd")
filePath := "../testdata/Z_resolve.csv"
want := uint32(3670)
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
count, err := db.GetClaimsInChannelCount(channelHash)
if err != nil {
t.Error(err)
@ -477,12 +470,11 @@ func TestGetShortClaimIdUrl(t *testing.T) {
var position uint16 = 0
filePath := "../testdata/F_resolve.csv"
log.Println(filePath)
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
shortUrl, err := db.GetShortClaimIdUrl(name, normalName, claimHash, rootTxNum, position)
if err != nil {
t.Error(err)
@ -496,11 +488,11 @@ func TestClaimShortIdIter(t *testing.T) {
filePath := "../testdata/F_cat.csv"
normalName := "cat"
claimId := "0"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
ch := db.ClaimShortIdIter(normalName, claimId)
@ -526,12 +518,11 @@ func TestGetTXOToClaim(t *testing.T) {
var txNum uint32 = 1456296
var position uint16 = 0
filePath := "../testdata/G_2.csv"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
val, err := db.GetCachedClaimHash(txNum, position)
if err != nil {
t.Error(err)
@ -555,11 +546,11 @@ func TestGetClaimToChannel(t *testing.T) {
var val []byte = nil
filePath := "../testdata/I_resolve.csv"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
val, err = db.GetChannelForClaim(claimHash, txNum, position)
if err != nil {
@ -580,68 +571,47 @@ func TestGetClaimToChannel(t *testing.T) {
}
func TestGetEffectiveAmountSupportOnly(t *testing.T) {
filePath := "../testdata/Si_resolve.csv"
want := uint64(20000006)
claimHashStr := "00000324e40fcb63a0b517a3660645e9bd99244a"
filePath := "../testdata/S_resolve.csv"
want := uint64(78999149300)
claimHashStr := "2556ed1cab9d17f2a9392030a9ad7f5d138f11bd"
claimHash, _ := hex.DecodeString(claimHashStr)
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
db.Height = 999999999
defer toDefer()
db.Height = 1116054
amount, err := db.GetEffectiveAmount(claimHash, true)
if err != nil {
t.Error(err)
}
if amount != want {
t.Errorf("Expected %d, got %d", want, amount)
}
// Cross-check against iterator-based implementation.
iteratorAmount, err := db.GetActiveAmount(claimHash, prefixes.ActivatedSupportTXOType, db.Height)
if err != nil {
t.Error(err)
}
if iteratorAmount != want {
t.Errorf("Expected %d, got %d", want, iteratorAmount)
}
}
func TestGetEffectiveAmount(t *testing.T) {
filePath := "../testdata/Si_resolve.csv"
want := uint64(21000006)
claimHashStr := "00000324e40fcb63a0b517a3660645e9bd99244a"
filePath := "../testdata/i_resolve.csv"
want := uint64(507171810600)
claimHashStr := "2556ed1cab9d17f2a9392030a9ad7f5d138f11bd"
claimHash, _ := hex.DecodeString(claimHashStr)
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
db.Height = 999999999
defer toDefer()
db.Height = 1116054
amount, err := db.GetEffectiveAmount(claimHash, false)
if err != nil {
t.Error(err)
}
if amount != want {
t.Errorf("Expected %d, got %d", want, amount)
}
// Cross-check against iterator-based implementation.
iteratorAmount1, err := db.GetActiveAmount(claimHash, prefixes.ActivatedSupportTXOType, db.Height)
if err != nil {
t.Error(err)
}
iteratorAmount2, err := db.GetActiveAmount(claimHash, prefixes.ActivateClaimTXOType, db.Height)
if err != nil {
t.Error(err)
}
if iteratorAmount1+iteratorAmount2 != want {
t.Errorf("Expected %d, got %d (%d + %d)", want, iteratorAmount1+iteratorAmount2, iteratorAmount1, iteratorAmount2)
}
}
func TestGetSupportAmount(t *testing.T) {
@ -652,12 +622,11 @@ func TestGetSupportAmount(t *testing.T) {
t.Error(err)
}
filePath := "../testdata/a_resolve.csv"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
res, err := db.GetSupportAmount(claimHash)
if err != nil {
t.Error(err)
@ -673,12 +642,11 @@ func TestGetTxHash(t *testing.T) {
want := "54e14ff0c404c29b3d39ae4d249435f167d5cd4ce5a428ecb745b3df1c8e3dde"
filePath := "../testdata/X_resolve.csv"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
resHash, err := db.GetTxHash(txNum)
if err != nil {
t.Error(err)
@ -716,12 +684,11 @@ func TestGetActivation(t *testing.T) {
txNum := uint32(0x6284e3)
position := uint16(0x0)
want := uint32(0xa6b65)
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
}
defer toDefer()
activation, err := db.GetActivation(txNum, position)
if err != nil {
t.Error(err)
@ -748,13 +715,12 @@ func TestGetClaimToTXO(t *testing.T) {
return
}
filePath := "../testdata/E_resolve.csv"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
return
}
defer toDefer()
res, err := db.GetCachedClaimTxo(claimHash, true)
if err != nil {
t.Error(err)
@ -778,13 +744,12 @@ func TestGetControllingClaim(t *testing.T) {
claimName := internal.NormalizeName("@Styxhexenhammer666")
claimHash := "2556ed1cab9d17f2a9392030a9ad7f5d138f11bd"
filePath := "../testdata/P_resolve.csv"
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
defer db.Shutdown()
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
if err != nil {
t.Error(err)
return
}
defer toDefer()
res, err := db.GetControllingClaim(claimName)
if err != nil {
t.Error(err)

View file

@ -6,7 +6,6 @@ import (
"bytes"
"github.com/lbryio/herald.go/db/prefixes"
"github.com/lbryio/lbry.go/v3/extras/stop"
"github.com/linxGnu/grocksdb"
log "github.com/sirupsen/logrus"
@ -23,8 +22,6 @@ type IterOptions struct {
IncludeValue bool
RawKey bool
RawValue bool
Grp *stop.Group
// DB *ReadOnlyDBColumnFamily
CfHandle *grocksdb.ColumnFamilyHandle
It *grocksdb.Iterator
Serializer *prefixes.SerializationAPI
@ -43,8 +40,6 @@ func NewIterateOptions() *IterOptions {
IncludeValue: false,
RawKey: false,
RawValue: false,
Grp: nil,
// DB: nil,
CfHandle: nil,
It: nil,
Serializer: prefixes.ProductionAPI,
@ -106,13 +101,6 @@ func (o *IterOptions) WithRawValue(rawValue bool) *IterOptions {
return o
}
func (o *IterOptions) WithDB(db *ReadOnlyDBColumnFamily) *IterOptions {
// o.Grp.AddNamed(1, iterKey)
o.Grp = stop.New(db.Grp)
o.Grp.Add(1)
return o
}
func (o *IterOptions) WithSerializer(serializer *prefixes.SerializationAPI) *IterOptions {
o.Serializer = serializer
return o

View file

@ -7,7 +7,6 @@ import (
"strings"
"github.com/go-restruct/restruct"
"github.com/lbryio/herald.go/internal"
"github.com/lbryio/lbcd/chaincfg/chainhash"
)
@ -60,34 +59,6 @@ func (kv *BlockTxsValue) Unpack(buf []byte, order binary.ByteOrder) ([]byte, err
return buf[offset:], nil
}
// Struct BigEndianChainHash is a chainhash.Hash stored in external
// byte-order (opposite of other 32 byte chainhash.Hash values). In order
// to reuse chainhash.Hash we need to correct the byte-order.
// Currently this type is used for field Genesis of DBStateValue.
func (kv *BigEndianChainHash) SizeOf() int {
return chainhash.HashSize
}
func (kv *BigEndianChainHash) Pack(buf []byte, order binary.ByteOrder) ([]byte, error) {
offset := 0
hash := kv.CloneBytes()
// HACK: Instances of chainhash.Hash use the internal byte-order.
// Python scribe writes bytes of genesis hash in external byte-order.
internal.ReverseBytesInPlace(hash)
offset += copy(buf[offset:chainhash.HashSize], hash[:])
return buf[offset:], nil
}
func (kv *BigEndianChainHash) Unpack(buf []byte, order binary.ByteOrder) ([]byte, error) {
offset := 0
offset += copy(kv.Hash[:], buf[offset:32])
// HACK: Instances of chainhash.Hash use the internal byte-order.
// Python scribe writes bytes of genesis hash in external byte-order.
internal.ReverseBytesInPlace(kv.Hash[:])
return buf[offset:], nil
}
func genericNew(prefix []byte, key bool) (interface{}, error) {
t, ok := prefixRegistry[prefix[0]]
if !ok {

View file

@ -12,13 +12,13 @@ import (
"encoding/binary"
"encoding/hex"
"fmt"
"log"
"reflect"
"sort"
"strings"
"github.com/lbryio/herald.go/internal"
"github.com/lbryio/lbcd/chaincfg/chainhash"
log "github.com/sirupsen/logrus"
)
const (
@ -182,25 +182,12 @@ func NewLengthEncodedPartialClaimId(s string) LengthEncodedPartialClaimId {
}
}
type BigEndianChainHash struct {
chainhash.Hash
}
func NewBigEndianChainHash(hash *chainhash.Hash) BigEndianChainHash {
if hash != nil {
return BigEndianChainHash{
*hash,
}
}
return BigEndianChainHash{}
}
type DBStateKey struct {
Prefix []byte `struct:"[1]byte" json:"prefix"`
}
type DBStateValue struct {
Genesis BigEndianChainHash
Genesis *chainhash.Hash
Height uint32
TxCount uint32
Tip *chainhash.Hash
@ -216,7 +203,7 @@ type DBStateValue struct {
func NewDBStateValue() *DBStateValue {
return &DBStateValue{
Genesis: NewBigEndianChainHash(nil),
Genesis: new(chainhash.Hash),
Height: 0,
TxCount: 0,
Tip: new(chainhash.Hash),
@ -250,11 +237,7 @@ func (v *DBStateValue) PackValue() []byte {
// b'>32sLL32sLLBBlllL'
n := 32 + 4 + 4 + 32 + 4 + 4 + 1 + 1 + 4 + 4 + 4 + 4
value := make([]byte, n)
genesis := v.Genesis.CloneBytes()
// HACK: Instances of chainhash.Hash use the internal byte-order.
// Python scribe writes bytes of genesis hash in external byte-order.
internal.ReverseBytesInPlace(genesis)
copy(value, genesis[:32])
copy(value, v.Genesis[:32])
binary.BigEndian.PutUint32(value[32:], v.Height)
binary.BigEndian.PutUint32(value[32+4:], v.TxCount)
copy(value[32+4+4:], v.Tip[:32])
@ -299,11 +282,8 @@ func DBStateKeyUnpack(key []byte) *DBStateKey {
func DBStateValueUnpack(value []byte) *DBStateValue {
genesis := (*chainhash.Hash)(value[:32])
tip := (*chainhash.Hash)(value[32+4+4 : 32+4+4+32])
// HACK: Python scribe writes bytes of genesis hash in external byte-order.
// Instances of chainhash.Hash should use the internal byte-order.
internal.ReverseBytesInPlace(genesis[:])
x := &DBStateValue{
Genesis: NewBigEndianChainHash(genesis),
Genesis: genesis,
Height: binary.BigEndian.Uint32(value[32:]),
TxCount: binary.BigEndian.Uint32(value[32+4:]),
Tip: tip,
@ -728,7 +708,7 @@ type BlockTxsKey struct {
}
type BlockTxsValue struct {
TxHashes []*chainhash.Hash `struct:"*[32]byte" struct-while:"!_eof" json:"tx_hashes"`
TxHashes []*chainhash.Hash `struct-while:"!_eof" json:"tx_hashes"`
}
func (k *BlockTxsKey) NewBlockTxsKey(height uint32) *BlockTxsKey {
@ -1070,6 +1050,84 @@ func TxNumValueUnpack(value []byte) *TxNumValue {
}
}
type TxKey struct {
Prefix []byte `struct:"[1]byte" json:"prefix"`
TxHash *chainhash.Hash `struct:"*[32]byte" json:"tx_hash"`
}
type TxValue struct {
RawTx []byte `struct-while:"!_eof" json:"raw_tx"`
}
func (k *TxKey) PackKey() []byte {
prefixLen := 1
// b'>L'
n := prefixLen + 32
key := make([]byte, n)
copy(key, k.Prefix)
copy(key[prefixLen:], k.TxHash[:32])
return key
}
func (v *TxValue) PackValue() []byte {
value := make([]byte, len(v.RawTx))
copy(value, v.RawTx)
return value
}
func (kv *TxKey) NumFields() int {
return 1
}
func (k *TxKey) PartialPack(fields int) []byte {
// Limit fields between 0 and number of fields, we always at least need
// the prefix, and we never need to iterate past the number of fields.
if fields > 1 {
fields = 1
}
if fields < 0 {
fields = 0
}
prefixLen := 1
var n = prefixLen
for i := 0; i <= fields; i++ {
switch i {
case 1:
n += 32
}
}
key := make([]byte, n)
for i := 0; i <= fields; i++ {
switch i {
case 0:
copy(key, k.Prefix)
case 1:
copy(key[prefixLen:], k.TxHash[:32])
}
}
return key
}
func TxKeyUnpack(key []byte) *TxKey {
prefixLen := 1
return &TxKey{
Prefix: key[:prefixLen],
TxHash: (*chainhash.Hash)(key[prefixLen : prefixLen+32]),
}
}
func TxValueUnpack(value []byte) *TxValue {
return &TxValue{
RawTx: value,
}
}
type BlockHeaderKey struct {
Prefix []byte `struct:"[1]byte" json:"prefix"`
Height uint32 `json:"height"`
@ -3293,12 +3351,9 @@ func (kv *TrendingNotificationValue) UnpackValue(buf []byte) {
offset += 8
}
type TxKey = MempoolTxKey
type TxValue = MempoolTxValue
type MempoolTxKey struct {
Prefix []byte `struct:"[1]byte" json:"prefix"`
TxHash *chainhash.Hash `struct:"*[32]byte" json:"tx_hash"`
TxHash []byte `struct:"[32]byte" json:"tx_hash"`
}
type MempoolTxValue struct {
@ -3331,7 +3386,7 @@ func (kv *MempoolTxKey) UnpackKey(buf []byte) {
offset := 0
kv.Prefix = buf[offset : offset+1]
offset += 1
kv.TxHash = (*chainhash.Hash)(buf[offset : offset+32])
kv.TxHash = buf[offset : offset+32]
offset += 32
}
@ -3413,7 +3468,7 @@ func (kv *TouchedHashXValue) UnpackValue(buf []byte) {
type HashXStatusKey struct {
Prefix []byte `struct:"[1]byte" json:"prefix"`
HashX []byte `struct:"[11]byte" json:"hashX"`
HashX []byte `struct:"[20]byte" json:"hashX"`
}
type HashXStatusValue struct {
@ -3425,15 +3480,15 @@ func (kv *HashXStatusKey) NumFields() int {
}
func (kv *HashXStatusKey) PartialPack(fields int) []byte {
// b'>20s' (really HASHX_LEN 11 bytes)
n := len(kv.Prefix) + 11
// b'>20s'
n := len(kv.Prefix) + 20
buf := make([]byte, n)
offset := 0
offset += copy(buf[offset:], kv.Prefix[:1])
if fields <= 0 {
return buf[:offset]
}
offset += copy(buf[offset:], kv.HashX[:11])
offset += copy(buf[offset:], kv.HashX[:20])
return buf[:offset]
}
@ -3442,12 +3497,12 @@ func (kv *HashXStatusKey) PackKey() []byte {
}
func (kv *HashXStatusKey) UnpackKey(buf []byte) {
// b'>20s' (really HASHX_LEN 11 bytes)
// b'>20s'
offset := 0
kv.Prefix = buf[offset : offset+1]
offset += 1
kv.HashX = buf[offset : offset+11]
offset += 11
kv.HashX = buf[offset : offset+20]
offset += 20
}
func (kv *HashXStatusValue) PackValue() []byte {
@ -3475,8 +3530,7 @@ type EffectiveAmountKey struct {
}
type EffectiveAmountValue struct {
ActivatedSum uint64 `json:"activated_sum"`
ActivatedSupportSum uint64 `json:"activated_support_sum"`
EffectiveAmount uint64 `json:"effective_amount"`
}
func (kv *EffectiveAmountKey) NumFields() int {
@ -3510,23 +3564,19 @@ func (kv *EffectiveAmountKey) UnpackKey(buf []byte) {
}
func (kv *EffectiveAmountValue) PackValue() []byte {
// b'>QQ'
n := 8 + 8
// b'>Q'
n := 8
buf := make([]byte, n)
offset := 0
binary.BigEndian.PutUint64(buf[offset:], kv.ActivatedSum)
offset += 8
binary.BigEndian.PutUint64(buf[offset:], kv.ActivatedSupportSum)
binary.BigEndian.PutUint64(buf[offset:], kv.EffectiveAmount)
offset += 8
return buf[:offset]
}
func (kv *EffectiveAmountValue) UnpackValue(buf []byte) {
// b'>QQ'
// b'>Q'
offset := 0
kv.ActivatedSum = binary.BigEndian.Uint64(buf[offset:])
offset += 8
kv.ActivatedSupportSum = binary.BigEndian.Uint64(buf[offset:])
kv.EffectiveAmount = binary.BigEndian.Uint64(buf[offset:])
offset += 8
}
@ -3870,6 +3920,12 @@ var prefixRegistry = map[byte]prefixMeta{
newValue: func() interface{} {
return &TxValue{}
},
newKeyUnpack: func(buf []byte) interface{} {
return TxKeyUnpack(buf)
},
newValueUnpack: func(buf []byte) interface{} {
return TxValueUnpack(buf)
},
},
BlockHash: {
newKey: func() interface{} {

View file

@ -6,6 +6,7 @@ import (
"encoding/csv"
"encoding/hex"
"fmt"
"log"
"math"
"math/big"
"os"
@ -15,7 +16,6 @@ import (
dbpkg "github.com/lbryio/herald.go/db"
prefixes "github.com/lbryio/herald.go/db/prefixes"
"github.com/linxGnu/grocksdb"
log "github.com/sirupsen/logrus"
)
func TestPrefixRegistry(t *testing.T) {
@ -429,7 +429,7 @@ func TestHashXMempoolStatus(t *testing.T) {
func TestEffectiveAmount(t *testing.T) {
prefix := byte(prefixes.EffectiveAmount)
filePath := fmt.Sprintf("../../testdata/%c.csv", prefix)
//synthesizeTestData([]byte{prefix}, filePath, []int{20}, []int{8, 8}, [][3]int{})
//synthesizeTestData([]byte{prefix}, filePath, []int{20}, []int{8}, [][3]int{})
key := &prefixes.EffectiveAmountKey{}
testGeneric(filePath, prefix, key.NumFields())(t)
}

View file

@ -1,13 +0,0 @@
FROM jeffreypicard/hub-github-env:dev
COPY scripts/integration_tests.sh /integration_tests.sh
COPY scripts/cicd_integration_test_runner.sh /cicd_integration_test_runner.sh
COPY herald /herald
RUN apt install -y jq curl
ENV CGO_LDFLAGS "-L/usr/local/lib -lrocksdb -lstdc++ -lm -lz -lsnappy -llz4 -lzstd"
ENV CGO_CFLAGS "-I/usr/local/include/rocksdb"
ENV LD_LIBRARY_PATH /usr/local/lib
ENTRYPOINT ["/cicd_integration_test_runner.sh"]

7
go.mod
View file

@ -24,11 +24,12 @@ require (
gopkg.in/karalabe/cookiejar.v1 v1.0.0-20141109175019-e1490cae028c
)
require (
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
@ -42,7 +43,7 @@ require (
github.com/prometheus/procfs v0.6.0 // indirect
github.com/stretchr/testify v1.7.0 // indirect
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 // indirect

6
go.sum
View file

@ -68,13 +68,11 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufo
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
@ -360,6 +358,8 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
github.com/lbryio/lbcd v0.22.100-beta/go.mod h1:u8SaFX4xdGMMR5xasBGfgApC8pvD4rnK2OujZnrq5gs=
github.com/lbryio/lbcd v0.22.100-beta-rc5/go.mod h1:9PbFSlHYX7WlnDQwcTxHVf1W35VAnRsattCSyKOO55g=
github.com/lbryio/lbcd v0.22.200-beta/go.mod h1:kNuzGWf808ipTGB0y0WogzsGv5BVM4Qv85Z+JYwC9FA=
github.com/lbryio/lbcd v0.22.201-beta-rc1 h1:FmzzApVj2RBXloLM2w9tLvN2xyTZjeyh+QC7GIw/wwo=
github.com/lbryio/lbcd v0.22.201-beta-rc1/go.mod h1:kNuzGWf808ipTGB0y0WogzsGv5BVM4Qv85Z+JYwC9FA=
github.com/lbryio/lbcd v0.22.201-beta-rc4 h1:Xh751Bh/GWRcP5bI6NJ2+zueo2otTcTWapFvFbryP5c=
github.com/lbryio/lbcd v0.22.201-beta-rc4/go.mod h1:Jgo48JDINhdOgHHR83J70Q6G42x3WAo9DI//QogcL+E=
github.com/lbryio/lbcutil v1.0.201/go.mod h1:gDHc/b+Rdz3J7+VB8e5/Bl9roVf8Q5/8FQCyuK9dXD0=
@ -375,6 +375,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/linxGnu/grocksdb v1.6.42 h1:nJLoXFuzwBwQQQrXTUgRGRz1QRm7y8pR6CNV/gwrbqs=
github.com/linxGnu/grocksdb v1.6.42/go.mod h1:JcMMDBFaDNhRXFYcYXmgQwb/RarSld1PulTI7UzE+w0=
github.com/linxGnu/grocksdb v1.7.0 h1:UyFDykX0CUfxDN10cqlFho/rwt9K6KoDaLXL9Ej5z9g=
github.com/linxGnu/grocksdb v1.7.0/go.mod h1:JcMMDBFaDNhRXFYcYXmgQwb/RarSld1PulTI7UzE+w0=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=

View file

@ -6,5 +6,4 @@ package internal
type HeightHash struct {
Height uint64
BlockHash []byte
BlockHeader []byte
}

27
main.go
View file

@ -10,10 +10,8 @@ import (
"github.com/lbryio/herald.go/internal"
pb "github.com/lbryio/herald.go/protobuf/go"
"github.com/lbryio/herald.go/server"
"github.com/lbryio/lbry.go/v3/extras/stop"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
func main() {
@ -29,22 +27,37 @@ func main() {
if args.CmdType == server.ServeCmd {
// This will cancel goroutines with the server finishes.
stopGroup := stop.New()
ctxWCancel, cancel := context.WithCancel(ctx)
defer cancel()
initsignals()
interrupt := interruptListener()
s := server.MakeHubServer(stopGroup, args)
s := server.MakeHubServer(ctxWCancel, args)
go s.Run()
defer s.Stop()
defer func() {
log.Println("Shutting down server...")
if s.EsClient != nil {
s.EsClient.Stop()
}
if s.GrpcServer != nil {
s.GrpcServer.GracefulStop()
}
if s.DB != nil {
s.DB.Shutdown()
}
log.Println("Returning from main...")
}()
<-interrupt
return
}
conn, err := grpc.Dial("localhost:"+fmt.Sprintf("%d", args.Port),
grpc.WithTransportCredentials(insecure.NewCredentials()),
conn, err := grpc.Dial("localhost:"+args.Port,
grpc.WithInsecure(),
grpc.WithBlock(),
)
if err != nil {

View file

@ -1,14 +0,0 @@
#!/bin/bash
#
# cicd_integration_test_runner.sh
#
# simple script to kick off herald and call the integration testing
# script
#
# N.B. this currently just works locally until we figure a way to have
# the data in the cicd environment.
#
./herald serve --db-path /mnt/sdb1/wallet_server/_data/lbry-rocksdb &
./integration_tests.sh

View file

@ -1,235 +0,0 @@
#!/bin/bash
#
# integration_testing.sh
#
# GitHub Action CI/CD based integration tests for herald.go
# These are smoke / sanity tests for the server behaving correctly on a "live"
# system, and looks for reasonable response codes, not specific correct
# behavior. Those are covered in unit tests.
#
# N.B.
# For the curl based json tests the `id` field existing is needed.
#
# global variables
RES=(0)
FINALRES=0
# functions
function logical_or {
for res in ${RES[@]}; do
if [ $res -eq 1 -o $FINALRES -eq 1 ]; then
FINALRES=1
return
fi
done
}
function want_got {
if [ "${WANT}" != "${GOT}" ]; then
echo "WANT: ${WANT}"
echo "GOT: ${GOT}"
RES+=(1)
else
RES+=(0)
fi
}
function want_greater {
if [ ${WANT} -ge ${GOT} ]; then
echo "WANT: ${WANT}"
echo "GOT: ${GOT}"
RES+=(1)
else
RES+=(0)
fi
}
function test_command_with_want {
echo $CMD
GOT=`eval $CMD`
want_got
}
# grpc endpoint testing
read -r -d '' CMD <<- EOM
grpcurl -plaintext -d '{"value": ["@Styxhexenhammer666:2"]}' 127.0.0.1:50051 pb.Hub.Resolve
| jq .txos[0].txHash | sed 's/"//g'
EOM
WANT="VOFP8MQEwps9Oa5NJJQ18WfVzUzlpCjst0Wz3xyOPd4="
test_command_with_want
# GOT=`eval $CMD`
#want_got
##
## N.B. This is a degenerate case that takes a long time to run.
## The runtime should be fixed, but in the meantime, we definitely should
## ensure this behaves as expected.
##
## TODO: Test runtime doesn't exceed worst case.
##
#WANT=806389
#read -r -d '' CMD <<- EOM
# grpcurl -plaintext -d '{"value": ["foo"]}' 127.0.0.1:50051 pb.Hub.Resolve | jq .txos[0].height
#EOM
# test_command_with_want
# json rpc endpoint testing
## blockchain.block
### blockchain.block.get_chunk
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.block.get_chunk", "params": [0]}'
| jq .result | sed 's/"//g' | head -c 100
EOM
WANT="010000000000000000000000000000000000000000000000000000000000000000000000cc59e59ff97ac092b55e423aa549"
test_command_with_want
### blockchain.block.get_header
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.block.get_header", "params": []}'
| jq .result.timestamp
EOM
WANT=1446058291
test_command_with_want
### blockchain.block.headers
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.block.headers", "params": []}'
| jq .result.count
EOM
WANT=0
test_command_with_want
## blockchain.claimtrie
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.claimtrie.resolve", "params":[{"Data": ["@Styxhexenhammer666:2"]}]}'
| jq .result.txos[0].tx_hash | sed 's/"//g'
EOM
WANT="VOFP8MQEwps9Oa5NJJQ18WfVzUzlpCjst0Wz3xyOPd4="
test_command_with_want
## blockchain.address
### blockchain.address.get_balance
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.address.get_balance", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
| jq .result.confirmed
EOM
WANT=44415602186
test_command_with_want
## blockchain.address.get_history
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.address.get_history", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
| jq '.result.confirmed | length'
EOM
WANT=82
test_command_with_want
## blockchain.address.listunspent
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.address.listunspent", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
| jq '.result | length'
EOM
WANT=32
test_command_with_want
# blockchain.scripthash
## blockchain.scripthash.get_mempool
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.scripthash.get_mempool", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
| jq .error | sed 's/"//g'
EOM
WANT="encoding/hex: invalid byte: U+0047 'G'"
test_command_with_want
## blockchain.scripthash.get_history
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.scripthash.get_history", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
| jq .error | sed 's/"//g'
EOM
WANT="encoding/hex: invalid byte: U+0047 'G'"
test_command_with_want
## blockchain.scripthash.listunspent
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "blockchain.scripthash.listunspent", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
| jq .error | sed 's/"//g'
EOM
WANT="encoding/hex: invalid byte: U+0047 'G'"
test_command_with_want
## server.banner
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "server.banner", "params":[]}'
| jq .result | sed 's/"//g'
EOM
WANT="You are connected to an 0.107.0 server."
test_command_with_want
## server.version
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "server.version", "params":[]}'
| jq .result | sed 's/"//g'
EOM
WANT="0.107.0"
test_command_with_want
## server.features
read -r -d '' CMD <<- EOM
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
--data '{"id": 1, "method": "server.features", "params":[]}'
EOM
WANT='{"result":{"hosts":{},"pruning":"","server_version":"0.107.0","protocol_min":"0.54.0","protocol_max":"0.199.0","genesis_hash":"9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463","description":"Herald","payment_address":"","donation_address":"","daily_fee":"1.0","hash_function":"sha256","trending_algorithm":"fast_ar"},"error":null,"id":1}'
test_command_with_want
# metrics endpoint testing
WANT=0
GOT=$(curl http://127.0.0.1:2112/metrics -s | grep requests | grep resolve | awk '{print $NF}')
want_greater
# caclulate return value
logical_or $RES
if [ $FINALRES -eq 1 ]; then
echo "Failed!"
exit 1
else
echo "Passed!"
exit 0
fi

View file

@ -1,11 +1,8 @@
package server
import (
"fmt"
"log"
"net/url"
"os"
"strconv"
"strings"
"github.com/akamensky/argparse"
@ -23,37 +20,21 @@ const (
type Args struct {
CmdType int
Host string
Port int
Port string
DBPath string
Chain *string
DaemonURL *url.URL
DaemonCAPath string
EsHost string
EsPort int
PrometheusPort int
NotifierPort int
JSONRPCPort int
JSONRPCHTTPPort int
MaxSessions int
SessionTimeout int
EsPort string
PrometheusPort string
NotifierPort string
JSONRPCPort string
EsIndex string
RefreshDelta int
CacheTTL int
PeerFile string
Banner *string
Country string
BlockingChannelIds []string
FilteringChannelIds []string
GenesisHash string
ServerVersion string
ProtocolMin string
ProtocolMax string
ServerDescription string
PaymentAddress string
DonationAddress string
DailyFee string
Debug bool
DisableEs bool
DisableLoadPeers bool
@ -70,31 +51,18 @@ type Args struct {
const (
DefaultHost = "0.0.0.0"
DefaultPort = 50051
DefaultPort = "50051"
DefaultDBPath = "/mnt/d/data/snapshot_1072108/lbry-rocksdb/" // FIXME
DefaultEsHost = "http://localhost"
DefaultEsIndex = "claims"
DefaultEsPort = 9200
DefaultPrometheusPort = 2112
DefaultNotifierPort = 18080
DefaultJSONRPCPort = 50001
DefaultJSONRPCHTTPPort = 50002
DefaultMaxSessions = 10000
DefaultSessionTimeout = 300
DefaultEsPort = "9200"
DefaultPrometheusPort = "2112"
DefaultNotifierPort = "18080"
DefaultJSONRPCPort = "50001"
DefaultRefreshDelta = 5
DefaultCacheTTL = 5
DefaultPeerFile = "peers.txt"
DefaultBannerFile = ""
DefaultCountry = "US"
HUB_PROTOCOL_VERSION = "0.107.0"
PROTOCOL_MIN = "0.54.0"
PROTOCOL_MAX = "0.199.0"
DefaultServerDescription = "Herald"
DefaultPaymentAddress = ""
DefaultDonationAddress = ""
DefaultDailyFee = "1.0"
DefaultDisableLoadPeers = false
DefaultDisableStartPrometheus = false
DefaultDisableStartUDP = false
@ -112,73 +80,6 @@ var (
DefaultFilteringChannelIds = []string{}
)
func loadBanner(bannerFile *string, serverVersion string) *string {
var banner string
data, err := os.ReadFile(*bannerFile)
if err != nil {
banner = fmt.Sprintf("You are connected to an %s server.", serverVersion)
} else {
banner = string(data)
}
/*
banner := os.Getenv("BANNER")
if banner == "" {
return nil
}
*/
return &banner
}
// MakeDefaultArgs creates a default set of arguments for testing the server.
func MakeDefaultTestArgs() *Args {
args := &Args{
CmdType: ServeCmd,
Host: DefaultHost,
Port: DefaultPort,
DBPath: DefaultDBPath,
EsHost: DefaultEsHost,
EsPort: DefaultEsPort,
PrometheusPort: DefaultPrometheusPort,
NotifierPort: DefaultNotifierPort,
JSONRPCPort: DefaultJSONRPCPort,
JSONRPCHTTPPort: DefaultJSONRPCHTTPPort,
MaxSessions: DefaultMaxSessions,
SessionTimeout: DefaultSessionTimeout,
EsIndex: DefaultEsIndex,
RefreshDelta: DefaultRefreshDelta,
CacheTTL: DefaultCacheTTL,
PeerFile: DefaultPeerFile,
Banner: nil,
Country: DefaultCountry,
GenesisHash: chaincfg.TestNet3Params.GenesisHash.String(),
ServerVersion: HUB_PROTOCOL_VERSION,
ProtocolMin: PROTOCOL_MIN,
ProtocolMax: PROTOCOL_MAX,
ServerDescription: DefaultServerDescription,
PaymentAddress: DefaultPaymentAddress,
DonationAddress: DefaultDonationAddress,
DailyFee: DefaultDailyFee,
DisableEs: true,
Debug: true,
DisableLoadPeers: true,
DisableStartPrometheus: true,
DisableStartUDP: true,
DisableWritePeers: true,
DisableRocksDBRefresh: true,
DisableResolve: true,
DisableBlockingAndFiltering: true,
DisableStartNotifier: true,
DisableStartJSONRPC: true,
}
return args
}
// GetEnvironment takes the environment variables as an array of strings
// and a getkeyval function to turn it into a map.
func GetEnvironment(data []string, getkeyval func(item string) (key, val string)) map[string]string {
@ -204,58 +105,30 @@ func GetEnvironmentStandard() map[string]string {
func ParseArgs(searchRequest *pb.SearchRequest) *Args {
environment := GetEnvironmentStandard()
parser := argparse.NewParser("herald", "herald server and client")
parser := argparse.NewParser("hub", "hub server and client")
serveCmd := parser.NewCommand("serve", "start the herald server")
serveCmd := parser.NewCommand("serve", "start the hub server")
searchCmd := parser.NewCommand("search", "claim search")
dbCmd := parser.NewCommand("db", "db testing")
defaultDaemonURL := "http://localhost:9245"
if url, ok := environment["DAEMON_URL"]; ok {
defaultDaemonURL = url
}
validateURL := func(arg []string) error {
_, err := url.Parse(arg[0])
return err
}
validatePort := func(arg []string) error {
_, err := strconv.ParseUint(arg[0], 10, 16)
return err
}
// main server config arguments
host := parser.String("", "rpchost", &argparse.Options{Required: false, Help: "RPC host", Default: DefaultHost})
port := parser.Int("", "rpcport", &argparse.Options{Required: false, Help: "RPC port", Validate: validatePort, Default: DefaultPort})
port := parser.String("", "rpcport", &argparse.Options{Required: false, Help: "RPC port", Default: DefaultPort})
dbPath := parser.String("", "db-path", &argparse.Options{Required: false, Help: "RocksDB path", Default: DefaultDBPath})
chain := parser.Selector("", "chain", []string{chaincfg.MainNetParams.Name, chaincfg.TestNet3Params.Name, chaincfg.RegressionNetParams.Name, "testnet"},
&argparse.Options{Required: false, Help: "Which chain to use, default is 'mainnet'. Values 'regtest' and 'testnet' are for testing", Default: chaincfg.MainNetParams.Name})
daemonURLStr := parser.String("", "daemon-url", &argparse.Options{Required: false, Help: "URL for rpc to lbrycrd or lbcd, <rpcuser>:<rpcpassword>@<lbcd rpc ip><lbrcd rpc port>.", Validate: validateURL, Default: defaultDaemonURL})
daemonCAPath := parser.String("", "daemon-ca-path", &argparse.Options{Required: false, Help: "Path to the lbcd CA file. Use SSL certificate to verify connection to lbcd."})
esHost := parser.String("", "eshost", &argparse.Options{Required: false, Help: "elasticsearch host", Default: DefaultEsHost})
esPort := parser.Int("", "esport", &argparse.Options{Required: false, Help: "elasticsearch port", Default: DefaultEsPort})
prometheusPort := parser.Int("", "prometheus-port", &argparse.Options{Required: false, Help: "prometheus port", Default: DefaultPrometheusPort})
notifierPort := parser.Int("", "notifier-port", &argparse.Options{Required: false, Help: "notifier port", Default: DefaultNotifierPort})
jsonRPCPort := parser.Int("", "json-rpc-port", &argparse.Options{Required: false, Help: "JSON RPC port", Validate: validatePort, Default: DefaultJSONRPCPort})
jsonRPCHTTPPort := parser.Int("", "json-rpc-http-port", &argparse.Options{Required: false, Help: "JSON RPC over HTTP port", Validate: validatePort, Default: DefaultJSONRPCHTTPPort})
maxSessions := parser.Int("", "max-sessions", &argparse.Options{Required: false, Help: "Maximum number of electrum clients that can be connected", Default: DefaultMaxSessions})
sessionTimeout := parser.Int("", "session-timeout", &argparse.Options{Required: false, Help: "Session inactivity timeout (seconds)", Default: DefaultSessionTimeout})
esPort := parser.String("", "esport", &argparse.Options{Required: false, Help: "elasticsearch port", Default: DefaultEsPort})
prometheusPort := parser.String("", "prometheus-port", &argparse.Options{Required: false, Help: "prometheus port", Default: DefaultPrometheusPort})
notifierPort := parser.String("", "notifier-port", &argparse.Options{Required: false, Help: "notifier port", Default: DefaultNotifierPort})
jsonRPCPort := parser.String("", "json-rpc-port", &argparse.Options{Required: false, Help: "JSON RPC port", Default: DefaultJSONRPCPort})
esIndex := parser.String("", "esindex", &argparse.Options{Required: false, Help: "elasticsearch index name", Default: DefaultEsIndex})
refreshDelta := parser.Int("", "refresh-delta", &argparse.Options{Required: false, Help: "elasticsearch index refresh delta in seconds", Default: DefaultRefreshDelta})
cacheTTL := parser.Int("", "cachettl", &argparse.Options{Required: false, Help: "Cache TTL in minutes", Default: DefaultCacheTTL})
peerFile := parser.String("", "peerfile", &argparse.Options{Required: false, Help: "Initial peer file for federation", Default: DefaultPeerFile})
bannerFile := parser.String("", "bannerfile", &argparse.Options{Required: false, Help: "Banner file server.banner", Default: DefaultBannerFile})
country := parser.String("", "country", &argparse.Options{Required: false, Help: "Country this node is running in. Default US.", Default: DefaultCountry})
blockingChannelIds := parser.StringList("", "blocking-channel-ids", &argparse.Options{Required: false, Help: "Blocking channel ids", Default: DefaultBlockingChannelIds})
filteringChannelIds := parser.StringList("", "filtering-channel-ids", &argparse.Options{Required: false, Help: "Filtering channel ids", Default: DefaultFilteringChannelIds})
// arguments for server features
serverDescription := parser.String("", "server-description", &argparse.Options{Required: false, Help: "Server description", Default: DefaultServerDescription})
paymentAddress := parser.String("", "payment-address", &argparse.Options{Required: false, Help: "Payment address", Default: DefaultPaymentAddress})
donationAddress := parser.String("", "donation-address", &argparse.Options{Required: false, Help: "Donation address", Default: DefaultDonationAddress})
dailyFee := parser.String("", "daily-fee", &argparse.Options{Required: false, Help: "Daily fee", Default: DefaultDailyFee})
// flags for disabling features
debug := parser.Flag("", "debug", &argparse.Options{Required: false, Help: "enable debug logging", Default: false})
disableEs := parser.Flag("", "disable-es", &argparse.Options{Required: false, Help: "Disable elastic search, for running/testing independently", Default: false})
disableLoadPeers := parser.Flag("", "disable-load-peers", &argparse.Options{Required: false, Help: "Disable load peers from disk at startup", Default: DefaultDisableLoadPeers})
@ -269,7 +142,6 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
disableStartNotifier := parser.Flag("", "disable-start-notifier", &argparse.Options{Required: false, Help: "Disable start notifier", Default: DisableStartNotifier})
disableStartJSONRPC := parser.Flag("", "disable-start-jsonrpc", &argparse.Options{Required: false, Help: "Disable start jsonrpc endpoint", Default: DisableStartJSONRPC})
// search command arguments
text := parser.String("", "text", &argparse.Options{Required: false, Help: "text query"})
name := parser.String("", "name", &argparse.Options{Required: false, Help: "name"})
claimType := parser.String("", "claim_type", &argparse.Options{Required: false, Help: "claim_type"})
@ -286,52 +158,24 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
log.Fatalln(parser.Usage(err))
}
// Use default JSON RPC port only if *neither* JSON RPC arg is specified.
if *jsonRPCPort == 0 && *jsonRPCHTTPPort == 0 {
*jsonRPCPort = DefaultJSONRPCPort
}
daemonURL, err := url.Parse(*daemonURLStr)
if err != nil {
log.Fatalf("URL parse failed: %v", err)
}
banner := loadBanner(bannerFile, HUB_PROTOCOL_VERSION)
args := &Args{
CmdType: SearchCmd,
Host: *host,
Port: *port,
DBPath: *dbPath,
Chain: chain,
DaemonURL: daemonURL,
DaemonCAPath: *daemonCAPath,
EsHost: *esHost,
EsPort: *esPort,
PrometheusPort: *prometheusPort,
NotifierPort: *notifierPort,
JSONRPCPort: *jsonRPCPort,
JSONRPCHTTPPort: *jsonRPCHTTPPort,
MaxSessions: *maxSessions,
SessionTimeout: *sessionTimeout,
EsIndex: *esIndex,
RefreshDelta: *refreshDelta,
CacheTTL: *cacheTTL,
PeerFile: *peerFile,
Banner: banner,
Country: *country,
BlockingChannelIds: *blockingChannelIds,
FilteringChannelIds: *filteringChannelIds,
GenesisHash: "",
ServerVersion: HUB_PROTOCOL_VERSION,
ProtocolMin: PROTOCOL_MIN,
ProtocolMax: PROTOCOL_MAX,
ServerDescription: *serverDescription,
PaymentAddress: *paymentAddress,
DonationAddress: *donationAddress,
DailyFee: *dailyFee,
Debug: *debug,
DisableEs: *disableEs,
DisableLoadPeers: *disableLoadPeers,
@ -355,17 +199,11 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
}
if esPort, ok := environment["ELASTIC_PORT"]; ok {
args.EsPort, err = strconv.Atoi(esPort)
if err != nil {
log.Fatal(err)
}
args.EsPort = esPort
}
if prometheusPort, ok := environment["GOHUB_PROMETHEUS_PORT"]; ok {
args.PrometheusPort, err = strconv.Atoi(prometheusPort)
if err != nil {
log.Fatal(err)
}
args.PrometheusPort = prometheusPort
}
/*

View file

@ -3,19 +3,17 @@ package server
import (
"bufio"
"context"
"log"
"math"
"net"
"os"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/lbryio/herald.go/internal/metrics"
pb "github.com/lbryio/herald.go/protobuf/go"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// Peer holds relevant information about peers that we know about.
@ -88,7 +86,7 @@ func (s *Server) getAndSetExternalIp(ip, port string) error {
// storing them as known peers. Returns a map of peerKey -> object
func (s *Server) loadPeers() error {
peerFile := s.Args.PeerFile
port := strconv.Itoa(s.Args.Port)
port := s.Args.Port
// First we make sure our server has come up, so we can answer back to peers.
var failures = 0
@ -100,7 +98,7 @@ retry:
time.Sleep(time.Second * time.Duration(math.Pow(float64(failures), 2)))
conn, err := grpc.DialContext(ctx,
"0.0.0.0:"+port,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithInsecure(),
grpc.WithBlock(),
)
@ -173,7 +171,7 @@ func (s *Server) subscribeToPeer(peer *Peer) error {
conn, err := grpc.DialContext(ctx,
peer.Address+":"+peer.Port,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithInsecure(),
grpc.WithBlock(),
)
if err != nil {
@ -183,12 +181,12 @@ func (s *Server) subscribeToPeer(peer *Peer) error {
msg := &pb.ServerMessage{
Address: s.ExternalIP.String(),
Port: strconv.Itoa(s.Args.Port),
Port: s.Args.Port,
}
c := pb.NewHubClient(conn)
log.Printf("%s:%d subscribing to %+v\n", s.ExternalIP, s.Args.Port, peer)
log.Printf("%s:%s subscribing to %+v\n", s.ExternalIP, s.Args.Port, peer)
_, err = c.PeerSubscribe(ctx, msg)
if err != nil {
return err
@ -209,7 +207,7 @@ func (s *Server) helloPeer(peer *Peer) (*pb.HelloMessage, error) {
conn, err := grpc.DialContext(ctx,
peer.Address+":"+peer.Port,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithInsecure(),
grpc.WithBlock(),
)
if err != nil {
@ -221,12 +219,12 @@ func (s *Server) helloPeer(peer *Peer) (*pb.HelloMessage, error) {
c := pb.NewHubClient(conn)
msg := &pb.HelloMessage{
Port: strconv.Itoa(s.Args.Port),
Port: s.Args.Port,
Host: s.ExternalIP.String(),
Servers: []*pb.ServerMessage{},
}
log.Printf("%s:%d saying hello to %+v\n", s.ExternalIP, s.Args.Port, peer)
log.Printf("%s:%s saying hello to %+v\n", s.ExternalIP, s.Args.Port, peer)
res, err := c.Hello(ctx, msg)
if err != nil {
log.Println(err)
@ -279,7 +277,7 @@ func (s *Server) notifyPeer(peerToNotify *Peer, newPeer *Peer) error {
conn, err := grpc.DialContext(ctx,
peerToNotify.Address+":"+peerToNotify.Port,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithInsecure(),
grpc.WithBlock(),
)
if err != nil {
@ -347,15 +345,15 @@ func (s *Server) addPeer(newPeer *Peer, ping bool, subscribe bool) error {
}
}
if strconv.Itoa(s.Args.Port) == newPeer.Port &&
if s.Args.Port == newPeer.Port &&
(localHosts[newPeer.Address] || newPeer.Address == s.ExternalIP.String()) {
log.Printf("%s:%d addPeer: Self peer, skipping...\n", s.ExternalIP, s.Args.Port)
log.Printf("%s:%s addPeer: Self peer, skipping...\n", s.ExternalIP, s.Args.Port)
return nil
}
k := peerKey(newPeer)
log.Printf("%s:%d adding peer %+v\n", s.ExternalIP, s.Args.Port, newPeer)
log.Printf("%s:%s adding peer %+v\n", s.ExternalIP, s.Args.Port, newPeer)
if oldServer, loaded := s.PeerServersLoadOrStore(newPeer); !loaded {
if ping {
_, err := s.helloPeer(newPeer)
@ -371,10 +369,6 @@ func (s *Server) addPeer(newPeer *Peer, ping bool, subscribe bool) error {
metrics.PeersKnown.Inc()
s.writePeers()
s.notifyPeerSubs(newPeer)
// This is weird because we're doing grpc and jsonrpc here.
// Do we still want to custom grpc?
log.Warn("Sending peer to NotifierChan")
s.NotifierChan <- peerNotification{newPeer.Address, newPeer.Port}
// Subscribe to all our peers for now
if subscribe {
@ -421,7 +415,7 @@ func (s *Server) makeHelloMessage() *pb.HelloMessage {
s.PeerServersMut.RUnlock()
return &pb.HelloMessage{
Port: strconv.Itoa(s.Args.Port),
Port: s.Args.Port,
Host: s.ExternalIP.String(),
Servers: servers,
}

View file

@ -4,31 +4,23 @@ import (
"bufio"
"context"
"fmt"
"log"
"net"
"os"
"strconv"
"strings"
"testing"
"github.com/lbryio/herald.go/internal/metrics"
pb "github.com/lbryio/herald.go/protobuf/go"
"github.com/lbryio/herald.go/server"
"github.com/lbryio/lbry.go/v3/extras/stop"
server "github.com/lbryio/herald.go/server"
dto "github.com/prometheus/client_model/go"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// lineCountFile takes a fileName and counts the number of lines in it.
func lineCountFile(fileName string) int {
f, err := os.Open(fileName)
defer func() {
err := f.Close()
if err != nil {
log.Warn(err)
}
}()
defer f.Close()
if err != nil {
log.Println(err)
return 0
@ -52,12 +44,43 @@ func removeFile(fileName string) {
}
}
// makeDefaultArgs creates a default set of arguments for testing the server.
func makeDefaultArgs() *server.Args {
args := &server.Args{
CmdType: server.ServeCmd,
Host: server.DefaultHost,
Port: server.DefaultPort,
DBPath: server.DefaultDBPath,
EsHost: server.DefaultEsHost,
EsPort: server.DefaultEsPort,
PrometheusPort: server.DefaultPrometheusPort,
NotifierPort: server.DefaultNotifierPort,
JSONRPCPort: server.DefaultJSONRPCPort,
EsIndex: server.DefaultEsIndex,
RefreshDelta: server.DefaultRefreshDelta,
CacheTTL: server.DefaultCacheTTL,
PeerFile: server.DefaultPeerFile,
Country: server.DefaultCountry,
DisableEs: true,
Debug: true,
DisableLoadPeers: true,
DisableStartPrometheus: true,
DisableStartUDP: true,
DisableWritePeers: true,
DisableRocksDBRefresh: true,
DisableResolve: true,
DisableBlockingAndFiltering: true,
DisableStartNotifier: true,
DisableStartJSONRPC: true,
}
return args
}
// TestAddPeer tests the ability to add peers
func TestAddPeer(t *testing.T) {
// ctx := context.Background()
ctx := stop.NewDebug()
args := server.MakeDefaultTestArgs()
args.DisableStartNotifier = false
ctx := context.Background()
args := makeDefaultArgs()
tests := []struct {
name string
@ -107,7 +130,6 @@ func TestAddPeer(t *testing.T) {
if got != tt.want {
t.Errorf("len(server.PeerServers) = %d, want %d\n", got, tt.want)
}
hubServer.Stop()
})
}
@ -115,10 +137,9 @@ func TestAddPeer(t *testing.T) {
// TestPeerWriter tests that peers get written properly
func TestPeerWriter(t *testing.T) {
ctx := stop.NewDebug()
args := server.MakeDefaultTestArgs()
ctx := context.Background()
args := makeDefaultArgs()
args.DisableWritePeers = false
args.DisableStartNotifier = false
tests := []struct {
name string
@ -153,16 +174,17 @@ func TestPeerWriter(t *testing.T) {
Port: "50051",
}
}
//log.Printf("Adding peer %+v\n", peer)
err := hubServer.AddPeerExported()(peer, false, false)
if err != nil {
log.Println(err)
}
}
//log.Println("Counting lines...")
got := lineCountFile(hubServer.Args.PeerFile)
if got != tt.want {
t.Errorf("lineCountFile(peers.txt) = %d, want %d", got, tt.want)
}
hubServer.Stop()
})
}
@ -171,13 +193,10 @@ func TestPeerWriter(t *testing.T) {
// TestAddPeerEndpoint tests the ability to add peers
func TestAddPeerEndpoint(t *testing.T) {
ctx := stop.NewDebug()
args := server.MakeDefaultTestArgs()
args.DisableStartNotifier = false
args2 := server.MakeDefaultTestArgs()
args2.DisableStartNotifier = false
args2.Port = 50052
args2.NotifierPort = 18081
ctx := context.Background()
args := makeDefaultArgs()
args2 := makeDefaultArgs()
args2.Port = "50052"
tests := []struct {
name string
@ -207,8 +226,9 @@ func TestAddPeerEndpoint(t *testing.T) {
metrics.PeersKnown.Set(0)
go hubServer.Run()
go hubServer2.Run()
conn, err := grpc.Dial("localhost:"+strconv.Itoa(args.Port),
grpc.WithTransportCredentials(insecure.NewCredentials()),
//go hubServer.Run()
conn, err := grpc.Dial("localhost:"+args.Port,
grpc.WithInsecure(),
grpc.WithBlock(),
)
if err != nil {
@ -227,6 +247,8 @@ func TestAddPeerEndpoint(t *testing.T) {
log.Println(err)
}
hubServer.GrpcServer.GracefulStop()
hubServer2.GrpcServer.GracefulStop()
got1 := hubServer.GetNumPeersExported()()
got2 := hubServer2.GetNumPeersExported()()
if got1 != tt.wantServerOne {
@ -235,8 +257,6 @@ func TestAddPeerEndpoint(t *testing.T) {
if got2 != tt.wantServerTwo {
t.Errorf("len(hubServer2.PeerServers) = %d, want %d\n", got2, tt.wantServerTwo)
}
hubServer.Stop()
hubServer2.Stop()
})
}
@ -244,17 +264,12 @@ func TestAddPeerEndpoint(t *testing.T) {
// TestAddPeerEndpoint2 tests the ability to add peers
func TestAddPeerEndpoint2(t *testing.T) {
ctx := stop.NewDebug()
args := server.MakeDefaultTestArgs()
args2 := server.MakeDefaultTestArgs()
args3 := server.MakeDefaultTestArgs()
args2.Port = 50052
args3.Port = 50053
args.DisableStartNotifier = false
args2.DisableStartNotifier = false
args3.DisableStartNotifier = false
args2.NotifierPort = 18081
args3.NotifierPort = 18082
ctx := context.Background()
args := makeDefaultArgs()
args2 := makeDefaultArgs()
args3 := makeDefaultArgs()
args2.Port = "50052"
args3.Port = "50053"
tests := []struct {
name string
@ -279,8 +294,8 @@ func TestAddPeerEndpoint2(t *testing.T) {
go hubServer.Run()
go hubServer2.Run()
go hubServer3.Run()
conn, err := grpc.Dial("localhost:"+strconv.Itoa(args.Port),
grpc.WithTransportCredentials(insecure.NewCredentials()),
conn, err := grpc.Dial("localhost:"+args.Port,
grpc.WithInsecure(),
grpc.WithBlock(),
)
if err != nil {
@ -308,6 +323,9 @@ func TestAddPeerEndpoint2(t *testing.T) {
log.Println(err)
}
hubServer.GrpcServer.GracefulStop()
hubServer2.GrpcServer.GracefulStop()
hubServer3.GrpcServer.GracefulStop()
got1 := hubServer.GetNumPeersExported()()
got2 := hubServer2.GetNumPeersExported()()
got3 := hubServer3.GetNumPeersExported()()
@ -320,9 +338,6 @@ func TestAddPeerEndpoint2(t *testing.T) {
if got3 != tt.wantServerThree {
t.Errorf("len(hubServer3.PeerServers) = %d, want %d\n", got3, tt.wantServerThree)
}
hubServer.Stop()
hubServer2.Stop()
hubServer3.Stop()
})
}
@ -330,17 +345,12 @@ func TestAddPeerEndpoint2(t *testing.T) {
// TestAddPeerEndpoint3 tests the ability to add peers
func TestAddPeerEndpoint3(t *testing.T) {
ctx := stop.NewDebug()
args := server.MakeDefaultTestArgs()
args2 := server.MakeDefaultTestArgs()
args3 := server.MakeDefaultTestArgs()
args2.Port = 50052
args3.Port = 50053
args.DisableStartNotifier = false
args2.DisableStartNotifier = false
args3.DisableStartNotifier = false
args2.NotifierPort = 18081
args3.NotifierPort = 18082
ctx := context.Background()
args := makeDefaultArgs()
args2 := makeDefaultArgs()
args3 := makeDefaultArgs()
args2.Port = "50052"
args3.Port = "50053"
tests := []struct {
name string
@ -365,15 +375,15 @@ func TestAddPeerEndpoint3(t *testing.T) {
go hubServer.Run()
go hubServer2.Run()
go hubServer3.Run()
conn, err := grpc.Dial("localhost:"+strconv.Itoa(args.Port),
grpc.WithTransportCredentials(insecure.NewCredentials()),
conn, err := grpc.Dial("localhost:"+args.Port,
grpc.WithInsecure(),
grpc.WithBlock(),
)
if err != nil {
log.Fatalf("did not connect: %v", err)
}
conn2, err := grpc.Dial("localhost:50052",
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithInsecure(),
grpc.WithBlock(),
)
if err != nil {
@ -402,9 +412,9 @@ func TestAddPeerEndpoint3(t *testing.T) {
log.Println(err)
}
hubServer.Stop()
hubServer2.Stop()
hubServer3.Stop()
hubServer.GrpcServer.GracefulStop()
hubServer2.GrpcServer.GracefulStop()
hubServer3.GrpcServer.GracefulStop()
got1 := hubServer.GetNumPeersExported()()
got2 := hubServer2.GetNumPeersExported()()
got3 := hubServer3.GetNumPeersExported()()
@ -424,11 +434,11 @@ func TestAddPeerEndpoint3(t *testing.T) {
// TestAddPeer tests the ability to add peers
func TestUDPServer(t *testing.T) {
ctx := stop.NewDebug()
args := server.MakeDefaultTestArgs()
args2 := server.MakeDefaultTestArgs()
args2.Port = 50052
ctx := context.Background()
args := makeDefaultArgs()
args.DisableStartUDP = false
args2 := makeDefaultArgs()
args2.Port = "50052"
args2.DisableStartUDP = false
tests := []struct {
@ -459,18 +469,18 @@ func TestUDPServer(t *testing.T) {
log.Println(err)
}
hubServer.Stop()
hubServer2.Stop()
hubServer.GrpcServer.GracefulStop()
hubServer2.GrpcServer.GracefulStop()
got1 := hubServer.ExternalIP.String()
if got1 != tt.want {
t.Errorf("hubServer.ExternalIP = %s, want %s\n", got1, tt.want)
t.Errorf("hubServer.Args.Port = %d\n", hubServer.Args.Port)
t.Errorf("hubServer.Args.Port = %s\n", hubServer.Args.Port)
}
got2 := hubServer2.ExternalIP.String()
if got2 != tt.want {
t.Errorf("hubServer2.ExternalIP = %s, want %s\n", got2, tt.want)
t.Errorf("hubServer2.Args.Port = %d\n", hubServer2.Args.Port)
t.Errorf("hubServer2.Args.Port = %s\n", hubServer2.Args.Port)
}
})
}

View file

@ -7,10 +7,12 @@ import (
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"github.com/gorilla/rpc"
"github.com/lbryio/herald.go/db"
"github.com/lbryio/herald.go/internal"
"github.com/lbryio/lbcd/chaincfg"
@ -19,49 +21,58 @@ import (
"github.com/lbryio/lbcd/wire"
"github.com/lbryio/lbcutil"
"golang.org/x/exp/constraints"
log "github.com/sirupsen/logrus"
)
// BlockchainBlockService methods handle "blockchain.block.*" RPCs
type BlockchainBlockService struct {
DB *db.ReadOnlyDBColumnFamily
Chain *chaincfg.Params
type BlockchainCodec struct {
rpc.Codec
}
// BlockchainBlockService methods handle "blockchain.headers.*" RPCs
type BlockchainHeadersService struct {
func (c *BlockchainCodec) NewRequest(r *http.Request) rpc.CodecRequest {
return &BlockchainCodecRequest{c.Codec.NewRequest(r)}
}
// BlockchainCodecRequest provides ability to rewrite the incoming
// request "method" field. For example:
// blockchain.block.get_header -> blockchain_block.Get_header
// blockchain.address.listunspent -> blockchain_address.Listunspent
// This makes the "method" string compatible with Gorilla/RPC
// requirements.
type BlockchainCodecRequest struct {
rpc.CodecRequest
}
func (cr *BlockchainCodecRequest) Method() (string, error) {
rawMethod, err := cr.CodecRequest.Method()
if err != nil {
return rawMethod, err
}
parts := strings.Split(rawMethod, ".")
if len(parts) < 2 {
return rawMethod, fmt.Errorf("blockchain rpc: service/method ill-formed: %q", rawMethod)
}
service := strings.Join(parts[0:len(parts)-1], "_")
method := parts[len(parts)-1]
if len(method) < 1 {
return rawMethod, fmt.Errorf("blockchain rpc: method ill-formed: %q", method)
}
method = strings.ToUpper(string(method[0])) + string(method[1:])
return service + "." + method, err
}
// BlockchainService methods handle "blockchain.block.*" RPCs
type BlockchainService struct {
DB *db.ReadOnlyDBColumnFamily
Chain *chaincfg.Params
// needed for subscribe/unsubscribe
sessionMgr *sessionManager
session *session
}
// BlockchainAddressService methods handle "blockchain.address.*" RPCs
type BlockchainAddressService struct {
DB *db.ReadOnlyDBColumnFamily
Chain *chaincfg.Params
// needed for subscribe/unsubscribe
sessionMgr *sessionManager
session *session
BlockchainService
}
// BlockchainScripthashService methods handle "blockchain.scripthash.*" RPCs
type BlockchainScripthashService struct {
DB *db.ReadOnlyDBColumnFamily
Chain *chaincfg.Params
// needed for subscribe/unsubscribe
sessionMgr *sessionManager
session *session
}
// BlockchainTransactionService methods handle "blockchain.transaction.*" RPCs
type BlockchainTransactionService struct {
DB *db.ReadOnlyDBColumnFamily
Chain *chaincfg.Params
// needed for broadcast TX
sessionMgr *sessionManager
BlockchainService
}
const CHUNK_SIZE = 96
@ -76,46 +87,10 @@ func min[Ord constraints.Ordered](x, y Ord) Ord {
return y
}
func max[Ord constraints.Ordered](x, y Ord) Ord {
if x > y {
return x
}
return y
}
type BlockHeaderElectrum struct {
Version uint32 `json:"version"`
PrevBlockHash string `json:"prev_block_hash"`
MerkleRoot string `json:"merkle_root"`
ClaimTrieRoot string `json:"claim_trie_root"`
Timestamp uint32 `json:"timestamp"`
Bits uint32 `json:"bits"`
Nonce uint32 `json:"nonce"`
BlockHeight uint32 `json:"block_height"`
}
func newBlockHeaderElectrum(header *[HEADER_SIZE]byte, height uint32) *BlockHeaderElectrum {
var h1, h2, h3 chainhash.Hash
h1.SetBytes(header[4:36])
h2.SetBytes(header[36:68])
h3.SetBytes(header[68:100])
return &BlockHeaderElectrum{
Version: binary.LittleEndian.Uint32(header[0:]),
PrevBlockHash: h1.String(),
MerkleRoot: h2.String(),
ClaimTrieRoot: h3.String(),
Timestamp: binary.LittleEndian.Uint32(header[100:]),
Bits: binary.LittleEndian.Uint32(header[104:]),
Nonce: binary.LittleEndian.Uint32(header[108:]),
BlockHeight: height,
}
}
type BlockGetServerHeightReq struct{}
type BlockGetServerHeightResp uint32
// blockchain.block.get_server_height
func (s *BlockchainBlockService) Get_server_height(req *BlockGetServerHeightReq, resp **BlockGetServerHeightResp) error {
func (s *BlockchainService) Get_server_height(r *http.Request, req *BlockGetServerHeightReq, resp **BlockGetServerHeightResp) error {
if s.DB == nil || s.DB.LastState == nil {
return fmt.Errorf("unknown height")
}
@ -128,11 +103,10 @@ type BlockGetChunkReq uint32
type BlockGetChunkResp string
// 'blockchain.block.get_chunk'
func (s *BlockchainBlockService) Get_chunk(req *BlockGetChunkReq, resp **BlockGetChunkResp) error {
func (s *BlockchainService) Get_chunk(r *http.Request, req *BlockGetChunkReq, resp **BlockGetChunkResp) error {
index := uint32(*req)
db_headers, err := s.DB.GetHeaders(index*CHUNK_SIZE, CHUNK_SIZE)
if err != nil {
log.Warn(err)
return err
}
raw := make([]byte, 0, HEADER_SIZE*len(db_headers))
@ -146,21 +120,43 @@ func (s *BlockchainBlockService) Get_chunk(req *BlockGetChunkReq, resp **BlockGe
type BlockGetHeaderReq uint32
type BlockGetHeaderResp struct {
BlockHeaderElectrum
Version uint32 `json:"version"`
PrevBlockHash string `json:"prev_block_hash"`
MerkleRoot string `json:"merkle_root"`
ClaimTrieRoot string `json:"claim_trie_root"`
Timestamp uint32 `json:"timestamp"`
Bits uint32 `json:"bits"`
Nonce uint32 `json:"nonce"`
BlockHeight uint32 `json:"block_height"`
}
// 'blockchain.block.get_header'
func (s *BlockchainBlockService) Get_header(req *BlockGetHeaderReq, resp **BlockGetHeaderResp) error {
func (s *BlockchainService) Get_header(r *http.Request, req *BlockGetHeaderReq, resp **BlockGetHeaderResp) error {
height := uint32(*req)
headers, err := s.DB.GetHeaders(height, 1)
if err != nil {
log.Warn(err)
return err
}
if len(headers) < 1 {
return errors.New("not found")
}
*resp = &BlockGetHeaderResp{*newBlockHeaderElectrum(&headers[0], height)}
decode := func(header *[HEADER_SIZE]byte, height uint32) *BlockGetHeaderResp {
var h1, h2, h3 chainhash.Hash
h1.SetBytes(header[4:36])
h2.SetBytes(header[36:68])
h3.SetBytes(header[68:100])
return &BlockGetHeaderResp{
Version: binary.LittleEndian.Uint32(header[0:]),
PrevBlockHash: h1.String(),
MerkleRoot: h2.String(),
ClaimTrieRoot: h3.String(),
Timestamp: binary.LittleEndian.Uint32(header[100:]),
Bits: binary.LittleEndian.Uint32(header[104:]),
Nonce: binary.LittleEndian.Uint32(header[108:]),
BlockHeight: height,
}
}
*resp = decode(&headers[0], height)
return err
}
@ -171,42 +167,9 @@ type BlockHeadersReq struct {
B64 bool `json:"b64"`
}
func (req *BlockHeadersReq) UnmarshalJSON(b []byte) error {
var params [4]interface{}
err := json.Unmarshal(b, &params)
if err != nil {
return err
}
switch params[0].(type) {
case float64:
req.StartHeight = uint32(params[0].(float64))
default:
return fmt.Errorf("expected numeric argument #0 (start_height)")
}
switch params[1].(type) {
case float64:
req.Count = uint32(params[1].(float64))
default:
return fmt.Errorf("expected numeric argument #1 (count)")
}
switch params[2].(type) {
case float64:
req.CpHeight = uint32(params[2].(float64))
default:
return fmt.Errorf("expected numeric argument #2 (cp_height)")
}
switch params[3].(type) {
case bool:
req.B64 = params[3].(bool)
default:
return fmt.Errorf("expected boolean argument #3 (b64)")
}
return nil
}
type BlockHeadersResp struct {
Base64 string `json:"base64,omitempty"`
Hex string `json:"hex"`
Hex string `json:"hex,omitempty"`
Count uint32 `json:"count"`
Max uint32 `json:"max"`
Branch string `json:"branch,omitempty"`
@ -214,11 +177,10 @@ type BlockHeadersResp struct {
}
// 'blockchain.block.headers'
func (s *BlockchainBlockService) Headers(req *BlockHeadersReq, resp **BlockHeadersResp) error {
func (s *BlockchainService) Headers(r *http.Request, req *BlockHeadersReq, resp **BlockHeadersResp) error {
count := min(req.Count, MAX_CHUNK_SIZE)
db_headers, err := s.DB.GetHeaders(req.StartHeight, count)
if err != nil {
log.Warn(err)
return err
}
count = uint32(len(db_headers))
@ -247,62 +209,6 @@ func (s *BlockchainBlockService) Headers(req *BlockHeadersReq, resp **BlockHeade
return err
}
type HeadersSubscribeReq struct {
Raw bool `json:"raw"`
}
func (req *HeadersSubscribeReq) UnmarshalJSON(b []byte) error {
var params [1]interface{}
err := json.Unmarshal(b, &params)
if err != nil {
return err
}
switch params[0].(type) {
case bool:
req.Raw = params[0].(bool)
default:
return fmt.Errorf("expected bool argument #0 (raw)")
}
return nil
}
type HeadersSubscribeResp struct {
BlockHeaderElectrum
}
type HeadersSubscribeRawResp struct {
Hex string `json:"hex"`
Height uint32 `json:"height"`
}
// 'blockchain.headers.subscribe'
func (s *BlockchainHeadersService) Subscribe(req *HeadersSubscribeReq, resp *interface{}) error {
if s.sessionMgr == nil || s.session == nil {
return errors.New("no session, rpc not supported")
}
s.sessionMgr.headersSubscribe(s.session, req.Raw, true /*subscribe*/)
height := s.DB.Height
if s.DB.LastState != nil {
height = s.DB.LastState.Height
}
headers, err := s.DB.GetHeaders(height, 1)
if err != nil {
s.sessionMgr.headersSubscribe(s.session, req.Raw, false /*subscribe*/)
return err
}
if len(headers) < 1 {
return errors.New("not found")
}
if req.Raw {
*resp = &HeadersSubscribeRawResp{
Hex: hex.EncodeToString(headers[0][:]),
Height: height,
}
} else {
*resp = &HeadersSubscribeResp{*newBlockHeaderElectrum(&headers[0], height)}
}
return err
}
func decodeScriptHash(scripthash string) ([]byte, error) {
sh, err := hex.DecodeString(scripthash)
if err != nil {
@ -343,25 +249,21 @@ type AddressGetBalanceResp struct {
}
// 'blockchain.address.get_balance'
func (s *BlockchainAddressService) Get_balance(req *AddressGetBalanceReq, resp **AddressGetBalanceResp) error {
func (s *BlockchainAddressService) Get_balance(r *http.Request, req *AddressGetBalanceReq, resp **AddressGetBalanceResp) error {
address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
if err != nil {
log.Warn(err)
return err
}
script, err := txscript.PayToAddrScript(address)
if err != nil {
log.Warn(err)
return err
}
hashX := hashXScript(script, s.Chain)
confirmed, unconfirmed, err := s.DB.GetBalance(hashX)
if err != nil {
log.Warn(err)
return err
}
*resp = &AddressGetBalanceResp{confirmed, unconfirmed}
return err
}
@ -374,16 +276,14 @@ type ScripthashGetBalanceResp struct {
}
// 'blockchain.scripthash.get_balance'
func (s *BlockchainScripthashService) Get_balance(req *scripthashGetBalanceReq, resp **ScripthashGetBalanceResp) error {
func (s *BlockchainScripthashService) Get_balance(r *http.Request, req *scripthashGetBalanceReq, resp **ScripthashGetBalanceResp) error {
scripthash, err := decodeScriptHash(req.ScriptHash)
if err != nil {
log.Warn(err)
return err
}
hashX := hashX(scripthash)
confirmed, unconfirmed, err := s.DB.GetBalance(hashX)
if err != nil {
log.Warn(err)
return err
}
*resp = &ScripthashGetBalanceResp{confirmed, unconfirmed}
@ -393,23 +293,6 @@ func (s *BlockchainScripthashService) Get_balance(req *scripthashGetBalanceReq,
type AddressGetHistoryReq struct {
Address string `json:"address"`
}
func (req *AddressGetHistoryReq) UnmarshalJSON(b []byte) error {
var params [1]interface{}
json.Unmarshal(b, &params)
err := json.Unmarshal(b, &params)
if err != nil {
return err
}
switch params[0].(type) {
case string:
req.Address = params[0].(string)
default:
return fmt.Errorf("expected string argument #0 (address)")
}
return nil
}
type TxInfo struct {
TxHash string `json:"tx_hash"`
Height uint32 `json:"height"`
@ -418,24 +301,24 @@ type TxInfoFee struct {
TxInfo
Fee uint64 `json:"fee"`
}
type AddressGetHistoryResp []TxInfoFee
type AddressGetHistoryResp struct {
Confirmed []TxInfo `json:"confirmed"`
Unconfirmed []TxInfoFee `json:"unconfirmed"`
}
// 'blockchain.address.get_history'
func (s *BlockchainAddressService) Get_history(req *AddressGetHistoryReq, resp **AddressGetHistoryResp) error {
func (s *BlockchainAddressService) Get_history(r *http.Request, req *AddressGetHistoryReq, resp **AddressGetHistoryResp) error {
address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
if err != nil {
log.Warn(err)
return err
}
script, err := txscript.PayToAddrScript(address)
if err != nil {
log.Warn(err)
return err
}
hashX := hashXScript(script, s.Chain)
dbTXs, err := s.DB.GetHistory(hashX)
if err != nil {
log.Warn(err)
return err
}
confirmed := make([]TxInfo, 0, len(dbTXs))
@ -446,18 +329,11 @@ func (s *BlockchainAddressService) Get_history(req *AddressGetHistoryReq, resp *
Height: tx.Height,
})
}
unconfirmed := []TxInfoFee{} // TODO
result := make(AddressGetHistoryResp, len(confirmed)+len(unconfirmed))
i := 0
for _, tx := range confirmed {
result[i].TxInfo = tx
i += 1
result := &AddressGetHistoryResp{
Confirmed: confirmed,
Unconfirmed: []TxInfoFee{}, // TODO
}
for _, tx := range unconfirmed {
result[i] = tx
i += 1
}
*resp = &result
*resp = result
return err
}
@ -470,16 +346,14 @@ type ScripthashGetHistoryResp struct {
}
// 'blockchain.scripthash.get_history'
func (s *BlockchainScripthashService) Get_history(req *ScripthashGetHistoryReq, resp **ScripthashGetHistoryResp) error {
func (s *BlockchainScripthashService) Get_history(r *http.Request, req *ScripthashGetHistoryReq, resp **ScripthashGetHistoryResp) error {
scripthash, err := decodeScriptHash(req.ScriptHash)
if err != nil {
log.Warn(err)
return err
}
hashX := hashX(scripthash)
dbTXs, err := s.DB.GetHistory(hashX)
if err != nil {
log.Warn(err)
return err
}
confirmed := make([]TxInfo, 0, len(dbTXs))
@ -504,15 +378,13 @@ type AddressGetMempoolReq struct {
type AddressGetMempoolResp []TxInfoFee
// 'blockchain.address.get_mempool'
func (s *BlockchainAddressService) Get_mempool(req *AddressGetMempoolReq, resp **AddressGetMempoolResp) error {
func (s *BlockchainAddressService) Get_mempool(r *http.Request, req *AddressGetMempoolReq, resp **AddressGetMempoolResp) error {
address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
if err != nil {
log.Warn(err)
return err
}
script, err := txscript.PayToAddrScript(address)
if err != nil {
log.Warn(err)
return err
}
hashX := hashXScript(script, s.Chain)
@ -530,10 +402,9 @@ type ScripthashGetMempoolReq struct {
type ScripthashGetMempoolResp []TxInfoFee
// 'blockchain.scripthash.get_mempool'
func (s *BlockchainScripthashService) Get_mempool(req *ScripthashGetMempoolReq, resp **ScripthashGetMempoolResp) error {
func (s *BlockchainScripthashService) Get_mempool(r *http.Request, req *ScripthashGetMempoolReq, resp **ScripthashGetMempoolResp) error {
scripthash, err := decodeScriptHash(req.ScriptHash)
if err != nil {
log.Warn(err)
return err
}
hashX := hashX(scripthash)
@ -557,15 +428,13 @@ type TXOInfo struct {
type AddressListUnspentResp []TXOInfo
// 'blockchain.address.listunspent'
func (s *BlockchainAddressService) Listunspent(req *AddressListUnspentReq, resp **AddressListUnspentResp) error {
func (s *BlockchainAddressService) Listunspent(r *http.Request, req *AddressListUnspentReq, resp **AddressListUnspentResp) error {
address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
if err != nil {
log.Warn(err)
return err
}
script, err := txscript.PayToAddrScript(address)
if err != nil {
log.Warn(err)
return err
}
hashX := hashXScript(script, s.Chain)
@ -591,10 +460,9 @@ type ScripthashListUnspentReq struct {
type ScripthashListUnspentResp []TXOInfo
// 'blockchain.scripthash.listunspent'
func (s *BlockchainScripthashService) Listunspent(req *ScripthashListUnspentReq, resp **ScripthashListUnspentResp) error {
func (s *BlockchainScripthashService) Listunspent(r *http.Request, req *ScripthashListUnspentReq, resp **ScripthashListUnspentResp) error {
scripthash, err := decodeScriptHash(req.ScriptHash)
if err != nil {
log.Warn(err)
return err
}
hashX := hashX(scripthash)
@ -613,312 +481,3 @@ func (s *BlockchainScripthashService) Listunspent(req *ScripthashListUnspentReq,
*resp = &result
return err
}
type AddressSubscribeReq []string
type AddressSubscribeResp []string
// 'blockchain.address.subscribe'
func (s *BlockchainAddressService) Subscribe(req *AddressSubscribeReq, resp **AddressSubscribeResp) error {
if s.sessionMgr == nil || s.session == nil {
return errors.New("no session, rpc not supported")
}
result := make([]string, 0, len(*req))
for _, addr := range *req {
address, err := lbcutil.DecodeAddress(addr, s.Chain)
if err != nil {
return err
}
script, err := txscript.PayToAddrScript(address)
if err != nil {
return err
}
hashX := hashXScript(script, s.Chain)
s.sessionMgr.hashXSubscribe(s.session, hashX, addr, true /*subscribe*/)
status, err := s.DB.GetStatus(hashX)
if err != nil {
return err
}
result = append(result, hex.EncodeToString(status))
}
*resp = (*AddressSubscribeResp)(&result)
return nil
}
// 'blockchain.address.unsubscribe'
func (s *BlockchainAddressService) Unsubscribe(req *AddressSubscribeReq, resp **AddressSubscribeResp) error {
if s.sessionMgr == nil || s.session == nil {
return errors.New("no session, rpc not supported")
}
for _, addr := range *req {
address, err := lbcutil.DecodeAddress(addr, s.Chain)
if err != nil {
return err
}
script, err := txscript.PayToAddrScript(address)
if err != nil {
return err
}
hashX := hashXScript(script, s.Chain)
s.sessionMgr.hashXSubscribe(s.session, hashX, addr, false /*subscribe*/)
}
*resp = (*AddressSubscribeResp)(nil)
return nil
}
type ScripthashSubscribeReq string
type ScripthashSubscribeResp string
// 'blockchain.scripthash.subscribe'
func (s *BlockchainScripthashService) Subscribe(req *ScripthashSubscribeReq, resp **ScripthashSubscribeResp) error {
if s.sessionMgr == nil || s.session == nil {
return errors.New("no session, rpc not supported")
}
var result string
scripthash, err := decodeScriptHash(string(*req))
if err != nil {
return err
}
hashX := hashX(scripthash)
s.sessionMgr.hashXSubscribe(s.session, hashX, string(*req), true /*subscribe*/)
status, err := s.DB.GetStatus(hashX)
if err != nil {
return err
}
result = hex.EncodeToString(status)
*resp = (*ScripthashSubscribeResp)(&result)
return nil
}
// 'blockchain.scripthash.unsubscribe'
func (s *BlockchainScripthashService) Unsubscribe(req *ScripthashSubscribeReq, resp **ScripthashSubscribeResp) error {
if s.sessionMgr == nil || s.session == nil {
return errors.New("no session, rpc not supported")
}
scripthash, err := decodeScriptHash(string(*req))
if err != nil {
return err
}
hashX := hashX(scripthash)
s.sessionMgr.hashXSubscribe(s.session, hashX, string(*req), false /*subscribe*/)
*resp = (*ScripthashSubscribeResp)(nil)
return nil
}
type TransactionBroadcastReq [1]string
type TransactionBroadcastResp string
// 'blockchain.transaction.broadcast'
func (s *BlockchainTransactionService) Broadcast(req *TransactionBroadcastReq, resp **TransactionBroadcastResp) error {
if s.sessionMgr == nil {
return errors.New("no session manager, rpc not supported")
}
strTx := string((*req)[0])
rawTx, err := hex.DecodeString(strTx)
if err != nil {
return err
}
txhash, err := s.sessionMgr.broadcastTx(rawTx)
if err != nil {
return err
}
result := txhash.String()
*resp = (*TransactionBroadcastResp)(&result)
return nil
}
type TransactionGetReq string
type TXFullDetail struct {
Height int `json:"block_height"`
Pos uint32 `json:"pos"`
Merkle []string `json:"merkle"`
}
type TXDetail struct {
Height int `json:"block_height"`
}
type TXGetItem struct {
TxHash string
TxRaw string
Detail interface{} // TXFullDetail or TXDetail struct
}
type TransactionGetResp TXGetItem
// 'blockchain.transaction.get'
func (s *BlockchainTransactionService) Get(req *TransactionGetReq, resp **TransactionGetResp) error {
txids := [1]string{string(*req)}
request := TransactionGetBatchReq(txids[:])
var response *TransactionGetBatchResp
err := s.Get_batch(&request, &response)
if err != nil {
return err
}
if len(*response) < 1 {
return errors.New("tx not found")
}
switch (*response)[0].Detail.(type) {
case TXFullDetail:
break
case TXDetail:
default:
return errors.New("tx not confirmed")
}
*resp = (*TransactionGetResp)(&(*response)[0])
return err
}
type TransactionGetBatchReq []string
func (req *TransactionGetBatchReq) UnmarshalJSON(b []byte) error {
var params []interface{}
json.Unmarshal(b, &params)
if len(params) > 100 {
return fmt.Errorf("too many tx hashes in request: %v", len(params))
}
for i, txhash := range params {
switch params[0].(type) {
case string:
*req = append(*req, txhash.(string))
default:
return fmt.Errorf("expected string argument #%d (tx_hash)", i)
}
}
return nil
}
type TransactionGetBatchResp []TXGetItem
func (resp *TransactionGetBatchResp) MarshalJSON() ([]byte, error) {
// encode key/value pairs as variable length JSON object
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
buf.WriteString("{")
for i, r := range *resp {
if i > 0 {
buf.WriteString(",")
}
txhash, raw, detail := r.TxHash, r.TxRaw, r.Detail
err := enc.Encode(txhash)
if err != nil {
return nil, err
}
buf.WriteString(":[")
err = enc.Encode(raw)
if err != nil {
return nil, err
}
buf.WriteString(",")
err = enc.Encode(detail)
if err != nil {
return nil, err
}
buf.WriteString("]")
}
buf.WriteString("}")
return buf.Bytes(), nil
}
// 'blockchain.transaction.get_batch'
func (s *BlockchainTransactionService) Get_batch(req *TransactionGetBatchReq, resp **TransactionGetBatchResp) error {
if len(*req) > 100 {
return fmt.Errorf("too many tx hashes in request: %v", len(*req))
}
tx_hashes := make([]chainhash.Hash, 0, len(*req))
for i, txid := range *req {
tx_hashes = append(tx_hashes, chainhash.Hash{})
if err := chainhash.Decode(&tx_hashes[i], txid); err != nil {
return err
}
}
dbResult, err := s.DB.GetTxMerkle(tx_hashes)
if err != nil {
return err
}
result := make([]TXGetItem, 0, len(dbResult))
for _, r := range dbResult {
merkles := make([]string, len(r.Merkle))
for i, m := range r.Merkle {
merkles[i] = m.String()
}
detail := TXFullDetail{
Height: r.Height,
Pos: r.Pos,
Merkle: merkles,
}
result = append(result, TXGetItem{r.TxHash.String(), hex.EncodeToString(r.RawTx), &detail})
}
*resp = (*TransactionGetBatchResp)(&result)
return err
}
type TransactionGetMerkleReq struct {
TxHash string `json:"tx_hash"`
Height uint32 `json:"height"`
}
type TransactionGetMerkleResp TXGetItem
// 'blockchain.transaction.get_merkle'
func (s *BlockchainTransactionService) Get_merkle(req *TransactionGetMerkleReq, resp **TransactionGetMerkleResp) error {
txids := [1]string{string(req.TxHash)}
request := TransactionGetBatchReq(txids[:])
var response *TransactionGetBatchResp
err := s.Get_batch(&request, &response)
if err != nil {
return err
}
if len(*response) < 1 {
return errors.New("tx not found")
}
switch (*response)[0].Detail.(type) {
case TXFullDetail:
break
case TXDetail:
default:
return errors.New("tx not confirmed")
}
*resp = (*TransactionGetMerkleResp)(&(*response)[0])
return err
}
type TransactionGetHeightReq string
type TransactionGetHeightResp uint32
// 'blockchain.transaction.get_height'
func (s *BlockchainTransactionService) Get_height(req *TransactionGetHeightReq, resp **TransactionGetHeightResp) error {
txid := string(*(req))
txhash, err := chainhash.NewHashFromStr(txid)
if err != nil {
return err
}
height, err := s.DB.GetTxHeight(txhash)
*resp = (*TransactionGetHeightResp)(&height)
return err
}
type TransactionInfoReq string
type TransactionInfoResp TXGetItem
// 'blockchain.transaction.info'
func (s *BlockchainTransactionService) Info(req *TransactionInfoReq, resp **TransactionInfoResp) error {
txids := [1]string{string(*req)}
request := TransactionGetBatchReq(txids[:])
var response *TransactionGetBatchResp
err := s.Get_batch(&request, &response)
if err != nil {
return err
}
if len(*response) < 1 {
return errors.New("tx not found")
}
switch (*response)[0].Detail.(type) {
case TXFullDetail:
break
case TXDetail:
default:
if (*response)[0].TxHash == "" {
return errors.New("no such mempool or blockchain transaction")
}
}
*resp = (*TransactionInfoResp)(&(*response)[0])
return err
}

View file

@ -1,19 +1,12 @@
package server
import (
"encoding/hex"
"encoding/json"
"net"
"strconv"
"sync"
"testing"
"github.com/lbryio/herald.go/db"
"github.com/lbryio/herald.go/internal"
"github.com/lbryio/lbcd/chaincfg"
"github.com/lbryio/lbcd/txscript"
"github.com/lbryio/lbcutil"
"github.com/lbryio/lbry.go/v3/extras/stop"
)
// Source: test_variety_of_transactions_and_longish_history (lbry-sdk/tests/integration/transactions)
@ -58,22 +51,21 @@ var regTestAddrs = [30]string{
func TestServerGetHeight(t *testing.T) {
secondaryPath := "asdf"
grp := stop.NewDebug()
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
defer db.Shutdown()
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
defer toDefer()
if err != nil {
t.Error(err)
return
}
s := &BlockchainBlockService{
s := &BlockchainService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
}
req := BlockGetServerHeightReq{}
var resp *BlockGetServerHeightResp
err = s.Get_server_height(&req, &resp)
err = s.Get_server_height(nil, &req, &resp)
if err != nil {
t.Errorf("handler err: %v", err)
}
@ -89,15 +81,14 @@ func TestServerGetHeight(t *testing.T) {
func TestGetChunk(t *testing.T) {
secondaryPath := "asdf"
grp := stop.NewDebug()
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
defer db.Shutdown()
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
defer toDefer()
if err != nil {
t.Error(err)
return
}
s := &BlockchainBlockService{
s := &BlockchainService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
}
@ -105,7 +96,7 @@ func TestGetChunk(t *testing.T) {
for index := 0; index < 10; index++ {
req := BlockGetChunkReq(index)
var resp *BlockGetChunkResp
err := s.Get_chunk(&req, &resp)
err := s.Get_chunk(nil, &req, &resp)
if err != nil {
t.Errorf("index: %v handler err: %v", index, err)
}
@ -133,15 +124,14 @@ func TestGetChunk(t *testing.T) {
func TestGetHeader(t *testing.T) {
secondaryPath := "asdf"
grp := stop.NewDebug()
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
defer db.Shutdown()
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
defer toDefer()
if err != nil {
t.Error(err)
return
}
s := &BlockchainBlockService{
s := &BlockchainService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
}
@ -149,7 +139,7 @@ func TestGetHeader(t *testing.T) {
for height := 0; height < 700; height += 100 {
req := BlockGetHeaderReq(height)
var resp *BlockGetHeaderResp
err := s.Get_header(&req, &resp)
err := s.Get_header(nil, &req, &resp)
if err != nil && height <= 500 {
t.Errorf("height: %v handler err: %v", height, err)
}
@ -161,151 +151,26 @@ func TestGetHeader(t *testing.T) {
}
}
func TestHeaders(t *testing.T) {
secondaryPath := "asdf"
grp := stop.NewDebug()
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
defer db.Shutdown()
if err != nil {
t.Error(err)
return
}
s := &BlockchainBlockService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
}
for height := uint32(0); height < 700; height += 100 {
req := BlockHeadersReq{
StartHeight: height,
Count: 1,
CpHeight: 0,
B64: false,
}
var resp *BlockHeadersResp
err := s.Headers(&req, &resp)
if err != nil {
t.Errorf("Headers: %v", err)
}
marshalled, err := json.MarshalIndent(resp, "", " ")
if err != nil {
t.Errorf("height: %v unmarshal err: %v", height, err)
}
t.Logf("height: %v resp: %v", height, string(marshalled))
}
}
func TestHeadersSubscribe(t *testing.T) {
args := MakeDefaultTestArgs()
grp := stop.NewDebug()
secondaryPath := "asdf"
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
defer db.Shutdown()
if err != nil {
t.Error(err)
return
}
sm := newSessionManager(nil, db, args, grp, &chaincfg.RegressionNetParams, nil)
sm.start()
defer sm.stop()
client1, server1 := net.Pipe()
sess1 := sm.addSession(server1)
client2, server2 := net.Pipe()
sess2 := sm.addSession(server2)
// Set up logic to read a notification.
var received sync.WaitGroup
recv := func(client net.Conn) {
defer received.Done()
buf := make([]byte, 1024)
len, err := client.Read(buf)
if err != nil {
t.Errorf("read err: %v", err)
}
t.Logf("len: %v notification: %v", len, string(buf))
}
received.Add(2)
go recv(client1)
go recv(client2)
s1 := &BlockchainHeadersService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
sessionMgr: sm,
session: sess1,
}
s2 := &BlockchainHeadersService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
sessionMgr: sm,
session: sess2,
}
// Subscribe with Raw: false.
req1 := HeadersSubscribeReq{Raw: false}
var r any
err = s1.Subscribe(&req1, &r)
if err != nil {
t.Errorf("handler err: %v", err)
}
resp1 := r.(*HeadersSubscribeResp)
marshalled1, err := json.MarshalIndent(resp1, "", " ")
if err != nil {
t.Errorf("unmarshal err: %v", err)
}
// Subscribe with Raw: true.
t.Logf("resp: %v", string(marshalled1))
req2 := HeadersSubscribeReq{Raw: true}
err = s2.Subscribe(&req2, &r)
if err != nil {
t.Errorf("handler err: %v", err)
}
resp2 := r.(*HeadersSubscribeRawResp)
marshalled2, err := json.MarshalIndent(resp2, "", " ")
if err != nil {
t.Errorf("unmarshal err: %v", err)
}
t.Logf("resp: %v", string(marshalled2))
// Now send a notification.
header500, err := hex.DecodeString("00000020e9537f98ae80a3aa0936dd424439b2b9305e5e9d9d5c7aa571b4422c447741e739b3109304ed4f0330d6854271db17da221559a46b68db4ceecfebd9f0c75dbe0100000000000000000000000000000000000000000000000000000000000000b3e02063ffff7f2001000000")
if err != nil {
t.Errorf("decode err: %v", err)
}
note1 := headerNotification{
HeightHash: internal.HeightHash{Height: 500, BlockHeader: header500},
blockHeaderElectrum: nil,
blockHeaderStr: "",
}
t.Logf("sending notification")
sm.doNotify(note1)
t.Logf("waiting to receive notification(s)...")
received.Wait()
}
func TestGetBalance(t *testing.T) {
secondaryPath := "asdf"
grp := stop.NewDebug()
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
defer db.Shutdown()
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
defer toDefer()
if err != nil {
t.Error(err)
return
}
s := &BlockchainAddressService{
BlockchainService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
},
}
for _, addr := range regTestAddrs {
req := AddressGetBalanceReq{addr}
var resp *AddressGetBalanceResp
err := s.Get_balance(&req, &resp)
err := s.Get_balance(nil, &req, &resp)
if err != nil {
t.Errorf("address: %v handler err: %v", addr, err)
}
@ -319,23 +184,24 @@ func TestGetBalance(t *testing.T) {
func TestGetHistory(t *testing.T) {
secondaryPath := "asdf"
grp := stop.NewDebug()
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
defer db.Shutdown()
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
defer toDefer()
if err != nil {
t.Error(err)
return
}
s := &BlockchainAddressService{
BlockchainService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
},
}
for _, addr := range regTestAddrs {
req := AddressGetHistoryReq{addr}
var resp *AddressGetHistoryResp
err := s.Get_history(&req, &resp)
err := s.Get_history(nil, &req, &resp)
if err != nil {
t.Errorf("address: %v handler err: %v", addr, err)
}
@ -349,23 +215,24 @@ func TestGetHistory(t *testing.T) {
func TestListUnspent(t *testing.T) {
secondaryPath := "asdf"
grp := stop.NewDebug()
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
defer db.Shutdown()
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
defer toDefer()
if err != nil {
t.Error(err)
return
}
s := &BlockchainAddressService{
BlockchainService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
},
}
for _, addr := range regTestAddrs {
req := AddressListUnspentReq{addr}
var resp *AddressListUnspentResp
err := s.Listunspent(&req, &resp)
err := s.Listunspent(nil, &req, &resp)
if err != nil {
t.Errorf("address: %v handler err: %v", addr, err)
}
@ -376,94 +243,3 @@ func TestListUnspent(t *testing.T) {
t.Logf("address: %v resp: %v", addr, string(marshalled))
}
}
func TestAddressSubscribe(t *testing.T) {
args := MakeDefaultTestArgs()
grp := stop.NewDebug()
secondaryPath := "asdf"
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
defer db.Shutdown()
if err != nil {
t.Error(err)
return
}
sm := newSessionManager(nil, db, args, grp, &chaincfg.RegressionNetParams, nil)
sm.start()
defer sm.stop()
client1, server1 := net.Pipe()
sess1 := sm.addSession(server1)
client2, server2 := net.Pipe()
sess2 := sm.addSession(server2)
// Set up logic to read a notification.
var received sync.WaitGroup
recv := func(client net.Conn) {
buf := make([]byte, 1024)
len, err := client.Read(buf)
if err != nil {
t.Errorf("read err: %v", err)
}
t.Logf("len: %v notification: %v", len, string(buf))
received.Done()
}
received.Add(2)
go recv(client1)
go recv(client2)
s1 := &BlockchainAddressService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
sessionMgr: sm,
session: sess1,
}
s2 := &BlockchainAddressService{
DB: db,
Chain: &chaincfg.RegressionNetParams,
sessionMgr: sm,
session: sess2,
}
addr1, addr2 := regTestAddrs[1], regTestAddrs[2]
// Subscribe to addr1 and addr2.
req1 := AddressSubscribeReq{addr1, addr2}
var resp1 *AddressSubscribeResp
err = s1.Subscribe(&req1, &resp1)
if err != nil {
t.Errorf("handler err: %v", err)
}
marshalled1, err := json.MarshalIndent(resp1, "", " ")
if err != nil {
t.Errorf("unmarshal err: %v", err)
}
// Subscribe to addr2 only.
t.Logf("resp: %v", string(marshalled1))
req2 := AddressSubscribeReq{addr2}
var resp2 *AddressSubscribeResp
err = s2.Subscribe(&req2, &resp2)
if err != nil {
t.Errorf("handler err: %v", err)
}
marshalled2, err := json.MarshalIndent(resp2, "", " ")
if err != nil {
t.Errorf("unmarshal err: %v", err)
}
t.Logf("resp: %v", string(marshalled2))
// Now send a notification for addr2.
address, _ := lbcutil.DecodeAddress(addr2, sm.chain)
script, _ := txscript.PayToAddrScript(address)
note := hashXNotification{}
copy(note.hashX[:], hashXScript(script, sm.chain))
status, err := hex.DecodeString((*resp1)[1])
if err != nil {
t.Errorf("decode err: %v", err)
}
note.status = append(note.status, []byte(status)...)
t.Logf("sending notification")
sm.doNotify(note)
t.Logf("waiting to receive notification(s)...")
received.Wait()
}

View file

@ -1,108 +0,0 @@
package server
import (
"context"
"fmt"
"github.com/lbryio/herald.go/db"
"github.com/lbryio/herald.go/internal/metrics"
pb "github.com/lbryio/herald.go/protobuf/go"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
type ClaimtrieService struct {
DB *db.ReadOnlyDBColumnFamily
Server *Server
}
type ResolveData struct {
Data []string `json:"data"`
}
type Result struct {
Data string `json:"data"`
}
type GetClaimByIDData struct {
ClaimID string `json:"claim_id"`
}
// Resolve is the json rpc endpoint for 'blockchain.claimtrie.resolve'.
func (t *ClaimtrieService) Resolve(args *ResolveData, result **pb.Outputs) error {
log.Println("Resolve")
res, err := InternalResolve(args.Data, t.DB)
*result = res
return err
}
// Search is the json rpc endpoint for 'blockchain.claimtrie.search'.
func (t *ClaimtrieService) Search(args *pb.SearchRequest, result **pb.Outputs) error {
log.Println("Search")
if t.Server == nil {
log.Warnln("Server is nil in Search")
*result = nil
return nil
}
ctx := context.Background()
res, err := t.Server.Search(ctx, args)
*result = res
return err
}
// GetClaimByID is the json rpc endpoint for 'blockchain.claimtrie.getclaimbyid'.
func (t *ClaimtrieService) GetClaimByID(args *GetClaimByIDData, result **pb.Outputs) error {
log.Println("GetClaimByID")
if len(args.ClaimID) != 40 {
*result = nil
return fmt.Errorf("len(claim_id) != 40")
}
rows, extras, err := t.DB.GetClaimByID(args.ClaimID)
if err != nil {
*result = nil
return err
}
metrics.RequestsCount.With(prometheus.Labels{"method": "blockchain.claimtrie.getclaimbyid"}).Inc()
// FIXME: this has txos and extras and so does GetClaimById?
allTxos := make([]*pb.Output, 0)
allExtraTxos := make([]*pb.Output, 0)
for _, row := range rows {
txos, extraTxos, err := row.ToOutputs()
if err != nil {
*result = nil
return err
}
// TODO: there may be a more efficient way to do this.
allTxos = append(allTxos, txos...)
allExtraTxos = append(allExtraTxos, extraTxos...)
}
for _, extra := range extras {
txos, extraTxos, err := extra.ToOutputs()
if err != nil {
*result = nil
return err
}
// TODO: there may be a more efficient way to do this.
allTxos = append(allTxos, txos...)
allExtraTxos = append(allExtraTxos, extraTxos...)
}
res := &pb.Outputs{
Txos: allTxos,
ExtraTxos: allExtraTxos,
Total: uint32(len(allTxos) + len(allExtraTxos)),
Offset: 0, //TODO
Blocked: nil, //TODO
BlockedTotal: 0, //TODO
}
log.Warn(res)
*result = res
return nil
}

View file

@ -1,39 +0,0 @@
package server
import (
"errors"
log "github.com/sirupsen/logrus"
)
type PeersService struct {
Server *Server
// needed for subscribe/unsubscribe
sessionMgr *sessionManager
session *session
}
type PeersSubscribeReq struct {
Ip string `json:"ip"`
Host string `json:"host"`
Details []string `json:"details"`
}
type PeersSubscribeResp string
// Features is the json rpc endpoint for 'server.peers.subcribe'.
func (t *PeersService) Subscribe(req *PeersSubscribeReq, res **PeersSubscribeResp) error {
log.Println("PeersSubscribe")
// var port = "50001"
// FIXME: Get the actual port from the request details
if t.sessionMgr == nil || t.session == nil {
*res = nil
return errors.New("no session, rpc not supported")
}
t.sessionMgr.peersSubscribe(t.session, true /*subscribe*/)
*res = nil
return nil
}

View file

@ -1,88 +0,0 @@
package server
import (
log "github.com/sirupsen/logrus"
)
type ServerService struct {
Args *Args
}
type ServerFeatureService struct {
Args *Args
}
type ServerFeaturesReq struct{}
type ServerFeaturesRes struct {
Hosts map[string]string `json:"hosts"`
Pruning string `json:"pruning"`
ServerVersion string `json:"server_version"`
ProtocolMin string `json:"protocol_min"`
ProtocolMax string `json:"protocol_max"`
GenesisHash string `json:"genesis_hash"`
Description string `json:"description"`
PaymentAddress string `json:"payment_address"`
DonationAddress string `json:"donation_address"`
DailyFee string `json:"daily_fee"`
HashFunction string `json:"hash_function"`
TrendingAlgorithm string `json:"trending_algorithm"`
}
// Features is the json rpc endpoint for 'server.features'.
func (t *ServerService) Features(req *ServerFeaturesReq, res **ServerFeaturesRes) error {
log.Println("Features")
features := &ServerFeaturesRes{
Hosts: map[string]string{},
Pruning: "",
ServerVersion: HUB_PROTOCOL_VERSION,
ProtocolMin: PROTOCOL_MIN,
ProtocolMax: PROTOCOL_MAX,
GenesisHash: t.Args.GenesisHash,
Description: t.Args.ServerDescription,
PaymentAddress: t.Args.PaymentAddress,
DonationAddress: t.Args.DonationAddress,
DailyFee: t.Args.DailyFee,
HashFunction: "sha256",
TrendingAlgorithm: "fast_ar",
}
*res = features
return nil
}
type ServerBannerService struct {
Args *Args
}
type ServerBannerReq struct{}
type ServerBannerRes string
// Banner is the json rpc endpoint for 'server.banner'.
func (t *ServerService) Banner(req *ServerBannerReq, res **ServerBannerRes) error {
log.Println("Banner")
*res = (*ServerBannerRes)(t.Args.Banner)
return nil
}
type ServerVersionService struct {
Args *Args
}
type ServerVersionReq [2]string // [client_name, client_version]
type ServerVersionRes [2]string // [version, protocol_version]
// Version is the json rpc endpoint for 'server.version'.
func (t *ServerService) Version(req *ServerVersionReq, res **ServerVersionRes) error {
// FIXME: We may need to do the computation of a negotiated version here.
// Also return an error if client is not supported?
result := [2]string{t.Args.ServerVersion, t.Args.ServerVersion}
*res = (*ServerVersionRes)(&result)
log.Printf("Version(%v) -> %v", *req, **res)
return nil
}

View file

@ -1,154 +1,69 @@
package server
import (
"fmt"
"net"
"net/http"
"strconv"
"strings"
gorilla_mux "github.com/gorilla/mux"
gorilla_rpc "github.com/gorilla/rpc"
gorilla_json "github.com/gorilla/rpc/json"
"github.com/gorilla/mux"
"github.com/gorilla/rpc"
"github.com/gorilla/rpc/json"
"github.com/lbryio/herald.go/db"
pb "github.com/lbryio/herald.go/protobuf/go"
log "github.com/sirupsen/logrus"
"golang.org/x/net/netutil"
)
type gorillaRpcCodec struct {
gorilla_rpc.Codec
type ClaimtrieService struct {
DB *db.ReadOnlyDBColumnFamily
}
func (c *gorillaRpcCodec) NewRequest(r *http.Request) gorilla_rpc.CodecRequest {
return &gorillaRpcCodecRequest{c.Codec.NewRequest(r)}
type ResolveData struct {
Data []string `json:"data"`
}
// gorillaRpcCodecRequest provides ability to rewrite the incoming
// request "method" field. For example:
// blockchain.block.get_header -> blockchain_block.Get_header
// blockchain.address.listunspent -> blockchain_address.Listunspent
// This makes the "method" string compatible with Gorilla/RPC
// requirements.
type gorillaRpcCodecRequest struct {
gorilla_rpc.CodecRequest
type Result struct {
Data string `json:"data"`
}
func (cr *gorillaRpcCodecRequest) Method() (string, error) {
rawMethod, err := cr.CodecRequest.Method()
if err != nil {
return rawMethod, err
}
parts := strings.Split(rawMethod, ".")
if len(parts) < 2 {
return rawMethod, fmt.Errorf("blockchain rpc: service/method ill-formed: %q", rawMethod)
}
service := strings.Join(parts[0:len(parts)-1], "_")
method := parts[len(parts)-1]
if len(method) < 1 {
return rawMethod, fmt.Errorf("blockchain rpc: method ill-formed: %q", method)
}
method = strings.ToUpper(string(method[0])) + string(method[1:])
return service + "." + method, err
// Resolve is the json rpc endpoint for 'blockchain.claimtrie.resolve'.
func (t *ClaimtrieService) Resolve(r *http.Request, args *ResolveData, result **pb.Outputs) error {
log.Println("Resolve")
res, err := InternalResolve(args.Data, t.DB)
*result = res
return err
}
// StartJsonRPC starts the json rpc server and registers the endpoints.
func (s *Server) StartJsonRPC() error {
// Set up the pure JSONRPC server with persistent connections/sessions.
if s.Args.JSONRPCPort != 0 {
port := ":" + strconv.Itoa(s.Args.JSONRPCPort)
laddr, err := net.ResolveTCPAddr("tcp4", port)
if err != nil {
log.Errorf("ResoveIPAddr: %v\n", err)
goto fail1
}
listener, err := net.ListenTCP("tcp4", laddr)
if err != nil {
log.Errorf("ListenTCP: %v\n", err)
goto fail1
}
log.Infof("JSONRPC server listening on %s", listener.Addr().String())
s.sessionManager.start()
acceptConnections := func(listener net.Listener) {
defer s.sessionManager.stop()
for {
conn, err := listener.Accept()
if err != nil {
log.Errorf("Accept: %v\n", err)
break
}
log.Infof("Accepted: %v", conn.RemoteAddr())
s.sessionManager.addSession(conn)
port := ":" + s.Args.JSONRPCPort
}
}
go acceptConnections(netutil.LimitListener(listener, s.sessionManager.sessionsMax))
}
fail1:
// Set up the JSONRPC over HTTP server.
if s.Args.JSONRPCHTTPPort != 0 {
s1 := gorilla_rpc.NewServer() // Create a new RPC server
s1 := rpc.NewServer() // Create a new RPC server
// Register the type of data requested as JSON, with custom codec.
s1.RegisterCodec(&gorillaRpcCodec{gorilla_json.NewCodec()}, "application/json")
s1.RegisterCodec(&BlockchainCodec{json.NewCodec()}, "application/json")
// Register "blockchain.claimtrie.*"" handlers.
claimtrieSvc := &ClaimtrieService{s.DB, s}
err := s1.RegisterTCPService(claimtrieSvc, "blockchain_claimtrie")
claimtrieSvc := &ClaimtrieService{s.DB}
err := s1.RegisterService(claimtrieSvc, "blockchain_claimtrie")
if err != nil {
log.Errorf("RegisterTCPService: %v\n", err)
goto fail2
log.Errorf("RegisterService: %v\n", err)
}
// Register "blockchain.{block,address,scripthash,transaction}.*" handlers.
blockchainSvc := &BlockchainBlockService{s.DB, s.Chain}
err = s1.RegisterTCPService(blockchainSvc, "blockchain_block")
// Register other "blockchain.{block,address,scripthash}.*" handlers.
blockchainSvc := &BlockchainService{s.DB, s.Chain}
err = s1.RegisterService(blockchainSvc, "blockchain_block")
if err != nil {
log.Errorf("RegisterTCPService: %v\n", err)
goto fail2
log.Errorf("RegisterService: %v\n", err)
}
err = s1.RegisterTCPService(&BlockchainHeadersService{s.DB, s.Chain, s.sessionManager, nil}, "blockchain_headers")
err = s1.RegisterService(&BlockchainAddressService{*blockchainSvc}, "blockchain_address")
if err != nil {
log.Errorf("RegisterTCPService: %v\n", err)
goto fail2
log.Errorf("RegisterService: %v\n", err)
}
err = s1.RegisterTCPService(&BlockchainAddressService{s.DB, s.Chain, s.sessionManager, nil}, "blockchain_address")
err = s1.RegisterService(&BlockchainScripthashService{*blockchainSvc}, "blockchain_scripthash")
if err != nil {
log.Errorf("RegisterTCPService: %v\n", err)
goto fail2
}
err = s1.RegisterTCPService(&BlockchainScripthashService{s.DB, s.Chain, s.sessionManager, nil}, "blockchain_scripthash")
if err != nil {
log.Errorf("RegisterTCPService: %v\n", err)
goto fail2
}
err = s1.RegisterTCPService(&BlockchainTransactionService{s.DB, s.Chain, s.sessionManager}, "blockchain_transaction")
if err != nil {
log.Errorf("RegisterTCPService: %v\n", err)
goto fail2
log.Errorf("RegisterService: %v\n", err)
}
// Register "server.{features,banner,version}" handlers.
serverSvc := &ServerService{s.Args}
err = s1.RegisterTCPService(serverSvc, "server")
if err != nil {
log.Errorf("RegisterTCPService: %v\n", err)
goto fail2
}
// Register "server.peers" handlers.
peersSvc := &PeersService{Server: s}
err = s1.RegisterTCPService(peersSvc, "server_peers")
if err != nil {
log.Errorf("RegisterTCPService: %v\n", err)
goto fail2
}
r := gorilla_mux.NewRouter()
r := mux.NewRouter()
r.Handle("/rpc", s1)
port := ":" + strconv.FormatUint(uint64(s.Args.JSONRPCHTTPPort), 10)
log.Infof("HTTP JSONRPC server listening on %s", port)
log.Fatal(http.ListenAndServe(port, r))
}
fail2:
return nil
}

View file

@ -2,7 +2,6 @@ package server
import (
"encoding/binary"
"fmt"
"net"
"github.com/lbryio/herald.go/internal"
@ -53,27 +52,15 @@ func (s *Server) DoNotify(heightHash *internal.HeightHash) error {
// RunNotifier Runs the notfying action forever
func (s *Server) RunNotifier() error {
for notification := range s.NotifierChan {
switch note := notification.(type) {
case internal.HeightHash:
heightHash := note
s.DoNotify(&heightHash)
// Do we need this?
// case peerNotification:
// peer, _ := notification.(peerNotification)
// s.notifyPeerSubs(&Peer{Address: peer.address, Port: peer.port})
default:
logrus.Warn("unknown notification type")
}
s.sessionManager.doNotify(notification)
for heightHash := range s.NotifierChan {
s.DoNotify(heightHash)
}
return nil
}
// NotifierServer implements the TCP protocol for height/blockheader notifications
func (s *Server) NotifierServer() error {
s.Grp.Add(1)
address := ":" + fmt.Sprintf("%d", s.Args.NotifierPort)
address := ":" + s.Args.NotifierPort
addr, err := net.ResolveTCPAddr("tcp", address)
if err != nil {
return err
@ -85,27 +72,11 @@ func (s *Server) NotifierServer() error {
}
defer listen.Close()
rdyCh := make(chan bool)
for {
var conn net.Conn
var err error
logrus.Info("Waiting for connection")
go func() {
conn, err = listen.Accept()
rdyCh <- true
}()
select {
case <-s.Grp.Ch():
s.Grp.Done()
return nil
case <-rdyCh:
logrus.Info("Connection accepted")
}
conn, err := listen.Accept()
if err != nil {
logrus.Warn(err)
continue

View file

@ -1,6 +1,7 @@
package server_test
import (
"context"
"encoding/hex"
"fmt"
"net"
@ -9,32 +10,11 @@ import (
"github.com/lbryio/herald.go/internal"
"github.com/lbryio/herald.go/server"
"github.com/lbryio/lbry.go/v3/extras/stop"
"github.com/sirupsen/logrus"
)
const defaultBufferSize = 1024
func subReady(s *server.Server) error {
sleepTime := time.Millisecond * 100
for {
if sleepTime > time.Second {
return fmt.Errorf("timeout")
}
s.HeightSubsMut.RLock()
if len(s.HeightSubs) > 0 {
s.HeightSubsMut.RUnlock()
return nil
}
s.HeightSubsMut.RUnlock()
logrus.Warn("waiting for subscriber")
time.Sleep(sleepTime)
sleepTime = sleepTime * 2
}
}
func tcpConnReady(addr string) (net.Conn, error) {
sleepTime := time.Millisecond * 100
for {
@ -67,14 +47,14 @@ func tcpRead(conn net.Conn) ([]byte, error) {
}
func TestNotifierServer(t *testing.T) {
args := server.MakeDefaultTestArgs()
ctx := stop.NewDebug()
args := makeDefaultArgs()
ctx := context.Background()
hub := server.MakeHubServer(ctx, args)
go hub.NotifierServer()
go hub.RunNotifier()
addr := fmt.Sprintf(":%d", args.NotifierPort)
addr := fmt.Sprintf(":%s", args.NotifierPort)
logrus.Info(addr)
conn, err := tcpConnReady(addr)
if err != nil {
@ -96,14 +76,11 @@ func TestNotifierServer(t *testing.T) {
// Hacky but needed because if the reader isn't ready
// before the writer sends it won't get the data
err = subReady(hub)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
hash, _ := hex.DecodeString("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
logrus.Warn("sending hash")
hub.NotifierChan <- internal.HeightHash{Height: 1, BlockHash: hash}
hub.NotifierChan <- &internal.HeightHash{Height: 1, BlockHash: hash}
res := <-resCh
logrus.Info(string(res))

View file

@ -11,7 +11,6 @@ import (
pb "github.com/lbryio/herald.go/protobuf/go"
server "github.com/lbryio/herald.go/server"
"github.com/lbryio/lbry.go/v3/extras/stop"
"github.com/olivere/elastic/v7"
)
@ -56,14 +55,13 @@ func TestSearch(t *testing.T) {
w.Write([]byte(resp))
}
ctx := context.Background()
stopGroup := stop.NewDebug()
args := server.MakeDefaultTestArgs()
hubServer := server.MakeHubServer(stopGroup, args)
context := context.Background()
args := makeDefaultArgs()
hubServer := server.MakeHubServer(context, args)
req := &pb.SearchRequest{
Text: "asdf",
}
out, err := hubServer.Search(ctx, req)
out, err := hubServer.Search(context, req)
if err != nil {
log.Println(err)
}

View file

@ -8,27 +8,25 @@ import (
"fmt"
"hash"
"io/ioutil"
golog "log"
"log"
"net"
"net/http"
"os"
"regexp"
"strconv"
"sync"
"time"
"github.com/ReneKroon/ttlcache/v2"
"github.com/lbryio/herald.go/db"
"github.com/lbryio/herald.go/internal"
"github.com/lbryio/herald.go/internal/metrics"
"github.com/lbryio/herald.go/meta"
pb "github.com/lbryio/herald.go/protobuf/go"
"github.com/lbryio/lbcd/chaincfg"
lbcd "github.com/lbryio/lbcd/rpcclient"
"github.com/lbryio/lbry.go/v3/extras/stop"
"github.com/olivere/elastic/v7"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
logrus "github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"
)
@ -40,7 +38,6 @@ type Server struct {
WeirdCharsRe *regexp.Regexp
DB *db.ReadOnlyDBColumnFamily
Chain *chaincfg.Params
DaemonClient *lbcd.Client
EsClient *elastic.Client
QueryCache *ttlcache.Cache
S256 *hash.Hash
@ -56,10 +53,7 @@ type Server struct {
ExternalIP net.IP
HeightSubs map[net.Addr]net.Conn
HeightSubsMut sync.RWMutex
NotifierChan chan interface{}
Grp *stop.Group
notiferListener *net.TCPListener
sessionManager *sessionManager
NotifierChan chan *internal.HeightHash
pb.UnimplementedHubServer
}
@ -140,8 +134,7 @@ func (s *Server) PeerServersLoadOrStore(peer *Peer) (actual *Peer, loaded bool)
// Run "main" function for starting the server. This blocks.
func (s *Server) Run() {
address := ":" + strconv.Itoa(s.Args.Port)
l, err := net.Listen("tcp", address)
l, err := net.Listen("tcp", ":"+s.Args.Port)
if err != nil {
log.Fatalf("failed to listen: %v", err)
}
@ -156,50 +149,31 @@ func (s *Server) Run() {
}
}
func (s *Server) Stop() {
log.Println("Shutting down server...")
if s.EsClient != nil {
log.Println("Stopping es client...")
s.EsClient.Stop()
}
if s.GrpcServer != nil {
log.Println("Stopping grpc server...")
s.GrpcServer.GracefulStop()
}
log.Println("Stopping other server threads...")
s.Grp.StopAndWait()
if s.DB != nil {
log.Println("Stopping database connection...")
s.DB.Shutdown()
}
log.Println("Returning from Stop...")
}
func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, error) {
tmpName, err := os.MkdirTemp("", "go-lbry-hub")
func LoadDatabase(args *Args) (*db.ReadOnlyDBColumnFamily, error) {
tmpName, err := ioutil.TempDir("", "go-lbry-hub")
if err != nil {
log.Info(err)
logrus.Info(err)
log.Fatal(err)
}
log.Info("tmpName", tmpName)
logrus.Info("tmpName", tmpName)
if err != nil {
log.Info(err)
logrus.Info(err)
}
myDB, err := db.GetProdDB(args.DBPath, tmpName, grp)
myDB, _, err := db.GetProdDB(args.DBPath, tmpName)
// dbShutdown = func() {
// db.Shutdown(myDB)
// }
if err != nil {
// Can't load the db, fail loudly
log.Info(err)
logrus.Info(err)
log.Fatalln(err)
}
if myDB.LastState != nil {
log.Infof("DB version: %v", myDB.LastState.DBVersion)
log.Infof("height: %v", myDB.LastState.Height)
log.Infof("genesis: %v", myDB.LastState.Genesis.String())
log.Infof("tip: %v", myDB.LastState.Tip.String())
log.Infof("tx count: %v", myDB.LastState.TxCount)
logrus.Infof("DB version: %v", myDB.LastState.DBVersion)
logrus.Infof("height: %v", myDB.LastState.Height)
logrus.Infof("tip: %v", myDB.LastState.Tip.String())
logrus.Infof("tx count: %v", myDB.LastState.TxCount)
}
blockingChannelHashes := make([][]byte, 0, 10)
@ -210,7 +184,7 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
for _, id := range args.BlockingChannelIds {
hash, err := hex.DecodeString(id)
if err != nil {
log.Warn("Invalid channel id: ", id)
logrus.Warn("Invalid channel id: ", id)
continue
}
blockingChannelHashes = append(blockingChannelHashes, hash)
@ -220,7 +194,7 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
for _, id := range args.FilteringChannelIds {
hash, err := hex.DecodeString(id)
if err != nil {
log.Warn("Invalid channel id: ", id)
logrus.Warn("Invalid channel id: ", id)
continue
}
filteringChannelHashes = append(filteringChannelHashes, hash)
@ -231,10 +205,10 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
myDB.FilteringChannelHashes = filteringChannelHashes
if len(filteringIds) > 0 {
log.Infof("filtering claims reposted by channels: %+s", filteringIds)
logrus.Infof("filtering claims reposted by channels: %+s", filteringIds)
}
if len(blockingIds) > 0 {
log.Infof("blocking claims reposted by channels: %+s", blockingIds)
logrus.Infof("blocking claims reposted by channels: %+s", blockingIds)
}
return myDB, nil
@ -243,7 +217,7 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
// MakeHubServer takes the arguments given to a hub when it's started and
// initializes everything. It loads information about previously known peers,
// creates needed internal data structures, and initializes goroutines.
func MakeHubServer(grp *stop.Group, args *Args) *Server {
func MakeHubServer(ctx context.Context, args *Args) *Server {
grpcServer := grpc.NewServer(grpc.NumStreamWorkers(0))
multiSpaceRe, err := regexp.Compile(`\s{2,}`)
@ -256,34 +230,9 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
log.Fatal(err)
}
var lbcdClient *lbcd.Client = nil
if args.DaemonURL != nil && args.DaemonURL.Host != "" {
var rpcCertificate []byte
if args.DaemonCAPath != "" {
rpcCertificate, err = ioutil.ReadFile(args.DaemonCAPath)
if err != nil {
log.Fatalf("failed to read SSL certificate from path: %v", args.DaemonCAPath)
}
}
log.Warnf("connecting to lbcd daemon at %v...", args.DaemonURL.Host)
password, _ := args.DaemonURL.User.Password()
cfg := &lbcd.ConnConfig{
Host: args.DaemonURL.Host,
User: args.DaemonURL.User.Username(),
Pass: password,
HTTPPostMode: true,
DisableTLS: rpcCertificate == nil,
Certificates: rpcCertificate,
}
lbcdClient, err = lbcd.New(cfg, nil)
if err != nil {
log.Fatalf("lbcd daemon connection failed: %v", err)
}
}
var esClient *elastic.Client = nil
var client *elastic.Client = nil
if !args.DisableEs {
esUrl := args.EsHost + ":" + fmt.Sprintf("%d", args.EsPort)
esUrl := args.EsHost + ":" + args.EsPort
opts := []elastic.ClientOptionFunc{
elastic.SetSniff(true),
elastic.SetSnifferTimeoutStartup(time.Second * 60),
@ -291,9 +240,9 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
elastic.SetURL(esUrl),
}
if args.Debug {
opts = append(opts, elastic.SetTraceLog(golog.New(os.Stderr, "[[ELASTIC]]", 0)))
opts = append(opts, elastic.SetTraceLog(log.New(os.Stderr, "[[ELASTIC]]", 0)))
}
esClient, err = elastic.NewClient(opts...)
client, err = elastic.NewClient(opts...)
if err != nil {
log.Fatal(err)
}
@ -317,18 +266,18 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
//TODO: is this the right place to load the db?
var myDB *db.ReadOnlyDBColumnFamily
// var dbShutdown = func() {}
if !args.DisableResolve {
myDB, err = LoadDatabase(args, grp)
myDB, err = LoadDatabase(args)
if err != nil {
log.Warning(err)
logrus.Warning(err)
}
}
// Determine which chain to use based on db and cli values
dbChain := (*chaincfg.Params)(nil)
if myDB != nil && myDB.LastState != nil {
if myDB != nil && myDB.LastState != nil && myDB.LastState.Genesis != nil {
// The chain params can be inferred from DBStateValue.
switch myDB.LastState.Genesis.Hash {
switch *myDB.LastState.Genesis {
case *chaincfg.MainNetParams.GenesisHash:
dbChain = &chaincfg.MainNetParams
case *chaincfg.TestNet3Params.GenesisHash:
@ -351,7 +300,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
chain := chaincfg.MainNetParams
if dbChain != nil && cliChain != nil {
if dbChain != cliChain {
log.Warnf("network: %v (from db) conflicts with %v (from cli)", dbChain.Name, cliChain.Name)
logrus.Warnf("network: %v (from db) conflicts with %v (from cli)", dbChain.Name, cliChain.Name)
}
chain = *dbChain
} else if dbChain != nil {
@ -359,11 +308,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
} else if cliChain != nil {
chain = *cliChain
}
log.Infof("network: %v", chain.Name)
args.GenesisHash = chain.GenesisHash.String()
sessionGrp := stop.New(grp)
logrus.Infof("network: %v", chain.Name)
s := &Server{
GrpcServer: grpcServer,
@ -372,8 +317,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
WeirdCharsRe: weirdCharsRe,
DB: myDB,
Chain: &chain,
DaemonClient: lbcdClient,
EsClient: esClient,
EsClient: client,
QueryCache: cache,
S256: &s256,
LastRefreshCheck: time.Now(),
@ -388,39 +332,27 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
ExternalIP: net.IPv4(127, 0, 0, 1),
HeightSubs: make(map[net.Addr]net.Conn),
HeightSubsMut: sync.RWMutex{},
NotifierChan: make(chan interface{}, 1),
Grp: grp,
sessionManager: nil,
NotifierChan: make(chan *internal.HeightHash),
}
// FIXME: HACK
s.sessionManager = newSessionManager(s, myDB, args, sessionGrp, &chain, lbcdClient)
// Start up our background services
if !args.DisableResolve && !args.DisableRocksDBRefresh {
log.Info("Running detect changes")
logrus.Info("Running detect changes")
myDB.RunDetectChanges(s.NotifierChan)
}
if !args.DisableBlockingAndFiltering {
myDB.RunGetBlocksAndFilters()
}
if !args.DisableStartPrometheus {
go s.prometheusEndpoint(fmt.Sprintf("%d", s.Args.PrometheusPort), "metrics")
go s.prometheusEndpoint(s.Args.PrometheusPort, "metrics")
}
if !args.DisableStartUDP {
go func() {
err := s.UDPServer(s.Args.Port)
err := s.UDPServer()
if err != nil {
log.Errorf("UDP Server (%d) failed! %v", s.Args.Port, err)
log.Println("UDP Server failed!", err)
}
}()
if s.Args.JSONRPCPort != 0 {
go func() {
err := s.UDPServer(s.Args.JSONRPCPort)
if err != nil {
log.Errorf("UDP Server (%d) failed! %v", s.Args.JSONRPCPort, err)
}
}()
}
}
if !args.DisableStartNotifier {
go func() {
@ -461,7 +393,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
// for this hub to allow for metric tracking.
func (s *Server) prometheusEndpoint(port string, endpoint string) {
http.Handle("/"+endpoint, promhttp.Handler())
log.Printf("listening on :%s /%s\n", port, endpoint)
log.Println(fmt.Sprintf("listening on :%s /%s", port, endpoint))
err := http.ListenAndServe(":"+port, nil)
log.Fatalln("Shouldn't happen??!?!", err)
}
@ -619,7 +551,7 @@ func InternalResolve(urls []string, DB *db.ReadOnlyDBColumnFamily) (*pb.Outputs,
BlockedTotal: 0, //TODO
}
log.Warn(res)
logrus.Warn(res)
return res, nil
}

View file

@ -1,11 +1,5 @@
package server
import (
"github.com/lbryio/herald.go/db"
"github.com/lbryio/lbcd/chaincfg"
"github.com/lbryio/lbry.go/v3/extras/stop"
)
func (s *Server) AddPeerExported() func(*Peer, bool, bool) error {
return s.addPeer
}
@ -13,7 +7,3 @@ func (s *Server) AddPeerExported() func(*Peer, bool, bool) error {
func (s *Server) GetNumPeersExported() func() int64 {
return s.getNumPeers
}
func NewSessionManagerExported(server *Server, db *db.ReadOnlyDBColumnFamily, args *Args, grp *stop.Group, chain *chaincfg.Params) *sessionManager {
return newSessionManager(server, db, args, grp, chain, nil)
}

View file

@ -1,631 +0,0 @@
package server
import (
"bytes"
"encoding/hex"
"encoding/json"
"fmt"
"net"
"net/rpc"
"net/rpc/jsonrpc"
"strings"
"sync"
"time"
"unsafe"
"github.com/lbryio/herald.go/db"
"github.com/lbryio/herald.go/internal"
"github.com/lbryio/lbcd/chaincfg"
"github.com/lbryio/lbcd/chaincfg/chainhash"
lbcd "github.com/lbryio/lbcd/rpcclient"
"github.com/lbryio/lbcd/wire"
"github.com/lbryio/lbry.go/v3/extras/stop"
log "github.com/sirupsen/logrus"
)
type headerNotification struct {
internal.HeightHash
blockHeaderElectrum *BlockHeaderElectrum
blockHeaderStr string
}
type hashXNotification struct {
hashX [HASHX_LEN]byte
status []byte
statusStr string
}
type peerNotification struct {
address string
port string
}
type session struct {
id uintptr
addr net.Addr
conn net.Conn
// hashXSubs maps hashX to the original subscription key (address or scripthash)
hashXSubs map[[HASHX_LEN]byte]string
// headersSub indicates header subscription
headersSub bool
// peersSub indicates peer subscription
peersSub bool
// headersSubRaw indicates the header subscription mode
headersSubRaw bool
// client provides the ability to send notifications
client rpc.ClientCodec
clientSeq uint64
// lastRecv records time of last incoming data
lastRecv time.Time
// lastSend records time of last outgoing data
lastSend time.Time
}
func (s *session) doNotify(notification interface{}) {
var method string
var params interface{}
switch note := notification.(type) {
case headerNotification:
if !s.headersSub {
return
}
heightHash := note.HeightHash
method = "blockchain.headers.subscribe"
if s.headersSubRaw {
header := note.blockHeaderStr
if len(header) == 0 {
header = hex.EncodeToString(note.BlockHeader[:])
}
params = &HeadersSubscribeRawResp{
Hex: header,
Height: uint32(heightHash.Height),
}
} else {
header := note.blockHeaderElectrum
if header == nil { // not initialized
header = newBlockHeaderElectrum((*[HEADER_SIZE]byte)(note.BlockHeader), uint32(heightHash.Height))
}
params = header
}
case hashXNotification:
orig, ok := s.hashXSubs[note.hashX]
if !ok {
return
}
if len(orig) == 64 {
method = "blockchain.scripthash.subscribe"
} else {
method = "blockchain.address.subscribe"
}
status := note.statusStr
if len(status) == 0 {
status = hex.EncodeToString(note.status)
}
params = []string{orig, status}
case peerNotification:
if !s.peersSub {
return
}
method = "server.peers.subscribe"
params = []string{note.address, note.port}
default:
log.Warnf("unknown notification type: %v", notification)
return
}
// Send the notification.
s.clientSeq += 1
req := &rpc.Request{
ServiceMethod: method,
Seq: s.clientSeq,
}
err := s.client.WriteRequest(req, params)
if err != nil {
log.Warnf("error: %v", err)
}
// Bump last send time.
s.lastSend = time.Now()
}
type sessionMap map[uintptr]*session
type sessionManager struct {
// sessionsMut protects sessions, headerSubs, hashXSubs state
sessionsMut sync.RWMutex
sessions sessionMap
// sessionsWait sync.WaitGroup
grp *stop.Group
sessionsMax int
sessionTimeout time.Duration
manageTicker *time.Ticker
db *db.ReadOnlyDBColumnFamily
args *Args
server *Server
chain *chaincfg.Params
lbcd *lbcd.Client
// peerSubs are sessions subscribed via 'blockchain.peers.subscribe'
peerSubs sessionMap
// headerSubs are sessions subscribed via 'blockchain.headers.subscribe'
headerSubs sessionMap
// hashXSubs are sessions subscribed via 'blockchain.{address,scripthash}.subscribe'
hashXSubs map[[HASHX_LEN]byte]sessionMap
}
func newSessionManager(server *Server, db *db.ReadOnlyDBColumnFamily, args *Args, grp *stop.Group, chain *chaincfg.Params, lbcd *lbcd.Client) *sessionManager {
return &sessionManager{
sessions: make(sessionMap),
grp: grp,
sessionsMax: args.MaxSessions,
sessionTimeout: time.Duration(args.SessionTimeout) * time.Second,
manageTicker: time.NewTicker(time.Duration(max(5, args.SessionTimeout/20)) * time.Second),
db: db,
args: args,
server: server,
chain: chain,
lbcd: lbcd,
peerSubs: make(sessionMap),
headerSubs: make(sessionMap),
hashXSubs: make(map[[HASHX_LEN]byte]sessionMap),
}
}
func (sm *sessionManager) start() {
sm.grp.Add(1)
go sm.manage()
}
func (sm *sessionManager) stop() {
sm.sessionsMut.Lock()
defer sm.sessionsMut.Unlock()
sm.headerSubs = make(sessionMap)
sm.hashXSubs = make(map[[HASHX_LEN]byte]sessionMap)
for _, sess := range sm.sessions {
sess.client.Close()
sess.conn.Close()
}
sm.sessions = make(sessionMap)
}
func (sm *sessionManager) manage() {
for {
sm.sessionsMut.Lock()
for _, sess := range sm.sessions {
if time.Since(sess.lastRecv) > sm.sessionTimeout {
sm.removeSessionLocked(sess)
log.Infof("session %v timed out", sess.addr.String())
}
}
sm.sessionsMut.Unlock()
// Wait for next management clock tick.
select {
case <-sm.grp.Ch():
sm.grp.Done()
return
case <-sm.manageTicker.C:
continue
}
}
}
func (sm *sessionManager) addSession(conn net.Conn) *session {
sm.sessionsMut.Lock()
sess := &session{
addr: conn.RemoteAddr(),
conn: conn,
hashXSubs: make(map[[11]byte]string),
client: jsonrpc.NewClientCodec(conn),
lastRecv: time.Now(),
}
sess.id = uintptr(unsafe.Pointer(sess))
sm.sessions[sess.id] = sess
sm.sessionsMut.Unlock()
// Create a new RPC server. These services are linked to the
// session, which allows RPC handlers to know the session for
// each request and update subscriptions.
s1 := rpc.NewServer()
// Register "server.{features,banner,version}" handlers.
serverSvc := &ServerService{sm.args}
err := s1.RegisterName("server", serverSvc)
if err != nil {
log.Errorf("RegisterName: %v\n", err)
}
// Register "server.peers" handlers.
peersSvc := &PeersService{Server: sm.server}
err = s1.RegisterName("server.peers", peersSvc)
if err != nil {
log.Errorf("RegisterName: %v\n", err)
}
// Register "blockchain.claimtrie.*"" handlers.
claimtrieSvc := &ClaimtrieService{sm.db, sm.server}
err = s1.RegisterName("blockchain.claimtrie", claimtrieSvc)
if err != nil {
log.Errorf("RegisterName: %v\n", err)
}
// Register "blockchain.{block,address,scripthash,transaction}.*" handlers.
blockchainSvc := &BlockchainBlockService{sm.db, sm.chain}
err = s1.RegisterName("blockchain.block", blockchainSvc)
if err != nil {
log.Errorf("RegisterName: %v\n", err)
goto fail
}
err = s1.RegisterName("blockchain.headers", &BlockchainHeadersService{sm.db, sm.chain, sm, sess})
if err != nil {
log.Errorf("RegisterName: %v\n", err)
goto fail
}
err = s1.RegisterName("blockchain.address", &BlockchainAddressService{sm.db, sm.chain, sm, sess})
if err != nil {
log.Errorf("RegisterName: %v\n", err)
goto fail
}
err = s1.RegisterName("blockchain.scripthash", &BlockchainScripthashService{sm.db, sm.chain, sm, sess})
if err != nil {
log.Errorf("RegisterName: %v\n", err)
goto fail
}
err = s1.RegisterName("blockchain.transaction", &BlockchainTransactionService{sm.db, sm.chain, sm})
if err != nil {
log.Errorf("RegisterName: %v\n", err)
goto fail
}
sm.grp.Add(1)
go func() {
s1.ServeCodec(&sessionServerCodec{jsonrpc.NewServerCodec(newJsonPatchingCodec(conn)), sess})
log.Infof("session %v goroutine exit", sess.addr.String())
sm.grp.Done()
}()
return sess
fail:
sm.removeSession(sess)
return nil
}
func (sm *sessionManager) removeSession(sess *session) {
sm.sessionsMut.Lock()
defer sm.sessionsMut.Unlock()
sm.removeSessionLocked(sess)
}
func (sm *sessionManager) removeSessionLocked(sess *session) {
if sess.headersSub {
delete(sm.headerSubs, sess.id)
}
for hashX := range sess.hashXSubs {
subs, ok := sm.hashXSubs[hashX]
if !ok {
continue
}
delete(subs, sess.id)
}
delete(sm.sessions, sess.id)
sess.client.Close()
sess.conn.Close()
}
func (sm *sessionManager) broadcastTx(rawTx []byte) (*chainhash.Hash, error) {
var msgTx wire.MsgTx
err := msgTx.Deserialize(bytes.NewReader(rawTx))
if err != nil {
return nil, err
}
return sm.lbcd.SendRawTransaction(&msgTx, false)
}
func (sm *sessionManager) peersSubscribe(sess *session, subscribe bool) {
sm.sessionsMut.Lock()
defer sm.sessionsMut.Unlock()
if subscribe {
sm.peerSubs[sess.id] = sess
sess.peersSub = true
return
}
delete(sm.peerSubs, sess.id)
sess.peersSub = false
}
func (sm *sessionManager) headersSubscribe(sess *session, raw bool, subscribe bool) {
sm.sessionsMut.Lock()
defer sm.sessionsMut.Unlock()
if subscribe {
sm.headerSubs[sess.id] = sess
sess.headersSub = true
sess.headersSubRaw = raw
return
}
delete(sm.headerSubs, sess.id)
sess.headersSub = false
sess.headersSubRaw = false
}
func (sm *sessionManager) hashXSubscribe(sess *session, hashX []byte, original string, subscribe bool) {
sm.sessionsMut.Lock()
defer sm.sessionsMut.Unlock()
var key [HASHX_LEN]byte
copy(key[:], hashX)
subs, ok := sm.hashXSubs[key]
if subscribe {
if !ok {
subs = make(sessionMap)
sm.hashXSubs[key] = subs
}
subs[sess.id] = sess
sess.hashXSubs[key] = original
return
}
if ok {
delete(subs, sess.id)
if len(subs) == 0 {
delete(sm.hashXSubs, key)
}
}
delete(sess.hashXSubs, key)
}
func (sm *sessionManager) doNotify(notification interface{}) {
switch note := notification.(type) {
case internal.HeightHash:
// The HeightHash notification translates to headerNotification.
notification = &headerNotification{HeightHash: note}
}
sm.sessionsMut.RLock()
var subsCopy sessionMap
switch note := notification.(type) {
case headerNotification:
log.Infof("header notification @ %#v", note)
subsCopy = sm.headerSubs
if len(subsCopy) > 0 {
hdr := [HEADER_SIZE]byte{}
copy(hdr[:], note.BlockHeader)
note.blockHeaderElectrum = newBlockHeaderElectrum(&hdr, uint32(note.Height))
note.blockHeaderStr = hex.EncodeToString(note.BlockHeader[:])
}
case hashXNotification:
log.Infof("hashX notification @ %#v", note)
hashXSubs, ok := sm.hashXSubs[note.hashX]
if ok {
subsCopy = hashXSubs
}
if len(subsCopy) > 0 {
note.statusStr = hex.EncodeToString(note.status)
}
case peerNotification:
subsCopy = sm.peerSubs
default:
log.Warnf("unknown notification type: %v", notification)
}
sm.sessionsMut.RUnlock()
// Deliver notification to relevant sessions.
for _, sess := range subsCopy {
sess.doNotify(notification)
}
// Produce secondary hashXNotification(s) corresponding to the headerNotification.
switch note := notification.(type) {
case headerNotification:
touched, err := sm.db.GetTouchedHashXs(uint32(note.Height))
if err != nil {
log.Errorf("failed to get touched hashXs at height %v, error: %v", note.Height, err)
break
}
for _, hashX := range touched {
hashXstatus, err := sm.db.GetStatus(hashX)
if err != nil {
log.Errorf("failed to get status of hashX %v, error: %v", hashX, err)
continue
}
note2 := hashXNotification{}
copy(note2.hashX[:], hashX)
note2.status = hashXstatus
sm.doNotify(note2)
}
}
}
type sessionServerCodec struct {
rpc.ServerCodec
sess *session
}
// ReadRequestHeader provides ability to rewrite the incoming
// request "method" field. For example:
//
// blockchain.block.get_header -> blockchain.block.Get_header
// blockchain.address.listunspent -> blockchain.address.Listunspent
//
// This makes the "method" string compatible with rpc.Server
// requirements.
func (c *sessionServerCodec) ReadRequestHeader(req *rpc.Request) error {
log.Infof("from %v receive header", c.sess.addr.String())
err := c.ServerCodec.ReadRequestHeader(req)
if err != nil {
log.Warnf("error: %v", err)
return err
}
log.Infof("from %v receive header: %#v", c.sess.addr.String(), *req)
rawMethod := req.ServiceMethod
parts := strings.Split(rawMethod, ".")
if len(parts) < 2 {
return fmt.Errorf("blockchain rpc: service/method ill-formed: %q", rawMethod)
}
service := strings.Join(parts[0:len(parts)-1], ".")
method := parts[len(parts)-1]
if len(method) < 1 {
return fmt.Errorf("blockchain rpc: method ill-formed: %q", method)
}
method = strings.ToUpper(string(method[0])) + string(method[1:])
req.ServiceMethod = service + "." + method
return err
}
// ReadRequestBody wraps the regular implementation, but updates session stats too.
func (c *sessionServerCodec) ReadRequestBody(params any) error {
log.Infof("from %v receive body", c.sess.addr.String())
err := c.ServerCodec.ReadRequestBody(params)
if err != nil {
log.Warnf("error: %v", err)
return err
}
log.Infof("from %v receive body: %#v", c.sess.addr.String(), params)
// Bump last receive time.
c.sess.lastRecv = time.Now()
return err
}
// WriteResponse wraps the regular implementation, but updates session stats too.
func (c *sessionServerCodec) WriteResponse(resp *rpc.Response, reply any) error {
log.Infof("respond to %v", c.sess.addr.String())
err := c.ServerCodec.WriteResponse(resp, reply)
if err != nil {
return err
}
// Bump last send time.
c.sess.lastSend = time.Now()
return err
}
// serverRequest is a duplicate of serverRequest from
// net/rpc/jsonrpc/server.go with an added Version which
// we can check.
type serverRequest struct {
Version string `json:"jsonrpc"`
Method string `json:"method"`
Params *json.RawMessage `json:"params"`
Id *json.RawMessage `json:"id"`
}
// serverResponse is a duplicate of serverResponse from
// net/rpc/jsonrpc/server.go with an added Version which
// we can set at will.
type serverResponse struct {
Version string `json:"jsonrpc"`
Id *json.RawMessage `json:"id"`
Result any `json:"result,omitempty"`
Error any `json:"error,omitempty"`
}
// jsonPatchingCodec is able to intercept the JSON requests/responses
// and tweak them. Currently, it appears we need to make several changes:
// 1) add "jsonrpc": "2.0" (or "jsonrpc": "1.0") in response
// 2) add newline to frame response
// 3) add "params": [] when "params" is missing
// 4) replace params ["arg1", "arg2", ...] with [["arg1", "arg2", ...]]
type jsonPatchingCodec struct {
conn net.Conn
inBuffer *bytes.Buffer
dec *json.Decoder
enc *json.Encoder
outBuffer *bytes.Buffer
}
func newJsonPatchingCodec(conn net.Conn) *jsonPatchingCodec {
buf1, buf2 := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
return &jsonPatchingCodec{
conn: conn,
inBuffer: buf1,
dec: json.NewDecoder(buf1),
enc: json.NewEncoder(buf2),
outBuffer: buf2,
}
}
func (c *jsonPatchingCodec) Read(p []byte) (n int, err error) {
if c.outBuffer.Len() > 0 {
// Return remaining decoded bytes.
return c.outBuffer.Read(p)
}
// Buffer contents consumed. Try to decode more JSON.
// Read until framing newline. This allows us to print the raw request.
for !bytes.ContainsAny(c.inBuffer.Bytes(), "\n") {
var buf [1024]byte
n, err = c.conn.Read(buf[:])
if err != nil {
return 0, err
}
c.inBuffer.Write(buf[:n])
}
log.Infof("raw request: %v", c.inBuffer.String())
var req serverRequest
err = c.dec.Decode(&req)
if err != nil {
return 0, err
}
if req.Params != nil {
n := len(*req.Params)
if n < 2 || (*req.Params)[0] != '[' && (*req.Params)[n-1] != ']' {
// This is an error, but we're not going to try to correct it.
goto encode
}
// FIXME: The heuristics here don't cover all possibilities.
// For example: [{obj1}, {obj2}] or ["foo,bar"] would not
// be handled correctly.
bracketed := (*req.Params)[1 : n-1]
n = len(bracketed)
if n > 1 && (bracketed[0] == '{' || bracketed[0] == '[') {
// Probable single object or list argument.
goto encode
}
// The params look like ["arg1", "arg2", "arg3", ...].
// We're in trouble because our jsonrpc library does not
// handle this. So pack these args in an inner list.
// The handler method will receive ONE list argument.
params := json.RawMessage(fmt.Sprintf("[[%s]]", bracketed))
req.Params = &params
} else {
// Add empty argument list if params omitted.
params := json.RawMessage("[]")
req.Params = &params
}
encode:
// Encode the request. This allows us to print the patched request.
buf, err := json.Marshal(req)
if err != nil {
return 0, err
}
log.Infof("patched request: %v", string(buf))
err = c.enc.Encode(req)
if err != nil {
return 0, err
}
return c.outBuffer.Read(p)
}
func (c *jsonPatchingCodec) Write(p []byte) (n int, err error) {
log.Infof("raw response: %v", string(p))
var resp serverResponse
err = json.Unmarshal(p, &resp)
if err != nil {
return 0, err
}
// Add "jsonrpc": "2.0" if missing.
if len(resp.Version) == 0 {
resp.Version = "2.0"
}
buf, err := json.Marshal(resp)
if err != nil {
return 0, err
}
log.Infof("patched response: %v", string(buf))
// Add newline for framing.
return c.conn.Write(append(buf, '\n'))
}
func (c *jsonPatchingCodec) Close() error {
return c.conn.Close()
}

View file

@ -219,9 +219,8 @@ func UDPPing(ip, port string) (*SPVPong, error) {
// UDPServer is a goroutine that starts an udp server that implements the hubs
// Ping/Pong protocol to find out about each other without making full TCP
// connections.
func (s *Server) UDPServer(port int) error {
address := ":" + strconv.Itoa(port)
func (s *Server) UDPServer() error {
address := ":" + s.Args.Port
tip := make([]byte, 32)
addr, err := net.ResolveUDPAddr("udp", address)
if err != nil {

View file

@ -11,7 +11,7 @@ import (
// TestUDPPing tests UDPPing correctness against prod server.
func TestUDPPing(t *testing.T) {
args := server.MakeDefaultTestArgs()
args := makeDefaultArgs()
args.DisableStartUDP = true
tests := []struct {

View file

@ -8,13 +8,12 @@ import (
"os"
"os/signal"
"github.com/lbryio/lbry.go/v3/extras/stop"
log "github.com/sirupsen/logrus"
)
// shutdownRequestChannel is used to initiate shutdown from one of the
// subsystems using the same code paths as when an interrupt signal is received.
var shutdownRequestChannel = make(stop.Chan)
var shutdownRequestChannel = make(chan struct{})
// interruptSignals defines the default signals to catch in order to do a proper
// shutdown. This may be modified during init depending on the platform.
@ -49,6 +48,7 @@ func interruptListener() <-chan struct{} {
case sig := <-interruptChannel:
log.Infof("Received signal (%s). Already "+
"shutting down...", sig)
case <-shutdownRequestChannel:
log.Info("Shutdown requested. Already " +
"shutting down...")

View file

@ -1,18 +0,0 @@
Si,,
S,53000000a420c44374f4f399ab4807fa1901eefc8701000e94ad0297ec210000,00000000000f4240
S,53000000c27eef5ea69e0d73f118826c7e326bb46901000773de00371d660000,000000001dcd6500
S,5300000110e40894573f528c393fbcec7a472ec85301000d069c01516b320000,0000000000989680
S,5300000324e40fcb63a0b517a3660645e9bd99244a01000f2fd8030bb6ba0000,00000000000f4240
S,5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff4030bc8a50000,0000000001312d00
S,5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff6030bc8b00000,0000000000000003
S,5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff7030bc8b10000,0000000000000002
S,5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff9030bc8cf0000,0000000000000001
S,53000003d1538a0f19f5cd4bc1a62cc294f5c8993401000c816a011c7c990000,00000000000f4240
S,53000008d47beeff8325e795a8604226145b01702b01000ef1ed02dbb2a20000,00000000000186a0
S,5300000906499e073e94370ceff37cb21c2821244401000fa7c40369842d0000,00000000000186a0
S,5300000906499e073e94370ceff37cb21c2821244402000fa7c403698fff0000,00000000000000a1
S,5300000906499e073e94370ceff37cb21c2821244402000fa7c80369f0010000,000000000000000f
S,53000009c3172e034a255f3c03566dca84bb9f046a01000e07020225c69c0000,000000000007a120
S,53000009ca6e0caaaef16872b4bd4f6f1b8c2363e201000eb5af02b169560000,00000000000f4240
i,6900000324e40fcb63a0b517a3660645e9bd99244a,0000000001406f460000000001312d06
i,6900000906499e073e94370ceff37cb21c28212444,000000000001875000000000000000b0
1 Si
2 S 53000000a420c44374f4f399ab4807fa1901eefc8701000e94ad0297ec210000 00000000000f4240
3 S 53000000c27eef5ea69e0d73f118826c7e326bb46901000773de00371d660000 000000001dcd6500
4 S 5300000110e40894573f528c393fbcec7a472ec85301000d069c01516b320000 0000000000989680
5 S 5300000324e40fcb63a0b517a3660645e9bd99244a01000f2fd8030bb6ba0000 00000000000f4240
6 S 5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff4030bc8a50000 0000000001312d00
7 S 5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff6030bc8b00000 0000000000000003
8 S 5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff7030bc8b10000 0000000000000002
9 S 5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff9030bc8cf0000 0000000000000001
10 S 53000003d1538a0f19f5cd4bc1a62cc294f5c8993401000c816a011c7c990000 00000000000f4240
11 S 53000008d47beeff8325e795a8604226145b01702b01000ef1ed02dbb2a20000 00000000000186a0
12 S 5300000906499e073e94370ceff37cb21c2821244401000fa7c40369842d0000 00000000000186a0
13 S 5300000906499e073e94370ceff37cb21c2821244402000fa7c403698fff0000 00000000000000a1
14 S 5300000906499e073e94370ceff37cb21c2821244402000fa7c80369f0010000 000000000000000f
15 S 53000009c3172e034a255f3c03566dca84bb9f046a01000e07020225c69c0000 000000000007a120
16 S 53000009ca6e0caaaef16872b4bd4f6f1b8c2363e201000eb5af02b169560000 00000000000f4240
17 i 6900000324e40fcb63a0b517a3660645e9bd99244a 0000000001406f460000000001312d06
18 i 6900000906499e073e94370ceff37cb21c28212444 000000000001875000000000000000b0

40
testdata/f.csv vendored
View file

@ -1,21 +1,21 @@
f,
660d649ba1defa4ab5ab71f8,919be5811844077f4660af66afa9a59a5ad17cf5c541524e780fe2137bfa250c
6623c6895027f70a5330bbcb,8dadcde1a6f676d4004eacd399f825006ddf136d1e92b1c92113377b3e1741b4
664f095b24484ebce8f31fbf,c0c4a751f569c1f9c01531f57ba674b2ad2338d9c08f9e9fc85b0209d15466b2
665201a38de7d7243df717c9,d9293577cc0d51fe3a5bee78fea9b2b2222e6c2aa0d26a4ef4bfb7dd095587e8
665328b2449e537b0ca4733f,624f80a361e47c7eb1b815e8714a40f67b4f642a5546547a3fcb5bf5593d8fab
665ec882021f55b1fbaa5fad,1e917fbc04385290d654f711bdef12773dd54b6b5ea26fe2a9d58ed051f2cb7f
6671c131cd433750ba6d3908,a2ebfbdf7a23024c340a45f201645aa46f48bc1fdd8d34ed83fcffbf1ee90523
667fb93d9ae877ba11f337f2,4710649e06619e13250754937e9c17c20b07434751171aac2f2f78b184aa0146
668ed5f39a5db059dc326137,8dd8ca749b87f43e290904749a546fe319c9d53e765f065bb8beb234a117655e
66951782f6ba94f2b71e46d0,4f5c9434dd0886c57c2530991cebd973e1b50d5ba8fcfc019e54561217a49bbb
66970565dfe2b01cad49b73a,f6ca0ae18c896d9bc97c5a9d0c3a06256485f59c77fb91780b213f933b80f48b
669f6a30a6712062da0cc271,5c6604bfd63b871daceb7893dd618850458974fe4108871c1a1323fb8ae34e4e
66a9a7b89b78553592acf3df,0561f28c3a5ea0027ecb3c53fa068772a6b7cb73d23104a14f9aba8cd1f070a2
66aba81567ba48f001f843f0,b0f6ae2c1db8263f7e11fc79423109e718d1f3c30bd123c4243401b5e4f1fee6
66b569cc3d28be4466fb28d1,ecee392ad8217f325508ba38d280436fb0a520b79a9627e5e18197bf55540885
66d4662cd100d66055917d63,5762a8ac767fa30d2ca76db7081f8a2e4f5da4f0bf92d29e1322da9a154cc3d6
66d6fa6ac71d0255dd3f185d,5fc193e5e51b3bd8e95f4eb9df63236da7abf678fc47c0b339ceb5c127d0f488
66e5b6c7c231a02a32eedd83,58c70ffbfada12550f24bf7931cee06eb2e267dec3560e2e46843e383415f163
66e673cce02c2163f756491e,b8db43d1f6e62361e2e3b8fa765f79c08ddfb3035caa06f8250d6d1b063a7140
66fc4ad75184e6029c805d94,fc7ac5e785f73732d95183d6bdc3423d41a074fc3f04b1304bae1efa652edde1
660d649ba1defa4ab5ab71f8a977d7f7cedb11056e,919be5811844077f4660af66afa9a59a5ad17cf5c541524e780fe2137bfa250c
6623c6895027f70a5330bbcb1153d635abcb4d5224,8dadcde1a6f676d4004eacd399f825006ddf136d1e92b1c92113377b3e1741b4
664f095b24484ebce8f31fbf008e63cc4aa163d401,c0c4a751f569c1f9c01531f57ba674b2ad2338d9c08f9e9fc85b0209d15466b2
665201a38de7d7243df717c9f9279cdd30105f0f77,d9293577cc0d51fe3a5bee78fea9b2b2222e6c2aa0d26a4ef4bfb7dd095587e8
665328b2449e537b0ca4733f87ac5ebcdf033c5ebd,624f80a361e47c7eb1b815e8714a40f67b4f642a5546547a3fcb5bf5593d8fab
665ec882021f55b1fbaa5fad00df5c5d07633b7af3,1e917fbc04385290d654f711bdef12773dd54b6b5ea26fe2a9d58ed051f2cb7f
6671c131cd433750ba6d3908150ca4910841164b74,a2ebfbdf7a23024c340a45f201645aa46f48bc1fdd8d34ed83fcffbf1ee90523
667fb93d9ae877ba11f337f21422b0679852580802,4710649e06619e13250754937e9c17c20b07434751171aac2f2f78b184aa0146
668ed5f39a5db059dc3261377f2a47728f7a357d33,8dd8ca749b87f43e290904749a546fe319c9d53e765f065bb8beb234a117655e
66951782f6ba94f2b71e46d0cc4a2411b14d81eb70,4f5c9434dd0886c57c2530991cebd973e1b50d5ba8fcfc019e54561217a49bbb
66970565dfe2b01cad49b73a085a3c3f7a3be61c4c,f6ca0ae18c896d9bc97c5a9d0c3a06256485f59c77fb91780b213f933b80f48b
669f6a30a6712062da0cc27181845c04d7430abf73,5c6604bfd63b871daceb7893dd618850458974fe4108871c1a1323fb8ae34e4e
66a9a7b89b78553592acf3dfc417c1d7654dab3273,0561f28c3a5ea0027ecb3c53fa068772a6b7cb73d23104a14f9aba8cd1f070a2
66aba81567ba48f001f843f01354d575c2e2687847,b0f6ae2c1db8263f7e11fc79423109e718d1f3c30bd123c4243401b5e4f1fee6
66b569cc3d28be4466fb28d147f66d6d8769598964,ecee392ad8217f325508ba38d280436fb0a520b79a9627e5e18197bf55540885
66d4662cd100d66055917d6342d48f49d948fcc255,5762a8ac767fa30d2ca76db7081f8a2e4f5da4f0bf92d29e1322da9a154cc3d6
66d6fa6ac71d0255dd3f185de6480d5b4316b6b050,5fc193e5e51b3bd8e95f4eb9df63236da7abf678fc47c0b339ceb5c127d0f488
66e5b6c7c231a02a32eedd8383a5750fd135244a03,58c70ffbfada12550f24bf7931cee06eb2e267dec3560e2e46843e383415f163
66e673cce02c2163f756491ef05d7535ceb578e215,b8db43d1f6e62361e2e3b8fa765f79c08ddfb3035caa06f8250d6d1b063a7140
66fc4ad75184e6029c805d9494eed4e81be770c002,fc7ac5e785f73732d95183d6bdc3423d41a074fc3f04b1304bae1efa652edde1

1 f
2 660d649ba1defa4ab5ab71f8 660d649ba1defa4ab5ab71f8a977d7f7cedb11056e 919be5811844077f4660af66afa9a59a5ad17cf5c541524e780fe2137bfa250c
3 6623c6895027f70a5330bbcb 6623c6895027f70a5330bbcb1153d635abcb4d5224 8dadcde1a6f676d4004eacd399f825006ddf136d1e92b1c92113377b3e1741b4
4 664f095b24484ebce8f31fbf 664f095b24484ebce8f31fbf008e63cc4aa163d401 c0c4a751f569c1f9c01531f57ba674b2ad2338d9c08f9e9fc85b0209d15466b2
5 665201a38de7d7243df717c9 665201a38de7d7243df717c9f9279cdd30105f0f77 d9293577cc0d51fe3a5bee78fea9b2b2222e6c2aa0d26a4ef4bfb7dd095587e8
6 665328b2449e537b0ca4733f 665328b2449e537b0ca4733f87ac5ebcdf033c5ebd 624f80a361e47c7eb1b815e8714a40f67b4f642a5546547a3fcb5bf5593d8fab
7 665ec882021f55b1fbaa5fad 665ec882021f55b1fbaa5fad00df5c5d07633b7af3 1e917fbc04385290d654f711bdef12773dd54b6b5ea26fe2a9d58ed051f2cb7f
8 6671c131cd433750ba6d3908 6671c131cd433750ba6d3908150ca4910841164b74 a2ebfbdf7a23024c340a45f201645aa46f48bc1fdd8d34ed83fcffbf1ee90523
9 667fb93d9ae877ba11f337f2 667fb93d9ae877ba11f337f21422b0679852580802 4710649e06619e13250754937e9c17c20b07434751171aac2f2f78b184aa0146
10 668ed5f39a5db059dc326137 668ed5f39a5db059dc3261377f2a47728f7a357d33 8dd8ca749b87f43e290904749a546fe319c9d53e765f065bb8beb234a117655e
11 66951782f6ba94f2b71e46d0 66951782f6ba94f2b71e46d0cc4a2411b14d81eb70 4f5c9434dd0886c57c2530991cebd973e1b50d5ba8fcfc019e54561217a49bbb
12 66970565dfe2b01cad49b73a 66970565dfe2b01cad49b73a085a3c3f7a3be61c4c f6ca0ae18c896d9bc97c5a9d0c3a06256485f59c77fb91780b213f933b80f48b
13 669f6a30a6712062da0cc271 669f6a30a6712062da0cc27181845c04d7430abf73 5c6604bfd63b871daceb7893dd618850458974fe4108871c1a1323fb8ae34e4e
14 66a9a7b89b78553592acf3df 66a9a7b89b78553592acf3dfc417c1d7654dab3273 0561f28c3a5ea0027ecb3c53fa068772a6b7cb73d23104a14f9aba8cd1f070a2
15 66aba81567ba48f001f843f0 66aba81567ba48f001f843f01354d575c2e2687847 b0f6ae2c1db8263f7e11fc79423109e718d1f3c30bd123c4243401b5e4f1fee6
16 66b569cc3d28be4466fb28d1 66b569cc3d28be4466fb28d147f66d6d8769598964 ecee392ad8217f325508ba38d280436fb0a520b79a9627e5e18197bf55540885
17 66d4662cd100d66055917d63 66d4662cd100d66055917d6342d48f49d948fcc255 5762a8ac767fa30d2ca76db7081f8a2e4f5da4f0bf92d29e1322da9a154cc3d6
18 66d6fa6ac71d0255dd3f185d 66d6fa6ac71d0255dd3f185de6480d5b4316b6b050 5fc193e5e51b3bd8e95f4eb9df63236da7abf678fc47c0b339ceb5c127d0f488
19 66e5b6c7c231a02a32eedd83 66e5b6c7c231a02a32eedd8383a5750fd135244a03 58c70ffbfada12550f24bf7931cee06eb2e267dec3560e2e46843e383415f163
20 66e673cce02c2163f756491e 66e673cce02c2163f756491ef05d7535ceb578e215 b8db43d1f6e62361e2e3b8fa765f79c08ddfb3035caa06f8250d6d1b063a7140
21 66fc4ad75184e6029c805d94 66fc4ad75184e6029c805d9494eed4e81be770c002 fc7ac5e785f73732d95183d6bdc3423d41a074fc3f04b1304bae1efa652edde1

40
testdata/g.csv vendored
View file

@ -1,21 +1,21 @@
g,
6702c124856d5168381a3297,575696fd653a4de2f9a8c1f580cf0c229631b0f5d95fceb354cda133e2eb2d34
6707f1511e3a2cb28493f91b,ba368e0f859ee36da8701df1c0b52cbf0c0f8a4b1a91f6d0db83a408f5a937d1
6707fd4213cae8d5342a98ba,bd3a44d30f66444f8732119bc7e0cf0bb47f8f0ab2840987fc06b629f3e6d3f4
6710294a5693224a6222404b,de35a8ea0a26d17445e2f509db23188961b5cd1229b96d2411565adf63731b5c
6716a9f84e02143b50d9034a,5823640ae4529f8df2dab20386c887d0a1ba1ffa4583b99dff761c01f670c2fa
672e51bc65c9b97d482b0b72,0687df449bd8cb8d8f526f4189973d084d786ab0927d81c127f56b03c61aa955
67682620db65932047689e5e,b262d40758edb28d1c04fa3a24d8268990516de6846ad94d002ce55640866239
676e8c320dbbf5eebc2969a9,c9e2a8e7181a70e2a488b884c8baadb4043a075c6876cb012c67fbec5aa9f615
6772e2ac48891ee3c2c72783,985a9c9ee7a0626d78dab431e663289762ce6959be314f91f7b08b1466097fd6
67847dd1dac117b85d1e20d9,62e6b1b8c2961703a90276dcde6dad182b2d14e23f27dccc927cca7770b9890e
678f49948c72b7295f12092a,2e7c456dac5206c5627736924e96ac016a09a88ec5f4835fbe0cf9e294611c88
67948b9633ab2ec07d752593,66b5c54b3a685de3ea18f9e69254eec065eb3207ac1f93494fdcd585e9a267a0
679674c162db8d3bb57c434f,05425880d80258f7441859b3494415a3fd7398c9e209a19674abd48372b283c6
67a8d3f17df85502bd644a36,1efce69a3a05c505e9f9cc5c2241d02099c043d934389b430fd8b185e6dfe6cb
67bad7f4fb3c6828b6fc4624,04a1c0a7ffe7acbf974ca18cf3debbd8e1be3d6703f842f57ef14af6d4c336d3
67c13fb0c65acca5520bc2f5,7fdc6989cd778baad45cd98358ea060237b169a4aeaeb14da6ac4686b7858c9f
67d4314588b4424b0ee02653,c63fd7a85a533b8591577bab805104708ba5458fab0e343d46b3e24a28b92cb5
67d734244f85f32a58e34e2d,d19a6307c24470b3973973319770bdb896218bb58d1f2d07c7226266075057d0
67d9c159c5d5e407e6b0a4ca,89cbdb903fdfe0b44e74b0a69eed3de7029f18c28f77e5509f8ace766ab86610
67fafc73d674250f11e559ab,1752ffbf9807bb2e4e480bf045b4bacc472befe755287384b5a526065a58c065
6702c124856d5168381a32971d8933440a1728fc41,575696fd653a4de2f9a8c1f580cf0c229631b0f5d95fceb354cda133e2eb2d34
6707f1511e3a2cb28493f91b85e9e4a9d9d07c86a5,ba368e0f859ee36da8701df1c0b52cbf0c0f8a4b1a91f6d0db83a408f5a937d1
6707fd4213cae8d5342a98ba49b255fa80b2a9a6e4,bd3a44d30f66444f8732119bc7e0cf0bb47f8f0ab2840987fc06b629f3e6d3f4
6710294a5693224a6222404ba45fd38eb2e77979a4,de35a8ea0a26d17445e2f509db23188961b5cd1229b96d2411565adf63731b5c
6716a9f84e02143b50d9034aec126b12d7f2708cc4,5823640ae4529f8df2dab20386c887d0a1ba1ffa4583b99dff761c01f670c2fa
672e51bc65c9b97d482b0b720e6cb673c41fe7b5c5,0687df449bd8cb8d8f526f4189973d084d786ab0927d81c127f56b03c61aa955
67682620db65932047689e5eaf392d6b85be801864,b262d40758edb28d1c04fa3a24d8268990516de6846ad94d002ce55640866239
676e8c320dbbf5eebc2969a93fbc51dd7f6062a7d1,c9e2a8e7181a70e2a488b884c8baadb4043a075c6876cb012c67fbec5aa9f615
6772e2ac48891ee3c2c727835702a374ad0cb70fd6,985a9c9ee7a0626d78dab431e663289762ce6959be314f91f7b08b1466097fd6
67847dd1dac117b85d1e20d93580cdf42f00001a77,62e6b1b8c2961703a90276dcde6dad182b2d14e23f27dccc927cca7770b9890e
678f49948c72b7295f12092a24d300eeff894f1dd7,2e7c456dac5206c5627736924e96ac016a09a88ec5f4835fbe0cf9e294611c88
67948b9633ab2ec07d7525936254e66f8c957d026c,66b5c54b3a685de3ea18f9e69254eec065eb3207ac1f93494fdcd585e9a267a0
679674c162db8d3bb57c434fe87825625c4d4daf63,05425880d80258f7441859b3494415a3fd7398c9e209a19674abd48372b283c6
67a8d3f17df85502bd644a364721e6364d61635b73,1efce69a3a05c505e9f9cc5c2241d02099c043d934389b430fd8b185e6dfe6cb
67bad7f4fb3c6828b6fc4624d43786fc8f55d6eb0f,04a1c0a7ffe7acbf974ca18cf3debbd8e1be3d6703f842f57ef14af6d4c336d3
67c13fb0c65acca5520bc2f59bd91ca3482dbec156,7fdc6989cd778baad45cd98358ea060237b169a4aeaeb14da6ac4686b7858c9f
67d4314588b4424b0ee026536b9bd7857f11cab2ee,c63fd7a85a533b8591577bab805104708ba5458fab0e343d46b3e24a28b92cb5
67d734244f85f32a58e34e2d9cadf225a56973d32f,d19a6307c24470b3973973319770bdb896218bb58d1f2d07c7226266075057d0
67d9c159c5d5e407e6b0a4cacf9d6fe62a55b0fedc,89cbdb903fdfe0b44e74b0a69eed3de7029f18c28f77e5509f8ace766ab86610
67fafc73d674250f11e559ab08b287f5714e531761,1752ffbf9807bb2e4e480bf045b4bacc472befe755287384b5a526065a58c065

1 g
2 6702c124856d5168381a3297 6702c124856d5168381a32971d8933440a1728fc41 575696fd653a4de2f9a8c1f580cf0c229631b0f5d95fceb354cda133e2eb2d34
3 6707f1511e3a2cb28493f91b 6707f1511e3a2cb28493f91b85e9e4a9d9d07c86a5 ba368e0f859ee36da8701df1c0b52cbf0c0f8a4b1a91f6d0db83a408f5a937d1
4 6707fd4213cae8d5342a98ba 6707fd4213cae8d5342a98ba49b255fa80b2a9a6e4 bd3a44d30f66444f8732119bc7e0cf0bb47f8f0ab2840987fc06b629f3e6d3f4
5 6710294a5693224a6222404b 6710294a5693224a6222404ba45fd38eb2e77979a4 de35a8ea0a26d17445e2f509db23188961b5cd1229b96d2411565adf63731b5c
6 6716a9f84e02143b50d9034a 6716a9f84e02143b50d9034aec126b12d7f2708cc4 5823640ae4529f8df2dab20386c887d0a1ba1ffa4583b99dff761c01f670c2fa
7 672e51bc65c9b97d482b0b72 672e51bc65c9b97d482b0b720e6cb673c41fe7b5c5 0687df449bd8cb8d8f526f4189973d084d786ab0927d81c127f56b03c61aa955
8 67682620db65932047689e5e 67682620db65932047689e5eaf392d6b85be801864 b262d40758edb28d1c04fa3a24d8268990516de6846ad94d002ce55640866239
9 676e8c320dbbf5eebc2969a9 676e8c320dbbf5eebc2969a93fbc51dd7f6062a7d1 c9e2a8e7181a70e2a488b884c8baadb4043a075c6876cb012c67fbec5aa9f615
10 6772e2ac48891ee3c2c72783 6772e2ac48891ee3c2c727835702a374ad0cb70fd6 985a9c9ee7a0626d78dab431e663289762ce6959be314f91f7b08b1466097fd6
11 67847dd1dac117b85d1e20d9 67847dd1dac117b85d1e20d93580cdf42f00001a77 62e6b1b8c2961703a90276dcde6dad182b2d14e23f27dccc927cca7770b9890e
12 678f49948c72b7295f12092a 678f49948c72b7295f12092a24d300eeff894f1dd7 2e7c456dac5206c5627736924e96ac016a09a88ec5f4835fbe0cf9e294611c88
13 67948b9633ab2ec07d752593 67948b9633ab2ec07d7525936254e66f8c957d026c 66b5c54b3a685de3ea18f9e69254eec065eb3207ac1f93494fdcd585e9a267a0
14 679674c162db8d3bb57c434f 679674c162db8d3bb57c434fe87825625c4d4daf63 05425880d80258f7441859b3494415a3fd7398c9e209a19674abd48372b283c6
15 67a8d3f17df85502bd644a36 67a8d3f17df85502bd644a364721e6364d61635b73 1efce69a3a05c505e9f9cc5c2241d02099c043d934389b430fd8b185e6dfe6cb
16 67bad7f4fb3c6828b6fc4624 67bad7f4fb3c6828b6fc4624d43786fc8f55d6eb0f 04a1c0a7ffe7acbf974ca18cf3debbd8e1be3d6703f842f57ef14af6d4c336d3
17 67c13fb0c65acca5520bc2f5 67c13fb0c65acca5520bc2f59bd91ca3482dbec156 7fdc6989cd778baad45cd98358ea060237b169a4aeaeb14da6ac4686b7858c9f
18 67d4314588b4424b0ee02653 67d4314588b4424b0ee026536b9bd7857f11cab2ee c63fd7a85a533b8591577bab805104708ba5458fab0e343d46b3e24a28b92cb5
19 67d734244f85f32a58e34e2d 67d734244f85f32a58e34e2d9cadf225a56973d32f d19a6307c24470b3973973319770bdb896218bb58d1f2d07c7226266075057d0
20 67d9c159c5d5e407e6b0a4ca 67d9c159c5d5e407e6b0a4cacf9d6fe62a55b0fedc 89cbdb903fdfe0b44e74b0a69eed3de7029f18c28f77e5509f8ace766ab86610
21 67fafc73d674250f11e559ab 67fafc73d674250f11e559ab08b287f5714e531761 1752ffbf9807bb2e4e480bf045b4bacc472befe755287384b5a526065a58c065

40
testdata/i.csv vendored
View file

@ -1,21 +1,21 @@
i,
691d3476414324a257c62079b055446cdfdb58fcb7,3fc1f36ad9acdae3160db55befe1fdcf
692514be0d49b250058c2c59c62b07383705d71c54,055bf209d8b7132f1743aee761d9c64d
6932de7e3a7bae762b9533e955fd2b2444f6656aa7,9371781e0655df3b914d18da42605b3d
6938e5c7d134233b0758e54f5eacb9dcee412f51f9,12079ef8dffde085385b2aafe7e8be53
693a56c48c3ec6bc90fdd02e0f0e2f01472c3f13f5,8c422f4f35e4170079a13f3e9f25a9db
693fe5c0979c7c4892486d478c8c57b75f0fa9bba3,8eeaafae4e906ccc36ec29bc3d4f1676
694abea2af1c27003a1c777e84891a0f81b3b5a382,fe24b3d28f8cf49fad2d4548726ac8bd
694c245cf621a28925da7d84e77137a1d54085d1b8,c04cf11c401c4fbc8976825e9b6db9ca
6951010e69f84df87d09cdae7706e25ecdc10a8a6f,a93d6f9c06d1e807c1d674828167cd7c
695661d8955be621b5f44c7965f517c17d2d1d48c6,8b945701a1d2920c3e6283ab7fda14ee
696fac500a5674eaa078abc71ceb48006c14a3f6aa,1f8000aec89b229349aa154e72fd4de3
697506379203bd2f8980d13c766966e400509e28f9,5ce938e06a98aa8b8bb0cfea5dce7e33
6975c5f2cdc6e8fdb64682557b1bcbb92f52c6113f,2817aa0f0806bb276e1af4af42504720
6984b87daaba891147b4c0f25c15703b2640df9833,169009ea3ff014d352a13152c8d39999
699f3d1a3f634bb035c626cdfa927caa59e2617bc4,8f3e2352ed874155a3aa3dd90a61430e
69b9dfcdaced1d6d696fab02be31cbee50cbffcdf9,281d9817a5360c1be0ac7710132adebe
69ca1fa3e939d061d74163b2da17c3d2b926484c5e,40ecc3bd5dc6b871ce3095456e043427
69cea59483161df9420027c4328f85382871798ee4,3919862cc4f0f910a954ffc4d08a6195
69d8ff3b5f44f5585e7d7e1349d1d62ba3fbe6831c,d48bd4f6c44ef8b6aabeb6b6e1a61894
69f22b1918f28b1e10d2946f84f6f3c8fa25865ba3,b49011d36a56e0dbe5a5cbce94159f66
6903dc9970183b8a05d118c84a37c355fe34d95d01,0e74dd23295a4610
6916bd29750d8d92b32677eda07e10af313c0019d9,ff0579207ec6e594
6926bdfcb4a1ad81f460ad561283584695cd6cea59,b834b13a8918262f
6930776827481ec15fa07e0dc266e376846467237d,4bf0a5127a1216dc
6955a1eaf08f9468a6c3565fe16b2ae4b726045538,e32029de8b58dd6e
69614fa6bc0cea1366377456bc88dda9ec7b6d4c3c,55bf2d8e0e262697
6971e7b039dde5797ae167b9783f87c4a67711799d,9197b827b560fc34
697765a71d8a4082d056baaae83f8b4af1e124f5e9,62c0d5dfde7ef884
6993e121642c01e2edca50d565ff3df1a656e83ebd,1796c74886d45045
69af0684076bc64adcbcd621de9c09fd15dade3e17,f9240ab9a9650d9f
69b70fdcc95d3b51ec28872a143c7a6fc947e6a58e,a8950968d95759a9
69bdb90916f78badec506a895b0edceb47533297f9,331c0ca597601ed7
69c276b7070ba79e75c82e4d97a70e4428dd3be058,85c61c842e5bfe7f
69cb8215b0c9440227a9e7e76bce468bdb4fa0f714,9c42e1ba41275362
69d2556fe7b8fce36a71c78e0001452298903ef81b,f61cf52e7e645bf8
69d2677078f1448c0e4295711e61a85a9fb6a280d1,28d57b45b1700cb3
69dd979490444cab3efc1e517f83369df6e2e279a3,dad1b5133cc15dd4
69e6e6834cf0da132c77af275cbab9885cbbc3f022,df55f9fd0278ca71
69ea25444b8d0ab22ac355f4b692b0d7006b672f4a,0703f38a1428ff8e
69fcb79c5e463ac4a0e078f9cd24e2c718b66c40d6,5be7b71b6dca5a20

1 i
2 691d3476414324a257c62079b055446cdfdb58fcb7 6903dc9970183b8a05d118c84a37c355fe34d95d01 3fc1f36ad9acdae3160db55befe1fdcf 0e74dd23295a4610
3 692514be0d49b250058c2c59c62b07383705d71c54 6916bd29750d8d92b32677eda07e10af313c0019d9 055bf209d8b7132f1743aee761d9c64d ff0579207ec6e594
4 6932de7e3a7bae762b9533e955fd2b2444f6656aa7 6926bdfcb4a1ad81f460ad561283584695cd6cea59 9371781e0655df3b914d18da42605b3d b834b13a8918262f
5 6938e5c7d134233b0758e54f5eacb9dcee412f51f9 6930776827481ec15fa07e0dc266e376846467237d 12079ef8dffde085385b2aafe7e8be53 4bf0a5127a1216dc
6 693a56c48c3ec6bc90fdd02e0f0e2f01472c3f13f5 6955a1eaf08f9468a6c3565fe16b2ae4b726045538 8c422f4f35e4170079a13f3e9f25a9db e32029de8b58dd6e
7 693fe5c0979c7c4892486d478c8c57b75f0fa9bba3 69614fa6bc0cea1366377456bc88dda9ec7b6d4c3c 8eeaafae4e906ccc36ec29bc3d4f1676 55bf2d8e0e262697
8 694abea2af1c27003a1c777e84891a0f81b3b5a382 6971e7b039dde5797ae167b9783f87c4a67711799d fe24b3d28f8cf49fad2d4548726ac8bd 9197b827b560fc34
9 694c245cf621a28925da7d84e77137a1d54085d1b8 697765a71d8a4082d056baaae83f8b4af1e124f5e9 c04cf11c401c4fbc8976825e9b6db9ca 62c0d5dfde7ef884
10 6951010e69f84df87d09cdae7706e25ecdc10a8a6f 6993e121642c01e2edca50d565ff3df1a656e83ebd a93d6f9c06d1e807c1d674828167cd7c 1796c74886d45045
11 695661d8955be621b5f44c7965f517c17d2d1d48c6 69af0684076bc64adcbcd621de9c09fd15dade3e17 8b945701a1d2920c3e6283ab7fda14ee f9240ab9a9650d9f
12 696fac500a5674eaa078abc71ceb48006c14a3f6aa 69b70fdcc95d3b51ec28872a143c7a6fc947e6a58e 1f8000aec89b229349aa154e72fd4de3 a8950968d95759a9
13 697506379203bd2f8980d13c766966e400509e28f9 69bdb90916f78badec506a895b0edceb47533297f9 5ce938e06a98aa8b8bb0cfea5dce7e33 331c0ca597601ed7
14 6975c5f2cdc6e8fdb64682557b1bcbb92f52c6113f 69c276b7070ba79e75c82e4d97a70e4428dd3be058 2817aa0f0806bb276e1af4af42504720 85c61c842e5bfe7f
15 6984b87daaba891147b4c0f25c15703b2640df9833 69cb8215b0c9440227a9e7e76bce468bdb4fa0f714 169009ea3ff014d352a13152c8d39999 9c42e1ba41275362
16 699f3d1a3f634bb035c626cdfa927caa59e2617bc4 69d2556fe7b8fce36a71c78e0001452298903ef81b 8f3e2352ed874155a3aa3dd90a61430e f61cf52e7e645bf8
17 69b9dfcdaced1d6d696fab02be31cbee50cbffcdf9 69d2677078f1448c0e4295711e61a85a9fb6a280d1 281d9817a5360c1be0ac7710132adebe 28d57b45b1700cb3
18 69ca1fa3e939d061d74163b2da17c3d2b926484c5e 69dd979490444cab3efc1e517f83369df6e2e279a3 40ecc3bd5dc6b871ce3095456e043427 dad1b5133cc15dd4
19 69cea59483161df9420027c4328f85382871798ee4 69e6e6834cf0da132c77af275cbab9885cbbc3f022 3919862cc4f0f910a954ffc4d08a6195 df55f9fd0278ca71
20 69d8ff3b5f44f5585e7d7e1349d1d62ba3fbe6831c 69ea25444b8d0ab22ac355f4b692b0d7006b672f4a d48bd4f6c44ef8b6aabeb6b6e1a61894 0703f38a1428ff8e
21 69f22b1918f28b1e10d2946f84f6f3c8fa25865ba3 69fcb79c5e463ac4a0e078f9cd24e2c718b66c40d6 b49011d36a56e0dbe5a5cbce94159f66 5be7b71b6dca5a20

View file

@ -1 +1 @@
v0.2022.10.05.1
v0.2022.08.16.1