Use time.Ticker object to drive management activity.

used in testing has no unique Addr.

.github/workflows/build-short.yml

@@ -7,6 +7,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
       - name: Build and Test
         uses: ./

db/db.go

@@ -15,7 +15,6 @@ import (
 	"github.com/lbryio/herald.go/internal"
 	"github.com/lbryio/herald.go/internal/metrics"
 	pb "github.com/lbryio/herald.go/protobuf/go"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	"github.com/linxGnu/grocksdb"
 
 	log "github.com/sirupsen/logrus"
@@ -59,7 +58,8 @@ type ReadOnlyDBColumnFamily struct {
 	BlockedChannels        map[string][]byte
 	FilteredStreams        map[string][]byte
 	FilteredChannels       map[string][]byte
-	Grp                    *stop.Group
+	ShutdownChan           chan struct{}
+	DoneChan               chan struct{}
 	Cleanup                func()
 }
 
@@ -318,29 +318,9 @@ func intMin(a, b int) int {
 	return b
 }
 
-// FIXME: This was copied from the signal.go file, maybe move it to a more common place?
-// interruptRequested returns true when the channel returned by
-// interruptListener was closed.  This simplifies early shutdown slightly since
-// the caller can just use an if statement instead of a select.
-func interruptRequested(interrupted <-chan struct{}) bool {
-	select {
-	case <-interrupted:
-		return true
-	default:
-	}
-
-	return false
-}
-
 func IterCF(db *grocksdb.DB, opts *IterOptions) <-chan *prefixes.PrefixRowKV {
 	ch := make(chan *prefixes.PrefixRowKV)
 
-	// Check if we've been told to shutdown in between getting created and getting here
-	if opts.Grp != nil && interruptRequested(opts.Grp.Ch()) {
-		opts.Grp.Done()
-		return ch
-	}
-
 	ro := grocksdb.NewDefaultReadOptions()
 	ro.SetFillCache(opts.FillCache)
 	it := db.NewIteratorCF(ro, opts.CfHandle)
@@ -356,10 +336,6 @@ func IterCF(db *grocksdb.DB, opts *IterOptions) <-chan *prefixes.PrefixRowKV {
 			it.Close()
 			close(ch)
 			ro.Destroy()
-			if opts.Grp != nil {
-				// opts.Grp.DoneNamed(iterKey)
-				opts.Grp.Done()
-			}
 		}()
 
 		var prevKey []byte
@@ -379,9 +355,6 @@ func IterCF(db *grocksdb.DB, opts *IterOptions) <-chan *prefixes.PrefixRowKV {
 			if kv = opts.ReadRow(&prevKey); kv != nil {
 				ch <- kv
 			}
-			if opts.Grp != nil && interruptRequested(opts.Grp.Ch()) {
-				return
-			}
 		}
 	}()
 
@@ -439,10 +412,10 @@ func (db *ReadOnlyDBColumnFamily) selectFrom(prefix []byte, startKey, stopKey pr
 		return nil, err
 	}
 	// Prefix and handle
-	options := NewIterateOptions().WithDB(db).WithPrefix(prefix).WithCfHandle(handle)
+	options := NewIterateOptions().WithPrefix(prefix).WithCfHandle(handle)
 	// Start and stop bounds
-	options = options.WithStart(startKey.PackKey()).WithStop(stopKey.PackKey()).WithIncludeStop(false)
-	// Include the key and value
+	options = options.WithStart(startKey.PackKey()).WithStop(stopKey.PackKey()).WithIncludeStop(true)
+	// Don't include the key
 	options = options.WithIncludeKey(true).WithIncludeValue(true)
 	return []*IterOptions{options}, nil
 }
@@ -455,7 +428,7 @@ func iterate(db *grocksdb.DB, opts []*IterOptions) <-chan []*prefixes.PrefixRowK
 			for kv := range IterCF(db, o) {
 				row := make([]*prefixes.PrefixRowKV, 0, 1)
 				row = append(row, kv)
-				log.Debugf("iterate[%v][%v] %#v -> %#v", i, j, kv.Key, kv.Value)
+				log.Debugf("iterate[%v][%v] %#v", i, j, kv)
 				out <- row
 				j++
 			}
@@ -481,7 +454,7 @@ func innerJoin(db *grocksdb.DB, in <-chan []*prefixes.PrefixRowKV, selectFn func
 				row = append(row, kvs...)
 				row = append(row, kv...)
 				for i, kv := range row {
-					log.Debugf("row[%v] %#v -> %#v", i, kv.Key, kv.Value)
+					log.Debugf("row[%v] %#v", i, kv)
 				}
 				out <- row
 			}
@@ -531,7 +504,7 @@ func GetWriteDBCF(name string) (*grocksdb.DB, []*grocksdb.ColumnFamilyHandle, er
 }
 
 // GetProdDB returns a db that is used for production.
-func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBColumnFamily, error) {
+func GetProdDB(name string, secondaryPath string) (*ReadOnlyDBColumnFamily, func(), error) {
 	prefixNames := prefixes.GetPrefixes()
 	// additional prefixes that aren't in the code explicitly
 	cfNames := []string{"default", "e", "d", "c"}
@@ -540,7 +513,7 @@ func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBC
 		cfNames = append(cfNames, cfName)
 	}
 
-	db, err := GetDBColumnFamilies(name, secondaryPath, cfNames, grp)
+	db, err := GetDBColumnFamilies(name, secondaryPath, cfNames)
 
 	cleanupFiles := func() {
 		err = os.RemoveAll(secondaryPath)
@@ -550,8 +523,7 @@ func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBC
 	}
 
 	if err != nil {
-		cleanupFiles()
-		return nil, err
+		return nil, cleanupFiles, err
 	}
 
 	cleanupDB := func() {
@@ -560,11 +532,11 @@ func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBC
 	}
 	db.Cleanup = cleanupDB
 
-	return db, nil
+	return db, cleanupDB, nil
 }
 
 // GetDBColumnFamilies gets a db with the specified column families and secondary path.
-func GetDBColumnFamilies(name string, secondayPath string, cfNames []string, grp *stop.Group) (*ReadOnlyDBColumnFamily, error) {
+func GetDBColumnFamilies(name string, secondayPath string, cfNames []string) (*ReadOnlyDBColumnFamily, error) {
 	opts := grocksdb.NewDefaultOptions()
 	roOpts := grocksdb.NewDefaultReadOptions()
 	cfOpt := grocksdb.NewDefaultOptions()
@@ -579,7 +551,6 @@ func GetDBColumnFamilies(name string, secondayPath string, cfNames []string, grp
 	// db, handles, err := grocksdb.OpenDbColumnFamilies(opts, name, cfNames, cfOpts)
 
 	if err != nil {
-		log.Errorf("open db as secondary failed: %v", err)
 		return nil, err
 	}
 
@@ -601,7 +572,8 @@ func GetDBColumnFamilies(name string, secondayPath string, cfNames []string, grp
 		LastState:        nil,
 		Height:           0,
 		Headers:          nil,
-		Grp:              grp,
+		ShutdownChan:     make(chan struct{}),
+		DoneChan:         make(chan struct{}),
 	}
 
 	err = myDB.ReadDBState() //TODO: Figure out right place for this
@@ -670,22 +642,21 @@ func (db *ReadOnlyDBColumnFamily) Unwind() {
 
 // Shutdown shuts down the db.
 func (db *ReadOnlyDBColumnFamily) Shutdown() {
-	log.Println("Calling cleanup...")
+	db.ShutdownChan <- struct{}{}
+	<-db.DoneChan
 	db.Cleanup()
-	log.Println("Leaving Shutdown...")
 }
 
 // RunDetectChanges Go routine the runs continuously while the hub is active
 // to keep the db readonly view up to date and handle reorgs on the
 // blockchain.
 func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan<- interface{}) {
-	db.Grp.Add(1)
 	go func() {
 		lastPrint := time.Now()
 		for {
 			// FIXME: Figure out best sleep interval
 			if time.Since(lastPrint) > time.Second {
-				log.Debugf("DetectChanges: %#v", db.LastState)
+				log.Debug("DetectChanges:", db.LastState)
 				lastPrint = time.Now()
 			}
 			err := db.detectChanges(notifCh)
@@ -693,8 +664,8 @@ func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan<- interface{}) {
 				log.Infof("Error detecting changes: %#v", err)
 			}
 			select {
-			case <-db.Grp.Ch():
-				db.Grp.Done()
+			case <-db.ShutdownChan:
+				db.DoneChan <- struct{}{}
 				return
 			case <-time.After(time.Millisecond * 10):
 			}
@@ -775,12 +746,7 @@ func (db *ReadOnlyDBColumnFamily) detectChanges(notifCh chan<- interface{}) erro
 				log.Info("error getting block hash: ", err)
 				return err
 			}
-			header, err := db.GetHeader(height)
-			if err != nil {
-				log.Info("error getting block header: ", err)
-				return err
-			}
-			notifCh <- &internal.HeightHash{Height: uint64(height), BlockHash: hash, BlockHeader: header}
+			notifCh <- &internal.HeightHash{Height: uint64(height), BlockHash: hash}
 		}
 		//TODO: ClearCache
 		log.Warn("implement cache clearing")
@@ -824,7 +790,7 @@ func (db *ReadOnlyDBColumnFamily) InitHeaders() error {
 	// endKey := prefixes.NewHeaderKey(db.LastState.Height)
 	startKeyRaw := startKey.PackKey()
 	// endKeyRaw := endKey.PackKey()
-	options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
+	options := NewIterateOptions().WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
 	options = options.WithIncludeKey(false).WithIncludeValue(true) //.WithIncludeStop(true)
 	options = options.WithStart(startKeyRaw)                       //.WithStop(endKeyRaw)
 
@@ -847,7 +813,7 @@ func (db *ReadOnlyDBColumnFamily) InitTxCounts() error {
 
 	db.TxCounts = stack.NewSliceBacked[uint32](InitialTxCountSize)
 
-	options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.TxCount}).WithCfHandle(handle)
+	options := NewIterateOptions().WithPrefix([]byte{prefixes.TxCount}).WithCfHandle(handle)
 	options = options.WithIncludeKey(false).WithIncludeValue(true).WithIncludeStop(true)
 
 	ch := IterCF(db.DB, options)

db/db_get.go

@@ -3,19 +3,16 @@ package db
 // db_get.go contains the basic access functions to the database.
 
 import (
-	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"log"
 	"math"
 
 	"github.com/lbryio/herald.go/db/prefixes"
 	"github.com/lbryio/herald.go/db/stack"
 	"github.com/lbryio/lbcd/chaincfg/chainhash"
-	"github.com/lbryio/lbcd/wire"
 	"github.com/linxGnu/grocksdb"
-
-	log "github.com/sirupsen/logrus"
 )
 
 // GetExpirationHeight returns the expiration height for the given height. Uses
@@ -68,57 +65,6 @@ func (db *ReadOnlyDBColumnFamily) GetBlockHash(height uint32) ([]byte, error) {
 	return rawValue, nil
 }
 
-func (db *ReadOnlyDBColumnFamily) GetBlockTXs(height uint32) ([]*chainhash.Hash, error) {
-	handle, err := db.EnsureHandle(prefixes.BlockTXs)
-	if err != nil {
-		return nil, err
-	}
-
-	key := prefixes.BlockTxsKey{
-		Prefix: []byte{prefixes.BlockTXs},
-		Height: height,
-	}
-	slice, err := db.DB.GetCF(db.Opts, handle, key.PackKey())
-	defer slice.Free()
-	if err != nil {
-		return nil, err
-	}
-	if slice.Size() == 0 {
-		return nil, nil
-	}
-
-	rawValue := make([]byte, len(slice.Data()))
-	copy(rawValue, slice.Data())
-	value := prefixes.BlockTxsValueUnpack(rawValue)
-	return value.TxHashes, nil
-}
-
-func (db *ReadOnlyDBColumnFamily) GetTouchedHashXs(height uint32) ([][]byte, error) {
-	handle, err := db.EnsureHandle(prefixes.TouchedHashX)
-	if err != nil {
-		return nil, err
-	}
-
-	key := prefixes.TouchedHashXKey{
-		Prefix: []byte{prefixes.TouchedHashX},
-		Height: height,
-	}
-	slice, err := db.DB.GetCF(db.Opts, handle, key.PackKey())
-	defer slice.Free()
-	if err != nil {
-		return nil, err
-	}
-	if slice.Size() == 0 {
-		return nil, nil
-	}
-
-	rawValue := make([]byte, len(slice.Data()))
-	copy(rawValue, slice.Data())
-	value := prefixes.TouchedHashXValue{}
-	value.UnpackValue(rawValue)
-	return value.TouchedHashXs, nil
-}
-
 func (db *ReadOnlyDBColumnFamily) GetHeader(height uint32) ([]byte, error) {
 	handle, err := db.EnsureHandle(prefixes.Header)
 	if err != nil {
@@ -148,7 +94,7 @@ func (db *ReadOnlyDBColumnFamily) GetHeaders(height uint32, count uint32) ([][11
 
 	startKeyRaw := prefixes.NewHeaderKey(height).PackKey()
 	endKeyRaw := prefixes.NewHeaderKey(height + count).PackKey()
-	options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
+	options := NewIterateOptions().WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
 	options = options.WithIncludeKey(false).WithIncludeValue(true) //.WithIncludeStop(true)
 	options = options.WithStart(startKeyRaw).WithStop(endKeyRaw)
 
@@ -184,7 +130,7 @@ func (db *ReadOnlyDBColumnFamily) GetBalance(hashX []byte) (uint64, uint64, erro
 	startKeyRaw := startKey.PackKey()
 	endKeyRaw := endKey.PackKey()
 	// Prefix and handle
-	options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.UTXO}).WithCfHandle(handle)
+	options := NewIterateOptions().WithPrefix([]byte{prefixes.UTXO}).WithCfHandle(handle)
 	// Start and stop bounds
 	options = options.WithStart(startKeyRaw).WithStop(endKeyRaw).WithIncludeStop(true)
 	// Don't include the key
@@ -325,7 +271,6 @@ func (db *ReadOnlyDBColumnFamily) GetStatus(hashX []byte) ([]byte, error) {
 	// Lookup in HashXMempoolStatus first.
 	status, err := db.getMempoolStatus(hashX)
 	if err == nil && status != nil {
-		log.Debugf("(mempool) status(%#v) -> %#v", hashX, status)
 		return status, err
 	}
 
@@ -346,7 +291,6 @@ func (db *ReadOnlyDBColumnFamily) GetStatus(hashX []byte) ([]byte, error) {
 		copy(rawValue, slice.Data())
 		value := prefixes.HashXStatusValue{}
 		value.UnpackValue(rawValue)
-		log.Debugf("status(%#v) -> %#v", hashX, value.Status)
 		return value.Status, nil
 	}
 
@@ -355,11 +299,6 @@ func (db *ReadOnlyDBColumnFamily) GetStatus(hashX []byte) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	if len(txs) == 0 {
-		return []byte{}, err
-	}
-
 	hash := sha256.New()
 	for _, tx := range txs {
 		hash.Write([]byte(fmt.Sprintf("%s:%d:", tx.TxHash.String(), tx.Height)))
@@ -407,7 +346,7 @@ func (db *ReadOnlyDBColumnFamily) GetStreamsAndChannelRepostedByChannelHashes(re
 	for _, reposterChannelHash := range reposterChannelHashes {
 		key := prefixes.NewChannelToClaimKeyWHash(reposterChannelHash)
 		rawKeyPrefix := key.PartialPack(1)
-		options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
+		options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 		options = options.WithIncludeKey(false).WithIncludeValue(true)
 		ch := IterCF(db.DB, options)
 		// for stream := range Iterate(db.DB, prefixes.ChannelToClaim, []byte{reposterChannelHash}, false) {
@@ -481,7 +420,7 @@ func (db *ReadOnlyDBColumnFamily) GetShortClaimIdUrl(name string, normalizedName
 		log.Printf("partialKey: %#v\n", partialKey)
 		keyPrefix := partialKey.PartialPack(2)
 		// Prefix and handle
-		options := NewIterateOptions().WithDB(db).WithPrefix(prefix).WithCfHandle(handle)
+		options := NewIterateOptions().WithPrefix(prefix).WithCfHandle(handle)
 		// Start and stop bounds
 		options = options.WithStart(keyPrefix).WithStop(keyPrefix)
 		// Don't include the key
@@ -579,7 +518,7 @@ func (db *ReadOnlyDBColumnFamily) GetActiveAmount(claimHash []byte, txoType uint
 	startKeyRaw := startKey.PartialPack(3)
 	endKeyRaw := endKey.PartialPack(3)
 	// Prefix and handle
-	options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.ActiveAmount}).WithCfHandle(handle)
+	options := NewIterateOptions().WithPrefix([]byte{prefixes.ActiveAmount}).WithCfHandle(handle)
 	// Start and stop bounds
 	options = options.WithStart(startKeyRaw).WithStop(endKeyRaw)
 	// Don't include the key
@@ -735,7 +674,7 @@ func (db *ReadOnlyDBColumnFamily) ControllingClaimIter() <-chan *prefixes.Prefix
 	key := prefixes.NewClaimTakeoverKey("")
 	var rawKeyPrefix []byte = nil
 	rawKeyPrefix = key.PartialPack(0)
-	options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
+	options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 	options = options.WithIncludeValue(true) //.WithIncludeStop(true)
 	ch := IterCF(db.DB, options)
 	return ch
@@ -792,70 +731,6 @@ func (db *ReadOnlyDBColumnFamily) FsGetClaimByHash(claimHash []byte) (*ResolveRe
 	)
 }
 
-func (db *ReadOnlyDBColumnFamily) GetTx(txhash *chainhash.Hash) ([]byte, *wire.MsgTx, error) {
-	// Lookup in MempoolTx first.
-	raw, tx, err := db.getMempoolTx(txhash)
-	if err == nil && raw != nil && tx != nil {
-		return raw, tx, err
-	}
-
-	handle, err := db.EnsureHandle(prefixes.Tx)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	key := prefixes.TxKey{Prefix: []byte{prefixes.Tx}, TxHash: txhash}
-	rawKey := key.PackKey()
-	slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
-	defer slice.Free()
-	if err != nil {
-		return nil, nil, err
-	}
-	if slice.Size() == 0 {
-		return nil, nil, nil
-	}
-
-	rawValue := make([]byte, len(slice.Data()))
-	copy(rawValue, slice.Data())
-	value := prefixes.TxValue{}
-	value.UnpackValue(rawValue)
-	var msgTx wire.MsgTx
-	err = msgTx.Deserialize(bytes.NewReader(value.RawTx))
-	if err != nil {
-		return nil, nil, err
-	}
-	return value.RawTx, &msgTx, nil
-}
-
-func (db *ReadOnlyDBColumnFamily) getMempoolTx(txhash *chainhash.Hash) ([]byte, *wire.MsgTx, error) {
-	handle, err := db.EnsureHandle(prefixes.MempoolTx)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	key := prefixes.MempoolTxKey{Prefix: []byte{prefixes.Tx}, TxHash: txhash}
-	rawKey := key.PackKey()
-	slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
-	defer slice.Free()
-	if err != nil {
-		return nil, nil, err
-	}
-	if slice.Size() == 0 {
-		return nil, nil, nil
-	}
-
-	rawValue := make([]byte, len(slice.Data()))
-	copy(rawValue, slice.Data())
-	value := prefixes.MempoolTxValue{}
-	value.UnpackValue(rawValue)
-	var msgTx wire.MsgTx
-	err = msgTx.Deserialize(bytes.NewReader(value.RawTx))
-	if err != nil {
-		return nil, nil, err
-	}
-	return value.RawTx, &msgTx, nil
-}
-
 func (db *ReadOnlyDBColumnFamily) GetTxCount(height uint32) (*prefixes.TxCountValue, error) {
 	handle, err := db.EnsureHandle(prefixes.TxCount)
 	if err != nil {
@@ -879,162 +754,6 @@ func (db *ReadOnlyDBColumnFamily) GetTxCount(height uint32) (*prefixes.TxCountVa
 	return value, nil
 }
 
-func (db *ReadOnlyDBColumnFamily) GetTxHeight(txhash *chainhash.Hash) (uint32, error) {
-	handle, err := db.EnsureHandle(prefixes.TxNum)
-	if err != nil {
-		return 0, err
-	}
-
-	key := prefixes.TxNumKey{Prefix: []byte{prefixes.TxNum}, TxHash: txhash}
-	rawKey := key.PackKey()
-	slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
-	defer slice.Free()
-	if err != nil {
-		return 0, err
-	}
-	if slice.Size() == 0 {
-		return 0, nil
-	}
-
-	// No slice copy needed. Value will be abandoned.
-	value := prefixes.TxNumValueUnpack(slice.Data())
-	height := stack.BisectRight(db.TxCounts, []uint32{value.TxNum})[0]
-	return height, nil
-}
-
-type TxMerkle struct {
-	TxHash *chainhash.Hash
-	RawTx  []byte
-	Height int
-	Pos    uint32
-	Merkle []*chainhash.Hash
-}
-
-// merklePath selects specific transactions by position within blockTxs.
-// The resulting merkle path (aka merkle branch, or merkle) is a list of TX hashes
-// which are in sibling relationship with TX nodes on the path to the root.
-func merklePath(pos uint32, blockTxs, partial []*chainhash.Hash) []*chainhash.Hash {
-	parent := func(p uint32) uint32 {
-		return p >> 1
-	}
-	sibling := func(p uint32) uint32 {
-		if p%2 == 0 {
-			return p + 1
-		} else {
-			return p - 1
-		}
-	}
-	p := parent(pos)
-	if p == 0 {
-		// No parent, path is complete.
-		return partial
-	}
-	// Add sibling to partial path and proceed to parent TX.
-	return merklePath(p, blockTxs, append(partial, blockTxs[sibling(pos)]))
-}
-
-func (db *ReadOnlyDBColumnFamily) GetTxMerkle(tx_hashes []chainhash.Hash) ([]TxMerkle, error) {
-
-	selectedTxNum := make([]*IterOptions, 0, len(tx_hashes))
-	for _, txhash := range tx_hashes {
-		key := prefixes.TxNumKey{Prefix: []byte{prefixes.TxNum}, TxHash: &txhash}
-		log.Debugf("%v", key)
-		opt, err := db.selectFrom(key.Prefix, &key, &key)
-		if err != nil {
-			return nil, err
-		}
-		selectedTxNum = append(selectedTxNum, opt...)
-	}
-
-	selectTxByTxNum := func(in []*prefixes.PrefixRowKV) ([]*IterOptions, error) {
-		txNumKey := in[0].Key.(*prefixes.TxNumKey)
-		log.Debugf("%v", txNumKey.TxHash.String())
-		out := make([]*IterOptions, 0, 100)
-		startKey := &prefixes.TxKey{
-			Prefix: []byte{prefixes.Tx},
-			TxHash: txNumKey.TxHash,
-		}
-		endKey := &prefixes.TxKey{
-			Prefix: []byte{prefixes.Tx},
-			TxHash: txNumKey.TxHash,
-		}
-		selectedTx, err := db.selectFrom([]byte{prefixes.Tx}, startKey, endKey)
-		if err != nil {
-			return nil, err
-		}
-		out = append(out, selectedTx...)
-		return out, nil
-	}
-
-	blockTxsCache := make(map[uint32][]*chainhash.Hash)
-	results := make([]TxMerkle, 0, 500)
-	for kvs := range innerJoin(db.DB, iterate(db.DB, selectedTxNum), selectTxByTxNum) {
-		if err := checkForError(kvs); err != nil {
-			return results, err
-		}
-		txNumKey, txNumVal := kvs[0].Key.(*prefixes.TxNumKey), kvs[0].Value.(*prefixes.TxNumValue)
-		_, txVal := kvs[1].Key.(*prefixes.TxKey), kvs[1].Value.(*prefixes.TxValue)
-		txHeight := stack.BisectRight(db.TxCounts, []uint32{txNumVal.TxNum})[0]
-		txPos := txNumVal.TxNum - db.TxCounts.Get(txHeight-1)
-		// We need all the TX hashes in order to select out the relevant ones.
-		if _, ok := blockTxsCache[txHeight]; !ok {
-			txs, err := db.GetBlockTXs(txHeight)
-			if err != nil {
-				return results, err
-			}
-			blockTxsCache[txHeight] = txs
-		}
-		blockTxs := blockTxsCache[txHeight]
-		results = append(results, TxMerkle{
-			TxHash: txNumKey.TxHash,
-			RawTx:  txVal.RawTx,
-			Height: int(txHeight),
-			Pos:    txPos,
-			Merkle: merklePath(txPos, blockTxs, []*chainhash.Hash{}),
-		})
-	}
-	return results, nil
-}
-
-func (db *ReadOnlyDBColumnFamily) GetClaimByID(claimID string) ([]*ExpandedResolveResult, []*ExpandedResolveResult, error) {
-	rows := make([]*ExpandedResolveResult, 0)
-	extras := make([]*ExpandedResolveResult, 0)
-	claimHash, err := hex.DecodeString(claimID)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	stream, err := db.FsGetClaimByHash(claimHash)
-	if err != nil {
-		return nil, nil, err
-	}
-	var res = NewExpandedResolveResult()
-	res.Stream = &optionalResolveResultOrError{res: stream}
-	rows = append(rows, res)
-
-	if stream != nil && stream.ChannelHash != nil {
-		channel, err := db.FsGetClaimByHash(stream.ChannelHash)
-		if err != nil {
-			return nil, nil, err
-		}
-		var res = NewExpandedResolveResult()
-		res.Channel = &optionalResolveResultOrError{res: channel}
-		extras = append(extras, res)
-	}
-
-	if stream != nil && stream.RepostedClaimHash != nil {
-		repost, err := db.FsGetClaimByHash(stream.RepostedClaimHash)
-		if err != nil {
-			return nil, nil, err
-		}
-		var res = NewExpandedResolveResult()
-		res.Repost = &optionalResolveResultOrError{res: repost}
-		extras = append(extras, res)
-	}
-
-	return rows, extras, nil
-}
-
 func (db *ReadOnlyDBColumnFamily) GetDBState() (*prefixes.DBStateValue, error) {
 	handle, err := db.EnsureHandle(prefixes.DBState)
 	if err != nil {
@@ -1066,7 +785,7 @@ func (db *ReadOnlyDBColumnFamily) BidOrderNameIter(normalizedName string) <-chan
 	key := prefixes.NewBidOrderKey(normalizedName)
 	var rawKeyPrefix []byte = nil
 	rawKeyPrefix = key.PartialPack(1)
-	options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
+	options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 	options = options.WithIncludeValue(true) //.WithIncludeStop(true)
 	ch := IterCF(db.DB, options)
 	return ch
@@ -1084,7 +803,7 @@ func (db *ReadOnlyDBColumnFamily) ClaimShortIdIter(normalizedName string, claimI
 	} else {
 		rawKeyPrefix = key.PartialPack(1)
 	}
-	options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
+	options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 	options = options.WithIncludeValue(true) //.WithIncludeStop(true)
 	ch := IterCF(db.DB, options)
 	return ch

db/db_resolve.go

@@ -326,7 +326,7 @@ func (db *ReadOnlyDBColumnFamily) ResolveClaimInChannel(channelHash []byte, norm
 
 	key := prefixes.NewChannelToClaimKey(channelHash, normalizedName)
 	rawKeyPrefix := key.PartialPack(2)
-	options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
+	options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 	options = options.WithIncludeValue(true) //.WithIncludeStop(true)
 	ch := IterCF(db.DB, options)
 	// TODO: what's a good default size for this?

db/db_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/csv"
 	"encoding/hex"
+	"log"
 	"os"
 	"strings"
 	"testing"
@@ -11,9 +12,7 @@ import (
 	dbpkg "github.com/lbryio/herald.go/db"
 	"github.com/lbryio/herald.go/db/prefixes"
 	"github.com/lbryio/herald.go/internal"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	"github.com/linxGnu/grocksdb"
-	log "github.com/sirupsen/logrus"
 )
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -21,7 +20,7 @@ import (
 ////////////////////////////////////////////////////////////////////////////////
 
 // OpenAndFillTmpDBColumnFamlies opens a db and fills it with data from a csv file using the given column family names
-func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFamily, [][]string, error) {
+func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFamily, [][]string, func(), error) {
 
 	log.Println(filePath)
 	file, err := os.Open(filePath)
@@ -31,7 +30,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
 	reader := csv.NewReader(file)
 	records, err := reader.ReadAll()
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	wOpts := grocksdb.NewDefaultWriteOptions()
@@ -39,7 +38,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
 	opts.SetCreateIfMissing(true)
 	db, err := grocksdb.OpenDb(opts, "tmp")
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	var handleMap map[string]*grocksdb.ColumnFamilyHandle = make(map[string]*grocksdb.ColumnFamilyHandle)
 
@@ -54,7 +53,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
 		log.Println(cfName)
 		handle, err := db.CreateColumnFamily(opts, cfName)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		handleMap[cfName] = handle
 	}
@@ -68,16 +67,16 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
 	for _, record := range records[1:] {
 		cf := record[0]
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		handle := handleMap[string(cf)]
 		key, err := hex.DecodeString(record[1])
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		val, err := hex.DecodeString(record[2])
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		db.PutCF(wOpts, handle, key, val)
 	}
@@ -94,8 +93,6 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
 		LastState:        nil,
 		Height:           0,
 		Headers:          nil,
-		Grp:              stop.New(),
-		Cleanup:          toDefer,
 	}
 
 	// err = dbpkg.ReadDBState(myDB) //TODO: Figure out right place for this
@@ -105,7 +102,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
 
 	err = myDB.InitTxCounts()
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	// err = dbpkg.InitHeaders(myDB)
@@ -113,7 +110,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
 	// 	return nil, nil, nil, err
 	// }
 
-	return myDB, records, nil
+	return myDB, records, toDefer, nil
 }
 
 // OpenAndFillTmpDBCF opens a db and fills it with data from a csv file
@@ -244,18 +241,17 @@ func CatCSV(filePath string) {
 
 func TestCatFullDB(t *testing.T) {
 	t.Skip("Skipping full db test")
-	grp := stop.New()
 	// url := "lbry://@lothrop#2/lothrop-livestream-games-and-code#c"
 	// "lbry://@lbry", "lbry://@lbry#3", "lbry://@lbry3f", "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a", "lbry://@lbry:1", "lbry://@lbry$1"
 	// url := "lbry://@Styxhexenhammer666#2/legacy-media-baron-les-moonves-(cbs#9"
 	// url := "lbry://@lbry"
 	// url := "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a"
-	dbPath := "/mnt/sda1/wallet_server/_data/lbry-rocksdb/"
+	dbPath := "/mnt/sda/wallet_server/_data/lbry-rocksdb/"
 	// dbPath := "/mnt/d/data/snapshot_1072108/lbry-rocksdb/"
 	secondaryPath := "asdf"
-	db, err := dbpkg.GetProdDB(dbPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := dbpkg.GetProdDB(dbPath, secondaryPath)
 
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
@@ -275,7 +271,6 @@ func TestCatFullDB(t *testing.T) {
 // TestOpenFullDB Tests running a resolve on a full db.
 func TestOpenFullDB(t *testing.T) {
 	t.Skip("Skipping full db test")
-	grp := stop.New()
 	// url := "lbry://@lothrop#2/lothrop-livestream-games-and-code#c"
 	// "lbry://@lbry", "lbry://@lbry#3", "lbry://@lbry3f", "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a", "lbry://@lbry:1", "lbry://@lbry$1"
 	// url := "lbry://@Styxhexenhammer666#2/legacy-media-baron-les-moonves-(cbs#9"
@@ -283,11 +278,11 @@ func TestOpenFullDB(t *testing.T) {
 	// url := "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a"
 	// url := "lbry://@lbry$1"
 	url := "https://lbry.tv/@lothrop:2/lothrop-livestream-games-and-code:c"
-	dbPath := "/mnt/sda1/wallet_server/_data/lbry-rocksdb/"
+	dbPath := "/mnt/sda/wallet_server/_data/lbry-rocksdb/"
 	// dbPath := "/mnt/d/data/snapshot_1072108/lbry-rocksdb/"
 	secondaryPath := "asdf"
-	db, err := dbpkg.GetProdDB(dbPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := dbpkg.GetProdDB(dbPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
@@ -301,13 +296,12 @@ func TestOpenFullDB(t *testing.T) {
 func TestResolve(t *testing.T) {
 	url := "lbry://@Styxhexenhammer666#2/legacy-media-baron-les-moonves-(cbs#9"
 	filePath := "../testdata/FULL_resolve.csv"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 		return
 	}
-
+	defer toDefer()
 	expandedResolveResult := db.Resolve(url)
 	log.Printf("%#v\n", expandedResolveResult)
 	if expandedResolveResult != nil && expandedResolveResult.Channel != nil {
@@ -321,11 +315,11 @@ func TestResolve(t *testing.T) {
 func TestGetDBState(t *testing.T) {
 	filePath := "../testdata/s_resolve.csv"
 	want := uint32(1072108)
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
+	defer toDefer()
 	state, err := db.GetDBState()
 	if err != nil {
 		t.Error(err)
@@ -343,11 +337,11 @@ func TestGetRepostedClaim(t *testing.T) {
 	// Should be non-existent
 	channelHash2, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bf")
 	filePath := "../testdata/W_resolve.csv"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
+	defer toDefer()
 
 	count, err := db.GetRepostedCount(channelHash)
 	if err != nil {
@@ -376,11 +370,11 @@ func TestGetRepostedCount(t *testing.T) {
 	// Should be non-existent
 	channelHash2, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bf")
 	filePath := "../testdata/j_resolve.csv"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
+	defer toDefer()
 
 	count, err := db.GetRepostedCount(channelHash)
 	if err != nil {
@@ -413,11 +407,11 @@ func TestGetRepost(t *testing.T) {
 	channelHash2, _ := hex.DecodeString("000009ca6e0caaaef16872b4bd4f6f1b8c2363e2")
 	filePath := "../testdata/V_resolve.csv"
 	// want := uint32(3670)
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
+	defer toDefer()
 
 	res, err := db.GetRepost(channelHash)
 	if err != nil {
@@ -447,12 +441,11 @@ func TestGetClaimsInChannelCount(t *testing.T) {
 	channelHash, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bd")
 	filePath := "../testdata/Z_resolve.csv"
 	want := uint32(3670)
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
-
+	defer toDefer()
 	count, err := db.GetClaimsInChannelCount(channelHash)
 	if err != nil {
 		t.Error(err)
@@ -477,12 +470,11 @@ func TestGetShortClaimIdUrl(t *testing.T) {
 	var position uint16 = 0
 	filePath := "../testdata/F_resolve.csv"
 	log.Println(filePath)
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
-
+	defer toDefer()
 	shortUrl, err := db.GetShortClaimIdUrl(name, normalName, claimHash, rootTxNum, position)
 	if err != nil {
 		t.Error(err)
@@ -496,11 +488,11 @@ func TestClaimShortIdIter(t *testing.T) {
 	filePath := "../testdata/F_cat.csv"
 	normalName := "cat"
 	claimId := "0"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
+	defer toDefer()
 
 	ch := db.ClaimShortIdIter(normalName, claimId)
 
@@ -526,12 +518,11 @@ func TestGetTXOToClaim(t *testing.T) {
 	var txNum uint32 = 1456296
 	var position uint16 = 0
 	filePath := "../testdata/G_2.csv"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
-
+	defer toDefer()
 	val, err := db.GetCachedClaimHash(txNum, position)
 	if err != nil {
 		t.Error(err)
@@ -555,11 +546,11 @@ func TestGetClaimToChannel(t *testing.T) {
 	var val []byte = nil
 
 	filePath := "../testdata/I_resolve.csv"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
+	defer toDefer()
 
 	val, err = db.GetChannelForClaim(claimHash, txNum, position)
 	if err != nil {
@@ -584,11 +575,11 @@ func TestGetEffectiveAmountSupportOnly(t *testing.T) {
 	want := uint64(20000006)
 	claimHashStr := "00000324e40fcb63a0b517a3660645e9bd99244a"
 	claimHash, _ := hex.DecodeString(claimHashStr)
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
+	defer toDefer()
 	db.Height = 999999999
 
 	amount, err := db.GetEffectiveAmount(claimHash, true)
@@ -614,12 +605,11 @@ func TestGetEffectiveAmount(t *testing.T) {
 	want := uint64(21000006)
 	claimHashStr := "00000324e40fcb63a0b517a3660645e9bd99244a"
 	claimHash, _ := hex.DecodeString(claimHashStr)
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
-
+	defer toDefer()
 	db.Height = 999999999
 
 	amount, err := db.GetEffectiveAmount(claimHash, false)
@@ -652,12 +642,11 @@ func TestGetSupportAmount(t *testing.T) {
 		t.Error(err)
 	}
 	filePath := "../testdata/a_resolve.csv"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
-
+	defer toDefer()
 	res, err := db.GetSupportAmount(claimHash)
 	if err != nil {
 		t.Error(err)
@@ -673,12 +662,11 @@ func TestGetTxHash(t *testing.T) {
 	want := "54e14ff0c404c29b3d39ae4d249435f167d5cd4ce5a428ecb745b3df1c8e3dde"
 
 	filePath := "../testdata/X_resolve.csv"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
-
+	defer toDefer()
 	resHash, err := db.GetTxHash(txNum)
 	if err != nil {
 		t.Error(err)
@@ -716,12 +704,11 @@ func TestGetActivation(t *testing.T) {
 	txNum := uint32(0x6284e3)
 	position := uint16(0x0)
 	want := uint32(0xa6b65)
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 	}
-
+	defer toDefer()
 	activation, err := db.GetActivation(txNum, position)
 	if err != nil {
 		t.Error(err)
@@ -748,13 +735,12 @@ func TestGetClaimToTXO(t *testing.T) {
 		return
 	}
 	filePath := "../testdata/E_resolve.csv"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 		return
 	}
-
+	defer toDefer()
 	res, err := db.GetCachedClaimTxo(claimHash, true)
 	if err != nil {
 		t.Error(err)
@@ -778,13 +764,12 @@ func TestGetControllingClaim(t *testing.T) {
 	claimName := internal.NormalizeName("@Styxhexenhammer666")
 	claimHash := "2556ed1cab9d17f2a9392030a9ad7f5d138f11bd"
 	filePath := "../testdata/P_resolve.csv"
-	db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
-	defer db.Shutdown()
+	db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
 	if err != nil {
 		t.Error(err)
 		return
 	}
-
+	defer toDefer()
 	res, err := db.GetControllingClaim(claimName)
 	if err != nil {
 		t.Error(err)

db/iteroptions.go

@@ -6,7 +6,6 @@ import (
 	"bytes"
 
 	"github.com/lbryio/herald.go/db/prefixes"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	"github.com/linxGnu/grocksdb"
 
 	log "github.com/sirupsen/logrus"
@@ -23,11 +22,9 @@ type IterOptions struct {
 	IncludeValue bool
 	RawKey       bool
 	RawValue     bool
-	Grp          *stop.Group
-	// DB           *ReadOnlyDBColumnFamily
-	CfHandle   *grocksdb.ColumnFamilyHandle
-	It         *grocksdb.Iterator
-	Serializer *prefixes.SerializationAPI
+	CfHandle     *grocksdb.ColumnFamilyHandle
+	It           *grocksdb.Iterator
+	Serializer   *prefixes.SerializationAPI
 }
 
 // NewIterateOptions creates a defualt options structure for a db iterator.
@@ -43,11 +40,9 @@ func NewIterateOptions() *IterOptions {
 		IncludeValue: false,
 		RawKey:       false,
 		RawValue:     false,
-		Grp:          nil,
-		// DB:           nil,
-		CfHandle:   nil,
-		It:         nil,
-		Serializer: prefixes.ProductionAPI,
+		CfHandle:     nil,
+		It:           nil,
+		Serializer:   prefixes.ProductionAPI,
 	}
 }
 
@@ -106,13 +101,6 @@ func (o *IterOptions) WithRawValue(rawValue bool) *IterOptions {
 	return o
 }
 
-func (o *IterOptions) WithDB(db *ReadOnlyDBColumnFamily) *IterOptions {
-	// o.Grp.AddNamed(1, iterKey)
-	o.Grp = stop.New(db.Grp)
-	o.Grp.Add(1)
-	return o
-}
-
 func (o *IterOptions) WithSerializer(serializer *prefixes.SerializationAPI) *IterOptions {
 	o.Serializer = serializer
 	return o

db/prefixes/generic.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 
 	"github.com/go-restruct/restruct"
-	"github.com/lbryio/herald.go/internal"
 	"github.com/lbryio/lbcd/chaincfg/chainhash"
 )
 
@@ -60,34 +59,6 @@ func (kv *BlockTxsValue) Unpack(buf []byte, order binary.ByteOrder) ([]byte, err
 	return buf[offset:], nil
 }
 
-// Struct BigEndianChainHash is a chainhash.Hash stored in external
-// byte-order (opposite of other 32 byte chainhash.Hash values). In order
-// to reuse chainhash.Hash we need to correct the byte-order.
-// Currently this type is used for field Genesis of DBStateValue.
-
-func (kv *BigEndianChainHash) SizeOf() int {
-	return chainhash.HashSize
-}
-
-func (kv *BigEndianChainHash) Pack(buf []byte, order binary.ByteOrder) ([]byte, error) {
-	offset := 0
-	hash := kv.CloneBytes()
-	// HACK: Instances of chainhash.Hash use the internal byte-order.
-	// Python scribe writes bytes of genesis hash in external byte-order.
-	internal.ReverseBytesInPlace(hash)
-	offset += copy(buf[offset:chainhash.HashSize], hash[:])
-	return buf[offset:], nil
-}
-
-func (kv *BigEndianChainHash) Unpack(buf []byte, order binary.ByteOrder) ([]byte, error) {
-	offset := 0
-	offset += copy(kv.Hash[:], buf[offset:32])
-	// HACK: Instances of chainhash.Hash use the internal byte-order.
-	// Python scribe writes bytes of genesis hash in external byte-order.
-	internal.ReverseBytesInPlace(kv.Hash[:])
-	return buf[offset:], nil
-}
-
 func genericNew(prefix []byte, key bool) (interface{}, error) {
 	t, ok := prefixRegistry[prefix[0]]
 	if !ok {

db/prefixes/prefixes.go

@@ -12,13 +12,13 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"log"
 	"reflect"
 	"sort"
 	"strings"
 
 	"github.com/lbryio/herald.go/internal"
 	"github.com/lbryio/lbcd/chaincfg/chainhash"
-	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -182,25 +182,12 @@ func NewLengthEncodedPartialClaimId(s string) LengthEncodedPartialClaimId {
 	}
 }
 
-type BigEndianChainHash struct {
-	chainhash.Hash
-}
-
-func NewBigEndianChainHash(hash *chainhash.Hash) BigEndianChainHash {
-	if hash != nil {
-		return BigEndianChainHash{
-			*hash,
-		}
-	}
-	return BigEndianChainHash{}
-}
-
 type DBStateKey struct {
 	Prefix []byte `struct:"[1]byte" json:"prefix"`
 }
 
 type DBStateValue struct {
-	Genesis        BigEndianChainHash
+	Genesis        *chainhash.Hash
 	Height         uint32
 	TxCount        uint32
 	Tip            *chainhash.Hash
@@ -216,7 +203,7 @@ type DBStateValue struct {
 
 func NewDBStateValue() *DBStateValue {
 	return &DBStateValue{
-		Genesis:        NewBigEndianChainHash(nil),
+		Genesis:        new(chainhash.Hash),
 		Height:         0,
 		TxCount:        0,
 		Tip:            new(chainhash.Hash),
@@ -250,11 +237,7 @@ func (v *DBStateValue) PackValue() []byte {
 	// b'>32sLL32sLLBBlllL'
 	n := 32 + 4 + 4 + 32 + 4 + 4 + 1 + 1 + 4 + 4 + 4 + 4
 	value := make([]byte, n)
-	genesis := v.Genesis.CloneBytes()
-	// HACK: Instances of chainhash.Hash use the internal byte-order.
-	// Python scribe writes bytes of genesis hash in external byte-order.
-	internal.ReverseBytesInPlace(genesis)
-	copy(value, genesis[:32])
+	copy(value, v.Genesis[:32])
 	binary.BigEndian.PutUint32(value[32:], v.Height)
 	binary.BigEndian.PutUint32(value[32+4:], v.TxCount)
 	copy(value[32+4+4:], v.Tip[:32])
@@ -299,11 +282,8 @@ func DBStateKeyUnpack(key []byte) *DBStateKey {
 func DBStateValueUnpack(value []byte) *DBStateValue {
 	genesis := (*chainhash.Hash)(value[:32])
 	tip := (*chainhash.Hash)(value[32+4+4 : 32+4+4+32])
-	// HACK: Python scribe writes bytes of genesis hash in external byte-order.
-	// Instances of chainhash.Hash should use the internal byte-order.
-	internal.ReverseBytesInPlace(genesis[:])
 	x := &DBStateValue{
-		Genesis:        NewBigEndianChainHash(genesis),
+		Genesis:        genesis,
 		Height:         binary.BigEndian.Uint32(value[32:]),
 		TxCount:        binary.BigEndian.Uint32(value[32+4:]),
 		Tip:            tip,
@@ -728,7 +708,7 @@ type BlockTxsKey struct {
 }
 
 type BlockTxsValue struct {
-	TxHashes []*chainhash.Hash `struct:"*[32]byte" struct-while:"!_eof" json:"tx_hashes"`
+	TxHashes []*chainhash.Hash `struct-while:"!_eof" json:"tx_hashes"`
 }
 
 func (k *BlockTxsKey) NewBlockTxsKey(height uint32) *BlockTxsKey {
@@ -1070,6 +1050,84 @@ func TxNumValueUnpack(value []byte) *TxNumValue {
 	}
 }
 
+type TxKey struct {
+	Prefix []byte          `struct:"[1]byte" json:"prefix"`
+	TxHash *chainhash.Hash `struct:"*[32]byte" json:"tx_hash"`
+}
+
+type TxValue struct {
+	RawTx []byte `struct-while:"!_eof" json:"raw_tx"`
+}
+
+func (k *TxKey) PackKey() []byte {
+	prefixLen := 1
+	// b'>L'
+	n := prefixLen + 32
+	key := make([]byte, n)
+	copy(key, k.Prefix)
+	copy(key[prefixLen:], k.TxHash[:32])
+
+	return key
+}
+
+func (v *TxValue) PackValue() []byte {
+	value := make([]byte, len(v.RawTx))
+	copy(value, v.RawTx)
+
+	return value
+}
+
+func (kv *TxKey) NumFields() int {
+	return 1
+}
+
+func (k *TxKey) PartialPack(fields int) []byte {
+	// Limit fields between 0 and number of fields, we always at least need
+	// the prefix, and we never need to iterate past the number of fields.
+	if fields > 1 {
+		fields = 1
+	}
+	if fields < 0 {
+		fields = 0
+	}
+
+	prefixLen := 1
+	var n = prefixLen
+	for i := 0; i <= fields; i++ {
+		switch i {
+		case 1:
+			n += 32
+		}
+	}
+
+	key := make([]byte, n)
+
+	for i := 0; i <= fields; i++ {
+		switch i {
+		case 0:
+			copy(key, k.Prefix)
+		case 1:
+			copy(key[prefixLen:], k.TxHash[:32])
+		}
+	}
+
+	return key
+}
+
+func TxKeyUnpack(key []byte) *TxKey {
+	prefixLen := 1
+	return &TxKey{
+		Prefix: key[:prefixLen],
+		TxHash: (*chainhash.Hash)(key[prefixLen : prefixLen+32]),
+	}
+}
+
+func TxValueUnpack(value []byte) *TxValue {
+	return &TxValue{
+		RawTx: value,
+	}
+}
+
 type BlockHeaderKey struct {
 	Prefix []byte `struct:"[1]byte" json:"prefix"`
 	Height uint32 `json:"height"`
@@ -3293,12 +3351,9 @@ func (kv *TrendingNotificationValue) UnpackValue(buf []byte) {
 	offset += 8
 }
 
-type TxKey = MempoolTxKey
-type TxValue = MempoolTxValue
-
 type MempoolTxKey struct {
-	Prefix []byte          `struct:"[1]byte" json:"prefix"`
-	TxHash *chainhash.Hash `struct:"*[32]byte" json:"tx_hash"`
+	Prefix []byte `struct:"[1]byte"  json:"prefix"`
+	TxHash []byte `struct:"[32]byte" json:"tx_hash"`
 }
 
 type MempoolTxValue struct {
@@ -3331,7 +3386,7 @@ func (kv *MempoolTxKey) UnpackKey(buf []byte) {
 	offset := 0
 	kv.Prefix = buf[offset : offset+1]
 	offset += 1
-	kv.TxHash = (*chainhash.Hash)(buf[offset : offset+32])
+	kv.TxHash = buf[offset : offset+32]
 	offset += 32
 }
 
@@ -3413,7 +3468,7 @@ func (kv *TouchedHashXValue) UnpackValue(buf []byte) {
 
 type HashXStatusKey struct {
 	Prefix []byte `struct:"[1]byte" json:"prefix"`
-	HashX  []byte `struct:"[11]byte" json:"hashX"`
+	HashX  []byte `struct:"[20]byte" json:"hashX"`
 }
 
 type HashXStatusValue struct {
@@ -3425,15 +3480,15 @@ func (kv *HashXStatusKey) NumFields() int {
 }
 
 func (kv *HashXStatusKey) PartialPack(fields int) []byte {
-	// b'>20s' (really HASHX_LEN 11 bytes)
-	n := len(kv.Prefix) + 11
+	// b'>20s'
+	n := len(kv.Prefix) + 20
 	buf := make([]byte, n)
 	offset := 0
 	offset += copy(buf[offset:], kv.Prefix[:1])
 	if fields <= 0 {
 		return buf[:offset]
 	}
-	offset += copy(buf[offset:], kv.HashX[:11])
+	offset += copy(buf[offset:], kv.HashX[:20])
 	return buf[:offset]
 }
 
@@ -3442,12 +3497,12 @@ func (kv *HashXStatusKey) PackKey() []byte {
 }
 
 func (kv *HashXStatusKey) UnpackKey(buf []byte) {
-	// b'>20s' (really HASHX_LEN 11 bytes)
+	// b'>20s'
 	offset := 0
 	kv.Prefix = buf[offset : offset+1]
 	offset += 1
-	kv.HashX = buf[offset : offset+11]
-	offset += 11
+	kv.HashX = buf[offset : offset+20]
+	offset += 20
 }
 
 func (kv *HashXStatusValue) PackValue() []byte {
@@ -3870,6 +3925,12 @@ var prefixRegistry = map[byte]prefixMeta{
 		newValue: func() interface{} {
 			return &TxValue{}
 		},
+		newKeyUnpack: func(buf []byte) interface{} {
+			return TxKeyUnpack(buf)
+		},
+		newValueUnpack: func(buf []byte) interface{} {
+			return TxValueUnpack(buf)
+		},
 	},
 	BlockHash: {
 		newKey: func() interface{} {

db/prefixes/prefixes_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/csv"
 	"encoding/hex"
 	"fmt"
+	"log"
 	"math"
 	"math/big"
 	"os"
@@ -15,7 +16,6 @@ import (
 	dbpkg "github.com/lbryio/herald.go/db"
 	prefixes "github.com/lbryio/herald.go/db/prefixes"
 	"github.com/linxGnu/grocksdb"
-	log "github.com/sirupsen/logrus"
 )
 
 func TestPrefixRegistry(t *testing.T) {

docker/Dockerfile.action.integration

@@ -1,13 +0,0 @@
-FROM jeffreypicard/hub-github-env:dev
-
-COPY scripts/integration_tests.sh /integration_tests.sh
-COPY scripts/cicd_integration_test_runner.sh /cicd_integration_test_runner.sh
-COPY herald /herald
-
-RUN apt install -y jq curl
-
-ENV CGO_LDFLAGS "-L/usr/local/lib -lrocksdb -lstdc++ -lm -lz -lsnappy -llz4 -lzstd"
-ENV CGO_CFLAGS "-I/usr/local/include/rocksdb"
-ENV LD_LIBRARY_PATH /usr/local/lib
-
-ENTRYPOINT ["/cicd_integration_test_runner.sh"]

go.mod

@@ -24,11 +24,12 @@ require (
 	gopkg.in/karalabe/cookiejar.v1 v1.0.0-20141109175019-e1490cae028c
 )
 
+require (
+)
+
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
-	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
-	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -42,7 +43,7 @@ require (
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
-	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
 	google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 // indirect

go.sum

@@ -68,13 +68,11 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufo
 github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o=
-github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
 github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
 github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
 github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I=
 github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
 github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
-github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
@@ -360,6 +358,8 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
 github.com/lbryio/lbcd v0.22.100-beta/go.mod h1:u8SaFX4xdGMMR5xasBGfgApC8pvD4rnK2OujZnrq5gs=
 github.com/lbryio/lbcd v0.22.100-beta-rc5/go.mod h1:9PbFSlHYX7WlnDQwcTxHVf1W35VAnRsattCSyKOO55g=
 github.com/lbryio/lbcd v0.22.200-beta/go.mod h1:kNuzGWf808ipTGB0y0WogzsGv5BVM4Qv85Z+JYwC9FA=
+github.com/lbryio/lbcd v0.22.201-beta-rc1 h1:FmzzApVj2RBXloLM2w9tLvN2xyTZjeyh+QC7GIw/wwo=
+github.com/lbryio/lbcd v0.22.201-beta-rc1/go.mod h1:kNuzGWf808ipTGB0y0WogzsGv5BVM4Qv85Z+JYwC9FA=
 github.com/lbryio/lbcd v0.22.201-beta-rc4 h1:Xh751Bh/GWRcP5bI6NJ2+zueo2otTcTWapFvFbryP5c=
 github.com/lbryio/lbcd v0.22.201-beta-rc4/go.mod h1:Jgo48JDINhdOgHHR83J70Q6G42x3WAo9DI//QogcL+E=
 github.com/lbryio/lbcutil v1.0.201/go.mod h1:gDHc/b+Rdz3J7+VB8e5/Bl9roVf8Q5/8FQCyuK9dXD0=
@@ -375,6 +375,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/linxGnu/grocksdb v1.6.42 h1:nJLoXFuzwBwQQQrXTUgRGRz1QRm7y8pR6CNV/gwrbqs=
 github.com/linxGnu/grocksdb v1.6.42/go.mod h1:JcMMDBFaDNhRXFYcYXmgQwb/RarSld1PulTI7UzE+w0=
+github.com/linxGnu/grocksdb v1.7.0 h1:UyFDykX0CUfxDN10cqlFho/rwt9K6KoDaLXL9Ej5z9g=
+github.com/linxGnu/grocksdb v1.7.0/go.mod h1:JcMMDBFaDNhRXFYcYXmgQwb/RarSld1PulTI7UzE+w0=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=

internal/types.go

@@ -4,7 +4,6 @@ package internal
 
 // HeightHash struct for the height subscription endpoint.
 type HeightHash struct {
-	Height      uint64
-	BlockHash   []byte
-	BlockHeader []byte
+	Height    uint64
+	BlockHash []byte
 }

main.go

@@ -10,10 +10,8 @@ import (
 	"github.com/lbryio/herald.go/internal"
 	pb "github.com/lbryio/herald.go/protobuf/go"
 	"github.com/lbryio/herald.go/server"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
 )
 
 func main() {
@@ -29,22 +27,37 @@ func main() {
 
 	if args.CmdType == server.ServeCmd {
 		// This will cancel goroutines with the server finishes.
-		stopGroup := stop.New()
+		ctxWCancel, cancel := context.WithCancel(ctx)
+		defer cancel()
 
 		initsignals()
 		interrupt := interruptListener()
 
-		s := server.MakeHubServer(stopGroup, args)
+		s := server.MakeHubServer(ctxWCancel, args)
 		go s.Run()
 
-		defer s.Stop()
+		defer func() {
+			log.Println("Shutting down server...")
+
+			if s.EsClient != nil {
+				s.EsClient.Stop()
+			}
+			if s.GrpcServer != nil {
+				s.GrpcServer.GracefulStop()
+			}
+			if s.DB != nil {
+				s.DB.Shutdown()
+			}
+
+			log.Println("Returning from main...")
+		}()
 
 		<-interrupt
 		return
 	}
 
-	conn, err := grpc.Dial("localhost:"+fmt.Sprintf("%d", args.Port),
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	conn, err := grpc.Dial("localhost:"+args.Port,
+		grpc.WithInsecure(),
 		grpc.WithBlock(),
 	)
 	if err != nil {

scripts/cicd_integration_test_runner.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-#
-# cicd_integration_test_runner.sh
-#
-# simple script to kick off herald and call the integration testing
-# script
-#
-# N.B. this currently just works locally until we figure a way to have
-# the data in the cicd environment.
-#
-
-./herald serve --db-path /mnt/sdb1/wallet_server/_data/lbry-rocksdb &
-
-./integration_tests.sh

scripts/integration_tests.sh

@@ -1,235 +0,0 @@
-#!/bin/bash
-#
-# integration_testing.sh
-#
-# GitHub Action CI/CD based integration tests for herald.go
-# These are smoke / sanity tests for the server behaving correctly on a "live"
-# system, and looks for reasonable response codes, not specific correct
-# behavior. Those are covered in unit tests.
-#
-# N.B.
-# For the curl based json tests the `id` field existing is needed.
-#
-
-# global variables
-
-RES=(0)
-FINALRES=0
-# functions
-
-
-function logical_or {
-	for res in ${RES[@]}; do
-		if [ $res -eq 1 -o $FINALRES -eq 1 ]; then
-			FINALRES=1
-			return
-		fi
-	done
-}
-
-function want_got {
-	if [ "${WANT}" != "${GOT}" ]; then
-		echo "WANT: ${WANT}"
-		echo "GOT: ${GOT}"
-		RES+=(1)
-	else
-		RES+=(0)
-	fi
-}
-
-function want_greater {
-	if [ ${WANT} -ge ${GOT} ]; then
-		echo "WANT: ${WANT}"
-		echo "GOT: ${GOT}"
-		RES+=(1)
-	else
-		RES+=(0)
-	fi
-}
-
-function test_command_with_want {
-	echo $CMD
-	GOT=`eval $CMD`
-
-	want_got
-}
-
-# grpc endpoint testing
-
-
-read -r -d '' CMD <<- EOM
-	grpcurl -plaintext -d '{"value": ["@Styxhexenhammer666:2"]}' 127.0.0.1:50051 pb.Hub.Resolve
-	| jq .txos[0].txHash | sed 's/"//g'
-EOM
-WANT="VOFP8MQEwps9Oa5NJJQ18WfVzUzlpCjst0Wz3xyOPd4="
-test_command_with_want
-
-# GOT=`eval $CMD`
-
-#want_got
-
-##
-## N.B. This is a degenerate case that takes a long time to run.
-## The runtime should be fixed, but in the meantime, we definitely should
-## ensure this behaves as expected.
-##
-## TODO: Test runtime doesn't exceed worst case.
-##
-
-#WANT=806389
-#read -r -d '' CMD <<- EOM
-#	grpcurl -plaintext -d '{"value": ["foo"]}' 127.0.0.1:50051 pb.Hub.Resolve | jq .txos[0].height
-#EOM
-# test_command_with_want
-
-# json rpc endpoint testing
-
-## blockchain.block
-
-### blockchain.block.get_chunk
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.block.get_chunk", "params": [0]}'
-	| jq .result | sed 's/"//g' | head -c 100
-EOM
-WANT="010000000000000000000000000000000000000000000000000000000000000000000000cc59e59ff97ac092b55e423aa549"
-test_command_with_want
-
-### blockchain.block.get_header
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.block.get_header", "params": []}'
-	| jq .result.timestamp
-EOM
-WANT=1446058291
-test_command_with_want
-
-### blockchain.block.headers
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.block.headers", "params": []}'
-	| jq .result.count
-EOM
-WANT=0
-test_command_with_want
-
-## blockchain.claimtrie
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.claimtrie.resolve", "params":[{"Data": ["@Styxhexenhammer666:2"]}]}'
-	| jq .result.txos[0].tx_hash | sed 's/"//g'
-EOM
-WANT="VOFP8MQEwps9Oa5NJJQ18WfVzUzlpCjst0Wz3xyOPd4="
-test_command_with_want
-
-## blockchain.address
-
-### blockchain.address.get_balance
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.address.get_balance", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
-	| jq .result.confirmed
-EOM
-WANT=44415602186
-test_command_with_want
-
-## blockchain.address.get_history
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.address.get_history", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
-	| jq '.result.confirmed | length'
-EOM
-WANT=82
-test_command_with_want
-
-## blockchain.address.listunspent
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.address.listunspent", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
-	| jq '.result | length'
-EOM
-WANT=32
-test_command_with_want
-
-# blockchain.scripthash
-
-## blockchain.scripthash.get_mempool
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.scripthash.get_mempool", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
-	| jq .error | sed 's/"//g'
-EOM
-WANT="encoding/hex: invalid byte: U+0047 'G'"
-test_command_with_want
-
-## blockchain.scripthash.get_history
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.scripthash.get_history", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
-	| jq .error | sed 's/"//g'
-EOM
-WANT="encoding/hex: invalid byte: U+0047 'G'"
-test_command_with_want
-
-## blockchain.scripthash.listunspent
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "blockchain.scripthash.listunspent", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
-	| jq .error | sed 's/"//g'
-EOM
-WANT="encoding/hex: invalid byte: U+0047 'G'"
-test_command_with_want
-
-## server.banner 
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "server.banner", "params":[]}'
-	| jq .result | sed 's/"//g'
-EOM
-WANT="You are connected to an 0.107.0 server."
-test_command_with_want
-
-## server.version
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "server.version", "params":[]}'
-	| jq .result | sed 's/"//g'
-EOM
-WANT="0.107.0"
-test_command_with_want
-
-## server.features
-
-read -r -d '' CMD <<- EOM
-	curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
-	--data '{"id": 1, "method": "server.features", "params":[]}'
-EOM
-WANT='{"result":{"hosts":{},"pruning":"","server_version":"0.107.0","protocol_min":"0.54.0","protocol_max":"0.199.0","genesis_hash":"9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463","description":"Herald","payment_address":"","donation_address":"","daily_fee":"1.0","hash_function":"sha256","trending_algorithm":"fast_ar"},"error":null,"id":1}'
-test_command_with_want
-
-# metrics endpoint testing
-
-WANT=0
-GOT=$(curl http://127.0.0.1:2112/metrics -s | grep requests | grep resolve | awk '{print $NF}')
-want_greater
-
-# caclulate return value
-
-logical_or $RES
-
-if [ $FINALRES -eq 1 ]; then
-	echo "Failed!"
-	exit 1
-else
-	echo "Passed!"
-	exit 0
-fi

server/args.go

@@ -1,9 +1,7 @@
 package server
 
 import (
-	"fmt"
 	"log"
-	"net/url"
 	"os"
 	"strconv"
 	"strings"
@@ -21,39 +19,26 @@ const (
 
 // Args struct contains the arguments to the hub server.
 type Args struct {
-	CmdType             int
-	Host                string
-	Port                int
-	DBPath              string
-	Chain               *string
-	DaemonURL           *url.URL
-	DaemonCAPath        string
-	EsHost              string
-	EsPort              int
-	PrometheusPort      int
-	NotifierPort        int
-	JSONRPCPort         int
-	JSONRPCHTTPPort     int
-	MaxSessions         int
-	SessionTimeout      int
-	EsIndex             string
-	RefreshDelta        int
-	CacheTTL            int
-	PeerFile            string
-	Banner              *string
-	Country             string
-	BlockingChannelIds  []string
-	FilteringChannelIds []string
-
-	GenesisHash       string
-	ServerVersion     string
-	ProtocolMin       string
-	ProtocolMax       string
-	ServerDescription string
-	PaymentAddress    string
-	DonationAddress   string
-	DailyFee          string
-
+	CmdType                     int
+	Host                        string
+	Port                        string
+	DBPath                      string
+	Chain                       *string
+	EsHost                      string
+	EsPort                      string
+	PrometheusPort              string
+	NotifierPort                string
+	JSONRPCPort                 int
+	JSONRPCHTTPPort             int
+	MaxSessions                 int
+	SessionTimeout              int
+	EsIndex                     string
+	RefreshDelta                int
+	CacheTTL                    int
+	PeerFile                    string
+	Country                     string
+	BlockingChannelIds          []string
+	FilteringChannelIds         []string
 	Debug                       bool
 	DisableEs                   bool
 	DisableLoadPeers            bool
@@ -69,32 +54,21 @@ type Args struct {
 }
 
 const (
-	DefaultHost            = "0.0.0.0"
-	DefaultPort            = 50051
-	DefaultDBPath          = "/mnt/d/data/snapshot_1072108/lbry-rocksdb/" // FIXME
-	DefaultEsHost          = "http://localhost"
-	DefaultEsIndex         = "claims"
-	DefaultEsPort          = 9200
-	DefaultPrometheusPort  = 2112
-	DefaultNotifierPort    = 18080
-	DefaultJSONRPCPort     = 50001
-	DefaultJSONRPCHTTPPort = 50002
-	DefaultMaxSessions     = 10000
-	DefaultSessionTimeout  = 300
-	DefaultRefreshDelta    = 5
-	DefaultCacheTTL        = 5
-	DefaultPeerFile        = "peers.txt"
-	DefaultBannerFile      = ""
-	DefaultCountry         = "US"
-
-	HUB_PROTOCOL_VERSION     = "0.107.0"
-	PROTOCOL_MIN             = "0.54.0"
-	PROTOCOL_MAX             = "0.199.0"
-	DefaultServerDescription = "Herald"
-	DefaultPaymentAddress    = ""
-	DefaultDonationAddress   = ""
-	DefaultDailyFee          = "1.0"
-
+	DefaultHost                        = "0.0.0.0"
+	DefaultPort                        = "50051"
+	DefaultDBPath                      = "/mnt/d/data/snapshot_1072108/lbry-rocksdb/" // FIXME
+	DefaultEsHost                      = "http://localhost"
+	DefaultEsIndex                     = "claims"
+	DefaultEsPort                      = "9200"
+	DefaultPrometheusPort              = "2112"
+	DefaultNotifierPort                = "18080"
+	DefaultJSONRPCPort                 = 50001
+	DefaultMaxSessions                 = 10000
+	DefaultSessionTimeout              = 300
+	DefaultRefreshDelta                = 5
+	DefaultCacheTTL                    = 5
+	DefaultPeerFile                    = "peers.txt"
+	DefaultCountry                     = "US"
 	DefaultDisableLoadPeers            = false
 	DefaultDisableStartPrometheus      = false
 	DefaultDisableStartUDP             = false
@@ -112,73 +86,6 @@ var (
 	DefaultFilteringChannelIds = []string{}
 )
 
-func loadBanner(bannerFile *string, serverVersion string) *string {
-	var banner string
-
-	data, err := os.ReadFile(*bannerFile)
-	if err != nil {
-		banner = fmt.Sprintf("You are connected to an %s server.", serverVersion)
-	} else {
-		banner = string(data)
-	}
-
-	/*
-		banner := os.Getenv("BANNER")
-		if banner == "" {
-			return nil
-		}
-	*/
-
-	return &banner
-}
-
-// MakeDefaultArgs creates a default set of arguments for testing the server.
-func MakeDefaultTestArgs() *Args {
-	args := &Args{
-		CmdType:         ServeCmd,
-		Host:            DefaultHost,
-		Port:            DefaultPort,
-		DBPath:          DefaultDBPath,
-		EsHost:          DefaultEsHost,
-		EsPort:          DefaultEsPort,
-		PrometheusPort:  DefaultPrometheusPort,
-		NotifierPort:    DefaultNotifierPort,
-		JSONRPCPort:     DefaultJSONRPCPort,
-		JSONRPCHTTPPort: DefaultJSONRPCHTTPPort,
-		MaxSessions:     DefaultMaxSessions,
-		SessionTimeout:  DefaultSessionTimeout,
-		EsIndex:         DefaultEsIndex,
-		RefreshDelta:    DefaultRefreshDelta,
-		CacheTTL:        DefaultCacheTTL,
-		PeerFile:        DefaultPeerFile,
-		Banner:          nil,
-		Country:         DefaultCountry,
-
-		GenesisHash:       chaincfg.TestNet3Params.GenesisHash.String(),
-		ServerVersion:     HUB_PROTOCOL_VERSION,
-		ProtocolMin:       PROTOCOL_MIN,
-		ProtocolMax:       PROTOCOL_MAX,
-		ServerDescription: DefaultServerDescription,
-		PaymentAddress:    DefaultPaymentAddress,
-		DonationAddress:   DefaultDonationAddress,
-		DailyFee:          DefaultDailyFee,
-
-		DisableEs:                   true,
-		Debug:                       true,
-		DisableLoadPeers:            true,
-		DisableStartPrometheus:      true,
-		DisableStartUDP:             true,
-		DisableWritePeers:           true,
-		DisableRocksDBRefresh:       true,
-		DisableResolve:              true,
-		DisableBlockingAndFiltering: true,
-		DisableStartNotifier:        true,
-		DisableStartJSONRPC:         true,
-	}
-
-	return args
-}
-
 // GetEnvironment takes the environment variables as an array of strings
 // and a getkeyval function to turn it into a map.
 func GetEnvironment(data []string, getkeyval func(item string) (key, val string)) map[string]string {
@@ -204,58 +111,38 @@ func GetEnvironmentStandard() map[string]string {
 func ParseArgs(searchRequest *pb.SearchRequest) *Args {
 
 	environment := GetEnvironmentStandard()
-	parser := argparse.NewParser("herald", "herald server and client")
+	parser := argparse.NewParser("hub", "hub server and client")
 
-	serveCmd := parser.NewCommand("serve", "start the herald server")
+	serveCmd := parser.NewCommand("serve", "start the hub server")
 	searchCmd := parser.NewCommand("search", "claim search")
 	dbCmd := parser.NewCommand("db", "db testing")
 
-	defaultDaemonURL := "http://localhost:9245"
-	if url, ok := environment["DAEMON_URL"]; ok {
-		defaultDaemonURL = url
-	}
-
-	validateURL := func(arg []string) error {
-		_, err := url.Parse(arg[0])
-		return err
-	}
 	validatePort := func(arg []string) error {
 		_, err := strconv.ParseUint(arg[0], 10, 16)
 		return err
 	}
 
-	// main server config arguments
 	host := parser.String("", "rpchost", &argparse.Options{Required: false, Help: "RPC host", Default: DefaultHost})
-	port := parser.Int("", "rpcport", &argparse.Options{Required: false, Help: "RPC port", Validate: validatePort, Default: DefaultPort})
+	port := parser.String("", "rpcport", &argparse.Options{Required: false, Help: "RPC port", Default: DefaultPort})
 	dbPath := parser.String("", "db-path", &argparse.Options{Required: false, Help: "RocksDB path", Default: DefaultDBPath})
 	chain := parser.Selector("", "chain", []string{chaincfg.MainNetParams.Name, chaincfg.TestNet3Params.Name, chaincfg.RegressionNetParams.Name, "testnet"},
 		&argparse.Options{Required: false, Help: "Which chain to use, default is 'mainnet'. Values 'regtest' and 'testnet' are for testing", Default: chaincfg.MainNetParams.Name})
-	daemonURLStr := parser.String("", "daemon-url", &argparse.Options{Required: false, Help: "URL for rpc to lbrycrd or lbcd, <rpcuser>:<rpcpassword>@<lbcd rpc ip><lbrcd rpc port>.", Validate: validateURL, Default: defaultDaemonURL})
-	daemonCAPath := parser.String("", "daemon-ca-path", &argparse.Options{Required: false, Help: "Path to the lbcd CA file. Use SSL certificate to verify connection to lbcd."})
 	esHost := parser.String("", "eshost", &argparse.Options{Required: false, Help: "elasticsearch host", Default: DefaultEsHost})
-	esPort := parser.Int("", "esport", &argparse.Options{Required: false, Help: "elasticsearch port", Default: DefaultEsPort})
-	prometheusPort := parser.Int("", "prometheus-port", &argparse.Options{Required: false, Help: "prometheus port", Default: DefaultPrometheusPort})
-	notifierPort := parser.Int("", "notifier-port", &argparse.Options{Required: false, Help: "notifier port", Default: DefaultNotifierPort})
-	jsonRPCPort := parser.Int("", "json-rpc-port", &argparse.Options{Required: false, Help: "JSON RPC port", Validate: validatePort, Default: DefaultJSONRPCPort})
-	jsonRPCHTTPPort := parser.Int("", "json-rpc-http-port", &argparse.Options{Required: false, Help: "JSON RPC over HTTP port", Validate: validatePort, Default: DefaultJSONRPCHTTPPort})
+	esPort := parser.String("", "esport", &argparse.Options{Required: false, Help: "elasticsearch port", Default: DefaultEsPort})
+	prometheusPort := parser.String("", "prometheus-port", &argparse.Options{Required: false, Help: "prometheus port", Default: DefaultPrometheusPort})
+	notifierPort := parser.String("", "notifier-port", &argparse.Options{Required: false, Help: "notifier port", Default: DefaultNotifierPort})
+	jsonRPCPort := parser.Int("", "json-rpc-port", &argparse.Options{Required: false, Help: "JSON RPC port", Validate: validatePort})
+	jsonRPCHTTPPort := parser.Int("", "json-rpc-http-port", &argparse.Options{Required: false, Help: "JSON RPC over HTTP port", Validate: validatePort})
 	maxSessions := parser.Int("", "max-sessions", &argparse.Options{Required: false, Help: "Maximum number of electrum clients that can be connected", Default: DefaultMaxSessions})
 	sessionTimeout := parser.Int("", "session-timeout", &argparse.Options{Required: false, Help: "Session inactivity timeout (seconds)", Default: DefaultSessionTimeout})
 	esIndex := parser.String("", "esindex", &argparse.Options{Required: false, Help: "elasticsearch index name", Default: DefaultEsIndex})
 	refreshDelta := parser.Int("", "refresh-delta", &argparse.Options{Required: false, Help: "elasticsearch index refresh delta in seconds", Default: DefaultRefreshDelta})
 	cacheTTL := parser.Int("", "cachettl", &argparse.Options{Required: false, Help: "Cache TTL in minutes", Default: DefaultCacheTTL})
 	peerFile := parser.String("", "peerfile", &argparse.Options{Required: false, Help: "Initial peer file for federation", Default: DefaultPeerFile})
-	bannerFile := parser.String("", "bannerfile", &argparse.Options{Required: false, Help: "Banner file server.banner", Default: DefaultBannerFile})
 	country := parser.String("", "country", &argparse.Options{Required: false, Help: "Country this node is running in. Default US.", Default: DefaultCountry})
 	blockingChannelIds := parser.StringList("", "blocking-channel-ids", &argparse.Options{Required: false, Help: "Blocking channel ids", Default: DefaultBlockingChannelIds})
 	filteringChannelIds := parser.StringList("", "filtering-channel-ids", &argparse.Options{Required: false, Help: "Filtering channel ids", Default: DefaultFilteringChannelIds})
 
-	// arguments for server features
-	serverDescription := parser.String("", "server-description", &argparse.Options{Required: false, Help: "Server description", Default: DefaultServerDescription})
-	paymentAddress := parser.String("", "payment-address", &argparse.Options{Required: false, Help: "Payment address", Default: DefaultPaymentAddress})
-	donationAddress := parser.String("", "donation-address", &argparse.Options{Required: false, Help: "Donation address", Default: DefaultDonationAddress})
-	dailyFee := parser.String("", "daily-fee", &argparse.Options{Required: false, Help: "Daily fee", Default: DefaultDailyFee})
-
-	// flags for disabling features
 	debug := parser.Flag("", "debug", &argparse.Options{Required: false, Help: "enable debug logging", Default: false})
 	disableEs := parser.Flag("", "disable-es", &argparse.Options{Required: false, Help: "Disable elastic search, for running/testing independently", Default: false})
 	disableLoadPeers := parser.Flag("", "disable-load-peers", &argparse.Options{Required: false, Help: "Disable load peers from disk at startup", Default: DefaultDisableLoadPeers})
@@ -269,7 +156,6 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
 	disableStartNotifier := parser.Flag("", "disable-start-notifier", &argparse.Options{Required: false, Help: "Disable start notifier", Default: DisableStartNotifier})
 	disableStartJSONRPC := parser.Flag("", "disable-start-jsonrpc", &argparse.Options{Required: false, Help: "Disable start jsonrpc endpoint", Default: DisableStartJSONRPC})
 
-	// search command arguments
 	text := parser.String("", "text", &argparse.Options{Required: false, Help: "text query"})
 	name := parser.String("", "name", &argparse.Options{Required: false, Help: "name"})
 	claimType := parser.String("", "claim_type", &argparse.Options{Required: false, Help: "claim_type"})
@@ -291,47 +177,27 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
 		*jsonRPCPort = DefaultJSONRPCPort
 	}
 
-	daemonURL, err := url.Parse(*daemonURLStr)
-	if err != nil {
-		log.Fatalf("URL parse failed: %v", err)
-	}
-
-	banner := loadBanner(bannerFile, HUB_PROTOCOL_VERSION)
-
 	args := &Args{
-		CmdType:             SearchCmd,
-		Host:                *host,
-		Port:                *port,
-		DBPath:              *dbPath,
-		Chain:               chain,
-		DaemonURL:           daemonURL,
-		DaemonCAPath:        *daemonCAPath,
-		EsHost:              *esHost,
-		EsPort:              *esPort,
-		PrometheusPort:      *prometheusPort,
-		NotifierPort:        *notifierPort,
-		JSONRPCPort:         *jsonRPCPort,
-		JSONRPCHTTPPort:     *jsonRPCHTTPPort,
-		MaxSessions:         *maxSessions,
-		SessionTimeout:      *sessionTimeout,
-		EsIndex:             *esIndex,
-		RefreshDelta:        *refreshDelta,
-		CacheTTL:            *cacheTTL,
-		PeerFile:            *peerFile,
-		Banner:              banner,
-		Country:             *country,
-		BlockingChannelIds:  *blockingChannelIds,
-		FilteringChannelIds: *filteringChannelIds,
-
-		GenesisHash:       "",
-		ServerVersion:     HUB_PROTOCOL_VERSION,
-		ProtocolMin:       PROTOCOL_MIN,
-		ProtocolMax:       PROTOCOL_MAX,
-		ServerDescription: *serverDescription,
-		PaymentAddress:    *paymentAddress,
-		DonationAddress:   *donationAddress,
-		DailyFee:          *dailyFee,
-
+		CmdType:                     SearchCmd,
+		Host:                        *host,
+		Port:                        *port,
+		DBPath:                      *dbPath,
+		Chain:                       chain,
+		EsHost:                      *esHost,
+		EsPort:                      *esPort,
+		PrometheusPort:              *prometheusPort,
+		NotifierPort:                *notifierPort,
+		JSONRPCPort:                 *jsonRPCPort,
+		JSONRPCHTTPPort:             *jsonRPCHTTPPort,
+		MaxSessions:                 *maxSessions,
+		SessionTimeout:              *sessionTimeout,
+		EsIndex:                     *esIndex,
+		RefreshDelta:                *refreshDelta,
+		CacheTTL:                    *cacheTTL,
+		PeerFile:                    *peerFile,
+		Country:                     *country,
+		BlockingChannelIds:          *blockingChannelIds,
+		FilteringChannelIds:         *filteringChannelIds,
 		Debug:                       *debug,
 		DisableEs:                   *disableEs,
 		DisableLoadPeers:            *disableLoadPeers,
@@ -355,17 +221,11 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
 	}
 
 	if esPort, ok := environment["ELASTIC_PORT"]; ok {
-		args.EsPort, err = strconv.Atoi(esPort)
-		if err != nil {
-			log.Fatal(err)
-		}
+		args.EsPort = esPort
 	}
 
 	if prometheusPort, ok := environment["GOHUB_PROMETHEUS_PORT"]; ok {
-		args.PrometheusPort, err = strconv.Atoi(prometheusPort)
-		if err != nil {
-			log.Fatal(err)
-		}
+		args.PrometheusPort = prometheusPort
 	}
 
 	/*

server/federation.go

@@ -3,19 +3,17 @@ package server
 import (
 	"bufio"
 	"context"
+	"log"
 	"math"
 	"net"
 	"os"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
 
 	"github.com/lbryio/herald.go/internal/metrics"
 	pb "github.com/lbryio/herald.go/protobuf/go"
-	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
 )
 
 // Peer holds relevant information about peers that we know about.
@@ -88,7 +86,7 @@ func (s *Server) getAndSetExternalIp(ip, port string) error {
 // storing them as known peers. Returns a map of peerKey -> object
 func (s *Server) loadPeers() error {
 	peerFile := s.Args.PeerFile
-	port := strconv.Itoa(s.Args.Port)
+	port := s.Args.Port
 
 	// First we make sure our server has come up, so we can answer back to peers.
 	var failures = 0
@@ -100,7 +98,7 @@ retry:
 	time.Sleep(time.Second * time.Duration(math.Pow(float64(failures), 2)))
 	conn, err := grpc.DialContext(ctx,
 		"0.0.0.0:"+port,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithInsecure(),
 		grpc.WithBlock(),
 	)
 
@@ -173,7 +171,7 @@ func (s *Server) subscribeToPeer(peer *Peer) error {
 
 	conn, err := grpc.DialContext(ctx,
 		peer.Address+":"+peer.Port,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithInsecure(),
 		grpc.WithBlock(),
 	)
 	if err != nil {
@@ -183,12 +181,12 @@ func (s *Server) subscribeToPeer(peer *Peer) error {
 
 	msg := &pb.ServerMessage{
 		Address: s.ExternalIP.String(),
-		Port:    strconv.Itoa(s.Args.Port),
+		Port:    s.Args.Port,
 	}
 
 	c := pb.NewHubClient(conn)
 
-	log.Printf("%s:%d subscribing to %+v\n", s.ExternalIP, s.Args.Port, peer)
+	log.Printf("%s:%s subscribing to %+v\n", s.ExternalIP, s.Args.Port, peer)
 	_, err = c.PeerSubscribe(ctx, msg)
 	if err != nil {
 		return err
@@ -209,7 +207,7 @@ func (s *Server) helloPeer(peer *Peer) (*pb.HelloMessage, error) {
 
 	conn, err := grpc.DialContext(ctx,
 		peer.Address+":"+peer.Port,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithInsecure(),
 		grpc.WithBlock(),
 	)
 	if err != nil {
@@ -221,12 +219,12 @@ func (s *Server) helloPeer(peer *Peer) (*pb.HelloMessage, error) {
 	c := pb.NewHubClient(conn)
 
 	msg := &pb.HelloMessage{
-		Port:    strconv.Itoa(s.Args.Port),
+		Port:    s.Args.Port,
 		Host:    s.ExternalIP.String(),
 		Servers: []*pb.ServerMessage{},
 	}
 
-	log.Printf("%s:%d saying hello to %+v\n", s.ExternalIP, s.Args.Port, peer)
+	log.Printf("%s:%s saying hello to %+v\n", s.ExternalIP, s.Args.Port, peer)
 	res, err := c.Hello(ctx, msg)
 	if err != nil {
 		log.Println(err)
@@ -279,7 +277,7 @@ func (s *Server) notifyPeer(peerToNotify *Peer, newPeer *Peer) error {
 
 	conn, err := grpc.DialContext(ctx,
 		peerToNotify.Address+":"+peerToNotify.Port,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithInsecure(),
 		grpc.WithBlock(),
 	)
 	if err != nil {
@@ -347,15 +345,15 @@ func (s *Server) addPeer(newPeer *Peer, ping bool, subscribe bool) error {
 		}
 	}
 
-	if strconv.Itoa(s.Args.Port) == newPeer.Port &&
+	if s.Args.Port == newPeer.Port &&
 		(localHosts[newPeer.Address] || newPeer.Address == s.ExternalIP.String()) {
-		log.Printf("%s:%d addPeer: Self peer, skipping...\n", s.ExternalIP, s.Args.Port)
+		log.Printf("%s:%s addPeer: Self peer, skipping...\n", s.ExternalIP, s.Args.Port)
 		return nil
 	}
 
 	k := peerKey(newPeer)
 
-	log.Printf("%s:%d adding peer %+v\n", s.ExternalIP, s.Args.Port, newPeer)
+	log.Printf("%s:%s adding peer %+v\n", s.ExternalIP, s.Args.Port, newPeer)
 	if oldServer, loaded := s.PeerServersLoadOrStore(newPeer); !loaded {
 		if ping {
 			_, err := s.helloPeer(newPeer)
@@ -371,10 +369,6 @@ func (s *Server) addPeer(newPeer *Peer, ping bool, subscribe bool) error {
 		metrics.PeersKnown.Inc()
 		s.writePeers()
 		s.notifyPeerSubs(newPeer)
-		// This is weird because we're doing grpc and jsonrpc here.
-		// Do we still want to custom grpc?
-		log.Warn("Sending peer to NotifierChan")
-		s.NotifierChan <- peerNotification{newPeer.Address, newPeer.Port}
 
 		// Subscribe to all our peers for now
 		if subscribe {
@@ -421,7 +415,7 @@ func (s *Server) makeHelloMessage() *pb.HelloMessage {
 	s.PeerServersMut.RUnlock()
 
 	return &pb.HelloMessage{
-		Port:    strconv.Itoa(s.Args.Port),
+		Port:    s.Args.Port,
 		Host:    s.ExternalIP.String(),
 		Servers: servers,
 	}

server/federation_test.go

@@ -4,31 +4,23 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"log"
 	"net"
 	"os"
-	"strconv"
 	"strings"
 	"testing"
 
 	"github.com/lbryio/herald.go/internal/metrics"
 	pb "github.com/lbryio/herald.go/protobuf/go"
-	"github.com/lbryio/herald.go/server"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
+	server "github.com/lbryio/herald.go/server"
 	dto "github.com/prometheus/client_model/go"
-	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
 )
 
 // lineCountFile takes a fileName and counts the number of lines in it.
 func lineCountFile(fileName string) int {
 	f, err := os.Open(fileName)
-	defer func() {
-		err := f.Close()
-		if err != nil {
-			log.Warn(err)
-		}
-	}()
+	defer f.Close()
 	if err != nil {
 		log.Println(err)
 		return 0
@@ -52,12 +44,43 @@ func removeFile(fileName string) {
 	}
 }
 
+// makeDefaultArgs creates a default set of arguments for testing the server.
+func makeDefaultArgs() *server.Args {
+	args := &server.Args{
+		CmdType:                     server.ServeCmd,
+		Host:                        server.DefaultHost,
+		Port:                        server.DefaultPort,
+		DBPath:                      server.DefaultDBPath,
+		EsHost:                      server.DefaultEsHost,
+		EsPort:                      server.DefaultEsPort,
+		PrometheusPort:              server.DefaultPrometheusPort,
+		NotifierPort:                server.DefaultNotifierPort,
+		JSONRPCPort:                 server.DefaultJSONRPCPort,
+		EsIndex:                     server.DefaultEsIndex,
+		RefreshDelta:                server.DefaultRefreshDelta,
+		CacheTTL:                    server.DefaultCacheTTL,
+		PeerFile:                    server.DefaultPeerFile,
+		Country:                     server.DefaultCountry,
+		DisableEs:                   true,
+		Debug:                       true,
+		DisableLoadPeers:            true,
+		DisableStartPrometheus:      true,
+		DisableStartUDP:             true,
+		DisableWritePeers:           true,
+		DisableRocksDBRefresh:       true,
+		DisableResolve:              true,
+		DisableBlockingAndFiltering: true,
+		DisableStartNotifier:        true,
+		DisableStartJSONRPC:         true,
+	}
+
+	return args
+}
+
 // TestAddPeer tests the ability to add peers
 func TestAddPeer(t *testing.T) {
-	// ctx := context.Background()
-	ctx := stop.NewDebug()
-	args := server.MakeDefaultTestArgs()
-	args.DisableStartNotifier = false
+	ctx := context.Background()
+	args := makeDefaultArgs()
 
 	tests := []struct {
 		name string
@@ -107,7 +130,6 @@ func TestAddPeer(t *testing.T) {
 			if got != tt.want {
 				t.Errorf("len(server.PeerServers) = %d, want %d\n", got, tt.want)
 			}
-			hubServer.Stop()
 		})
 	}
 
@@ -115,10 +137,9 @@ func TestAddPeer(t *testing.T) {
 
 // TestPeerWriter tests that peers get written properly
 func TestPeerWriter(t *testing.T) {
-	ctx := stop.NewDebug()
-	args := server.MakeDefaultTestArgs()
+	ctx := context.Background()
+	args := makeDefaultArgs()
 	args.DisableWritePeers = false
-	args.DisableStartNotifier = false
 
 	tests := []struct {
 		name string
@@ -153,16 +174,17 @@ func TestPeerWriter(t *testing.T) {
 						Port:    "50051",
 					}
 				}
+				//log.Printf("Adding peer %+v\n", peer)
 				err := hubServer.AddPeerExported()(peer, false, false)
 				if err != nil {
 					log.Println(err)
 				}
 			}
+			//log.Println("Counting lines...")
 			got := lineCountFile(hubServer.Args.PeerFile)
 			if got != tt.want {
 				t.Errorf("lineCountFile(peers.txt) = %d, want %d", got, tt.want)
 			}
-			hubServer.Stop()
 		})
 	}
 
@@ -171,13 +193,10 @@ func TestPeerWriter(t *testing.T) {
 
 // TestAddPeerEndpoint tests the ability to add peers
 func TestAddPeerEndpoint(t *testing.T) {
-	ctx := stop.NewDebug()
-	args := server.MakeDefaultTestArgs()
-	args.DisableStartNotifier = false
-	args2 := server.MakeDefaultTestArgs()
-	args2.DisableStartNotifier = false
-	args2.Port = 50052
-	args2.NotifierPort = 18081
+	ctx := context.Background()
+	args := makeDefaultArgs()
+	args2 := makeDefaultArgs()
+	args2.Port = "50052"
 
 	tests := []struct {
 		name          string
@@ -207,8 +226,9 @@ func TestAddPeerEndpoint(t *testing.T) {
 			metrics.PeersKnown.Set(0)
 			go hubServer.Run()
 			go hubServer2.Run()
-			conn, err := grpc.Dial("localhost:"+strconv.Itoa(args.Port),
-				grpc.WithTransportCredentials(insecure.NewCredentials()),
+			//go hubServer.Run()
+			conn, err := grpc.Dial("localhost:"+args.Port,
+				grpc.WithInsecure(),
 				grpc.WithBlock(),
 			)
 			if err != nil {
@@ -227,6 +247,8 @@ func TestAddPeerEndpoint(t *testing.T) {
 				log.Println(err)
 			}
 
+			hubServer.GrpcServer.GracefulStop()
+			hubServer2.GrpcServer.GracefulStop()
 			got1 := hubServer.GetNumPeersExported()()
 			got2 := hubServer2.GetNumPeersExported()()
 			if got1 != tt.wantServerOne {
@@ -235,8 +257,6 @@ func TestAddPeerEndpoint(t *testing.T) {
 			if got2 != tt.wantServerTwo {
 				t.Errorf("len(hubServer2.PeerServers) = %d, want %d\n", got2, tt.wantServerTwo)
 			}
-			hubServer.Stop()
-			hubServer2.Stop()
 		})
 	}
 
@@ -244,17 +264,12 @@ func TestAddPeerEndpoint(t *testing.T) {
 
 // TestAddPeerEndpoint2 tests the ability to add peers
 func TestAddPeerEndpoint2(t *testing.T) {
-	ctx := stop.NewDebug()
-	args := server.MakeDefaultTestArgs()
-	args2 := server.MakeDefaultTestArgs()
-	args3 := server.MakeDefaultTestArgs()
-	args2.Port = 50052
-	args3.Port = 50053
-	args.DisableStartNotifier = false
-	args2.DisableStartNotifier = false
-	args3.DisableStartNotifier = false
-	args2.NotifierPort = 18081
-	args3.NotifierPort = 18082
+	ctx := context.Background()
+	args := makeDefaultArgs()
+	args2 := makeDefaultArgs()
+	args3 := makeDefaultArgs()
+	args2.Port = "50052"
+	args3.Port = "50053"
 
 	tests := []struct {
 		name            string
@@ -279,8 +294,8 @@ func TestAddPeerEndpoint2(t *testing.T) {
 			go hubServer.Run()
 			go hubServer2.Run()
 			go hubServer3.Run()
-			conn, err := grpc.Dial("localhost:"+strconv.Itoa(args.Port),
-				grpc.WithTransportCredentials(insecure.NewCredentials()),
+			conn, err := grpc.Dial("localhost:"+args.Port,
+				grpc.WithInsecure(),
 				grpc.WithBlock(),
 			)
 			if err != nil {
@@ -308,6 +323,9 @@ func TestAddPeerEndpoint2(t *testing.T) {
 				log.Println(err)
 			}
 
+			hubServer.GrpcServer.GracefulStop()
+			hubServer2.GrpcServer.GracefulStop()
+			hubServer3.GrpcServer.GracefulStop()
 			got1 := hubServer.GetNumPeersExported()()
 			got2 := hubServer2.GetNumPeersExported()()
 			got3 := hubServer3.GetNumPeersExported()()
@@ -320,9 +338,6 @@ func TestAddPeerEndpoint2(t *testing.T) {
 			if got3 != tt.wantServerThree {
 				t.Errorf("len(hubServer3.PeerServers) = %d, want %d\n", got3, tt.wantServerThree)
 			}
-			hubServer.Stop()
-			hubServer2.Stop()
-			hubServer3.Stop()
 		})
 	}
 
@@ -330,17 +345,12 @@ func TestAddPeerEndpoint2(t *testing.T) {
 
 // TestAddPeerEndpoint3 tests the ability to add peers
 func TestAddPeerEndpoint3(t *testing.T) {
-	ctx := stop.NewDebug()
-	args := server.MakeDefaultTestArgs()
-	args2 := server.MakeDefaultTestArgs()
-	args3 := server.MakeDefaultTestArgs()
-	args2.Port = 50052
-	args3.Port = 50053
-	args.DisableStartNotifier = false
-	args2.DisableStartNotifier = false
-	args3.DisableStartNotifier = false
-	args2.NotifierPort = 18081
-	args3.NotifierPort = 18082
+	ctx := context.Background()
+	args := makeDefaultArgs()
+	args2 := makeDefaultArgs()
+	args3 := makeDefaultArgs()
+	args2.Port = "50052"
+	args3.Port = "50053"
 
 	tests := []struct {
 		name            string
@@ -365,15 +375,15 @@ func TestAddPeerEndpoint3(t *testing.T) {
 			go hubServer.Run()
 			go hubServer2.Run()
 			go hubServer3.Run()
-			conn, err := grpc.Dial("localhost:"+strconv.Itoa(args.Port),
-				grpc.WithTransportCredentials(insecure.NewCredentials()),
+			conn, err := grpc.Dial("localhost:"+args.Port,
+				grpc.WithInsecure(),
 				grpc.WithBlock(),
 			)
 			if err != nil {
 				log.Fatalf("did not connect: %v", err)
 			}
 			conn2, err := grpc.Dial("localhost:50052",
-				grpc.WithTransportCredentials(insecure.NewCredentials()),
+				grpc.WithInsecure(),
 				grpc.WithBlock(),
 			)
 			if err != nil {
@@ -402,9 +412,9 @@ func TestAddPeerEndpoint3(t *testing.T) {
 				log.Println(err)
 			}
 
-			hubServer.Stop()
-			hubServer2.Stop()
-			hubServer3.Stop()
+			hubServer.GrpcServer.GracefulStop()
+			hubServer2.GrpcServer.GracefulStop()
+			hubServer3.GrpcServer.GracefulStop()
 			got1 := hubServer.GetNumPeersExported()()
 			got2 := hubServer2.GetNumPeersExported()()
 			got3 := hubServer3.GetNumPeersExported()()
@@ -424,11 +434,11 @@ func TestAddPeerEndpoint3(t *testing.T) {
 
 // TestAddPeer tests the ability to add peers
 func TestUDPServer(t *testing.T) {
-	ctx := stop.NewDebug()
-	args := server.MakeDefaultTestArgs()
-	args2 := server.MakeDefaultTestArgs()
-	args2.Port = 50052
+	ctx := context.Background()
+	args := makeDefaultArgs()
 	args.DisableStartUDP = false
+	args2 := makeDefaultArgs()
+	args2.Port = "50052"
 	args2.DisableStartUDP = false
 
 	tests := []struct {
@@ -459,18 +469,18 @@ func TestUDPServer(t *testing.T) {
 				log.Println(err)
 			}
 
-			hubServer.Stop()
-			hubServer2.Stop()
+			hubServer.GrpcServer.GracefulStop()
+			hubServer2.GrpcServer.GracefulStop()
 
 			got1 := hubServer.ExternalIP.String()
 			if got1 != tt.want {
 				t.Errorf("hubServer.ExternalIP = %s, want %s\n", got1, tt.want)
-				t.Errorf("hubServer.Args.Port = %d\n", hubServer.Args.Port)
+				t.Errorf("hubServer.Args.Port = %s\n", hubServer.Args.Port)
 			}
 			got2 := hubServer2.ExternalIP.String()
 			if got2 != tt.want {
 				t.Errorf("hubServer2.ExternalIP = %s, want %s\n", got2, tt.want)
-				t.Errorf("hubServer2.Args.Port = %d\n", hubServer2.Args.Port)
+				t.Errorf("hubServer2.Args.Port = %s\n", hubServer2.Args.Port)
 			}
 		})
 	}

server/jsonrpc_blockchain.go

@@ -7,7 +7,6 @@ import (
 	"encoding/base64"
 	"encoding/binary"
 	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"fmt"
 
@@ -19,8 +18,6 @@ import (
 	"github.com/lbryio/lbcd/wire"
 	"github.com/lbryio/lbcutil"
 	"golang.org/x/exp/constraints"
-
-	log "github.com/sirupsen/logrus"
 )
 
 // BlockchainBlockService methods handle "blockchain.block.*" RPCs
@@ -56,14 +53,6 @@ type BlockchainScripthashService struct {
 	session    *session
 }
 
-// BlockchainTransactionService methods handle "blockchain.transaction.*" RPCs
-type BlockchainTransactionService struct {
-	DB    *db.ReadOnlyDBColumnFamily
-	Chain *chaincfg.Params
-	// needed for broadcast TX
-	sessionMgr *sessionManager
-}
-
 const CHUNK_SIZE = 96
 const MAX_CHUNK_SIZE = 40960
 const HEADER_SIZE = wire.MaxBlockHeaderPayload
@@ -114,7 +103,6 @@ func newBlockHeaderElectrum(header *[HEADER_SIZE]byte, height uint32) *BlockHead
 type BlockGetServerHeightReq struct{}
 type BlockGetServerHeightResp uint32
 
-// blockchain.block.get_server_height
 func (s *BlockchainBlockService) Get_server_height(req *BlockGetServerHeightReq, resp **BlockGetServerHeightResp) error {
 	if s.DB == nil || s.DB.LastState == nil {
 		return fmt.Errorf("unknown height")
@@ -132,7 +120,6 @@ func (s *BlockchainBlockService) Get_chunk(req *BlockGetChunkReq, resp **BlockGe
 	index := uint32(*req)
 	db_headers, err := s.DB.GetHeaders(index*CHUNK_SIZE, CHUNK_SIZE)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	raw := make([]byte, 0, HEADER_SIZE*len(db_headers))
@@ -154,7 +141,6 @@ func (s *BlockchainBlockService) Get_header(req *BlockGetHeaderReq, resp **Block
 	height := uint32(*req)
 	headers, err := s.DB.GetHeaders(height, 1)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	if len(headers) < 1 {
@@ -171,42 +157,9 @@ type BlockHeadersReq struct {
 	B64         bool   `json:"b64"`
 }
 
-func (req *BlockHeadersReq) UnmarshalJSON(b []byte) error {
-	var params [4]interface{}
-	err := json.Unmarshal(b, &params)
-	if err != nil {
-		return err
-	}
-	switch params[0].(type) {
-	case float64:
-		req.StartHeight = uint32(params[0].(float64))
-	default:
-		return fmt.Errorf("expected numeric argument #0 (start_height)")
-	}
-	switch params[1].(type) {
-	case float64:
-		req.Count = uint32(params[1].(float64))
-	default:
-		return fmt.Errorf("expected numeric argument #1 (count)")
-	}
-	switch params[2].(type) {
-	case float64:
-		req.CpHeight = uint32(params[2].(float64))
-	default:
-		return fmt.Errorf("expected numeric argument #2 (cp_height)")
-	}
-	switch params[3].(type) {
-	case bool:
-		req.B64 = params[3].(bool)
-	default:
-		return fmt.Errorf("expected boolean argument #3 (b64)")
-	}
-	return nil
-}
-
 type BlockHeadersResp struct {
 	Base64 string `json:"base64,omitempty"`
-	Hex    string `json:"hex"`
+	Hex    string `json:"hex,omitempty"`
 	Count  uint32 `json:"count"`
 	Max    uint32 `json:"max"`
 	Branch string `json:"branch,omitempty"`
@@ -218,7 +171,6 @@ func (s *BlockchainBlockService) Headers(req *BlockHeadersReq, resp **BlockHeade
 	count := min(req.Count, MAX_CHUNK_SIZE)
 	db_headers, err := s.DB.GetHeaders(req.StartHeight, count)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	count = uint32(len(db_headers))
@@ -251,21 +203,6 @@ type HeadersSubscribeReq struct {
 	Raw bool `json:"raw"`
 }
 
-func (req *HeadersSubscribeReq) UnmarshalJSON(b []byte) error {
-	var params [1]interface{}
-	err := json.Unmarshal(b, &params)
-	if err != nil {
-		return err
-	}
-	switch params[0].(type) {
-	case bool:
-		req.Raw = params[0].(bool)
-	default:
-		return fmt.Errorf("expected bool argument #0 (raw)")
-	}
-	return nil
-}
-
 type HeadersSubscribeResp struct {
 	BlockHeaderElectrum
 }
@@ -346,22 +283,18 @@ type AddressGetBalanceResp struct {
 func (s *BlockchainAddressService) Get_balance(req *AddressGetBalanceReq, resp **AddressGetBalanceResp) error {
 	address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	script, err := txscript.PayToAddrScript(address)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	hashX := hashXScript(script, s.Chain)
 	confirmed, unconfirmed, err := s.DB.GetBalance(hashX)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	*resp = &AddressGetBalanceResp{confirmed, unconfirmed}
-
 	return err
 }
 
@@ -377,13 +310,11 @@ type ScripthashGetBalanceResp struct {
 func (s *BlockchainScripthashService) Get_balance(req *scripthashGetBalanceReq, resp **ScripthashGetBalanceResp) error {
 	scripthash, err := decodeScriptHash(req.ScriptHash)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	hashX := hashX(scripthash)
 	confirmed, unconfirmed, err := s.DB.GetBalance(hashX)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	*resp = &ScripthashGetBalanceResp{confirmed, unconfirmed}
@@ -393,23 +324,6 @@ func (s *BlockchainScripthashService) Get_balance(req *scripthashGetBalanceReq,
 type AddressGetHistoryReq struct {
 	Address string `json:"address"`
 }
-
-func (req *AddressGetHistoryReq) UnmarshalJSON(b []byte) error {
-	var params [1]interface{}
-	json.Unmarshal(b, &params)
-	err := json.Unmarshal(b, &params)
-	if err != nil {
-		return err
-	}
-	switch params[0].(type) {
-	case string:
-		req.Address = params[0].(string)
-	default:
-		return fmt.Errorf("expected string argument #0 (address)")
-	}
-	return nil
-}
-
 type TxInfo struct {
 	TxHash string `json:"tx_hash"`
 	Height uint32 `json:"height"`
@@ -418,24 +332,24 @@ type TxInfoFee struct {
 	TxInfo
 	Fee uint64 `json:"fee"`
 }
-type AddressGetHistoryResp []TxInfoFee
+type AddressGetHistoryResp struct {
+	Confirmed   []TxInfo    `json:"confirmed"`
+	Unconfirmed []TxInfoFee `json:"unconfirmed"`
+}
 
 // 'blockchain.address.get_history'
 func (s *BlockchainAddressService) Get_history(req *AddressGetHistoryReq, resp **AddressGetHistoryResp) error {
 	address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	script, err := txscript.PayToAddrScript(address)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	hashX := hashXScript(script, s.Chain)
 	dbTXs, err := s.DB.GetHistory(hashX)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	confirmed := make([]TxInfo, 0, len(dbTXs))
@@ -446,18 +360,11 @@ func (s *BlockchainAddressService) Get_history(req *AddressGetHistoryReq, resp *
 				Height: tx.Height,
 			})
 	}
-	unconfirmed := []TxInfoFee{} // TODO
-	result := make(AddressGetHistoryResp, len(confirmed)+len(unconfirmed))
-	i := 0
-	for _, tx := range confirmed {
-		result[i].TxInfo = tx
-		i += 1
+	result := &AddressGetHistoryResp{
+		Confirmed:   confirmed,
+		Unconfirmed: []TxInfoFee{}, // TODO
 	}
-	for _, tx := range unconfirmed {
-		result[i] = tx
-		i += 1
-	}
-	*resp = &result
+	*resp = result
 	return err
 }
 
@@ -473,13 +380,11 @@ type ScripthashGetHistoryResp struct {
 func (s *BlockchainScripthashService) Get_history(req *ScripthashGetHistoryReq, resp **ScripthashGetHistoryResp) error {
 	scripthash, err := decodeScriptHash(req.ScriptHash)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	hashX := hashX(scripthash)
 	dbTXs, err := s.DB.GetHistory(hashX)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	confirmed := make([]TxInfo, 0, len(dbTXs))
@@ -507,12 +412,10 @@ type AddressGetMempoolResp []TxInfoFee
 func (s *BlockchainAddressService) Get_mempool(req *AddressGetMempoolReq, resp **AddressGetMempoolResp) error {
 	address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	script, err := txscript.PayToAddrScript(address)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	hashX := hashXScript(script, s.Chain)
@@ -533,7 +436,6 @@ type ScripthashGetMempoolResp []TxInfoFee
 func (s *BlockchainScripthashService) Get_mempool(req *ScripthashGetMempoolReq, resp **ScripthashGetMempoolResp) error {
 	scripthash, err := decodeScriptHash(req.ScriptHash)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	hashX := hashX(scripthash)
@@ -560,12 +462,10 @@ type AddressListUnspentResp []TXOInfo
 func (s *BlockchainAddressService) Listunspent(req *AddressListUnspentReq, resp **AddressListUnspentResp) error {
 	address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	script, err := txscript.PayToAddrScript(address)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	hashX := hashXScript(script, s.Chain)
@@ -594,7 +494,6 @@ type ScripthashListUnspentResp []TXOInfo
 func (s *BlockchainScripthashService) Listunspent(req *ScripthashListUnspentReq, resp **ScripthashListUnspentResp) error {
 	scripthash, err := decodeScriptHash(req.ScriptHash)
 	if err != nil {
-		log.Warn(err)
 		return err
 	}
 	hashX := hashX(scripthash)
@@ -704,221 +603,3 @@ func (s *BlockchainScripthashService) Unsubscribe(req *ScripthashSubscribeReq, r
 	*resp = (*ScripthashSubscribeResp)(nil)
 	return nil
 }
-
-type TransactionBroadcastReq [1]string
-type TransactionBroadcastResp string
-
-// 'blockchain.transaction.broadcast'
-func (s *BlockchainTransactionService) Broadcast(req *TransactionBroadcastReq, resp **TransactionBroadcastResp) error {
-	if s.sessionMgr == nil {
-		return errors.New("no session manager, rpc not supported")
-	}
-	strTx := string((*req)[0])
-	rawTx, err := hex.DecodeString(strTx)
-	if err != nil {
-		return err
-	}
-	txhash, err := s.sessionMgr.broadcastTx(rawTx)
-	if err != nil {
-		return err
-	}
-	result := txhash.String()
-	*resp = (*TransactionBroadcastResp)(&result)
-	return nil
-}
-
-type TransactionGetReq string
-type TXFullDetail struct {
-	Height int      `json:"block_height"`
-	Pos    uint32   `json:"pos"`
-	Merkle []string `json:"merkle"`
-}
-type TXDetail struct {
-	Height int `json:"block_height"`
-}
-
-type TXGetItem struct {
-	TxHash string
-	TxRaw  string
-	Detail interface{} // TXFullDetail or TXDetail struct
-}
-type TransactionGetResp TXGetItem
-
-// 'blockchain.transaction.get'
-func (s *BlockchainTransactionService) Get(req *TransactionGetReq, resp **TransactionGetResp) error {
-	txids := [1]string{string(*req)}
-	request := TransactionGetBatchReq(txids[:])
-	var response *TransactionGetBatchResp
-	err := s.Get_batch(&request, &response)
-	if err != nil {
-		return err
-	}
-	if len(*response) < 1 {
-		return errors.New("tx not found")
-	}
-	switch (*response)[0].Detail.(type) {
-	case TXFullDetail:
-		break
-	case TXDetail:
-	default:
-		return errors.New("tx not confirmed")
-	}
-	*resp = (*TransactionGetResp)(&(*response)[0])
-	return err
-}
-
-type TransactionGetBatchReq []string
-
-func (req *TransactionGetBatchReq) UnmarshalJSON(b []byte) error {
-	var params []interface{}
-	json.Unmarshal(b, &params)
-	if len(params) > 100 {
-		return fmt.Errorf("too many tx hashes in request: %v", len(params))
-	}
-	for i, txhash := range params {
-		switch params[0].(type) {
-		case string:
-			*req = append(*req, txhash.(string))
-		default:
-			return fmt.Errorf("expected string argument #%d (tx_hash)", i)
-		}
-	}
-	return nil
-}
-
-type TransactionGetBatchResp []TXGetItem
-
-func (resp *TransactionGetBatchResp) MarshalJSON() ([]byte, error) {
-	// encode key/value pairs as variable length JSON object
-	var buf bytes.Buffer
-	enc := json.NewEncoder(&buf)
-	buf.WriteString("{")
-	for i, r := range *resp {
-		if i > 0 {
-			buf.WriteString(",")
-		}
-		txhash, raw, detail := r.TxHash, r.TxRaw, r.Detail
-		err := enc.Encode(txhash)
-		if err != nil {
-			return nil, err
-		}
-		buf.WriteString(":[")
-		err = enc.Encode(raw)
-		if err != nil {
-			return nil, err
-		}
-		buf.WriteString(",")
-		err = enc.Encode(detail)
-		if err != nil {
-			return nil, err
-		}
-		buf.WriteString("]")
-	}
-	buf.WriteString("}")
-	return buf.Bytes(), nil
-}
-
-// 'blockchain.transaction.get_batch'
-func (s *BlockchainTransactionService) Get_batch(req *TransactionGetBatchReq, resp **TransactionGetBatchResp) error {
-	if len(*req) > 100 {
-		return fmt.Errorf("too many tx hashes in request: %v", len(*req))
-	}
-	tx_hashes := make([]chainhash.Hash, 0, len(*req))
-	for i, txid := range *req {
-		tx_hashes = append(tx_hashes, chainhash.Hash{})
-		if err := chainhash.Decode(&tx_hashes[i], txid); err != nil {
-			return err
-		}
-	}
-	dbResult, err := s.DB.GetTxMerkle(tx_hashes)
-	if err != nil {
-		return err
-	}
-	result := make([]TXGetItem, 0, len(dbResult))
-	for _, r := range dbResult {
-		merkles := make([]string, len(r.Merkle))
-		for i, m := range r.Merkle {
-			merkles[i] = m.String()
-		}
-		detail := TXFullDetail{
-			Height: r.Height,
-			Pos:    r.Pos,
-			Merkle: merkles,
-		}
-		result = append(result, TXGetItem{r.TxHash.String(), hex.EncodeToString(r.RawTx), &detail})
-	}
-	*resp = (*TransactionGetBatchResp)(&result)
-	return err
-}
-
-type TransactionGetMerkleReq struct {
-	TxHash string `json:"tx_hash"`
-	Height uint32 `json:"height"`
-}
-type TransactionGetMerkleResp TXGetItem
-
-// 'blockchain.transaction.get_merkle'
-func (s *BlockchainTransactionService) Get_merkle(req *TransactionGetMerkleReq, resp **TransactionGetMerkleResp) error {
-	txids := [1]string{string(req.TxHash)}
-	request := TransactionGetBatchReq(txids[:])
-	var response *TransactionGetBatchResp
-	err := s.Get_batch(&request, &response)
-	if err != nil {
-		return err
-	}
-	if len(*response) < 1 {
-		return errors.New("tx not found")
-	}
-	switch (*response)[0].Detail.(type) {
-	case TXFullDetail:
-		break
-	case TXDetail:
-	default:
-		return errors.New("tx not confirmed")
-	}
-	*resp = (*TransactionGetMerkleResp)(&(*response)[0])
-	return err
-}
-
-type TransactionGetHeightReq string
-type TransactionGetHeightResp uint32
-
-// 'blockchain.transaction.get_height'
-func (s *BlockchainTransactionService) Get_height(req *TransactionGetHeightReq, resp **TransactionGetHeightResp) error {
-	txid := string(*(req))
-	txhash, err := chainhash.NewHashFromStr(txid)
-	if err != nil {
-		return err
-	}
-	height, err := s.DB.GetTxHeight(txhash)
-	*resp = (*TransactionGetHeightResp)(&height)
-	return err
-}
-
-type TransactionInfoReq string
-type TransactionInfoResp TXGetItem
-
-// 'blockchain.transaction.info'
-func (s *BlockchainTransactionService) Info(req *TransactionInfoReq, resp **TransactionInfoResp) error {
-	txids := [1]string{string(*req)}
-	request := TransactionGetBatchReq(txids[:])
-	var response *TransactionGetBatchResp
-	err := s.Get_batch(&request, &response)
-	if err != nil {
-		return err
-	}
-	if len(*response) < 1 {
-		return errors.New("tx not found")
-	}
-	switch (*response)[0].Detail.(type) {
-	case TXFullDetail:
-		break
-	case TXDetail:
-	default:
-		if (*response)[0].TxHash == "" {
-			return errors.New("no such mempool or blockchain transaction")
-		}
-	}
-	*resp = (*TransactionInfoResp)(&(*response)[0])
-	return err
-}

server/jsonrpc_blockchain_test.go

@@ -13,7 +13,6 @@ import (
 	"github.com/lbryio/lbcd/chaincfg"
 	"github.com/lbryio/lbcd/txscript"
 	"github.com/lbryio/lbcutil"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 )
 
 // Source: test_variety_of_transactions_and_longish_history (lbry-sdk/tests/integration/transactions)
@@ -58,9 +57,8 @@ var regTestAddrs = [30]string{
 
 func TestServerGetHeight(t *testing.T) {
 	secondaryPath := "asdf"
-	grp := stop.NewDebug()
-	db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
@@ -89,9 +87,8 @@ func TestServerGetHeight(t *testing.T) {
 
 func TestGetChunk(t *testing.T) {
 	secondaryPath := "asdf"
-	grp := stop.NewDebug()
-	db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
@@ -133,9 +130,8 @@ func TestGetChunk(t *testing.T) {
 
 func TestGetHeader(t *testing.T) {
 	secondaryPath := "asdf"
-	grp := stop.NewDebug()
-	db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
@@ -163,9 +159,8 @@ func TestGetHeader(t *testing.T) {
 
 func TestHeaders(t *testing.T) {
 	secondaryPath := "asdf"
-	grp := stop.NewDebug()
-	db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
@@ -185,9 +180,6 @@ func TestHeaders(t *testing.T) {
 		}
 		var resp *BlockHeadersResp
 		err := s.Headers(&req, &resp)
-		if err != nil {
-			t.Errorf("Headers: %v", err)
-		}
 		marshalled, err := json.MarshalIndent(resp, "", "    ")
 		if err != nil {
 			t.Errorf("height: %v unmarshal err: %v", height, err)
@@ -197,17 +189,15 @@ func TestHeaders(t *testing.T) {
 }
 
 func TestHeadersSubscribe(t *testing.T) {
-	args := MakeDefaultTestArgs()
-	grp := stop.NewDebug()
 	secondaryPath := "asdf"
-	db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
 	}
 
-	sm := newSessionManager(nil, db, args, grp, &chaincfg.RegressionNetParams, nil)
+	sm := newSessionManager(db, &chaincfg.RegressionNetParams, DefaultMaxSessions, DefaultSessionTimeout)
 	sm.start()
 	defer sm.stop()
 
@@ -219,13 +209,13 @@ func TestHeadersSubscribe(t *testing.T) {
 	// Set up logic to read a notification.
 	var received sync.WaitGroup
 	recv := func(client net.Conn) {
-		defer received.Done()
 		buf := make([]byte, 1024)
 		len, err := client.Read(buf)
 		if err != nil {
 			t.Errorf("read err: %v", err)
 		}
 		t.Logf("len: %v notification: %v", len, string(buf))
+		received.Done()
 	}
 	received.Add(2)
 	go recv(client1)
@@ -276,10 +266,12 @@ func TestHeadersSubscribe(t *testing.T) {
 		t.Errorf("decode err: %v", err)
 	}
 	note1 := headerNotification{
-		HeightHash:          internal.HeightHash{Height: 500, BlockHeader: header500},
+		HeightHash:          internal.HeightHash{Height: 500},
+		blockHeader:         [112]byte{},
 		blockHeaderElectrum: nil,
 		blockHeaderStr:      "",
 	}
+	copy(note1.blockHeader[:], header500)
 	t.Logf("sending notification")
 	sm.doNotify(note1)
 
@@ -289,9 +281,8 @@ func TestHeadersSubscribe(t *testing.T) {
 
 func TestGetBalance(t *testing.T) {
 	secondaryPath := "asdf"
-	grp := stop.NewDebug()
-	db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
@@ -319,9 +310,8 @@ func TestGetBalance(t *testing.T) {
 
 func TestGetHistory(t *testing.T) {
 	secondaryPath := "asdf"
-	grp := stop.NewDebug()
-	db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
@@ -349,9 +339,8 @@ func TestGetHistory(t *testing.T) {
 
 func TestListUnspent(t *testing.T) {
 	secondaryPath := "asdf"
-	grp := stop.NewDebug()
-	db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
@@ -378,17 +367,15 @@ func TestListUnspent(t *testing.T) {
 }
 
 func TestAddressSubscribe(t *testing.T) {
-	args := MakeDefaultTestArgs()
-	grp := stop.NewDebug()
 	secondaryPath := "asdf"
-	db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
-	defer db.Shutdown()
+	db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
+	defer toDefer()
 	if err != nil {
 		t.Error(err)
 		return
 	}
 
-	sm := newSessionManager(nil, db, args, grp, &chaincfg.RegressionNetParams, nil)
+	sm := newSessionManager(db, &chaincfg.RegressionNetParams, DefaultMaxSessions, DefaultSessionTimeout)
 	sm.start()
 	defer sm.stop()
 

server/jsonrpc_claimtrie.go

@@ -1,19 +1,13 @@
 package server
 
 import (
-	"context"
-	"fmt"
-
 	"github.com/lbryio/herald.go/db"
-	"github.com/lbryio/herald.go/internal/metrics"
 	pb "github.com/lbryio/herald.go/protobuf/go"
-	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 )
 
 type ClaimtrieService struct {
-	DB     *db.ReadOnlyDBColumnFamily
-	Server *Server
+	DB *db.ReadOnlyDBColumnFamily
 }
 
 type ResolveData struct {
@@ -24,10 +18,6 @@ type Result struct {
 	Data string `json:"data"`
 }
 
-type GetClaimByIDData struct {
-	ClaimID string `json:"claim_id"`
-}
-
 // Resolve is the json rpc endpoint for 'blockchain.claimtrie.resolve'.
 func (t *ClaimtrieService) Resolve(args *ResolveData, result **pb.Outputs) error {
 	log.Println("Resolve")
@@ -35,74 +25,3 @@ func (t *ClaimtrieService) Resolve(args *ResolveData, result **pb.Outputs) error
 	*result = res
 	return err
 }
-
-// Search is the json rpc endpoint for 'blockchain.claimtrie.search'.
-func (t *ClaimtrieService) Search(args *pb.SearchRequest, result **pb.Outputs) error {
-	log.Println("Search")
-	if t.Server == nil {
-		log.Warnln("Server is nil in Search")
-		*result = nil
-		return nil
-	}
-	ctx := context.Background()
-	res, err := t.Server.Search(ctx, args)
-	*result = res
-	return err
-}
-
-// GetClaimByID is the json rpc endpoint for 'blockchain.claimtrie.getclaimbyid'.
-func (t *ClaimtrieService) GetClaimByID(args *GetClaimByIDData, result **pb.Outputs) error {
-	log.Println("GetClaimByID")
-	if len(args.ClaimID) != 40 {
-		*result = nil
-		return fmt.Errorf("len(claim_id) != 40")
-	}
-
-	rows, extras, err := t.DB.GetClaimByID(args.ClaimID)
-	if err != nil {
-		*result = nil
-		return err
-	}
-
-	metrics.RequestsCount.With(prometheus.Labels{"method": "blockchain.claimtrie.getclaimbyid"}).Inc()
-
-	// FIXME: this has txos and extras and so does GetClaimById?
-	allTxos := make([]*pb.Output, 0)
-	allExtraTxos := make([]*pb.Output, 0)
-
-	for _, row := range rows {
-		txos, extraTxos, err := row.ToOutputs()
-		if err != nil {
-			*result = nil
-			return err
-		}
-		// TODO: there may be a more efficient way to do this.
-		allTxos = append(allTxos, txos...)
-		allExtraTxos = append(allExtraTxos, extraTxos...)
-	}
-
-	for _, extra := range extras {
-		txos, extraTxos, err := extra.ToOutputs()
-		if err != nil {
-			*result = nil
-			return err
-		}
-		// TODO: there may be a more efficient way to do this.
-		allTxos = append(allTxos, txos...)
-		allExtraTxos = append(allExtraTxos, extraTxos...)
-	}
-
-	res := &pb.Outputs{
-		Txos:         allTxos,
-		ExtraTxos:    allExtraTxos,
-		Total:        uint32(len(allTxos) + len(allExtraTxos)),
-		Offset:       0,   //TODO
-		Blocked:      nil, //TODO
-		BlockedTotal: 0,   //TODO
-	}
-
-	log.Warn(res)
-
-	*result = res
-	return nil
-}

server/jsonrpc_federation.go

@@ -1,39 +0,0 @@
-package server
-
-import (
-	"errors"
-
-	log "github.com/sirupsen/logrus"
-)
-
-type PeersService struct {
-	Server *Server
-	// needed for subscribe/unsubscribe
-	sessionMgr *sessionManager
-	session    *session
-}
-
-type PeersSubscribeReq struct {
-	Ip      string   `json:"ip"`
-	Host    string   `json:"host"`
-	Details []string `json:"details"`
-}
-
-type PeersSubscribeResp string
-
-// Features is the json rpc endpoint for 'server.peers.subcribe'.
-func (t *PeersService) Subscribe(req *PeersSubscribeReq, res **PeersSubscribeResp) error {
-	log.Println("PeersSubscribe")
-	// var port = "50001"
-
-	// FIXME: Get the actual port from the request details
-
-	if t.sessionMgr == nil || t.session == nil {
-		*res = nil
-		return errors.New("no session, rpc not supported")
-	}
-	t.sessionMgr.peersSubscribe(t.session, true /*subscribe*/)
-
-	*res = nil
-	return nil
-}

server/jsonrpc_server.go

@@ -1,88 +0,0 @@
-package server
-
-import (
-	log "github.com/sirupsen/logrus"
-)
-
-type ServerService struct {
-	Args *Args
-}
-
-type ServerFeatureService struct {
-	Args *Args
-}
-
-type ServerFeaturesReq struct{}
-
-type ServerFeaturesRes struct {
-	Hosts             map[string]string `json:"hosts"`
-	Pruning           string            `json:"pruning"`
-	ServerVersion     string            `json:"server_version"`
-	ProtocolMin       string            `json:"protocol_min"`
-	ProtocolMax       string            `json:"protocol_max"`
-	GenesisHash       string            `json:"genesis_hash"`
-	Description       string            `json:"description"`
-	PaymentAddress    string            `json:"payment_address"`
-	DonationAddress   string            `json:"donation_address"`
-	DailyFee          string            `json:"daily_fee"`
-	HashFunction      string            `json:"hash_function"`
-	TrendingAlgorithm string            `json:"trending_algorithm"`
-}
-
-// Features is the json rpc endpoint for 'server.features'.
-func (t *ServerService) Features(req *ServerFeaturesReq, res **ServerFeaturesRes) error {
-	log.Println("Features")
-
-	features := &ServerFeaturesRes{
-		Hosts:             map[string]string{},
-		Pruning:           "",
-		ServerVersion:     HUB_PROTOCOL_VERSION,
-		ProtocolMin:       PROTOCOL_MIN,
-		ProtocolMax:       PROTOCOL_MAX,
-		GenesisHash:       t.Args.GenesisHash,
-		Description:       t.Args.ServerDescription,
-		PaymentAddress:    t.Args.PaymentAddress,
-		DonationAddress:   t.Args.DonationAddress,
-		DailyFee:          t.Args.DailyFee,
-		HashFunction:      "sha256",
-		TrendingAlgorithm: "fast_ar",
-	}
-	*res = features
-
-	return nil
-}
-
-type ServerBannerService struct {
-	Args *Args
-}
-
-type ServerBannerReq struct{}
-
-type ServerBannerRes string
-
-// Banner is the json rpc endpoint for 'server.banner'.
-func (t *ServerService) Banner(req *ServerBannerReq, res **ServerBannerRes) error {
-	log.Println("Banner")
-
-	*res = (*ServerBannerRes)(t.Args.Banner)
-
-	return nil
-}
-
-type ServerVersionService struct {
-	Args *Args
-}
-
-type ServerVersionReq [2]string // [client_name, client_version]
-
-type ServerVersionRes [2]string // [version, protocol_version]
-
-// Version is the json rpc endpoint for 'server.version'.
-func (t *ServerService) Version(req *ServerVersionReq, res **ServerVersionRes) error {
-	// FIXME: We may need to do the computation of a negotiated version here.
-	// Also return an error if client is not supported?
-	result := [2]string{t.Args.ServerVersion, t.Args.ServerVersion}
-	*res = (*ServerVersionRes)(&result)
-	log.Printf("Version(%v) -> %v", *req, **res)
-	return nil
-}

server/jsonrpc_service.go

@@ -52,23 +52,24 @@ func (cr *gorillaRpcCodecRequest) Method() (string, error) {
 
 // StartJsonRPC starts the json rpc server and registers the endpoints.
 func (s *Server) StartJsonRPC() error {
+	s.sessionManager.start()
+	defer s.sessionManager.stop()
+
 	// Set up the pure JSONRPC server with persistent connections/sessions.
 	if s.Args.JSONRPCPort != 0 {
-		port := ":" + strconv.Itoa(s.Args.JSONRPCPort)
-		laddr, err := net.ResolveTCPAddr("tcp4", port)
+		port := ":" + strconv.FormatUint(uint64(s.Args.JSONRPCPort), 10)
+		laddr, err := net.ResolveTCPAddr("tcp", port)
 		if err != nil {
 			log.Errorf("ResoveIPAddr: %v\n", err)
 			goto fail1
 		}
-		listener, err := net.ListenTCP("tcp4", laddr)
+		listener, err := net.ListenTCP("tcp", laddr)
 		if err != nil {
 			log.Errorf("ListenTCP: %v\n", err)
 			goto fail1
 		}
 		log.Infof("JSONRPC server listening on %s", listener.Addr().String())
-		s.sessionManager.start()
 		acceptConnections := func(listener net.Listener) {
-			defer s.sessionManager.stop()
 			for {
 				conn, err := listener.Accept()
 				if err != nil {
@@ -77,7 +78,6 @@ func (s *Server) StartJsonRPC() error {
 				}
 				log.Infof("Accepted: %v", conn.RemoteAddr())
 				s.sessionManager.addSession(conn)
-
 			}
 		}
 		go acceptConnections(netutil.LimitListener(listener, s.sessionManager.sessionsMax))
@@ -91,52 +91,31 @@ fail1:
 		s1.RegisterCodec(&gorillaRpcCodec{gorilla_json.NewCodec()}, "application/json")
 
 		// Register "blockchain.claimtrie.*"" handlers.
-		claimtrieSvc := &ClaimtrieService{s.DB, s}
+		claimtrieSvc := &ClaimtrieService{s.DB}
 		err := s1.RegisterTCPService(claimtrieSvc, "blockchain_claimtrie")
 		if err != nil {
 			log.Errorf("RegisterTCPService: %v\n", err)
 			goto fail2
 		}
 
-		// Register "blockchain.{block,address,scripthash,transaction}.*" handlers.
+		// Register other "blockchain.{block,address,scripthash}.*" handlers.
 		blockchainSvc := &BlockchainBlockService{s.DB, s.Chain}
 		err = s1.RegisterTCPService(blockchainSvc, "blockchain_block")
 		if err != nil {
 			log.Errorf("RegisterTCPService: %v\n", err)
 			goto fail2
 		}
-		err = s1.RegisterTCPService(&BlockchainHeadersService{s.DB, s.Chain, s.sessionManager, nil}, "blockchain_headers")
+		err = s1.RegisterTCPService(&BlockchainHeadersService{s.DB, s.Chain, nil, nil}, "blockchain_headers")
 		if err != nil {
 			log.Errorf("RegisterTCPService: %v\n", err)
 			goto fail2
 		}
-		err = s1.RegisterTCPService(&BlockchainAddressService{s.DB, s.Chain, s.sessionManager, nil}, "blockchain_address")
+		err = s1.RegisterTCPService(&BlockchainAddressService{s.DB, s.Chain, nil, nil}, "blockchain_address")
 		if err != nil {
 			log.Errorf("RegisterTCPService: %v\n", err)
 			goto fail2
 		}
-		err = s1.RegisterTCPService(&BlockchainScripthashService{s.DB, s.Chain, s.sessionManager, nil}, "blockchain_scripthash")
-		if err != nil {
-			log.Errorf("RegisterTCPService: %v\n", err)
-			goto fail2
-		}
-		err = s1.RegisterTCPService(&BlockchainTransactionService{s.DB, s.Chain, s.sessionManager}, "blockchain_transaction")
-		if err != nil {
-			log.Errorf("RegisterTCPService: %v\n", err)
-			goto fail2
-		}
-
-		// Register "server.{features,banner,version}" handlers.
-		serverSvc := &ServerService{s.Args}
-		err = s1.RegisterTCPService(serverSvc, "server")
-		if err != nil {
-			log.Errorf("RegisterTCPService: %v\n", err)
-			goto fail2
-		}
-
-		// Register "server.peers" handlers.
-		peersSvc := &PeersService{Server: s}
-		err = s1.RegisterTCPService(peersSvc, "server_peers")
+		err = s1.RegisterTCPService(&BlockchainScripthashService{s.DB, s.Chain, nil, nil}, "blockchain_scripthash")
 		if err != nil {
 			log.Errorf("RegisterTCPService: %v\n", err)
 			goto fail2

server/notifier.go

@@ -2,7 +2,6 @@ package server
 
 import (
 	"encoding/binary"
-	"fmt"
 	"net"
 
 	"github.com/lbryio/herald.go/internal"
@@ -54,16 +53,10 @@ func (s *Server) DoNotify(heightHash *internal.HeightHash) error {
 // RunNotifier Runs the notfying action forever
 func (s *Server) RunNotifier() error {
 	for notification := range s.NotifierChan {
-		switch note := notification.(type) {
+		switch notification.(type) {
 		case internal.HeightHash:
-			heightHash := note
+			heightHash, _ := notification.(internal.HeightHash)
 			s.DoNotify(&heightHash)
-		// Do we need this?
-		// case peerNotification:
-		// 	peer, _ := notification.(peerNotification)
-		// 	s.notifyPeerSubs(&Peer{Address: peer.address, Port: peer.port})
-		default:
-			logrus.Warn("unknown notification type")
 		}
 		s.sessionManager.doNotify(notification)
 	}
@@ -72,8 +65,7 @@ func (s *Server) RunNotifier() error {
 
 // NotifierServer implements the TCP protocol for height/blockheader notifications
 func (s *Server) NotifierServer() error {
-	s.Grp.Add(1)
-	address := ":" + fmt.Sprintf("%d", s.Args.NotifierPort)
+	address := ":" + s.Args.NotifierPort
 	addr, err := net.ResolveTCPAddr("tcp", address)
 	if err != nil {
 		return err
@@ -85,27 +77,11 @@ func (s *Server) NotifierServer() error {
 	}
 
 	defer listen.Close()
-	rdyCh := make(chan bool)
 
 	for {
-		var conn net.Conn
-		var err error
 
 		logrus.Info("Waiting for connection")
-
-		go func() {
-			conn, err = listen.Accept()
-			rdyCh <- true
-		}()
-
-		select {
-		case <-s.Grp.Ch():
-			s.Grp.Done()
-			return nil
-		case <-rdyCh:
-			logrus.Info("Connection accepted")
-		}
-
+		conn, err := listen.Accept()
 		if err != nil {
 			logrus.Warn(err)
 			continue

server/notifier_test.go

@@ -1,6 +1,7 @@
 package server_test
 
 import (
+	"context"
 	"encoding/hex"
 	"fmt"
 	"net"
@@ -9,32 +10,11 @@ import (
 
 	"github.com/lbryio/herald.go/internal"
 	"github.com/lbryio/herald.go/server"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	"github.com/sirupsen/logrus"
 )
 
 const defaultBufferSize = 1024
 
-func subReady(s *server.Server) error {
-	sleepTime := time.Millisecond * 100
-	for {
-		if sleepTime > time.Second {
-			return fmt.Errorf("timeout")
-		}
-		s.HeightSubsMut.RLock()
-		if len(s.HeightSubs) > 0 {
-			s.HeightSubsMut.RUnlock()
-			return nil
-		}
-		s.HeightSubsMut.RUnlock()
-
-		logrus.Warn("waiting for subscriber")
-		time.Sleep(sleepTime)
-		sleepTime = sleepTime * 2
-
-	}
-}
-
 func tcpConnReady(addr string) (net.Conn, error) {
 	sleepTime := time.Millisecond * 100
 	for {
@@ -67,14 +47,14 @@ func tcpRead(conn net.Conn) ([]byte, error) {
 }
 
 func TestNotifierServer(t *testing.T) {
-	args := server.MakeDefaultTestArgs()
-	ctx := stop.NewDebug()
+	args := makeDefaultArgs()
+	ctx := context.Background()
 	hub := server.MakeHubServer(ctx, args)
 
 	go hub.NotifierServer()
 	go hub.RunNotifier()
 
-	addr := fmt.Sprintf(":%d", args.NotifierPort)
+	addr := fmt.Sprintf(":%s", args.NotifierPort)
 	logrus.Info(addr)
 	conn, err := tcpConnReady(addr)
 	if err != nil {
@@ -96,10 +76,7 @@ func TestNotifierServer(t *testing.T) {
 
 	// Hacky but needed because if the reader isn't ready
 	// before the writer sends it won't get the data
-	err = subReady(hub)
-	if err != nil {
-		t.Fatal(err)
-	}
+	time.Sleep(time.Second)
 
 	hash, _ := hex.DecodeString("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
 	logrus.Warn("sending hash")

server/search_test.go

@@ -11,7 +11,6 @@ import (
 
 	pb "github.com/lbryio/herald.go/protobuf/go"
 	server "github.com/lbryio/herald.go/server"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	"github.com/olivere/elastic/v7"
 )
 
@@ -56,14 +55,13 @@ func TestSearch(t *testing.T) {
 		w.Write([]byte(resp))
 	}
 
-	ctx := context.Background()
-	stopGroup := stop.NewDebug()
-	args := server.MakeDefaultTestArgs()
-	hubServer := server.MakeHubServer(stopGroup, args)
+	context := context.Background()
+	args := makeDefaultArgs()
+	hubServer := server.MakeHubServer(context, args)
 	req := &pb.SearchRequest{
 		Text: "asdf",
 	}
-	out, err := hubServer.Search(ctx, req)
+	out, err := hubServer.Search(context, req)
 	if err != nil {
 		log.Println(err)
 	}

server/server.go

@@ -8,12 +8,11 @@ import (
 	"fmt"
 	"hash"
 	"io/ioutil"
-	golog "log"
+	"log"
 	"net"
 	"net/http"
 	"os"
 	"regexp"
-	"strconv"
 	"sync"
 	"time"
 
@@ -23,12 +22,10 @@ import (
 	"github.com/lbryio/herald.go/meta"
 	pb "github.com/lbryio/herald.go/protobuf/go"
 	"github.com/lbryio/lbcd/chaincfg"
-	lbcd "github.com/lbryio/lbcd/rpcclient"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	"github.com/olivere/elastic/v7"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	log "github.com/sirupsen/logrus"
+	logrus "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
 )
@@ -40,7 +37,6 @@ type Server struct {
 	WeirdCharsRe     *regexp.Regexp
 	DB               *db.ReadOnlyDBColumnFamily
 	Chain            *chaincfg.Params
-	DaemonClient     *lbcd.Client
 	EsClient         *elastic.Client
 	QueryCache       *ttlcache.Cache
 	S256             *hash.Hash
@@ -57,8 +53,6 @@ type Server struct {
 	HeightSubs       map[net.Addr]net.Conn
 	HeightSubsMut    sync.RWMutex
 	NotifierChan     chan interface{}
-	Grp              *stop.Group
-	notiferListener  *net.TCPListener
 	sessionManager   *sessionManager
 	pb.UnimplementedHubServer
 }
@@ -140,8 +134,7 @@ func (s *Server) PeerServersLoadOrStore(peer *Peer) (actual *Peer, loaded bool)
 
 // Run "main" function for starting the server. This blocks.
 func (s *Server) Run() {
-	address := ":" + strconv.Itoa(s.Args.Port)
-	l, err := net.Listen("tcp", address)
+	l, err := net.Listen("tcp", ":"+s.Args.Port)
 	if err != nil {
 		log.Fatalf("failed to listen: %v", err)
 	}
@@ -156,50 +149,31 @@ func (s *Server) Run() {
 	}
 }
 
-func (s *Server) Stop() {
-	log.Println("Shutting down server...")
-
-	if s.EsClient != nil {
-		log.Println("Stopping es client...")
-		s.EsClient.Stop()
-	}
-	if s.GrpcServer != nil {
-		log.Println("Stopping grpc server...")
-		s.GrpcServer.GracefulStop()
-	}
-	log.Println("Stopping other server threads...")
-	s.Grp.StopAndWait()
-	if s.DB != nil {
-		log.Println("Stopping database connection...")
-		s.DB.Shutdown()
-	}
-
-	log.Println("Returning from Stop...")
-}
-
-func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, error) {
-	tmpName, err := os.MkdirTemp("", "go-lbry-hub")
+func LoadDatabase(args *Args) (*db.ReadOnlyDBColumnFamily, error) {
+	tmpName, err := ioutil.TempDir("", "go-lbry-hub")
 	if err != nil {
-		log.Info(err)
+		logrus.Info(err)
 		log.Fatal(err)
 	}
-	log.Info("tmpName", tmpName)
+	logrus.Info("tmpName", tmpName)
 	if err != nil {
-		log.Info(err)
+		logrus.Info(err)
 	}
-	myDB, err := db.GetProdDB(args.DBPath, tmpName, grp)
+	myDB, _, err := db.GetProdDB(args.DBPath, tmpName)
+	// dbShutdown = func() {
+	// 	db.Shutdown(myDB)
+	// }
 	if err != nil {
 		// Can't load the db, fail loudly
-		log.Info(err)
+		logrus.Info(err)
 		log.Fatalln(err)
 	}
 
 	if myDB.LastState != nil {
-		log.Infof("DB version: %v", myDB.LastState.DBVersion)
-		log.Infof("height: %v", myDB.LastState.Height)
-		log.Infof("genesis: %v", myDB.LastState.Genesis.String())
-		log.Infof("tip: %v", myDB.LastState.Tip.String())
-		log.Infof("tx count: %v", myDB.LastState.TxCount)
+		logrus.Infof("DB version: %v", myDB.LastState.DBVersion)
+		logrus.Infof("height: %v", myDB.LastState.Height)
+		logrus.Infof("tip: %v", myDB.LastState.Tip.String())
+		logrus.Infof("tx count: %v", myDB.LastState.TxCount)
 	}
 
 	blockingChannelHashes := make([][]byte, 0, 10)
@@ -210,7 +184,7 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
 	for _, id := range args.BlockingChannelIds {
 		hash, err := hex.DecodeString(id)
 		if err != nil {
-			log.Warn("Invalid channel id: ", id)
+			logrus.Warn("Invalid channel id: ", id)
 			continue
 		}
 		blockingChannelHashes = append(blockingChannelHashes, hash)
@@ -220,7 +194,7 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
 	for _, id := range args.FilteringChannelIds {
 		hash, err := hex.DecodeString(id)
 		if err != nil {
-			log.Warn("Invalid channel id: ", id)
+			logrus.Warn("Invalid channel id: ", id)
 			continue
 		}
 		filteringChannelHashes = append(filteringChannelHashes, hash)
@@ -231,10 +205,10 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
 	myDB.FilteringChannelHashes = filteringChannelHashes
 
 	if len(filteringIds) > 0 {
-		log.Infof("filtering claims reposted by channels: %+s", filteringIds)
+		logrus.Infof("filtering claims reposted by channels: %+s", filteringIds)
 	}
 	if len(blockingIds) > 0 {
-		log.Infof("blocking claims reposted by channels: %+s", blockingIds)
+		logrus.Infof("blocking claims reposted by channels: %+s", blockingIds)
 	}
 
 	return myDB, nil
@@ -243,7 +217,7 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
 // MakeHubServer takes the arguments given to a hub when it's started and
 // initializes everything. It loads information about previously known peers,
 // creates needed internal data structures, and initializes goroutines.
-func MakeHubServer(grp *stop.Group, args *Args) *Server {
+func MakeHubServer(ctx context.Context, args *Args) *Server {
 	grpcServer := grpc.NewServer(grpc.NumStreamWorkers(0))
 
 	multiSpaceRe, err := regexp.Compile(`\s{2,}`)
@@ -256,34 +230,9 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
 		log.Fatal(err)
 	}
 
-	var lbcdClient *lbcd.Client = nil
-	if args.DaemonURL != nil && args.DaemonURL.Host != "" {
-		var rpcCertificate []byte
-		if args.DaemonCAPath != "" {
-			rpcCertificate, err = ioutil.ReadFile(args.DaemonCAPath)
-			if err != nil {
-				log.Fatalf("failed to read SSL certificate from path: %v", args.DaemonCAPath)
-			}
-		}
-		log.Warnf("connecting to lbcd daemon at %v...", args.DaemonURL.Host)
-		password, _ := args.DaemonURL.User.Password()
-		cfg := &lbcd.ConnConfig{
-			Host:         args.DaemonURL.Host,
-			User:         args.DaemonURL.User.Username(),
-			Pass:         password,
-			HTTPPostMode: true,
-			DisableTLS:   rpcCertificate == nil,
-			Certificates: rpcCertificate,
-		}
-		lbcdClient, err = lbcd.New(cfg, nil)
-		if err != nil {
-			log.Fatalf("lbcd daemon connection failed: %v", err)
-		}
-	}
-
-	var esClient *elastic.Client = nil
+	var client *elastic.Client = nil
 	if !args.DisableEs {
-		esUrl := args.EsHost + ":" + fmt.Sprintf("%d", args.EsPort)
+		esUrl := args.EsHost + ":" + args.EsPort
 		opts := []elastic.ClientOptionFunc{
 			elastic.SetSniff(true),
 			elastic.SetSnifferTimeoutStartup(time.Second * 60),
@@ -291,9 +240,9 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
 			elastic.SetURL(esUrl),
 		}
 		if args.Debug {
-			opts = append(opts, elastic.SetTraceLog(golog.New(os.Stderr, "[[ELASTIC]]", 0)))
+			opts = append(opts, elastic.SetTraceLog(log.New(os.Stderr, "[[ELASTIC]]", 0)))
 		}
-		esClient, err = elastic.NewClient(opts...)
+		client, err = elastic.NewClient(opts...)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -317,18 +266,18 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
 
 	//TODO: is this the right place to load the db?
 	var myDB *db.ReadOnlyDBColumnFamily
+	// var dbShutdown = func() {}
 	if !args.DisableResolve {
-		myDB, err = LoadDatabase(args, grp)
+		myDB, err = LoadDatabase(args)
 		if err != nil {
-			log.Warning(err)
+			logrus.Warning(err)
 		}
 	}
 
-	// Determine which chain to use based on db and cli values
 	dbChain := (*chaincfg.Params)(nil)
-	if myDB != nil && myDB.LastState != nil {
+	if myDB != nil && myDB.LastState != nil && myDB.LastState.Genesis != nil {
 		// The chain params can be inferred from DBStateValue.
-		switch myDB.LastState.Genesis.Hash {
+		switch *myDB.LastState.Genesis {
 		case *chaincfg.MainNetParams.GenesisHash:
 			dbChain = &chaincfg.MainNetParams
 		case *chaincfg.TestNet3Params.GenesisHash:
@@ -351,7 +300,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
 	chain := chaincfg.MainNetParams
 	if dbChain != nil && cliChain != nil {
 		if dbChain != cliChain {
-			log.Warnf("network: %v (from db) conflicts with %v (from cli)", dbChain.Name, cliChain.Name)
+			logrus.Warnf("network: %v (from db) conflicts with %v (from cli)", dbChain.Name, cliChain.Name)
 		}
 		chain = *dbChain
 	} else if dbChain != nil {
@@ -359,11 +308,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
 	} else if cliChain != nil {
 		chain = *cliChain
 	}
-	log.Infof("network: %v", chain.Name)
-
-	args.GenesisHash = chain.GenesisHash.String()
-
-	sessionGrp := stop.New(grp)
+	logrus.Infof("network: %v", chain.Name)
 
 	s := &Server{
 		GrpcServer:       grpcServer,
@@ -372,8 +317,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
 		WeirdCharsRe:     weirdCharsRe,
 		DB:               myDB,
 		Chain:            &chain,
-		DaemonClient:     lbcdClient,
-		EsClient:         esClient,
+		EsClient:         client,
 		QueryCache:       cache,
 		S256:             &s256,
 		LastRefreshCheck: time.Now(),
@@ -388,39 +332,28 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
 		ExternalIP:       net.IPv4(127, 0, 0, 1),
 		HeightSubs:       make(map[net.Addr]net.Conn),
 		HeightSubsMut:    sync.RWMutex{},
-		NotifierChan:     make(chan interface{}, 1),
-		Grp:              grp,
-		sessionManager:   nil,
+		NotifierChan:     make(chan interface{}),
+		sessionManager:   newSessionManager(myDB, &chain, args.MaxSessions, args.SessionTimeout),
 	}
-	// FIXME: HACK
-	s.sessionManager = newSessionManager(s, myDB, args, sessionGrp, &chain, lbcdClient)
 
 	// Start up our background services
 	if !args.DisableResolve && !args.DisableRocksDBRefresh {
-		log.Info("Running detect changes")
+		logrus.Info("Running detect changes")
 		myDB.RunDetectChanges(s.NotifierChan)
 	}
 	if !args.DisableBlockingAndFiltering {
 		myDB.RunGetBlocksAndFilters()
 	}
 	if !args.DisableStartPrometheus {
-		go s.prometheusEndpoint(fmt.Sprintf("%d", s.Args.PrometheusPort), "metrics")
+		go s.prometheusEndpoint(s.Args.PrometheusPort, "metrics")
 	}
 	if !args.DisableStartUDP {
 		go func() {
-			err := s.UDPServer(s.Args.Port)
+			err := s.UDPServer()
 			if err != nil {
-				log.Errorf("UDP Server (%d) failed! %v", s.Args.Port, err)
+				log.Println("UDP Server failed!", err)
 			}
 		}()
-		if s.Args.JSONRPCPort != 0 {
-			go func() {
-				err := s.UDPServer(s.Args.JSONRPCPort)
-				if err != nil {
-					log.Errorf("UDP Server (%d) failed! %v", s.Args.JSONRPCPort, err)
-				}
-			}()
-		}
 	}
 	if !args.DisableStartNotifier {
 		go func() {
@@ -461,7 +394,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
 // for this hub to allow for metric tracking.
 func (s *Server) prometheusEndpoint(port string, endpoint string) {
 	http.Handle("/"+endpoint, promhttp.Handler())
-	log.Printf("listening on :%s /%s\n", port, endpoint)
+	log.Println(fmt.Sprintf("listening on :%s /%s", port, endpoint))
 	err := http.ListenAndServe(":"+port, nil)
 	log.Fatalln("Shouldn't happen??!?!", err)
 }
@@ -619,7 +552,7 @@ func InternalResolve(urls []string, DB *db.ReadOnlyDBColumnFamily) (*pb.Outputs,
 		BlockedTotal: 0,   //TODO
 	}
 
-	log.Warn(res)
+	logrus.Warn(res)
 
 	return res, nil
 }

server/server_test_pkg.go

@@ -1,11 +1,5 @@
 package server
 
-import (
-	"github.com/lbryio/herald.go/db"
-	"github.com/lbryio/lbcd/chaincfg"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
-)
-
 func (s *Server) AddPeerExported() func(*Peer, bool, bool) error {
 	return s.addPeer
 }
@@ -13,7 +7,3 @@ func (s *Server) AddPeerExported() func(*Peer, bool, bool) error {
 func (s *Server) GetNumPeersExported() func() int64 {
 	return s.getNumPeers
 }
-
-func NewSessionManagerExported(server *Server, db *db.ReadOnlyDBColumnFamily, args *Args, grp *stop.Group, chain *chaincfg.Params) *sessionManager {
-	return newSessionManager(server, db, args, grp, chain, nil)
-}

server/session.go

@@ -1,9 +1,7 @@
 package server
 
 import (
-	"bytes"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
 	"net"
 	"net/rpc"
@@ -16,15 +14,12 @@ import (
 	"github.com/lbryio/herald.go/db"
 	"github.com/lbryio/herald.go/internal"
 	"github.com/lbryio/lbcd/chaincfg"
-	"github.com/lbryio/lbcd/chaincfg/chainhash"
-	lbcd "github.com/lbryio/lbcd/rpcclient"
-	"github.com/lbryio/lbcd/wire"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	log "github.com/sirupsen/logrus"
 )
 
 type headerNotification struct {
 	internal.HeightHash
+	blockHeader         [HEADER_SIZE]byte
 	blockHeaderElectrum *BlockHeaderElectrum
 	blockHeaderStr      string
 }
@@ -35,11 +30,6 @@ type hashXNotification struct {
 	statusStr string
 }
 
-type peerNotification struct {
-	address string
-	port    string
-}
-
 type session struct {
 	id   uintptr
 	addr net.Addr
@@ -48,8 +38,6 @@ type session struct {
 	hashXSubs map[[HASHX_LEN]byte]string
 	// headersSub indicates header subscription
 	headersSub bool
-	// peersSub indicates peer subscription
-	peersSub bool
 	// headersSubRaw indicates the header subscription mode
 	headersSubRaw bool
 	// client provides the ability to send notifications
@@ -64,17 +52,18 @@ type session struct {
 func (s *session) doNotify(notification interface{}) {
 	var method string
 	var params interface{}
-	switch note := notification.(type) {
+	switch notification.(type) {
 	case headerNotification:
 		if !s.headersSub {
 			return
 		}
+		note, _ := notification.(headerNotification)
 		heightHash := note.HeightHash
 		method = "blockchain.headers.subscribe"
 		if s.headersSubRaw {
 			header := note.blockHeaderStr
 			if len(header) == 0 {
-				header = hex.EncodeToString(note.BlockHeader[:])
+				header = hex.EncodeToString(note.blockHeader[:])
 			}
 			params = &HeadersSubscribeRawResp{
 				Hex:    header,
@@ -83,11 +72,12 @@ func (s *session) doNotify(notification interface{}) {
 		} else {
 			header := note.blockHeaderElectrum
 			if header == nil { // not initialized
-				header = newBlockHeaderElectrum((*[HEADER_SIZE]byte)(note.BlockHeader), uint32(heightHash.Height))
+				header = newBlockHeaderElectrum(&note.blockHeader, uint32(heightHash.Height))
 			}
 			params = header
 		}
 	case hashXNotification:
+		note, _ := notification.(hashXNotification)
 		orig, ok := s.hashXSubs[note.hashX]
 		if !ok {
 			return
@@ -102,13 +92,6 @@ func (s *session) doNotify(notification interface{}) {
 			status = hex.EncodeToString(note.status)
 		}
 		params = []string{orig, status}
-	case peerNotification:
-		if !s.peersSub {
-			return
-		}
-		method = "server.peers.subscribe"
-		params = []string{note.address, note.port}
-
 	default:
 		log.Warnf("unknown notification type: %v", notification)
 		return
@@ -131,46 +114,34 @@ type sessionMap map[uintptr]*session
 
 type sessionManager struct {
 	// sessionsMut protects sessions, headerSubs, hashXSubs state
-	sessionsMut sync.RWMutex
-	sessions    sessionMap
-	// sessionsWait   sync.WaitGroup
-	grp            *stop.Group
+	sessionsMut    sync.RWMutex
+	sessions       sessionMap
+	sessionsWait   sync.WaitGroup
 	sessionsMax    int
 	sessionTimeout time.Duration
 	manageTicker   *time.Ticker
 	db             *db.ReadOnlyDBColumnFamily
-	args           *Args
-	server         *Server
 	chain          *chaincfg.Params
-	lbcd           *lbcd.Client
-	// peerSubs are sessions subscribed via 'blockchain.peers.subscribe'
-	peerSubs sessionMap
 	// headerSubs are sessions subscribed via 'blockchain.headers.subscribe'
 	headerSubs sessionMap
 	// hashXSubs are sessions subscribed via 'blockchain.{address,scripthash}.subscribe'
 	hashXSubs map[[HASHX_LEN]byte]sessionMap
 }
 
-func newSessionManager(server *Server, db *db.ReadOnlyDBColumnFamily, args *Args, grp *stop.Group, chain *chaincfg.Params, lbcd *lbcd.Client) *sessionManager {
+func newSessionManager(db *db.ReadOnlyDBColumnFamily, chain *chaincfg.Params, sessionsMax, sessionTimeout int) *sessionManager {
 	return &sessionManager{
 		sessions:       make(sessionMap),
-		grp:            grp,
-		sessionsMax:    args.MaxSessions,
-		sessionTimeout: time.Duration(args.SessionTimeout) * time.Second,
-		manageTicker:   time.NewTicker(time.Duration(max(5, args.SessionTimeout/20)) * time.Second),
+		sessionsMax:    sessionsMax,
+		sessionTimeout: time.Duration(sessionTimeout) * time.Second,
+		manageTicker:   time.NewTicker(time.Duration(max(5, sessionTimeout/20)) * time.Second),
 		db:             db,
-		args:           args,
-		server:         server,
 		chain:          chain,
-		lbcd:           lbcd,
-		peerSubs:       make(sessionMap),
 		headerSubs:     make(sessionMap),
 		hashXSubs:      make(map[[HASHX_LEN]byte]sessionMap),
 	}
 }
 
 func (sm *sessionManager) start() {
-	sm.grp.Add(1)
 	go sm.manage()
 }
 
@@ -197,13 +168,7 @@ func (sm *sessionManager) manage() {
 		}
 		sm.sessionsMut.Unlock()
 		// Wait for next management clock tick.
-		select {
-		case <-sm.grp.Ch():
-			sm.grp.Done()
-			return
-		case <-sm.manageTicker.C:
-			continue
-		}
+		<-sm.manageTicker.C
 	}
 }
 
@@ -225,28 +190,14 @@ func (sm *sessionManager) addSession(conn net.Conn) *session {
 	// each request and update subscriptions.
 	s1 := rpc.NewServer()
 
-	// Register "server.{features,banner,version}" handlers.
-	serverSvc := &ServerService{sm.args}
-	err := s1.RegisterName("server", serverSvc)
-	if err != nil {
-		log.Errorf("RegisterName: %v\n", err)
-	}
-
-	// Register "server.peers" handlers.
-	peersSvc := &PeersService{Server: sm.server}
-	err = s1.RegisterName("server.peers", peersSvc)
-	if err != nil {
-		log.Errorf("RegisterName: %v\n", err)
-	}
-
 	// Register "blockchain.claimtrie.*"" handlers.
-	claimtrieSvc := &ClaimtrieService{sm.db, sm.server}
-	err = s1.RegisterName("blockchain.claimtrie", claimtrieSvc)
+	claimtrieSvc := &ClaimtrieService{sm.db}
+	err := s1.RegisterName("blockchain.claimtrie", claimtrieSvc)
 	if err != nil {
-		log.Errorf("RegisterName: %v\n", err)
+		log.Errorf("RegisterService: %v\n", err)
 	}
 
-	// Register "blockchain.{block,address,scripthash,transaction}.*" handlers.
+	// Register other "blockchain.{block,address,scripthash}.*" handlers.
 	blockchainSvc := &BlockchainBlockService{sm.db, sm.chain}
 	err = s1.RegisterName("blockchain.block", blockchainSvc)
 	if err != nil {
@@ -268,17 +219,12 @@ func (sm *sessionManager) addSession(conn net.Conn) *session {
 		log.Errorf("RegisterName: %v\n", err)
 		goto fail
 	}
-	err = s1.RegisterName("blockchain.transaction", &BlockchainTransactionService{sm.db, sm.chain, sm})
-	if err != nil {
-		log.Errorf("RegisterName: %v\n", err)
-		goto fail
-	}
 
-	sm.grp.Add(1)
+	sm.sessionsWait.Add(1)
 	go func() {
-		s1.ServeCodec(&sessionServerCodec{jsonrpc.NewServerCodec(newJsonPatchingCodec(conn)), sess})
+		s1.ServeCodec(&SessionServerCodec{jsonrpc.NewServerCodec(conn), sess})
 		log.Infof("session %v goroutine exit", sess.addr.String())
-		sm.grp.Done()
+		sm.sessionsWait.Done()
 	}()
 	return sess
 
@@ -309,27 +255,6 @@ func (sm *sessionManager) removeSessionLocked(sess *session) {
 	sess.conn.Close()
 }
 
-func (sm *sessionManager) broadcastTx(rawTx []byte) (*chainhash.Hash, error) {
-	var msgTx wire.MsgTx
-	err := msgTx.Deserialize(bytes.NewReader(rawTx))
-	if err != nil {
-		return nil, err
-	}
-	return sm.lbcd.SendRawTransaction(&msgTx, false)
-}
-
-func (sm *sessionManager) peersSubscribe(sess *session, subscribe bool) {
-	sm.sessionsMut.Lock()
-	defer sm.sessionsMut.Unlock()
-	if subscribe {
-		sm.peerSubs[sess.id] = sess
-		sess.peersSub = true
-		return
-	}
-	delete(sm.peerSubs, sess.id)
-	sess.peersSub = false
-}
-
 func (sm *sessionManager) headersSubscribe(sess *session, raw bool, subscribe bool) {
 	sm.sessionsMut.Lock()
 	defer sm.sessionsMut.Unlock()
@@ -369,26 +294,18 @@ func (sm *sessionManager) hashXSubscribe(sess *session, hashX []byte, original s
 }
 
 func (sm *sessionManager) doNotify(notification interface{}) {
-	switch note := notification.(type) {
-	case internal.HeightHash:
-		// The HeightHash notification translates to headerNotification.
-		notification = &headerNotification{HeightHash: note}
-	}
-
 	sm.sessionsMut.RLock()
 	var subsCopy sessionMap
-	switch note := notification.(type) {
+	switch notification.(type) {
 	case headerNotification:
-		log.Infof("header notification @ %#v", note)
+		note, _ := notification.(headerNotification)
 		subsCopy = sm.headerSubs
 		if len(subsCopy) > 0 {
-			hdr := [HEADER_SIZE]byte{}
-			copy(hdr[:], note.BlockHeader)
-			note.blockHeaderElectrum = newBlockHeaderElectrum(&hdr, uint32(note.Height))
-			note.blockHeaderStr = hex.EncodeToString(note.BlockHeader[:])
+			note.blockHeaderElectrum = newBlockHeaderElectrum(&note.blockHeader, uint32(note.Height))
+			note.blockHeaderStr = hex.EncodeToString(note.blockHeader[:])
 		}
 	case hashXNotification:
-		log.Infof("hashX notification @ %#v", note)
+		note, _ := notification.(hashXNotification)
 		hashXSubs, ok := sm.hashXSubs[note.hashX]
 		if ok {
 			subsCopy = hashXSubs
@@ -396,8 +313,6 @@ func (sm *sessionManager) doNotify(notification interface{}) {
 		if len(subsCopy) > 0 {
 			note.statusStr = hex.EncodeToString(note.status)
 		}
-	case peerNotification:
-		subsCopy = sm.peerSubs
 	default:
 		log.Warnf("unknown notification type: %v", notification)
 	}
@@ -407,50 +322,26 @@ func (sm *sessionManager) doNotify(notification interface{}) {
 	for _, sess := range subsCopy {
 		sess.doNotify(notification)
 	}
-
-	// Produce secondary hashXNotification(s) corresponding to the headerNotification.
-	switch note := notification.(type) {
-	case headerNotification:
-		touched, err := sm.db.GetTouchedHashXs(uint32(note.Height))
-		if err != nil {
-			log.Errorf("failed to get touched hashXs at height %v, error: %v", note.Height, err)
-			break
-		}
-		for _, hashX := range touched {
-			hashXstatus, err := sm.db.GetStatus(hashX)
-			if err != nil {
-				log.Errorf("failed to get status of hashX %v, error: %v", hashX, err)
-				continue
-			}
-			note2 := hashXNotification{}
-			copy(note2.hashX[:], hashX)
-			note2.status = hashXstatus
-			sm.doNotify(note2)
-		}
-	}
 }
 
-type sessionServerCodec struct {
+type SessionServerCodec struct {
 	rpc.ServerCodec
 	sess *session
 }
 
 // ReadRequestHeader provides ability to rewrite the incoming
 // request "method" field. For example:
-//
-//	blockchain.block.get_header -> blockchain.block.Get_header
-//	blockchain.address.listunspent -> blockchain.address.Listunspent
-//
+//     blockchain.block.get_header -> blockchain.block.Get_header
+//     blockchain.address.listunspent -> blockchain.address.Listunspent
 // This makes the "method" string compatible with rpc.Server
 // requirements.
-func (c *sessionServerCodec) ReadRequestHeader(req *rpc.Request) error {
-	log.Infof("from %v receive header", c.sess.addr.String())
+func (c *SessionServerCodec) ReadRequestHeader(req *rpc.Request) error {
+	log.Infof("receive header from %v", c.sess.addr.String())
 	err := c.ServerCodec.ReadRequestHeader(req)
 	if err != nil {
 		log.Warnf("error: %v", err)
 		return err
 	}
-	log.Infof("from %v receive header: %#v", c.sess.addr.String(), *req)
 	rawMethod := req.ServiceMethod
 	parts := strings.Split(rawMethod, ".")
 	if len(parts) < 2 {
@@ -467,21 +358,20 @@ func (c *sessionServerCodec) ReadRequestHeader(req *rpc.Request) error {
 }
 
 // ReadRequestBody wraps the regular implementation, but updates session stats too.
-func (c *sessionServerCodec) ReadRequestBody(params any) error {
-	log.Infof("from %v receive body", c.sess.addr.String())
+func (c *SessionServerCodec) ReadRequestBody(params any) error {
 	err := c.ServerCodec.ReadRequestBody(params)
 	if err != nil {
 		log.Warnf("error: %v", err)
 		return err
 	}
-	log.Infof("from %v receive body: %#v", c.sess.addr.String(), params)
+	log.Infof("receive body from %v", c.sess.addr.String())
 	// Bump last receive time.
 	c.sess.lastRecv = time.Now()
 	return err
 }
 
 // WriteResponse wraps the regular implementation, but updates session stats too.
-func (c *sessionServerCodec) WriteResponse(resp *rpc.Response, reply any) error {
+func (c *SessionServerCodec) WriteResponse(resp *rpc.Response, reply any) error {
 	log.Infof("respond to %v", c.sess.addr.String())
 	err := c.ServerCodec.WriteResponse(resp, reply)
 	if err != nil {
@@ -491,141 +381,3 @@ func (c *sessionServerCodec) WriteResponse(resp *rpc.Response, reply any) error
 	c.sess.lastSend = time.Now()
 	return err
 }
-
-// serverRequest is a duplicate of serverRequest from
-// net/rpc/jsonrpc/server.go  with an added Version which
-// we can check.
-type serverRequest struct {
-	Version string           `json:"jsonrpc"`
-	Method  string           `json:"method"`
-	Params  *json.RawMessage `json:"params"`
-	Id      *json.RawMessage `json:"id"`
-}
-
-// serverResponse is a duplicate of serverResponse from
-// net/rpc/jsonrpc/server.go with an added Version which
-// we can set at will.
-type serverResponse struct {
-	Version string           `json:"jsonrpc"`
-	Id      *json.RawMessage `json:"id"`
-	Result  any              `json:"result,omitempty"`
-	Error   any              `json:"error,omitempty"`
-}
-
-// jsonPatchingCodec is able to intercept the JSON requests/responses
-// and tweak them. Currently, it appears we need to make several changes:
-// 1) add "jsonrpc": "2.0" (or "jsonrpc": "1.0") in response
-// 2) add newline to frame response
-// 3) add "params": [] when "params" is missing
-// 4) replace params ["arg1", "arg2", ...] with [["arg1", "arg2", ...]]
-type jsonPatchingCodec struct {
-	conn      net.Conn
-	inBuffer  *bytes.Buffer
-	dec       *json.Decoder
-	enc       *json.Encoder
-	outBuffer *bytes.Buffer
-}
-
-func newJsonPatchingCodec(conn net.Conn) *jsonPatchingCodec {
-	buf1, buf2 := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
-	return &jsonPatchingCodec{
-		conn:      conn,
-		inBuffer:  buf1,
-		dec:       json.NewDecoder(buf1),
-		enc:       json.NewEncoder(buf2),
-		outBuffer: buf2,
-	}
-}
-
-func (c *jsonPatchingCodec) Read(p []byte) (n int, err error) {
-	if c.outBuffer.Len() > 0 {
-		// Return remaining decoded bytes.
-		return c.outBuffer.Read(p)
-	}
-	// Buffer contents consumed. Try to decode more JSON.
-
-	// Read until framing newline. This allows us to print the raw request.
-	for !bytes.ContainsAny(c.inBuffer.Bytes(), "\n") {
-		var buf [1024]byte
-		n, err = c.conn.Read(buf[:])
-		if err != nil {
-			return 0, err
-		}
-		c.inBuffer.Write(buf[:n])
-	}
-	log.Infof("raw request: %v", c.inBuffer.String())
-
-	var req serverRequest
-	err = c.dec.Decode(&req)
-	if err != nil {
-		return 0, err
-	}
-
-	if req.Params != nil {
-		n := len(*req.Params)
-		if n < 2 || (*req.Params)[0] != '[' && (*req.Params)[n-1] != ']' {
-			// This is an error, but we're not going to try to correct it.
-			goto encode
-		}
-		// FIXME: The heuristics here don't cover all possibilities.
-		// For example: [{obj1}, {obj2}] or ["foo,bar"] would not
-		// be handled correctly.
-		bracketed := (*req.Params)[1 : n-1]
-		n = len(bracketed)
-		if n > 1 && (bracketed[0] == '{' || bracketed[0] == '[') {
-			// Probable single object or list argument.
-			goto encode
-		}
-		// The params look like ["arg1", "arg2", "arg3", ...].
-		// We're in trouble because our jsonrpc library does not
-		// handle this. So pack these args in an inner list.
-		// The handler method will receive ONE list argument.
-		params := json.RawMessage(fmt.Sprintf("[[%s]]", bracketed))
-		req.Params = &params
-	} else {
-		// Add empty argument list if params omitted.
-		params := json.RawMessage("[]")
-		req.Params = &params
-	}
-
-encode:
-	// Encode the request. This allows us to print the patched request.
-	buf, err := json.Marshal(req)
-	if err != nil {
-		return 0, err
-	}
-	log.Infof("patched request: %v", string(buf))
-
-	err = c.enc.Encode(req)
-	if err != nil {
-		return 0, err
-	}
-	return c.outBuffer.Read(p)
-}
-
-func (c *jsonPatchingCodec) Write(p []byte) (n int, err error) {
-	log.Infof("raw response: %v", string(p))
-	var resp serverResponse
-	err = json.Unmarshal(p, &resp)
-	if err != nil {
-		return 0, err
-	}
-
-	// Add "jsonrpc": "2.0" if missing.
-	if len(resp.Version) == 0 {
-		resp.Version = "2.0"
-	}
-
-	buf, err := json.Marshal(resp)
-	if err != nil {
-		return 0, err
-	}
-	log.Infof("patched response: %v", string(buf))
-
-	// Add newline for framing.
-	return c.conn.Write(append(buf, '\n'))
-}
-
-func (c *jsonPatchingCodec) Close() error {
-	return c.conn.Close()
-}

server/udp.go

@@ -219,9 +219,8 @@ func UDPPing(ip, port string) (*SPVPong, error) {
 // UDPServer is a goroutine that starts an udp server that implements the hubs
 // Ping/Pong protocol to find out about each other without making full TCP
 // connections.
-func (s *Server) UDPServer(port int) error {
-	address := ":" + strconv.Itoa(port)
-
+func (s *Server) UDPServer() error {
+	address := ":" + s.Args.Port
 	tip := make([]byte, 32)
 	addr, err := net.ResolveUDPAddr("udp", address)
 	if err != nil {

server/udp_test.go

@@ -11,7 +11,7 @@ import (
 
 // TestUDPPing tests UDPPing correctness against prod server.
 func TestUDPPing(t *testing.T) {
-	args := server.MakeDefaultTestArgs()
+	args := makeDefaultArgs()
 	args.DisableStartUDP = true
 
 	tests := []struct {

signal.go

@@ -8,13 +8,12 @@ import (
 	"os"
 	"os/signal"
 
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	log "github.com/sirupsen/logrus"
 )
 
 // shutdownRequestChannel is used to initiate shutdown from one of the
 // subsystems using the same code paths as when an interrupt signal is received.
-var shutdownRequestChannel = make(stop.Chan)
+var shutdownRequestChannel = make(chan struct{})
 
 // interruptSignals defines the default signals to catch in order to do a proper
 // shutdown.  This may be modified during init depending on the platform.
@@ -49,6 +48,7 @@ func interruptListener() <-chan struct{} {
 			case sig := <-interruptChannel:
 				log.Infof("Received signal (%s).  Already "+
 					"shutting down...", sig)
+
 			case <-shutdownRequestChannel:
 				log.Info("Shutdown requested.  Already " +
 					"shutting down...")

testdata/f.csv

@@ -1,21 +1,21 @@
 f,
-660d649ba1defa4ab5ab71f8,919be5811844077f4660af66afa9a59a5ad17cf5c541524e780fe2137bfa250c
-6623c6895027f70a5330bbcb,8dadcde1a6f676d4004eacd399f825006ddf136d1e92b1c92113377b3e1741b4
-664f095b24484ebce8f31fbf,c0c4a751f569c1f9c01531f57ba674b2ad2338d9c08f9e9fc85b0209d15466b2
-665201a38de7d7243df717c9,d9293577cc0d51fe3a5bee78fea9b2b2222e6c2aa0d26a4ef4bfb7dd095587e8
-665328b2449e537b0ca4733f,624f80a361e47c7eb1b815e8714a40f67b4f642a5546547a3fcb5bf5593d8fab
-665ec882021f55b1fbaa5fad,1e917fbc04385290d654f711bdef12773dd54b6b5ea26fe2a9d58ed051f2cb7f
-6671c131cd433750ba6d3908,a2ebfbdf7a23024c340a45f201645aa46f48bc1fdd8d34ed83fcffbf1ee90523
-667fb93d9ae877ba11f337f2,4710649e06619e13250754937e9c17c20b07434751171aac2f2f78b184aa0146
-668ed5f39a5db059dc326137,8dd8ca749b87f43e290904749a546fe319c9d53e765f065bb8beb234a117655e
-66951782f6ba94f2b71e46d0,4f5c9434dd0886c57c2530991cebd973e1b50d5ba8fcfc019e54561217a49bbb
-66970565dfe2b01cad49b73a,f6ca0ae18c896d9bc97c5a9d0c3a06256485f59c77fb91780b213f933b80f48b
-669f6a30a6712062da0cc271,5c6604bfd63b871daceb7893dd618850458974fe4108871c1a1323fb8ae34e4e
-66a9a7b89b78553592acf3df,0561f28c3a5ea0027ecb3c53fa068772a6b7cb73d23104a14f9aba8cd1f070a2
-66aba81567ba48f001f843f0,b0f6ae2c1db8263f7e11fc79423109e718d1f3c30bd123c4243401b5e4f1fee6
-66b569cc3d28be4466fb28d1,ecee392ad8217f325508ba38d280436fb0a520b79a9627e5e18197bf55540885
-66d4662cd100d66055917d63,5762a8ac767fa30d2ca76db7081f8a2e4f5da4f0bf92d29e1322da9a154cc3d6
-66d6fa6ac71d0255dd3f185d,5fc193e5e51b3bd8e95f4eb9df63236da7abf678fc47c0b339ceb5c127d0f488
-66e5b6c7c231a02a32eedd83,58c70ffbfada12550f24bf7931cee06eb2e267dec3560e2e46843e383415f163
-66e673cce02c2163f756491e,b8db43d1f6e62361e2e3b8fa765f79c08ddfb3035caa06f8250d6d1b063a7140
-66fc4ad75184e6029c805d94,fc7ac5e785f73732d95183d6bdc3423d41a074fc3f04b1304bae1efa652edde1
+660d649ba1defa4ab5ab71f8a977d7f7cedb11056e,919be5811844077f4660af66afa9a59a5ad17cf5c541524e780fe2137bfa250c
+6623c6895027f70a5330bbcb1153d635abcb4d5224,8dadcde1a6f676d4004eacd399f825006ddf136d1e92b1c92113377b3e1741b4
+664f095b24484ebce8f31fbf008e63cc4aa163d401,c0c4a751f569c1f9c01531f57ba674b2ad2338d9c08f9e9fc85b0209d15466b2
+665201a38de7d7243df717c9f9279cdd30105f0f77,d9293577cc0d51fe3a5bee78fea9b2b2222e6c2aa0d26a4ef4bfb7dd095587e8
+665328b2449e537b0ca4733f87ac5ebcdf033c5ebd,624f80a361e47c7eb1b815e8714a40f67b4f642a5546547a3fcb5bf5593d8fab
+665ec882021f55b1fbaa5fad00df5c5d07633b7af3,1e917fbc04385290d654f711bdef12773dd54b6b5ea26fe2a9d58ed051f2cb7f
+6671c131cd433750ba6d3908150ca4910841164b74,a2ebfbdf7a23024c340a45f201645aa46f48bc1fdd8d34ed83fcffbf1ee90523
+667fb93d9ae877ba11f337f21422b0679852580802,4710649e06619e13250754937e9c17c20b07434751171aac2f2f78b184aa0146
+668ed5f39a5db059dc3261377f2a47728f7a357d33,8dd8ca749b87f43e290904749a546fe319c9d53e765f065bb8beb234a117655e
+66951782f6ba94f2b71e46d0cc4a2411b14d81eb70,4f5c9434dd0886c57c2530991cebd973e1b50d5ba8fcfc019e54561217a49bbb
+66970565dfe2b01cad49b73a085a3c3f7a3be61c4c,f6ca0ae18c896d9bc97c5a9d0c3a06256485f59c77fb91780b213f933b80f48b
+669f6a30a6712062da0cc27181845c04d7430abf73,5c6604bfd63b871daceb7893dd618850458974fe4108871c1a1323fb8ae34e4e
+66a9a7b89b78553592acf3dfc417c1d7654dab3273,0561f28c3a5ea0027ecb3c53fa068772a6b7cb73d23104a14f9aba8cd1f070a2
+66aba81567ba48f001f843f01354d575c2e2687847,b0f6ae2c1db8263f7e11fc79423109e718d1f3c30bd123c4243401b5e4f1fee6
+66b569cc3d28be4466fb28d147f66d6d8769598964,ecee392ad8217f325508ba38d280436fb0a520b79a9627e5e18197bf55540885
+66d4662cd100d66055917d6342d48f49d948fcc255,5762a8ac767fa30d2ca76db7081f8a2e4f5da4f0bf92d29e1322da9a154cc3d6
+66d6fa6ac71d0255dd3f185de6480d5b4316b6b050,5fc193e5e51b3bd8e95f4eb9df63236da7abf678fc47c0b339ceb5c127d0f488
+66e5b6c7c231a02a32eedd8383a5750fd135244a03,58c70ffbfada12550f24bf7931cee06eb2e267dec3560e2e46843e383415f163
+66e673cce02c2163f756491ef05d7535ceb578e215,b8db43d1f6e62361e2e3b8fa765f79c08ddfb3035caa06f8250d6d1b063a7140
+66fc4ad75184e6029c805d9494eed4e81be770c002,fc7ac5e785f73732d95183d6bdc3423d41a074fc3f04b1304bae1efa652edde1

testdata/g.csv

@@ -1,21 +1,21 @@
 g,
-6702c124856d5168381a3297,575696fd653a4de2f9a8c1f580cf0c229631b0f5d95fceb354cda133e2eb2d34
-6707f1511e3a2cb28493f91b,ba368e0f859ee36da8701df1c0b52cbf0c0f8a4b1a91f6d0db83a408f5a937d1
-6707fd4213cae8d5342a98ba,bd3a44d30f66444f8732119bc7e0cf0bb47f8f0ab2840987fc06b629f3e6d3f4
-6710294a5693224a6222404b,de35a8ea0a26d17445e2f509db23188961b5cd1229b96d2411565adf63731b5c
-6716a9f84e02143b50d9034a,5823640ae4529f8df2dab20386c887d0a1ba1ffa4583b99dff761c01f670c2fa
-672e51bc65c9b97d482b0b72,0687df449bd8cb8d8f526f4189973d084d786ab0927d81c127f56b03c61aa955
-67682620db65932047689e5e,b262d40758edb28d1c04fa3a24d8268990516de6846ad94d002ce55640866239
-676e8c320dbbf5eebc2969a9,c9e2a8e7181a70e2a488b884c8baadb4043a075c6876cb012c67fbec5aa9f615
-6772e2ac48891ee3c2c72783,985a9c9ee7a0626d78dab431e663289762ce6959be314f91f7b08b1466097fd6
-67847dd1dac117b85d1e20d9,62e6b1b8c2961703a90276dcde6dad182b2d14e23f27dccc927cca7770b9890e
-678f49948c72b7295f12092a,2e7c456dac5206c5627736924e96ac016a09a88ec5f4835fbe0cf9e294611c88
-67948b9633ab2ec07d752593,66b5c54b3a685de3ea18f9e69254eec065eb3207ac1f93494fdcd585e9a267a0
-679674c162db8d3bb57c434f,05425880d80258f7441859b3494415a3fd7398c9e209a19674abd48372b283c6
-67a8d3f17df85502bd644a36,1efce69a3a05c505e9f9cc5c2241d02099c043d934389b430fd8b185e6dfe6cb
-67bad7f4fb3c6828b6fc4624,04a1c0a7ffe7acbf974ca18cf3debbd8e1be3d6703f842f57ef14af6d4c336d3
-67c13fb0c65acca5520bc2f5,7fdc6989cd778baad45cd98358ea060237b169a4aeaeb14da6ac4686b7858c9f
-67d4314588b4424b0ee02653,c63fd7a85a533b8591577bab805104708ba5458fab0e343d46b3e24a28b92cb5
-67d734244f85f32a58e34e2d,d19a6307c24470b3973973319770bdb896218bb58d1f2d07c7226266075057d0
-67d9c159c5d5e407e6b0a4ca,89cbdb903fdfe0b44e74b0a69eed3de7029f18c28f77e5509f8ace766ab86610
-67fafc73d674250f11e559ab,1752ffbf9807bb2e4e480bf045b4bacc472befe755287384b5a526065a58c065
+6702c124856d5168381a32971d8933440a1728fc41,575696fd653a4de2f9a8c1f580cf0c229631b0f5d95fceb354cda133e2eb2d34
+6707f1511e3a2cb28493f91b85e9e4a9d9d07c86a5,ba368e0f859ee36da8701df1c0b52cbf0c0f8a4b1a91f6d0db83a408f5a937d1
+6707fd4213cae8d5342a98ba49b255fa80b2a9a6e4,bd3a44d30f66444f8732119bc7e0cf0bb47f8f0ab2840987fc06b629f3e6d3f4
+6710294a5693224a6222404ba45fd38eb2e77979a4,de35a8ea0a26d17445e2f509db23188961b5cd1229b96d2411565adf63731b5c
+6716a9f84e02143b50d9034aec126b12d7f2708cc4,5823640ae4529f8df2dab20386c887d0a1ba1ffa4583b99dff761c01f670c2fa
+672e51bc65c9b97d482b0b720e6cb673c41fe7b5c5,0687df449bd8cb8d8f526f4189973d084d786ab0927d81c127f56b03c61aa955
+67682620db65932047689e5eaf392d6b85be801864,b262d40758edb28d1c04fa3a24d8268990516de6846ad94d002ce55640866239
+676e8c320dbbf5eebc2969a93fbc51dd7f6062a7d1,c9e2a8e7181a70e2a488b884c8baadb4043a075c6876cb012c67fbec5aa9f615
+6772e2ac48891ee3c2c727835702a374ad0cb70fd6,985a9c9ee7a0626d78dab431e663289762ce6959be314f91f7b08b1466097fd6
+67847dd1dac117b85d1e20d93580cdf42f00001a77,62e6b1b8c2961703a90276dcde6dad182b2d14e23f27dccc927cca7770b9890e
+678f49948c72b7295f12092a24d300eeff894f1dd7,2e7c456dac5206c5627736924e96ac016a09a88ec5f4835fbe0cf9e294611c88
+67948b9633ab2ec07d7525936254e66f8c957d026c,66b5c54b3a685de3ea18f9e69254eec065eb3207ac1f93494fdcd585e9a267a0
+679674c162db8d3bb57c434fe87825625c4d4daf63,05425880d80258f7441859b3494415a3fd7398c9e209a19674abd48372b283c6
+67a8d3f17df85502bd644a364721e6364d61635b73,1efce69a3a05c505e9f9cc5c2241d02099c043d934389b430fd8b185e6dfe6cb
+67bad7f4fb3c6828b6fc4624d43786fc8f55d6eb0f,04a1c0a7ffe7acbf974ca18cf3debbd8e1be3d6703f842f57ef14af6d4c336d3
+67c13fb0c65acca5520bc2f59bd91ca3482dbec156,7fdc6989cd778baad45cd98358ea060237b169a4aeaeb14da6ac4686b7858c9f
+67d4314588b4424b0ee026536b9bd7857f11cab2ee,c63fd7a85a533b8591577bab805104708ba5458fab0e343d46b3e24a28b92cb5
+67d734244f85f32a58e34e2d9cadf225a56973d32f,d19a6307c24470b3973973319770bdb896218bb58d1f2d07c7226266075057d0
+67d9c159c5d5e407e6b0a4cacf9d6fe62a55b0fedc,89cbdb903fdfe0b44e74b0a69eed3de7029f18c28f77e5509f8ace766ab86610
+67fafc73d674250f11e559ab08b287f5714e531761,1752ffbf9807bb2e4e480bf045b4bacc472befe755287384b5a526065a58c065

version.txt

@@ -1 +1 @@
-v0.2022.10.05.1
+v0.2022.08.16.1