Compare commits
No commits in common. "master" and "blockchain_rpc1" have entirely different histories.
master
...
blockchain
39 changed files with 688 additions and 3260 deletions
2
.github/workflows/build-short.yml
vendored
2
.github/workflows/build-short.yml
vendored
|
@ -7,6 +7,6 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
- name: Build and Test
|
||||
uses: ./
|
||||
|
|
82
db/db.go
82
db/db.go
|
@ -15,7 +15,6 @@ import (
|
|||
"github.com/lbryio/herald.go/internal"
|
||||
"github.com/lbryio/herald.go/internal/metrics"
|
||||
pb "github.com/lbryio/herald.go/protobuf/go"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
"github.com/linxGnu/grocksdb"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
@ -59,7 +58,8 @@ type ReadOnlyDBColumnFamily struct {
|
|||
BlockedChannels map[string][]byte
|
||||
FilteredStreams map[string][]byte
|
||||
FilteredChannels map[string][]byte
|
||||
Grp *stop.Group
|
||||
ShutdownChan chan struct{}
|
||||
DoneChan chan struct{}
|
||||
Cleanup func()
|
||||
}
|
||||
|
||||
|
@ -318,29 +318,9 @@ func intMin(a, b int) int {
|
|||
return b
|
||||
}
|
||||
|
||||
// FIXME: This was copied from the signal.go file, maybe move it to a more common place?
|
||||
// interruptRequested returns true when the channel returned by
|
||||
// interruptListener was closed. This simplifies early shutdown slightly since
|
||||
// the caller can just use an if statement instead of a select.
|
||||
func interruptRequested(interrupted <-chan struct{}) bool {
|
||||
select {
|
||||
case <-interrupted:
|
||||
return true
|
||||
default:
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func IterCF(db *grocksdb.DB, opts *IterOptions) <-chan *prefixes.PrefixRowKV {
|
||||
ch := make(chan *prefixes.PrefixRowKV)
|
||||
|
||||
// Check if we've been told to shutdown in between getting created and getting here
|
||||
if opts.Grp != nil && interruptRequested(opts.Grp.Ch()) {
|
||||
opts.Grp.Done()
|
||||
return ch
|
||||
}
|
||||
|
||||
ro := grocksdb.NewDefaultReadOptions()
|
||||
ro.SetFillCache(opts.FillCache)
|
||||
it := db.NewIteratorCF(ro, opts.CfHandle)
|
||||
|
@ -356,10 +336,6 @@ func IterCF(db *grocksdb.DB, opts *IterOptions) <-chan *prefixes.PrefixRowKV {
|
|||
it.Close()
|
||||
close(ch)
|
||||
ro.Destroy()
|
||||
if opts.Grp != nil {
|
||||
// opts.Grp.DoneNamed(iterKey)
|
||||
opts.Grp.Done()
|
||||
}
|
||||
}()
|
||||
|
||||
var prevKey []byte
|
||||
|
@ -379,9 +355,6 @@ func IterCF(db *grocksdb.DB, opts *IterOptions) <-chan *prefixes.PrefixRowKV {
|
|||
if kv = opts.ReadRow(&prevKey); kv != nil {
|
||||
ch <- kv
|
||||
}
|
||||
if opts.Grp != nil && interruptRequested(opts.Grp.Ch()) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
|
@ -439,10 +412,10 @@ func (db *ReadOnlyDBColumnFamily) selectFrom(prefix []byte, startKey, stopKey pr
|
|||
return nil, err
|
||||
}
|
||||
// Prefix and handle
|
||||
options := NewIterateOptions().WithDB(db).WithPrefix(prefix).WithCfHandle(handle)
|
||||
options := NewIterateOptions().WithPrefix(prefix).WithCfHandle(handle)
|
||||
// Start and stop bounds
|
||||
options = options.WithStart(startKey.PackKey()).WithStop(stopKey.PackKey()).WithIncludeStop(false)
|
||||
// Include the key and value
|
||||
options = options.WithStart(startKey.PackKey()).WithStop(stopKey.PackKey()).WithIncludeStop(true)
|
||||
// Don't include the key
|
||||
options = options.WithIncludeKey(true).WithIncludeValue(true)
|
||||
return []*IterOptions{options}, nil
|
||||
}
|
||||
|
@ -455,7 +428,7 @@ func iterate(db *grocksdb.DB, opts []*IterOptions) <-chan []*prefixes.PrefixRowK
|
|||
for kv := range IterCF(db, o) {
|
||||
row := make([]*prefixes.PrefixRowKV, 0, 1)
|
||||
row = append(row, kv)
|
||||
log.Debugf("iterate[%v][%v] %#v -> %#v", i, j, kv.Key, kv.Value)
|
||||
log.Debugf("iterate[%v][%v] %#v", i, j, kv)
|
||||
out <- row
|
||||
j++
|
||||
}
|
||||
|
@ -481,7 +454,7 @@ func innerJoin(db *grocksdb.DB, in <-chan []*prefixes.PrefixRowKV, selectFn func
|
|||
row = append(row, kvs...)
|
||||
row = append(row, kv...)
|
||||
for i, kv := range row {
|
||||
log.Debugf("row[%v] %#v -> %#v", i, kv.Key, kv.Value)
|
||||
log.Debugf("row[%v] %#v", i, kv)
|
||||
}
|
||||
out <- row
|
||||
}
|
||||
|
@ -531,7 +504,7 @@ func GetWriteDBCF(name string) (*grocksdb.DB, []*grocksdb.ColumnFamilyHandle, er
|
|||
}
|
||||
|
||||
// GetProdDB returns a db that is used for production.
|
||||
func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBColumnFamily, error) {
|
||||
func GetProdDB(name string, secondaryPath string) (*ReadOnlyDBColumnFamily, func(), error) {
|
||||
prefixNames := prefixes.GetPrefixes()
|
||||
// additional prefixes that aren't in the code explicitly
|
||||
cfNames := []string{"default", "e", "d", "c"}
|
||||
|
@ -540,7 +513,7 @@ func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBC
|
|||
cfNames = append(cfNames, cfName)
|
||||
}
|
||||
|
||||
db, err := GetDBColumnFamilies(name, secondaryPath, cfNames, grp)
|
||||
db, err := GetDBColumnFamilies(name, secondaryPath, cfNames)
|
||||
|
||||
cleanupFiles := func() {
|
||||
err = os.RemoveAll(secondaryPath)
|
||||
|
@ -550,8 +523,7 @@ func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBC
|
|||
}
|
||||
|
||||
if err != nil {
|
||||
cleanupFiles()
|
||||
return nil, err
|
||||
return nil, cleanupFiles, err
|
||||
}
|
||||
|
||||
cleanupDB := func() {
|
||||
|
@ -560,11 +532,11 @@ func GetProdDB(name string, secondaryPath string, grp *stop.Group) (*ReadOnlyDBC
|
|||
}
|
||||
db.Cleanup = cleanupDB
|
||||
|
||||
return db, nil
|
||||
return db, cleanupDB, nil
|
||||
}
|
||||
|
||||
// GetDBColumnFamilies gets a db with the specified column families and secondary path.
|
||||
func GetDBColumnFamilies(name string, secondayPath string, cfNames []string, grp *stop.Group) (*ReadOnlyDBColumnFamily, error) {
|
||||
func GetDBColumnFamilies(name string, secondayPath string, cfNames []string) (*ReadOnlyDBColumnFamily, error) {
|
||||
opts := grocksdb.NewDefaultOptions()
|
||||
roOpts := grocksdb.NewDefaultReadOptions()
|
||||
cfOpt := grocksdb.NewDefaultOptions()
|
||||
|
@ -579,7 +551,6 @@ func GetDBColumnFamilies(name string, secondayPath string, cfNames []string, grp
|
|||
// db, handles, err := grocksdb.OpenDbColumnFamilies(opts, name, cfNames, cfOpts)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("open db as secondary failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
@ -601,7 +572,8 @@ func GetDBColumnFamilies(name string, secondayPath string, cfNames []string, grp
|
|||
LastState: nil,
|
||||
Height: 0,
|
||||
Headers: nil,
|
||||
Grp: grp,
|
||||
ShutdownChan: make(chan struct{}),
|
||||
DoneChan: make(chan struct{}),
|
||||
}
|
||||
|
||||
err = myDB.ReadDBState() //TODO: Figure out right place for this
|
||||
|
@ -670,22 +642,21 @@ func (db *ReadOnlyDBColumnFamily) Unwind() {
|
|||
|
||||
// Shutdown shuts down the db.
|
||||
func (db *ReadOnlyDBColumnFamily) Shutdown() {
|
||||
log.Println("Calling cleanup...")
|
||||
db.ShutdownChan <- struct{}{}
|
||||
<-db.DoneChan
|
||||
db.Cleanup()
|
||||
log.Println("Leaving Shutdown...")
|
||||
}
|
||||
|
||||
// RunDetectChanges Go routine the runs continuously while the hub is active
|
||||
// to keep the db readonly view up to date and handle reorgs on the
|
||||
// blockchain.
|
||||
func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan<- interface{}) {
|
||||
db.Grp.Add(1)
|
||||
func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan *internal.HeightHash) {
|
||||
go func() {
|
||||
lastPrint := time.Now()
|
||||
for {
|
||||
// FIXME: Figure out best sleep interval
|
||||
if time.Since(lastPrint) > time.Second {
|
||||
log.Debugf("DetectChanges: %#v", db.LastState)
|
||||
log.Debug("DetectChanges:", db.LastState)
|
||||
lastPrint = time.Now()
|
||||
}
|
||||
err := db.detectChanges(notifCh)
|
||||
|
@ -693,8 +664,8 @@ func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan<- interface{}) {
|
|||
log.Infof("Error detecting changes: %#v", err)
|
||||
}
|
||||
select {
|
||||
case <-db.Grp.Ch():
|
||||
db.Grp.Done()
|
||||
case <-db.ShutdownChan:
|
||||
db.DoneChan <- struct{}{}
|
||||
return
|
||||
case <-time.After(time.Millisecond * 10):
|
||||
}
|
||||
|
@ -703,7 +674,7 @@ func (db *ReadOnlyDBColumnFamily) RunDetectChanges(notifCh chan<- interface{}) {
|
|||
}
|
||||
|
||||
// DetectChanges keep the rocksdb db in sync and handle reorgs
|
||||
func (db *ReadOnlyDBColumnFamily) detectChanges(notifCh chan<- interface{}) error {
|
||||
func (db *ReadOnlyDBColumnFamily) detectChanges(notifCh chan *internal.HeightHash) error {
|
||||
err := db.DB.TryCatchUpWithPrimary()
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -775,12 +746,7 @@ func (db *ReadOnlyDBColumnFamily) detectChanges(notifCh chan<- interface{}) erro
|
|||
log.Info("error getting block hash: ", err)
|
||||
return err
|
||||
}
|
||||
header, err := db.GetHeader(height)
|
||||
if err != nil {
|
||||
log.Info("error getting block header: ", err)
|
||||
return err
|
||||
}
|
||||
notifCh <- &internal.HeightHash{Height: uint64(height), BlockHash: hash, BlockHeader: header}
|
||||
notifCh <- &internal.HeightHash{Height: uint64(height), BlockHash: hash}
|
||||
}
|
||||
//TODO: ClearCache
|
||||
log.Warn("implement cache clearing")
|
||||
|
@ -824,7 +790,7 @@ func (db *ReadOnlyDBColumnFamily) InitHeaders() error {
|
|||
// endKey := prefixes.NewHeaderKey(db.LastState.Height)
|
||||
startKeyRaw := startKey.PackKey()
|
||||
// endKeyRaw := endKey.PackKey()
|
||||
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
|
||||
options := NewIterateOptions().WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
|
||||
options = options.WithIncludeKey(false).WithIncludeValue(true) //.WithIncludeStop(true)
|
||||
options = options.WithStart(startKeyRaw) //.WithStop(endKeyRaw)
|
||||
|
||||
|
@ -847,7 +813,7 @@ func (db *ReadOnlyDBColumnFamily) InitTxCounts() error {
|
|||
|
||||
db.TxCounts = stack.NewSliceBacked[uint32](InitialTxCountSize)
|
||||
|
||||
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.TxCount}).WithCfHandle(handle)
|
||||
options := NewIterateOptions().WithPrefix([]byte{prefixes.TxCount}).WithCfHandle(handle)
|
||||
options = options.WithIncludeKey(false).WithIncludeValue(true).WithIncludeStop(true)
|
||||
|
||||
ch := IterCF(db.DB, options)
|
||||
|
|
384
db/db_get.go
384
db/db_get.go
|
@ -3,19 +3,15 @@ package db
|
|||
// db_get.go contains the basic access functions to the database.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
|
||||
"github.com/lbryio/herald.go/db/prefixes"
|
||||
"github.com/lbryio/herald.go/db/stack"
|
||||
"github.com/lbryio/lbcd/chaincfg/chainhash"
|
||||
"github.com/lbryio/lbcd/wire"
|
||||
"github.com/linxGnu/grocksdb"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// GetExpirationHeight returns the expiration height for the given height. Uses
|
||||
|
@ -68,57 +64,6 @@ func (db *ReadOnlyDBColumnFamily) GetBlockHash(height uint32) ([]byte, error) {
|
|||
return rawValue, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetBlockTXs(height uint32) ([]*chainhash.Hash, error) {
|
||||
handle, err := db.EnsureHandle(prefixes.BlockTXs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key := prefixes.BlockTxsKey{
|
||||
Prefix: []byte{prefixes.BlockTXs},
|
||||
Height: height,
|
||||
}
|
||||
slice, err := db.DB.GetCF(db.Opts, handle, key.PackKey())
|
||||
defer slice.Free()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if slice.Size() == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rawValue := make([]byte, len(slice.Data()))
|
||||
copy(rawValue, slice.Data())
|
||||
value := prefixes.BlockTxsValueUnpack(rawValue)
|
||||
return value.TxHashes, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetTouchedHashXs(height uint32) ([][]byte, error) {
|
||||
handle, err := db.EnsureHandle(prefixes.TouchedHashX)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key := prefixes.TouchedHashXKey{
|
||||
Prefix: []byte{prefixes.TouchedHashX},
|
||||
Height: height,
|
||||
}
|
||||
slice, err := db.DB.GetCF(db.Opts, handle, key.PackKey())
|
||||
defer slice.Free()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if slice.Size() == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rawValue := make([]byte, len(slice.Data()))
|
||||
copy(rawValue, slice.Data())
|
||||
value := prefixes.TouchedHashXValue{}
|
||||
value.UnpackValue(rawValue)
|
||||
return value.TouchedHashXs, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetHeader(height uint32) ([]byte, error) {
|
||||
handle, err := db.EnsureHandle(prefixes.Header)
|
||||
if err != nil {
|
||||
|
@ -148,7 +93,7 @@ func (db *ReadOnlyDBColumnFamily) GetHeaders(height uint32, count uint32) ([][11
|
|||
|
||||
startKeyRaw := prefixes.NewHeaderKey(height).PackKey()
|
||||
endKeyRaw := prefixes.NewHeaderKey(height + count).PackKey()
|
||||
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
|
||||
options := NewIterateOptions().WithPrefix([]byte{prefixes.Header}).WithCfHandle(handle)
|
||||
options = options.WithIncludeKey(false).WithIncludeValue(true) //.WithIncludeStop(true)
|
||||
options = options.WithStart(startKeyRaw).WithStop(endKeyRaw)
|
||||
|
||||
|
@ -184,7 +129,7 @@ func (db *ReadOnlyDBColumnFamily) GetBalance(hashX []byte) (uint64, uint64, erro
|
|||
startKeyRaw := startKey.PackKey()
|
||||
endKeyRaw := endKey.PackKey()
|
||||
// Prefix and handle
|
||||
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.UTXO}).WithCfHandle(handle)
|
||||
options := NewIterateOptions().WithPrefix([]byte{prefixes.UTXO}).WithCfHandle(handle)
|
||||
// Start and stop bounds
|
||||
options = options.WithStart(startKeyRaw).WithStop(endKeyRaw).WithIncludeStop(true)
|
||||
// Don't include the key
|
||||
|
@ -321,79 +266,6 @@ func (db *ReadOnlyDBColumnFamily) GetHistory(hashX []byte) ([]TxInfo, error) {
|
|||
return results, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetStatus(hashX []byte) ([]byte, error) {
|
||||
// Lookup in HashXMempoolStatus first.
|
||||
status, err := db.getMempoolStatus(hashX)
|
||||
if err == nil && status != nil {
|
||||
log.Debugf("(mempool) status(%#v) -> %#v", hashX, status)
|
||||
return status, err
|
||||
}
|
||||
|
||||
// No indexed mempool status. Lookup in HashXStatus second.
|
||||
handle, err := db.EnsureHandle(prefixes.HashXStatus)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := &prefixes.HashXStatusKey{
|
||||
Prefix: []byte{prefixes.HashXStatus},
|
||||
HashX: hashX,
|
||||
}
|
||||
rawKey := key.PackKey()
|
||||
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
|
||||
defer slice.Free()
|
||||
if err == nil && slice.Size() > 0 {
|
||||
rawValue := make([]byte, len(slice.Data()))
|
||||
copy(rawValue, slice.Data())
|
||||
value := prefixes.HashXStatusValue{}
|
||||
value.UnpackValue(rawValue)
|
||||
log.Debugf("status(%#v) -> %#v", hashX, value.Status)
|
||||
return value.Status, nil
|
||||
}
|
||||
|
||||
// No indexed status. Fall back to enumerating HashXHistory.
|
||||
txs, err := db.GetHistory(hashX)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(txs) == 0 {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
hash := sha256.New()
|
||||
for _, tx := range txs {
|
||||
hash.Write([]byte(fmt.Sprintf("%s:%d:", tx.TxHash.String(), tx.Height)))
|
||||
}
|
||||
// TODO: Mempool history
|
||||
return hash.Sum(nil), err
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) getMempoolStatus(hashX []byte) ([]byte, error) {
|
||||
handle, err := db.EnsureHandle(prefixes.HashXMempoolStatus)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key := &prefixes.HashXMempoolStatusKey{
|
||||
Prefix: []byte{prefixes.HashXMempoolStatus},
|
||||
HashX: hashX,
|
||||
}
|
||||
rawKey := key.PackKey()
|
||||
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
|
||||
defer slice.Free()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if slice.Size() == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rawValue := make([]byte, len(slice.Data()))
|
||||
copy(rawValue, slice.Data())
|
||||
value := prefixes.HashXMempoolStatusValue{}
|
||||
value.UnpackValue(rawValue)
|
||||
return value.Status, nil
|
||||
}
|
||||
|
||||
// GetStreamsAndChannelRepostedByChannelHashes returns a map of streams and channel hashes that are reposted by the given channel hashes.
|
||||
func (db *ReadOnlyDBColumnFamily) GetStreamsAndChannelRepostedByChannelHashes(reposterChannelHashes [][]byte) (map[string][]byte, map[string][]byte, error) {
|
||||
handle, err := db.EnsureHandle(prefixes.ChannelToClaim)
|
||||
|
@ -407,7 +279,7 @@ func (db *ReadOnlyDBColumnFamily) GetStreamsAndChannelRepostedByChannelHashes(re
|
|||
for _, reposterChannelHash := range reposterChannelHashes {
|
||||
key := prefixes.NewChannelToClaimKeyWHash(reposterChannelHash)
|
||||
rawKeyPrefix := key.PartialPack(1)
|
||||
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options = options.WithIncludeKey(false).WithIncludeValue(true)
|
||||
ch := IterCF(db.DB, options)
|
||||
// for stream := range Iterate(db.DB, prefixes.ChannelToClaim, []byte{reposterChannelHash}, false) {
|
||||
|
@ -481,7 +353,7 @@ func (db *ReadOnlyDBColumnFamily) GetShortClaimIdUrl(name string, normalizedName
|
|||
log.Printf("partialKey: %#v\n", partialKey)
|
||||
keyPrefix := partialKey.PartialPack(2)
|
||||
// Prefix and handle
|
||||
options := NewIterateOptions().WithDB(db).WithPrefix(prefix).WithCfHandle(handle)
|
||||
options := NewIterateOptions().WithPrefix(prefix).WithCfHandle(handle)
|
||||
// Start and stop bounds
|
||||
options = options.WithStart(keyPrefix).WithStop(keyPrefix)
|
||||
// Don't include the key
|
||||
|
@ -574,12 +446,12 @@ func (db *ReadOnlyDBColumnFamily) GetActiveAmount(claimHash []byte, txoType uint
|
|||
}
|
||||
|
||||
startKey := prefixes.NewActiveAmountKey(claimHash, txoType, 0)
|
||||
endKey := prefixes.NewActiveAmountKey(claimHash, txoType, height+1)
|
||||
endKey := prefixes.NewActiveAmountKey(claimHash, txoType, height)
|
||||
|
||||
startKeyRaw := startKey.PartialPack(3)
|
||||
endKeyRaw := endKey.PartialPack(3)
|
||||
// Prefix and handle
|
||||
options := NewIterateOptions().WithDB(db).WithPrefix([]byte{prefixes.ActiveAmount}).WithCfHandle(handle)
|
||||
options := NewIterateOptions().WithPrefix([]byte{prefixes.ActiveAmount}).WithCfHandle(handle)
|
||||
// Start and stop bounds
|
||||
options = options.WithStart(startKeyRaw).WithStop(endKeyRaw)
|
||||
// Don't include the key
|
||||
|
@ -595,6 +467,14 @@ func (db *ReadOnlyDBColumnFamily) GetActiveAmount(claimHash []byte, txoType uint
|
|||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetEffectiveAmount(claimHash []byte, supportOnly bool) (uint64, error) {
|
||||
if supportOnly {
|
||||
supportAmount, err := db.GetActiveAmount(claimHash, prefixes.ActivatedSupportTXOType, db.Height+1)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return supportAmount, nil
|
||||
}
|
||||
|
||||
handle, err := db.EnsureHandle(prefixes.EffectiveAmount)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
|
@ -612,13 +492,7 @@ func (db *ReadOnlyDBColumnFamily) GetEffectiveAmount(claimHash []byte, supportOn
|
|||
|
||||
value := prefixes.EffectiveAmountValue{}
|
||||
value.UnpackValue(slice.Data())
|
||||
var amount uint64
|
||||
if supportOnly {
|
||||
amount += value.ActivatedSupportSum
|
||||
} else {
|
||||
amount += value.ActivatedSum
|
||||
}
|
||||
return amount, nil
|
||||
return value.EffectiveAmount, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetSupportAmount(claimHash []byte) (uint64, error) {
|
||||
|
@ -735,7 +609,7 @@ func (db *ReadOnlyDBColumnFamily) ControllingClaimIter() <-chan *prefixes.Prefix
|
|||
key := prefixes.NewClaimTakeoverKey("")
|
||||
var rawKeyPrefix []byte = nil
|
||||
rawKeyPrefix = key.PartialPack(0)
|
||||
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options = options.WithIncludeValue(true) //.WithIncludeStop(true)
|
||||
ch := IterCF(db.DB, options)
|
||||
return ch
|
||||
|
@ -792,70 +666,6 @@ func (db *ReadOnlyDBColumnFamily) FsGetClaimByHash(claimHash []byte) (*ResolveRe
|
|||
)
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetTx(txhash *chainhash.Hash) ([]byte, *wire.MsgTx, error) {
|
||||
// Lookup in MempoolTx first.
|
||||
raw, tx, err := db.getMempoolTx(txhash)
|
||||
if err == nil && raw != nil && tx != nil {
|
||||
return raw, tx, err
|
||||
}
|
||||
|
||||
handle, err := db.EnsureHandle(prefixes.Tx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
key := prefixes.TxKey{Prefix: []byte{prefixes.Tx}, TxHash: txhash}
|
||||
rawKey := key.PackKey()
|
||||
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
|
||||
defer slice.Free()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if slice.Size() == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
rawValue := make([]byte, len(slice.Data()))
|
||||
copy(rawValue, slice.Data())
|
||||
value := prefixes.TxValue{}
|
||||
value.UnpackValue(rawValue)
|
||||
var msgTx wire.MsgTx
|
||||
err = msgTx.Deserialize(bytes.NewReader(value.RawTx))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return value.RawTx, &msgTx, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) getMempoolTx(txhash *chainhash.Hash) ([]byte, *wire.MsgTx, error) {
|
||||
handle, err := db.EnsureHandle(prefixes.MempoolTx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
key := prefixes.MempoolTxKey{Prefix: []byte{prefixes.Tx}, TxHash: txhash}
|
||||
rawKey := key.PackKey()
|
||||
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
|
||||
defer slice.Free()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if slice.Size() == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
rawValue := make([]byte, len(slice.Data()))
|
||||
copy(rawValue, slice.Data())
|
||||
value := prefixes.MempoolTxValue{}
|
||||
value.UnpackValue(rawValue)
|
||||
var msgTx wire.MsgTx
|
||||
err = msgTx.Deserialize(bytes.NewReader(value.RawTx))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return value.RawTx, &msgTx, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetTxCount(height uint32) (*prefixes.TxCountValue, error) {
|
||||
handle, err := db.EnsureHandle(prefixes.TxCount)
|
||||
if err != nil {
|
||||
|
@ -879,162 +689,6 @@ func (db *ReadOnlyDBColumnFamily) GetTxCount(height uint32) (*prefixes.TxCountVa
|
|||
return value, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetTxHeight(txhash *chainhash.Hash) (uint32, error) {
|
||||
handle, err := db.EnsureHandle(prefixes.TxNum)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
key := prefixes.TxNumKey{Prefix: []byte{prefixes.TxNum}, TxHash: txhash}
|
||||
rawKey := key.PackKey()
|
||||
slice, err := db.DB.GetCF(db.Opts, handle, rawKey)
|
||||
defer slice.Free()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if slice.Size() == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// No slice copy needed. Value will be abandoned.
|
||||
value := prefixes.TxNumValueUnpack(slice.Data())
|
||||
height := stack.BisectRight(db.TxCounts, []uint32{value.TxNum})[0]
|
||||
return height, nil
|
||||
}
|
||||
|
||||
type TxMerkle struct {
|
||||
TxHash *chainhash.Hash
|
||||
RawTx []byte
|
||||
Height int
|
||||
Pos uint32
|
||||
Merkle []*chainhash.Hash
|
||||
}
|
||||
|
||||
// merklePath selects specific transactions by position within blockTxs.
|
||||
// The resulting merkle path (aka merkle branch, or merkle) is a list of TX hashes
|
||||
// which are in sibling relationship with TX nodes on the path to the root.
|
||||
func merklePath(pos uint32, blockTxs, partial []*chainhash.Hash) []*chainhash.Hash {
|
||||
parent := func(p uint32) uint32 {
|
||||
return p >> 1
|
||||
}
|
||||
sibling := func(p uint32) uint32 {
|
||||
if p%2 == 0 {
|
||||
return p + 1
|
||||
} else {
|
||||
return p - 1
|
||||
}
|
||||
}
|
||||
p := parent(pos)
|
||||
if p == 0 {
|
||||
// No parent, path is complete.
|
||||
return partial
|
||||
}
|
||||
// Add sibling to partial path and proceed to parent TX.
|
||||
return merklePath(p, blockTxs, append(partial, blockTxs[sibling(pos)]))
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetTxMerkle(tx_hashes []chainhash.Hash) ([]TxMerkle, error) {
|
||||
|
||||
selectedTxNum := make([]*IterOptions, 0, len(tx_hashes))
|
||||
for _, txhash := range tx_hashes {
|
||||
key := prefixes.TxNumKey{Prefix: []byte{prefixes.TxNum}, TxHash: &txhash}
|
||||
log.Debugf("%v", key)
|
||||
opt, err := db.selectFrom(key.Prefix, &key, &key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
selectedTxNum = append(selectedTxNum, opt...)
|
||||
}
|
||||
|
||||
selectTxByTxNum := func(in []*prefixes.PrefixRowKV) ([]*IterOptions, error) {
|
||||
txNumKey := in[0].Key.(*prefixes.TxNumKey)
|
||||
log.Debugf("%v", txNumKey.TxHash.String())
|
||||
out := make([]*IterOptions, 0, 100)
|
||||
startKey := &prefixes.TxKey{
|
||||
Prefix: []byte{prefixes.Tx},
|
||||
TxHash: txNumKey.TxHash,
|
||||
}
|
||||
endKey := &prefixes.TxKey{
|
||||
Prefix: []byte{prefixes.Tx},
|
||||
TxHash: txNumKey.TxHash,
|
||||
}
|
||||
selectedTx, err := db.selectFrom([]byte{prefixes.Tx}, startKey, endKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, selectedTx...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
blockTxsCache := make(map[uint32][]*chainhash.Hash)
|
||||
results := make([]TxMerkle, 0, 500)
|
||||
for kvs := range innerJoin(db.DB, iterate(db.DB, selectedTxNum), selectTxByTxNum) {
|
||||
if err := checkForError(kvs); err != nil {
|
||||
return results, err
|
||||
}
|
||||
txNumKey, txNumVal := kvs[0].Key.(*prefixes.TxNumKey), kvs[0].Value.(*prefixes.TxNumValue)
|
||||
_, txVal := kvs[1].Key.(*prefixes.TxKey), kvs[1].Value.(*prefixes.TxValue)
|
||||
txHeight := stack.BisectRight(db.TxCounts, []uint32{txNumVal.TxNum})[0]
|
||||
txPos := txNumVal.TxNum - db.TxCounts.Get(txHeight-1)
|
||||
// We need all the TX hashes in order to select out the relevant ones.
|
||||
if _, ok := blockTxsCache[txHeight]; !ok {
|
||||
txs, err := db.GetBlockTXs(txHeight)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
blockTxsCache[txHeight] = txs
|
||||
}
|
||||
blockTxs := blockTxsCache[txHeight]
|
||||
results = append(results, TxMerkle{
|
||||
TxHash: txNumKey.TxHash,
|
||||
RawTx: txVal.RawTx,
|
||||
Height: int(txHeight),
|
||||
Pos: txPos,
|
||||
Merkle: merklePath(txPos, blockTxs, []*chainhash.Hash{}),
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetClaimByID(claimID string) ([]*ExpandedResolveResult, []*ExpandedResolveResult, error) {
|
||||
rows := make([]*ExpandedResolveResult, 0)
|
||||
extras := make([]*ExpandedResolveResult, 0)
|
||||
claimHash, err := hex.DecodeString(claimID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
stream, err := db.FsGetClaimByHash(claimHash)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var res = NewExpandedResolveResult()
|
||||
res.Stream = &optionalResolveResultOrError{res: stream}
|
||||
rows = append(rows, res)
|
||||
|
||||
if stream != nil && stream.ChannelHash != nil {
|
||||
channel, err := db.FsGetClaimByHash(stream.ChannelHash)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var res = NewExpandedResolveResult()
|
||||
res.Channel = &optionalResolveResultOrError{res: channel}
|
||||
extras = append(extras, res)
|
||||
}
|
||||
|
||||
if stream != nil && stream.RepostedClaimHash != nil {
|
||||
repost, err := db.FsGetClaimByHash(stream.RepostedClaimHash)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var res = NewExpandedResolveResult()
|
||||
res.Repost = &optionalResolveResultOrError{res: repost}
|
||||
extras = append(extras, res)
|
||||
}
|
||||
|
||||
return rows, extras, nil
|
||||
}
|
||||
|
||||
func (db *ReadOnlyDBColumnFamily) GetDBState() (*prefixes.DBStateValue, error) {
|
||||
handle, err := db.EnsureHandle(prefixes.DBState)
|
||||
if err != nil {
|
||||
|
@ -1066,7 +720,7 @@ func (db *ReadOnlyDBColumnFamily) BidOrderNameIter(normalizedName string) <-chan
|
|||
key := prefixes.NewBidOrderKey(normalizedName)
|
||||
var rawKeyPrefix []byte = nil
|
||||
rawKeyPrefix = key.PartialPack(1)
|
||||
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options = options.WithIncludeValue(true) //.WithIncludeStop(true)
|
||||
ch := IterCF(db.DB, options)
|
||||
return ch
|
||||
|
@ -1084,7 +738,7 @@ func (db *ReadOnlyDBColumnFamily) ClaimShortIdIter(normalizedName string, claimI
|
|||
} else {
|
||||
rawKeyPrefix = key.PartialPack(1)
|
||||
}
|
||||
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options = options.WithIncludeValue(true) //.WithIncludeStop(true)
|
||||
ch := IterCF(db.DB, options)
|
||||
return ch
|
||||
|
|
|
@ -326,7 +326,7 @@ func (db *ReadOnlyDBColumnFamily) ResolveClaimInChannel(channelHash []byte, norm
|
|||
|
||||
key := prefixes.NewChannelToClaimKey(channelHash, normalizedName)
|
||||
rawKeyPrefix := key.PartialPack(2)
|
||||
options := NewIterateOptions().WithDB(db).WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
|
||||
options = options.WithIncludeValue(true) //.WithIncludeStop(true)
|
||||
ch := IterCF(db.DB, options)
|
||||
// TODO: what's a good default size for this?
|
||||
|
|
155
db/db_test.go
155
db/db_test.go
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/hex"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
@ -11,9 +12,7 @@ import (
|
|||
dbpkg "github.com/lbryio/herald.go/db"
|
||||
"github.com/lbryio/herald.go/db/prefixes"
|
||||
"github.com/lbryio/herald.go/internal"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
"github.com/linxGnu/grocksdb"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
@ -21,7 +20,7 @@ import (
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// OpenAndFillTmpDBColumnFamlies opens a db and fills it with data from a csv file using the given column family names
|
||||
func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFamily, [][]string, error) {
|
||||
func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFamily, [][]string, func(), error) {
|
||||
|
||||
log.Println(filePath)
|
||||
file, err := os.Open(filePath)
|
||||
|
@ -31,7 +30,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
|
|||
reader := csv.NewReader(file)
|
||||
records, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
wOpts := grocksdb.NewDefaultWriteOptions()
|
||||
|
@ -39,7 +38,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
|
|||
opts.SetCreateIfMissing(true)
|
||||
db, err := grocksdb.OpenDb(opts, "tmp")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
var handleMap map[string]*grocksdb.ColumnFamilyHandle = make(map[string]*grocksdb.ColumnFamilyHandle)
|
||||
|
||||
|
@ -54,7 +53,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
|
|||
log.Println(cfName)
|
||||
handle, err := db.CreateColumnFamily(opts, cfName)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
handleMap[cfName] = handle
|
||||
}
|
||||
|
@ -68,16 +67,16 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
|
|||
for _, record := range records[1:] {
|
||||
cf := record[0]
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
handle := handleMap[string(cf)]
|
||||
key, err := hex.DecodeString(record[1])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
val, err := hex.DecodeString(record[2])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
db.PutCF(wOpts, handle, key, val)
|
||||
}
|
||||
|
@ -94,8 +93,6 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
|
|||
LastState: nil,
|
||||
Height: 0,
|
||||
Headers: nil,
|
||||
Grp: stop.New(),
|
||||
Cleanup: toDefer,
|
||||
}
|
||||
|
||||
// err = dbpkg.ReadDBState(myDB) //TODO: Figure out right place for this
|
||||
|
@ -105,7 +102,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
|
|||
|
||||
err = myDB.InitTxCounts()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
// err = dbpkg.InitHeaders(myDB)
|
||||
|
@ -113,7 +110,7 @@ func OpenAndFillTmpDBColumnFamlies(filePath string) (*dbpkg.ReadOnlyDBColumnFami
|
|||
// return nil, nil, nil, err
|
||||
// }
|
||||
|
||||
return myDB, records, nil
|
||||
return myDB, records, toDefer, nil
|
||||
}
|
||||
|
||||
// OpenAndFillTmpDBCF opens a db and fills it with data from a csv file
|
||||
|
@ -244,18 +241,17 @@ func CatCSV(filePath string) {
|
|||
|
||||
func TestCatFullDB(t *testing.T) {
|
||||
t.Skip("Skipping full db test")
|
||||
grp := stop.New()
|
||||
// url := "lbry://@lothrop#2/lothrop-livestream-games-and-code#c"
|
||||
// "lbry://@lbry", "lbry://@lbry#3", "lbry://@lbry3f", "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a", "lbry://@lbry:1", "lbry://@lbry$1"
|
||||
// url := "lbry://@Styxhexenhammer666#2/legacy-media-baron-les-moonves-(cbs#9"
|
||||
// url := "lbry://@lbry"
|
||||
// url := "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a"
|
||||
dbPath := "/mnt/sda1/wallet_server/_data/lbry-rocksdb/"
|
||||
dbPath := "/mnt/sda/wallet_server/_data/lbry-rocksdb/"
|
||||
// dbPath := "/mnt/d/data/snapshot_1072108/lbry-rocksdb/"
|
||||
secondaryPath := "asdf"
|
||||
db, err := dbpkg.GetProdDB(dbPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
db, toDefer, err := dbpkg.GetProdDB(dbPath, secondaryPath)
|
||||
|
||||
defer toDefer()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
|
@ -275,7 +271,6 @@ func TestCatFullDB(t *testing.T) {
|
|||
// TestOpenFullDB Tests running a resolve on a full db.
|
||||
func TestOpenFullDB(t *testing.T) {
|
||||
t.Skip("Skipping full db test")
|
||||
grp := stop.New()
|
||||
// url := "lbry://@lothrop#2/lothrop-livestream-games-and-code#c"
|
||||
// "lbry://@lbry", "lbry://@lbry#3", "lbry://@lbry3f", "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a", "lbry://@lbry:1", "lbry://@lbry$1"
|
||||
// url := "lbry://@Styxhexenhammer666#2/legacy-media-baron-les-moonves-(cbs#9"
|
||||
|
@ -283,11 +278,11 @@ func TestOpenFullDB(t *testing.T) {
|
|||
// url := "lbry://@lbry#3fda836a92faaceedfe398225fb9b2ee2ed1f01a"
|
||||
// url := "lbry://@lbry$1"
|
||||
url := "https://lbry.tv/@lothrop:2/lothrop-livestream-games-and-code:c"
|
||||
dbPath := "/mnt/sda1/wallet_server/_data/lbry-rocksdb/"
|
||||
dbPath := "/mnt/sda/wallet_server/_data/lbry-rocksdb/"
|
||||
// dbPath := "/mnt/d/data/snapshot_1072108/lbry-rocksdb/"
|
||||
secondaryPath := "asdf"
|
||||
db, err := dbpkg.GetProdDB(dbPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
db, toDefer, err := dbpkg.GetProdDB(dbPath, secondaryPath)
|
||||
defer toDefer()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
|
@ -301,13 +296,12 @@ func TestOpenFullDB(t *testing.T) {
|
|||
func TestResolve(t *testing.T) {
|
||||
url := "lbry://@Styxhexenhammer666#2/legacy-media-baron-les-moonves-(cbs#9"
|
||||
filePath := "../testdata/FULL_resolve.csv"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
defer toDefer()
|
||||
expandedResolveResult := db.Resolve(url)
|
||||
log.Printf("%#v\n", expandedResolveResult)
|
||||
if expandedResolveResult != nil && expandedResolveResult.Channel != nil {
|
||||
|
@ -321,11 +315,11 @@ func TestResolve(t *testing.T) {
|
|||
func TestGetDBState(t *testing.T) {
|
||||
filePath := "../testdata/s_resolve.csv"
|
||||
want := uint32(1072108)
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer toDefer()
|
||||
state, err := db.GetDBState()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
@ -343,11 +337,11 @@ func TestGetRepostedClaim(t *testing.T) {
|
|||
// Should be non-existent
|
||||
channelHash2, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bf")
|
||||
filePath := "../testdata/W_resolve.csv"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer toDefer()
|
||||
|
||||
count, err := db.GetRepostedCount(channelHash)
|
||||
if err != nil {
|
||||
|
@ -376,11 +370,11 @@ func TestGetRepostedCount(t *testing.T) {
|
|||
// Should be non-existent
|
||||
channelHash2, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bf")
|
||||
filePath := "../testdata/j_resolve.csv"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer toDefer()
|
||||
|
||||
count, err := db.GetRepostedCount(channelHash)
|
||||
if err != nil {
|
||||
|
@ -413,11 +407,11 @@ func TestGetRepost(t *testing.T) {
|
|||
channelHash2, _ := hex.DecodeString("000009ca6e0caaaef16872b4bd4f6f1b8c2363e2")
|
||||
filePath := "../testdata/V_resolve.csv"
|
||||
// want := uint32(3670)
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer toDefer()
|
||||
|
||||
res, err := db.GetRepost(channelHash)
|
||||
if err != nil {
|
||||
|
@ -447,12 +441,11 @@ func TestGetClaimsInChannelCount(t *testing.T) {
|
|||
channelHash, _ := hex.DecodeString("2556ed1cab9d17f2a9392030a9ad7f5d138f11bd")
|
||||
filePath := "../testdata/Z_resolve.csv"
|
||||
want := uint32(3670)
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
defer toDefer()
|
||||
count, err := db.GetClaimsInChannelCount(channelHash)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
@ -477,12 +470,11 @@ func TestGetShortClaimIdUrl(t *testing.T) {
|
|||
var position uint16 = 0
|
||||
filePath := "../testdata/F_resolve.csv"
|
||||
log.Println(filePath)
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
defer toDefer()
|
||||
shortUrl, err := db.GetShortClaimIdUrl(name, normalName, claimHash, rootTxNum, position)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
@ -496,11 +488,11 @@ func TestClaimShortIdIter(t *testing.T) {
|
|||
filePath := "../testdata/F_cat.csv"
|
||||
normalName := "cat"
|
||||
claimId := "0"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer toDefer()
|
||||
|
||||
ch := db.ClaimShortIdIter(normalName, claimId)
|
||||
|
||||
|
@ -526,12 +518,11 @@ func TestGetTXOToClaim(t *testing.T) {
|
|||
var txNum uint32 = 1456296
|
||||
var position uint16 = 0
|
||||
filePath := "../testdata/G_2.csv"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
defer toDefer()
|
||||
val, err := db.GetCachedClaimHash(txNum, position)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
@ -555,11 +546,11 @@ func TestGetClaimToChannel(t *testing.T) {
|
|||
var val []byte = nil
|
||||
|
||||
filePath := "../testdata/I_resolve.csv"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer toDefer()
|
||||
|
||||
val, err = db.GetChannelForClaim(claimHash, txNum, position)
|
||||
if err != nil {
|
||||
|
@ -580,68 +571,47 @@ func TestGetClaimToChannel(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestGetEffectiveAmountSupportOnly(t *testing.T) {
|
||||
filePath := "../testdata/Si_resolve.csv"
|
||||
want := uint64(20000006)
|
||||
claimHashStr := "00000324e40fcb63a0b517a3660645e9bd99244a"
|
||||
filePath := "../testdata/S_resolve.csv"
|
||||
want := uint64(78999149300)
|
||||
claimHashStr := "2556ed1cab9d17f2a9392030a9ad7f5d138f11bd"
|
||||
claimHash, _ := hex.DecodeString(claimHashStr)
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
db.Height = 999999999
|
||||
defer toDefer()
|
||||
db.Height = 1116054
|
||||
|
||||
amount, err := db.GetEffectiveAmount(claimHash, true)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if amount != want {
|
||||
t.Errorf("Expected %d, got %d", want, amount)
|
||||
}
|
||||
|
||||
// Cross-check against iterator-based implementation.
|
||||
iteratorAmount, err := db.GetActiveAmount(claimHash, prefixes.ActivatedSupportTXOType, db.Height)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if iteratorAmount != want {
|
||||
t.Errorf("Expected %d, got %d", want, iteratorAmount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetEffectiveAmount(t *testing.T) {
|
||||
filePath := "../testdata/Si_resolve.csv"
|
||||
want := uint64(21000006)
|
||||
claimHashStr := "00000324e40fcb63a0b517a3660645e9bd99244a"
|
||||
filePath := "../testdata/i_resolve.csv"
|
||||
want := uint64(507171810600)
|
||||
claimHashStr := "2556ed1cab9d17f2a9392030a9ad7f5d138f11bd"
|
||||
claimHash, _ := hex.DecodeString(claimHashStr)
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
db.Height = 999999999
|
||||
defer toDefer()
|
||||
db.Height = 1116054
|
||||
|
||||
amount, err := db.GetEffectiveAmount(claimHash, false)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if amount != want {
|
||||
t.Errorf("Expected %d, got %d", want, amount)
|
||||
}
|
||||
|
||||
// Cross-check against iterator-based implementation.
|
||||
iteratorAmount1, err := db.GetActiveAmount(claimHash, prefixes.ActivatedSupportTXOType, db.Height)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
iteratorAmount2, err := db.GetActiveAmount(claimHash, prefixes.ActivateClaimTXOType, db.Height)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if iteratorAmount1+iteratorAmount2 != want {
|
||||
t.Errorf("Expected %d, got %d (%d + %d)", want, iteratorAmount1+iteratorAmount2, iteratorAmount1, iteratorAmount2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSupportAmount(t *testing.T) {
|
||||
|
@ -652,12 +622,11 @@ func TestGetSupportAmount(t *testing.T) {
|
|||
t.Error(err)
|
||||
}
|
||||
filePath := "../testdata/a_resolve.csv"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
defer toDefer()
|
||||
res, err := db.GetSupportAmount(claimHash)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
@ -673,12 +642,11 @@ func TestGetTxHash(t *testing.T) {
|
|||
want := "54e14ff0c404c29b3d39ae4d249435f167d5cd4ce5a428ecb745b3df1c8e3dde"
|
||||
|
||||
filePath := "../testdata/X_resolve.csv"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
defer toDefer()
|
||||
resHash, err := db.GetTxHash(txNum)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
@ -716,12 +684,11 @@ func TestGetActivation(t *testing.T) {
|
|||
txNum := uint32(0x6284e3)
|
||||
position := uint16(0x0)
|
||||
want := uint32(0xa6b65)
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
defer toDefer()
|
||||
activation, err := db.GetActivation(txNum, position)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
@ -748,13 +715,12 @@ func TestGetClaimToTXO(t *testing.T) {
|
|||
return
|
||||
}
|
||||
filePath := "../testdata/E_resolve.csv"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
defer toDefer()
|
||||
res, err := db.GetCachedClaimTxo(claimHash, true)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
@ -778,13 +744,12 @@ func TestGetControllingClaim(t *testing.T) {
|
|||
claimName := internal.NormalizeName("@Styxhexenhammer666")
|
||||
claimHash := "2556ed1cab9d17f2a9392030a9ad7f5d138f11bd"
|
||||
filePath := "../testdata/P_resolve.csv"
|
||||
db, _, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
defer db.Shutdown()
|
||||
db, _, toDefer, err := OpenAndFillTmpDBColumnFamlies(filePath)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
defer toDefer()
|
||||
res, err := db.GetControllingClaim(claimName)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
|
|
@ -6,7 +6,6 @@ import (
|
|||
"bytes"
|
||||
|
||||
"github.com/lbryio/herald.go/db/prefixes"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
"github.com/linxGnu/grocksdb"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
@ -23,8 +22,6 @@ type IterOptions struct {
|
|||
IncludeValue bool
|
||||
RawKey bool
|
||||
RawValue bool
|
||||
Grp *stop.Group
|
||||
// DB *ReadOnlyDBColumnFamily
|
||||
CfHandle *grocksdb.ColumnFamilyHandle
|
||||
It *grocksdb.Iterator
|
||||
Serializer *prefixes.SerializationAPI
|
||||
|
@ -43,8 +40,6 @@ func NewIterateOptions() *IterOptions {
|
|||
IncludeValue: false,
|
||||
RawKey: false,
|
||||
RawValue: false,
|
||||
Grp: nil,
|
||||
// DB: nil,
|
||||
CfHandle: nil,
|
||||
It: nil,
|
||||
Serializer: prefixes.ProductionAPI,
|
||||
|
@ -106,13 +101,6 @@ func (o *IterOptions) WithRawValue(rawValue bool) *IterOptions {
|
|||
return o
|
||||
}
|
||||
|
||||
func (o *IterOptions) WithDB(db *ReadOnlyDBColumnFamily) *IterOptions {
|
||||
// o.Grp.AddNamed(1, iterKey)
|
||||
o.Grp = stop.New(db.Grp)
|
||||
o.Grp.Add(1)
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *IterOptions) WithSerializer(serializer *prefixes.SerializationAPI) *IterOptions {
|
||||
o.Serializer = serializer
|
||||
return o
|
||||
|
|
|
@ -7,7 +7,6 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/go-restruct/restruct"
|
||||
"github.com/lbryio/herald.go/internal"
|
||||
"github.com/lbryio/lbcd/chaincfg/chainhash"
|
||||
)
|
||||
|
||||
|
@ -60,34 +59,6 @@ func (kv *BlockTxsValue) Unpack(buf []byte, order binary.ByteOrder) ([]byte, err
|
|||
return buf[offset:], nil
|
||||
}
|
||||
|
||||
// Struct BigEndianChainHash is a chainhash.Hash stored in external
|
||||
// byte-order (opposite of other 32 byte chainhash.Hash values). In order
|
||||
// to reuse chainhash.Hash we need to correct the byte-order.
|
||||
// Currently this type is used for field Genesis of DBStateValue.
|
||||
|
||||
func (kv *BigEndianChainHash) SizeOf() int {
|
||||
return chainhash.HashSize
|
||||
}
|
||||
|
||||
func (kv *BigEndianChainHash) Pack(buf []byte, order binary.ByteOrder) ([]byte, error) {
|
||||
offset := 0
|
||||
hash := kv.CloneBytes()
|
||||
// HACK: Instances of chainhash.Hash use the internal byte-order.
|
||||
// Python scribe writes bytes of genesis hash in external byte-order.
|
||||
internal.ReverseBytesInPlace(hash)
|
||||
offset += copy(buf[offset:chainhash.HashSize], hash[:])
|
||||
return buf[offset:], nil
|
||||
}
|
||||
|
||||
func (kv *BigEndianChainHash) Unpack(buf []byte, order binary.ByteOrder) ([]byte, error) {
|
||||
offset := 0
|
||||
offset += copy(kv.Hash[:], buf[offset:32])
|
||||
// HACK: Instances of chainhash.Hash use the internal byte-order.
|
||||
// Python scribe writes bytes of genesis hash in external byte-order.
|
||||
internal.ReverseBytesInPlace(kv.Hash[:])
|
||||
return buf[offset:], nil
|
||||
}
|
||||
|
||||
func genericNew(prefix []byte, key bool) (interface{}, error) {
|
||||
t, ok := prefixRegistry[prefix[0]]
|
||||
if !ok {
|
||||
|
|
|
@ -12,13 +12,13 @@ import (
|
|||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/herald.go/internal"
|
||||
"github.com/lbryio/lbcd/chaincfg/chainhash"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -182,25 +182,12 @@ func NewLengthEncodedPartialClaimId(s string) LengthEncodedPartialClaimId {
|
|||
}
|
||||
}
|
||||
|
||||
type BigEndianChainHash struct {
|
||||
chainhash.Hash
|
||||
}
|
||||
|
||||
func NewBigEndianChainHash(hash *chainhash.Hash) BigEndianChainHash {
|
||||
if hash != nil {
|
||||
return BigEndianChainHash{
|
||||
*hash,
|
||||
}
|
||||
}
|
||||
return BigEndianChainHash{}
|
||||
}
|
||||
|
||||
type DBStateKey struct {
|
||||
Prefix []byte `struct:"[1]byte" json:"prefix"`
|
||||
}
|
||||
|
||||
type DBStateValue struct {
|
||||
Genesis BigEndianChainHash
|
||||
Genesis *chainhash.Hash
|
||||
Height uint32
|
||||
TxCount uint32
|
||||
Tip *chainhash.Hash
|
||||
|
@ -216,7 +203,7 @@ type DBStateValue struct {
|
|||
|
||||
func NewDBStateValue() *DBStateValue {
|
||||
return &DBStateValue{
|
||||
Genesis: NewBigEndianChainHash(nil),
|
||||
Genesis: new(chainhash.Hash),
|
||||
Height: 0,
|
||||
TxCount: 0,
|
||||
Tip: new(chainhash.Hash),
|
||||
|
@ -250,11 +237,7 @@ func (v *DBStateValue) PackValue() []byte {
|
|||
// b'>32sLL32sLLBBlllL'
|
||||
n := 32 + 4 + 4 + 32 + 4 + 4 + 1 + 1 + 4 + 4 + 4 + 4
|
||||
value := make([]byte, n)
|
||||
genesis := v.Genesis.CloneBytes()
|
||||
// HACK: Instances of chainhash.Hash use the internal byte-order.
|
||||
// Python scribe writes bytes of genesis hash in external byte-order.
|
||||
internal.ReverseBytesInPlace(genesis)
|
||||
copy(value, genesis[:32])
|
||||
copy(value, v.Genesis[:32])
|
||||
binary.BigEndian.PutUint32(value[32:], v.Height)
|
||||
binary.BigEndian.PutUint32(value[32+4:], v.TxCount)
|
||||
copy(value[32+4+4:], v.Tip[:32])
|
||||
|
@ -299,11 +282,8 @@ func DBStateKeyUnpack(key []byte) *DBStateKey {
|
|||
func DBStateValueUnpack(value []byte) *DBStateValue {
|
||||
genesis := (*chainhash.Hash)(value[:32])
|
||||
tip := (*chainhash.Hash)(value[32+4+4 : 32+4+4+32])
|
||||
// HACK: Python scribe writes bytes of genesis hash in external byte-order.
|
||||
// Instances of chainhash.Hash should use the internal byte-order.
|
||||
internal.ReverseBytesInPlace(genesis[:])
|
||||
x := &DBStateValue{
|
||||
Genesis: NewBigEndianChainHash(genesis),
|
||||
Genesis: genesis,
|
||||
Height: binary.BigEndian.Uint32(value[32:]),
|
||||
TxCount: binary.BigEndian.Uint32(value[32+4:]),
|
||||
Tip: tip,
|
||||
|
@ -728,7 +708,7 @@ type BlockTxsKey struct {
|
|||
}
|
||||
|
||||
type BlockTxsValue struct {
|
||||
TxHashes []*chainhash.Hash `struct:"*[32]byte" struct-while:"!_eof" json:"tx_hashes"`
|
||||
TxHashes []*chainhash.Hash `struct-while:"!_eof" json:"tx_hashes"`
|
||||
}
|
||||
|
||||
func (k *BlockTxsKey) NewBlockTxsKey(height uint32) *BlockTxsKey {
|
||||
|
@ -1070,6 +1050,84 @@ func TxNumValueUnpack(value []byte) *TxNumValue {
|
|||
}
|
||||
}
|
||||
|
||||
type TxKey struct {
|
||||
Prefix []byte `struct:"[1]byte" json:"prefix"`
|
||||
TxHash *chainhash.Hash `struct:"*[32]byte" json:"tx_hash"`
|
||||
}
|
||||
|
||||
type TxValue struct {
|
||||
RawTx []byte `struct-while:"!_eof" json:"raw_tx"`
|
||||
}
|
||||
|
||||
func (k *TxKey) PackKey() []byte {
|
||||
prefixLen := 1
|
||||
// b'>L'
|
||||
n := prefixLen + 32
|
||||
key := make([]byte, n)
|
||||
copy(key, k.Prefix)
|
||||
copy(key[prefixLen:], k.TxHash[:32])
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func (v *TxValue) PackValue() []byte {
|
||||
value := make([]byte, len(v.RawTx))
|
||||
copy(value, v.RawTx)
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
func (kv *TxKey) NumFields() int {
|
||||
return 1
|
||||
}
|
||||
|
||||
func (k *TxKey) PartialPack(fields int) []byte {
|
||||
// Limit fields between 0 and number of fields, we always at least need
|
||||
// the prefix, and we never need to iterate past the number of fields.
|
||||
if fields > 1 {
|
||||
fields = 1
|
||||
}
|
||||
if fields < 0 {
|
||||
fields = 0
|
||||
}
|
||||
|
||||
prefixLen := 1
|
||||
var n = prefixLen
|
||||
for i := 0; i <= fields; i++ {
|
||||
switch i {
|
||||
case 1:
|
||||
n += 32
|
||||
}
|
||||
}
|
||||
|
||||
key := make([]byte, n)
|
||||
|
||||
for i := 0; i <= fields; i++ {
|
||||
switch i {
|
||||
case 0:
|
||||
copy(key, k.Prefix)
|
||||
case 1:
|
||||
copy(key[prefixLen:], k.TxHash[:32])
|
||||
}
|
||||
}
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func TxKeyUnpack(key []byte) *TxKey {
|
||||
prefixLen := 1
|
||||
return &TxKey{
|
||||
Prefix: key[:prefixLen],
|
||||
TxHash: (*chainhash.Hash)(key[prefixLen : prefixLen+32]),
|
||||
}
|
||||
}
|
||||
|
||||
func TxValueUnpack(value []byte) *TxValue {
|
||||
return &TxValue{
|
||||
RawTx: value,
|
||||
}
|
||||
}
|
||||
|
||||
type BlockHeaderKey struct {
|
||||
Prefix []byte `struct:"[1]byte" json:"prefix"`
|
||||
Height uint32 `json:"height"`
|
||||
|
@ -3293,12 +3351,9 @@ func (kv *TrendingNotificationValue) UnpackValue(buf []byte) {
|
|||
offset += 8
|
||||
}
|
||||
|
||||
type TxKey = MempoolTxKey
|
||||
type TxValue = MempoolTxValue
|
||||
|
||||
type MempoolTxKey struct {
|
||||
Prefix []byte `struct:"[1]byte" json:"prefix"`
|
||||
TxHash *chainhash.Hash `struct:"*[32]byte" json:"tx_hash"`
|
||||
TxHash []byte `struct:"[32]byte" json:"tx_hash"`
|
||||
}
|
||||
|
||||
type MempoolTxValue struct {
|
||||
|
@ -3331,7 +3386,7 @@ func (kv *MempoolTxKey) UnpackKey(buf []byte) {
|
|||
offset := 0
|
||||
kv.Prefix = buf[offset : offset+1]
|
||||
offset += 1
|
||||
kv.TxHash = (*chainhash.Hash)(buf[offset : offset+32])
|
||||
kv.TxHash = buf[offset : offset+32]
|
||||
offset += 32
|
||||
}
|
||||
|
||||
|
@ -3413,7 +3468,7 @@ func (kv *TouchedHashXValue) UnpackValue(buf []byte) {
|
|||
|
||||
type HashXStatusKey struct {
|
||||
Prefix []byte `struct:"[1]byte" json:"prefix"`
|
||||
HashX []byte `struct:"[11]byte" json:"hashX"`
|
||||
HashX []byte `struct:"[20]byte" json:"hashX"`
|
||||
}
|
||||
|
||||
type HashXStatusValue struct {
|
||||
|
@ -3425,15 +3480,15 @@ func (kv *HashXStatusKey) NumFields() int {
|
|||
}
|
||||
|
||||
func (kv *HashXStatusKey) PartialPack(fields int) []byte {
|
||||
// b'>20s' (really HASHX_LEN 11 bytes)
|
||||
n := len(kv.Prefix) + 11
|
||||
// b'>20s'
|
||||
n := len(kv.Prefix) + 20
|
||||
buf := make([]byte, n)
|
||||
offset := 0
|
||||
offset += copy(buf[offset:], kv.Prefix[:1])
|
||||
if fields <= 0 {
|
||||
return buf[:offset]
|
||||
}
|
||||
offset += copy(buf[offset:], kv.HashX[:11])
|
||||
offset += copy(buf[offset:], kv.HashX[:20])
|
||||
return buf[:offset]
|
||||
}
|
||||
|
||||
|
@ -3442,12 +3497,12 @@ func (kv *HashXStatusKey) PackKey() []byte {
|
|||
}
|
||||
|
||||
func (kv *HashXStatusKey) UnpackKey(buf []byte) {
|
||||
// b'>20s' (really HASHX_LEN 11 bytes)
|
||||
// b'>20s'
|
||||
offset := 0
|
||||
kv.Prefix = buf[offset : offset+1]
|
||||
offset += 1
|
||||
kv.HashX = buf[offset : offset+11]
|
||||
offset += 11
|
||||
kv.HashX = buf[offset : offset+20]
|
||||
offset += 20
|
||||
}
|
||||
|
||||
func (kv *HashXStatusValue) PackValue() []byte {
|
||||
|
@ -3475,8 +3530,7 @@ type EffectiveAmountKey struct {
|
|||
}
|
||||
|
||||
type EffectiveAmountValue struct {
|
||||
ActivatedSum uint64 `json:"activated_sum"`
|
||||
ActivatedSupportSum uint64 `json:"activated_support_sum"`
|
||||
EffectiveAmount uint64 `json:"effective_amount"`
|
||||
}
|
||||
|
||||
func (kv *EffectiveAmountKey) NumFields() int {
|
||||
|
@ -3510,23 +3564,19 @@ func (kv *EffectiveAmountKey) UnpackKey(buf []byte) {
|
|||
}
|
||||
|
||||
func (kv *EffectiveAmountValue) PackValue() []byte {
|
||||
// b'>QQ'
|
||||
n := 8 + 8
|
||||
// b'>Q'
|
||||
n := 8
|
||||
buf := make([]byte, n)
|
||||
offset := 0
|
||||
binary.BigEndian.PutUint64(buf[offset:], kv.ActivatedSum)
|
||||
offset += 8
|
||||
binary.BigEndian.PutUint64(buf[offset:], kv.ActivatedSupportSum)
|
||||
binary.BigEndian.PutUint64(buf[offset:], kv.EffectiveAmount)
|
||||
offset += 8
|
||||
return buf[:offset]
|
||||
}
|
||||
|
||||
func (kv *EffectiveAmountValue) UnpackValue(buf []byte) {
|
||||
// b'>QQ'
|
||||
// b'>Q'
|
||||
offset := 0
|
||||
kv.ActivatedSum = binary.BigEndian.Uint64(buf[offset:])
|
||||
offset += 8
|
||||
kv.ActivatedSupportSum = binary.BigEndian.Uint64(buf[offset:])
|
||||
kv.EffectiveAmount = binary.BigEndian.Uint64(buf[offset:])
|
||||
offset += 8
|
||||
}
|
||||
|
||||
|
@ -3870,6 +3920,12 @@ var prefixRegistry = map[byte]prefixMeta{
|
|||
newValue: func() interface{} {
|
||||
return &TxValue{}
|
||||
},
|
||||
newKeyUnpack: func(buf []byte) interface{} {
|
||||
return TxKeyUnpack(buf)
|
||||
},
|
||||
newValueUnpack: func(buf []byte) interface{} {
|
||||
return TxValueUnpack(buf)
|
||||
},
|
||||
},
|
||||
BlockHash: {
|
||||
newKey: func() interface{} {
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
"encoding/csv"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"math/big"
|
||||
"os"
|
||||
|
@ -15,7 +16,6 @@ import (
|
|||
dbpkg "github.com/lbryio/herald.go/db"
|
||||
prefixes "github.com/lbryio/herald.go/db/prefixes"
|
||||
"github.com/linxGnu/grocksdb"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func TestPrefixRegistry(t *testing.T) {
|
||||
|
@ -429,7 +429,7 @@ func TestHashXMempoolStatus(t *testing.T) {
|
|||
func TestEffectiveAmount(t *testing.T) {
|
||||
prefix := byte(prefixes.EffectiveAmount)
|
||||
filePath := fmt.Sprintf("../../testdata/%c.csv", prefix)
|
||||
//synthesizeTestData([]byte{prefix}, filePath, []int{20}, []int{8, 8}, [][3]int{})
|
||||
//synthesizeTestData([]byte{prefix}, filePath, []int{20}, []int{8}, [][3]int{})
|
||||
key := &prefixes.EffectiveAmountKey{}
|
||||
testGeneric(filePath, prefix, key.NumFields())(t)
|
||||
}
|
||||
|
|
|
@ -1,13 +0,0 @@
|
|||
FROM jeffreypicard/hub-github-env:dev
|
||||
|
||||
COPY scripts/integration_tests.sh /integration_tests.sh
|
||||
COPY scripts/cicd_integration_test_runner.sh /cicd_integration_test_runner.sh
|
||||
COPY herald /herald
|
||||
|
||||
RUN apt install -y jq curl
|
||||
|
||||
ENV CGO_LDFLAGS "-L/usr/local/lib -lrocksdb -lstdc++ -lm -lz -lsnappy -llz4 -lzstd"
|
||||
ENV CGO_CFLAGS "-I/usr/local/include/rocksdb"
|
||||
ENV LD_LIBRARY_PATH /usr/local/lib
|
||||
|
||||
ENTRYPOINT ["/cicd_integration_test_runner.sh"]
|
7
go.mod
7
go.mod
|
@ -24,11 +24,12 @@ require (
|
|||
gopkg.in/karalabe/cookiejar.v1 v1.0.0-20141109175019-e1490cae028c
|
||||
)
|
||||
|
||||
require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
|
@ -42,7 +43,7 @@ require (
|
|||
github.com/prometheus/procfs v0.6.0 // indirect
|
||||
github.com/stretchr/testify v1.7.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
|
||||
google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 // indirect
|
||||
|
|
6
go.sum
6
go.sum
|
@ -68,13 +68,11 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufo
|
|||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
||||
|
@ -360,6 +358,8 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
|
|||
github.com/lbryio/lbcd v0.22.100-beta/go.mod h1:u8SaFX4xdGMMR5xasBGfgApC8pvD4rnK2OujZnrq5gs=
|
||||
github.com/lbryio/lbcd v0.22.100-beta-rc5/go.mod h1:9PbFSlHYX7WlnDQwcTxHVf1W35VAnRsattCSyKOO55g=
|
||||
github.com/lbryio/lbcd v0.22.200-beta/go.mod h1:kNuzGWf808ipTGB0y0WogzsGv5BVM4Qv85Z+JYwC9FA=
|
||||
github.com/lbryio/lbcd v0.22.201-beta-rc1 h1:FmzzApVj2RBXloLM2w9tLvN2xyTZjeyh+QC7GIw/wwo=
|
||||
github.com/lbryio/lbcd v0.22.201-beta-rc1/go.mod h1:kNuzGWf808ipTGB0y0WogzsGv5BVM4Qv85Z+JYwC9FA=
|
||||
github.com/lbryio/lbcd v0.22.201-beta-rc4 h1:Xh751Bh/GWRcP5bI6NJ2+zueo2otTcTWapFvFbryP5c=
|
||||
github.com/lbryio/lbcd v0.22.201-beta-rc4/go.mod h1:Jgo48JDINhdOgHHR83J70Q6G42x3WAo9DI//QogcL+E=
|
||||
github.com/lbryio/lbcutil v1.0.201/go.mod h1:gDHc/b+Rdz3J7+VB8e5/Bl9roVf8Q5/8FQCyuK9dXD0=
|
||||
|
@ -375,6 +375,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
|
|||
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
|
||||
github.com/linxGnu/grocksdb v1.6.42 h1:nJLoXFuzwBwQQQrXTUgRGRz1QRm7y8pR6CNV/gwrbqs=
|
||||
github.com/linxGnu/grocksdb v1.6.42/go.mod h1:JcMMDBFaDNhRXFYcYXmgQwb/RarSld1PulTI7UzE+w0=
|
||||
github.com/linxGnu/grocksdb v1.7.0 h1:UyFDykX0CUfxDN10cqlFho/rwt9K6KoDaLXL9Ej5z9g=
|
||||
github.com/linxGnu/grocksdb v1.7.0/go.mod h1:JcMMDBFaDNhRXFYcYXmgQwb/RarSld1PulTI7UzE+w0=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
|
|
|
@ -6,5 +6,4 @@ package internal
|
|||
type HeightHash struct {
|
||||
Height uint64
|
||||
BlockHash []byte
|
||||
BlockHeader []byte
|
||||
}
|
||||
|
|
27
main.go
27
main.go
|
@ -10,10 +10,8 @@ import (
|
|||
"github.com/lbryio/herald.go/internal"
|
||||
pb "github.com/lbryio/herald.go/protobuf/go"
|
||||
"github.com/lbryio/herald.go/server"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
@ -29,22 +27,37 @@ func main() {
|
|||
|
||||
if args.CmdType == server.ServeCmd {
|
||||
// This will cancel goroutines with the server finishes.
|
||||
stopGroup := stop.New()
|
||||
ctxWCancel, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
initsignals()
|
||||
interrupt := interruptListener()
|
||||
|
||||
s := server.MakeHubServer(stopGroup, args)
|
||||
s := server.MakeHubServer(ctxWCancel, args)
|
||||
go s.Run()
|
||||
|
||||
defer s.Stop()
|
||||
defer func() {
|
||||
log.Println("Shutting down server...")
|
||||
|
||||
if s.EsClient != nil {
|
||||
s.EsClient.Stop()
|
||||
}
|
||||
if s.GrpcServer != nil {
|
||||
s.GrpcServer.GracefulStop()
|
||||
}
|
||||
if s.DB != nil {
|
||||
s.DB.Shutdown()
|
||||
}
|
||||
|
||||
log.Println("Returning from main...")
|
||||
}()
|
||||
|
||||
<-interrupt
|
||||
return
|
||||
}
|
||||
|
||||
conn, err := grpc.Dial("localhost:"+fmt.Sprintf("%d", args.Port),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
conn, err := grpc.Dial("localhost:"+args.Port,
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
if err != nil {
|
||||
|
|
|
@ -1,14 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# cicd_integration_test_runner.sh
|
||||
#
|
||||
# simple script to kick off herald and call the integration testing
|
||||
# script
|
||||
#
|
||||
# N.B. this currently just works locally until we figure a way to have
|
||||
# the data in the cicd environment.
|
||||
#
|
||||
|
||||
./herald serve --db-path /mnt/sdb1/wallet_server/_data/lbry-rocksdb &
|
||||
|
||||
./integration_tests.sh
|
|
@ -1,235 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# integration_testing.sh
|
||||
#
|
||||
# GitHub Action CI/CD based integration tests for herald.go
|
||||
# These are smoke / sanity tests for the server behaving correctly on a "live"
|
||||
# system, and looks for reasonable response codes, not specific correct
|
||||
# behavior. Those are covered in unit tests.
|
||||
#
|
||||
# N.B.
|
||||
# For the curl based json tests the `id` field existing is needed.
|
||||
#
|
||||
|
||||
# global variables
|
||||
|
||||
RES=(0)
|
||||
FINALRES=0
|
||||
# functions
|
||||
|
||||
|
||||
function logical_or {
|
||||
for res in ${RES[@]}; do
|
||||
if [ $res -eq 1 -o $FINALRES -eq 1 ]; then
|
||||
FINALRES=1
|
||||
return
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function want_got {
|
||||
if [ "${WANT}" != "${GOT}" ]; then
|
||||
echo "WANT: ${WANT}"
|
||||
echo "GOT: ${GOT}"
|
||||
RES+=(1)
|
||||
else
|
||||
RES+=(0)
|
||||
fi
|
||||
}
|
||||
|
||||
function want_greater {
|
||||
if [ ${WANT} -ge ${GOT} ]; then
|
||||
echo "WANT: ${WANT}"
|
||||
echo "GOT: ${GOT}"
|
||||
RES+=(1)
|
||||
else
|
||||
RES+=(0)
|
||||
fi
|
||||
}
|
||||
|
||||
function test_command_with_want {
|
||||
echo $CMD
|
||||
GOT=`eval $CMD`
|
||||
|
||||
want_got
|
||||
}
|
||||
|
||||
# grpc endpoint testing
|
||||
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
grpcurl -plaintext -d '{"value": ["@Styxhexenhammer666:2"]}' 127.0.0.1:50051 pb.Hub.Resolve
|
||||
| jq .txos[0].txHash | sed 's/"//g'
|
||||
EOM
|
||||
WANT="VOFP8MQEwps9Oa5NJJQ18WfVzUzlpCjst0Wz3xyOPd4="
|
||||
test_command_with_want
|
||||
|
||||
# GOT=`eval $CMD`
|
||||
|
||||
#want_got
|
||||
|
||||
##
|
||||
## N.B. This is a degenerate case that takes a long time to run.
|
||||
## The runtime should be fixed, but in the meantime, we definitely should
|
||||
## ensure this behaves as expected.
|
||||
##
|
||||
## TODO: Test runtime doesn't exceed worst case.
|
||||
##
|
||||
|
||||
#WANT=806389
|
||||
#read -r -d '' CMD <<- EOM
|
||||
# grpcurl -plaintext -d '{"value": ["foo"]}' 127.0.0.1:50051 pb.Hub.Resolve | jq .txos[0].height
|
||||
#EOM
|
||||
# test_command_with_want
|
||||
|
||||
# json rpc endpoint testing
|
||||
|
||||
## blockchain.block
|
||||
|
||||
### blockchain.block.get_chunk
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.block.get_chunk", "params": [0]}'
|
||||
| jq .result | sed 's/"//g' | head -c 100
|
||||
EOM
|
||||
WANT="010000000000000000000000000000000000000000000000000000000000000000000000cc59e59ff97ac092b55e423aa549"
|
||||
test_command_with_want
|
||||
|
||||
### blockchain.block.get_header
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.block.get_header", "params": []}'
|
||||
| jq .result.timestamp
|
||||
EOM
|
||||
WANT=1446058291
|
||||
test_command_with_want
|
||||
|
||||
### blockchain.block.headers
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.block.headers", "params": []}'
|
||||
| jq .result.count
|
||||
EOM
|
||||
WANT=0
|
||||
test_command_with_want
|
||||
|
||||
## blockchain.claimtrie
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.claimtrie.resolve", "params":[{"Data": ["@Styxhexenhammer666:2"]}]}'
|
||||
| jq .result.txos[0].tx_hash | sed 's/"//g'
|
||||
EOM
|
||||
WANT="VOFP8MQEwps9Oa5NJJQ18WfVzUzlpCjst0Wz3xyOPd4="
|
||||
test_command_with_want
|
||||
|
||||
## blockchain.address
|
||||
|
||||
### blockchain.address.get_balance
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.address.get_balance", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
|
||||
| jq .result.confirmed
|
||||
EOM
|
||||
WANT=44415602186
|
||||
test_command_with_want
|
||||
|
||||
## blockchain.address.get_history
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.address.get_history", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
|
||||
| jq '.result.confirmed | length'
|
||||
EOM
|
||||
WANT=82
|
||||
test_command_with_want
|
||||
|
||||
## blockchain.address.listunspent
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.address.listunspent", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
|
||||
| jq '.result | length'
|
||||
EOM
|
||||
WANT=32
|
||||
test_command_with_want
|
||||
|
||||
# blockchain.scripthash
|
||||
|
||||
## blockchain.scripthash.get_mempool
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.scripthash.get_mempool", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
|
||||
| jq .error | sed 's/"//g'
|
||||
EOM
|
||||
WANT="encoding/hex: invalid byte: U+0047 'G'"
|
||||
test_command_with_want
|
||||
|
||||
## blockchain.scripthash.get_history
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.scripthash.get_history", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
|
||||
| jq .error | sed 's/"//g'
|
||||
EOM
|
||||
WANT="encoding/hex: invalid byte: U+0047 'G'"
|
||||
test_command_with_want
|
||||
|
||||
## blockchain.scripthash.listunspent
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "blockchain.scripthash.listunspent", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
|
||||
| jq .error | sed 's/"//g'
|
||||
EOM
|
||||
WANT="encoding/hex: invalid byte: U+0047 'G'"
|
||||
test_command_with_want
|
||||
|
||||
## server.banner
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "server.banner", "params":[]}'
|
||||
| jq .result | sed 's/"//g'
|
||||
EOM
|
||||
WANT="You are connected to an 0.107.0 server."
|
||||
test_command_with_want
|
||||
|
||||
## server.version
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "server.version", "params":[]}'
|
||||
| jq .result | sed 's/"//g'
|
||||
EOM
|
||||
WANT="0.107.0"
|
||||
test_command_with_want
|
||||
|
||||
## server.features
|
||||
|
||||
read -r -d '' CMD <<- EOM
|
||||
curl http://127.0.0.1:50002/rpc -s -H "Content-Type: application/json"
|
||||
--data '{"id": 1, "method": "server.features", "params":[]}'
|
||||
EOM
|
||||
WANT='{"result":{"hosts":{},"pruning":"","server_version":"0.107.0","protocol_min":"0.54.0","protocol_max":"0.199.0","genesis_hash":"9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463","description":"Herald","payment_address":"","donation_address":"","daily_fee":"1.0","hash_function":"sha256","trending_algorithm":"fast_ar"},"error":null,"id":1}'
|
||||
test_command_with_want
|
||||
|
||||
# metrics endpoint testing
|
||||
|
||||
WANT=0
|
||||
GOT=$(curl http://127.0.0.1:2112/metrics -s | grep requests | grep resolve | awk '{print $NF}')
|
||||
want_greater
|
||||
|
||||
# caclulate return value
|
||||
|
||||
logical_or $RES
|
||||
|
||||
if [ $FINALRES -eq 1 ]; then
|
||||
echo "Failed!"
|
||||
exit 1
|
||||
else
|
||||
echo "Passed!"
|
||||
exit 0
|
||||
fi
|
200
server/args.go
200
server/args.go
|
@ -1,11 +1,8 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/akamensky/argparse"
|
||||
|
@ -23,37 +20,21 @@ const (
|
|||
type Args struct {
|
||||
CmdType int
|
||||
Host string
|
||||
Port int
|
||||
Port string
|
||||
DBPath string
|
||||
Chain *string
|
||||
DaemonURL *url.URL
|
||||
DaemonCAPath string
|
||||
EsHost string
|
||||
EsPort int
|
||||
PrometheusPort int
|
||||
NotifierPort int
|
||||
JSONRPCPort int
|
||||
JSONRPCHTTPPort int
|
||||
MaxSessions int
|
||||
SessionTimeout int
|
||||
EsPort string
|
||||
PrometheusPort string
|
||||
NotifierPort string
|
||||
JSONRPCPort string
|
||||
EsIndex string
|
||||
RefreshDelta int
|
||||
CacheTTL int
|
||||
PeerFile string
|
||||
Banner *string
|
||||
Country string
|
||||
BlockingChannelIds []string
|
||||
FilteringChannelIds []string
|
||||
|
||||
GenesisHash string
|
||||
ServerVersion string
|
||||
ProtocolMin string
|
||||
ProtocolMax string
|
||||
ServerDescription string
|
||||
PaymentAddress string
|
||||
DonationAddress string
|
||||
DailyFee string
|
||||
|
||||
Debug bool
|
||||
DisableEs bool
|
||||
DisableLoadPeers bool
|
||||
|
@ -70,31 +51,18 @@ type Args struct {
|
|||
|
||||
const (
|
||||
DefaultHost = "0.0.0.0"
|
||||
DefaultPort = 50051
|
||||
DefaultPort = "50051"
|
||||
DefaultDBPath = "/mnt/d/data/snapshot_1072108/lbry-rocksdb/" // FIXME
|
||||
DefaultEsHost = "http://localhost"
|
||||
DefaultEsIndex = "claims"
|
||||
DefaultEsPort = 9200
|
||||
DefaultPrometheusPort = 2112
|
||||
DefaultNotifierPort = 18080
|
||||
DefaultJSONRPCPort = 50001
|
||||
DefaultJSONRPCHTTPPort = 50002
|
||||
DefaultMaxSessions = 10000
|
||||
DefaultSessionTimeout = 300
|
||||
DefaultEsPort = "9200"
|
||||
DefaultPrometheusPort = "2112"
|
||||
DefaultNotifierPort = "18080"
|
||||
DefaultJSONRPCPort = "50001"
|
||||
DefaultRefreshDelta = 5
|
||||
DefaultCacheTTL = 5
|
||||
DefaultPeerFile = "peers.txt"
|
||||
DefaultBannerFile = ""
|
||||
DefaultCountry = "US"
|
||||
|
||||
HUB_PROTOCOL_VERSION = "0.107.0"
|
||||
PROTOCOL_MIN = "0.54.0"
|
||||
PROTOCOL_MAX = "0.199.0"
|
||||
DefaultServerDescription = "Herald"
|
||||
DefaultPaymentAddress = ""
|
||||
DefaultDonationAddress = ""
|
||||
DefaultDailyFee = "1.0"
|
||||
|
||||
DefaultDisableLoadPeers = false
|
||||
DefaultDisableStartPrometheus = false
|
||||
DefaultDisableStartUDP = false
|
||||
|
@ -112,73 +80,6 @@ var (
|
|||
DefaultFilteringChannelIds = []string{}
|
||||
)
|
||||
|
||||
func loadBanner(bannerFile *string, serverVersion string) *string {
|
||||
var banner string
|
||||
|
||||
data, err := os.ReadFile(*bannerFile)
|
||||
if err != nil {
|
||||
banner = fmt.Sprintf("You are connected to an %s server.", serverVersion)
|
||||
} else {
|
||||
banner = string(data)
|
||||
}
|
||||
|
||||
/*
|
||||
banner := os.Getenv("BANNER")
|
||||
if banner == "" {
|
||||
return nil
|
||||
}
|
||||
*/
|
||||
|
||||
return &banner
|
||||
}
|
||||
|
||||
// MakeDefaultArgs creates a default set of arguments for testing the server.
|
||||
func MakeDefaultTestArgs() *Args {
|
||||
args := &Args{
|
||||
CmdType: ServeCmd,
|
||||
Host: DefaultHost,
|
||||
Port: DefaultPort,
|
||||
DBPath: DefaultDBPath,
|
||||
EsHost: DefaultEsHost,
|
||||
EsPort: DefaultEsPort,
|
||||
PrometheusPort: DefaultPrometheusPort,
|
||||
NotifierPort: DefaultNotifierPort,
|
||||
JSONRPCPort: DefaultJSONRPCPort,
|
||||
JSONRPCHTTPPort: DefaultJSONRPCHTTPPort,
|
||||
MaxSessions: DefaultMaxSessions,
|
||||
SessionTimeout: DefaultSessionTimeout,
|
||||
EsIndex: DefaultEsIndex,
|
||||
RefreshDelta: DefaultRefreshDelta,
|
||||
CacheTTL: DefaultCacheTTL,
|
||||
PeerFile: DefaultPeerFile,
|
||||
Banner: nil,
|
||||
Country: DefaultCountry,
|
||||
|
||||
GenesisHash: chaincfg.TestNet3Params.GenesisHash.String(),
|
||||
ServerVersion: HUB_PROTOCOL_VERSION,
|
||||
ProtocolMin: PROTOCOL_MIN,
|
||||
ProtocolMax: PROTOCOL_MAX,
|
||||
ServerDescription: DefaultServerDescription,
|
||||
PaymentAddress: DefaultPaymentAddress,
|
||||
DonationAddress: DefaultDonationAddress,
|
||||
DailyFee: DefaultDailyFee,
|
||||
|
||||
DisableEs: true,
|
||||
Debug: true,
|
||||
DisableLoadPeers: true,
|
||||
DisableStartPrometheus: true,
|
||||
DisableStartUDP: true,
|
||||
DisableWritePeers: true,
|
||||
DisableRocksDBRefresh: true,
|
||||
DisableResolve: true,
|
||||
DisableBlockingAndFiltering: true,
|
||||
DisableStartNotifier: true,
|
||||
DisableStartJSONRPC: true,
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
// GetEnvironment takes the environment variables as an array of strings
|
||||
// and a getkeyval function to turn it into a map.
|
||||
func GetEnvironment(data []string, getkeyval func(item string) (key, val string)) map[string]string {
|
||||
|
@ -204,58 +105,30 @@ func GetEnvironmentStandard() map[string]string {
|
|||
func ParseArgs(searchRequest *pb.SearchRequest) *Args {
|
||||
|
||||
environment := GetEnvironmentStandard()
|
||||
parser := argparse.NewParser("herald", "herald server and client")
|
||||
parser := argparse.NewParser("hub", "hub server and client")
|
||||
|
||||
serveCmd := parser.NewCommand("serve", "start the herald server")
|
||||
serveCmd := parser.NewCommand("serve", "start the hub server")
|
||||
searchCmd := parser.NewCommand("search", "claim search")
|
||||
dbCmd := parser.NewCommand("db", "db testing")
|
||||
|
||||
defaultDaemonURL := "http://localhost:9245"
|
||||
if url, ok := environment["DAEMON_URL"]; ok {
|
||||
defaultDaemonURL = url
|
||||
}
|
||||
|
||||
validateURL := func(arg []string) error {
|
||||
_, err := url.Parse(arg[0])
|
||||
return err
|
||||
}
|
||||
validatePort := func(arg []string) error {
|
||||
_, err := strconv.ParseUint(arg[0], 10, 16)
|
||||
return err
|
||||
}
|
||||
|
||||
// main server config arguments
|
||||
host := parser.String("", "rpchost", &argparse.Options{Required: false, Help: "RPC host", Default: DefaultHost})
|
||||
port := parser.Int("", "rpcport", &argparse.Options{Required: false, Help: "RPC port", Validate: validatePort, Default: DefaultPort})
|
||||
port := parser.String("", "rpcport", &argparse.Options{Required: false, Help: "RPC port", Default: DefaultPort})
|
||||
dbPath := parser.String("", "db-path", &argparse.Options{Required: false, Help: "RocksDB path", Default: DefaultDBPath})
|
||||
chain := parser.Selector("", "chain", []string{chaincfg.MainNetParams.Name, chaincfg.TestNet3Params.Name, chaincfg.RegressionNetParams.Name, "testnet"},
|
||||
&argparse.Options{Required: false, Help: "Which chain to use, default is 'mainnet'. Values 'regtest' and 'testnet' are for testing", Default: chaincfg.MainNetParams.Name})
|
||||
daemonURLStr := parser.String("", "daemon-url", &argparse.Options{Required: false, Help: "URL for rpc to lbrycrd or lbcd, <rpcuser>:<rpcpassword>@<lbcd rpc ip><lbrcd rpc port>.", Validate: validateURL, Default: defaultDaemonURL})
|
||||
daemonCAPath := parser.String("", "daemon-ca-path", &argparse.Options{Required: false, Help: "Path to the lbcd CA file. Use SSL certificate to verify connection to lbcd."})
|
||||
esHost := parser.String("", "eshost", &argparse.Options{Required: false, Help: "elasticsearch host", Default: DefaultEsHost})
|
||||
esPort := parser.Int("", "esport", &argparse.Options{Required: false, Help: "elasticsearch port", Default: DefaultEsPort})
|
||||
prometheusPort := parser.Int("", "prometheus-port", &argparse.Options{Required: false, Help: "prometheus port", Default: DefaultPrometheusPort})
|
||||
notifierPort := parser.Int("", "notifier-port", &argparse.Options{Required: false, Help: "notifier port", Default: DefaultNotifierPort})
|
||||
jsonRPCPort := parser.Int("", "json-rpc-port", &argparse.Options{Required: false, Help: "JSON RPC port", Validate: validatePort, Default: DefaultJSONRPCPort})
|
||||
jsonRPCHTTPPort := parser.Int("", "json-rpc-http-port", &argparse.Options{Required: false, Help: "JSON RPC over HTTP port", Validate: validatePort, Default: DefaultJSONRPCHTTPPort})
|
||||
maxSessions := parser.Int("", "max-sessions", &argparse.Options{Required: false, Help: "Maximum number of electrum clients that can be connected", Default: DefaultMaxSessions})
|
||||
sessionTimeout := parser.Int("", "session-timeout", &argparse.Options{Required: false, Help: "Session inactivity timeout (seconds)", Default: DefaultSessionTimeout})
|
||||
esPort := parser.String("", "esport", &argparse.Options{Required: false, Help: "elasticsearch port", Default: DefaultEsPort})
|
||||
prometheusPort := parser.String("", "prometheus-port", &argparse.Options{Required: false, Help: "prometheus port", Default: DefaultPrometheusPort})
|
||||
notifierPort := parser.String("", "notifier-port", &argparse.Options{Required: false, Help: "notifier port", Default: DefaultNotifierPort})
|
||||
jsonRPCPort := parser.String("", "json-rpc-port", &argparse.Options{Required: false, Help: "JSON RPC port", Default: DefaultJSONRPCPort})
|
||||
esIndex := parser.String("", "esindex", &argparse.Options{Required: false, Help: "elasticsearch index name", Default: DefaultEsIndex})
|
||||
refreshDelta := parser.Int("", "refresh-delta", &argparse.Options{Required: false, Help: "elasticsearch index refresh delta in seconds", Default: DefaultRefreshDelta})
|
||||
cacheTTL := parser.Int("", "cachettl", &argparse.Options{Required: false, Help: "Cache TTL in minutes", Default: DefaultCacheTTL})
|
||||
peerFile := parser.String("", "peerfile", &argparse.Options{Required: false, Help: "Initial peer file for federation", Default: DefaultPeerFile})
|
||||
bannerFile := parser.String("", "bannerfile", &argparse.Options{Required: false, Help: "Banner file server.banner", Default: DefaultBannerFile})
|
||||
country := parser.String("", "country", &argparse.Options{Required: false, Help: "Country this node is running in. Default US.", Default: DefaultCountry})
|
||||
blockingChannelIds := parser.StringList("", "blocking-channel-ids", &argparse.Options{Required: false, Help: "Blocking channel ids", Default: DefaultBlockingChannelIds})
|
||||
filteringChannelIds := parser.StringList("", "filtering-channel-ids", &argparse.Options{Required: false, Help: "Filtering channel ids", Default: DefaultFilteringChannelIds})
|
||||
|
||||
// arguments for server features
|
||||
serverDescription := parser.String("", "server-description", &argparse.Options{Required: false, Help: "Server description", Default: DefaultServerDescription})
|
||||
paymentAddress := parser.String("", "payment-address", &argparse.Options{Required: false, Help: "Payment address", Default: DefaultPaymentAddress})
|
||||
donationAddress := parser.String("", "donation-address", &argparse.Options{Required: false, Help: "Donation address", Default: DefaultDonationAddress})
|
||||
dailyFee := parser.String("", "daily-fee", &argparse.Options{Required: false, Help: "Daily fee", Default: DefaultDailyFee})
|
||||
|
||||
// flags for disabling features
|
||||
debug := parser.Flag("", "debug", &argparse.Options{Required: false, Help: "enable debug logging", Default: false})
|
||||
disableEs := parser.Flag("", "disable-es", &argparse.Options{Required: false, Help: "Disable elastic search, for running/testing independently", Default: false})
|
||||
disableLoadPeers := parser.Flag("", "disable-load-peers", &argparse.Options{Required: false, Help: "Disable load peers from disk at startup", Default: DefaultDisableLoadPeers})
|
||||
|
@ -269,7 +142,6 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
|
|||
disableStartNotifier := parser.Flag("", "disable-start-notifier", &argparse.Options{Required: false, Help: "Disable start notifier", Default: DisableStartNotifier})
|
||||
disableStartJSONRPC := parser.Flag("", "disable-start-jsonrpc", &argparse.Options{Required: false, Help: "Disable start jsonrpc endpoint", Default: DisableStartJSONRPC})
|
||||
|
||||
// search command arguments
|
||||
text := parser.String("", "text", &argparse.Options{Required: false, Help: "text query"})
|
||||
name := parser.String("", "name", &argparse.Options{Required: false, Help: "name"})
|
||||
claimType := parser.String("", "claim_type", &argparse.Options{Required: false, Help: "claim_type"})
|
||||
|
@ -286,52 +158,24 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
|
|||
log.Fatalln(parser.Usage(err))
|
||||
}
|
||||
|
||||
// Use default JSON RPC port only if *neither* JSON RPC arg is specified.
|
||||
if *jsonRPCPort == 0 && *jsonRPCHTTPPort == 0 {
|
||||
*jsonRPCPort = DefaultJSONRPCPort
|
||||
}
|
||||
|
||||
daemonURL, err := url.Parse(*daemonURLStr)
|
||||
if err != nil {
|
||||
log.Fatalf("URL parse failed: %v", err)
|
||||
}
|
||||
|
||||
banner := loadBanner(bannerFile, HUB_PROTOCOL_VERSION)
|
||||
|
||||
args := &Args{
|
||||
CmdType: SearchCmd,
|
||||
Host: *host,
|
||||
Port: *port,
|
||||
DBPath: *dbPath,
|
||||
Chain: chain,
|
||||
DaemonURL: daemonURL,
|
||||
DaemonCAPath: *daemonCAPath,
|
||||
EsHost: *esHost,
|
||||
EsPort: *esPort,
|
||||
PrometheusPort: *prometheusPort,
|
||||
NotifierPort: *notifierPort,
|
||||
JSONRPCPort: *jsonRPCPort,
|
||||
JSONRPCHTTPPort: *jsonRPCHTTPPort,
|
||||
MaxSessions: *maxSessions,
|
||||
SessionTimeout: *sessionTimeout,
|
||||
EsIndex: *esIndex,
|
||||
RefreshDelta: *refreshDelta,
|
||||
CacheTTL: *cacheTTL,
|
||||
PeerFile: *peerFile,
|
||||
Banner: banner,
|
||||
Country: *country,
|
||||
BlockingChannelIds: *blockingChannelIds,
|
||||
FilteringChannelIds: *filteringChannelIds,
|
||||
|
||||
GenesisHash: "",
|
||||
ServerVersion: HUB_PROTOCOL_VERSION,
|
||||
ProtocolMin: PROTOCOL_MIN,
|
||||
ProtocolMax: PROTOCOL_MAX,
|
||||
ServerDescription: *serverDescription,
|
||||
PaymentAddress: *paymentAddress,
|
||||
DonationAddress: *donationAddress,
|
||||
DailyFee: *dailyFee,
|
||||
|
||||
Debug: *debug,
|
||||
DisableEs: *disableEs,
|
||||
DisableLoadPeers: *disableLoadPeers,
|
||||
|
@ -355,17 +199,11 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
|
|||
}
|
||||
|
||||
if esPort, ok := environment["ELASTIC_PORT"]; ok {
|
||||
args.EsPort, err = strconv.Atoi(esPort)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
args.EsPort = esPort
|
||||
}
|
||||
|
||||
if prometheusPort, ok := environment["GOHUB_PROMETHEUS_PORT"]; ok {
|
||||
args.PrometheusPort, err = strconv.Atoi(prometheusPort)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
args.PrometheusPort = prometheusPort
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
@ -3,19 +3,17 @@ package server
|
|||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"log"
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/herald.go/internal/metrics"
|
||||
pb "github.com/lbryio/herald.go/protobuf/go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// Peer holds relevant information about peers that we know about.
|
||||
|
@ -88,7 +86,7 @@ func (s *Server) getAndSetExternalIp(ip, port string) error {
|
|||
// storing them as known peers. Returns a map of peerKey -> object
|
||||
func (s *Server) loadPeers() error {
|
||||
peerFile := s.Args.PeerFile
|
||||
port := strconv.Itoa(s.Args.Port)
|
||||
port := s.Args.Port
|
||||
|
||||
// First we make sure our server has come up, so we can answer back to peers.
|
||||
var failures = 0
|
||||
|
@ -100,7 +98,7 @@ retry:
|
|||
time.Sleep(time.Second * time.Duration(math.Pow(float64(failures), 2)))
|
||||
conn, err := grpc.DialContext(ctx,
|
||||
"0.0.0.0:"+port,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
|
||||
|
@ -173,7 +171,7 @@ func (s *Server) subscribeToPeer(peer *Peer) error {
|
|||
|
||||
conn, err := grpc.DialContext(ctx,
|
||||
peer.Address+":"+peer.Port,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
if err != nil {
|
||||
|
@ -183,12 +181,12 @@ func (s *Server) subscribeToPeer(peer *Peer) error {
|
|||
|
||||
msg := &pb.ServerMessage{
|
||||
Address: s.ExternalIP.String(),
|
||||
Port: strconv.Itoa(s.Args.Port),
|
||||
Port: s.Args.Port,
|
||||
}
|
||||
|
||||
c := pb.NewHubClient(conn)
|
||||
|
||||
log.Printf("%s:%d subscribing to %+v\n", s.ExternalIP, s.Args.Port, peer)
|
||||
log.Printf("%s:%s subscribing to %+v\n", s.ExternalIP, s.Args.Port, peer)
|
||||
_, err = c.PeerSubscribe(ctx, msg)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -209,7 +207,7 @@ func (s *Server) helloPeer(peer *Peer) (*pb.HelloMessage, error) {
|
|||
|
||||
conn, err := grpc.DialContext(ctx,
|
||||
peer.Address+":"+peer.Port,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
if err != nil {
|
||||
|
@ -221,12 +219,12 @@ func (s *Server) helloPeer(peer *Peer) (*pb.HelloMessage, error) {
|
|||
c := pb.NewHubClient(conn)
|
||||
|
||||
msg := &pb.HelloMessage{
|
||||
Port: strconv.Itoa(s.Args.Port),
|
||||
Port: s.Args.Port,
|
||||
Host: s.ExternalIP.String(),
|
||||
Servers: []*pb.ServerMessage{},
|
||||
}
|
||||
|
||||
log.Printf("%s:%d saying hello to %+v\n", s.ExternalIP, s.Args.Port, peer)
|
||||
log.Printf("%s:%s saying hello to %+v\n", s.ExternalIP, s.Args.Port, peer)
|
||||
res, err := c.Hello(ctx, msg)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
|
@ -279,7 +277,7 @@ func (s *Server) notifyPeer(peerToNotify *Peer, newPeer *Peer) error {
|
|||
|
||||
conn, err := grpc.DialContext(ctx,
|
||||
peerToNotify.Address+":"+peerToNotify.Port,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
if err != nil {
|
||||
|
@ -347,15 +345,15 @@ func (s *Server) addPeer(newPeer *Peer, ping bool, subscribe bool) error {
|
|||
}
|
||||
}
|
||||
|
||||
if strconv.Itoa(s.Args.Port) == newPeer.Port &&
|
||||
if s.Args.Port == newPeer.Port &&
|
||||
(localHosts[newPeer.Address] || newPeer.Address == s.ExternalIP.String()) {
|
||||
log.Printf("%s:%d addPeer: Self peer, skipping...\n", s.ExternalIP, s.Args.Port)
|
||||
log.Printf("%s:%s addPeer: Self peer, skipping...\n", s.ExternalIP, s.Args.Port)
|
||||
return nil
|
||||
}
|
||||
|
||||
k := peerKey(newPeer)
|
||||
|
||||
log.Printf("%s:%d adding peer %+v\n", s.ExternalIP, s.Args.Port, newPeer)
|
||||
log.Printf("%s:%s adding peer %+v\n", s.ExternalIP, s.Args.Port, newPeer)
|
||||
if oldServer, loaded := s.PeerServersLoadOrStore(newPeer); !loaded {
|
||||
if ping {
|
||||
_, err := s.helloPeer(newPeer)
|
||||
|
@ -371,10 +369,6 @@ func (s *Server) addPeer(newPeer *Peer, ping bool, subscribe bool) error {
|
|||
metrics.PeersKnown.Inc()
|
||||
s.writePeers()
|
||||
s.notifyPeerSubs(newPeer)
|
||||
// This is weird because we're doing grpc and jsonrpc here.
|
||||
// Do we still want to custom grpc?
|
||||
log.Warn("Sending peer to NotifierChan")
|
||||
s.NotifierChan <- peerNotification{newPeer.Address, newPeer.Port}
|
||||
|
||||
// Subscribe to all our peers for now
|
||||
if subscribe {
|
||||
|
@ -421,7 +415,7 @@ func (s *Server) makeHelloMessage() *pb.HelloMessage {
|
|||
s.PeerServersMut.RUnlock()
|
||||
|
||||
return &pb.HelloMessage{
|
||||
Port: strconv.Itoa(s.Args.Port),
|
||||
Port: s.Args.Port,
|
||||
Host: s.ExternalIP.String(),
|
||||
Servers: servers,
|
||||
}
|
||||
|
|
|
@ -4,31 +4,23 @@ import (
|
|||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/herald.go/internal/metrics"
|
||||
pb "github.com/lbryio/herald.go/protobuf/go"
|
||||
"github.com/lbryio/herald.go/server"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
server "github.com/lbryio/herald.go/server"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// lineCountFile takes a fileName and counts the number of lines in it.
|
||||
func lineCountFile(fileName string) int {
|
||||
f, err := os.Open(fileName)
|
||||
defer func() {
|
||||
err := f.Close()
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
}
|
||||
}()
|
||||
defer f.Close()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return 0
|
||||
|
@ -52,12 +44,43 @@ func removeFile(fileName string) {
|
|||
}
|
||||
}
|
||||
|
||||
// makeDefaultArgs creates a default set of arguments for testing the server.
|
||||
func makeDefaultArgs() *server.Args {
|
||||
args := &server.Args{
|
||||
CmdType: server.ServeCmd,
|
||||
Host: server.DefaultHost,
|
||||
Port: server.DefaultPort,
|
||||
DBPath: server.DefaultDBPath,
|
||||
EsHost: server.DefaultEsHost,
|
||||
EsPort: server.DefaultEsPort,
|
||||
PrometheusPort: server.DefaultPrometheusPort,
|
||||
NotifierPort: server.DefaultNotifierPort,
|
||||
JSONRPCPort: server.DefaultJSONRPCPort,
|
||||
EsIndex: server.DefaultEsIndex,
|
||||
RefreshDelta: server.DefaultRefreshDelta,
|
||||
CacheTTL: server.DefaultCacheTTL,
|
||||
PeerFile: server.DefaultPeerFile,
|
||||
Country: server.DefaultCountry,
|
||||
DisableEs: true,
|
||||
Debug: true,
|
||||
DisableLoadPeers: true,
|
||||
DisableStartPrometheus: true,
|
||||
DisableStartUDP: true,
|
||||
DisableWritePeers: true,
|
||||
DisableRocksDBRefresh: true,
|
||||
DisableResolve: true,
|
||||
DisableBlockingAndFiltering: true,
|
||||
DisableStartNotifier: true,
|
||||
DisableStartJSONRPC: true,
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
// TestAddPeer tests the ability to add peers
|
||||
func TestAddPeer(t *testing.T) {
|
||||
// ctx := context.Background()
|
||||
ctx := stop.NewDebug()
|
||||
args := server.MakeDefaultTestArgs()
|
||||
args.DisableStartNotifier = false
|
||||
ctx := context.Background()
|
||||
args := makeDefaultArgs()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
@ -107,7 +130,6 @@ func TestAddPeer(t *testing.T) {
|
|||
if got != tt.want {
|
||||
t.Errorf("len(server.PeerServers) = %d, want %d\n", got, tt.want)
|
||||
}
|
||||
hubServer.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -115,10 +137,9 @@ func TestAddPeer(t *testing.T) {
|
|||
|
||||
// TestPeerWriter tests that peers get written properly
|
||||
func TestPeerWriter(t *testing.T) {
|
||||
ctx := stop.NewDebug()
|
||||
args := server.MakeDefaultTestArgs()
|
||||
ctx := context.Background()
|
||||
args := makeDefaultArgs()
|
||||
args.DisableWritePeers = false
|
||||
args.DisableStartNotifier = false
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
@ -153,16 +174,17 @@ func TestPeerWriter(t *testing.T) {
|
|||
Port: "50051",
|
||||
}
|
||||
}
|
||||
//log.Printf("Adding peer %+v\n", peer)
|
||||
err := hubServer.AddPeerExported()(peer, false, false)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
//log.Println("Counting lines...")
|
||||
got := lineCountFile(hubServer.Args.PeerFile)
|
||||
if got != tt.want {
|
||||
t.Errorf("lineCountFile(peers.txt) = %d, want %d", got, tt.want)
|
||||
}
|
||||
hubServer.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -171,13 +193,10 @@ func TestPeerWriter(t *testing.T) {
|
|||
|
||||
// TestAddPeerEndpoint tests the ability to add peers
|
||||
func TestAddPeerEndpoint(t *testing.T) {
|
||||
ctx := stop.NewDebug()
|
||||
args := server.MakeDefaultTestArgs()
|
||||
args.DisableStartNotifier = false
|
||||
args2 := server.MakeDefaultTestArgs()
|
||||
args2.DisableStartNotifier = false
|
||||
args2.Port = 50052
|
||||
args2.NotifierPort = 18081
|
||||
ctx := context.Background()
|
||||
args := makeDefaultArgs()
|
||||
args2 := makeDefaultArgs()
|
||||
args2.Port = "50052"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
@ -207,8 +226,9 @@ func TestAddPeerEndpoint(t *testing.T) {
|
|||
metrics.PeersKnown.Set(0)
|
||||
go hubServer.Run()
|
||||
go hubServer2.Run()
|
||||
conn, err := grpc.Dial("localhost:"+strconv.Itoa(args.Port),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
//go hubServer.Run()
|
||||
conn, err := grpc.Dial("localhost:"+args.Port,
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
if err != nil {
|
||||
|
@ -227,6 +247,8 @@ func TestAddPeerEndpoint(t *testing.T) {
|
|||
log.Println(err)
|
||||
}
|
||||
|
||||
hubServer.GrpcServer.GracefulStop()
|
||||
hubServer2.GrpcServer.GracefulStop()
|
||||
got1 := hubServer.GetNumPeersExported()()
|
||||
got2 := hubServer2.GetNumPeersExported()()
|
||||
if got1 != tt.wantServerOne {
|
||||
|
@ -235,8 +257,6 @@ func TestAddPeerEndpoint(t *testing.T) {
|
|||
if got2 != tt.wantServerTwo {
|
||||
t.Errorf("len(hubServer2.PeerServers) = %d, want %d\n", got2, tt.wantServerTwo)
|
||||
}
|
||||
hubServer.Stop()
|
||||
hubServer2.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -244,17 +264,12 @@ func TestAddPeerEndpoint(t *testing.T) {
|
|||
|
||||
// TestAddPeerEndpoint2 tests the ability to add peers
|
||||
func TestAddPeerEndpoint2(t *testing.T) {
|
||||
ctx := stop.NewDebug()
|
||||
args := server.MakeDefaultTestArgs()
|
||||
args2 := server.MakeDefaultTestArgs()
|
||||
args3 := server.MakeDefaultTestArgs()
|
||||
args2.Port = 50052
|
||||
args3.Port = 50053
|
||||
args.DisableStartNotifier = false
|
||||
args2.DisableStartNotifier = false
|
||||
args3.DisableStartNotifier = false
|
||||
args2.NotifierPort = 18081
|
||||
args3.NotifierPort = 18082
|
||||
ctx := context.Background()
|
||||
args := makeDefaultArgs()
|
||||
args2 := makeDefaultArgs()
|
||||
args3 := makeDefaultArgs()
|
||||
args2.Port = "50052"
|
||||
args3.Port = "50053"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
@ -279,8 +294,8 @@ func TestAddPeerEndpoint2(t *testing.T) {
|
|||
go hubServer.Run()
|
||||
go hubServer2.Run()
|
||||
go hubServer3.Run()
|
||||
conn, err := grpc.Dial("localhost:"+strconv.Itoa(args.Port),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
conn, err := grpc.Dial("localhost:"+args.Port,
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
if err != nil {
|
||||
|
@ -308,6 +323,9 @@ func TestAddPeerEndpoint2(t *testing.T) {
|
|||
log.Println(err)
|
||||
}
|
||||
|
||||
hubServer.GrpcServer.GracefulStop()
|
||||
hubServer2.GrpcServer.GracefulStop()
|
||||
hubServer3.GrpcServer.GracefulStop()
|
||||
got1 := hubServer.GetNumPeersExported()()
|
||||
got2 := hubServer2.GetNumPeersExported()()
|
||||
got3 := hubServer3.GetNumPeersExported()()
|
||||
|
@ -320,9 +338,6 @@ func TestAddPeerEndpoint2(t *testing.T) {
|
|||
if got3 != tt.wantServerThree {
|
||||
t.Errorf("len(hubServer3.PeerServers) = %d, want %d\n", got3, tt.wantServerThree)
|
||||
}
|
||||
hubServer.Stop()
|
||||
hubServer2.Stop()
|
||||
hubServer3.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -330,17 +345,12 @@ func TestAddPeerEndpoint2(t *testing.T) {
|
|||
|
||||
// TestAddPeerEndpoint3 tests the ability to add peers
|
||||
func TestAddPeerEndpoint3(t *testing.T) {
|
||||
ctx := stop.NewDebug()
|
||||
args := server.MakeDefaultTestArgs()
|
||||
args2 := server.MakeDefaultTestArgs()
|
||||
args3 := server.MakeDefaultTestArgs()
|
||||
args2.Port = 50052
|
||||
args3.Port = 50053
|
||||
args.DisableStartNotifier = false
|
||||
args2.DisableStartNotifier = false
|
||||
args3.DisableStartNotifier = false
|
||||
args2.NotifierPort = 18081
|
||||
args3.NotifierPort = 18082
|
||||
ctx := context.Background()
|
||||
args := makeDefaultArgs()
|
||||
args2 := makeDefaultArgs()
|
||||
args3 := makeDefaultArgs()
|
||||
args2.Port = "50052"
|
||||
args3.Port = "50053"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
@ -365,15 +375,15 @@ func TestAddPeerEndpoint3(t *testing.T) {
|
|||
go hubServer.Run()
|
||||
go hubServer2.Run()
|
||||
go hubServer3.Run()
|
||||
conn, err := grpc.Dial("localhost:"+strconv.Itoa(args.Port),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
conn, err := grpc.Dial("localhost:"+args.Port,
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("did not connect: %v", err)
|
||||
}
|
||||
conn2, err := grpc.Dial("localhost:50052",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
if err != nil {
|
||||
|
@ -402,9 +412,9 @@ func TestAddPeerEndpoint3(t *testing.T) {
|
|||
log.Println(err)
|
||||
}
|
||||
|
||||
hubServer.Stop()
|
||||
hubServer2.Stop()
|
||||
hubServer3.Stop()
|
||||
hubServer.GrpcServer.GracefulStop()
|
||||
hubServer2.GrpcServer.GracefulStop()
|
||||
hubServer3.GrpcServer.GracefulStop()
|
||||
got1 := hubServer.GetNumPeersExported()()
|
||||
got2 := hubServer2.GetNumPeersExported()()
|
||||
got3 := hubServer3.GetNumPeersExported()()
|
||||
|
@ -424,11 +434,11 @@ func TestAddPeerEndpoint3(t *testing.T) {
|
|||
|
||||
// TestAddPeer tests the ability to add peers
|
||||
func TestUDPServer(t *testing.T) {
|
||||
ctx := stop.NewDebug()
|
||||
args := server.MakeDefaultTestArgs()
|
||||
args2 := server.MakeDefaultTestArgs()
|
||||
args2.Port = 50052
|
||||
ctx := context.Background()
|
||||
args := makeDefaultArgs()
|
||||
args.DisableStartUDP = false
|
||||
args2 := makeDefaultArgs()
|
||||
args2.Port = "50052"
|
||||
args2.DisableStartUDP = false
|
||||
|
||||
tests := []struct {
|
||||
|
@ -459,18 +469,18 @@ func TestUDPServer(t *testing.T) {
|
|||
log.Println(err)
|
||||
}
|
||||
|
||||
hubServer.Stop()
|
||||
hubServer2.Stop()
|
||||
hubServer.GrpcServer.GracefulStop()
|
||||
hubServer2.GrpcServer.GracefulStop()
|
||||
|
||||
got1 := hubServer.ExternalIP.String()
|
||||
if got1 != tt.want {
|
||||
t.Errorf("hubServer.ExternalIP = %s, want %s\n", got1, tt.want)
|
||||
t.Errorf("hubServer.Args.Port = %d\n", hubServer.Args.Port)
|
||||
t.Errorf("hubServer.Args.Port = %s\n", hubServer.Args.Port)
|
||||
}
|
||||
got2 := hubServer2.ExternalIP.String()
|
||||
if got2 != tt.want {
|
||||
t.Errorf("hubServer2.ExternalIP = %s, want %s\n", got2, tt.want)
|
||||
t.Errorf("hubServer2.Args.Port = %d\n", hubServer2.Args.Port)
|
||||
t.Errorf("hubServer2.Args.Port = %s\n", hubServer2.Args.Port)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
@ -7,10 +7,12 @@ import (
|
|||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/rpc"
|
||||
"github.com/lbryio/herald.go/db"
|
||||
"github.com/lbryio/herald.go/internal"
|
||||
"github.com/lbryio/lbcd/chaincfg"
|
||||
|
@ -19,49 +21,58 @@ import (
|
|||
"github.com/lbryio/lbcd/wire"
|
||||
"github.com/lbryio/lbcutil"
|
||||
"golang.org/x/exp/constraints"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// BlockchainBlockService methods handle "blockchain.block.*" RPCs
|
||||
type BlockchainBlockService struct {
|
||||
DB *db.ReadOnlyDBColumnFamily
|
||||
Chain *chaincfg.Params
|
||||
type BlockchainCodec struct {
|
||||
rpc.Codec
|
||||
}
|
||||
|
||||
// BlockchainBlockService methods handle "blockchain.headers.*" RPCs
|
||||
type BlockchainHeadersService struct {
|
||||
func (c *BlockchainCodec) NewRequest(r *http.Request) rpc.CodecRequest {
|
||||
return &BlockchainCodecRequest{c.Codec.NewRequest(r)}
|
||||
}
|
||||
|
||||
// BlockchainCodecRequest provides ability to rewrite the incoming
|
||||
// request "method" field. For example:
|
||||
// blockchain.block.get_header -> blockchain_block.Get_header
|
||||
// blockchain.address.listunspent -> blockchain_address.Listunspent
|
||||
// This makes the "method" string compatible with Gorilla/RPC
|
||||
// requirements.
|
||||
type BlockchainCodecRequest struct {
|
||||
rpc.CodecRequest
|
||||
}
|
||||
|
||||
func (cr *BlockchainCodecRequest) Method() (string, error) {
|
||||
rawMethod, err := cr.CodecRequest.Method()
|
||||
if err != nil {
|
||||
return rawMethod, err
|
||||
}
|
||||
parts := strings.Split(rawMethod, ".")
|
||||
if len(parts) < 2 {
|
||||
return rawMethod, fmt.Errorf("blockchain rpc: service/method ill-formed: %q", rawMethod)
|
||||
}
|
||||
service := strings.Join(parts[0:len(parts)-1], "_")
|
||||
method := parts[len(parts)-1]
|
||||
if len(method) < 1 {
|
||||
return rawMethod, fmt.Errorf("blockchain rpc: method ill-formed: %q", method)
|
||||
}
|
||||
method = strings.ToUpper(string(method[0])) + string(method[1:])
|
||||
return service + "." + method, err
|
||||
}
|
||||
|
||||
// BlockchainService methods handle "blockchain.block.*" RPCs
|
||||
type BlockchainService struct {
|
||||
DB *db.ReadOnlyDBColumnFamily
|
||||
Chain *chaincfg.Params
|
||||
// needed for subscribe/unsubscribe
|
||||
sessionMgr *sessionManager
|
||||
session *session
|
||||
}
|
||||
|
||||
// BlockchainAddressService methods handle "blockchain.address.*" RPCs
|
||||
type BlockchainAddressService struct {
|
||||
DB *db.ReadOnlyDBColumnFamily
|
||||
Chain *chaincfg.Params
|
||||
// needed for subscribe/unsubscribe
|
||||
sessionMgr *sessionManager
|
||||
session *session
|
||||
BlockchainService
|
||||
}
|
||||
|
||||
// BlockchainScripthashService methods handle "blockchain.scripthash.*" RPCs
|
||||
type BlockchainScripthashService struct {
|
||||
DB *db.ReadOnlyDBColumnFamily
|
||||
Chain *chaincfg.Params
|
||||
// needed for subscribe/unsubscribe
|
||||
sessionMgr *sessionManager
|
||||
session *session
|
||||
}
|
||||
|
||||
// BlockchainTransactionService methods handle "blockchain.transaction.*" RPCs
|
||||
type BlockchainTransactionService struct {
|
||||
DB *db.ReadOnlyDBColumnFamily
|
||||
Chain *chaincfg.Params
|
||||
// needed for broadcast TX
|
||||
sessionMgr *sessionManager
|
||||
BlockchainService
|
||||
}
|
||||
|
||||
const CHUNK_SIZE = 96
|
||||
|
@ -76,46 +87,10 @@ func min[Ord constraints.Ordered](x, y Ord) Ord {
|
|||
return y
|
||||
}
|
||||
|
||||
func max[Ord constraints.Ordered](x, y Ord) Ord {
|
||||
if x > y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
|
||||
type BlockHeaderElectrum struct {
|
||||
Version uint32 `json:"version"`
|
||||
PrevBlockHash string `json:"prev_block_hash"`
|
||||
MerkleRoot string `json:"merkle_root"`
|
||||
ClaimTrieRoot string `json:"claim_trie_root"`
|
||||
Timestamp uint32 `json:"timestamp"`
|
||||
Bits uint32 `json:"bits"`
|
||||
Nonce uint32 `json:"nonce"`
|
||||
BlockHeight uint32 `json:"block_height"`
|
||||
}
|
||||
|
||||
func newBlockHeaderElectrum(header *[HEADER_SIZE]byte, height uint32) *BlockHeaderElectrum {
|
||||
var h1, h2, h3 chainhash.Hash
|
||||
h1.SetBytes(header[4:36])
|
||||
h2.SetBytes(header[36:68])
|
||||
h3.SetBytes(header[68:100])
|
||||
return &BlockHeaderElectrum{
|
||||
Version: binary.LittleEndian.Uint32(header[0:]),
|
||||
PrevBlockHash: h1.String(),
|
||||
MerkleRoot: h2.String(),
|
||||
ClaimTrieRoot: h3.String(),
|
||||
Timestamp: binary.LittleEndian.Uint32(header[100:]),
|
||||
Bits: binary.LittleEndian.Uint32(header[104:]),
|
||||
Nonce: binary.LittleEndian.Uint32(header[108:]),
|
||||
BlockHeight: height,
|
||||
}
|
||||
}
|
||||
|
||||
type BlockGetServerHeightReq struct{}
|
||||
type BlockGetServerHeightResp uint32
|
||||
|
||||
// blockchain.block.get_server_height
|
||||
func (s *BlockchainBlockService) Get_server_height(req *BlockGetServerHeightReq, resp **BlockGetServerHeightResp) error {
|
||||
func (s *BlockchainService) Get_server_height(r *http.Request, req *BlockGetServerHeightReq, resp **BlockGetServerHeightResp) error {
|
||||
if s.DB == nil || s.DB.LastState == nil {
|
||||
return fmt.Errorf("unknown height")
|
||||
}
|
||||
|
@ -128,11 +103,10 @@ type BlockGetChunkReq uint32
|
|||
type BlockGetChunkResp string
|
||||
|
||||
// 'blockchain.block.get_chunk'
|
||||
func (s *BlockchainBlockService) Get_chunk(req *BlockGetChunkReq, resp **BlockGetChunkResp) error {
|
||||
func (s *BlockchainService) Get_chunk(r *http.Request, req *BlockGetChunkReq, resp **BlockGetChunkResp) error {
|
||||
index := uint32(*req)
|
||||
db_headers, err := s.DB.GetHeaders(index*CHUNK_SIZE, CHUNK_SIZE)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
raw := make([]byte, 0, HEADER_SIZE*len(db_headers))
|
||||
|
@ -146,21 +120,43 @@ func (s *BlockchainBlockService) Get_chunk(req *BlockGetChunkReq, resp **BlockGe
|
|||
|
||||
type BlockGetHeaderReq uint32
|
||||
type BlockGetHeaderResp struct {
|
||||
BlockHeaderElectrum
|
||||
Version uint32 `json:"version"`
|
||||
PrevBlockHash string `json:"prev_block_hash"`
|
||||
MerkleRoot string `json:"merkle_root"`
|
||||
ClaimTrieRoot string `json:"claim_trie_root"`
|
||||
Timestamp uint32 `json:"timestamp"`
|
||||
Bits uint32 `json:"bits"`
|
||||
Nonce uint32 `json:"nonce"`
|
||||
BlockHeight uint32 `json:"block_height"`
|
||||
}
|
||||
|
||||
// 'blockchain.block.get_header'
|
||||
func (s *BlockchainBlockService) Get_header(req *BlockGetHeaderReq, resp **BlockGetHeaderResp) error {
|
||||
func (s *BlockchainService) Get_header(r *http.Request, req *BlockGetHeaderReq, resp **BlockGetHeaderResp) error {
|
||||
height := uint32(*req)
|
||||
headers, err := s.DB.GetHeaders(height, 1)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
if len(headers) < 1 {
|
||||
return errors.New("not found")
|
||||
}
|
||||
*resp = &BlockGetHeaderResp{*newBlockHeaderElectrum(&headers[0], height)}
|
||||
decode := func(header *[HEADER_SIZE]byte, height uint32) *BlockGetHeaderResp {
|
||||
var h1, h2, h3 chainhash.Hash
|
||||
h1.SetBytes(header[4:36])
|
||||
h2.SetBytes(header[36:68])
|
||||
h3.SetBytes(header[68:100])
|
||||
return &BlockGetHeaderResp{
|
||||
Version: binary.LittleEndian.Uint32(header[0:]),
|
||||
PrevBlockHash: h1.String(),
|
||||
MerkleRoot: h2.String(),
|
||||
ClaimTrieRoot: h3.String(),
|
||||
Timestamp: binary.LittleEndian.Uint32(header[100:]),
|
||||
Bits: binary.LittleEndian.Uint32(header[104:]),
|
||||
Nonce: binary.LittleEndian.Uint32(header[108:]),
|
||||
BlockHeight: height,
|
||||
}
|
||||
}
|
||||
*resp = decode(&headers[0], height)
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -171,42 +167,9 @@ type BlockHeadersReq struct {
|
|||
B64 bool `json:"b64"`
|
||||
}
|
||||
|
||||
func (req *BlockHeadersReq) UnmarshalJSON(b []byte) error {
|
||||
var params [4]interface{}
|
||||
err := json.Unmarshal(b, ¶ms)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch params[0].(type) {
|
||||
case float64:
|
||||
req.StartHeight = uint32(params[0].(float64))
|
||||
default:
|
||||
return fmt.Errorf("expected numeric argument #0 (start_height)")
|
||||
}
|
||||
switch params[1].(type) {
|
||||
case float64:
|
||||
req.Count = uint32(params[1].(float64))
|
||||
default:
|
||||
return fmt.Errorf("expected numeric argument #1 (count)")
|
||||
}
|
||||
switch params[2].(type) {
|
||||
case float64:
|
||||
req.CpHeight = uint32(params[2].(float64))
|
||||
default:
|
||||
return fmt.Errorf("expected numeric argument #2 (cp_height)")
|
||||
}
|
||||
switch params[3].(type) {
|
||||
case bool:
|
||||
req.B64 = params[3].(bool)
|
||||
default:
|
||||
return fmt.Errorf("expected boolean argument #3 (b64)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type BlockHeadersResp struct {
|
||||
Base64 string `json:"base64,omitempty"`
|
||||
Hex string `json:"hex"`
|
||||
Hex string `json:"hex,omitempty"`
|
||||
Count uint32 `json:"count"`
|
||||
Max uint32 `json:"max"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
|
@ -214,11 +177,10 @@ type BlockHeadersResp struct {
|
|||
}
|
||||
|
||||
// 'blockchain.block.headers'
|
||||
func (s *BlockchainBlockService) Headers(req *BlockHeadersReq, resp **BlockHeadersResp) error {
|
||||
func (s *BlockchainService) Headers(r *http.Request, req *BlockHeadersReq, resp **BlockHeadersResp) error {
|
||||
count := min(req.Count, MAX_CHUNK_SIZE)
|
||||
db_headers, err := s.DB.GetHeaders(req.StartHeight, count)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
count = uint32(len(db_headers))
|
||||
|
@ -247,62 +209,6 @@ func (s *BlockchainBlockService) Headers(req *BlockHeadersReq, resp **BlockHeade
|
|||
return err
|
||||
}
|
||||
|
||||
type HeadersSubscribeReq struct {
|
||||
Raw bool `json:"raw"`
|
||||
}
|
||||
|
||||
func (req *HeadersSubscribeReq) UnmarshalJSON(b []byte) error {
|
||||
var params [1]interface{}
|
||||
err := json.Unmarshal(b, ¶ms)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch params[0].(type) {
|
||||
case bool:
|
||||
req.Raw = params[0].(bool)
|
||||
default:
|
||||
return fmt.Errorf("expected bool argument #0 (raw)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type HeadersSubscribeResp struct {
|
||||
BlockHeaderElectrum
|
||||
}
|
||||
type HeadersSubscribeRawResp struct {
|
||||
Hex string `json:"hex"`
|
||||
Height uint32 `json:"height"`
|
||||
}
|
||||
|
||||
// 'blockchain.headers.subscribe'
|
||||
func (s *BlockchainHeadersService) Subscribe(req *HeadersSubscribeReq, resp *interface{}) error {
|
||||
if s.sessionMgr == nil || s.session == nil {
|
||||
return errors.New("no session, rpc not supported")
|
||||
}
|
||||
s.sessionMgr.headersSubscribe(s.session, req.Raw, true /*subscribe*/)
|
||||
height := s.DB.Height
|
||||
if s.DB.LastState != nil {
|
||||
height = s.DB.LastState.Height
|
||||
}
|
||||
headers, err := s.DB.GetHeaders(height, 1)
|
||||
if err != nil {
|
||||
s.sessionMgr.headersSubscribe(s.session, req.Raw, false /*subscribe*/)
|
||||
return err
|
||||
}
|
||||
if len(headers) < 1 {
|
||||
return errors.New("not found")
|
||||
}
|
||||
if req.Raw {
|
||||
*resp = &HeadersSubscribeRawResp{
|
||||
Hex: hex.EncodeToString(headers[0][:]),
|
||||
Height: height,
|
||||
}
|
||||
} else {
|
||||
*resp = &HeadersSubscribeResp{*newBlockHeaderElectrum(&headers[0], height)}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func decodeScriptHash(scripthash string) ([]byte, error) {
|
||||
sh, err := hex.DecodeString(scripthash)
|
||||
if err != nil {
|
||||
|
@ -343,25 +249,21 @@ type AddressGetBalanceResp struct {
|
|||
}
|
||||
|
||||
// 'blockchain.address.get_balance'
|
||||
func (s *BlockchainAddressService) Get_balance(req *AddressGetBalanceReq, resp **AddressGetBalanceResp) error {
|
||||
func (s *BlockchainAddressService) Get_balance(r *http.Request, req *AddressGetBalanceReq, resp **AddressGetBalanceResp) error {
|
||||
address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
script, err := txscript.PayToAddrScript(address)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
hashX := hashXScript(script, s.Chain)
|
||||
confirmed, unconfirmed, err := s.DB.GetBalance(hashX)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
*resp = &AddressGetBalanceResp{confirmed, unconfirmed}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -374,16 +276,14 @@ type ScripthashGetBalanceResp struct {
|
|||
}
|
||||
|
||||
// 'blockchain.scripthash.get_balance'
|
||||
func (s *BlockchainScripthashService) Get_balance(req *scripthashGetBalanceReq, resp **ScripthashGetBalanceResp) error {
|
||||
func (s *BlockchainScripthashService) Get_balance(r *http.Request, req *scripthashGetBalanceReq, resp **ScripthashGetBalanceResp) error {
|
||||
scripthash, err := decodeScriptHash(req.ScriptHash)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
hashX := hashX(scripthash)
|
||||
confirmed, unconfirmed, err := s.DB.GetBalance(hashX)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
*resp = &ScripthashGetBalanceResp{confirmed, unconfirmed}
|
||||
|
@ -393,23 +293,6 @@ func (s *BlockchainScripthashService) Get_balance(req *scripthashGetBalanceReq,
|
|||
type AddressGetHistoryReq struct {
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
func (req *AddressGetHistoryReq) UnmarshalJSON(b []byte) error {
|
||||
var params [1]interface{}
|
||||
json.Unmarshal(b, ¶ms)
|
||||
err := json.Unmarshal(b, ¶ms)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch params[0].(type) {
|
||||
case string:
|
||||
req.Address = params[0].(string)
|
||||
default:
|
||||
return fmt.Errorf("expected string argument #0 (address)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type TxInfo struct {
|
||||
TxHash string `json:"tx_hash"`
|
||||
Height uint32 `json:"height"`
|
||||
|
@ -418,24 +301,24 @@ type TxInfoFee struct {
|
|||
TxInfo
|
||||
Fee uint64 `json:"fee"`
|
||||
}
|
||||
type AddressGetHistoryResp []TxInfoFee
|
||||
type AddressGetHistoryResp struct {
|
||||
Confirmed []TxInfo `json:"confirmed"`
|
||||
Unconfirmed []TxInfoFee `json:"unconfirmed"`
|
||||
}
|
||||
|
||||
// 'blockchain.address.get_history'
|
||||
func (s *BlockchainAddressService) Get_history(req *AddressGetHistoryReq, resp **AddressGetHistoryResp) error {
|
||||
func (s *BlockchainAddressService) Get_history(r *http.Request, req *AddressGetHistoryReq, resp **AddressGetHistoryResp) error {
|
||||
address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
script, err := txscript.PayToAddrScript(address)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
hashX := hashXScript(script, s.Chain)
|
||||
dbTXs, err := s.DB.GetHistory(hashX)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
confirmed := make([]TxInfo, 0, len(dbTXs))
|
||||
|
@ -446,18 +329,11 @@ func (s *BlockchainAddressService) Get_history(req *AddressGetHistoryReq, resp *
|
|||
Height: tx.Height,
|
||||
})
|
||||
}
|
||||
unconfirmed := []TxInfoFee{} // TODO
|
||||
result := make(AddressGetHistoryResp, len(confirmed)+len(unconfirmed))
|
||||
i := 0
|
||||
for _, tx := range confirmed {
|
||||
result[i].TxInfo = tx
|
||||
i += 1
|
||||
result := &AddressGetHistoryResp{
|
||||
Confirmed: confirmed,
|
||||
Unconfirmed: []TxInfoFee{}, // TODO
|
||||
}
|
||||
for _, tx := range unconfirmed {
|
||||
result[i] = tx
|
||||
i += 1
|
||||
}
|
||||
*resp = &result
|
||||
*resp = result
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -470,16 +346,14 @@ type ScripthashGetHistoryResp struct {
|
|||
}
|
||||
|
||||
// 'blockchain.scripthash.get_history'
|
||||
func (s *BlockchainScripthashService) Get_history(req *ScripthashGetHistoryReq, resp **ScripthashGetHistoryResp) error {
|
||||
func (s *BlockchainScripthashService) Get_history(r *http.Request, req *ScripthashGetHistoryReq, resp **ScripthashGetHistoryResp) error {
|
||||
scripthash, err := decodeScriptHash(req.ScriptHash)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
hashX := hashX(scripthash)
|
||||
dbTXs, err := s.DB.GetHistory(hashX)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
confirmed := make([]TxInfo, 0, len(dbTXs))
|
||||
|
@ -504,15 +378,13 @@ type AddressGetMempoolReq struct {
|
|||
type AddressGetMempoolResp []TxInfoFee
|
||||
|
||||
// 'blockchain.address.get_mempool'
|
||||
func (s *BlockchainAddressService) Get_mempool(req *AddressGetMempoolReq, resp **AddressGetMempoolResp) error {
|
||||
func (s *BlockchainAddressService) Get_mempool(r *http.Request, req *AddressGetMempoolReq, resp **AddressGetMempoolResp) error {
|
||||
address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
script, err := txscript.PayToAddrScript(address)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
hashX := hashXScript(script, s.Chain)
|
||||
|
@ -530,10 +402,9 @@ type ScripthashGetMempoolReq struct {
|
|||
type ScripthashGetMempoolResp []TxInfoFee
|
||||
|
||||
// 'blockchain.scripthash.get_mempool'
|
||||
func (s *BlockchainScripthashService) Get_mempool(req *ScripthashGetMempoolReq, resp **ScripthashGetMempoolResp) error {
|
||||
func (s *BlockchainScripthashService) Get_mempool(r *http.Request, req *ScripthashGetMempoolReq, resp **ScripthashGetMempoolResp) error {
|
||||
scripthash, err := decodeScriptHash(req.ScriptHash)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
hashX := hashX(scripthash)
|
||||
|
@ -557,15 +428,13 @@ type TXOInfo struct {
|
|||
type AddressListUnspentResp []TXOInfo
|
||||
|
||||
// 'blockchain.address.listunspent'
|
||||
func (s *BlockchainAddressService) Listunspent(req *AddressListUnspentReq, resp **AddressListUnspentResp) error {
|
||||
func (s *BlockchainAddressService) Listunspent(r *http.Request, req *AddressListUnspentReq, resp **AddressListUnspentResp) error {
|
||||
address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
script, err := txscript.PayToAddrScript(address)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
hashX := hashXScript(script, s.Chain)
|
||||
|
@ -591,10 +460,9 @@ type ScripthashListUnspentReq struct {
|
|||
type ScripthashListUnspentResp []TXOInfo
|
||||
|
||||
// 'blockchain.scripthash.listunspent'
|
||||
func (s *BlockchainScripthashService) Listunspent(req *ScripthashListUnspentReq, resp **ScripthashListUnspentResp) error {
|
||||
func (s *BlockchainScripthashService) Listunspent(r *http.Request, req *ScripthashListUnspentReq, resp **ScripthashListUnspentResp) error {
|
||||
scripthash, err := decodeScriptHash(req.ScriptHash)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return err
|
||||
}
|
||||
hashX := hashX(scripthash)
|
||||
|
@ -613,312 +481,3 @@ func (s *BlockchainScripthashService) Listunspent(req *ScripthashListUnspentReq,
|
|||
*resp = &result
|
||||
return err
|
||||
}
|
||||
|
||||
type AddressSubscribeReq []string
|
||||
type AddressSubscribeResp []string
|
||||
|
||||
// 'blockchain.address.subscribe'
|
||||
func (s *BlockchainAddressService) Subscribe(req *AddressSubscribeReq, resp **AddressSubscribeResp) error {
|
||||
if s.sessionMgr == nil || s.session == nil {
|
||||
return errors.New("no session, rpc not supported")
|
||||
}
|
||||
result := make([]string, 0, len(*req))
|
||||
for _, addr := range *req {
|
||||
address, err := lbcutil.DecodeAddress(addr, s.Chain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
script, err := txscript.PayToAddrScript(address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashX := hashXScript(script, s.Chain)
|
||||
s.sessionMgr.hashXSubscribe(s.session, hashX, addr, true /*subscribe*/)
|
||||
status, err := s.DB.GetStatus(hashX)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result = append(result, hex.EncodeToString(status))
|
||||
}
|
||||
*resp = (*AddressSubscribeResp)(&result)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 'blockchain.address.unsubscribe'
|
||||
func (s *BlockchainAddressService) Unsubscribe(req *AddressSubscribeReq, resp **AddressSubscribeResp) error {
|
||||
if s.sessionMgr == nil || s.session == nil {
|
||||
return errors.New("no session, rpc not supported")
|
||||
}
|
||||
for _, addr := range *req {
|
||||
address, err := lbcutil.DecodeAddress(addr, s.Chain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
script, err := txscript.PayToAddrScript(address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashX := hashXScript(script, s.Chain)
|
||||
s.sessionMgr.hashXSubscribe(s.session, hashX, addr, false /*subscribe*/)
|
||||
}
|
||||
*resp = (*AddressSubscribeResp)(nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
type ScripthashSubscribeReq string
|
||||
type ScripthashSubscribeResp string
|
||||
|
||||
// 'blockchain.scripthash.subscribe'
|
||||
func (s *BlockchainScripthashService) Subscribe(req *ScripthashSubscribeReq, resp **ScripthashSubscribeResp) error {
|
||||
if s.sessionMgr == nil || s.session == nil {
|
||||
return errors.New("no session, rpc not supported")
|
||||
}
|
||||
var result string
|
||||
scripthash, err := decodeScriptHash(string(*req))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashX := hashX(scripthash)
|
||||
s.sessionMgr.hashXSubscribe(s.session, hashX, string(*req), true /*subscribe*/)
|
||||
|
||||
status, err := s.DB.GetStatus(hashX)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result = hex.EncodeToString(status)
|
||||
*resp = (*ScripthashSubscribeResp)(&result)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 'blockchain.scripthash.unsubscribe'
|
||||
func (s *BlockchainScripthashService) Unsubscribe(req *ScripthashSubscribeReq, resp **ScripthashSubscribeResp) error {
|
||||
if s.sessionMgr == nil || s.session == nil {
|
||||
return errors.New("no session, rpc not supported")
|
||||
}
|
||||
scripthash, err := decodeScriptHash(string(*req))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashX := hashX(scripthash)
|
||||
s.sessionMgr.hashXSubscribe(s.session, hashX, string(*req), false /*subscribe*/)
|
||||
*resp = (*ScripthashSubscribeResp)(nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
type TransactionBroadcastReq [1]string
|
||||
type TransactionBroadcastResp string
|
||||
|
||||
// 'blockchain.transaction.broadcast'
|
||||
func (s *BlockchainTransactionService) Broadcast(req *TransactionBroadcastReq, resp **TransactionBroadcastResp) error {
|
||||
if s.sessionMgr == nil {
|
||||
return errors.New("no session manager, rpc not supported")
|
||||
}
|
||||
strTx := string((*req)[0])
|
||||
rawTx, err := hex.DecodeString(strTx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
txhash, err := s.sessionMgr.broadcastTx(rawTx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result := txhash.String()
|
||||
*resp = (*TransactionBroadcastResp)(&result)
|
||||
return nil
|
||||
}
|
||||
|
||||
type TransactionGetReq string
|
||||
type TXFullDetail struct {
|
||||
Height int `json:"block_height"`
|
||||
Pos uint32 `json:"pos"`
|
||||
Merkle []string `json:"merkle"`
|
||||
}
|
||||
type TXDetail struct {
|
||||
Height int `json:"block_height"`
|
||||
}
|
||||
|
||||
type TXGetItem struct {
|
||||
TxHash string
|
||||
TxRaw string
|
||||
Detail interface{} // TXFullDetail or TXDetail struct
|
||||
}
|
||||
type TransactionGetResp TXGetItem
|
||||
|
||||
// 'blockchain.transaction.get'
|
||||
func (s *BlockchainTransactionService) Get(req *TransactionGetReq, resp **TransactionGetResp) error {
|
||||
txids := [1]string{string(*req)}
|
||||
request := TransactionGetBatchReq(txids[:])
|
||||
var response *TransactionGetBatchResp
|
||||
err := s.Get_batch(&request, &response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(*response) < 1 {
|
||||
return errors.New("tx not found")
|
||||
}
|
||||
switch (*response)[0].Detail.(type) {
|
||||
case TXFullDetail:
|
||||
break
|
||||
case TXDetail:
|
||||
default:
|
||||
return errors.New("tx not confirmed")
|
||||
}
|
||||
*resp = (*TransactionGetResp)(&(*response)[0])
|
||||
return err
|
||||
}
|
||||
|
||||
type TransactionGetBatchReq []string
|
||||
|
||||
func (req *TransactionGetBatchReq) UnmarshalJSON(b []byte) error {
|
||||
var params []interface{}
|
||||
json.Unmarshal(b, ¶ms)
|
||||
if len(params) > 100 {
|
||||
return fmt.Errorf("too many tx hashes in request: %v", len(params))
|
||||
}
|
||||
for i, txhash := range params {
|
||||
switch params[0].(type) {
|
||||
case string:
|
||||
*req = append(*req, txhash.(string))
|
||||
default:
|
||||
return fmt.Errorf("expected string argument #%d (tx_hash)", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type TransactionGetBatchResp []TXGetItem
|
||||
|
||||
func (resp *TransactionGetBatchResp) MarshalJSON() ([]byte, error) {
|
||||
// encode key/value pairs as variable length JSON object
|
||||
var buf bytes.Buffer
|
||||
enc := json.NewEncoder(&buf)
|
||||
buf.WriteString("{")
|
||||
for i, r := range *resp {
|
||||
if i > 0 {
|
||||
buf.WriteString(",")
|
||||
}
|
||||
txhash, raw, detail := r.TxHash, r.TxRaw, r.Detail
|
||||
err := enc.Encode(txhash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf.WriteString(":[")
|
||||
err = enc.Encode(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf.WriteString(",")
|
||||
err = enc.Encode(detail)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf.WriteString("]")
|
||||
}
|
||||
buf.WriteString("}")
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// 'blockchain.transaction.get_batch'
|
||||
func (s *BlockchainTransactionService) Get_batch(req *TransactionGetBatchReq, resp **TransactionGetBatchResp) error {
|
||||
if len(*req) > 100 {
|
||||
return fmt.Errorf("too many tx hashes in request: %v", len(*req))
|
||||
}
|
||||
tx_hashes := make([]chainhash.Hash, 0, len(*req))
|
||||
for i, txid := range *req {
|
||||
tx_hashes = append(tx_hashes, chainhash.Hash{})
|
||||
if err := chainhash.Decode(&tx_hashes[i], txid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
dbResult, err := s.DB.GetTxMerkle(tx_hashes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result := make([]TXGetItem, 0, len(dbResult))
|
||||
for _, r := range dbResult {
|
||||
merkles := make([]string, len(r.Merkle))
|
||||
for i, m := range r.Merkle {
|
||||
merkles[i] = m.String()
|
||||
}
|
||||
detail := TXFullDetail{
|
||||
Height: r.Height,
|
||||
Pos: r.Pos,
|
||||
Merkle: merkles,
|
||||
}
|
||||
result = append(result, TXGetItem{r.TxHash.String(), hex.EncodeToString(r.RawTx), &detail})
|
||||
}
|
||||
*resp = (*TransactionGetBatchResp)(&result)
|
||||
return err
|
||||
}
|
||||
|
||||
type TransactionGetMerkleReq struct {
|
||||
TxHash string `json:"tx_hash"`
|
||||
Height uint32 `json:"height"`
|
||||
}
|
||||
type TransactionGetMerkleResp TXGetItem
|
||||
|
||||
// 'blockchain.transaction.get_merkle'
|
||||
func (s *BlockchainTransactionService) Get_merkle(req *TransactionGetMerkleReq, resp **TransactionGetMerkleResp) error {
|
||||
txids := [1]string{string(req.TxHash)}
|
||||
request := TransactionGetBatchReq(txids[:])
|
||||
var response *TransactionGetBatchResp
|
||||
err := s.Get_batch(&request, &response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(*response) < 1 {
|
||||
return errors.New("tx not found")
|
||||
}
|
||||
switch (*response)[0].Detail.(type) {
|
||||
case TXFullDetail:
|
||||
break
|
||||
case TXDetail:
|
||||
default:
|
||||
return errors.New("tx not confirmed")
|
||||
}
|
||||
*resp = (*TransactionGetMerkleResp)(&(*response)[0])
|
||||
return err
|
||||
}
|
||||
|
||||
type TransactionGetHeightReq string
|
||||
type TransactionGetHeightResp uint32
|
||||
|
||||
// 'blockchain.transaction.get_height'
|
||||
func (s *BlockchainTransactionService) Get_height(req *TransactionGetHeightReq, resp **TransactionGetHeightResp) error {
|
||||
txid := string(*(req))
|
||||
txhash, err := chainhash.NewHashFromStr(txid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
height, err := s.DB.GetTxHeight(txhash)
|
||||
*resp = (*TransactionGetHeightResp)(&height)
|
||||
return err
|
||||
}
|
||||
|
||||
type TransactionInfoReq string
|
||||
type TransactionInfoResp TXGetItem
|
||||
|
||||
// 'blockchain.transaction.info'
|
||||
func (s *BlockchainTransactionService) Info(req *TransactionInfoReq, resp **TransactionInfoResp) error {
|
||||
txids := [1]string{string(*req)}
|
||||
request := TransactionGetBatchReq(txids[:])
|
||||
var response *TransactionGetBatchResp
|
||||
err := s.Get_batch(&request, &response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(*response) < 1 {
|
||||
return errors.New("tx not found")
|
||||
}
|
||||
switch (*response)[0].Detail.(type) {
|
||||
case TXFullDetail:
|
||||
break
|
||||
case TXDetail:
|
||||
default:
|
||||
if (*response)[0].TxHash == "" {
|
||||
return errors.New("no such mempool or blockchain transaction")
|
||||
}
|
||||
}
|
||||
*resp = (*TransactionInfoResp)(&(*response)[0])
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -1,19 +1,12 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/herald.go/db"
|
||||
"github.com/lbryio/herald.go/internal"
|
||||
"github.com/lbryio/lbcd/chaincfg"
|
||||
"github.com/lbryio/lbcd/txscript"
|
||||
"github.com/lbryio/lbcutil"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
)
|
||||
|
||||
// Source: test_variety_of_transactions_and_longish_history (lbry-sdk/tests/integration/transactions)
|
||||
|
@ -58,22 +51,21 @@ var regTestAddrs = [30]string{
|
|||
|
||||
func TestServerGetHeight(t *testing.T) {
|
||||
secondaryPath := "asdf"
|
||||
grp := stop.NewDebug()
|
||||
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
|
||||
defer toDefer()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
s := &BlockchainBlockService{
|
||||
s := &BlockchainService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
}
|
||||
|
||||
req := BlockGetServerHeightReq{}
|
||||
var resp *BlockGetServerHeightResp
|
||||
err = s.Get_server_height(&req, &resp)
|
||||
err = s.Get_server_height(nil, &req, &resp)
|
||||
if err != nil {
|
||||
t.Errorf("handler err: %v", err)
|
||||
}
|
||||
|
@ -89,15 +81,14 @@ func TestServerGetHeight(t *testing.T) {
|
|||
|
||||
func TestGetChunk(t *testing.T) {
|
||||
secondaryPath := "asdf"
|
||||
grp := stop.NewDebug()
|
||||
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
|
||||
defer toDefer()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
s := &BlockchainBlockService{
|
||||
s := &BlockchainService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
}
|
||||
|
@ -105,7 +96,7 @@ func TestGetChunk(t *testing.T) {
|
|||
for index := 0; index < 10; index++ {
|
||||
req := BlockGetChunkReq(index)
|
||||
var resp *BlockGetChunkResp
|
||||
err := s.Get_chunk(&req, &resp)
|
||||
err := s.Get_chunk(nil, &req, &resp)
|
||||
if err != nil {
|
||||
t.Errorf("index: %v handler err: %v", index, err)
|
||||
}
|
||||
|
@ -133,15 +124,14 @@ func TestGetChunk(t *testing.T) {
|
|||
|
||||
func TestGetHeader(t *testing.T) {
|
||||
secondaryPath := "asdf"
|
||||
grp := stop.NewDebug()
|
||||
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
|
||||
defer toDefer()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
s := &BlockchainBlockService{
|
||||
s := &BlockchainService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
}
|
||||
|
@ -149,7 +139,7 @@ func TestGetHeader(t *testing.T) {
|
|||
for height := 0; height < 700; height += 100 {
|
||||
req := BlockGetHeaderReq(height)
|
||||
var resp *BlockGetHeaderResp
|
||||
err := s.Get_header(&req, &resp)
|
||||
err := s.Get_header(nil, &req, &resp)
|
||||
if err != nil && height <= 500 {
|
||||
t.Errorf("height: %v handler err: %v", height, err)
|
||||
}
|
||||
|
@ -161,151 +151,26 @@ func TestGetHeader(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestHeaders(t *testing.T) {
|
||||
secondaryPath := "asdf"
|
||||
grp := stop.NewDebug()
|
||||
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
s := &BlockchainBlockService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
}
|
||||
|
||||
for height := uint32(0); height < 700; height += 100 {
|
||||
req := BlockHeadersReq{
|
||||
StartHeight: height,
|
||||
Count: 1,
|
||||
CpHeight: 0,
|
||||
B64: false,
|
||||
}
|
||||
var resp *BlockHeadersResp
|
||||
err := s.Headers(&req, &resp)
|
||||
if err != nil {
|
||||
t.Errorf("Headers: %v", err)
|
||||
}
|
||||
marshalled, err := json.MarshalIndent(resp, "", " ")
|
||||
if err != nil {
|
||||
t.Errorf("height: %v unmarshal err: %v", height, err)
|
||||
}
|
||||
t.Logf("height: %v resp: %v", height, string(marshalled))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadersSubscribe(t *testing.T) {
|
||||
args := MakeDefaultTestArgs()
|
||||
grp := stop.NewDebug()
|
||||
secondaryPath := "asdf"
|
||||
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
sm := newSessionManager(nil, db, args, grp, &chaincfg.RegressionNetParams, nil)
|
||||
sm.start()
|
||||
defer sm.stop()
|
||||
|
||||
client1, server1 := net.Pipe()
|
||||
sess1 := sm.addSession(server1)
|
||||
client2, server2 := net.Pipe()
|
||||
sess2 := sm.addSession(server2)
|
||||
|
||||
// Set up logic to read a notification.
|
||||
var received sync.WaitGroup
|
||||
recv := func(client net.Conn) {
|
||||
defer received.Done()
|
||||
buf := make([]byte, 1024)
|
||||
len, err := client.Read(buf)
|
||||
if err != nil {
|
||||
t.Errorf("read err: %v", err)
|
||||
}
|
||||
t.Logf("len: %v notification: %v", len, string(buf))
|
||||
}
|
||||
received.Add(2)
|
||||
go recv(client1)
|
||||
go recv(client2)
|
||||
|
||||
s1 := &BlockchainHeadersService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
sessionMgr: sm,
|
||||
session: sess1,
|
||||
}
|
||||
s2 := &BlockchainHeadersService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
sessionMgr: sm,
|
||||
session: sess2,
|
||||
}
|
||||
|
||||
// Subscribe with Raw: false.
|
||||
req1 := HeadersSubscribeReq{Raw: false}
|
||||
var r any
|
||||
err = s1.Subscribe(&req1, &r)
|
||||
if err != nil {
|
||||
t.Errorf("handler err: %v", err)
|
||||
}
|
||||
resp1 := r.(*HeadersSubscribeResp)
|
||||
marshalled1, err := json.MarshalIndent(resp1, "", " ")
|
||||
if err != nil {
|
||||
t.Errorf("unmarshal err: %v", err)
|
||||
}
|
||||
// Subscribe with Raw: true.
|
||||
t.Logf("resp: %v", string(marshalled1))
|
||||
req2 := HeadersSubscribeReq{Raw: true}
|
||||
err = s2.Subscribe(&req2, &r)
|
||||
if err != nil {
|
||||
t.Errorf("handler err: %v", err)
|
||||
}
|
||||
resp2 := r.(*HeadersSubscribeRawResp)
|
||||
marshalled2, err := json.MarshalIndent(resp2, "", " ")
|
||||
if err != nil {
|
||||
t.Errorf("unmarshal err: %v", err)
|
||||
}
|
||||
t.Logf("resp: %v", string(marshalled2))
|
||||
|
||||
// Now send a notification.
|
||||
header500, err := hex.DecodeString("00000020e9537f98ae80a3aa0936dd424439b2b9305e5e9d9d5c7aa571b4422c447741e739b3109304ed4f0330d6854271db17da221559a46b68db4ceecfebd9f0c75dbe0100000000000000000000000000000000000000000000000000000000000000b3e02063ffff7f2001000000")
|
||||
if err != nil {
|
||||
t.Errorf("decode err: %v", err)
|
||||
}
|
||||
note1 := headerNotification{
|
||||
HeightHash: internal.HeightHash{Height: 500, BlockHeader: header500},
|
||||
blockHeaderElectrum: nil,
|
||||
blockHeaderStr: "",
|
||||
}
|
||||
t.Logf("sending notification")
|
||||
sm.doNotify(note1)
|
||||
|
||||
t.Logf("waiting to receive notification(s)...")
|
||||
received.Wait()
|
||||
}
|
||||
|
||||
func TestGetBalance(t *testing.T) {
|
||||
secondaryPath := "asdf"
|
||||
grp := stop.NewDebug()
|
||||
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
|
||||
defer toDefer()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
s := &BlockchainAddressService{
|
||||
BlockchainService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
},
|
||||
}
|
||||
|
||||
for _, addr := range regTestAddrs {
|
||||
req := AddressGetBalanceReq{addr}
|
||||
var resp *AddressGetBalanceResp
|
||||
err := s.Get_balance(&req, &resp)
|
||||
err := s.Get_balance(nil, &req, &resp)
|
||||
if err != nil {
|
||||
t.Errorf("address: %v handler err: %v", addr, err)
|
||||
}
|
||||
|
@ -319,23 +184,24 @@ func TestGetBalance(t *testing.T) {
|
|||
|
||||
func TestGetHistory(t *testing.T) {
|
||||
secondaryPath := "asdf"
|
||||
grp := stop.NewDebug()
|
||||
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
|
||||
defer toDefer()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
s := &BlockchainAddressService{
|
||||
BlockchainService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
},
|
||||
}
|
||||
|
||||
for _, addr := range regTestAddrs {
|
||||
req := AddressGetHistoryReq{addr}
|
||||
var resp *AddressGetHistoryResp
|
||||
err := s.Get_history(&req, &resp)
|
||||
err := s.Get_history(nil, &req, &resp)
|
||||
if err != nil {
|
||||
t.Errorf("address: %v handler err: %v", addr, err)
|
||||
}
|
||||
|
@ -349,23 +215,24 @@ func TestGetHistory(t *testing.T) {
|
|||
|
||||
func TestListUnspent(t *testing.T) {
|
||||
secondaryPath := "asdf"
|
||||
grp := stop.NewDebug()
|
||||
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
db, toDefer, err := db.GetProdDB(regTestDBPath, secondaryPath)
|
||||
defer toDefer()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
s := &BlockchainAddressService{
|
||||
BlockchainService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
},
|
||||
}
|
||||
|
||||
for _, addr := range regTestAddrs {
|
||||
req := AddressListUnspentReq{addr}
|
||||
var resp *AddressListUnspentResp
|
||||
err := s.Listunspent(&req, &resp)
|
||||
err := s.Listunspent(nil, &req, &resp)
|
||||
if err != nil {
|
||||
t.Errorf("address: %v handler err: %v", addr, err)
|
||||
}
|
||||
|
@ -376,94 +243,3 @@ func TestListUnspent(t *testing.T) {
|
|||
t.Logf("address: %v resp: %v", addr, string(marshalled))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddressSubscribe(t *testing.T) {
|
||||
args := MakeDefaultTestArgs()
|
||||
grp := stop.NewDebug()
|
||||
secondaryPath := "asdf"
|
||||
db, err := db.GetProdDB(regTestDBPath, secondaryPath, grp)
|
||||
defer db.Shutdown()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
sm := newSessionManager(nil, db, args, grp, &chaincfg.RegressionNetParams, nil)
|
||||
sm.start()
|
||||
defer sm.stop()
|
||||
|
||||
client1, server1 := net.Pipe()
|
||||
sess1 := sm.addSession(server1)
|
||||
client2, server2 := net.Pipe()
|
||||
sess2 := sm.addSession(server2)
|
||||
|
||||
// Set up logic to read a notification.
|
||||
var received sync.WaitGroup
|
||||
recv := func(client net.Conn) {
|
||||
buf := make([]byte, 1024)
|
||||
len, err := client.Read(buf)
|
||||
if err != nil {
|
||||
t.Errorf("read err: %v", err)
|
||||
}
|
||||
t.Logf("len: %v notification: %v", len, string(buf))
|
||||
received.Done()
|
||||
}
|
||||
received.Add(2)
|
||||
go recv(client1)
|
||||
go recv(client2)
|
||||
|
||||
s1 := &BlockchainAddressService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
sessionMgr: sm,
|
||||
session: sess1,
|
||||
}
|
||||
s2 := &BlockchainAddressService{
|
||||
DB: db,
|
||||
Chain: &chaincfg.RegressionNetParams,
|
||||
sessionMgr: sm,
|
||||
session: sess2,
|
||||
}
|
||||
|
||||
addr1, addr2 := regTestAddrs[1], regTestAddrs[2]
|
||||
// Subscribe to addr1 and addr2.
|
||||
req1 := AddressSubscribeReq{addr1, addr2}
|
||||
var resp1 *AddressSubscribeResp
|
||||
err = s1.Subscribe(&req1, &resp1)
|
||||
if err != nil {
|
||||
t.Errorf("handler err: %v", err)
|
||||
}
|
||||
marshalled1, err := json.MarshalIndent(resp1, "", " ")
|
||||
if err != nil {
|
||||
t.Errorf("unmarshal err: %v", err)
|
||||
}
|
||||
// Subscribe to addr2 only.
|
||||
t.Logf("resp: %v", string(marshalled1))
|
||||
req2 := AddressSubscribeReq{addr2}
|
||||
var resp2 *AddressSubscribeResp
|
||||
err = s2.Subscribe(&req2, &resp2)
|
||||
if err != nil {
|
||||
t.Errorf("handler err: %v", err)
|
||||
}
|
||||
marshalled2, err := json.MarshalIndent(resp2, "", " ")
|
||||
if err != nil {
|
||||
t.Errorf("unmarshal err: %v", err)
|
||||
}
|
||||
t.Logf("resp: %v", string(marshalled2))
|
||||
|
||||
// Now send a notification for addr2.
|
||||
address, _ := lbcutil.DecodeAddress(addr2, sm.chain)
|
||||
script, _ := txscript.PayToAddrScript(address)
|
||||
note := hashXNotification{}
|
||||
copy(note.hashX[:], hashXScript(script, sm.chain))
|
||||
status, err := hex.DecodeString((*resp1)[1])
|
||||
if err != nil {
|
||||
t.Errorf("decode err: %v", err)
|
||||
}
|
||||
note.status = append(note.status, []byte(status)...)
|
||||
t.Logf("sending notification")
|
||||
sm.doNotify(note)
|
||||
|
||||
t.Logf("waiting to receive notification(s)...")
|
||||
received.Wait()
|
||||
}
|
||||
|
|
|
@ -1,108 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/herald.go/db"
|
||||
"github.com/lbryio/herald.go/internal/metrics"
|
||||
pb "github.com/lbryio/herald.go/protobuf/go"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type ClaimtrieService struct {
|
||||
DB *db.ReadOnlyDBColumnFamily
|
||||
Server *Server
|
||||
}
|
||||
|
||||
type ResolveData struct {
|
||||
Data []string `json:"data"`
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
type GetClaimByIDData struct {
|
||||
ClaimID string `json:"claim_id"`
|
||||
}
|
||||
|
||||
// Resolve is the json rpc endpoint for 'blockchain.claimtrie.resolve'.
|
||||
func (t *ClaimtrieService) Resolve(args *ResolveData, result **pb.Outputs) error {
|
||||
log.Println("Resolve")
|
||||
res, err := InternalResolve(args.Data, t.DB)
|
||||
*result = res
|
||||
return err
|
||||
}
|
||||
|
||||
// Search is the json rpc endpoint for 'blockchain.claimtrie.search'.
|
||||
func (t *ClaimtrieService) Search(args *pb.SearchRequest, result **pb.Outputs) error {
|
||||
log.Println("Search")
|
||||
if t.Server == nil {
|
||||
log.Warnln("Server is nil in Search")
|
||||
*result = nil
|
||||
return nil
|
||||
}
|
||||
ctx := context.Background()
|
||||
res, err := t.Server.Search(ctx, args)
|
||||
*result = res
|
||||
return err
|
||||
}
|
||||
|
||||
// GetClaimByID is the json rpc endpoint for 'blockchain.claimtrie.getclaimbyid'.
|
||||
func (t *ClaimtrieService) GetClaimByID(args *GetClaimByIDData, result **pb.Outputs) error {
|
||||
log.Println("GetClaimByID")
|
||||
if len(args.ClaimID) != 40 {
|
||||
*result = nil
|
||||
return fmt.Errorf("len(claim_id) != 40")
|
||||
}
|
||||
|
||||
rows, extras, err := t.DB.GetClaimByID(args.ClaimID)
|
||||
if err != nil {
|
||||
*result = nil
|
||||
return err
|
||||
}
|
||||
|
||||
metrics.RequestsCount.With(prometheus.Labels{"method": "blockchain.claimtrie.getclaimbyid"}).Inc()
|
||||
|
||||
// FIXME: this has txos and extras and so does GetClaimById?
|
||||
allTxos := make([]*pb.Output, 0)
|
||||
allExtraTxos := make([]*pb.Output, 0)
|
||||
|
||||
for _, row := range rows {
|
||||
txos, extraTxos, err := row.ToOutputs()
|
||||
if err != nil {
|
||||
*result = nil
|
||||
return err
|
||||
}
|
||||
// TODO: there may be a more efficient way to do this.
|
||||
allTxos = append(allTxos, txos...)
|
||||
allExtraTxos = append(allExtraTxos, extraTxos...)
|
||||
}
|
||||
|
||||
for _, extra := range extras {
|
||||
txos, extraTxos, err := extra.ToOutputs()
|
||||
if err != nil {
|
||||
*result = nil
|
||||
return err
|
||||
}
|
||||
// TODO: there may be a more efficient way to do this.
|
||||
allTxos = append(allTxos, txos...)
|
||||
allExtraTxos = append(allExtraTxos, extraTxos...)
|
||||
}
|
||||
|
||||
res := &pb.Outputs{
|
||||
Txos: allTxos,
|
||||
ExtraTxos: allExtraTxos,
|
||||
Total: uint32(len(allTxos) + len(allExtraTxos)),
|
||||
Offset: 0, //TODO
|
||||
Blocked: nil, //TODO
|
||||
BlockedTotal: 0, //TODO
|
||||
}
|
||||
|
||||
log.Warn(res)
|
||||
|
||||
*result = res
|
||||
return nil
|
||||
}
|
|
@ -1,39 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type PeersService struct {
|
||||
Server *Server
|
||||
// needed for subscribe/unsubscribe
|
||||
sessionMgr *sessionManager
|
||||
session *session
|
||||
}
|
||||
|
||||
type PeersSubscribeReq struct {
|
||||
Ip string `json:"ip"`
|
||||
Host string `json:"host"`
|
||||
Details []string `json:"details"`
|
||||
}
|
||||
|
||||
type PeersSubscribeResp string
|
||||
|
||||
// Features is the json rpc endpoint for 'server.peers.subcribe'.
|
||||
func (t *PeersService) Subscribe(req *PeersSubscribeReq, res **PeersSubscribeResp) error {
|
||||
log.Println("PeersSubscribe")
|
||||
// var port = "50001"
|
||||
|
||||
// FIXME: Get the actual port from the request details
|
||||
|
||||
if t.sessionMgr == nil || t.session == nil {
|
||||
*res = nil
|
||||
return errors.New("no session, rpc not supported")
|
||||
}
|
||||
t.sessionMgr.peersSubscribe(t.session, true /*subscribe*/)
|
||||
|
||||
*res = nil
|
||||
return nil
|
||||
}
|
|
@ -1,88 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type ServerService struct {
|
||||
Args *Args
|
||||
}
|
||||
|
||||
type ServerFeatureService struct {
|
||||
Args *Args
|
||||
}
|
||||
|
||||
type ServerFeaturesReq struct{}
|
||||
|
||||
type ServerFeaturesRes struct {
|
||||
Hosts map[string]string `json:"hosts"`
|
||||
Pruning string `json:"pruning"`
|
||||
ServerVersion string `json:"server_version"`
|
||||
ProtocolMin string `json:"protocol_min"`
|
||||
ProtocolMax string `json:"protocol_max"`
|
||||
GenesisHash string `json:"genesis_hash"`
|
||||
Description string `json:"description"`
|
||||
PaymentAddress string `json:"payment_address"`
|
||||
DonationAddress string `json:"donation_address"`
|
||||
DailyFee string `json:"daily_fee"`
|
||||
HashFunction string `json:"hash_function"`
|
||||
TrendingAlgorithm string `json:"trending_algorithm"`
|
||||
}
|
||||
|
||||
// Features is the json rpc endpoint for 'server.features'.
|
||||
func (t *ServerService) Features(req *ServerFeaturesReq, res **ServerFeaturesRes) error {
|
||||
log.Println("Features")
|
||||
|
||||
features := &ServerFeaturesRes{
|
||||
Hosts: map[string]string{},
|
||||
Pruning: "",
|
||||
ServerVersion: HUB_PROTOCOL_VERSION,
|
||||
ProtocolMin: PROTOCOL_MIN,
|
||||
ProtocolMax: PROTOCOL_MAX,
|
||||
GenesisHash: t.Args.GenesisHash,
|
||||
Description: t.Args.ServerDescription,
|
||||
PaymentAddress: t.Args.PaymentAddress,
|
||||
DonationAddress: t.Args.DonationAddress,
|
||||
DailyFee: t.Args.DailyFee,
|
||||
HashFunction: "sha256",
|
||||
TrendingAlgorithm: "fast_ar",
|
||||
}
|
||||
*res = features
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type ServerBannerService struct {
|
||||
Args *Args
|
||||
}
|
||||
|
||||
type ServerBannerReq struct{}
|
||||
|
||||
type ServerBannerRes string
|
||||
|
||||
// Banner is the json rpc endpoint for 'server.banner'.
|
||||
func (t *ServerService) Banner(req *ServerBannerReq, res **ServerBannerRes) error {
|
||||
log.Println("Banner")
|
||||
|
||||
*res = (*ServerBannerRes)(t.Args.Banner)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type ServerVersionService struct {
|
||||
Args *Args
|
||||
}
|
||||
|
||||
type ServerVersionReq [2]string // [client_name, client_version]
|
||||
|
||||
type ServerVersionRes [2]string // [version, protocol_version]
|
||||
|
||||
// Version is the json rpc endpoint for 'server.version'.
|
||||
func (t *ServerService) Version(req *ServerVersionReq, res **ServerVersionRes) error {
|
||||
// FIXME: We may need to do the computation of a negotiated version here.
|
||||
// Also return an error if client is not supported?
|
||||
result := [2]string{t.Args.ServerVersion, t.Args.ServerVersion}
|
||||
*res = (*ServerVersionRes)(&result)
|
||||
log.Printf("Version(%v) -> %v", *req, **res)
|
||||
return nil
|
||||
}
|
|
@ -1,154 +1,69 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
gorilla_mux "github.com/gorilla/mux"
|
||||
gorilla_rpc "github.com/gorilla/rpc"
|
||||
gorilla_json "github.com/gorilla/rpc/json"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/gorilla/rpc"
|
||||
"github.com/gorilla/rpc/json"
|
||||
"github.com/lbryio/herald.go/db"
|
||||
pb "github.com/lbryio/herald.go/protobuf/go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/net/netutil"
|
||||
)
|
||||
|
||||
type gorillaRpcCodec struct {
|
||||
gorilla_rpc.Codec
|
||||
type ClaimtrieService struct {
|
||||
DB *db.ReadOnlyDBColumnFamily
|
||||
}
|
||||
|
||||
func (c *gorillaRpcCodec) NewRequest(r *http.Request) gorilla_rpc.CodecRequest {
|
||||
return &gorillaRpcCodecRequest{c.Codec.NewRequest(r)}
|
||||
type ResolveData struct {
|
||||
Data []string `json:"data"`
|
||||
}
|
||||
|
||||
// gorillaRpcCodecRequest provides ability to rewrite the incoming
|
||||
// request "method" field. For example:
|
||||
// blockchain.block.get_header -> blockchain_block.Get_header
|
||||
// blockchain.address.listunspent -> blockchain_address.Listunspent
|
||||
// This makes the "method" string compatible with Gorilla/RPC
|
||||
// requirements.
|
||||
type gorillaRpcCodecRequest struct {
|
||||
gorilla_rpc.CodecRequest
|
||||
type Result struct {
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
func (cr *gorillaRpcCodecRequest) Method() (string, error) {
|
||||
rawMethod, err := cr.CodecRequest.Method()
|
||||
if err != nil {
|
||||
return rawMethod, err
|
||||
}
|
||||
parts := strings.Split(rawMethod, ".")
|
||||
if len(parts) < 2 {
|
||||
return rawMethod, fmt.Errorf("blockchain rpc: service/method ill-formed: %q", rawMethod)
|
||||
}
|
||||
service := strings.Join(parts[0:len(parts)-1], "_")
|
||||
method := parts[len(parts)-1]
|
||||
if len(method) < 1 {
|
||||
return rawMethod, fmt.Errorf("blockchain rpc: method ill-formed: %q", method)
|
||||
}
|
||||
method = strings.ToUpper(string(method[0])) + string(method[1:])
|
||||
return service + "." + method, err
|
||||
// Resolve is the json rpc endpoint for 'blockchain.claimtrie.resolve'.
|
||||
func (t *ClaimtrieService) Resolve(r *http.Request, args *ResolveData, result **pb.Outputs) error {
|
||||
log.Println("Resolve")
|
||||
res, err := InternalResolve(args.Data, t.DB)
|
||||
*result = res
|
||||
return err
|
||||
}
|
||||
|
||||
// StartJsonRPC starts the json rpc server and registers the endpoints.
|
||||
func (s *Server) StartJsonRPC() error {
|
||||
// Set up the pure JSONRPC server with persistent connections/sessions.
|
||||
if s.Args.JSONRPCPort != 0 {
|
||||
port := ":" + strconv.Itoa(s.Args.JSONRPCPort)
|
||||
laddr, err := net.ResolveTCPAddr("tcp4", port)
|
||||
if err != nil {
|
||||
log.Errorf("ResoveIPAddr: %v\n", err)
|
||||
goto fail1
|
||||
}
|
||||
listener, err := net.ListenTCP("tcp4", laddr)
|
||||
if err != nil {
|
||||
log.Errorf("ListenTCP: %v\n", err)
|
||||
goto fail1
|
||||
}
|
||||
log.Infof("JSONRPC server listening on %s", listener.Addr().String())
|
||||
s.sessionManager.start()
|
||||
acceptConnections := func(listener net.Listener) {
|
||||
defer s.sessionManager.stop()
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
log.Errorf("Accept: %v\n", err)
|
||||
break
|
||||
}
|
||||
log.Infof("Accepted: %v", conn.RemoteAddr())
|
||||
s.sessionManager.addSession(conn)
|
||||
port := ":" + s.Args.JSONRPCPort
|
||||
|
||||
}
|
||||
}
|
||||
go acceptConnections(netutil.LimitListener(listener, s.sessionManager.sessionsMax))
|
||||
}
|
||||
|
||||
fail1:
|
||||
// Set up the JSONRPC over HTTP server.
|
||||
if s.Args.JSONRPCHTTPPort != 0 {
|
||||
s1 := gorilla_rpc.NewServer() // Create a new RPC server
|
||||
s1 := rpc.NewServer() // Create a new RPC server
|
||||
// Register the type of data requested as JSON, with custom codec.
|
||||
s1.RegisterCodec(&gorillaRpcCodec{gorilla_json.NewCodec()}, "application/json")
|
||||
s1.RegisterCodec(&BlockchainCodec{json.NewCodec()}, "application/json")
|
||||
|
||||
// Register "blockchain.claimtrie.*"" handlers.
|
||||
claimtrieSvc := &ClaimtrieService{s.DB, s}
|
||||
err := s1.RegisterTCPService(claimtrieSvc, "blockchain_claimtrie")
|
||||
claimtrieSvc := &ClaimtrieService{s.DB}
|
||||
err := s1.RegisterService(claimtrieSvc, "blockchain_claimtrie")
|
||||
if err != nil {
|
||||
log.Errorf("RegisterTCPService: %v\n", err)
|
||||
goto fail2
|
||||
log.Errorf("RegisterService: %v\n", err)
|
||||
}
|
||||
|
||||
// Register "blockchain.{block,address,scripthash,transaction}.*" handlers.
|
||||
blockchainSvc := &BlockchainBlockService{s.DB, s.Chain}
|
||||
err = s1.RegisterTCPService(blockchainSvc, "blockchain_block")
|
||||
// Register other "blockchain.{block,address,scripthash}.*" handlers.
|
||||
blockchainSvc := &BlockchainService{s.DB, s.Chain}
|
||||
err = s1.RegisterService(blockchainSvc, "blockchain_block")
|
||||
if err != nil {
|
||||
log.Errorf("RegisterTCPService: %v\n", err)
|
||||
goto fail2
|
||||
log.Errorf("RegisterService: %v\n", err)
|
||||
}
|
||||
err = s1.RegisterTCPService(&BlockchainHeadersService{s.DB, s.Chain, s.sessionManager, nil}, "blockchain_headers")
|
||||
err = s1.RegisterService(&BlockchainAddressService{*blockchainSvc}, "blockchain_address")
|
||||
if err != nil {
|
||||
log.Errorf("RegisterTCPService: %v\n", err)
|
||||
goto fail2
|
||||
log.Errorf("RegisterService: %v\n", err)
|
||||
}
|
||||
err = s1.RegisterTCPService(&BlockchainAddressService{s.DB, s.Chain, s.sessionManager, nil}, "blockchain_address")
|
||||
err = s1.RegisterService(&BlockchainScripthashService{*blockchainSvc}, "blockchain_scripthash")
|
||||
if err != nil {
|
||||
log.Errorf("RegisterTCPService: %v\n", err)
|
||||
goto fail2
|
||||
}
|
||||
err = s1.RegisterTCPService(&BlockchainScripthashService{s.DB, s.Chain, s.sessionManager, nil}, "blockchain_scripthash")
|
||||
if err != nil {
|
||||
log.Errorf("RegisterTCPService: %v\n", err)
|
||||
goto fail2
|
||||
}
|
||||
err = s1.RegisterTCPService(&BlockchainTransactionService{s.DB, s.Chain, s.sessionManager}, "blockchain_transaction")
|
||||
if err != nil {
|
||||
log.Errorf("RegisterTCPService: %v\n", err)
|
||||
goto fail2
|
||||
log.Errorf("RegisterService: %v\n", err)
|
||||
}
|
||||
|
||||
// Register "server.{features,banner,version}" handlers.
|
||||
serverSvc := &ServerService{s.Args}
|
||||
err = s1.RegisterTCPService(serverSvc, "server")
|
||||
if err != nil {
|
||||
log.Errorf("RegisterTCPService: %v\n", err)
|
||||
goto fail2
|
||||
}
|
||||
|
||||
// Register "server.peers" handlers.
|
||||
peersSvc := &PeersService{Server: s}
|
||||
err = s1.RegisterTCPService(peersSvc, "server_peers")
|
||||
if err != nil {
|
||||
log.Errorf("RegisterTCPService: %v\n", err)
|
||||
goto fail2
|
||||
}
|
||||
|
||||
r := gorilla_mux.NewRouter()
|
||||
r := mux.NewRouter()
|
||||
r.Handle("/rpc", s1)
|
||||
port := ":" + strconv.FormatUint(uint64(s.Args.JSONRPCHTTPPort), 10)
|
||||
log.Infof("HTTP JSONRPC server listening on %s", port)
|
||||
log.Fatal(http.ListenAndServe(port, r))
|
||||
}
|
||||
|
||||
fail2:
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -2,7 +2,6 @@ package server
|
|||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"net"
|
||||
|
||||
"github.com/lbryio/herald.go/internal"
|
||||
|
@ -53,27 +52,15 @@ func (s *Server) DoNotify(heightHash *internal.HeightHash) error {
|
|||
|
||||
// RunNotifier Runs the notfying action forever
|
||||
func (s *Server) RunNotifier() error {
|
||||
for notification := range s.NotifierChan {
|
||||
switch note := notification.(type) {
|
||||
case internal.HeightHash:
|
||||
heightHash := note
|
||||
s.DoNotify(&heightHash)
|
||||
// Do we need this?
|
||||
// case peerNotification:
|
||||
// peer, _ := notification.(peerNotification)
|
||||
// s.notifyPeerSubs(&Peer{Address: peer.address, Port: peer.port})
|
||||
default:
|
||||
logrus.Warn("unknown notification type")
|
||||
}
|
||||
s.sessionManager.doNotify(notification)
|
||||
for heightHash := range s.NotifierChan {
|
||||
s.DoNotify(heightHash)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NotifierServer implements the TCP protocol for height/blockheader notifications
|
||||
func (s *Server) NotifierServer() error {
|
||||
s.Grp.Add(1)
|
||||
address := ":" + fmt.Sprintf("%d", s.Args.NotifierPort)
|
||||
address := ":" + s.Args.NotifierPort
|
||||
addr, err := net.ResolveTCPAddr("tcp", address)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -85,27 +72,11 @@ func (s *Server) NotifierServer() error {
|
|||
}
|
||||
|
||||
defer listen.Close()
|
||||
rdyCh := make(chan bool)
|
||||
|
||||
for {
|
||||
var conn net.Conn
|
||||
var err error
|
||||
|
||||
logrus.Info("Waiting for connection")
|
||||
|
||||
go func() {
|
||||
conn, err = listen.Accept()
|
||||
rdyCh <- true
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-s.Grp.Ch():
|
||||
s.Grp.Done()
|
||||
return nil
|
||||
case <-rdyCh:
|
||||
logrus.Info("Connection accepted")
|
||||
}
|
||||
|
||||
conn, err := listen.Accept()
|
||||
if err != nil {
|
||||
logrus.Warn(err)
|
||||
continue
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package server_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net"
|
||||
|
@ -9,32 +10,11 @@ import (
|
|||
|
||||
"github.com/lbryio/herald.go/internal"
|
||||
"github.com/lbryio/herald.go/server"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const defaultBufferSize = 1024
|
||||
|
||||
func subReady(s *server.Server) error {
|
||||
sleepTime := time.Millisecond * 100
|
||||
for {
|
||||
if sleepTime > time.Second {
|
||||
return fmt.Errorf("timeout")
|
||||
}
|
||||
s.HeightSubsMut.RLock()
|
||||
if len(s.HeightSubs) > 0 {
|
||||
s.HeightSubsMut.RUnlock()
|
||||
return nil
|
||||
}
|
||||
s.HeightSubsMut.RUnlock()
|
||||
|
||||
logrus.Warn("waiting for subscriber")
|
||||
time.Sleep(sleepTime)
|
||||
sleepTime = sleepTime * 2
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func tcpConnReady(addr string) (net.Conn, error) {
|
||||
sleepTime := time.Millisecond * 100
|
||||
for {
|
||||
|
@ -67,14 +47,14 @@ func tcpRead(conn net.Conn) ([]byte, error) {
|
|||
}
|
||||
|
||||
func TestNotifierServer(t *testing.T) {
|
||||
args := server.MakeDefaultTestArgs()
|
||||
ctx := stop.NewDebug()
|
||||
args := makeDefaultArgs()
|
||||
ctx := context.Background()
|
||||
hub := server.MakeHubServer(ctx, args)
|
||||
|
||||
go hub.NotifierServer()
|
||||
go hub.RunNotifier()
|
||||
|
||||
addr := fmt.Sprintf(":%d", args.NotifierPort)
|
||||
addr := fmt.Sprintf(":%s", args.NotifierPort)
|
||||
logrus.Info(addr)
|
||||
conn, err := tcpConnReady(addr)
|
||||
if err != nil {
|
||||
|
@ -96,14 +76,11 @@ func TestNotifierServer(t *testing.T) {
|
|||
|
||||
// Hacky but needed because if the reader isn't ready
|
||||
// before the writer sends it won't get the data
|
||||
err = subReady(hub)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
|
||||
hash, _ := hex.DecodeString("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
|
||||
logrus.Warn("sending hash")
|
||||
hub.NotifierChan <- internal.HeightHash{Height: 1, BlockHash: hash}
|
||||
hub.NotifierChan <- &internal.HeightHash{Height: 1, BlockHash: hash}
|
||||
|
||||
res := <-resCh
|
||||
logrus.Info(string(res))
|
||||
|
|
|
@ -11,7 +11,6 @@ import (
|
|||
|
||||
pb "github.com/lbryio/herald.go/protobuf/go"
|
||||
server "github.com/lbryio/herald.go/server"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
"github.com/olivere/elastic/v7"
|
||||
)
|
||||
|
||||
|
@ -56,14 +55,13 @@ func TestSearch(t *testing.T) {
|
|||
w.Write([]byte(resp))
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
stopGroup := stop.NewDebug()
|
||||
args := server.MakeDefaultTestArgs()
|
||||
hubServer := server.MakeHubServer(stopGroup, args)
|
||||
context := context.Background()
|
||||
args := makeDefaultArgs()
|
||||
hubServer := server.MakeHubServer(context, args)
|
||||
req := &pb.SearchRequest{
|
||||
Text: "asdf",
|
||||
}
|
||||
out, err := hubServer.Search(ctx, req)
|
||||
out, err := hubServer.Search(context, req)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
|
154
server/server.go
154
server/server.go
|
@ -8,27 +8,25 @@ import (
|
|||
"fmt"
|
||||
"hash"
|
||||
"io/ioutil"
|
||||
golog "log"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ReneKroon/ttlcache/v2"
|
||||
"github.com/lbryio/herald.go/db"
|
||||
"github.com/lbryio/herald.go/internal"
|
||||
"github.com/lbryio/herald.go/internal/metrics"
|
||||
"github.com/lbryio/herald.go/meta"
|
||||
pb "github.com/lbryio/herald.go/protobuf/go"
|
||||
"github.com/lbryio/lbcd/chaincfg"
|
||||
lbcd "github.com/lbryio/lbcd/rpcclient"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
"github.com/olivere/elastic/v7"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
logrus "github.com/sirupsen/logrus"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/reflection"
|
||||
)
|
||||
|
@ -40,7 +38,6 @@ type Server struct {
|
|||
WeirdCharsRe *regexp.Regexp
|
||||
DB *db.ReadOnlyDBColumnFamily
|
||||
Chain *chaincfg.Params
|
||||
DaemonClient *lbcd.Client
|
||||
EsClient *elastic.Client
|
||||
QueryCache *ttlcache.Cache
|
||||
S256 *hash.Hash
|
||||
|
@ -56,10 +53,7 @@ type Server struct {
|
|||
ExternalIP net.IP
|
||||
HeightSubs map[net.Addr]net.Conn
|
||||
HeightSubsMut sync.RWMutex
|
||||
NotifierChan chan interface{}
|
||||
Grp *stop.Group
|
||||
notiferListener *net.TCPListener
|
||||
sessionManager *sessionManager
|
||||
NotifierChan chan *internal.HeightHash
|
||||
pb.UnimplementedHubServer
|
||||
}
|
||||
|
||||
|
@ -140,8 +134,7 @@ func (s *Server) PeerServersLoadOrStore(peer *Peer) (actual *Peer, loaded bool)
|
|||
|
||||
// Run "main" function for starting the server. This blocks.
|
||||
func (s *Server) Run() {
|
||||
address := ":" + strconv.Itoa(s.Args.Port)
|
||||
l, err := net.Listen("tcp", address)
|
||||
l, err := net.Listen("tcp", ":"+s.Args.Port)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to listen: %v", err)
|
||||
}
|
||||
|
@ -156,50 +149,31 @@ func (s *Server) Run() {
|
|||
}
|
||||
}
|
||||
|
||||
func (s *Server) Stop() {
|
||||
log.Println("Shutting down server...")
|
||||
|
||||
if s.EsClient != nil {
|
||||
log.Println("Stopping es client...")
|
||||
s.EsClient.Stop()
|
||||
}
|
||||
if s.GrpcServer != nil {
|
||||
log.Println("Stopping grpc server...")
|
||||
s.GrpcServer.GracefulStop()
|
||||
}
|
||||
log.Println("Stopping other server threads...")
|
||||
s.Grp.StopAndWait()
|
||||
if s.DB != nil {
|
||||
log.Println("Stopping database connection...")
|
||||
s.DB.Shutdown()
|
||||
}
|
||||
|
||||
log.Println("Returning from Stop...")
|
||||
}
|
||||
|
||||
func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, error) {
|
||||
tmpName, err := os.MkdirTemp("", "go-lbry-hub")
|
||||
func LoadDatabase(args *Args) (*db.ReadOnlyDBColumnFamily, error) {
|
||||
tmpName, err := ioutil.TempDir("", "go-lbry-hub")
|
||||
if err != nil {
|
||||
log.Info(err)
|
||||
logrus.Info(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Info("tmpName", tmpName)
|
||||
logrus.Info("tmpName", tmpName)
|
||||
if err != nil {
|
||||
log.Info(err)
|
||||
logrus.Info(err)
|
||||
}
|
||||
myDB, err := db.GetProdDB(args.DBPath, tmpName, grp)
|
||||
myDB, _, err := db.GetProdDB(args.DBPath, tmpName)
|
||||
// dbShutdown = func() {
|
||||
// db.Shutdown(myDB)
|
||||
// }
|
||||
if err != nil {
|
||||
// Can't load the db, fail loudly
|
||||
log.Info(err)
|
||||
logrus.Info(err)
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
if myDB.LastState != nil {
|
||||
log.Infof("DB version: %v", myDB.LastState.DBVersion)
|
||||
log.Infof("height: %v", myDB.LastState.Height)
|
||||
log.Infof("genesis: %v", myDB.LastState.Genesis.String())
|
||||
log.Infof("tip: %v", myDB.LastState.Tip.String())
|
||||
log.Infof("tx count: %v", myDB.LastState.TxCount)
|
||||
logrus.Infof("DB version: %v", myDB.LastState.DBVersion)
|
||||
logrus.Infof("height: %v", myDB.LastState.Height)
|
||||
logrus.Infof("tip: %v", myDB.LastState.Tip.String())
|
||||
logrus.Infof("tx count: %v", myDB.LastState.TxCount)
|
||||
}
|
||||
|
||||
blockingChannelHashes := make([][]byte, 0, 10)
|
||||
|
@ -210,7 +184,7 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
|
|||
for _, id := range args.BlockingChannelIds {
|
||||
hash, err := hex.DecodeString(id)
|
||||
if err != nil {
|
||||
log.Warn("Invalid channel id: ", id)
|
||||
logrus.Warn("Invalid channel id: ", id)
|
||||
continue
|
||||
}
|
||||
blockingChannelHashes = append(blockingChannelHashes, hash)
|
||||
|
@ -220,7 +194,7 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
|
|||
for _, id := range args.FilteringChannelIds {
|
||||
hash, err := hex.DecodeString(id)
|
||||
if err != nil {
|
||||
log.Warn("Invalid channel id: ", id)
|
||||
logrus.Warn("Invalid channel id: ", id)
|
||||
continue
|
||||
}
|
||||
filteringChannelHashes = append(filteringChannelHashes, hash)
|
||||
|
@ -231,10 +205,10 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
|
|||
myDB.FilteringChannelHashes = filteringChannelHashes
|
||||
|
||||
if len(filteringIds) > 0 {
|
||||
log.Infof("filtering claims reposted by channels: %+s", filteringIds)
|
||||
logrus.Infof("filtering claims reposted by channels: %+s", filteringIds)
|
||||
}
|
||||
if len(blockingIds) > 0 {
|
||||
log.Infof("blocking claims reposted by channels: %+s", blockingIds)
|
||||
logrus.Infof("blocking claims reposted by channels: %+s", blockingIds)
|
||||
}
|
||||
|
||||
return myDB, nil
|
||||
|
@ -243,7 +217,7 @@ func LoadDatabase(args *Args, grp *stop.Group) (*db.ReadOnlyDBColumnFamily, erro
|
|||
// MakeHubServer takes the arguments given to a hub when it's started and
|
||||
// initializes everything. It loads information about previously known peers,
|
||||
// creates needed internal data structures, and initializes goroutines.
|
||||
func MakeHubServer(grp *stop.Group, args *Args) *Server {
|
||||
func MakeHubServer(ctx context.Context, args *Args) *Server {
|
||||
grpcServer := grpc.NewServer(grpc.NumStreamWorkers(0))
|
||||
|
||||
multiSpaceRe, err := regexp.Compile(`\s{2,}`)
|
||||
|
@ -256,34 +230,9 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
|
|||
log.Fatal(err)
|
||||
}
|
||||
|
||||
var lbcdClient *lbcd.Client = nil
|
||||
if args.DaemonURL != nil && args.DaemonURL.Host != "" {
|
||||
var rpcCertificate []byte
|
||||
if args.DaemonCAPath != "" {
|
||||
rpcCertificate, err = ioutil.ReadFile(args.DaemonCAPath)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to read SSL certificate from path: %v", args.DaemonCAPath)
|
||||
}
|
||||
}
|
||||
log.Warnf("connecting to lbcd daemon at %v...", args.DaemonURL.Host)
|
||||
password, _ := args.DaemonURL.User.Password()
|
||||
cfg := &lbcd.ConnConfig{
|
||||
Host: args.DaemonURL.Host,
|
||||
User: args.DaemonURL.User.Username(),
|
||||
Pass: password,
|
||||
HTTPPostMode: true,
|
||||
DisableTLS: rpcCertificate == nil,
|
||||
Certificates: rpcCertificate,
|
||||
}
|
||||
lbcdClient, err = lbcd.New(cfg, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("lbcd daemon connection failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
var esClient *elastic.Client = nil
|
||||
var client *elastic.Client = nil
|
||||
if !args.DisableEs {
|
||||
esUrl := args.EsHost + ":" + fmt.Sprintf("%d", args.EsPort)
|
||||
esUrl := args.EsHost + ":" + args.EsPort
|
||||
opts := []elastic.ClientOptionFunc{
|
||||
elastic.SetSniff(true),
|
||||
elastic.SetSnifferTimeoutStartup(time.Second * 60),
|
||||
|
@ -291,9 +240,9 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
|
|||
elastic.SetURL(esUrl),
|
||||
}
|
||||
if args.Debug {
|
||||
opts = append(opts, elastic.SetTraceLog(golog.New(os.Stderr, "[[ELASTIC]]", 0)))
|
||||
opts = append(opts, elastic.SetTraceLog(log.New(os.Stderr, "[[ELASTIC]]", 0)))
|
||||
}
|
||||
esClient, err = elastic.NewClient(opts...)
|
||||
client, err = elastic.NewClient(opts...)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
@ -317,18 +266,18 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
|
|||
|
||||
//TODO: is this the right place to load the db?
|
||||
var myDB *db.ReadOnlyDBColumnFamily
|
||||
// var dbShutdown = func() {}
|
||||
if !args.DisableResolve {
|
||||
myDB, err = LoadDatabase(args, grp)
|
||||
myDB, err = LoadDatabase(args)
|
||||
if err != nil {
|
||||
log.Warning(err)
|
||||
logrus.Warning(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine which chain to use based on db and cli values
|
||||
dbChain := (*chaincfg.Params)(nil)
|
||||
if myDB != nil && myDB.LastState != nil {
|
||||
if myDB != nil && myDB.LastState != nil && myDB.LastState.Genesis != nil {
|
||||
// The chain params can be inferred from DBStateValue.
|
||||
switch myDB.LastState.Genesis.Hash {
|
||||
switch *myDB.LastState.Genesis {
|
||||
case *chaincfg.MainNetParams.GenesisHash:
|
||||
dbChain = &chaincfg.MainNetParams
|
||||
case *chaincfg.TestNet3Params.GenesisHash:
|
||||
|
@ -351,7 +300,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
|
|||
chain := chaincfg.MainNetParams
|
||||
if dbChain != nil && cliChain != nil {
|
||||
if dbChain != cliChain {
|
||||
log.Warnf("network: %v (from db) conflicts with %v (from cli)", dbChain.Name, cliChain.Name)
|
||||
logrus.Warnf("network: %v (from db) conflicts with %v (from cli)", dbChain.Name, cliChain.Name)
|
||||
}
|
||||
chain = *dbChain
|
||||
} else if dbChain != nil {
|
||||
|
@ -359,11 +308,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
|
|||
} else if cliChain != nil {
|
||||
chain = *cliChain
|
||||
}
|
||||
log.Infof("network: %v", chain.Name)
|
||||
|
||||
args.GenesisHash = chain.GenesisHash.String()
|
||||
|
||||
sessionGrp := stop.New(grp)
|
||||
logrus.Infof("network: %v", chain.Name)
|
||||
|
||||
s := &Server{
|
||||
GrpcServer: grpcServer,
|
||||
|
@ -372,8 +317,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
|
|||
WeirdCharsRe: weirdCharsRe,
|
||||
DB: myDB,
|
||||
Chain: &chain,
|
||||
DaemonClient: lbcdClient,
|
||||
EsClient: esClient,
|
||||
EsClient: client,
|
||||
QueryCache: cache,
|
||||
S256: &s256,
|
||||
LastRefreshCheck: time.Now(),
|
||||
|
@ -388,39 +332,27 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
|
|||
ExternalIP: net.IPv4(127, 0, 0, 1),
|
||||
HeightSubs: make(map[net.Addr]net.Conn),
|
||||
HeightSubsMut: sync.RWMutex{},
|
||||
NotifierChan: make(chan interface{}, 1),
|
||||
Grp: grp,
|
||||
sessionManager: nil,
|
||||
NotifierChan: make(chan *internal.HeightHash),
|
||||
}
|
||||
// FIXME: HACK
|
||||
s.sessionManager = newSessionManager(s, myDB, args, sessionGrp, &chain, lbcdClient)
|
||||
|
||||
// Start up our background services
|
||||
if !args.DisableResolve && !args.DisableRocksDBRefresh {
|
||||
log.Info("Running detect changes")
|
||||
logrus.Info("Running detect changes")
|
||||
myDB.RunDetectChanges(s.NotifierChan)
|
||||
}
|
||||
if !args.DisableBlockingAndFiltering {
|
||||
myDB.RunGetBlocksAndFilters()
|
||||
}
|
||||
if !args.DisableStartPrometheus {
|
||||
go s.prometheusEndpoint(fmt.Sprintf("%d", s.Args.PrometheusPort), "metrics")
|
||||
go s.prometheusEndpoint(s.Args.PrometheusPort, "metrics")
|
||||
}
|
||||
if !args.DisableStartUDP {
|
||||
go func() {
|
||||
err := s.UDPServer(s.Args.Port)
|
||||
err := s.UDPServer()
|
||||
if err != nil {
|
||||
log.Errorf("UDP Server (%d) failed! %v", s.Args.Port, err)
|
||||
log.Println("UDP Server failed!", err)
|
||||
}
|
||||
}()
|
||||
if s.Args.JSONRPCPort != 0 {
|
||||
go func() {
|
||||
err := s.UDPServer(s.Args.JSONRPCPort)
|
||||
if err != nil {
|
||||
log.Errorf("UDP Server (%d) failed! %v", s.Args.JSONRPCPort, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
if !args.DisableStartNotifier {
|
||||
go func() {
|
||||
|
@ -461,7 +393,7 @@ func MakeHubServer(grp *stop.Group, args *Args) *Server {
|
|||
// for this hub to allow for metric tracking.
|
||||
func (s *Server) prometheusEndpoint(port string, endpoint string) {
|
||||
http.Handle("/"+endpoint, promhttp.Handler())
|
||||
log.Printf("listening on :%s /%s\n", port, endpoint)
|
||||
log.Println(fmt.Sprintf("listening on :%s /%s", port, endpoint))
|
||||
err := http.ListenAndServe(":"+port, nil)
|
||||
log.Fatalln("Shouldn't happen??!?!", err)
|
||||
}
|
||||
|
@ -619,7 +551,7 @@ func InternalResolve(urls []string, DB *db.ReadOnlyDBColumnFamily) (*pb.Outputs,
|
|||
BlockedTotal: 0, //TODO
|
||||
}
|
||||
|
||||
log.Warn(res)
|
||||
logrus.Warn(res)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
|
|
@ -1,11 +1,5 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"github.com/lbryio/herald.go/db"
|
||||
"github.com/lbryio/lbcd/chaincfg"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
)
|
||||
|
||||
func (s *Server) AddPeerExported() func(*Peer, bool, bool) error {
|
||||
return s.addPeer
|
||||
}
|
||||
|
@ -13,7 +7,3 @@ func (s *Server) AddPeerExported() func(*Peer, bool, bool) error {
|
|||
func (s *Server) GetNumPeersExported() func() int64 {
|
||||
return s.getNumPeers
|
||||
}
|
||||
|
||||
func NewSessionManagerExported(server *Server, db *db.ReadOnlyDBColumnFamily, args *Args, grp *stop.Group, chain *chaincfg.Params) *sessionManager {
|
||||
return newSessionManager(server, db, args, grp, chain, nil)
|
||||
}
|
||||
|
|
|
@ -1,631 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/rpc"
|
||||
"net/rpc/jsonrpc"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/lbryio/herald.go/db"
|
||||
"github.com/lbryio/herald.go/internal"
|
||||
"github.com/lbryio/lbcd/chaincfg"
|
||||
"github.com/lbryio/lbcd/chaincfg/chainhash"
|
||||
lbcd "github.com/lbryio/lbcd/rpcclient"
|
||||
"github.com/lbryio/lbcd/wire"
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type headerNotification struct {
|
||||
internal.HeightHash
|
||||
blockHeaderElectrum *BlockHeaderElectrum
|
||||
blockHeaderStr string
|
||||
}
|
||||
|
||||
type hashXNotification struct {
|
||||
hashX [HASHX_LEN]byte
|
||||
status []byte
|
||||
statusStr string
|
||||
}
|
||||
|
||||
type peerNotification struct {
|
||||
address string
|
||||
port string
|
||||
}
|
||||
|
||||
type session struct {
|
||||
id uintptr
|
||||
addr net.Addr
|
||||
conn net.Conn
|
||||
// hashXSubs maps hashX to the original subscription key (address or scripthash)
|
||||
hashXSubs map[[HASHX_LEN]byte]string
|
||||
// headersSub indicates header subscription
|
||||
headersSub bool
|
||||
// peersSub indicates peer subscription
|
||||
peersSub bool
|
||||
// headersSubRaw indicates the header subscription mode
|
||||
headersSubRaw bool
|
||||
// client provides the ability to send notifications
|
||||
client rpc.ClientCodec
|
||||
clientSeq uint64
|
||||
// lastRecv records time of last incoming data
|
||||
lastRecv time.Time
|
||||
// lastSend records time of last outgoing data
|
||||
lastSend time.Time
|
||||
}
|
||||
|
||||
func (s *session) doNotify(notification interface{}) {
|
||||
var method string
|
||||
var params interface{}
|
||||
switch note := notification.(type) {
|
||||
case headerNotification:
|
||||
if !s.headersSub {
|
||||
return
|
||||
}
|
||||
heightHash := note.HeightHash
|
||||
method = "blockchain.headers.subscribe"
|
||||
if s.headersSubRaw {
|
||||
header := note.blockHeaderStr
|
||||
if len(header) == 0 {
|
||||
header = hex.EncodeToString(note.BlockHeader[:])
|
||||
}
|
||||
params = &HeadersSubscribeRawResp{
|
||||
Hex: header,
|
||||
Height: uint32(heightHash.Height),
|
||||
}
|
||||
} else {
|
||||
header := note.blockHeaderElectrum
|
||||
if header == nil { // not initialized
|
||||
header = newBlockHeaderElectrum((*[HEADER_SIZE]byte)(note.BlockHeader), uint32(heightHash.Height))
|
||||
}
|
||||
params = header
|
||||
}
|
||||
case hashXNotification:
|
||||
orig, ok := s.hashXSubs[note.hashX]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if len(orig) == 64 {
|
||||
method = "blockchain.scripthash.subscribe"
|
||||
} else {
|
||||
method = "blockchain.address.subscribe"
|
||||
}
|
||||
status := note.statusStr
|
||||
if len(status) == 0 {
|
||||
status = hex.EncodeToString(note.status)
|
||||
}
|
||||
params = []string{orig, status}
|
||||
case peerNotification:
|
||||
if !s.peersSub {
|
||||
return
|
||||
}
|
||||
method = "server.peers.subscribe"
|
||||
params = []string{note.address, note.port}
|
||||
|
||||
default:
|
||||
log.Warnf("unknown notification type: %v", notification)
|
||||
return
|
||||
}
|
||||
// Send the notification.
|
||||
s.clientSeq += 1
|
||||
req := &rpc.Request{
|
||||
ServiceMethod: method,
|
||||
Seq: s.clientSeq,
|
||||
}
|
||||
err := s.client.WriteRequest(req, params)
|
||||
if err != nil {
|
||||
log.Warnf("error: %v", err)
|
||||
}
|
||||
// Bump last send time.
|
||||
s.lastSend = time.Now()
|
||||
}
|
||||
|
||||
type sessionMap map[uintptr]*session
|
||||
|
||||
type sessionManager struct {
|
||||
// sessionsMut protects sessions, headerSubs, hashXSubs state
|
||||
sessionsMut sync.RWMutex
|
||||
sessions sessionMap
|
||||
// sessionsWait sync.WaitGroup
|
||||
grp *stop.Group
|
||||
sessionsMax int
|
||||
sessionTimeout time.Duration
|
||||
manageTicker *time.Ticker
|
||||
db *db.ReadOnlyDBColumnFamily
|
||||
args *Args
|
||||
server *Server
|
||||
chain *chaincfg.Params
|
||||
lbcd *lbcd.Client
|
||||
// peerSubs are sessions subscribed via 'blockchain.peers.subscribe'
|
||||
peerSubs sessionMap
|
||||
// headerSubs are sessions subscribed via 'blockchain.headers.subscribe'
|
||||
headerSubs sessionMap
|
||||
// hashXSubs are sessions subscribed via 'blockchain.{address,scripthash}.subscribe'
|
||||
hashXSubs map[[HASHX_LEN]byte]sessionMap
|
||||
}
|
||||
|
||||
func newSessionManager(server *Server, db *db.ReadOnlyDBColumnFamily, args *Args, grp *stop.Group, chain *chaincfg.Params, lbcd *lbcd.Client) *sessionManager {
|
||||
return &sessionManager{
|
||||
sessions: make(sessionMap),
|
||||
grp: grp,
|
||||
sessionsMax: args.MaxSessions,
|
||||
sessionTimeout: time.Duration(args.SessionTimeout) * time.Second,
|
||||
manageTicker: time.NewTicker(time.Duration(max(5, args.SessionTimeout/20)) * time.Second),
|
||||
db: db,
|
||||
args: args,
|
||||
server: server,
|
||||
chain: chain,
|
||||
lbcd: lbcd,
|
||||
peerSubs: make(sessionMap),
|
||||
headerSubs: make(sessionMap),
|
||||
hashXSubs: make(map[[HASHX_LEN]byte]sessionMap),
|
||||
}
|
||||
}
|
||||
|
||||
func (sm *sessionManager) start() {
|
||||
sm.grp.Add(1)
|
||||
go sm.manage()
|
||||
}
|
||||
|
||||
func (sm *sessionManager) stop() {
|
||||
sm.sessionsMut.Lock()
|
||||
defer sm.sessionsMut.Unlock()
|
||||
sm.headerSubs = make(sessionMap)
|
||||
sm.hashXSubs = make(map[[HASHX_LEN]byte]sessionMap)
|
||||
for _, sess := range sm.sessions {
|
||||
sess.client.Close()
|
||||
sess.conn.Close()
|
||||
}
|
||||
sm.sessions = make(sessionMap)
|
||||
}
|
||||
|
||||
func (sm *sessionManager) manage() {
|
||||
for {
|
||||
sm.sessionsMut.Lock()
|
||||
for _, sess := range sm.sessions {
|
||||
if time.Since(sess.lastRecv) > sm.sessionTimeout {
|
||||
sm.removeSessionLocked(sess)
|
||||
log.Infof("session %v timed out", sess.addr.String())
|
||||
}
|
||||
}
|
||||
sm.sessionsMut.Unlock()
|
||||
// Wait for next management clock tick.
|
||||
select {
|
||||
case <-sm.grp.Ch():
|
||||
sm.grp.Done()
|
||||
return
|
||||
case <-sm.manageTicker.C:
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sm *sessionManager) addSession(conn net.Conn) *session {
|
||||
sm.sessionsMut.Lock()
|
||||
sess := &session{
|
||||
addr: conn.RemoteAddr(),
|
||||
conn: conn,
|
||||
hashXSubs: make(map[[11]byte]string),
|
||||
client: jsonrpc.NewClientCodec(conn),
|
||||
lastRecv: time.Now(),
|
||||
}
|
||||
sess.id = uintptr(unsafe.Pointer(sess))
|
||||
sm.sessions[sess.id] = sess
|
||||
sm.sessionsMut.Unlock()
|
||||
|
||||
// Create a new RPC server. These services are linked to the
|
||||
// session, which allows RPC handlers to know the session for
|
||||
// each request and update subscriptions.
|
||||
s1 := rpc.NewServer()
|
||||
|
||||
// Register "server.{features,banner,version}" handlers.
|
||||
serverSvc := &ServerService{sm.args}
|
||||
err := s1.RegisterName("server", serverSvc)
|
||||
if err != nil {
|
||||
log.Errorf("RegisterName: %v\n", err)
|
||||
}
|
||||
|
||||
// Register "server.peers" handlers.
|
||||
peersSvc := &PeersService{Server: sm.server}
|
||||
err = s1.RegisterName("server.peers", peersSvc)
|
||||
if err != nil {
|
||||
log.Errorf("RegisterName: %v\n", err)
|
||||
}
|
||||
|
||||
// Register "blockchain.claimtrie.*"" handlers.
|
||||
claimtrieSvc := &ClaimtrieService{sm.db, sm.server}
|
||||
err = s1.RegisterName("blockchain.claimtrie", claimtrieSvc)
|
||||
if err != nil {
|
||||
log.Errorf("RegisterName: %v\n", err)
|
||||
}
|
||||
|
||||
// Register "blockchain.{block,address,scripthash,transaction}.*" handlers.
|
||||
blockchainSvc := &BlockchainBlockService{sm.db, sm.chain}
|
||||
err = s1.RegisterName("blockchain.block", blockchainSvc)
|
||||
if err != nil {
|
||||
log.Errorf("RegisterName: %v\n", err)
|
||||
goto fail
|
||||
}
|
||||
err = s1.RegisterName("blockchain.headers", &BlockchainHeadersService{sm.db, sm.chain, sm, sess})
|
||||
if err != nil {
|
||||
log.Errorf("RegisterName: %v\n", err)
|
||||
goto fail
|
||||
}
|
||||
err = s1.RegisterName("blockchain.address", &BlockchainAddressService{sm.db, sm.chain, sm, sess})
|
||||
if err != nil {
|
||||
log.Errorf("RegisterName: %v\n", err)
|
||||
goto fail
|
||||
}
|
||||
err = s1.RegisterName("blockchain.scripthash", &BlockchainScripthashService{sm.db, sm.chain, sm, sess})
|
||||
if err != nil {
|
||||
log.Errorf("RegisterName: %v\n", err)
|
||||
goto fail
|
||||
}
|
||||
err = s1.RegisterName("blockchain.transaction", &BlockchainTransactionService{sm.db, sm.chain, sm})
|
||||
if err != nil {
|
||||
log.Errorf("RegisterName: %v\n", err)
|
||||
goto fail
|
||||
}
|
||||
|
||||
sm.grp.Add(1)
|
||||
go func() {
|
||||
s1.ServeCodec(&sessionServerCodec{jsonrpc.NewServerCodec(newJsonPatchingCodec(conn)), sess})
|
||||
log.Infof("session %v goroutine exit", sess.addr.String())
|
||||
sm.grp.Done()
|
||||
}()
|
||||
return sess
|
||||
|
||||
fail:
|
||||
sm.removeSession(sess)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sm *sessionManager) removeSession(sess *session) {
|
||||
sm.sessionsMut.Lock()
|
||||
defer sm.sessionsMut.Unlock()
|
||||
sm.removeSessionLocked(sess)
|
||||
}
|
||||
|
||||
func (sm *sessionManager) removeSessionLocked(sess *session) {
|
||||
if sess.headersSub {
|
||||
delete(sm.headerSubs, sess.id)
|
||||
}
|
||||
for hashX := range sess.hashXSubs {
|
||||
subs, ok := sm.hashXSubs[hashX]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
delete(subs, sess.id)
|
||||
}
|
||||
delete(sm.sessions, sess.id)
|
||||
sess.client.Close()
|
||||
sess.conn.Close()
|
||||
}
|
||||
|
||||
func (sm *sessionManager) broadcastTx(rawTx []byte) (*chainhash.Hash, error) {
|
||||
var msgTx wire.MsgTx
|
||||
err := msgTx.Deserialize(bytes.NewReader(rawTx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sm.lbcd.SendRawTransaction(&msgTx, false)
|
||||
}
|
||||
|
||||
func (sm *sessionManager) peersSubscribe(sess *session, subscribe bool) {
|
||||
sm.sessionsMut.Lock()
|
||||
defer sm.sessionsMut.Unlock()
|
||||
if subscribe {
|
||||
sm.peerSubs[sess.id] = sess
|
||||
sess.peersSub = true
|
||||
return
|
||||
}
|
||||
delete(sm.peerSubs, sess.id)
|
||||
sess.peersSub = false
|
||||
}
|
||||
|
||||
func (sm *sessionManager) headersSubscribe(sess *session, raw bool, subscribe bool) {
|
||||
sm.sessionsMut.Lock()
|
||||
defer sm.sessionsMut.Unlock()
|
||||
if subscribe {
|
||||
sm.headerSubs[sess.id] = sess
|
||||
sess.headersSub = true
|
||||
sess.headersSubRaw = raw
|
||||
return
|
||||
}
|
||||
delete(sm.headerSubs, sess.id)
|
||||
sess.headersSub = false
|
||||
sess.headersSubRaw = false
|
||||
}
|
||||
|
||||
func (sm *sessionManager) hashXSubscribe(sess *session, hashX []byte, original string, subscribe bool) {
|
||||
sm.sessionsMut.Lock()
|
||||
defer sm.sessionsMut.Unlock()
|
||||
var key [HASHX_LEN]byte
|
||||
copy(key[:], hashX)
|
||||
subs, ok := sm.hashXSubs[key]
|
||||
if subscribe {
|
||||
if !ok {
|
||||
subs = make(sessionMap)
|
||||
sm.hashXSubs[key] = subs
|
||||
}
|
||||
subs[sess.id] = sess
|
||||
sess.hashXSubs[key] = original
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
delete(subs, sess.id)
|
||||
if len(subs) == 0 {
|
||||
delete(sm.hashXSubs, key)
|
||||
}
|
||||
}
|
||||
delete(sess.hashXSubs, key)
|
||||
}
|
||||
|
||||
func (sm *sessionManager) doNotify(notification interface{}) {
|
||||
switch note := notification.(type) {
|
||||
case internal.HeightHash:
|
||||
// The HeightHash notification translates to headerNotification.
|
||||
notification = &headerNotification{HeightHash: note}
|
||||
}
|
||||
|
||||
sm.sessionsMut.RLock()
|
||||
var subsCopy sessionMap
|
||||
switch note := notification.(type) {
|
||||
case headerNotification:
|
||||
log.Infof("header notification @ %#v", note)
|
||||
subsCopy = sm.headerSubs
|
||||
if len(subsCopy) > 0 {
|
||||
hdr := [HEADER_SIZE]byte{}
|
||||
copy(hdr[:], note.BlockHeader)
|
||||
note.blockHeaderElectrum = newBlockHeaderElectrum(&hdr, uint32(note.Height))
|
||||
note.blockHeaderStr = hex.EncodeToString(note.BlockHeader[:])
|
||||
}
|
||||
case hashXNotification:
|
||||
log.Infof("hashX notification @ %#v", note)
|
||||
hashXSubs, ok := sm.hashXSubs[note.hashX]
|
||||
if ok {
|
||||
subsCopy = hashXSubs
|
||||
}
|
||||
if len(subsCopy) > 0 {
|
||||
note.statusStr = hex.EncodeToString(note.status)
|
||||
}
|
||||
case peerNotification:
|
||||
subsCopy = sm.peerSubs
|
||||
default:
|
||||
log.Warnf("unknown notification type: %v", notification)
|
||||
}
|
||||
sm.sessionsMut.RUnlock()
|
||||
|
||||
// Deliver notification to relevant sessions.
|
||||
for _, sess := range subsCopy {
|
||||
sess.doNotify(notification)
|
||||
}
|
||||
|
||||
// Produce secondary hashXNotification(s) corresponding to the headerNotification.
|
||||
switch note := notification.(type) {
|
||||
case headerNotification:
|
||||
touched, err := sm.db.GetTouchedHashXs(uint32(note.Height))
|
||||
if err != nil {
|
||||
log.Errorf("failed to get touched hashXs at height %v, error: %v", note.Height, err)
|
||||
break
|
||||
}
|
||||
for _, hashX := range touched {
|
||||
hashXstatus, err := sm.db.GetStatus(hashX)
|
||||
if err != nil {
|
||||
log.Errorf("failed to get status of hashX %v, error: %v", hashX, err)
|
||||
continue
|
||||
}
|
||||
note2 := hashXNotification{}
|
||||
copy(note2.hashX[:], hashX)
|
||||
note2.status = hashXstatus
|
||||
sm.doNotify(note2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type sessionServerCodec struct {
|
||||
rpc.ServerCodec
|
||||
sess *session
|
||||
}
|
||||
|
||||
// ReadRequestHeader provides ability to rewrite the incoming
|
||||
// request "method" field. For example:
|
||||
//
|
||||
// blockchain.block.get_header -> blockchain.block.Get_header
|
||||
// blockchain.address.listunspent -> blockchain.address.Listunspent
|
||||
//
|
||||
// This makes the "method" string compatible with rpc.Server
|
||||
// requirements.
|
||||
func (c *sessionServerCodec) ReadRequestHeader(req *rpc.Request) error {
|
||||
log.Infof("from %v receive header", c.sess.addr.String())
|
||||
err := c.ServerCodec.ReadRequestHeader(req)
|
||||
if err != nil {
|
||||
log.Warnf("error: %v", err)
|
||||
return err
|
||||
}
|
||||
log.Infof("from %v receive header: %#v", c.sess.addr.String(), *req)
|
||||
rawMethod := req.ServiceMethod
|
||||
parts := strings.Split(rawMethod, ".")
|
||||
if len(parts) < 2 {
|
||||
return fmt.Errorf("blockchain rpc: service/method ill-formed: %q", rawMethod)
|
||||
}
|
||||
service := strings.Join(parts[0:len(parts)-1], ".")
|
||||
method := parts[len(parts)-1]
|
||||
if len(method) < 1 {
|
||||
return fmt.Errorf("blockchain rpc: method ill-formed: %q", method)
|
||||
}
|
||||
method = strings.ToUpper(string(method[0])) + string(method[1:])
|
||||
req.ServiceMethod = service + "." + method
|
||||
return err
|
||||
}
|
||||
|
||||
// ReadRequestBody wraps the regular implementation, but updates session stats too.
|
||||
func (c *sessionServerCodec) ReadRequestBody(params any) error {
|
||||
log.Infof("from %v receive body", c.sess.addr.String())
|
||||
err := c.ServerCodec.ReadRequestBody(params)
|
||||
if err != nil {
|
||||
log.Warnf("error: %v", err)
|
||||
return err
|
||||
}
|
||||
log.Infof("from %v receive body: %#v", c.sess.addr.String(), params)
|
||||
// Bump last receive time.
|
||||
c.sess.lastRecv = time.Now()
|
||||
return err
|
||||
}
|
||||
|
||||
// WriteResponse wraps the regular implementation, but updates session stats too.
|
||||
func (c *sessionServerCodec) WriteResponse(resp *rpc.Response, reply any) error {
|
||||
log.Infof("respond to %v", c.sess.addr.String())
|
||||
err := c.ServerCodec.WriteResponse(resp, reply)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Bump last send time.
|
||||
c.sess.lastSend = time.Now()
|
||||
return err
|
||||
}
|
||||
|
||||
// serverRequest is a duplicate of serverRequest from
|
||||
// net/rpc/jsonrpc/server.go with an added Version which
|
||||
// we can check.
|
||||
type serverRequest struct {
|
||||
Version string `json:"jsonrpc"`
|
||||
Method string `json:"method"`
|
||||
Params *json.RawMessage `json:"params"`
|
||||
Id *json.RawMessage `json:"id"`
|
||||
}
|
||||
|
||||
// serverResponse is a duplicate of serverResponse from
|
||||
// net/rpc/jsonrpc/server.go with an added Version which
|
||||
// we can set at will.
|
||||
type serverResponse struct {
|
||||
Version string `json:"jsonrpc"`
|
||||
Id *json.RawMessage `json:"id"`
|
||||
Result any `json:"result,omitempty"`
|
||||
Error any `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// jsonPatchingCodec is able to intercept the JSON requests/responses
|
||||
// and tweak them. Currently, it appears we need to make several changes:
|
||||
// 1) add "jsonrpc": "2.0" (or "jsonrpc": "1.0") in response
|
||||
// 2) add newline to frame response
|
||||
// 3) add "params": [] when "params" is missing
|
||||
// 4) replace params ["arg1", "arg2", ...] with [["arg1", "arg2", ...]]
|
||||
type jsonPatchingCodec struct {
|
||||
conn net.Conn
|
||||
inBuffer *bytes.Buffer
|
||||
dec *json.Decoder
|
||||
enc *json.Encoder
|
||||
outBuffer *bytes.Buffer
|
||||
}
|
||||
|
||||
func newJsonPatchingCodec(conn net.Conn) *jsonPatchingCodec {
|
||||
buf1, buf2 := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
|
||||
return &jsonPatchingCodec{
|
||||
conn: conn,
|
||||
inBuffer: buf1,
|
||||
dec: json.NewDecoder(buf1),
|
||||
enc: json.NewEncoder(buf2),
|
||||
outBuffer: buf2,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *jsonPatchingCodec) Read(p []byte) (n int, err error) {
|
||||
if c.outBuffer.Len() > 0 {
|
||||
// Return remaining decoded bytes.
|
||||
return c.outBuffer.Read(p)
|
||||
}
|
||||
// Buffer contents consumed. Try to decode more JSON.
|
||||
|
||||
// Read until framing newline. This allows us to print the raw request.
|
||||
for !bytes.ContainsAny(c.inBuffer.Bytes(), "\n") {
|
||||
var buf [1024]byte
|
||||
n, err = c.conn.Read(buf[:])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
c.inBuffer.Write(buf[:n])
|
||||
}
|
||||
log.Infof("raw request: %v", c.inBuffer.String())
|
||||
|
||||
var req serverRequest
|
||||
err = c.dec.Decode(&req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if req.Params != nil {
|
||||
n := len(*req.Params)
|
||||
if n < 2 || (*req.Params)[0] != '[' && (*req.Params)[n-1] != ']' {
|
||||
// This is an error, but we're not going to try to correct it.
|
||||
goto encode
|
||||
}
|
||||
// FIXME: The heuristics here don't cover all possibilities.
|
||||
// For example: [{obj1}, {obj2}] or ["foo,bar"] would not
|
||||
// be handled correctly.
|
||||
bracketed := (*req.Params)[1 : n-1]
|
||||
n = len(bracketed)
|
||||
if n > 1 && (bracketed[0] == '{' || bracketed[0] == '[') {
|
||||
// Probable single object or list argument.
|
||||
goto encode
|
||||
}
|
||||
// The params look like ["arg1", "arg2", "arg3", ...].
|
||||
// We're in trouble because our jsonrpc library does not
|
||||
// handle this. So pack these args in an inner list.
|
||||
// The handler method will receive ONE list argument.
|
||||
params := json.RawMessage(fmt.Sprintf("[[%s]]", bracketed))
|
||||
req.Params = ¶ms
|
||||
} else {
|
||||
// Add empty argument list if params omitted.
|
||||
params := json.RawMessage("[]")
|
||||
req.Params = ¶ms
|
||||
}
|
||||
|
||||
encode:
|
||||
// Encode the request. This allows us to print the patched request.
|
||||
buf, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
log.Infof("patched request: %v", string(buf))
|
||||
|
||||
err = c.enc.Encode(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.outBuffer.Read(p)
|
||||
}
|
||||
|
||||
func (c *jsonPatchingCodec) Write(p []byte) (n int, err error) {
|
||||
log.Infof("raw response: %v", string(p))
|
||||
var resp serverResponse
|
||||
err = json.Unmarshal(p, &resp)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Add "jsonrpc": "2.0" if missing.
|
||||
if len(resp.Version) == 0 {
|
||||
resp.Version = "2.0"
|
||||
}
|
||||
|
||||
buf, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
log.Infof("patched response: %v", string(buf))
|
||||
|
||||
// Add newline for framing.
|
||||
return c.conn.Write(append(buf, '\n'))
|
||||
}
|
||||
|
||||
func (c *jsonPatchingCodec) Close() error {
|
||||
return c.conn.Close()
|
||||
}
|
|
@ -219,9 +219,8 @@ func UDPPing(ip, port string) (*SPVPong, error) {
|
|||
// UDPServer is a goroutine that starts an udp server that implements the hubs
|
||||
// Ping/Pong protocol to find out about each other without making full TCP
|
||||
// connections.
|
||||
func (s *Server) UDPServer(port int) error {
|
||||
address := ":" + strconv.Itoa(port)
|
||||
|
||||
func (s *Server) UDPServer() error {
|
||||
address := ":" + s.Args.Port
|
||||
tip := make([]byte, 32)
|
||||
addr, err := net.ResolveUDPAddr("udp", address)
|
||||
if err != nil {
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
|
||||
// TestUDPPing tests UDPPing correctness against prod server.
|
||||
func TestUDPPing(t *testing.T) {
|
||||
args := server.MakeDefaultTestArgs()
|
||||
args := makeDefaultArgs()
|
||||
args.DisableStartUDP = true
|
||||
|
||||
tests := []struct {
|
||||
|
|
|
@ -8,13 +8,12 @@ import (
|
|||
"os"
|
||||
"os/signal"
|
||||
|
||||
"github.com/lbryio/lbry.go/v3/extras/stop"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// shutdownRequestChannel is used to initiate shutdown from one of the
|
||||
// subsystems using the same code paths as when an interrupt signal is received.
|
||||
var shutdownRequestChannel = make(stop.Chan)
|
||||
var shutdownRequestChannel = make(chan struct{})
|
||||
|
||||
// interruptSignals defines the default signals to catch in order to do a proper
|
||||
// shutdown. This may be modified during init depending on the platform.
|
||||
|
@ -49,6 +48,7 @@ func interruptListener() <-chan struct{} {
|
|||
case sig := <-interruptChannel:
|
||||
log.Infof("Received signal (%s). Already "+
|
||||
"shutting down...", sig)
|
||||
|
||||
case <-shutdownRequestChannel:
|
||||
log.Info("Shutdown requested. Already " +
|
||||
"shutting down...")
|
||||
|
|
18
testdata/Si_resolve.csv
vendored
18
testdata/Si_resolve.csv
vendored
|
@ -1,18 +0,0 @@
|
|||
Si,,
|
||||
S,53000000a420c44374f4f399ab4807fa1901eefc8701000e94ad0297ec210000,00000000000f4240
|
||||
S,53000000c27eef5ea69e0d73f118826c7e326bb46901000773de00371d660000,000000001dcd6500
|
||||
S,5300000110e40894573f528c393fbcec7a472ec85301000d069c01516b320000,0000000000989680
|
||||
S,5300000324e40fcb63a0b517a3660645e9bd99244a01000f2fd8030bb6ba0000,00000000000f4240
|
||||
S,5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff4030bc8a50000,0000000001312d00
|
||||
S,5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff6030bc8b00000,0000000000000003
|
||||
S,5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff7030bc8b10000,0000000000000002
|
||||
S,5300000324e40fcb63a0b517a3660645e9bd99244a02000f2ff9030bc8cf0000,0000000000000001
|
||||
S,53000003d1538a0f19f5cd4bc1a62cc294f5c8993401000c816a011c7c990000,00000000000f4240
|
||||
S,53000008d47beeff8325e795a8604226145b01702b01000ef1ed02dbb2a20000,00000000000186a0
|
||||
S,5300000906499e073e94370ceff37cb21c2821244401000fa7c40369842d0000,00000000000186a0
|
||||
S,5300000906499e073e94370ceff37cb21c2821244402000fa7c403698fff0000,00000000000000a1
|
||||
S,5300000906499e073e94370ceff37cb21c2821244402000fa7c80369f0010000,000000000000000f
|
||||
S,53000009c3172e034a255f3c03566dca84bb9f046a01000e07020225c69c0000,000000000007a120
|
||||
S,53000009ca6e0caaaef16872b4bd4f6f1b8c2363e201000eb5af02b169560000,00000000000f4240
|
||||
i,6900000324e40fcb63a0b517a3660645e9bd99244a,0000000001406f460000000001312d06
|
||||
i,6900000906499e073e94370ceff37cb21c28212444,000000000001875000000000000000b0
|
|
40
testdata/f.csv
vendored
40
testdata/f.csv
vendored
|
@ -1,21 +1,21 @@
|
|||
f,
|
||||
660d649ba1defa4ab5ab71f8,919be5811844077f4660af66afa9a59a5ad17cf5c541524e780fe2137bfa250c
|
||||
6623c6895027f70a5330bbcb,8dadcde1a6f676d4004eacd399f825006ddf136d1e92b1c92113377b3e1741b4
|
||||
664f095b24484ebce8f31fbf,c0c4a751f569c1f9c01531f57ba674b2ad2338d9c08f9e9fc85b0209d15466b2
|
||||
665201a38de7d7243df717c9,d9293577cc0d51fe3a5bee78fea9b2b2222e6c2aa0d26a4ef4bfb7dd095587e8
|
||||
665328b2449e537b0ca4733f,624f80a361e47c7eb1b815e8714a40f67b4f642a5546547a3fcb5bf5593d8fab
|
||||
665ec882021f55b1fbaa5fad,1e917fbc04385290d654f711bdef12773dd54b6b5ea26fe2a9d58ed051f2cb7f
|
||||
6671c131cd433750ba6d3908,a2ebfbdf7a23024c340a45f201645aa46f48bc1fdd8d34ed83fcffbf1ee90523
|
||||
667fb93d9ae877ba11f337f2,4710649e06619e13250754937e9c17c20b07434751171aac2f2f78b184aa0146
|
||||
668ed5f39a5db059dc326137,8dd8ca749b87f43e290904749a546fe319c9d53e765f065bb8beb234a117655e
|
||||
66951782f6ba94f2b71e46d0,4f5c9434dd0886c57c2530991cebd973e1b50d5ba8fcfc019e54561217a49bbb
|
||||
66970565dfe2b01cad49b73a,f6ca0ae18c896d9bc97c5a9d0c3a06256485f59c77fb91780b213f933b80f48b
|
||||
669f6a30a6712062da0cc271,5c6604bfd63b871daceb7893dd618850458974fe4108871c1a1323fb8ae34e4e
|
||||
66a9a7b89b78553592acf3df,0561f28c3a5ea0027ecb3c53fa068772a6b7cb73d23104a14f9aba8cd1f070a2
|
||||
66aba81567ba48f001f843f0,b0f6ae2c1db8263f7e11fc79423109e718d1f3c30bd123c4243401b5e4f1fee6
|
||||
66b569cc3d28be4466fb28d1,ecee392ad8217f325508ba38d280436fb0a520b79a9627e5e18197bf55540885
|
||||
66d4662cd100d66055917d63,5762a8ac767fa30d2ca76db7081f8a2e4f5da4f0bf92d29e1322da9a154cc3d6
|
||||
66d6fa6ac71d0255dd3f185d,5fc193e5e51b3bd8e95f4eb9df63236da7abf678fc47c0b339ceb5c127d0f488
|
||||
66e5b6c7c231a02a32eedd83,58c70ffbfada12550f24bf7931cee06eb2e267dec3560e2e46843e383415f163
|
||||
66e673cce02c2163f756491e,b8db43d1f6e62361e2e3b8fa765f79c08ddfb3035caa06f8250d6d1b063a7140
|
||||
66fc4ad75184e6029c805d94,fc7ac5e785f73732d95183d6bdc3423d41a074fc3f04b1304bae1efa652edde1
|
||||
660d649ba1defa4ab5ab71f8a977d7f7cedb11056e,919be5811844077f4660af66afa9a59a5ad17cf5c541524e780fe2137bfa250c
|
||||
6623c6895027f70a5330bbcb1153d635abcb4d5224,8dadcde1a6f676d4004eacd399f825006ddf136d1e92b1c92113377b3e1741b4
|
||||
664f095b24484ebce8f31fbf008e63cc4aa163d401,c0c4a751f569c1f9c01531f57ba674b2ad2338d9c08f9e9fc85b0209d15466b2
|
||||
665201a38de7d7243df717c9f9279cdd30105f0f77,d9293577cc0d51fe3a5bee78fea9b2b2222e6c2aa0d26a4ef4bfb7dd095587e8
|
||||
665328b2449e537b0ca4733f87ac5ebcdf033c5ebd,624f80a361e47c7eb1b815e8714a40f67b4f642a5546547a3fcb5bf5593d8fab
|
||||
665ec882021f55b1fbaa5fad00df5c5d07633b7af3,1e917fbc04385290d654f711bdef12773dd54b6b5ea26fe2a9d58ed051f2cb7f
|
||||
6671c131cd433750ba6d3908150ca4910841164b74,a2ebfbdf7a23024c340a45f201645aa46f48bc1fdd8d34ed83fcffbf1ee90523
|
||||
667fb93d9ae877ba11f337f21422b0679852580802,4710649e06619e13250754937e9c17c20b07434751171aac2f2f78b184aa0146
|
||||
668ed5f39a5db059dc3261377f2a47728f7a357d33,8dd8ca749b87f43e290904749a546fe319c9d53e765f065bb8beb234a117655e
|
||||
66951782f6ba94f2b71e46d0cc4a2411b14d81eb70,4f5c9434dd0886c57c2530991cebd973e1b50d5ba8fcfc019e54561217a49bbb
|
||||
66970565dfe2b01cad49b73a085a3c3f7a3be61c4c,f6ca0ae18c896d9bc97c5a9d0c3a06256485f59c77fb91780b213f933b80f48b
|
||||
669f6a30a6712062da0cc27181845c04d7430abf73,5c6604bfd63b871daceb7893dd618850458974fe4108871c1a1323fb8ae34e4e
|
||||
66a9a7b89b78553592acf3dfc417c1d7654dab3273,0561f28c3a5ea0027ecb3c53fa068772a6b7cb73d23104a14f9aba8cd1f070a2
|
||||
66aba81567ba48f001f843f01354d575c2e2687847,b0f6ae2c1db8263f7e11fc79423109e718d1f3c30bd123c4243401b5e4f1fee6
|
||||
66b569cc3d28be4466fb28d147f66d6d8769598964,ecee392ad8217f325508ba38d280436fb0a520b79a9627e5e18197bf55540885
|
||||
66d4662cd100d66055917d6342d48f49d948fcc255,5762a8ac767fa30d2ca76db7081f8a2e4f5da4f0bf92d29e1322da9a154cc3d6
|
||||
66d6fa6ac71d0255dd3f185de6480d5b4316b6b050,5fc193e5e51b3bd8e95f4eb9df63236da7abf678fc47c0b339ceb5c127d0f488
|
||||
66e5b6c7c231a02a32eedd8383a5750fd135244a03,58c70ffbfada12550f24bf7931cee06eb2e267dec3560e2e46843e383415f163
|
||||
66e673cce02c2163f756491ef05d7535ceb578e215,b8db43d1f6e62361e2e3b8fa765f79c08ddfb3035caa06f8250d6d1b063a7140
|
||||
66fc4ad75184e6029c805d9494eed4e81be770c002,fc7ac5e785f73732d95183d6bdc3423d41a074fc3f04b1304bae1efa652edde1
|
||||
|
|
|
40
testdata/g.csv
vendored
40
testdata/g.csv
vendored
|
@ -1,21 +1,21 @@
|
|||
g,
|
||||
6702c124856d5168381a3297,575696fd653a4de2f9a8c1f580cf0c229631b0f5d95fceb354cda133e2eb2d34
|
||||
6707f1511e3a2cb28493f91b,ba368e0f859ee36da8701df1c0b52cbf0c0f8a4b1a91f6d0db83a408f5a937d1
|
||||
6707fd4213cae8d5342a98ba,bd3a44d30f66444f8732119bc7e0cf0bb47f8f0ab2840987fc06b629f3e6d3f4
|
||||
6710294a5693224a6222404b,de35a8ea0a26d17445e2f509db23188961b5cd1229b96d2411565adf63731b5c
|
||||
6716a9f84e02143b50d9034a,5823640ae4529f8df2dab20386c887d0a1ba1ffa4583b99dff761c01f670c2fa
|
||||
672e51bc65c9b97d482b0b72,0687df449bd8cb8d8f526f4189973d084d786ab0927d81c127f56b03c61aa955
|
||||
67682620db65932047689e5e,b262d40758edb28d1c04fa3a24d8268990516de6846ad94d002ce55640866239
|
||||
676e8c320dbbf5eebc2969a9,c9e2a8e7181a70e2a488b884c8baadb4043a075c6876cb012c67fbec5aa9f615
|
||||
6772e2ac48891ee3c2c72783,985a9c9ee7a0626d78dab431e663289762ce6959be314f91f7b08b1466097fd6
|
||||
67847dd1dac117b85d1e20d9,62e6b1b8c2961703a90276dcde6dad182b2d14e23f27dccc927cca7770b9890e
|
||||
678f49948c72b7295f12092a,2e7c456dac5206c5627736924e96ac016a09a88ec5f4835fbe0cf9e294611c88
|
||||
67948b9633ab2ec07d752593,66b5c54b3a685de3ea18f9e69254eec065eb3207ac1f93494fdcd585e9a267a0
|
||||
679674c162db8d3bb57c434f,05425880d80258f7441859b3494415a3fd7398c9e209a19674abd48372b283c6
|
||||
67a8d3f17df85502bd644a36,1efce69a3a05c505e9f9cc5c2241d02099c043d934389b430fd8b185e6dfe6cb
|
||||
67bad7f4fb3c6828b6fc4624,04a1c0a7ffe7acbf974ca18cf3debbd8e1be3d6703f842f57ef14af6d4c336d3
|
||||
67c13fb0c65acca5520bc2f5,7fdc6989cd778baad45cd98358ea060237b169a4aeaeb14da6ac4686b7858c9f
|
||||
67d4314588b4424b0ee02653,c63fd7a85a533b8591577bab805104708ba5458fab0e343d46b3e24a28b92cb5
|
||||
67d734244f85f32a58e34e2d,d19a6307c24470b3973973319770bdb896218bb58d1f2d07c7226266075057d0
|
||||
67d9c159c5d5e407e6b0a4ca,89cbdb903fdfe0b44e74b0a69eed3de7029f18c28f77e5509f8ace766ab86610
|
||||
67fafc73d674250f11e559ab,1752ffbf9807bb2e4e480bf045b4bacc472befe755287384b5a526065a58c065
|
||||
6702c124856d5168381a32971d8933440a1728fc41,575696fd653a4de2f9a8c1f580cf0c229631b0f5d95fceb354cda133e2eb2d34
|
||||
6707f1511e3a2cb28493f91b85e9e4a9d9d07c86a5,ba368e0f859ee36da8701df1c0b52cbf0c0f8a4b1a91f6d0db83a408f5a937d1
|
||||
6707fd4213cae8d5342a98ba49b255fa80b2a9a6e4,bd3a44d30f66444f8732119bc7e0cf0bb47f8f0ab2840987fc06b629f3e6d3f4
|
||||
6710294a5693224a6222404ba45fd38eb2e77979a4,de35a8ea0a26d17445e2f509db23188961b5cd1229b96d2411565adf63731b5c
|
||||
6716a9f84e02143b50d9034aec126b12d7f2708cc4,5823640ae4529f8df2dab20386c887d0a1ba1ffa4583b99dff761c01f670c2fa
|
||||
672e51bc65c9b97d482b0b720e6cb673c41fe7b5c5,0687df449bd8cb8d8f526f4189973d084d786ab0927d81c127f56b03c61aa955
|
||||
67682620db65932047689e5eaf392d6b85be801864,b262d40758edb28d1c04fa3a24d8268990516de6846ad94d002ce55640866239
|
||||
676e8c320dbbf5eebc2969a93fbc51dd7f6062a7d1,c9e2a8e7181a70e2a488b884c8baadb4043a075c6876cb012c67fbec5aa9f615
|
||||
6772e2ac48891ee3c2c727835702a374ad0cb70fd6,985a9c9ee7a0626d78dab431e663289762ce6959be314f91f7b08b1466097fd6
|
||||
67847dd1dac117b85d1e20d93580cdf42f00001a77,62e6b1b8c2961703a90276dcde6dad182b2d14e23f27dccc927cca7770b9890e
|
||||
678f49948c72b7295f12092a24d300eeff894f1dd7,2e7c456dac5206c5627736924e96ac016a09a88ec5f4835fbe0cf9e294611c88
|
||||
67948b9633ab2ec07d7525936254e66f8c957d026c,66b5c54b3a685de3ea18f9e69254eec065eb3207ac1f93494fdcd585e9a267a0
|
||||
679674c162db8d3bb57c434fe87825625c4d4daf63,05425880d80258f7441859b3494415a3fd7398c9e209a19674abd48372b283c6
|
||||
67a8d3f17df85502bd644a364721e6364d61635b73,1efce69a3a05c505e9f9cc5c2241d02099c043d934389b430fd8b185e6dfe6cb
|
||||
67bad7f4fb3c6828b6fc4624d43786fc8f55d6eb0f,04a1c0a7ffe7acbf974ca18cf3debbd8e1be3d6703f842f57ef14af6d4c336d3
|
||||
67c13fb0c65acca5520bc2f59bd91ca3482dbec156,7fdc6989cd778baad45cd98358ea060237b169a4aeaeb14da6ac4686b7858c9f
|
||||
67d4314588b4424b0ee026536b9bd7857f11cab2ee,c63fd7a85a533b8591577bab805104708ba5458fab0e343d46b3e24a28b92cb5
|
||||
67d734244f85f32a58e34e2d9cadf225a56973d32f,d19a6307c24470b3973973319770bdb896218bb58d1f2d07c7226266075057d0
|
||||
67d9c159c5d5e407e6b0a4cacf9d6fe62a55b0fedc,89cbdb903fdfe0b44e74b0a69eed3de7029f18c28f77e5509f8ace766ab86610
|
||||
67fafc73d674250f11e559ab08b287f5714e531761,1752ffbf9807bb2e4e480bf045b4bacc472befe755287384b5a526065a58c065
|
||||
|
|
|
40
testdata/i.csv
vendored
40
testdata/i.csv
vendored
|
@ -1,21 +1,21 @@
|
|||
i,
|
||||
691d3476414324a257c62079b055446cdfdb58fcb7,3fc1f36ad9acdae3160db55befe1fdcf
|
||||
692514be0d49b250058c2c59c62b07383705d71c54,055bf209d8b7132f1743aee761d9c64d
|
||||
6932de7e3a7bae762b9533e955fd2b2444f6656aa7,9371781e0655df3b914d18da42605b3d
|
||||
6938e5c7d134233b0758e54f5eacb9dcee412f51f9,12079ef8dffde085385b2aafe7e8be53
|
||||
693a56c48c3ec6bc90fdd02e0f0e2f01472c3f13f5,8c422f4f35e4170079a13f3e9f25a9db
|
||||
693fe5c0979c7c4892486d478c8c57b75f0fa9bba3,8eeaafae4e906ccc36ec29bc3d4f1676
|
||||
694abea2af1c27003a1c777e84891a0f81b3b5a382,fe24b3d28f8cf49fad2d4548726ac8bd
|
||||
694c245cf621a28925da7d84e77137a1d54085d1b8,c04cf11c401c4fbc8976825e9b6db9ca
|
||||
6951010e69f84df87d09cdae7706e25ecdc10a8a6f,a93d6f9c06d1e807c1d674828167cd7c
|
||||
695661d8955be621b5f44c7965f517c17d2d1d48c6,8b945701a1d2920c3e6283ab7fda14ee
|
||||
696fac500a5674eaa078abc71ceb48006c14a3f6aa,1f8000aec89b229349aa154e72fd4de3
|
||||
697506379203bd2f8980d13c766966e400509e28f9,5ce938e06a98aa8b8bb0cfea5dce7e33
|
||||
6975c5f2cdc6e8fdb64682557b1bcbb92f52c6113f,2817aa0f0806bb276e1af4af42504720
|
||||
6984b87daaba891147b4c0f25c15703b2640df9833,169009ea3ff014d352a13152c8d39999
|
||||
699f3d1a3f634bb035c626cdfa927caa59e2617bc4,8f3e2352ed874155a3aa3dd90a61430e
|
||||
69b9dfcdaced1d6d696fab02be31cbee50cbffcdf9,281d9817a5360c1be0ac7710132adebe
|
||||
69ca1fa3e939d061d74163b2da17c3d2b926484c5e,40ecc3bd5dc6b871ce3095456e043427
|
||||
69cea59483161df9420027c4328f85382871798ee4,3919862cc4f0f910a954ffc4d08a6195
|
||||
69d8ff3b5f44f5585e7d7e1349d1d62ba3fbe6831c,d48bd4f6c44ef8b6aabeb6b6e1a61894
|
||||
69f22b1918f28b1e10d2946f84f6f3c8fa25865ba3,b49011d36a56e0dbe5a5cbce94159f66
|
||||
6903dc9970183b8a05d118c84a37c355fe34d95d01,0e74dd23295a4610
|
||||
6916bd29750d8d92b32677eda07e10af313c0019d9,ff0579207ec6e594
|
||||
6926bdfcb4a1ad81f460ad561283584695cd6cea59,b834b13a8918262f
|
||||
6930776827481ec15fa07e0dc266e376846467237d,4bf0a5127a1216dc
|
||||
6955a1eaf08f9468a6c3565fe16b2ae4b726045538,e32029de8b58dd6e
|
||||
69614fa6bc0cea1366377456bc88dda9ec7b6d4c3c,55bf2d8e0e262697
|
||||
6971e7b039dde5797ae167b9783f87c4a67711799d,9197b827b560fc34
|
||||
697765a71d8a4082d056baaae83f8b4af1e124f5e9,62c0d5dfde7ef884
|
||||
6993e121642c01e2edca50d565ff3df1a656e83ebd,1796c74886d45045
|
||||
69af0684076bc64adcbcd621de9c09fd15dade3e17,f9240ab9a9650d9f
|
||||
69b70fdcc95d3b51ec28872a143c7a6fc947e6a58e,a8950968d95759a9
|
||||
69bdb90916f78badec506a895b0edceb47533297f9,331c0ca597601ed7
|
||||
69c276b7070ba79e75c82e4d97a70e4428dd3be058,85c61c842e5bfe7f
|
||||
69cb8215b0c9440227a9e7e76bce468bdb4fa0f714,9c42e1ba41275362
|
||||
69d2556fe7b8fce36a71c78e0001452298903ef81b,f61cf52e7e645bf8
|
||||
69d2677078f1448c0e4295711e61a85a9fb6a280d1,28d57b45b1700cb3
|
||||
69dd979490444cab3efc1e517f83369df6e2e279a3,dad1b5133cc15dd4
|
||||
69e6e6834cf0da132c77af275cbab9885cbbc3f022,df55f9fd0278ca71
|
||||
69ea25444b8d0ab22ac355f4b692b0d7006b672f4a,0703f38a1428ff8e
|
||||
69fcb79c5e463ac4a0e078f9cd24e2c718b66c40d6,5be7b71b6dca5a20
|
||||
|
|
|
|
@ -1 +1 @@
|
|||
v0.2022.10.05.1
|
||||
v0.2022.08.16.1
|
Loading…
Reference in a new issue