server.xxx endpoints

Additional server endpoints in jsonrpc and also some refactoring

go.mod

@@ -24,9 +24,6 @@ require (
 	gopkg.in/karalabe/cookiejar.v1 v1.0.0-20141109175019-e1490cae028c
 )
 
-require (
-)
-
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
@@ -43,7 +40,7 @@ require (
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
-	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
 	google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 // indirect

go.sum

@@ -358,8 +358,6 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
 github.com/lbryio/lbcd v0.22.100-beta/go.mod h1:u8SaFX4xdGMMR5xasBGfgApC8pvD4rnK2OujZnrq5gs=
 github.com/lbryio/lbcd v0.22.100-beta-rc5/go.mod h1:9PbFSlHYX7WlnDQwcTxHVf1W35VAnRsattCSyKOO55g=
 github.com/lbryio/lbcd v0.22.200-beta/go.mod h1:kNuzGWf808ipTGB0y0WogzsGv5BVM4Qv85Z+JYwC9FA=
-github.com/lbryio/lbcd v0.22.201-beta-rc1 h1:FmzzApVj2RBXloLM2w9tLvN2xyTZjeyh+QC7GIw/wwo=
-github.com/lbryio/lbcd v0.22.201-beta-rc1/go.mod h1:kNuzGWf808ipTGB0y0WogzsGv5BVM4Qv85Z+JYwC9FA=
 github.com/lbryio/lbcd v0.22.201-beta-rc4 h1:Xh751Bh/GWRcP5bI6NJ2+zueo2otTcTWapFvFbryP5c=
 github.com/lbryio/lbcd v0.22.201-beta-rc4/go.mod h1:Jgo48JDINhdOgHHR83J70Q6G42x3WAo9DI//QogcL+E=
 github.com/lbryio/lbcutil v1.0.201/go.mod h1:gDHc/b+Rdz3J7+VB8e5/Bl9roVf8Q5/8FQCyuK9dXD0=
@@ -375,8 +373,6 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/linxGnu/grocksdb v1.6.42 h1:nJLoXFuzwBwQQQrXTUgRGRz1QRm7y8pR6CNV/gwrbqs=
 github.com/linxGnu/grocksdb v1.6.42/go.mod h1:JcMMDBFaDNhRXFYcYXmgQwb/RarSld1PulTI7UzE+w0=
-github.com/linxGnu/grocksdb v1.7.0 h1:UyFDykX0CUfxDN10cqlFho/rwt9K6KoDaLXL9Ej5z9g=
-github.com/linxGnu/grocksdb v1.7.0/go.mod h1:JcMMDBFaDNhRXFYcYXmgQwb/RarSld1PulTI7UzE+w0=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=

server/args.go

@@ -39,6 +39,16 @@ type Args struct {
 	Country             string
 	BlockingChannelIds  []string
 	FilteringChannelIds []string
+
+	GenesisHash       string
+	ServerVersion     string
+	ProtocolMin       string
+	ProtocolMax       string
+	ServerDescription string
+	PaymentAddress    string
+	DonationAddress   string
+	DailyFee          string
+
 	Debug                       bool
 	DisableEs                   bool
 	DisableLoadPeers            bool
@@ -69,6 +79,16 @@ const (
 	DefaultCacheTTL       = 5
 	DefaultPeerFile       = "peers.txt"
 	DefaultCountry        = "US"
+
+	GENESIS_HASH             = "9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463"
+	HUB_PROTOCOL_VERSION     = "0.107.0"
+	PROTOCOL_MIN             = "0.54.0"
+	PROTOCOL_MAX             = "0.199.0"
+	DefaultServerDescription = "Herald"
+	DefaultPaymentAddress    = ""
+	DefaultDonationAddress   = ""
+	DefaultDailyFee          = "1.0"
+
 	DefaultDisableLoadPeers            = false
 	DefaultDisableStartPrometheus      = false
 	DefaultDisableStartUDP             = false
@@ -86,6 +106,49 @@ var (
 	DefaultFilteringChannelIds = []string{}
 )
 
+// MakeDefaultArgs creates a default set of arguments for testing the server.
+func MakeDefaultTestArgs() *Args {
+	args := &Args{
+		CmdType:        ServeCmd,
+		Host:           DefaultHost,
+		Port:           DefaultPort,
+		DBPath:         DefaultDBPath,
+		EsHost:         DefaultEsHost,
+		EsPort:         DefaultEsPort,
+		PrometheusPort: DefaultPrometheusPort,
+		NotifierPort:   DefaultNotifierPort,
+		JSONRPCPort:    DefaultJSONRPCPort,
+		EsIndex:        DefaultEsIndex,
+		RefreshDelta:   DefaultRefreshDelta,
+		CacheTTL:       DefaultCacheTTL,
+		PeerFile:       DefaultPeerFile,
+		Country:        DefaultCountry,
+
+		GenesisHash:       GENESIS_HASH,
+		ServerVersion:     HUB_PROTOCOL_VERSION,
+		ProtocolMin:       PROTOCOL_MIN,
+		ProtocolMax:       PROTOCOL_MAX,
+		ServerDescription: DefaultServerDescription,
+		PaymentAddress:    DefaultPaymentAddress,
+		DonationAddress:   DefaultDonationAddress,
+		DailyFee:          DefaultDailyFee,
+
+		DisableEs:                   true,
+		Debug:                       true,
+		DisableLoadPeers:            true,
+		DisableStartPrometheus:      true,
+		DisableStartUDP:             true,
+		DisableWritePeers:           true,
+		DisableRocksDBRefresh:       true,
+		DisableResolve:              true,
+		DisableBlockingAndFiltering: true,
+		DisableStartNotifier:        true,
+		DisableStartJSONRPC:         true,
+	}
+
+	return args
+}
+
 // GetEnvironment takes the environment variables as an array of strings
 // and a getkeyval function to turn it into a map.
 func GetEnvironment(data []string, getkeyval func(item string) (key, val string)) map[string]string {
@@ -111,7 +174,7 @@ func GetEnvironmentStandard() map[string]string {
 func ParseArgs(searchRequest *pb.SearchRequest) *Args {
 
 	environment := GetEnvironmentStandard()
-	parser := argparse.NewParser("hub", "hub server and client")
+	parser := argparse.NewParser("herald", "herald server and client")
 
 	serveCmd := parser.NewCommand("serve", "start the hub server")
 	searchCmd := parser.NewCommand("search", "claim search")
@@ -122,6 +185,7 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
 		return err
 	}
 
+	// main server config arguments
 	host := parser.String("", "rpchost", &argparse.Options{Required: false, Help: "RPC host", Default: DefaultHost})
 	port := parser.String("", "rpcport", &argparse.Options{Required: false, Help: "RPC port", Default: DefaultPort})
 	dbPath := parser.String("", "db-path", &argparse.Options{Required: false, Help: "RocksDB path", Default: DefaultDBPath})
@@ -143,6 +207,13 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
 	blockingChannelIds := parser.StringList("", "blocking-channel-ids", &argparse.Options{Required: false, Help: "Blocking channel ids", Default: DefaultBlockingChannelIds})
 	filteringChannelIds := parser.StringList("", "filtering-channel-ids", &argparse.Options{Required: false, Help: "Filtering channel ids", Default: DefaultFilteringChannelIds})
 
+	// arguments for server features
+	serverDescription := parser.String("", "server-description", &argparse.Options{Required: false, Help: "Server description", Default: DefaultServerDescription})
+	paymentAddress := parser.String("", "payment-address", &argparse.Options{Required: false, Help: "Payment address", Default: DefaultPaymentAddress})
+	donationAddress := parser.String("", "donation-address", &argparse.Options{Required: false, Help: "Donation address", Default: DefaultDonationAddress})
+	dailyFee := parser.String("", "daily-fee", &argparse.Options{Required: false, Help: "Daily fee", Default: DefaultDailyFee})
+
+	// flags for disabling features
 	debug := parser.Flag("", "debug", &argparse.Options{Required: false, Help: "enable debug logging", Default: false})
 	disableEs := parser.Flag("", "disable-es", &argparse.Options{Required: false, Help: "Disable elastic search, for running/testing independently", Default: false})
 	disableLoadPeers := parser.Flag("", "disable-load-peers", &argparse.Options{Required: false, Help: "Disable load peers from disk at startup", Default: DefaultDisableLoadPeers})
@@ -156,6 +227,7 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
 	disableStartNotifier := parser.Flag("", "disable-start-notifier", &argparse.Options{Required: false, Help: "Disable start notifier", Default: DisableStartNotifier})
 	disableStartJSONRPC := parser.Flag("", "disable-start-jsonrpc", &argparse.Options{Required: false, Help: "Disable start jsonrpc endpoint", Default: DisableStartJSONRPC})
 
+	// search command arguments
 	text := parser.String("", "text", &argparse.Options{Required: false, Help: "text query"})
 	name := parser.String("", "name", &argparse.Options{Required: false, Help: "name"})
 	claimType := parser.String("", "claim_type", &argparse.Options{Required: false, Help: "claim_type"})
@@ -198,6 +270,16 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
 		Country:             *country,
 		BlockingChannelIds:  *blockingChannelIds,
 		FilteringChannelIds: *filteringChannelIds,
+
+		GenesisHash:       GENESIS_HASH,
+		ServerVersion:     HUB_PROTOCOL_VERSION,
+		ProtocolMin:       PROTOCOL_MIN,
+		ProtocolMax:       PROTOCOL_MAX,
+		ServerDescription: *serverDescription,
+		PaymentAddress:    *paymentAddress,
+		DonationAddress:   *donationAddress,
+		DailyFee:          *dailyFee,
+
 		Debug:                       *debug,
 		DisableEs:                   *disableEs,
 		DisableLoadPeers:            *disableLoadPeers,

server/federation_test.go

@@ -44,43 +44,10 @@ func removeFile(fileName string) {
 	}
 }
 
-// makeDefaultArgs creates a default set of arguments for testing the server.
-func makeDefaultArgs() *server.Args {
-	args := &server.Args{
-		CmdType:                     server.ServeCmd,
-		Host:                        server.DefaultHost,
-		Port:                        server.DefaultPort,
-		DBPath:                      server.DefaultDBPath,
-		EsHost:                      server.DefaultEsHost,
-		EsPort:                      server.DefaultEsPort,
-		PrometheusPort:              server.DefaultPrometheusPort,
-		NotifierPort:                server.DefaultNotifierPort,
-		JSONRPCPort:                 server.DefaultJSONRPCPort,
-		EsIndex:                     server.DefaultEsIndex,
-		RefreshDelta:                server.DefaultRefreshDelta,
-		CacheTTL:                    server.DefaultCacheTTL,
-		PeerFile:                    server.DefaultPeerFile,
-		Country:                     server.DefaultCountry,
-		DisableEs:                   true,
-		Debug:                       true,
-		DisableLoadPeers:            true,
-		DisableStartPrometheus:      true,
-		DisableStartUDP:             true,
-		DisableWritePeers:           true,
-		DisableRocksDBRefresh:       true,
-		DisableResolve:              true,
-		DisableBlockingAndFiltering: true,
-		DisableStartNotifier:        true,
-		DisableStartJSONRPC:         true,
-	}
-
-	return args
-}
-
 // TestAddPeer tests the ability to add peers
 func TestAddPeer(t *testing.T) {
 	ctx := context.Background()
-	args := makeDefaultArgs()
+	args := server.MakeDefaultTestArgs()
 
 	tests := []struct {
 		name string
@@ -138,7 +105,7 @@ func TestAddPeer(t *testing.T) {
 // TestPeerWriter tests that peers get written properly
 func TestPeerWriter(t *testing.T) {
 	ctx := context.Background()
-	args := makeDefaultArgs()
+	args := server.MakeDefaultTestArgs()
 	args.DisableWritePeers = false
 
 	tests := []struct {
@@ -194,8 +161,8 @@ func TestPeerWriter(t *testing.T) {
 // TestAddPeerEndpoint tests the ability to add peers
 func TestAddPeerEndpoint(t *testing.T) {
 	ctx := context.Background()
-	args := makeDefaultArgs()
+	args := server.MakeDefaultTestArgs()
-	args2 := makeDefaultArgs()
+	args2 := server.MakeDefaultTestArgs()
 	args2.Port = "50052"
 
 	tests := []struct {
@@ -265,9 +232,9 @@ func TestAddPeerEndpoint(t *testing.T) {
 // TestAddPeerEndpoint2 tests the ability to add peers
 func TestAddPeerEndpoint2(t *testing.T) {
 	ctx := context.Background()
-	args := makeDefaultArgs()
+	args := server.MakeDefaultTestArgs()
-	args2 := makeDefaultArgs()
+	args2 := server.MakeDefaultTestArgs()
-	args3 := makeDefaultArgs()
+	args3 := server.MakeDefaultTestArgs()
 	args2.Port = "50052"
 	args3.Port = "50053"
 
@@ -346,9 +313,9 @@ func TestAddPeerEndpoint2(t *testing.T) {
 // TestAddPeerEndpoint3 tests the ability to add peers
 func TestAddPeerEndpoint3(t *testing.T) {
 	ctx := context.Background()
-	args := makeDefaultArgs()
+	args := server.MakeDefaultTestArgs()
-	args2 := makeDefaultArgs()
+	args2 := server.MakeDefaultTestArgs()
-	args3 := makeDefaultArgs()
+	args3 := server.MakeDefaultTestArgs()
 	args2.Port = "50052"
 	args3.Port = "50053"
 
@@ -435,9 +402,9 @@ func TestAddPeerEndpoint3(t *testing.T) {
 // TestAddPeer tests the ability to add peers
 func TestUDPServer(t *testing.T) {
 	ctx := context.Background()
-	args := makeDefaultArgs()
+	args := server.MakeDefaultTestArgs()
 	args.DisableStartUDP = false
-	args2 := makeDefaultArgs()
+	args2 := server.MakeDefaultTestArgs()
 	args2.Port = "50052"
 	args2.DisableStartUDP = false
 

server/jsonrpc_blockchain.go

@@ -105,6 +105,7 @@ func newBlockHeaderElectrum(header *[HEADER_SIZE]byte, height uint32) *BlockHead
 type BlockGetServerHeightReq struct{}
 type BlockGetServerHeightResp uint32
 
+// blockchain.block.get_server_height
 func (s *BlockchainBlockService) Get_server_height(req *BlockGetServerHeightReq, resp **BlockGetServerHeightResp) error {
 	if s.DB == nil || s.DB.LastState == nil {
 		return fmt.Errorf("unknown height")

server/jsonrpc_server.go

@@ -0,0 +1,74 @@
+package server
+
+import (
+	"github.com/lbryio/herald.go/db"
+	log "github.com/sirupsen/logrus"
+)
+
+// const (
+// 	GENESIS_HASH         = "9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463"
+// 	HUB_PROTOCOL_VERSION = "0.107.0"
+// 	PROTOCOL_MIN         = "0.54.0"
+// 	PROTOCOL_MAX         = "0.199.0"
+// )
+
+type ServerService struct {
+	DB   *db.ReadOnlyDBColumnFamily
+	Args *Args
+}
+
+type ServerFeaturesReq struct{}
+
+/*
+   cls.cached_server_features.update({
+       'hosts': {},
+       'pruning': None,
+       'server_version': cls.version,
+       'protocol_min': min_str,
+       'protocol_max': max_str,
+       'genesis_hash': env.coin.GENESIS_HASH,
+       'description': env.description,
+       'payment_address': env.payment_address,
+       'donation_address': env.donation_address,
+       'daily_fee': env.daily_fee,
+       'hash_function': 'sha256',
+       'trending_algorithm': 'fast_ar'
+   })
+*/
+type ServerFeaturesRes struct {
+	Hosts             map[string]string `json:"hosts"`
+	Pruning           string            `json:"pruning"`
+	ServerVersion     string            `json:"server_version"`
+	ProtocolMin       string            `json:"protocol_min"`
+	ProtocolMax       string            `json:"protocol_max"`
+	GenesisHash       string            `json:"genesis_hash"`
+	Description       string            `json:"description"`
+	PaymentAddress    string            `json:"payment_address"`
+	DonationAddress   string            `json:"donation_address"`
+	DailyFee          string            `json:"daily_fee"`
+	HashFunction      string            `json:"hash_function"`
+	TrendingAlgorithm string            `json:"trending_algorithm"`
+}
+
+// Features is the json rpc endpoint for 'server.features'.
+func (t *ServerService) Features(req *ServerFeaturesReq, res **ServerFeaturesRes) error {
+	log.Println("Features")
+
+	features := &ServerFeaturesRes{
+		Hosts:             map[string]string{},
+		Pruning:           "",
+		ServerVersion:     HUB_PROTOCOL_VERSION,
+		ProtocolMin:       PROTOCOL_MIN,
+		ProtocolMax:       PROTOCOL_MAX,
+		GenesisHash:       GENESIS_HASH,
+		Description:       "Herald",
+		PaymentAddress:    "",
+		DonationAddress:   "",
+		DailyFee:          "1.0",
+		HashFunction:      "sha256",
+		TrendingAlgorithm: "fast_ar",
+	}
+	*res = features
+
+	return nil
+}

server/jsonrpc_service.go

@@ -121,6 +121,14 @@ fail1:
 			goto fail2
 		}
 
+		// Register "server.{features,version}"" handlers.
+		serverSvc := &ServerService{s.DB, s.Args}
+		err = s1.RegisterTCPService(serverSvc, "server")
+		if err != nil {
+			log.Errorf("RegisterTCPService: %v\n", err)
+			goto fail2
+		}
+
 		r := gorilla_mux.NewRouter()
 		r.Handle("/rpc", s1)
 		port := ":" + strconv.FormatUint(uint64(s.Args.JSONRPCHTTPPort), 10)

server/notifier_test.go

@@ -47,7 +47,7 @@ func tcpRead(conn net.Conn) ([]byte, error) {
 }
 
 func TestNotifierServer(t *testing.T) {
-	args := makeDefaultArgs()
+	args := server.MakeDefaultTestArgs()
 	ctx := context.Background()
 	hub := server.MakeHubServer(ctx, args)
 

server/search_test.go

@@ -56,7 +56,7 @@ func TestSearch(t *testing.T) {
 	}
 
 	context := context.Background()
-	args := makeDefaultArgs()
+	args := server.MakeDefaultTestArgs()
 	hubServer := server.MakeHubServer(context, args)
 	req := &pb.SearchRequest{
 		Text: "asdf",

server/udp_test.go

@@ -11,7 +11,7 @@ import (
 
 // TestUDPPing tests UDPPing correctness against prod server.
 func TestUDPPing(t *testing.T) {
-	args := makeDefaultArgs()
+	args := server.MakeDefaultTestArgs()
 	args.DisableStartUDP = true
 
 	tests := []struct {