LBRYCommunity/herald.go
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases 14 Packages Wiki Activity

Run gofmt

Browse source
This commit is contained in:
Jeffrey Picard 2021-11-24 18:24:06 -05:00
parent 0a1ba43d66
commit 72ea236d86
8 changed files with 150 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
internal/metrics/metrics.go
View file

@ -18,8 +18,8 @@ var (
Help: "Number of errors by type", Help: "Number of errors by type",
}, []string{"error_type"}) }, []string{"error_type"})
QueryTime = promauto.NewHistogramVec(prometheus.HistogramOpts{ QueryTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: "query_time", Name: "query_time",
Help: "Histogram of query times", Help: "Histogram of query times",
Buckets: HistogramBuckets, Buckets: HistogramBuckets,
}, []string{"method"}) }, []string{"method"})
PeersKnown = promauto.NewGauge(prometheus.GaugeOpts{ PeersKnown = promauto.NewGauge(prometheus.GaugeOpts{
@ -31,4 +31,3 @@ var (
Help: "Number of peers that are subscribed to us.", Help: "Number of peers that are subscribed to us.",
}) })
) )

37
server/args.go
View file

@ -16,23 +16,24 @@ const (
// Args struct contains the arguments to the hub server. // Args struct contains the arguments to the hub server.
type Args struct { type Args struct {
CmdType int CmdType int
Host string Host string
Port string Port string
EsHost string EsHost string
EsPort string EsPort string
PrometheusPort string PrometheusPort string
EsIndex string EsIndex string
RefreshDelta int RefreshDelta int
CacheTTL int CacheTTL int
PeerFile string PeerFile string
Country string Country string
DisableEs bool DisableEs bool
Debug bool Debug bool
LoadPeers bool LoadPeers bool
StartPrometheus bool StartPrometheus bool
StartUDP bool StartUDP bool
WritePeers bool WritePeers bool
DisableFederation bool
} }
const ( const (
@ -153,7 +154,7 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
} }
/* /*
Verify no invalid argument combinations Verify no invalid argument combinations
*/ */
if len(*channelIds) > 0 && *channelId != "" { if len(*channelIds) > 0 && *channelId != "" {
log.Fatal("Cannot specify both channel_id and channel_ids") log.Fatal("Cannot specify both channel_id and channel_ids")

24
server/federation.go
View file

@ -26,13 +26,12 @@ type FederatedServer struct {
var ( var (
localHosts = map[string]bool{ localHosts = map[string]bool{
"127.0.0.1": true, "127.0.0.1": true,
"0.0.0.0": true, "0.0.0.0": true,
"localhost": true, "localhost": true,
"<nil>": true, "<nil>": true,
} }
) )
// peerKey takes a ServerMessage object and returns the key that for that peer // peerKey takes a ServerMessage object and returns the key that for that peer
// in our peer table. // in our peer table.
func peerKey(msg *pb.ServerMessage) string { func peerKey(msg *pb.ServerMessage) string {
@ -117,7 +116,6 @@ retry:
} }
cancel() cancel()
f, err := os.Open(peerFile) f, err := os.Open(peerFile)
if err != nil { if err != nil {
log.Println(err) log.Println(err)
@ -135,7 +133,7 @@ retry:
} }
for _, line := range text { for _, line := range text {
ipPort := strings.Split(line,":") ipPort := strings.Split(line, ":")
if len(ipPort) != 2 { if len(ipPort) != 2 {
log.Println("Malformed entry in peer file") log.Println("Malformed entry in peer file")
continue continue
@ -220,8 +218,8 @@ func (s *Server) helloPeer(server *FederatedServer) (*pb.HelloMessage, error) {
c := pb.NewHubClient(conn) c := pb.NewHubClient(conn)
msg := &pb.HelloMessage{ msg := &pb.HelloMessage{
Port: s.Args.Port, Port: s.Args.Port,
Host: s.ExternalIP.String(), Host: s.ExternalIP.String(),
Servers: []*pb.ServerMessage{}, Servers: []*pb.ServerMessage{},
} }
@ -333,7 +331,7 @@ func (s *Server) addPeer(msg *pb.ServerMessage, ping bool, subscribe bool) error
// could end up subscribed to our self, which is silly. // could end up subscribed to our self, which is silly.
nilIP := net.IP{} nilIP := net.IP{}
//localIP0 := net.IPv4(0,0,0,0) //localIP0 := net.IPv4(0,0,0,0)
localIP1 := net.IPv4(127,0,0,1) localIP1 := net.IPv4(127, 0, 0, 1)
if s.ExternalIP.Equal(nilIP) || s.ExternalIP.Equal(localIP1) { if s.ExternalIP.Equal(nilIP) || s.ExternalIP.Equal(localIP1) {
err := s.getAndSetExternalIp(msg) err := s.getAndSetExternalIp(msg)
if err != nil { if err != nil {
@ -351,8 +349,8 @@ func (s *Server) addPeer(msg *pb.ServerMessage, ping bool, subscribe bool) error
k := peerKey(msg) k := peerKey(msg)
newServer := &FederatedServer{ newServer := &FederatedServer{
Address: msg.Address, Address: msg.Address,
Port: msg.Port, Port: msg.Port,
Ts: time.Now(), Ts: time.Now(),
} }
log.Printf("%s:%s adding peer %+v\n", s.ExternalIP, s.Args.Port, msg) log.Printf("%s:%s adding peer %+v\n", s.ExternalIP, s.Args.Port, msg)
@ -406,14 +404,14 @@ func (s *Server) makeHelloMessage() *pb.HelloMessage {
for _, peer := range s.PeerServers { for _, peer := range s.PeerServers {
servers = append(servers, &pb.ServerMessage{ servers = append(servers, &pb.ServerMessage{
Address: peer.Address, Address: peer.Address,
Port: peer.Port, Port: peer.Port,
}) })
} }
s.PeerServersMut.RUnlock() s.PeerServersMut.RUnlock()
return &pb.HelloMessage{ return &pb.HelloMessage{
Port: s.Args.Port, Port: s.Args.Port,
Host: s.ExternalIP.String(), Host: s.ExternalIP.String(),
Servers: servers, Servers: servers,
} }
} }

45
server/federation_test.go
View file

@ -75,7 +75,7 @@ func TestAddPeer(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
want int want int
} { }{
{ {
name: "Add 10 peers", name: "Add 10 peers",
want: 10, want: 10,
@ -87,9 +87,9 @@ func TestAddPeer(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){ t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args) server := MakeHubServer(ctx, args)
server.ExternalIP = net.IPv4(0,0,0,0) server.ExternalIP = net.IPv4(0, 0, 0, 0)
metrics.PeersKnown.Set(0) metrics.PeersKnown.Set(0)
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
@ -134,7 +134,7 @@ func TestPeerWriter(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
want int want int
} { }{
{ {
name: "Add 10 peers", name: "Add 10 peers",
want: 10, want: 10,
@ -146,9 +146,9 @@ func TestPeerWriter(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){ t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args) server := MakeHubServer(ctx, args)
server.ExternalIP = net.IPv4(0,0,0,0) server.ExternalIP = net.IPv4(0, 0, 0, 0)
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
var msg *pb.ServerMessage var msg *pb.ServerMessage
@ -188,12 +188,11 @@ func TestAddPeerEndpoint(t *testing.T) {
args2 := makeDefaultArgs() args2 := makeDefaultArgs()
args2.Port = "50052" args2.Port = "50052"
tests := []struct { tests := []struct {
name string name string
wantServerOne int64 wantServerOne int64
wantServerTwo int64 wantServerTwo int64
} { }{
{ {
// outside -> server1.AddPeer(server2, ping=true) : server1 = 1, server2 = 0 // outside -> server1.AddPeer(server2, ping=true) : server1 = 1, server2 = 0
// server1 -> server2.Hello(server1) : server1 = 1, server2 = 0 // server1 -> server2.Hello(server1) : server1 = 1, server2 = 0
@ -204,14 +203,14 @@ func TestAddPeerEndpoint(t *testing.T) {
// server1 -> server2.AddPeer(server2) : server1 = 1, server2 = 1 // server1 -> server2.AddPeer(server2) : server1 = 1, server2 = 1
// server2 self peer, skipping : server1 = 1, server2 = 1 // server2 self peer, skipping : server1 = 1, server2 = 1
// server1 -> server2.PeerSubscribe(server1) : server1 = 1, server2 = 1 // server1 -> server2.PeerSubscribe(server1) : server1 = 1, server2 = 1
name: "Add 1 peer", name: "Add 1 peer",
wantServerOne: 1, wantServerOne: 1,
wantServerTwo: 1, wantServerTwo: 1,
}, },
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){ t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args) server := MakeHubServer(ctx, args)
server2 := MakeHubServer(ctx, args2) server2 := MakeHubServer(ctx, args2)
metrics.PeersKnown.Set(0) metrics.PeersKnown.Set(0)
@ -262,15 +261,14 @@ func TestAddPeerEndpoint2(t *testing.T) {
args2.Port = "50052" args2.Port = "50052"
args3.Port = "50053" args3.Port = "50053"
tests := []struct { tests := []struct {
name string name string
wantServerOne int64 wantServerOne int64
wantServerTwo int64 wantServerTwo int64
wantServerThree int64 wantServerThree int64
} { }{
{ {
name: "Add 2 peers", name: "Add 2 peers",
wantServerOne: 2, wantServerOne: 2,
wantServerTwo: 2, wantServerTwo: 2,
wantServerThree: 2, wantServerThree: 2,
@ -278,7 +276,7 @@ func TestAddPeerEndpoint2(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){ t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args) server := MakeHubServer(ctx, args)
server2 := MakeHubServer(ctx, args2) server2 := MakeHubServer(ctx, args2)
server3 := MakeHubServer(ctx, args3) server3 := MakeHubServer(ctx, args3)
@ -335,7 +333,6 @@ func TestAddPeerEndpoint2(t *testing.T) {
} }
// TestAddPeerEndpoint3 tests the ability to add peers // TestAddPeerEndpoint3 tests the ability to add peers
func TestAddPeerEndpoint3(t *testing.T) { func TestAddPeerEndpoint3(t *testing.T) {
ctx := context.Background() ctx := context.Background()
@ -345,15 +342,14 @@ func TestAddPeerEndpoint3(t *testing.T) {
args2.Port = "50052" args2.Port = "50052"
args3.Port = "50053" args3.Port = "50053"
tests := []struct { tests := []struct {
name string name string
wantServerOne int64 wantServerOne int64
wantServerTwo int64 wantServerTwo int64
wantServerThree int64 wantServerThree int64
} { }{
{ {
name: "Add 1 peer to each", name: "Add 1 peer to each",
wantServerOne: 2, wantServerOne: 2,
wantServerTwo: 2, wantServerTwo: 2,
wantServerThree: 2, wantServerThree: 2,
@ -361,7 +357,7 @@ func TestAddPeerEndpoint3(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){ t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args) server := MakeHubServer(ctx, args)
server2 := MakeHubServer(ctx, args2) server2 := MakeHubServer(ctx, args2)
server3 := MakeHubServer(ctx, args3) server3 := MakeHubServer(ctx, args3)
@ -426,7 +422,6 @@ func TestAddPeerEndpoint3(t *testing.T) {
} }
// TestAddPeer tests the ability to add peers // TestAddPeer tests the ability to add peers
func TestUDPServer(t *testing.T) { func TestUDPServer(t *testing.T) {
ctx := context.Background() ctx := context.Background()
@ -439,7 +434,7 @@ func TestUDPServer(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
want string want string
} { }{
{ {
name: "hubs server external ip", name: "hubs server external ip",
want: "127.0.0.1", want: "127.0.0.1",
@ -447,7 +442,7 @@ func TestUDPServer(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){ t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args) server := MakeHubServer(ctx, args)
server2 := MakeHubServer(ctx, args2) server2 := MakeHubServer(ctx, args2)
go server.Run() go server.Run()

37
server/search.go
View file

@ -275,16 +275,16 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
setPageVars(in, &pageSize, &from) setPageVars(in, &pageSize, &from)
/* /*
cache based on search request params cache based on search request params
include from value and number of results. include from value and number of results.
When another search request comes in with same search params When another search request comes in with same search params
and same or increased offset (which we currently don't even use?) and same or increased offset (which we currently don't even use?)
that will be a cache hit. that will be a cache hit.
FIXME: For now the cache is turned off when in debugging mode FIXME: For now the cache is turned off when in debugging mode
(for unit tests) because it breaks on some of them. (for unit tests) because it breaks on some of them.
FIXME: Currently the cache just skips the initial search, FIXME: Currently the cache just skips the initial search,
the mgets and post processing are still done. There's probably the mgets and post processing are still done. There's probably
a more efficient way to store the final result. a more efficient way to store the final result.
*/ */
if val, err := s.QueryCache.Get(cacheKey); err != nil { if val, err := s.QueryCache.Get(cacheKey); err != nil {
@ -518,15 +518,15 @@ func (s *Server) setupEsQuery(
} }
replacements := map[string]string{ replacements := map[string]string{
"name": "normalized_name", "name": "normalized_name",
"normalized": "normalized_name", "normalized": "normalized_name",
"claim_name": "normalized_name", "claim_name": "normalized_name",
"txid": "tx_id", "txid": "tx_id",
"nout": "tx_nout", "nout": "tx_nout",
"reposted": "repost_count", "reposted": "repost_count",
"valid_channel_signature": "is_signature_valid", "valid_channel_signature": "is_signature_valid",
"claim_id": "_id", "claim_id": "_id",
"signature_digest": "signature", "signature_digest": "signature",
} }
textFields := map[string]bool{ textFields := map[string]bool{
@ -967,4 +967,3 @@ func removeBlocked(searchHits []*record) ([]*record, []*record, map[string]*pb.B
return newHits, blockedHits, blockedChannels return newHits, blockedHits, blockedChannels
} }

7
server/server.go
View file

@ -45,7 +45,6 @@ type Server struct {
pb.UnimplementedHubServer pb.UnimplementedHubServer
} }
func getVersion() string { func getVersion() string {
return meta.Version return meta.Version
} }
@ -202,7 +201,7 @@ func MakeHubServer(ctx context.Context, args *Args) *Server {
PeerSubs: make(map[string]*FederatedServer), PeerSubs: make(map[string]*FederatedServer),
PeerSubsMut: sync.RWMutex{}, PeerSubsMut: sync.RWMutex{},
NumPeerSubs: numSubs, NumPeerSubs: numSubs,
ExternalIP: net.IPv4(127,0,0,1), ExternalIP: net.IPv4(127, 0, 0, 1),
} }
// Start up our background services // Start up our background services
@ -248,8 +247,8 @@ func (s *Server) Hello(ctx context.Context, args *pb.HelloMessage) (*pb.HelloMes
host := args.Host host := args.Host
server := &FederatedServer{ server := &FederatedServer{
Address: host, Address: host,
Port: port, Port: port,
Ts: time.Now(), Ts: time.Now(),
} }
log.Println(server) log.Println(server)

23
server/udp.go
View file

@ -11,9 +11,10 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/extras/errors"
) )
const maxBufferSize = 1024 const maxBufferSize = 1024
// genesis blocktime (which is actually wrong) // genesis blocktime (which is actually wrong)
const magic = 1446058291 const magic = 1446058291
const protocolVersion = 1 const protocolVersion = 1
// SPVPing is a struct for the format of how to ping another hub over udp. // SPVPing is a struct for the format of how to ping another hub over udp.
@ -31,7 +32,7 @@ type SPVPong struct {
flags byte flags byte
height uint32 height uint32
tip []byte // 32 tip []byte // 32
srcAddrRaw []byte // 4 srcAddrRaw []byte // 4
country uint16 country uint16
} }
@ -55,7 +56,7 @@ func decodeSPVPing(data []byte) *SPVPing {
parsedMagic := binary.BigEndian.Uint32(data) parsedMagic := binary.BigEndian.Uint32(data)
parsedProtocalVersion := data[4] parsedProtocalVersion := data[4]
return &SPVPing{ return &SPVPing{
magic: parsedMagic, magic: parsedMagic,
version: parsedProtocalVersion, version: parsedProtocalVersion,
} }
} }
@ -65,7 +66,7 @@ func decodeSPVPing(data []byte) *SPVPing {
func (pong *SPVPong) Encode() []byte { func (pong *SPVPong) Encode() []byte {
data := make([]byte, 44) data := make([]byte, 44)
data[0] = pong.protocolVersion data[0] = pong.protocolVersion
data[1] = pong.flags data[1] = pong.flags
binary.BigEndian.PutUint32(data[2:], pong.height) binary.BigEndian.PutUint32(data[2:], pong.height)
copy(data[6:], pong.tip) copy(data[6:], pong.tip)
@ -110,11 +111,11 @@ func decodeSPVPong(data []byte) *SPVPong {
country := binary.BigEndian.Uint16(data[42:]) country := binary.BigEndian.Uint16(data[42:])
return &SPVPong{ return &SPVPong{
protocolVersion: parsedProtocalVersion, protocolVersion: parsedProtocalVersion,
flags: flags, flags: flags,
height: height, height: height,
tip: tip, tip: tip,
srcAddrRaw: srcRawAddr, srcAddrRaw: srcRawAddr,
country: country, country: country,
} }
} }
@ -240,7 +241,7 @@ func UDPServer(args *Args) error {
} }
sAddr := addr.IP.String() sAddr := addr.IP.String()
pong := makeSPVPong(0,0, tip, sAddr, args.Country) pong := makeSPVPong(0, 0, tip, sAddr, args.Country)
data := pong.Encode() data := pong.Encode()
_, err = conn.WriteToUDP(data, addr) _, err = conn.WriteToUDP(data, addr)

128
server/udp_test.go
View file

@ -1,83 +1,83 @@
package server package server
import ( import (
"log" "log"
"os/exec" "os/exec"
"strings" "strings"
"testing" "testing"
) )
// TestAddPeer tests the ability to add peers // TestAddPeer tests the ability to add peers
func TestUDPPing(t *testing.T) { func TestUDPPing(t *testing.T) {
args := makeDefaultArgs() args := makeDefaultArgs()
args.StartUDP = false args.StartUDP = false
tests := []struct { tests := []struct {
name string name string
wantIP string wantIP string
wantCountry string wantCountry string
wantProtocolVersion int wantProtocolVersion int
wantHeightMin int wantHeightMin int
wantFlags byte wantFlags byte
} { }{
{ {
name: "Correctly parse information from production server.", name: "Correctly parse information from production server.",
wantIP: "SETME", wantIP: "SETME",
wantCountry: "US", wantCountry: "US",
wantProtocolVersion: 1, wantProtocolVersion: 1,
wantHeightMin: 1060000, wantHeightMin: 1060000,
wantFlags: 1, wantFlags: 1,
}, },
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){ t.Run(tt.name, func(t *testing.T) {
toAddr := "spv16.lbry.com" toAddr := "spv16.lbry.com"
toPort := "50001" toPort := "50001"
pong, err := UDPPing(toAddr, toPort) pong, err := UDPPing(toAddr, toPort)
gotCountry := pong.DecodeCountry() gotCountry := pong.DecodeCountry()
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
res, err := exec.Command("dig", "@resolver4.opendns.com", "myip.opendns.com", "+short").Output() res, err := exec.Command("dig", "@resolver4.opendns.com", "myip.opendns.com", "+short").Output()
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
digIP := strings.TrimSpace(string(res)) digIP := strings.TrimSpace(string(res))
udpIP := pong.DecodeAddress().String() udpIP := pong.DecodeAddress().String()
tt.wantIP = digIP tt.wantIP = digIP
log.Println("Height:", pong.DecodeHeight()) log.Println("Height:", pong.DecodeHeight())
log.Printf("Flags: %x\n", pong.DecodeFlags()) log.Printf("Flags: %x\n", pong.DecodeFlags())
log.Println("ProtocolVersion:", pong.DecodeProtocolVersion()) log.Println("ProtocolVersion:", pong.DecodeProtocolVersion())
log.Printf("Tip: %x\n", pong.DecodeTip()) log.Printf("Tip: %x\n", pong.DecodeTip())
gotHeight := pong.DecodeHeight() gotHeight := pong.DecodeHeight()
gotProtocolVersion := pong.DecodeProtocolVersion() gotProtocolVersion := pong.DecodeProtocolVersion()
gotFlags := pong.DecodeFlags() gotFlags := pong.DecodeFlags()
gotIP := udpIP gotIP := udpIP
if gotIP != tt.wantIP { if gotIP != tt.wantIP {
t.Errorf("ip: got: '%s', want: '%s'\n", gotIP, tt.wantIP) t.Errorf("ip: got: '%s', want: '%s'\n", gotIP, tt.wantIP)
} }
if gotCountry != tt.wantCountry { if gotCountry != tt.wantCountry {
t.Errorf("country: got: '%s', want: '%s'\n", gotCountry, tt.wantCountry) t.Errorf("country: got: '%s', want: '%s'\n", gotCountry, tt.wantCountry)
} }
if gotHeight < tt.wantHeightMin { if gotHeight < tt.wantHeightMin {
t.Errorf("height: got: %d, want >=: %d\n", gotHeight, tt.wantHeightMin) t.Errorf("height: got: %d, want >=: %d\n", gotHeight, tt.wantHeightMin)
} }
if gotProtocolVersion != tt.wantProtocolVersion { if gotProtocolVersion != tt.wantProtocolVersion {
t.Errorf("protocolVersion: got: %d, want: %d\n", gotProtocolVersion, tt.wantProtocolVersion) t.Errorf("protocolVersion: got: %d, want: %d\n", gotProtocolVersion, tt.wantProtocolVersion)
} }
if gotFlags != tt.wantFlags { if gotFlags != tt.wantFlags {
t.Errorf("flags: got: %d, want: %d\n", gotFlags, tt.wantFlags) t.Errorf("flags: got: %d, want: %d\n", gotFlags, tt.wantFlags)
} }
}) })
} }
} }