LBRYCommunity/herald.go
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases 14 Packages Wiki Activity

Run gofmt

Browse source
This commit is contained in:
Jeffrey Picard 2021-11-24 18:24:06 -05:00
parent 0a1ba43d66
commit 72ea236d86
8 changed files with 150 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
internal/metrics/metrics.go
View file

@ -18,8 +18,8 @@ var (
Help: "Number of errors by type",
}, []string{"error_type"})
QueryTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: "query_time",
Help: "Histogram of query times",
Name: "query_time",
Help: "Histogram of query times",
Buckets: HistogramBuckets,
}, []string{"method"})
PeersKnown = promauto.NewGauge(prometheus.GaugeOpts{
@ -31,4 +31,3 @@ var (
Help: "Number of peers that are subscribed to us.",
})
)

39
server/args.go
View file

@ -16,23 +16,24 @@ const (
// Args struct contains the arguments to the hub server.
type Args struct {
CmdType int
Host string
Port string
EsHost string
EsPort string
PrometheusPort string
EsIndex string
RefreshDelta int
CacheTTL int
PeerFile string
Country string
DisableEs bool
Debug bool
LoadPeers bool
StartPrometheus bool
StartUDP bool
WritePeers bool
CmdType int
Host string
Port string
EsHost string
EsPort string
PrometheusPort string
EsIndex string
RefreshDelta int
CacheTTL int
PeerFile string
Country string
DisableEs bool
Debug bool
LoadPeers bool
StartPrometheus bool
StartUDP bool
WritePeers bool
DisableFederation bool
}
const (
@ -153,7 +154,7 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
}
/*
Verify no invalid argument combinations
Verify no invalid argument combinations
*/
if len(*channelIds) > 0 && *channelId != "" {
log.Fatal("Cannot specify both channel_id and channel_ids")
@ -194,4 +195,4 @@ func ParseArgs(searchRequest *pb.SearchRequest) *Args {
}
return args
}
}

24
server/federation.go
View file

@ -26,13 +26,12 @@ type FederatedServer struct {
var (
localHosts = map[string]bool{
"127.0.0.1": true,
"0.0.0.0": true,
"0.0.0.0": true,
"localhost": true,
"<nil>": true,
"<nil>": true,
}
)
// peerKey takes a ServerMessage object and returns the key that for that peer
// in our peer table.
func peerKey(msg *pb.ServerMessage) string {
@ -117,7 +116,6 @@ retry:
}
cancel()
f, err := os.Open(peerFile)
if err != nil {
log.Println(err)
@ -135,7 +133,7 @@ retry:
}
for _, line := range text {
ipPort := strings.Split(line,":")
ipPort := strings.Split(line, ":")
if len(ipPort) != 2 {
log.Println("Malformed entry in peer file")
continue
@ -220,8 +218,8 @@ func (s *Server) helloPeer(server *FederatedServer) (*pb.HelloMessage, error) {
c := pb.NewHubClient(conn)
msg := &pb.HelloMessage{
Port: s.Args.Port,
Host: s.ExternalIP.String(),
Port: s.Args.Port,
Host: s.ExternalIP.String(),
Servers: []*pb.ServerMessage{},
}
@ -333,7 +331,7 @@ func (s *Server) addPeer(msg *pb.ServerMessage, ping bool, subscribe bool) error
// could end up subscribed to our self, which is silly.
nilIP := net.IP{}
//localIP0 := net.IPv4(0,0,0,0)
localIP1 := net.IPv4(127,0,0,1)
localIP1 := net.IPv4(127, 0, 0, 1)
if s.ExternalIP.Equal(nilIP) || s.ExternalIP.Equal(localIP1) {
err := s.getAndSetExternalIp(msg)
if err != nil {
@ -351,8 +349,8 @@ func (s *Server) addPeer(msg *pb.ServerMessage, ping bool, subscribe bool) error
k := peerKey(msg)
newServer := &FederatedServer{
Address: msg.Address,
Port: msg.Port,
Ts: time.Now(),
Port: msg.Port,
Ts: time.Now(),
}
log.Printf("%s:%s adding peer %+v\n", s.ExternalIP, s.Args.Port, msg)
@ -406,14 +404,14 @@ func (s *Server) makeHelloMessage() *pb.HelloMessage {
for _, peer := range s.PeerServers {
servers = append(servers, &pb.ServerMessage{
Address: peer.Address,
Port: peer.Port,
Port: peer.Port,
})
}
s.PeerServersMut.RUnlock()
return &pb.HelloMessage{
Port: s.Args.Port,
Host: s.ExternalIP.String(),
Port: s.Args.Port,
Host: s.ExternalIP.String(),
Servers: servers,
}
}

45
server/federation_test.go
View file

@ -75,7 +75,7 @@ func TestAddPeer(t *testing.T) {
tests := []struct {
name string
want int
} {
}{
{
name: "Add 10 peers",
want: 10,
@ -87,9 +87,9 @@ func TestAddPeer(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){
t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args)
server.ExternalIP = net.IPv4(0,0,0,0)
server.ExternalIP = net.IPv4(0, 0, 0, 0)
metrics.PeersKnown.Set(0)
for i := 0; i < 10; i++ {
@ -134,7 +134,7 @@ func TestPeerWriter(t *testing.T) {
tests := []struct {
name string
want int
} {
}{
{
name: "Add 10 peers",
want: 10,
@ -146,9 +146,9 @@ func TestPeerWriter(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){
t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args)
server.ExternalIP = net.IPv4(0,0,0,0)
server.ExternalIP = net.IPv4(0, 0, 0, 0)
for i := 0; i < 10; i++ {
var msg *pb.ServerMessage
@ -188,12 +188,11 @@ func TestAddPeerEndpoint(t *testing.T) {
args2 := makeDefaultArgs()
args2.Port = "50052"
tests := []struct {
name string
name string
wantServerOne int64
wantServerTwo int64
} {
}{
{
// outside -> server1.AddPeer(server2, ping=true) : server1 = 1, server2 = 0
// server1 -> server2.Hello(server1) : server1 = 1, server2 = 0
@ -204,14 +203,14 @@ func TestAddPeerEndpoint(t *testing.T) {
// server1 -> server2.AddPeer(server2) : server1 = 1, server2 = 1
// server2 self peer, skipping : server1 = 1, server2 = 1
// server1 -> server2.PeerSubscribe(server1) : server1 = 1, server2 = 1
name: "Add 1 peer",
name: "Add 1 peer",
wantServerOne: 1,
wantServerTwo: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){
t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args)
server2 := MakeHubServer(ctx, args2)
metrics.PeersKnown.Set(0)
@ -262,15 +261,14 @@ func TestAddPeerEndpoint2(t *testing.T) {
args2.Port = "50052"
args3.Port = "50053"
tests := []struct {
name string
name string
wantServerOne int64
wantServerTwo int64
wantServerThree int64
} {
}{
{
name: "Add 2 peers",
name: "Add 2 peers",
wantServerOne: 2,
wantServerTwo: 2,
wantServerThree: 2,
@ -278,7 +276,7 @@ func TestAddPeerEndpoint2(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){
t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args)
server2 := MakeHubServer(ctx, args2)
server3 := MakeHubServer(ctx, args3)
@ -335,7 +333,6 @@ func TestAddPeerEndpoint2(t *testing.T) {
}
// TestAddPeerEndpoint3 tests the ability to add peers
func TestAddPeerEndpoint3(t *testing.T) {
ctx := context.Background()
@ -345,15 +342,14 @@ func TestAddPeerEndpoint3(t *testing.T) {
args2.Port = "50052"
args3.Port = "50053"
tests := []struct {
name string
name string
wantServerOne int64
wantServerTwo int64
wantServerThree int64
} {
}{
{
name: "Add 1 peer to each",
name: "Add 1 peer to each",
wantServerOne: 2,
wantServerTwo: 2,
wantServerThree: 2,
@ -361,7 +357,7 @@ func TestAddPeerEndpoint3(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){
t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args)
server2 := MakeHubServer(ctx, args2)
server3 := MakeHubServer(ctx, args3)
@ -426,7 +422,6 @@ func TestAddPeerEndpoint3(t *testing.T) {
}
// TestAddPeer tests the ability to add peers
func TestUDPServer(t *testing.T) {
ctx := context.Background()
@ -439,7 +434,7 @@ func TestUDPServer(t *testing.T) {
tests := []struct {
name string
want string
} {
}{
{
name: "hubs server external ip",
want: "127.0.0.1",
@ -447,7 +442,7 @@ func TestUDPServer(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){
t.Run(tt.name, func(t *testing.T) {
server := MakeHubServer(ctx, args)
server2 := MakeHubServer(ctx, args2)
go server.Run()

37
server/search.go
View file

@ -275,16 +275,16 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
setPageVars(in, &pageSize, &from)
/*
cache based on search request params
include from value and number of results.
When another search request comes in with same search params
and same or increased offset (which we currently don't even use?)
that will be a cache hit.
FIXME: For now the cache is turned off when in debugging mode
(for unit tests) because it breaks on some of them.
FIXME: Currently the cache just skips the initial search,
the mgets and post processing are still done. There's probably
a more efficient way to store the final result.
cache based on search request params
include from value and number of results.
When another search request comes in with same search params
and same or increased offset (which we currently don't even use?)
that will be a cache hit.
FIXME: For now the cache is turned off when in debugging mode
(for unit tests) because it breaks on some of them.
FIXME: Currently the cache just skips the initial search,
the mgets and post processing are still done. There's probably
a more efficient way to store the final result.
*/
if val, err := s.QueryCache.Get(cacheKey); err != nil {
@ -518,15 +518,15 @@ func (s *Server) setupEsQuery(
}
replacements := map[string]string{
"name": "normalized_name",
"normalized": "normalized_name",
"claim_name": "normalized_name",
"txid": "tx_id",
"nout": "tx_nout",
"reposted": "repost_count",
"name": "normalized_name",
"normalized": "normalized_name",
"claim_name": "normalized_name",
"txid": "tx_id",
"nout": "tx_nout",
"reposted": "repost_count",
"valid_channel_signature": "is_signature_valid",
"claim_id": "_id",
"signature_digest": "signature",
"claim_id": "_id",
"signature_digest": "signature",
}
textFields := map[string]bool{
@ -967,4 +967,3 @@ func removeBlocked(searchHits []*record) ([]*record, []*record, map[string]*pb.B
return newHits, blockedHits, blockedChannels
}

7
server/server.go
View file

@ -45,7 +45,6 @@ type Server struct {
pb.UnimplementedHubServer
}
func getVersion() string {
return meta.Version
}
@ -202,7 +201,7 @@ func MakeHubServer(ctx context.Context, args *Args) *Server {
PeerSubs: make(map[string]*FederatedServer),
PeerSubsMut: sync.RWMutex{},
NumPeerSubs: numSubs,
ExternalIP: net.IPv4(127,0,0,1),
ExternalIP: net.IPv4(127, 0, 0, 1),
}
// Start up our background services
@ -248,8 +247,8 @@ func (s *Server) Hello(ctx context.Context, args *pb.HelloMessage) (*pb.HelloMes
host := args.Host
server := &FederatedServer{
Address: host,
Port: port,
Ts: time.Now(),
Port: port,
Ts: time.Now(),
}
log.Println(server)

23
server/udp.go
View file

@ -11,9 +11,10 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
)
const maxBufferSize = 1024
const maxBufferSize = 1024
// genesis blocktime (which is actually wrong)
const magic = 1446058291
const magic = 1446058291
const protocolVersion = 1
// SPVPing is a struct for the format of how to ping another hub over udp.
@ -31,7 +32,7 @@ type SPVPong struct {
flags byte
height uint32
tip []byte // 32
srcAddrRaw []byte // 4
srcAddrRaw []byte // 4
country uint16
}
@ -55,7 +56,7 @@ func decodeSPVPing(data []byte) *SPVPing {
parsedMagic := binary.BigEndian.Uint32(data)
parsedProtocalVersion := data[4]
return &SPVPing{
magic: parsedMagic,
magic: parsedMagic,
version: parsedProtocalVersion,
}
}
@ -65,7 +66,7 @@ func decodeSPVPing(data []byte) *SPVPing {
func (pong *SPVPong) Encode() []byte {
data := make([]byte, 44)
data[0] = pong.protocolVersion
data[0] = pong.protocolVersion
data[1] = pong.flags
binary.BigEndian.PutUint32(data[2:], pong.height)
copy(data[6:], pong.tip)
@ -110,11 +111,11 @@ func decodeSPVPong(data []byte) *SPVPong {
country := binary.BigEndian.Uint16(data[42:])
return &SPVPong{
protocolVersion: parsedProtocalVersion,
flags: flags,
height: height,
tip: tip,
srcAddrRaw: srcRawAddr,
country: country,
flags: flags,
height: height,
tip: tip,
srcAddrRaw: srcRawAddr,
country: country,
}
}
@ -240,7 +241,7 @@ func UDPServer(args *Args) error {
}
sAddr := addr.IP.String()
pong := makeSPVPong(0,0, tip, sAddr, args.Country)
pong := makeSPVPong(0, 0, tip, sAddr, args.Country)
data := pong.Encode()
_, err = conn.WriteToUDP(data, addr)

128
server/udp_test.go
View file

@ -1,83 +1,83 @@
package server
import (
"log"
"os/exec"
"strings"
"testing"
"log"
"os/exec"
"strings"
"testing"
)
// TestAddPeer tests the ability to add peers
func TestUDPPing(t *testing.T) {
args := makeDefaultArgs()
args.StartUDP = false
args := makeDefaultArgs()
args.StartUDP = false
tests := []struct {
name string
wantIP string
wantCountry string
wantProtocolVersion int
wantHeightMin int
wantFlags byte
} {
{
name: "Correctly parse information from production server.",
wantIP: "SETME",
wantCountry: "US",
wantProtocolVersion: 1,
wantHeightMin: 1060000,
wantFlags: 1,
},
}
tests := []struct {
name string
wantIP string
wantCountry string
wantProtocolVersion int
wantHeightMin int
wantFlags byte
}{
{
name: "Correctly parse information from production server.",
wantIP: "SETME",
wantCountry: "US",
wantProtocolVersion: 1,
wantHeightMin: 1060000,
wantFlags: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T){
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
toAddr := "spv16.lbry.com"
toPort := "50001"
toAddr := "spv16.lbry.com"
toPort := "50001"
pong, err := UDPPing(toAddr, toPort)
gotCountry := pong.DecodeCountry()
if err != nil {
log.Println(err)
}
pong, err := UDPPing(toAddr, toPort)
gotCountry := pong.DecodeCountry()
if err != nil {
log.Println(err)
}
res, err := exec.Command("dig", "@resolver4.opendns.com", "myip.opendns.com", "+short").Output()
res, err := exec.Command("dig", "@resolver4.opendns.com", "myip.opendns.com", "+short").Output()
if err != nil {
log.Println(err)
}
if err != nil {
log.Println(err)
}
digIP := strings.TrimSpace(string(res))
udpIP := pong.DecodeAddress().String()
tt.wantIP = digIP
digIP := strings.TrimSpace(string(res))
udpIP := pong.DecodeAddress().String()
tt.wantIP = digIP
log.Println("Height:", pong.DecodeHeight())
log.Printf("Flags: %x\n", pong.DecodeFlags())
log.Println("ProtocolVersion:", pong.DecodeProtocolVersion())
log.Printf("Tip: %x\n", pong.DecodeTip())
log.Println("Height:", pong.DecodeHeight())
log.Printf("Flags: %x\n", pong.DecodeFlags())
log.Println("ProtocolVersion:", pong.DecodeProtocolVersion())
log.Printf("Tip: %x\n", pong.DecodeTip())
gotHeight := pong.DecodeHeight()
gotProtocolVersion := pong.DecodeProtocolVersion()
gotFlags := pong.DecodeFlags()
gotIP := udpIP
gotHeight := pong.DecodeHeight()
gotProtocolVersion := pong.DecodeProtocolVersion()
gotFlags := pong.DecodeFlags()
gotIP := udpIP
if gotIP != tt.wantIP {
t.Errorf("ip: got: '%s', want: '%s'\n", gotIP, tt.wantIP)
}
if gotCountry != tt.wantCountry {
t.Errorf("country: got: '%s', want: '%s'\n", gotCountry, tt.wantCountry)
}
if gotHeight < tt.wantHeightMin {
t.Errorf("height: got: %d, want >=: %d\n", gotHeight, tt.wantHeightMin)
}
if gotProtocolVersion != tt.wantProtocolVersion {
t.Errorf("protocolVersion: got: %d, want: %d\n", gotProtocolVersion, tt.wantProtocolVersion)
}
if gotFlags != tt.wantFlags {
t.Errorf("flags: got: %d, want: %d\n", gotFlags, tt.wantFlags)
}
})
}
if gotIP != tt.wantIP {
t.Errorf("ip: got: '%s', want: '%s'\n", gotIP, tt.wantIP)
}
if gotCountry != tt.wantCountry {
t.Errorf("country: got: '%s', want: '%s'\n", gotCountry, tt.wantCountry)
}
if gotHeight < tt.wantHeightMin {
t.Errorf("height: got: %d, want >=: %d\n", gotHeight, tt.wantHeightMin)
}
if gotProtocolVersion != tt.wantProtocolVersion {
t.Errorf("protocolVersion: got: %d, want: %d\n", gotProtocolVersion, tt.wantProtocolVersion)
}
if gotFlags != tt.wantFlags {
t.Errorf("flags: got: %d, want: %d\n", gotFlags, tt.wantFlags)
}
})
}
}