423 lines
9.6 KiB
Go
423 lines
9.6 KiB
Go
package server
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"log"
|
|
"math"
|
|
"net"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/lbryio/herald.go/internal/metrics"
|
|
pb "github.com/lbryio/herald.go/protobuf/go"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
// Peer holds relevant information about peers that we know about.
|
|
type Peer struct {
|
|
Address string
|
|
Port string
|
|
LastSeen time.Time
|
|
}
|
|
|
|
var (
|
|
localHosts = map[string]bool{
|
|
"127.0.0.1": true,
|
|
"0.0.0.0": true,
|
|
"localhost": true,
|
|
"<nil>": true, // Empty net.IP turned into a string
|
|
}
|
|
)
|
|
|
|
// peerKey takes a peer and returns the key that for that peer
|
|
// in our peer table.
|
|
func peerKey(peer *Peer) string {
|
|
return peer.Address + ":" + peer.Port
|
|
}
|
|
|
|
// peerKey is a function on a FederatedServer struct to return the key for that
|
|
// peer is out peer table.
|
|
func (peer *Peer) peerKey() string {
|
|
return peer.Address + ":" + peer.Port
|
|
}
|
|
|
|
func (s *Server) incNumPeers() {
|
|
atomic.AddInt64(s.NumPeerServers, 1)
|
|
}
|
|
|
|
func (s *Server) decNumPeers() {
|
|
atomic.AddInt64(s.NumPeerServers, -1)
|
|
}
|
|
|
|
func (s *Server) getNumPeers() int64 {
|
|
return *s.NumPeerServers
|
|
}
|
|
|
|
func (s *Server) incNumSubs() {
|
|
atomic.AddInt64(s.NumPeerSubs, 1)
|
|
}
|
|
|
|
func (s *Server) decNumSubs() {
|
|
atomic.AddInt64(s.NumPeerSubs, -1)
|
|
}
|
|
|
|
func (s *Server) getNumSubs() int64 {
|
|
return *s.NumPeerSubs
|
|
}
|
|
|
|
// getAndSetExternalIp detects the server's external IP and stores it.
|
|
func (s *Server) getAndSetExternalIp(ip, port string) error {
|
|
pong, err := UDPPing(ip, port)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
myIp := pong.DecodeAddress()
|
|
log.Println("my ip: ", myIp)
|
|
s.ExternalIP = myIp
|
|
|
|
return nil
|
|
}
|
|
|
|
// loadPeers takes the arguments given to the hub at startup and loads the
|
|
// previously known peers from disk and verifies their existence before
|
|
// storing them as known peers. Returns a map of peerKey -> object
|
|
func (s *Server) loadPeers() error {
|
|
peerFile := s.Args.PeerFile
|
|
port := strconv.Itoa(s.Args.Port)
|
|
|
|
// First we make sure our server has come up, so we can answer back to peers.
|
|
var failures = 0
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
|
|
// log.Println("loadPeers #### waiting for server to come up")
|
|
retry:
|
|
time.Sleep(time.Second * time.Duration(math.Pow(float64(failures), 2)))
|
|
conn, err := grpc.DialContext(ctx,
|
|
"0.0.0.0:"+port,
|
|
grpc.WithInsecure(),
|
|
grpc.WithBlock(),
|
|
)
|
|
|
|
if err != nil {
|
|
if failures > 3 {
|
|
log.Println("Warning! Our endpoint doesn't seem to have come up, didn't load peers")
|
|
return err
|
|
}
|
|
failures += 1
|
|
goto retry
|
|
}
|
|
if err = conn.Close(); err != nil {
|
|
log.Println(err)
|
|
}
|
|
cancel()
|
|
// log.Println("loadPeers #### Past checking for server to come up")
|
|
|
|
f, err := os.Open(peerFile)
|
|
if err != nil {
|
|
// log.Println(err)
|
|
return err
|
|
}
|
|
scanner := bufio.NewScanner(f)
|
|
scanner.Split(bufio.ScanLines)
|
|
var text []string
|
|
for scanner.Scan() {
|
|
text = append(text, scanner.Text())
|
|
}
|
|
err = f.Close()
|
|
if err != nil {
|
|
log.Println("peer file failed to close: ", err)
|
|
}
|
|
|
|
for _, line := range text {
|
|
ipPort := strings.Split(line, ":")
|
|
if len(ipPort) != 2 {
|
|
log.Println("Malformed entry in peer file")
|
|
continue
|
|
}
|
|
// If the peer is us, skip
|
|
log.Println(ipPort)
|
|
if ipPort[1] == port &&
|
|
(localHosts[ipPort[0]] || ipPort[0] == s.ExternalIP.String()) {
|
|
log.Println("Self peer, skipping ...")
|
|
continue
|
|
}
|
|
|
|
newPeer := &Peer{
|
|
Address: ipPort[0],
|
|
Port: ipPort[1],
|
|
LastSeen: time.Now(),
|
|
}
|
|
log.Printf("pinging peer %+v\n", newPeer)
|
|
err = s.addPeer(newPeer, true, true)
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
|
|
}
|
|
|
|
log.Println("Returning from loadPeers")
|
|
return nil
|
|
}
|
|
|
|
// subscribeToPeer subscribes us to a peer to we'll get updates about their
|
|
// known peers.
|
|
func (s *Server) subscribeToPeer(peer *Peer) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
|
|
conn, err := grpc.DialContext(ctx,
|
|
peer.Address+":"+peer.Port,
|
|
grpc.WithInsecure(),
|
|
grpc.WithBlock(),
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer conn.Close()
|
|
|
|
msg := &pb.ServerMessage{
|
|
Address: s.ExternalIP.String(),
|
|
Port: strconv.Itoa(s.Args.Port),
|
|
}
|
|
|
|
c := pb.NewHubClient(conn)
|
|
|
|
log.Printf("%s:%d subscribing to %+v\n", s.ExternalIP, s.Args.Port, peer)
|
|
_, err = c.PeerSubscribe(ctx, msg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// helloPeer takes a peer to say hello to and sends a hello message
|
|
// containing all the peers we know about and information about us.
|
|
// This is used to confirm existence of peers on start and let them
|
|
// know about us. Returns the response from the server on success,
|
|
// nil otherwise.
|
|
func (s *Server) helloPeer(peer *Peer) (*pb.HelloMessage, error) {
|
|
log.Println("In helloPeer")
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
|
|
conn, err := grpc.DialContext(ctx,
|
|
peer.Address+":"+peer.Port,
|
|
grpc.WithInsecure(),
|
|
grpc.WithBlock(),
|
|
)
|
|
if err != nil {
|
|
log.Println(err)
|
|
return nil, err
|
|
}
|
|
defer conn.Close()
|
|
|
|
c := pb.NewHubClient(conn)
|
|
|
|
msg := &pb.HelloMessage{
|
|
Port: strconv.Itoa(s.Args.Port),
|
|
Host: s.ExternalIP.String(),
|
|
Servers: []*pb.ServerMessage{},
|
|
}
|
|
|
|
log.Printf("%s:%d saying hello to %+v\n", s.ExternalIP, s.Args.Port, peer)
|
|
res, err := c.Hello(ctx, msg)
|
|
if err != nil {
|
|
log.Println(err)
|
|
return nil, err
|
|
}
|
|
|
|
log.Println(res)
|
|
|
|
return res, nil
|
|
}
|
|
|
|
// writePeers writes our current known peers to disk.
|
|
func (s *Server) writePeers() {
|
|
if s.Args.DisableWritePeers {
|
|
return
|
|
}
|
|
f, err := os.Create(s.Args.PeerFile)
|
|
if err != nil {
|
|
log.Println(err)
|
|
return
|
|
}
|
|
writer := bufio.NewWriter(f)
|
|
|
|
for key := range s.PeerServers {
|
|
line := key + "\n"
|
|
_, err := writer.WriteString(line)
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
}
|
|
|
|
err = writer.Flush()
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
err = f.Close()
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
}
|
|
|
|
// notifyPeer takes a peer to notify and a new peer we just learned about
|
|
// and informs the already known peer about the new peer.
|
|
func (s *Server) notifyPeer(peerToNotify *Peer, newPeer *Peer) error {
|
|
if s.Args.DisableFederation {
|
|
return nil
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
|
|
conn, err := grpc.DialContext(ctx,
|
|
peerToNotify.Address+":"+peerToNotify.Port,
|
|
grpc.WithInsecure(),
|
|
grpc.WithBlock(),
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer conn.Close()
|
|
|
|
msg := &pb.ServerMessage{
|
|
Address: newPeer.Address,
|
|
Port: newPeer.Port,
|
|
}
|
|
|
|
c := pb.NewHubClient(conn)
|
|
|
|
_, err = c.AddPeer(ctx, msg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// notifyPeerSubs takes a new peer server we just learned about and notifies
|
|
// all the peers that have subscribed to us about it.
|
|
func (s *Server) notifyPeerSubs(newPeer *Peer) {
|
|
var unsubscribe []string
|
|
s.PeerSubsMut.RLock()
|
|
for key, peer := range s.PeerSubs {
|
|
log.Printf("Notifying peer %s of new node %+v\n", key, newPeer)
|
|
err := s.notifyPeer(peer, newPeer)
|
|
if err != nil {
|
|
log.Println("Failed to send data to ", key)
|
|
log.Println(err)
|
|
unsubscribe = append(unsubscribe, key)
|
|
}
|
|
}
|
|
s.PeerSubsMut.RUnlock()
|
|
|
|
s.PeerSubsMut.Lock()
|
|
for _, key := range unsubscribe {
|
|
if _, ok := s.PeerSubs[key]; ok {
|
|
delete(s.PeerSubs, key)
|
|
s.decNumSubs()
|
|
metrics.PeersSubscribed.Dec()
|
|
}
|
|
}
|
|
s.PeerSubsMut.Unlock()
|
|
}
|
|
|
|
// addPeer takes a new peer, optionally checks to see if they're online, and
|
|
// adds them to our list of peers. It will also optionally subscribe to it.
|
|
func (s *Server) addPeer(newPeer *Peer, ping bool, subscribe bool) error {
|
|
if s.Args.DisableFederation {
|
|
return nil
|
|
}
|
|
// First thing we get our external ip if we don't have it, otherwise we
|
|
// could end up subscribed to our self, which is silly.
|
|
nilIP := net.IP{}
|
|
localIP1 := net.IPv4(127, 0, 0, 1)
|
|
if s.ExternalIP.Equal(nilIP) || s.ExternalIP.Equal(localIP1) {
|
|
err := s.getAndSetExternalIp(newPeer.Address, newPeer.Port)
|
|
if err != nil {
|
|
log.Println(err)
|
|
log.Println("WARNING: can't determine external IP, continuing with ", s.Args.Host)
|
|
}
|
|
}
|
|
|
|
if strconv.Itoa(s.Args.Port) == newPeer.Port &&
|
|
(localHosts[newPeer.Address] || newPeer.Address == s.ExternalIP.String()) {
|
|
log.Printf("%s:%d addPeer: Self peer, skipping...\n", s.ExternalIP, s.Args.Port)
|
|
return nil
|
|
}
|
|
|
|
k := peerKey(newPeer)
|
|
|
|
log.Printf("%s:%d adding peer %+v\n", s.ExternalIP, s.Args.Port, newPeer)
|
|
if oldServer, loaded := s.PeerServersLoadOrStore(newPeer); !loaded {
|
|
if ping {
|
|
_, err := s.helloPeer(newPeer)
|
|
if err != nil {
|
|
s.PeerServersMut.Lock()
|
|
delete(s.PeerServers, k)
|
|
s.PeerServersMut.Unlock()
|
|
return err
|
|
}
|
|
}
|
|
|
|
s.incNumPeers()
|
|
metrics.PeersKnown.Inc()
|
|
s.writePeers()
|
|
s.notifyPeerSubs(newPeer)
|
|
|
|
// Subscribe to all our peers for now
|
|
if subscribe {
|
|
err := s.subscribeToPeer(newPeer)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
} else {
|
|
oldServer.LastSeen = time.Now()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// mergePeers is an internal convenience function to add a list of
|
|
// peers.
|
|
func (s *Server) mergePeers(servers []*pb.ServerMessage) {
|
|
for _, srvMsg := range servers {
|
|
newPeer := &Peer{
|
|
Address: srvMsg.Address,
|
|
Port: srvMsg.Port,
|
|
LastSeen: time.Now(),
|
|
}
|
|
err := s.addPeer(newPeer, false, true)
|
|
// This shouldn't happen because we're not pinging them.
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// makeHelloMessage makes a message for this hub to call the Hello endpoint
|
|
// on another hub.
|
|
func (s *Server) makeHelloMessage() *pb.HelloMessage {
|
|
servers := make([]*pb.ServerMessage, 0, 10)
|
|
|
|
s.PeerServersMut.RLock()
|
|
for _, peer := range s.PeerServers {
|
|
servers = append(servers, &pb.ServerMessage{
|
|
Address: peer.Address,
|
|
Port: peer.Port,
|
|
})
|
|
}
|
|
s.PeerServersMut.RUnlock()
|
|
|
|
return &pb.HelloMessage{
|
|
Port: strconv.Itoa(s.Args.Port),
|
|
Host: s.ExternalIP.String(),
|
|
Servers: servers,
|
|
}
|
|
}
|