From fb0004bac482060952f9339fcf14eb41b0108eec Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Thu, 19 Mar 2020 05:30:21 +0100 Subject: [PATCH 01/18] increase packet size add retro-compatibility remove unused SPV servers remove travis test failure --- .travis.yml | 2 +- cmd/reflector.go | 25 ++- go.mod | 1 + go.sum | 15 ++ internal/metrics/metrics.go | 4 + peer/quic/client.go | 213 ++++++++++++++++++ peer/quic/server.go | 427 ++++++++++++++++++++++++++++++++++++ peer/quic/store.go | 61 ++++++ reflector/blocklist.go | 4 +- 9 files changed, 744 insertions(+), 8 deletions(-) create mode 100644 peer/quic/client.go create mode 100644 peer/quic/server.go create mode 100644 peer/quic/store.go diff --git a/.travis.yml b/.travis.yml index c9f1b44..933ce39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: true # set -e enabled in bash. before_script: # All the .go files, excluding vendor/ and model (auto generated) - - GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) +# - GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p - go get golang.org/x/tools/cmd/goimports # Used in build script for generated files # - go get github.com/golang/lint/golint # Linter # - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter diff --git a/cmd/reflector.go b/cmd/reflector.go index 8177d1e..19802f0 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -7,10 +7,12 @@ import ( "syscall" "time" + "github.com/lbryio/reflector.go/peer" + "github.com/lbryio/reflector.go/peer/quic" + "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/meta" - "github.com/lbryio/reflector.go/peer" "github.com/lbryio/reflector.go/reflector" "github.com/lbryio/reflector.go/store" @@ -19,6 +21,10 @@ import ( ) var reflectorCmdCacheDir string +var peerPort int +var quicPeerPort int +var reflectorPort int +var metricsPort int func init() { var cmd = &cobra.Command{ @@ -27,6 +33,10 @@ func init() { Run: reflectorCmd, } cmd.Flags().StringVar(&reflectorCmdCacheDir, "cache", "", "Enable disk cache for blobs. Store them in this directory") + cmd.Flags().IntVar(&peerPort, "peer-port", 5567, "The port reflector will distribute content from") + cmd.Flags().IntVar(&quicPeerPort, "quic-peer-port", 5568, "The port reflector will distribute content from over QUIC protocol") + cmd.Flags().IntVar(&reflectorPort, "reflector-port", 5566, "The port reflector will receive content from") + cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics") rootCmd.AddCommand(cmd) } @@ -57,7 +67,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) { reflectorServer.Timeout = 3 * time.Minute reflectorServer.EnableBlocklist = true - err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort)) + err = reflectorServer.Start(":" + strconv.Itoa(reflectorPort)) if err != nil { log.Fatal(err) } @@ -72,12 +82,18 @@ func reflectorCmd(cmd *cobra.Command, args []string) { } peerServer := peer.NewServer(blobStore) - err = peerServer.Start(":5567") + err = peerServer.Start(":"+ strconv.Itoa(peerPort)) if err != nil { log.Fatal(err) } - metricsServer := metrics.NewServer(":2112", "/metrics") + quicPeerServer := quic.NewServer(blobStore) + err = quicPeerServer.Start(":"+ strconv.Itoa(quicPeerPort)) + if err != nil { + log.Fatal(err) + } + + metricsServer := metrics.NewServer(":" + strconv.Itoa(metricsPort), "/metrics") metricsServer.Start() interruptChan := make(chan os.Signal, 1) @@ -85,6 +101,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) { <-interruptChan metricsServer.Shutdown() peerServer.Shutdown() + quicPeerServer.Shutdown() if reflectorServer != nil { reflectorServer.Shutdown() } diff --git a/go.mod b/go.mod index a83d35d..73abece 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/lbryio/lbry.go/v2 v2.4.5 github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec + github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.2 github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index 0ff2261..785df1f 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= +github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -60,6 +62,8 @@ github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= @@ -126,6 +130,7 @@ github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0 github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= @@ -179,6 +184,8 @@ github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiV github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c= github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 h1:tbuodUh2vuhOVZAdW3NEUvosFHUMJwUNl7jk/VSEiwc= +github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns= @@ -187,6 +194,8 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcX github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA= +github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= @@ -210,8 +219,10 @@ github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA= github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= @@ -307,6 +318,7 @@ go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -350,6 +362,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -396,6 +409,7 @@ google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRn gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gorp.v1 v1.7.1 h1:GBB9KrWRATQZh95HJyVGUZrWwOPswitEYEyqlK8JbAA= gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= @@ -404,6 +418,7 @@ gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8= gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 93e0756..617bd55 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -70,6 +70,7 @@ const ( errReadConnReset = "read_conn_reset" errWriteConnReset = "write_conn_reset" errReadConnTimedOut = "read_conn_timed_out" + errNoNetworkActivity = "no_network_activity" errWriteConnTimedOut = "write_conn_timed_out" errWriteBrokenPipe = "write_broken_pipe" errEPipe = "e_pipe" @@ -133,6 +134,9 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a } else if strings.Contains(err.Error(), "read: connection timed out") { // the other side closed the connection using TCP reset //log.Warnln("read conn timed out is not the same as ETIMEDOUT") errType = errReadConnTimedOut + }else if strings.Contains(err.Error(), "NO_ERROR: No recent network activity") { // the other side closed the QUIC connection + //log.Warnln("read conn timed out is not the same as ETIMEDOUT") + errType = errNoNetworkActivity } else if strings.Contains(err.Error(), "write: connection timed out") { errType = errWriteConnTimedOut } else if errors.Is(e, io.ErrUnexpectedEOF) { diff --git a/peer/quic/client.go b/peer/quic/client.go new file mode 100644 index 0000000..08eb753 --- /dev/null +++ b/peer/quic/client.go @@ -0,0 +1,213 @@ +package quic + +import ( + "bufio" + "crypto/tls" + "encoding/hex" + "encoding/json" + "io" + "time" + + "github.com/lucas-clemente/quic-go" + + "github.com/lbryio/reflector.go/store" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" + + log "github.com/sirupsen/logrus" +) + +// ErrBlobExists is a default error for when a blob already exists on the reflector server. +var ErrBlobExists = errors.Base("blob exists on server") + +// Client is an instance of a client connected to a server. +type Client struct { + Timeout time.Duration + conn quic.Session + stream quic.Stream + buf *bufio.Reader + connected bool +} + +// Connect connects to a specific clients and errors if it cannot be contacted. +func (c *Client) Connect(address string) error { + var err error + if c.Timeout == 0 { + c.Timeout = 5 * time.Second + } + tlsConf := &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{"quic-echo-example"}, + } + + c.conn, err = quic.DialAddr(address, tlsConf, nil) + if err != nil { + return err + } + c.connected = true + c.stream, err = c.conn.OpenStream() + if err != nil { + return errors.Err(err) + } + c.buf = bufio.NewReader(c.stream) + return nil +} + +// Close closes the connection with the client. +func (c *Client) Close() error { + c.connected = false + return c.conn.Close() +} + +// GetStream gets a stream +func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) { + if !c.connected { + return nil, errors.Err("not connected") + } + + var sd stream.SDBlob + + b, err := c.GetBlob(sdHash) + if err != nil { + return nil, err + } + + err = sd.FromBlob(b) + if err != nil { + return nil, err + } + + s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob + s[0] = b + + for i := 0; i < len(sd.BlobInfos)-1; i++ { + s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash)) + if err != nil { + return nil, err + } + } + + return s, nil +} + +// HasBlob checks if the blob is available +func (c *Client) HasBlob(hash string) (bool, error) { + if !c.connected { + return false, errors.Err("not connected") + } + + sendRequest, err := json.Marshal(availabilityRequest{ + RequestedBlobs: []string{hash}, + }) + if err != nil { + return false, err + } + + err = c.write(sendRequest) + if err != nil { + return false, err + } + + var resp availabilityResponse + err = c.read(&resp) + if err != nil { + return false, err + } + + for _, h := range resp.AvailableBlobs { + if h == hash { + return true, nil + } + } + + return false, nil +} + +// GetBlob gets a blob +func (c *Client) GetBlob(hash string) (stream.Blob, error) { + if !c.connected { + return nil, errors.Err("not connected") + } + + sendRequest, err := json.Marshal(blobRequest{ + RequestedBlob: hash, + }) + if err != nil { + return nil, err + } + + err = c.write(sendRequest) + if err != nil { + return nil, err + } + + var resp blobResponse + err = c.read(&resp) + if err != nil { + return nil, err + } + + if resp.IncomingBlob.Error != "" { + return nil, errors.Prefix(hash[:8], resp.IncomingBlob.Error) + } + if resp.IncomingBlob.BlobHash != hash { + return nil, errors.Prefix(hash[:8], "blob hash in response does not match requested hash") + } + if resp.IncomingBlob.Length <= 0 { + return nil, errors.Prefix(hash[:8], "length reported as <= 0") + } + + log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr()) + + blob, err := c.readRawBlob(resp.IncomingBlob.Length) + if err != nil { + return nil, err + } + + return stream.Blob(blob), nil +} + +func (c *Client) read(v interface{}) error { + err := c.stream.SetReadDeadline(time.Now().Add(c.Timeout)) + if err != nil { + return errors.Err(err) + } + + m, err := readNextMessage(c.buf) + if err != nil { + return err + } + + log.Debugf("read %d bytes from %s", len(m), c.conn.RemoteAddr()) + + err = json.Unmarshal(m, v) + return errors.Err(err) +} + +func (c *Client) readRawBlob(blobSize int) ([]byte, error) { + err := c.stream.SetReadDeadline(time.Now().Add(c.Timeout)) + if err != nil { + return nil, errors.Err(err) + } + + blob := make([]byte, blobSize) + n, err := io.ReadFull(c.buf, blob) + log.Debugf("read %d bytes from %s", n, c.conn.RemoteAddr()) + return blob, errors.Err(err) +} + +func (c *Client) write(b []byte) error { + err := c.stream.SetWriteDeadline(time.Now().Add(c.Timeout)) + if err != nil { + return errors.Err(err) + } + + log.Debugf("writing %d bytes to %s", len(b), c.conn.RemoteAddr()) + + n, err := c.stream.Write(b) + if err == nil && n != len(b) { + err = io.ErrShortWrite + } + return errors.Err(err) +} diff --git a/peer/quic/server.go b/peer/quic/server.go new file mode 100644 index 0000000..601e904 --- /dev/null +++ b/peer/quic/server.go @@ -0,0 +1,427 @@ +package quic + +import ( + "bufio" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "io" + "math/big" + "strings" + "time" + + "github.com/lucas-clemente/quic-go" + + "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/reflector" + "github.com/lbryio/reflector.go/store" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/extras/stop" + "github.com/lbryio/lbry.go/v2/stream" + + log "github.com/sirupsen/logrus" +) + +const ( + // DefaultPort is the port the peer server listens on if not passed in. + DefaultPort = 3333 + // LbrycrdAddress to be used when paying for data. Not implemented yet. + LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct" +) + +// Server is an instance of a peer server that houses the listener and store. +type Server struct { + store store.BlobStore + closed bool + + grp *stop.Group +} + +// NewServer returns an initialized Server pointer. +func NewServer(store store.BlobStore) *Server { + return &Server{ + store: store, + grp: stop.New(), + } +} + +// Shutdown gracefully shuts down the peer server. +func (s *Server) Shutdown() { + log.Debug("shutting down peer server") + s.grp.StopAndWait() + log.Debug("peer server stopped") +} + +// Start starts the server listener to handle connections. +func (s *Server) Start(address string) error { + log.Println("QUIC peer listening on " + address) + + l, err := quic.ListenAddr(address, generateTLSConfig(), nil) + if err != nil { + return err + } + + go s.listenForShutdown(l) + s.grp.Add(1) + go func() { + s.listenAndServe(l) + s.grp.Done() + }() + + return nil +} + +func (s *Server) listenForShutdown(listener quic.Listener) { + <-s.grp.Ch() + s.closed = true + err := listener.Close() + if err != nil { + log.Error("error closing listener for peer server - ", err) + } +} + +func (s *Server) listenAndServe(listener quic.Listener) { + for { + conn, err := listener.Accept() + if err != nil { + if s.closed { + return + } + log.Error(errors.Prefix("accepting conn", err)) + } else { + s.grp.Add(1) + go func() { + s.handleConnection(conn) + s.grp.Done() + }() + } + } +} + +func (s *Server) handleConnection(conn quic.Session) { + defer func() { + if err := conn.Close(); err != nil { + log.Error(errors.Prefix("closing peer conn", err)) + } + }() + + timeoutDuration := 1 * time.Minute + stream, err := conn.AcceptStream() + if err != nil { + log.Error(err) + return + } + buf := bufio.NewReader(stream) + for { + var request []byte + var response []byte + + err := stream.SetReadDeadline(time.Now().Add(timeoutDuration)) + if err != nil { + log.Error(errors.FullTrace(err)) + } + + request, err = readNextMessage(buf) + if err != nil { + if err != io.EOF { + s.logError(err) + } + return + } + + err = stream.SetReadDeadline(time.Time{}) + if err != nil { + log.Error(errors.FullTrace(err)) + } + + response, err = s.handleCompositeRequest(request) + if err != nil { + log.Error(errors.FullTrace(err)) + return + } + + err = stream.SetWriteDeadline(time.Now().Add(timeoutDuration)) + if err != nil { + log.Error(errors.FullTrace(err)) + } + + n, err := stream.Write(response) + if err != nil { + if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset + s.logError(err) + } + return + } else if n != len(response) { + log.Errorln(io.ErrShortWrite) + return + } + + err = stream.SetWriteDeadline(time.Time{}) + if err != nil { + log.Error(errors.FullTrace(err)) + } + } +} + +func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) { + var request availabilityRequest + err := json.Unmarshal(data, &request) + if err != nil { + return nil, errors.Err(err) + } + + availableBlobs := []string{} + for _, blobHash := range request.RequestedBlobs { + exists, err := s.store.Has(blobHash) + if err != nil { + return nil, err + } + if exists { + availableBlobs = append(availableBlobs, blobHash) + } + } + + return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs}) +} + +//func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) { +// var request paymentRateRequest +// err := json.Unmarshal(data, &request) +// if err != nil { +// return nil, err +// } +// +// offerReply := paymentRateAccepted +// if request.BlobDataPaymentRate < 0 { +// offerReply = paymentRateTooLow +// } +// +// return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply}) +//} +// +//func (s *Server) handleBlobRequest(data []byte) ([]byte, error) { +// var request blobRequest +// err := json.Unmarshal(data, &request) +// if err != nil { +// return nil, err +// } +// +// log.Debugln("Sending blob " + request.RequestedBlob[:8]) +// +// blob, err := s.store.Get(request.RequestedBlob) +// if err != nil { +// return nil, err +// } +// +// response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{ +// BlobHash: reflector.BlobHash(blob), +// Length: len(blob), +// }}) +// if err != nil { +// return nil, err +// } +// +// return append(response, blob...), nil +//} + +func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) { + var request compositeRequest + err := json.Unmarshal(data, &request) + if err != nil { + return nil, errors.Err(err) + } + + response := compositeResponse{ + LbrycrdAddress: LbrycrdAddress, + } + + if len(request.RequestedBlobs) > 0 { + var availableBlobs []string + for _, blobHash := range request.RequestedBlobs { + exists, err := s.store.Has(blobHash) + if err != nil { + return nil, err + } + if exists { + availableBlobs = append(availableBlobs, blobHash) + } + } + response.AvailableBlobs = availableBlobs + } + + response.BlobDataPaymentRate = paymentRateAccepted + if request.BlobDataPaymentRate < 0 { + response.BlobDataPaymentRate = paymentRateTooLow + } + + var blob []byte + if request.RequestedBlob != "" { + if len(request.RequestedBlob) != stream.BlobHashHexLength { + return nil, errors.Err("Invalid blob hash length") + } + + log.Debugln("Sending blob " + request.RequestedBlob[:8]) + + blob, err = s.store.Get(request.RequestedBlob) + if errors.Is(err, store.ErrBlobNotFound) { + response.IncomingBlob = incomingBlob{ + Error: err.Error(), + } + } else if err != nil { + return nil, err + } else { + response.IncomingBlob = incomingBlob{ + BlobHash: reflector.BlobHash(blob), + Length: len(blob), + } + metrics.BlobDownloadCount.Inc() + } + } + + respData, err := json.Marshal(response) + if err != nil { + return nil, err + } + + return append(respData, blob...), nil +} + +func (s *Server) logError(e error) { + if e == nil { + return + } + shouldLog := metrics.TrackError(metrics.DirectionDownload, e) + if shouldLog { + log.Errorln(errors.FullTrace(e)) + } +} + +func readNextMessage(buf *bufio.Reader) ([]byte, error) { + msg := make([]byte, 0) + eof := false + + for { + chunk, err := buf.ReadBytes('}') + if err != nil { + if err != io.EOF { + //log.Errorln("readBytes error:", err) // logged by caller + return msg, err + } + eof = true + } + + //log.Debugln("got", len(chunk), "bytes.") + //spew.Dump(chunk) + + if len(chunk) > 0 { + msg = append(msg, chunk...) + + if len(msg) > maxRequestSize { + return msg, errRequestTooLarge + } + + // yes, this is how the peer protocol knows when the request finishes + if reflector.IsValidJSON(msg) { + break + } + } + + if eof { + break + } + } + + //log.Debugln("total size:", len(request)) + //if len(request) > 0 { + // spew.Dump(request) + //} + + if len(msg) == 0 && eof { + return nil, io.EOF + } + + return msg, nil +} + +const ( + maxRequestSize = 64 * (2 ^ 10) // 64kb + paymentRateAccepted = "RATE_ACCEPTED" + paymentRateTooLow = "RATE_TOO_LOW" + //ToDo: paymentRateUnset is not used but exists in the protocol. + //paymentRateUnset = "RATE_UNSET" +) + +var errRequestTooLarge = errors.Base("request is too large") + +type availabilityRequest struct { + LbrycrdAddress bool `json:"lbrycrd_address"` + RequestedBlobs []string `json:"requested_blobs"` +} + +type availabilityResponse struct { + LbrycrdAddress string `json:"lbrycrd_address"` + AvailableBlobs []string `json:"available_blobs"` +} + +type paymentRateRequest struct { + BlobDataPaymentRate float64 `json:"blob_data_payment_rate"` +} + +type paymentRateResponse struct { + BlobDataPaymentRate string `json:"blob_data_payment_rate"` +} + +type blobRequest struct { + RequestedBlob string `json:"requested_blob"` +} + +type incomingBlob struct { + Error string `json:"error,omitempty"` + BlobHash string `json:"blob_hash"` + Length int `json:"length"` +} +type blobResponse struct { + IncomingBlob incomingBlob `json:"incoming_blob"` +} + +type compositeRequest struct { + LbrycrdAddress bool `json:"lbrycrd_address"` + RequestedBlobs []string `json:"requested_blobs"` + BlobDataPaymentRate float64 `json:"blob_data_payment_rate"` + RequestedBlob string `json:"requested_blob"` +} + +type compositeResponse struct { + LbrycrdAddress string `json:"lbrycrd_address,omitempty"` + AvailableBlobs []string `json:"available_blobs,omitempty"` + BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"` + IncomingBlob incomingBlob `json:"incoming_blob,omitempty"` +} + +// Setup a bare-bones TLS config for the server +func generateTLSConfig() *tls.Config { + key, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + panic(err) + } + template := x509.Certificate{SerialNumber: big.NewInt(1)} + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + panic(err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + panic(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + NextProtos: []string{"quic-echo-example"}, + } +} diff --git a/peer/quic/store.go b/peer/quic/store.go new file mode 100644 index 0000000..dd3ffe5 --- /dev/null +++ b/peer/quic/store.go @@ -0,0 +1,61 @@ +package quic + +import ( + "time" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" +) + +// Store is a blob store that gets blobs from a peer. +// It satisfies the store.BlobStore interface but cannot put or delete blobs. +type Store struct { + client *Client + connErr error +} + +// StoreOpts allows to set options for a new Store. +type StoreOpts struct { + Address string + Timeout time.Duration +} + +// NewStore makes a new peer store. +func NewStore(opts StoreOpts) *Store { + c := &Client{Timeout: opts.Timeout} + err := c.Connect(opts.Address) + return &Store{client: c, connErr: err} +} + +// Has asks the peer if they have a hash +func (p *Store) Has(hash string) (bool, error) { + if p.connErr != nil { + return false, errors.Prefix("connection error", p.connErr) + } + + return p.client.HasBlob(hash) +} + +// Get downloads the blob from the peer +func (p *Store) Get(hash string) (stream.Blob, error) { + if p.connErr != nil { + return nil, errors.Prefix("connection error", p.connErr) + } + + return p.client.GetBlob(hash) +} + +// Put is not supported +func (p *Store) Put(hash string, blob stream.Blob) error { + panic("PeerStore cannot put or delete blobs") +} + +// PutSD is not supported +func (p *Store) PutSD(hash string, blob stream.Blob) error { + panic("PeerStore cannot put or delete blobs") +} + +// Delete is not supported +func (p *Store) Delete(hash string) error { + panic("PeerStore cannot put or delete blobs") +} diff --git a/reflector/blocklist.go b/reflector/blocklist.go index 9f8f4ad..42aa2ce 100644 --- a/reflector/blocklist.go +++ b/reflector/blocklist.go @@ -41,7 +41,7 @@ func updateBlocklist(b store.Blocklister) { for name, v := range values { if v.Err != nil { - log.Error(errors.Err("blocklist: %s: %s", name, v.Err)) + log.Error(errors.FullTrace(errors.Err("blocklist: %s: %s", name, v.Err))) continue } @@ -101,9 +101,7 @@ func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) { "spv9.lbry.com:50001", "spv15.lbry.com:50001", "spv17.lbry.com:50001", - "spv19.1bry.com:50001", "spv25.lbry.com:50001", - "spv26.lbry.com:50001", }, nil) if err != nil { return nil, errors.Err(err) -- 2.45.2 From e98794e1254608afd3e0cc7c0ccaddf3f5eaecf5 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sat, 21 Mar 2020 23:37:01 -0400 Subject: [PATCH 02/18] Add close function for quic store so routines do not hang waiting for timeout. --- internal/metrics/metrics.go | 7 +++++-- peer/quic/store.go | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 617bd55..94bac40 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -70,7 +70,7 @@ const ( errReadConnReset = "read_conn_reset" errWriteConnReset = "write_conn_reset" errReadConnTimedOut = "read_conn_timed_out" - errNoNetworkActivity = "no_network_activity" + errNoNetworkActivity = "no_network_activity" errWriteConnTimedOut = "write_conn_timed_out" errWriteBrokenPipe = "write_broken_pipe" errEPipe = "e_pipe" @@ -84,6 +84,7 @@ const ( errHashMismatch = "hash_mismatch" errZeroByteBlob = "zero_byte_blob" errInvalidCharacter = "invalid_character" + errNoErr = "no_error" errOther = "other" ) @@ -134,7 +135,7 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a } else if strings.Contains(err.Error(), "read: connection timed out") { // the other side closed the connection using TCP reset //log.Warnln("read conn timed out is not the same as ETIMEDOUT") errType = errReadConnTimedOut - }else if strings.Contains(err.Error(), "NO_ERROR: No recent network activity") { // the other side closed the QUIC connection + } else if strings.Contains(err.Error(), "NO_ERROR: No recent network activity") { // the other side closed the QUIC connection //log.Warnln("read conn timed out is not the same as ETIMEDOUT") errType = errNoNetworkActivity } else if strings.Contains(err.Error(), "write: connection timed out") { @@ -161,6 +162,8 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a errType = errInvalidCharacter } else if _, ok := e.(*json.SyntaxError); ok { errType = errJSONSyntax + } else if strings.Contains(err.Error(), "NO_ERROR") { + errType = errNoErr } else { log.Warnf("error '%s' for direction '%s' is not being tracked", err.TypeName(), direction) shouldLog = true diff --git a/peer/quic/store.go b/peer/quic/store.go index dd3ffe5..12fd7d2 100644 --- a/peer/quic/store.go +++ b/peer/quic/store.go @@ -27,6 +27,15 @@ func NewStore(opts StoreOpts) *Store { return &Store{client: c, connErr: err} } +// CloseStore closes the client that gets initialized when the store is initialized +func (p *Store) CloseStore() error { + err := p.client.stream.Close() + if err != nil { + return errors.Err(err) + } + return p.client.Close() +} + // Has asks the peer if they have a hash func (p *Store) Has(hash string) (bool, error) { if p.connErr != nil { -- 2.45.2 From de1fb63a1c5776457db8b3a7ed332453ef715cf8 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sun, 22 Mar 2020 00:38:46 -0400 Subject: [PATCH 03/18] fix potentially missing client or stream for a quic store --- peer/quic/store.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/peer/quic/store.go b/peer/quic/store.go index 12fd7d2..bf4c10e 100644 --- a/peer/quic/store.go +++ b/peer/quic/store.go @@ -29,11 +29,15 @@ func NewStore(opts StoreOpts) *Store { // CloseStore closes the client that gets initialized when the store is initialized func (p *Store) CloseStore() error { - err := p.client.stream.Close() - if err != nil { - return errors.Err(err) + if p.client != nil && p.client.stream != nil { + err := p.client.stream.Close() + if err != nil { + return errors.Err(err) + } + return p.client.Close() } - return p.client.Close() + return nil + } // Has asks the peer if they have a hash -- 2.45.2 From 4a9f127ecc50fd128088cfb8d73c641b03f16307 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sun, 22 Mar 2020 01:12:55 -0400 Subject: [PATCH 04/18] Add gops to reflector --- go.mod | 1 + go.sum | 16 ++++++++++++++++ main.go | 6 ++++++ 3 files changed, 23 insertions(+) diff --git a/go.mod b/go.mod index 73abece..acedd1e 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/go-sql-driver/mysql v1.4.1 github.com/golang/protobuf v1.3.2 github.com/google/btree v1.0.0 // indirect + github.com/google/gops v0.3.7 github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/hashicorp/memberlist v0.1.4 // indirect diff --git a/go.sum b/go.sum index 785df1f..4548ae8 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -54,6 +55,7 @@ github.com/go-ini/ini v1.38.2 h1:6Hl/z3p3iFkA0dlDfzYxuFuUGD+kaweypF6btsR2/Q4= github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw= github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= @@ -79,6 +81,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw= +github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000= @@ -152,6 +156,10 @@ github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro= +github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999 h1:2d+FLQbz4xRTi36DO1qYNUwfORax9XcQ0jhbO81Vago= +github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= @@ -260,6 +268,9 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf h1:6uhCmVhF7UeIpvdPUtuU91QwYTUi+TGS8wp/7QA+KmM= github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk= +github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7 h1:80VN+vGkqM773Br/uNNTSheo3KatTgV8IpjIKjvVLng= +github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -307,6 +318,8 @@ github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dl github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ= github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU= github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 h1:YdYsPAZ2pC6Tow/nPZOPQ96O3hm/ToAkGsPLzedXERk= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= @@ -356,6 +369,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -430,3 +444,5 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/goversion v1.0.0 h1:/IhXBiai89TyuerPquiZZ39IQkTfAUbZB2awsyYZ/2c= +rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= diff --git a/main.go b/main.go index f268821..7b76efe 100644 --- a/main.go +++ b/main.go @@ -5,9 +5,15 @@ import ( "time" "github.com/lbryio/reflector.go/cmd" + + "github.com/google/gops/agent" + log "github.com/sirupsen/logrus" ) func main() { + if err := agent.Listen(agent.Options{}); err != nil { + log.Fatal(err) + } rand.Seed(time.Now().UnixNano()) cmd.Execute() } -- 2.45.2 From 1bf3cb81b3f3abf00f102685c41c43c98ccd9444 Mon Sep 17 00:00:00 2001 From: Andrey Beletsky Date: Fri, 3 Apr 2020 19:47:45 +0700 Subject: [PATCH 05/18] Use ModTime on systems that don't provide Atim file stat field --- store/atime_linux.go | 17 +++++++++++++++++ store/atime_universal.go | 12 ++++++++++++ store/disk.go | 8 -------- 3 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 store/atime_linux.go create mode 100644 store/atime_universal.go diff --git a/store/atime_linux.go b/store/atime_linux.go new file mode 100644 index 0000000..f3be390 --- /dev/null +++ b/store/atime_linux.go @@ -0,0 +1,17 @@ +// +build linux + +package store + +import ( + "os" + "syscall" + "time" +) + +func timespecToTime(ts syscall.Timespec) time.Time { + return time.Unix(int64(ts.Sec), int64(ts.Nsec)) +} + +func atime(fi os.FileInfo) time.Time { + return timespecToTime(fi.Sys().(*syscall.Stat_t).Atim) +} diff --git a/store/atime_universal.go b/store/atime_universal.go new file mode 100644 index 0000000..237a013 --- /dev/null +++ b/store/atime_universal.go @@ -0,0 +1,12 @@ +// +build !linux + +package store + +import ( + "os" + "time" +) + +func atime(fi os.FileInfo) time.Time { + return fi.ModTime() +} diff --git a/store/disk.go b/store/disk.go index 0dbc30d..b7dea85 100644 --- a/store/disk.go +++ b/store/disk.go @@ -227,11 +227,3 @@ func (d *DiskBlobStore) WipeOldestBlobs() (err error) { } return nil } - -func timespecToTime(ts syscall.Timespec) time.Time { - return time.Unix(int64(ts.Sec), int64(ts.Nsec)) -} - -func atime(fi os.FileInfo) time.Time { - return timespecToTime(fi.Sys().(*syscall.Stat_t).Atim) -} -- 2.45.2 From 3ffe7a10c7be0d7b53a2f7e2f7d6fb49611607e9 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Sat, 9 May 2020 03:06:51 +0200 Subject: [PATCH 06/18] add other reflector store add flags improve disk cleanup --- cmd/reflector.go | 42 +++++++++++++++++++++++++++++++----------- store/disk.go | 7 ++++--- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index 19802f0..e411781 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -25,6 +25,11 @@ var peerPort int var quicPeerPort int var reflectorPort int var metricsPort int +var disableUploads bool +var reflectorServerAddress string +var reflectorServerPort string +var reflectorServerProtocol string +var useDB bool func init() { var cmd = &cobra.Command{ @@ -33,25 +38,40 @@ func init() { Run: reflectorCmd, } cmd.Flags().StringVar(&reflectorCmdCacheDir, "cache", "", "Enable disk cache for blobs. Store them in this directory") + cmd.Flags().StringVar(&reflectorServerAddress, "reflector-server-address", "", "address of another reflector server where blobs are fetched from") + cmd.Flags().StringVar(&reflectorServerPort, "reflector-server-port", "5567", "port of another reflector server where blobs are fetched from") + cmd.Flags().StringVar(&reflectorServerProtocol, "reflector-server-protocol", "tcp", "protocol used to fetch blobs from another reflector server (tcp/udp)") cmd.Flags().IntVar(&peerPort, "peer-port", 5567, "The port reflector will distribute content from") cmd.Flags().IntVar(&quicPeerPort, "quic-peer-port", 5568, "The port reflector will distribute content from over QUIC protocol") cmd.Flags().IntVar(&reflectorPort, "reflector-port", 5566, "The port reflector will receive content from") cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics") + cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server") + cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not") rootCmd.AddCommand(cmd) } func reflectorCmd(cmd *cobra.Command, args []string) { log.Printf("reflector version %s, built %s", meta.Version, meta.BuildTime.Format(time.RFC3339)) - // flip this flag to false when doing db maintenance. uploads will not work (as reflector server wont be running) - // but downloads will still work straight from s3 - useDB := true - - s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName) + var blobStore store.BlobStore + if reflectorServerAddress != "" { + switch reflectorServerProtocol { + case "tcp": + blobStore = peer.NewStore(peer.StoreOpts{ + Address: reflectorServerAddress + ":" + reflectorServerPort, + Timeout: 30 * time.Second, + }) + case "udp": + blobStore = quic.NewStore(quic.StoreOpts{ + Address: reflectorServerAddress + ":" + reflectorServerPort, + Timeout: 30 * time.Second, + }) + } + } else { + blobStore = store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName) + } var err error - - var blobStore store.BlobStore = s3 var reflectorServer *reflector.Server if useDB { @@ -61,7 +81,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) { log.Fatal(err) } - blobStore = store.NewDBBackedStore(s3, db) + blobStore = store.NewDBBackedStore(blobStore, db) reflectorServer = reflector.NewServer(blobStore) reflectorServer.Timeout = 3 * time.Minute @@ -82,18 +102,18 @@ func reflectorCmd(cmd *cobra.Command, args []string) { } peerServer := peer.NewServer(blobStore) - err = peerServer.Start(":"+ strconv.Itoa(peerPort)) + err = peerServer.Start(":" + strconv.Itoa(peerPort)) if err != nil { log.Fatal(err) } quicPeerServer := quic.NewServer(blobStore) - err = quicPeerServer.Start(":"+ strconv.Itoa(quicPeerPort)) + err = quicPeerServer.Start(":" + strconv.Itoa(quicPeerPort)) if err != nil { log.Fatal(err) } - metricsServer := metrics.NewServer(":" + strconv.Itoa(metricsPort), "/metrics") + metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics") metricsServer.Start() interruptChan := make(chan os.Signal, 1) diff --git a/store/disk.go b/store/disk.go index b7dea85..18f7bd7 100644 --- a/store/disk.go +++ b/store/disk.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "sort" "syscall" "time" @@ -191,10 +192,10 @@ func (d *DiskBlobStore) WipeOldestBlobs() (err error) { File *os.FileInfo FullPath string } - datedFiles := make([]datedFile, 0, 500) + datedFiles := make([]datedFile, 0, 5000) for _, dir := range dirs { if dir.IsDir() { - files, err := ioutil.ReadDir(d.blobDir + "/" + dir.Name()) + files, err := ioutil.ReadDir(filepath.Join(d.blobDir, dir.Name())) if err != nil { return err } @@ -203,7 +204,7 @@ func (d *DiskBlobStore) WipeOldestBlobs() (err error) { datedFiles = append(datedFiles, datedFile{ Atime: atime(file), File: &file, - FullPath: d.blobDir + "/" + dir.Name() + "/" + file.Name(), + FullPath: filepath.Join(d.blobDir, dir.Name(), file.Name()), }) } } -- 2.45.2 From a80599413c44bee0a6f03e1161182cd96ca5ec0b Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Sat, 9 May 2020 05:14:26 +0200 Subject: [PATCH 07/18] improve params description --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index da5e4f5..0f59306 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -49,7 +49,7 @@ var rootCmd = &cobra.Command{ func init() { rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components") - rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config") + rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config. Use 'none' to disable") } // Execute adds all child commands to the root command and sets flags appropriately. -- 2.45.2 From e0da2674a1cbe2a932901982a4cde064de9ac2b9 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 12 May 2020 03:32:11 +0200 Subject: [PATCH 08/18] use new connections for each action reduce timeout --- cmd/reflector.go | 4 ++-- peer/store.go | 31 ++++++++++++++++++------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index e411781..869d757 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -59,12 +59,12 @@ func reflectorCmd(cmd *cobra.Command, args []string) { case "tcp": blobStore = peer.NewStore(peer.StoreOpts{ Address: reflectorServerAddress + ":" + reflectorServerPort, - Timeout: 30 * time.Second, + Timeout: 10 * time.Second, }) case "udp": blobStore = quic.NewStore(quic.StoreOpts{ Address: reflectorServerAddress + ":" + reflectorServerPort, - Timeout: 30 * time.Second, + Timeout: 10 * time.Second, }) } } else { diff --git a/peer/store.go b/peer/store.go index c5d3d5a..3857426 100644 --- a/peer/store.go +++ b/peer/store.go @@ -10,8 +10,7 @@ import ( // Store is a blob store that gets blobs from a peer. // It satisfies the store.BlobStore interface but cannot put or delete blobs. type Store struct { - client *Client - connErr error + opts StoreOpts } // StoreOpts allows to set options for a new Store. @@ -22,27 +21,33 @@ type StoreOpts struct { // NewStore makes a new peer store. func NewStore(opts StoreOpts) *Store { - c := &Client{Timeout: opts.Timeout} - err := c.Connect(opts.Address) - return &Store{client: c, connErr: err} + return &Store{opts: opts} +} + +func (p *Store) getClient() (*Client, error) { + c := &Client{Timeout: p.opts.Timeout} + err := c.Connect(p.opts.Address) + return c, errors.Prefix("connection error", err) } // Has asks the peer if they have a hash func (p *Store) Has(hash string) (bool, error) { - if p.connErr != nil { - return false, errors.Prefix("connection error", p.connErr) + c, err := p.getClient() + if err != nil { + return false, err } - - return p.client.HasBlob(hash) + defer c.Close() + return c.HasBlob(hash) } // Get downloads the blob from the peer func (p *Store) Get(hash string) (stream.Blob, error) { - if p.connErr != nil { - return nil, errors.Prefix("connection error", p.connErr) + c, err := p.getClient() + if err != nil { + return nil, err } - - return p.client.GetBlob(hash) + defer c.Close() + return c.GetBlob(hash) } // Put is not supported -- 2.45.2 From 90997b99182ce820bbdd7c3d53d70affaf2cab2f Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 13 May 2020 21:53:57 +0200 Subject: [PATCH 09/18] use new connections for each blob --- cmd/reflector.go | 4 ++-- peer/quic/client.go | 3 +++ peer/quic/store.go | 40 ++++++++++++++++------------------------ 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index 869d757..e411781 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -59,12 +59,12 @@ func reflectorCmd(cmd *cobra.Command, args []string) { case "tcp": blobStore = peer.NewStore(peer.StoreOpts{ Address: reflectorServerAddress + ":" + reflectorServerPort, - Timeout: 10 * time.Second, + Timeout: 30 * time.Second, }) case "udp": blobStore = quic.NewStore(quic.StoreOpts{ Address: reflectorServerAddress + ":" + reflectorServerPort, - Timeout: 10 * time.Second, + Timeout: 30 * time.Second, }) } } else { diff --git a/peer/quic/client.go b/peer/quic/client.go index 08eb753..83f50c0 100644 --- a/peer/quic/client.go +++ b/peer/quic/client.go @@ -149,6 +149,9 @@ func (c *Client) GetBlob(hash string) (stream.Blob, error) { } if resp.IncomingBlob.Error != "" { + if resp.IncomingBlob.Error == store.ErrBlobNotFound.Error() { + return nil, errors.Err(store.ErrBlobNotFound) + } return nil, errors.Prefix(hash[:8], resp.IncomingBlob.Error) } if resp.IncomingBlob.BlobHash != hash { diff --git a/peer/quic/store.go b/peer/quic/store.go index bf4c10e..9300c4a 100644 --- a/peer/quic/store.go +++ b/peer/quic/store.go @@ -10,8 +10,7 @@ import ( // Store is a blob store that gets blobs from a peer. // It satisfies the store.BlobStore interface but cannot put or delete blobs. type Store struct { - client *Client - connErr error + opts StoreOpts } // StoreOpts allows to set options for a new Store. @@ -22,40 +21,33 @@ type StoreOpts struct { // NewStore makes a new peer store. func NewStore(opts StoreOpts) *Store { - c := &Client{Timeout: opts.Timeout} - err := c.Connect(opts.Address) - return &Store{client: c, connErr: err} + return &Store{opts: opts} } -// CloseStore closes the client that gets initialized when the store is initialized -func (p *Store) CloseStore() error { - if p.client != nil && p.client.stream != nil { - err := p.client.stream.Close() - if err != nil { - return errors.Err(err) - } - return p.client.Close() - } - return nil - +func (p *Store) getClient() (*Client, error) { + c := &Client{Timeout: p.opts.Timeout} + err := c.Connect(p.opts.Address) + return c, errors.Prefix("connection error", err) } // Has asks the peer if they have a hash func (p *Store) Has(hash string) (bool, error) { - if p.connErr != nil { - return false, errors.Prefix("connection error", p.connErr) + c, err := p.getClient() + if err != nil { + return false, err } - - return p.client.HasBlob(hash) + defer c.Close() + return c.HasBlob(hash) } // Get downloads the blob from the peer func (p *Store) Get(hash string) (stream.Blob, error) { - if p.connErr != nil { - return nil, errors.Prefix("connection error", p.connErr) + c, err := p.getClient() + if err != nil { + return nil, err } - - return p.client.GetBlob(hash) + defer c.Close() + return c.GetBlob(hash) } // Put is not supported -- 2.45.2 From 8a5f57b14f39fb523d3234f34fc160808afe94d4 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 29 Jun 2020 21:13:52 +0200 Subject: [PATCH 10/18] remove QUIC protocol add HTTP3/QUIC protocol --- cmd/reflector.go | 11 +- go.mod | 6 +- go.sum | 155 +++++++++++- peer/http3/client.go | 97 ++++++++ peer/http3/server.go | 162 +++++++++++++ peer/{quic => http3}/store.go | 29 ++- peer/quic/client.go | 216 ----------------- peer/quic/server.go | 427 ---------------------------------- reflector/blocklist.go | 6 - 9 files changed, 441 insertions(+), 668 deletions(-) create mode 100644 peer/http3/client.go create mode 100644 peer/http3/server.go rename peer/{quic => http3}/store.go (71%) delete mode 100644 peer/quic/client.go delete mode 100644 peer/quic/server.go diff --git a/cmd/reflector.go b/cmd/reflector.go index e411781..ebdbe5b 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -8,7 +8,7 @@ import ( "time" "github.com/lbryio/reflector.go/peer" - "github.com/lbryio/reflector.go/peer/quic" + "github.com/lbryio/reflector.go/peer/http3" "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/internal/metrics" @@ -62,7 +62,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) { Timeout: 30 * time.Second, }) case "udp": - blobStore = quic.NewStore(quic.StoreOpts{ + blobStore = http3.NewStore(http3.StoreOpts{ Address: reflectorServerAddress + ":" + reflectorServerPort, Timeout: 30 * time.Second, }) @@ -107,8 +107,8 @@ func reflectorCmd(cmd *cobra.Command, args []string) { log.Fatal(err) } - quicPeerServer := quic.NewServer(blobStore) - err = quicPeerServer.Start(":" + strconv.Itoa(quicPeerPort)) + http3PeerServer := http3.NewServer(blobStore) + err = http3PeerServer.Start(":" + strconv.Itoa(quicPeerPort)) if err != nil { log.Fatal(err) } @@ -121,7 +121,8 @@ func reflectorCmd(cmd *cobra.Command, args []string) { <-interruptChan metricsServer.Shutdown() peerServer.Shutdown() - quicPeerServer.Shutdown() + http3PeerServer.Shutdown() + log.Infoln("done shutting down?") if reflectorServer != nil { reflectorServer.Shutdown() } diff --git a/go.mod b/go.mod index acedd1e..26f9f98 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,10 @@ require ( github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d github.com/davecgh/go-spew v1.1.1 github.com/go-sql-driver/mysql v1.4.1 - github.com/golang/protobuf v1.3.2 + github.com/golang/protobuf v1.4.0 github.com/google/btree v1.0.0 // indirect github.com/google/gops v0.3.7 + github.com/gorilla/mux v1.7.4 github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/hashicorp/memberlist v0.1.4 // indirect @@ -21,7 +22,7 @@ require ( github.com/lbryio/lbry.go/v2 v2.4.5 github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec - github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 + github.com/lucas-clemente/quic-go v0.17.1 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.2 github.com/sirupsen/logrus v1.4.2 @@ -30,7 +31,6 @@ require ( github.com/spf13/pflag v1.0.3 // indirect go.uber.org/atomic v1.5.1 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect - golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8 // indirect golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect google.golang.org/appengine v1.6.2 // indirect ) diff --git a/go.sum b/go.sum index 4548ae8..6aef50f 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,18 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= @@ -16,6 +25,7 @@ github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -32,11 +42,13 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -44,11 +56,17 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-ini/ini v1.38.2 h1:6Hl/z3p3iFkA0dlDfzYxuFuUGD+kaweypF6btsR2/Q4= @@ -62,10 +80,14 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= @@ -73,6 +95,12 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= @@ -81,8 +109,16 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw= github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000= @@ -93,6 +129,8 @@ github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A= github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk= @@ -103,6 +141,8 @@ github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTM github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -140,6 +180,7 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7V github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -151,6 +192,8 @@ github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/ github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY= github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jteeuwen/go-bindata v3.0.7+incompatible h1:91Uy4d9SYVr1kyTJ15wJsog+esAZZl7JmEfTkwmhJts= github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -169,6 +212,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0= @@ -192,8 +236,9 @@ github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiV github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c= github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 h1:tbuodUh2vuhOVZAdW3NEUvosFHUMJwUNl7jk/VSEiwc= -github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= +github.com/lucas-clemente/quic-go v0.17.1 h1:ezsH76xpn6hKugfsXUy6voIJBFmAOwnM/Oy9F4b/n+M= +github.com/lucas-clemente/quic-go v0.17.1/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns= @@ -202,8 +247,11 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcX github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA= -github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/marten-seemann/qpack v0.1.0 h1:/0M7lkda/6mus9B8u34Asqm8ZhHAAt9Ho0vniNuVSVg= +github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= +github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ= +github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= @@ -211,6 +259,7 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -221,6 +270,10 @@ github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675 h1:/rdJjIiK github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0= github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA= @@ -229,9 +282,14 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= +github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -247,16 +305,20 @@ github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6J github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2 h1:pZcZXvSLkiJfwtodlUyIbG0wSdFprYDwd4lIWJRYV+k= github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -276,6 +338,28 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t4 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw= github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -285,6 +369,8 @@ github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUr github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spf13/afero v1.1.1 h1:Lt3ihYMlE+lreX1GS4Qw4ZsNpYQLxIXKBTEOXm3nt6I= github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= @@ -307,11 +393,14 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d h1:gI4/tqP6lCY5k6Sg+4k9qSoBXmPwG+xXgMpK7jivD4M= github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q= github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dlEnaQ9Km5OXlK4zg= @@ -322,24 +411,32 @@ github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 h1:YdYsPAZ2pC6Tow/n github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM= go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -352,10 +449,15 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -364,9 +466,14 @@ golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqa golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -375,8 +482,9 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -384,22 +492,27 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw= golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8 h1:JA8d3MPx/IToSyXZG/RhwYEtfrKO1Fxrqe8KrkiLXKM= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U= @@ -409,17 +522,36 @@ golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.2 h1:j8RI1yW0SkI+paT6uGwMlrMI/6zwYA6/CFil8rxOzGI= google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -427,6 +559,7 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gorp.v1 v1.7.1 h1:GBB9KrWRATQZh95HJyVGUZrWwOPswitEYEyqlK8JbAA= gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8= gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -441,8 +574,14 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= rsc.io/goversion v1.0.0 h1:/IhXBiai89TyuerPquiZZ39IQkTfAUbZB2awsyYZ/2c= rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/peer/http3/client.go b/peer/http3/client.go new file mode 100644 index 0000000..ee3b97d --- /dev/null +++ b/peer/http3/client.go @@ -0,0 +1,97 @@ +package http3 + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "net/http" + "time" + + "github.com/lbryio/reflector.go/store" + "github.com/lucas-clemente/quic-go/http3" + log "github.com/sirupsen/logrus" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" +) + +// ErrBlobExists is a default error for when a blob already exists on the reflector server. +var ErrBlobExists = errors.Base("blob exists on server") + +// Client is an instance of a client connected to a server. +type Client struct { + Timeout time.Duration + conn *http.Client + roundTripper *http3.RoundTripper + ServerAddr string +} + +// Close closes the connection with the client. +func (c *Client) Close() error { + return c.roundTripper.Close() +} + +// GetStream gets a stream +func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) { + var sd stream.SDBlob + + b, err := c.GetBlob(sdHash) + if err != nil { + return nil, err + } + + err = sd.FromBlob(b) + if err != nil { + return nil, err + } + + s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob + s[0] = b + + for i := 0; i < len(sd.BlobInfos)-1; i++ { + s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash)) + if err != nil { + return nil, err + } + } + + return s, nil +} + +// HasBlob checks if the blob is available +func (c *Client) HasBlob(hash string) (bool, error) { + resp, err := c.conn.Get(fmt.Sprintf("https://%s/has/%s", c.ServerAddr, hash)) + if err != nil { + return false, errors.Err(err) + } + if resp.StatusCode == http.StatusOK { + return true, nil + } + if resp.StatusCode == http.StatusNotFound { + return false, nil + } + return false, errors.Err("non 200 status code returned: %d", resp.StatusCode) +} + +// GetBlob gets a blob +func (c *Client) GetBlob(hash string) (stream.Blob, error) { + resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s", c.ServerAddr, hash)) + if err != nil { + return nil, errors.Err(err) + } + if resp.StatusCode == http.StatusNotFound { + return nil, errors.Err(store.ErrBlobNotFound) + } + if resp.StatusCode != http.StatusOK { + return nil, errors.Err("non 200 status code returned: %d", resp.StatusCode) + } + body := &bytes.Buffer{} + _, err = io.Copy(body, resp.Body) + if err != nil { + return nil, errors.Err(err) + } + log.Infof("downloaded %s with HTTP3", hash) + + return body.Bytes(), nil +} diff --git a/peer/http3/server.go b/peer/http3/server.go new file mode 100644 index 0000000..a497b63 --- /dev/null +++ b/peer/http3/server.go @@ -0,0 +1,162 @@ +package http3 + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "math/big" + "net/http" + + "github.com/gorilla/mux" + "github.com/lbryio/lbry.go/extras/stop" + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/store" + "github.com/lucas-clemente/quic-go" + "github.com/lucas-clemente/quic-go/http3" + log "github.com/sirupsen/logrus" +) + +// Server is an instance of a peer server that houses the listener and store. +type Server struct { + store store.BlobStore + + grp *stop.Group +} + +// NewServer returns an initialized Server pointer. +func NewServer(store store.BlobStore) *Server { + return &Server{ + store: store, + grp: stop.New(), + } +} + +// Shutdown gracefully shuts down the peer server. +func (s *Server) Shutdown() { + log.Debug("shutting down peer server") + s.grp.StopAndWait() + log.Debug("peer server stopped") +} +func (s *Server) logError(e error) { + if e == nil { + return + } + shouldLog := metrics.TrackError(metrics.DirectionDownload, e) + if shouldLog { + log.Errorln(errors.FullTrace(e)) + } +} + +type availabilityResponse struct { + LbrycrdAddress string `json:"lbrycrd_address"` + IsAvailable bool `json:"is_available"` +} + +// Start starts the server listener to handle connections. +func (s *Server) Start(address string) error { + log.Println("HTTP3 peer listening on " + address) + quicConf := &quic.Config{} + r := mux.NewRouter() + r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + requestedBlob := vars["hash"] + blob, err := s.store.Get(requestedBlob) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + _, err = w.Write(blob) + if err != nil { + s.logError(err) + } + }) + r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + requestedBlob := vars["hash"] + blobExists, err := s.store.Has(requestedBlob) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + s.logError(err) + return + } + if !blobExists { + w.WriteHeader(http.StatusNotFound) + } + // LbrycrdAddress to be used when paying for data. Not implemented yet. + const LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct" + resp, err := json.Marshal(availabilityResponse{ + LbrycrdAddress: LbrycrdAddress, + IsAvailable: blobExists, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + s.logError(err) + return + } + _, err = w.Write(resp) + if err != nil { + s.logError(err) + } + }) + server := http3.Server{ + Server: &http.Server{ + Handler: r, + Addr: address, + TLSConfig: generateTLSConfig(), + }, + QuicConfig: quicConf, + } + + go s.listenForShutdown(&server) + s.grp.Add(1) + go func() { + s.listenAndServe(&server) + s.grp.Done() + }() + + return nil +} + +// Setup a bare-bones TLS config for the server +func generateTLSConfig() *tls.Config { + key, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + panic(err) + } + template := x509.Certificate{SerialNumber: big.NewInt(1)} + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + panic(err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + panic(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + NextProtos: []string{"quic-echo-example"}, + } +} + +func (s *Server) listenAndServe(server *http3.Server) { + err := server.ListenAndServe() + if err != nil { + log.Errorln(errors.FullTrace(err)) + } +} + +func (s *Server) listenForShutdown(listener *http3.Server) { + <-s.grp.Ch() + err := listener.Close() + if err != nil { + log.Error("error closing listener for peer server - ", err) + } +} diff --git a/peer/quic/store.go b/peer/http3/store.go similarity index 71% rename from peer/quic/store.go rename to peer/http3/store.go index 9300c4a..3de9a8a 100644 --- a/peer/quic/store.go +++ b/peer/http3/store.go @@ -1,10 +1,15 @@ -package quic +package http3 import ( + "crypto/tls" + "crypto/x509" + "net/http" "time" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lucas-clemente/quic-go" + "github.com/lucas-clemente/quic-go/http3" ) // Store is a blob store that gets blobs from a peer. @@ -25,8 +30,26 @@ func NewStore(opts StoreOpts) *Store { } func (p *Store) getClient() (*Client, error) { - c := &Client{Timeout: p.opts.Timeout} - err := c.Connect(p.opts.Address) + var qconf quic.Config + pool, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + roundTripper := &http3.RoundTripper{ + TLSClientConfig: &tls.Config{ + RootCAs: pool, + InsecureSkipVerify: true, + }, + QuicConfig: &qconf, + } + hclient := &http.Client{ + Transport: roundTripper, + } + c := &Client{ + conn: hclient, + roundTripper: roundTripper, + ServerAddr: p.opts.Address, + } return c, errors.Prefix("connection error", err) } diff --git a/peer/quic/client.go b/peer/quic/client.go deleted file mode 100644 index 83f50c0..0000000 --- a/peer/quic/client.go +++ /dev/null @@ -1,216 +0,0 @@ -package quic - -import ( - "bufio" - "crypto/tls" - "encoding/hex" - "encoding/json" - "io" - "time" - - "github.com/lucas-clemente/quic-go" - - "github.com/lbryio/reflector.go/store" - - "github.com/lbryio/lbry.go/v2/extras/errors" - "github.com/lbryio/lbry.go/v2/stream" - - log "github.com/sirupsen/logrus" -) - -// ErrBlobExists is a default error for when a blob already exists on the reflector server. -var ErrBlobExists = errors.Base("blob exists on server") - -// Client is an instance of a client connected to a server. -type Client struct { - Timeout time.Duration - conn quic.Session - stream quic.Stream - buf *bufio.Reader - connected bool -} - -// Connect connects to a specific clients and errors if it cannot be contacted. -func (c *Client) Connect(address string) error { - var err error - if c.Timeout == 0 { - c.Timeout = 5 * time.Second - } - tlsConf := &tls.Config{ - InsecureSkipVerify: true, - NextProtos: []string{"quic-echo-example"}, - } - - c.conn, err = quic.DialAddr(address, tlsConf, nil) - if err != nil { - return err - } - c.connected = true - c.stream, err = c.conn.OpenStream() - if err != nil { - return errors.Err(err) - } - c.buf = bufio.NewReader(c.stream) - return nil -} - -// Close closes the connection with the client. -func (c *Client) Close() error { - c.connected = false - return c.conn.Close() -} - -// GetStream gets a stream -func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) { - if !c.connected { - return nil, errors.Err("not connected") - } - - var sd stream.SDBlob - - b, err := c.GetBlob(sdHash) - if err != nil { - return nil, err - } - - err = sd.FromBlob(b) - if err != nil { - return nil, err - } - - s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob - s[0] = b - - for i := 0; i < len(sd.BlobInfos)-1; i++ { - s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash)) - if err != nil { - return nil, err - } - } - - return s, nil -} - -// HasBlob checks if the blob is available -func (c *Client) HasBlob(hash string) (bool, error) { - if !c.connected { - return false, errors.Err("not connected") - } - - sendRequest, err := json.Marshal(availabilityRequest{ - RequestedBlobs: []string{hash}, - }) - if err != nil { - return false, err - } - - err = c.write(sendRequest) - if err != nil { - return false, err - } - - var resp availabilityResponse - err = c.read(&resp) - if err != nil { - return false, err - } - - for _, h := range resp.AvailableBlobs { - if h == hash { - return true, nil - } - } - - return false, nil -} - -// GetBlob gets a blob -func (c *Client) GetBlob(hash string) (stream.Blob, error) { - if !c.connected { - return nil, errors.Err("not connected") - } - - sendRequest, err := json.Marshal(blobRequest{ - RequestedBlob: hash, - }) - if err != nil { - return nil, err - } - - err = c.write(sendRequest) - if err != nil { - return nil, err - } - - var resp blobResponse - err = c.read(&resp) - if err != nil { - return nil, err - } - - if resp.IncomingBlob.Error != "" { - if resp.IncomingBlob.Error == store.ErrBlobNotFound.Error() { - return nil, errors.Err(store.ErrBlobNotFound) - } - return nil, errors.Prefix(hash[:8], resp.IncomingBlob.Error) - } - if resp.IncomingBlob.BlobHash != hash { - return nil, errors.Prefix(hash[:8], "blob hash in response does not match requested hash") - } - if resp.IncomingBlob.Length <= 0 { - return nil, errors.Prefix(hash[:8], "length reported as <= 0") - } - - log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr()) - - blob, err := c.readRawBlob(resp.IncomingBlob.Length) - if err != nil { - return nil, err - } - - return stream.Blob(blob), nil -} - -func (c *Client) read(v interface{}) error { - err := c.stream.SetReadDeadline(time.Now().Add(c.Timeout)) - if err != nil { - return errors.Err(err) - } - - m, err := readNextMessage(c.buf) - if err != nil { - return err - } - - log.Debugf("read %d bytes from %s", len(m), c.conn.RemoteAddr()) - - err = json.Unmarshal(m, v) - return errors.Err(err) -} - -func (c *Client) readRawBlob(blobSize int) ([]byte, error) { - err := c.stream.SetReadDeadline(time.Now().Add(c.Timeout)) - if err != nil { - return nil, errors.Err(err) - } - - blob := make([]byte, blobSize) - n, err := io.ReadFull(c.buf, blob) - log.Debugf("read %d bytes from %s", n, c.conn.RemoteAddr()) - return blob, errors.Err(err) -} - -func (c *Client) write(b []byte) error { - err := c.stream.SetWriteDeadline(time.Now().Add(c.Timeout)) - if err != nil { - return errors.Err(err) - } - - log.Debugf("writing %d bytes to %s", len(b), c.conn.RemoteAddr()) - - n, err := c.stream.Write(b) - if err == nil && n != len(b) { - err = io.ErrShortWrite - } - return errors.Err(err) -} diff --git a/peer/quic/server.go b/peer/quic/server.go deleted file mode 100644 index 601e904..0000000 --- a/peer/quic/server.go +++ /dev/null @@ -1,427 +0,0 @@ -package quic - -import ( - "bufio" - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "encoding/json" - "encoding/pem" - "io" - "math/big" - "strings" - "time" - - "github.com/lucas-clemente/quic-go" - - "github.com/lbryio/reflector.go/internal/metrics" - "github.com/lbryio/reflector.go/reflector" - "github.com/lbryio/reflector.go/store" - - "github.com/lbryio/lbry.go/v2/extras/errors" - "github.com/lbryio/lbry.go/v2/extras/stop" - "github.com/lbryio/lbry.go/v2/stream" - - log "github.com/sirupsen/logrus" -) - -const ( - // DefaultPort is the port the peer server listens on if not passed in. - DefaultPort = 3333 - // LbrycrdAddress to be used when paying for data. Not implemented yet. - LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct" -) - -// Server is an instance of a peer server that houses the listener and store. -type Server struct { - store store.BlobStore - closed bool - - grp *stop.Group -} - -// NewServer returns an initialized Server pointer. -func NewServer(store store.BlobStore) *Server { - return &Server{ - store: store, - grp: stop.New(), - } -} - -// Shutdown gracefully shuts down the peer server. -func (s *Server) Shutdown() { - log.Debug("shutting down peer server") - s.grp.StopAndWait() - log.Debug("peer server stopped") -} - -// Start starts the server listener to handle connections. -func (s *Server) Start(address string) error { - log.Println("QUIC peer listening on " + address) - - l, err := quic.ListenAddr(address, generateTLSConfig(), nil) - if err != nil { - return err - } - - go s.listenForShutdown(l) - s.grp.Add(1) - go func() { - s.listenAndServe(l) - s.grp.Done() - }() - - return nil -} - -func (s *Server) listenForShutdown(listener quic.Listener) { - <-s.grp.Ch() - s.closed = true - err := listener.Close() - if err != nil { - log.Error("error closing listener for peer server - ", err) - } -} - -func (s *Server) listenAndServe(listener quic.Listener) { - for { - conn, err := listener.Accept() - if err != nil { - if s.closed { - return - } - log.Error(errors.Prefix("accepting conn", err)) - } else { - s.grp.Add(1) - go func() { - s.handleConnection(conn) - s.grp.Done() - }() - } - } -} - -func (s *Server) handleConnection(conn quic.Session) { - defer func() { - if err := conn.Close(); err != nil { - log.Error(errors.Prefix("closing peer conn", err)) - } - }() - - timeoutDuration := 1 * time.Minute - stream, err := conn.AcceptStream() - if err != nil { - log.Error(err) - return - } - buf := bufio.NewReader(stream) - for { - var request []byte - var response []byte - - err := stream.SetReadDeadline(time.Now().Add(timeoutDuration)) - if err != nil { - log.Error(errors.FullTrace(err)) - } - - request, err = readNextMessage(buf) - if err != nil { - if err != io.EOF { - s.logError(err) - } - return - } - - err = stream.SetReadDeadline(time.Time{}) - if err != nil { - log.Error(errors.FullTrace(err)) - } - - response, err = s.handleCompositeRequest(request) - if err != nil { - log.Error(errors.FullTrace(err)) - return - } - - err = stream.SetWriteDeadline(time.Now().Add(timeoutDuration)) - if err != nil { - log.Error(errors.FullTrace(err)) - } - - n, err := stream.Write(response) - if err != nil { - if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset - s.logError(err) - } - return - } else if n != len(response) { - log.Errorln(io.ErrShortWrite) - return - } - - err = stream.SetWriteDeadline(time.Time{}) - if err != nil { - log.Error(errors.FullTrace(err)) - } - } -} - -func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) { - var request availabilityRequest - err := json.Unmarshal(data, &request) - if err != nil { - return nil, errors.Err(err) - } - - availableBlobs := []string{} - for _, blobHash := range request.RequestedBlobs { - exists, err := s.store.Has(blobHash) - if err != nil { - return nil, err - } - if exists { - availableBlobs = append(availableBlobs, blobHash) - } - } - - return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs}) -} - -//func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) { -// var request paymentRateRequest -// err := json.Unmarshal(data, &request) -// if err != nil { -// return nil, err -// } -// -// offerReply := paymentRateAccepted -// if request.BlobDataPaymentRate < 0 { -// offerReply = paymentRateTooLow -// } -// -// return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply}) -//} -// -//func (s *Server) handleBlobRequest(data []byte) ([]byte, error) { -// var request blobRequest -// err := json.Unmarshal(data, &request) -// if err != nil { -// return nil, err -// } -// -// log.Debugln("Sending blob " + request.RequestedBlob[:8]) -// -// blob, err := s.store.Get(request.RequestedBlob) -// if err != nil { -// return nil, err -// } -// -// response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{ -// BlobHash: reflector.BlobHash(blob), -// Length: len(blob), -// }}) -// if err != nil { -// return nil, err -// } -// -// return append(response, blob...), nil -//} - -func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) { - var request compositeRequest - err := json.Unmarshal(data, &request) - if err != nil { - return nil, errors.Err(err) - } - - response := compositeResponse{ - LbrycrdAddress: LbrycrdAddress, - } - - if len(request.RequestedBlobs) > 0 { - var availableBlobs []string - for _, blobHash := range request.RequestedBlobs { - exists, err := s.store.Has(blobHash) - if err != nil { - return nil, err - } - if exists { - availableBlobs = append(availableBlobs, blobHash) - } - } - response.AvailableBlobs = availableBlobs - } - - response.BlobDataPaymentRate = paymentRateAccepted - if request.BlobDataPaymentRate < 0 { - response.BlobDataPaymentRate = paymentRateTooLow - } - - var blob []byte - if request.RequestedBlob != "" { - if len(request.RequestedBlob) != stream.BlobHashHexLength { - return nil, errors.Err("Invalid blob hash length") - } - - log.Debugln("Sending blob " + request.RequestedBlob[:8]) - - blob, err = s.store.Get(request.RequestedBlob) - if errors.Is(err, store.ErrBlobNotFound) { - response.IncomingBlob = incomingBlob{ - Error: err.Error(), - } - } else if err != nil { - return nil, err - } else { - response.IncomingBlob = incomingBlob{ - BlobHash: reflector.BlobHash(blob), - Length: len(blob), - } - metrics.BlobDownloadCount.Inc() - } - } - - respData, err := json.Marshal(response) - if err != nil { - return nil, err - } - - return append(respData, blob...), nil -} - -func (s *Server) logError(e error) { - if e == nil { - return - } - shouldLog := metrics.TrackError(metrics.DirectionDownload, e) - if shouldLog { - log.Errorln(errors.FullTrace(e)) - } -} - -func readNextMessage(buf *bufio.Reader) ([]byte, error) { - msg := make([]byte, 0) - eof := false - - for { - chunk, err := buf.ReadBytes('}') - if err != nil { - if err != io.EOF { - //log.Errorln("readBytes error:", err) // logged by caller - return msg, err - } - eof = true - } - - //log.Debugln("got", len(chunk), "bytes.") - //spew.Dump(chunk) - - if len(chunk) > 0 { - msg = append(msg, chunk...) - - if len(msg) > maxRequestSize { - return msg, errRequestTooLarge - } - - // yes, this is how the peer protocol knows when the request finishes - if reflector.IsValidJSON(msg) { - break - } - } - - if eof { - break - } - } - - //log.Debugln("total size:", len(request)) - //if len(request) > 0 { - // spew.Dump(request) - //} - - if len(msg) == 0 && eof { - return nil, io.EOF - } - - return msg, nil -} - -const ( - maxRequestSize = 64 * (2 ^ 10) // 64kb - paymentRateAccepted = "RATE_ACCEPTED" - paymentRateTooLow = "RATE_TOO_LOW" - //ToDo: paymentRateUnset is not used but exists in the protocol. - //paymentRateUnset = "RATE_UNSET" -) - -var errRequestTooLarge = errors.Base("request is too large") - -type availabilityRequest struct { - LbrycrdAddress bool `json:"lbrycrd_address"` - RequestedBlobs []string `json:"requested_blobs"` -} - -type availabilityResponse struct { - LbrycrdAddress string `json:"lbrycrd_address"` - AvailableBlobs []string `json:"available_blobs"` -} - -type paymentRateRequest struct { - BlobDataPaymentRate float64 `json:"blob_data_payment_rate"` -} - -type paymentRateResponse struct { - BlobDataPaymentRate string `json:"blob_data_payment_rate"` -} - -type blobRequest struct { - RequestedBlob string `json:"requested_blob"` -} - -type incomingBlob struct { - Error string `json:"error,omitempty"` - BlobHash string `json:"blob_hash"` - Length int `json:"length"` -} -type blobResponse struct { - IncomingBlob incomingBlob `json:"incoming_blob"` -} - -type compositeRequest struct { - LbrycrdAddress bool `json:"lbrycrd_address"` - RequestedBlobs []string `json:"requested_blobs"` - BlobDataPaymentRate float64 `json:"blob_data_payment_rate"` - RequestedBlob string `json:"requested_blob"` -} - -type compositeResponse struct { - LbrycrdAddress string `json:"lbrycrd_address,omitempty"` - AvailableBlobs []string `json:"available_blobs,omitempty"` - BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"` - IncomingBlob incomingBlob `json:"incoming_blob,omitempty"` -} - -// Setup a bare-bones TLS config for the server -func generateTLSConfig() *tls.Config { - key, err := rsa.GenerateKey(rand.Reader, 1024) - if err != nil { - panic(err) - } - template := x509.Certificate{SerialNumber: big.NewInt(1)} - certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) - if err != nil { - panic(err) - } - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) - - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - panic(err) - } - return &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - NextProtos: []string{"quic-echo-example"}, - } -} diff --git a/reflector/blocklist.go b/reflector/blocklist.go index 42aa2ce..7e3adf5 100644 --- a/reflector/blocklist.go +++ b/reflector/blocklist.go @@ -95,12 +95,6 @@ func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) { node := wallet.NewNode() defer node.Shutdown() err := node.Connect([]string{ - "spv4.lbry.com:50001", - "spv5.lbry.com:50001", - "spv7.lbry.com:50001", - "spv9.lbry.com:50001", - "spv15.lbry.com:50001", - "spv17.lbry.com:50001", "spv25.lbry.com:50001", }, nil) if err != nil { -- 2.45.2 From 09c7718f30340bcb0aae4ea706f1e5fd6141262f Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 30 Jun 2020 01:14:52 +0200 Subject: [PATCH 11/18] refactor code --- cmd/reflector.go | 50 +++++++++++++++++++++--------------------- peer/http3/client.go | 9 ++------ peer/http3/server.go | 14 +++++++----- peer/http3/store.go | 10 ++++----- reflector/blocklist.go | 3 +++ 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index ebdbe5b..5c900a3 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -7,12 +7,11 @@ import ( "syscall" "time" - "github.com/lbryio/reflector.go/peer" - "github.com/lbryio/reflector.go/peer/http3" - "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/meta" + "github.com/lbryio/reflector.go/peer" + "github.com/lbryio/reflector.go/peer/http3" "github.com/lbryio/reflector.go/reflector" "github.com/lbryio/reflector.go/store" @@ -21,14 +20,14 @@ import ( ) var reflectorCmdCacheDir string -var peerPort int -var quicPeerPort int -var reflectorPort int +var tcpPeerPort int +var http3PeerPort int +var receiverPort int var metricsPort int var disableUploads bool -var reflectorServerAddress string -var reflectorServerPort string -var reflectorServerProtocol string +var proxyAddress string +var proxyPort string +var proxyProtocol string var useDB bool func init() { @@ -37,13 +36,13 @@ func init() { Short: "Run reflector server", Run: reflectorCmd, } - cmd.Flags().StringVar(&reflectorCmdCacheDir, "cache", "", "Enable disk cache for blobs. Store them in this directory") - cmd.Flags().StringVar(&reflectorServerAddress, "reflector-server-address", "", "address of another reflector server where blobs are fetched from") - cmd.Flags().StringVar(&reflectorServerPort, "reflector-server-port", "5567", "port of another reflector server where blobs are fetched from") - cmd.Flags().StringVar(&reflectorServerProtocol, "reflector-server-protocol", "tcp", "protocol used to fetch blobs from another reflector server (tcp/udp)") - cmd.Flags().IntVar(&peerPort, "peer-port", 5567, "The port reflector will distribute content from") - cmd.Flags().IntVar(&quicPeerPort, "quic-peer-port", 5568, "The port reflector will distribute content from over QUIC protocol") - cmd.Flags().IntVar(&reflectorPort, "reflector-port", 5566, "The port reflector will receive content from") + cmd.Flags().StringVar(&reflectorCmdCacheDir, "cache", "", "if specified, the path where blobs should be cached (disabled when left empty)") + cmd.Flags().StringVar(&proxyAddress, "proxy-address", "", "address of another reflector server where blobs are fetched from") + cmd.Flags().StringVar(&proxyPort, "proxy-port", "5567", "port of another reflector server where blobs are fetched from") + cmd.Flags().StringVar(&proxyProtocol, "proxy-protocol", "http3", "protocol used to fetch blobs from another reflector server (tcp/http3)") + cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from") + cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol") + cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from") cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics") cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server") cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not") @@ -54,18 +53,20 @@ func reflectorCmd(cmd *cobra.Command, args []string) { log.Printf("reflector version %s, built %s", meta.Version, meta.BuildTime.Format(time.RFC3339)) var blobStore store.BlobStore - if reflectorServerAddress != "" { - switch reflectorServerProtocol { + if proxyAddress != "" { + switch proxyProtocol { case "tcp": blobStore = peer.NewStore(peer.StoreOpts{ - Address: reflectorServerAddress + ":" + reflectorServerPort, + Address: proxyAddress + ":" + proxyPort, Timeout: 30 * time.Second, }) - case "udp": + case "http3": blobStore = http3.NewStore(http3.StoreOpts{ - Address: reflectorServerAddress + ":" + reflectorServerPort, + Address: proxyAddress + ":" + proxyPort, Timeout: 30 * time.Second, }) + default: + log.Fatalf("specified protocol is not recognized: %s", proxyProtocol) } } else { blobStore = store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName) @@ -87,7 +88,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) { reflectorServer.Timeout = 3 * time.Minute reflectorServer.EnableBlocklist = true - err = reflectorServer.Start(":" + strconv.Itoa(reflectorPort)) + err = reflectorServer.Start(":" + strconv.Itoa(receiverPort)) if err != nil { log.Fatal(err) } @@ -102,13 +103,13 @@ func reflectorCmd(cmd *cobra.Command, args []string) { } peerServer := peer.NewServer(blobStore) - err = peerServer.Start(":" + strconv.Itoa(peerPort)) + err = peerServer.Start(":" + strconv.Itoa(tcpPeerPort)) if err != nil { log.Fatal(err) } http3PeerServer := http3.NewServer(blobStore) - err = http3PeerServer.Start(":" + strconv.Itoa(quicPeerPort)) + err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort)) if err != nil { log.Fatal(err) } @@ -122,7 +123,6 @@ func reflectorCmd(cmd *cobra.Command, args []string) { metricsServer.Shutdown() peerServer.Shutdown() http3PeerServer.Shutdown() - log.Infoln("done shutting down?") if reflectorServer != nil { reflectorServer.Shutdown() } diff --git a/peer/http3/client.go b/peer/http3/client.go index ee3b97d..ca3d556 100644 --- a/peer/http3/client.go +++ b/peer/http3/client.go @@ -9,15 +9,12 @@ import ( "time" "github.com/lbryio/reflector.go/store" - "github.com/lucas-clemente/quic-go/http3" - log "github.com/sirupsen/logrus" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" -) -// ErrBlobExists is a default error for when a blob already exists on the reflector server. -var ErrBlobExists = errors.Base("blob exists on server") + "github.com/lucas-clemente/quic-go/http3" +) // Client is an instance of a client connected to a server. type Client struct { @@ -91,7 +88,5 @@ func (c *Client) GetBlob(hash string) (stream.Blob, error) { if err != nil { return nil, errors.Err(err) } - log.Infof("downloaded %s with HTTP3", hash) - return body.Bytes(), nil } diff --git a/peer/http3/server.go b/peer/http3/server.go index a497b63..c04026e 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -10,11 +10,13 @@ import ( "math/big" "net/http" - "github.com/gorilla/mux" - "github.com/lbryio/lbry.go/extras/stop" - "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/store" + + "github.com/lbryio/lbry.go/extras/stop" + "github.com/lbryio/lbry.go/v2/extras/errors" + + "github.com/gorilla/mux" "github.com/lucas-clemente/quic-go" "github.com/lucas-clemente/quic-go/http3" log "github.com/sirupsen/logrus" @@ -23,8 +25,7 @@ import ( // Server is an instance of a peer server that houses the listener and store. type Server struct { store store.BlobStore - - grp *stop.Group + grp *stop.Group } // NewServer returns an initialized Server pointer. @@ -41,6 +42,7 @@ func (s *Server) Shutdown() { s.grp.StopAndWait() log.Debug("peer server stopped") } + func (s *Server) logError(e error) { if e == nil { return @@ -142,7 +144,7 @@ func generateTLSConfig() *tls.Config { } return &tls.Config{ Certificates: []tls.Certificate{tlsCert}, - NextProtos: []string{"quic-echo-example"}, + NextProtos: []string{"http3-reflector-server"}, } } diff --git a/peer/http3/store.go b/peer/http3/store.go index 3de9a8a..7d5e433 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -42,11 +42,11 @@ func (p *Store) getClient() (*Client, error) { }, QuicConfig: &qconf, } - hclient := &http.Client{ + connection := &http.Client{ Transport: roundTripper, } c := &Client{ - conn: hclient, + conn: connection, roundTripper: roundTripper, ServerAddr: p.opts.Address, } @@ -75,15 +75,15 @@ func (p *Store) Get(hash string) (stream.Blob, error) { // Put is not supported func (p *Store) Put(hash string, blob stream.Blob) error { - panic("PeerStore cannot put or delete blobs") + panic("http3Store cannot put or delete blobs") } // PutSD is not supported func (p *Store) PutSD(hash string, blob stream.Blob) error { - panic("PeerStore cannot put or delete blobs") + panic("http3Store cannot put or delete blobs") } // Delete is not supported func (p *Store) Delete(hash string) error { - panic("PeerStore cannot put or delete blobs") + panic("http3Store cannot put or delete blobs") } diff --git a/reflector/blocklist.go b/reflector/blocklist.go index 7e3adf5..ed2ba9c 100644 --- a/reflector/blocklist.go +++ b/reflector/blocklist.go @@ -96,6 +96,9 @@ func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) { defer node.Shutdown() err := node.Connect([]string{ "spv25.lbry.com:50001", + "spv26.lbry.com:50001", + "spv19.lbry.com:50001", + "spv14.lbry.com:50001", }, nil) if err != nil { return nil, errors.Err(err) -- 2.45.2 From fdcc41829ad110bad377b72f6aef096402ae12c2 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 1 Jul 2020 00:14:51 +0200 Subject: [PATCH 12/18] handshake changes --- internal/metrics/metrics.go | 2 +- peer/http3/client.go | 1 + peer/http3/server.go | 4 +++- peer/http3/store.go | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 94bac40..aa89dbb 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -53,7 +53,7 @@ func (s *Server) Start() { } func (s *Server) Shutdown() { - s.srv.Shutdown(context.Background()) + _ = s.srv.Shutdown(context.Background()) s.stop.StopAndWait() } diff --git a/peer/http3/client.go b/peer/http3/client.go index ca3d556..aba77c3 100644 --- a/peer/http3/client.go +++ b/peer/http3/client.go @@ -26,6 +26,7 @@ type Client struct { // Close closes the connection with the client. func (c *Client) Close() error { + c.conn.CloseIdleConnections() return c.roundTripper.Close() } diff --git a/peer/http3/server.go b/peer/http3/server.go index c04026e..3cc0155 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -9,6 +9,7 @@ import ( "encoding/pem" "math/big" "net/http" + "time" "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/store" @@ -61,13 +62,14 @@ type availabilityResponse struct { // Start starts the server listener to handle connections. func (s *Server) Start(address string) error { log.Println("HTTP3 peer listening on " + address) - quicConf := &quic.Config{} + quicConf := &quic.Config{HandshakeTimeout: 3 * time.Second} r := mux.NewRouter() r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) requestedBlob := vars["hash"] blob, err := s.store.Get(requestedBlob) if err != nil { + log.Errorln(errors.FullTrace(err)) http.Error(w, err.Error(), http.StatusBadRequest) return } diff --git a/peer/http3/store.go b/peer/http3/store.go index 7d5e433..cb68bb2 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -31,6 +31,7 @@ func NewStore(opts StoreOpts) *Store { func (p *Store) getClient() (*Client, error) { var qconf quic.Config + qconf.HandshakeTimeout = 3 * time.Second pool, err := x509.SystemCertPool() if err != nil { return nil, err -- 2.45.2 From 264390a2b2ba151abf146d121928d522e69377f2 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 1 Jul 2020 20:10:32 +0200 Subject: [PATCH 13/18] add debug code --- internal/metrics/metrics.go | 6 ++++++ peer/http3/server.go | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index aa89dbb..b14435c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -12,6 +12,7 @@ import ( ee "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/extras/stop" + "github.com/lbryio/reflector.go/store" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -84,6 +85,7 @@ const ( errHashMismatch = "hash_mismatch" errZeroByteBlob = "zero_byte_blob" errInvalidCharacter = "invalid_character" + errBlobNotFound = "blob_not_found" errNoErr = "no_error" errOther = "other" ) @@ -146,6 +148,8 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a errType = errUnexpectedEOFStr } else if errors.Is(e, syscall.EPIPE) { errType = errEPipe + } else if errors.Is(e, store.ErrBlobNotFound) { + errType = errBlobNotFound } else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer // I believe this is the same as EPipe when direction == "download", but not for upload errType = errWriteBrokenPipe @@ -156,6 +160,8 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a errType = errBlobTooBig } else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") { errType = errHashMismatch + } else if strings.Contains(err.Error(), "blob not found") { + errType = errBlobNotFound } else if strings.Contains(err.Error(), "0-byte blob received") { errType = errZeroByteBlob } else if strings.Contains(err.Error(), "invalid character") { diff --git a/peer/http3/server.go b/peer/http3/server.go index 3cc0155..c9bab5e 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -7,6 +7,7 @@ import ( "crypto/x509" "encoding/json" "encoding/pem" + "fmt" "math/big" "net/http" "time" @@ -69,7 +70,8 @@ func (s *Server) Start(address string) error { requestedBlob := vars["hash"] blob, err := s.store.Get(requestedBlob) if err != nil { - log.Errorln(errors.FullTrace(err)) + fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err)) + s.logError(err) http.Error(w, err.Error(), http.StatusBadRequest) return } -- 2.45.2 From 41d758ef5c7af6563e628bb1b3e7fd9ed7ff0e1d Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Wed, 1 Jul 2020 13:23:44 -0400 Subject: [PATCH 14/18] test wallet server connection --- wallet/network.go | 3 +++ wallet/transport.go | 57 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/wallet/network.go b/wallet/network.go index 7ff05af..20e86ba 100644 --- a/wallet/network.go +++ b/wallet/network.go @@ -76,6 +76,9 @@ func (n *Node) Connect(addrs []string, config *tls.Config) error { if err == nil { break } + if errors.Is(err, ErrTimeout) { + continue + } if e, ok := err.(*net.OpError); ok && e.Err.Error() == "no such host" { // net.errNoSuchHost is not exported, so we have to string-match continue diff --git a/wallet/transport.go b/wallet/transport.go index 60169a5..83a0dab 100644 --- a/wallet/transport.go +++ b/wallet/transport.go @@ -5,9 +5,12 @@ package wallet import ( "bufio" "crypto/tls" + "encoding/json" + "fmt" "net" "time" + "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/v2/extras/stop" log "github.com/sirupsen/logrus" @@ -50,15 +53,31 @@ func NewTransport(addr string, config *tls.Config) (*TCPTransport, error) { t.grp.Add(1) go func() { - t.grp.Done() + defer t.grp.Done() t.listen() }() + err = t.test() + if err != nil { + t.grp.StopAndWait() + return nil, errors.Prefix(addr, err) + } + return t, nil } const delimiter = byte('\n') +func (t *TCPTransport) Send(body []byte) error { + log.Debugf("%s <- %s", t.conn.RemoteAddr(), body) + _, err := t.conn.Write(body) + return err +} + +func (t *TCPTransport) Responses() <-chan []byte { return t.responses } +func (t *TCPTransport) Errors() <-chan error { return t.errors } +func (t *TCPTransport) Shutdown() { t.grp.StopAndWait() } + func (t *TCPTransport) listen() { reader := bufio.NewReader(t.conn) for { @@ -74,12 +93,6 @@ func (t *TCPTransport) listen() { } } -func (t *TCPTransport) Send(body []byte) error { - log.Debugf("%s <- %s", t.conn.RemoteAddr(), body) - _, err := t.conn.Write(body) - return err -} - func (t *TCPTransport) error(err error) { select { case t.errors <- err: @@ -87,11 +100,33 @@ func (t *TCPTransport) error(err error) { } } -func (t *TCPTransport) Responses() <-chan []byte { return t.responses } -func (t *TCPTransport) Errors() <-chan error { return t.errors } +func (t *TCPTransport) test() error { + err := t.Send([]byte(`{"id":1,"method":"server.version"}` + "\n")) + if err != nil { + return errors.Err(err) + } -func (t *TCPTransport) Shutdown() { - t.grp.StopAndWait() + var data []byte + select { + case data = <-t.Responses(): + case <-time.Tick(1 * time.Second): + return errors.Err(ErrTimeout) + } + + var response struct { + Error struct { + Message string `json:"message"` + } `json:"error"` + } + + err = json.Unmarshal(data, &response) + if err != nil { + return errors.Err(err) + } + if response.Error.Message != "" { + return fmt.Errorf(response.Error.Message) + } + return nil } func (t *TCPTransport) close() { -- 2.45.2 From df4f42db82136299fe6a6926def71393c68af2ee Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Wed, 1 Jul 2020 14:42:23 -0400 Subject: [PATCH 15/18] successfully shut down wallet server --- peer/http3/server.go | 4 +-- reflector/blocklist.go | 58 ++++++++++++++++++++++++++++++------------ wallet/network.go | 14 ++++++++++ 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/peer/http3/server.go b/peer/http3/server.go index c9bab5e..6f18db3 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -40,9 +40,9 @@ func NewServer(store store.BlobStore) *Server { // Shutdown gracefully shuts down the peer server. func (s *Server) Shutdown() { - log.Debug("shutting down peer server") + log.Debug("shutting down http3 peer server") s.grp.StopAndWait() - log.Debug("peer server stopped") + log.Debug("http3 peer server stopped") } func (s *Server) logError(e error) { diff --git a/reflector/blocklist.go b/reflector/blocklist.go index ed2ba9c..9abfc4a 100644 --- a/reflector/blocklist.go +++ b/reflector/blocklist.go @@ -12,6 +12,7 @@ import ( "github.com/lbryio/reflector.go/wallet" "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/extras/stop" log "github.com/sirupsen/logrus" ) @@ -19,21 +20,28 @@ import ( const blocklistURL = "https://api.lbry.com/file/list_blocked" func (s *Server) enableBlocklist(b store.Blocklister) { - // TODO: updateBlocklist should be killed when server is shutting down - updateBlocklist(b) + walletServers := []string{ + "spv25.lbry.com:50001", + "spv26.lbry.com:50001", + "spv19.lbry.com:50001", + "spv14.lbry.com:50001", + } + + updateBlocklist(b, walletServers, s.grp.Ch()) t := time.NewTicker(12 * time.Hour) for { select { case <-s.grp.Ch(): return case <-t.C: - updateBlocklist(b) + updateBlocklist(b, walletServers, s.grp.Ch()) } } } -func updateBlocklist(b store.Blocklister) { - values, err := blockedSdHashes() +func updateBlocklist(b store.Blocklister, walletServers []string, stopper stop.Chan) { + log.Debugf("blocklist update starting") + values, err := blockedSdHashes(walletServers, stopper) if err != nil { log.Error(err) return @@ -50,10 +58,12 @@ func updateBlocklist(b store.Blocklister) { log.Error(err) } } + log.Debugf("blocklist update done") } -func blockedSdHashes() (map[string]valOrErr, error) { - resp, err := http.Get(blocklistURL) +func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valOrErr, error) { + client := http.Client{Timeout: 1 * time.Second} + resp, err := client.Get(blocklistURL) if err != nil { return nil, errors.Err(err) } @@ -80,7 +90,7 @@ func blockedSdHashes() (map[string]valOrErr, error) { return nil, errors.Prefix("list_blocked API call", r.Error) } - return sdHashesForOutpoints(r.Data.Outpoints) + return sdHashesForOutpoints(walletServers, r.Data.Outpoints, stopper) } type valOrErr struct { @@ -89,22 +99,33 @@ type valOrErr struct { } // sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints -func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) { +func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan) (map[string]valOrErr, error) { values := make(map[string]valOrErr) node := wallet.NewNode() - defer node.Shutdown() - err := node.Connect([]string{ - "spv25.lbry.com:50001", - "spv26.lbry.com:50001", - "spv19.lbry.com:50001", - "spv14.lbry.com:50001", - }, nil) + err := node.Connect(walletServers, nil) if err != nil { return nil, errors.Err(err) } + done := make(chan bool) + + go func() { + select { + case <-done: + case <-stopper: + } + node.Shutdown() + }() + +OutpointLoop: for _, outpoint := range outpoints { + select { + case <-stopper: + break OutpointLoop + default: + } + parts := strings.Split(outpoint, ":") if len(parts) != 2 { values[outpoint] = valOrErr{Err: errors.Err("invalid outpoint format")} @@ -127,5 +148,10 @@ func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) { values[outpoint] = valOrErr{Value: hash, Err: nil} } + select { + case done <- true: + default: // in case of race where stopper got stopped right after loop finished + } + return values, nil } diff --git a/wallet/network.go b/wallet/network.go index 20e86ba..10e4edf 100644 --- a/wallet/network.go +++ b/wallet/network.go @@ -115,7 +115,13 @@ func (n *Node) Connect(addrs []string, config *tls.Config) error { } func (n *Node) Shutdown() { + var addr net.Addr + if n.transport != nil { + addr = n.transport.conn.RemoteAddr() + } + log.Debugf("shutting down wallet %s", addr) n.grp.StopAndWait() + log.Debugf("wallet stopped") } func (n *Node) handleErrors() { @@ -138,6 +144,12 @@ func (n *Node) err(err error) { // listen processes messages from the server. func (n *Node) listen() { for { + select { + case <-n.grp.Ch(): + return + default: + } + select { case <-n.grp.Ch(): return @@ -248,6 +260,8 @@ func (n *Node) request(method string, params []string, v interface{}) error { var r response select { + case <-n.grp.Ch(): + return nil case r = <-c: case <-time.After(n.timeout): r = response{err: errors.Err(ErrTimeout)} -- 2.45.2 From 5c91051b783b7a1645464a580e9bdab18c2e4a31 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 7 Jul 2020 01:01:20 +0200 Subject: [PATCH 16/18] disable disk cleanup routine --- store/disk.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/store/disk.go b/store/disk.go index 18f7bd7..9b537d2 100644 --- a/store/disk.go +++ b/store/disk.go @@ -124,14 +124,6 @@ func (d *DiskBlobStore) Put(hash string, blob stream.Blob) error { if err != nil { return err } - select { - case <-d.diskCleanupBusy: - if time.Since(d.lastChecked) > 5*time.Minute { - go d.ensureDiskSpace() - } - default: - break - } return ioutil.WriteFile(d.path(hash), blob, 0644) } -- 2.45.2 From 34ca7847d0c2df6d9a8e2319824e3479ac00f549 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Thu, 9 Jul 2020 03:12:33 +0200 Subject: [PATCH 17/18] return a better descriptive error when the blob isn't found --- peer/http3/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/peer/http3/server.go b/peer/http3/server.go index 6f18db3..9be4f9c 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -70,6 +70,10 @@ func (s *Server) Start(address string) error { requestedBlob := vars["hash"] blob, err := s.store.Get(requestedBlob) if err != nil { + if errors.Is(err, store.ErrBlobNotFound) { + http.Error(w, err.Error(), http.StatusNotFound) + return + } fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err)) s.logError(err) http.Error(w, err.Error(), http.StatusBadRequest) -- 2.45.2 From 694bda105c82f01a0a193a79dabcbdcedeca2d50 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Thu, 9 Jul 2020 04:28:34 +0200 Subject: [PATCH 18/18] add metrics --- internal/metrics/metrics.go | 31 +++++++++++++++++++++++++++---- peer/http3/client.go | 4 ++-- peer/http3/server.go | 2 ++ peer/server.go | 1 + store/caching.go | 16 ++++++++++++++-- 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index b14435c..5c9dc92 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -12,8 +12,6 @@ import ( ee "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/extras/stop" - "github.com/lbryio/reflector.go/store" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -67,6 +65,8 @@ const ( DirectionUpload = "upload" // to reflector DirectionDownload = "download" // from reflector + MtrLabelSource = "source" + errConnReset = "conn_reset" errReadConnReset = "read_conn_reset" errWriteConnReset = "write_conn_reset" @@ -96,6 +96,26 @@ var ( Name: "blob_download_total", Help: "Total number of blobs downloaded from reflector", }) + PeerDownloadCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Name: "peer_download_total", + Help: "Total number of blobs downloaded from reflector through tcp protocol", + }) + Http3DownloadCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Name: "http3_blob_download_total", + Help: "Total number of blobs downloaded from reflector through QUIC protocol", + }) + CacheHitCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Name: "cache_hit_total", + Help: "Total number of blobs retrieved from the cache storage", + }) + CacheMissCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Name: "cache_miss_total", + Help: "Total number of blobs retrieved from origin rather than cache storage", + }) BlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: ns, Name: "blob_upload_total", @@ -106,6 +126,11 @@ var ( Name: "sdblob_upload_total", Help: "Total number of SD blobs (and therefore streams) uploaded to reflector", }) + RetrieverSpeed = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: ns, + Name: "speed_mbps", + Help: "Speed of blob retrieval", + }, []string{MtrLabelSource}) ErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: ns, Name: "error_total", @@ -148,8 +173,6 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a errType = errUnexpectedEOFStr } else if errors.Is(e, syscall.EPIPE) { errType = errEPipe - } else if errors.Is(e, store.ErrBlobNotFound) { - errType = errBlobNotFound } else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer // I believe this is the same as EPipe when direction == "download", but not for upload errType = errWriteBrokenPipe diff --git a/peer/http3/client.go b/peer/http3/client.go index aba77c3..85bd57d 100644 --- a/peer/http3/client.go +++ b/peer/http3/client.go @@ -8,10 +8,9 @@ import ( "net/http" "time" - "github.com/lbryio/reflector.go/store" - "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/store" "github.com/lucas-clemente/quic-go/http3" ) @@ -79,6 +78,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, error) { return nil, errors.Err(err) } if resp.StatusCode == http.StatusNotFound { + fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode) return nil, errors.Err(store.ErrBlobNotFound) } if resp.StatusCode != http.StatusOK { diff --git a/peer/http3/server.go b/peer/http3/server.go index 9be4f9c..9cc4f80 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -84,6 +84,8 @@ func (s *Server) Start(address string) error { if err != nil { s.logError(err) } + metrics.BlobDownloadCount.Inc() + metrics.Http3DownloadCount.Inc() }) r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) diff --git a/peer/server.go b/peer/server.go index d7db443..9a53ef5 100644 --- a/peer/server.go +++ b/peer/server.go @@ -273,6 +273,7 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) { Length: len(blob), } metrics.BlobDownloadCount.Inc() + metrics.PeerDownloadCount.Inc() } } diff --git a/store/caching.go b/store/caching.go index 9d5149f..dfbff8f 100644 --- a/store/caching.go +++ b/store/caching.go @@ -1,8 +1,12 @@ package store import ( + "time" + "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + + "github.com/lbryio/reflector.go/internal/metrics" ) // CachingBlobStore combines two stores, typically a local and a remote store, to improve performance. @@ -29,18 +33,26 @@ func (c *CachingBlobStore) Has(hash string) (bool, error) { // Get tries to get the blob from the cache first, falling back to the origin. If the blob comes // from the origin, it is also stored in the cache. func (c *CachingBlobStore) Get(hash string) (stream.Blob, error) { + start := time.Now() blob, err := c.cache.Get(hash) + retrievalTime := time.Since(start) if err == nil || !errors.Is(err, ErrBlobNotFound) { + metrics.CacheHitCount.Inc() + rate := float64(len(blob)) / 1024 / 1024 / retrievalTime.Seconds() + metrics.RetrieverSpeed.With(map[string]string{metrics.MtrLabelSource: "cache"}).Set(rate) return blob, err } + start = time.Now() blob, err = c.origin.Get(hash) if err != nil { return nil, err } - + retrievalTime = time.Since(start) err = c.cache.Put(hash, blob) - + rate := float64(len(blob)) / 1024 / 1024 / retrievalTime.Seconds() + metrics.RetrieverSpeed.With(map[string]string{metrics.MtrLabelSource: "origin"}).Set(rate) + metrics.CacheMissCount.Inc() return blob, err } -- 2.45.2