From 9146c8b08427864ccb4fb6657065d500c985e140 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 18 Nov 2020 05:03:02 +0100 Subject: [PATCH 01/51] update quic don't wait for a blob to be written to disk before sending it downstream don't wait for the disk store to be walked before starting everything up --- go.mod | 6 +++--- go.sum | 15 +++++++-------- store/caching.go | 13 ++++++++++--- store/lru.go | 13 +++++++------ store/speedwalk/speedwalk.go | 3 +-- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 9df120d..b98b2ad 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/golang/protobuf v1.4.2 github.com/google/btree v1.0.0 // indirect github.com/google/gops v0.3.7 + github.com/google/martian v2.1.0+incompatible github.com/gorilla/mux v1.7.4 github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/golang-lru v0.5.4 @@ -24,11 +25,11 @@ require ( github.com/lbryio/lbry.go v1.1.2 // indirect github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec - github.com/lucas-clemente/quic-go v0.18.1 + github.com/lucas-clemente/quic-go v0.19.1 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.2 github.com/sirupsen/logrus v1.4.2 - github.com/spf13/afero v1.4.1 + github.com/spf13/afero v1.4.1 // indirect github.com/spf13/cast v1.3.0 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect @@ -40,7 +41,6 @@ require ( golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect google.golang.org/appengine v1.6.2 // indirect - gotest.tools v2.2.0+incompatible ) go 1.15 diff --git a/go.sum b/go.sum index 84b207a..fceb2df 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,7 @@ github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+u github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw= github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -150,8 +151,6 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce h1:xdsDDbiBDQTKASoGEZ+pEmF1OnWuu8AQ9I8iNbHNeno= @@ -222,8 +221,8 @@ github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiV github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c= github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys= -github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= +github.com/lucas-clemente/quic-go v0.19.1 h1:J9TkQJGJVOR3UmGhd4zdVYwKSA0EoXbLRf15uQJ6gT4= +github.com/lucas-clemente/quic-go v0.19.1/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= @@ -232,12 +231,12 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCW github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.2.0 h1:/r1rhZoOmgxVKBqPNnYilZBDEyw+6OUHCbBzA5jc2y0= -github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= +github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= -github.com/marten-seemann/qtls-go1-15 v0.1.0 h1:i/YPXVxz8q9umso/5y474CNcHmTpA+5DH+mFPjx6PZg= -github.com/marten-seemann/qtls-go1-15 v0.1.0/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ= +github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= diff --git a/store/caching.go b/store/caching.go index 0a06395..c8e7d45 100644 --- a/store/caching.go +++ b/store/caching.go @@ -5,6 +5,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + log "github.com/sirupsen/logrus" "github.com/lbryio/reflector.go/internal/metrics" ) @@ -62,9 +63,15 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) { if err != nil { return nil, err } - - err = c.cache.Put(hash, blob) - return blob, err + // there is no need to wait for the blob to be stored before we return it + // TODO: however this should be refactored to limit the amount of routines that the process can spawn to avoid a possible DoS + go func() { + err = c.cache.Put(hash, blob) + if err != nil { + log.Errorf("error saving blob to underlying cache: %s", err.Error()) + } + }() + return blob, nil } // Put stores the blob in the origin and the cache diff --git a/store/lru.go b/store/lru.go index 4ef4908..6d06aa6 100644 --- a/store/lru.go +++ b/store/lru.go @@ -30,13 +30,14 @@ func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore { store: store, lru: lru, } - - if lstr, ok := store.(lister); ok { - err = l.loadExisting(lstr, maxItems) - if err != nil { - panic(err) // TODO: what should happen here? panic? return nil? just keep going? + go func() { + if lstr, ok := store.(lister); ok { + err = l.loadExisting(lstr, maxItems) + if err != nil { + panic(err) // TODO: what should happen here? panic? return nil? just keep going? + } } - } + }() return l } diff --git a/store/speedwalk/speedwalk.go b/store/speedwalk/speedwalk.go index e2563ba..23230df 100644 --- a/store/speedwalk/speedwalk.go +++ b/store/speedwalk/speedwalk.go @@ -60,7 +60,6 @@ func AllFiles(startDir string, basename bool) ([]string, error) { walkerWG.Done() goroutineLimiter <- struct{}{} }() - err = godirwalk.Walk(filepath.Join(startDir, dir), &godirwalk.Options{ Unsorted: true, // faster this way Callback: func(osPathname string, de *godirwalk.Dirent) error { @@ -84,6 +83,6 @@ func AllFiles(startDir string, basename bool) ([]string, error) { close(pathChan) pathWG.Wait() - + logrus.Infoln("loaded LRU") return paths, nil } -- 2.45.2 From a574fecf4ee3595325b2f84d711b9532c527838f Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 20 Nov 2020 19:39:08 +0100 Subject: [PATCH 02/51] add buffer cache for nvme drive --- cmd/reflector.go | 50 +++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index 2f687d0..cff66f5 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -22,19 +22,20 @@ import ( ) var ( - tcpPeerPort int - http3PeerPort int - receiverPort int - metricsPort int - disableUploads bool - disableBlocklist bool - proxyAddress string - proxyPort string - proxyProtocol string - useDB bool - cloudFrontEndpoint string - reflectorCmdDiskCache string - reflectorCmdMemCache int + tcpPeerPort int + http3PeerPort int + receiverPort int + metricsPort int + disableUploads bool + disableBlocklist bool + proxyAddress string + proxyPort string + proxyProtocol string + useDB bool + cloudFrontEndpoint string + reflectorCmdDiskCache string + bufferReflectorCmdDiskCache string + reflectorCmdMemCache int ) func init() { @@ -56,6 +57,8 @@ func init() { cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not") cmd.Flags().StringVar(&reflectorCmdDiskCache, "disk-cache", "", "enable disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'") + cmd.Flags().StringVar(&bufferReflectorCmdDiskCache, "buffer-disk-cache", "", + "enable buffer disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'") cmd.Flags().IntVar(&reflectorCmdMemCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs") rootCmd.AddCommand(cmd) } @@ -147,7 +150,19 @@ func setupStore() store.BlobStore { func wrapWithCache(s store.BlobStore) store.BlobStore { wrapped := s - diskCacheMaxSize, diskCachePath := diskCacheParams() + diskCacheMaxSize, diskCachePath := diskCacheParams(reflectorCmdDiskCache) + if diskCacheMaxSize > 0 { + err := os.MkdirAll(diskCachePath, os.ModePerm) + if err != nil { + log.Fatal(err) + } + wrapped = store.NewCachingStore( + "reflector", + wrapped, + store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize), + ) + } + diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache) if diskCacheMaxSize > 0 { err := os.MkdirAll(diskCachePath, os.ModePerm) if err != nil { @@ -159,7 +174,6 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize), ) } - if reflectorCmdMemCache > 0 { wrapped = store.NewCachingStore( "reflector", @@ -171,12 +185,12 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { return wrapped } -func diskCacheParams() (int, string) { - if reflectorCmdDiskCache == "" { +func diskCacheParams(diskParams string) (int, string) { + if diskParams == "" { return 0, "" } - parts := strings.Split(reflectorCmdDiskCache, ":") + parts := strings.Split(diskParams, ":") if len(parts) != 2 { log.Fatalf("--disk-cache must be a number, followed by ':', followed by a string") } -- 2.45.2 From bb41a84bb7ac4335497cae43fa3f00535edfe02b Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Fri, 20 Nov 2020 15:01:33 -0500 Subject: [PATCH 03/51] rename cahces --- cmd/reflector.go | 8 +++++--- store/lru.go | 18 ++++++++++-------- store/singleflight.go | 10 +++++----- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index cff66f5..a047b85 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -159,9 +159,10 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize), + store.NewLRUStore("hdd", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize), ) } + diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache) if diskCacheMaxSize > 0 { err := os.MkdirAll(diskCachePath, os.ModePerm) @@ -171,14 +172,15 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize), + store.NewLRUStore("nvme", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize), ) } + if reflectorCmdMemCache > 0 { wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewLRUStore("peer_server", store.NewMemStore(), reflectorCmdMemCache), + store.NewLRUStore("mem", store.NewMemStore(), reflectorCmdMemCache), ) } diff --git a/store/lru.go b/store/lru.go index 6d06aa6..edbe3a5 100644 --- a/store/lru.go +++ b/store/lru.go @@ -18,18 +18,20 @@ type LRUStore struct { // NewLRUStore initialize a new LRUStore func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore { + l := &LRUStore{ + store: store, + } + lru, err := golru.NewWithEvict(maxItems, func(key interface{}, value interface{}) { - metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc() + metrics.CacheLRUEvictCount.With(metrics.CacheLabels(l.Name(), component)).Inc() _ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there }) if err != nil { panic(err) } - l := &LRUStore{ - store: store, - lru: lru, - } + l.lru = lru + go func() { if lstr, ok := store.(lister); ok { err = l.loadExisting(lstr, maxItems) @@ -42,10 +44,10 @@ func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore { return l } -const nameLRU = "lru" - // Name is the cache type name -func (l *LRUStore) Name() string { return nameLRU } +func (l *LRUStore) Name() string { + return "lru_" + l.store.Name() +} // Has returns whether the blob is in the store, without updating the recent-ness. func (l *LRUStore) Has(hash string) (bool, error) { diff --git a/store/singleflight.go b/store/singleflight.go index fbe314f..cca1e2f 100644 --- a/store/singleflight.go +++ b/store/singleflight.go @@ -32,8 +32,8 @@ func (s *singleflightStore) Name() string { // Get ensures that only one request per hash is sent to the origin at a time, // thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem func (s *singleflightStore) Get(hash string) (stream.Blob, error) { - metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Inc() - defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Dec() + metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc() + defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec() blob, err, _ := s.sf.Do(hash, s.getter(hash)) if err != nil { @@ -46,8 +46,8 @@ func (s *singleflightStore) Get(hash string) (stream.Blob, error) { // only one getter per hash will be executing at a time func (s *singleflightStore) getter(hash string) func() (interface{}, error) { return func() (interface{}, error) { - metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Inc() - defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Dec() + metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc() + defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec() start := time.Now() blob, err := s.BlobStore.Get(hash) @@ -57,7 +57,7 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) { rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds() metrics.CacheRetrievalSpeed.With(map[string]string{ - metrics.LabelCacheType: s.BlobStore.Name(), + metrics.LabelCacheType: s.Name(), metrics.LabelComponent: s.component, metrics.LabelSource: "origin", }).Set(rate) -- 2.45.2 From bc54601dde0b295cef801c62263b6321fe15bfc0 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Sat, 21 Nov 2020 01:26:32 +0100 Subject: [PATCH 04/51] add LFUDA store update quic fix tests --- go.mod | 3 +- go.sum | 4 ++ store/caching_test.go | 1 + store/lfuda.go | 118 +++++++++++++++++++++++++++++++++++ store/lfuda_test.go | 139 ++++++++++++++++++++++++++++++++++++++++++ store/lru_test.go | 4 +- 6 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 store/lfuda.go create mode 100644 store/lfuda_test.go diff --git a/go.mod b/go.mod index b98b2ad..5ab0eb5 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203 require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/aws/aws-sdk-go v1.16.11 + github.com/bparli/lfuda-go v0.3.0 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d github.com/davecgh/go-spew v1.1.1 @@ -25,7 +26,7 @@ require ( github.com/lbryio/lbry.go v1.1.2 // indirect github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec - github.com/lucas-clemente/quic-go v0.19.1 + github.com/lucas-clemente/quic-go v0.19.2 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.2 github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index fceb2df..25e908f 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bparli/lfuda-go v0.3.0 h1:b6qPjEb0BN006oQnj2nuGfz94yY3iYo0bmuFM079tQg= +github.com/bparli/lfuda-go v0.3.0/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -223,6 +225,8 @@ github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiV github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucas-clemente/quic-go v0.19.1 h1:J9TkQJGJVOR3UmGhd4zdVYwKSA0EoXbLRf15uQJ6gT4= github.com/lucas-clemente/quic-go v0.19.1/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0= +github.com/lucas-clemente/quic-go v0.19.2 h1:w8BBYUx5Z+kNpeaOeQW/KzcNsKWhh4O6PeQhb0nURPg= +github.com/lucas-clemente/quic-go v0.19.2/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= diff --git a/store/caching_test.go b/store/caching_test.go index 34f928c..1168a92 100644 --- a/store/caching_test.go +++ b/store/caching_test.go @@ -58,6 +58,7 @@ func TestCachingStore_CacheMiss(t *testing.T) { if !bytes.Equal(b, res) { t.Errorf("expected Get() to return %s, got %s", string(b), string(res)) } + time.Sleep(10 * time.Millisecond) //storing to cache is done async so let's give it some time has, err := cache.Has(hash) if err != nil { diff --git a/store/lfuda.go b/store/lfuda.go new file mode 100644 index 0000000..9715954 --- /dev/null +++ b/store/lfuda.go @@ -0,0 +1,118 @@ +package store + +import ( + "github.com/bparli/lfuda-go" + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/internal/metrics" +) + +// LRUStore adds a max cache size and LRU eviction to a BlobStore +type LFUDAStore struct { + // underlying store + store BlobStore + // lfuda implementation + lfuda *lfuda.Cache +} + +// NewLRUStore initialize a new LRUStore +func NewLFUDAStore(component string, store BlobStore, maxSize float64) *LFUDAStore { + lfuda := lfuda.NewGDSFWithEvict(maxSize, func(key interface{}, value interface{}) { + metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc() + _ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there + }) + lfuda.Age() + l := &LFUDAStore{ + store: store, + lfuda: lfuda, + } + go func() { + if lstr, ok := store.(lister); ok { + err := l.loadExisting(lstr, int(maxSize/2000000.0)) + if err != nil { + panic(err) // TODO: what should happen here? panic? return nil? just keep going? + } + } + }() + + return l +} + +const nameLFUDA = "lfuda" + +var fakeTrue = []byte{'t'} + +// Name is the cache type name +func (l *LFUDAStore) Name() string { return nameLFUDA } + +// Has returns whether the blob is in the store, without updating the recent-ness. +func (l *LFUDAStore) Has(hash string) (bool, error) { + return l.lfuda.Contains(hash), nil +} + +// Get returns the blob or an error if the blob doesn't exist. +func (l *LFUDAStore) Get(hash string) (stream.Blob, error) { + _, has := l.lfuda.Get(hash) + if !has { + return nil, errors.Err(ErrBlobNotFound) + } + blob, err := l.store.Get(hash) + if errors.Is(err, ErrBlobNotFound) { + // Blob disappeared from underlying store + l.lfuda.Remove(hash) + } + return blob, err +} + +// Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!! +func (l *LFUDAStore) Put(hash string, blob stream.Blob) error { + err := l.store.Put(hash, blob) + if err != nil { + return err + } + l.lfuda.Set(hash, fakeTrue) + return nil +} + +// PutSD stores the sd blob +func (l *LFUDAStore) PutSD(hash string, blob stream.Blob) error { + err := l.store.PutSD(hash, blob) + if err != nil { + return err + } + + l.lfuda.Set(hash, fakeTrue) + return nil +} + +// Delete deletes the blob from the store +func (l *LFUDAStore) Delete(hash string) error { + err := l.store.Delete(hash) + if err != nil { + return err + } + + // This must come after store.Delete() + // Remove triggers onEvict function, which also tries to delete blob from store + // We need to delete it manually first so any errors can be propagated up + l.lfuda.Remove(hash) + return nil +} + +// loadExisting imports existing blobs from the underlying store into the LRU cache +func (l *LFUDAStore) loadExisting(store lister, maxItems int) error { + existing, err := store.list() + if err != nil { + return err + } + + added := 0 + for _, h := range existing { + l.lfuda.Set(h, fakeTrue) + added++ + if maxItems > 0 && added >= maxItems { // underlying cache is bigger than LRU cache + break + } + } + return nil +} diff --git a/store/lfuda_test.go b/store/lfuda_test.go new file mode 100644 index 0000000..9b1eb58 --- /dev/null +++ b/store/lfuda_test.go @@ -0,0 +1,139 @@ +package store + +import ( + "io/ioutil" + "os" + "reflect" + "testing" + "time" + + "github.com/lbryio/lbry.go/v2/extras/errors" + log "github.com/sirupsen/logrus" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const cacheMaxSize = 3 + +func getTestLFUDAStore() (*LFUDAStore, *MemStore) { + m := NewMemStore() + return NewLFUDAStore("test", m, cacheMaxSize), m +} + +func TestFUDAStore_Eviction(t *testing.T) { + lfuda, mem := getTestLFUDAStore() + b := []byte("x") + err := lfuda.Put("one", b) + require.NoError(t, err) + err = lfuda.Put("two", b) + require.NoError(t, err) + err = lfuda.Put("three", b) + require.NoError(t, err) + err = lfuda.Put("four", b) + require.NoError(t, err) + err = lfuda.Put("five", b) + require.NoError(t, err) + err = lfuda.Put("five", b) + require.NoError(t, err) + err = lfuda.Put("four", b) + require.NoError(t, err) + err = lfuda.Put("two", b) + require.NoError(t, err) + + _, err = lfuda.Get("five") + require.NoError(t, err) + _, err = lfuda.Get("four") + require.NoError(t, err) + _, err = lfuda.Get("two") + require.NoError(t, err) + assert.Equal(t, cacheMaxBlobs, len(mem.Debug())) + + for k, v := range map[string]bool{ + "one": false, + "two": true, + "three": false, + "four": true, + "five": true, + "six": false, + } { + has, err := lfuda.Has(k) + assert.NoError(t, err) + assert.Equal(t, v, has) + } + + lfuda.Get("two") // touch so it stays in cache + lfuda.Get("five") // touch so it stays in cache + lfuda.Put("six", b) + + assert.Equal(t, cacheMaxBlobs, len(mem.Debug())) + + keys := lfuda.lfuda.Keys() + log.Infof("%+v", keys) + for k, v := range map[string]bool{ + "one": false, + "two": true, + "three": false, + "four": false, + "five": true, + "six": true, + } { + has, err := lfuda.Has(k) + assert.NoError(t, err) + assert.Equal(t, v, has) + } + + err = lfuda.Delete("six") + assert.NoError(t, err) + err = lfuda.Delete("five") + assert.NoError(t, err) + err = lfuda.Delete("two") + assert.NoError(t, err) + assert.Equal(t, 0, len(mem.Debug())) +} + +func TestFUDAStore_UnderlyingBlobMissing(t *testing.T) { + lfuda, mem := getTestLFUDAStore() + hash := "hash" + b := []byte("this is a blob of stuff") + err := lfuda.Put(hash, b) + require.NoError(t, err) + + err = mem.Delete(hash) + require.NoError(t, err) + + // hash still exists in lru + assert.True(t, lfuda.lfuda.Contains(hash)) + + blob, err := lfuda.Get(hash) + assert.Nil(t, blob) + assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s", + reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(), + reflect.TypeOf(err).String(), err.Error()) + + // lru.Get() removes hash if underlying store doesn't have it + assert.False(t, lfuda.lfuda.Contains(hash)) +} + +func TestFUDAStore_loadExisting(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "reflector_test_*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + d := NewDiskStore(tmpDir, 2) + + hash := "hash" + b := []byte("this is a blob of stuff") + err = d.Put(hash, b) + require.NoError(t, err) + + existing, err := d.list() + require.NoError(t, err) + require.Equal(t, 1, len(existing), "blob should exist in cache") + assert.Equal(t, hash, existing[0]) + + lfuda := NewLFUDAStore("test", d, 3) // lru should load existing blobs when it's created + time.Sleep(100 * time.Millisecond) // async load so let's wait... + has, err := lfuda.Has(hash) + require.NoError(t, err) + assert.True(t, has, "hash should be loaded from disk store but it's not") +} diff --git a/store/lru_test.go b/store/lru_test.go index 968956c..2bd934c 100644 --- a/store/lru_test.go +++ b/store/lru_test.go @@ -5,6 +5,7 @@ import ( "os" "reflect" "testing" + "time" "github.com/lbryio/lbry.go/v2/extras/errors" @@ -114,7 +115,8 @@ func TestLRUStore_loadExisting(t *testing.T) { require.Equal(t, 1, len(existing), "blob should exist in cache") assert.Equal(t, hash, existing[0]) - lru := NewLRUStore("test", d, 3) // lru should load existing blobs when it's created + lru := NewLRUStore("test", d, 3) // lru should load existing blobs when it's created + time.Sleep(100 * time.Millisecond) // async load so let's wait... has, err := lru.Has(hash) require.NoError(t, err) assert.True(t, has, "hash should be loaded from disk store but it's not") -- 2.45.2 From d45abdbdb0e0351e212940f56021437ae2e3836a Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Sat, 21 Nov 2020 01:39:50 +0100 Subject: [PATCH 05/51] use LFUDA store swap size to bytes --- cmd/reflector.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index a047b85..4d47970 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -151,6 +151,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { wrapped := s diskCacheMaxSize, diskCachePath := diskCacheParams(reflectorCmdDiskCache) + cacheMaxSizeInBytes := float64(diskCacheMaxSize * 2 * 1000 * 1000) if diskCacheMaxSize > 0 { err := os.MkdirAll(diskCachePath, os.ModePerm) if err != nil { @@ -159,7 +160,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewLRUStore("hdd", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize), + store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), cacheMaxSizeInBytes), ) } @@ -172,7 +173,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewLRUStore("nvme", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize), + store.NewLFUDAStore("nvme", store.NewDiskStore(diskCachePath, 2), cacheMaxSizeInBytes), ) } -- 2.45.2 From 7b80b2d4d24f82f115e26bdc0d2e8ae6edd2a49f Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Sun, 22 Nov 2020 04:04:44 +0100 Subject: [PATCH 06/51] fix buffer cache running out of space --- cmd/reflector.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/reflector.go b/cmd/reflector.go index 4d47970..55d77b3 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -165,6 +165,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { } diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache) + cacheMaxSizeInBytes = float64(diskCacheMaxSize * 2 * 1000 * 1000) if diskCacheMaxSize > 0 { err := os.MkdirAll(diskCachePath, os.ModePerm) if err != nil { -- 2.45.2 From ff9b61b034bd75ed23b2c2826b5b5d4690e67cae Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Sun, 22 Nov 2020 05:06:36 +0100 Subject: [PATCH 07/51] fix cache size mess --- cmd/reflector.go | 34 +++++++++++++++++++++------------- go.mod | 2 +- go.sum | 4 ++-- store/lfuda.go | 5 +++-- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index 55d77b3..7551f77 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -16,8 +16,10 @@ import ( "github.com/lbryio/reflector.go/reflector" "github.com/lbryio/reflector.go/store" + "github.com/lbryio/lbry.go/v2/stream" + + "github.com/c2h5oh/datasize" log "github.com/sirupsen/logrus" - "github.com/spf13/cast" "github.com/spf13/cobra" ) @@ -56,9 +58,9 @@ func init() { cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating") cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not") cmd.Flags().StringVar(&reflectorCmdDiskCache, "disk-cache", "", - "enable disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'") + "enable disk cache, setting max size and path where to store blobs. format is 'sizeGB:CACHE_PATH'") cmd.Flags().StringVar(&bufferReflectorCmdDiskCache, "buffer-disk-cache", "", - "enable buffer disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'") + "enable buffer disk cache, setting max size and path where to store blobs. format is 'sizeGB:CACHE_PATH'") cmd.Flags().IntVar(&reflectorCmdMemCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs") rootCmd.AddCommand(cmd) } @@ -151,7 +153,9 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { wrapped := s diskCacheMaxSize, diskCachePath := diskCacheParams(reflectorCmdDiskCache) - cacheMaxSizeInBytes := float64(diskCacheMaxSize * 2 * 1000 * 1000) + //we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte + // so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup + realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize) if diskCacheMaxSize > 0 { err := os.MkdirAll(diskCachePath, os.ModePerm) if err != nil { @@ -160,12 +164,12 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), cacheMaxSizeInBytes), + store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), realCacheSize), ) } diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache) - cacheMaxSizeInBytes = float64(diskCacheMaxSize * 2 * 1000 * 1000) + realCacheSize = float64(diskCacheMaxSize) / float64(stream.MaxBlobSize) if diskCacheMaxSize > 0 { err := os.MkdirAll(diskCachePath, os.ModePerm) if err != nil { @@ -174,7 +178,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewLFUDAStore("nvme", store.NewDiskStore(diskCachePath, 2), cacheMaxSizeInBytes), + store.NewLFUDAStore("nvme", store.NewDiskStore(diskCachePath, 2), realCacheSize), ) } @@ -199,15 +203,19 @@ func diskCacheParams(diskParams string) (int, string) { log.Fatalf("--disk-cache must be a number, followed by ':', followed by a string") } - maxSize := cast.ToInt(parts[0]) - if maxSize <= 0 { - log.Fatalf("--disk-cache max size must be more than 0") - } - + diskCacheSize := parts[0] path := parts[1] if len(path) == 0 || path[0] != '/' { log.Fatalf("--disk-cache path must start with '/'") } - return maxSize, path + var maxSize datasize.ByteSize + err := maxSize.UnmarshalText([]byte(diskCacheSize)) + if err != nil { + log.Fatal(err) + } + if maxSize <= 0 { + log.Fatal("--disk-cache size must be more than 0") + } + return int(maxSize), path } diff --git a/go.mod b/go.mod index 5ab0eb5..df0c53d 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,12 @@ require ( github.com/bparli/lfuda-go v0.3.0 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d + github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 github.com/davecgh/go-spew v1.1.1 github.com/go-sql-driver/mysql v1.4.1 github.com/golang/protobuf v1.4.2 github.com/google/btree v1.0.0 // indirect github.com/google/gops v0.3.7 - github.com/google/martian v2.1.0+incompatible github.com/gorilla/mux v1.7.4 github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/golang-lru v0.5.4 diff --git a/go.sum b/go.sum index 25e908f..5965f78 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak= +github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= @@ -223,8 +225,6 @@ github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiV github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c= github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lucas-clemente/quic-go v0.19.1 h1:J9TkQJGJVOR3UmGhd4zdVYwKSA0EoXbLRf15uQJ6gT4= -github.com/lucas-clemente/quic-go v0.19.1/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0= github.com/lucas-clemente/quic-go v0.19.2 h1:w8BBYUx5Z+kNpeaOeQW/KzcNsKWhh4O6PeQhb0nURPg= github.com/lucas-clemente/quic-go v0.19.2/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= diff --git a/store/lfuda.go b/store/lfuda.go index 9715954..2a12e99 100644 --- a/store/lfuda.go +++ b/store/lfuda.go @@ -5,6 +5,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/internal/metrics" + "github.com/sirupsen/logrus" ) // LRUStore adds a max cache size and LRU eviction to a BlobStore @@ -21,14 +22,13 @@ func NewLFUDAStore(component string, store BlobStore, maxSize float64) *LFUDASto metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc() _ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there }) - lfuda.Age() l := &LFUDAStore{ store: store, lfuda: lfuda, } go func() { if lstr, ok := store.(lister); ok { - err := l.loadExisting(lstr, int(maxSize/2000000.0)) + err := l.loadExisting(lstr, int(maxSize)) if err != nil { panic(err) // TODO: what should happen here? panic? return nil? just keep going? } @@ -101,6 +101,7 @@ func (l *LFUDAStore) Delete(hash string) error { // loadExisting imports existing blobs from the underlying store into the LRU cache func (l *LFUDAStore) loadExisting(store lister, maxItems int) error { + logrus.Infof("loading at most %d items", maxItems) existing, err := store.list() if err != nil { return err -- 2.45.2 From 9fc96ac01b61607bde39a66f407e54283eb8d351 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 25 Nov 2020 19:24:32 +0100 Subject: [PATCH 08/51] only store the blobs in the underlying storage if LFUDA accepted them --- store/lfuda.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/store/lfuda.go b/store/lfuda.go index 2a12e99..e33afa0 100644 --- a/store/lfuda.go +++ b/store/lfuda.go @@ -66,11 +66,14 @@ func (l *LFUDAStore) Get(hash string) (stream.Blob, error) { // Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!! func (l *LFUDAStore) Put(hash string, blob stream.Blob) error { - err := l.store.Put(hash, blob) - if err != nil { - return err - } l.lfuda.Set(hash, fakeTrue) + has, _ := l.Has(hash) + if has { + err := l.store.Put(hash, blob) + if err != nil { + return err + } + } return nil } -- 2.45.2 From 2c0df2ca8a497b13829e13790fe9b1b9106464a2 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 27 Nov 2020 16:19:17 +0100 Subject: [PATCH 09/51] update lfuda library --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index df0c53d..95a4a5a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203 require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/aws/aws-sdk-go v1.16.11 - github.com/bparli/lfuda-go v0.3.0 + github.com/bparli/lfuda-go v0.3.1 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 diff --git a/go.sum b/go.sum index 5965f78..4879450 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,8 @@ github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bparli/lfuda-go v0.3.0 h1:b6qPjEb0BN006oQnj2nuGfz94yY3iYo0bmuFM079tQg= -github.com/bparli/lfuda-go v0.3.0/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo= +github.com/bparli/lfuda-go v0.3.1 h1:nO9Szo627RC8/z+R+MMPBItNwHCOonchmpjQuQi8jVY= +github.com/bparli/lfuda-go v0.3.1/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -- 2.45.2 From 74b76a11e47360023059883fc9635d6a7ffb800f Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Thu, 17 Dec 2020 23:49:37 +0100 Subject: [PATCH 10/51] upgrade quic --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 95a4a5a..04d07c2 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/lbryio/lbry.go v1.1.2 // indirect github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec - github.com/lucas-clemente/quic-go v0.19.2 + github.com/lucas-clemente/quic-go v0.19.3 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.2 github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index 4879450..18420b2 100644 --- a/go.sum +++ b/go.sum @@ -225,8 +225,8 @@ github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiV github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c= github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lucas-clemente/quic-go v0.19.2 h1:w8BBYUx5Z+kNpeaOeQW/KzcNsKWhh4O6PeQhb0nURPg= -github.com/lucas-clemente/quic-go v0.19.2/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0= +github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4= +github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= -- 2.45.2 From def551cc895b078aa01714daeb49f8e943559a5a Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 22 Dec 2020 20:53:48 +0100 Subject: [PATCH 11/51] add option to run with RO-CF only as upstream increase idle timeout to avoid errors downstream add option to delete blobs from DB if storage doesn't have it (for future local tracking) --- cmd/peer.go | 2 +- cmd/reflector.go | 18 ++++-- cmd/start.go | 2 +- cmd/upload.go | 2 +- go.mod | 5 +- go.sum | 149 +++++++++++++++++++++++++++++++++++++++++++ peer/http3/server.go | 2 +- peer/http3/store.go | 2 +- store/dbbacked.go | 23 +++++-- 9 files changed, 188 insertions(+), 17 deletions(-) diff --git a/cmd/peer.go b/cmd/peer.go index 3cb2a19..86af279 100644 --- a/cmd/peer.go +++ b/cmd/peer.go @@ -37,7 +37,7 @@ func peerCmd(cmd *cobra.Command, args []string) { err = db.Connect(globalConfig.DBConn) checkErr(err) - combo := store.NewDBBackedStore(s3, db) + combo := store.NewDBBackedStore(s3, db, false) peerServer = peer.NewServer(combo) } diff --git a/cmd/reflector.go b/cmd/reflector.go index 7551f77..58c3f6f 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -127,11 +127,21 @@ func setupStore() store.BlobStore { log.Fatalf("protocol is not recognized: %s", proxyProtocol) } } else { - s3Store := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName) + var s3Store *store.S3Store + if conf != "none" { + s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName) + } if cloudFrontEndpoint != "" { - s = store.NewCloudFrontRWStore(store.NewCloudFrontROStore(cloudFrontEndpoint), s3Store) - } else { + cfs := store.NewCloudFrontROStore(cloudFrontEndpoint) + if s3Store != nil { + s = store.NewCloudFrontRWStore(cfs, s3Store) + } else { + s = cfs + } + } else if s3Store != nil { s = s3Store + } else { + log.Fatalf("this configuration does not include a valid upstream source") } } @@ -143,7 +153,7 @@ func setupStore() store.BlobStore { log.Fatal(err) } - s = store.NewDBBackedStore(s, db) + s = store.NewDBBackedStore(s, db, false) } return s diff --git a/cmd/start.go b/cmd/start.go index 4d4a56b..476e9fd 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -56,7 +56,7 @@ func startCmd(cmd *cobra.Command, args []string) { err := db.Connect(globalConfig.DBConn) checkErr(err) s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName) - comboStore := store.NewDBBackedStore(s3, db) + comboStore := store.NewDBBackedStore(s3, db, false) conf := prism.DefaultConf() diff --git a/cmd/upload.go b/cmd/upload.go index 636e824..e5e2cec 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -36,7 +36,7 @@ func uploadCmd(cmd *cobra.Command, args []string) { st := store.NewDBBackedStore( store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName), - db) + db, false) uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload) diff --git a/go.mod b/go.mod index 04d07c2..d3384a1 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/go-sql-driver/mysql v1.4.1 github.com/golang/protobuf v1.4.2 github.com/google/btree v1.0.0 // indirect - github.com/google/gops v0.3.7 + github.com/google/gops v0.3.7 // indirect github.com/gorilla/mux v1.7.4 github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/golang-lru v0.5.4 @@ -28,12 +28,13 @@ require ( github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec github.com/lucas-clemente/quic-go v0.19.3 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 - github.com/prometheus/client_golang v0.9.2 + github.com/prometheus/client_golang v0.9.3 github.com/sirupsen/logrus v1.4.2 github.com/spf13/afero v1.4.1 // indirect github.com/spf13/cast v1.3.0 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect + github.com/spf13/viper v1.7.1 // indirect github.com/stretchr/testify v1.4.0 github.com/volatiletech/null v8.0.0+incompatible go.uber.org/atomic v1.5.1 diff --git a/go.sum b/go.sum index 18420b2..b7dbdb2 100644 --- a/go.sum +++ b/go.sum @@ -2,16 +2,31 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -24,7 +39,10 @@ github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyEleg github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bparli/lfuda-go v0.3.1 h1:nO9Szo627RC8/z+R+MMPBItNwHCOonchmpjQuQi8jVY= github.com/bparli/lfuda-go v0.3.1/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -43,17 +61,24 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak= github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= @@ -68,23 +93,31 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw= github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= @@ -116,8 +149,12 @@ github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQl github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000= @@ -134,11 +171,19 @@ github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlI github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -147,6 +192,7 @@ github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -155,10 +201,13 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce h1:xdsDDbiBDQTKASoGEZ+pEmF1OnWuu8AQ9I8iNbHNeno= github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= @@ -182,6 +231,7 @@ github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4 github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo= github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY= github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -189,17 +239,20 @@ github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro= github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -234,6 +287,8 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcX github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= @@ -250,6 +305,8 @@ github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00v github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -258,6 +315,7 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= @@ -265,6 +323,7 @@ github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA= github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -293,16 +352,30 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -351,10 +424,15 @@ github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUr github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.1 h1:Lt3ihYMlE+lreX1GS4Qw4ZsNpYQLxIXKBTEOXm3nt6I= github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= @@ -364,18 +442,25 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig= github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.0.2 h1:Ncr3ZIuJn322w2k1qmzXDnkLAdQMlJqBa9kfAH+irso= github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= @@ -386,15 +471,21 @@ github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dl github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ= github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU= github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM= go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -403,6 +494,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -412,13 +504,26 @@ golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -428,11 +533,14 @@ golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -445,6 +553,7 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -461,13 +570,18 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -487,19 +601,30 @@ golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -510,10 +635,17 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.2 h1:j8RI1yW0SkI+paT6uGwMlrMI/6zwYA6/CFil8rxOzGI= google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -522,13 +654,21 @@ google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -537,10 +677,12 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= @@ -548,10 +690,14 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8= gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= @@ -564,7 +710,10 @@ grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJd honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/peer/http3/server.go b/peer/http3/server.go index 55ab5fc..aaadac0 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -65,7 +65,7 @@ func (s *Server) Start(address string) error { log.Println("HTTP3 peer listening on " + address) quicConf := &quic.Config{ HandshakeTimeout: 4 * time.Second, - MaxIdleTimeout: 10 * time.Second, + MaxIdleTimeout: 20 * time.Second, } r := mux.NewRouter() r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) { diff --git a/peer/http3/store.go b/peer/http3/store.go index 4749ab0..76a58c1 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -32,7 +32,7 @@ func NewStore(opts StoreOpts) *Store { func (p *Store) getClient() (*Client, error) { var qconf quic.Config qconf.HandshakeTimeout = 4 * time.Second - qconf.MaxIdleTimeout = 10 * time.Second + qconf.MaxIdleTimeout = 20 * time.Second pool, err := x509.SystemCertPool() if err != nil { return nil, err diff --git a/store/dbbacked.go b/store/dbbacked.go index 59c47b5..25cc446 100644 --- a/store/dbbacked.go +++ b/store/dbbacked.go @@ -14,15 +14,16 @@ import ( // DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store. type DBBackedStore struct { - blobs BlobStore - db *db.SQL - blockedMu sync.RWMutex - blocked map[string]bool + blobs BlobStore + db *db.SQL + blockedMu sync.RWMutex + blocked map[string]bool + deleteOnMiss bool } // NewDBBackedStore returns an initialized store pointer. -func NewDBBackedStore(blobs BlobStore, db *db.SQL) *DBBackedStore { - return &DBBackedStore{blobs: blobs, db: db} +func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore { + return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss} } const nameDBBacked = "db-backed" @@ -44,6 +45,16 @@ func (d *DBBackedStore) Get(hash string) (stream.Blob, error) { if !has { return nil, ErrBlobNotFound } + if d.deleteOnMiss { + b, err := d.blobs.Get(hash) + if err != nil && errors.Is(err, ErrBlobNotFound) { + e2 := d.Delete(hash) + if e2 != nil { + log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err)) + } + return b, err + } + } return d.blobs.Get(hash) } -- 2.45.2 From 869030fc584ce8db93f2817e56f604eefa52c391 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 22 Dec 2020 21:19:48 +0100 Subject: [PATCH 12/51] address some review comments --- store/lfuda.go | 23 ++++++++++++----------- store/lfuda_test.go | 3 --- store/lru.go | 4 +++- store/speedwalk/speedwalk.go | 1 - 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/store/lfuda.go b/store/lfuda.go index e33afa0..cfdb155 100644 --- a/store/lfuda.go +++ b/store/lfuda.go @@ -40,8 +40,6 @@ func NewLFUDAStore(component string, store BlobStore, maxSize float64) *LFUDASto const nameLFUDA = "lfuda" -var fakeTrue = []byte{'t'} - // Name is the cache type name func (l *LFUDAStore) Name() string { return nameLFUDA } @@ -66,7 +64,7 @@ func (l *LFUDAStore) Get(hash string) (stream.Blob, error) { // Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!! func (l *LFUDAStore) Put(hash string, blob stream.Blob) error { - l.lfuda.Set(hash, fakeTrue) + l.lfuda.Set(hash, true) has, _ := l.Has(hash) if has { err := l.store.Put(hash, blob) @@ -77,14 +75,16 @@ func (l *LFUDAStore) Put(hash string, blob stream.Blob) error { return nil } -// PutSD stores the sd blob +// PutSD stores the sd blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!! func (l *LFUDAStore) PutSD(hash string, blob stream.Blob) error { - err := l.store.PutSD(hash, blob) - if err != nil { - return err + l.lfuda.Set(hash, true) + has, _ := l.Has(hash) + if has { + err := l.store.PutSD(hash, blob) + if err != nil { + return err + } } - - l.lfuda.Set(hash, fakeTrue) return nil } @@ -109,12 +109,13 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error { if err != nil { return err } + logrus.Infof("read %d files from disk", len(existing)) added := 0 for _, h := range existing { - l.lfuda.Set(h, fakeTrue) + l.lfuda.Set(h, true) added++ - if maxItems > 0 && added >= maxItems { // underlying cache is bigger than LRU cache + if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache break } } diff --git a/store/lfuda_test.go b/store/lfuda_test.go index 9b1eb58..d5eae47 100644 --- a/store/lfuda_test.go +++ b/store/lfuda_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/lbryio/lbry.go/v2/extras/errors" - log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -68,8 +67,6 @@ func TestFUDAStore_Eviction(t *testing.T) { assert.Equal(t, cacheMaxBlobs, len(mem.Debug())) - keys := lfuda.lfuda.Keys() - log.Infof("%+v", keys) for k, v := range map[string]bool{ "one": false, "two": true, diff --git a/store/lru.go b/store/lru.go index edbe3a5..071e2b9 100644 --- a/store/lru.go +++ b/store/lru.go @@ -6,6 +6,7 @@ import ( "github.com/lbryio/reflector.go/internal/metrics" golru "github.com/hashicorp/golang-lru" + "github.com/sirupsen/logrus" ) // LRUStore adds a max cache size and LRU eviction to a BlobStore @@ -106,11 +107,12 @@ func (l *LRUStore) Delete(hash string) error { // loadExisting imports existing blobs from the underlying store into the LRU cache func (l *LRUStore) loadExisting(store lister, maxItems int) error { + logrus.Infof("loading at most %d items", maxItems) existing, err := store.list() if err != nil { return err } - + logrus.Infof("read %d files from disk", len(existing)) added := 0 for _, h := range existing { l.lru.Add(h, true) diff --git a/store/speedwalk/speedwalk.go b/store/speedwalk/speedwalk.go index 23230df..9b899e4 100644 --- a/store/speedwalk/speedwalk.go +++ b/store/speedwalk/speedwalk.go @@ -83,6 +83,5 @@ func AllFiles(startDir string, basename bool) ([]string, error) { close(pathChan) pathWG.Wait() - logrus.Infoln("loaded LRU") return paths, nil } -- 2.45.2 From 03304312e8e4f032bf135696e2113c1f3d5f5ff6 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 23 Dec 2020 06:04:42 +0100 Subject: [PATCH 13/51] add PoC for litedb to avoid all the overhead --- cmd/reflector.go | 11 +- lite_db/db.go | 348 +++++++++++++++++++++++++++++++++++++++++ peer/http3/store.go | 5 + peer/store.go | 5 + store/caching.go | 9 +- store/cloudfront_ro.go | 5 + store/cloudfront_rw.go | 7 + store/dbbacked.go | 6 + store/disk.go | 5 + store/lfuda.go | 7 +- store/litedbbacked.go | 160 +++++++++++++++++++ store/lru.go | 5 + store/memory.go | 5 + store/noop.go | 1 + store/s3.go | 5 + store/singleflight.go | 6 + store/store.go | 2 + 17 files changed, 589 insertions(+), 3 deletions(-) create mode 100644 lite_db/db.go create mode 100644 store/litedbbacked.go diff --git a/cmd/reflector.go b/cmd/reflector.go index 58c3f6f..3bb6b4e 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -10,6 +10,7 @@ import ( "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/lite_db" "github.com/lbryio/reflector.go/meta" "github.com/lbryio/reflector.go/peer" "github.com/lbryio/reflector.go/peer/http3" @@ -101,6 +102,8 @@ func reflectorCmd(cmd *cobra.Command, args []string) { metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics") metricsServer.Start() defer metricsServer.Shutdown() + defer underlyingStore.Shutdown() + defer outerStore.Shutdown() interruptChan := make(chan os.Signal, 1) signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM) @@ -171,10 +174,16 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { if err != nil { log.Fatal(err) } + localDb := new(lite_db.SQL) + localDb.TrackAccessTime = true + err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector") + if err != nil { + log.Fatal(err) + } wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), realCacheSize), + store.NewLiteDBBackedStore("hdd", store.NewDiskStore(diskCachePath, 2), localDb, int(realCacheSize)), ) } diff --git a/lite_db/db.go b/lite_db/db.go new file mode 100644 index 0000000..6be9837 --- /dev/null +++ b/lite_db/db.go @@ -0,0 +1,348 @@ +package lite_db + +import ( + "database/sql" + "time" + + "github.com/lbryio/lbry.go/v2/extras/errors" + qt "github.com/lbryio/lbry.go/v2/extras/query" + + "github.com/go-sql-driver/mysql" + _ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used + log "github.com/sirupsen/logrus" + "github.com/volatiletech/null" +) + +// SdBlob is a special blob that contains information on the rest of the blobs in the stream +type SdBlob struct { + StreamName string `json:"stream_name"` + Blobs []struct { + Length int `json:"length"` + BlobNum int `json:"blob_num"` + BlobHash string `json:"blob_hash,omitempty"` + IV string `json:"iv"` + } `json:"blobs"` + StreamType string `json:"stream_type"` + Key string `json:"key"` + SuggestedFileName string `json:"suggested_file_name"` + StreamHash string `json:"stream_hash"` +} + +// SQL implements the DB interface +type SQL struct { + conn *sql.DB + + TrackAccessTime bool +} + +func logQuery(query string, args ...interface{}) { + s, err := qt.InterpolateParams(query, args...) + if err != nil { + log.Errorln(err) + } else { + log.Debugln(s) + } +} + +// Connect will create a connection to the database +func (s *SQL) Connect(dsn string) error { + var err error + // interpolateParams is necessary. otherwise uploading a stream with thousands of blobs + // will hit MySQL's max_prepared_stmt_count limit because the prepared statements are all + // opened inside a transaction. closing them manually doesn't seem to help + dsn += "?parseTime=1&collation=utf8mb4_unicode_ci&interpolateParams=1" + s.conn, err = sql.Open("mysql", dsn) + if err != nil { + return errors.Err(err) + } + + s.conn.SetMaxIdleConns(12) + + return errors.Err(s.conn.Ping()) +} + +// AddBlob adds a blob to the database. +func (s *SQL) AddBlob(hash string, length int) error { + if s.conn == nil { + return errors.Err("not connected") + } + + _, err := s.insertBlob(hash, length) + return err +} + +func (s *SQL) insertBlob(hash string, length int) (int64, error) { + if length <= 0 { + return 0, errors.Err("length must be positive") + } + const isStored = true + now := time.Now() + args := []interface{}{hash, isStored, length, now} + blobID, err := s.exec( + "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at=VALUES(last_accessed_at)", + args..., + ) + if err != nil { + return 0, err + } + + if blobID == 0 { + err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID) + if err != nil { + return 0, errors.Err(err) + } + if blobID == 0 { + return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing") + } + } + + return blobID, nil +} + +// HasBlob checks if the database contains the blob information. +func (s *SQL) HasBlob(hash string) (bool, error) { + exists, err := s.HasBlobs([]string{hash}) + if err != nil { + return false, err + } + return exists[hash], nil +} + +// HasBlobs checks if the database contains the set of blobs and returns a bool map. +func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) { + exists, streamsNeedingTouch, err := s.hasBlobs(hashes) + s.touch(streamsNeedingTouch) + return exists, err +} + +func (s *SQL) touch(blobIDs []uint64) error { + if len(blobIDs) == 0 { + return nil + } + + query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")" + args := make([]interface{}, len(blobIDs)+1) + args[0] = time.Now() + for i := range blobIDs { + args[i+1] = blobIDs[i] + } + + startTime := time.Now() + _, err := s.exec(query, args...) + log.Debugf("blobs access query touched %d blobs and took %s", len(blobIDs), time.Since(startTime)) + return errors.Err(err) +} + +func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) { + if s.conn == nil { + return nil, nil, errors.Err("not connected") + } + + var ( + hash string + blobID uint64 + lastAccessedAt null.Time + ) + + var needsTouch []uint64 + exists := make(map[string]bool) + + touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time + maxBatchSize := 10000 + doneIndex := 0 + + for len(hashes) > doneIndex { + sliceEnd := doneIndex + maxBatchSize + if sliceEnd > len(hashes) { + sliceEnd = len(hashes) + } + log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes)) + batch := hashes[doneIndex:sliceEnd] + + // TODO: this query doesn't work for SD blobs, which are not in the stream_blob table + + query := `SELECT hash, id, last_accessed_at +FROM blob_ +WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)` + args := make([]interface{}, len(batch)+1) + args[0] = true + for i := range batch { + args[i+1] = batch[i] + } + + logQuery(query, args...) + + err := func() error { + startTime := time.Now() + rows, err := s.conn.Query(query, args...) + log.Debugf("hashes query took %s", time.Since(startTime)) + if err != nil { + return errors.Err(err) + } + defer closeRows(rows) + + for rows.Next() { + err := rows.Scan(&hash, &blobID, &lastAccessedAt) + if err != nil { + return errors.Err(err) + } + exists[hash] = true + if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) { + needsTouch = append(needsTouch, blobID) + } + } + + err = rows.Err() + if err != nil { + return errors.Err(err) + } + + doneIndex += len(batch) + return nil + }() + if err != nil { + return nil, nil, err + } + } + + return exists, needsTouch, nil +} + +// Delete will remove the blob from the db +func (s *SQL) Delete(hash string) error { + _, err := s.exec("UPDATE blob_ set is_stored = ? WHERE hash = ?", 0, hash) + return errors.Err(err) +} + +// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored", +// but they are tracked so reflector knows what it is missing. +func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int) error { + if s.conn == nil { + return errors.Err("not connected") + } + + _, err := s.insertBlob(sdHash, sdBlobLength) + return err +} + +// GetHashRange gets the smallest and biggest hashes in the db +func (s *SQL) GetLRUBlobs(maxBlobs int) ([]string, error) { + if s.conn == nil { + return nil, errors.Err("not connected") + } + + query := "SELECT hash from blob_ where is_stored = ? order by last_accessed_at limit ?" + const isStored = true + logQuery(query, isStored, maxBlobs) + rows, err := s.conn.Query(query, isStored, maxBlobs) + if err != nil { + return nil, errors.Err(err) + } + defer closeRows(rows) + blobs := make([]string, 0, maxBlobs) + for rows.Next() { + var hash string + err := rows.Scan(&hash) + if err != nil { + return nil, errors.Err(err) + } + blobs = append(blobs, hash) + } + return blobs, nil +} + +func (s *SQL) AllBlobs() ([]string, error) { + if s.conn == nil { + return nil, errors.Err("not connected") + } + + query := "SELECT hash from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense? + const isStored = true + logQuery(query, isStored) + rows, err := s.conn.Query(query, isStored) + if err != nil { + return nil, errors.Err(err) + } + defer closeRows(rows) + totalBlobs, err := s.BlobsCount() + if err != nil { + return nil, err + } + blobs := make([]string, 0, totalBlobs) + for rows.Next() { + var hash string + err := rows.Scan(&hash) + if err != nil { + return nil, errors.Err(err) + } + blobs = append(blobs, hash) + } + return blobs, nil +} + +func (s *SQL) BlobsCount() (int, error) { + if s.conn == nil { + return 0, errors.Err("not connected") + } + + query := "SELECT count(id) from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense? + const isStored = true + logQuery(query, isStored) + var count int + err := s.conn.QueryRow(query, isStored).Scan(&count) + return count, errors.Err(err) +} + +func closeRows(rows *sql.Rows) { + if rows != nil { + err := rows.Close() + if err != nil { + log.Error("error closing rows: ", err) + } + } +} + +func (s *SQL) exec(query string, args ...interface{}) (int64, error) { + logQuery(query, args...) + attempt, maxAttempts := 0, 3 +Retry: + attempt++ + result, err := s.conn.Exec(query, args...) + if isLockTimeoutError(err) { + if attempt <= maxAttempts { + //Error 1205: Lock wait timeout exceeded; try restarting transaction + goto Retry + } + err = errors.Prefix("Lock timeout for query "+query, err) + } + + if err != nil { + return 0, errors.Err(err) + } + + lastID, err := result.LastInsertId() + return lastID, errors.Err(err) +} + +func isLockTimeoutError(err error) bool { + e, ok := err.(*mysql.MySQLError) + return ok && e != nil && e.Number == 1205 +} + +/* SQL schema + +in prod make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space. + +CREATE TABLE `blob_` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `hash` char(96) NOT NULL, + `is_stored` tinyint(1) NOT NULL DEFAULT '0', + `length` bigint unsigned DEFAULT NULL, + `last_accessed_at` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`), + UNIQUE KEY `blob_hash_idx` (`hash`), + KEY `blob_last_accessed_idx` (`last_accessed_at`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 + +*/ diff --git a/peer/http3/store.go b/peer/http3/store.go index 76a58c1..b681443 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -91,3 +91,8 @@ func (p *Store) PutSD(hash string, blob stream.Blob) error { func (p *Store) Delete(hash string) error { panic("http3Store cannot put or delete blobs") } + +// Delete is not supported +func (p *Store) Shutdown() { + return +} diff --git a/peer/store.go b/peer/store.go index b8abedd..3ef9622 100644 --- a/peer/store.go +++ b/peer/store.go @@ -66,3 +66,8 @@ func (p *Store) PutSD(hash string, blob stream.Blob) error { func (p *Store) Delete(hash string) error { panic("PeerStore cannot put or delete blobs") } + +// Delete is not supported +func (p *Store) Shutdown() { + return +} diff --git a/store/caching.go b/store/caching.go index c8e7d45..53b692c 100644 --- a/store/caching.go +++ b/store/caching.go @@ -68,7 +68,7 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) { go func() { err = c.cache.Put(hash, blob) if err != nil { - log.Errorf("error saving blob to underlying cache: %s", err.Error()) + log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err)) } }() return blob, nil @@ -100,3 +100,10 @@ func (c *CachingStore) Delete(hash string) error { } return c.cache.Delete(hash) } + +// Shutdown shuts down the store gracefully +func (c *CachingStore) Shutdown() { + c.origin.Shutdown() + c.cache.Shutdown() + return +} diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go index a914285..3dcffc0 100644 --- a/store/cloudfront_ro.go +++ b/store/cloudfront_ro.go @@ -103,3 +103,8 @@ func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error { func (c *CloudFrontROStore) Delete(_ string) error { panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore") } + +// Shutdown shuts down the store gracefully +func (c *CloudFrontROStore) Shutdown() { + return +} diff --git a/store/cloudfront_rw.go b/store/cloudfront_rw.go index ee771da..6de64af 100644 --- a/store/cloudfront_rw.go +++ b/store/cloudfront_rw.go @@ -48,3 +48,10 @@ func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error { func (c *CloudFrontRWStore) Delete(hash string) error { return c.s3.Delete(hash) } + +// Shutdown shuts down the store gracefully +func (c *CloudFrontRWStore) Shutdown() { + c.s3.Shutdown() + c.cf.Shutdown() + return +} diff --git a/store/dbbacked.go b/store/dbbacked.go index 25cc446..0ebaaed 100644 --- a/store/dbbacked.go +++ b/store/dbbacked.go @@ -193,3 +193,9 @@ func (d *DBBackedStore) initBlocked() error { return err } + +// Shutdown shuts down the store gracefully +func (d *DBBackedStore) Shutdown() { + d.blobs.Shutdown() + return +} diff --git a/store/disk.go b/store/disk.go index 2234971..03cc393 100644 --- a/store/disk.go +++ b/store/disk.go @@ -147,3 +147,8 @@ func (d *DiskStore) initOnce() error { d.initialized = true return nil } + +// Shutdown shuts down the store gracefully +func (d *DiskStore) Shutdown() { + return +} diff --git a/store/lfuda.go b/store/lfuda.go index cfdb155..0eb7296 100644 --- a/store/lfuda.go +++ b/store/lfuda.go @@ -109,7 +109,7 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error { if err != nil { return err } - logrus.Infof("read %d files from disk", len(existing)) + logrus.Infof("read %d files from underlying store", len(existing)) added := 0 for _, h := range existing { @@ -121,3 +121,8 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error { } return nil } + +// Shutdown shuts down the store gracefully +func (l *LFUDAStore) Shutdown() { + return +} diff --git a/store/litedbbacked.go b/store/litedbbacked.go new file mode 100644 index 0000000..93790d6 --- /dev/null +++ b/store/litedbbacked.go @@ -0,0 +1,160 @@ +package store + +import ( + "encoding/json" + "time" + + "github.com/lbryio/lbry.go/v2/extras/stop" + "github.com/lbryio/reflector.go/db" + "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/lite_db" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" + + log "github.com/sirupsen/logrus" +) + +// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store. +type LiteDBBackedStore struct { + blobs BlobStore + db *lite_db.SQL + maxItems int + stopper *stop.Stopper + component string +} + +// NewDBBackedStore returns an initialized store pointer. +func NewLiteDBBackedStore(component string, blobs BlobStore, db *lite_db.SQL, maxItems int) *LiteDBBackedStore { + instance := &LiteDBBackedStore{blobs: blobs, db: db, maxItems: maxItems, stopper: stop.New(), component: component} + go func() { + instance.selfClean() + }() + return instance +} + +const nameLiteDBBacked = "lite-db-backed" + +// Name is the cache type name +func (d *LiteDBBackedStore) Name() string { return nameDBBacked } + +// Has returns true if the blob is in the store +func (d *LiteDBBackedStore) Has(hash string) (bool, error) { + return d.db.HasBlob(hash) +} + +// Get gets the blob +func (d *LiteDBBackedStore) Get(hash string) (stream.Blob, error) { + has, err := d.db.HasBlob(hash) + if err != nil { + return nil, err + } + if !has { + return nil, ErrBlobNotFound + } + b, err := d.blobs.Get(hash) + if err != nil && errors.Is(err, ErrBlobNotFound) { + e2 := d.db.Delete(hash) + if e2 != nil { + log.Errorf("error while deleting blob from db: %s", errors.FullTrace(e2)) + } + return b, err + } + + return d.blobs.Get(hash) +} + +// Put stores the blob in the S3 store and stores the blob information in the DB. +func (d *LiteDBBackedStore) Put(hash string, blob stream.Blob) error { + err := d.blobs.Put(hash, blob) + if err != nil { + return err + } + + return d.db.AddBlob(hash, len(blob)) +} + +// PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if +// there is an error storing the blob information in the DB. +func (d *LiteDBBackedStore) PutSD(hash string, blob stream.Blob) error { + var blobContents db.SdBlob + err := json.Unmarshal(blob, &blobContents) + if err != nil { + return errors.Err(err) + } + if blobContents.StreamHash == "" { + return errors.Err("sd blob is missing stream hash") + } + + err = d.blobs.PutSD(hash, blob) + if err != nil { + return err + } + + return d.db.AddSDBlob(hash, len(blob)) +} + +func (d *LiteDBBackedStore) Delete(hash string) error { + err := d.blobs.Delete(hash) + if err != nil { + return err + } + + return d.db.Delete(hash) +} + +// list returns the hashes of blobs that already exist in the database +func (d *LiteDBBackedStore) list() ([]string, error) { + blobs, err := d.db.AllBlobs() + return blobs, err +} + +func (d *LiteDBBackedStore) selfClean() { + d.stopper.Add(1) + defer d.stopper.Done() + lastCleanup := time.Now() + const cleanupInterval = 10 * time.Second + for { + select { + case <-d.stopper.Ch(): + log.Infoln("stopping self cleanup") + return + default: + time.Sleep(1 * time.Second) + } + if time.Since(lastCleanup) < cleanupInterval { + continue + } + blobsCount, err := d.db.BlobsCount() + if err != nil { + log.Errorf(errors.FullTrace(err)) + } + if blobsCount >= d.maxItems { + itemsToDelete := blobsCount / 100 * 10 + blobs, err := d.db.GetLRUBlobs(itemsToDelete) + if err != nil { + log.Errorf(errors.FullTrace(err)) + } + for _, hash := range blobs { + select { + case <-d.stopper.Ch(): + return + default: + + } + err = d.Delete(hash) + if err != nil { + log.Errorf(errors.FullTrace(err)) + } + metrics.CacheLRUEvictCount.With(metrics.CacheLabels(d.Name(), d.component)).Inc() + } + } + lastCleanup = time.Now() + } +} + +// Shutdown shuts down the store gracefully +func (d *LiteDBBackedStore) Shutdown() { + d.stopper.StopAndWait() + return +} diff --git a/store/lru.go b/store/lru.go index 071e2b9..ce0d83d 100644 --- a/store/lru.go +++ b/store/lru.go @@ -123,3 +123,8 @@ func (l *LRUStore) loadExisting(store lister, maxItems int) error { } return nil } + +// Shutdown shuts down the store gracefully +func (l *LRUStore) Shutdown() { + return +} diff --git a/store/memory.go b/store/memory.go index f462a8d..62b93b6 100644 --- a/store/memory.go +++ b/store/memory.go @@ -71,3 +71,8 @@ func (m *MemStore) Debug() map[string]stream.Blob { defer m.mu.RUnlock() return m.blobs } + +// Shutdown shuts down the store gracefully +func (m *MemStore) Shutdown() { + return +} diff --git a/store/noop.go b/store/noop.go index 9e5d815..2792a0d 100644 --- a/store/noop.go +++ b/store/noop.go @@ -13,3 +13,4 @@ func (n *NoopStore) Get(_ string) (stream.Blob, error) { return nil, nil } func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil } func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil } func (n *NoopStore) Delete(_ string) error { return nil } +func (n *NoopStore) Shutdown() { return } diff --git a/store/s3.go b/store/s3.go index 53451be..02a23c5 100644 --- a/store/s3.go +++ b/store/s3.go @@ -158,3 +158,8 @@ func (s *S3Store) initOnce() error { s.session = sess return nil } + +// Shutdown shuts down the store gracefully +func (s *S3Store) Shutdown() { + return +} diff --git a/store/singleflight.go b/store/singleflight.go index cca1e2f..7d0139d 100644 --- a/store/singleflight.go +++ b/store/singleflight.go @@ -65,3 +65,9 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) { return blob, nil } } + +// Shutdown shuts down the store gracefully +func (s *singleflightStore) Shutdown() { + s.BlobStore.Shutdown() + return +} diff --git a/store/store.go b/store/store.go index 200ff61..36d388d 100644 --- a/store/store.go +++ b/store/store.go @@ -19,6 +19,8 @@ type BlobStore interface { PutSD(hash string, blob stream.Blob) error // Delete the blob from the store. Delete(hash string) error + // Shutdown the store gracefully + Shutdown() } // Blocklister is a store that supports blocking blobs to prevent their inclusion in the store. -- 2.45.2 From 3a1d9d33041b9d43adf2a08cb83e17014ce8a86e Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Wed, 23 Dec 2020 17:08:13 -0500 Subject: [PATCH 14/51] something like this --- cmd/reflector.go | 74 ++++++++++-- db/db.go | 265 ++++++++++++++++++++++++++++++++++++----- go.mod | 4 +- peer/http3/store.go | 5 + peer/store.go | 5 + store/caching.go | 9 +- store/caching_test.go | 4 + store/cloudfront_ro.go | 5 + store/cloudfront_rw.go | 7 ++ store/dbbacked.go | 9 +- store/disk.go | 5 + store/lfuda.go | 7 +- store/lru.go | 5 + store/memory.go | 5 + store/noop.go | 1 + store/s3.go | 5 + store/singleflight.go | 6 + store/store.go | 2 + 18 files changed, 380 insertions(+), 43 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index 58c3f6f..47a27ff 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -8,6 +8,8 @@ import ( "syscall" "time" + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/extras/stop" "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/meta" @@ -67,10 +69,11 @@ func init() { func reflectorCmd(cmd *cobra.Command, args []string) { log.Printf("reflector %s", meta.VersionString()) + cleanerStopper := stop.New() // the blocklist logic requires the db backed store to be the outer-most store underlyingStore := setupStore() - outerStore := wrapWithCache(underlyingStore) + outerStore := wrapWithCache(underlyingStore, cleanerStopper) if !disableUploads { reflectorServer := reflector.NewServer(underlyingStore) @@ -101,11 +104,14 @@ func reflectorCmd(cmd *cobra.Command, args []string) { metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics") metricsServer.Start() defer metricsServer.Shutdown() + defer outerStore.Shutdown() + defer underlyingStore.Shutdown() interruptChan := make(chan os.Signal, 1) signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM) <-interruptChan // deferred shutdowns happen now + cleanerStopper.StopAndWait() } func setupStore() store.BlobStore { @@ -146,20 +152,20 @@ func setupStore() store.BlobStore { } if useDB { - db := new(db.SQL) - db.TrackAccessTime = true - err := db.Connect(globalConfig.DBConn) + dbInst := new(db.SQL) + dbInst.TrackAccess = db.TrackAccessStreams + err := dbInst.Connect(globalConfig.DBConn) if err != nil { log.Fatal(err) } - s = store.NewDBBackedStore(s, db, false) + s = store.NewDBBackedStore(s, dbInst, false) } return s } -func wrapWithCache(s store.BlobStore) store.BlobStore { +func wrapWithCache(s store.BlobStore, cleanerStopper *stop.Group) store.BlobStore { wrapped := s diskCacheMaxSize, diskCachePath := diskCacheParams(reflectorCmdDiskCache) @@ -171,11 +177,20 @@ func wrapWithCache(s store.BlobStore) store.BlobStore { if err != nil { log.Fatal(err) } + + localDb := new(db.SQL) + localDb.TrackAccess = db.TrackAccessBlobs + err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector") + if err != nil { + log.Fatal(err) + } wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), realCacheSize), + store.NewDBBackedStore(store.NewDiskStore(diskCachePath, 2), localDb, false), ) + + go cleanOldestBlobs(int(realCacheSize), localDb, wrapped, cleanerStopper) } diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache) @@ -229,3 +244,48 @@ func diskCacheParams(diskParams string) (int, string) { } return int(maxSize), path } + +func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) { + const cleanupInterval = 10 * time.Second + + for { + select { + case <-stopper.Ch(): + log.Infoln("stopping self cleanup") + return + case <-time.After(cleanupInterval): + err := doClean(maxItems, db, store, stopper) + if err != nil { + log.Error(errors.FullTrace(err)) + } + } + } +} + +func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error { + blobsCount, err := db.Count() + if err != nil { + return err + } + + if blobsCount >= maxItems { + itemsToDelete := blobsCount / 10 + blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete) + if err != nil { + return err + } + + for _, hash := range blobs { + select { + case <-stopper.Ch(): + return nil + default: + } + + err = store.Delete(hash) + if err != nil { + return err + } + } + } +} diff --git a/db/db.go b/db/db.go index c3bea24..3ac2dab 100644 --- a/db/db.go +++ b/db/db.go @@ -30,11 +30,26 @@ type SdBlob struct { StreamHash string `json:"stream_hash"` } +type trackAccess int + +const ( + // Don't track accesses + TrackAccessNone trackAccess = iota + // Track accesses at the stream level + TrackAccessStreams + // Track accesses at the blob level + TrackAccessBlobs +) + // SQL implements the DB interface type SQL struct { conn *sql.DB - TrackAccessTime bool + // Track the approx last time a blob or stream was accessed + TrackAccess trackAccess + + // Instead of deleting a blob, marked it as not stored in the db + SoftDelete bool } func logQuery(query string, args ...interface{}) { @@ -78,11 +93,19 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) return 0, errors.Err("length must be positive") } - args := []interface{}{hash, isStored, length} - blobID, err := s.exec( - "INSERT INTO blob_ (hash, is_stored, length) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))", - args..., + var ( + q string + args []interface{} ) + if s.TrackAccess == TrackAccessBlobs { + args = []interface{}{hash, isStored, length, time.Now()} + q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)" + } else { + args = []interface{}{hash, isStored, length} + q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))" + } + + blobID, err := s.exec(q, args...) if err != nil { return 0, err } @@ -95,17 +118,33 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) if blobID == 0 { return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing") } + + if s.TrackAccess == TrackAccessBlobs { + err := s.touchBlobs([]uint64{uint64(blobID)}) + if err != nil { + return 0, errors.Err(err) + } + } } return blobID, nil } func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) { - args := []interface{}{hash, sdBlobID, time.Now()} - streamID, err := s.exec( - "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES ("+qt.Qs(len(args))+")", - args..., + var ( + q string + args []interface{} ) + + if s.TrackAccess == TrackAccessStreams { + args = []interface{}{hash, sdBlobID, time.Now()} + q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")" + } else { + args = []interface{}{hash, sdBlobID} + q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")" + } + + streamID, err := s.exec(q, args...) if err != nil { return 0, errors.Err(err) } @@ -119,8 +158,8 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) { return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing") } - if s.TrackAccessTime { - err := s.touch([]uint64{uint64(streamID)}) + if s.TrackAccess == TrackAccessStreams { + err := s.touchStreams([]uint64{uint64(streamID)}) if err != nil { return 0, errors.Err(err) } @@ -140,12 +179,36 @@ func (s *SQL) HasBlob(hash string) (bool, error) { // HasBlobs checks if the database contains the set of blobs and returns a bool map. func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) { - exists, streamsNeedingTouch, err := s.hasBlobs(hashes) - s.touch(streamsNeedingTouch) + exists, idsNeedingTouch, err := s.hasBlobs(hashes) + + if s.TrackAccess == TrackAccessBlobs { + s.touchBlobs(idsNeedingTouch) + } else if s.TrackAccess == TrackAccessStreams { + s.touchStreams(idsNeedingTouch) + } + return exists, err } -func (s *SQL) touch(streamIDs []uint64) error { +func (s *SQL) touchBlobs(blobIDs []uint64) error { + if len(blobIDs) == 0 { + return nil + } + + query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")" + args := make([]interface{}, len(blobIDs)+1) + args[0] = time.Now() + for i := range blobIDs { + args[i+1] = blobIDs[i] + } + + startTime := time.Now() + _, err := s.exec(query, args...) + log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime)) + return errors.Err(err) +} + +func (s *SQL) touchStreams(streamIDs []uint64) error { if len(streamIDs) == 0 { return nil } @@ -159,7 +222,7 @@ func (s *SQL) touch(streamIDs []uint64) error { startTime := time.Now() _, err := s.exec(query, args...) - log.Debugf("stream access query touched %d streams and took %s", len(streamIDs), time.Since(startTime)) + log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime)) return errors.Err(err) } @@ -169,9 +232,9 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) { } var ( - hash string - streamID uint64 - lastAccessedAt null.Time + hash string + blobID, streamID uint64 + lastAccessedAt null.Time ) var needsTouch []uint64 @@ -189,17 +252,23 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) { log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes)) batch := hashes[doneIndex:sliceEnd] - // TODO: this query doesn't work for SD blobs, which are not in the stream_blob table + var lastAccessedAtSelect string + if s.TrackAccess == TrackAccessBlobs { + lastAccessedAtSelect = "b.last_accessed_at" + } else if s.TrackAccess == TrackAccessStreams { + lastAccessedAtSelect = "s.last_accessed_at" + } else { + lastAccessedAtSelect = "NULL" + } - query := `SELECT b.hash, s.id, s.last_accessed_at + query := `SELECT b.hash, b.id, s.id, ` + lastAccessedAtSelect + ` FROM blob_ b LEFT JOIN stream_blob sb ON b.id = sb.blob_id -INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id) -WHERE b.is_stored = ? and b.hash IN (` + qt.Qs(len(batch)) + `)` - args := make([]interface{}, len(batch)+1) - args[0] = true +LEFT JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id) +WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)` + args := make([]interface{}, len(batch)) for i := range batch { - args[i+1] = batch[i] + args[i] = batch[i] } logQuery(query, args...) @@ -214,13 +283,17 @@ WHERE b.is_stored = ? and b.hash IN (` + qt.Qs(len(batch)) + `)` defer closeRows(rows) for rows.Next() { - err := rows.Scan(&hash, &streamID, &lastAccessedAt) + err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt) if err != nil { return errors.Err(err) } exists[hash] = true - if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) { - needsTouch = append(needsTouch, streamID) + if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) { + if s.TrackAccess == TrackAccessBlobs { + needsTouch = append(needsTouch, blobID) + } else if s.TrackAccess == TrackAccessStreams { + needsTouch = append(needsTouch, streamID) + } } } @@ -240,8 +313,14 @@ WHERE b.is_stored = ? and b.hash IN (` + qt.Qs(len(batch)) + `)` return exists, needsTouch, nil } -// Delete will remove the blob from the db +// Delete will remove (or soft-delete) the blob from the db +// NOTE: If SoftDelete is enabled, streams will never be deleted func (s *SQL) Delete(hash string) error { + if s.SoftDelete { + _, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash) + return errors.Err(err) + } + _, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash) if err != nil { return errors.Err(err) @@ -251,6 +330,88 @@ func (s *SQL) Delete(hash string) error { return errors.Err(err) } +// GetHashRange gets the smallest and biggest hashes in the db +func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) { + if s.conn == nil { + return nil, errors.Err("not connected") + } + + if s.TrackAccess != TrackAccessBlobs { + return nil, errors.Err("blob access tracking is disabled") + } + + query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?" + logQuery(query, maxBlobs) + + rows, err := s.conn.Query(query, maxBlobs) + if err != nil { + return nil, errors.Err(err) + } + defer closeRows(rows) + + blobs := make([]string, 0, maxBlobs) + for rows.Next() { + var hash string + err := rows.Scan(&hash) + if err != nil { + return nil, errors.Err(err) + } + blobs = append(blobs, hash) + } + + return blobs, nil +} + +// AllHashes writes all hashes from the db into the channel. +// It does not close the channel when it finishes. +//func (s *SQL) AllHashes(ch chan<- string) error { +// if s.conn == nil { +// return errors.Err("not connected") +// } +// +// query := "SELECT hash from blob_" +// if s.SoftDelete { +// query += " where is_stored = 1" +// } +// logQuery(query) +// +// rows, err := s.conn.Query(query) +// if err != nil { +// return errors.Err(err) +// } +// defer closeRows(rows) +// +// for rows.Next() { +// var hash string +// err := rows.Scan(&hash) +// if err != nil { +// return errors.Err(err) +// } +// ch <- hash +// // TODO: this needs testing +// // TODO: need a way to cancel this early (e.g. in case of shutdown) +// } +// +// close(ch) +// return nil +//} + +func (s *SQL) Count() (int, error) { + if s.conn == nil { + return 0, errors.Err("not connected") + } + + query := "SELECT count(id) from blob_" + if s.SoftDelete { + query += " where is_stored = 1" + } + logQuery(query) + + var count int + err := s.conn.QueryRow(query).Scan(&count) + return count, errors.Err(err) +} + // Block will mark a blob as blocked func (s *SQL) Block(hash string) error { query := "INSERT IGNORE INTO blocked SET hash = ?" @@ -528,8 +689,10 @@ CREATE TABLE blob_ ( hash char(96) NOT NULL, is_stored TINYINT(1) NOT NULL DEFAULT 0, length bigint(20) unsigned DEFAULT NULL, + last_accessed_at TIMESTAMP NULL DEFAULT NULL, PRIMARY KEY (id), UNIQUE KEY blob_hash_idx (hash) + KEY `blob_last_accessed_idx` (`last_accessed_at`) ); CREATE TABLE stream ( @@ -560,3 +723,47 @@ CREATE TABLE blocked ( ); */ + +//func (d *LiteDBBackedStore) selfClean() { +// d.stopper.Add(1) +// defer d.stopper.Done() +// lastCleanup := time.Now() +// const cleanupInterval = 10 * time.Second +// for { +// select { +// case <-d.stopper.Ch(): +// log.Infoln("stopping self cleanup") +// return +// default: +// time.Sleep(1 * time.Second) +// } +// if time.Since(lastCleanup) < cleanupInterval { +// continue +// +// blobsCount, err := d.db.BlobsCount() +// if err != nil { +// log.Errorf(errors.FullTrace(err)) +// } +// if blobsCount >= d.maxItems { +// itemsToDelete := blobsCount / 100 * 10 +// blobs, err := d.db.GetLRUBlobs(itemsToDelete) +// if err != nil { +// log.Errorf(errors.FullTrace(err)) +// } +// for _, hash := range blobs { +// select { +// case <-d.stopper.Ch(): +// return +// default: +// +// } +// err = d.Delete(hash) +// if err != nil { +// log.Errorf(errors.FullTrace(err)) +// } +// metrics.CacheLRUEvictCount.With(metrics.CacheLabels(d.Name(), d.component)).Inc() +// } +// } +// lastCleanup = time.Now() +// } +//} diff --git a/go.mod b/go.mod index d3384a1..4570f1d 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/go-sql-driver/mysql v1.4.1 github.com/golang/protobuf v1.4.2 - github.com/google/btree v1.0.0 // indirect - github.com/google/gops v0.3.7 // indirect + github.com/google/gops v0.3.7 github.com/gorilla/mux v1.7.4 github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/golang-lru v0.5.4 @@ -33,7 +32,6 @@ require ( github.com/spf13/afero v1.4.1 // indirect github.com/spf13/cast v1.3.0 github.com/spf13/cobra v0.0.3 - github.com/spf13/pflag v1.0.3 // indirect github.com/spf13/viper v1.7.1 // indirect github.com/stretchr/testify v1.4.0 github.com/volatiletech/null v8.0.0+incompatible diff --git a/peer/http3/store.go b/peer/http3/store.go index 76a58c1..b681443 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -91,3 +91,8 @@ func (p *Store) PutSD(hash string, blob stream.Blob) error { func (p *Store) Delete(hash string) error { panic("http3Store cannot put or delete blobs") } + +// Delete is not supported +func (p *Store) Shutdown() { + return +} diff --git a/peer/store.go b/peer/store.go index b8abedd..3ef9622 100644 --- a/peer/store.go +++ b/peer/store.go @@ -66,3 +66,8 @@ func (p *Store) PutSD(hash string, blob stream.Blob) error { func (p *Store) Delete(hash string) error { panic("PeerStore cannot put or delete blobs") } + +// Delete is not supported +func (p *Store) Shutdown() { + return +} diff --git a/store/caching.go b/store/caching.go index c8e7d45..53b692c 100644 --- a/store/caching.go +++ b/store/caching.go @@ -68,7 +68,7 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) { go func() { err = c.cache.Put(hash, blob) if err != nil { - log.Errorf("error saving blob to underlying cache: %s", err.Error()) + log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err)) } }() return blob, nil @@ -100,3 +100,10 @@ func (c *CachingStore) Delete(hash string) error { } return c.cache.Delete(hash) } + +// Shutdown shuts down the store gracefully +func (c *CachingStore) Shutdown() { + c.origin.Shutdown() + c.cache.Shutdown() + return +} diff --git a/store/caching_test.go b/store/caching_test.go index 1168a92..0636583 100644 --- a/store/caching_test.go +++ b/store/caching_test.go @@ -168,3 +168,7 @@ func (s *SlowBlobStore) Delete(hash string) error { time.Sleep(s.delay) return s.mem.Delete(hash) } + +func (s *SlowBlobStore) Shutdown() { + return +} diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go index a914285..3dcffc0 100644 --- a/store/cloudfront_ro.go +++ b/store/cloudfront_ro.go @@ -103,3 +103,8 @@ func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error { func (c *CloudFrontROStore) Delete(_ string) error { panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore") } + +// Shutdown shuts down the store gracefully +func (c *CloudFrontROStore) Shutdown() { + return +} diff --git a/store/cloudfront_rw.go b/store/cloudfront_rw.go index ee771da..6de64af 100644 --- a/store/cloudfront_rw.go +++ b/store/cloudfront_rw.go @@ -48,3 +48,10 @@ func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error { func (c *CloudFrontRWStore) Delete(hash string) error { return c.s3.Delete(hash) } + +// Shutdown shuts down the store gracefully +func (c *CloudFrontRWStore) Shutdown() { + c.s3.Shutdown() + c.cf.Shutdown() + return +} diff --git a/store/dbbacked.go b/store/dbbacked.go index 25cc446..1c5088a 100644 --- a/store/dbbacked.go +++ b/store/dbbacked.go @@ -4,10 +4,9 @@ import ( "encoding/json" "sync" - "github.com/lbryio/reflector.go/db" - "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/db" log "github.com/sirupsen/logrus" ) @@ -193,3 +192,9 @@ func (d *DBBackedStore) initBlocked() error { return err } + +// Shutdown shuts down the store gracefully +func (d *DBBackedStore) Shutdown() { + d.blobs.Shutdown() + return +} diff --git a/store/disk.go b/store/disk.go index 2234971..03cc393 100644 --- a/store/disk.go +++ b/store/disk.go @@ -147,3 +147,8 @@ func (d *DiskStore) initOnce() error { d.initialized = true return nil } + +// Shutdown shuts down the store gracefully +func (d *DiskStore) Shutdown() { + return +} diff --git a/store/lfuda.go b/store/lfuda.go index cfdb155..0eb7296 100644 --- a/store/lfuda.go +++ b/store/lfuda.go @@ -109,7 +109,7 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error { if err != nil { return err } - logrus.Infof("read %d files from disk", len(existing)) + logrus.Infof("read %d files from underlying store", len(existing)) added := 0 for _, h := range existing { @@ -121,3 +121,8 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error { } return nil } + +// Shutdown shuts down the store gracefully +func (l *LFUDAStore) Shutdown() { + return +} diff --git a/store/lru.go b/store/lru.go index 071e2b9..ce0d83d 100644 --- a/store/lru.go +++ b/store/lru.go @@ -123,3 +123,8 @@ func (l *LRUStore) loadExisting(store lister, maxItems int) error { } return nil } + +// Shutdown shuts down the store gracefully +func (l *LRUStore) Shutdown() { + return +} diff --git a/store/memory.go b/store/memory.go index f462a8d..62b93b6 100644 --- a/store/memory.go +++ b/store/memory.go @@ -71,3 +71,8 @@ func (m *MemStore) Debug() map[string]stream.Blob { defer m.mu.RUnlock() return m.blobs } + +// Shutdown shuts down the store gracefully +func (m *MemStore) Shutdown() { + return +} diff --git a/store/noop.go b/store/noop.go index 9e5d815..2792a0d 100644 --- a/store/noop.go +++ b/store/noop.go @@ -13,3 +13,4 @@ func (n *NoopStore) Get(_ string) (stream.Blob, error) { return nil, nil } func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil } func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil } func (n *NoopStore) Delete(_ string) error { return nil } +func (n *NoopStore) Shutdown() { return } diff --git a/store/s3.go b/store/s3.go index 53451be..02a23c5 100644 --- a/store/s3.go +++ b/store/s3.go @@ -158,3 +158,8 @@ func (s *S3Store) initOnce() error { s.session = sess return nil } + +// Shutdown shuts down the store gracefully +func (s *S3Store) Shutdown() { + return +} diff --git a/store/singleflight.go b/store/singleflight.go index cca1e2f..7d0139d 100644 --- a/store/singleflight.go +++ b/store/singleflight.go @@ -65,3 +65,9 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) { return blob, nil } } + +// Shutdown shuts down the store gracefully +func (s *singleflightStore) Shutdown() { + s.BlobStore.Shutdown() + return +} diff --git a/store/store.go b/store/store.go index 200ff61..36d388d 100644 --- a/store/store.go +++ b/store/store.go @@ -19,6 +19,8 @@ type BlobStore interface { PutSD(hash string, blob stream.Blob) error // Delete the blob from the store. Delete(hash string) error + // Shutdown the store gracefully + Shutdown() } // Blocklister is a store that supports blocking blobs to prevent their inclusion in the store. -- 2.45.2 From 0d5004a83bc5cd6eaca1d05976403f74ae1dadfb Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 30 Dec 2020 04:24:11 +0100 Subject: [PATCH 15/51] add cmd to populate db fix store init try fixing unreasonable db bottleneck --- cmd/populatedb.go | 45 ++++++++++++ cmd/reflector.go | 7 +- db/db.go | 56 +++++++++++++-- store/dbbacked.go | 6 +- store/litedbbacked.go | 155 ------------------------------------------ 5 files changed, 101 insertions(+), 168 deletions(-) create mode 100644 cmd/populatedb.go delete mode 100644 store/litedbbacked.go diff --git a/cmd/populatedb.go b/cmd/populatedb.go new file mode 100644 index 0000000..d30bafe --- /dev/null +++ b/cmd/populatedb.go @@ -0,0 +1,45 @@ +package cmd + +import ( + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/reflector.go/db" + "github.com/lbryio/reflector.go/meta" + "github.com/lbryio/reflector.go/store/speedwalk" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var ( + diskStorePath string +) + +func init() { + var cmd = &cobra.Command{ + Use: "populate-db", + Short: "populate local database with blobs from a disk storage", + Run: populateDbCmd, + } + cmd.Flags().StringVar(&diskStorePath, "store-path", "", + "path of the store where all blobs are cached") + rootCmd.AddCommand(cmd) +} + +func populateDbCmd(cmd *cobra.Command, args []string) { + log.Printf("reflector %s", meta.VersionString()) + if diskStorePath == "" { + log.Fatal("store-path must be defined") + } + localDb := new(db.SQL) + localDb.SoftDelete = true + localDb.TrackAccess = db.TrackAccessBlobs + err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector") + if err != nil { + log.Fatal(err) + } + blobs, err := speedwalk.AllFiles(diskStorePath, true) + err = localDb.AddBlobs(blobs) + if err != nil { + log.Errorf("error while storing to db: %s", errors.FullTrace(err)) + } +} diff --git a/cmd/reflector.go b/cmd/reflector.go index ab9f53e..aacef08 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -185,11 +185,11 @@ func wrapWithCache(s store.BlobStore, cleanerStopper *stop.Group) store.BlobStor if err != nil { log.Fatal(err) } - dbBackedDiskStore := store.NewDBBackedStore(store.NewDiskStore(diskCachePath, 2), localDb, false) + dbBackedDiskStore := store.NewDBBackedStore(store.NewDiskStore(diskCachePath, 2), localDb, true) wrapped = store.NewCachingStore( "reflector", wrapped, - store.NewDBBackedStore(store.NewDiskStore(diskCachePath, 2), localDb, false), + dbBackedDiskStore, ) go cleanOldestBlobs(int(realCacheSize), localDb, dbBackedDiskStore, cleanerStopper) @@ -248,8 +248,7 @@ func diskCacheParams(diskParams string) (int, string) { } func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) { - const cleanupInterval = 10 * time.Second - + const cleanupInterval = 10 * time.Minute for { select { case <-stopper.Ch(): diff --git a/db/db.go b/db/db.go index 40b5c7d..9e19eba 100644 --- a/db/db.go +++ b/db/db.go @@ -3,11 +3,13 @@ package db import ( "context" "database/sql" + "strings" "time" "github.com/lbryio/lbry.go/v2/dht/bits" "github.com/lbryio/lbry.go/v2/extras/errors" qt "github.com/lbryio/lbry.go/v2/extras/query" + "github.com/lbryio/lbry.go/v2/stream" "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used @@ -88,6 +90,27 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error { return err } +// AddBlob adds a blob to the database. +func (s *SQL) AddBlobs(hash []string) error { + if s.conn == nil { + return errors.Err("not connected") + } + // Split the slice into batches of 20 items. + batch := 10000 + + for i := 0; i < len(hash); i += batch { + j := i + batch + if j > len(hash) { + j = len(hash) + } + err := s.insertBlobs(hash[i:j]) // Process the batch. + if err != nil { + log.Errorf("error while inserting batch: %s", errors.FullTrace(err)) + } + } + return nil +} + func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) { if length <= 0 { return 0, errors.Err("length must be positive") @@ -130,6 +153,26 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) return blobID, nil } +func (s *SQL) insertBlobs(hashes []string) error { + var ( + q string + args []interface{} + ) + dayAgo := time.Now().AddDate(0, 0, -1) + q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values " + for _, hash := range hashes { + q += "(?,?,?,?)," + args = append(args, hash, true, stream.MaxBlobSize, dayAgo) + } + q = strings.TrimSuffix(q, ",") + _, err := s.exec(q, args...) + if err != nil { + return err + } + + return nil +} + func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) { var ( q string @@ -180,12 +223,13 @@ func (s *SQL) HasBlob(hash string) (bool, error) { // HasBlobs checks if the database contains the set of blobs and returns a bool map. func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) { exists, idsNeedingTouch, err := s.hasBlobs(hashes) - - if s.TrackAccess == TrackAccessBlobs { - s.touchBlobs(idsNeedingTouch) - } else if s.TrackAccess == TrackAccessStreams { - s.touchStreams(idsNeedingTouch) - } + go func() { + if s.TrackAccess == TrackAccessBlobs { + s.touchBlobs(idsNeedingTouch) + } else if s.TrackAccess == TrackAccessStreams { + s.touchStreams(idsNeedingTouch) + } + }() return exists, err } diff --git a/store/dbbacked.go b/store/dbbacked.go index 1c5088a..dc88ad2 100644 --- a/store/dbbacked.go +++ b/store/dbbacked.go @@ -44,18 +44,18 @@ func (d *DBBackedStore) Get(hash string) (stream.Blob, error) { if !has { return nil, ErrBlobNotFound } + + b, err := d.blobs.Get(hash) if d.deleteOnMiss { - b, err := d.blobs.Get(hash) if err != nil && errors.Is(err, ErrBlobNotFound) { e2 := d.Delete(hash) if e2 != nil { log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err)) } - return b, err } } - return d.blobs.Get(hash) + return b, err } // Put stores the blob in the S3 store and stores the blob information in the DB. diff --git a/store/litedbbacked.go b/store/litedbbacked.go deleted file mode 100644 index 2e2d726..0000000 --- a/store/litedbbacked.go +++ /dev/null @@ -1,155 +0,0 @@ -package store - -import ( - "encoding/json" - - "github.com/lbryio/lbry.go/v2/extras/stop" - "github.com/lbryio/reflector.go/db" - - "github.com/lbryio/lbry.go/v2/extras/errors" - "github.com/lbryio/lbry.go/v2/stream" - - log "github.com/sirupsen/logrus" -) - -// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store. -type LiteDBBackedStore struct { - blobs BlobStore - db *db.SQL - maxItems int - stopper *stop.Stopper - component string -} - -// NewDBBackedStore returns an initialized store pointer. -func NewLiteDBBackedStore(component string, blobs BlobStore, db *db.SQL, maxItems int) *LiteDBBackedStore { - instance := &LiteDBBackedStore{blobs: blobs, db: db, maxItems: maxItems, stopper: stop.New(), component: component} - return instance -} - -const nameLiteDBBacked = "lite-db-backed" - -// Name is the cache type name -func (d *LiteDBBackedStore) Name() string { return nameDBBacked } - -// Has returns true if the blob is in the store -func (d *LiteDBBackedStore) Has(hash string) (bool, error) { - return d.db.HasBlob(hash) -} - -// Get gets the blob -func (d *LiteDBBackedStore) Get(hash string) (stream.Blob, error) { - has, err := d.db.HasBlob(hash) - if err != nil { - return nil, err - } - if !has { - return nil, ErrBlobNotFound - } - b, err := d.blobs.Get(hash) - if err != nil && errors.Is(err, ErrBlobNotFound) { - e2 := d.db.Delete(hash) - if e2 != nil { - log.Errorf("error while deleting blob from db: %s", errors.FullTrace(e2)) - } - return b, err - } - - return d.blobs.Get(hash) -} - -// Put stores the blob in the S3 store and stores the blob information in the DB. -func (d *LiteDBBackedStore) Put(hash string, blob stream.Blob) error { - err := d.blobs.Put(hash, blob) - if err != nil { - return err - } - - return d.db.AddBlob(hash, len(blob), true) -} - -// PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if -// there is an error storing the blob information in the DB. -func (d *LiteDBBackedStore) PutSD(hash string, blob stream.Blob) error { - var blobContents db.SdBlob - err := json.Unmarshal(blob, &blobContents) - if err != nil { - return errors.Err(err) - } - if blobContents.StreamHash == "" { - return errors.Err("sd blob is missing stream hash") - } - - err = d.blobs.PutSD(hash, blob) - if err != nil { - return err - } - - return d.db.AddSDBlob(hash, len(blob), blobContents) -} - -func (d *LiteDBBackedStore) Delete(hash string) error { - err := d.blobs.Delete(hash) - if err != nil { - return err - } - - return d.db.Delete(hash) -} - -// list returns the hashes of blobs that already exist in the database -func (d *LiteDBBackedStore) list() ([]string, error) { - //blobs, err := d.db.AllBlobs() - //return blobs, err - return nil, nil -} - -//func (d *LiteDBBackedStore) selfClean() { -// d.stopper.Add(1) -// defer d.stopper.Done() -// lastCleanup := time.Now() -// const cleanupInterval = 10 * time.Second -// for { -// select { -// case <-d.stopper.Ch(): -// log.Infoln("stopping self cleanup") -// return -// default: -// time.Sleep(1 * time.Second) -// } -// if time.Since(lastCleanup) < cleanupInterval { -// continue -// } -// blobsCount, err := d.db.BlobsCount() -// if err != nil { -// log.Errorf(errors.FullTrace(err)) -// } -// if blobsCount >= d.maxItems { -// itemsToDelete := blobsCount / 100 * 10 -// blobs, err := d.db.GetLRUBlobs(itemsToDelete) -// if err != nil { -// log.Errorf(errors.FullTrace(err)) -// } -// for _, hash := range blobs { -// select { -// case <-d.stopper.Ch(): -// return -// default: -// -// } -// err = d.Delete(hash) -// if err != nil { -// log.Errorf(errors.FullTrace(err)) -// } -// metrics.CacheLRUEvictCount.With(metrics.CacheLabels(d.Name(), d.component)).Inc() -// } -// } -// lastCleanup = time.Now() -// } -//} - -// Shutdown shuts down the store gracefully -func (d *LiteDBBackedStore) Shutdown() { - d.stopper.StopAndWait() - return -} -- 2.45.2 From b33651ae2640012da93f8d29ea986fa0c3913d0e Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 5 Jan 2021 05:09:55 +0100 Subject: [PATCH 16/51] save uploaded blobs and work around the blocklist issue --- cmd/reflector.go | 2 +- cmd/test.go | 2 +- prism/prism.go | 2 +- reflector/server.go | 26 ++++++++++++++------------ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index aacef08..49bef3e 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -76,7 +76,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) { outerStore := wrapWithCache(underlyingStore, cleanerStopper) if !disableUploads { - reflectorServer := reflector.NewServer(underlyingStore) + reflectorServer := reflector.NewServer(underlyingStore, outerStore) reflectorServer.Timeout = 3 * time.Minute reflectorServer.EnableBlocklist = !disableBlocklist diff --git a/cmd/test.go b/cmd/test.go index a330856..afa2ff2 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -31,7 +31,7 @@ func testCmd(cmd *cobra.Command, args []string) { memStore := store.NewMemStore() - reflectorServer := reflector.NewServer(memStore) + reflectorServer := reflector.NewServer(memStore, memStore) reflectorServer.Timeout = 3 * time.Minute err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort)) diff --git a/prism/prism.go b/prism/prism.go index 12d0329..ee18ba1 100644 --- a/prism/prism.go +++ b/prism/prism.go @@ -79,7 +79,7 @@ func New(conf *Config) *Prism { dht: d, cluster: c, peer: peer.NewServer(conf.Blobs), - reflector: reflector.NewServer(conf.Blobs), + reflector: reflector.NewServer(conf.Blobs, conf.Blobs), grp: stop.New(), } diff --git a/reflector/server.go b/reflector/server.go index bbe3d33..592ba3d 100644 --- a/reflector/server.go +++ b/reflector/server.go @@ -40,16 +40,18 @@ type Server struct { EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled - store store.BlobStore - grp *stop.Group + underlyingStore store.BlobStore + outerStore store.BlobStore + grp *stop.Group } // NewServer returns an initialized reflector server pointer. -func NewServer(store store.BlobStore) *Server { +func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server { return &Server{ - Timeout: DefaultTimeout, - store: store, - grp: stop.New(), + Timeout: DefaultTimeout, + underlyingStore: underlying, + outerStore: outer, + grp: stop.New(), } } @@ -85,7 +87,7 @@ func (s *Server) Start(address string) error { }() if s.EnableBlocklist { - if b, ok := s.store.(store.Blocklister); ok { + if b, ok := s.underlyingStore.(store.Blocklister); ok { s.grp.Add(1) go func() { s.enableBlocklist(b) @@ -190,13 +192,13 @@ func (s *Server) receiveBlob(conn net.Conn) error { } var wantsBlob bool - if bl, ok := s.store.(store.Blocklister); ok { + if bl, ok := s.underlyingStore.(store.Blocklister); ok { wantsBlob, err = bl.Wants(blobHash) if err != nil { return err } } else { - blobExists, err := s.store.Has(blobHash) + blobExists, err := s.underlyingStore.Has(blobHash) if err != nil { return err } @@ -206,7 +208,7 @@ func (s *Server) receiveBlob(conn net.Conn) error { var neededBlobs []string if isSdBlob && !wantsBlob { - if nbc, ok := s.store.(neededBlobChecker); ok { + if nbc, ok := s.underlyingStore.(neededBlobChecker); ok { neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash) if err != nil { return err @@ -249,9 +251,9 @@ func (s *Server) receiveBlob(conn net.Conn) error { log.Debugln("Got blob " + blobHash[:8]) if isSdBlob { - err = s.store.PutSD(blobHash, blob) + err = s.outerStore.PutSD(blobHash, blob) } else { - err = s.store.Put(blobHash, blob) + err = s.outerStore.Put(blobHash, blob) } if err != nil { return err -- 2.45.2 From 49714c02a61256d0af8303f3ed749cf56d5b7977 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Tue, 5 Jan 2021 11:36:33 -0500 Subject: [PATCH 17/51] only touch blobs when you get them --- db/db.go | 51 ++++++++++++++++++++++--------------------- reflector/uploader.go | 2 +- store/dbbacked.go | 16 ++++++-------- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/db/db.go b/db/db.go index 9e19eba..e48bcaf 100644 --- a/db/db.go +++ b/db/db.go @@ -111,6 +111,26 @@ func (s *SQL) AddBlobs(hash []string) error { return nil } +func (s *SQL) insertBlobs(hashes []string) error { + var ( + q string + args []interface{} + ) + dayAgo := time.Now().AddDate(0, 0, -1) + q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values " + for _, hash := range hashes { + q += "(?,?,?,?)," + args = append(args, hash, true, stream.MaxBlobSize, dayAgo) + } + q = strings.TrimSuffix(q, ",") + _, err := s.exec(q, args...) + if err != nil { + return err + } + + return nil +} + func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) { if length <= 0 { return 0, errors.Err("length must be positive") @@ -153,26 +173,6 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) return blobID, nil } -func (s *SQL) insertBlobs(hashes []string) error { - var ( - q string - args []interface{} - ) - dayAgo := time.Now().AddDate(0, 0, -1) - q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values " - for _, hash := range hashes { - q += "(?,?,?,?)," - args = append(args, hash, true, stream.MaxBlobSize, dayAgo) - } - q = strings.TrimSuffix(q, ",") - _, err := s.exec(q, args...) - if err != nil { - return err - } - - return nil -} - func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) { var ( q string @@ -212,8 +212,8 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) { } // HasBlob checks if the database contains the blob information. -func (s *SQL) HasBlob(hash string) (bool, error) { - exists, err := s.HasBlobs([]string{hash}) +func (s *SQL) HasBlob(hash string, touch bool) (bool, error) { + exists, err := s.HasBlobs([]string{hash}, touch) if err != nil { return false, err } @@ -221,15 +221,16 @@ func (s *SQL) HasBlob(hash string) (bool, error) { } // HasBlobs checks if the database contains the set of blobs and returns a bool map. -func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) { +func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) { exists, idsNeedingTouch, err := s.hasBlobs(hashes) - go func() { + + if touch { if s.TrackAccess == TrackAccessBlobs { s.touchBlobs(idsNeedingTouch) } else if s.TrackAccess == TrackAccessStreams { s.touchStreams(idsNeedingTouch) } - }() + } return exists, err } diff --git a/reflector/uploader.go b/reflector/uploader.go index ae762f4..e5b2688 100644 --- a/reflector/uploader.go +++ b/reflector/uploader.go @@ -74,7 +74,7 @@ func (u *Uploader) Upload(dirOrFilePath string) error { var exists map[string]bool if !u.skipExistsCheck { - exists, err = u.db.HasBlobs(hashes) + exists, err = u.db.HasBlobs(hashes, false) if err != nil { return err } diff --git a/store/dbbacked.go b/store/dbbacked.go index dc88ad2..1484c49 100644 --- a/store/dbbacked.go +++ b/store/dbbacked.go @@ -32,12 +32,12 @@ func (d *DBBackedStore) Name() string { return nameDBBacked } // Has returns true if the blob is in the store func (d *DBBackedStore) Has(hash string) (bool, error) { - return d.db.HasBlob(hash) + return d.db.HasBlob(hash, false) } // Get gets the blob func (d *DBBackedStore) Get(hash string) (stream.Blob, error) { - has, err := d.db.HasBlob(hash) + has, err := d.db.HasBlob(hash, true) if err != nil { return nil, err } @@ -46,12 +46,10 @@ func (d *DBBackedStore) Get(hash string) (stream.Blob, error) { } b, err := d.blobs.Get(hash) - if d.deleteOnMiss { - if err != nil && errors.Is(err, ErrBlobNotFound) { - e2 := d.Delete(hash) - if e2 != nil { - log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err)) - } + if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) { + e2 := d.Delete(hash) + if e2 != nil { + log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err)) } } @@ -110,7 +108,7 @@ func (d *DBBackedStore) Block(hash string) error { return err } - has, err := d.db.HasBlob(hash) + has, err := d.db.HasBlob(hash, false) if err != nil { return err } -- 2.45.2 From cc504e6c4493c74be4fc2f1be4a86823ccf37627 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Tue, 5 Jan 2021 12:16:44 -0500 Subject: [PATCH 18/51] fix long query --- db/db.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/db/db.go b/db/db.go index e48bcaf..d1c8b7b 100644 --- a/db/db.go +++ b/db/db.go @@ -298,20 +298,23 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) { log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes)) batch := hashes[doneIndex:sliceEnd] - var lastAccessedAtSelect string + var query string if s.TrackAccess == TrackAccessBlobs { - lastAccessedAtSelect = "b.last_accessed_at" + query = `SELECT b.hash, b.id, NULL, b.last_accessed_at +FROM blob_ b +WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)` } else if s.TrackAccess == TrackAccessStreams { - lastAccessedAtSelect = "s.last_accessed_at" - } else { - lastAccessedAtSelect = "NULL" - } - - query := `SELECT b.hash, b.id, s.id, ` + lastAccessedAtSelect + ` + query = `SELECT b.hash, b.id, s.id, s.last_accessed_at FROM blob_ b LEFT JOIN stream_blob sb ON b.id = sb.blob_id -LEFT JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id) +INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id) WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)` + } else { + query = `SELECT b.hash, b.id, NULL, NULL +FROM blob_ b +WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)` + } + args := make([]interface{}, len(batch)) for i := range batch { args[i] = batch[i] -- 2.45.2 From c4504631bc896b67611fa4a9ff02ddfedefbcc39 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Wed, 6 Jan 2021 10:43:35 -0500 Subject: [PATCH 19/51] avoid heavy interpolateparams call --- cmd/peer.go | 4 +++- cmd/populatedb.go | 8 +++++--- cmd/reflector.go | 14 +++++++++----- cmd/start.go | 4 +++- cmd/upload.go | 5 ++++- db/db.go | 33 ++++++++++++++++++++------------- reflector/server_test.go | 6 +++--- 7 files changed, 47 insertions(+), 27 deletions(-) diff --git a/cmd/peer.go b/cmd/peer.go index 86af279..c631bfe 100644 --- a/cmd/peer.go +++ b/cmd/peer.go @@ -33,7 +33,9 @@ func peerCmd(cmd *cobra.Command, args []string) { peerServer := peer.NewServer(s3) if !peerNoDB { - db := new(db.SQL) + db := &db.SQL{ + LogQueries: log.GetLevel() == log.DebugLevel, + } err = db.Connect(globalConfig.DBConn) checkErr(err) diff --git a/cmd/populatedb.go b/cmd/populatedb.go index d30bafe..7199e42 100644 --- a/cmd/populatedb.go +++ b/cmd/populatedb.go @@ -30,9 +30,11 @@ func populateDbCmd(cmd *cobra.Command, args []string) { if diskStorePath == "" { log.Fatal("store-path must be defined") } - localDb := new(db.SQL) - localDb.SoftDelete = true - localDb.TrackAccess = db.TrackAccessBlobs + localDb := &db.SQL{ + SoftDelete: true, + TrackAccess: db.TrackAccessBlobs, + LogQueries: log.GetLevel() == log.DebugLevel, + } err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector") if err != nil { log.Fatal(err) diff --git a/cmd/reflector.go b/cmd/reflector.go index 49bef3e..738a066 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -152,8 +152,10 @@ func setupStore() store.BlobStore { } if useDB { - dbInst := new(db.SQL) - dbInst.TrackAccess = db.TrackAccessStreams + dbInst := &db.SQL{ + TrackAccess: db.TrackAccessStreams, + LogQueries: log.GetLevel() == log.DebugLevel, + } err := dbInst.Connect(globalConfig.DBConn) if err != nil { log.Fatal(err) @@ -178,9 +180,11 @@ func wrapWithCache(s store.BlobStore, cleanerStopper *stop.Group) store.BlobStor log.Fatal(err) } - localDb := new(db.SQL) - localDb.SoftDelete = true - localDb.TrackAccess = db.TrackAccessBlobs + localDb := &db.SQL{ + SoftDelete: true, + TrackAccess: db.TrackAccessBlobs, + LogQueries: log.GetLevel() == log.DebugLevel, + } err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector") if err != nil { log.Fatal(err) diff --git a/cmd/start.go b/cmd/start.go index 476e9fd..7b5facd 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -52,7 +52,9 @@ func init() { } func startCmd(cmd *cobra.Command, args []string) { - db := new(db.SQL) + db := &db.SQL{ + LogQueries: log.GetLevel() == log.DebugLevel, + } err := db.Connect(globalConfig.DBConn) checkErr(err) s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName) diff --git a/cmd/upload.go b/cmd/upload.go index e5e2cec..e2a49cd 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -8,6 +8,7 @@ import ( "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/reflector" "github.com/lbryio/reflector.go/store" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -30,7 +31,9 @@ func init() { } func uploadCmd(cmd *cobra.Command, args []string) { - db := new(db.SQL) + db := &db.SQL{ + LogQueries: log.GetLevel() == log.DebugLevel, + } err := db.Connect(globalConfig.DBConn) checkErr(err) diff --git a/db/db.go b/db/db.go index d1c8b7b..afa708b 100644 --- a/db/db.go +++ b/db/db.go @@ -52,14 +52,21 @@ type SQL struct { // Instead of deleting a blob, marked it as not stored in the db SoftDelete bool + + // Log executed queries. qt.InterpolateParams is cpu-heavy. This avoids that call if not needed. + LogQueries bool } -func logQuery(query string, args ...interface{}) { - s, err := qt.InterpolateParams(query, args...) +func (s SQL) logQuery(query string, args ...interface{}) { + if !s.LogQueries { + return + } + + qStr, err := qt.InterpolateParams(query, args...) if err != nil { log.Errorln(err) } else { - log.Debugln(s) + log.Debugln(qStr) } } @@ -320,7 +327,7 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)` args[i] = batch[i] } - logQuery(query, args...) + s.logQuery(query, args...) err := func() error { startTime := time.Now() @@ -390,7 +397,7 @@ func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) { } query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?" - logQuery(query, maxBlobs) + s.logQuery(query, maxBlobs) rows, err := s.conn.Query(query, maxBlobs) if err != nil { @@ -422,7 +429,7 @@ func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) { // if s.SoftDelete { // query += " where is_stored = 1" // } -// logQuery(query) +// s.logQuery(query) // // rows, err := s.conn.Query(query) // if err != nil { @@ -454,7 +461,7 @@ func (s *SQL) Count() (int, error) { if s.SoftDelete { query += " where is_stored = 1" } - logQuery(query) + s.logQuery(query) var count int err := s.conn.QueryRow(query).Scan(&count) @@ -465,7 +472,7 @@ func (s *SQL) Count() (int, error) { func (s *SQL) Block(hash string) error { query := "INSERT IGNORE INTO blocked SET hash = ?" args := []interface{}{hash} - logQuery(query, args...) + s.logQuery(query, args...) _, err := s.conn.Exec(query, args...) return errors.Err(err) } @@ -473,7 +480,7 @@ func (s *SQL) Block(hash string) error { // GetBlocked will return a list of blocked hashes func (s *SQL) GetBlocked() (map[string]bool, error) { query := "SELECT hash FROM blocked" - logQuery(query) + s.logQuery(query) rows, err := s.conn.Query(query) if err != nil { return nil, errors.Err(err) @@ -516,7 +523,7 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) { ` args := []interface{}{sdHash} - logQuery(query, args...) + s.logQuery(query, args...) rows, err := s.conn.Query(query, args...) if err != nil { @@ -595,7 +602,7 @@ func (s *SQL) GetHashRange() (string, string, error) { query := "SELECT MIN(hash), MAX(hash) from blob_" - logQuery(query) + s.logQuery(query) err := s.conn.QueryRow(query).Scan(&min, &max) return min, max, err @@ -619,7 +626,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap query := "SELECT hash FROM blob_ WHERE hash >= ? AND hash <= ? AND is_stored = 1" args := []interface{}{start.Hex(), end.Hex()} - logQuery(query, args...) + s.logQuery(query, args...) rows, err := s.conn.Query(query, args...) defer closeRows(rows) @@ -700,7 +707,7 @@ func closeRows(rows *sql.Rows) { } func (s *SQL) exec(query string, args ...interface{}) (int64, error) { - logQuery(query, args...) + s.logQuery(query, args...) attempt, maxAttempts := 0, 3 Retry: attempt++ diff --git a/reflector/server_test.go b/reflector/server_test.go index 0de200e..9004d84 100644 --- a/reflector/server_test.go +++ b/reflector/server_test.go @@ -22,7 +22,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) { t.Fatal(err) } - srv := NewServer(store.NewMemStore()) + srv := NewServer(store.NewMemStore(), store.NewMemStore()) err = srv.Start("127.0.0.1:" + strconv.Itoa(port)) if err != nil { t.Fatal(err) @@ -119,7 +119,7 @@ func TestServer_Timeout(t *testing.T) { t.Fatal(err) } - srv := NewServer(store.NewMemStore()) + srv := NewServer(store.NewMemStore(), store.NewMemStore()) srv.Timeout = testTimeout err = srv.Start("127.0.0.1:" + strconv.Itoa(port)) if err != nil { @@ -190,7 +190,7 @@ func TestServer_PartialUpload(t *testing.T) { t.Fatal(err) } - srv := NewServer(st) + srv := NewServer(st, st) err = srv.Start("127.0.0.1:" + strconv.Itoa(port)) if err != nil { t.Fatal(err) -- 2.45.2 From 3e475e537b37566ac3c35ab1649f6cfd694516ec Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Thu, 7 Jan 2021 01:26:30 +0100 Subject: [PATCH 20/51] optimize batch insertions reduce touch time to every 6 hours --- db/db.go | 65 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/db/db.go b/db/db.go index afa708b..3e7d4ca 100644 --- a/db/db.go +++ b/db/db.go @@ -3,18 +3,22 @@ package db import ( "context" "database/sql" + "fmt" + "runtime" "strings" "time" "github.com/lbryio/lbry.go/v2/dht/bits" "github.com/lbryio/lbry.go/v2/extras/errors" qt "github.com/lbryio/lbry.go/v2/extras/query" + "github.com/lbryio/lbry.go/v2/extras/stop" "github.com/lbryio/lbry.go/v2/stream" "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used log "github.com/sirupsen/logrus" "github.com/volatiletech/null" + "go.uber.org/atomic" ) // SdBlob is a special blob that contains information on the rest of the blobs in the stream @@ -104,33 +108,62 @@ func (s *SQL) AddBlobs(hash []string) error { } // Split the slice into batches of 20 items. batch := 10000 - - for i := 0; i < len(hash); i += batch { - j := i + batch - if j > len(hash) { - j = len(hash) - } - err := s.insertBlobs(hash[i:j]) // Process the batch. - if err != nil { - log.Errorf("error while inserting batch: %s", errors.FullTrace(err)) + totalBlobs := int64(len(hash)) + work := make(chan []string, 1000) + stopper := stop.New() + var totalInserted atomic.Int64 + start := time.Now() + go func() { + for i := 0; i < len(hash); i += batch { + j := i + batch + if j > len(hash) { + j = len(hash) + } + work <- hash[i:j] } + log.Infof("done loading %d hashes in the work queue", len(hash)) + close(work) + }() + for i := 0; i < runtime.NumCPU(); i++ { + stopper.Add(1) + go func(worker int) { + log.Infof("starting worker %d", worker) + defer stopper.Done() + for hashes := range work { + inserted := totalInserted.Load() + remaining := totalBlobs - inserted + if inserted > 0 { + timePerBlob := time.Since(start).Microseconds() / inserted + remainingTime := time.Duration(remaining*timePerBlob) * time.Microsecond + log.Infof("[T%d] processing batch of %d items. ETA: %s", worker, len(hashes), remainingTime.String()) + } + err := s.insertBlobs(hashes) // Process the batch. + if err != nil { + log.Errorf("error while inserting batch: %s", errors.FullTrace(err)) + } + totalInserted.Add(int64(len(hashes))) + } + }(i) } + stopper.Wait() return nil } func (s *SQL) insertBlobs(hashes []string) error { var ( - q string - args []interface{} + q string + //args []interface{} ) - dayAgo := time.Now().AddDate(0, 0, -1) + dayAgo := time.Now().AddDate(0, 0, -1).Format("2006-01-02 15:04:05") q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values " for _, hash := range hashes { - q += "(?,?,?,?)," - args = append(args, hash, true, stream.MaxBlobSize, dayAgo) + // prepared statements slow everything down by a lot due to reflection + // for this specific instance we'll go ahead and hardcode the query to make it go faster + q += fmt.Sprintf("('%s',1,%d,'%s'),", hash, stream.MaxBlobSize, dayAgo) + //args = append(args, hash, true, stream.MaxBlobSize, dayAgo) } q = strings.TrimSuffix(q, ",") - _, err := s.exec(q, args...) + _, err := s.exec(q) if err != nil { return err } @@ -293,7 +326,7 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) { var needsTouch []uint64 exists := make(map[string]bool) - touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time + touchDeadline := time.Now().Add(-6 * time.Hour) // touch blob if last accessed before this time maxBatchSize := 10000 doneIndex := 0 -- 2.45.2 From 6291e33ee184ad334f993ea062384787db966643 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Sat, 9 Jan 2021 05:08:20 +0100 Subject: [PATCH 21/51] add tracing to blobs --- cmd/getstream.go | 4 +-- peer/client.go | 32 ++++++++++------- peer/http3/client.go | 37 ++++++++++++------- peer/http3/server.go | 22 +++++++++++- peer/http3/store.go | 6 ++-- peer/server.go | 6 +++- peer/store.go | 6 ++-- shared/shared.go | 82 ++++++++++++++++++++++++++++++++++++++++++ shared/shared_test.go | 32 +++++++++++++++++ store/caching.go | 13 +++---- store/caching_test.go | 11 +++--- store/cloudfront_ro.go | 22 ++++++------ store/cloudfront_rw.go | 9 +++-- store/dbbacked.go | 13 ++++--- store/disk.go | 13 ++++--- store/disk_test.go | 4 +-- store/lfuda.go | 12 ++++--- store/lfuda_test.go | 8 ++--- store/lru.go | 12 ++++--- store/lru_test.go | 2 +- store/memory.go | 9 +++-- store/memory_test.go | 4 +-- store/noop.go | 15 +++++--- store/s3.go | 16 +++++---- store/singleflight.go | 28 +++++++++++---- store/store.go | 3 +- 26 files changed, 317 insertions(+), 104 deletions(-) create mode 100644 shared/shared.go create mode 100644 shared/shared_test.go diff --git a/cmd/getstream.go b/cmd/getstream.go index c02ad0b..1c25df5 100644 --- a/cmd/getstream.go +++ b/cmd/getstream.go @@ -41,7 +41,7 @@ func getStreamCmd(cmd *cobra.Command, args []string) { var sd stream.SDBlob - sdb, err := s.Get(sdHash) + sdb, _, err := s.Get(sdHash) if err != nil { log.Fatal(err) } @@ -62,7 +62,7 @@ func getStreamCmd(cmd *cobra.Command, args []string) { } for i := 0; i < len(sd.BlobInfos)-1; i++ { - b, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash)) + b, _, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash)) if err != nil { log.Fatal(err) } diff --git a/peer/client.go b/peer/client.go index cdafd1c..9eae088 100644 --- a/peer/client.go +++ b/peer/client.go @@ -9,6 +9,7 @@ import ( "time" "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/shared" "github.com/lbryio/reflector.go/store" "github.com/lbryio/lbry.go/v2/extras/errors" @@ -57,10 +58,11 @@ func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Str var sd stream.SDBlob - b, err := c.GetBlob(sdHash) + b, trace, err := c.GetBlob(sdHash) if err != nil { return nil, err } + log.Debug(trace.String()) err = sd.FromBlob(b) if err != nil { @@ -71,10 +73,11 @@ func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Str s[0] = b for i := 0; i < len(sd.BlobInfos)-1; i++ { - s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash)) + s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash)) if err != nil { return nil, err } + log.Debug(trace.String()) } return s, nil @@ -114,47 +117,52 @@ func (c *Client) HasBlob(hash string) (bool, error) { } // GetBlob gets a blob -func (c *Client) GetBlob(hash string) (stream.Blob, error) { +func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() if !c.connected { - return nil, errors.Err("not connected") + return nil, shared.NewBlobTrace(time.Since(start), "tcp"), errors.Err("not connected") } sendRequest, err := json.Marshal(blobRequest{ RequestedBlob: hash, }) if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err } err = c.write(sendRequest) if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err } var resp blobResponse err = c.read(&resp) if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err } + trace := shared.NewBlobTrace(time.Since(start), "tcp") + if resp.RequestTrace != nil { + trace = *resp.RequestTrace + } if resp.IncomingBlob.Error != "" { - return nil, errors.Prefix(hash[:8], resp.IncomingBlob.Error) + return nil, trace, errors.Prefix(hash[:8], resp.IncomingBlob.Error) } if resp.IncomingBlob.BlobHash != hash { - return nil, errors.Prefix(hash[:8], "blob hash in response does not match requested hash") + return nil, trace.Stack(time.Since(start), "tcp"), errors.Prefix(hash[:8], "blob hash in response does not match requested hash") } if resp.IncomingBlob.Length <= 0 { - return nil, errors.Prefix(hash[:8], "length reported as <= 0") + return nil, trace, errors.Prefix(hash[:8], "length reported as <= 0") } log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr()) blob, err := c.readRawBlob(resp.IncomingBlob.Length) if err != nil { - return nil, err + return nil, (*resp.RequestTrace).Stack(time.Since(start), "tcp"), err } metrics.MtrInBytesTcp.Add(float64(len(blob))) - return blob, nil + return blob, trace.Stack(time.Since(start), "tcp"), nil } func (c *Client) read(v interface{}) error { diff --git a/peer/http3/client.go b/peer/http3/client.go index ad5c741..d9b1495 100644 --- a/peer/http3/client.go +++ b/peer/http3/client.go @@ -9,12 +9,14 @@ import ( "sync" "time" - "github.com/lbryio/lbry.go/v2/extras/errors" - "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/shared" "github.com/lbryio/reflector.go/store" + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" "github.com/lucas-clemente/quic-go/http3" + log "github.com/sirupsen/logrus" ) // Client is an instance of a client connected to a server. @@ -35,7 +37,7 @@ func (c *Client) Close() error { func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) { var sd stream.SDBlob - b, err := c.GetBlob(sdHash) + b, _, err := c.GetBlob(sdHash) if err != nil { return nil, err } @@ -49,10 +51,12 @@ func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Str s[0] = b for i := 0; i < len(sd.BlobInfos)-1; i++ { - s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash)) + var trace shared.BlobTrace + s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash)) if err != nil { return nil, err } + log.Debug(trace.String()) } return s, nil @@ -75,26 +79,35 @@ func (c *Client) HasBlob(hash string) (bool, error) { } // GetBlob gets a blob -func (c *Client) GetBlob(hash string) (stream.Blob, error) { - resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s", c.ServerAddr, hash)) +func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() + resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s?trace=true", c.ServerAddr, hash)) if err != nil { - return nil, errors.Err(err) + return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err) } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode) - return nil, errors.Err(store.ErrBlobNotFound) + return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(store.ErrBlobNotFound) } else if resp.StatusCode != http.StatusOK { - return nil, errors.Err("non 200 status code returned: %d", resp.StatusCode) + return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err("non 200 status code returned: %d", resp.StatusCode) } tmp := getBuffer() defer putBuffer(tmp) - + serialized := resp.Header.Get("Via") + trace := shared.NewBlobTrace(time.Since(start), "http3") + if serialized != "" { + parsedTrace, err := shared.Deserialize(serialized) + if err != nil { + return nil, shared.NewBlobTrace(time.Since(start), "http3"), err + } + trace = *parsedTrace + } written, err := io.Copy(tmp, resp.Body) if err != nil { - return nil, errors.Err(err) + return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err) } blob := make([]byte, written) @@ -102,7 +115,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, error) { metrics.MtrInBytesUdp.Add(float64(len(blob))) - return blob, nil + return blob, trace.Stack(time.Since(start), "http3"), nil } // buffer pool to reduce GC diff --git a/peer/http3/server.go b/peer/http3/server.go index aaadac0..c33a676 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -10,6 +10,7 @@ import ( "fmt" "math/big" "net/http" + "strconv" "time" "github.com/lbryio/reflector.go/internal/metrics" @@ -71,7 +72,26 @@ func (s *Server) Start(address string) error { r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) requestedBlob := vars["hash"] - blob, err := s.store.Get(requestedBlob) + traceParam := r.URL.Query().Get("trace") + var err error + wantsTrace := false + if traceParam != "" { + wantsTrace, err = strconv.ParseBool(traceParam) + if err != nil { + wantsTrace = false + } + } + blob, trace, err := s.store.Get(requestedBlob) + + if wantsTrace { + serialized, err := trace.Serialize() + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + w.Header().Add("Via", serialized) + log.Debug(trace.String()) + } if err != nil { if errors.Is(err, store.ErrBlobNotFound) { http.Error(w, err.Error(), http.StatusNotFound) diff --git a/peer/http3/store.go b/peer/http3/store.go index b681443..2d9a23c 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -8,6 +8,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/shared" "github.com/lucas-clemente/quic-go" "github.com/lucas-clemente/quic-go/http3" ) @@ -68,10 +69,11 @@ func (p *Store) Has(hash string) (bool, error) { } // Get downloads the blob from the peer -func (p *Store) Get(hash string) (stream.Blob, error) { +func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() c, err := p.getClient() if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err } defer c.Close() return c.GetBlob(hash) diff --git a/peer/server.go b/peer/server.go index 44916cb..708fe53 100644 --- a/peer/server.go +++ b/peer/server.go @@ -12,6 +12,7 @@ import ( "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/reflector" + "github.com/lbryio/reflector.go/shared" "github.com/lbryio/reflector.go/store" "github.com/lbryio/lbry.go/v2/extras/errors" @@ -253,6 +254,7 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) { } var blob []byte + var trace shared.BlobTrace if request.RequestedBlob != "" { if len(request.RequestedBlob) != stream.BlobHashHexLength { return nil, errors.Err("Invalid blob hash length") @@ -260,7 +262,8 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) { log.Debugln("Sending blob " + request.RequestedBlob[:8]) - blob, err = s.store.Get(request.RequestedBlob) + blob, trace, err = s.store.Get(request.RequestedBlob) + log.Debug(trace.String()) if errors.Is(err, store.ErrBlobNotFound) { response.IncomingBlob = incomingBlob{ Error: err.Error(), @@ -382,6 +385,7 @@ type incomingBlob struct { } type blobResponse struct { IncomingBlob incomingBlob `json:"incoming_blob"` + RequestTrace *shared.BlobTrace } type compositeRequest struct { diff --git a/peer/store.go b/peer/store.go index 3ef9622..c227884 100644 --- a/peer/store.go +++ b/peer/store.go @@ -5,6 +5,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/shared" ) // Store is a blob store that gets blobs from a peer. @@ -43,10 +44,11 @@ func (p *Store) Has(hash string) (bool, error) { } // Get downloads the blob from the peer -func (p *Store) Get(hash string) (stream.Blob, error) { +func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() c, err := p.getClient() if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err } defer c.Close() return c.GetBlob(hash) diff --git a/shared/shared.go b/shared/shared.go new file mode 100644 index 0000000..c232cf6 --- /dev/null +++ b/shared/shared.go @@ -0,0 +1,82 @@ +package shared + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "github.com/lbryio/lbry.go/v2/extras/errors" +) + +type BlobStack struct { + Timing time.Duration `json:"timing"` + OriginName string `json:"origin_name"` + HostName string `json:"host_name"` +} +type BlobTrace struct { + Stacks []BlobStack `json:"stacks"` +} + +var hostName *string + +func getHostName() string { + if hostName == nil { + hn, err := os.Hostname() + if err != nil { + hn = "unknown" + } + hostName = &hn + } + return *hostName +} +func (b *BlobTrace) Stack(timing time.Duration, originName string) BlobTrace { + b.Stacks = append(b.Stacks, BlobStack{ + Timing: timing, + OriginName: originName, + HostName: getHostName(), + }) + return *b +} +func (b *BlobTrace) Merge(otherTrance BlobTrace) BlobTrace { + b.Stacks = append(b.Stacks, otherTrance.Stacks...) + return *b +} +func NewBlobTrace(timing time.Duration, originName string) BlobTrace { + b := BlobTrace{} + b.Stacks = append(b.Stacks, BlobStack{ + Timing: timing, + OriginName: originName, + HostName: getHostName(), + }) + return b +} + +func (b BlobTrace) String() string { + var fullTrace string + for i, stack := range b.Stacks { + delta := time.Duration(0) + if i > 0 { + delta = stack.Timing - b.Stacks[i-1].Timing + } + fullTrace += fmt.Sprintf("[%d](%s) origin: %s - timing: %s - delta: %s\n", i, stack.HostName, stack.OriginName, stack.Timing.String(), delta.String()) + } + return fullTrace +} + +func (b BlobTrace) Serialize() (string, error) { + t, err := json.Marshal(b) + if err != nil { + return "", errors.Err(err) + } + return string(t), nil +} + +func Deserialize(serializedData string) (*BlobTrace, error) { + var trace BlobTrace + err := json.Unmarshal([]byte(serializedData), &trace) + if err != nil { + return nil, errors.Err(err) + } + return &trace, nil +} diff --git a/shared/shared_test.go b/shared/shared_test.go new file mode 100644 index 0000000..47c8eef --- /dev/null +++ b/shared/shared_test.go @@ -0,0 +1,32 @@ +package shared + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBlobTrace_Serialize(t *testing.T) { + stack := NewBlobTrace(10*time.Second, "test") + stack.Stack(20*time.Second, "test2") + stack.Stack(30*time.Second, "test3") + serialized, err := stack.Serialize() + assert.NoError(t, err) + t.Log(serialized) + expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}" + assert.Equal(t, expected, serialized) +} + +func TestBlobTrace_Deserialize(t *testing.T) { + serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}" + stack, err := Deserialize(serialized) + assert.NoError(t, err) + assert.Len(t, stack.Stacks, 3) + assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second) + assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second) + assert.Equal(t, stack.Stacks[2].Timing, 30*time.Second) + assert.Equal(t, stack.Stacks[0].OriginName, "test") + assert.Equal(t, stack.Stacks[1].OriginName, "test2") + assert.Equal(t, stack.Stacks[2].OriginName, "test3") +} diff --git a/store/caching.go b/store/caching.go index 53b692c..6f1d4d4 100644 --- a/store/caching.go +++ b/store/caching.go @@ -5,6 +5,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/shared" log "github.com/sirupsen/logrus" "github.com/lbryio/reflector.go/internal/metrics" @@ -43,9 +44,9 @@ func (c *CachingStore) Has(hash string) (bool, error) { // Get tries to get the blob from the cache first, falling back to the origin. If the blob comes // from the origin, it is also stored in the cache. -func (c *CachingStore) Get(hash string) (stream.Blob, error) { +func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { start := time.Now() - blob, err := c.cache.Get(hash) + blob, trace, err := c.cache.Get(hash) if err == nil || !errors.Is(err, ErrBlobNotFound) { metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc() rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds() @@ -54,14 +55,14 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) { metrics.LabelComponent: c.component, metrics.LabelSource: "cache", }).Set(rate) - return blob, err + return blob, trace.Stack(time.Since(start), c.Name()), err } metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc() - blob, err = c.origin.Get(hash) + blob, trace, err = c.origin.Get(hash) if err != nil { - return nil, err + return nil, trace.Stack(time.Since(start), c.Name()), err } // there is no need to wait for the blob to be stored before we return it // TODO: however this should be refactored to limit the amount of routines that the process can spawn to avoid a possible DoS @@ -71,7 +72,7 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) { log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err)) } }() - return blob, nil + return blob, trace.Stack(time.Since(start), c.Name()), nil } // Put stores the blob in the origin and the cache diff --git a/store/caching_test.go b/store/caching_test.go index 0636583..66a42fd 100644 --- a/store/caching_test.go +++ b/store/caching_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/shared" ) func TestCachingStore_Put(t *testing.T) { @@ -51,7 +52,7 @@ func TestCachingStore_CacheMiss(t *testing.T) { t.Fatal(err) } - res, err := s.Get(hash) + res, stack, err := s.Get(hash) if err != nil { t.Fatal(err) } @@ -67,14 +68,16 @@ func TestCachingStore_CacheMiss(t *testing.T) { if !has { t.Errorf("Get() did not copy blob to cache") } + t.Logf("stack: %s", stack.String()) - res, err = cache.Get(hash) + res, stack, err = cache.Get(hash) if err != nil { t.Fatal(err) } if !bytes.Equal(b, res) { t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res)) } + t.Logf("stack: %s", stack.String()) } func TestCachingStore_ThunderingHerd(t *testing.T) { @@ -93,7 +96,7 @@ func TestCachingStore_ThunderingHerd(t *testing.T) { wg := &sync.WaitGroup{} getNoErr := func() { - res, err := s.Get(hash) + res, _, err := s.Get(hash) if err != nil { t.Fatal(err) } @@ -149,7 +152,7 @@ func (s *SlowBlobStore) Has(hash string) (bool, error) { return s.mem.Has(hash) } -func (s *SlowBlobStore) Get(hash string) (stream.Blob, error) { +func (s *SlowBlobStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { time.Sleep(s.delay) return s.mem.Get(hash) } diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go index 3dcffc0..51fe4b8 100644 --- a/store/cloudfront_ro.go +++ b/store/cloudfront_ro.go @@ -6,11 +6,11 @@ import ( "net/http" "time" - "github.com/lbryio/reflector.go/internal/metrics" - "github.com/lbryio/reflector.go/meta" - "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/meta" + "github.com/lbryio/reflector.go/shared" log "github.com/sirupsen/logrus" ) @@ -49,30 +49,30 @@ func (c *CloudFrontROStore) Has(hash string) (bool, error) { } // Get gets the blob from Cloudfront. -func (c *CloudFrontROStore) Get(hash string) (stream.Blob, error) { +func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { log.Debugf("Getting %s from S3", hash[:8]) + start := time.Now() defer func(t time.Time) { log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String()) - }(time.Now()) + }(start) status, body, err := c.cfRequest(http.MethodGet, hash) if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err } defer body.Close() - switch status { case http.StatusNotFound, http.StatusForbidden: - return nil, errors.Err(ErrBlobNotFound) + return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound) case http.StatusOK: b, err := ioutil.ReadAll(body) if err != nil { - return nil, errors.Err(err) + return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err) } metrics.MtrInBytesS3.Add(float64(len(b))) - return b, nil + return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil default: - return nil, errors.Err("unexpected status %d", status) + return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status) } } diff --git a/store/cloudfront_rw.go b/store/cloudfront_rw.go index 6de64af..32ee418 100644 --- a/store/cloudfront_rw.go +++ b/store/cloudfront_rw.go @@ -1,7 +1,10 @@ package store import ( + "time" + "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/shared" ) // CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront, writes go to S3. @@ -30,8 +33,10 @@ func (c *CloudFrontRWStore) Has(hash string) (bool, error) { } // Get gets the blob from Cloudfront. -func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, error) { - return c.cf.Get(hash) +func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() + blob, trace, err := c.cf.Get(hash) + return blob, trace.Stack(time.Since(start), c.Name()), err } // Put stores the blob on S3 diff --git a/store/dbbacked.go b/store/dbbacked.go index 1484c49..90c9fed 100644 --- a/store/dbbacked.go +++ b/store/dbbacked.go @@ -3,10 +3,12 @@ package store import ( "encoding/json" "sync" + "time" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/db" + "github.com/lbryio/reflector.go/shared" log "github.com/sirupsen/logrus" ) @@ -36,16 +38,17 @@ func (d *DBBackedStore) Has(hash string) (bool, error) { } // Get gets the blob -func (d *DBBackedStore) Get(hash string) (stream.Blob, error) { +func (d *DBBackedStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() has, err := d.db.HasBlob(hash, true) if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err } if !has { - return nil, ErrBlobNotFound + return nil, shared.NewBlobTrace(time.Since(start), d.Name()), ErrBlobNotFound } - b, err := d.blobs.Get(hash) + b, stack, err := d.blobs.Get(hash) if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) { e2 := d.Delete(hash) if e2 != nil { @@ -53,7 +56,7 @@ func (d *DBBackedStore) Get(hash string) (stream.Blob, error) { } } - return b, err + return b, stack.Stack(time.Since(start), d.Name()), err } // Put stores the blob in the S3 store and stores the blob information in the DB. diff --git a/store/disk.go b/store/disk.go index 03cc393..412d326 100644 --- a/store/disk.go +++ b/store/disk.go @@ -4,9 +4,11 @@ import ( "io/ioutil" "os" "path" + "time" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/shared" "github.com/lbryio/reflector.go/store/speedwalk" ) @@ -52,21 +54,22 @@ func (d *DiskStore) Has(hash string) (bool, error) { } // Get returns the blob or an error if the blob doesn't exist. -func (d *DiskStore) Get(hash string) (stream.Blob, error) { +func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() err := d.initOnce() if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err } blob, err := ioutil.ReadFile(d.path(hash)) if err != nil { if os.IsNotExist(err) { - return nil, errors.Err(ErrBlobNotFound) + return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound) } - return nil, errors.Err(err) + return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err) } - return blob, nil + return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil } // Put stores the blob on disk diff --git a/store/disk_test.go b/store/disk_test.go index 3bc088a..72c7755 100644 --- a/store/disk_test.go +++ b/store/disk_test.go @@ -28,7 +28,7 @@ func TestDiskStore_Get(t *testing.T) { err = ioutil.WriteFile(expectedPath, data, os.ModePerm) require.NoError(t, err) - blob, err := d.Get(hash) + blob, _, err := d.Get(hash) assert.NoError(t, err) assert.EqualValues(t, data, blob) } @@ -39,7 +39,7 @@ func TestDiskStore_GetNonexistentBlob(t *testing.T) { defer os.RemoveAll(tmpDir) d := NewDiskStore(tmpDir, 2) - blob, err := d.Get("nonexistent") + blob, _, err := d.Get("nonexistent") assert.Nil(t, blob) assert.True(t, errors.Is(err, ErrBlobNotFound)) } diff --git a/store/lfuda.go b/store/lfuda.go index 0eb7296..b0437b0 100644 --- a/store/lfuda.go +++ b/store/lfuda.go @@ -1,10 +1,13 @@ package store import ( + "time" + "github.com/bparli/lfuda-go" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/shared" "github.com/sirupsen/logrus" ) @@ -49,17 +52,18 @@ func (l *LFUDAStore) Has(hash string) (bool, error) { } // Get returns the blob or an error if the blob doesn't exist. -func (l *LFUDAStore) Get(hash string) (stream.Blob, error) { +func (l *LFUDAStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() _, has := l.lfuda.Get(hash) if !has { - return nil, errors.Err(ErrBlobNotFound) + return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound) } - blob, err := l.store.Get(hash) + blob, stack, err := l.store.Get(hash) if errors.Is(err, ErrBlobNotFound) { // Blob disappeared from underlying store l.lfuda.Remove(hash) } - return blob, err + return blob, stack.Stack(time.Since(start), l.Name()), err } // Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!! diff --git a/store/lfuda_test.go b/store/lfuda_test.go index d5eae47..d8657ac 100644 --- a/store/lfuda_test.go +++ b/store/lfuda_test.go @@ -40,11 +40,11 @@ func TestFUDAStore_Eviction(t *testing.T) { err = lfuda.Put("two", b) require.NoError(t, err) - _, err = lfuda.Get("five") + _, _, err = lfuda.Get("five") require.NoError(t, err) - _, err = lfuda.Get("four") + _, _, err = lfuda.Get("four") require.NoError(t, err) - _, err = lfuda.Get("two") + _, _, err = lfuda.Get("two") require.NoError(t, err) assert.Equal(t, cacheMaxBlobs, len(mem.Debug())) @@ -102,7 +102,7 @@ func TestFUDAStore_UnderlyingBlobMissing(t *testing.T) { // hash still exists in lru assert.True(t, lfuda.lfuda.Contains(hash)) - blob, err := lfuda.Get(hash) + blob, _, err := lfuda.Get(hash) assert.Nil(t, blob) assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s", reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(), diff --git a/store/lru.go b/store/lru.go index ce0d83d..e96931b 100644 --- a/store/lru.go +++ b/store/lru.go @@ -1,9 +1,12 @@ package store import ( + "time" + "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/shared" golru "github.com/hashicorp/golang-lru" "github.com/sirupsen/logrus" @@ -56,17 +59,18 @@ func (l *LRUStore) Has(hash string) (bool, error) { } // Get returns the blob or an error if the blob doesn't exist. -func (l *LRUStore) Get(hash string) (stream.Blob, error) { +func (l *LRUStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() _, has := l.lru.Get(hash) if !has { - return nil, errors.Err(ErrBlobNotFound) + return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound) } - blob, err := l.store.Get(hash) + blob, stack, err := l.store.Get(hash) if errors.Is(err, ErrBlobNotFound) { // Blob disappeared from underlying store l.lru.Remove(hash) } - return blob, err + return blob, stack.Stack(time.Since(start), l.Name()), err } // Put stores the blob diff --git a/store/lru_test.go b/store/lru_test.go index 2bd934c..6aec768 100644 --- a/store/lru_test.go +++ b/store/lru_test.go @@ -89,7 +89,7 @@ func TestLRUStore_UnderlyingBlobMissing(t *testing.T) { // hash still exists in lru assert.True(t, lru.lru.Contains(hash)) - blob, err := lru.Get(hash) + blob, _, err := lru.Get(hash) assert.Nil(t, blob) assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s", reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(), diff --git a/store/memory.go b/store/memory.go index 62b93b6..30d8dea 100644 --- a/store/memory.go +++ b/store/memory.go @@ -2,9 +2,11 @@ package store import ( "sync" + "time" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/shared" ) // MemStore is an in memory only blob store with no persistence. @@ -34,14 +36,15 @@ func (m *MemStore) Has(hash string) (bool, error) { } // Get returns the blob byte slice if present and errors if the blob is not found. -func (m *MemStore) Get(hash string) (stream.Blob, error) { +func (m *MemStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() m.mu.RLock() defer m.mu.RUnlock() blob, ok := m.blobs[hash] if !ok { - return nil, errors.Err(ErrBlobNotFound) + return nil, shared.NewBlobTrace(time.Since(start), m.Name()), errors.Err(ErrBlobNotFound) } - return blob, nil + return blob, shared.NewBlobTrace(time.Since(start), m.Name()), nil } // Put stores the blob in memory diff --git a/store/memory_test.go b/store/memory_test.go index 8d85114..775850b 100644 --- a/store/memory_test.go +++ b/store/memory_test.go @@ -25,7 +25,7 @@ func TestMemStore_Get(t *testing.T) { t.Error("error getting memory blob - ", err) } - gotBlob, err := s.Get(hash) + gotBlob, _, err := s.Get(hash) if err != nil { t.Errorf("Expected no error, got %v", err) } @@ -33,7 +33,7 @@ func TestMemStore_Get(t *testing.T) { t.Error("Got blob that is different from expected blob") } - missingBlob, err := s.Get("nonexistent hash") + missingBlob, _, err := s.Get("nonexistent hash") if err == nil { t.Errorf("Expected ErrBlobNotFound, got nil") } diff --git a/store/noop.go b/store/noop.go index 2792a0d..9cadf5c 100644 --- a/store/noop.go +++ b/store/noop.go @@ -1,15 +1,22 @@ package store -import "github.com/lbryio/lbry.go/v2/stream" +import ( + "time" + + "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/shared" +) // NoopStore is a store that does nothing type NoopStore struct{} const nameNoop = "noop" -func (n *NoopStore) Name() string { return nameNoop } -func (n *NoopStore) Has(_ string) (bool, error) { return false, nil } -func (n *NoopStore) Get(_ string) (stream.Blob, error) { return nil, nil } +func (n *NoopStore) Name() string { return nameNoop } +func (n *NoopStore) Has(_ string) (bool, error) { return false, nil } +func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) { + return nil, shared.NewBlobTrace(time.Since(time.Now()), n.Name()), nil +} func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil } func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil } func (n *NoopStore) Delete(_ string) error { return nil } diff --git a/store/s3.go b/store/s3.go index 02a23c5..75d66cc 100644 --- a/store/s3.go +++ b/store/s3.go @@ -8,6 +8,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/shared" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -65,17 +66,18 @@ func (s *S3Store) Has(hash string) (bool, error) { } // Get returns the blob slice if present or errors on S3. -func (s *S3Store) Get(hash string) (stream.Blob, error) { +func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() //Todo-Need to handle error for blob doesn't exist for consistency. err := s.initOnce() if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err } log.Debugf("Getting %s from S3", hash[:8]) defer func(t time.Time) { log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String()) - }(time.Now()) + }(start) buf := &aws.WriteAtBuffer{} _, err = s3manager.NewDownloader(s.session).Download(buf, &s3.GetObjectInput{ @@ -86,15 +88,15 @@ func (s *S3Store) Get(hash string) (stream.Blob, error) { if aerr, ok := err.(awserr.Error); ok { switch aerr.Code() { case s3.ErrCodeNoSuchBucket: - return nil, errors.Err("bucket %s does not exist", s.bucket) + return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket) case s3.ErrCodeNoSuchKey: - return nil, errors.Err(ErrBlobNotFound) + return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound) } } - return buf.Bytes(), err + return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), err } - return buf.Bytes(), nil + return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), nil } // Put stores the blob on S3 or errors if S3 connection errors. diff --git a/store/singleflight.go b/store/singleflight.go index 7d0139d..362182d 100644 --- a/store/singleflight.go +++ b/store/singleflight.go @@ -4,6 +4,7 @@ import ( "time" "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/shared" "github.com/lbryio/lbry.go/v2/stream" @@ -29,17 +30,24 @@ func (s *singleflightStore) Name() string { return "sf_" + s.BlobStore.Name() } +type getterResponse struct { + blob stream.Blob + stack shared.BlobTrace +} + // Get ensures that only one request per hash is sent to the origin at a time, // thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem -func (s *singleflightStore) Get(hash string) (stream.Blob, error) { +func (s *singleflightStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc() defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec() - blob, err, _ := s.sf.Do(hash, s.getter(hash)) + gr, err, _ := s.sf.Do(hash, s.getter(hash)) if err != nil { - return nil, err + return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err } - return blob.(stream.Blob), nil + rsp := gr.(getterResponse) + return rsp.blob, rsp.stack, nil } // getter returns a function that gets a blob from the origin @@ -50,9 +58,12 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) { defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec() start := time.Now() - blob, err := s.BlobStore.Get(hash) + blob, stack, err := s.BlobStore.Get(hash) if err != nil { - return nil, err + return getterResponse{ + blob: nil, + stack: stack.Stack(time.Since(start), s.Name()), + }, err } rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds() @@ -62,7 +73,10 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) { metrics.LabelSource: "origin", }).Set(rate) - return blob, nil + return getterResponse{ + blob: blob, + stack: stack.Stack(time.Since(start), s.Name()), + }, nil } } diff --git a/store/store.go b/store/store.go index 36d388d..bd9223b 100644 --- a/store/store.go +++ b/store/store.go @@ -3,6 +3,7 @@ package store import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/shared" ) // BlobStore is an interface for handling blob storage. @@ -12,7 +13,7 @@ type BlobStore interface { // Does blob exist in the store. Has(hash string) (bool, error) // Get the blob from the store. Must return ErrBlobNotFound if blob is not in store. - Get(hash string) (stream.Blob, error) + Get(hash string) (stream.Blob, shared.BlobTrace, error) // Put the blob into the store. Put(hash string, blob stream.Blob) error // Put an SD blob into the store. -- 2.45.2 From 7b49dd115b75b1fd16362248dcee1d7e631926eb Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 23 Feb 2021 15:08:32 +0100 Subject: [PATCH 22/51] remove panics --- peer/http3/store.go | 9 ++++++--- peer/store.go | 9 ++++++--- store/cloudfront_ro.go | 6 +++--- store/store.go | 3 +++ 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/peer/http3/store.go b/peer/http3/store.go index 2d9a23c..f842437 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -13,6 +13,9 @@ import ( "github.com/lucas-clemente/quic-go/http3" ) +//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method +var ErrNotImplemented = errors.Base("this store does not implement this method") + // Store is a blob store that gets blobs from a peer. // It satisfies the store.BlobStore interface but cannot put or delete blobs. type Store struct { @@ -81,17 +84,17 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) { // Put is not supported func (p *Store) Put(hash string, blob stream.Blob) error { - panic("http3Store cannot put or delete blobs") + return errors.Err(ErrNotImplemented) } // PutSD is not supported func (p *Store) PutSD(hash string, blob stream.Blob) error { - panic("http3Store cannot put or delete blobs") + return errors.Err(ErrNotImplemented) } // Delete is not supported func (p *Store) Delete(hash string) error { - panic("http3Store cannot put or delete blobs") + return errors.Err(ErrNotImplemented) } // Delete is not supported diff --git a/peer/store.go b/peer/store.go index c227884..6bf3b38 100644 --- a/peer/store.go +++ b/peer/store.go @@ -8,6 +8,9 @@ import ( "github.com/lbryio/reflector.go/shared" ) +//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method +var ErrNotImplemented = errors.Base("this store does not implement this method") + // Store is a blob store that gets blobs from a peer. // It satisfies the store.BlobStore interface but cannot put or delete blobs. type Store struct { @@ -56,17 +59,17 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) { // Put is not supported func (p *Store) Put(hash string, blob stream.Blob) error { - panic("PeerStore cannot put or delete blobs") + return errors.Err(ErrNotImplemented) } // PutSD is not supported func (p *Store) PutSD(hash string, blob stream.Blob) error { - panic("PeerStore cannot put or delete blobs") + return errors.Err(ErrNotImplemented) } // Delete is not supported func (p *Store) Delete(hash string) error { - panic("PeerStore cannot put or delete blobs") + return errors.Err(ErrNotImplemented) } // Delete is not supported diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go index 51fe4b8..fcc8ed0 100644 --- a/store/cloudfront_ro.go +++ b/store/cloudfront_ro.go @@ -93,15 +93,15 @@ func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, } func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error { - panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore") + return errors.Err(ErrNotImplemented) } func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error { - panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore") + return errors.Err(ErrNotImplemented) } func (c *CloudFrontROStore) Delete(_ string) error { - panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore") + return errors.Err(ErrNotImplemented) } // Shutdown shuts down the store gracefully diff --git a/store/store.go b/store/store.go index bd9223b..3a2fc6f 100644 --- a/store/store.go +++ b/store/store.go @@ -40,3 +40,6 @@ type lister interface { //ErrBlobNotFound is a standard error when a blob is not found in the store. var ErrBlobNotFound = errors.Base("blob not found") + +//ErrNotImplemented is a standard error when a store that implements this interface does not implement a method +var ErrNotImplemented = errors.Base("this store does not implement this method") -- 2.45.2 From 8cb73896199b9ec808505e6b42435d40f75654b3 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 23 Feb 2021 15:23:46 +0100 Subject: [PATCH 23/51] make it simpler --- peer/http3/store.go | 9 +++------ peer/store.go | 9 +++------ shared/errors.go | 6 ++++++ store/cloudfront_ro.go | 6 +++--- store/store.go | 3 --- 5 files changed, 15 insertions(+), 18 deletions(-) create mode 100644 shared/errors.go diff --git a/peer/http3/store.go b/peer/http3/store.go index f842437..8a5c812 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -13,9 +13,6 @@ import ( "github.com/lucas-clemente/quic-go/http3" ) -//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method -var ErrNotImplemented = errors.Base("this store does not implement this method") - // Store is a blob store that gets blobs from a peer. // It satisfies the store.BlobStore interface but cannot put or delete blobs. type Store struct { @@ -84,17 +81,17 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) { // Put is not supported func (p *Store) Put(hash string, blob stream.Blob) error { - return errors.Err(ErrNotImplemented) + return errors.Err(shared.ErrNotImplemented) } // PutSD is not supported func (p *Store) PutSD(hash string, blob stream.Blob) error { - return errors.Err(ErrNotImplemented) + return errors.Err(shared.ErrNotImplemented) } // Delete is not supported func (p *Store) Delete(hash string) error { - return errors.Err(ErrNotImplemented) + return errors.Err(shared.ErrNotImplemented) } // Delete is not supported diff --git a/peer/store.go b/peer/store.go index 6bf3b38..3068794 100644 --- a/peer/store.go +++ b/peer/store.go @@ -8,9 +8,6 @@ import ( "github.com/lbryio/reflector.go/shared" ) -//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method -var ErrNotImplemented = errors.Base("this store does not implement this method") - // Store is a blob store that gets blobs from a peer. // It satisfies the store.BlobStore interface but cannot put or delete blobs. type Store struct { @@ -59,17 +56,17 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) { // Put is not supported func (p *Store) Put(hash string, blob stream.Blob) error { - return errors.Err(ErrNotImplemented) + return errors.Err(shared.ErrNotImplemented) } // PutSD is not supported func (p *Store) PutSD(hash string, blob stream.Blob) error { - return errors.Err(ErrNotImplemented) + return errors.Err(shared.ErrNotImplemented) } // Delete is not supported func (p *Store) Delete(hash string) error { - return errors.Err(ErrNotImplemented) + return errors.Err(shared.ErrNotImplemented) } // Delete is not supported diff --git a/shared/errors.go b/shared/errors.go new file mode 100644 index 0000000..f103394 --- /dev/null +++ b/shared/errors.go @@ -0,0 +1,6 @@ +package shared + +import "github.com/lbryio/lbry.go/v2/extras/errors" + +//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method +var ErrNotImplemented = errors.Base("this store does not implement this method") diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go index fcc8ed0..757174c 100644 --- a/store/cloudfront_ro.go +++ b/store/cloudfront_ro.go @@ -93,15 +93,15 @@ func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, } func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error { - return errors.Err(ErrNotImplemented) + return errors.Err(shared.ErrNotImplemented) } func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error { - return errors.Err(ErrNotImplemented) + return errors.Err(shared.ErrNotImplemented) } func (c *CloudFrontROStore) Delete(_ string) error { - return errors.Err(ErrNotImplemented) + return errors.Err(shared.ErrNotImplemented) } // Shutdown shuts down the store gracefully diff --git a/store/store.go b/store/store.go index 3a2fc6f..bd9223b 100644 --- a/store/store.go +++ b/store/store.go @@ -40,6 +40,3 @@ type lister interface { //ErrBlobNotFound is a standard error when a blob is not found in the store. var ErrBlobNotFound = errors.Base("blob not found") - -//ErrNotImplemented is a standard error when a store that implements this interface does not implement a method -var ErrNotImplemented = errors.Base("this store does not implement this method") -- 2.45.2 From ebb62d0a245e256fdeb7c9ae4affcf92a4eac5d5 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 29 Mar 2021 19:44:27 +0200 Subject: [PATCH 24/51] run go mod tidy --- go.sum | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/go.sum b/go.sum index b7dbdb2..2b6ef87 100644 --- a/go.sum +++ b/go.sum @@ -37,7 +37,6 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyElegc= github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -83,9 +82,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -118,7 +115,6 @@ github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200j github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= @@ -130,7 +126,6 @@ github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= @@ -146,7 +141,6 @@ github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+u github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw= github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -169,7 +163,6 @@ github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk= github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -204,7 +197,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce h1:xdsDDbiBDQTKASoGEZ+pEmF1OnWuu8AQ9I8iNbHNeno= github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -215,7 +207,6 @@ github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0 github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= @@ -262,7 +253,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0= github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= -github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f h1:ovd2wPXzkT80vdP/FX5xcQeXu0i9RAo80SQ6qIsrAjM= github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c= github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= @@ -285,7 +275,6 @@ github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKR github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0= github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -353,23 +342,19 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -422,7 +407,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -430,7 +414,6 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.1 h1:Lt3ihYMlE+lreX1GS4Qw4ZsNpYQLxIXKBTEOXm3nt6I= github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= @@ -440,14 +423,12 @@ github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig= github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.0.2 h1:Ncr3ZIuJn322w2k1qmzXDnkLAdQMlJqBa9kfAH+irso= github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= @@ -544,7 +525,6 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko= golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= @@ -559,7 +539,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -587,7 +566,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= @@ -595,7 +573,6 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -683,12 +660,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8= gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -700,7 +675,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -- 2.45.2 From 3a441aed3abbb13a685e72acb32957353c0a5432 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 29 Mar 2021 19:56:18 +0200 Subject: [PATCH 25/51] fix issues caused by beamer's renaming --- cmd/decode.go | 4 ++-- go.mod | 6 +++--- go.sum | 14 ++++++++------ wallet/client.go | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cmd/decode.go b/cmd/decode.go index 4a54668..7cb677f 100644 --- a/cmd/decode.go +++ b/cmd/decode.go @@ -4,7 +4,7 @@ import ( "encoding/hex" "fmt" - "github.com/lbryio/lbry.go/v2/schema/claim" + "github.com/lbryio/lbry.go/v2/schema/stake" "github.com/davecgh/go-spew/spew" "github.com/golang/protobuf/jsonpb" @@ -23,7 +23,7 @@ func init() { } func decodeCmd(cmd *cobra.Command, args []string) { - c, err := claim.DecodeClaimHex(args[0], "") + c, err := stake.DecodeClaimHex(args[0], "") if err != nil { log.Fatal(err) } diff --git a/go.mod b/go.mod index 4570f1d..ba4f821 100644 --- a/go.mod +++ b/go.mod @@ -23,8 +23,8 @@ require ( github.com/karrick/godirwalk v1.16.1 github.com/lbryio/chainquery v1.9.0 github.com/lbryio/lbry.go v1.1.2 // indirect - github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 - github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec + github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011 + github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 github.com/lucas-clemente/quic-go v0.19.3 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.3 @@ -33,7 +33,7 @@ require ( github.com/spf13/cast v1.3.0 github.com/spf13/cobra v0.0.3 github.com/spf13/viper v1.7.1 // indirect - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.7.0 github.com/volatiletech/null v8.0.0+incompatible go.uber.org/atomic v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 diff --git a/go.sum b/go.sum index 2b6ef87..5895055 100644 --- a/go.sum +++ b/go.sum @@ -256,17 +256,16 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c= github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY= -github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4= +github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011 h1:r1NoX3NQu/Me+/qw4OzJGw8bhOUnTZHUneCbIV6SC+Y= +github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011/go.mod h1:sUhhSKqPNkiwgBqvBzJIqfLLzGH8hkDGrrO/HcaXzFc= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= -github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= -github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c= -github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= +github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M= +github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= @@ -436,8 +435,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -678,6 +678,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= diff --git a/wallet/client.go b/wallet/client.go index a4cb165..3dc0e8c 100644 --- a/wallet/client.go +++ b/wallet/client.go @@ -6,7 +6,7 @@ import ( "github.com/lbryio/chainquery/lbrycrd" "github.com/lbryio/lbry.go/v2/extras/errors" - "github.com/lbryio/lbry.go/v2/schema/claim" + "github.com/lbryio/lbry.go/v2/schema/stake" types "github.com/lbryio/types/v2/go" "github.com/btcsuite/btcutil" @@ -140,7 +140,7 @@ func (n *Node) GetClaimInTx(txid string, nout int) (*types.Claim, error) { return nil, errors.Err(err) } - ch, err := claim.DecodeClaimBytes(value, "") + ch, err := stake.DecodeClaimBytes(value, "") if err != nil { return nil, errors.Err(err) } -- 2.45.2 From 90c36fbe243f0f2fc0560206685792682bc68ae9 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 31 Mar 2021 04:53:27 +0200 Subject: [PATCH 26/51] upgrade quic-go add cache for blobs not found --- go.mod | 2 +- go.sum | 27 ++++++++++++--------------- peer/http3/server.go | 4 ++-- peer/http3/store.go | 18 +++++++++++++++--- 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index ba4f821..ed3ebcb 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/lbryio/lbry.go v1.1.2 // indirect github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011 github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 - github.com/lucas-clemente/quic-go v0.19.3 + github.com/lucas-clemente/quic-go v0.20.0 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.3 github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index 5895055..f61f06b 100644 --- a/go.sum +++ b/go.sum @@ -109,15 +109,13 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -267,8 +265,8 @@ github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiV github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M= github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4= -github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= +github.com/lucas-clemente/quic-go v0.20.0 h1:FSU3YN5VnLafHR27Ejs1r1CYMS7XMyIVDzRewkDLNBw= +github.com/lucas-clemente/quic-go v0.20.0/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= @@ -280,10 +278,10 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc= -github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= -github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ= -github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.4 h1:RehYMOyRW8hPVEja1KBVsFVNSm35Jj9Mvs5yNoZZ28A= +github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJGYmwfwKQN2gU= +github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -460,7 +458,6 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM= @@ -506,6 +503,7 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -568,9 +566,9 @@ golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg= +golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -602,6 +600,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -691,7 +690,5 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/peer/http3/server.go b/peer/http3/server.go index c33a676..16fa156 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -65,8 +65,8 @@ type availabilityResponse struct { func (s *Server) Start(address string) error { log.Println("HTTP3 peer listening on " + address) quicConf := &quic.Config{ - HandshakeTimeout: 4 * time.Second, - MaxIdleTimeout: 20 * time.Second, + HandshakeIdleTimeout: 4 * time.Second, + MaxIdleTimeout: 20 * time.Second, } r := mux.NewRouter() r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) { diff --git a/peer/http3/store.go b/peer/http3/store.go index 8a5c812..bcc7cae 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -4,11 +4,14 @@ import ( "crypto/tls" "crypto/x509" "net/http" + "strings" + "sync" "time" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/shared" + "github.com/lbryio/reflector.go/store" "github.com/lucas-clemente/quic-go" "github.com/lucas-clemente/quic-go/http3" ) @@ -16,7 +19,8 @@ import ( // Store is a blob store that gets blobs from a peer. // It satisfies the store.BlobStore interface but cannot put or delete blobs. type Store struct { - opts StoreOpts + opts StoreOpts + NotFoundCache *sync.Map } // StoreOpts allows to set options for a new Store. @@ -27,12 +31,12 @@ type StoreOpts struct { // NewStore makes a new peer store. func NewStore(opts StoreOpts) *Store { - return &Store{opts: opts} + return &Store{opts: opts, NotFoundCache: &sync.Map{}} } func (p *Store) getClient() (*Client, error) { var qconf quic.Config - qconf.HandshakeTimeout = 4 * time.Second + qconf.HandshakeIdleTimeout = 4 * time.Second qconf.MaxIdleTimeout = 20 * time.Second pool, err := x509.SystemCertPool() if err != nil { @@ -71,7 +75,15 @@ func (p *Store) Has(hash string) (bool, error) { // Get downloads the blob from the peer func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) { start := time.Now() + if lastChecked, ok := p.NotFoundCache.Load(hash); ok { + if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) { + return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound + } + } c, err := p.getClient() + if err != nil && strings.Contains(err.Error(), "blob not found") { + p.NotFoundCache.Store(hash, time.Now()) + } if err != nil { return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err } -- 2.45.2 From 38b44218f2a6fd843faa643165f9df431fd78672 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 5 Apr 2021 23:34:45 +0200 Subject: [PATCH 27/51] check blobs when reading them --- store/disk.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/store/disk.go b/store/disk.go index 412d326..08cdbd1 100644 --- a/store/disk.go +++ b/store/disk.go @@ -1,6 +1,9 @@ package store import ( + "crypto/sha512" + "encoding/hex" + "fmt" "io/ioutil" "os" "path" @@ -10,6 +13,7 @@ import ( "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/shared" "github.com/lbryio/reflector.go/store/speedwalk" + log "github.com/sirupsen/logrus" ) // DiskStore stores blobs on a local disk @@ -68,6 +72,17 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { } return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err) } + hashBytes := sha512.Sum384(blob) + readHash := hex.EncodeToString(hashBytes[:]) + if hash != readHash { + message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash) + log.Errorf("%s", message) + err := d.Delete(hash) + if err != nil { + return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err + } + return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message) + } return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil } -- 2.45.2 From bd13836897b0b69e2216517415e665b6f15687e5 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 6 Apr 2021 14:00:36 -0400 Subject: [PATCH 28/51] Add request queue for blob cache --- internal/metrics/metrics.go | 5 +++ peer/http3/server.go | 87 ++++++++++++++++++++----------------- peer/http3/worker.go | 40 +++++++++++++++++ 3 files changed, 91 insertions(+), 41 deletions(-) create mode 100644 peer/http3/worker.go diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 8d46266..2793fc0 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -201,6 +201,11 @@ var ( Name: "s3_in_bytes", Help: "Total number of incoming bytes (from S3-CF)", }) + Http3BlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: ns, + Name: "http3_blob_request_queue_size", + Help: "Blob requests of https queue size", + }) ) func CacheLabels(name, component string) prometheus.Labels { diff --git a/peer/http3/server.go b/peer/http3/server.go index 16fa156..da8f485 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -70,46 +70,7 @@ func (s *Server) Start(address string) error { } r := mux.NewRouter() r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - requestedBlob := vars["hash"] - traceParam := r.URL.Query().Get("trace") - var err error - wantsTrace := false - if traceParam != "" { - wantsTrace, err = strconv.ParseBool(traceParam) - if err != nil { - wantsTrace = false - } - } - blob, trace, err := s.store.Get(requestedBlob) - - if wantsTrace { - serialized, err := trace.Serialize() - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - w.Header().Add("Via", serialized) - log.Debug(trace.String()) - } - if err != nil { - if errors.Is(err, store.ErrBlobNotFound) { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err)) - s.logError(err) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - _, err = w.Write(blob) - if err != nil { - s.logError(err) - } - metrics.MtrOutBytesUdp.Add(float64(len(blob))) - metrics.BlobDownloadCount.Inc() - metrics.Http3DownloadCount.Inc() + enqueue(&blobRequest{request: r, reply: w}) }) r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -147,7 +108,7 @@ func (s *Server) Start(address string) error { }, QuicConfig: quicConf, } - + go InitWorkers(s, 100) go s.listenForShutdown(&server) s.grp.Add(1) go func() { @@ -196,3 +157,47 @@ func (s *Server) listenForShutdown(listener *http3.Server) { log.Error("error closing listener for peer server - ", err) } } + +func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + requestedBlob := vars["hash"] + traceParam := r.URL.Query().Get("trace") + var err error + wantsTrace := false + if traceParam != "" { + wantsTrace, err = strconv.ParseBool(traceParam) + if err != nil { + wantsTrace = false + } + } + + blob, trace, err := s.store.Get(requestedBlob) + + if wantsTrace { + serialized, err := trace.Serialize() + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + w.Header().Add("Via", serialized) + log.Debug(trace.String()) + } + if err != nil { + if errors.Is(err, store.ErrBlobNotFound) { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err)) + s.logError(err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + _, err = w.Write(blob) + if err != nil { + s.logError(err) + } + metrics.MtrOutBytesUdp.Add(float64(len(blob))) + metrics.BlobDownloadCount.Inc() + metrics.Http3DownloadCount.Inc() +} diff --git a/peer/http3/worker.go b/peer/http3/worker.go new file mode 100644 index 0000000..182ca08 --- /dev/null +++ b/peer/http3/worker.go @@ -0,0 +1,40 @@ +package http3 + +import ( + "net/http" + + "github.com/lbryio/reflector.go/internal/metrics" + + "github.com/lbryio/lbry.go/v2/extras/stop" +) + +type blobRequest struct { + request *http.Request + reply http.ResponseWriter +} + +var getReqCh = make(chan *blobRequest) + +func InitWorkers(server *Server, workers int) error { + stopper := stop.New(server.grp) + for i := 0; i < workers; i++ { + go func(worker int) { + select { + case <-stopper.Ch(): + case r := <-getReqCh: + metrics.Http3BlobReqQueue.Dec() + process(server, r) + } + }(i) + } + return nil +} + +func enqueue(b *blobRequest) { + metrics.Http3BlobReqQueue.Inc() + getReqCh <- b +} + +func process(server *Server, r *blobRequest) { + server.HandleGetBlob(r.reply, r.request) +} -- 2.45.2 From b97595311f077b6a40b11fdb182d92dcd07a4efa Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 6 Apr 2021 14:21:05 -0400 Subject: [PATCH 29/51] Wait for request to be handled before returning --- peer/http3/server.go | 6 +++++- peer/http3/worker.go | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/peer/http3/server.go b/peer/http3/server.go index da8f485..2147421 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -70,7 +70,11 @@ func (s *Server) Start(address string) error { } r := mux.NewRouter() r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) { - enqueue(&blobRequest{request: r, reply: w}) + waiter := stop.New() + enqueue(&blobRequest{request: r, reply: w, finished: waiter}) + select { + case <-waiter.Ch(): + } }) r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) diff --git a/peer/http3/worker.go b/peer/http3/worker.go index 182ca08..62c4c4d 100644 --- a/peer/http3/worker.go +++ b/peer/http3/worker.go @@ -9,8 +9,9 @@ import ( ) type blobRequest struct { - request *http.Request - reply http.ResponseWriter + request *http.Request + reply http.ResponseWriter + finished *stop.Group } var getReqCh = make(chan *blobRequest) @@ -37,4 +38,5 @@ func enqueue(b *blobRequest) { func process(server *Server, r *blobRequest) { server.HandleGetBlob(r.reply, r.request) + r.finished.Done() } -- 2.45.2 From 25a7fac4f0a2cd6287bab7a6da7be4250f81d956 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 6 Apr 2021 14:28:29 -0400 Subject: [PATCH 30/51] use wait group not stopper --- peer/http3/server.go | 8 ++++---- peer/http3/worker.go | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/peer/http3/server.go b/peer/http3/server.go index 2147421..89a64db 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -11,6 +11,7 @@ import ( "math/big" "net/http" "strconv" + "sync" "time" "github.com/lbryio/reflector.go/internal/metrics" @@ -70,11 +71,10 @@ func (s *Server) Start(address string) error { } r := mux.NewRouter() r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) { - waiter := stop.New() + waiter := &sync.WaitGroup{} + waiter.Add(1) enqueue(&blobRequest{request: r, reply: w, finished: waiter}) - select { - case <-waiter.Ch(): - } + waiter.Wait() }) r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) diff --git a/peer/http3/worker.go b/peer/http3/worker.go index 62c4c4d..dac2893 100644 --- a/peer/http3/worker.go +++ b/peer/http3/worker.go @@ -2,6 +2,7 @@ package http3 import ( "net/http" + "sync" "github.com/lbryio/reflector.go/internal/metrics" @@ -11,7 +12,7 @@ import ( type blobRequest struct { request *http.Request reply http.ResponseWriter - finished *stop.Group + finished *sync.WaitGroup } var getReqCh = make(chan *blobRequest) -- 2.45.2 From dc95351cf30c3332185c929b18fc0eba29f54afa Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 7 Apr 2021 04:46:18 +0200 Subject: [PATCH 31/51] add integrity check cmd throttle live integrity checks bug fixes --- cmd/integrity.go | 93 ++++++++++++++++++++++++++++++++++++++++++++ peer/http3/worker.go | 18 +++++---- store/disk.go | 30 +++++++++----- 3 files changed, 124 insertions(+), 17 deletions(-) create mode 100644 cmd/integrity.go diff --git a/cmd/integrity.go b/cmd/integrity.go new file mode 100644 index 0000000..8be9e2b --- /dev/null +++ b/cmd/integrity.go @@ -0,0 +1,93 @@ +package cmd + +import ( + "crypto/sha512" + "encoding/hex" + "io/ioutil" + "os" + "path" + "runtime" + "sync/atomic" + "time" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/reflector.go/meta" + "github.com/lbryio/reflector.go/store/speedwalk" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var threads int + +func init() { + var cmd = &cobra.Command{ + Use: "check-integrity", + Short: "check blobs integrity for a given path", + Run: integrityCheckCmd, + } + cmd.Flags().StringVar(&diskStorePath, "store-path", "", "path of the store where all blobs are cached") + cmd.Flags().IntVar(&threads, "threads", runtime.NumCPU()-1, "number of concurrent threads to process blobs") + rootCmd.AddCommand(cmd) +} + +func integrityCheckCmd(cmd *cobra.Command, args []string) { + log.Printf("reflector %s", meta.VersionString()) + if diskStorePath == "" { + log.Fatal("store-path must be defined") + } + + blobs, err := speedwalk.AllFiles(diskStorePath, true) + if err != nil { + log.Errorf("error while reading blobs from disk %s", errors.FullTrace(err)) + } + tasks := make(chan string, len(blobs)) + done := make(chan bool) + processed := new(int32) + go produce(tasks, blobs) + cpus := runtime.NumCPU() + for i := 0; i < cpus-1; i++ { + go consume(i, tasks, done, len(blobs), processed) + } + <-done +} + +func produce(tasks chan<- string, blobs []string) { + for _, b := range blobs { + tasks <- b + } + close(tasks) +} + +func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int, processed *int32) { + start := time.Now() + + for b := range tasks { + checked := atomic.AddInt32(processed, 1) + if worker == 0 { + remaining := int32(totalTasks) - checked + timePerBlob := time.Since(start).Microseconds() / int64(checked) + remainingTime := time.Duration(int64(remaining)*timePerBlob) * time.Microsecond + log.Infof("[T%d] %d/%d blobs checked. ETA: %s", worker, checked, totalTasks, remainingTime.String()) + } + blobPath := path.Join(diskStorePath, b[:2], b) + blob, err := ioutil.ReadFile(blobPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + log.Errorf("[Worker %d] Error looking up blob %s: %s", worker, b, err.Error()) + continue + } + hashBytes := sha512.Sum384(blob) + readHash := hex.EncodeToString(hashBytes[:]) + if readHash != b { + log.Infof("[%s] found a broken blob while reading from disk. Actual hash: %s", b, readHash) + err := os.Remove(blobPath) + if err != nil { + log.Errorf("Error while deleting broken blob %s: %s", b, err.Error()) + } + } + } + done <- true +} diff --git a/peer/http3/worker.go b/peer/http3/worker.go index dac2893..d2b625b 100644 --- a/peer/http3/worker.go +++ b/peer/http3/worker.go @@ -15,21 +15,23 @@ type blobRequest struct { finished *sync.WaitGroup } -var getReqCh = make(chan *blobRequest) +var getReqCh = make(chan *blobRequest, 20000) -func InitWorkers(server *Server, workers int) error { +func InitWorkers(server *Server, workers int) { stopper := stop.New(server.grp) for i := 0; i < workers; i++ { go func(worker int) { - select { - case <-stopper.Ch(): - case r := <-getReqCh: - metrics.Http3BlobReqQueue.Dec() - process(server, r) + for { + select { + case <-stopper.Ch(): + case r := <-getReqCh: + metrics.Http3BlobReqQueue.Dec() + process(server, r) + } } }(i) } - return nil + return } func enqueue(b *blobRequest) { diff --git a/store/disk.go b/store/disk.go index 08cdbd1..eaecfbb 100644 --- a/store/disk.go +++ b/store/disk.go @@ -14,6 +14,7 @@ import ( "github.com/lbryio/reflector.go/shared" "github.com/lbryio/reflector.go/store/speedwalk" log "github.com/sirupsen/logrus" + "go.uber.org/atomic" ) // DiskStore stores blobs on a local disk @@ -25,8 +26,12 @@ type DiskStore struct { // true if initOnce ran, false otherwise initialized bool + + concurrentChecks atomic.Int32 } +const maxConcurrentChecks = 3 + // NewDiskStore returns an initialized file disk store pointer. func NewDiskStore(dir string, prefixLength int) *DiskStore { return &DiskStore{ @@ -72,16 +77,23 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { } return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err) } - hashBytes := sha512.Sum384(blob) - readHash := hex.EncodeToString(hashBytes[:]) - if hash != readHash { - message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash) - log.Errorf("%s", message) - err := d.Delete(hash) - if err != nil { - return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err + + // this is a rather poor yet effective way of throttling how many blobs can be checked concurrently + // poor because there is a possible race condition between the check and the actual +1 + if d.concurrentChecks.Load() < maxConcurrentChecks { + d.concurrentChecks.Add(1) + defer d.concurrentChecks.Sub(1) + hashBytes := sha512.Sum384(blob) + readHash := hex.EncodeToString(hashBytes[:]) + if hash != readHash { + message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash) + log.Errorf("%s", message) + err := d.Delete(hash) + if err != nil { + return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err + } + return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message) } - return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message) } return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil -- 2.45.2 From ec3aae33ba8d091c85c73fb384f9fb43c3686496 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 12 Apr 2021 23:05:50 +0200 Subject: [PATCH 32/51] add if this than that store switch to wasabi for uploads --- cmd/reflector.go | 10 +++-- internal/metrics/metrics.go | 13 +++++++ peer/http3/server.go | 2 +- store/cloudfront_rw.go | 6 +-- store/ittt.go | 75 +++++++++++++++++++++++++++++++++++++ store/s3.go | 10 +++-- 6 files changed, 104 insertions(+), 12 deletions(-) create mode 100644 store/ittt.go diff --git a/cmd/reflector.go b/cmd/reflector.go index 738a066..6fe44ad 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -37,6 +37,7 @@ var ( proxyProtocol string useDB bool cloudFrontEndpoint string + WasabiEndpoint string reflectorCmdDiskCache string bufferReflectorCmdDiskCache string reflectorCmdMemCache int @@ -52,6 +53,7 @@ func init() { cmd.Flags().StringVar(&proxyPort, "proxy-port", "5567", "port of another reflector server where blobs are fetched from") cmd.Flags().StringVar(&proxyProtocol, "proxy-protocol", "http3", "protocol used to fetch blobs from another reflector server (tcp/http3)") cmd.Flags().StringVar(&cloudFrontEndpoint, "cloudfront-endpoint", "", "CloudFront edge endpoint for standard HTTP retrieval") + cmd.Flags().StringVar(&WasabiEndpoint, "wasabi-endpoint", "", "Wasabi edge endpoint for standard HTTP retrieval") cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from") cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol") cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from") @@ -137,12 +139,12 @@ func setupStore() store.BlobStore { if conf != "none" { s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName) } - if cloudFrontEndpoint != "" { - cfs := store.NewCloudFrontROStore(cloudFrontEndpoint) + if cloudFrontEndpoint != "" && WasabiEndpoint != "" { + ittt := store.NewITTTStore(store.NewCloudFrontROStore(WasabiEndpoint), store.NewCloudFrontROStore(cloudFrontEndpoint)) if s3Store != nil { - s = store.NewCloudFrontRWStore(cfs, s3Store) + s = store.NewCloudFrontRWStore(ittt, s3Store) } else { - s = cfs + s = ittt } } else if s3Store != nil { s = s3Store diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 2793fc0..2570677 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -60,6 +60,7 @@ func (s *Server) Shutdown() { const ( ns = "reflector" subsystemCache = "cache" + subsystemITTT = "ittt" labelDirection = "direction" labelErrorType = "error_type" @@ -124,6 +125,18 @@ var ( Name: "hit_total", Help: "Total number of blobs retrieved from the cache storage", }, []string{LabelCacheType, LabelComponent}) + ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Subsystem: subsystemITTT, + Name: "this_hit_total", + Help: "Total number of blobs retrieved from the this storage", + }) + ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Subsystem: subsystemITTT, + Name: "that_hit_total", + Help: "Total number of blobs retrieved from the that storage", + }) CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: ns, Subsystem: subsystemCache, diff --git a/peer/http3/server.go b/peer/http3/server.go index 89a64db..30ae3fd 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -112,7 +112,7 @@ func (s *Server) Start(address string) error { }, QuicConfig: quicConf, } - go InitWorkers(s, 100) + go InitWorkers(s, 200) go s.listenForShutdown(&server) s.grp.Add(1) go func() { diff --git a/store/cloudfront_rw.go b/store/cloudfront_rw.go index 32ee418..6b293a8 100644 --- a/store/cloudfront_rw.go +++ b/store/cloudfront_rw.go @@ -7,15 +7,15 @@ import ( "github.com/lbryio/reflector.go/shared" ) -// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront, writes go to S3. +// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3. type CloudFrontRWStore struct { - cf *CloudFrontROStore + cf *ITTTStore s3 *S3Store } // NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer. // NOTE: It panics if either argument is nil. -func NewCloudFrontRWStore(cf *CloudFrontROStore, s3 *S3Store) *CloudFrontRWStore { +func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore { if cf == nil || s3 == nil { panic("both stores must be set") } diff --git a/store/ittt.go b/store/ittt.go new file mode 100644 index 0000000..5edc67d --- /dev/null +++ b/store/ittt.go @@ -0,0 +1,75 @@ +package store + +import ( + "time" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" + + "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/shared" +) + +// ITTT store performs an operation on this storage, if this fails, it attempts to run it on that +type ITTTStore struct { + this, that BlobStore +} + +// NewCachingStore makes a new caching disk store and returns a pointer to it. +func NewITTTStore(this, that BlobStore) *ITTTStore { + return &ITTTStore{ + this: this, + that: that, + } +} + +const nameIttt = "ittt" + +// Name is the cache type name +func (c *ITTTStore) Name() string { return nameIttt } + +// Has checks the cache and then the origin for a hash. It returns true if either store has it. +func (c *ITTTStore) Has(hash string) (bool, error) { + has, err := c.this.Has(hash) + if err != nil || !has { + has, err = c.that.Has(hash) + } + return has, err +} + +// Get tries to get the blob from this first, falling back to that. +func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() + blob, trace, err := c.this.Get(hash) + if err == nil { + metrics.ThisHitCount.Inc() + return blob, trace.Stack(time.Since(start), c.Name()), err + } + + blob, trace, err = c.that.Get(hash) + if err != nil { + return nil, trace.Stack(time.Since(start), c.Name()), err + } + metrics.ThatHitCount.Inc() + return blob, trace.Stack(time.Since(start), c.Name()), nil +} + +// Put not implemented +func (c *ITTTStore) Put(hash string, blob stream.Blob) error { + return errors.Err(shared.ErrNotImplemented) +} + +// PutSD not implemented +func (c *ITTTStore) PutSD(hash string, blob stream.Blob) error { + return errors.Err(shared.ErrNotImplemented) +} + +// Delete not implemented +func (c *ITTTStore) Delete(hash string) error { + return errors.Err(shared.ErrNotImplemented) +} + +// Shutdown shuts down the store gracefully +func (c *ITTTStore) Shutdown() { + return +} diff --git a/store/s3.go b/store/s3.go index 75d66cc..75ba459 100644 --- a/store/s3.go +++ b/store/s3.go @@ -112,10 +112,11 @@ func (s *S3Store) Put(hash string, blob stream.Blob) error { }(time.Now()) _, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(hash), - Body: bytes.NewBuffer(blob), - StorageClass: aws.String(s3.StorageClassIntelligentTiering), + Bucket: aws.String(s.bucket), + Key: aws.String(hash), + Body: bytes.NewBuffer(blob), + ACL: aws.String("public-read"), + //StorageClass: aws.String(s3.StorageClassIntelligentTiering), }) metrics.MtrOutBytesReflector.Add(float64(blob.Size())) @@ -152,6 +153,7 @@ func (s *S3Store) initOnce() error { sess, err := session.NewSession(&aws.Config{ Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""), Region: aws.String(s.region), + Endpoint: aws.String("https://s3.wasabisys.com"), }) if err != nil { return err -- 2.45.2 From 4392c9724262b60ed378cd727c2a2d9b35e4c638 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 13 Apr 2021 00:52:56 +0200 Subject: [PATCH 33/51] fix mess with lbry.go --- cmd/root.go | 6 ++ cmd/send.go | 160 +++++++++++++++++++++++++++++++++++++++++++++ cmd/sendblob.go | 6 +- go.mod | 7 +- go.sum | 11 ++-- publish/publish.go | 76 ++++++++++----------- 6 files changed, 211 insertions(+), 55 deletions(-) create mode 100644 cmd/send.go diff --git a/cmd/root.go b/cmd/root.go index ce28f8f..1e422a7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -163,3 +163,9 @@ func mustGetFlagInt64(cmd *cobra.Command, name string) int64 { checkErr(err) return v } + +func mustGetFlagBool(cmd *cobra.Command, name string) bool { + v, err := cmd.Flags().GetBool(name) + checkErr(err) + return v +} diff --git a/cmd/send.go b/cmd/send.go new file mode 100644 index 0000000..7021866 --- /dev/null +++ b/cmd/send.go @@ -0,0 +1,160 @@ +package cmd + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/signal" + "path" + "syscall" + + "github.com/lbryio/reflector.go/reflector" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" + + "github.com/spf13/cobra" +) + +func init() { + var cmd = &cobra.Command{ + Use: "send ADDRESS:PORT PATH", + Short: "Send a file to a reflector", + Args: cobra.ExactArgs(2), + Run: sendCmd, + } + cmd.PersistentFlags().String("sd-cache", "", "path to dir where sd blobs will be cached") + rootCmd.AddCommand(cmd) +} + +// todo: if retrying a large file is slow, we can add the ability to seek ahead in the file so we're not +// re-uploading blobs that already exist + +var hackyReflector reflector.Client + +func sendCmd(cmd *cobra.Command, args []string) { + reflectorAddress := args[0] + err := hackyReflector.Connect(reflectorAddress) + checkErr(err) + defer hackyReflector.Close() + + filePath := args[1] + file, err := os.Open(filePath) + checkErr(err) + defer file.Close() + + sdCachePath := "" + sdCacheDir := mustGetFlagString(cmd, "sd-cache") + if sdCacheDir != "" { + if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) { + err = os.MkdirAll(sdCacheDir, 0777) + checkErr(err) + } + sdCachePath = path.Join(sdCacheDir, filePath+".sdblob") + } + + var enc *stream.Encoder + + if sdCachePath != "" { + if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) { + sdBlob, err := ioutil.ReadFile(sdCachePath) + checkErr(err) + cachedSDBlob := &stream.SDBlob{} + err = cachedSDBlob.FromBlob(sdBlob) + checkErr(err) + enc = stream.NewEncoderFromSD(file, cachedSDBlob) + } + } + if enc == nil { + enc = stream.NewEncoder(file) + } + + exitCode := 0 + + var killed bool + interruptChan := make(chan os.Signal, 1) + signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM) + go func() { + sig := <-interruptChan + fmt.Printf("caught %s, exiting...\n", sig.String()) + killed = true + exitCode = 1 + }() + + for { + if killed { + break + } + + b, err := enc.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + fmt.Printf("error reading next blob: %v\n", err) + exitCode = 1 + break + } + + err = hackyReflect(b, false) + if err != nil { + fmt.Printf("error reflecting blob %s: %v\n", b.HashHex()[:8], err) + exitCode = 1 + break + } + } + + sd := enc.SDBlob() + //sd.StreamName = filepath.Base(filePath) + //sd.SuggestedFileName = filepath.Base(filePath) + err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666) + if err != nil { + fmt.Printf("error saving sd blob: %v\n", err) + fmt.Println(sd.ToJson()) + exitCode = 1 + } + + if killed { + os.Exit(exitCode) + } + + if reflectorAddress != "" { + err = hackyReflect(sd.ToBlob(), true) + if err != nil { + fmt.Printf("error reflecting sd blob %s: %v\n", sd.HashHex()[:8], err) + exitCode = 1 + } + } + + ret := struct { + SDHash string `json:"sd_hash"` + SourceHash string `json:"source_hash"` + }{ + SDHash: sd.HashHex(), + SourceHash: hex.EncodeToString(enc.SourceHash()), + } + + j, err := json.MarshalIndent(ret, "", " ") + checkErr(err) + fmt.Println(string(j)) + os.Exit(exitCode) +} + +func hackyReflect(b stream.Blob, sd bool) error { + var err error + if sd { + err = hackyReflector.SendSDBlob(b) + } else { + err = hackyReflector.SendBlob(b) + } + + if errors.Is(err, reflector.ErrBlobExists) { + //fmt.Printf("%s already reflected\n", b.HashHex()[:8]) + return nil + } + + return err +} diff --git a/cmd/sendblob.go b/cmd/sendblob.go index 3cd9f05..f37f390 100644 --- a/cmd/sendblob.go +++ b/cmd/sendblob.go @@ -2,7 +2,6 @@ package cmd import ( "crypto/rand" - "io/ioutil" "os" "github.com/lbryio/reflector.go/reflector" @@ -52,9 +51,8 @@ func sendBlobCmd(cmd *cobra.Command, args []string) { file, err := os.Open(path) checkErr(err) - data, err := ioutil.ReadAll(file) - checkErr(err) - s, err := stream.New(data) + defer file.Close() + s, err := stream.New(file) checkErr(err) sdBlob := &stream.SDBlob{} diff --git a/go.mod b/go.mod index ed3ebcb..4ed4f23 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/lbryio/reflector.go replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 +//replace github.com/lbryio/lbry.go/v2 => ../lbry.go + require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/aws/aws-sdk-go v1.16.11 @@ -23,9 +25,9 @@ require ( github.com/karrick/godirwalk v1.16.1 github.com/lbryio/chainquery v1.9.0 github.com/lbryio/lbry.go v1.1.2 // indirect - github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011 + github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 - github.com/lucas-clemente/quic-go v0.20.0 + github.com/lucas-clemente/quic-go v0.20.1 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.3 github.com/sirupsen/logrus v1.4.2 @@ -36,7 +38,6 @@ require ( github.com/stretchr/testify v1.7.0 github.com/volatiletech/null v8.0.0+incompatible go.uber.org/atomic v1.5.1 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect diff --git a/go.sum b/go.sum index f61f06b..c51b403 100644 --- a/go.sum +++ b/go.sum @@ -88,8 +88,9 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= +github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw= @@ -254,8 +255,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c= github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011 h1:r1NoX3NQu/Me+/qw4OzJGw8bhOUnTZHUneCbIV6SC+Y= -github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011/go.mod h1:sUhhSKqPNkiwgBqvBzJIqfLLzGH8hkDGrrO/HcaXzFc= +github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d h1:VUaOZ3cbCe7gfpycN/srCOk6U2bBS9NZHEz9RiRxd4E= +github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= @@ -265,8 +266,8 @@ github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiV github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M= github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lucas-clemente/quic-go v0.20.0 h1:FSU3YN5VnLafHR27Ejs1r1CYMS7XMyIVDzRewkDLNBw= -github.com/lucas-clemente/quic-go v0.20.0/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk= +github.com/lucas-clemente/quic-go v0.20.1 h1:hb5m76V8QS/8Nw/suHvXqo3BMHAozvIkcnzpJdpanSk= +github.com/lucas-clemente/quic-go v0.20.1/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= diff --git a/publish/publish.go b/publish/publish.go index 417978e..9e585e8 100644 --- a/publish/publish.go +++ b/publish/publish.go @@ -3,7 +3,6 @@ package publish import ( "bytes" "encoding/json" - "io/ioutil" "os" "path/filepath" "sort" @@ -21,7 +20,6 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" "github.com/golang/protobuf/proto" - "golang.org/x/crypto/sha3" ) var TODO = ` @@ -43,6 +41,14 @@ var TODO = ` } ` +type Details struct { + Title string + Description string + Author string + Tags []string + ReleaseTime int64 +} + func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) { if name == "" { return nil, nil, errors.Err("name required") @@ -69,11 +75,20 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details return nil, nil, err } - claim, st, err := makeClaimAndStream(path, details) + st, stPB, err := makeStream(path) if err != nil { return nil, nil, err } + stPB.Author = details.Author + stPB.ReleaseTime = details.ReleaseTime + + claim := &pb.Claim{ + Title: details.Title, + Description: details.Description, + Type: &pb.Claim_Stream{Stream: stPB}, + } + err = addClaimToTx(tx, claim, name, amount, addr) if err != nil { return nil, nil, err @@ -203,50 +218,31 @@ func reflect(st stream.Stream, reflectorAddress string) error { return nil } -type Details struct { - Title string - Description string - Author string - Tags []string - ReleaseTime int64 -} - -func makeClaimAndStream(path string, details Details) (*pb.Claim, stream.Stream, error) { +func makeStream(path string) (stream.Stream, *pb.Stream, error) { file, err := os.Open(path) if err != nil { return nil, nil, errors.Err(err) } - data, err := ioutil.ReadAll(file) - if err != nil { - return nil, nil, errors.Err(err) - } - s, err := stream.New(data) + defer file.Close() + + enc := stream.NewEncoder(file) + + s, err := enc.Stream() if err != nil { return nil, nil, errors.Err(err) } - // make the claim - sdBlob := &stream.SDBlob{} - err = sdBlob.FromBlob(s[0]) - if err != nil { - return nil, nil, errors.Err(err) - } - - filehash := sha3.Sum384(data) - - streamPB := &pb.Stream{ - Author: details.Author, - ReleaseTime: details.ReleaseTime, + streamProto := &pb.Stream{ Source: &pb.Source{ - SdHash: s[0].Hash(), + SdHash: enc.SDBlob().Hash(), Name: filepath.Base(file.Name()), - Size: uint64(len(data)), - Hash: filehash[:], + Size: uint64(enc.SourceLen()), + Hash: enc.SourceHash(), }, } mimeType, category := guessMimeType(filepath.Ext(file.Name())) - streamPB.Source.MediaType = mimeType + streamProto.Source.MediaType = mimeType switch category { case "video": @@ -254,20 +250,14 @@ func makeClaimAndStream(path string, details Details) (*pb.Claim, stream.Stream, //if err != nil { // return nil, nil, err //} - streamPB.Type = &pb.Stream_Video{} + streamProto.Type = &pb.Stream_Video{} case "audio": - streamPB.Type = &pb.Stream_Audio{} + streamProto.Type = &pb.Stream_Audio{} case "image": - streamPB.Type = &pb.Stream_Image{} + streamProto.Type = &pb.Stream_Image{} } - claim := &pb.Claim{ - Title: details.Title, - Description: details.Description, - Type: &pb.Claim_Stream{Stream: streamPB}, - } - - return claim, s, nil + return s, streamProto, nil } func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) { -- 2.45.2 From c4084eeb6849c4e6903285b18463e1c1c0432cd7 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Thu, 29 Apr 2021 03:41:18 +0200 Subject: [PATCH 34/51] improve disk cleanup add index to is_stored fix test replace LRU cache --- cmd/reflector.go | 46 +++++++++++++++++++++++++++++++++++----------- db/db.go | 3 ++- go.mod | 3 ++- go.sum | 2 ++ store/disk_test.go | 2 +- store/lru.go | 35 +++++++++++++++-------------------- store/lru_test.go | 4 ++-- 7 files changed, 59 insertions(+), 36 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index 6fe44ad..a642c7a 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -254,6 +254,11 @@ func diskCacheParams(diskParams string) (int, string) { } func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) { + // this is so that it runs on startup without having to wait for 10 minutes + err := doClean(maxItems, db, store, stopper) + if err != nil { + log.Error(errors.FullTrace(err)) + } const cleanupInterval = 10 * time.Minute for { select { @@ -281,19 +286,38 @@ func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Grou if err != nil { return err } - - for _, hash := range blobs { - select { - case <-stopper.Ch(): - return nil - default: - } - - err = store.Delete(hash) - if err != nil { - return err + blobsChan := make(chan string, len(blobs)) + wg := &stop.Group{} + go func() { + for _, hash := range blobs { + select { + case <-stopper.Ch(): + return + default: + } + blobsChan <- hash } + close(blobsChan) + }() + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for h := range blobsChan { + select { + case <-stopper.Ch(): + return + default: + } + err = store.Delete(h) + if err != nil { + log.Errorf("error pruning %s: %s", h, errors.FullTrace(err)) + continue + } + } + }() } + wg.Wait() } return nil } diff --git a/db/db.go b/db/db.go index 3e7d4ca..babe641 100644 --- a/db/db.go +++ b/db/db.go @@ -781,7 +781,8 @@ CREATE TABLE blob_ ( last_accessed_at TIMESTAMP NULL DEFAULT NULL, PRIMARY KEY (id), UNIQUE KEY blob_hash_idx (hash), - KEY `blob_last_accessed_idx` (`last_accessed_at`) + KEY `blob_last_accessed_idx` (`last_accessed_at`), + KEY `is_stored_idx` (`is_stored`) ); CREATE TABLE stream ( diff --git a/go.mod b/go.mod index 4ed4f23..eceb7ba 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203 require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/aws/aws-sdk-go v1.16.11 + github.com/bluele/gcache v0.0.2 github.com/bparli/lfuda-go v0.3.1 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d @@ -17,7 +18,7 @@ require ( github.com/google/gops v0.3.7 github.com/gorilla/mux v1.7.4 github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/golang-lru v0.5.4 + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/memberlist v0.1.4 // indirect github.com/hashicorp/serf v0.8.2 github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf diff --git a/go.sum b/go.sum index c51b403..48680c9 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bparli/lfuda-go v0.3.1 h1:nO9Szo627RC8/z+R+MMPBItNwHCOonchmpjQuQi8jVY= github.com/bparli/lfuda-go v0.3.1/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= diff --git a/store/disk_test.go b/store/disk_test.go index 72c7755..1ab3f05 100644 --- a/store/disk_test.go +++ b/store/disk_test.go @@ -19,7 +19,7 @@ func TestDiskStore_Get(t *testing.T) { defer os.RemoveAll(tmpDir) d := NewDiskStore(tmpDir, 2) - hash := "1234567890" + hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b" data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf") expectedPath := path.Join(tmpDir, hash[:2], hash) diff --git a/store/lru.go b/store/lru.go index e96931b..be6cd82 100644 --- a/store/lru.go +++ b/store/lru.go @@ -3,12 +3,13 @@ package store import ( "time" - "github.com/lbryio/lbry.go/v2/extras/errors" - "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/shared" - golru "github.com/hashicorp/golang-lru" + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" + + "github.com/bluele/gcache" "github.com/sirupsen/logrus" ) @@ -17,7 +18,7 @@ type LRUStore struct { // underlying store store BlobStore // lru implementation - lru *golru.Cache + lru gcache.Cache } // NewLRUStore initialize a new LRUStore @@ -25,20 +26,14 @@ func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore { l := &LRUStore{ store: store, } - - lru, err := golru.NewWithEvict(maxItems, func(key interface{}, value interface{}) { + l.lru = gcache.New(maxItems).ARC().EvictedFunc(func(key, value interface{}) { metrics.CacheLRUEvictCount.With(metrics.CacheLabels(l.Name(), component)).Inc() - _ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there - }) - if err != nil { - panic(err) - } - - l.lru = lru + _ = store.Delete(key.(string)) + }).Build() go func() { if lstr, ok := store.(lister); ok { - err = l.loadExisting(lstr, maxItems) + err := l.loadExisting(lstr, maxItems) if err != nil { panic(err) // TODO: what should happen here? panic? return nil? just keep going? } @@ -55,14 +50,14 @@ func (l *LRUStore) Name() string { // Has returns whether the blob is in the store, without updating the recent-ness. func (l *LRUStore) Has(hash string) (bool, error) { - return l.lru.Contains(hash), nil + return l.lru.Has(hash), nil } // Get returns the blob or an error if the blob doesn't exist. func (l *LRUStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { start := time.Now() - _, has := l.lru.Get(hash) - if !has { + _, err := l.lru.Get(hash) + if err != nil { return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound) } blob, stack, err := l.store.Get(hash) @@ -80,7 +75,7 @@ func (l *LRUStore) Put(hash string, blob stream.Blob) error { return err } - l.lru.Add(hash, true) + l.lru.Set(hash, true) return nil } @@ -91,7 +86,7 @@ func (l *LRUStore) PutSD(hash string, blob stream.Blob) error { return err } - l.lru.Add(hash, true) + _ = l.lru.Set(hash, true) return nil } @@ -119,7 +114,7 @@ func (l *LRUStore) loadExisting(store lister, maxItems int) error { logrus.Infof("read %d files from disk", len(existing)) added := 0 for _, h := range existing { - l.lru.Add(h, true) + l.lru.Set(h, true) added++ if maxItems > 0 && added >= maxItems { // underlying cache is bigger than LRU cache break diff --git a/store/lru_test.go b/store/lru_test.go index 6aec768..798d5c2 100644 --- a/store/lru_test.go +++ b/store/lru_test.go @@ -87,7 +87,7 @@ func TestLRUStore_UnderlyingBlobMissing(t *testing.T) { require.NoError(t, err) // hash still exists in lru - assert.True(t, lru.lru.Contains(hash)) + assert.True(t, lru.lru.Has(hash)) blob, _, err := lru.Get(hash) assert.Nil(t, blob) @@ -96,7 +96,7 @@ func TestLRUStore_UnderlyingBlobMissing(t *testing.T) { reflect.TypeOf(err).String(), err.Error()) // lru.Get() removes hash if underlying store doesn't have it - assert.False(t, lru.lru.Contains(hash)) + assert.False(t, lru.lru.Has(hash)) } func TestLRUStore_loadExisting(t *testing.T) { -- 2.45.2 From 070938e12afba8e26881259e04a6654102443413 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Thu, 6 May 2021 22:53:18 +0200 Subject: [PATCH 35/51] increase window size --- peer/http3/server.go | 9 +++++++-- peer/http3/store.go | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/peer/http3/server.go b/peer/http3/server.go index 30ae3fd..d81f0f8 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -65,9 +65,14 @@ type availabilityResponse struct { // Start starts the server listener to handle connections. func (s *Server) Start(address string) error { log.Println("HTTP3 peer listening on " + address) + window500M := 500 * 1 << 20 + quicConf := &quic.Config{ - HandshakeIdleTimeout: 4 * time.Second, - MaxIdleTimeout: 20 * time.Second, + MaxStreamReceiveWindow: uint64(window500M), + MaxConnectionReceiveWindow: uint64(window500M), + EnableDatagrams: true, + HandshakeIdleTimeout: 4 * time.Second, + MaxIdleTimeout: 20 * time.Second, } r := mux.NewRouter() r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) { diff --git a/peer/http3/store.go b/peer/http3/store.go index bcc7cae..7d8fc34 100644 --- a/peer/http3/store.go +++ b/peer/http3/store.go @@ -36,6 +36,10 @@ func NewStore(opts StoreOpts) *Store { func (p *Store) getClient() (*Client, error) { var qconf quic.Config + window500M := 500 * 1 << 20 + qconf.MaxStreamReceiveWindow = uint64(window500M) + qconf.MaxConnectionReceiveWindow = uint64(window500M) + qconf.EnableDatagrams = true qconf.HandshakeIdleTimeout = 4 * time.Second qconf.MaxIdleTimeout = 20 * time.Second pool, err := x509.SystemCertPool() -- 2.45.2 From 50c077a9cba8d0ac22d2be4328308ed611895ac6 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Thu, 20 May 2021 23:17:18 +0200 Subject: [PATCH 36/51] request queue size param --- cmd/reflector.go | 4 +++- peer/http3/server.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/reflector.go b/cmd/reflector.go index a642c7a..d2acfed 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -41,6 +41,7 @@ var ( reflectorCmdDiskCache string bufferReflectorCmdDiskCache string reflectorCmdMemCache int + requestQueueSize int ) func init() { @@ -58,6 +59,7 @@ func init() { cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol") cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from") cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics") + cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests should be submitted to upstream") cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server") cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating") cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not") @@ -96,7 +98,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) { } defer peerServer.Shutdown() - http3PeerServer := http3.NewServer(outerStore) + http3PeerServer := http3.NewServer(outerStore, requestQueueSize) err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort)) if err != nil { log.Fatal(err) diff --git a/peer/http3/server.go b/peer/http3/server.go index d81f0f8..3d49d17 100644 --- a/peer/http3/server.go +++ b/peer/http3/server.go @@ -33,7 +33,7 @@ type Server struct { } // NewServer returns an initialized Server pointer. -func NewServer(store store.BlobStore) *Server { +func NewServer(store store.BlobStore, requestQueueSize int) *Server { return &Server{ store: store, grp: stop.New(), -- 2.45.2 From eafc62f2a6103ed1435c906553cf0f5b7e42ebbd Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 20 May 2021 17:21:35 -0400 Subject: [PATCH 37/51] add gops to reflector server --- go.mod | 2 +- go.sum | 25 +++++++++++++------------ reflector/server.go | 5 ++++- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index eceb7ba..82eaa1e 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/go-sql-driver/mysql v1.4.1 github.com/golang/protobuf v1.4.2 - github.com/google/gops v0.3.7 + github.com/google/gops v0.3.18 github.com/gorilla/mux v1.7.4 github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect diff --git a/go.sum b/go.sum index 48680c9..6af413e 100644 --- a/go.sum +++ b/go.sum @@ -23,7 +23,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -100,7 +100,7 @@ github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= @@ -140,8 +140,8 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw= -github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY= +github.com/google/gops v0.3.18 h1:my259V+172PVFmduS2RAsq4FKH+HjKqdh7pLr17Ot8c= +github.com/google/gops v0.3.18/go.mod h1:Pfp8hWGIFdV/7rY9/O/U5WgdjYQXf/GiEK4NVuVd2ZE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -232,11 +232,9 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro= -github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ= +github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= @@ -372,8 +370,7 @@ github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywN github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk= -github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shirou/gopsutil/v3 v3.21.2/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw= github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw= @@ -437,11 +434,14 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= +github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= @@ -454,7 +454,7 @@ github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05 github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU= github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -543,7 +543,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -572,6 +571,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg= golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -692,6 +693,6 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= +rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/reflector/server.go b/reflector/server.go index 592ba3d..8e78e22 100644 --- a/reflector/server.go +++ b/reflector/server.go @@ -10,6 +10,7 @@ import ( "net" "time" + "github.com/google/gops/agent" "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/store" @@ -69,7 +70,9 @@ func (s *Server) Start(address string) error { return errors.Err(err) } log.Println("reflector listening on " + address) - + if err := agent.Listen(agent.Options{}); err != nil { + log.Fatal(err) + } s.grp.Add(1) go func() { <-s.grp.Ch() -- 2.45.2 From 0152300d8df2570344b333161be3c915e83cdcf4 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 20 May 2021 18:01:13 -0400 Subject: [PATCH 38/51] add guage metrics for go routines in reflector package --- internal/metrics/metrics.go | 5 +++++ reflector/blocklist.go | 5 ++++- reflector/server.go | 10 ++++++++++ reflector/uploader.go | 6 ++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 2570677..4c30606 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -219,6 +219,11 @@ var ( Name: "http3_blob_request_queue_size", Help: "Blob requests of https queue size", }) + RoutinesQueue = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: ns, + Name: "routines", + Help: "routines running by type", + }, []string{"package", "kind"}) ) func CacheLabels(name, component string) prometheus.Labels { diff --git a/reflector/blocklist.go b/reflector/blocklist.go index 9abfc4a..26edff6 100644 --- a/reflector/blocklist.go +++ b/reflector/blocklist.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/store" "github.com/lbryio/reflector.go/wallet" @@ -109,8 +111,9 @@ func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan) } done := make(chan bool) - + metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Inc() go func() { + defer metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Dec() select { case <-done: case <-stopper: diff --git a/reflector/server.go b/reflector/server.go index 8e78e22..3171ced 100644 --- a/reflector/server.go +++ b/reflector/server.go @@ -74,7 +74,9 @@ func (s *Server) Start(address string) error { log.Fatal(err) } s.grp.Add(1) + metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Inc() go func() { + defer metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Dec() <-s.grp.Ch() err := l.Close() if err != nil { @@ -84,7 +86,9 @@ func (s *Server) Start(address string) error { }() s.grp.Add(1) + metrics.RoutinesQueue.WithLabelValues("reflector", "start").Inc() go func() { + defer metrics.RoutinesQueue.WithLabelValues("reflector", "start").Dec() s.listenAndServe(l) s.grp.Done() }() @@ -92,7 +96,9 @@ func (s *Server) Start(address string) error { if s.EnableBlocklist { if b, ok := s.underlyingStore.(store.Blocklister); ok { s.grp.Add(1) + metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc() go func() { + defer metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Dec() s.enableBlocklist(b) s.grp.Done() }() @@ -115,7 +121,9 @@ func (s *Server) listenAndServe(listener net.Listener) { log.Error(err) } else { s.grp.Add(1) + metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc() go func() { + defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc() s.handleConn(conn) s.grp.Done() }() @@ -130,7 +138,9 @@ func (s *Server) handleConn(conn net.Conn) { close(connNeedsClosing) }() s.grp.Add(1) + metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Inc() go func() { + defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Dec() defer s.grp.Done() select { case <-connNeedsClosing: diff --git a/reflector/uploader.go b/reflector/uploader.go index e5b2688..b421272 100644 --- a/reflector/uploader.go +++ b/reflector/uploader.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/store" @@ -88,7 +90,9 @@ func (u *Uploader) Upload(dirOrFilePath string) error { for i := 0; i < u.workers; i++ { workerWG.Add(1) + metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Inc() go func(i int) { + defer metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Dec() defer workerWG.Done() defer func(i int) { log.Debugf("worker %d quitting", i) }(i) u.worker(pathChan) @@ -97,7 +101,9 @@ func (u *Uploader) Upload(dirOrFilePath string) error { countWG := sync.WaitGroup{} countWG.Add(1) + metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Inc() go func() { + defer metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Dec() defer countWG.Done() u.counter() }() -- 2.45.2 From 4ecce75e2372c690fc85ea801dedbaa81e90cd78 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 20 May 2021 18:12:30 -0400 Subject: [PATCH 39/51] add metric calls for other packages --- peer/http3/worker.go | 2 ++ peer/server.go | 2 ++ store/caching.go | 2 ++ store/speedwalk/speedwalk.go | 3 +++ 4 files changed, 9 insertions(+) diff --git a/peer/http3/worker.go b/peer/http3/worker.go index d2b625b..91dfd02 100644 --- a/peer/http3/worker.go +++ b/peer/http3/worker.go @@ -20,7 +20,9 @@ var getReqCh = make(chan *blobRequest, 20000) func InitWorkers(server *Server, workers int) { stopper := stop.New(server.grp) for i := 0; i < workers; i++ { + metrics.RoutinesQueue.WithLabelValues("http3", "worker").Inc() go func(worker int) { + defer metrics.RoutinesQueue.WithLabelValues("http3", "worker").Dec() for { select { case <-stopper.Ch(): diff --git a/peer/server.go b/peer/server.go index 708fe53..4061ea5 100644 --- a/peer/server.go +++ b/peer/server.go @@ -89,7 +89,9 @@ func (s *Server) listenAndServe(listener net.Listener) { log.Error(errors.Prefix("accepting conn", err)) } else { s.grp.Add(1) + metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Inc() go func() { + defer metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Dec() s.handleConnection(conn) s.grp.Done() }() diff --git a/store/caching.go b/store/caching.go index 6f1d4d4..cd05b41 100644 --- a/store/caching.go +++ b/store/caching.go @@ -66,7 +66,9 @@ func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { } // there is no need to wait for the blob to be stored before we return it // TODO: however this should be refactored to limit the amount of routines that the process can spawn to avoid a possible DoS + metrics.RoutinesQueue.WithLabelValues("store", "cache-put").Inc() go func() { + defer metrics.RoutinesQueue.WithLabelValues("store", "cache-put").Dec() err = c.cache.Put(hash, blob) if err != nil { log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err)) diff --git a/store/speedwalk/speedwalk.go b/store/speedwalk/speedwalk.go index 9b899e4..1d4b981 100644 --- a/store/speedwalk/speedwalk.go +++ b/store/speedwalk/speedwalk.go @@ -6,6 +6,8 @@ import ( "runtime" "sync" + "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/karrick/godirwalk" @@ -24,6 +26,7 @@ func AllFiles(startDir string, basename bool) ([]string, error) { paths := make([]string, 0, 1000) pathWG := &sync.WaitGroup{} pathWG.Add(1) + metrics.RoutinesQueue.WithLabelValues("speedwalk", "worker").Inc() go func() { defer pathWG.Done() for { -- 2.45.2 From 45130499cd2e2fbfd566832c92baf802c5dce8a2 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 20 May 2021 19:05:48 -0400 Subject: [PATCH 40/51] Add single flight for cache not just origin --- store/caching.go | 2 +- store/singleflight.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/store/caching.go b/store/caching.go index cd05b41..65b6397 100644 --- a/store/caching.go +++ b/store/caching.go @@ -24,7 +24,7 @@ func NewCachingStore(component string, origin, cache BlobStore) *CachingStore { return &CachingStore{ component: component, origin: WithSingleFlight(component, origin), - cache: cache, + cache: WithSingleFlight(component, cache), } } diff --git a/store/singleflight.go b/store/singleflight.go index 362182d..6de91f4 100644 --- a/store/singleflight.go +++ b/store/singleflight.go @@ -80,6 +80,43 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) { } } +// Put ensures that only one request per hash is sent to the origin at a time, +// thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem +func (s *singleflightStore) Put(hash string, blob stream.Blob) error { + metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc() + defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec() + + _, err, _ := s.sf.Do(hash, s.putter(hash, blob)) + if err != nil { + return err + } + return nil +} + +// putter returns a function that puts a blob from the origin +// only one putter per hash will be executing at a time +func (s *singleflightStore) putter(hash string, blob stream.Blob) func() (interface{}, error) { + return func() (interface{}, error) { + metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc() + defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec() + + start := time.Now() + err := s.BlobStore.Put(hash, blob) + if err != nil { + return nil, err + } + + rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds() + metrics.CacheRetrievalSpeed.With(map[string]string{ + metrics.LabelCacheType: s.Name(), + metrics.LabelComponent: s.component, + metrics.LabelSource: "origin", + }).Set(rate) + + return nil, nil + } +} + // Shutdown shuts down the store gracefully func (s *singleflightStore) Shutdown() { s.BlobStore.Shutdown() -- 2.45.2 From 006b04f6e9bc140572c4a9b1acdb2a7a1232adb6 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 21 May 2021 01:48:46 +0200 Subject: [PATCH 41/51] add a lot of extra heavy debugging --- store/disk.go | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/store/disk.go b/store/disk.go index eaecfbb..f108204 100644 --- a/store/disk.go +++ b/store/disk.go @@ -101,6 +101,12 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { // Put stores the blob on disk func (d *DiskStore) Put(hash string, blob stream.Blob) error { + start := time.Now() + defer func() { + if time.Since(start) > 100*time.Millisecond { + log.Infof("it took %s to write %s", time.Since(start), hash) + } + }() err := d.initOnce() if err != nil { return err @@ -110,8 +116,26 @@ func (d *DiskStore) Put(hash string, blob stream.Blob) error { if err != nil { return err } - + hashBytes := sha512.Sum384(blob) + readHash := hex.EncodeToString(hashBytes[:]) + matchesBeforeWriting := readHash == hash err = ioutil.WriteFile(d.path(hash), blob, 0644) + if err != nil { + log.Errorf("Error saving to disk: %s", err.Error()) + } + readBlob, err := ioutil.ReadFile(d.path(hash)) + matchesAfterReading := false + if err != nil { + log.Errorf("for some fucking reasons I can't read the blob I just wrote %s", err.Error()) + } else { + hashBytes = sha512.Sum384(readBlob) + readHash = hex.EncodeToString(hashBytes[:]) + matchesAfterReading = readHash == hash + } + + log.Infof(`writing %s to disk: hash match: %t, error: %t +reading after writing: hash match: %t`, hash, matchesBeforeWriting, err == nil, matchesAfterReading) + return errors.Err(err) } -- 2.45.2 From 213d21b0214f210c581e744e30afc0e158826c75 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 20 May 2021 19:59:50 -0400 Subject: [PATCH 42/51] Add locks to disk store. --- locks/multiplelock.go | 131 +++++++++++++++++++++++++++++++++++++ locks/multiplelock_test.go | 56 ++++++++++++++++ store/disk.go | 13 ++++ 3 files changed, 200 insertions(+) create mode 100644 locks/multiplelock.go create mode 100644 locks/multiplelock_test.go diff --git a/locks/multiplelock.go b/locks/multiplelock.go new file mode 100644 index 0000000..c87b645 --- /dev/null +++ b/locks/multiplelock.go @@ -0,0 +1,131 @@ +package locks + +import ( + "fmt" + "sync" + "time" +) + +// MultipleLock is the main interface for multiLock based on key +type MultipleLock interface { + Lock(key string) + RLock(key string) + Unlock(key string) + RUnlock(key string) +} + +func NewMultipleLock() MultipleLock { + return &multiLock{ + locks: make(map[string]*itemLock), + mu: sync.Mutex{}, + } +} + +type itemLock struct { + lk *sync.RWMutex + cnt int64 +} + +// multiLock is an optimized locking system per locking key +type multiLock struct { + locks map[string]*itemLock + mu sync.Mutex // synchronize reads/writes to locks map +} + +func debugPrint(format string, a ...interface{}) { + debugEnabled := false + if debugEnabled { + a = append(a, time.Now().Format("04:05.000000")) + fmt.Printf(format+" at %s\n", a...) + } +} + +func (ml *multiLock) Lock(key string) { + debugPrint("mutex requested %s", key) + ml.mu.Lock() + debugPrint("mutex acquired %s", key) + + itmLock, exists := ml.locks[key] + if !exists { + debugPrint("new lock created %s", key) + itmLock = &itemLock{&sync.RWMutex{}, 0} + ml.locks[key] = itmLock + } + itmLock.cnt++ + debugPrint("releasing mutex %s", key) + ml.mu.Unlock() + + debugPrint("Lock requested %s", key) + itmLock.lk.Lock() + debugPrint("Lock acquired %s", key) +} + +func (ml *multiLock) RLock(key string) { + debugPrint("mutex requested %s", key) + ml.mu.Lock() + debugPrint("mutex acquired %s", key) + + itmLock, exists := ml.locks[key] + if !exists { + debugPrint("new lock created for %s", key) + itmLock = &itemLock{&sync.RWMutex{}, 0} + ml.locks[key] = itmLock + } + itmLock.cnt++ + debugPrint("releasing mutex %s", key) + ml.mu.Unlock() + + debugPrint("RLock requested %s", key) + itmLock.lk.RLock() + debugPrint("RLock acquired %s", key) +} + +func (ml *multiLock) Unlock(key string) { + debugPrint("mutex requested %s", key) + ml.mu.Lock() + debugPrint("mutex acquired %s", key) + + itmLock, exists := ml.locks[key] + if !exists { + panic("sync Unlock of non existent lock!!") + } + + debugPrint("Unlock %s", key) + itmLock.lk.Unlock() + itmLock.cnt-- + if itmLock.cnt == 0 { + debugPrint("delete lock %s", key) + delete(ml.locks, key) + } + if itmLock.cnt < 0 { + panic("sync Unlock of free Lock!!") + } + + debugPrint("releasing mutex %s", key) + ml.mu.Unlock() +} + +func (ml *multiLock) RUnlock(key string) { + debugPrint("mutex requested %s", key) + ml.mu.Lock() + debugPrint("mutex acquired %s", key) + + itmLock, exists := ml.locks[key] + if !exists { + panic("sync Unlock of non existent lock!!") + } + + debugPrint("RUnlock %s", key) + itmLock.lk.RUnlock() + itmLock.cnt-- + if itmLock.cnt == 0 { + debugPrint("delete lock %s", key) + delete(ml.locks, key) + } + if itmLock.cnt < 0 { + panic("sync Unlock of free Lock!!") + } + + debugPrint("releasing mutex %s", key) + ml.mu.Unlock() +} diff --git a/locks/multiplelock_test.go b/locks/multiplelock_test.go new file mode 100644 index 0000000..a7a42c7 --- /dev/null +++ b/locks/multiplelock_test.go @@ -0,0 +1,56 @@ +package locks + +import ( + "math/rand" + "strconv" + "testing" + "time" + + "github.com/lbryio/lbry.go/v2/extras/stop" +) + +var lock = NewMultipleLock() + +func TestNewMultipleLock(t *testing.T) { + grp := stop.New() + for i := 0; i < 100; i++ { + grp.Add(2) + go doRWWork(i, i%10, grp) + go doRWork(i, i%10, grp) + } + time.Sleep(5 * time.Second) + grp.StopAndWait() +} +func doRWWork(worker int, resource int, grp *stop.Group) { + for { + select { + case <-grp.Ch(): + grp.Done() + return + default: + //log.Printf("RW - worker %d doing work on resource %d\n", worker, resource) + lock.Lock(strconv.Itoa(resource)) + randomTime := time.Duration(rand.Int()%10+1) * time.Microsecond + time.Sleep(randomTime) + //log.Printf("RW - worker %d releasing %d\n", worker, resource) + lock.Unlock(strconv.Itoa(resource)) + } + } +} + +func doRWork(worker int, resource int, grp *stop.Group) { + for { + select { + case <-grp.Ch(): + grp.Done() + return + default: + //log.Printf("R - worker %d doing work on resource %d\n", worker, resource) + lock.RLock(strconv.Itoa(resource)) + randomTime := time.Duration(rand.Int()%10+1) * time.Microsecond + time.Sleep(randomTime) + //log.Printf("R - worker %d releasing %d\n", worker, resource) + lock.RUnlock(strconv.Itoa(resource)) + } + } +} diff --git a/store/disk.go b/store/disk.go index f108204..1515135 100644 --- a/store/disk.go +++ b/store/disk.go @@ -11,6 +11,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/locks" "github.com/lbryio/reflector.go/shared" "github.com/lbryio/reflector.go/store/speedwalk" log "github.com/sirupsen/logrus" @@ -28,6 +29,7 @@ type DiskStore struct { initialized bool concurrentChecks atomic.Int32 + lock locks.MultipleLock } const maxConcurrentChecks = 3 @@ -37,6 +39,7 @@ func NewDiskStore(dir string, prefixLength int) *DiskStore { return &DiskStore{ blobDir: dir, prefixLength: prefixLength, + lock: locks.NewMultipleLock(), } } @@ -47,6 +50,8 @@ func (d *DiskStore) Name() string { return nameDisk } // Has returns T/F or Error if it the blob stored already. It will error with any IO disk error. func (d *DiskStore) Has(hash string) (bool, error) { + d.lock.RLock(hash) + defer d.lock.RUnlock(hash) err := d.initOnce() if err != nil { return false, err @@ -64,6 +69,8 @@ func (d *DiskStore) Has(hash string) (bool, error) { // Get returns the blob or an error if the blob doesn't exist. func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + d.lock.RLock(hash) + defer d.lock.RUnlock(hash) start := time.Now() err := d.initOnce() if err != nil { @@ -101,6 +108,8 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { // Put stores the blob on disk func (d *DiskStore) Put(hash string, blob stream.Blob) error { + d.lock.Lock(hash) + defer d.lock.Unlock(hash) start := time.Now() defer func() { if time.Since(start) > 100*time.Millisecond { @@ -141,11 +150,15 @@ reading after writing: hash match: %t`, hash, matchesBeforeWriting, err == nil, // PutSD stores the sd blob on the disk func (d *DiskStore) PutSD(hash string, blob stream.Blob) error { + d.lock.Lock(hash) + defer d.lock.Unlock(hash) return d.Put(hash, blob) } // Delete deletes the blob from the store func (d *DiskStore) Delete(hash string) error { + d.lock.Lock(hash) + defer d.lock.Unlock(hash) err := d.initOnce() if err != nil { return err -- 2.45.2 From c1caf1938c35082fe18920f59cbabb276297fba4 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 20 May 2021 20:41:47 -0400 Subject: [PATCH 43/51] Add queue to prevent writing too many files at once. --- store/disk.go | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/store/disk.go b/store/disk.go index 1515135..e52731a 100644 --- a/store/disk.go +++ b/store/disk.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "path" + "runtime" "time" "github.com/lbryio/lbry.go/v2/extras/errors" @@ -18,6 +19,23 @@ import ( "go.uber.org/atomic" ) +func init() { + writeCh = make(chan writeRequest) + for i := 0; i < runtime.NumCPU(); i++ { + go func() { + select { + case r := <-writeCh: + err := ioutil.WriteFile(r.filename, r.data, r.perm) + if err != nil { + log.Errorf("could not write file %s to disk, failed with error: %s", r.filename, err.Error()) + } + } + }() + } +} + +var writeCh chan writeRequest + // DiskStore stores blobs on a local disk type DiskStore struct { // the location of blobs on disk @@ -128,10 +146,7 @@ func (d *DiskStore) Put(hash string, blob stream.Blob) error { hashBytes := sha512.Sum384(blob) readHash := hex.EncodeToString(hashBytes[:]) matchesBeforeWriting := readHash == hash - err = ioutil.WriteFile(d.path(hash), blob, 0644) - if err != nil { - log.Errorf("Error saving to disk: %s", err.Error()) - } + writeFile(d.path(hash), blob, 0644) readBlob, err := ioutil.ReadFile(d.path(hash)) matchesAfterReading := false if err != nil { @@ -215,7 +230,21 @@ func (d *DiskStore) initOnce() error { return nil } +type writeRequest struct { + filename string + data []byte + perm os.FileMode +} + // Shutdown shuts down the store gracefully func (d *DiskStore) Shutdown() { return } + +func writeFile(filename string, data []byte, perm os.FileMode) { + writeCh <- writeRequest{ + filename: filename, + data: data, + perm: perm, + } +} -- 2.45.2 From 76ece1e1178377373093dba8ae4b8e9ea23cfdc7 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 20 May 2021 20:43:01 -0400 Subject: [PATCH 44/51] Add queue to prevent writing too many files at once. --- store/disk.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/store/disk.go b/store/disk.go index e52731a..7123b65 100644 --- a/store/disk.go +++ b/store/disk.go @@ -23,11 +23,13 @@ func init() { writeCh = make(chan writeRequest) for i := 0; i < runtime.NumCPU(); i++ { go func() { - select { - case r := <-writeCh: - err := ioutil.WriteFile(r.filename, r.data, r.perm) - if err != nil { - log.Errorf("could not write file %s to disk, failed with error: %s", r.filename, err.Error()) + for { + select { + case r := <-writeCh: + err := ioutil.WriteFile(r.filename, r.data, r.perm) + if err != nil { + log.Errorf("could not write file %s to disk, failed with error: %s", r.filename, err.Error()) + } } } }() -- 2.45.2 From a7086a00f3a5afa9b164db59b62078550a17071f Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 21 May 2021 05:49:02 +0200 Subject: [PATCH 45/51] add http server/client --- cmd/reflector.go | 12 +++ go.mod | 1 + go.sum | 29 +++++++ internal/metrics/metrics.go | 5 ++ peer/store.go | 9 ++- server/http/routes.go | 53 +++++++++++++ server/http/server.go | 68 ++++++++++++++++ store/disk.go | 22 ------ store/http.go | 152 ++++++++++++++++++++++++++++++++++++ store/singleflight.go | 4 + 10 files changed, 332 insertions(+), 23 deletions(-) create mode 100644 server/http/routes.go create mode 100644 server/http/server.go create mode 100644 store/http.go diff --git a/cmd/reflector.go b/cmd/reflector.go index d2acfed..087c918 100644 --- a/cmd/reflector.go +++ b/cmd/reflector.go @@ -16,6 +16,7 @@ import ( "github.com/lbryio/reflector.go/peer" "github.com/lbryio/reflector.go/peer/http3" "github.com/lbryio/reflector.go/reflector" + "github.com/lbryio/reflector.go/server/http" "github.com/lbryio/reflector.go/store" "github.com/lbryio/lbry.go/v2/stream" @@ -28,6 +29,7 @@ import ( var ( tcpPeerPort int http3PeerPort int + httpPort int receiverPort int metricsPort int disableUploads bool @@ -57,6 +59,7 @@ func init() { cmd.Flags().StringVar(&WasabiEndpoint, "wasabi-endpoint", "", "Wasabi edge endpoint for standard HTTP retrieval") cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from") cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol") + cmd.Flags().IntVar(&httpPort, "http-port", 5569, "The port reflector will distribute content from over HTTP protocol") cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from") cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics") cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests should be submitted to upstream") @@ -105,6 +108,13 @@ func reflectorCmd(cmd *cobra.Command, args []string) { } defer http3PeerServer.Shutdown() + httpServer := http.NewServer(outerStore) + err = httpServer.Start(":" + strconv.Itoa(httpPort)) + if err != nil { + log.Fatal(err) + } + defer httpServer.Shutdown() + metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics") metricsServer.Start() defer metricsServer.Shutdown() @@ -133,6 +143,8 @@ func setupStore() store.BlobStore { Address: proxyAddress + ":" + proxyPort, Timeout: 30 * time.Second, }) + case "http": + s = store.NewHttpStore(proxyAddress + ":" + proxyPort) default: log.Fatalf("protocol is not recognized: %s", proxyProtocol) } diff --git a/go.mod b/go.mod index 82eaa1e..08137fa 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 github.com/davecgh/go-spew v1.1.1 + github.com/gin-gonic/gin v1.7.1 // indirect github.com/go-sql-driver/mysql v1.4.1 github.com/golang/protobuf v1.4.2 github.com/google/gops v0.3.18 diff --git a/go.sum b/go.sum index 6af413e..810e719 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,10 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.7.1 h1:qC89GU3p8TvKWMAVhEpmpB2CIb1hnqt2UdKZaP93mS8= +github.com/gin-gonic/gin v1.7.1/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= @@ -103,6 +107,13 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -123,6 +134,7 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -140,6 +152,7 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gops v0.3.18 h1:my259V+172PVFmduS2RAsq4FKH+HjKqdh7pLr17Ot8c= github.com/google/gops v0.3.18/go.mod h1:Pfp8hWGIFdV/7rY9/O/U5WgdjYQXf/GiEK4NVuVd2ZE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -226,6 +239,8 @@ github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -265,6 +280,8 @@ github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:f github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M= github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucas-clemente/quic-go v0.20.1 h1:hb5m76V8QS/8Nw/suHvXqo3BMHAozvIkcnzpJdpanSk= github.com/lucas-clemente/quic-go v0.20.1/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk= @@ -285,6 +302,8 @@ github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJG github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -300,7 +319,11 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= @@ -445,6 +468,10 @@ github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1g github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d h1:gI4/tqP6lCY5k6Sg+4k9qSoBXmPwG+xXgMpK7jivD4M= @@ -566,6 +593,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -679,6 +707,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 4c30606..ca65021 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -194,6 +194,11 @@ var ( Name: "udp_in_bytes", Help: "Total number of bytes downloaded through UDP", }) + MtrInBytesHttp = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Name: "http_in_bytes", + Help: "Total number of bytes downloaded through HTTP", + }) MtrOutBytesUdp = promauto.NewCounter(prometheus.CounterOpts{ Namespace: ns, Name: "udp_out_bytes", diff --git a/peer/store.go b/peer/store.go index 3068794..689d1c0 100644 --- a/peer/store.go +++ b/peer/store.go @@ -1,11 +1,13 @@ package peer import ( + "strings" "time" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/shared" + "github.com/lbryio/reflector.go/store" ) // Store is a blob store that gets blobs from a peer. @@ -51,7 +53,12 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) { return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err } defer c.Close() - return c.GetBlob(hash) + blob, trace, err := c.GetBlob(hash) + if err != nil && strings.Contains(err.Error(), "blob not found") { + return nil, trace, store.ErrBlobNotFound + } + + return blob, trace, err } // Put is not supported diff --git a/server/http/routes.go b/server/http/routes.go new file mode 100644 index 0000000..eee696f --- /dev/null +++ b/server/http/routes.go @@ -0,0 +1,53 @@ +package http + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/reflector.go/store" + log "github.com/sirupsen/logrus" +) + +func (s *Server) getBlob(c *gin.Context) { + hash := c.Query("hash") + blob, trace, err := s.store.Get(hash) + if err != nil { + serialized, serializeErr := trace.Serialize() + if serializeErr != nil { + _ = c.AbortWithError(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err)) + return + } + c.Header("Via", serialized) + + if errors.Is(err, store.ErrBlobNotFound) { + log.Errorf("wtf: %s", err.Error()) + c.AbortWithStatus(http.StatusNotFound) + return + } + _ = c.AbortWithError(http.StatusInternalServerError, err) + return + } + serialized, err := trace.Serialize() + if err != nil { + _ = c.AbortWithError(http.StatusInternalServerError, err) + return + } + c.Header("Via", serialized) + c.Header("Content-Disposition", "filename="+hash) + c.Data(http.StatusOK, "application/octet-stream", blob) +} + +func (s *Server) hasBlob(c *gin.Context) { + hash := c.Query("hash") + has, err := s.store.Has(hash) + if err != nil { + _ = c.AbortWithError(http.StatusInternalServerError, err) + return + } + if has { + c.Status(http.StatusNoContent) + return + } + c.Status(http.StatusNotFound) +} diff --git a/server/http/server.go b/server/http/server.go new file mode 100644 index 0000000..adc104f --- /dev/null +++ b/server/http/server.go @@ -0,0 +1,68 @@ +package http + +import ( + "context" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/lbryio/lbry.go/v2/extras/stop" + "github.com/lbryio/reflector.go/store" + log "github.com/sirupsen/logrus" +) + +// Server is an instance of a peer server that houses the listener and store. +type Server struct { + store store.BlobStore + grp *stop.Group +} + +// NewServer returns an initialized Server pointer. +func NewServer(store store.BlobStore) *Server { + return &Server{ + store: store, + grp: stop.New(), + } +} + +// Shutdown gracefully shuts down the peer server. +func (s *Server) Shutdown() { + log.Debug("shutting down HTTP server") + s.grp.StopAndWait() + log.Debug("HTTP server stopped") +} + +// Start starts the server listener to handle connections. +func (s *Server) Start(address string) error { + gin.SetMode(gin.ReleaseMode) + router := gin.Default() + router.GET("/blob", s.getBlob) + router.HEAD("/blob", s.hasBlob) + srv := &http.Server{ + Addr: address, + Handler: router, + } + go s.listenForShutdown(srv) + // Initializing the server in a goroutine so that + // it won't block the graceful shutdown handling below + s.grp.Add(1) + go func() { + defer s.grp.Done() + log.Println("HTTP server listening on " + address) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("listen: %s\n", err) + } + }() + return nil +} + +func (s *Server) listenForShutdown(listener *http.Server) { + <-s.grp.Ch() + // The context is used to inform the server it has 5 seconds to finish + // the request it is currently handling + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := listener.Shutdown(ctx); err != nil { + log.Fatal("Server forced to shutdown:", err) + } +} diff --git a/store/disk.go b/store/disk.go index 7123b65..fb5e326 100644 --- a/store/disk.go +++ b/store/disk.go @@ -130,12 +130,6 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { func (d *DiskStore) Put(hash string, blob stream.Blob) error { d.lock.Lock(hash) defer d.lock.Unlock(hash) - start := time.Now() - defer func() { - if time.Since(start) > 100*time.Millisecond { - log.Infof("it took %s to write %s", time.Since(start), hash) - } - }() err := d.initOnce() if err != nil { return err @@ -145,23 +139,7 @@ func (d *DiskStore) Put(hash string, blob stream.Blob) error { if err != nil { return err } - hashBytes := sha512.Sum384(blob) - readHash := hex.EncodeToString(hashBytes[:]) - matchesBeforeWriting := readHash == hash writeFile(d.path(hash), blob, 0644) - readBlob, err := ioutil.ReadFile(d.path(hash)) - matchesAfterReading := false - if err != nil { - log.Errorf("for some fucking reasons I can't read the blob I just wrote %s", err.Error()) - } else { - hashBytes = sha512.Sum384(readBlob) - readHash = hex.EncodeToString(hashBytes[:]) - matchesAfterReading = readHash == hash - } - - log.Infof(`writing %s to disk: hash match: %t, error: %t -reading after writing: hash match: %t`, hash, matchesBeforeWriting, err == nil, matchesAfterReading) - return errors.Err(err) } diff --git a/store/http.go b/store/http.go new file mode 100644 index 0000000..8a934d7 --- /dev/null +++ b/store/http.go @@ -0,0 +1,152 @@ +package store + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "sync" + "time" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" + "github.com/lbryio/reflector.go/internal/metrics" + "github.com/lbryio/reflector.go/shared" +) + +// NoopStore is a store that does nothing +type HttpStore struct { + upstream string + httpClient *http.Client +} + +func NewHttpStore(upstream string) *HttpStore { + return &HttpStore{ + upstream: upstream, + httpClient: getClient(), + } +} + +const nameHttp = "http" + +func (n *HttpStore) Name() string { return nameNoop } +func (n *HttpStore) Has(hash string) (bool, error) { + url := n.upstream + "/blob?hash=" + hash + + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return false, errors.Err(err) + } + + res, err := n.httpClient.Do(req) + if err != nil { + return false, errors.Err(err) + } + defer res.Body.Close() + if res.StatusCode == http.StatusNotFound { + return false, nil + } + if res.StatusCode == http.StatusNoContent { + return true, nil + } + var body []byte + if res.Body != nil { + body, _ = ioutil.ReadAll(res.Body) + } + return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body)) +} + +func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { + start := time.Now() + url := n.upstream + "/blob?hash=" + hash + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err) + } + + res, err := n.httpClient.Do(req) + if err != nil { + return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err) + } + defer res.Body.Close() + tmp := getBuffer() + defer putBuffer(tmp) + serialized := res.Header.Get("Via") + trace := shared.NewBlobTrace(time.Since(start), n.Name()) + if serialized != "" { + parsedTrace, err := shared.Deserialize(serialized) + if err != nil { + return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err + } + trace = *parsedTrace + } + + if res.StatusCode == http.StatusNotFound { + return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound + } + if res.StatusCode == http.StatusOK { + written, err := io.Copy(tmp, res.Body) + if err != nil { + return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err) + } + + blob := make([]byte, written) + copy(blob, tmp.Bytes()) + metrics.MtrInBytesHttp.Add(float64(len(blob))) + return blob, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound + } + var body []byte + if res.Body != nil { + body, _ = ioutil.ReadAll(res.Body) + } + + return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body)) +} + +func (n *HttpStore) Put(string, stream.Blob) error { + return shared.ErrNotImplemented +} +func (n *HttpStore) PutSD(string, stream.Blob) error { + return shared.ErrNotImplemented +} +func (n *HttpStore) Delete(string) error { + return shared.ErrNotImplemented +} +func (n *HttpStore) Shutdown() { return } + +// buffer pool to reduce GC +// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha +var buffers = sync.Pool{ + // New is called when a new instance is needed + New: func() interface{} { + buf := make([]byte, 0, stream.MaxBlobSize) + return bytes.NewBuffer(buf) + }, +} + +// getBuffer fetches a buffer from the pool +func getBuffer() *bytes.Buffer { + return buffers.Get().(*bytes.Buffer) +} + +// putBuffer returns a buffer to the pool +func putBuffer(buf *bytes.Buffer) { + buf.Reset() + buffers.Put(buf) +} + +// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs) +func getClient() *http.Client { + // Customize the Transport to have larger connection pool + defaultRoundTripper := http.DefaultTransport + defaultTransportPointer := defaultRoundTripper.(*http.Transport) + + defaultTransport := *defaultTransportPointer // dereference it to get a copy of the struct that the pointer points to + defaultTransport.MaxIdleConns = 100 + defaultTransport.DisableCompression = true + defaultTransport.MaxIdleConnsPerHost = 100 + defaultTransport.ReadBufferSize = stream.MaxBlobSize + 1024*10 //add an extra few KBs to make sure it fits the extra information + + return &http.Client{Transport: &defaultTransport} +} diff --git a/store/singleflight.go b/store/singleflight.go index 6de91f4..02337a2 100644 --- a/store/singleflight.go +++ b/store/singleflight.go @@ -3,6 +3,7 @@ package store import ( "time" + "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/shared" @@ -46,6 +47,9 @@ func (s *singleflightStore) Get(hash string) (stream.Blob, shared.BlobTrace, err if err != nil { return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err } + if gr == nil { + return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("getter response is nil") + } rsp := gr.(getterResponse) return rsp.blob, rsp.stack, nil } -- 2.45.2 From 5cc1e84adb77756bfa83b5e32e44ad7aa38ebf45 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 21 May 2021 05:53:13 +0200 Subject: [PATCH 46/51] remove locks causing deadlocks --- locks/multiplelock.go | 131 ------------------------------------- locks/multiplelock_test.go | 56 ---------------- store/disk.go | 13 ---- store/http.go | 2 +- 4 files changed, 1 insertion(+), 201 deletions(-) delete mode 100644 locks/multiplelock.go delete mode 100644 locks/multiplelock_test.go diff --git a/locks/multiplelock.go b/locks/multiplelock.go deleted file mode 100644 index c87b645..0000000 --- a/locks/multiplelock.go +++ /dev/null @@ -1,131 +0,0 @@ -package locks - -import ( - "fmt" - "sync" - "time" -) - -// MultipleLock is the main interface for multiLock based on key -type MultipleLock interface { - Lock(key string) - RLock(key string) - Unlock(key string) - RUnlock(key string) -} - -func NewMultipleLock() MultipleLock { - return &multiLock{ - locks: make(map[string]*itemLock), - mu: sync.Mutex{}, - } -} - -type itemLock struct { - lk *sync.RWMutex - cnt int64 -} - -// multiLock is an optimized locking system per locking key -type multiLock struct { - locks map[string]*itemLock - mu sync.Mutex // synchronize reads/writes to locks map -} - -func debugPrint(format string, a ...interface{}) { - debugEnabled := false - if debugEnabled { - a = append(a, time.Now().Format("04:05.000000")) - fmt.Printf(format+" at %s\n", a...) - } -} - -func (ml *multiLock) Lock(key string) { - debugPrint("mutex requested %s", key) - ml.mu.Lock() - debugPrint("mutex acquired %s", key) - - itmLock, exists := ml.locks[key] - if !exists { - debugPrint("new lock created %s", key) - itmLock = &itemLock{&sync.RWMutex{}, 0} - ml.locks[key] = itmLock - } - itmLock.cnt++ - debugPrint("releasing mutex %s", key) - ml.mu.Unlock() - - debugPrint("Lock requested %s", key) - itmLock.lk.Lock() - debugPrint("Lock acquired %s", key) -} - -func (ml *multiLock) RLock(key string) { - debugPrint("mutex requested %s", key) - ml.mu.Lock() - debugPrint("mutex acquired %s", key) - - itmLock, exists := ml.locks[key] - if !exists { - debugPrint("new lock created for %s", key) - itmLock = &itemLock{&sync.RWMutex{}, 0} - ml.locks[key] = itmLock - } - itmLock.cnt++ - debugPrint("releasing mutex %s", key) - ml.mu.Unlock() - - debugPrint("RLock requested %s", key) - itmLock.lk.RLock() - debugPrint("RLock acquired %s", key) -} - -func (ml *multiLock) Unlock(key string) { - debugPrint("mutex requested %s", key) - ml.mu.Lock() - debugPrint("mutex acquired %s", key) - - itmLock, exists := ml.locks[key] - if !exists { - panic("sync Unlock of non existent lock!!") - } - - debugPrint("Unlock %s", key) - itmLock.lk.Unlock() - itmLock.cnt-- - if itmLock.cnt == 0 { - debugPrint("delete lock %s", key) - delete(ml.locks, key) - } - if itmLock.cnt < 0 { - panic("sync Unlock of free Lock!!") - } - - debugPrint("releasing mutex %s", key) - ml.mu.Unlock() -} - -func (ml *multiLock) RUnlock(key string) { - debugPrint("mutex requested %s", key) - ml.mu.Lock() - debugPrint("mutex acquired %s", key) - - itmLock, exists := ml.locks[key] - if !exists { - panic("sync Unlock of non existent lock!!") - } - - debugPrint("RUnlock %s", key) - itmLock.lk.RUnlock() - itmLock.cnt-- - if itmLock.cnt == 0 { - debugPrint("delete lock %s", key) - delete(ml.locks, key) - } - if itmLock.cnt < 0 { - panic("sync Unlock of free Lock!!") - } - - debugPrint("releasing mutex %s", key) - ml.mu.Unlock() -} diff --git a/locks/multiplelock_test.go b/locks/multiplelock_test.go deleted file mode 100644 index a7a42c7..0000000 --- a/locks/multiplelock_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package locks - -import ( - "math/rand" - "strconv" - "testing" - "time" - - "github.com/lbryio/lbry.go/v2/extras/stop" -) - -var lock = NewMultipleLock() - -func TestNewMultipleLock(t *testing.T) { - grp := stop.New() - for i := 0; i < 100; i++ { - grp.Add(2) - go doRWWork(i, i%10, grp) - go doRWork(i, i%10, grp) - } - time.Sleep(5 * time.Second) - grp.StopAndWait() -} -func doRWWork(worker int, resource int, grp *stop.Group) { - for { - select { - case <-grp.Ch(): - grp.Done() - return - default: - //log.Printf("RW - worker %d doing work on resource %d\n", worker, resource) - lock.Lock(strconv.Itoa(resource)) - randomTime := time.Duration(rand.Int()%10+1) * time.Microsecond - time.Sleep(randomTime) - //log.Printf("RW - worker %d releasing %d\n", worker, resource) - lock.Unlock(strconv.Itoa(resource)) - } - } -} - -func doRWork(worker int, resource int, grp *stop.Group) { - for { - select { - case <-grp.Ch(): - grp.Done() - return - default: - //log.Printf("R - worker %d doing work on resource %d\n", worker, resource) - lock.RLock(strconv.Itoa(resource)) - randomTime := time.Duration(rand.Int()%10+1) * time.Microsecond - time.Sleep(randomTime) - //log.Printf("R - worker %d releasing %d\n", worker, resource) - lock.RUnlock(strconv.Itoa(resource)) - } - } -} diff --git a/store/disk.go b/store/disk.go index fb5e326..5ef7006 100644 --- a/store/disk.go +++ b/store/disk.go @@ -12,7 +12,6 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" - "github.com/lbryio/reflector.go/locks" "github.com/lbryio/reflector.go/shared" "github.com/lbryio/reflector.go/store/speedwalk" log "github.com/sirupsen/logrus" @@ -49,7 +48,6 @@ type DiskStore struct { initialized bool concurrentChecks atomic.Int32 - lock locks.MultipleLock } const maxConcurrentChecks = 3 @@ -59,7 +57,6 @@ func NewDiskStore(dir string, prefixLength int) *DiskStore { return &DiskStore{ blobDir: dir, prefixLength: prefixLength, - lock: locks.NewMultipleLock(), } } @@ -70,8 +67,6 @@ func (d *DiskStore) Name() string { return nameDisk } // Has returns T/F or Error if it the blob stored already. It will error with any IO disk error. func (d *DiskStore) Has(hash string) (bool, error) { - d.lock.RLock(hash) - defer d.lock.RUnlock(hash) err := d.initOnce() if err != nil { return false, err @@ -89,8 +84,6 @@ func (d *DiskStore) Has(hash string) (bool, error) { // Get returns the blob or an error if the blob doesn't exist. func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { - d.lock.RLock(hash) - defer d.lock.RUnlock(hash) start := time.Now() err := d.initOnce() if err != nil { @@ -128,8 +121,6 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { // Put stores the blob on disk func (d *DiskStore) Put(hash string, blob stream.Blob) error { - d.lock.Lock(hash) - defer d.lock.Unlock(hash) err := d.initOnce() if err != nil { return err @@ -145,15 +136,11 @@ func (d *DiskStore) Put(hash string, blob stream.Blob) error { // PutSD stores the sd blob on the disk func (d *DiskStore) PutSD(hash string, blob stream.Blob) error { - d.lock.Lock(hash) - defer d.lock.Unlock(hash) return d.Put(hash, blob) } // Delete deletes the blob from the store func (d *DiskStore) Delete(hash string) error { - d.lock.Lock(hash) - defer d.lock.Unlock(hash) err := d.initOnce() if err != nil { return err diff --git a/store/http.go b/store/http.go index 8a934d7..34461de 100644 --- a/store/http.go +++ b/store/http.go @@ -94,7 +94,7 @@ func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) { blob := make([]byte, written) copy(blob, tmp.Bytes()) metrics.MtrInBytesHttp.Add(float64(len(blob))) - return blob, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound + return blob, trace.Stack(time.Since(start), n.Name()), nil } var body []byte if res.Body != nil { -- 2.45.2 From 1ec218483397f6e472030e4ab9b5b7d86a4217eb Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 21 May 2021 17:58:33 +0200 Subject: [PATCH 47/51] upgrade singleflight http store fix --- go.mod | 4 ++-- go.sum | 5 +++-- store/http.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 08137fa..0bc0bd2 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 github.com/davecgh/go-spew v1.1.1 - github.com/gin-gonic/gin v1.7.1 // indirect + github.com/gin-gonic/gin v1.7.1 github.com/go-sql-driver/mysql v1.4.1 github.com/golang/protobuf v1.4.2 github.com/google/gops v0.3.18 @@ -41,7 +41,7 @@ require ( github.com/volatiletech/null v8.0.0+incompatible go.uber.org/atomic v1.5.1 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect - golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect google.golang.org/appengine v1.6.2 // indirect ) diff --git a/go.sum b/go.sum index 810e719..64c9a07 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -568,8 +569,9 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -597,7 +599,6 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg= golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU= golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/store/http.go b/store/http.go index 34461de..11f506d 100644 --- a/store/http.go +++ b/store/http.go @@ -22,7 +22,7 @@ type HttpStore struct { func NewHttpStore(upstream string) *HttpStore { return &HttpStore{ - upstream: upstream, + upstream: "http://" + upstream, httpClient: getClient(), } } -- 2.45.2 From bc889001bbe924ff1dad13852b66352b070e9ea6 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 21 May 2021 18:05:37 +0200 Subject: [PATCH 48/51] update lbry.go dep --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0bc0bd2..31386d5 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/karrick/godirwalk v1.16.1 github.com/lbryio/chainquery v1.9.0 github.com/lbryio/lbry.go v1.1.2 // indirect - github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d + github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3 github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 github.com/lucas-clemente/quic-go v0.20.1 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 diff --git a/go.sum b/go.sum index 64c9a07..351f737 100644 --- a/go.sum +++ b/go.sum @@ -271,8 +271,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c= github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d h1:VUaOZ3cbCe7gfpycN/srCOk6U2bBS9NZHEz9RiRxd4E= -github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk= +github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3 h1:hkVViG8qbOKJMf/M6mN+Hrtns4j55IU2dRavpJZWbxw= +github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= -- 2.45.2 From df881e16b5b39cb8dee4d29a6517f48e7ccfa6d5 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 21 May 2021 19:09:02 +0200 Subject: [PATCH 49/51] add metrics --- internal/metrics/metrics.go | 10 ++++++++++ server/http/routes.go | 6 ++++++ store/http.go | 5 +++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index ca65021..b6d5a57 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -118,6 +118,11 @@ var ( Name: "http3_blob_download_total", Help: "Total number of blobs downloaded from reflector through QUIC protocol", }) + HttpDownloadCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Name: "http_blob_download_total", + Help: "Total number of blobs downloaded from reflector through HTTP protocol", + }) CacheHitCount = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: ns, @@ -204,6 +209,11 @@ var ( Name: "udp_out_bytes", Help: "Total number of bytes streamed out through UDP", }) + MtrOutBytesHttp = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Name: "http_out_bytes", + Help: "Total number of bytes streamed out through UDP", + }) MtrInBytesReflector = promauto.NewCounter(prometheus.CounterOpts{ Namespace: ns, Name: "reflector_in_bytes", diff --git a/server/http/routes.go b/server/http/routes.go index eee696f..e9124a9 100644 --- a/server/http/routes.go +++ b/server/http/routes.go @@ -5,7 +5,10 @@ import ( "github.com/gin-gonic/gin" "github.com/lbryio/lbry.go/v2/extras/errors" + + "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/store" + log "github.com/sirupsen/logrus" ) @@ -33,6 +36,9 @@ func (s *Server) getBlob(c *gin.Context) { _ = c.AbortWithError(http.StatusInternalServerError, err) return } + metrics.MtrOutBytesHttp.Add(float64(len(blob))) + metrics.BlobDownloadCount.Inc() + metrics.HttpDownloadCount.Inc() c.Header("Via", serialized) c.Header("Content-Disposition", "filename="+hash) c.Data(http.StatusOK, "application/octet-stream", blob) diff --git a/store/http.go b/store/http.go index 11f506d..f79612e 100644 --- a/store/http.go +++ b/store/http.go @@ -8,10 +8,11 @@ import ( "sync" "time" - "github.com/lbryio/lbry.go/v2/extras/errors" - "github.com/lbryio/lbry.go/v2/stream" "github.com/lbryio/reflector.go/internal/metrics" "github.com/lbryio/reflector.go/shared" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" ) // NoopStore is a store that does nothing -- 2.45.2 From 9670bc14f88d835dca248dc030a7001dc773350b Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 21 May 2021 21:06:59 +0200 Subject: [PATCH 50/51] fix unsafe dereference --- store/http.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/store/http.go b/store/http.go index f79612e..c24169a 100644 --- a/store/http.go +++ b/store/http.go @@ -4,6 +4,7 @@ import ( "bytes" "io" "io/ioutil" + "net" "net/http" "sync" "time" @@ -140,14 +141,20 @@ func putBuffer(buf *bytes.Buffer) { // getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs) func getClient() *http.Client { // Customize the Transport to have larger connection pool - defaultRoundTripper := http.DefaultTransport - defaultTransportPointer := defaultRoundTripper.(*http.Transport) + defaultTransport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DisableCompression: true, + MaxIdleConnsPerHost: 100, + ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information + } - defaultTransport := *defaultTransportPointer // dereference it to get a copy of the struct that the pointer points to - defaultTransport.MaxIdleConns = 100 - defaultTransport.DisableCompression = true - defaultTransport.MaxIdleConnsPerHost = 100 - defaultTransport.ReadBufferSize = stream.MaxBlobSize + 1024*10 //add an extra few KBs to make sure it fits the extra information - - return &http.Client{Transport: &defaultTransport} + return &http.Client{Transport: defaultTransport} } -- 2.45.2 From 2a1557845da5e6e4acf06882d4976cd5f0e01092 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Thu, 20 May 2021 17:15:45 -0400 Subject: [PATCH 51/51] wrap blob insertion in tx. fixes lbryio/lbry-sdk#3296 The problem is that inserting an sd blob with ~5k blobs takes longer than 30 seconds. So the client times out and retries the request. At that point, reflector is not done inserting so it replies with a smaller number of blobs than it should. The client uploads that many blobs and marks the stream as reflected. The remaining blobs never get uploaded. Doing the insert inside a transaction should be faster than doing 10k (2 per blob) inserts independently. --- db/db.go | 125 +++++++++++++++++++++--------------------- db/interfaces.go | 45 +++++++++++++++ shared/shared_test.go | 22 ++++++-- 3 files changed, 125 insertions(+), 67 deletions(-) create mode 100644 db/interfaces.go diff --git a/db/db.go b/db/db.go index babe641..11f4d53 100644 --- a/db/db.go +++ b/db/db.go @@ -97,7 +97,7 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error { return errors.Err("not connected") } - _, err := s.insertBlob(hash, length, isStored) + _, err := s.insertBlob(s.conn, hash, length, isStored) return err } @@ -163,7 +163,7 @@ func (s *SQL) insertBlobs(hashes []string) error { //args = append(args, hash, true, stream.MaxBlobSize, dayAgo) } q = strings.TrimSuffix(q, ",") - _, err := s.exec(q) + _, err := s.exec(s.conn, q) if err != nil { return err } @@ -171,7 +171,7 @@ func (s *SQL) insertBlobs(hashes []string) error { return nil } -func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) { +func (s *SQL) insertBlob(ex Executor, hash string, length int, isStored bool) (int64, error) { if length <= 0 { return 0, errors.Err("length must be positive") } @@ -188,13 +188,13 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))" } - blobID, err := s.exec(q, args...) + blobID, err := s.exec(ex, q, args...) if err != nil { return 0, err } if blobID == 0 { - err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID) + err = ex.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID) if err != nil { return 0, errors.Err(err) } @@ -203,7 +203,7 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) } if s.TrackAccess == TrackAccessBlobs { - err := s.touchBlobs([]uint64{uint64(blobID)}) + err := s.touchBlobs(ex, []uint64{uint64(blobID)}) if err != nil { return 0, errors.Err(err) } @@ -227,7 +227,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) { q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")" } - streamID, err := s.exec(q, args...) + streamID, err := s.exec(s.conn, q, args...) if err != nil { return 0, errors.Err(err) } @@ -266,16 +266,16 @@ func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) { if touch { if s.TrackAccess == TrackAccessBlobs { - s.touchBlobs(idsNeedingTouch) + _ = s.touchBlobs(s.conn, idsNeedingTouch) } else if s.TrackAccess == TrackAccessStreams { - s.touchStreams(idsNeedingTouch) + _ = s.touchStreams(idsNeedingTouch) } } return exists, err } -func (s *SQL) touchBlobs(blobIDs []uint64) error { +func (s *SQL) touchBlobs(ex Executor, blobIDs []uint64) error { if len(blobIDs) == 0 { return nil } @@ -288,7 +288,7 @@ func (s *SQL) touchBlobs(blobIDs []uint64) error { } startTime := time.Now() - _, err := s.exec(query, args...) + _, err := s.exec(ex, query, args...) log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime)) return errors.Err(err) } @@ -306,7 +306,7 @@ func (s *SQL) touchStreams(streamIDs []uint64) error { } startTime := time.Now() - _, err := s.exec(query, args...) + _, err := s.exec(s.conn, query, args...) log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime)) return errors.Err(err) } @@ -406,16 +406,16 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)` // NOTE: If SoftDelete is enabled, streams will never be deleted func (s *SQL) Delete(hash string) error { if s.SoftDelete { - _, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash) + _, err := s.exec(s.conn, "UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash) return errors.Err(err) } - _, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash) + _, err := s.exec(s.conn, "DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash) if err != nil { return errors.Err(err) } - _, err = s.exec("DELETE FROM blob_ WHERE hash = ?", hash) + _, err = s.exec(s.conn, "DELETE FROM blob_ WHERE hash = ?", hash) return errors.Err(err) } @@ -590,7 +590,7 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error { return errors.Err("not connected") } - sdBlobID, err := s.insertBlob(sdHash, sdBlobLength, true) + sdBlobID, err := s.insertBlob(s.conn, sdHash, sdBlobLength, true) if err != nil { return err } @@ -600,28 +600,30 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error { return err } - // insert content blobs and connect them to stream - for _, contentBlob := range sdBlob.Blobs { - if contentBlob.BlobHash == "" { - // null terminator blob - continue - } + return withTx(s.conn, func(tx Transactor) error { + // insert content blobs and connect them to stream + for _, contentBlob := range sdBlob.Blobs { + if contentBlob.BlobHash == "" { + // null terminator blob + continue + } - blobID, err := s.insertBlob(contentBlob.BlobHash, contentBlob.Length, false) - if err != nil { - return err - } + blobID, err := s.insertBlob(tx, contentBlob.BlobHash, contentBlob.Length, false) + if err != nil { + return err + } - args := []interface{}{streamID, blobID, contentBlob.BlobNum} - _, err = s.exec( - "INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")", - args..., - ) - if err != nil { - return errors.Err(err) + args := []interface{}{streamID, blobID, contentBlob.BlobNum} + _, err = s.exec(tx, + "INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")", + args..., + ) + if err != nil { + return errors.Err(err) + } } - } - return nil + return nil + }) } // GetHashRange gets the smallest and biggest hashes in the db @@ -694,39 +696,38 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap } // txFunc is a function that can be wrapped in a transaction -type txFunc func(tx *sql.Tx) error +type txFunc func(tx Transactor) error -// withTx wraps a function in an sql transaction. the transaction is committed if there's no error, or rolled back if there is one. -// if dbOrTx is an sql.DB, a new transaction is started +// withTx wraps a function in an sql transaction. the transaction is committed if there's +// no error, or rolled back if there is one. if dbOrTx is not a Transactor (e.g. if it's +// an *sql.DB), withTx attempts to start a new transaction to use. func withTx(dbOrTx interface{}, f txFunc) (err error) { - var tx *sql.Tx + var tx Transactor + var ok bool - switch t := dbOrTx.(type) { - case *sql.Tx: - tx = t - case *sql.DB: - tx, err = t.Begin() + tx, ok = dbOrTx.(Transactor) + if !ok { + tx, err = Begin(dbOrTx) if err != nil { return err } - defer func() { - if p := recover(); p != nil { - if rollBackError := tx.Rollback(); rollBackError != nil { - log.Error("failed to rollback tx on panic - ", rollBackError) - } - panic(p) - } else if err != nil { - if rollBackError := tx.Rollback(); rollBackError != nil { - log.Error("failed to rollback tx on panic - ", rollBackError) - } - } else { - err = errors.Err(tx.Commit()) - } - }() - default: - return errors.Err("db or tx required") } + defer func() { + if p := recover(); p != nil { + if rollBackError := tx.Rollback(); rollBackError != nil { + log.Error("failed to rollback tx on panic: ", rollBackError) + } + err = errors.Prefix("panic", p) + } else if err != nil { + if rollBackError := tx.Rollback(); rollBackError != nil { + log.Error("failed to rollback tx: ", rollBackError) + } + } else { + err = errors.Err(tx.Commit()) + } + }() + return f(tx) } @@ -739,12 +740,12 @@ func closeRows(rows *sql.Rows) { } } -func (s *SQL) exec(query string, args ...interface{}) (int64, error) { +func (s *SQL) exec(ex Executor, query string, args ...interface{}) (int64, error) { s.logQuery(query, args...) attempt, maxAttempts := 0, 3 Retry: attempt++ - result, err := s.conn.Exec(query, args...) + result, err := ex.Exec(query, args...) if isLockTimeoutError(err) { if attempt <= maxAttempts { //Error 1205: Lock wait timeout exceeded; try restarting transaction diff --git a/db/interfaces.go b/db/interfaces.go new file mode 100644 index 0000000..11b2ee6 --- /dev/null +++ b/db/interfaces.go @@ -0,0 +1,45 @@ +package db + +import ( + "database/sql" + + "github.com/lbryio/lbry.go/v2/extras/errors" +) + +// Executor can perform SQL queries. +type Executor interface { + Exec(query string, args ...interface{}) (sql.Result, error) + Query(query string, args ...interface{}) (*sql.Rows, error) + QueryRow(query string, args ...interface{}) *sql.Row +} + +// Transactor can commit and rollback, on top of being able to execute queries. +type Transactor interface { + Commit() error + Rollback() error + + Executor +} + +// Begin begins a transaction +func Begin(db interface{}) (Transactor, error) { + type beginner interface { + Begin() (Transactor, error) + } + + creator, ok := db.(beginner) + if ok { + return creator.Begin() + } + + type sqlBeginner interface { + Begin() (*sql.Tx, error) + } + + creator2, ok := db.(sqlBeginner) + if ok { + return creator2.Begin() + } + + return nil, errors.Err("database does not support transactions") +} diff --git a/shared/shared_test.go b/shared/shared_test.go index 47c8eef..50a4eb7 100644 --- a/shared/shared_test.go +++ b/shared/shared_test.go @@ -1,27 +1,39 @@ package shared import ( + "os" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestBlobTrace_Serialize(t *testing.T) { + hostName, err := os.Hostname() + require.NoError(t, err) + stack := NewBlobTrace(10*time.Second, "test") stack.Stack(20*time.Second, "test2") stack.Stack(30*time.Second, "test3") serialized, err := stack.Serialize() - assert.NoError(t, err) - t.Log(serialized) - expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}" + require.NoError(t, err) + + expected := `{"stacks":[{"timing":10000000000,"origin_name":"test","host_name":"` + + hostName + + `"},{"timing":20000000000,"origin_name":"test2","host_name":"` + + hostName + + `"},{"timing":30000000000,"origin_name":"test3","host_name":"` + + hostName + + `"}]}` assert.Equal(t, expected, serialized) } func TestBlobTrace_Deserialize(t *testing.T) { - serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}" + serialized := `{"stacks":[{"timing":10000000000,"origin_name":"test"},{"timing":20000000000,"origin_name":"test2"},{"timing":30000000000,"origin_name":"test3"}]}` stack, err := Deserialize(serialized) - assert.NoError(t, err) + require.NoError(t, err) + assert.Len(t, stack.Stacks, 3) assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second) assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second) -- 2.45.2