simplify resolve call on wallet node

.github/workflows/go.yml

@@ -1,37 +0,0 @@
-name: Go
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.20.x
-
-      - name: Build linux
-        run: make linux
-
-      - name: Build macos
-        run: make macos
-
-      - name: Test
-        run: make test
-
-      - name: Lint
-        run: make lint
-
-      - name: retrieve all tags
-        run: git fetch --prune --unshallow --tags
-
-      - name: Print changes since last version
-        run: git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline

.github/workflows/release.yml

@@ -1,62 +0,0 @@
-name: release
-
-on:
-  push:
-    tags:
-      - "*.*.*"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.20.x
-
-      - name: Build linux
-        run: make linux
-
-      - name: Build macos
-        run: make macos
-
-      - name: Test
-        run: make test
-
-      - name: Lint
-        run: make lint
-
-      - name: Zip macos
-        run: zip -r reflector_darwin_amd64.zip ./dist/darwin_amd64
-
-      - name: Zip linux
-        run: zip -r reflector_linux_amd64.zip ./dist/linux_amd64
-
-      - name: retrieve all tags
-        run: git fetch --prune --unshallow --tags
-
-      - name: Generate Changelog
-        run: git log $(git describe --tags --abbrev=0 @^)..@ --no-merges --oneline > ${{ github.workspace }}-CHANGELOG.txt
-
-      - name: upload to github releases
-        uses: softprops/action-gh-release@v1
-        with:
-          files: |
-            ./reflector_linux_amd64.zip
-            ./reflector_darwin_amd64.zip
-          body_path: ${{ github.workspace }}-CHANGELOG.txt
-
-#      - name: Login to DockerHub
-#        uses: docker/login-action@v2
-#        with:
-#          username: ${{ secrets.DOCKERHUB_USERNAME }}
-#          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-#      - name: Generate docker image
-#        run: make image
-
-#      - name: Docker push
-#        run: make publish_image

.gitignore

@@ -1,4 +1,3 @@
 /vendor
 /config.json*
-/dist
 /bin

.travis.yml

@@ -1,9 +1,12 @@
 os: linux
-dist: bionic
+dist: trusty
 language: go
 
+env:
+  - GO111MODULE=on
+
 go:
-  - 1.20.x
+  - 1.15.x
 
 cache:
   directories:
@@ -14,7 +17,7 @@ notifications:
   email: false
 
 # Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
-#install: true
+install: true
 
 # Anything in before_script that returns a nonzero exit code will
 # flunk the build and immediately stop. It's sorta like having
@@ -22,14 +25,14 @@ notifications:
 before_script:
 # All the .go files, excluding vendor/ and model (auto generated)
   - GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p
-  - go install golang.org/x/tools/cmd/goimports                     # Used in build script for generated files
+  - go get golang.org/x/tools/cmd/goimports                     # Used in build script for generated files
 # - go get github.com/golang/lint/golint                        # Linter
 #  - go get honnef.co/go/tools/cmd/megacheck                     # Badass static analyzer/linter
-  - go install github.com/fzipp/gocyclo/cmd/gocyclo@latest                 # Check against high complexity
+  - go get github.com/jgautheron/gocyclo                        # Check against high complexity
-  - go install github.com/mdempsky/unconvert@latest                        # Identifies unnecessary type conversions
+  - go get github.com/mdempsky/unconvert                        # Identifies unnecessary type conversions
-  - go install github.com/kisielk/errcheck@latest                          # Checks for unhandled errors
+  - go get github.com/kisielk/errcheck                          # Checks for unhandled errors
-  - go install github.com/opennota/check/cmd/varcheck@latest               # Checks for unused vars
+  - go get github.com/opennota/check/cmd/varcheck               # Checks for unused vars
-  - go install github.com/opennota/check/cmd/structcheck@latest            # Checks for unused fields in structs
+  - go get github.com/opennota/check/cmd/structcheck            # Checks for unused fields in structs
 
 
 
@@ -37,7 +40,7 @@ before_script:
 # in a modern Go project.
 script:
   # Fail if a .go file hasn't been formatted with gofmt
-  - for i in $GO_FILES; do test -z $(gofmt -s -l $i); done
+  - test -z $(gofmt -s -l $GO_FILES)
   # Run unit tests
   - make test
   # Checks for unused vars and fields on structs
@@ -56,11 +59,11 @@ script:
   # one last linter - ignore autogen code
   #- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
   # Finally, build the binary
-  - make linux
+  - make
 
 deploy:
   - provider: s3
-    local_dir: ./dist/linux_amd64
+    local_dir: ./bin
     skip_cleanup: true
     on:
       repo: lbryio/reflector.go

Dockerfile

@@ -3,7 +3,7 @@ EXPOSE 8080
 
 RUN mkdir /app
 WORKDIR /app
-COPY dist/linux_amd64/prism-bin ./prism
+COPY bin/prism-bin ./prism
 RUN chmod +x prism
 
 ENTRYPOINT [ "/app/prism" ]

Makefile

@@ -1,33 +1,25 @@
-version := $(shell git describe --dirty --always --long --abbrev=7)
-commit := $(shell git rev-parse --short HEAD)
-commit_long := $(shell git rev-parse HEAD)
-branch := $(shell git rev-parse --abbrev-ref HEAD)
-curTime := $(shell date +%s)
-
 BINARY=prism-bin
-IMPORT_PATH = github.com/lbryio/reflector.go
+
-LDFLAGS="-X ${IMPORT_PATH}/meta.version=$(version) -X ${IMPORT_PATH}/meta.commit=$(commit) -X ${IMPORT_PATH}/meta.commitLong=$(commit_long) -X ${IMPORT_PATH}/meta.branch=$(branch) -X '${IMPORT_PATH}/meta.Time=$(curTime)'"
 DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
-BIN_DIR = $(DIR)/dist
+BIN_DIR = ${DIR}/bin
+IMPORT_PATH = github.com/lbryio/reflector.go
 
-.DEFAULT_GOAL := linux
+VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
+LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
+
+
+.PHONY: build clean test lint
+.DEFAULT_GOAL: build
+
+
+build:
+	mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
+
+clean:
+	if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
 
-.PHONY: test
 test:
-	go test -cover -v ./...
+	go test ./... -v -cover
 
-.PHONY: lint
 lint:
-	./scripts/lint.sh
+	go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
-
-.PHONY: linux
-linux:
-	GOARCH=amd64 GOOS=linux go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/linux_amd64/${BINARY}
-
-.PHONY: macos
-macos:
-	GOARCH=amd64 GOOS=darwin go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/darwin_amd64/${BINARY}
-
-.PHONY: image
-image:
-	docker buildx build -t lbry/reflector:$(version) -t lbry/reflector:latest --platform linux/amd64 .

cluster/cluster.go

@@ -1,7 +1,7 @@
 package cluster
 
 import (
-	"io"
+	"io/ioutil"
 	baselog "log"
 	"sort"
 	"time"
@@ -52,7 +52,7 @@ func (c *Cluster) Connect() error {
 	conf.MemberlistConfig.AdvertisePort = c.port
 	conf.NodeName = c.name
 
-	nullLogger := baselog.New(io.Discard, "", 0)
+	nullLogger := baselog.New(ioutil.Discard, "", 0)
 	conf.Logger = nullLogger
 
 	c.eventCh = make(chan serf.Event)

cmd/cluster.go

@@ -7,7 +7,6 @@ import (
 	"syscall"
 
 	"github.com/lbryio/lbry.go/v2/extras/crypto"
-
 	"github.com/lbryio/reflector.go/cluster"
 
 	log "github.com/sirupsen/logrus"

cmd/decode.go

@@ -4,10 +4,10 @@ import (
 	"encoding/hex"
 	"fmt"
 
-	"github.com/lbryio/lbry.go/v2/schema/stake"
+	"github.com/lbryio/lbry.go/v2/schema/claim"
 
 	"github.com/davecgh/go-spew/spew"
-	"github.com/gogo/protobuf/jsonpb"
+	"github.com/golang/protobuf/jsonpb"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -23,7 +23,7 @@ func init() {
 }
 
 func decodeCmd(cmd *cobra.Command, args []string) {
-	c, err := stake.DecodeClaimHex(args[0], "")
+	c, err := claim.DecodeClaimHex(args[0], "")
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/getstream.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/lbryio/reflector.go/server/peer"
+	"github.com/lbryio/reflector.go/peer"
 	"github.com/lbryio/reflector.go/store"
 
 	"github.com/lbryio/lbry.go/v2/stream"
@@ -41,7 +41,7 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
 
 	var sd stream.SDBlob
 
-	sdb, _, err := s.Get(sdHash)
+	sdb, err := s.Get(sdHash)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -62,7 +62,7 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
 	}
 
 	for i := 0; i < len(sd.BlobInfos)-1; i++ {
-		b, _, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
+		b, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/integrity.go

@@ -1,93 +0,0 @@
-package cmd
-
-import (
-	"crypto/sha512"
-	"encoding/hex"
-	"os"
-	"path"
-	"runtime"
-	"sync/atomic"
-	"time"
-
-	"github.com/lbryio/reflector.go/meta"
-	"github.com/lbryio/reflector.go/store/speedwalk"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-var threads int
-
-func init() {
-	var cmd = &cobra.Command{
-		Use:   "check-integrity",
-		Short: "check blobs integrity for a given path",
-		Run:   integrityCheckCmd,
-	}
-	cmd.Flags().StringVar(&diskStorePath, "store-path", "", "path of the store where all blobs are cached")
-	cmd.Flags().IntVar(&threads, "threads", runtime.NumCPU()-1, "number of concurrent threads to process blobs")
-	rootCmd.AddCommand(cmd)
-}
-
-func integrityCheckCmd(cmd *cobra.Command, args []string) {
-	log.Printf("reflector %s", meta.VersionString())
-	if diskStorePath == "" {
-		log.Fatal("store-path must be defined")
-	}
-
-	blobs, err := speedwalk.AllFiles(diskStorePath, true)
-	if err != nil {
-		log.Fatalf("error while reading blobs from disk %s", errors.FullTrace(err))
-	}
-	tasks := make(chan string, len(blobs))
-	done := make(chan bool)
-	processed := new(int32)
-	go produce(tasks, blobs)
-	cpus := runtime.NumCPU()
-	for i := 0; i < cpus-1; i++ {
-		go consume(i, tasks, done, len(blobs), processed)
-	}
-	<-done
-}
-
-func produce(tasks chan<- string, blobs []string) {
-	for _, b := range blobs {
-		tasks <- b
-	}
-	close(tasks)
-}
-
-func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int, processed *int32) {
-	start := time.Now()
-
-	for b := range tasks {
-		processedSoFar := atomic.AddInt32(processed, 1)
-		if worker == 0 {
-			remaining := int32(totalTasks) - processedSoFar
-			timePerBlob := time.Since(start).Microseconds() / int64(processedSoFar)
-			remainingTime := time.Duration(int64(remaining)*timePerBlob) * time.Microsecond
-			log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
-		}
-		blobPath := path.Join(diskStorePath, b[:2], b)
-		blob, err := os.ReadFile(blobPath)
-		if err != nil {
-			if os.IsNotExist(err) {
-				continue
-			}
-			log.Errorf("[Worker %d] Error looking up blob %s: %s", worker, b, err.Error())
-			continue
-		}
-		hashBytes := sha512.Sum384(blob)
-		readHash := hex.EncodeToString(hashBytes[:])
-		if readHash != b {
-			log.Infof("[%s] found a broken blob while reading from disk. Actual hash: %s", b, readHash)
-			err := os.Remove(blobPath)
-			if err != nil {
-				log.Errorf("Error while deleting broken blob %s: %s", b, err.Error())
-			}
-		}
-	}
-	done <- true
-}

cmd/peer.go

@@ -7,7 +7,7 @@ import (
 	"syscall"
 
 	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/server/peer"
+	"github.com/lbryio/reflector.go/peer"
 	"github.com/lbryio/reflector.go/store"
 
 	log "github.com/sirupsen/logrus"
@@ -33,13 +33,11 @@ func peerCmd(cmd *cobra.Command, args []string) {
 	peerServer := peer.NewServer(s3)
 
 	if !peerNoDB {
-		db := &db.SQL{
+		db := new(db.SQL)
-			LogQueries: log.GetLevel() == log.DebugLevel,
-		}
 		err = db.Connect(globalConfig.DBConn)
 		checkErr(err)
 
-		combo := store.NewDBBackedStore(s3, db, false)
+		combo := store.NewDBBackedStore(s3, db)
 		peerServer = peer.NewServer(combo)
 	}
 

cmd/populatedb.go

@@ -1,51 +0,0 @@
-package cmd
-
-import (
-	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/meta"
-	"github.com/lbryio/reflector.go/store/speedwalk"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-var (
-	diskStorePath string
-)
-
-func init() {
-	var cmd = &cobra.Command{
-		Use:   "populate-db",
-		Short: "populate local database with blobs from a disk storage",
-		Run:   populateDbCmd,
-	}
-	cmd.Flags().StringVar(&diskStorePath, "store-path", "",
-		"path of the store where all blobs are cached")
-	rootCmd.AddCommand(cmd)
-}
-
-func populateDbCmd(cmd *cobra.Command, args []string) {
-	log.Printf("reflector %s", meta.VersionString())
-	if diskStorePath == "" {
-		log.Fatal("store-path must be defined")
-	}
-	localDb := &db.SQL{
-		SoftDelete:  true,
-		TrackAccess: db.TrackAccessBlobs,
-		LogQueries:  log.GetLevel() == log.DebugLevel,
-	}
-	err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
-	if err != nil {
-		log.Fatal(err)
-	}
-	blobs, err := speedwalk.AllFiles(diskStorePath, true)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = localDb.AddBlobs(blobs)
-	if err != nil {
-		log.Errorf("error while storing to db: %s", errors.FullTrace(err))
-	}
-}

cmd/reflector.go

@@ -8,63 +8,34 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/lbryio/reflector.go/db"
 	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/meta"
+	"github.com/lbryio/reflector.go/peer"
+	"github.com/lbryio/reflector.go/peer/http3"
 	"github.com/lbryio/reflector.go/reflector"
-	"github.com/lbryio/reflector.go/server/http"
-	"github.com/lbryio/reflector.go/server/http3"
-	"github.com/lbryio/reflector.go/server/peer"
 	"github.com/lbryio/reflector.go/store"
 
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-	"github.com/lbryio/lbry.go/v2/stream"
-
-	"github.com/c2h5oh/datasize"
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cast"
 	"github.com/spf13/cobra"
 )
 
 var (
-	//port configuration
+	tcpPeerPort           int
-	tcpPeerPort   int
+	http3PeerPort         int
-	http3PeerPort int
+	receiverPort          int
-	httpPeerPort  int
+	metricsPort           int
-	receiverPort  int
+	disableUploads        bool
-	metricsPort   int
+	disableBlocklist      bool
-
+	proxyAddress          string
-	//flags configuration
+	proxyPort             string
-	disableUploads   bool
+	proxyProtocol         string
-	disableBlocklist bool
+	useDB                 bool
-	useDB            bool
+	cloudFrontEndpoint    string
-
+	reflectorCmdDiskCache string
-	//upstream configuration
+	reflectorCmdMemCache  int
-	upstreamReflector string
-	upstreamProtocol  string
-	upstreamEdgeToken string
-
-	//downstream configuration
-	requestQueueSize int
-
-	//upstream edge configuration (to "cold" storage)
-	originEndpoint         string
-	originEndpointFallback string
-
-	//cache configuration
-	diskCache          string
-	secondaryDiskCache string
-	memCache           int
 )
-var cacheManagers = []string{"localdb", "lfu", "arc", "lru", "simple"}
-
-var cacheMangerToGcache = map[string]store.EvictionStrategy{
-	"lfu":    store.LFU,
-	"arc":    store.ARC,
-	"lru":    store.LRU,
-	"simple": store.SIMPLE,
-}
 
 func init() {
 	var cmd = &cobra.Command{
@@ -72,30 +43,20 @@ func init() {
 		Short: "Run reflector server",
 		Run:   reflectorCmd,
 	}
-
+	cmd.Flags().StringVar(&proxyAddress, "proxy-address", "", "address of another reflector server where blobs are fetched from")
-	cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from for the TCP (LBRY) protocol")
+	cmd.Flags().StringVar(&proxyPort, "proxy-port", "5567", "port of another reflector server where blobs are fetched from")
+	cmd.Flags().StringVar(&proxyProtocol, "proxy-protocol", "http3", "protocol used to fetch blobs from another reflector server (tcp/http3)")
+	cmd.Flags().StringVar(&cloudFrontEndpoint, "cloudfront-endpoint", "", "CloudFront edge endpoint for standard HTTP retrieval")
+	cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from")
 	cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
-	cmd.Flags().IntVar(&httpPeerPort, "http-peer-port", 5569, "The port reflector will distribute content from over HTTP protocol")
 	cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
-	cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
+	cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics")
-
 	cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
 	cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
-	cmd.Flags().BoolVar(&useDB, "use-db", true, "Whether to connect to the reflector db or not")
+	cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not")
-
+	cmd.Flags().StringVar(&reflectorCmdDiskCache, "disk-cache", "",
-	cmd.Flags().StringVar(&upstreamReflector, "upstream-reflector", "", "host:port of a reflector server where blobs are fetched from")
+		"enable disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'")
-	cmd.Flags().StringVar(&upstreamProtocol, "upstream-protocol", "http", "protocol used to fetch blobs from another upstream reflector server (tcp/http3/http)")
+	cmd.Flags().IntVar(&reflectorCmdMemCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
-	cmd.Flags().StringVar(&upstreamEdgeToken, "upstream-edge-token", "", "token used to retrieve/authenticate protected content")
-
-	cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests from downstream should be handled at once (the rest will wait)")
-
-	cmd.Flags().StringVar(&originEndpoint, "origin-endpoint", "", "HTTP edge endpoint for standard HTTP retrieval")
-	cmd.Flags().StringVar(&originEndpointFallback, "origin-endpoint-fallback", "", "HTTP edge endpoint for standard HTTP retrieval if first origin fails")
-
-	cmd.Flags().StringVar(&diskCache, "disk-cache", "100GB:/tmp/downloaded_blobs:localdb", "Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru)")
-	cmd.Flags().StringVar(&secondaryDiskCache, "optional-disk-cache", "", "Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru) (this would get hit before the one specified in disk-cache)")
-	cmd.Flags().IntVar(&memCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
-
 	rootCmd.AddCommand(cmd)
 }
 
@@ -103,11 +64,11 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
 	log.Printf("reflector %s", meta.VersionString())
 
 	// the blocklist logic requires the db backed store to be the outer-most store
-	underlyingStore := initStores()
+	underlyingStore := setupStore()
-	underlyingStoreWithCaches, cleanerStopper := initCaches(underlyingStore)
+	outerStore := wrapWithCache(underlyingStore)
 
 	if !disableUploads {
-		reflectorServer := reflector.NewServer(underlyingStore, underlyingStoreWithCaches)
+		reflectorServer := reflector.NewServer(underlyingStore)
 		reflectorServer.Timeout = 3 * time.Minute
 		reflectorServer.EnableBlocklist = !disableBlocklist
 
@@ -118,264 +79,117 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
 		defer reflectorServer.Shutdown()
 	}
 
-	peerServer := peer.NewServer(underlyingStoreWithCaches)
+	peerServer := peer.NewServer(outerStore)
 	err := peerServer.Start(":" + strconv.Itoa(tcpPeerPort))
 	if err != nil {
 		log.Fatal(err)
 	}
 	defer peerServer.Shutdown()
 
-	http3PeerServer := http3.NewServer(underlyingStoreWithCaches, requestQueueSize)
+	http3PeerServer := http3.NewServer(outerStore)
 	err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
 	if err != nil {
 		log.Fatal(err)
 	}
 	defer http3PeerServer.Shutdown()
 
-	httpServer := http.NewServer(store.WithSingleFlight("sf-http", underlyingStoreWithCaches), requestQueueSize, upstreamEdgeToken)
-	err = httpServer.Start(":" + strconv.Itoa(httpPeerPort))
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer httpServer.Shutdown()
-
 	metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
 	metricsServer.Start()
 	defer metricsServer.Shutdown()
-	defer underlyingStoreWithCaches.Shutdown()
-	defer underlyingStore.Shutdown() //do we actually need this? Oo
 
 	interruptChan := make(chan os.Signal, 1)
 	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
 	<-interruptChan
 	// deferred shutdowns happen now
-	cleanerStopper.StopAndWait()
 }
 
-func initUpstreamStore() store.BlobStore {
+func setupStore() store.BlobStore {
-	var s store.BlobStore
-	if upstreamReflector == "" {
-		return nil
-	}
-	switch upstreamProtocol {
-	case "tcp":
-		s = peer.NewStore(peer.StoreOpts{
-			Address: upstreamReflector,
-			Timeout: 30 * time.Second,
-		})
-	case "http3":
-		s = http3.NewStore(http3.StoreOpts{
-			Address: upstreamReflector,
-			Timeout: 30 * time.Second,
-		})
-	case "http":
-		s = store.NewHttpStore(upstreamReflector, upstreamEdgeToken)
-	default:
-		log.Fatalf("protocol is not recognized: %s", upstreamProtocol)
-	}
-	return s
-}
-func initEdgeStore() store.BlobStore {
-	var s3Store *store.S3Store
 	var s store.BlobStore
 
-	if conf != "none" {
+	if proxyAddress != "" {
-		s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
+		switch proxyProtocol {
-	}
+		case "tcp":
-	if originEndpointFallback != "" && originEndpoint != "" {
+			s = peer.NewStore(peer.StoreOpts{
-		ittt := store.NewITTTStore(store.NewCloudFrontROStore(originEndpoint), store.NewCloudFrontROStore(originEndpointFallback))
+				Address: proxyAddress + ":" + proxyPort,
-		if s3Store != nil {
+				Timeout: 30 * time.Second,
-			s = store.NewCloudFrontRWStore(ittt, s3Store)
+			})
-		} else {
+		case "http3":
-			s = ittt
+			s = http3.NewStore(http3.StoreOpts{
+				Address: proxyAddress + ":" + proxyPort,
+				Timeout: 30 * time.Second,
+			})
+		default:
+			log.Fatalf("protocol is not recognized: %s", proxyProtocol)
 		}
-	} else if s3Store != nil {
-		s = s3Store
 	} else {
-		log.Fatalf("this configuration does not include a valid upstream source")
+		s3Store := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
-	}
+		if cloudFrontEndpoint != "" {
-	return s
+			s = store.NewCloudFrontRWStore(store.NewCloudFrontROStore(cloudFrontEndpoint), s3Store)
-}
+		} else {
-
+			s = s3Store
-func initDBStore(s store.BlobStore) store.BlobStore {
-	if useDB {
-		dbInst := &db.SQL{
-			TrackAccess: db.TrackAccessStreams,
-			LogQueries:  log.GetLevel() == log.DebugLevel,
 		}
-		err := dbInst.Connect(globalConfig.DBConn)
+	}
+
+	if useDB {
+		db := new(db.SQL)
+		db.TrackAccessTime = true
+		err := db.Connect(globalConfig.DBConn)
 		if err != nil {
 			log.Fatal(err)
 		}
-		s = store.NewDBBackedStore(s, dbInst, false)
+
+		s = store.NewDBBackedStore(s, db)
 	}
+
 	return s
 }
 
-func initStores() store.BlobStore {
+func wrapWithCache(s store.BlobStore) store.BlobStore {
-	s := initUpstreamStore()
+	wrapped := s
-	if s == nil {
-		s = initEdgeStore()
-	}
-	s = initDBStore(s)
-	return s
-}
 
-// initCaches returns a store wrapped with caches and a stop group to execute a clean shutdown
+	diskCacheMaxSize, diskCachePath := diskCacheParams()
-func initCaches(s store.BlobStore) (store.BlobStore, *stop.Group) {
+	if diskCacheMaxSize > 0 {
-	stopper := stop.New()
+		err := os.MkdirAll(diskCachePath, os.ModePerm)
-	diskStore := initDiskStore(s, diskCache, stopper)
+		if err != nil {
-	finalStore := initDiskStore(diskStore, secondaryDiskCache, stopper)
+			log.Fatal(err)
-	stop.New()
+		}
-	if memCache > 0 {
+		wrapped = store.NewCachingStore(
-		finalStore = store.NewCachingStore(
 			"reflector",
-			finalStore,
+			wrapped,
-			store.NewGcacheStore("mem", store.NewMemStore(), memCache, store.LRU),
+			store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
 		)
 	}
-	return finalStore, stopper
-}
 
-func initDiskStore(upstreamStore store.BlobStore, diskParams string, stopper *stop.Group) store.BlobStore {
+	if reflectorCmdMemCache > 0 {
-	diskCacheMaxSize, diskCachePath, cacheManager := diskCacheParams(diskParams)
+		wrapped = store.NewCachingStore(
-	//we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte
+			"reflector",
-	// so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup
+			wrapped,
-	realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
+			store.NewLRUStore("peer_server", store.NewMemStore(), reflectorCmdMemCache),
-	if diskCacheMaxSize == 0 {
+		)
-		return upstreamStore
-	}
-	err := os.MkdirAll(diskCachePath, os.ModePerm)
-	if err != nil {
-		log.Fatal(err)
 	}
 
-	diskStore := store.NewDiskStore(diskCachePath, 2)
-	var unwrappedStore store.BlobStore
-	cleanerStopper := stop.New(stopper)
-
-	if cacheManager == "localdb" {
-		localDb := &db.SQL{
-			SoftDelete:  true,
-			TrackAccess: db.TrackAccessBlobs,
-			LogQueries:  log.GetLevel() == log.DebugLevel,
-		}
-		err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
-		if err != nil {
-			log.Fatal(err)
-		}
-		unwrappedStore = store.NewDBBackedStore(diskStore, localDb, true)
-		go cleanOldestBlobs(int(realCacheSize), localDb, unwrappedStore, cleanerStopper)
-	} else {
-		unwrappedStore = store.NewGcacheStore("nvme", store.NewDiskStore(diskCachePath, 2), int(realCacheSize), cacheMangerToGcache[cacheManager])
-	}
-
-	wrapped := store.NewCachingStore(
-		"reflector",
-		upstreamStore,
-		unwrappedStore,
-	)
 	return wrapped
 }
 
-func diskCacheParams(diskParams string) (int, string, string) {
+func diskCacheParams() (int, string) {
-	if diskParams == "" {
+	if reflectorCmdDiskCache == "" {
-		return 0, "", ""
+		return 0, ""
 	}
 
-	parts := strings.Split(diskParams, ":")
+	parts := strings.Split(reflectorCmdDiskCache, ":")
-	if len(parts) != 3 {
+	if len(parts) != 2 {
-		log.Fatalf("%s does is formatted incorrectly. Expected format: 'sizeGB:CACHE_PATH:cachemanager' for example: '100GB:/tmp/downloaded_blobs:localdb'", diskParams)
+		log.Fatalf("--disk-cache must be a number, followed by ':', followed by a string")
 	}
 
-	diskCacheSize := parts[0]
+	maxSize := cast.ToInt(parts[0])
-	path := parts[1]
-	cacheManager := parts[2]
-
-	if len(path) == 0 || path[0] != '/' {
-		log.Fatalf("disk cache paths must start with '/'")
-	}
-
-	if !util.InSlice(cacheManager, cacheManagers) {
-		log.Fatalf("specified cache manager '%s' is not supported. Use one of the following: %v", cacheManager, cacheManagers)
-	}
-
-	var maxSize datasize.ByteSize
-	err := maxSize.UnmarshalText([]byte(diskCacheSize))
-	if err != nil {
-		log.Fatal(err)
-	}
 	if maxSize <= 0 {
-		log.Fatal("disk cache size must be more than 0")
+		log.Fatalf("--disk-cache max size must be more than 0")
-	}
-	return int(maxSize), path, cacheManager
-}
-
-func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
-	// this is so that it runs on startup without having to wait for 10 minutes
-	err := doClean(maxItems, db, store, stopper)
-	if err != nil {
-		log.Error(errors.FullTrace(err))
-	}
-	const cleanupInterval = 10 * time.Minute
-	for {
-		select {
-		case <-stopper.Ch():
-			log.Infoln("stopping self cleanup")
-			return
-		case <-time.After(cleanupInterval):
-			err := doClean(maxItems, db, store, stopper)
-			if err != nil {
-				log.Error(errors.FullTrace(err))
-			}
-		}
-	}
-}
-
-func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error {
-	blobsCount, err := db.Count()
-	if err != nil {
-		return err
 	}
 
-	if blobsCount >= maxItems {
+	path := parts[1]
-		itemsToDelete := blobsCount / 10
+	if len(path) == 0 || path[0] != '/' {
-		blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete)
+		log.Fatalf("--disk-cache path must start with '/'")
-		if err != nil {
-			return err
-		}
-		blobsChan := make(chan string, len(blobs))
-		wg := &stop.Group{}
-		go func() {
-			for _, hash := range blobs {
-				select {
-				case <-stopper.Ch():
-					return
-				default:
-				}
-				blobsChan <- hash
-			}
-			close(blobsChan)
-		}()
-		for i := 0; i < 3; i++ {
-			wg.Add(1)
-			go func() {
-				defer wg.Done()
-				for h := range blobsChan {
-					select {
-					case <-stopper.Ch():
-						return
-					default:
-					}
-					err = store.Delete(h)
-					if err != nil {
-						log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
-						continue
-					}
-				}
-			}()
-		}
-		wg.Wait()
 	}
-	return nil
+
+	return maxSize, path
 }

cmd/resolve.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 
@@ -29,10 +28,7 @@ func resolveCmd(cmd *cobra.Command, args []string) {
 	err := node.Connect([]string{addr}, nil)
 	checkErr(err)
 
-	output, err := node.Resolve(url)
+	claim, _, err := node.ResolveToClaim(url)
-	checkErr(err)
-
-	claim, err := node.GetClaimInTx(hex.EncodeToString(rev(output.GetTxHash())), int(output.GetNout()))
 	checkErr(err)
 
 	jsonClaim, err := json.MarshalIndent(claim, "", "  ")
@@ -40,11 +36,3 @@ func resolveCmd(cmd *cobra.Command, args []string) {
 
 	fmt.Println(string(jsonClaim))
 }
-
-func rev(b []byte) []byte {
-	r := make([]byte, len(b))
-	for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 {
-		r[left], r[right] = b[right], b[left]
-	}
-	return r
-}

cmd/root.go

@@ -2,14 +2,14 @@ package cmd
 
 import (
 	"encoding/json"
+	"io/ioutil"
 	"os"
 	"strings"
 
-	"github.com/lbryio/reflector.go/updater"
-
 	"github.com/lbryio/lbry.go/v2/dht"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/reflector.go/updater"
 
 	"github.com/johntdyer/slackrus"
 	"github.com/sirupsen/logrus"
@@ -24,7 +24,6 @@ type Config struct {
 	BucketName   string `json:"bucket_name"`
 	DBConn       string `json:"db_conn"`
 	SlackHookURL string `json:"slack_hook_url"`
-	SlackChannel string `json:"slack_channel"`
 	UpdateBinURL string `json:"update_bin_url"`
 	UpdateCmd    string `json:"update_cmd"`
 }
@@ -102,7 +101,7 @@ func preRun(cmd *cobra.Command, args []string) {
 		hook := &slackrus.SlackrusHook{
 			HookURL:        globalConfig.SlackHookURL,
 			AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
-			Channel:        globalConfig.SlackChannel,
+			Channel:        "#reflector-logs",
 			//IconEmoji:      ":ghost:",
 			//Username:       "reflector.go",
 		}
@@ -141,7 +140,7 @@ func argFuncs(funcs ...cobra.PositionalArgs) cobra.PositionalArgs {
 func loadConfig(path string) (Config, error) {
 	var c Config
 
-	raw, err := os.ReadFile(path)
+	raw, err := ioutil.ReadFile(path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return c, errors.Err("config file not found")
@@ -164,9 +163,3 @@ func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
 	checkErr(err)
 	return v
 }
-
-//func mustGetFlagBool(cmd *cobra.Command, name string) bool {
-//	v, err := cmd.Flags().GetBool(name)
-//	checkErr(err)
-//	return v
-//}

cmd/send.go

@@ -1,158 +0,0 @@
-package cmd
-
-import (
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"io"
-	"os"
-	"os/signal"
-	"path"
-	"syscall"
-
-	"github.com/lbryio/reflector.go/reflector"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
-
-	"github.com/spf13/cobra"
-)
-
-func init() {
-	var cmd = &cobra.Command{
-		Use:   "send ADDRESS:PORT PATH",
-		Short: "Send a file to a reflector",
-		Args:  cobra.ExactArgs(2),
-		Run:   sendCmd,
-	}
-	cmd.PersistentFlags().String("sd-cache", "", "path to dir where sd blobs will be cached")
-	rootCmd.AddCommand(cmd)
-}
-
-// todo: if retrying a large file is slow, we can add the ability to seek ahead in the file so we're not
-// re-uploading blobs that already exist
-
-var hackyReflector reflector.Client
-
-func sendCmd(cmd *cobra.Command, args []string) {
-	reflectorAddress := args[0]
-	err := hackyReflector.Connect(reflectorAddress)
-	checkErr(err)
-	defer func() { _ = hackyReflector.Close() }()
-
-	filePath := args[1]
-	file, err := os.Open(filePath)
-	checkErr(err)
-	defer func() { _ = file.Close() }()
-	sdCachePath := ""
-	sdCacheDir := mustGetFlagString(cmd, "sd-cache")
-	if sdCacheDir != "" {
-		if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) {
-			err = os.MkdirAll(sdCacheDir, 0777)
-			checkErr(err)
-		}
-		sdCachePath = path.Join(sdCacheDir, filePath+".sdblob")
-	}
-
-	var enc *stream.Encoder
-
-	if sdCachePath != "" {
-		if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
-			sdBlob, err := os.ReadFile(sdCachePath)
-			checkErr(err)
-			cachedSDBlob := &stream.SDBlob{}
-			err = cachedSDBlob.FromBlob(sdBlob)
-			checkErr(err)
-			enc = stream.NewEncoderFromSD(file, cachedSDBlob)
-		}
-	}
-	if enc == nil {
-		enc = stream.NewEncoder(file)
-	}
-
-	exitCode := 0
-
-	var killed bool
-	interruptChan := make(chan os.Signal, 1)
-	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		sig := <-interruptChan
-		fmt.Printf("caught %s, exiting...\n", sig.String())
-		killed = true
-		exitCode = 1
-	}()
-
-	for {
-		if killed {
-			break
-		}
-
-		b, err := enc.Next()
-		if errors.Is(err, io.EOF) {
-			break
-		}
-		if err != nil {
-			fmt.Printf("error reading next blob: %v\n", err)
-			exitCode = 1
-			break
-		}
-
-		err = hackyReflect(b, false)
-		if err != nil {
-			fmt.Printf("error reflecting blob %s: %v\n", b.HashHex()[:8], err)
-			exitCode = 1
-			break
-		}
-	}
-
-	sd := enc.SDBlob()
-	//sd.StreamName = filepath.Base(filePath)
-	//sd.SuggestedFileName = filepath.Base(filePath)
-	err = os.WriteFile(sdCachePath, sd.ToBlob(), 0666)
-	if err != nil {
-		fmt.Printf("error saving sd blob: %v\n", err)
-		fmt.Println(sd.ToJson())
-		exitCode = 1
-	}
-
-	if killed {
-		os.Exit(exitCode)
-	}
-
-	if reflectorAddress != "" {
-		err = hackyReflect(sd.ToBlob(), true)
-		if err != nil {
-			fmt.Printf("error reflecting sd blob %s: %v\n", sd.HashHex()[:8], err)
-			exitCode = 1
-		}
-	}
-
-	ret := struct {
-		SDHash     string `json:"sd_hash"`
-		SourceHash string `json:"source_hash"`
-	}{
-		SDHash:     sd.HashHex(),
-		SourceHash: hex.EncodeToString(enc.SourceHash()),
-	}
-
-	j, err := json.MarshalIndent(ret, "", "  ")
-	checkErr(err)
-	fmt.Println(string(j))
-	os.Exit(exitCode)
-}
-
-func hackyReflect(b stream.Blob, sd bool) error {
-	var err error
-	if sd {
-		err = hackyReflector.SendSDBlob(b)
-	} else {
-		err = hackyReflector.SendBlob(b)
-	}
-
-	if errors.Is(err, reflector.ErrBlobExists) {
-		//fmt.Printf("%s already reflected\n", b.HashHex()[:8])
-		return nil
-	}
-
-	return err
-}

cmd/sendblob.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"crypto/rand"
+	"io/ioutil"
 	"os"
 
 	"github.com/lbryio/reflector.go/reflector"
@@ -51,8 +52,9 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
 
 	file, err := os.Open(path)
 	checkErr(err)
-	defer func() { _ = file.Close() }()
+	data, err := ioutil.ReadAll(file)
-	s, err := stream.New(file)
+	checkErr(err)
+	s, err := stream.New(data)
 	checkErr(err)
 
 	sdBlob := &stream.SDBlob{}

cmd/start.go

@@ -7,15 +7,14 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/lbryio/reflector.go/cluster"
-	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/prism"
-	"github.com/lbryio/reflector.go/reflector"
-	"github.com/lbryio/reflector.go/server/peer"
-	"github.com/lbryio/reflector.go/store"
-
 	"github.com/lbryio/lbry.go/v2/dht"
 	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/reflector.go/cluster"
+	"github.com/lbryio/reflector.go/db"
+	"github.com/lbryio/reflector.go/peer"
+	"github.com/lbryio/reflector.go/prism"
+	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/store"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -53,13 +52,11 @@ func init() {
 }
 
 func startCmd(cmd *cobra.Command, args []string) {
-	db := &db.SQL{
+	db := new(db.SQL)
-		LogQueries: log.GetLevel() == log.DebugLevel,
-	}
 	err := db.Connect(globalConfig.DBConn)
 	checkErr(err)
 	s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
-	comboStore := store.NewDBBackedStore(s3, db, false)
+	comboStore := store.NewDBBackedStore(s3, db)
 
 	conf := prism.DefaultConf()
 

cmd/test.go

@@ -9,8 +9,8 @@ import (
 	"time"
 
 	"github.com/lbryio/reflector.go/meta"
+	"github.com/lbryio/reflector.go/peer"
 	"github.com/lbryio/reflector.go/reflector"
-	"github.com/lbryio/reflector.go/server/peer"
 	"github.com/lbryio/reflector.go/store"
 
 	log "github.com/sirupsen/logrus"
@@ -31,7 +31,7 @@ func testCmd(cmd *cobra.Command, args []string) {
 
 	memStore := store.NewMemStore()
 
-	reflectorServer := reflector.NewServer(memStore, memStore)
+	reflectorServer := reflector.NewServer(memStore)
 	reflectorServer.Timeout = 3 * time.Minute
 
 	err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))

cmd/upload.go

@@ -9,7 +9,6 @@ import (
 	"github.com/lbryio/reflector.go/reflector"
 	"github.com/lbryio/reflector.go/store"
 
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -31,15 +30,13 @@ func init() {
 }
 
 func uploadCmd(cmd *cobra.Command, args []string) {
-	db := &db.SQL{
+	db := new(db.SQL)
-		LogQueries: log.GetLevel() == log.DebugLevel,
-	}
 	err := db.Connect(globalConfig.DBConn)
 	checkErr(err)
 
 	st := store.NewDBBackedStore(
 		store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
-		db, false)
+		db)
 
 	uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
 

cmd/version.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/lbryio/reflector.go/meta"
-
 	"github.com/spf13/cobra"
 )
 
@@ -18,5 +17,5 @@ func init() {
 }
 
 func versionCmd(cmd *cobra.Command, args []string) {
-	fmt.Println(meta.FullName())
+	fmt.Println(meta.VersionString())
 }

db/db.go

@@ -3,22 +3,16 @@ package db
 import (
 	"context"
 	"database/sql"
-	"fmt"
-	"runtime"
-	"strings"
 	"time"
 
 	"github.com/lbryio/lbry.go/v2/dht/bits"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	qt "github.com/lbryio/lbry.go/v2/extras/query"
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-	"github.com/lbryio/lbry.go/v2/stream"
 
 	"github.com/go-sql-driver/mysql"
 	_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
 	log "github.com/sirupsen/logrus"
-	"github.com/volatiletech/null/v8"
+	"github.com/volatiletech/null"
-	"go.uber.org/atomic"
 )
 
 // SdBlob is a special blob that contains information on the rest of the blobs in the stream
@@ -36,38 +30,19 @@ type SdBlob struct {
 	StreamHash        string `json:"stream_hash"`
 }
 
-type trackAccess int
-
-const (
-	TrackAccessNone    trackAccess = iota // Don't track accesses
-	TrackAccessStreams                    // Track accesses at the stream level
-	TrackAccessBlobs                      // Track accesses at the blob level
-)
-
 // SQL implements the DB interface
 type SQL struct {
 	conn *sql.DB
 
-	// Track the approx last time a blob or stream was accessed
+	TrackAccessTime bool
-	TrackAccess trackAccess
-
-	// Instead of deleting a blob, marked it as not stored in the db
-	SoftDelete bool
-
-	// Log executed queries. qt.InterpolateParams is cpu-heavy. This avoids that call if not needed.
-	LogQueries bool
 }
 
-func (s SQL) logQuery(query string, args ...interface{}) {
+func logQuery(query string, args ...interface{}) {
-	if !s.LogQueries {
+	s, err := qt.InterpolateParams(query, args...)
-		return
-	}
-
-	qStr, err := qt.InterpolateParams(query, args...)
 	if err != nil {
 		log.Errorln(err)
 	} else {
-		log.Debugln(qStr)
+		log.Debugln(s)
 	}
 }
 
@@ -98,97 +73,16 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
 	return err
 }
 
-//AddBlobs adds blobs to the database.
-func (s *SQL) AddBlobs(hash []string) error {
-	if s.conn == nil {
-		return errors.Err("not connected")
-	}
-
-	batch := 10000
-	totalBlobs := int64(len(hash))
-	work := make(chan []string, 1000)
-	stopper := stop.New()
-	var totalInserted atomic.Int64
-	start := time.Now()
-
-	go func() {
-		for i := 0; i < len(hash); i += batch {
-			j := i + batch
-			if j > len(hash) {
-				j = len(hash)
-			}
-			work <- hash[i:j]
-		}
-		log.Infof("done loading %d hashes in the work queue", len(hash))
-		close(work)
-	}()
-
-	for i := 0; i < runtime.NumCPU(); i++ {
-		stopper.Add(1)
-		go func(worker int) {
-			log.Infof("starting worker %d", worker)
-			defer stopper.Done()
-			for hashes := range work {
-				inserted := totalInserted.Load()
-				remaining := totalBlobs - inserted
-				if inserted > 0 {
-					timePerBlob := time.Since(start).Microseconds() / inserted
-					remainingTime := time.Duration(remaining*timePerBlob) * time.Microsecond
-					log.Infof("[T%d] processing batch of %d items. ETA: %s", worker, len(hashes), remainingTime.String())
-				}
-				err := s.insertBlobs(hashes) // Process the batch.
-				if err != nil {
-					log.Errorf("error while inserting batch: %s", errors.FullTrace(err))
-				}
-				totalInserted.Add(int64(len(hashes)))
-			}
-		}(i)
-	}
-
-	stopper.Wait()
-	return nil
-}
-
-func (s *SQL) insertBlobs(hashes []string) error {
-	var (
-		q string
-		//args []interface{}
-	)
-	dayAgo := time.Now().AddDate(0, 0, -1).Format("2006-01-02 15:04:05")
-	q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
-	for _, hash := range hashes {
-		// prepared statements slow everything down by a lot due to reflection
-		// for this specific instance we'll go ahead and hardcode the query to make it go faster
-		q += fmt.Sprintf("('%s',1,%d,'%s'),", hash, stream.MaxBlobSize, dayAgo)
-		//args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
-	}
-	q = strings.TrimSuffix(q, ",")
-	_, err := s.exec(q)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
 	if length <= 0 {
 		return 0, errors.Err("length must be positive")
 	}
 
-	var (
+	args := []interface{}{hash, isStored, length}
-		q    string
+	blobID, err := s.exec(
-		args []interface{}
+		"INSERT INTO blob_ (hash, is_stored, length) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
+		args...,
 	)
-	if s.TrackAccess == TrackAccessBlobs {
-		args = []interface{}{hash, isStored, length, time.Now()}
-		q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
-	} else {
-		args = []interface{}{hash, isStored, length}
-		q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))"
-	}
-
-	blobID, err := s.exec(q, args...)
 	if err != nil {
 		return 0, err
 	}
@@ -201,33 +95,17 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
 		if blobID == 0 {
 			return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
 		}
-
-		if s.TrackAccess == TrackAccessBlobs {
-			err := s.touchBlobs([]uint64{uint64(blobID)})
-			if err != nil {
-				return 0, errors.Err(err)
-			}
-		}
 	}
 
 	return blobID, nil
 }
 
 func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
-	var (
+	args := []interface{}{hash, sdBlobID, time.Now()}
-		q    string
+	streamID, err := s.exec(
-		args []interface{}
+		"INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES ("+qt.Qs(len(args))+")",
+		args...,
 	)
-
-	if s.TrackAccess == TrackAccessStreams {
-		args = []interface{}{hash, sdBlobID, time.Now()}
-		q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
-	} else {
-		args = []interface{}{hash, sdBlobID}
-		q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")"
-	}
-
-	streamID, err := s.exec(q, args...)
 	if err != nil {
 		return 0, errors.Err(err)
 	}
@@ -241,8 +119,8 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
 			return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
 		}
 
-		if s.TrackAccess == TrackAccessStreams {
+		if s.TrackAccessTime {
-			err := s.touchStreams([]uint64{uint64(streamID)})
+			err := s.touch([]uint64{uint64(streamID)})
 			if err != nil {
 				return 0, errors.Err(err)
 			}
@@ -252,8 +130,8 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
 }
 
 // HasBlob checks if the database contains the blob information.
-func (s *SQL) HasBlob(hash string, touch bool) (bool, error) {
+func (s *SQL) HasBlob(hash string) (bool, error) {
-	exists, err := s.HasBlobs([]string{hash}, touch)
+	exists, err := s.HasBlobs([]string{hash})
 	if err != nil {
 		return false, err
 	}
@@ -261,39 +139,13 @@ func (s *SQL) HasBlob(hash string, touch bool) (bool, error) {
 }
 
 // HasBlobs checks if the database contains the set of blobs and returns a bool map.
-func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
+func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
-	exists, idsNeedingTouch, err := s.hasBlobs(hashes)
+	exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
-
+	s.touch(streamsNeedingTouch)
-	if touch {
-		if s.TrackAccess == TrackAccessBlobs {
-			_ = s.touchBlobs(idsNeedingTouch)
-		} else if s.TrackAccess == TrackAccessStreams {
-			_ = s.touchStreams(idsNeedingTouch)
-		}
-	}
-
 	return exists, err
 }
 
-func (s *SQL) touchBlobs(blobIDs []uint64) error {
+func (s *SQL) touch(streamIDs []uint64) error {
-	if len(blobIDs) == 0 {
-		return nil
-	}
-
-	query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
-	args := make([]interface{}, len(blobIDs)+1)
-	args[0] = time.Now()
-	for i := range blobIDs {
-		args[i+1] = blobIDs[i]
-	}
-
-	startTime := time.Now()
-	_, err := s.exec(query, args...)
-	log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
-	return errors.Err(err)
-}
-
-func (s *SQL) touchStreams(streamIDs []uint64) error {
 	if len(streamIDs) == 0 {
 		return nil
 	}
@@ -307,7 +159,7 @@ func (s *SQL) touchStreams(streamIDs []uint64) error {
 
 	startTime := time.Now()
 	_, err := s.exec(query, args...)
-	log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime))
+	log.Debugf("stream access query touched %d streams and took %s", len(streamIDs), time.Since(startTime))
 	return errors.Err(err)
 }
 
@@ -318,15 +170,14 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
 
 	var (
 		hash           string
-		blobID         uint64
+		streamID       uint64
-		streamID       null.Uint64
 		lastAccessedAt null.Time
 	)
 
 	var needsTouch []uint64
 	exists := make(map[string]bool)
 
-	touchDeadline := time.Now().Add(-6 * time.Hour) // touch blob if last accessed before this time
+	touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
 	maxBatchSize := 10000
 	doneIndex := 0
 
@@ -338,29 +189,20 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
 		log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
 		batch := hashes[doneIndex:sliceEnd]
 
-		var query string
+		// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
-		if s.TrackAccess == TrackAccessBlobs {
+
-			query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
+		query := `SELECT b.hash, s.id, s.last_accessed_at
-FROM blob_ b
-WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
-		} else if s.TrackAccess == TrackAccessStreams {
-			query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
 FROM blob_ b
 LEFT JOIN stream_blob sb ON b.id = sb.blob_id
 INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
-WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
+WHERE b.is_stored = ? and b.hash IN (` + qt.Qs(len(batch)) + `)`
-		} else {
+		args := make([]interface{}, len(batch)+1)
-			query = `SELECT b.hash, b.id, NULL, NULL
+		args[0] = true
-FROM blob_ b
-WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
-		}
-
-		args := make([]interface{}, len(batch))
 		for i := range batch {
-			args[i] = batch[i]
+			args[i+1] = batch[i]
 		}
 
-		s.logQuery(query, args...)
+		logQuery(query, args...)
 
 		err := func() error {
 			startTime := time.Now()
@@ -372,17 +214,13 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
 			defer closeRows(rows)
 
 			for rows.Next() {
-				err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
+				err := rows.Scan(&hash, &streamID, &lastAccessedAt)
 				if err != nil {
 					return errors.Err(err)
 				}
 				exists[hash] = true
-				if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
+				if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
-					if s.TrackAccess == TrackAccessBlobs {
+					needsTouch = append(needsTouch, streamID)
-						needsTouch = append(needsTouch, blobID)
-					} else if s.TrackAccess == TrackAccessStreams && !streamID.IsZero() {
-						needsTouch = append(needsTouch, streamID.Uint64)
-					}
 				}
 			}
 
@@ -402,14 +240,8 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
 	return exists, needsTouch, nil
 }
 
-// Delete will remove (or soft-delete) the blob from the db
+// Delete will remove the blob from the db
-// NOTE: If SoftDelete is enabled, streams will never be deleted
 func (s *SQL) Delete(hash string) error {
-	if s.SoftDelete {
-		_, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
-		return errors.Err(err)
-	}
-
 	_, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
 	if err != nil {
 		return errors.Err(err)
@@ -419,59 +251,11 @@ func (s *SQL) Delete(hash string) error {
 	return errors.Err(err)
 }
 
-//LeastRecentlyAccessedHashes gets the least recently accessed blobs
-func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
-	if s.conn == nil {
-		return nil, errors.Err("not connected")
-	}
-
-	if s.TrackAccess != TrackAccessBlobs {
-		return nil, errors.Err("blob access tracking is disabled")
-	}
-
-	query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?"
-	s.logQuery(query, maxBlobs)
-
-	rows, err := s.conn.Query(query, maxBlobs)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-	defer closeRows(rows)
-
-	blobs := make([]string, 0, maxBlobs)
-	for rows.Next() {
-		var hash string
-		err := rows.Scan(&hash)
-		if err != nil {
-			return nil, errors.Err(err)
-		}
-		blobs = append(blobs, hash)
-	}
-
-	return blobs, nil
-}
-
-func (s *SQL) Count() (int, error) {
-	if s.conn == nil {
-		return 0, errors.Err("not connected")
-	}
-
-	query := "SELECT count(id) from blob_"
-	if s.SoftDelete {
-		query += " where is_stored = 1"
-	}
-	s.logQuery(query)
-
-	var count int
-	err := s.conn.QueryRow(query).Scan(&count)
-	return count, errors.Err(err)
-}
-
 // Block will mark a blob as blocked
 func (s *SQL) Block(hash string) error {
 	query := "INSERT IGNORE INTO blocked SET hash = ?"
 	args := []interface{}{hash}
-	s.logQuery(query, args...)
+	logQuery(query, args...)
 	_, err := s.conn.Exec(query, args...)
 	return errors.Err(err)
 }
@@ -479,7 +263,7 @@ func (s *SQL) Block(hash string) error {
 // GetBlocked will return a list of blocked hashes
 func (s *SQL) GetBlocked() (map[string]bool, error) {
 	query := "SELECT hash FROM blocked"
-	s.logQuery(query)
+	logQuery(query)
 	rows, err := s.conn.Query(query)
 	if err != nil {
 		return nil, errors.Err(err)
@@ -522,7 +306,7 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
 	`
 	args := []interface{}{sdHash}
 
-	s.logQuery(query, args...)
+	logQuery(query, args...)
 
 	rows, err := s.conn.Query(query, args...)
 	if err != nil {
@@ -601,7 +385,7 @@ func (s *SQL) GetHashRange() (string, string, error) {
 
 	query := "SELECT MIN(hash), MAX(hash) from blob_"
 
-	s.logQuery(query)
+	logQuery(query)
 
 	err := s.conn.QueryRow(query).Scan(&min, &max)
 	return min, max, err
@@ -625,7 +409,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
 		query := "SELECT hash FROM blob_ WHERE hash >= ? AND hash <= ? AND is_stored = 1"
 		args := []interface{}{start.Hex(), end.Hex()}
 
-		s.logQuery(query, args...)
+		logQuery(query, args...)
 
 		rows, err := s.conn.Query(query, args...)
 		defer closeRows(rows)
@@ -706,7 +490,7 @@ func closeRows(rows *sql.Rows) {
 }
 
 func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
-	s.logQuery(query, args...)
+	logQuery(query, args...)
 	attempt, maxAttempts := 0, 3
 Retry:
 	attempt++
@@ -744,11 +528,8 @@ CREATE TABLE blob_ (
   hash char(96) NOT NULL,
   is_stored TINYINT(1) NOT NULL DEFAULT 0,
   length bigint(20) unsigned DEFAULT NULL,
-  last_accessed_at TIMESTAMP NULL DEFAULT NULL,
   PRIMARY KEY (id),
-  UNIQUE KEY blob_hash_idx (hash),
+  UNIQUE KEY blob_hash_idx (hash)
-  KEY `blob_last_accessed_idx` (`last_accessed_at`),
-  KEY `is_stored_idx` (`is_stored`)
 );
 
 CREATE TABLE stream (

go.mod

@@ -1,124 +1,46 @@
 module github.com/lbryio/reflector.go
 
-go 1.20
-
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
 
 require (
-	github.com/aws/aws-sdk-go v1.45.24
+	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
-	github.com/bluele/gcache v0.0.2
+	github.com/aws/aws-sdk-go v1.16.11
-	github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7
 	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
-	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
 	github.com/davecgh/go-spew v1.1.1
-	github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
+	github.com/go-sql-driver/mysql v1.4.1
-	github.com/gin-gonic/gin v1.9.1
+	github.com/golang/protobuf v1.4.2
-	github.com/go-sql-driver/mysql v1.7.1
+	github.com/google/btree v1.0.0 // indirect
-	github.com/gogo/protobuf v1.3.2
+	github.com/google/gops v0.3.7
-	github.com/golang/protobuf v1.5.3
+	github.com/gorilla/mux v1.7.4
-	github.com/google/gops v0.3.28
+	github.com/hashicorp/go-msgpack v0.5.5 // indirect
-	github.com/gorilla/mux v1.8.0
+	github.com/hashicorp/golang-lru v0.5.4
-	github.com/hashicorp/serf v0.10.1
+	github.com/hashicorp/memberlist v0.1.4 // indirect
+	github.com/hashicorp/serf v0.8.2
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
-	github.com/johntdyer/slackrus v0.0.0-20230315191314-80bc92dee4fc
+	github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07
-	github.com/karrick/godirwalk v1.17.0
+	github.com/karrick/godirwalk v1.16.1
-	github.com/lbryio/chainquery v1.9.1-0.20230515181855-2fcba3115cfe
+	github.com/lbryio/chainquery v1.9.0
-	github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629
+	github.com/lbryio/lbry.go v1.1.2 // indirect
-	github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
+	github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
-	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
+	github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec
-	github.com/prometheus/client_golang v1.16.0
+	github.com/lucas-clemente/quic-go v0.18.1
-	github.com/quic-go/quic-go v0.39.0
+	github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
-	github.com/sirupsen/logrus v1.9.3
+	github.com/prometheus/client_golang v0.9.2
-	github.com/spf13/cast v1.5.1
+	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/afero v1.4.1
-	github.com/stretchr/testify v1.8.4
+	github.com/spf13/cast v1.3.0
-	github.com/volatiletech/null/v8 v8.1.2
+	github.com/spf13/cobra v0.0.3
-	go.uber.org/atomic v1.11.0
+	github.com/spf13/pflag v1.0.3 // indirect
-	golang.org/x/sync v0.4.0
+	github.com/stretchr/testify v1.4.0
+	github.com/volatiletech/null v8.0.0+incompatible
+	go.uber.org/atomic v1.5.1
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect
+	google.golang.org/appengine v1.6.2 // indirect
+	gotest.tools v2.2.0+incompatible
 )
 
-require (
+go 1.15
-	github.com/armon/go-metrics v0.4.0 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
-	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
-	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
-	github.com/bytedance/sonic v1.9.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
-	github.com/friendsofgo/errors v0.9.2 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
-	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/go-playground/locales v0.14.1 // indirect
-	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.14.0 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/gofrs/uuid v4.2.0+incompatible // indirect
-	github.com/google/btree v1.0.0 // indirect
-	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
-	github.com/gorilla/rpc v1.2.0 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack v0.5.3 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-sockaddr v1.0.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/hashicorp/memberlist v0.5.0 // indirect
-	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/johntdyer/slack-go v0.0.0-20230314151037-c5bf334f9b6e // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
-	github.com/leodido/go-urn v1.2.4 // indirect
-	github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
-	github.com/magiconair/properties v1.8.7 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/miekg/dns v1.1.41 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/quic-go/qpack v0.4.0 // indirect
-	github.com/quic-go/qtls-go1-20 v0.3.4 // indirect
-	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
-	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/slack-go/slack v0.12.1 // indirect
-	github.com/spf13/afero v1.9.3 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.15.0 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
-	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/ugorji/go/codec v1.2.11 // indirect
-	github.com/volatiletech/inflect v0.0.1 // indirect
-	github.com/volatiletech/randomize v0.0.1 // indirect
-	github.com/volatiletech/strmangle v0.0.4 // indirect
-	go.uber.org/mock v0.3.0 // indirect
-	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
-	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.9.1 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-)

internal/metrics/metrics.go

@@ -60,7 +60,6 @@ func (s *Server) Shutdown() {
 const (
 	ns             = "reflector"
 	subsystemCache = "cache"
-	subsystemITTT  = "ittt"
 
 	labelDirection = "direction"
 	labelErrorType = "error_type"
@@ -86,13 +85,8 @@ const (
 	errUnexpectedEOFStr  = "unexpected_eof_str"
 	errJSONSyntax        = "json_syntax"
 	errBlobTooBig        = "blob_too_big"
-	errInvalidPeerJSON   = "invalid_peer_json"
-	errInvalidPeerData   = "invalid_peer_data"
-	errRequestTooLarge   = "request_too_large"
 	errDeadlineExceeded  = "deadline_exceeded"
 	errHashMismatch      = "hash_mismatch"
-	errProtectedBlob     = "protected_blob"
-	errInvalidBlobHash   = "invalid_blob_hash"
 	errZeroByteBlob      = "zero_byte_blob"
 	errInvalidCharacter  = "invalid_character"
 	errBlobNotFound      = "blob_not_found"
@@ -123,11 +117,6 @@ var (
 		Name:      "http3_blob_download_total",
 		Help:      "Total number of blobs downloaded from reflector through QUIC protocol",
 	})
-	HttpDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: ns,
-		Name:      "http_blob_download_total",
-		Help:      "Total number of blobs downloaded from reflector through HTTP protocol",
-	})
 
 	CacheHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: ns,
@@ -135,18 +124,6 @@ var (
 		Name:      "hit_total",
 		Help:      "Total number of blobs retrieved from the cache storage",
 	}, []string{LabelCacheType, LabelComponent})
-	ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: ns,
-		Subsystem: subsystemITTT,
-		Name:      "this_hit_total",
-		Help:      "Total number of blobs retrieved from the this storage",
-	})
-	ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: ns,
-		Subsystem: subsystemITTT,
-		Name:      "that_hit_total",
-		Help:      "Total number of blobs retrieved from the that storage",
-	})
 	CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: ns,
 		Subsystem: subsystemCache,
@@ -159,7 +136,7 @@ var (
 		Name:      "origin_requests_total",
 		Help:      "How many Get requests are in flight from the cache to the origin",
 	}, []string{LabelCacheType, LabelComponent})
-	//during thundering-herd situations, the metric below should be a lot smaller than the metric above
+	// during thundering-herd situations, the metric below should be a lot smaller than the metric above
 	CacheWaitingRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: ns,
 		Subsystem: subsystemCache,
@@ -204,21 +181,11 @@ var (
 		Name:      "udp_in_bytes",
 		Help:      "Total number of bytes downloaded through UDP",
 	})
-	MtrInBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: ns,
-		Name:      "http_in_bytes",
-		Help:      "Total number of bytes downloaded through HTTP",
-	})
 	MtrOutBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
 		Namespace: ns,
 		Name:      "udp_out_bytes",
 		Help:      "Total number of bytes streamed out through UDP",
 	})
-	MtrOutBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: ns,
-		Name:      "http_out_bytes",
-		Help:      "Total number of bytes streamed out through UDP",
-	})
 	MtrInBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
 		Namespace: ns,
 		Name:      "reflector_in_bytes",
@@ -234,21 +201,6 @@ var (
 		Name:      "s3_in_bytes",
 		Help:      "Total number of incoming bytes (from S3-CF)",
 	})
-	Http3BlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: ns,
-		Name:      "http3_blob_request_queue_size",
-		Help:      "Blob requests of https queue size",
-	})
-	HttpBlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: ns,
-		Name:      "http_blob_request_queue_size",
-		Help:      "Blob requests queue size of the HTTP protocol",
-	})
-	RoutinesQueue = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: ns,
-		Name:      "routines",
-		Help:      "routines running by type",
-	}, []string{"package", "kind"})
 )
 
 func CacheLabels(name, component string) prometheus.Labels {
@@ -301,20 +253,10 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a
 	} else if strings.Contains(err.Error(), "blob must be at most") {
 		//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
 		errType = errBlobTooBig
-	} else if strings.Contains(err.Error(), "invalid json request") {
-		errType = errInvalidPeerJSON
-	} else if strings.Contains(err.Error(), "Invalid data") {
-		errType = errInvalidPeerData
-	} else if strings.Contains(err.Error(), "request is too large") {
-		errType = errRequestTooLarge
-	} else if strings.Contains(err.Error(), "Invalid blob hash length") {
-		errType = errInvalidBlobHash
 	} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
 		errType = errHashMismatch
 	} else if strings.Contains(err.Error(), "blob not found") {
 		errType = errBlobNotFound
-	} else if strings.Contains(err.Error(), "requested blob is protected") {
-		errType = errProtectedBlob
 	} else if strings.Contains(err.Error(), "0-byte blob received") {
 		errType = errZeroByteBlob
 	} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {

lite_db/db.go

@@ -1,348 +0,0 @@
-package lite_db
-
-import (
-	"database/sql"
-	"time"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	qt "github.com/lbryio/lbry.go/v2/extras/query"
-
-	"github.com/go-sql-driver/mysql"
-	_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
-	log "github.com/sirupsen/logrus"
-	"github.com/volatiletech/null/v8"
-)
-
-// SdBlob is a special blob that contains information on the rest of the blobs in the stream
-type SdBlob struct {
-	StreamName string `json:"stream_name"`
-	Blobs      []struct {
-		Length   int    `json:"length"`
-		BlobNum  int    `json:"blob_num"`
-		BlobHash string `json:"blob_hash,omitempty"`
-		IV       string `json:"iv"`
-	} `json:"blobs"`
-	StreamType        string `json:"stream_type"`
-	Key               string `json:"key"`
-	SuggestedFileName string `json:"suggested_file_name"`
-	StreamHash        string `json:"stream_hash"`
-}
-
-// SQL implements the DB interface
-type SQL struct {
-	conn *sql.DB
-
-	TrackAccessTime bool
-}
-
-func logQuery(query string, args ...interface{}) {
-	s, err := qt.InterpolateParams(query, args...)
-	if err != nil {
-		log.Errorln(err)
-	} else {
-		log.Debugln(s)
-	}
-}
-
-// Connect will create a connection to the database
-func (s *SQL) Connect(dsn string) error {
-	var err error
-	// interpolateParams is necessary. otherwise uploading a stream with thousands of blobs
-	// will hit MySQL's max_prepared_stmt_count limit because the prepared statements are all
-	// opened inside a transaction. closing them manually doesn't seem to help
-	dsn += "?parseTime=1&collation=utf8mb4_unicode_ci&interpolateParams=1"
-	s.conn, err = sql.Open("mysql", dsn)
-	if err != nil {
-		return errors.Err(err)
-	}
-
-	s.conn.SetMaxIdleConns(12)
-
-	return errors.Err(s.conn.Ping())
-}
-
-// AddBlob adds a blob to the database.
-func (s *SQL) AddBlob(hash string, length int) error {
-	if s.conn == nil {
-		return errors.Err("not connected")
-	}
-
-	_, err := s.insertBlob(hash, length)
-	return err
-}
-
-func (s *SQL) insertBlob(hash string, length int) (int64, error) {
-	if length <= 0 {
-		return 0, errors.Err("length must be positive")
-	}
-	const isStored = true
-	now := time.Now()
-	args := []interface{}{hash, isStored, length, now}
-	blobID, err := s.exec(
-		"INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at=VALUES(last_accessed_at)",
-		args...,
-	)
-	if err != nil {
-		return 0, err
-	}
-
-	if blobID == 0 {
-		err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
-		if err != nil {
-			return 0, errors.Err(err)
-		}
-		if blobID == 0 {
-			return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
-		}
-	}
-
-	return blobID, nil
-}
-
-// HasBlob checks if the database contains the blob information.
-func (s *SQL) HasBlob(hash string) (bool, error) {
-	exists, err := s.HasBlobs([]string{hash})
-	if err != nil {
-		return false, err
-	}
-	return exists[hash], nil
-}
-
-// HasBlobs checks if the database contains the set of blobs and returns a bool map.
-func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
-	exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
-	_ = s.touch(streamsNeedingTouch)
-	return exists, err
-}
-
-func (s *SQL) touch(blobIDs []uint64) error {
-	if len(blobIDs) == 0 {
-		return nil
-	}
-
-	query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
-	args := make([]interface{}, len(blobIDs)+1)
-	args[0] = time.Now()
-	for i := range blobIDs {
-		args[i+1] = blobIDs[i]
-	}
-
-	startTime := time.Now()
-	_, err := s.exec(query, args...)
-	log.Debugf("blobs access query touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
-	return errors.Err(err)
-}
-
-func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
-	if s.conn == nil {
-		return nil, nil, errors.Err("not connected")
-	}
-
-	var (
-		hash           string
-		blobID         uint64
-		lastAccessedAt null.Time
-	)
-
-	var needsTouch []uint64
-	exists := make(map[string]bool)
-
-	touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
-	maxBatchSize := 10000
-	doneIndex := 0
-
-	for len(hashes) > doneIndex {
-		sliceEnd := doneIndex + maxBatchSize
-		if sliceEnd > len(hashes) {
-			sliceEnd = len(hashes)
-		}
-		log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
-		batch := hashes[doneIndex:sliceEnd]
-
-		// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
-
-		query := `SELECT hash, id, last_accessed_at
-FROM blob_ 
-WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
-		args := make([]interface{}, len(batch)+1)
-		args[0] = true
-		for i := range batch {
-			args[i+1] = batch[i]
-		}
-
-		logQuery(query, args...)
-
-		err := func() error {
-			startTime := time.Now()
-			rows, err := s.conn.Query(query, args...)
-			log.Debugf("hashes query took %s", time.Since(startTime))
-			if err != nil {
-				return errors.Err(err)
-			}
-			defer closeRows(rows)
-
-			for rows.Next() {
-				err := rows.Scan(&hash, &blobID, &lastAccessedAt)
-				if err != nil {
-					return errors.Err(err)
-				}
-				exists[hash] = true
-				if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
-					needsTouch = append(needsTouch, blobID)
-				}
-			}
-
-			err = rows.Err()
-			if err != nil {
-				return errors.Err(err)
-			}
-
-			doneIndex += len(batch)
-			return nil
-		}()
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-
-	return exists, needsTouch, nil
-}
-
-// Delete will remove the blob from the db
-func (s *SQL) Delete(hash string) error {
-	_, err := s.exec("UPDATE blob_ set is_stored = ? WHERE hash = ?", 0, hash)
-	return errors.Err(err)
-}
-
-// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
-// but they are tracked so reflector knows what it is missing.
-func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int) error {
-	if s.conn == nil {
-		return errors.Err("not connected")
-	}
-
-	_, err := s.insertBlob(sdHash, sdBlobLength)
-	return err
-}
-
-// GetHashRange gets the smallest and biggest hashes in the db
-func (s *SQL) GetLRUBlobs(maxBlobs int) ([]string, error) {
-	if s.conn == nil {
-		return nil, errors.Err("not connected")
-	}
-
-	query := "SELECT hash from blob_ where is_stored = ? order by last_accessed_at limit ?"
-	const isStored = true
-	logQuery(query, isStored, maxBlobs)
-	rows, err := s.conn.Query(query, isStored, maxBlobs)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-	defer closeRows(rows)
-	blobs := make([]string, 0, maxBlobs)
-	for rows.Next() {
-		var hash string
-		err := rows.Scan(&hash)
-		if err != nil {
-			return nil, errors.Err(err)
-		}
-		blobs = append(blobs, hash)
-	}
-	return blobs, nil
-}
-
-func (s *SQL) AllBlobs() ([]string, error) {
-	if s.conn == nil {
-		return nil, errors.Err("not connected")
-	}
-
-	query := "SELECT hash from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
-	const isStored = true
-	logQuery(query, isStored)
-	rows, err := s.conn.Query(query, isStored)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-	defer closeRows(rows)
-	totalBlobs, err := s.BlobsCount()
-	if err != nil {
-		return nil, err
-	}
-	blobs := make([]string, 0, totalBlobs)
-	for rows.Next() {
-		var hash string
-		err := rows.Scan(&hash)
-		if err != nil {
-			return nil, errors.Err(err)
-		}
-		blobs = append(blobs, hash)
-	}
-	return blobs, nil
-}
-
-func (s *SQL) BlobsCount() (int, error) {
-	if s.conn == nil {
-		return 0, errors.Err("not connected")
-	}
-
-	query := "SELECT count(id) from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
-	const isStored = true
-	logQuery(query, isStored)
-	var count int
-	err := s.conn.QueryRow(query, isStored).Scan(&count)
-	return count, errors.Err(err)
-}
-
-func closeRows(rows *sql.Rows) {
-	if rows != nil {
-		err := rows.Close()
-		if err != nil {
-			log.Error("error closing rows: ", err)
-		}
-	}
-}
-
-func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
-	logQuery(query, args...)
-	attempt, maxAttempts := 0, 3
-Retry:
-	attempt++
-	result, err := s.conn.Exec(query, args...)
-	if isLockTimeoutError(err) {
-		if attempt <= maxAttempts {
-			//Error 1205: Lock wait timeout exceeded; try restarting transaction
-			goto Retry
-		}
-		err = errors.Prefix("Lock timeout for query "+query, err)
-	}
-
-	if err != nil {
-		return 0, errors.Err(err)
-	}
-
-	lastID, err := result.LastInsertId()
-	return lastID, errors.Err(err)
-}
-
-func isLockTimeoutError(err error) bool {
-	e, ok := err.(*mysql.MySQLError)
-	return ok && e != nil && e.Number == 1205
-}
-
-/*  SQL schema
-
-in prod make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
-
-CREATE TABLE `blob_` (
-  `id` bigint unsigned NOT NULL AUTO_INCREMENT,
-  `hash` char(96) NOT NULL,
-  `is_stored` tinyint(1) NOT NULL DEFAULT '0',
-  `length` bigint unsigned DEFAULT NULL,
-  `last_accessed_at` datetime DEFAULT CURRENT_TIMESTAMP,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `id` (`id`),
-  UNIQUE KEY `blob_hash_idx` (`hash`),
-  KEY `blob_last_accessed_idx` (`last_accessed_at`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
-
-*/

meta/meta.go

@@ -6,36 +6,9 @@ import (
 	"time"
 )
 
-var (
+var Version = ""
-	name       = "prism-bin"
+var Time = ""
-	version    = "unknown"
+var BuildTime time.Time
-	commit     = "unknown"
-	commitLong = "unknown"
-	branch     = "unknown"
-	Time       = "unknown"
-	BuildTime  time.Time
-)
-
-// Name returns main application name
-func Name() string {
-	return name
-}
-
-// Version returns current application version
-func Version() string {
-	return version
-}
-
-// FullName returns current app version, commit and build time
-func FullName() string {
-	return fmt.Sprintf(
-		`Name: %v
-Version: %v
-branch: %v
-commit: %v
-commit long: %v
-build date: %v`, Name(), Version(), branch, commit, commitLong, BuildTime.String())
-}
 
 func init() {
 	if Time != "" {
@@ -47,6 +20,11 @@ func init() {
 }
 
 func VersionString() string {
+	version := Version
+	if version == "" {
+		version = "<unset>"
+	}
+
 	var buildTime string
 	if BuildTime.IsZero() {
 		buildTime = "<now>"

peer/client.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -18,6 +17,9 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// ErrBlobExists is a default error for when a blob already exists on the reflector server.
+var ErrBlobExists = errors.Base("blob exists on server")
+
 // Client is an instance of a client connected to a server.
 type Client struct {
 	Timeout   time.Duration
@@ -55,11 +57,10 @@ func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Str
 
 	var sd stream.SDBlob
 
-	b, trace, err := c.GetBlob(sdHash)
+	b, err := c.GetBlob(sdHash)
 	if err != nil {
 		return nil, err
 	}
-	log.Debug(trace.String())
 
 	err = sd.FromBlob(b)
 	if err != nil {
@@ -70,11 +71,10 @@ func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Str
 	s[0] = b
 
 	for i := 0; i < len(sd.BlobInfos)-1; i++ {
-		s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
+		s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
 		if err != nil {
 			return nil, err
 		}
-		log.Debug(trace.String())
 	}
 
 	return s, nil
@@ -114,52 +114,47 @@ func (c *Client) HasBlob(hash string) (bool, error) {
 }
 
 // GetBlob gets a blob
-func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (c *Client) GetBlob(hash string) (stream.Blob, error) {
-	start := time.Now()
 	if !c.connected {
-		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), errors.Err("not connected")
+		return nil, errors.Err("not connected")
 	}
 
 	sendRequest, err := json.Marshal(blobRequest{
 		RequestedBlob: hash,
 	})
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
+		return nil, err
 	}
 
 	err = c.write(sendRequest)
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
+		return nil, err
 	}
 
 	var resp blobResponse
 	err = c.read(&resp)
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
+		return nil, err
 	}
 
-	trace := shared.NewBlobTrace(time.Since(start), "tcp")
-	if resp.RequestTrace != nil {
-		trace = *resp.RequestTrace
-	}
 	if resp.IncomingBlob.Error != "" {
-		return nil, trace, errors.Prefix(hash[:8], resp.IncomingBlob.Error)
+		return nil, errors.Prefix(hash[:8], resp.IncomingBlob.Error)
 	}
 	if resp.IncomingBlob.BlobHash != hash {
-		return nil, trace.Stack(time.Since(start), "tcp"), errors.Prefix(hash[:8], "blob hash in response does not match requested hash")
+		return nil, errors.Prefix(hash[:8], "blob hash in response does not match requested hash")
 	}
 	if resp.IncomingBlob.Length <= 0 {
-		return nil, trace, errors.Prefix(hash[:8], "length reported as <= 0")
+		return nil, errors.Prefix(hash[:8], "length reported as <= 0")
 	}
 
 	log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr())
 
 	blob, err := c.readRawBlob(resp.IncomingBlob.Length)
 	if err != nil {
-		return nil, (*resp.RequestTrace).Stack(time.Since(start), "tcp"), err
+		return nil, err
 	}
 	metrics.MtrInBytesTcp.Add(float64(len(blob)))
-	return blob, trace.Stack(time.Since(start), "tcp"), nil
+	return blob, nil
 }
 
 func (c *Client) read(v interface{}) error {

peer/http3/client.go

@@ -0,0 +1,97 @@
+package http3
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lucas-clemente/quic-go/http3"
+)
+
+// Client is an instance of a client connected to a server.
+type Client struct {
+	Timeout      time.Duration
+	conn         *http.Client
+	roundTripper *http3.RoundTripper
+	ServerAddr   string
+}
+
+// Close closes the connection with the client.
+func (c *Client) Close() error {
+	c.conn.CloseIdleConnections()
+	return c.roundTripper.Close()
+}
+
+// GetStream gets a stream
+func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
+	var sd stream.SDBlob
+
+	b, err := c.GetBlob(sdHash)
+	if err != nil {
+		return nil, err
+	}
+
+	err = sd.FromBlob(b)
+	if err != nil {
+		return nil, err
+	}
+
+	s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
+	s[0] = b
+
+	for i := 0; i < len(sd.BlobInfos)-1; i++ {
+		s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return s, nil
+}
+
+// HasBlob checks if the blob is available
+func (c *Client) HasBlob(hash string) (bool, error) {
+	resp, err := c.conn.Get(fmt.Sprintf("https://%s/has/%s", c.ServerAddr, hash))
+	if err != nil {
+		return false, errors.Err(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusOK {
+		return true, nil
+	}
+	if resp.StatusCode == http.StatusNotFound {
+		return false, nil
+	}
+	return false, errors.Err("non 200 status code returned: %d", resp.StatusCode)
+}
+
+// GetBlob gets a blob
+func (c *Client) GetBlob(hash string) (stream.Blob, error) {
+	resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s", c.ServerAddr, hash))
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNotFound {
+		fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
+		return nil, errors.Err(store.ErrBlobNotFound)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Err("non 200 status code returned: %d", resp.StatusCode)
+	}
+	body := &bytes.Buffer{}
+	_, err = io.Copy(body, resp.Body)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	metrics.MtrInBytesUdp.Add(float64(len(body.Bytes())))
+	return body.Bytes(), nil
+}

peer/http3/server.go

@@ -10,36 +10,31 @@ import (
 	"fmt"
 	"math/big"
 	"net/http"
-	"strconv"
-	"sync"
 	"time"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/reflector"
 	"github.com/lbryio/reflector.go/store"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 
 	"github.com/gorilla/mux"
-	"github.com/quic-go/quic-go"
+	"github.com/lucas-clemente/quic-go"
-	"github.com/quic-go/quic-go/http3"
+	"github.com/lucas-clemente/quic-go/http3"
 	log "github.com/sirupsen/logrus"
 )
 
 // Server is an instance of a peer server that houses the listener and store.
 type Server struct {
-	store              store.BlobStore
+	store store.BlobStore
-	grp                *stop.Group
+	grp   *stop.Group
-	concurrentRequests int
 }
 
 // NewServer returns an initialized Server pointer.
-func NewServer(store store.BlobStore, requestQueueSize int) *Server {
+func NewServer(store store.BlobStore) *Server {
 	return &Server{
-		store:              store,
+		store: store,
-		grp:                stop.New(),
+		grp:   stop.New(),
-		concurrentRequests: requestQueueSize,
 	}
 }
 
@@ -68,21 +63,33 @@ type availabilityResponse struct {
 // Start starts the server listener to handle connections.
 func (s *Server) Start(address string) error {
 	log.Println("HTTP3 peer listening on " + address)
-	window500M := 500 * 1 << 20
-
 	quicConf := &quic.Config{
-		MaxStreamReceiveWindow:     uint64(window500M),
+		HandshakeTimeout: 4 * time.Second,
-		MaxConnectionReceiveWindow: uint64(window500M),
+		MaxIdleTimeout:   10 * time.Second,
-		EnableDatagrams:            true,
-		HandshakeIdleTimeout:       4 * time.Second,
-		MaxIdleTimeout:             20 * time.Second,
 	}
 	r := mux.NewRouter()
 	r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
-		waiter := &sync.WaitGroup{}
+		vars := mux.Vars(r)
-		waiter.Add(1)
+		requestedBlob := vars["hash"]
-		enqueue(&blobRequest{request: r, reply: w, finished: waiter})
+		blob, err := s.store.Get(requestedBlob)
-		waiter.Wait()
+		if err != nil {
+			if errors.Is(err, store.ErrBlobNotFound) {
+				http.Error(w, err.Error(), http.StatusNotFound)
+				return
+			}
+			fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
+			s.logError(err)
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		_, err = w.Write(blob)
+		if err != nil {
+			s.logError(err)
+		}
+		metrics.MtrOutBytesUdp.Add(float64(len(blob)))
+		metrics.BlobDownloadCount.Inc()
+		metrics.Http3DownloadCount.Inc()
 	})
 	r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
@@ -113,12 +120,14 @@ func (s *Server) Start(address string) error {
 		}
 	})
 	server := http3.Server{
-		Addr:       address,
+		Server: &http.Server{
-		Handler:    r,
+			Handler:   r,
-		TLSConfig:  generateTLSConfig(),
+			Addr:      address,
+			TLSConfig: generateTLSConfig(),
+		},
 		QuicConfig: quicConf,
 	}
-	go InitWorkers(s, s.concurrentRequests)
+
 	go s.listenForShutdown(&server)
 	s.grp.Add(1)
 	go func() {
@@ -155,7 +164,7 @@ func generateTLSConfig() *tls.Config {
 
 func (s *Server) listenAndServe(server *http3.Server) {
 	err := server.ListenAndServe()
-	if err != nil && err != quic.ErrServerClosed {
+	if err != nil && err.Error() != "server closed" {
 		log.Errorln(errors.FullTrace(err))
 	}
 }
@@ -167,50 +176,3 @@ func (s *Server) listenForShutdown(listener *http3.Server) {
 		log.Error("error closing listener for peer server - ", err)
 	}
 }
-
-func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	requestedBlob := vars["hash"]
-	traceParam := r.URL.Query().Get("trace")
-	var err error
-	wantsTrace := false
-	if traceParam != "" {
-		wantsTrace, err = strconv.ParseBool(traceParam)
-		if err != nil {
-			wantsTrace = false
-		}
-	}
-	if reflector.IsProtected(requestedBlob) {
-		http.Error(w, "requested blob is protected", http.StatusForbidden)
-		return
-	}
-	blob, trace, err := s.store.Get(requestedBlob)
-
-	if wantsTrace {
-		serialized, err := trace.Serialize()
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusNotFound)
-			return
-		}
-		w.Header().Add("Via", serialized)
-		log.Debug(trace.String())
-	}
-	if err != nil {
-		if errors.Is(err, store.ErrBlobNotFound) {
-			http.Error(w, err.Error(), http.StatusNotFound)
-			return
-		}
-		fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
-		s.logError(err)
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	_, err = w.Write(blob)
-	if err != nil {
-		s.logError(err)
-	}
-	metrics.MtrOutBytesUdp.Add(float64(len(blob)))
-	metrics.BlobDownloadCount.Inc()
-	metrics.Http3DownloadCount.Inc()
-}

peer/http3/store.go

@@ -4,25 +4,18 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"net/http"
-	"strings"
-	"sync"
 	"time"
 
-	"github.com/lbryio/reflector.go/shared"
-	"github.com/lbryio/reflector.go/store"
-
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
-
+	"github.com/lucas-clemente/quic-go"
-	"github.com/quic-go/quic-go"
+	"github.com/lucas-clemente/quic-go/http3"
-	"github.com/quic-go/quic-go/http3"
 )
 
 // Store is a blob store that gets blobs from a peer.
 // It satisfies the store.BlobStore interface but cannot put or delete blobs.
 type Store struct {
-	opts          StoreOpts
+	opts StoreOpts
-	NotFoundCache *sync.Map
 }
 
 // StoreOpts allows to set options for a new Store.
@@ -33,17 +26,13 @@ type StoreOpts struct {
 
 // NewStore makes a new peer store.
 func NewStore(opts StoreOpts) *Store {
-	return &Store{opts: opts, NotFoundCache: &sync.Map{}}
+	return &Store{opts: opts}
 }
 
 func (p *Store) getClient() (*Client, error) {
 	var qconf quic.Config
-	window500M := 500 * 1 << 20
+	qconf.HandshakeTimeout = 4 * time.Second
-	qconf.MaxStreamReceiveWindow = uint64(window500M)
+	qconf.MaxIdleTimeout = 10 * time.Second
-	qconf.MaxConnectionReceiveWindow = uint64(window500M)
-	qconf.EnableDatagrams = true
-	qconf.HandshakeIdleTimeout = 4 * time.Second
-	qconf.MaxIdleTimeout = 20 * time.Second
 	pool, err := x509.SystemCertPool()
 	if err != nil {
 		return nil, err
@@ -74,44 +63,31 @@ func (p *Store) Has(hash string) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	defer func() { _ = c.Close() }()
+	defer c.Close()
 	return c.HasBlob(hash)
 }
 
 // Get downloads the blob from the peer
-func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (p *Store) Get(hash string) (stream.Blob, error) {
-	start := time.Now()
-	if lastChecked, ok := p.NotFoundCache.Load(hash); ok {
-		if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
-			return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound
-		}
-	}
 	c, err := p.getClient()
-	if err != nil && strings.Contains(err.Error(), "blob not found") {
-		p.NotFoundCache.Store(hash, time.Now())
-	}
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
+		return nil, err
 	}
-	defer func() { _ = c.Close() }()
+	defer c.Close()
 	return c.GetBlob(hash)
 }
 
 // Put is not supported
 func (p *Store) Put(hash string, blob stream.Blob) error {
-	return errors.Err(shared.ErrNotImplemented)
+	panic("http3Store cannot put or delete blobs")
 }
 
 // PutSD is not supported
 func (p *Store) PutSD(hash string, blob stream.Blob) error {
-	return errors.Err(shared.ErrNotImplemented)
+	panic("http3Store cannot put or delete blobs")
 }
 
 // Delete is not supported
 func (p *Store) Delete(hash string) error {
-	return errors.Err(shared.ErrNotImplemented)
+	panic("http3Store cannot put or delete blobs")
-}
-
-// Shutdown is not supported
-func (p *Store) Shutdown() {
 }

peer/server.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/reflector"
-	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -89,9 +88,7 @@ func (s *Server) listenAndServe(listener net.Listener) {
 			log.Error(errors.Prefix("accepting conn", err))
 		} else {
 			s.grp.Add(1)
-			metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Inc()
 			go func() {
-				defer metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Dec()
 				s.handleConnection(conn)
 				s.grp.Done()
 			}()
@@ -227,40 +224,35 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
 	if err != nil {
 		var je *json.SyntaxError
 		if ee.As(err, &je) {
-			return nil, errors.Err("invalid json request: offset %d in data %s", je.Offset, hex.EncodeToString(data))
+			return nil, errors.Err("invalid json at offset %d in data %s", je.Offset, hex.EncodeToString(data))
 		}
 		return nil, errors.Err(err)
 	}
 
 	response := compositeResponse{
 		LbrycrdAddress: LbrycrdAddress,
-		AvailableBlobs: []string{},
 	}
 
 	if len(request.RequestedBlobs) > 0 {
+		var availableBlobs []string
 		for _, blobHash := range request.RequestedBlobs {
-			if reflector.IsProtected(blobHash) {
-				return nil, errors.Err("requested blob is protected")
-			}
 			exists, err := s.store.Has(blobHash)
 			if err != nil {
 				return nil, err
 			}
 			if exists {
-				response.AvailableBlobs = append(response.AvailableBlobs, blobHash)
+				availableBlobs = append(availableBlobs, blobHash)
 			}
 		}
+		response.AvailableBlobs = availableBlobs
 	}
 
-	if request.BlobDataPaymentRate != nil {
+	response.BlobDataPaymentRate = paymentRateAccepted
-		response.BlobDataPaymentRate = paymentRateAccepted
+	if request.BlobDataPaymentRate < 0 {
-		if *request.BlobDataPaymentRate < 0 {
+		response.BlobDataPaymentRate = paymentRateTooLow
-			response.BlobDataPaymentRate = paymentRateTooLow
-		}
 	}
 
 	var blob []byte
-	var trace shared.BlobTrace
 	if request.RequestedBlob != "" {
 		if len(request.RequestedBlob) != stream.BlobHashHexLength {
 			return nil, errors.Err("Invalid blob hash length")
@@ -268,17 +260,16 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
 
 		log.Debugln("Sending blob " + request.RequestedBlob[:8])
 
-		blob, trace, err = s.store.Get(request.RequestedBlob)
+		blob, err = s.store.Get(request.RequestedBlob)
-		log.Debug(trace.String())
 		if errors.Is(err, store.ErrBlobNotFound) {
-			response.IncomingBlob = &incomingBlob{
+			response.IncomingBlob = incomingBlob{
 				Error: err.Error(),
 			}
 		} else if err != nil {
 			return nil, err
 		} else {
-			response.IncomingBlob = &incomingBlob{
+			response.IncomingBlob = incomingBlob{
-				BlobHash: request.RequestedBlob,
+				BlobHash: reflector.BlobHash(blob),
 				Length:   len(blob),
 			}
 			metrics.MtrOutBytesTcp.Add(float64(len(blob)))
@@ -306,15 +297,7 @@ func (s *Server) logError(e error) {
 }
 
 func readNextMessage(buf *bufio.Reader) ([]byte, error) {
-	first_byte, err := buf.ReadByte()
+	msg := make([]byte, 0)
-	if err != nil {
-		return nil, err
-	}
-	if first_byte != '{' {
-		// every request starts with '{'. Checking here disconnects earlier, so we don't wait until timeout
-		return nil, errInvalidData
-	}
-	msg := []byte("{")
 	eof := false
 
 	for {
@@ -335,8 +318,6 @@ func readNextMessage(buf *bufio.Reader) ([]byte, error) {
 
 			if len(msg) > maxRequestSize {
 				return msg, errRequestTooLarge
-			} else if len(msg) > 0 && msg[0] != '{' {
-				return msg, errInvalidData
 			}
 
 			// yes, this is how the peer protocol knows when the request finishes
@@ -371,7 +352,6 @@ const (
 )
 
 var errRequestTooLarge = errors.Base("request is too large")
-var errInvalidData = errors.Base("Invalid data")
 
 type availabilityRequest struct {
 	LbrycrdAddress bool     `json:"lbrycrd_address"`
@@ -402,19 +382,18 @@ type incomingBlob struct {
 }
 type blobResponse struct {
 	IncomingBlob incomingBlob `json:"incoming_blob"`
-	RequestTrace *shared.BlobTrace
 }
 
 type compositeRequest struct {
 	LbrycrdAddress      bool     `json:"lbrycrd_address"`
 	RequestedBlobs      []string `json:"requested_blobs"`
-	BlobDataPaymentRate *float64 `json:"blob_data_payment_rate"`
+	BlobDataPaymentRate float64  `json:"blob_data_payment_rate"`
 	RequestedBlob       string   `json:"requested_blob"`
 }
 
 type compositeResponse struct {
-	LbrycrdAddress      string        `json:"lbrycrd_address,omitempty"`
+	LbrycrdAddress      string       `json:"lbrycrd_address,omitempty"`
-	AvailableBlobs      []string      `json:"available_blobs"`
+	AvailableBlobs      []string     `json:"available_blobs,omitempty"`
-	BlobDataPaymentRate string        `json:"blob_data_payment_rate,omitempty"`
+	BlobDataPaymentRate string       `json:"blob_data_payment_rate,omitempty"`
-	IncomingBlob        *incomingBlob `json:"incoming_blob,omitempty"`
+	IncomingBlob        incomingBlob `json:"incoming_blob,omitempty"`
 }

peer/server_test.go

@@ -2,10 +2,7 @@ package peer
 
 import (
 	"bytes"
-	"io"
-	"net"
 	"testing"
-	"time"
 
 	"github.com/lbryio/reflector.go/store"
 )
@@ -78,62 +75,3 @@ func TestAvailabilityRequest_WithBlobs(t *testing.T) {
 		}
 	}
 }
-
-func TestRequestFromConnection(t *testing.T) {
-	s := getServer(t, true)
-	err := s.Start("127.0.0.1:50505")
-	defer s.Shutdown()
-	if err != nil {
-		t.Error("error starting server", err)
-	}
-
-	for _, p := range availabilityRequests {
-		conn, err := net.Dial("tcp", "127.0.0.1:50505")
-		if err != nil {
-			t.Error("error opening connection", err)
-		}
-		defer func() { _ = conn.Close() }()
-
-		response := make([]byte, 8192)
-		_, err = conn.Write(p.request)
-		if err != nil {
-			t.Error("error writing", err)
-		}
-		_, err = conn.Read(response)
-		if err != nil {
-			t.Error("error reading", err)
-		}
-		if !bytes.Equal(response[:len(p.response)], p.response) {
-			t.Errorf("Response did not match expected response.\nExpected: %s\nGot: %s", string(p.response), string(response))
-		}
-	}
-}
-
-func TestInvalidData(t *testing.T) {
-	s := getServer(t, true)
-	err := s.Start("127.0.0.1:50503")
-	defer s.Shutdown()
-	if err != nil {
-		t.Error("error starting server", err)
-	}
-	conn, err := net.Dial("tcp", "127.0.0.1:50503")
-	if err != nil {
-		t.Error("error opening connection", err)
-	}
-	defer func() { _ = conn.Close() }()
-
-	response := make([]byte, 8192)
-	_, err = conn.Write([]byte("hello dear server, I would like blobs. Please"))
-	if err != nil {
-		t.Error("error writing", err)
-	}
-	err = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
-	if err != nil {
-		t.Error("error setting read deadline", err)
-	}
-	_, err = conn.Read(response)
-	if err != io.EOF {
-		t.Error("error reading", err)
-	}
-	println(response)
-}

peer/store.go

@@ -1,12 +1,8 @@
 package peer
 
 import (
-	"strings"
 	"time"
 
-	"github.com/lbryio/reflector.go/shared"
-	"github.com/lbryio/reflector.go/store"
-
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 )
@@ -42,41 +38,31 @@ func (p *Store) Has(hash string) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	defer func() { _ = c.Close() }()
+	defer c.Close()
 	return c.HasBlob(hash)
 }
 
 // Get downloads the blob from the peer
-func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (p *Store) Get(hash string) (stream.Blob, error) {
-	start := time.Now()
 	c, err := p.getClient()
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
+		return nil, err
 	}
-	defer func() { _ = c.Close() }()
+	defer c.Close()
-	blob, trace, err := c.GetBlob(hash)
+	return c.GetBlob(hash)
-	if err != nil && strings.Contains(err.Error(), "blob not found") {
-		return nil, trace, store.ErrBlobNotFound
-	}
-
-	return blob, trace, err
 }
 
 // Put is not supported
 func (p *Store) Put(hash string, blob stream.Blob) error {
-	return errors.Err(shared.ErrNotImplemented)
+	panic("PeerStore cannot put or delete blobs")
 }
 
 // PutSD is not supported
 func (p *Store) PutSD(hash string, blob stream.Blob) error {
-	return errors.Err(shared.ErrNotImplemented)
+	panic("PeerStore cannot put or delete blobs")
 }
 
 // Delete is not supported
 func (p *Store) Delete(hash string) error {
-	return errors.Err(shared.ErrNotImplemented)
+	panic("PeerStore cannot put or delete blobs")
-}
-
-// Shutdown is not supported
-func (p *Store) Shutdown() {
 }

prism/prism.go

@@ -5,14 +5,14 @@ import (
 	"strconv"
 	"sync"
 
-	"github.com/lbryio/reflector.go/cluster"
-	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/reflector"
-	"github.com/lbryio/reflector.go/server/peer"
-	"github.com/lbryio/reflector.go/store"
-
 	"github.com/lbryio/lbry.go/v2/dht"
 	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/reflector.go/cluster"
+	"github.com/lbryio/reflector.go/db"
+	"github.com/lbryio/reflector.go/peer"
+	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/store"
+
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 
@@ -79,7 +79,7 @@ func New(conf *Config) *Prism {
 		dht:       d,
 		cluster:   c,
 		peer:      peer.NewServer(conf.Blobs),
-		reflector: reflector.NewServer(conf.Blobs, conf.Blobs),
+		reflector: reflector.NewServer(conf.Blobs),
 
 		grp: stop.New(),
 	}

prism/prism_test.go

@@ -4,9 +4,8 @@ import (
 	"math/big"
 	"testing"
 
-	"github.com/lbryio/lbry.go/v2/dht/bits"
-
 	"github.com/davecgh/go-spew/spew"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 )
 
 func TestAnnounceRange(t *testing.T) {

publish/publish.go

@@ -3,6 +3,7 @@ package publish
 import (
 	"bytes"
 	"encoding/json"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -20,33 +21,27 @@ import (
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
 	"github.com/golang/protobuf/proto"
+	"golang.org/x/crypto/sha3"
 )
 
-/* TODO:
+var TODO = `
-import cert from wallet
+	import cert from wallet
-get all utxos from chainquery
+	get all utxos from chainquery
-create transaction
+	create transaction
-sign it with the channel
+	sign it with the channel
-track state of utxos across publishes from this channel so that we can just do one query to get utxos
+	track state of utxos across publishes from this channel so that we can just do one query to get utxos
-prioritize only confirmed utxos
+	prioritize only confirmed utxos
 
-Handling all the issues we handle currently with lbrynet:
+	Handling all the issues we handle currently with lbrynet:
-	"Couldn't find private key for id",
+		"Couldn't find private key for id",
-	"You already have a stream claim published under the name",
+		"You already have a stream claim published under the name",
-	"Cannot publish using channel",
+		"Cannot publish using channel",
-	"txn-mempool-conflict",
+		"txn-mempool-conflict",
-	"too-long-mempool-chain",
+		"too-long-mempool-chain",
-	"Missing inputs",
+		"Missing inputs",
-	"Not enough funds to cover this transaction",
+		"Not enough funds to cover this transaction",
-*/
-
-type Details struct {
-	Title       string
-	Description string
-	Author      string
-	Tags        []string
-	ReleaseTime int64
 }
+`
 
 func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) {
 	if name == "" {
@@ -74,20 +69,11 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details
 		return nil, nil, err
 	}
 
-	st, stPB, err := makeStream(path)
+	claim, st, err := makeClaimAndStream(path, details)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	stPB.Author = details.Author
-	stPB.ReleaseTime = details.ReleaseTime
-
-	claim := &pb.Claim{
-		Title:       details.Title,
-		Description: details.Description,
-		Type:        &pb.Claim_Stream{Stream: stPB},
-	}
-
 	err = addClaimToTx(tx, claim, name, amount, addr)
 	if err != nil {
 		return nil, nil, err
@@ -115,7 +101,7 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details
 	return signedTx, txid, nil
 }
 
-// TODO: lots of assumptions. hardcoded values need to be passed in or calculated
+//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
 func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
 	txFee := 0.0002 // TODO: estimate this better?
 
@@ -217,30 +203,50 @@ func reflect(st stream.Stream, reflectorAddress string) error {
 	return nil
 }
 
-func makeStream(path string) (stream.Stream, *pb.Stream, error) {
+type Details struct {
+	Title       string
+	Description string
+	Author      string
+	Tags        []string
+	ReleaseTime int64
+}
+
+func makeClaimAndStream(path string, details Details) (*pb.Claim, stream.Stream, error) {
 	file, err := os.Open(path)
 	if err != nil {
 		return nil, nil, errors.Err(err)
 	}
-	defer func() { _ = file.Close() }()
+	data, err := ioutil.ReadAll(file)
-	enc := stream.NewEncoder(file)
+	if err != nil {
-
+		return nil, nil, errors.Err(err)
-	s, err := enc.Stream()
+	}
+	s, err := stream.New(data)
 	if err != nil {
 		return nil, nil, errors.Err(err)
 	}
 
-	streamProto := &pb.Stream{
+	// make the claim
+	sdBlob := &stream.SDBlob{}
+	err = sdBlob.FromBlob(s[0])
+	if err != nil {
+		return nil, nil, errors.Err(err)
+	}
+
+	filehash := sha3.Sum384(data)
+
+	streamPB := &pb.Stream{
+		Author:      details.Author,
+		ReleaseTime: details.ReleaseTime,
 		Source: &pb.Source{
-			SdHash: enc.SDBlob().Hash(),
+			SdHash: s[0].Hash(),
 			Name:   filepath.Base(file.Name()),
-			Size:   uint64(enc.SourceLen()),
+			Size:   uint64(len(data)),
-			Hash:   enc.SourceHash(),
+			Hash:   filehash[:],
 		},
 	}
 
 	mimeType, category := guessMimeType(filepath.Ext(file.Name()))
-	streamProto.Source.MediaType = mimeType
+	streamPB.Source.MediaType = mimeType
 
 	switch category {
 	case "video":
@@ -248,14 +254,20 @@ func makeStream(path string) (stream.Stream, *pb.Stream, error) {
 		//if err != nil {
 		//	return nil, nil, err
 		//}
-		streamProto.Type = &pb.Stream_Video{}
+		streamPB.Type = &pb.Stream_Video{}
 	case "audio":
-		streamProto.Type = &pb.Stream_Audio{}
+		streamPB.Type = &pb.Stream_Audio{}
 	case "image":
-		streamProto.Type = &pb.Stream_Image{}
+		streamPB.Type = &pb.Stream_Image{}
 	}
 
-	return s, streamProto, nil
+	claim := &pb.Claim{
+		Title:       details.Title,
+		Description: details.Description,
+		Type:        &pb.Claim_Stream{Stream: streamPB},
+	}
+
+	return claim, s, nil
 }
 
 func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) {

readme.md

@@ -1,110 +1,25 @@
 # Reflector
 
-Reflector is a central piece of software that providers LBRY with the following features:
+A reflector cluster to accept LBRY content for hosting en masse, rehost the content, and make money on data fees (TODO). 
-- Blobs reflection: when something is published, we capture the data and store it on our servers for quicker retrieval
+This code includes Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
-- Blobs distribution: when a piece of content is requested and the LBRY network doesn't have it, reflector will retrieve it from its storage and distribute it
-- Blobs caching: reflectors can be chained together in multiple regions or servers to form a chain of cached content. We call those "blobcaches". They are layered so that content distribution is favorable in all the regions we deploy it to
-
-There are a few other features embedded in reflector.go including publishing streams from Go, downloading or upload blobs, resolving content and more unfinished tools.
-
-This code includes a Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
 
 ## Installation
 
-- Install mysql 8 (5.7 might work too)
+coming soon
-- add a reflector user and database with password `reflector` with localhost access only
-- Create the tables as described [here](https://github.com/lbryio/reflector.go/blob/master/db/db.go#L735) (the link might not update as the code does so just look for the schema in that file)
-
-#### We do not support running reflector.go as a blob receiver, however if you want to run it as a private blobcache you may compile it yourself and run it as following:
-```bash
-./prism-bin reflector \
---conf="none" \
---disable-uploads=true \
---use-db=false \
---upstream-reflector="reflector.lbry.com" \
---upstream-protocol="http" \
---request-queue-size=200 \
---disk-cache="2GB:/path/to/your/storage/:localdb" \
-```
-
-Create a systemd script if you want to run it automatically on startup or as a service.
 
 ## Usage
 
-Usage as reflector/blobcache:
+coming soon
-```bash
-Run reflector server
 
-Usage:
-  prism reflector [flags]
-
-Flags:
-      --disable-blocklist                 Disable blocklist watching/updating
-      --disable-uploads                   Disable uploads to this reflector server
-      --disk-cache string                 Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (default "100GB:/tmp/downloaded_blobs:localdb")
-  -h, --help                              help for reflector
-      --http-peer-port int                The port reflector will distribute content from over HTTP protocol (default 5569)
-      --http3-peer-port int               The port reflector will distribute content from over HTTP3 protocol (default 5568)
-      --mem-cache int                     enable in-memory cache with a max size of this many blobs
-      --metrics-port int                  The port reflector will use for prometheus metrics (default 2112)
-      --optional-disk-cache string        Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (this would get hit before the one specified in disk-cache)
-      --origin-endpoint string            HTTP edge endpoint for standard HTTP retrieval
-      --origin-endpoint-fallback string   HTTP edge endpoint for standard HTTP retrieval if first origin fails
-      --receiver-port int                 The port reflector will receive content from (default 5566)
-      --request-queue-size int            How many concurrent requests from downstream should be handled at once (the rest will wait) (default 200)
-      --tcp-peer-port int                 The port reflector will distribute content from for the TCP (LBRY) protocol (default 5567)
-      --upstream-protocol string          protocol used to fetch blobs from another upstream reflector server (tcp/http3/http) (default "http")
-      --upstream-reflector string         host:port of a reflector server where blobs are fetched from
-      --use-db                            Whether to connect to the reflector db or not (default true)
-
-Global Flags:
-      --conf string       Path to config. Use 'none' to disable (default "config.json")
-  -v, --verbose strings   Verbose logging for specific components
-```
-
-Other uses:
-
-```bash
-Prism is a single entry point application with multiple sub modules which can be leveraged individually or together
-
-Usage:
-  prism [command]
-
-Available Commands:
-  check-integrity check blobs integrity for a given path
-  cluster         Start(join) to or Start a new cluster
-  decode          Decode a claim value
-  dht             Run dht node
-  getstream       Get a stream from a reflector server
-  help            Help about any command
-  peer            Run peer server
-  populate-db     populate local database with blobs from a disk storage
-  publish         Publish a file
-  reflector       Run reflector server
-  resolve         Resolve a URL
-  send            Send a file to a reflector
-  sendblob        Send a random blob to a reflector server
-  start           Runs full prism application with cluster, dht, peer server, and reflector server.
-  test            Test things
-  upload          Upload blobs to S3
-  version         Print the version
-
-Flags:
-      --conf string       Path to config. Use 'none' to disable (default "config.json")
-  -h, --help              help for prism
-  -v, --verbose strings   Verbose logging for specific components
-```
 ## Running from Source
 
-This project requires [Go v1.20](https://golang.org/doc/install).
+This project requires [Go v1.11](https://golang.org/doc/install) or higher because it uses Go modules.
-
-On Ubuntu you can install it with `sudo snap install go --classic`
 
 ```
 git clone git@github.com:lbryio/reflector.go.git
 cd reflector.go
 make
-./dist/linux_amd64/prism-bin
+./bin/prism-bin
 ```
 
 ## Contributing
@@ -118,7 +33,8 @@ This project is MIT licensed.
 ## Security
 
 We take security seriously. Please contact security@lbry.com regarding any security issues.
-Our PGP key is [here](https://lbry.com/faq/pgp-key) if you need it.
+Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
 
 ## Contact
-The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777) (niko-at-lbry.com)
+
+The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.com)

reflector/blocklist.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/store"
 	"github.com/lbryio/reflector.go/wallet"
 
@@ -110,9 +109,8 @@ func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan)
 	}
 
 	done := make(chan bool)
-	metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Inc()
+
 	go func() {
-		defer metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Dec()
 		select {
 		case <-done:
 		case <-stopper:

reflector/protected_content.go

@@ -1,81 +0,0 @@
-package reflector
-
-import (
-	"encoding/json"
-	"net/http"
-	"time"
-
-	"github.com/bluele/gcache"
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"golang.org/x/sync/singleflight"
-)
-
-const protectedListURL = "https://api.odysee.com/file/list_protected"
-
-type ProtectedContent struct {
-	SDHash  string `json:"sd_hash"`
-	ClaimID string `json:"claim_id"`
-}
-
-var protectedCache = gcache.New(10).Expiration(2 * time.Minute).Build()
-
-func GetProtectedContent() (interface{}, error) {
-	cachedVal, err := protectedCache.Get("protected")
-	if err == nil && cachedVal != nil {
-		return cachedVal.(map[string]bool), nil
-	}
-
-	method := "GET"
-	var r struct {
-		Success bool               `json:"success"`
-		Error   string             `json:"error"`
-		Data    []ProtectedContent `json:"data"`
-	}
-
-	client := &http.Client{}
-	req, err := http.NewRequest(method, protectedListURL, nil)
-
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-	res, err := client.Do(req)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-	defer func() { _ = res.Body.Close() }()
-	if res.StatusCode != http.StatusOK {
-		return nil, errors.Err("unexpected status code %d", res.StatusCode)
-	}
-	if err = json.NewDecoder(res.Body).Decode(&r); err != nil {
-		return nil, errors.Err(err)
-	}
-
-	if !r.Success {
-		return nil, errors.Prefix("file/list_protected API call", r.Error)
-	}
-
-	protectedMap := make(map[string]bool, len(r.Data))
-	for _, pc := range r.Data {
-		protectedMap[pc.SDHash] = true
-	}
-	err = protectedCache.Set("protected", protectedMap)
-	if err != nil {
-		return protectedMap, errors.Err(err)
-	}
-	return protectedMap, nil
-}
-
-var sf = singleflight.Group{}
-
-func IsProtected(sdHash string) bool {
-	val, err, _ := sf.Do("protected", GetProtectedContent)
-	if err != nil {
-		return false
-	}
-	cachedMap, ok := val.(map[string]bool)
-	if !ok {
-		return false
-	}
-
-	return cachedMap[sdHash]
-}

reflector/server.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"io"
+	"io/ioutil"
 	"net"
 	"time"
 
@@ -39,18 +40,16 @@ type Server struct {
 
 	EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
 
-	underlyingStore store.BlobStore
+	store store.BlobStore
-	outerStore      store.BlobStore
+	grp   *stop.Group
-	grp             *stop.Group
 }
 
 // NewServer returns an initialized reflector server pointer.
-func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server {
+func NewServer(store store.BlobStore) *Server {
 	return &Server{
-		Timeout:         DefaultTimeout,
+		Timeout: DefaultTimeout,
-		underlyingStore: underlying,
+		store:   store,
-		outerStore:      outer,
+		grp:     stop.New(),
-		grp:             stop.New(),
 	}
 }
 
@@ -61,17 +60,16 @@ func (s *Server) Shutdown() {
 	log.Println("reflector server stopped")
 }
 
-// Start starts the server to handle connections.
+//Start starts the server to handle connections.
 func (s *Server) Start(address string) error {
 	l, err := net.Listen(network, address)
 	if err != nil {
 		return errors.Err(err)
 	}
 	log.Println("reflector listening on " + address)
+
 	s.grp.Add(1)
-	metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Inc()
 	go func() {
-		defer metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Dec()
 		<-s.grp.Ch()
 		err := l.Close()
 		if err != nil {
@@ -81,19 +79,15 @@ func (s *Server) Start(address string) error {
 	}()
 
 	s.grp.Add(1)
-	metrics.RoutinesQueue.WithLabelValues("reflector", "start").Inc()
 	go func() {
-		defer metrics.RoutinesQueue.WithLabelValues("reflector", "start").Dec()
 		s.listenAndServe(l)
 		s.grp.Done()
 	}()
 
 	if s.EnableBlocklist {
-		if b, ok := s.underlyingStore.(store.Blocklister); ok {
+		if b, ok := s.store.(store.Blocklister); ok {
 			s.grp.Add(1)
-			metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
 			go func() {
-				defer metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Dec()
 				s.enableBlocklist(b)
 				s.grp.Done()
 			}()
@@ -116,9 +110,7 @@ func (s *Server) listenAndServe(listener net.Listener) {
 			log.Error(err)
 		} else {
 			s.grp.Add(1)
-			metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
 			go func() {
-				defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Dec()
 				s.handleConn(conn)
 				s.grp.Done()
 			}()
@@ -133,9 +125,7 @@ func (s *Server) handleConn(conn net.Conn) {
 		close(connNeedsClosing)
 	}()
 	s.grp.Add(1)
-	metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Inc()
 	go func() {
-		defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Dec()
 		defer s.grp.Done()
 		select {
 		case <-connNeedsClosing:
@@ -200,13 +190,13 @@ func (s *Server) receiveBlob(conn net.Conn) error {
 	}
 
 	var wantsBlob bool
-	if bl, ok := s.underlyingStore.(store.Blocklister); ok {
+	if bl, ok := s.store.(store.Blocklister); ok {
 		wantsBlob, err = bl.Wants(blobHash)
 		if err != nil {
 			return err
 		}
 	} else {
-		blobExists, err := s.underlyingStore.Has(blobHash)
+		blobExists, err := s.store.Has(blobHash)
 		if err != nil {
 			return err
 		}
@@ -216,7 +206,7 @@ func (s *Server) receiveBlob(conn net.Conn) error {
 	var neededBlobs []string
 
 	if isSdBlob && !wantsBlob {
-		if nbc, ok := s.underlyingStore.(neededBlobChecker); ok {
+		if nbc, ok := s.store.(neededBlobChecker); ok {
 			neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
 			if err != nil {
 				return err
@@ -259,9 +249,9 @@ func (s *Server) receiveBlob(conn net.Conn) error {
 	log.Debugln("Got blob " + blobHash[:8])
 
 	if isSdBlob {
-		err = s.outerStore.PutSD(blobHash, blob)
+		err = s.store.PutSD(blobHash, blob)
 	} else {
-		err = s.outerStore.Put(blobHash, blob)
+		err = s.store.Put(blobHash, blob)
 	}
 	if err != nil {
 		return err
@@ -366,7 +356,7 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
 	dec := json.NewDecoder(conn)
 	err = dec.Decode(v)
 	if err != nil {
-		data, _ := io.ReadAll(dec.Buffered())
+		data, _ := ioutil.ReadAll(dec.Buffered())
 		if len(data) > 0 {
 			return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
 		}

reflector/server_test.go

@@ -9,9 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/lbryio/reflector.go/store"
-
 	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/reflector.go/store"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/phayes/freeport"
@@ -23,7 +22,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
 		t.Fatal(err)
 	}
 
-	srv := NewServer(store.NewMemStore(), store.NewMemStore())
+	srv := NewServer(store.NewMemStore())
 	err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
 	if err != nil {
 		t.Fatal(err)
@@ -120,7 +119,7 @@ func TestServer_Timeout(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	srv := NewServer(store.NewMemStore(), store.NewMemStore())
+	srv := NewServer(store.NewMemStore())
 	srv.Timeout = testTimeout
 	err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
 	if err != nil {
@@ -191,7 +190,7 @@ func TestServer_PartialUpload(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	srv := NewServer(st, st)
+	srv := NewServer(st)
 	err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
 	if err != nil {
 		t.Fatal(err)

reflector/uploader.go

@@ -1,13 +1,13 @@
 package reflector
 
 import (
+	"io/ioutil"
 	"os"
 	"path"
 	"sync"
 	"time"
 
 	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/store"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -74,7 +74,7 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
 
 	var exists map[string]bool
 	if !u.skipExistsCheck {
-		exists, err = u.db.HasBlobs(hashes, false)
+		exists, err = u.db.HasBlobs(hashes)
 		if err != nil {
 			return err
 		}
@@ -88,9 +88,7 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
 
 	for i := 0; i < u.workers; i++ {
 		workerWG.Add(1)
-		metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Inc()
 		go func(i int) {
-			defer metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Dec()
 			defer workerWG.Done()
 			defer func(i int) { log.Debugf("worker %d quitting", i) }(i)
 			u.worker(pathChan)
@@ -99,9 +97,7 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
 
 	countWG := sync.WaitGroup{}
 	countWG.Add(1)
-	metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Inc()
 	go func() {
-		defer metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Dec()
 		defer countWG.Done()
 		u.counter()
 	}()
@@ -164,7 +160,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
 		}
 	}()
 
-	blob, err := os.ReadFile(filepath)
+	blob, err := ioutil.ReadFile(filepath)
 	if err != nil {
 		return errors.Err(err)
 	}

scripts/lint.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-err=0
-trap 'err=1' ERR
-# All the .go files, excluding auto generated folders
-GO_FILES=$(find . -iname '*.go' -type f)
-(
-	go install golang.org/x/tools/cmd/goimports@latest                   # Used in build script for generated files
-#	go install golang.org/x/lint/golint@latest                           # Linter
-	go install github.com/jgautheron/gocyclo@latest                      # Check against high complexity
-	go install github.com/mdempsky/unconvert@latest                      # Identifies unnecessary type conversions
-	go install github.com/kisielk/errcheck@latest                        # Checks for unhandled errors
-	go install github.com/opennota/check/cmd/varcheck@latest             # Checks for unused vars
-	go install github.com/opennota/check/cmd/structcheck@latest          # Checks for unused fields in structs
-)
-echo "Running varcheck..." && varcheck $(go list ./...)
-echo "Running structcheck..." && structcheck $(go list ./...)
-# go vet is the official Go static analyzer
-echo "Running go vet..." && go vet $(go list ./...)
-# checks for unhandled errors
-echo "Running errcheck..." && errcheck $(go list ./...)
-# check for unnecessary conversions - ignore autogen code
-echo "Running unconvert..." && unconvert -v $(go list ./...)
-echo "Running gocyclo..." && gocyclo -ignore "_test" -avg -over 28 $GO_FILES
-#echo "Running golint..." && golint -set_exit_status $(go list ./...)
-test $err = 0 # Return non-zero if any command failed

server/http/routes.go

@@ -1,105 +0,0 @@
-package http
-
-import (
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/reflector"
-	"github.com/lbryio/reflector.go/shared"
-	"github.com/lbryio/reflector.go/store"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-
-	"github.com/gin-gonic/gin"
-	log "github.com/sirupsen/logrus"
-)
-
-func (s *Server) getBlob(c *gin.Context) {
-	waiter := &sync.WaitGroup{}
-	waiter.Add(1)
-	enqueue(&blobRequest{c: c, finished: waiter})
-	waiter.Wait()
-}
-
-func (s *Server) HandleGetBlob(c *gin.Context) {
-	defer func() {
-		if r := recover(); r != nil {
-			log.Errorf("Recovered from panic: %v", r)
-		}
-	}()
-	start := time.Now()
-	hash := c.Query("hash")
-	edgeToken := c.Query("edge_token")
-
-	if reflector.IsProtected(hash) && edgeToken != s.edgeToken {
-		_ = c.Error(errors.Err("requested blob is protected"))
-		c.String(http.StatusForbidden, "requested blob is protected")
-		return
-	}
-	if s.missesCache.Has(hash) {
-		serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
-		c.Header("Via", serialized)
-		if err != nil {
-			_ = c.Error(errors.Err(err))
-			c.String(http.StatusInternalServerError, err.Error())
-			return
-		}
-		c.AbortWithStatus(http.StatusNotFound)
-		return
-	}
-	blob, trace, err := s.store.Get(hash)
-	if err != nil {
-		serialized, serializeErr := trace.Serialize()
-		if serializeErr != nil {
-			_ = c.Error(errors.Prefix(serializeErr.Error(), err))
-			c.String(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err).Error())
-			return
-		}
-		c.Header("Via", serialized)
-
-		if errors.Is(err, store.ErrBlobNotFound) {
-			_ = s.missesCache.Set(hash, true)
-			c.AbortWithStatus(http.StatusNotFound)
-			return
-		}
-		_ = c.Error(err)
-		c.String(http.StatusInternalServerError, err.Error())
-		return
-	}
-	serialized, err := trace.Serialize()
-	if err != nil {
-		_ = c.Error(err)
-		c.String(http.StatusInternalServerError, err.Error())
-		return
-	}
-	metrics.MtrOutBytesHttp.Add(float64(len(blob)))
-	metrics.BlobDownloadCount.Inc()
-	metrics.HttpDownloadCount.Inc()
-	c.Header("Via", serialized)
-	c.Header("Content-Disposition", "filename="+hash)
-	c.Data(http.StatusOK, "application/octet-stream", blob)
-}
-
-func (s *Server) hasBlob(c *gin.Context) {
-	hash := c.Query("hash")
-	has, err := s.store.Has(hash)
-	if err != nil {
-		_ = c.Error(err)
-		c.String(http.StatusInternalServerError, err.Error())
-		return
-	}
-	if has {
-		c.Status(http.StatusNoContent)
-		return
-	}
-	c.Status(http.StatusNotFound)
-}
-
-func (s *Server) recoveryHandler(c *gin.Context, err interface{}) {
-	c.JSON(500, gin.H{
-		"title": "Error",
-		"err":   err,
-	})
-}

server/http/server.go

@@ -1,82 +0,0 @@
-package http
-
-import (
-	"context"
-	"net/http"
-	"time"
-
-	"github.com/lbryio/reflector.go/store"
-
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-
-	"github.com/bluele/gcache"
-	nice "github.com/ekyoung/gin-nice-recovery"
-	"github.com/gin-gonic/gin"
-	log "github.com/sirupsen/logrus"
-)
-
-// Server is an instance of a peer server that houses the listener and store.
-type Server struct {
-	store              store.BlobStore
-	grp                *stop.Group
-	concurrentRequests int
-	missesCache        gcache.Cache
-	edgeToken          string
-}
-
-// NewServer returns an initialized Server pointer.
-func NewServer(store store.BlobStore, requestQueueSize int, edgeToken string) *Server {
-	return &Server{
-		store:              store,
-		grp:                stop.New(),
-		concurrentRequests: requestQueueSize,
-		missesCache:        gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
-		edgeToken:          edgeToken,
-	}
-}
-
-// Shutdown gracefully shuts down the peer server.
-func (s *Server) Shutdown() {
-	log.Debug("shutting down HTTP server")
-	s.grp.StopAndWait()
-	log.Debug("HTTP server stopped")
-}
-
-// Start starts the server listener to handle connections.
-func (s *Server) Start(address string) error {
-	gin.SetMode(gin.ReleaseMode)
-	router := gin.New()
-	router.Use(gin.Logger())
-	// Install nice.Recovery, passing the handler to call after recovery
-	router.Use(nice.Recovery(s.recoveryHandler))
-	router.GET("/blob", s.getBlob)
-	router.HEAD("/blob", s.hasBlob)
-	srv := &http.Server{
-		Addr:    address,
-		Handler: router,
-	}
-	go s.listenForShutdown(srv)
-	go InitWorkers(s, s.concurrentRequests)
-	// Initializing the server in a goroutine so that
-	// it won't block the graceful shutdown handling below
-	s.grp.Add(1)
-	go func() {
-		defer s.grp.Done()
-		log.Println("HTTP server listening on " + address)
-		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			log.Fatalf("listen: %s\n", err)
-		}
-	}()
-	return nil
-}
-
-func (s *Server) listenForShutdown(listener *http.Server) {
-	<-s.grp.Ch()
-	// The context is used to inform the server it has 5 seconds to finish
-	// the request it is currently handling
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	if err := listener.Shutdown(ctx); err != nil {
-		log.Fatal("Server forced to shutdown:", err)
-	}
-}

server/http/worker.go

@@ -1,46 +0,0 @@
-package http
-
-import (
-	"sync"
-
-	"github.com/lbryio/reflector.go/internal/metrics"
-
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-
-	"github.com/gin-gonic/gin"
-)
-
-type blobRequest struct {
-	c        *gin.Context
-	finished *sync.WaitGroup
-}
-
-var getReqCh = make(chan *blobRequest, 20000)
-
-func InitWorkers(server *Server, workers int) {
-	stopper := stop.New(server.grp)
-	for i := 0; i < workers; i++ {
-		metrics.RoutinesQueue.WithLabelValues("http", "worker").Inc()
-		go func(worker int) {
-			defer metrics.RoutinesQueue.WithLabelValues("http", "worker").Dec()
-			for {
-				select {
-				case <-stopper.Ch():
-				case r := <-getReqCh:
-					process(server, r)
-					metrics.HttpBlobReqQueue.Dec()
-				}
-			}
-		}(i)
-	}
-}
-
-func enqueue(b *blobRequest) {
-	metrics.HttpBlobReqQueue.Inc()
-	getReqCh <- b
-}
-
-func process(server *Server, r *blobRequest) {
-	server.HandleGetBlob(r.c)
-	r.finished.Done()
-}

server/http3/client.go

@@ -1,141 +0,0 @@
-package http3
-
-import (
-	"bytes"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/shared"
-	"github.com/lbryio/reflector.go/store"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
-
-	"github.com/quic-go/quic-go/http3"
-	log "github.com/sirupsen/logrus"
-)
-
-// Client is an instance of a client connected to a server.
-type Client struct {
-	Timeout      time.Duration
-	conn         *http.Client
-	roundTripper *http3.RoundTripper
-	ServerAddr   string
-}
-
-// Close closes the connection with the client.
-func (c *Client) Close() error {
-	c.conn.CloseIdleConnections()
-	return c.roundTripper.Close()
-}
-
-// GetStream gets a stream
-func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
-	var sd stream.SDBlob
-
-	b, _, err := c.GetBlob(sdHash)
-	if err != nil {
-		return nil, err
-	}
-
-	err = sd.FromBlob(b)
-	if err != nil {
-		return nil, err
-	}
-
-	s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
-	s[0] = b
-
-	for i := 0; i < len(sd.BlobInfos)-1; i++ {
-		var trace shared.BlobTrace
-		s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
-		if err != nil {
-			return nil, err
-		}
-		log.Debug(trace.String())
-	}
-
-	return s, nil
-}
-
-// HasBlob checks if the blob is available
-func (c *Client) HasBlob(hash string) (bool, error) {
-	resp, err := c.conn.Get(fmt.Sprintf("https://%s/has/%s", c.ServerAddr, hash))
-	if err != nil {
-		return false, errors.Err(err)
-	}
-	defer func() { _ = resp.Body.Close() }()
-	if resp.StatusCode == http.StatusOK {
-		return true, nil
-	}
-	if resp.StatusCode == http.StatusNotFound {
-		return false, nil
-	}
-	return false, errors.Err("non 200 status code returned: %d", resp.StatusCode)
-}
-
-// GetBlob gets a blob
-func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
-	start := time.Now()
-	resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s?trace=true", c.ServerAddr, hash))
-	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
-	}
-	defer func() { _ = resp.Body.Close() }()
-
-	if resp.StatusCode == http.StatusNotFound {
-		fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
-		return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(store.ErrBlobNotFound)
-	} else if resp.StatusCode != http.StatusOK {
-		return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err("non 200 status code returned: %d", resp.StatusCode)
-	}
-
-	tmp := getBuffer()
-	defer putBuffer(tmp)
-	serialized := resp.Header.Get("Via")
-	trace := shared.NewBlobTrace(time.Since(start), "http3")
-	if serialized != "" {
-		parsedTrace, err := shared.Deserialize(serialized)
-		if err != nil {
-			return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
-		}
-		trace = *parsedTrace
-	}
-	written, err := io.Copy(tmp, resp.Body)
-	if err != nil {
-		return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
-	}
-
-	blob := make([]byte, written)
-	copy(blob, tmp.Bytes())
-
-	metrics.MtrInBytesUdp.Add(float64(len(blob)))
-
-	return blob, trace.Stack(time.Since(start), "http3"), nil
-}
-
-// buffer pool to reduce GC
-// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
-var buffers = sync.Pool{
-	// New is called when a new instance is needed
-	New: func() interface{} {
-		buf := make([]byte, 0, stream.MaxBlobSize)
-		return bytes.NewBuffer(buf)
-	},
-}
-
-// getBuffer fetches a buffer from the pool
-func getBuffer() *bytes.Buffer {
-	return buffers.Get().(*bytes.Buffer)
-}
-
-// putBuffer returns a buffer to the pool
-func putBuffer(buf *bytes.Buffer) {
-	buf.Reset()
-	buffers.Put(buf)
-}

server/http3/worker.go

@@ -1,46 +0,0 @@
-package http3
-
-import (
-	"net/http"
-	"sync"
-
-	"github.com/lbryio/reflector.go/internal/metrics"
-
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-)
-
-type blobRequest struct {
-	request  *http.Request
-	reply    http.ResponseWriter
-	finished *sync.WaitGroup
-}
-
-var getReqCh = make(chan *blobRequest, 20000)
-
-func InitWorkers(server *Server, workers int) {
-	stopper := stop.New(server.grp)
-	for i := 0; i < workers; i++ {
-		metrics.RoutinesQueue.WithLabelValues("http3", "worker").Inc()
-		go func(worker int) {
-			defer metrics.RoutinesQueue.WithLabelValues("http3", "worker").Dec()
-			for {
-				select {
-				case <-stopper.Ch():
-				case r := <-getReqCh:
-					metrics.Http3BlobReqQueue.Dec()
-					process(server, r)
-				}
-			}
-		}(i)
-	}
-}
-
-func enqueue(b *blobRequest) {
-	metrics.Http3BlobReqQueue.Inc()
-	getReqCh <- b
-}
-
-func process(server *Server, r *blobRequest) {
-	server.HandleGetBlob(r.reply, r.request)
-	r.finished.Done()
-}

shared/errors.go

@@ -1,6 +0,0 @@
-package shared
-
-import "github.com/lbryio/lbry.go/v2/extras/errors"
-
-//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
-var ErrNotImplemented = errors.Base("this store does not implement this method")

shared/shared.go

@@ -1,82 +0,0 @@
-package shared
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"time"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-)
-
-type BlobStack struct {
-	Timing     time.Duration `json:"timing"`
-	OriginName string        `json:"origin_name"`
-	HostName   string        `json:"host_name"`
-}
-type BlobTrace struct {
-	Stacks []BlobStack `json:"stacks"`
-}
-
-var hostName *string
-
-func getHostName() string {
-	if hostName == nil {
-		hn, err := os.Hostname()
-		if err != nil {
-			hn = "unknown"
-		}
-		hostName = &hn
-	}
-	return *hostName
-}
-func (b *BlobTrace) Stack(timing time.Duration, originName string) BlobTrace {
-	b.Stacks = append(b.Stacks, BlobStack{
-		Timing:     timing,
-		OriginName: originName,
-		HostName:   getHostName(),
-	})
-	return *b
-}
-func (b *BlobTrace) Merge(otherTrance BlobTrace) BlobTrace {
-	b.Stacks = append(b.Stacks, otherTrance.Stacks...)
-	return *b
-}
-func NewBlobTrace(timing time.Duration, originName string) BlobTrace {
-	b := BlobTrace{}
-	b.Stacks = append(b.Stacks, BlobStack{
-		Timing:     timing,
-		OriginName: originName,
-		HostName:   getHostName(),
-	})
-	return b
-}
-
-func (b BlobTrace) String() string {
-	var fullTrace string
-	for i, stack := range b.Stacks {
-		delta := time.Duration(0)
-		if i > 0 {
-			delta = stack.Timing - b.Stacks[i-1].Timing
-		}
-		fullTrace += fmt.Sprintf("[%d](%s) origin: %s - timing: %s - delta: %s\n", i, stack.HostName, stack.OriginName, stack.Timing.String(), delta.String())
-	}
-	return fullTrace
-}
-
-func (b BlobTrace) Serialize() (string, error) {
-	t, err := json.Marshal(b)
-	if err != nil {
-		return "", errors.Err(err)
-	}
-	return string(t), nil
-}
-
-func Deserialize(serializedData string) (*BlobTrace, error) {
-	var trace BlobTrace
-	err := json.Unmarshal([]byte(serializedData), &trace)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-	return &trace, nil
-}

shared/shared_test.go

@@ -1,36 +0,0 @@
-package shared
-
-import (
-	"testing"
-	"time"
-
-	"github.com/lbryio/lbry.go/v2/extras/util"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBlobTrace_Serialize(t *testing.T) {
-	hostName = util.PtrToString("test_machine")
-	stack := NewBlobTrace(10*time.Second, "test")
-	stack.Stack(20*time.Second, "test2")
-	stack.Stack(30*time.Second, "test3")
-	serialized, err := stack.Serialize()
-	assert.NoError(t, err)
-	t.Log(serialized)
-	expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\",\"host_name\":\"test_machine\"},{\"timing\":20000000000,\"origin_name\":\"test2\",\"host_name\":\"test_machine\"},{\"timing\":30000000000,\"origin_name\":\"test3\",\"host_name\":\"test_machine\"}]}"
-	assert.Equal(t, expected, serialized)
-}
-
-func TestBlobTrace_Deserialize(t *testing.T) {
-	hostName = util.PtrToString("test_machine")
-	serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
-	stack, err := Deserialize(serialized)
-	assert.NoError(t, err)
-	assert.Len(t, stack.Stacks, 3)
-	assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second)
-	assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second)
-	assert.Equal(t, stack.Stacks[2].Timing, 30*time.Second)
-	assert.Equal(t, stack.Stacks[0].OriginName, "test")
-	assert.Equal(t, stack.Stacks[1].OriginName, "test2")
-	assert.Equal(t, stack.Stacks[2].OriginName, "test3")
-}

store/atime_linux.go

@@ -1,4 +1,3 @@
-//go:build linux
 // +build linux
 
 package store
@@ -10,7 +9,7 @@ import (
 )
 
 func timespecToTime(ts syscall.Timespec) time.Time {
-	return time.Unix(ts.Sec, ts.Nsec)
+	return time.Unix(int64(ts.Sec), int64(ts.Nsec))
 }
 
 func atime(fi os.FileInfo) time.Time {

store/caching.go

@@ -3,13 +3,10 @@ package store
 import (
 	"time"
 
-	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/shared"
-
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/lbryio/reflector.go/internal/metrics"
 )
 
 // CachingStore combines two stores, typically a local and a remote store, to improve performance.
@@ -25,7 +22,7 @@ func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
 	return &CachingStore{
 		component: component,
 		origin:    WithSingleFlight(component, origin),
-		cache:     WithSingleFlight(component, cache),
+		cache:     cache,
 	}
 }
 
@@ -45,9 +42,9 @@ func (c *CachingStore) Has(hash string) (bool, error) {
 
 // Get tries to get the blob from the cache first, falling back to the origin. If the blob comes
 // from the origin, it is also stored in the cache.
-func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (c *CachingStore) Get(hash string) (stream.Blob, error) {
 	start := time.Now()
-	blob, trace, err := c.cache.Get(hash)
+	blob, err := c.cache.Get(hash)
 	if err == nil || !errors.Is(err, ErrBlobNotFound) {
 		metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
 		rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
@@ -56,21 +53,18 @@ func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 			metrics.LabelComponent: c.component,
 			metrics.LabelSource:    "cache",
 		}).Set(rate)
-		return blob, trace.Stack(time.Since(start), c.Name()), err
+		return blob, err
 	}
 
 	metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
 
-	blob, trace, err = c.origin.Get(hash)
+	blob, err = c.origin.Get(hash)
 	if err != nil {
-		return nil, trace.Stack(time.Since(start), c.Name()), err
+		return nil, err
 	}
-	// do not do this async unless you're prepared to deal with mayhem
+
 	err = c.cache.Put(hash, blob)
-	if err != nil {
+	return blob, err
-		log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
-	}
-	return blob, trace.Stack(time.Since(start), c.Name()), nil
 }
 
 // Put stores the blob in the origin and the cache
@@ -99,9 +93,3 @@ func (c *CachingStore) Delete(hash string) error {
 	}
 	return c.cache.Delete(hash)
 }
-
-// Shutdown shuts down the store gracefully
-func (c *CachingStore) Shutdown() {
-	c.origin.Shutdown()
-	c.cache.Shutdown()
-}

store/caching_test.go

@@ -6,8 +6,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/lbryio/reflector.go/shared"
-
 	"github.com/lbryio/lbry.go/v2/stream"
 )
 
@@ -53,14 +51,13 @@ func TestCachingStore_CacheMiss(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	res, stack, err := s.Get(hash)
+	res, err := s.Get(hash)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !bytes.Equal(b, res) {
 		t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
 	}
-	time.Sleep(10 * time.Millisecond) //storing to cache is done async so let's give it some time
 
 	has, err := cache.Has(hash)
 	if err != nil {
@@ -69,16 +66,14 @@ func TestCachingStore_CacheMiss(t *testing.T) {
 	if !has {
 		t.Errorf("Get() did not copy blob to cache")
 	}
-	t.Logf("stack: %s", stack.String())
 
-	res, stack, err = cache.Get(hash)
+	res, err = cache.Get(hash)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !bytes.Equal(b, res) {
 		t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res))
 	}
-	t.Logf("stack: %s", stack.String())
 }
 
 func TestCachingStore_ThunderingHerd(t *testing.T) {
@@ -97,7 +92,7 @@ func TestCachingStore_ThunderingHerd(t *testing.T) {
 	wg := &sync.WaitGroup{}
 
 	getNoErr := func() {
-		res, _, err := s.Get(hash)
+		res, err := s.Get(hash)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -153,7 +148,7 @@ func (s *SlowBlobStore) Has(hash string) (bool, error) {
 	return s.mem.Has(hash)
 }
 
-func (s *SlowBlobStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (s *SlowBlobStore) Get(hash string) (stream.Blob, error) {
 	time.Sleep(s.delay)
 	return s.mem.Get(hash)
 }
@@ -172,7 +167,3 @@ func (s *SlowBlobStore) Delete(hash string) error {
 	time.Sleep(s.delay)
 	return s.mem.Delete(hash)
 }
-
-func (s *SlowBlobStore) Shutdown() {
-	return
-}

store/cloudfront_ro.go

@@ -2,12 +2,12 @@ package store
 
 import (
 	"io"
+	"io/ioutil"
 	"net/http"
 	"time"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/meta"
-	"github.com/lbryio/reflector.go/shared"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
@@ -36,7 +36,8 @@ func (c *CloudFrontROStore) Has(hash string) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	defer func() { _ = body.Close() }()
+	defer body.Close()
+
 	switch status {
 	case http.StatusNotFound, http.StatusForbidden:
 		return false, nil
@@ -48,30 +49,30 @@ func (c *CloudFrontROStore) Has(hash string) (bool, error) {
 }
 
 // Get gets the blob from Cloudfront.
-func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (c *CloudFrontROStore) Get(hash string) (stream.Blob, error) {
 	log.Debugf("Getting %s from S3", hash[:8])
-	start := time.Now()
 	defer func(t time.Time) {
 		log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
-	}(start)
+	}(time.Now())
 
 	status, body, err := c.cfRequest(http.MethodGet, hash)
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
+		return nil, err
 	}
-	defer func() { _ = body.Close() }()
+	defer body.Close()
+
 	switch status {
 	case http.StatusNotFound, http.StatusForbidden:
-		return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
+		return nil, errors.Err(ErrBlobNotFound)
 	case http.StatusOK:
-		b, err := io.ReadAll(body)
+		b, err := ioutil.ReadAll(body)
 		if err != nil {
-			return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
+			return nil, errors.Err(err)
 		}
 		metrics.MtrInBytesS3.Add(float64(len(b)))
-		return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil
+		return b, nil
 	default:
-		return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status)
+		return nil, errors.Err("unexpected status %d", status)
 	}
 }
 
@@ -81,7 +82,7 @@ func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser,
 	if err != nil {
 		return 0, nil, errors.Err(err)
 	}
-	req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
+	req.Header.Add("User-Agent", "reflector.go/"+meta.Version)
 
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
@@ -92,17 +93,13 @@ func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser,
 }
 
 func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
-	return errors.Err(shared.ErrNotImplemented)
+	panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
 }
 
 func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
-	return errors.Err(shared.ErrNotImplemented)
+	panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
 }
 
 func (c *CloudFrontROStore) Delete(_ string) error {
-	return errors.Err(shared.ErrNotImplemented)
+	panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
-}
-
-// Shutdown shuts down the store gracefully
-func (c *CloudFrontROStore) Shutdown() {
 }

store/cloudfront_rw.go

@@ -1,22 +1,18 @@
 package store
 
 import (
-	"time"
-
-	"github.com/lbryio/reflector.go/shared"
-
 	"github.com/lbryio/lbry.go/v2/stream"
 )
 
-// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3.
+// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront, writes go to S3.
 type CloudFrontRWStore struct {
-	cf *ITTTStore
+	cf *CloudFrontROStore
 	s3 *S3Store
 }
 
 // NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
 // NOTE: It panics if either argument is nil.
-func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore {
+func NewCloudFrontRWStore(cf *CloudFrontROStore, s3 *S3Store) *CloudFrontRWStore {
 	if cf == nil || s3 == nil {
 		panic("both stores must be set")
 	}
@@ -34,10 +30,8 @@ func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
 }
 
 // Get gets the blob from Cloudfront.
-func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, error) {
-	start := time.Now()
+	return c.cf.Get(hash)
-	blob, trace, err := c.cf.Get(hash)
-	return blob, trace.Stack(time.Since(start), c.Name()), err
 }
 
 // Put stores the blob on S3
@@ -54,9 +48,3 @@ func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
 func (c *CloudFrontRWStore) Delete(hash string) error {
 	return c.s3.Delete(hash)
 }
-
-// Shutdown shuts down the store gracefully
-func (c *CloudFrontRWStore) Shutdown() {
-	c.s3.Shutdown()
-	c.cf.Shutdown()
-}

store/dbbacked.go

@@ -3,10 +3,8 @@ package store
 import (
 	"encoding/json"
 	"sync"
-	"time"
 
 	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/shared"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
@@ -16,16 +14,15 @@ import (
 
 // DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
 type DBBackedStore struct {
-	blobs        BlobStore
+	blobs     BlobStore
-	db           *db.SQL
+	db        *db.SQL
-	blockedMu    sync.RWMutex
+	blockedMu sync.RWMutex
-	blocked      map[string]bool
+	blocked   map[string]bool
-	deleteOnMiss bool
 }
 
 // NewDBBackedStore returns an initialized store pointer.
-func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore {
+func NewDBBackedStore(blobs BlobStore, db *db.SQL) *DBBackedStore {
-	return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss}
+	return &DBBackedStore{blobs: blobs, db: db}
 }
 
 const nameDBBacked = "db-backed"
@@ -35,29 +32,20 @@ func (d *DBBackedStore) Name() string { return nameDBBacked }
 
 // Has returns true if the blob is in the store
 func (d *DBBackedStore) Has(hash string) (bool, error) {
-	return d.db.HasBlob(hash, false)
+	return d.db.HasBlob(hash)
 }
 
 // Get gets the blob
-func (d *DBBackedStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
-	start := time.Now()
+	has, err := d.db.HasBlob(hash)
-	has, err := d.db.HasBlob(hash, true)
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
+		return nil, err
 	}
 	if !has {
-		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), ErrBlobNotFound
+		return nil, ErrBlobNotFound
 	}
 
-	b, stack, err := d.blobs.Get(hash)
+	return d.blobs.Get(hash)
-	if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) {
-		e2 := d.Delete(hash)
-		if e2 != nil {
-			log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
-		}
-	}
-
-	return b, stack.Stack(time.Since(start), d.Name()), err
 }
 
 // Put stores the blob in the S3 store and stores the blob information in the DB.
@@ -112,22 +100,22 @@ func (d *DBBackedStore) Block(hash string) error {
 		return err
 	}
 
-	//has, err := d.db.HasBlob(hash, false)
+	has, err := d.db.HasBlob(hash)
-	//if err != nil {
+	if err != nil {
-	//	return err
+		return err
-	//}
+	}
-	//
+
-	//if has {
+	if has {
-	//	err = d.blobs.Delete(hash)
+		err = d.blobs.Delete(hash)
-	//	if err != nil {
+		if err != nil {
-	//		return err
+			return err
-	//	}
+		}
-	//
+
-	//	err = d.db.Delete(hash)
+		err = d.db.Delete(hash)
-	//	if err != nil {
+		if err != nil {
-	//		return err
+			return err
-	//	}
+		}
-	//}
+	}
 
 	return d.markBlocked(hash)
 }
@@ -194,8 +182,3 @@ func (d *DBBackedStore) initBlocked() error {
 
 	return err
 }
-
-// Shutdown shuts down the store gracefully
-func (d *DBBackedStore) Shutdown() {
-	d.blobs.Shutdown()
-}

store/disk.go

@@ -1,15 +1,13 @@
 package store
 
 import (
+	"io/ioutil"
 	"os"
 	"path"
-	"time"
-
-	"github.com/lbryio/reflector.go/shared"
-	"github.com/lbryio/reflector.go/store/speedwalk"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/store/speedwalk"
 )
 
 // DiskStore stores blobs on a local disk
@@ -54,21 +52,39 @@ func (d *DiskStore) Has(hash string) (bool, error) {
 }
 
 // Get returns the blob or an error if the blob doesn't exist.
-func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (d *DiskStore) Get(hash string) (stream.Blob, error) {
-	start := time.Now()
 	err := d.initOnce()
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
+		return nil, err
 	}
 
-	blob, err := os.ReadFile(d.path(hash))
+	file, err := os.Open(d.path(hash))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
+			return nil, errors.Err(ErrBlobNotFound)
 		}
-		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
+		return nil, err
 	}
-	return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
+	defer file.Close()
+
+	blob, err := ioutil.ReadAll(file)
+	return blob, errors.Err(err)
+}
+
+// Put stores the blob on disk
+func (d *DiskStore) Put(hash string, blob stream.Blob) error {
+	err := d.initOnce()
+	if err != nil {
+		return err
+	}
+
+	err = d.ensureDirExists(d.dir(hash))
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(d.path(hash), blob, 0644)
+	return errors.Err(err)
 }
 
 // PutSD stores the sd blob on the disk
@@ -111,15 +127,11 @@ func (d *DiskStore) dir(hash string) string {
 	}
 	return path.Join(d.blobDir, hash[:d.prefixLength])
 }
-func (d *DiskStore) tmpDir(hash string) string {
+
-	return path.Join(d.blobDir, "tmp")
-}
 func (d *DiskStore) path(hash string) string {
 	return path.Join(d.dir(hash), hash)
 }
-func (d *DiskStore) tmpPath(hash string) string {
+
-	return path.Join(d.tmpDir(hash), hash)
-}
 func (d *DiskStore) ensureDirExists(dir string) error {
 	return errors.Err(os.MkdirAll(dir, 0755))
 }
@@ -133,14 +145,7 @@ func (d *DiskStore) initOnce() error {
 	if err != nil {
 		return err
 	}
-	err = d.ensureDirExists(path.Join(d.blobDir, "tmp"))
+
-	if err != nil {
-		return err
-	}
 	d.initialized = true
 	return nil
 }
-
-// Shutdown shuts down the store gracefully
-func (d *DiskStore) Shutdown() {
-}

store/disk_test.go

@@ -1,44 +0,0 @@
-package store
-
-import (
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestDiskStore_Get(t *testing.T) {
-	tmpDir, err := os.MkdirTemp("", "reflector_test_*")
-	require.NoError(t, err)
-	defer func() { _ = os.RemoveAll(tmpDir) }()
-	d := NewDiskStore(tmpDir, 2)
-
-	hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
-	data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
-
-	expectedPath := path.Join(tmpDir, hash[:2], hash)
-	err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
-	require.NoError(t, err)
-	err = os.WriteFile(expectedPath, data, os.ModePerm)
-	require.NoError(t, err)
-
-	blob, _, err := d.Get(hash)
-	assert.NoError(t, err)
-	assert.EqualValues(t, data, blob)
-}
-
-func TestDiskStore_GetNonexistentBlob(t *testing.T) {
-	tmpDir, err := os.MkdirTemp("", "reflector_test_*")
-	require.NoError(t, err)
-	defer func() { _ = os.RemoveAll(tmpDir) }()
-	d := NewDiskStore(tmpDir, 2)
-
-	blob, _, err := d.Get("nonexistent")
-	assert.Nil(t, blob)
-	assert.True(t, errors.Is(err, ErrBlobNotFound))
-}

store/diskstore_put_darwin.go

@@ -1,42 +0,0 @@
-//go:build darwin
-// +build darwin
-
-package store
-
-import (
-	"bytes"
-	"io"
-	"os"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
-)
-
-var openFileFlags = os.O_WRONLY | os.O_CREATE
-
-// Put stores the blob on disk
-func (d *DiskStore) Put(hash string, blob stream.Blob) error {
-	err := d.initOnce()
-	if err != nil {
-		return err
-	}
-
-	err = d.ensureDirExists(d.dir(hash))
-	if err != nil {
-		return err
-	}
-
-	// Open file with O_DIRECT
-	f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
-	if err != nil {
-		return errors.Err(err)
-	}
-	defer f.Close()
-
-	_, err = io.Copy(f, bytes.NewReader(blob))
-	if err != nil {
-		return errors.Err(err)
-	}
-	err = os.Rename(d.tmpPath(hash), d.path(hash))
-	return errors.Err(err)
-}

store/diskstore_put_linux.go

@@ -1,49 +0,0 @@
-//go:build linux
-// +build linux
-
-package store
-
-import (
-	"bytes"
-	"io"
-	"os"
-	"syscall"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
-
-	"github.com/brk0v/directio"
-)
-
-var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
-
-// Put stores the blob on disk
-func (d *DiskStore) Put(hash string, blob stream.Blob) error {
-	err := d.initOnce()
-	if err != nil {
-		return err
-	}
-
-	err = d.ensureDirExists(d.dir(hash))
-	if err != nil {
-		return err
-	}
-
-	// Open file with O_DIRECT
-	f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
-	if err != nil {
-		return errors.Err(err)
-	}
-	defer func() { _ = f.Close() }()
-	dio, err := directio.New(f)
-	if err != nil {
-		return errors.Err(err)
-	}
-	defer func() { _ = dio.Flush() }()
-	_, err = io.Copy(dio, bytes.NewReader(blob))
-	if err != nil {
-		return errors.Err(err)
-	}
-	err = os.Rename(d.tmpPath(hash), d.path(hash))
-	return errors.Err(err)
-}

store/gcache.go

@@ -1,163 +0,0 @@
-package store
-
-import (
-	"time"
-
-	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/shared"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
-
-	"github.com/bluele/gcache"
-	"github.com/sirupsen/logrus"
-)
-
-// GcacheStore adds a max cache size and Greedy-Dual-Size-Frequency cache eviction strategy to a BlobStore
-type GcacheStore struct {
-	// underlying store
-	store BlobStore
-	// cache implementation
-	cache gcache.Cache
-}
-type EvictionStrategy int
-
-const (
-	//LFU Discards the least frequently used items first.
-	LFU EvictionStrategy = iota
-	//ARC Constantly balances between LRU and LFU, to improve the combined result.
-	ARC
-	//LRU Discards the least recently used items first.
-	LRU
-	//SIMPLE has no clear priority for evict cache. It depends on key-value map order.
-	SIMPLE
-)
-
-// NewGcacheStore initialize a new LRUStore
-func NewGcacheStore(component string, store BlobStore, maxSize int, strategy EvictionStrategy) *GcacheStore {
-	cacheBuilder := gcache.New(maxSize)
-	var cache gcache.Cache
-	evictFunc := func(key interface{}, value interface{}) {
-		logrus.Infof("evicting %s", key)
-		metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
-		_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
-	}
-	switch strategy {
-	case LFU:
-		cache = cacheBuilder.LFU().EvictedFunc(evictFunc).Build()
-	case ARC:
-		cache = cacheBuilder.ARC().EvictedFunc(evictFunc).Build()
-	case LRU:
-		cache = cacheBuilder.LRU().EvictedFunc(evictFunc).Build()
-	case SIMPLE:
-		cache = cacheBuilder.Simple().EvictedFunc(evictFunc).Build()
-
-	}
-	l := &GcacheStore{
-		store: store,
-		cache: cache,
-	}
-	go func() {
-		if lstr, ok := store.(lister); ok {
-			err := l.loadExisting(lstr, maxSize)
-			if err != nil {
-				panic(err) // TODO: what should happen here? panic? return nil? just keep going?
-			}
-		}
-	}()
-
-	return l
-}
-
-const nameGcache = "gcache"
-
-// Name is the cache type name
-func (l *GcacheStore) Name() string { return nameGcache }
-
-// Has returns whether the blob is in the store, without updating the recent-ness.
-func (l *GcacheStore) Has(hash string) (bool, error) {
-	return l.cache.Has(hash), nil
-}
-
-// Get returns the blob or an error if the blob doesn't exist.
-func (l *GcacheStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
-	start := time.Now()
-	_, err := l.cache.Get(hash)
-	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
-	}
-	blob, stack, err := l.store.Get(hash)
-	if errors.Is(err, ErrBlobNotFound) {
-		// Blob disappeared from underlying store
-		l.cache.Remove(hash)
-	}
-	return blob, stack.Stack(time.Since(start), l.Name()), err
-}
-
-// Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
-func (l *GcacheStore) Put(hash string, blob stream.Blob) error {
-	_ = l.cache.Set(hash, true)
-	has, _ := l.Has(hash)
-	if has {
-		err := l.store.Put(hash, blob)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// PutSD stores the sd blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
-func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
-	_ = l.cache.Set(hash, true)
-	has, _ := l.Has(hash)
-	if has {
-		err := l.store.PutSD(hash, blob)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// Delete deletes the blob from the store
-func (l *GcacheStore) Delete(hash string) error {
-	err := l.store.Delete(hash)
-	if err != nil {
-		return err
-	}
-
-	// This must come after store.Delete()
-	// Remove triggers onEvict function, which also tries to delete blob from store
-	// We need to delete it manually first so any errors can be propagated up
-	l.cache.Remove(hash)
-	return nil
-}
-
-// loadExisting imports existing blobs from the underlying store into the LRU cache
-func (l *GcacheStore) loadExisting(store lister, maxItems int) error {
-	logrus.Infof("loading at most %d items", maxItems)
-	existing, err := store.list()
-	if err != nil {
-		return err
-	}
-	logrus.Infof("read %d files from underlying store", len(existing))
-
-	added := 0
-	for i, h := range existing {
-		_ = l.cache.Set(h, true)
-		added++
-		if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
-			err := l.Delete(h)
-			logrus.Infof("deleted overflowing blob: %s (%d/%d)", h, i, len(existing))
-			if err != nil {
-				logrus.Warnf("error while deleting a blob that's overflowing the cache: %s", err.Error())
-			}
-		}
-	}
-	return nil
-}
-
-// Shutdown shuts down the store gracefully
-func (l *GcacheStore) Shutdown() {
-}

store/http.go

@@ -1,170 +0,0 @@
-package store
-
-import (
-	"bytes"
-	"context"
-	"io"
-	"net"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/shared"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
-)
-
-// HttpStore is a store that works on top of the HTTP protocol
-type HttpStore struct {
-	upstream   string
-	httpClient *http.Client
-	edgeToken  string
-}
-
-func NewHttpStore(upstream, edgeToken string) *HttpStore {
-	return &HttpStore{
-		upstream:   "http://" + upstream,
-		httpClient: getClient(),
-		edgeToken:  edgeToken,
-	}
-}
-
-const nameHttp = "http"
-
-func (n *HttpStore) Name() string { return nameHttp }
-func (n *HttpStore) Has(hash string) (bool, error) {
-	url := n.upstream + "/blob?hash=" + hash
-
-	req, err := http.NewRequest("HEAD", url, nil)
-	if err != nil {
-		return false, errors.Err(err)
-	}
-
-	res, err := n.httpClient.Do(req)
-	if err != nil {
-		return false, errors.Err(err)
-	}
-	defer func() { _ = res.Body.Close() }()
-	if res.StatusCode == http.StatusNotFound {
-		return false, nil
-	}
-	if res.StatusCode == http.StatusNoContent {
-		return true, nil
-	}
-	var body []byte
-	if res.Body != nil {
-		body, _ = io.ReadAll(res.Body)
-	}
-	return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
-}
-
-func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
-	start := time.Now()
-	url := n.upstream + "/blob?hash=" + hash
-	if n.edgeToken != "" {
-		url += "&edge_token=" + n.edgeToken
-	}
-
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
-	}
-
-	res, err := n.httpClient.Do(req)
-	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
-	}
-	defer func() { _ = res.Body.Close() }()
-	tmp := getBuffer()
-	defer putBuffer(tmp)
-	serialized := res.Header.Get("Via")
-	trace := shared.NewBlobTrace(time.Since(start), n.Name())
-	if serialized != "" {
-		parsedTrace, err := shared.Deserialize(serialized)
-		if err != nil {
-			return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
-		}
-		trace = *parsedTrace
-	}
-
-	if res.StatusCode == http.StatusNotFound {
-		return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
-	}
-	if res.StatusCode == http.StatusOK {
-		written, err := io.Copy(tmp, res.Body)
-		if err != nil {
-			return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
-		}
-
-		blob := make([]byte, written)
-		copy(blob, tmp.Bytes())
-		metrics.MtrInBytesHttp.Add(float64(len(blob)))
-		return blob, trace.Stack(time.Since(start), n.Name()), nil
-	}
-	var body []byte
-	if res.Body != nil {
-		body, _ = io.ReadAll(res.Body)
-	}
-
-	return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
-}
-
-func (n *HttpStore) Put(string, stream.Blob) error {
-	return shared.ErrNotImplemented
-}
-func (n *HttpStore) PutSD(string, stream.Blob) error {
-	return shared.ErrNotImplemented
-}
-func (n *HttpStore) Delete(string) error {
-	return shared.ErrNotImplemented
-}
-func (n *HttpStore) Shutdown() {}
-
-// buffer pool to reduce GC
-// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
-var buffers = sync.Pool{
-	// New is called when a new instance is needed
-	New: func() interface{} {
-		buf := make([]byte, 0, stream.MaxBlobSize)
-		return bytes.NewBuffer(buf)
-	},
-}
-
-// getBuffer fetches a buffer from the pool
-func getBuffer() *bytes.Buffer {
-	return buffers.Get().(*bytes.Buffer)
-}
-
-// putBuffer returns a buffer to the pool
-func putBuffer(buf *bytes.Buffer) {
-	buf.Reset()
-	buffers.Put(buf)
-}
-
-func dialContext(ctx context.Context, network, address string) (net.Conn, error) {
-	dialer := &net.Dialer{
-		Timeout:   30 * time.Second,
-		KeepAlive: 30 * time.Second,
-	}
-	return dialer.DialContext(ctx, network, address)
-}
-
-// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
-func getClient() *http.Client {
-	// Customize the Transport to have larger connection pool
-	defaultTransport := &http.Transport{
-		DialContext:           dialContext,
-		ForceAttemptHTTP2:     true,
-		MaxIdleConns:          100,
-		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-		DisableCompression:    true,
-		MaxIdleConnsPerHost:   100,
-		ReadBufferSize:        stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
-	}
-
-	return &http.Client{Transport: defaultTransport}
-}

store/ittt.go

@@ -1,73 +0,0 @@
-package store
-
-import (
-	"time"
-
-	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/shared"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
-)
-
-// ITTTStore performs an operation on this storage, if this fails, it attempts to run it on that
-type ITTTStore struct {
-	this, that BlobStore
-}
-
-// NewITTTStore returns a new instance of the IF THIS THAN THAT store
-func NewITTTStore(this, that BlobStore) *ITTTStore {
-	return &ITTTStore{
-		this: this,
-		that: that,
-	}
-}
-
-const nameIttt = "ittt"
-
-// Name is the cache type name
-func (c *ITTTStore) Name() string { return nameIttt }
-
-// Has checks in this for a hash, if it fails it checks in that. It returns true if either store has it.
-func (c *ITTTStore) Has(hash string) (bool, error) {
-	has, err := c.this.Has(hash)
-	if err != nil || !has {
-		has, err = c.that.Has(hash)
-	}
-	return has, err
-}
-
-// Get tries to get the blob from this first, falling back to that.
-func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
-	start := time.Now()
-	blob, trace, err := c.this.Get(hash)
-	if err == nil {
-		metrics.ThisHitCount.Inc()
-		return blob, trace.Stack(time.Since(start), c.Name()), err
-	}
-
-	blob, trace, err = c.that.Get(hash)
-	if err != nil {
-		return nil, trace.Stack(time.Since(start), c.Name()), err
-	}
-	metrics.ThatHitCount.Inc()
-	return blob, trace.Stack(time.Since(start), c.Name()), nil
-}
-
-// Put not implemented
-func (c *ITTTStore) Put(hash string, blob stream.Blob) error {
-	return errors.Err(shared.ErrNotImplemented)
-}
-
-// PutSD not implemented
-func (c *ITTTStore) PutSD(hash string, blob stream.Blob) error {
-	return errors.Err(shared.ErrNotImplemented)
-}
-
-// Delete not implemented
-func (c *ITTTStore) Delete(hash string) error {
-	return errors.Err(shared.ErrNotImplemented)
-}
-
-// Shutdown shuts down the store gracefully
-func (c *ITTTStore) Shutdown() {}

store/lru.go

@@ -0,0 +1,120 @@
+package store
+
+import (
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/internal/metrics"
+
+	golru "github.com/hashicorp/golang-lru"
+)
+
+// LRUStore adds a max cache size and LRU eviction to a BlobStore
+type LRUStore struct {
+	// underlying store
+	store BlobStore
+	// lru implementation
+	lru *golru.Cache
+}
+
+// NewLRUStore initialize a new LRUStore
+func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore {
+	lru, err := golru.NewWithEvict(maxItems, func(key interface{}, value interface{}) {
+		metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
+		_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	l := &LRUStore{
+		store: store,
+		lru:   lru,
+	}
+
+	if lstr, ok := store.(lister); ok {
+		err = l.loadExisting(lstr, maxItems)
+		if err != nil {
+			panic(err) // TODO: what should happen here? panic? return nil? just keep going?
+		}
+	}
+
+	return l
+}
+
+const nameLRU = "lru"
+
+// Name is the cache type name
+func (l *LRUStore) Name() string { return nameLRU }
+
+// Has returns whether the blob is in the store, without updating the recent-ness.
+func (l *LRUStore) Has(hash string) (bool, error) {
+	return l.lru.Contains(hash), nil
+}
+
+// Get returns the blob or an error if the blob doesn't exist.
+func (l *LRUStore) Get(hash string) (stream.Blob, error) {
+	_, has := l.lru.Get(hash)
+	if !has {
+		return nil, errors.Err(ErrBlobNotFound)
+	}
+	blob, err := l.store.Get(hash)
+	if errors.Is(err, ErrBlobNotFound) {
+		// Blob disappeared from underlying store
+		l.lru.Remove(hash)
+	}
+	return blob, err
+}
+
+// Put stores the blob
+func (l *LRUStore) Put(hash string, blob stream.Blob) error {
+	err := l.store.Put(hash, blob)
+	if err != nil {
+		return err
+	}
+
+	l.lru.Add(hash, true)
+	return nil
+}
+
+// PutSD stores the sd blob
+func (l *LRUStore) PutSD(hash string, blob stream.Blob) error {
+	err := l.store.PutSD(hash, blob)
+	if err != nil {
+		return err
+	}
+
+	l.lru.Add(hash, true)
+	return nil
+}
+
+// Delete deletes the blob from the store
+func (l *LRUStore) Delete(hash string) error {
+	err := l.store.Delete(hash)
+	if err != nil {
+		return err
+	}
+
+	// This must come after store.Delete()
+	// Remove triggers onEvict function, which also tries to delete blob from store
+	// We need to delete it manually first so any errors can be propagated up
+	l.lru.Remove(hash)
+	return nil
+}
+
+// loadExisting imports existing blobs from the underlying store into the LRU cache
+func (l *LRUStore) loadExisting(store lister, maxItems int) error {
+	existing, err := store.list()
+	if err != nil {
+		return err
+	}
+
+	added := 0
+	for _, h := range existing {
+		l.lru.Add(h, true)
+		added++
+		if maxItems > 0 && added >= maxItems { // underlying cache is bigger than LRU cache
+			break
+		}
+	}
+	return nil
+}

store/lru_test.go

@@ -1,11 +1,10 @@
 package store
 
 import (
-	"fmt"
+	"io/ioutil"
 	"os"
 	"reflect"
 	"testing"
-	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
@@ -13,83 +12,96 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const cacheMaxSize = 3
+const cacheMaxBlobs = 3
 
-func getTestGcacheStore() (*GcacheStore, *MemStore) {
+func getTestLRUStore() (*LRUStore, *MemStore) {
 	m := NewMemStore()
-	return NewGcacheStore("test", m, cacheMaxSize, LFU), m
+	return NewLRUStore("test", m, 3), m
 }
 
-func TestGcacheStore_Eviction(t *testing.T) {
+func TestLRUStore_Eviction(t *testing.T) {
-	lfu, mem := getTestGcacheStore()
+	lru, mem := getTestLRUStore()
 	b := []byte("x")
-	for i := 0; i < 3; i++ {
+	err := lru.Put("one", b)
-		err := lfu.Put(fmt.Sprintf("%d", i), b)
-		require.NoError(t, err)
-		for j := 0; j < 3-i; j++ {
-			_, _, err = lfu.Get(fmt.Sprintf("%d", i))
-			require.NoError(t, err)
-		}
-	}
-	for k, v := range map[string]bool{
-		"0": true,
-		"1": true,
-		"2": true,
-	} {
-		has, err := lfu.Has(k)
-		assert.NoError(t, err)
-		assert.Equal(t, v, has)
-	}
-	err := lfu.Put("3", b)
 	require.NoError(t, err)
+	err = lru.Put("two", b)
+	require.NoError(t, err)
+	err = lru.Put("three", b)
+	require.NoError(t, err)
+	err = lru.Put("four", b)
+	require.NoError(t, err)
+	err = lru.Put("five", b)
+	require.NoError(t, err)
+
+	assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
+
 	for k, v := range map[string]bool{
-		"0": true,
+		"one":   false,
-		"1": true,
+		"two":   false,
-		"2": false,
+		"three": true,
-		"3": true,
+		"four":  true,
+		"five":  true,
+		"six":   false,
 	} {
-		has, err := lfu.Has(k)
+		has, err := lru.Has(k)
 		assert.NoError(t, err)
 		assert.Equal(t, v, has)
 	}
-	assert.Equal(t, cacheMaxSize, len(mem.Debug()))
 
-	err = lfu.Delete("0")
+	lru.Get("three") // touch so it stays in cache
+	lru.Put("six", b)
+
+	assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
+
+	for k, v := range map[string]bool{
+		"one":   false,
+		"two":   false,
+		"three": true,
+		"four":  false,
+		"five":  true,
+		"six":   true,
+	} {
+		has, err := lru.Has(k)
+		assert.NoError(t, err)
+		assert.Equal(t, v, has)
+	}
+
+	err = lru.Delete("three")
 	assert.NoError(t, err)
-	err = lfu.Delete("1")
+	err = lru.Delete("five")
 	assert.NoError(t, err)
-	err = lfu.Delete("3")
+	err = lru.Delete("six")
 	assert.NoError(t, err)
 	assert.Equal(t, 0, len(mem.Debug()))
 }
 
-func TestGcacheStore_UnderlyingBlobMissing(t *testing.T) {
+func TestLRUStore_UnderlyingBlobMissing(t *testing.T) {
-	lfu, mem := getTestGcacheStore()
+	lru, mem := getTestLRUStore()
 	hash := "hash"
 	b := []byte("this is a blob of stuff")
-	err := lfu.Put(hash, b)
+	err := lru.Put(hash, b)
 	require.NoError(t, err)
 
 	err = mem.Delete(hash)
 	require.NoError(t, err)
 
 	// hash still exists in lru
-	assert.True(t, lfu.cache.Has(hash))
+	assert.True(t, lru.lru.Contains(hash))
 
-	blob, _, err := lfu.Get(hash)
+	blob, err := lru.Get(hash)
 	assert.Nil(t, blob)
 	assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
 		reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
 		reflect.TypeOf(err).String(), err.Error())
 
 	// lru.Get() removes hash if underlying store doesn't have it
-	assert.False(t, lfu.cache.Has(hash))
+	assert.False(t, lru.lru.Contains(hash))
 }
 
-func TestGcacheStore_loadExisting(t *testing.T) {
+func TestLRUStore_loadExisting(t *testing.T) {
-	tmpDir, err := os.MkdirTemp("", "reflector_test_*")
+	tmpDir, err := ioutil.TempDir("", "reflector_test_*")
 	require.NoError(t, err)
-	defer func() { _ = os.RemoveAll(tmpDir) }()
+	defer os.RemoveAll(tmpDir)
 	d := NewDiskStore(tmpDir, 2)
 
 	hash := "hash"
@@ -102,9 +114,8 @@ func TestGcacheStore_loadExisting(t *testing.T) {
 	require.Equal(t, 1, len(existing), "blob should exist in cache")
 	assert.Equal(t, hash, existing[0])
 
-	lfu := NewGcacheStore("test", d, 3, LFU) // lru should load existing blobs when it's created
+	lru := NewLRUStore("test", d, 3) // lru should load existing blobs when it's created
-	time.Sleep(100 * time.Millisecond)       // async load so let's wait...
+	has, err := lru.Has(hash)
-	has, err := lfu.Has(hash)
 	require.NoError(t, err)
 	assert.True(t, has, "hash should be loaded from disk store but it's not")
 }

store/memory.go

@@ -2,9 +2,6 @@ package store
 
 import (
 	"sync"
-	"time"
-
-	"github.com/lbryio/reflector.go/shared"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
@@ -37,15 +34,14 @@ func (m *MemStore) Has(hash string) (bool, error) {
 }
 
 // Get returns the blob byte slice if present and errors if the blob is not found.
-func (m *MemStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (m *MemStore) Get(hash string) (stream.Blob, error) {
-	start := time.Now()
 	m.mu.RLock()
 	defer m.mu.RUnlock()
 	blob, ok := m.blobs[hash]
 	if !ok {
-		return nil, shared.NewBlobTrace(time.Since(start), m.Name()), errors.Err(ErrBlobNotFound)
+		return nil, errors.Err(ErrBlobNotFound)
 	}
-	return blob, shared.NewBlobTrace(time.Since(start), m.Name()), nil
+	return blob, nil
 }
 
 // Put stores the blob in memory
@@ -75,6 +71,3 @@ func (m *MemStore) Debug() map[string]stream.Blob {
 	defer m.mu.RUnlock()
 	return m.blobs
 }
-
-// Shutdown shuts down the store gracefully
-func (m *MemStore) Shutdown() {}

store/memory_test.go

@@ -25,7 +25,7 @@ func TestMemStore_Get(t *testing.T) {
 		t.Error("error getting memory blob - ", err)
 	}
 
-	gotBlob, _, err := s.Get(hash)
+	gotBlob, err := s.Get(hash)
 	if err != nil {
 		t.Errorf("Expected no error, got %v", err)
 	}
@@ -33,7 +33,7 @@ func TestMemStore_Get(t *testing.T) {
 		t.Error("Got blob that is different from expected blob")
 	}
 
-	missingBlob, _, err := s.Get("nonexistent hash")
+	missingBlob, err := s.Get("nonexistent hash")
 	if err == nil {
 		t.Errorf("Expected ErrBlobNotFound, got nil")
 	}

store/noop.go

@@ -1,24 +1,15 @@
 package store
 
-import (
+import "github.com/lbryio/lbry.go/v2/stream"
-	"time"
-
-	"github.com/lbryio/reflector.go/shared"
-
-	"github.com/lbryio/lbry.go/v2/stream"
-)
 
 // NoopStore is a store that does nothing
 type NoopStore struct{}
 
 const nameNoop = "noop"
 
-func (n *NoopStore) Name() string               { return nameNoop }
+func (n *NoopStore) Name() string                        { return nameNoop }
-func (n *NoopStore) Has(_ string) (bool, error) { return false, nil }
+func (n *NoopStore) Has(_ string) (bool, error)          { return false, nil }
-func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
+func (n *NoopStore) Get(_ string) (stream.Blob, error)   { return nil, nil }
-	return nil, shared.NewBlobTrace(time.Since(time.Now()), n.Name()), nil
-}
 func (n *NoopStore) Put(_ string, _ stream.Blob) error   { return nil }
 func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
 func (n *NoopStore) Delete(_ string) error               { return nil }
-func (n *NoopStore) Shutdown()                           { return }

store/s3.go

@@ -5,11 +5,9 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/shared"
-
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/internal/metrics"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -67,18 +65,17 @@ func (s *S3Store) Has(hash string) (bool, error) {
 }
 
 // Get returns the blob slice if present or errors on S3.
-func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (s *S3Store) Get(hash string) (stream.Blob, error) {
-	start := time.Now()
 	//Todo-Need to handle error for blob doesn't exist for consistency.
 	err := s.initOnce()
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
+		return nil, err
 	}
 
 	log.Debugf("Getting %s from S3", hash[:8])
 	defer func(t time.Time) {
 		log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
-	}(start)
+	}(time.Now())
 
 	buf := &aws.WriteAtBuffer{}
 	_, err = s3manager.NewDownloader(s.session).Download(buf, &s3.GetObjectInput{
@@ -89,15 +86,15 @@ func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 		if aerr, ok := err.(awserr.Error); ok {
 			switch aerr.Code() {
 			case s3.ErrCodeNoSuchBucket:
-				return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
+				return nil, errors.Err("bucket %s does not exist", s.bucket)
 			case s3.ErrCodeNoSuchKey:
-				return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
+				return nil, errors.Err(ErrBlobNotFound)
 			}
 		}
-		return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), err
+		return buf.Bytes(), err
 	}
 
-	return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), nil
+	return buf.Bytes(), nil
 }
 
 // Put stores the blob on S3 or errors if S3 connection errors.
@@ -113,11 +110,10 @@ func (s *S3Store) Put(hash string, blob stream.Blob) error {
 	}(time.Now())
 
 	_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
-		Bucket: aws.String(s.bucket),
+		Bucket:       aws.String(s.bucket),
-		Key:    aws.String(hash),
+		Key:          aws.String(hash),
-		Body:   bytes.NewBuffer(blob),
+		Body:         bytes.NewBuffer(blob),
-		ACL:    aws.String("public-read"),
+		StorageClass: aws.String(s3.StorageClassIntelligentTiering),
-		//StorageClass: aws.String(s3.StorageClassIntelligentTiering),
 	})
 	metrics.MtrOutBytesReflector.Add(float64(blob.Size()))
 
@@ -154,7 +150,6 @@ func (s *S3Store) initOnce() error {
 	sess, err := session.NewSession(&aws.Config{
 		Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
 		Region:      aws.String(s.region),
-		Endpoint:    aws.String("https://s3.wasabisys.com"),
 	})
 	if err != nil {
 		return err
@@ -163,8 +158,3 @@ func (s *S3Store) initOnce() error {
 	s.session = sess
 	return nil
 }
-
-// Shutdown shuts down the store gracefully
-func (s *S3Store) Shutdown() {
-	return
-}

store/singleflight.go

@@ -4,9 +4,7 @@ import (
 	"time"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/shared"
 
-	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 
 	"golang.org/x/sync/singleflight"
@@ -31,98 +29,39 @@ func (s *singleflightStore) Name() string {
 	return "sf_" + s.BlobStore.Name()
 }
 
-type getterResponse struct {
-	blob  stream.Blob
-	stack shared.BlobTrace
-}
-
 // Get ensures that only one request per hash is sent to the origin at a time,
 // thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
-func (s *singleflightStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+func (s *singleflightStore) Get(hash string) (stream.Blob, error) {
-	start := time.Now()
+	metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Inc()
-	metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
+	defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Dec()
-	defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
 
-	gr, err, _ := s.sf.Do(hash, s.getter(hash))
+	blob, err, _ := s.sf.Do(hash, s.getter(hash))
 	if err != nil {
-		return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
+		return nil, err
 	}
-	if gr == nil {
+	return blob.(stream.Blob), nil
-		return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("getter response is nil")
-	}
-	rsp := gr.(getterResponse)
-	return rsp.blob, rsp.stack, nil
 }
 
 // getter returns a function that gets a blob from the origin
 // only one getter per hash will be executing at a time
 func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
 	return func() (interface{}, error) {
-		metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
+		metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Inc()
-		defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
+		defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Dec()
 
 		start := time.Now()
-		blob, stack, err := s.BlobStore.Get(hash)
+		blob, err := s.BlobStore.Get(hash)
-		if err != nil {
-			return getterResponse{
-				blob:  nil,
-				stack: stack.Stack(time.Since(start), s.Name()),
-			}, err
-		}
-
-		rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
-		metrics.CacheRetrievalSpeed.With(map[string]string{
-			metrics.LabelCacheType: s.Name(),
-			metrics.LabelComponent: s.component,
-			metrics.LabelSource:    "origin",
-		}).Set(rate)
-
-		return getterResponse{
-			blob:  blob,
-			stack: stack.Stack(time.Since(start), s.Name()),
-		}, nil
-	}
-}
-
-// Put ensures that only one request per hash is sent to the origin at a time,
-// thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
-func (s *singleflightStore) Put(hash string, blob stream.Blob) error {
-	metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
-	defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
-
-	_, err, _ := s.sf.Do(hash, s.putter(hash, blob))
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// putter returns a function that puts a blob from the origin
-// only one putter per hash will be executing at a time
-func (s *singleflightStore) putter(hash string, blob stream.Blob) func() (interface{}, error) {
-	return func() (interface{}, error) {
-		metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
-		defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
-
-		start := time.Now()
-		err := s.BlobStore.Put(hash, blob)
 		if err != nil {
 			return nil, err
 		}
 
 		rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
 		metrics.CacheRetrievalSpeed.With(map[string]string{
-			metrics.LabelCacheType: s.Name(),
+			metrics.LabelCacheType: s.BlobStore.Name(),
 			metrics.LabelComponent: s.component,
 			metrics.LabelSource:    "origin",
 		}).Set(rate)
 
-		return nil, nil
+		return blob, nil
 	}
 }
-
-// Shutdown shuts down the store gracefully
-func (s *singleflightStore) Shutdown() {
-	s.BlobStore.Shutdown()
-	return
-}

store/speedwalk/speedwalk.go

@@ -1,14 +1,11 @@
 package speedwalk
 
 import (
-	"io/fs"
+	"io/ioutil"
-	"os"
 	"path/filepath"
 	"runtime"
 	"sync"
 
-	"github.com/lbryio/reflector.go/internal/metrics"
-
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
 	"github.com/karrick/godirwalk"
@@ -18,19 +15,7 @@ import (
 // AllFiles recursively lists every file in every subdirectory of a given directory
 // If basename is true, return the basename of each file. Otherwise return the full path starting at startDir.
 func AllFiles(startDir string, basename bool) ([]string, error) {
-	entries, err := os.ReadDir(startDir)
+	items, err := ioutil.ReadDir(startDir)
-	if err != nil {
-		return nil, err
-	}
-	items := make([]fs.FileInfo, 0, len(entries))
-	for _, entry := range entries {
-		info, err := entry.Info()
-		if err != nil {
-			return nil, err
-		}
-		items = append(items, info)
-	}
-
 	if err != nil {
 		return nil, err
 	}
@@ -39,7 +24,6 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
 	paths := make([]string, 0, 1000)
 	pathWG := &sync.WaitGroup{}
 	pathWG.Add(1)
-	metrics.RoutinesQueue.WithLabelValues("speedwalk", "worker").Inc()
 	go func() {
 		defer pathWG.Done()
 		for {
@@ -76,6 +60,7 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
 				walkerWG.Done()
 				goroutineLimiter <- struct{}{}
 			}()
+
 			err = godirwalk.Walk(filepath.Join(startDir, dir), &godirwalk.Options{
 				Unsorted: true, // faster this way
 				Callback: func(osPathname string, de *godirwalk.Dirent) error {
@@ -99,5 +84,6 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
 
 	close(pathChan)
 	pathWG.Wait()
+
 	return paths, nil
 }

store/store.go

@@ -1,8 +1,6 @@
 package store
 
 import (
-	"github.com/lbryio/reflector.go/shared"
-
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 )
@@ -11,18 +9,16 @@ import (
 type BlobStore interface {
 	// Name of blob store (useful for metrics)
 	Name() string
-	// Has Does blob exist in the store.
+	// Does blob exist in the store.
 	Has(hash string) (bool, error)
 	// Get the blob from the store. Must return ErrBlobNotFound if blob is not in store.
-	Get(hash string) (stream.Blob, shared.BlobTrace, error)
+	Get(hash string) (stream.Blob, error)
 	// Put the blob into the store.
 	Put(hash string, blob stream.Blob) error
-	// PutSD an SD blob into the store.
+	// Put an SD blob into the store.
 	PutSD(hash string, blob stream.Blob) error
 	// Delete the blob from the store.
 	Delete(hash string) error
-	// Shutdown the store gracefully
-	Shutdown()
 }
 
 // Blocklister is a store that supports blocking blobs to prevent their inclusion in the store.

wallet/client.go

@@ -3,10 +3,11 @@ package wallet
 import (
 	"encoding/base64"
 	"encoding/hex"
+	"strings"
 
 	"github.com/lbryio/chainquery/lbrycrd"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/schema/stake"
+	"github.com/lbryio/lbry.go/v2/schema/claim"
 	types "github.com/lbryio/types/v2/go"
 
 	"github.com/btcsuite/btcutil"
@@ -14,6 +15,8 @@ import (
 	"github.com/spf13/cast"
 )
 
+var ErrClaimNotFound = errors.Base("claim not found")
+
 // Raw makes a raw wallet server request
 func (n *Node) Raw(method string, params []string, v interface{}) error {
 	return n.request(method, params, v)
@@ -61,6 +64,12 @@ func (n *Node) Resolve(url string) (*types.Output, error) {
 	}
 
 	if e := outputs.GetTxos()[0].GetError(); e != nil {
+		//  TODO: return these errors as real error values so callers don't have to string-match
+		//  https://github.com/lbryio/types/blob/master/v2/proto/result.proto#L45
+		//  UNKNOWN_CODE = 0;
+		//  NOT_FOUND = 1;
+		//  INVALID = 2;
+		//  BLOCKED = 3;
 		return nil, errors.Err("%s: %s", e.GetCode(), e.GetText())
 	}
 
@@ -140,10 +149,35 @@ func (n *Node) GetClaimInTx(txid string, nout int) (*types.Claim, error) {
 		return nil, errors.Err(err)
 	}
 
-	ch, err := stake.DecodeClaimBytes(value, "")
+	ch, err := claim.DecodeClaimBytes(value, "")
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 
 	return ch.Claim, nil
 }
+
+func (n *Node) ResolveToClaim(url string) (*types.Claim, *types.Output, error) {
+	output, err := n.Resolve(url)
+	if err != nil {
+		if strings.Contains(err.Error(), "NOT_FOUND") {
+			return nil, nil, errors.Err(ErrClaimNotFound)
+		}
+		return nil, nil, err
+	}
+
+	claim, err := n.GetClaimInTx(hex.EncodeToString(rev(output.GetTxHash())), int(output.GetNout()))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return claim, output, nil
+}
+
+func rev(b []byte) []byte {
+	r := make([]byte, len(b))
+	for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 {
+		r[left], r[right] = b[right], b[left]
+	}
+	return r
+}

wallet/network.go

@@ -23,10 +23,10 @@ const (
 )
 
 var (
-	ErrNotImplemented = errors.Base("not implemented")
+	ErrNotImplemented   = errors.Base("not implemented")
-	ErrNodeConnected  = errors.Base("node already connected")
+	ErrAlreadyConnected = errors.Base("node already connected")
-	ErrConnectFailed  = errors.Base("failed to connect")
+	ErrConnectFailed    = errors.Base("failed to connect")
-	ErrTimeout        = errors.Base("timeout")
+	ErrTimeout          = errors.Base("timeout")
 )
 
 type response struct {
@@ -35,6 +35,8 @@ type response struct {
 }
 
 type Node struct {
+	ErrFn func(error)
+
 	transport *TCPTransport
 	nextId    atomic.Uint32
 	grp       *stop.Group
@@ -63,7 +65,7 @@ func NewNode() *Node {
 // Connect creates a new connection to the specified address.
 func (n *Node) Connect(addrs []string, config *tls.Config) error {
 	if n.transport != nil {
-		return errors.Err(ErrNodeConnected)
+		return errors.Err(ErrAlreadyConnected)
 	}
 
 	// shuffle addresses for load balancing
@@ -135,10 +137,13 @@ func (n *Node) handleErrors() {
 	}
 }
 
-// err handles errors produced by the foreign node.
+// err handles errors produced by the server
-func (n *Node) err(err error) {
+func (n *Node) err(e error) {
-	// TODO: Better error handling.
+	if n.ErrFn != nil {
-	log.Error(errors.FullTrace(err))
+		n.ErrFn(e)
+	} else {
+		log.Error(errors.FullTrace(e))
+	}
 }
 
 // listen processes messages from the server.