update dependencies

update dependencies

.github/workflows/go.yml

@@ -0,0 +1,37 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.x
+
+      - name: Build linux
+        run: make linux
+
+      - name: Build macos
+        run: make macos
+
+      - name: Test
+        run: make test
+
+      - name: Lint
+        run: make lint
+
+      - name: retrieve all tags
+        run: git fetch --prune --unshallow --tags
+
+      - name: Print changes since last version
+        run: git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline

.github/workflows/release.yml

@@ -0,0 +1,62 @@
+name: release
+
+on:
+  push:
+    tags:
+      - "*.*.*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.x
+
+      - name: Build linux
+        run: make linux
+
+      - name: Build macos
+        run: make macos
+
+      - name: Test
+        run: make test
+
+      - name: Lint
+        run: make lint
+
+      - name: Zip macos
+        run: zip -r reflector_darwin_amd64.zip ./dist/darwin_amd64
+
+      - name: Zip linux
+        run: zip -r reflector_linux_amd64.zip ./dist/linux_amd64
+
+      - name: retrieve all tags
+        run: git fetch --prune --unshallow --tags
+
+      - name: Generate Changelog
+        run: git log $(git describe --tags --abbrev=0 @^)..@ --no-merges --oneline > ${{ github.workspace }}-CHANGELOG.txt
+
+      - name: upload to github releases
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ./reflector_linux_amd64.zip
+            ./reflector_darwin_amd64.zip
+          body_path: ${{ github.workspace }}-CHANGELOG.txt
+
+#      - name: Login to DockerHub
+#        uses: docker/login-action@v2
+#        with:
+#          username: ${{ secrets.DOCKERHUB_USERNAME }}
+#          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+#      - name: Generate docker image
+#        run: make image
+
+#      - name: Docker push
+#        run: make publish_image

.gitignore

@@ -1,4 +1,4 @@
 /vendor
-/blobs
 /config.json*
-/prism-bin
+/dist
+/bin

.gometalinter.json

@@ -0,0 +1,4 @@
+{
+  "Disable": ["golint"],
+  "Vendor": true
+}

.travis.yml

@@ -1,59 +1,45 @@
 os: linux
-dist: trusty
+dist: bionic
 language: go
 
-# Only the last two Go releases are supported by the Go team with security
-# updates. Any versions older than that should be considered deprecated.
-# Don't bother testing with them. tip builds your code with the latest
-# development version of Go. This can warn you that your code will break
-# in the next version of Go. Don't worry! Later we declare that test runs
-# are allowed to fail on Go tip.
-matrix:
-  include:
-    - go: 1.10.2
-      env: DEPLOY=true # dont deploy tip build
-    - go: master
-      env: DEPLOY=false # dont deploy tip build
-  allow_failures:
-    - go: master
-  # Don't wait for tip tests to finish. Mark the test run green if the
-  # tests pass on the stable versions of Go.
-  fast_finish: true
+go:
+  - 1.20.x
+
+cache:
+  directories:
+    - $HOME/.cache/go-build
+    - $HOME/gopath/pkg/mod
 
-# Don't email me the results of the test runs.
 notifications:
   email: false
 
-# Skip the install step. Don't `go get` dependencies. Only build with the
-# code in vendor/
-install: true
+# Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
+#install: true
 
 # Anything in before_script that returns a nonzero exit code will
 # flunk the build and immediately stop. It's sorta like having
 # set -e enabled in bash.
 before_script:
 # All the .go files, excluding vendor/ and model (auto generated)
-  - GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ )
-  - go get golang.org/x/tools/cmd/goimports                     # Used in build script for generated files
-  - go get github.com/golang/lint/golint                        # Linter
-  - go get honnef.co/go/tools/cmd/megacheck                     # Badass static analyzer/linter
-  - go get github.com/jgautheron/gocyclo                        # Check against high complexity
-  - go get github.com/mdempsky/unconvert                        # Identifies unnecessary type conversions
-  - go get github.com/kisielk/errcheck                          # Checks for unhandled errors
-  - go get github.com/opennota/check/cmd/varcheck               # Checks for unused vars
-  - go get github.com/opennota/check/cmd/structcheck            # Checks for unused fields in structs
+  - GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p
+  - go install golang.org/x/tools/cmd/goimports                     # Used in build script for generated files
+# - go get github.com/golang/lint/golint                        # Linter
+#  - go get honnef.co/go/tools/cmd/megacheck                     # Badass static analyzer/linter
+  - go install github.com/fzipp/gocyclo/cmd/gocyclo@latest                 # Check against high complexity
+  - go install github.com/mdempsky/unconvert@latest                        # Identifies unnecessary type conversions
+  - go install github.com/kisielk/errcheck@latest                          # Checks for unhandled errors
+  - go install github.com/opennota/check/cmd/varcheck@latest               # Checks for unused vars
+  - go install github.com/opennota/check/cmd/structcheck@latest            # Checks for unused fields in structs
 
 
 
 # script always run to completion (set +e). All of these code checks are must haves
 # in a modern Go project.
 script:
-  # Build Prism successfully
-  - make
   # Fail if a .go file hasn't been formatted with gofmt
-  - test -z $(gofmt -s -l $GO_FILES)
+  - for i in $GO_FILES; do test -z $(gofmt -s -l $i); done
   # Run unit tests
-  - go test ./...
+  - make test
   # Checks for unused vars and fields on structs
   - varcheck ./...
   - structcheck ./...
@@ -62,21 +48,33 @@ script:
   # forbid code with huge functions
   #- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
   # checks for unhandled errors
-  - errcheck ./...
+  # - errcheck ./... # COMMENTED OUT UNTIL https://github.com/kisielk/errcheck/issues/155 is fixed
   # "go vet on steroids" + linter - ignore autogen code
-  - megacheck -simple.exit-non-zero=true ./...
+  # - megacheck -simple.exit-non-zero=true ./... # DISABLED UNTIL https://github.com/dominikh/go-tools/issues/328 is fixed
   # check for unnecessary conversions - ignore autogen code
-  - unconvert ./...
+  # - unconvert ./... # SEEMS TO BE BROKEN WITH GO MODULES
   # one last linter - ignore autogen code
   #- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
+  # Finally, build the binary
+  - make linux
 
 deploy:
-  - provider: releases
-    file: prism-bin
+  - provider: s3
+    local_dir: ./dist/linux_amd64
     skip_cleanup: true
     on:
       repo: lbryio/reflector.go
-      tags: true
-      condition: "$DEPLOY = true"
-    api_key:
-      secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
+    access_key_id:
+      secure: "ABmoSTxTee1GubJmmi+MlyBcnqRT0ywEOtTL4hrH7T+Vj5UPcNBhjqGRiMXzO3MdrVWom19OnqMvsvAdehrB4uQJVlX/+zB26gTpEiiU5dWJgerNAYmfxDGNz/p9pyYWLvVY/cymSlJ4HkggJVg+dMTBTOP+gTNgQUhHL7pAd/Z5HMPmPW/rlNlQgDdczoY44OdiniuHny4Lr7mo8dcT4dcyJysPpHLCPGWp3kXcffIEbgO02aQZ14BbX9oU9xm1BacYnr4t8gRCU+uK0mt7TREHUzZ8bQBRXUymmpeeEQshXdifdwtk6sdRGIZiTi0NmQ7kYter6oh6623TH6rpNy/lkxKi823FbSPC1XZGH7CwcbnSIEwY++7e9PT0EA14VuDUF7+iXC/gpRkVQwUoMcoC9rHBjkXW+SiwHWoUeDKXrfVO9QGEJTi5bDRNwklwu2zkYWeRIPzfDaVayUBYwjunjT1eST997ygmJwJTJ6DeOWLx+WFlmwBJfQpBwbrlMrceiNjreMwGB6ffXEBzuM6S5inXGIChllGDpNyJeFVcH9zPpU0JsIpWoRjwLNFOiEhFsnzK5rvA2SKmlOOnSlgOCBQnFi2SmLDM790KoY8qjan2s984chQz2qsKnhYuVpJhG6LKgrI1QwE0KHr80spVFfewq11AU4Q+lfA9aAQ="
+    secret_access_key:
+      secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
+    bucket: "lbry-reflector-binary"
+
+#  - provider: releases
+#    file: ./bin/prism-bin
+#    skip_cleanup: true
+#    on:
+#      repo: lbryio/reflector.go
+#      tags: true
+#    api_key:
+#      secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=

Dockerfile

@@ -0,0 +1,9 @@
+FROM alpine
+EXPOSE 8080
+
+RUN mkdir /app
+WORKDIR /app
+COPY dist/linux_amd64/prism-bin ./prism
+RUN chmod +x prism
+
+ENTRYPOINT [ "/app/prism" ]

Gopkg.lock

@@ -1,435 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  digest = "1:d64110a78451e373c5a952d2625323dbbe3bfe41c67f9652ea9668a6ceb4f645"
-  name = "github.com/armon/go-metrics"
-  packages = ["."]
-  pruneopts = ""
-  revision = "3c58d8115a78a6879e5df75ae900846768d36895"
-
-[[projects]]
-  branch = "master"
-  digest = "1:c0b6dbbb56a745020d5939bdde2197241a1c6109f226cca57b16f46916be5e94"
-  name = "github.com/aws/aws-sdk-go"
-  packages = [
-    "aws",
-    "aws/awserr",
-    "aws/awsutil",
-    "aws/client",
-    "aws/client/metadata",
-    "aws/corehandlers",
-    "aws/credentials",
-    "aws/credentials/ec2rolecreds",
-    "aws/credentials/endpointcreds",
-    "aws/credentials/stscreds",
-    "aws/csm",
-    "aws/defaults",
-    "aws/ec2metadata",
-    "aws/endpoints",
-    "aws/request",
-    "aws/session",
-    "aws/signer/v4",
-    "internal/sdkio",
-    "internal/sdkrand",
-    "internal/sdkuri",
-    "internal/shareddefaults",
-    "private/protocol",
-    "private/protocol/eventstream",
-    "private/protocol/eventstream/eventstreamapi",
-    "private/protocol/query",
-    "private/protocol/query/queryutil",
-    "private/protocol/rest",
-    "private/protocol/restxml",
-    "private/protocol/xml/xmlutil",
-    "service/s3",
-    "service/s3/s3iface",
-    "service/s3/s3manager",
-    "service/sts",
-  ]
-  pruneopts = ""
-  revision = "c0447dbaaf195bb477fd2d511b8e4665e04b9017"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e250643be8120824556f39df6ef128fc2be490fc96e0cb64b1a8ecf96bbe3ce6"
-  name = "github.com/btcsuite/btcutil"
-  packages = ["base58"]
-  pruneopts = ""
-  revision = "ab6388e0c60ae4834a1f57511e20c17b5f78be4b"
-
-[[projects]]
-  digest = "1:0a39ec8bf5629610a4bc7873a92039ee509246da3cef1a0ea60f1ed7e5f9cea5"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = ""
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a"
-  name = "github.com/go-errors/errors"
-  packages = ["."]
-  pruneopts = ""
-  revision = "a6af135bd4e28680facf08a3d206b454abc877a4"
-  version = "v1.0.1"
-
-[[projects]]
-  digest = "1:d19c78214e03e297e9e30d2eb11892f731358b2951f2a5c7374658a156373e4c"
-  name = "github.com/go-ini/ini"
-  packages = ["."]
-  pruneopts = ""
-  revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
-  version = "v1.38.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:27b11ca1ad214ead955ff5480e8575e74c5df4e4dc02b04256a8d92131e1d3ad"
-  name = "github.com/go-sql-driver/mysql"
-  packages = ["."]
-  pruneopts = ""
-  revision = "99ff426eb706cffe92ff3d058e168b278cabf7c7"
-
-[[projects]]
-  digest = "1:dbbeb8ddb0be949954c8157ee8439c2adfd8dc1c9510eb44a6e58cb68c3dce28"
-  name = "github.com/gorilla/context"
-  packages = ["."]
-  pruneopts = ""
-  revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42"
-  version = "v1.1.1"
-
-[[projects]]
-  digest = "1:c2c8666b4836c81a1d247bdf21c6a6fc1ab586538ab56f74437c2e0df5c375e1"
-  name = "github.com/gorilla/mux"
-  packages = ["."]
-  pruneopts = ""
-  revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf"
-  version = "v1.6.2"
-
-[[projects]]
-  digest = "1:b4817bdb0da3054166de058111943ac58b315aead2fed4ee838625a4e304f74c"
-  name = "github.com/gorilla/rpc"
-  packages = [
-    "v2",
-    "v2/json",
-  ]
-  pruneopts = ""
-  revision = "22c016f3df3febe0c1f6727598b6389507e03a18"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:fe1b4d4cbe48c0d55507c55f8663aa4185576cc58fa0c8be03bb8f19dfe17a9c"
-  name = "github.com/gorilla/websocket"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:4fe55793760295fbef367890352b720784243e0ad19b5ee242519a4682bb9ef8"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  pruneopts = ""
-  revision = "d6c0cd88035724dd42e0f335ae30161c20575ecc"
-
-[[projects]]
-  branch = "master"
-  digest = "1:4423ee95d6ee30bb22f680445c58889bb5b91e1b955405bf34374a053784a8a2"
-  name = "github.com/hashicorp/go-immutable-radix"
-  packages = ["."]
-  pruneopts = ""
-  revision = "7f3cd4390caab3250a57f30efdb2a65dd7649ecf"
-
-[[projects]]
-  branch = "master"
-  digest = "1:6a611e691e739173805cb54019b5c39bb9d46455526dff31e0e6fe3aaca52776"
-  name = "github.com/hashicorp/go-msgpack"
-  packages = ["codec"]
-  pruneopts = ""
-  revision = "fa3f63826f7c23912c15263591e65d54d080b458"
-
-[[projects]]
-  branch = "master"
-  digest = "1:deaebb4a98ca748bbad7eb653f3a675749500020823a086448ffcd7ba6b8b02d"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  pruneopts = ""
-  revision = "3d5d8f294aa03d8e98859feac328afbdf1ae0703"
-
-[[projects]]
-  branch = "master"
-  digest = "1:74f54e6ef2339f1de1e8c4b6674442118bd89e619b2fbd949ef2337330067994"
-  name = "github.com/hashicorp/go-sockaddr"
-  packages = ["."]
-  pruneopts = ""
-  revision = "6d291a969b86c4b633730bfc6b8b9d64c3aafed9"
-
-[[projects]]
-  branch = "master"
-  digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
-  name = "github.com/hashicorp/golang-lru"
-  packages = ["simplelru"]
-  pruneopts = ""
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  digest = "1:d7ce65372f495908f80fc1f80f4dab5d763d9a1de544abd95aa719e4262d0dd5"
-  name = "github.com/hashicorp/memberlist"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ce8abaa0c60c2d6bee7219f5ddf500e0a1457b28"
-  version = "v0.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:4fd01ac3766b886665cfd335cc63819ec4e4538dcc1180c05d6edc089619962c"
-  name = "github.com/hashicorp/serf"
-  packages = [
-    "coordinate",
-    "serf",
-  ]
-  pruneopts = ""
-  revision = "984a73625de3138f44deb38d00878fab39eb6447"
-
-[[projects]]
-  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  pruneopts = ""
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  digest = "1:4f767a115bc8e08576f6d38ab73c376fc1b1cd3bb5041171c9e8668cc7739b52"
-  name = "github.com/jmespath/go-jmespath"
-  packages = ["."]
-  pruneopts = ""
-  revision = "0b12d6b5"
-
-[[projects]]
-  branch = "master"
-  digest = "1:1e4cb5c6d4e92e2fb34f1545e071fb61e81c71fc0c325445db19b9f3981cb343"
-  name = "github.com/johntdyer/slack-go"
-  packages = ["."]
-  pruneopts = ""
-  revision = "95fac1160b220c5abcf8b0ef88e9c3cb213c09f4"
-
-[[projects]]
-  branch = "master"
-  digest = "1:132aae3fa5ad407b53f57cf6ebe274e414ec22d1700899023319931fa90b63e2"
-  name = "github.com/johntdyer/slackrus"
-  packages = ["."]
-  pruneopts = ""
-  revision = "f7aae3243a0786c5a974bce71ed951c459876e64"
-
-[[projects]]
-  branch = "master"
-  digest = "1:65bb92382468ce7a1ee218b5f352cd0927d358569be7521dd7cb42e74589a308"
-  name = "github.com/lbryio/errors.go"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ad03d3cc6a5c27c94bbe3412cf7de0eae1a9bd7c"
-
-[[projects]]
-  branch = "master"
-  digest = "1:76fd7507e6014c598a01f1b3d558774d2a3114c438403bc98123870d2aecec62"
-  name = "github.com/lbryio/lbry.go"
-  packages = [
-    "crypto",
-    "errors",
-    "null",
-    "querytools",
-    "stop",
-    "util",
-  ]
-  pruneopts = ""
-  revision = "e2c96944fc485d3ab5e164da78f8439a94c5aa85"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cabf2bf5e49edfe0c34cb9c6a256f2a99e6cc8c5e660855c8f3dafe1f81d5dcd"
-  name = "github.com/lyoshenka/bencode"
-  packages = ["."]
-  pruneopts = ""
-  revision = "b7abd7672df533e627eddbf3a5a529786e8bda7f"
-
-[[projects]]
-  digest = "1:f0bad0fece0fb73c6ea249c18d8e80ffbe86be0457715b04463068f04686cf39"
-  name = "github.com/miekg/dns"
-  packages = ["."]
-  pruneopts = ""
-  revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1"
-  version = "v1.0.8"
-
-[[projects]]
-  digest = "1:ba3b2eb0ae6fd3deac4386c02fe9d2279c9520738eb9db2f0667e74d5c7a0a61"
-  name = "github.com/nlopes/slack"
-  packages = ["."]
-  pruneopts = ""
-  revision = "0db1d5eae1116bf7c8ed96c6749acfbf4daaec3e"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:af967afd3cbc6b0145937f4dcab78bcd93e7b2f2b618fb2bcaf7069ad5c638fa"
-  name = "github.com/phayes/freeport"
-  packages = ["."]
-  pruneopts = ""
-  revision = "e27662a4a9d6b2083dfd7e7b5d0e30985daca925"
-
-[[projects]]
-  branch = "master"
-  digest = "1:6ee36f2cea425916d81fdaaf983469fc18f91b3cf090cfe90fa0a9d85b8bfab7"
-  name = "github.com/sean-/seed"
-  packages = ["."]
-  pruneopts = ""
-  revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
-
-[[projects]]
-  branch = "master"
-  digest = "1:65d0c541c450f6f8bb9654a3f1938407a835bf41faade00bb0fa1416de215d00"
-  name = "github.com/sebdah/goldie"
-  packages = ["."]
-  pruneopts = ""
-  revision = "8784dd1ab561dcf43d141f6678e9e41f3d0dff55"
-
-[[projects]]
-  branch = "master"
-  digest = "1:1747c026a603e3c9f33f238e1d1390df2c8f48876b6bcb7a9c52c7b479e040f4"
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  pruneopts = ""
-  revision = "d329d24db4313262a3b0a24d8aeb1dc4bd294fb0"
-
-[[projects]]
-  digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
-  name = "github.com/spf13/cast"
-  packages = ["."]
-  pruneopts = ""
-  revision = "8965335b8c7107321228e3e3702cab9832751bac"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b5212c335a490d958a6b1b5b48901b46e682b82fa9af3a238fa88df6eaa60873"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  pruneopts = ""
-  revision = "7c4570c3ebeb8129a1f7456d0908a8b676b6f9f1"
-
-[[projects]]
-  digest = "1:8e243c568f36b09031ec18dff5f7d2769dcf5ca4d624ea511c8e3197dc3d352d"
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  pruneopts = ""
-  revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:f05efcac20bea32e1fcefde9a3cbefb07e02053666c4a67681ad18c8efc682d3"
-  name = "github.com/uber-go/atomic"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ca680462431f8c7f96686d91b1d9e5203c4b075b"
-
-[[projects]]
-  branch = "master"
-  digest = "1:16b935c128f178647036048862a21e8bfd66d1e83fb19787a8b356bdcf0de899"
-  name = "golang.org/x/crypto"
-  packages = [
-    "ed25519",
-    "ed25519/internal/edwards25519",
-    "ripemd160",
-    "sha3",
-    "ssh/terminal",
-  ]
-  pruneopts = ""
-  revision = "f027049dab0ad238e394a753dba2d14753473a04"
-
-[[projects]]
-  branch = "master"
-  digest = "1:15f2fc8cc79d90b0d4d712f04bd3eb3a3856ff3dd6b610a1425113ead3501610"
-  name = "golang.org/x/net"
-  packages = [
-    "bpf",
-    "context",
-    "internal/iana",
-    "internal/socket",
-    "ipv4",
-    "ipv6",
-  ]
-  pruneopts = ""
-  revision = "19491d39cadbd9cd33f26ca22cc89ba4ba38251c"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e8f649ecfae7835a0a27ef39fd2180f6d3c12bc422e2ae55cd611f0b283b3e6e"
-  name = "golang.org/x/sys"
-  packages = [
-    "unix",
-    "windows",
-  ]
-  pruneopts = ""
-  revision = "0718ef2ef256118d53a01598f179001ec2af7626"
-
-[[projects]]
-  branch = "master"
-  digest = "1:55a681cb66f28755765fa5fa5104cbd8dc85c55c02d206f9f89566451e3fe1aa"
-  name = "golang.org/x/time"
-  packages = ["rate"]
-  pruneopts = ""
-  revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
-
-[[projects]]
-  digest = "1:eede11c81b63c8f6fd06ef24ba0a640dc077196ec9b7a58ecde03c82eee2f151"
-  name = "google.golang.org/appengine"
-  packages = ["cloudsql"]
-  pruneopts = ""
-  revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:05eca53b271663de74078b5484b1995a8d56668a51434a698dc5d0863035d575"
-  name = "gopkg.in/nullbio/null.v6"
-  packages = ["convert"]
-  pruneopts = ""
-  revision = "40264a2e6b7972d183906cf17663983c23231c82"
-  version = "v6.3"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/aws/aws-sdk-go/aws",
-    "github.com/aws/aws-sdk-go/aws/awserr",
-    "github.com/aws/aws-sdk-go/aws/credentials",
-    "github.com/aws/aws-sdk-go/aws/session",
-    "github.com/aws/aws-sdk-go/service/s3",
-    "github.com/aws/aws-sdk-go/service/s3/s3manager",
-    "github.com/davecgh/go-spew/spew",
-    "github.com/go-sql-driver/mysql",
-    "github.com/gorilla/mux",
-    "github.com/gorilla/rpc/v2",
-    "github.com/gorilla/rpc/v2/json",
-    "github.com/hashicorp/serf/serf",
-    "github.com/johntdyer/slackrus",
-    "github.com/lbryio/errors.go",
-    "github.com/lbryio/lbry.go/crypto",
-    "github.com/lbryio/lbry.go/errors",
-    "github.com/lbryio/lbry.go/querytools",
-    "github.com/lbryio/lbry.go/stop",
-    "github.com/lbryio/lbry.go/util",
-    "github.com/lyoshenka/bencode",
-    "github.com/phayes/freeport",
-    "github.com/sebdah/goldie",
-    "github.com/sirupsen/logrus",
-    "github.com/spf13/cast",
-    "github.com/spf13/cobra",
-    "github.com/uber-go/atomic",
-    "golang.org/x/time/rate",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,39 +0,0 @@
-[[constraint]]
-  name = "github.com/aws/aws-sdk-go"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/go-sql-driver/mysql"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/hashicorp/serf"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/lbryio/lbry.go"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/sirupsen/logrus"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/spf13/cobra"
-  branch = "master"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/lyoshenka/bencode"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/uber-go/atomic"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/phayes/freeport"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/johntdyer/slackrus"

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2018 LBRY Inc
+Copyright (c) 2016-2020 LBRY Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

Makefile

@@ -1,27 +1,33 @@
+version := $(shell git describe --dirty --always --long --abbrev=7)
+commit := $(shell git rev-parse --short HEAD)
+commit_long := $(shell git rev-parse HEAD)
+branch := $(shell git rev-parse --abbrev-ref HEAD)
+curTime := $(shell date +%s)
+
 BINARY=prism-bin
-
-DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
-VENDOR_DIR = vendor
 IMPORT_PATH = github.com/lbryio/reflector.go
+LDFLAGS="-X ${IMPORT_PATH}/meta.version=$(version) -X ${IMPORT_PATH}/meta.commit=$(commit) -X ${IMPORT_PATH}/meta.commitLong=$(commit_long) -X ${IMPORT_PATH}/meta.branch=$(branch) -X '${IMPORT_PATH}/meta.Time=$(curTime)'"
+DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
+BIN_DIR = $(DIR)/dist
 
-VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
-LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION}"
-
-
-.PHONY: build dep clean test
-.DEFAULT_GOAL: build
-
-
-build: dep
-	CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${DIR}/${BINARY} main.go
-
-dep: | $(VENDOR_DIR)
-
-$(VENDOR_DIR):
-	go get github.com/golang/dep/cmd/dep && dep ensure
-
-clean:
-	if [ -f ${DIR}/${BINARY} ]; then rm ${DIR}/${BINARY}; fi
+.DEFAULT_GOAL := linux
 
+.PHONY: test
 test:
-	go test ./... -v -cover
+	go test -cover -v ./...
+
+.PHONY: lint
+lint:
+	./scripts/lint.sh
+
+.PHONY: linux
+linux:
+	GOARCH=amd64 GOOS=linux go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/linux_amd64/${BINARY}
+
+.PHONY: macos
+macos:
+	GOARCH=amd64 GOOS=darwin go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/darwin_amd64/${BINARY}
+
+.PHONY: image
+image:
+	docker buildx build -t lbry/reflector:$(version) -t lbry/reflector:latest --platform linux/amd64 .

cluster/cluster.go

@@ -1,14 +1,14 @@
 package cluster
 
 import (
-	"io/ioutil"
+	"io"
 	baselog "log"
 	"sort"
 	"time"
 
-	"github.com/lbryio/lbry.go/crypto"
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/stop"
+	"github.com/lbryio/lbry.go/v2/extras/crypto"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
 
 	"github.com/hashicorp/serf/serf"
 	log "github.com/sirupsen/logrus"
@@ -52,7 +52,7 @@ func (c *Cluster) Connect() error {
 	conf.MemberlistConfig.AdvertisePort = c.port
 	conf.NodeName = c.name
 
-	nullLogger := baselog.New(ioutil.Discard, "", 0)
+	nullLogger := baselog.New(io.Discard, "", 0)
 	conf.Logger = nullLogger
 
 	c.eventCh = make(chan serf.Event)

cmd/cluster.go

@@ -6,7 +6,8 @@ import (
 	"strconv"
 	"syscall"
 
-	"github.com/lbryio/lbry.go/crypto"
+	"github.com/lbryio/lbry.go/v2/extras/crypto"
+
 	"github.com/lbryio/reflector.go/cluster"
 
 	log "github.com/sirupsen/logrus"

cmd/decode.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	"encoding/hex"
+	"fmt"
+
+	"github.com/lbryio/lbry.go/v2/schema/stake"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/gogo/protobuf/jsonpb"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "decode VALUE",
+		Short: "Decode a claim value",
+		Args:  cobra.ExactArgs(1),
+		Run:   decodeCmd,
+	}
+	rootCmd.AddCommand(cmd)
+}
+
+func decodeCmd(cmd *cobra.Command, args []string) {
+	c, err := stake.DecodeClaimHex(args[0], "")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	m := jsonpb.Marshaler{Indent: "  "}
+
+	if stream := c.Claim.GetStream(); stream != nil {
+		json, err := m.MarshalToString(stream)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println(json)
+		fmt.Printf("SD hash as hex: %s\n", hex.EncodeToString(stream.GetSource().GetSdHash()))
+	} else if channel := c.Claim.GetChannel(); channel != nil {
+		json, err := m.MarshalToString(channel)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println(json)
+	} else if repost := c.Claim.GetRepost(); repost != nil {
+		json, err := m.MarshalToString(repost)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println(json)
+	} else {
+		spew.Dump(c)
+	}
+}

cmd/dht.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"log"
 	"net"
 	"os"
 	"os/signal"
@@ -9,15 +8,16 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/lbryio/reflector.go/dht"
-	"github.com/lbryio/reflector.go/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
 var dhtNodeID string
 var dhtPort int
-var dhtRpcPort int
+var dhtRPCPort int
 var dhtSeeds []string
 
 func init() {
@@ -30,7 +30,7 @@ func init() {
 	}
 	cmd.PersistentFlags().StringVar(&dhtNodeID, "nodeID", "", "nodeID in hex")
 	cmd.PersistentFlags().IntVar(&dhtPort, "port", 4567, "Port to start DHT on")
-	cmd.PersistentFlags().IntVar(&dhtRpcPort, "rpcPort", 0, "Port to listen for rpc commands on")
+	cmd.PersistentFlags().IntVar(&dhtRPCPort, "rpcPort", 0, "Port to listen for rpc commands on")
 	cmd.PersistentFlags().StringSliceVar(&dhtSeeds, "seeds", []string{}, "Addresses of seed nodes")
 	rootCmd.AddCommand(cmd)
 }
@@ -56,7 +56,7 @@ func dhtCmd(cmd *cobra.Command, args []string) {
 
 		dhtConf := dht.NewStandardConfig()
 		dhtConf.Address = "0.0.0.0:" + strconv.Itoa(dhtPort)
-		dhtConf.RPCPort = dhtRpcPort
+		dhtConf.RPCPort = dhtRPCPort
 		if len(dhtSeeds) > 0 {
 			dhtConf.SeedNodes = dhtSeeds
 		}

cmd/getstream.go

@@ -0,0 +1,80 @@
+package cmd
+
+import (
+	"encoding/hex"
+	"os"
+	"time"
+
+	"github.com/lbryio/reflector.go/server/peer"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "getstream ADDRESS:PORT SDHASH",
+		Short: "Get a stream from a reflector server",
+		Args:  cobra.ExactArgs(2),
+		Run:   getStreamCmd,
+	}
+	rootCmd.AddCommand(cmd)
+}
+
+func getStreamCmd(cmd *cobra.Command, args []string) {
+	addr := args[0]
+	sdHash := args[1]
+
+	s := store.NewCachingStore(
+		"getstream",
+		peer.NewStore(peer.StoreOpts{Address: addr}),
+		store.NewDiskStore("/tmp/lbry_downloaded_blobs", 2),
+	)
+
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var sd stream.SDBlob
+
+	sdb, _, err := s.Get(sdHash)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = sd.FromBlob(sdb)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	filename := sd.SuggestedFileName
+	if filename == "" {
+		filename = "stream_" + time.Now().Format("20060102_150405")
+	}
+
+	f, err := os.Create(wd + "/" + filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for i := 0; i < len(sd.BlobInfos)-1; i++ {
+		b, _, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		data, err := b.Plaintext(sd.Key, sd.BlobInfos[i].IV)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		_, err = f.Write(data)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}

cmd/integrity.go

@@ -0,0 +1,93 @@
+package cmd
+
+import (
+	"crypto/sha512"
+	"encoding/hex"
+	"os"
+	"path"
+	"runtime"
+	"sync/atomic"
+	"time"
+
+	"github.com/lbryio/reflector.go/meta"
+	"github.com/lbryio/reflector.go/store/speedwalk"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var threads int
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "check-integrity",
+		Short: "check blobs integrity for a given path",
+		Run:   integrityCheckCmd,
+	}
+	cmd.Flags().StringVar(&diskStorePath, "store-path", "", "path of the store where all blobs are cached")
+	cmd.Flags().IntVar(&threads, "threads", runtime.NumCPU()-1, "number of concurrent threads to process blobs")
+	rootCmd.AddCommand(cmd)
+}
+
+func integrityCheckCmd(cmd *cobra.Command, args []string) {
+	log.Printf("reflector %s", meta.VersionString())
+	if diskStorePath == "" {
+		log.Fatal("store-path must be defined")
+	}
+
+	blobs, err := speedwalk.AllFiles(diskStorePath, true)
+	if err != nil {
+		log.Fatalf("error while reading blobs from disk %s", errors.FullTrace(err))
+	}
+	tasks := make(chan string, len(blobs))
+	done := make(chan bool)
+	processed := new(int32)
+	go produce(tasks, blobs)
+	cpus := runtime.NumCPU()
+	for i := 0; i < cpus-1; i++ {
+		go consume(i, tasks, done, len(blobs), processed)
+	}
+	<-done
+}
+
+func produce(tasks chan<- string, blobs []string) {
+	for _, b := range blobs {
+		tasks <- b
+	}
+	close(tasks)
+}
+
+func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int, processed *int32) {
+	start := time.Now()
+
+	for b := range tasks {
+		processedSoFar := atomic.AddInt32(processed, 1)
+		if worker == 0 {
+			remaining := int32(totalTasks) - processedSoFar
+			timePerBlob := time.Since(start).Microseconds() / int64(processedSoFar)
+			remainingTime := time.Duration(int64(remaining)*timePerBlob) * time.Microsecond
+			log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
+		}
+		blobPath := path.Join(diskStorePath, b[:2], b)
+		blob, err := os.ReadFile(blobPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			log.Errorf("[Worker %d] Error looking up blob %s: %s", worker, b, err.Error())
+			continue
+		}
+		hashBytes := sha512.Sum384(blob)
+		readHash := hex.EncodeToString(hashBytes[:])
+		if readHash != b {
+			log.Infof("[%s] found a broken blob while reading from disk. Actual hash: %s", b, readHash)
+			err := os.Remove(blobPath)
+			if err != nil {
+				log.Errorf("Error while deleting broken blob %s: %s", b, err.Error())
+			}
+		}
+	}
+	done <- true
+}

cmd/peer.go

@@ -7,30 +7,41 @@ import (
 	"syscall"
 
 	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/peer"
+	"github.com/lbryio/reflector.go/server/peer"
 	"github.com/lbryio/reflector.go/store"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
+var peerNoDB bool
+
 func init() {
 	var cmd = &cobra.Command{
 		Use:   "peer",
 		Short: "Run peer server",
 		Run:   peerCmd,
 	}
+	cmd.Flags().BoolVar(&peerNoDB, "nodb", false, "Don't connect to a db and don't use a db-backed blob store")
 	rootCmd.AddCommand(cmd)
 }
 
 func peerCmd(cmd *cobra.Command, args []string) {
-	db := new(db.SQL)
-	err := db.Connect(globalConfig.DBConn)
-	checkErr(err)
+	var err error
 
-	s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
-	combo := store.NewDBBackedS3Store(s3, db)
-	peerServer := peer.NewServer(combo)
+	s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
+	peerServer := peer.NewServer(s3)
+
+	if !peerNoDB {
+		db := &db.SQL{
+			LogQueries: log.GetLevel() == log.DebugLevel,
+		}
+		err = db.Connect(globalConfig.DBConn)
+		checkErr(err)
+
+		combo := store.NewDBBackedStore(s3, db, false)
+		peerServer = peer.NewServer(combo)
+	}
 
 	err = peerServer.Start(":" + strconv.Itoa(peer.DefaultPort))
 	if err != nil {

cmd/populatedb.go

@@ -0,0 +1,51 @@
+package cmd
+
+import (
+	"github.com/lbryio/reflector.go/db"
+	"github.com/lbryio/reflector.go/meta"
+	"github.com/lbryio/reflector.go/store/speedwalk"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var (
+	diskStorePath string
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "populate-db",
+		Short: "populate local database with blobs from a disk storage",
+		Run:   populateDbCmd,
+	}
+	cmd.Flags().StringVar(&diskStorePath, "store-path", "",
+		"path of the store where all blobs are cached")
+	rootCmd.AddCommand(cmd)
+}
+
+func populateDbCmd(cmd *cobra.Command, args []string) {
+	log.Printf("reflector %s", meta.VersionString())
+	if diskStorePath == "" {
+		log.Fatal("store-path must be defined")
+	}
+	localDb := &db.SQL{
+		SoftDelete:  true,
+		TrackAccess: db.TrackAccessBlobs,
+		LogQueries:  log.GetLevel() == log.DebugLevel,
+	}
+	err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
+	if err != nil {
+		log.Fatal(err)
+	}
+	blobs, err := speedwalk.AllFiles(diskStorePath, true)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = localDb.AddBlobs(blobs)
+	if err != nil {
+		log.Errorf("error while storing to db: %s", errors.FullTrace(err))
+	}
+}

cmd/publish.go

@@ -0,0 +1,65 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/lbryio/reflector.go/publish"
+
+	"github.com/lbryio/lbry.go/v2/lbrycrd"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "publish FILE",
+		Short: "Publish a file",
+		Args:  cobra.ExactArgs(1),
+		Run:   publishCmd,
+	}
+	cmd.Flags().String("name", "", "Claim name")
+	cmd.Flags().String("title", "", "Title of the content")
+	cmd.Flags().String("description", "", "Description of the content")
+	cmd.Flags().String("author", "", "Content author")
+	cmd.Flags().String("tags", "", "Comma-separated list of tags")
+	cmd.Flags().Int64("release-time", 0, "original public release of content, seconds since UNIX epoch")
+	rootCmd.AddCommand(cmd)
+}
+
+func publishCmd(cmd *cobra.Command, args []string) {
+	var err error
+
+	claimName := mustGetFlagString(cmd, "name")
+	if claimName == "" {
+		log.Errorln("--name required")
+		return
+	}
+
+	path := args[0]
+
+	client, err := lbrycrd.NewWithDefaultURL(nil)
+	checkErr(err)
+
+	tx, txid, err := publish.Publish(
+		client,
+		path,
+		claimName,
+		"bSzpgkTnAoiT2YAhUShPpfpajPESfNXVTu",
+		publish.Details{
+			Title:       mustGetFlagString(cmd, "title"),
+			Description: mustGetFlagString(cmd, "description"),
+			Author:      mustGetFlagString(cmd, "author"),
+			Tags:        nil,
+			ReleaseTime: mustGetFlagInt64(cmd, "release-time"),
+		},
+		"reflector.lbry.com:5566",
+	)
+	checkErr(err)
+
+	decoded, err := publish.Decode(client, tx)
+	checkErr(err)
+
+	fmt.Printf("TX: %s\n\n", decoded)
+	fmt.Printf("TXID: %s\n", txid.String())
+}

cmd/reflector.go

@@ -4,52 +4,378 @@ import (
 	"os"
 	"os/signal"
 	"strconv"
+	"strings"
 	"syscall"
 	"time"
 
+	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/lbryio/reflector.go/db"
+	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/meta"
 	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/server/http"
+	"github.com/lbryio/reflector.go/server/http3"
+	"github.com/lbryio/reflector.go/server/peer"
 	"github.com/lbryio/reflector.go/store"
 
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	"github.com/c2h5oh/datasize"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
+var (
+	//port configuration
+	tcpPeerPort   int
+	http3PeerPort int
+	httpPeerPort  int
+	receiverPort  int
+	metricsPort   int
+
+	//flags configuration
+	disableUploads   bool
+	disableBlocklist bool
+	useDB            bool
+
+	//upstream configuration
+	upstreamReflector string
+	upstreamProtocol  string
+	upstreamEdgeToken string
+
+	//downstream configuration
+	requestQueueSize int
+
+	//upstream edge configuration (to "cold" storage)
+	originEndpoint         string
+	originEndpointFallback string
+
+	//cache configuration
+	diskCache          string
+	secondaryDiskCache string
+	memCache           int
+)
+var cacheManagers = []string{"localdb", "lfu", "arc", "lru", "simple"}
+
+var cacheMangerToGcache = map[string]store.EvictionStrategy{
+	"lfu":    store.LFU,
+	"arc":    store.ARC,
+	"lru":    store.LRU,
+	"simple": store.SIMPLE,
+}
+
 func init() {
 	var cmd = &cobra.Command{
 		Use:   "reflector",
 		Short: "Run reflector server",
 		Run:   reflectorCmd,
 	}
+
+	cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from for the TCP (LBRY) protocol")
+	cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
+	cmd.Flags().IntVar(&httpPeerPort, "http-peer-port", 5569, "The port reflector will distribute content from over HTTP protocol")
+	cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
+	cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
+
+	cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
+	cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
+	cmd.Flags().BoolVar(&useDB, "use-db", true, "Whether to connect to the reflector db or not")
+
+	cmd.Flags().StringVar(&upstreamReflector, "upstream-reflector", "", "host:port of a reflector server where blobs are fetched from")
+	cmd.Flags().StringVar(&upstreamProtocol, "upstream-protocol", "http", "protocol used to fetch blobs from another upstream reflector server (tcp/http3/http)")
+	cmd.Flags().StringVar(&upstreamEdgeToken, "upstream-edge-token", "", "token used to retrieve/authenticate protected content")
+
+	cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests from downstream should be handled at once (the rest will wait)")
+
+	cmd.Flags().StringVar(&originEndpoint, "origin-endpoint", "", "HTTP edge endpoint for standard HTTP retrieval")
+	cmd.Flags().StringVar(&originEndpointFallback, "origin-endpoint-fallback", "", "HTTP edge endpoint for standard HTTP retrieval if first origin fails")
+
+	cmd.Flags().StringVar(&diskCache, "disk-cache", "100GB:/tmp/downloaded_blobs:localdb", "Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru)")
+	cmd.Flags().StringVar(&secondaryDiskCache, "optional-disk-cache", "", "Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru) (this would get hit before the one specified in disk-cache)")
+	cmd.Flags().IntVar(&memCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
+
 	rootCmd.AddCommand(cmd)
 }
 
 func reflectorCmd(cmd *cobra.Command, args []string) {
-	log.Printf("reflector version %s", meta.Version)
-	db := new(db.SQL)
-	err := db.Connect(globalConfig.DBConn)
+	log.Printf("reflector %s", meta.VersionString())
+
+	// the blocklist logic requires the db backed store to be the outer-most store
+	underlyingStore := initStores()
+	underlyingStoreWithCaches, cleanerStopper := initCaches(underlyingStore)
+
+	if !disableUploads {
+		reflectorServer := reflector.NewServer(underlyingStore, underlyingStoreWithCaches)
+		reflectorServer.Timeout = 3 * time.Minute
+		reflectorServer.EnableBlocklist = !disableBlocklist
+
+		err := reflectorServer.Start(":" + strconv.Itoa(receiverPort))
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer reflectorServer.Shutdown()
+	}
+
+	peerServer := peer.NewServer(underlyingStoreWithCaches)
+	err := peerServer.Start(":" + strconv.Itoa(tcpPeerPort))
 	if err != nil {
 		log.Fatal(err)
 	}
+	defer peerServer.Shutdown()
 
-	s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
-	combo := store.NewDBBackedS3Store(s3, db)
-
-	reflectorServer := reflector.NewServer(combo)
-	reflectorServer.Timeout = 30 * time.Second
-	if globalConfig.SlackHookURL != "" {
-		reflectorServer.StatLogger = log.StandardLogger()
-		reflectorServer.StatReportFrequency = 10 * time.Minute
-	}
-
-	err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
+	http3PeerServer := http3.NewServer(underlyingStoreWithCaches, requestQueueSize)
+	err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
 	if err != nil {
 		log.Fatal(err)
 	}
+	defer http3PeerServer.Shutdown()
+
+	httpServer := http.NewServer(store.WithSingleFlight("sf-http", underlyingStoreWithCaches), requestQueueSize, upstreamEdgeToken)
+	err = httpServer.Start(":" + strconv.Itoa(httpPeerPort))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer httpServer.Shutdown()
+
+	metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
+	metricsServer.Start()
+	defer metricsServer.Shutdown()
+	defer underlyingStoreWithCaches.Shutdown()
+	defer underlyingStore.Shutdown() //do we actually need this? Oo
 
 	interruptChan := make(chan os.Signal, 1)
 	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
 	<-interruptChan
-	reflectorServer.Shutdown()
+	// deferred shutdowns happen now
+	cleanerStopper.StopAndWait()
+}
+
+func initUpstreamStore() store.BlobStore {
+	var s store.BlobStore
+	if upstreamReflector == "" {
+		return nil
+	}
+	switch upstreamProtocol {
+	case "tcp":
+		s = peer.NewStore(peer.StoreOpts{
+			Address: upstreamReflector,
+			Timeout: 30 * time.Second,
+		})
+	case "http3":
+		s = http3.NewStore(http3.StoreOpts{
+			Address: upstreamReflector,
+			Timeout: 30 * time.Second,
+		})
+	case "http":
+		s = store.NewHttpStore(upstreamReflector, upstreamEdgeToken)
+	default:
+		log.Fatalf("protocol is not recognized: %s", upstreamProtocol)
+	}
+	return s
+}
+func initEdgeStore() store.BlobStore {
+	var s3Store *store.S3Store
+	var s store.BlobStore
+
+	if conf != "none" {
+		s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
+	}
+	if originEndpointFallback != "" && originEndpoint != "" {
+		ittt := store.NewITTTStore(store.NewCloudFrontROStore(originEndpoint), store.NewCloudFrontROStore(originEndpointFallback))
+		if s3Store != nil {
+			s = store.NewCloudFrontRWStore(ittt, s3Store)
+		} else {
+			s = ittt
+		}
+	} else if s3Store != nil {
+		s = s3Store
+	} else {
+		log.Fatalf("this configuration does not include a valid upstream source")
+	}
+	return s
+}
+
+func initDBStore(s store.BlobStore) store.BlobStore {
+	if useDB {
+		dbInst := &db.SQL{
+			TrackAccess: db.TrackAccessStreams,
+			LogQueries:  log.GetLevel() == log.DebugLevel,
+		}
+		err := dbInst.Connect(globalConfig.DBConn)
+		if err != nil {
+			log.Fatal(err)
+		}
+		s = store.NewDBBackedStore(s, dbInst, false)
+	}
+	return s
+}
+
+func initStores() store.BlobStore {
+	s := initUpstreamStore()
+	if s == nil {
+		s = initEdgeStore()
+	}
+	s = initDBStore(s)
+	return s
+}
+
+// initCaches returns a store wrapped with caches and a stop group to execute a clean shutdown
+func initCaches(s store.BlobStore) (store.BlobStore, *stop.Group) {
+	stopper := stop.New()
+	diskStore := initDiskStore(s, diskCache, stopper)
+	finalStore := initDiskStore(diskStore, secondaryDiskCache, stopper)
+	stop.New()
+	if memCache > 0 {
+		finalStore = store.NewCachingStore(
+			"reflector",
+			finalStore,
+			store.NewGcacheStore("mem", store.NewMemStore(), memCache, store.LRU),
+		)
+	}
+	return finalStore, stopper
+}
+
+func initDiskStore(upstreamStore store.BlobStore, diskParams string, stopper *stop.Group) store.BlobStore {
+	diskCacheMaxSize, diskCachePath, cacheManager := diskCacheParams(diskParams)
+	//we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte
+	// so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup
+	realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
+	if diskCacheMaxSize == 0 {
+		return upstreamStore
+	}
+	err := os.MkdirAll(diskCachePath, os.ModePerm)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	diskStore := store.NewDiskStore(diskCachePath, 2)
+	var unwrappedStore store.BlobStore
+	cleanerStopper := stop.New(stopper)
+
+	if cacheManager == "localdb" {
+		localDb := &db.SQL{
+			SoftDelete:  true,
+			TrackAccess: db.TrackAccessBlobs,
+			LogQueries:  log.GetLevel() == log.DebugLevel,
+		}
+		err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
+		if err != nil {
+			log.Fatal(err)
+		}
+		unwrappedStore = store.NewDBBackedStore(diskStore, localDb, true)
+		go cleanOldestBlobs(int(realCacheSize), localDb, unwrappedStore, cleanerStopper)
+	} else {
+		unwrappedStore = store.NewGcacheStore("nvme", store.NewDiskStore(diskCachePath, 2), int(realCacheSize), cacheMangerToGcache[cacheManager])
+	}
+
+	wrapped := store.NewCachingStore(
+		"reflector",
+		upstreamStore,
+		unwrappedStore,
+	)
+	return wrapped
+}
+
+func diskCacheParams(diskParams string) (int, string, string) {
+	if diskParams == "" {
+		return 0, "", ""
+	}
+
+	parts := strings.Split(diskParams, ":")
+	if len(parts) != 3 {
+		log.Fatalf("%s does is formatted incorrectly. Expected format: 'sizeGB:CACHE_PATH:cachemanager' for example: '100GB:/tmp/downloaded_blobs:localdb'", diskParams)
+	}
+
+	diskCacheSize := parts[0]
+	path := parts[1]
+	cacheManager := parts[2]
+
+	if len(path) == 0 || path[0] != '/' {
+		log.Fatalf("disk cache paths must start with '/'")
+	}
+
+	if !util.InSlice(cacheManager, cacheManagers) {
+		log.Fatalf("specified cache manager '%s' is not supported. Use one of the following: %v", cacheManager, cacheManagers)
+	}
+
+	var maxSize datasize.ByteSize
+	err := maxSize.UnmarshalText([]byte(diskCacheSize))
+	if err != nil {
+		log.Fatal(err)
+	}
+	if maxSize <= 0 {
+		log.Fatal("disk cache size must be more than 0")
+	}
+	return int(maxSize), path, cacheManager
+}
+
+func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
+	// this is so that it runs on startup without having to wait for 10 minutes
+	err := doClean(maxItems, db, store, stopper)
+	if err != nil {
+		log.Error(errors.FullTrace(err))
+	}
+	const cleanupInterval = 10 * time.Minute
+	for {
+		select {
+		case <-stopper.Ch():
+			log.Infoln("stopping self cleanup")
+			return
+		case <-time.After(cleanupInterval):
+			err := doClean(maxItems, db, store, stopper)
+			if err != nil {
+				log.Error(errors.FullTrace(err))
+			}
+		}
+	}
+}
+
+func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error {
+	blobsCount, err := db.Count()
+	if err != nil {
+		return err
+	}
+
+	if blobsCount >= maxItems {
+		itemsToDelete := blobsCount / 10
+		blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete)
+		if err != nil {
+			return err
+		}
+		blobsChan := make(chan string, len(blobs))
+		wg := &stop.Group{}
+		go func() {
+			for _, hash := range blobs {
+				select {
+				case <-stopper.Ch():
+					return
+				default:
+				}
+				blobsChan <- hash
+			}
+			close(blobsChan)
+		}()
+		for i := 0; i < 3; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for h := range blobsChan {
+					select {
+					case <-stopper.Ch():
+						return
+					default:
+					}
+					err = store.Delete(h)
+					if err != nil {
+						log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
+						continue
+					}
+				}
+			}()
+		}
+		wg.Wait()
+	}
+	return nil
 }

cmd/resolve.go

@@ -0,0 +1,50 @@
+package cmd
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+
+	"github.com/lbryio/reflector.go/wallet"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "resolve ADDRESS:PORT URL",
+		Short: "Resolve a URL",
+		Args:  cobra.ExactArgs(2),
+		Run:   resolveCmd,
+	}
+	rootCmd.AddCommand(cmd)
+}
+
+func resolveCmd(cmd *cobra.Command, args []string) {
+	addr := args[0]
+	url := args[1]
+
+	node := wallet.NewNode()
+	defer node.Shutdown()
+	err := node.Connect([]string{addr}, nil)
+	checkErr(err)
+
+	output, err := node.Resolve(url)
+	checkErr(err)
+
+	claim, err := node.GetClaimInTx(hex.EncodeToString(rev(output.GetTxHash())), int(output.GetNout()))
+	checkErr(err)
+
+	jsonClaim, err := json.MarshalIndent(claim, "", "  ")
+	checkErr(err)
+
+	fmt.Println(string(jsonClaim))
+}
+
+func rev(b []byte) []byte {
+	r := make([]byte, len(b))
+	for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 {
+		r[left], r[right] = b[right], b[left]
+	}
+	return r
+}

cmd/root.go

@@ -2,12 +2,14 @@ package cmd
 
 import (
 	"encoding/json"
-	"io/ioutil"
 	"os"
+	"strings"
 
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/util"
-	"github.com/lbryio/reflector.go/dht"
+	"github.com/lbryio/reflector.go/updater"
+
+	"github.com/lbryio/lbry.go/v2/dht"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/util"
 
 	"github.com/johntdyer/slackrus"
 	"github.com/sirupsen/logrus"
@@ -22,6 +24,9 @@ type Config struct {
 	BucketName   string `json:"bucket_name"`
 	DBConn       string `json:"db_conn"`
 	SlackHookURL string `json:"slack_hook_url"`
+	SlackChannel string `json:"slack_channel"`
+	UpdateBinURL string `json:"update_bin_url"`
+	UpdateCmd    string `json:"update_cmd"`
 }
 
 var verbose []string
@@ -46,7 +51,7 @@ var rootCmd = &cobra.Command{
 
 func init() {
 	rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
-	rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config")
+	rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config. Use 'none' to disable")
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
@@ -65,8 +70,11 @@ func preRun(cmd *cobra.Command, args []string) {
 	debugLogger.SetOutput(os.Stderr)
 
 	if util.InSlice(verboseAll, verbose) {
+		logrus.Info("global verbose logging enabled")
 		logrus.SetLevel(logrus.DebugLevel)
 		verbose = []string{verboseDHT, verboseNodeFinder}
+	} else if len(verbose) > 0 {
+		logrus.Infof("verbose logging enabled for: %s", strings.Join(verbose, ", "))
 	}
 
 	for _, debugType := range verbose {
@@ -94,7 +102,7 @@ func preRun(cmd *cobra.Command, args []string) {
 		hook := &slackrus.SlackrusHook{
 			HookURL:        globalConfig.SlackHookURL,
 			AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
-			Channel:        "#reflector-logs",
+			Channel:        globalConfig.SlackChannel,
 			//IconEmoji:      ":ghost:",
 			//Username:       "reflector.go",
 		}
@@ -102,6 +110,14 @@ func preRun(cmd *cobra.Command, args []string) {
 		logrus.AddHook(hook)
 		debugLogger.AddHook(hook)
 	}
+
+	if globalConfig.UpdateBinURL != "" {
+		if globalConfig.UpdateCmd == "" {
+			logrus.Warnln("update_cmd is empty in conf file")
+		}
+		logrus.Println("starting update checker")
+		go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
+	}
 }
 
 func checkErr(err error) {
@@ -125,14 +141,32 @@ func argFuncs(funcs ...cobra.PositionalArgs) cobra.PositionalArgs {
 func loadConfig(path string) (Config, error) {
 	var c Config
 
-	raw, err := ioutil.ReadFile(path)
+	raw, err := os.ReadFile(path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return c, errors.Err("config file not found")
 		}
-		return c, err
+		return c, errors.Err(err)
 	}
 
 	err = json.Unmarshal(raw, &c)
-	return c, err
+	return c, errors.Err(err)
 }
+
+func mustGetFlagString(cmd *cobra.Command, name string) string {
+	v, err := cmd.Flags().GetString(name)
+	checkErr(err)
+	return v
+}
+
+func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
+	v, err := cmd.Flags().GetInt64(name)
+	checkErr(err)
+	return v
+}
+
+//func mustGetFlagBool(cmd *cobra.Command, name string) bool {
+//	v, err := cmd.Flags().GetBool(name)
+//	checkErr(err)
+//	return v
+//}

cmd/send.go

@@ -0,0 +1,158 @@
+package cmd
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"path"
+	"syscall"
+
+	"github.com/lbryio/reflector.go/reflector"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "send ADDRESS:PORT PATH",
+		Short: "Send a file to a reflector",
+		Args:  cobra.ExactArgs(2),
+		Run:   sendCmd,
+	}
+	cmd.PersistentFlags().String("sd-cache", "", "path to dir where sd blobs will be cached")
+	rootCmd.AddCommand(cmd)
+}
+
+// todo: if retrying a large file is slow, we can add the ability to seek ahead in the file so we're not
+// re-uploading blobs that already exist
+
+var hackyReflector reflector.Client
+
+func sendCmd(cmd *cobra.Command, args []string) {
+	reflectorAddress := args[0]
+	err := hackyReflector.Connect(reflectorAddress)
+	checkErr(err)
+	defer func() { _ = hackyReflector.Close() }()
+
+	filePath := args[1]
+	file, err := os.Open(filePath)
+	checkErr(err)
+	defer func() { _ = file.Close() }()
+	sdCachePath := ""
+	sdCacheDir := mustGetFlagString(cmd, "sd-cache")
+	if sdCacheDir != "" {
+		if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) {
+			err = os.MkdirAll(sdCacheDir, 0777)
+			checkErr(err)
+		}
+		sdCachePath = path.Join(sdCacheDir, filePath+".sdblob")
+	}
+
+	var enc *stream.Encoder
+
+	if sdCachePath != "" {
+		if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
+			sdBlob, err := os.ReadFile(sdCachePath)
+			checkErr(err)
+			cachedSDBlob := &stream.SDBlob{}
+			err = cachedSDBlob.FromBlob(sdBlob)
+			checkErr(err)
+			enc = stream.NewEncoderFromSD(file, cachedSDBlob)
+		}
+	}
+	if enc == nil {
+		enc = stream.NewEncoder(file)
+	}
+
+	exitCode := 0
+
+	var killed bool
+	interruptChan := make(chan os.Signal, 1)
+	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		sig := <-interruptChan
+		fmt.Printf("caught %s, exiting...\n", sig.String())
+		killed = true
+		exitCode = 1
+	}()
+
+	for {
+		if killed {
+			break
+		}
+
+		b, err := enc.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			fmt.Printf("error reading next blob: %v\n", err)
+			exitCode = 1
+			break
+		}
+
+		err = hackyReflect(b, false)
+		if err != nil {
+			fmt.Printf("error reflecting blob %s: %v\n", b.HashHex()[:8], err)
+			exitCode = 1
+			break
+		}
+	}
+
+	sd := enc.SDBlob()
+	//sd.StreamName = filepath.Base(filePath)
+	//sd.SuggestedFileName = filepath.Base(filePath)
+	err = os.WriteFile(sdCachePath, sd.ToBlob(), 0666)
+	if err != nil {
+		fmt.Printf("error saving sd blob: %v\n", err)
+		fmt.Println(sd.ToJson())
+		exitCode = 1
+	}
+
+	if killed {
+		os.Exit(exitCode)
+	}
+
+	if reflectorAddress != "" {
+		err = hackyReflect(sd.ToBlob(), true)
+		if err != nil {
+			fmt.Printf("error reflecting sd blob %s: %v\n", sd.HashHex()[:8], err)
+			exitCode = 1
+		}
+	}
+
+	ret := struct {
+		SDHash     string `json:"sd_hash"`
+		SourceHash string `json:"source_hash"`
+	}{
+		SDHash:     sd.HashHex(),
+		SourceHash: hex.EncodeToString(enc.SourceHash()),
+	}
+
+	j, err := json.MarshalIndent(ret, "", "  ")
+	checkErr(err)
+	fmt.Println(string(j))
+	os.Exit(exitCode)
+}
+
+func hackyReflect(b stream.Blob, sd bool) error {
+	var err error
+	if sd {
+		err = hackyReflector.SendSDBlob(b)
+	} else {
+		err = hackyReflector.SendBlob(b)
+	}
+
+	if errors.Is(err, reflector.ErrBlobExists) {
+		//fmt.Printf("%s already reflected\n", b.HashHex()[:8])
+		return nil
+	}
+
+	return err
+}

cmd/sendblob.go

@@ -2,18 +2,21 @@ package cmd
 
 import (
 	"crypto/rand"
+	"os"
 
 	"github.com/lbryio/reflector.go/reflector"
 
+	"github.com/lbryio/lbry.go/v2/stream"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
 func init() {
 	var cmd = &cobra.Command{
-		Use:   "sendblob ADDRESS:PORT",
+		Use:   "sendblob ADDRESS:PORT [PATH]",
 		Short: "Send a random blob to a reflector server",
-		Args:  cobra.ExactArgs(1),
+		Args:  cobra.RangeArgs(1, 2),
 		Run:   sendBlobCmd,
 	}
 	rootCmd.AddCommand(cmd)
@@ -21,21 +24,47 @@ func init() {
 
 func sendBlobCmd(cmd *cobra.Command, args []string) {
 	addr := args[0]
+	var path string
+	if len(args) >= 2 {
+		path = args[1]
+	}
 
 	c := reflector.Client{}
 	err := c.Connect(addr)
 	if err != nil {
-		log.Fatal("error connecting client to server", err)
+		log.Fatal("error connecting client to server: ", err)
 	}
 
-	blob := make([]byte, 1024)
-	_, err = rand.Read(blob)
-	if err != nil {
-		log.Fatal("failed to make random blob", err)
+	if path == "" {
+		blob := make(stream.Blob, 1024)
+		_, err = rand.Read(blob)
+		if err != nil {
+			log.Fatal("failed to make random blob: ", err)
+		}
+
+		err = c.SendBlob(blob)
+		if err != nil {
+			log.Error(err)
+		}
+		return
 	}
 
-	err = c.SendBlob(blob)
-	if err != nil {
-		log.Error(err)
+	file, err := os.Open(path)
+	checkErr(err)
+	defer func() { _ = file.Close() }()
+	s, err := stream.New(file)
+	checkErr(err)
+
+	sdBlob := &stream.SDBlob{}
+	err = sdBlob.FromBlob(s[0])
+	checkErr(err)
+
+	for i, b := range s {
+		if i == 0 {
+			err = c.SendSDBlob(b)
+		} else {
+			err = c.SendBlob(b)
+		}
+		checkErr(err)
 	}
 }

cmd/start.go

@@ -9,13 +9,14 @@ import (
 
 	"github.com/lbryio/reflector.go/cluster"
 	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/dht"
-	"github.com/lbryio/reflector.go/dht/bits"
-	"github.com/lbryio/reflector.go/peer"
 	"github.com/lbryio/reflector.go/prism"
 	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/server/peer"
 	"github.com/lbryio/reflector.go/store"
 
+	"github.com/lbryio/lbry.go/v2/dht"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -52,11 +53,13 @@ func init() {
 }
 
 func startCmd(cmd *cobra.Command, args []string) {
-	db := new(db.SQL)
+	db := &db.SQL{
+		LogQueries: log.GetLevel() == log.DebugLevel,
+	}
 	err := db.Connect(globalConfig.DBConn)
 	checkErr(err)
-	s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
-	comboStore := store.NewDBBackedS3Store(s3, db)
+	s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
+	comboStore := store.NewDBBackedStore(s3, db, false)
 
 	conf := prism.DefaultConf()
 

cmd/test.go

@@ -0,0 +1,58 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/lbryio/reflector.go/meta"
+	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/server/peer"
+	"github.com/lbryio/reflector.go/store"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "test",
+		Short: "Test things",
+		Run:   testCmd,
+	}
+	rootCmd.AddCommand(cmd)
+}
+
+func testCmd(cmd *cobra.Command, args []string) {
+	log.Printf("reflector %s", meta.VersionString())
+
+	memStore := store.NewMemStore()
+
+	reflectorServer := reflector.NewServer(memStore, memStore)
+	reflectorServer.Timeout = 3 * time.Minute
+
+	err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	peerServer := peer.NewServer(memStore)
+	err = peerServer.Start(":" + strconv.Itoa(reflector.DefaultPort+1))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	interruptChan := make(chan os.Signal, 1)
+	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
+	<-interruptChan
+	peerServer.Shutdown()
+	reflectorServer.Shutdown()
+
+	fmt.Println("Blobs in store")
+	for hash, blob := range memStore.Debug() {
+		fmt.Printf("%s: %d bytes\n", hash, len(blob))
+	}
+}

cmd/upload.go

@@ -1,44 +1,21 @@
 package cmd
 
 import (
-	"encoding/json"
-	"io/ioutil"
 	"os"
 	"os/signal"
-	"path"
-	"sync"
 	"syscall"
-	"time"
 
 	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/peer"
+	"github.com/lbryio/reflector.go/reflector"
 	"github.com/lbryio/reflector.go/store"
 
-	"github.com/lbryio/lbry.go/stop"
-
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
 var uploadWorkers int
 var uploadSkipExistsCheck bool
-
-const (
-	sdInc   = 1
-	blobInc = 2
-	errInc  = 3
-)
-
-type uploaderParams struct {
-	workerWG  *sync.WaitGroup
-	counterWG *sync.WaitGroup
-	stopper   *stop.Group
-	pathChan  chan string
-	countChan chan int
-	sdCount   int
-	blobCount int
-	errCount  int
-}
+var uploadDeleteBlobsAfterUpload bool
 
 func init() {
 	var cmd = &cobra.Command{
@@ -49,220 +26,30 @@ func init() {
 	}
 	cmd.PersistentFlags().IntVar(&uploadWorkers, "workers", 1, "How many worker threads to run at once")
 	cmd.PersistentFlags().BoolVar(&uploadSkipExistsCheck, "skipExistsCheck", false, "Dont check if blobs exist before uploading")
+	cmd.PersistentFlags().BoolVar(&uploadDeleteBlobsAfterUpload, "deleteBlobsAfterUpload", false, "Delete blobs after uploading them")
 	rootCmd.AddCommand(cmd)
 }
 
 func uploadCmd(cmd *cobra.Command, args []string) {
-	startTime := time.Now()
-	db := new(db.SQL)
+	db := &db.SQL{
+		LogQueries: log.GetLevel() == log.DebugLevel,
+	}
 	err := db.Connect(globalConfig.DBConn)
 	checkErr(err)
 
-	params := uploaderParams{
-		workerWG:  &sync.WaitGroup{},
-		counterWG: &sync.WaitGroup{},
-		pathChan:  make(chan string),
-		countChan: make(chan int),
-		stopper:   stop.New()}
+	st := store.NewDBBackedStore(
+		store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
+		db, false)
 
-	setInterrupt(params.stopper)
+	uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
 
-	paths, err := getPaths(args[0])
-	checkErr(err)
-
-	totalCount := len(paths)
-
-	hashes := make([]string, len(paths))
-	for i, p := range paths {
-		hashes[i] = path.Base(p)
-	}
-
-	log.Println("checking for existing blobs")
-
-	exists := make(map[string]bool)
-	if !uploadSkipExistsCheck {
-		exists, err = db.HasBlobs(hashes)
-		checkErr(err)
-	}
-	existsCount := len(exists)
-
-	log.Printf("%d new blobs to upload", totalCount-existsCount)
-
-	startUploadWorkers(&params)
-	params.counterWG.Add(1)
-	go func() {
-		defer params.counterWG.Done()
-		runCountReceiver(&params, startTime, totalCount, existsCount)
-	}()
-
-Upload:
-	for _, f := range paths {
-		if exists[path.Base(f)] {
-			continue
-		}
-
-		select {
-		case params.pathChan <- f:
-		case <-params.stopper.Ch():
-			log.Warnln("Caught interrupt, quitting at first opportunity...")
-			break Upload
-		}
-	}
-
-	close(params.pathChan)
-	params.workerWG.Wait()
-	close(params.countChan)
-	params.counterWG.Wait()
-	params.stopper.Stop()
-
-	log.Println("SUMMARY")
-	log.Printf("%d blobs total", totalCount)
-	log.Printf("%d SD blobs uploaded", params.sdCount)
-	log.Printf("%d content blobs uploaded", params.blobCount)
-	log.Printf("%d blobs already stored", existsCount)
-	log.Printf("%d errors encountered", params.errCount)
-}
-
-func isJSON(data []byte) bool {
-	var js json.RawMessage
-	return json.Unmarshal(data, &js) == nil
-}
-
-func newBlobStore() *store.DBBackedS3Store {
-	db := new(db.SQL)
-	err := db.Connect(globalConfig.DBConn)
-	checkErr(err)
-
-	s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
-	return store.NewDBBackedS3Store(s3, db)
-}
-
-func setInterrupt(stopper *stop.Group) {
 	interruptChan := make(chan os.Signal, 1)
 	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-interruptChan
-		stopper.Stop()
+		uploader.Stop()
 	}()
-}
-
-func startUploadWorkers(params *uploaderParams) {
-	for i := 0; i < uploadWorkers; i++ {
-		params.workerWG.Add(1)
-		go func(i int) {
-			defer params.workerWG.Done()
-			defer func(i int) {
-				log.Printf("worker %d quitting", i)
-			}(i)
-
-			blobStore := newBlobStore()
-			launchFileUploader(params, blobStore, i)
-		}(i)
-	}
-}
-
-func launchFileUploader(params *uploaderParams, blobStore *store.DBBackedS3Store, worker int) {
-	for {
-		select {
-		case <-params.stopper.Ch():
-			return
-		case filepath, ok := <-params.pathChan:
-			if !ok {
-				return
-			}
-
-			blob, err := ioutil.ReadFile(filepath)
-			checkErr(err)
-
-			hash := peer.GetBlobHash(blob)
-			if hash != path.Base(filepath) {
-				log.Errorf("worker %d: file name does not match hash (%s != %s), skipping", worker, filepath, hash)
-				select {
-				case params.countChan <- errInc:
-				case <-params.stopper.Ch():
-				}
-				continue
-			}
-
-			if isJSON(blob) {
-				log.Printf("worker %d: PUTTING SD BLOB %s", worker, hash)
-				err := blobStore.PutSD(hash, blob)
-				if err != nil {
-					log.Error("PutSD Error: ", err)
-				}
-				select {
-				case params.countChan <- sdInc:
-				case <-params.stopper.Ch():
-				}
-			} else {
-				log.Printf("worker %d: putting %s", worker, hash)
-				err = blobStore.Put(hash, blob)
-				if err != nil {
-					log.Error("put Blob Error: ", err)
-				}
-				select {
-				case params.countChan <- blobInc:
-				case <-params.stopper.Ch():
-				}
-			}
-		}
-	}
-}
-
-func runCountReceiver(params *uploaderParams, startTime time.Time, totalCount int, existsCount int) {
-	for {
-		select {
-		case <-params.stopper.Ch():
-			return
-		case countType, ok := <-params.countChan:
-			if !ok {
-				return
-			}
-			switch countType {
-			case sdInc:
-				params.sdCount++
-			case blobInc:
-				params.blobCount++
-			case errInc:
-				params.errCount++
-			}
-		}
-		if (params.sdCount+params.blobCount)%50 == 0 {
-			log.Printf("%d of %d done (%s elapsed, %.3fs per blob)", params.sdCount+params.blobCount, totalCount-existsCount, time.Since(startTime).String(), time.Since(startTime).Seconds()/float64(params.sdCount+params.blobCount))
-		}
-	}
-}
-
-func getPaths(path string) ([]string, error) {
-	info, err := os.Stat(path)
-	if err != nil {
-		return nil, err
-	}
-
-	if info.Mode().IsRegular() {
-		return []string{path}, nil
-	}
-
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-
-	files, err := f.Readdir(-1)
-	if err != nil {
-		return nil, err
-	}
-	err = f.Close()
-	if err != nil {
-		return nil, err
-	}
-
-	var filenames []string
-	for _, file := range files {
-		if !file.IsDir() {
-			filenames = append(filenames, path+"/"+file.Name())
-		}
-	}
-
-	return filenames, nil
+
+	err = uploader.Upload(args[0])
+	checkErr(err)
 }

cmd/version.go

@@ -1,7 +1,7 @@
 package cmd
 
 import (
-	"log"
+	"fmt"
 
 	"github.com/lbryio/reflector.go/meta"
 
@@ -18,5 +18,5 @@ func init() {
 }
 
 func versionCmd(cmd *cobra.Command, args []string) {
-	log.Printf("version %s\n", meta.Version)
+	fmt.Println(meta.FullName())
 }

config.tmpl

@@ -0,0 +1,8 @@
+{
+  "aws_id": "YOUR-AWS-ID",
+  "aws_secret": "YOUR-AWS-SECRET",
+  "bucket_region": "YOUR-BUCKET-REGION",
+  "bucket_name": "YOUR-BUCKET-NAME",
+  "db_conn": "USER:PASSWORD@tcp(localhost:3306)/DBNAME",
+  "slack_hook_url": ""
+}

db/db.go

@@ -3,13 +3,22 @@ package db
 import (
 	"context"
 	"database/sql"
+	"fmt"
+	"runtime"
+	"strings"
+	"time"
 
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/querytools"
-	"github.com/lbryio/reflector.go/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	qt "github.com/lbryio/lbry.go/v2/extras/query"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/stream"
 
-	_ "github.com/go-sql-driver/mysql" // blank import for db driver
+	"github.com/go-sql-driver/mysql"
+	_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
 	log "github.com/sirupsen/logrus"
+	"github.com/volatiletech/null/v8"
+	"go.uber.org/atomic"
 )
 
 // SdBlob is a special blob that contains information on the rest of the blobs in the stream
@@ -19,7 +28,7 @@ type SdBlob struct {
 		Length   int    `json:"length"`
 		BlobNum  int    `json:"blob_num"`
 		BlobHash string `json:"blob_hash,omitempty"`
-		Iv       string `json:"iv"`
+		IV       string `json:"iv"`
 	} `json:"blobs"`
 	StreamType        string `json:"stream_type"`
 	Key               string `json:"key"`
@@ -27,17 +36,38 @@ type SdBlob struct {
 	StreamHash        string `json:"stream_hash"`
 }
 
+type trackAccess int
+
+const (
+	TrackAccessNone    trackAccess = iota // Don't track accesses
+	TrackAccessStreams                    // Track accesses at the stream level
+	TrackAccessBlobs                      // Track accesses at the blob level
+)
+
 // SQL implements the DB interface
 type SQL struct {
 	conn *sql.DB
+
+	// Track the approx last time a blob or stream was accessed
+	TrackAccess trackAccess
+
+	// Instead of deleting a blob, marked it as not stored in the db
+	SoftDelete bool
+
+	// Log executed queries. qt.InterpolateParams is cpu-heavy. This avoids that call if not needed.
+	LogQueries bool
 }
 
-func logQuery(query string, args ...interface{}) {
-	s, err := querytools.InterpolateParams(query, args...)
+func (s SQL) logQuery(query string, args ...interface{}) {
+	if !s.LogQueries {
+		return
+	}
+
+	qStr, err := qt.InterpolateParams(query, args...)
 	if err != nil {
 		log.Errorln(err)
 	} else {
-		log.Debugln(s)
+		log.Debugln(qStr)
 	}
 }
 
@@ -53,6 +83,8 @@ func (s *SQL) Connect(dsn string) error {
 		return errors.Err(err)
 	}
 
+	s.conn.SetMaxIdleConns(12)
+
 	return errors.Err(s.conn.Ping())
 }
 
@@ -62,55 +94,240 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
 		return errors.Err("not connected")
 	}
 
-	return withTx(s.conn, func(tx *sql.Tx) error {
-		return addBlob(tx, hash, length, isStored)
-	})
+	_, err := s.insertBlob(hash, length, isStored)
+	return err
 }
 
-func addBlob(tx *sql.Tx, hash string, length int, isStored bool) error {
-	if length <= 0 {
-		return errors.Err("length must be positive")
+//AddBlobs adds blobs to the database.
+func (s *SQL) AddBlobs(hash []string) error {
+	if s.conn == nil {
+		return errors.Err("not connected")
 	}
 
-	err := execTx(tx,
-		"INSERT INTO blob_ (hash, is_stored, length) VALUES (?,?,?) ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
-		[]interface{}{hash, isStored, length},
+	batch := 10000
+	totalBlobs := int64(len(hash))
+	work := make(chan []string, 1000)
+	stopper := stop.New()
+	var totalInserted atomic.Int64
+	start := time.Now()
+
+	go func() {
+		for i := 0; i < len(hash); i += batch {
+			j := i + batch
+			if j > len(hash) {
+				j = len(hash)
+			}
+			work <- hash[i:j]
+		}
+		log.Infof("done loading %d hashes in the work queue", len(hash))
+		close(work)
+	}()
+
+	for i := 0; i < runtime.NumCPU(); i++ {
+		stopper.Add(1)
+		go func(worker int) {
+			log.Infof("starting worker %d", worker)
+			defer stopper.Done()
+			for hashes := range work {
+				inserted := totalInserted.Load()
+				remaining := totalBlobs - inserted
+				if inserted > 0 {
+					timePerBlob := time.Since(start).Microseconds() / inserted
+					remainingTime := time.Duration(remaining*timePerBlob) * time.Microsecond
+					log.Infof("[T%d] processing batch of %d items. ETA: %s", worker, len(hashes), remainingTime.String())
+				}
+				err := s.insertBlobs(hashes) // Process the batch.
+				if err != nil {
+					log.Errorf("error while inserting batch: %s", errors.FullTrace(err))
+				}
+				totalInserted.Add(int64(len(hashes)))
+			}
+		}(i)
+	}
+
+	stopper.Wait()
+	return nil
+}
+
+func (s *SQL) insertBlobs(hashes []string) error {
+	var (
+		q string
+		//args []interface{}
 	)
+	dayAgo := time.Now().AddDate(0, 0, -1).Format("2006-01-02 15:04:05")
+	q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
+	for _, hash := range hashes {
+		// prepared statements slow everything down by a lot due to reflection
+		// for this specific instance we'll go ahead and hardcode the query to make it go faster
+		q += fmt.Sprintf("('%s',1,%d,'%s'),", hash, stream.MaxBlobSize, dayAgo)
+		//args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
+	}
+	q = strings.TrimSuffix(q, ",")
+	_, err := s.exec(q)
 	if err != nil {
-		return errors.Err(err)
+		return err
 	}
 
 	return nil
 }
 
-// HasBlob checks if the database contains the blob information.
-func (s *SQL) HasBlob(hash string) (bool, error) {
-	if s.conn == nil {
-		return false, errors.Err("not connected")
+func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
+	if length <= 0 {
+		return 0, errors.Err("length must be positive")
 	}
 
-	query := "SELECT EXISTS(SELECT 1 FROM blob_ WHERE hash = ? AND is_stored = ?)"
-	args := []interface{}{hash, true}
+	var (
+		q    string
+		args []interface{}
+	)
+	if s.TrackAccess == TrackAccessBlobs {
+		args = []interface{}{hash, isStored, length, time.Now()}
+		q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
+	} else {
+		args = []interface{}{hash, isStored, length}
+		q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))"
+	}
 
-	logQuery(query, args...)
+	blobID, err := s.exec(q, args...)
+	if err != nil {
+		return 0, err
+	}
 
-	row := s.conn.QueryRow(query, args...)
+	if blobID == 0 {
+		err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
+		if err != nil {
+			return 0, errors.Err(err)
+		}
+		if blobID == 0 {
+			return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
+		}
 
-	exists := false
-	err := row.Scan(&exists)
+		if s.TrackAccess == TrackAccessBlobs {
+			err := s.touchBlobs([]uint64{uint64(blobID)})
+			if err != nil {
+				return 0, errors.Err(err)
+			}
+		}
+	}
 
-	return exists, errors.Err(err)
+	return blobID, nil
+}
+
+func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
+	var (
+		q    string
+		args []interface{}
+	)
+
+	if s.TrackAccess == TrackAccessStreams {
+		args = []interface{}{hash, sdBlobID, time.Now()}
+		q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
+	} else {
+		args = []interface{}{hash, sdBlobID}
+		q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")"
+	}
+
+	streamID, err := s.exec(q, args...)
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+
+	if streamID == 0 {
+		err = s.conn.QueryRow("SELECT id FROM stream WHERE sd_blob_id = ?", sdBlobID).Scan(&streamID)
+		if err != nil {
+			return 0, errors.Err(err)
+		}
+		if streamID == 0 {
+			return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
+		}
+
+		if s.TrackAccess == TrackAccessStreams {
+			err := s.touchStreams([]uint64{uint64(streamID)})
+			if err != nil {
+				return 0, errors.Err(err)
+			}
+		}
+	}
+	return streamID, nil
+}
+
+// HasBlob checks if the database contains the blob information.
+func (s *SQL) HasBlob(hash string, touch bool) (bool, error) {
+	exists, err := s.HasBlobs([]string{hash}, touch)
+	if err != nil {
+		return false, err
+	}
+	return exists[hash], nil
 }
 
 // HasBlobs checks if the database contains the set of blobs and returns a bool map.
-func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
-	if s.conn == nil {
-		return nil, errors.Err("not connected")
+func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
+	exists, idsNeedingTouch, err := s.hasBlobs(hashes)
+
+	if touch {
+		if s.TrackAccess == TrackAccessBlobs {
+			_ = s.touchBlobs(idsNeedingTouch)
+		} else if s.TrackAccess == TrackAccessStreams {
+			_ = s.touchStreams(idsNeedingTouch)
+		}
 	}
 
-	var hash string
+	return exists, err
+}
+
+func (s *SQL) touchBlobs(blobIDs []uint64) error {
+	if len(blobIDs) == 0 {
+		return nil
+	}
+
+	query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
+	args := make([]interface{}, len(blobIDs)+1)
+	args[0] = time.Now()
+	for i := range blobIDs {
+		args[i+1] = blobIDs[i]
+	}
+
+	startTime := time.Now()
+	_, err := s.exec(query, args...)
+	log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
+	return errors.Err(err)
+}
+
+func (s *SQL) touchStreams(streamIDs []uint64) error {
+	if len(streamIDs) == 0 {
+		return nil
+	}
+
+	query := "UPDATE stream SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(streamIDs)) + ")"
+	args := make([]interface{}, len(streamIDs)+1)
+	args[0] = time.Now()
+	for i := range streamIDs {
+		args[i+1] = streamIDs[i]
+	}
+
+	startTime := time.Now()
+	_, err := s.exec(query, args...)
+	log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime))
+	return errors.Err(err)
+}
+
+func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
+	if s.conn == nil {
+		return nil, nil, errors.Err("not connected")
+	}
+
+	var (
+		hash           string
+		blobID         uint64
+		streamID       null.Uint64
+		lastAccessedAt null.Time
+	)
+
+	var needsTouch []uint64
 	exists := make(map[string]bool)
-	maxBatchSize := 100
+
+	touchDeadline := time.Now().Add(-6 * time.Hour) // touch blob if last accessed before this time
+	maxBatchSize := 10000
 	doneIndex := 0
 
 	for len(hashes) > doneIndex {
@@ -121,41 +338,171 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
 		log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
 		batch := hashes[doneIndex:sliceEnd]
 
-		query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + querytools.Qs(len(batch)) + ")"
-		args := make([]interface{}, len(batch)+1)
-		args[0] = true
+		var query string
+		if s.TrackAccess == TrackAccessBlobs {
+			query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
+FROM blob_ b
+WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
+		} else if s.TrackAccess == TrackAccessStreams {
+			query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
+FROM blob_ b
+LEFT JOIN stream_blob sb ON b.id = sb.blob_id
+INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
+WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
+		} else {
+			query = `SELECT b.hash, b.id, NULL, NULL
+FROM blob_ b
+WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
+		}
+
+		args := make([]interface{}, len(batch))
 		for i := range batch {
-			args[i+1] = batch[i]
+			args[i] = batch[i]
 		}
 
-		logQuery(query, args...)
+		s.logQuery(query, args...)
 
-		rows, err := s.conn.Query(query, args...)
-		if err != nil {
-			closeRows(rows)
-			return exists, err
-		}
-
-		for rows.Next() {
-			err := rows.Scan(&hash)
+		err := func() error {
+			startTime := time.Now()
+			rows, err := s.conn.Query(query, args...)
+			log.Debugf("hashes query took %s", time.Since(startTime))
 			if err != nil {
-				closeRows(rows)
-				return exists, err
+				return errors.Err(err)
 			}
-			exists[hash] = true
-		}
+			defer closeRows(rows)
 
-		err = rows.Err()
+			for rows.Next() {
+				err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
+				if err != nil {
+					return errors.Err(err)
+				}
+				exists[hash] = true
+				if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
+					if s.TrackAccess == TrackAccessBlobs {
+						needsTouch = append(needsTouch, blobID)
+					} else if s.TrackAccess == TrackAccessStreams && !streamID.IsZero() {
+						needsTouch = append(needsTouch, streamID.Uint64)
+					}
+				}
+			}
+
+			err = rows.Err()
+			if err != nil {
+				return errors.Err(err)
+			}
+
+			doneIndex += len(batch)
+			return nil
+		}()
 		if err != nil {
-			closeRows(rows)
-			return exists, err
+			return nil, nil, err
 		}
-
-		closeRows(rows)
-		doneIndex += len(batch)
 	}
 
-	return exists, nil
+	return exists, needsTouch, nil
+}
+
+// Delete will remove (or soft-delete) the blob from the db
+// NOTE: If SoftDelete is enabled, streams will never be deleted
+func (s *SQL) Delete(hash string) error {
+	if s.SoftDelete {
+		_, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
+		return errors.Err(err)
+	}
+
+	_, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	_, err = s.exec("DELETE FROM blob_ WHERE hash = ?", hash)
+	return errors.Err(err)
+}
+
+//LeastRecentlyAccessedHashes gets the least recently accessed blobs
+func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
+	if s.conn == nil {
+		return nil, errors.Err("not connected")
+	}
+
+	if s.TrackAccess != TrackAccessBlobs {
+		return nil, errors.Err("blob access tracking is disabled")
+	}
+
+	query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?"
+	s.logQuery(query, maxBlobs)
+
+	rows, err := s.conn.Query(query, maxBlobs)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer closeRows(rows)
+
+	blobs := make([]string, 0, maxBlobs)
+	for rows.Next() {
+		var hash string
+		err := rows.Scan(&hash)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		blobs = append(blobs, hash)
+	}
+
+	return blobs, nil
+}
+
+func (s *SQL) Count() (int, error) {
+	if s.conn == nil {
+		return 0, errors.Err("not connected")
+	}
+
+	query := "SELECT count(id) from blob_"
+	if s.SoftDelete {
+		query += " where is_stored = 1"
+	}
+	s.logQuery(query)
+
+	var count int
+	err := s.conn.QueryRow(query).Scan(&count)
+	return count, errors.Err(err)
+}
+
+// Block will mark a blob as blocked
+func (s *SQL) Block(hash string) error {
+	query := "INSERT IGNORE INTO blocked SET hash = ?"
+	args := []interface{}{hash}
+	s.logQuery(query, args...)
+	_, err := s.conn.Exec(query, args...)
+	return errors.Err(err)
+}
+
+// GetBlocked will return a list of blocked hashes
+func (s *SQL) GetBlocked() (map[string]bool, error) {
+	query := "SELECT hash FROM blocked"
+	s.logQuery(query)
+	rows, err := s.conn.Query(query)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer closeRows(rows)
+
+	blocked := make(map[string]bool)
+
+	var hash string
+	for rows.Next() {
+		err := rows.Scan(&hash)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		blocked[hash] = true
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	return blocked, nil
 }
 
 // MissingBlobsForKnownStream returns missing blobs for an existing stream
@@ -168,13 +515,14 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
 
 	query := `
 		SELECT b.hash FROM blob_ b
-		INNER JOIN stream_blob sb ON b.hash = sb.blob_hash
-		INNER JOIN stream s ON s.hash = sb.stream_hash AND s.sd_hash = ?
+		INNER JOIN stream_blob sb ON b.id = sb.blob_id
+		INNER JOIN stream s ON s.id = sb.stream_id
+		INNER JOIN blob_ sdb ON sdb.id = s.sd_blob_id AND sdb.hash = ?
 		WHERE b.is_stored = 0
 	`
 	args := []interface{}{sdHash}
 
-	logQuery(query, args...)
+	s.logQuery(query, args...)
 
 	rows, err := s.conn.Query(query, args...)
 	if err != nil {
@@ -201,52 +549,45 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
 	return missingBlobs, errors.Err(err)
 }
 
-// AddSDBlob takes the SD Hash number of blobs and the set of blobs. In a single db tx it inserts the sdblob information
-// into a stream, and inserts the associated blobs' information in the database. If a blob fails the transaction is
-// rolled back and error(s) are returned.
+// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
+// but they are tracked so reflector knows what it is missing.
 func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
 	if s.conn == nil {
 		return errors.Err("not connected")
 	}
 
-	return withTx(s.conn, func(tx *sql.Tx) error {
-		// insert sd blob
-		err := addBlob(tx, sdHash, sdBlobLength, true)
+	sdBlobID, err := s.insertBlob(sdHash, sdBlobLength, true)
+	if err != nil {
+		return err
+	}
+
+	streamID, err := s.insertStream(sdBlob.StreamHash, sdBlobID)
+	if err != nil {
+		return err
+	}
+
+	// insert content blobs and connect them to stream
+	for _, contentBlob := range sdBlob.Blobs {
+		if contentBlob.BlobHash == "" {
+			// null terminator blob
+			continue
+		}
+
+		blobID, err := s.insertBlob(contentBlob.BlobHash, contentBlob.Length, false)
 		if err != nil {
 			return err
 		}
 
-		// insert stream
-		err = execTx(tx,
-			"INSERT IGNORE INTO stream (hash, sd_hash) VALUES (?,?)",
-			[]interface{}{sdBlob.StreamHash, sdHash},
+		args := []interface{}{streamID, blobID, contentBlob.BlobNum}
+		_, err = s.exec(
+			"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
+			args...,
 		)
 		if err != nil {
 			return errors.Err(err)
 		}
-
-		// insert content blobs and connect them to stream
-		for _, contentBlob := range sdBlob.Blobs {
-			if contentBlob.BlobHash == "" {
-				// null terminator blob
-				continue
-			}
-
-			err := addBlob(tx, contentBlob.BlobHash, contentBlob.Length, false)
-			if err != nil {
-				return err
-			}
-
-			err = execTx(tx,
-				"INSERT IGNORE INTO stream_blob (stream_hash, blob_hash, num) VALUES (?,?,?)",
-				[]interface{}{sdBlob.StreamHash, contentBlob.BlobHash, contentBlob.BlobNum},
-			)
-			if err != nil {
-				return errors.Err(err)
-			}
-		}
-		return nil
-	})
+	}
+	return nil
 }
 
 // GetHashRange gets the smallest and biggest hashes in the db
@@ -260,7 +601,7 @@ func (s *SQL) GetHashRange() (string, string, error) {
 
 	query := "SELECT MIN(hash), MAX(hash) from blob_"
 
-	logQuery(query)
+	s.logQuery(query)
 
 	err := s.conn.QueryRow(query).Scan(&min, &max)
 	return min, max, err
@@ -284,7 +625,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
 		query := "SELECT hash FROM blob_ WHERE hash >= ? AND hash <= ? AND is_stored = 1"
 		args := []interface{}{start.Hex(), end.Hex()}
 
-		logQuery(query, args...)
+		s.logQuery(query, args...)
 
 		rows, err := s.conn.Query(query, args...)
 		defer closeRows(rows)
@@ -364,38 +705,77 @@ func closeRows(rows *sql.Rows) {
 	}
 }
 
-func execTx(tx *sql.Tx, query string, args []interface{}) error {
-	logQuery(query, args...)
-	_, err := tx.Exec(query, args...)
-	return errors.Err(err)
+func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
+	s.logQuery(query, args...)
+	attempt, maxAttempts := 0, 3
+Retry:
+	attempt++
+	result, err := s.conn.Exec(query, args...)
+	if isLockTimeoutError(err) {
+		if attempt <= maxAttempts {
+			//Error 1205: Lock wait timeout exceeded; try restarting transaction
+			goto Retry
+		}
+		err = errors.Prefix("Lock timeout for query "+query, err)
+	}
+
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+
+	lastID, err := result.LastInsertId()
+	return lastID, errors.Err(err)
+}
+
+func isLockTimeoutError(err error) bool {
+	e, ok := err.(*mysql.MySQLError)
+	return ok && e != nil && e.Number == 1205
 }
 
 /*  SQL schema
 
+in prod, set tx_isolation to READ-COMMITTED to improve db performance
+make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
+
+todo: could add UNIQUE KEY (stream_hash, num) to stream_blob ...
+
 CREATE TABLE blob_ (
+  id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
   hash char(96) NOT NULL,
   is_stored TINYINT(1) NOT NULL DEFAULT 0,
   length bigint(20) unsigned DEFAULT NULL,
-  PRIMARY KEY (hash)
+  last_accessed_at TIMESTAMP NULL DEFAULT NULL,
+  PRIMARY KEY (id),
+  UNIQUE KEY blob_hash_idx (hash),
+  KEY `blob_last_accessed_idx` (`last_accessed_at`),
+  KEY `is_stored_idx` (`is_stored`)
 );
 
 CREATE TABLE stream (
+  id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
   hash char(96) NOT NULL,
-  sd_hash char(96) NOT NULL,
-  PRIMARY KEY (hash),
-  KEY sd_hash_idx (sd_hash),
-  FOREIGN KEY (sd_hash) REFERENCES blob_ (hash) ON DELETE RESTRICT ON UPDATE CASCADE
+  sd_blob_id BIGINT UNSIGNED NOT NULL,
+  last_accessed_at TIMESTAMP NULL DEFAULT NULL,
+  PRIMARY KEY (id),
+  UNIQUE KEY stream_hash_idx (hash),
+  KEY stream_sd_blob_id_idx (sd_blob_id),
+  KEY last_accessed_at_idx (last_accessed_at),
+  FOREIGN KEY (sd_blob_id) REFERENCES blob_ (id) ON DELETE RESTRICT ON UPDATE CASCADE
 );
 
 CREATE TABLE stream_blob (
-  stream_hash char(96) NOT NULL,
-  blob_hash char(96) NOT NULL,
+  stream_id BIGINT UNSIGNED NOT NULL,
+  blob_id BIGINT UNSIGNED NOT NULL,
   num int NOT NULL,
-  PRIMARY KEY (stream_hash, blob_hash),
-  FOREIGN KEY (stream_hash) REFERENCES stream (hash) ON DELETE CASCADE ON UPDATE CASCADE,
-  FOREIGN KEY (blob_hash) REFERENCES blob_ (hash) ON DELETE CASCADE ON UPDATE CASCADE
+  PRIMARY KEY (stream_id, blob_id),
+  KEY stream_blob_blob_id_idx (blob_id),
+  FOREIGN KEY (stream_id) REFERENCES stream (id) ON DELETE CASCADE ON UPDATE CASCADE,
+  FOREIGN KEY (blob_id) REFERENCES blob_ (id) ON DELETE CASCADE ON UPDATE CASCADE
 );
 
-could add UNIQUE KEY (stream_hash, num) to stream_blob ...
+CREATE TABLE blocked (
+  hash char(96) NOT NULL,
+  PRIMARY KEY (hash)
+);
 
 */

dht/bits/bitmap.go

@@ -1,399 +0,0 @@
-package bits
-
-import (
-	"crypto/rand"
-	"encoding/hex"
-	"math/big"
-	"strconv"
-	"strings"
-
-	"github.com/lbryio/lbry.go/errors"
-
-	"github.com/lyoshenka/bencode"
-)
-
-// TODO: http://roaringbitmap.org/
-
-const (
-	NumBytes = 48 // bytes
-	NumBits  = NumBytes * 8
-)
-
-// Bitmap is a generalized representation of an identifier or data that can be sorted, compared fast. Used by the DHT
-// package as a way to handle the unique identifiers of a DHT node.
-type Bitmap [NumBytes]byte
-
-func (b Bitmap) RawString() string {
-	return string(b[:])
-}
-
-func (b Bitmap) String() string {
-	return b.Hex()
-}
-
-// BString returns the bitmap as a string of 0s and 1s
-func (b Bitmap) BString() string {
-	var s string
-	for _, byte := range b {
-		s += strconv.FormatInt(int64(byte), 2)
-	}
-	return s
-}
-
-// Hex returns a hexadecimal representation of the bitmap.
-func (b Bitmap) Hex() string {
-	return hex.EncodeToString(b[:])
-}
-
-// HexShort returns a hexadecimal representation of the first 4 bytes.
-func (b Bitmap) HexShort() string {
-	return hex.EncodeToString(b[:4])
-}
-
-// HexSimplified returns the hexadecimal representation with all leading 0's removed
-func (b Bitmap) HexSimplified() string {
-	simple := strings.TrimLeft(b.Hex(), "0")
-	if simple == "" {
-		simple = "0"
-	}
-	return simple
-}
-
-func (b Bitmap) Big() *big.Int {
-	i := new(big.Int)
-	i.SetString(b.Hex(), 16)
-	return i
-}
-
-// Cmp compares b and other and returns:
-//
-//   -1 if b < other
-//    0 if b == other
-//   +1 if b > other
-//
-func (b Bitmap) Cmp(other Bitmap) int {
-	for k := range b {
-		if b[k] < other[k] {
-			return -1
-		} else if b[k] > other[k] {
-			return 1
-		}
-	}
-	return 0
-}
-
-// Closer returns true if dist(b,x) < dist(b,y)
-func (b Bitmap) Closer(x, y Bitmap) bool {
-	return x.Xor(b).Cmp(y.Xor(b)) < 0
-}
-
-// Equals returns true if every byte in bitmap are equal, false otherwise
-func (b Bitmap) Equals(other Bitmap) bool {
-	return b.Cmp(other) == 0
-}
-
-// Copy returns a duplicate value for the bitmap.
-func (b Bitmap) Copy() Bitmap {
-	var ret Bitmap
-	copy(ret[:], b[:])
-	return ret
-}
-
-// Xor returns a diff bitmap. If they are equal, the returned bitmap will be all 0's. If 100% unique the returned
-// bitmap will be all 1's.
-func (b Bitmap) Xor(other Bitmap) Bitmap {
-	var ret Bitmap
-	for k := range b {
-		ret[k] = b[k] ^ other[k]
-	}
-	return ret
-}
-
-// And returns a comparison bitmap, that for each byte returns the AND true table result
-func (b Bitmap) And(other Bitmap) Bitmap {
-	var ret Bitmap
-	for k := range b {
-		ret[k] = b[k] & other[k]
-	}
-	return ret
-}
-
-// Or returns a comparison bitmap, that for each byte returns the OR true table result
-func (b Bitmap) Or(other Bitmap) Bitmap {
-	var ret Bitmap
-	for k := range b {
-		ret[k] = b[k] | other[k]
-	}
-	return ret
-}
-
-// Not returns a complimentary bitmap that is an inverse. So b.NOT.NOT = b
-func (b Bitmap) Not() Bitmap {
-	var ret Bitmap
-	for k := range b {
-		ret[k] = ^b[k]
-	}
-	return ret
-}
-
-func (b Bitmap) add(other Bitmap) (Bitmap, bool) {
-	var ret Bitmap
-	carry := false
-	for i := NumBits - 1; i >= 0; i-- {
-		bBit := getBit(b[:], i)
-		oBit := getBit(other[:], i)
-		setBit(ret[:], i, bBit != oBit != carry)
-		carry = (bBit && oBit) || (bBit && carry) || (oBit && carry)
-	}
-	return ret, carry
-}
-
-// Add returns a bitmap that treats both bitmaps as numbers and adding them together. Since the size of a bitmap is
-// limited, an overflow is possible when adding bitmaps.
-func (b Bitmap) Add(other Bitmap) Bitmap {
-	ret, carry := b.add(other)
-	if carry {
-		panic("overflow in bitmap addition. limited to " + strconv.Itoa(NumBits) + " bits.")
-	}
-	return ret
-}
-
-// Sub returns a bitmap that treats both bitmaps as numbers and subtracts then via the inverse of the other and adding
-// then together a + (-b). Negative bitmaps are not supported so other must be greater than this.
-func (b Bitmap) Sub(other Bitmap) Bitmap {
-	if b.Cmp(other) < 0 {
-		// ToDo: Why is this not supported? Should it say not implemented? BitMap might have a generic use case outside of dht.
-		panic("negative bitmaps not supported")
-	}
-	complement, _ := other.Not().add(FromShortHexP("1"))
-	ret, _ := b.add(complement)
-	return ret
-}
-
-// Get returns the binary bit at the position passed.
-func (b Bitmap) Get(n int) bool {
-	return getBit(b[:], n)
-}
-
-// Set sets the binary bit at the position passed.
-func (b Bitmap) Set(n int, one bool) Bitmap {
-	ret := b.Copy()
-	setBit(ret[:], n, one)
-	return ret
-}
-
-// PrefixLen returns the number of leading 0 bits
-func (b Bitmap) PrefixLen() int {
-	for i := range b {
-		for j := 0; j < 8; j++ {
-			if (b[i]>>uint8(7-j))&0x1 != 0 {
-				return i*8 + j
-			}
-		}
-	}
-	return NumBits
-}
-
-// Prefix returns a copy of b with the first n bits set to 1 (if `one` is true) or 0 (if `one` is false)
-// https://stackoverflow.com/a/23192263/182709
-func (b Bitmap) Prefix(n int, one bool) Bitmap {
-	ret := b.Copy()
-
-Outer:
-	for i := range ret {
-		for j := 0; j < 8; j++ {
-			if i*8+j < n {
-				if one {
-					ret[i] |= 1 << uint(7-j)
-				} else {
-					ret[i] &= ^(1 << uint(7-j))
-				}
-			} else {
-				break Outer
-			}
-		}
-	}
-
-	return ret
-}
-
-// Suffix returns a copy of b with the last n bits set to 1 (if `one` is true) or 0 (if `one` is false)
-// https://stackoverflow.com/a/23192263/182709
-func (b Bitmap) Suffix(n int, one bool) Bitmap {
-	ret := b.Copy()
-
-Outer:
-	for i := len(ret) - 1; i >= 0; i-- {
-		for j := 7; j >= 0; j-- {
-			if i*8+j >= NumBits-n {
-				if one {
-					ret[i] |= 1 << uint(7-j)
-				} else {
-					ret[i] &= ^(1 << uint(7-j))
-				}
-			} else {
-				break Outer
-			}
-		}
-	}
-
-	return ret
-}
-
-// MarshalBencode implements the Marshaller(bencode)/Message interface.
-func (b Bitmap) MarshalBencode() ([]byte, error) {
-	str := string(b[:])
-	return bencode.EncodeBytes(str)
-}
-
-// UnmarshalBencode implements the Marshaller(bencode)/Message interface.
-func (b *Bitmap) UnmarshalBencode(encoded []byte) error {
-	var str string
-	err := bencode.DecodeBytes(encoded, &str)
-	if err != nil {
-		return err
-	}
-	if len(str) != NumBytes {
-		return errors.Err("invalid bitmap length")
-	}
-	copy(b[:], str)
-	return nil
-}
-
-// FromBytes returns a bitmap as long as the byte array is of a specific length specified in the parameters.
-func FromBytes(data []byte) (Bitmap, error) {
-	var bmp Bitmap
-
-	if len(data) != len(bmp) {
-		return bmp, errors.Err("invalid bitmap of length %d", len(data))
-	}
-
-	copy(bmp[:], data)
-	return bmp, nil
-}
-
-// FromBytesP returns a bitmap as long as the byte array is of a specific length specified in the parameters
-// otherwise it wil panic.
-func FromBytesP(data []byte) Bitmap {
-	bmp, err := FromBytes(data)
-	if err != nil {
-		panic(err)
-	}
-	return bmp
-}
-
-//FromString returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
-// is of a specific length specified in the parameters
-func FromString(data string) (Bitmap, error) {
-	return FromBytes([]byte(data))
-}
-
-//FromStringP returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
-// is of a specific length specified in the parameters otherwise it wil panic.
-func FromStringP(data string) Bitmap {
-	bmp, err := FromString(data)
-	if err != nil {
-		panic(err)
-	}
-	return bmp
-}
-
-//FromHex returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
-// is of a specific length specified in the parameters
-func FromHex(hexStr string) (Bitmap, error) {
-	decoded, err := hex.DecodeString(hexStr)
-	if err != nil {
-		return Bitmap{}, errors.Err(err)
-	}
-	return FromBytes(decoded)
-}
-
-//FromHexP returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
-// is of a specific length specified in the parameters otherwise it wil panic.
-func FromHexP(hexStr string) Bitmap {
-	bmp, err := FromHex(hexStr)
-	if err != nil {
-		panic(err)
-	}
-	return bmp
-}
-
-//FromShortHex returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
-// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
-func FromShortHex(hexStr string) (Bitmap, error) {
-	return FromHex(strings.Repeat("0", NumBytes*2-len(hexStr)) + hexStr)
-}
-
-//FromShortHexP returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
-// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
-// otherwise it wil panic.
-func FromShortHexP(hexStr string) Bitmap {
-	bmp, err := FromShortHex(hexStr)
-	if err != nil {
-		panic(err)
-	}
-	return bmp
-}
-
-func FromBigP(b *big.Int) Bitmap {
-	return FromShortHexP(b.Text(16))
-}
-
-// Max returns a bitmap with all bits set to 1
-func MaxP() Bitmap {
-	return FromHexP(strings.Repeat("f", NumBytes*2))
-}
-
-// Rand generates a cryptographically random bitmap with the confines of the parameters specified.
-func Rand() Bitmap {
-	var id Bitmap
-	_, err := rand.Read(id[:])
-	if err != nil {
-		panic(err)
-	}
-	return id
-}
-
-// RandInRangeP generates a cryptographically random bitmap and while it is greater than the high threshold
-// bitmap will subtract the diff between high and low until it is no longer greater that the high.
-func RandInRangeP(low, high Bitmap) Bitmap {
-	diff := high.Sub(low)
-	r := Rand()
-	for r.Cmp(diff) > 0 {
-		r = r.Sub(diff)
-	}
-	//ToDo - Adding the low at this point doesn't gurantee it will be within the range. Consider bitmaps as numbers and
-	// I have a range of 50-100. If get to say 60, and add 50, I would be at 110. Should protect against this?
-	return r.Add(low)
-}
-
-func getBit(b []byte, n int) bool {
-	i := n / 8
-	j := n % 8
-	return b[i]&(1<<uint(7-j)) > 0
-}
-
-func setBit(b []byte, n int, one bool) {
-	i := n / 8
-	j := n % 8
-	if one {
-		b[i] |= 1 << uint(7-j)
-	} else {
-		b[i] &= ^(1 << uint(7-j))
-	}
-}
-
-// CLosest returns the closest bitmap to target. if no bitmaps are provided, target itself is returned
-func Closest(target Bitmap, bitmaps ...Bitmap) Bitmap {
-	if len(bitmaps) == 0 {
-		return target
-	}
-
-	var closest *Bitmap
-	for _, b := range bitmaps {
-		if closest == nil || target.Closer(b, *closest) {
-			closest = &b
-		}
-	}
-	return *closest
-}

dht/bits/bitmap_test.go

@@ -1,386 +0,0 @@
-package bits
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/lyoshenka/bencode"
-)
-
-func TestBitmap(t *testing.T) {
-	a := Bitmap{
-		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
-		12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
-		24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
-		36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
-	}
-	b := Bitmap{
-		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
-		12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
-		24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
-		36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 47, 46,
-	}
-	c := Bitmap{
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
-	}
-
-	if !a.Equals(a) {
-		t.Error("bitmap does not equal itself")
-	}
-	if a.Equals(b) {
-		t.Error("bitmap equals another bitmap with different id")
-	}
-
-	if !a.Xor(b).Equals(c) {
-		t.Error(a.Xor(b))
-	}
-
-	if c.PrefixLen() != 375 {
-		t.Error(c.PrefixLen())
-	}
-
-	if b.Cmp(a) < 0 {
-		t.Error("bitmap fails Cmp test")
-	}
-
-	if a.Closer(c, b) || !a.Closer(b, c) || c.Closer(a, b) || c.Closer(b, c) {
-		t.Error("bitmap fails Closer test")
-	}
-
-	id := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
-	if FromHexP(id).Hex() != id {
-		t.Error(FromHexP(id).Hex())
-	}
-}
-
-func TestBitmap_GetBit(t *testing.T) {
-	tt := []struct {
-		bit      int
-		expected bool
-		panic    bool
-	}{
-		{bit: 383, expected: false, panic: false},
-		{bit: 382, expected: true, panic: false},
-		{bit: 381, expected: false, panic: false},
-		{bit: 380, expected: true, panic: false},
-	}
-
-	b := FromShortHexP("a")
-
-	for _, test := range tt {
-		actual := getBit(b[:], test.bit)
-		if test.expected != actual {
-			t.Errorf("getting bit %d of %s: expected %t, got %t", test.bit, b.HexSimplified(), test.expected, actual)
-		}
-	}
-}
-
-func TestBitmap_SetBit(t *testing.T) {
-	tt := []struct {
-		hex      string
-		bit      int
-		one      bool
-		expected string
-		panic    bool
-	}{
-		{hex: "0", bit: 383, one: true, expected: "1", panic: false},
-		{hex: "0", bit: 382, one: true, expected: "2", panic: false},
-		{hex: "0", bit: 381, one: true, expected: "4", panic: false},
-		{hex: "0", bit: 385, one: true, expected: "1", panic: true},
-		{hex: "0", bit: 384, one: true, expected: "1", panic: true},
-	}
-
-	for _, test := range tt {
-		expected := FromShortHexP(test.expected)
-		actual := FromShortHexP(test.hex)
-		if test.panic {
-			assertPanic(t, fmt.Sprintf("setting bit %d to %t", test.bit, test.one), func() { setBit(actual[:], test.bit, test.one) })
-		} else {
-			setBit(actual[:], test.bit, test.one)
-			if !expected.Equals(actual) {
-				t.Errorf("setting bit %d to %t: expected %s, got %s", test.bit, test.one, test.expected, actual.HexSimplified())
-			}
-
-		}
-	}
-}
-
-func TestBitmap_FromHexShort(t *testing.T) {
-	tt := []struct {
-		short string
-		long  string
-	}{
-		{short: "", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{short: "0", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{short: "00000", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{short: "9473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
-		{short: "09473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
-		{short: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
-			long: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"},
-	}
-
-	for _, test := range tt {
-		short := FromShortHexP(test.short)
-		long := FromHexP(test.long)
-		if !short.Equals(long) {
-			t.Errorf("short hex %s: expected %s, got %s", test.short, long.Hex(), short.Hex())
-		}
-	}
-}
-
-func TestBitmapMarshal(t *testing.T) {
-	b := FromStringP("123456789012345678901234567890123456789012345678")
-	encoded, err := bencode.EncodeBytes(b)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if string(encoded) != "48:123456789012345678901234567890123456789012345678" {
-		t.Error("encoding does not match expected")
-	}
-}
-
-func TestBitmapMarshalEmbedded(t *testing.T) {
-	e := struct {
-		A string
-		B Bitmap
-		C int
-	}{
-		A: "1",
-		B: FromStringP("222222222222222222222222222222222222222222222222"),
-		C: 3,
-	}
-
-	encoded, err := bencode.EncodeBytes(e)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if string(encoded) != "d1:A1:11:B48:2222222222222222222222222222222222222222222222221:Ci3ee" {
-		t.Error("encoding does not match expected")
-	}
-}
-
-func TestBitmapMarshalEmbedded2(t *testing.T) {
-	encoded, err := bencode.EncodeBytes([]interface{}{
-		FromStringP("333333333333333333333333333333333333333333333333"),
-	})
-	if err != nil {
-		t.Error(err)
-	}
-
-	if string(encoded) != "l48:333333333333333333333333333333333333333333333333e" {
-		t.Error("encoding does not match expected")
-	}
-}
-
-func TestBitmap_PrefixLen(t *testing.T) {
-	tt := []struct {
-		hex string
-		len int
-	}{
-		{len: 0, hex: "F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{len: 0, hex: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{len: 1, hex: "700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{len: 1, hex: "400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{len: 384, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{len: 383, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
-		{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"},
-		{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"},
-	}
-
-	for _, test := range tt {
-		len := FromHexP(test.hex).PrefixLen()
-		if len != test.len {
-			t.Errorf("got prefix len %d; expected %d for %s", len, test.len, test.hex)
-		}
-	}
-}
-
-func TestBitmap_Prefix(t *testing.T) {
-	allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
-
-	zerosTT := []struct {
-		zeros    int
-		expected string
-	}{
-		{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-		{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-		{zeros: 1, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-		{zeros: 69, expected: "000000000000000007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-		{zeros: 383, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
-		{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-	}
-
-	for _, test := range zerosTT {
-		expected := FromHexP(test.expected)
-		actual := allOne.Prefix(test.zeros, false)
-		if !actual.Equals(expected) {
-			t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
-		}
-	}
-
-	for i := 0; i < NumBits; i++ {
-		b := allOne.Prefix(i, false)
-		if b.PrefixLen() != i {
-			t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
-		}
-	}
-
-	allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
-
-	onesTT := []struct {
-		ones     int
-		expected string
-	}{
-		{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{ones: 1, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{ones: 69, expected: "fffffffffffffffff8000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{ones: 383, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
-		{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-		{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-	}
-
-	for _, test := range onesTT {
-		expected := FromHexP(test.expected)
-		actual := allZero.Prefix(test.ones, true)
-		if !actual.Equals(expected) {
-			t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
-		}
-	}
-}
-
-func TestBitmap_Suffix(t *testing.T) {
-	allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
-
-	zerosTT := []struct {
-		zeros    int
-		expected string
-	}{
-		{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-		{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-		{zeros: 1, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
-		{zeros: 69, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe00000000000000000"},
-		{zeros: 383, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-	}
-
-	for _, test := range zerosTT {
-		expected := FromHexP(test.expected)
-		actual := allOne.Suffix(test.zeros, false)
-		if !actual.Equals(expected) {
-			t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
-		}
-	}
-
-	for i := 0; i < NumBits; i++ {
-		b := allOne.Prefix(i, false)
-		if b.PrefixLen() != i {
-			t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
-		}
-	}
-
-	allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
-
-	onesTT := []struct {
-		ones     int
-		expected string
-	}{
-		{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
-		{ones: 1, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
-		{ones: 69, expected: "0000000000000000000000000000000000000000000000000000000000000000000000000000001fffffffffffffffff"},
-		{ones: 383, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-		{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-		{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
-	}
-
-	for _, test := range onesTT {
-		expected := FromHexP(test.expected)
-		actual := allZero.Suffix(test.ones, true)
-		if !actual.Equals(expected) {
-			t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
-		}
-	}
-}
-
-func TestBitmap_Add(t *testing.T) {
-	tt := []struct {
-		a, b, sum string
-		panic     bool
-	}{
-		{"0", "0", "0", false},
-		{"0", "1", "1", false},
-		{"1", "0", "1", false},
-		{"1", "1", "2", false},
-		{"8", "4", "c", false},
-		{"1000", "0010", "1010", false},
-		{"1111", "1111", "2222", false},
-		{"ffff", "1", "10000", false},
-		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
-		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "1", "", true},
-	}
-
-	for _, test := range tt {
-		a := FromShortHexP(test.a)
-		b := FromShortHexP(test.b)
-		expected := FromShortHexP(test.sum)
-		if test.panic {
-			assertPanic(t, fmt.Sprintf("adding %s and %s", test.a, test.b), func() { a.Add(b) })
-		} else {
-			actual := a.Add(b)
-			if !expected.Equals(actual) {
-				t.Errorf("adding %s and %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
-			}
-		}
-	}
-}
-
-func TestBitmap_Sub(t *testing.T) {
-	tt := []struct {
-		a, b, sum string
-		panic     bool
-	}{
-		{"0", "0", "0", false},
-		{"1", "0", "1", false},
-		{"1", "1", "0", false},
-		{"8", "4", "4", false},
-		{"f", "9", "6", false},
-		{"f", "e", "1", false},
-		{"10", "f", "1", false},
-		{"2222", "1111", "1111", false},
-		{"ffff", "1", "fffe", false},
-		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
-		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", false},
-		{"0", "1", "", true},
-	}
-
-	for _, test := range tt {
-		a := FromShortHexP(test.a)
-		b := FromShortHexP(test.b)
-		expected := FromShortHexP(test.sum)
-		if test.panic {
-			assertPanic(t, fmt.Sprintf("subtracting %s - %s", test.a, test.b), func() { a.Sub(b) })
-		} else {
-			actual := a.Sub(b)
-			if !expected.Equals(actual) {
-				t.Errorf("subtracting %s - %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
-			}
-		}
-	}
-}
-
-func assertPanic(t *testing.T, text string, f func()) {
-	defer func() {
-		if r := recover(); r == nil {
-			t.Errorf("%s: did not panic as expected", text)
-		}
-	}()
-	f()
-}

dht/bits/range.go

@@ -1,65 +0,0 @@
-package bits
-
-import (
-	"math/big"
-
-	"github.com/lbryio/errors.go"
-)
-
-// Range has a start and end
-type Range struct {
-	Start Bitmap
-	End   Bitmap
-}
-
-func MaxRange() Range {
-	return Range{
-		Start: Bitmap{},
-		End:   MaxP(),
-	}
-}
-
-// IntervalP divides the range into `num` intervals and returns the `n`th one
-// intervals are approximately the same size, but may not be exact because of rounding issues
-// the first interval always starts at the beginning of the range, and the last interval always ends at the end
-func (r Range) IntervalP(n, num int) Range {
-	if num < 1 || n < 1 || n > num {
-		panic(errors.Err("invalid interval %d of %d", n, num))
-	}
-
-	start := r.intervalStart(n, num)
-	end := r.End.Big()
-	if n < num {
-		end = r.intervalStart(n+1, num)
-		end.Sub(end, big.NewInt(1))
-	}
-
-	return Range{FromBigP(start), FromBigP(end)}
-}
-
-func (r Range) intervalStart(n, num int) *big.Int {
-	// formula:
-	// size = (end - start) / num
-	// rem = (end - start) % num
-	// intervalStart = rangeStart + (size * n-1) + ((rem * n-1) % num)
-
-	size := new(big.Int)
-	rem := new(big.Int)
-	size.Sub(r.End.Big(), r.Start.Big()).DivMod(size, big.NewInt(int64(num)), rem)
-
-	size.Mul(size, big.NewInt(int64(n-1)))
-	rem.Mul(rem, big.NewInt(int64(n-1))).Mod(rem, big.NewInt(int64(num)))
-
-	start := r.Start.Big()
-	start.Add(start, size).Add(start, rem)
-
-	return start
-}
-
-func (r Range) IntervalSize() *big.Int {
-	return (&big.Int{}).Sub(r.End.Big(), r.Start.Big())
-}
-
-func (r Range) Contains(b Bitmap) bool {
-	return r.Start.Cmp(b) <= 0 && r.End.Cmp(b) >= 0
-}

dht/bits/range_test.go

@@ -1,48 +0,0 @@
-package bits
-
-import (
-	"math/big"
-	"testing"
-)
-
-func TestMaxRange(t *testing.T) {
-	start := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
-	end := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
-	r := MaxRange()
-
-	if !r.Start.Equals(start) {
-		t.Error("max range does not start at the beginning")
-	}
-	if !r.End.Equals(end) {
-		t.Error("max range does not end at the end")
-	}
-}
-
-func TestRange_IntervalP(t *testing.T) {
-	max := MaxRange()
-
-	numIntervals := 97
-	expectedAvg := (&big.Int{}).Div(max.IntervalSize(), big.NewInt(int64(numIntervals)))
-	maxDiff := big.NewInt(int64(numIntervals))
-
-	var lastEnd Bitmap
-
-	for i := 1; i <= numIntervals; i++ {
-		ival := max.IntervalP(i, numIntervals)
-		if i == 1 && !ival.Start.Equals(max.Start) {
-			t.Error("first interval does not start at 0")
-		}
-		if i == numIntervals && !ival.End.Equals(max.End) {
-			t.Error("last interval does not end at max")
-		}
-		if i > 1 && !ival.Start.Equals(lastEnd.Add(FromShortHexP("1"))) {
-			t.Errorf("interval %d of %d: last end was %s, this start is %s", i, numIntervals, lastEnd.Hex(), ival.Start.Hex())
-		}
-
-		if ival.IntervalSize().Cmp((&big.Int{}).Add(expectedAvg, maxDiff)) > 0 || ival.IntervalSize().Cmp((&big.Int{}).Sub(expectedAvg, maxDiff)) < 0 {
-			t.Errorf("interval %d of %d: interval size is outside the normal range", i, numIntervals)
-		}
-
-		lastEnd = ival.End
-	}
-}

dht/bootstrap.go

@@ -1,212 +0,0 @@
-package dht
-
-import (
-	"math/rand"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-)
-
-const (
-	bootstrapDefaultRefreshDuration = 15 * time.Minute
-)
-
-// BootstrapNode is a configured node setup for testing.
-type BootstrapNode struct {
-	Node
-
-	initialPingInterval time.Duration
-	checkInterval       time.Duration
-
-	nlock   *sync.RWMutex
-	peers   map[bits.Bitmap]*peer
-	nodeIDs []bits.Bitmap // necessary for efficient random ID selection
-}
-
-// NewBootstrapNode returns a BootstrapNode pointer.
-func NewBootstrapNode(id bits.Bitmap, initialPingInterval, rePingInterval time.Duration) *BootstrapNode {
-	b := &BootstrapNode{
-		Node: *NewNode(id),
-
-		initialPingInterval: initialPingInterval,
-		checkInterval:       rePingInterval,
-
-		nlock:   &sync.RWMutex{},
-		peers:   make(map[bits.Bitmap]*peer),
-		nodeIDs: make([]bits.Bitmap, 0),
-	}
-
-	b.requestHandler = b.handleRequest
-
-	return b
-}
-
-// Add manually adds a contact
-func (b *BootstrapNode) Add(c Contact) {
-	b.upsert(c)
-}
-
-// Connect connects to the given connection and starts any background threads necessary
-func (b *BootstrapNode) Connect(conn UDPConn) error {
-	err := b.Node.Connect(conn)
-	if err != nil {
-		return err
-	}
-
-	log.Infof("[%s] bootstrap: node connected", b.id.HexShort())
-
-	go func() {
-		t := time.NewTicker(b.checkInterval / 5)
-		for {
-			select {
-			case <-t.C:
-				b.check()
-			case <-b.grp.Ch():
-				return
-			}
-		}
-	}()
-
-	return nil
-}
-
-// upsert adds the contact to the list, or updates the lastPinged time
-func (b *BootstrapNode) upsert(c Contact) {
-	b.nlock.Lock()
-	defer b.nlock.Unlock()
-
-	if peer, exists := b.peers[c.ID]; exists {
-		log.Debugf("[%s] bootstrap: touching contact %s", b.id.HexShort(), peer.Contact.ID.HexShort())
-		peer.Touch()
-		return
-	}
-
-	log.Debugf("[%s] bootstrap: adding new contact %s", b.id.HexShort(), c.ID.HexShort())
-	b.peers[c.ID] = &peer{c, b.id.Xor(c.ID), time.Now(), 0}
-	b.nodeIDs = append(b.nodeIDs, c.ID)
-}
-
-// remove removes the contact from the list
-func (b *BootstrapNode) remove(c Contact) {
-	b.nlock.Lock()
-	defer b.nlock.Unlock()
-
-	_, exists := b.peers[c.ID]
-	if !exists {
-		return
-	}
-
-	log.Debugf("[%s] bootstrap: removing contact %s", b.id.HexShort(), c.ID.HexShort())
-	delete(b.peers, c.ID)
-	for i := range b.nodeIDs {
-		if b.nodeIDs[i].Equals(c.ID) {
-			b.nodeIDs = append(b.nodeIDs[:i], b.nodeIDs[i+1:]...)
-			break
-		}
-	}
-}
-
-// get returns up to `limit` random contacts from the list
-func (b *BootstrapNode) get(limit int) []Contact {
-	b.nlock.RLock()
-	defer b.nlock.RUnlock()
-
-	if len(b.peers) < limit {
-		limit = len(b.peers)
-	}
-
-	ret := make([]Contact, limit)
-	for i, k := range randKeys(len(b.nodeIDs))[:limit] {
-		ret[i] = b.peers[b.nodeIDs[k]].Contact
-	}
-
-	return ret
-}
-
-// ping pings a node. if the node responds, it is added to the list. otherwise, it is removed
-func (b *BootstrapNode) ping(c Contact) {
-	log.Debugf("[%s] bootstrap: pinging %s", b.id.HexShort(), c.ID.HexShort())
-	b.grp.Add(1)
-	defer b.grp.Done()
-
-	resCh := b.SendAsync(c, Request{Method: pingMethod})
-
-	var res *Response
-
-	select {
-	case res = <-resCh:
-	case <-b.grp.Ch():
-		return
-	}
-
-	if res != nil && res.Data == pingSuccessResponse {
-		b.upsert(c)
-	} else {
-		b.remove(c)
-	}
-}
-
-func (b *BootstrapNode) check() {
-	b.nlock.RLock()
-	defer b.nlock.RUnlock()
-
-	for i := range b.peers {
-		if !b.peers[i].ActiveInLast(b.checkInterval) {
-			go b.ping(b.peers[i].Contact)
-		}
-	}
-}
-
-// handleRequest handles the requests received from udp.
-func (b *BootstrapNode) handleRequest(addr *net.UDPAddr, request Request) {
-	switch request.Method {
-	case pingMethod:
-		err := b.sendMessage(addr, Response{ID: request.ID, NodeID: b.id, Data: pingSuccessResponse})
-		if err != nil {
-			log.Error("error sending response message - ", err)
-		}
-	case findNodeMethod:
-		if request.Arg == nil {
-			log.Errorln("request is missing arg")
-			return
-		}
-
-		err := b.sendMessage(addr, Response{
-			ID:       request.ID,
-			NodeID:   b.id,
-			Contacts: b.get(bucketSize),
-		})
-		if err != nil {
-			log.Error("error sending 'findnodemethod' response message - ", err)
-		}
-	}
-
-	go func() {
-		b.nlock.RLock()
-		_, exists := b.peers[request.NodeID]
-		b.nlock.RUnlock()
-		if !exists {
-			log.Debugf("[%s] bootstrap: queuing %s to ping", b.id.HexShort(), request.NodeID.HexShort())
-			<-time.After(b.initialPingInterval)
-			b.nlock.RLock()
-			_, exists = b.peers[request.NodeID]
-			b.nlock.RUnlock()
-			if !exists {
-				b.ping(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
-			}
-		}
-	}()
-}
-
-func randKeys(max int) []int {
-	keys := make([]int, max)
-	for k := range keys {
-		keys[k] = k
-	}
-	rand.Shuffle(max, func(i, j int) {
-		keys[i], keys[j] = keys[j], keys[i]
-	})
-	return keys
-}

dht/bootstrap_test.go

@@ -1,24 +0,0 @@
-package dht
-
-import (
-	"net"
-	"testing"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-)
-
-func TestBootstrapPing(t *testing.T) {
-	b := NewBootstrapNode(bits.Rand(), 10, bootstrapDefaultRefreshDuration)
-
-	listener, err := net.ListenPacket(Network, "127.0.0.1:54320")
-	if err != nil {
-		panic(err)
-	}
-
-	err = b.Connect(listener.(*net.UDPConn))
-	if err != nil {
-		t.Error(err)
-	}
-
-	b.Shutdown()
-}

dht/config.go

@@ -1,76 +0,0 @@
-package dht
-
-import (
-	"strconv"
-	"time"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-	peerproto "github.com/lbryio/reflector.go/peer"
-)
-
-const (
-	Network     = "udp4"
-	DefaultPort = 4444
-
-	DefaultAnnounceRate   = 10               // send at most this many announces per second
-	DefaultReannounceTime = 50 * time.Minute // should be a bit less than hash expiration time
-
-	// TODO: all these constants should be defaults, and should be used to set values in the standard Config. then the code should use values in the config
-	// TODO: alternatively, have a global Config for constants. at least that way tests can modify the values
-	alpha           = 3             // this is the constant alpha in the spec
-	bucketSize      = 8             // this is the constant k in the spec
-	nodeIDLength    = bits.NumBytes // bytes. this is the constant B in the spec
-	messageIDLength = 20            // bytes.
-
-	udpRetry            = 1
-	udpTimeout          = 5 * time.Second
-	udpMaxMessageLength = 4096 // bytes. I think our longest message is ~676 bytes, so I rounded up to 1024
-	//                            scratch that. a findValue could return more than K results if a lot of nodes are storing that value, so we need more buffer
-
-	maxPeerFails = 3 // after this many failures, a peer is considered bad and will be removed from the routing table
-	//tExpire     = 60 * time.Minute // the time after which a key/value pair expires; this is a time-to-live (TTL) from the original publication date
-	tRefresh = 1 * time.Hour // the time after which an otherwise unaccessed bucket must be refreshed
-	//tReplicate   = 1 * time.Hour    // the interval between Kademlia replication events, when a node is required to publish its entire database
-	//tNodeRefresh = 15 * time.Minute // the time after which a good node becomes questionable if it has not messaged us
-
-	compactNodeInfoLength = nodeIDLength + 6 // nodeID + 4 for IP + 2 for port
-
-	tokenSecretRotationInterval = 5 * time.Minute // how often the token-generating secret is rotated
-)
-
-// Config represents the configure of dht.
-type Config struct {
-	// this node's address. format is `ip:port`
-	Address string
-	// the seed nodes through which we can join in dht network
-	SeedNodes []string
-	// the hex-encoded node id for this node. if string is empty, a random id will be generated
-	NodeID string
-	// print the state of the dht every X time
-	PrintState time.Duration
-	// the port that clients can use to download blobs using the LBRY peer protocol
-	PeerProtocolPort int
-	// if nonzero, an RPC server will listen to requests on this port and respond to them
-	RPCPort int
-	// the time after which the original publisher must reannounce a key/value pair
-	ReannounceTime time.Duration
-	// send at most this many announces per second
-	AnnounceRate int
-	// channel that will receive notifications about announcements
-	AnnounceNotificationCh chan announceNotification
-}
-
-// NewStandardConfig returns a Config pointer with default values.
-func NewStandardConfig() *Config {
-	return &Config{
-		Address: "0.0.0.0:" + strconv.Itoa(DefaultPort),
-		SeedNodes: []string{
-			"lbrynet1.lbry.io:4444",
-			"lbrynet2.lbry.io:4444",
-			"lbrynet3.lbry.io:4444",
-		},
-		PeerProtocolPort: peerproto.DefaultPort,
-		ReannounceTime:   DefaultReannounceTime,
-		AnnounceRate:     DefaultAnnounceRate,
-	}
-}

dht/contact.go

@@ -1,118 +0,0 @@
-package dht
-
-import (
-	"bytes"
-	"net"
-	"sort"
-	"strconv"
-
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/reflector.go/dht/bits"
-
-	"github.com/lyoshenka/bencode"
-)
-
-// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
-
-// Contact contains information for contacting another node on the network
-type Contact struct {
-	ID       bits.Bitmap
-	IP       net.IP
-	Port     int // the udp port used for the dht
-	PeerPort int // the tcp port a peer can be contacted on for blob requests
-}
-
-// Equals returns true if two contacts are the same.
-func (c Contact) Equals(other Contact, checkID bool) bool {
-	return c.IP.Equal(other.IP) && c.Port == other.Port && (!checkID || c.ID == other.ID)
-}
-
-// Addr returns the address of the contact.
-func (c Contact) Addr() *net.UDPAddr {
-	return &net.UDPAddr{IP: c.IP, Port: c.Port}
-}
-
-// String returns a short string representation of the contact
-func (c Contact) String() string {
-	str := c.ID.HexShort() + "@" + c.Addr().String()
-	if c.PeerPort != 0 {
-		str += "(" + strconv.Itoa(c.PeerPort) + ")"
-	}
-	return str
-}
-
-// MarshalCompact returns a compact byteslice representation of the contact
-// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
-func (c Contact) MarshalCompact() ([]byte, error) {
-	if c.IP.To4() == nil {
-		return nil, errors.Err("ip not set")
-	}
-	if c.PeerPort < 0 || c.PeerPort > 65535 {
-		return nil, errors.Err("invalid port")
-	}
-
-	var buf bytes.Buffer
-	buf.Write(c.IP.To4())
-	buf.WriteByte(byte(c.PeerPort >> 8))
-	buf.WriteByte(byte(c.PeerPort))
-	buf.Write(c.ID[:])
-
-	if buf.Len() != compactNodeInfoLength {
-		return nil, errors.Err("i dont know how this happened")
-	}
-
-	return buf.Bytes(), nil
-}
-
-// UnmarshalCompact unmarshals the compact byteslice representation of a contact.
-// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
-func (c *Contact) UnmarshalCompact(b []byte) error {
-	if len(b) != compactNodeInfoLength {
-		return errors.Err("invalid compact length")
-	}
-	c.IP = net.IPv4(b[0], b[1], b[2], b[3]).To4()
-	c.PeerPort = int(uint16(b[5]) | uint16(b[4])<<8)
-	c.ID = bits.FromBytesP(b[6:])
-	return nil
-}
-
-// MarshalBencode returns the serialized byte slice representation of a contact.
-func (c Contact) MarshalBencode() ([]byte, error) {
-	return bencode.EncodeBytes([]interface{}{c.ID, c.IP.String(), c.Port})
-}
-
-// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the contact.
-func (c *Contact) UnmarshalBencode(b []byte) error {
-	var raw []bencode.RawMessage
-	err := bencode.DecodeBytes(b, &raw)
-	if err != nil {
-		return err
-	}
-
-	if len(raw) != 3 {
-		return errors.Err("contact must have 3 elements; got %d", len(raw))
-	}
-
-	err = bencode.DecodeBytes(raw[0], &c.ID)
-	if err != nil {
-		return err
-	}
-
-	var ipStr string
-	err = bencode.DecodeBytes(raw[1], &ipStr)
-	if err != nil {
-		return err
-	}
-	c.IP = net.ParseIP(ipStr).To4()
-	if c.IP == nil {
-		return errors.Err("invalid IP")
-	}
-
-	return bencode.DecodeBytes(raw[2], &c.Port)
-}
-
-func sortByDistance(contacts []Contact, target bits.Bitmap) {
-	sort.Slice(contacts, func(i, j int) bool {
-		return contacts[i].ID.Xor(target).Cmp(contacts[j].ID.Xor(target)) < 0
-	})
-}

dht/contact_test.go

@@ -1,31 +0,0 @@
-package dht
-
-import (
-	"net"
-	"reflect"
-	"testing"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-)
-
-func TestCompactEncoding(t *testing.T) {
-	c := Contact{
-		ID:       bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"),
-		IP:       net.ParseIP("1.2.3.4"),
-		PeerPort: int(55<<8 + 66),
-	}
-
-	var compact []byte
-	compact, err := c.MarshalCompact()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if len(compact) != compactNodeInfoLength {
-		t.Fatalf("got length of %d; expected %d", len(compact), compactNodeInfoLength)
-	}
-
-	if !reflect.DeepEqual(compact, append([]byte{1, 2, 3, 4, 55, 66}, c.ID[:]...)) {
-		t.Errorf("compact bytes not encoded correctly")
-	}
-}

dht/dht.go

@@ -1,232 +0,0 @@
-package dht
-
-import (
-	"fmt"
-	"net"
-	"strings"
-	"time"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/stop"
-
-	"github.com/sirupsen/logrus"
-	"github.com/spf13/cast"
-)
-
-var log *logrus.Logger
-
-func UseLogger(l *logrus.Logger) {
-	log = l
-}
-
-func init() {
-	log = logrus.StandardLogger()
-	//log.SetFormatter(&log.TextFormatter{ForceColors: true})
-	//log.SetLevel(log.DebugLevel)
-}
-
-// DHT represents a DHT node.
-type DHT struct {
-	// config
-	conf *Config
-	// local contact
-	contact Contact
-	// node
-	node *Node
-	// stopGroup to shut down DHT
-	grp *stop.Group
-	// channel is closed when DHT joins network
-	joined chan struct{}
-	// cache for store tokens
-	tokenCache *tokenCache
-	// hashes that need to be put into the announce queue or removed from the queue
-	announceAddRemove chan queueEdit
-}
-
-// New returns a DHT pointer. If config is nil, then config will be set to the default config.
-func New(config *Config) *DHT {
-	if config == nil {
-		config = NewStandardConfig()
-	}
-
-	d := &DHT{
-		conf:              config,
-		grp:               stop.New(),
-		joined:            make(chan struct{}),
-		announceAddRemove: make(chan queueEdit),
-	}
-	return d
-}
-
-func (dht *DHT) connect(conn UDPConn) error {
-	contact, err := getContact(dht.conf.NodeID, dht.conf.Address)
-	if err != nil {
-		return err
-	}
-
-	dht.contact = contact
-	dht.node = NewNode(contact.ID)
-	dht.tokenCache = newTokenCache(dht.node, tokenSecretRotationInterval)
-
-	return dht.node.Connect(conn)
-}
-
-// Start starts the dht
-func (dht *DHT) Start() error {
-	listener, err := net.ListenPacket(Network, dht.conf.Address)
-	if err != nil {
-		return errors.Err(err)
-	}
-	conn := listener.(*net.UDPConn)
-
-	err = dht.connect(conn)
-	if err != nil {
-		return err
-	}
-
-	dht.join()
-	log.Infof("[%s] DHT ready on %s (%d nodes found during join)",
-		dht.node.id.HexShort(), dht.contact.Addr().String(), dht.node.rt.Count())
-
-	dht.grp.Add(1)
-	go func() {
-		dht.runAnnouncer()
-		dht.grp.Done()
-	}()
-
-	if dht.conf.RPCPort > 0 {
-		dht.grp.Add(1)
-		go func() {
-			dht.runRPCServer(dht.conf.RPCPort)
-			dht.grp.Done()
-		}()
-	}
-
-	return nil
-}
-
-// join makes current node join the dht network.
-func (dht *DHT) join() {
-	defer close(dht.joined) // if anyone's waiting for join to finish, they'll know its done
-
-	log.Infof("[%s] joining DHT network", dht.node.id.HexShort())
-
-	// ping nodes, which gets their real node IDs and adds them to the routing table
-	atLeastOneNodeResponded := false
-	for _, addr := range dht.conf.SeedNodes {
-		err := dht.Ping(addr)
-		if err != nil {
-			log.Error(errors.Prefix(fmt.Sprintf("[%s] join", dht.node.id.HexShort()), err))
-		} else {
-			atLeastOneNodeResponded = true
-		}
-	}
-
-	if !atLeastOneNodeResponded {
-		log.Errorf("[%s] join: no nodes responded to initial ping", dht.node.id.HexShort())
-		return
-	}
-
-	// now call iterativeFind on yourself
-	_, _, err := FindContacts(dht.node, dht.node.id, false, dht.grp.Child())
-	if err != nil {
-		log.Errorf("[%s] join: %s", dht.node.id.HexShort(), err.Error())
-	}
-
-	// TODO: after joining, refresh all buckets further away than our closest neighbor
-	// http://xlattice.sourceforge.net/components/protocol/kademlia/specs.html#join
-}
-
-// WaitUntilJoined blocks until the node joins the network.
-func (dht *DHT) WaitUntilJoined() {
-	if dht.joined == nil {
-		panic("dht not initialized")
-	}
-	<-dht.joined
-}
-
-// Shutdown shuts down the dht
-func (dht *DHT) Shutdown() {
-	log.Debugf("[%s] DHT shutting down", dht.node.id.HexShort())
-	dht.grp.StopAndWait()
-	dht.node.Shutdown()
-	log.Debugf("[%s] DHT stopped", dht.node.id.HexShort())
-}
-
-// Ping pings a given address, creates a temporary contact for sending a message, and returns an error if communication
-// fails.
-func (dht *DHT) Ping(addr string) error {
-	raddr, err := net.ResolveUDPAddr(Network, addr)
-	if err != nil {
-		return err
-	}
-
-	tmpNode := Contact{ID: bits.Rand(), IP: raddr.IP, Port: raddr.Port}
-	res := dht.node.Send(tmpNode, Request{Method: pingMethod}, SendOptions{skipIDCheck: true})
-	if res == nil {
-		return errors.Err("no response from node %s", addr)
-	}
-
-	return nil
-}
-
-// Get returns the list of nodes that have the blob for the given hash
-func (dht *DHT) Get(hash bits.Bitmap) ([]Contact, error) {
-	contacts, found, err := FindContacts(dht.node, hash, true, dht.grp.Child())
-	if err != nil {
-		return nil, err
-	}
-
-	if found {
-		return contacts, nil
-	}
-	return nil, nil
-}
-
-// PrintState prints the current state of the DHT including address, nr outstanding transactions, stored hashes as well
-// as current bucket information.
-func (dht *DHT) PrintState() {
-	log.Printf("DHT node %s at %s", dht.contact.String(), time.Now().Format(time.RFC822Z))
-	log.Printf("Outstanding transactions: %d", dht.node.CountActiveTransactions())
-	log.Printf("Stored hashes: %d", dht.node.store.CountStoredHashes())
-	log.Printf("Buckets:")
-	for _, line := range strings.Split(dht.node.rt.BucketInfo(), "\n") {
-		log.Println(line)
-	}
-}
-
-func (dht DHT) ID() bits.Bitmap {
-	return dht.contact.ID
-}
-
-func getContact(nodeID, addr string) (Contact, error) {
-	var c Contact
-	if nodeID == "" {
-		c.ID = bits.Rand()
-	} else {
-		c.ID = bits.FromHexP(nodeID)
-	}
-
-	ip, port, err := net.SplitHostPort(addr)
-	if err != nil {
-		return c, errors.Err(err)
-	} else if ip == "" {
-		return c, errors.Err("address does not contain an IP")
-	} else if port == "" {
-		return c, errors.Err("address does not contain a port")
-	}
-
-	c.IP = net.ParseIP(ip)
-	if c.IP == nil {
-		return c, errors.Err("invalid ip")
-	}
-
-	c.Port, err = cast.ToIntE(port)
-	if err != nil {
-		return c, errors.Err(err)
-	}
-
-	return c, nil
-}

dht/dht_announce.go

@@ -1,214 +0,0 @@
-package dht
-
-import (
-	"container/ring"
-	"context"
-	"math"
-	"sync"
-	"time"
-
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/reflector.go/dht/bits"
-
-	"golang.org/x/time/rate"
-)
-
-type queueEdit struct {
-	hash bits.Bitmap
-	add  bool
-}
-
-const (
-	announceStarted = "started"
-	announceFinishd = "finished"
-)
-
-type announceNotification struct {
-	hash   bits.Bitmap
-	action string
-	err    error
-}
-
-// Add adds the hash to the list of hashes this node is announcing
-func (dht *DHT) Add(hash bits.Bitmap) {
-	dht.announceAddRemove <- queueEdit{hash: hash, add: true}
-}
-
-// Remove removes the hash from the list of hashes this node is announcing
-func (dht *DHT) Remove(hash bits.Bitmap) {
-	dht.announceAddRemove <- queueEdit{hash: hash, add: false}
-}
-
-func (dht *DHT) runAnnouncer() {
-	type hashAndTime struct {
-		hash         bits.Bitmap
-		lastAnnounce time.Time
-	}
-
-	var queue *ring.Ring
-	hashes := make(map[bits.Bitmap]*ring.Ring)
-
-	var announceNextHash <-chan time.Time
-	timer := time.NewTimer(math.MaxInt64)
-	timer.Stop()
-
-	limitCh := make(chan time.Time)
-	dht.grp.Add(1)
-	go func() {
-		defer dht.grp.Done()
-		limiter := rate.NewLimiter(rate.Limit(dht.conf.AnnounceRate), dht.conf.AnnounceRate)
-		for {
-			err := limiter.Wait(context.Background()) // TODO: should use grp.ctx somehow? so when grp is closed, wait returns
-			if err != nil {
-				log.Error(errors.Prefix("rate limiter", err))
-				continue
-			}
-			select {
-			case limitCh <- time.Now():
-			case <-dht.grp.Ch():
-				return
-			}
-		}
-	}()
-
-	maintenance := time.NewTicker(1 * time.Minute)
-
-	// TODO: work to space hash announces out so they aren't bunched up around the reannounce time. track time since last announce. if its been more than the ideal time (reannounce time / numhashes), start announcing hashes early
-
-	for {
-		select {
-		case <-dht.grp.Ch():
-			return
-
-		case <-maintenance.C:
-			maxAnnounce := dht.conf.AnnounceRate * int(dht.conf.ReannounceTime.Seconds())
-			if len(hashes) > maxAnnounce {
-				// TODO: send this to slack
-				log.Warnf("DHT has %d hashes, but can only announce %d hashes in the %s reannounce window. Raise the announce rate or spawn more nodes.",
-					len(hashes), maxAnnounce, dht.conf.ReannounceTime.String())
-			}
-
-		case change := <-dht.announceAddRemove:
-			if change.add {
-				if _, exists := hashes[change.hash]; exists {
-					continue
-				}
-
-				r := ring.New(1)
-				r.Value = hashAndTime{hash: change.hash}
-				if queue != nil {
-					queue.Prev().Link(r)
-				}
-				queue = r
-				hashes[change.hash] = r
-				announceNextHash = limitCh // announce next hash ASAP
-			} else {
-				r, exists := hashes[change.hash]
-				if !exists {
-					continue
-				}
-
-				delete(hashes, change.hash)
-
-				if len(hashes) == 0 {
-					queue = ring.New(0)
-					announceNextHash = nil // no hashes to announce, wait indefinitely
-				} else {
-					if r == queue {
-						queue = queue.Next() // don't lose our pointer
-					}
-					r.Prev().Link(r.Next())
-				}
-			}
-
-		case <-announceNextHash:
-			dht.grp.Add(1)
-			ht := queue.Value.(hashAndTime)
-
-			if !ht.lastAnnounce.IsZero() {
-				nextAnnounce := ht.lastAnnounce.Add(dht.conf.ReannounceTime)
-				if nextAnnounce.After(time.Now()) {
-					timer.Reset(time.Until(nextAnnounce))
-					announceNextHash = timer.C // wait until next hash should be announced
-					continue
-				}
-			}
-
-			if dht.conf.AnnounceNotificationCh != nil {
-				dht.conf.AnnounceNotificationCh <- announceNotification{
-					hash:   ht.hash,
-					action: announceStarted,
-				}
-			}
-
-			go func(hash bits.Bitmap) {
-				defer dht.grp.Done()
-				err := dht.announce(hash)
-				if err != nil {
-					log.Error(errors.Prefix("announce", err))
-				}
-
-				if dht.conf.AnnounceNotificationCh != nil {
-					dht.conf.AnnounceNotificationCh <- announceNotification{
-						hash:   ht.hash,
-						action: announceFinishd,
-						err:    err,
-					}
-				}
-			}(ht.hash)
-
-			queue.Value = hashAndTime{hash: ht.hash, lastAnnounce: time.Now()}
-			queue = queue.Next()
-			announceNextHash = limitCh // announce next hash ASAP
-		}
-	}
-}
-
-// Announce announces to the DHT that this node has the blob for the given hash
-func (dht *DHT) announce(hash bits.Bitmap) error {
-	contacts, _, err := FindContacts(dht.node, hash, false, dht.grp.Child())
-	if err != nil {
-		return err
-	}
-
-	// self-store if we found less than K contacts, or we're closer than the farthest contact
-	if len(contacts) < bucketSize {
-		contacts = append(contacts, dht.contact)
-	} else if hash.Closer(dht.node.id, contacts[bucketSize-1].ID) {
-		contacts[bucketSize-1] = dht.contact
-	}
-
-	wg := &sync.WaitGroup{}
-	for _, c := range contacts {
-		wg.Add(1)
-		go func(c Contact) {
-			dht.store(hash, c)
-			wg.Done()
-		}(c)
-	}
-
-	wg.Wait()
-
-	return nil
-}
-
-func (dht *DHT) store(hash bits.Bitmap, c Contact) {
-	if dht.contact.ID == c.ID {
-		// self-store
-		c.PeerPort = dht.conf.PeerProtocolPort
-		dht.node.Store(hash, c)
-		return
-	}
-
-	dht.node.SendAsync(c, Request{
-		Method: storeMethod,
-		StoreArgs: &storeArgs{
-			BlobHash: hash,
-			Value: storeArgsValue{
-				Token:  dht.tokenCache.Get(c, hash, dht.grp.Ch()),
-				LbryID: dht.contact.ID,
-				Port:   dht.conf.PeerProtocolPort,
-			},
-		},
-	})
-}

dht/dht_test.go

@@ -1,181 +0,0 @@
-package dht
-
-import (
-	"net"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-)
-
-func TestNodeFinder_FindNodes(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping slow nodeFinder test")
-	}
-
-	bs, dhts := TestingCreateNetwork(t, 3, true, false)
-	defer func() {
-		for i := range dhts {
-			dhts[i].Shutdown()
-		}
-		bs.Shutdown()
-	}()
-
-	contacts, found, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if found {
-		t.Fatal("something was found, but it should not have been")
-	}
-
-	if len(contacts) != 3 {
-		t.Errorf("expected 3 node, found %d", len(contacts))
-	}
-
-	foundBootstrap := false
-	foundOne := false
-	foundTwo := false
-
-	for _, n := range contacts {
-		if n.ID.Equals(bs.id) {
-			foundBootstrap = true
-		}
-		if n.ID.Equals(dhts[0].node.id) {
-			foundOne = true
-		}
-		if n.ID.Equals(dhts[1].node.id) {
-			foundTwo = true
-		}
-	}
-
-	if !foundBootstrap {
-		t.Errorf("did not find bootstrap node %s", bs.id.Hex())
-	}
-	if !foundOne {
-		t.Errorf("did not find first node %s", dhts[0].node.id.Hex())
-	}
-	if !foundTwo {
-		t.Errorf("did not find second node %s", dhts[1].node.id.Hex())
-	}
-}
-
-func TestNodeFinder_FindNodes_NoBootstrap(t *testing.T) {
-	_, dhts := TestingCreateNetwork(t, 3, false, false)
-	defer func() {
-		for i := range dhts {
-			dhts[i].Shutdown()
-		}
-	}()
-
-	_, _, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
-	if err == nil {
-		t.Fatal("contact finder should have errored saying that there are no contacts in the routing table")
-	}
-}
-
-func TestNodeFinder_FindValue(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping slow nodeFinder test")
-	}
-
-	bs, dhts := TestingCreateNetwork(t, 3, true, false)
-	defer func() {
-		for i := range dhts {
-			dhts[i].Shutdown()
-		}
-		bs.Shutdown()
-	}()
-
-	blobHashToFind := bits.Rand()
-	nodeToFind := Contact{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4), Port: 5678}
-	dhts[0].node.store.Upsert(blobHashToFind, nodeToFind)
-
-	contacts, found, err := FindContacts(dhts[2].node, blobHashToFind, true, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !found {
-		t.Fatal("node was not found")
-	}
-
-	if len(contacts) != 1 {
-		t.Fatalf("expected one node, found %d", len(contacts))
-	}
-
-	if !contacts[0].ID.Equals(nodeToFind.ID) {
-		t.Fatalf("found node id %s, expected %s", contacts[0].ID.Hex(), nodeToFind.ID.Hex())
-	}
-}
-
-func TestDHT_LargeDHT(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping large DHT test")
-	}
-
-	nodes := 100
-	bs, dhts := TestingCreateNetwork(t, nodes, true, true)
-	defer func() {
-		for _, d := range dhts {
-			go d.Shutdown()
-		}
-		bs.Shutdown()
-		time.Sleep(1 * time.Second)
-	}()
-
-	wg := &sync.WaitGroup{}
-	ids := make([]bits.Bitmap, nodes)
-	for i := range ids {
-		ids[i] = bits.Rand()
-		wg.Add(1)
-		go func(index int) {
-			defer wg.Done()
-			err := dhts[index].announce(ids[index])
-			if err != nil {
-				t.Error("error announcing random bitmap - ", err)
-			}
-		}(i)
-	}
-	wg.Wait()
-
-	// check that each node is in at learst 1 other routing table
-	rtCounts := make(map[bits.Bitmap]int)
-	for _, d := range dhts {
-		for _, d2 := range dhts {
-			if d.node.id.Equals(d2.node.id) {
-				continue
-			}
-			c := d2.node.rt.GetClosest(d.node.id, 1)
-			if len(c) > 1 {
-				t.Error("rt returned more than one node when only one requested")
-			} else if len(c) == 1 && c[0].ID.Equals(d.node.id) {
-				rtCounts[d.node.id]++
-			}
-		}
-	}
-
-	for k, v := range rtCounts {
-		if v == 0 {
-			t.Errorf("%s was not in any routing tables", k.HexShort())
-		}
-	}
-
-	// check that each ID is stored by at least 3 nodes
-	storeCounts := make(map[bits.Bitmap]int)
-	for _, d := range dhts {
-		for _, id := range ids {
-			if len(d.node.store.Get(id)) > 0 {
-				storeCounts[id]++
-			}
-		}
-	}
-
-	for k, v := range storeCounts {
-		if v == 0 {
-			t.Errorf("%s was not stored by any nodes", k.HexShort())
-		}
-	}
-}

dht/message.go

@@ -1,463 +0,0 @@
-package dht
-
-import (
-	"crypto/rand"
-	"encoding/hex"
-	"reflect"
-	"strconv"
-	"strings"
-
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/reflector.go/dht/bits"
-
-	"github.com/lyoshenka/bencode"
-	"github.com/spf13/cast"
-)
-
-const (
-	pingMethod      = "ping"
-	storeMethod     = "store"
-	findNodeMethod  = "findNode"
-	findValueMethod = "findValue"
-)
-
-const (
-	pingSuccessResponse  = "pong"
-	storeSuccessResponse = "OK"
-)
-
-const (
-	requestType  = 0
-	responseType = 1
-	errorType    = 2
-)
-
-const (
-	// these are strings because bencode requires bytestring keys
-	headerTypeField      = "0"
-	headerMessageIDField = "1" // message id is 20 bytes long
-	headerNodeIDField    = "2" // node id is 48 bytes long
-	headerPayloadField   = "3"
-	headerArgsField      = "4"
-	contactsField        = "contacts"
-	tokenField           = "token"
-	protocolVersionField = "protocolVersion"
-)
-
-// Message is a DHT message
-type Message interface {
-	bencode.Marshaler
-}
-
-type messageID [messageIDLength]byte
-
-// HexShort returns the first 8 hex characters of the hex encoded message id.
-func (m messageID) HexShort() string {
-	return hex.EncodeToString(m[:])[:8]
-}
-
-// UnmarshalBencode takes a byte slice and unmarshals the message id.
-func (m *messageID) UnmarshalBencode(encoded []byte) error {
-	var str string
-	err := bencode.DecodeBytes(encoded, &str)
-	if err != nil {
-		return err
-	}
-	copy(m[:], str)
-	return nil
-}
-
-// MarshallBencode returns the encoded byte slice of the message id.
-func (m messageID) MarshalBencode() ([]byte, error) {
-	str := string(m[:])
-	return bencode.EncodeBytes(str)
-}
-
-func newMessageID() messageID {
-	var m messageID
-	_, err := rand.Read(m[:])
-	if err != nil {
-		panic(err)
-	}
-	return m
-}
-
-// Request represents a DHT request message
-type Request struct {
-	ID              messageID
-	NodeID          bits.Bitmap
-	Method          string
-	Arg             *bits.Bitmap
-	StoreArgs       *storeArgs
-	ProtocolVersion int
-}
-
-// MarshalBencode returns the serialized byte slice representation of the request
-func (r Request) MarshalBencode() ([]byte, error) {
-	var args interface{}
-	if r.StoreArgs != nil {
-		args = r.StoreArgs
-	} else if r.Arg != nil {
-		args = []bits.Bitmap{*r.Arg}
-	} else {
-		args = []string{} // request must always have keys 0-4, so we use an empty list for PING
-	}
-	return bencode.EncodeBytes(map[string]interface{}{
-		headerTypeField:      requestType,
-		headerMessageIDField: r.ID,
-		headerNodeIDField:    r.NodeID,
-		headerPayloadField:   r.Method,
-		headerArgsField:      args,
-	})
-}
-
-// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the request.
-func (r *Request) UnmarshalBencode(b []byte) error {
-	var raw struct {
-		ID     messageID          `bencode:"1"`
-		NodeID bits.Bitmap        `bencode:"2"`
-		Method string             `bencode:"3"`
-		Args   bencode.RawMessage `bencode:"4"`
-	}
-	err := bencode.DecodeBytes(b, &raw)
-	if err != nil {
-		return errors.Prefix("request unmarshal", err)
-	}
-
-	r.ID = raw.ID
-	r.NodeID = raw.NodeID
-	r.Method = raw.Method
-
-	if r.Method == storeMethod {
-		r.StoreArgs = &storeArgs{} // bencode wont find the unmarshaler on a null pointer. need to fix it.
-		err = bencode.DecodeBytes(raw.Args, &r.StoreArgs)
-		if err != nil {
-			return errors.Prefix("request unmarshal", err)
-		}
-	} else if len(raw.Args) > 2 { // 2 because an empty list is `le`
-		r.Arg, r.ProtocolVersion, err = processArgsAndProtoVersion(raw.Args)
-		if err != nil {
-			return errors.Prefix("request unmarshal", err)
-		}
-	}
-
-	return nil
-}
-
-func processArgsAndProtoVersion(raw bencode.RawMessage) (arg *bits.Bitmap, version int, err error) {
-	var args []bencode.RawMessage
-	err = bencode.DecodeBytes(raw, &args)
-	if err != nil {
-		return nil, 0, err
-	}
-
-	if len(args) == 0 {
-		return nil, 0, nil
-	}
-
-	var extras map[string]int
-	err = bencode.DecodeBytes(args[len(args)-1], &extras)
-	if err == nil {
-		if v, exists := extras[protocolVersionField]; exists {
-			version = v
-			args = args[:len(args)-1]
-		}
-	}
-
-	if len(args) > 0 {
-		var b bits.Bitmap
-		err = bencode.DecodeBytes(args[0], &b)
-		if err != nil {
-			return nil, 0, err
-		}
-		arg = &b
-	}
-
-	return arg, version, nil
-}
-
-func (r Request) argsDebug() string {
-	if r.StoreArgs != nil {
-		return r.StoreArgs.BlobHash.HexShort() + ", " + r.StoreArgs.Value.LbryID.HexShort() + ":" + strconv.Itoa(r.StoreArgs.Value.Port)
-	} else if r.Arg != nil {
-		return r.Arg.HexShort()
-	}
-	return ""
-}
-
-type storeArgsValue struct {
-	Token  string      `bencode:"token"`
-	LbryID bits.Bitmap `bencode:"lbryid"`
-	Port   int         `bencode:"port"`
-}
-
-type storeArgs struct {
-	BlobHash  bits.Bitmap
-	Value     storeArgsValue
-	NodeID    bits.Bitmap // original publisher id? I think this is getting fixed in the new dht stuff
-	SelfStore bool        // this is an int on the wire
-}
-
-// MarshalBencode returns the serialized byte slice representation of the storage arguments.
-func (s storeArgs) MarshalBencode() ([]byte, error) {
-	encodedValue, err := bencode.EncodeString(s.Value)
-	if err != nil {
-		return nil, err
-	}
-
-	selfStoreStr := 0
-	if s.SelfStore {
-		selfStoreStr = 1
-	}
-
-	return bencode.EncodeBytes([]interface{}{
-		s.BlobHash,
-		bencode.RawMessage(encodedValue),
-		s.NodeID,
-		selfStoreStr,
-	})
-}
-
-// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
-func (s *storeArgs) UnmarshalBencode(b []byte) error {
-	var argsInt []bencode.RawMessage
-	err := bencode.DecodeBytes(b, &argsInt)
-	if err != nil {
-		return errors.Prefix("storeArgs unmarshal", err)
-	}
-
-	if len(argsInt) != 4 {
-		return errors.Err("unexpected number of fields for store args. got " + cast.ToString(len(argsInt)))
-	}
-
-	err = bencode.DecodeBytes(argsInt[0], &s.BlobHash)
-	if err != nil {
-		return errors.Prefix("storeArgs unmarshal", err)
-	}
-
-	err = bencode.DecodeBytes(argsInt[1], &s.Value)
-	if err != nil {
-		return errors.Prefix("storeArgs unmarshal", err)
-	}
-
-	err = bencode.DecodeBytes(argsInt[2], &s.NodeID)
-	if err != nil {
-		return errors.Prefix("storeArgs unmarshal", err)
-	}
-
-	var selfStore int
-	err = bencode.DecodeBytes(argsInt[3], &selfStore)
-	if err != nil {
-		return errors.Prefix("storeArgs unmarshal", err)
-	}
-	if selfStore == 0 {
-		s.SelfStore = false
-	} else if selfStore == 1 {
-		s.SelfStore = true
-	} else {
-		return errors.Err("selfstore must be 1 or 0")
-	}
-
-	return nil
-}
-
-// Response represents a DHT response message
-type Response struct {
-	ID              messageID
-	NodeID          bits.Bitmap
-	Data            string
-	Contacts        []Contact
-	FindValueKey    string
-	Token           string
-	ProtocolVersion int
-}
-
-func (r Response) argsDebug() string {
-	if r.Data != "" {
-		return r.Data
-	}
-
-	str := "contacts "
-	if r.FindValueKey != "" {
-		str = "value for " + hex.EncodeToString([]byte(r.FindValueKey))[:8] + " "
-	}
-
-	str += "|"
-	for _, c := range r.Contacts {
-		str += c.String() + ","
-	}
-	str = strings.TrimRight(str, ",") + "|"
-
-	if r.Token != "" {
-		str += " token: " + hex.EncodeToString([]byte(r.Token))[:8]
-	}
-
-	return str
-}
-
-// MarshalBencode returns the serialized byte slice representation of the response.
-func (r Response) MarshalBencode() ([]byte, error) {
-	data := map[string]interface{}{
-		headerTypeField:      responseType,
-		headerMessageIDField: r.ID,
-		headerNodeIDField:    r.NodeID,
-	}
-
-	if r.Data != "" {
-		// ping or store
-		data[headerPayloadField] = r.Data
-	} else if r.FindValueKey != "" {
-		// findValue success
-		if r.Token == "" {
-			return nil, errors.Err("response to findValue must have a token")
-		}
-
-		var contacts [][]byte
-		for _, c := range r.Contacts {
-			compact, err := c.MarshalCompact()
-			if err != nil {
-				return nil, err
-			}
-			contacts = append(contacts, compact)
-		}
-		data[headerPayloadField] = map[string]interface{}{
-			r.FindValueKey: contacts,
-			tokenField:     r.Token,
-		}
-	} else if r.Token != "" {
-		// findValue failure falling back to findNode
-		data[headerPayloadField] = map[string]interface{}{
-			contactsField: r.Contacts,
-			tokenField:    r.Token,
-		}
-	} else {
-		// straight up findNode
-		data[headerPayloadField] = r.Contacts
-	}
-
-	return bencode.EncodeBytes(data)
-}
-
-// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
-func (r *Response) UnmarshalBencode(b []byte) error {
-	var raw struct {
-		ID     messageID          `bencode:"1"`
-		NodeID bits.Bitmap        `bencode:"2"`
-		Data   bencode.RawMessage `bencode:"3"`
-	}
-	err := bencode.DecodeBytes(b, &raw)
-	if err != nil {
-		return err
-	}
-
-	r.ID = raw.ID
-	r.NodeID = raw.NodeID
-
-	// maybe data is a string (response to ping or store)?
-	err = bencode.DecodeBytes(raw.Data, &r.Data)
-	if err == nil {
-		return nil
-	}
-
-	// maybe data is a list of contacts (response to findNode)?
-	err = bencode.DecodeBytes(raw.Data, &r.Contacts)
-	if err == nil {
-		return nil
-	}
-
-	// it must be a response to findValue
-	var rawData map[string]bencode.RawMessage
-	err = bencode.DecodeBytes(raw.Data, &rawData)
-	if err != nil {
-		return err
-	}
-
-	if token, ok := rawData[tokenField]; ok {
-		err = bencode.DecodeBytes(token, &r.Token)
-		if err != nil {
-			return err
-		}
-		delete(rawData, tokenField) // so it doesnt mess up findValue key finding below
-	}
-
-	if protocolVersion, ok := rawData[protocolVersionField]; ok {
-		err = bencode.DecodeBytes(protocolVersion, &r.ProtocolVersion)
-		if err != nil {
-			return err
-		}
-		delete(rawData, protocolVersionField) // so it doesnt mess up findValue key finding below
-	}
-
-	if contacts, ok := rawData[contactsField]; ok {
-		err = bencode.DecodeBytes(contacts, &r.Contacts)
-		if err != nil {
-			return err
-		}
-	} else {
-		for k, v := range rawData {
-			r.FindValueKey = k
-			var compactContacts [][]byte
-			err = bencode.DecodeBytes(v, &compactContacts)
-			if err != nil {
-				return err
-			}
-			for _, compact := range compactContacts {
-				var c Contact
-				err = c.UnmarshalCompact(compact)
-				if err != nil {
-					return err
-				}
-				r.Contacts = append(r.Contacts, c)
-			}
-			break
-		}
-	}
-
-	return nil
-}
-
-// Error represents a DHT error response
-type Error struct {
-	ID            messageID
-	NodeID        bits.Bitmap
-	ExceptionType string
-	Response      []string
-}
-
-// MarshalBencode returns the serialized byte slice representation of an error message.
-func (e Error) MarshalBencode() ([]byte, error) {
-	return bencode.EncodeBytes(map[string]interface{}{
-		headerTypeField:      errorType,
-		headerMessageIDField: e.ID,
-		headerNodeIDField:    e.NodeID,
-		headerPayloadField:   e.ExceptionType,
-		headerArgsField:      e.Response,
-	})
-}
-
-// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the error message.
-func (e *Error) UnmarshalBencode(b []byte) error {
-	var raw struct {
-		ID            messageID   `bencode:"1"`
-		NodeID        bits.Bitmap `bencode:"2"`
-		ExceptionType string      `bencode:"3"`
-		Args          interface{} `bencode:"4"`
-	}
-	err := bencode.DecodeBytes(b, &raw)
-	if err != nil {
-		return err
-	}
-
-	e.ID = raw.ID
-	e.NodeID = raw.NodeID
-	e.ExceptionType = raw.ExceptionType
-
-	if reflect.TypeOf(raw.Args).Kind() == reflect.Slice {
-		v := reflect.ValueOf(raw.Args)
-		for i := 0; i < v.Len(); i++ {
-			e.Response = append(e.Response, cast.ToString(v.Index(i).Interface()))
-		}
-	}
-
-	return nil
-}

dht/node.go

@@ -1,474 +0,0 @@
-package dht
-
-import (
-	"encoding/hex"
-	"net"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/lbryio/errors.go"
-	"github.com/lbryio/lbry.go/stop"
-	"github.com/lbryio/lbry.go/util"
-	"github.com/lbryio/reflector.go/dht/bits"
-
-	"github.com/davecgh/go-spew/spew"
-	"github.com/lyoshenka/bencode"
-)
-
-// packet represents the information receive from udp.
-type packet struct {
-	data  []byte
-	raddr *net.UDPAddr
-}
-
-// UDPConn allows using a mocked connection to test sending/receiving data
-// TODO: stop mocking this and use the real thing
-type UDPConn interface {
-	ReadFromUDP([]byte) (int, *net.UDPAddr, error)
-	WriteToUDP([]byte, *net.UDPAddr) (int, error)
-	SetReadDeadline(time.Time) error
-	SetWriteDeadline(time.Time) error
-	Close() error
-}
-
-// RequestHandlerFunc is exported handler for requests.
-type RequestHandlerFunc func(addr *net.UDPAddr, request Request)
-
-// Node is a type representation of a node on the network.
-type Node struct {
-	// the node's id
-	id bits.Bitmap
-	// UDP connection for sending and receiving data
-	conn UDPConn
-	// true if we've closed the connection on purpose
-	connClosed bool
-	// token manager
-	tokens *tokenManager
-
-	// map of outstanding transactions + mutex
-	txLock       *sync.RWMutex
-	transactions map[messageID]*transaction
-
-	// routing table
-	rt *routingTable
-	// data store
-	store *contactStore
-
-	// overrides for request handlers
-	requestHandler RequestHandlerFunc
-
-	// stop the node neatly and clean up after itself
-	grp *stop.Group
-}
-
-// NewNode returns an initialized Node's pointer.
-func NewNode(id bits.Bitmap) *Node {
-	return &Node{
-		id:    id,
-		rt:    newRoutingTable(id),
-		store: newStore(),
-
-		txLock:       &sync.RWMutex{},
-		transactions: make(map[messageID]*transaction),
-
-		grp:    stop.New(),
-		tokens: &tokenManager{},
-	}
-}
-
-// Connect connects to the given connection and starts any background threads necessary
-func (n *Node) Connect(conn UDPConn) error {
-	n.conn = conn
-
-	n.tokens.Start(tokenSecretRotationInterval)
-
-	go func() {
-		// stop tokens and close the connection when we're shutting down
-		<-n.grp.Ch()
-		n.tokens.Stop()
-		n.connClosed = true
-		err := n.conn.Close()
-		if err != nil {
-			log.Error("error closing node connection on shutdown - ", err)
-		}
-	}()
-
-	packets := make(chan packet)
-
-	n.grp.Add(1)
-	go func() {
-		defer n.grp.Done()
-
-		buf := make([]byte, udpMaxMessageLength)
-
-		for {
-			bytesRead, raddr, err := n.conn.ReadFromUDP(buf)
-			if err != nil {
-				if n.connClosed {
-					return
-				}
-				log.Errorf("udp read error: %v", err)
-				continue
-			} else if raddr == nil {
-				log.Errorf("udp read with no raddr")
-				continue
-			}
-
-			data := make([]byte, bytesRead)
-			copy(data, buf[:bytesRead]) // slices use the same underlying array, so we need a new one for each packet
-
-			select { // needs select here because packet consumer can quit and the packets channel gets filled up and blocks
-			case packets <- packet{data: data, raddr: raddr}:
-			case <-n.grp.Ch():
-				return
-			}
-		}
-	}()
-
-	n.grp.Add(1)
-	go func() {
-		defer n.grp.Done()
-
-		var pkt packet
-
-		for {
-			select {
-			case pkt = <-packets:
-				n.handlePacket(pkt)
-			case <-n.grp.Ch():
-				return
-			}
-		}
-	}()
-
-	// TODO: turn this back on when you're sure it works right
-	n.grp.Add(1)
-	go func() {
-		defer n.grp.Done()
-		n.startRoutingTableGrooming()
-	}()
-
-	return nil
-}
-
-// Shutdown shuts down the node
-func (n *Node) Shutdown() {
-	log.Debugf("[%s] node shutting down", n.id.HexShort())
-	n.grp.StopAndWait()
-	log.Debugf("[%s] node stopped", n.id.HexShort())
-}
-
-// handlePacket handles packets received from udp.
-func (n *Node) handlePacket(pkt packet) {
-	//log.Debugf("[%s] Received message from %s (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), len(pkt.data), hex.EncodeToString(pkt.data))
-
-	if !util.InSlice(string(pkt.data[0:5]), []string{"d1:0i", "di0ei"}) {
-		log.Errorf("[%s] data is not a well-formatted dict: (%d bytes) %s", n.id.HexShort(), len(pkt.data), hex.EncodeToString(pkt.data))
-		return
-	}
-
-	// the following is a bit of a hack, but it lets us avoid decoding every message twice
-	// it depends on the data being a dict with 0 as the first key (so it starts with "d1:0i") and the message type as the first value
-	// TODO: test this more thoroughly
-
-	switch pkt.data[5] {
-	case '0' + requestType:
-		request := Request{}
-		err := bencode.DecodeBytes(pkt.data, &request)
-		if err != nil {
-			log.Errorf("[%s] error decoding request from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
-			return
-		}
-		log.Debugf("[%s] query %s: received request from %s: %s(%s)", n.id.HexShort(), request.ID.HexShort(), request.NodeID.HexShort(), request.Method, request.argsDebug())
-		n.handleRequest(pkt.raddr, request)
-
-	case '0' + responseType:
-		response := Response{}
-		err := bencode.DecodeBytes(pkt.data, &response)
-		if err != nil {
-			log.Errorf("[%s] error decoding response from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
-			return
-		}
-		log.Debugf("[%s] query %s: received response from %s: %s", n.id.HexShort(), response.ID.HexShort(), response.NodeID.HexShort(), response.argsDebug())
-		n.handleResponse(pkt.raddr, response)
-
-	case '0' + errorType:
-		e := Error{}
-		err := bencode.DecodeBytes(pkt.data, &e)
-		if err != nil {
-			log.Errorf("[%s] error decoding error from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
-			return
-		}
-		log.Debugf("[%s] query %s: received error from %s: %s", n.id.HexShort(), e.ID.HexShort(), e.NodeID.HexShort(), e.ExceptionType)
-		n.handleError(pkt.raddr, e)
-
-	default:
-		log.Errorf("[%s] invalid message type: %s", n.id.HexShort(), string(pkt.data[5]))
-		return
-	}
-}
-
-// handleRequest handles the requests received from udp.
-func (n *Node) handleRequest(addr *net.UDPAddr, request Request) {
-	if request.NodeID.Equals(n.id) {
-		log.Warn("ignoring self-request")
-		return
-	}
-
-	// if a handler is overridden, call it instead
-	if n.requestHandler != nil {
-		n.requestHandler(addr, request)
-		return
-	}
-
-	switch request.Method {
-	default:
-		//n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-request-method"})
-		log.Errorln("invalid request method")
-		return
-	case pingMethod:
-		err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: pingSuccessResponse})
-		if err != nil {
-			log.Error("error sending 'pingmethod' response message - ", err)
-		}
-	case storeMethod:
-		// TODO: we should be sending the IP in the request, not just using the sender's IP
-		// TODO: should we be using StoreArgs.NodeID or StoreArgs.Value.LbryID ???
-		if n.tokens.Verify(request.StoreArgs.Value.Token, request.NodeID, addr) {
-			n.Store(request.StoreArgs.BlobHash, Contact{ID: request.StoreArgs.NodeID, IP: addr.IP, Port: addr.Port, PeerPort: request.StoreArgs.Value.Port})
-
-			err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: storeSuccessResponse})
-			if err != nil {
-				log.Error("error sending 'storemethod' response message - ", err)
-			}
-		} else {
-			err := n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-token"})
-			if err != nil {
-				log.Error("error sending 'storemethod'response message for invalid-token - ", err)
-			}
-		}
-	case findNodeMethod:
-		if request.Arg == nil {
-			log.Errorln("request is missing arg")
-			return
-		}
-		err := n.sendMessage(addr, Response{
-			ID:       request.ID,
-			NodeID:   n.id,
-			Contacts: n.rt.GetClosest(*request.Arg, bucketSize),
-		})
-		if err != nil {
-			log.Error("error sending 'findnodemethod' response message - ", err)
-		}
-
-	case findValueMethod:
-		if request.Arg == nil {
-			log.Errorln("request is missing arg")
-			return
-		}
-
-		res := Response{
-			ID:     request.ID,
-			NodeID: n.id,
-			Token:  n.tokens.Get(request.NodeID, addr),
-		}
-
-		if contacts := n.store.Get(*request.Arg); len(contacts) > 0 {
-			res.FindValueKey = request.Arg.RawString()
-			res.Contacts = contacts
-		} else {
-			res.Contacts = n.rt.GetClosest(*request.Arg, bucketSize)
-		}
-
-		err := n.sendMessage(addr, res)
-		if err != nil {
-			log.Error("error sending 'findvaluemethod' response message - ", err)
-		}
-	}
-
-	// nodes that send us requests should not be inserted, only refreshed.
-	// the routing table must only contain "good" nodes, which are nodes that reply to our requests
-	// if a node is already good (aka in the table), its fine to refresh it
-	// http://www.bittorrent.org/beps/bep_0005.html#routing-table
-	n.rt.Fresh(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
-}
-
-// handleResponse handles responses received from udp.
-func (n *Node) handleResponse(addr *net.UDPAddr, response Response) {
-	tx := n.txFind(response.ID, Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
-	if tx != nil {
-		select {
-		case tx.res <- response:
-		default:
-			//log.Errorf("[%s] query %s: response received, but tx has no listener or multiple responses to the same tx", n.id.HexShort(), response.ID.HexShort())
-		}
-	}
-
-	n.rt.Update(Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
-}
-
-// handleError handles errors received from udp.
-func (n *Node) handleError(addr *net.UDPAddr, e Error) {
-	spew.Dump(e)
-	n.rt.Fresh(Contact{ID: e.NodeID, IP: addr.IP, Port: addr.Port})
-}
-
-// send sends data to a udp address
-func (n *Node) sendMessage(addr *net.UDPAddr, data Message) error {
-	encoded, err := bencode.EncodeBytes(data)
-	if err != nil {
-		return errors.Err(err)
-	}
-
-	if req, ok := data.(Request); ok {
-		log.Debugf("[%s] query %s: sending request to %s (%d bytes) %s(%s)",
-			n.id.HexShort(), req.ID.HexShort(), addr.String(), len(encoded), req.Method, req.argsDebug())
-	} else if res, ok := data.(Response); ok {
-		log.Debugf("[%s] query %s: sending response to %s (%d bytes) %s",
-			n.id.HexShort(), res.ID.HexShort(), addr.String(), len(encoded), res.argsDebug())
-	} else {
-		log.Debugf("[%s] (%d bytes) %s", n.id.HexShort(), len(encoded), spew.Sdump(data))
-	}
-
-	err = n.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
-	if err != nil {
-		if n.connClosed {
-			return nil
-		}
-		log.Error("error setting write deadline - ", err)
-	}
-
-	_, err = n.conn.WriteToUDP(encoded, addr)
-	return errors.Err(err)
-}
-
-// transaction represents a single query to the dht. it stores the queried contact, the request, and the response channel
-type transaction struct {
-	contact     Contact
-	req         Request
-	res         chan Response
-	skipIDCheck bool
-}
-
-// insert adds a transaction to the manager.
-func (n *Node) txInsert(tx *transaction) {
-	n.txLock.Lock()
-	defer n.txLock.Unlock()
-	n.transactions[tx.req.ID] = tx
-}
-
-// delete removes a transaction from the manager.
-func (n *Node) txDelete(id messageID) {
-	n.txLock.Lock()
-	defer n.txLock.Unlock()
-	delete(n.transactions, id)
-}
-
-// Find finds a transaction for the given id and contact
-func (n *Node) txFind(id messageID, c Contact) *transaction {
-	n.txLock.RLock()
-	defer n.txLock.RUnlock()
-
-	t, ok := n.transactions[id]
-	if !ok || !t.contact.Equals(c, !t.skipIDCheck) {
-		return nil
-	}
-
-	return t
-}
-
-// SendOptions controls the behavior of send calls
-type SendOptions struct {
-	skipIDCheck bool
-}
-
-// SendAsync sends a transaction and returns a channel that will eventually contain the transaction response
-// The response channel is closed when the transaction is completed or times out.
-func (n *Node) SendAsync(contact Contact, req Request, options ...SendOptions) <-chan *Response {
-	ch := make(chan *Response, 1)
-
-	if contact.ID.Equals(n.id) {
-		log.Error("sending query to self")
-		close(ch)
-		return ch
-	}
-
-	go func() {
-		defer close(ch)
-
-		req.ID = newMessageID()
-		req.NodeID = n.id
-		tx := &transaction{
-			contact: contact,
-			req:     req,
-			res:     make(chan Response),
-		}
-
-		if len(options) > 0 && options[0].skipIDCheck {
-			tx.skipIDCheck = true
-		}
-
-		n.txInsert(tx)
-		defer n.txDelete(tx.req.ID)
-
-		for i := 0; i < udpRetry; i++ {
-			err := n.sendMessage(contact.Addr(), tx.req)
-			if err != nil {
-				if !strings.Contains(err.Error(), "use of closed network connection") { // this only happens on localhost. real UDP has no connections
-					log.Error("send error: ", err)
-				}
-				continue
-			}
-
-			select {
-			case res := <-tx.res:
-				ch <- &res
-				return
-			case <-n.grp.Ch():
-				return
-			case <-time.After(udpTimeout):
-			}
-		}
-
-		// notify routing table about a failure to respond
-		n.rt.Fail(tx.contact)
-	}()
-
-	return ch
-}
-
-// Send sends a transaction and blocks until the response is available. It returns a response, or nil
-// if the transaction timed out.
-func (n *Node) Send(contact Contact, req Request, options ...SendOptions) *Response {
-	return <-n.SendAsync(contact, req, options...)
-}
-
-// CountActiveTransactions returns the number of transactions in the manager
-func (n *Node) CountActiveTransactions() int {
-	n.txLock.Lock()
-	defer n.txLock.Unlock()
-	return len(n.transactions)
-}
-
-func (n *Node) startRoutingTableGrooming() {
-	refreshTicker := time.NewTicker(tRefresh / 5) // how often to check for buckets that need to be refreshed
-	for {
-		select {
-		case <-refreshTicker.C:
-			RoutingTableRefresh(n, tRefresh, n.grp.Child())
-		case <-n.grp.Ch():
-			return
-		}
-	}
-}
-
-// Store stores a node contact in the node's contact store.
-func (n *Node) Store(hash bits.Bitmap, c Contact) {
-	n.store.Upsert(hash, c)
-}
-
-//AddKnownNode adds a known-good node to the routing table
-func (n *Node) AddKnownNode(c Contact) {
-	n.rt.Update(c)
-}

dht/node_finder.go

@@ -1,338 +0,0 @@
-package dht
-
-import (
-	"sync"
-	"time"
-
-	"github.com/lbryio/lbry.go/crypto"
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/stop"
-	"github.com/lbryio/reflector.go/dht/bits"
-
-	"github.com/sirupsen/logrus"
-	"github.com/uber-go/atomic"
-)
-
-// TODO: iterativeFindValue may be stopping early. if it gets a response with one peer, it should keep going because other nodes may know about more peers that have that blob
-// TODO: or, it should try a tcp handshake with peers as it finds them, to make sure they are still online and have the blob
-
-var cfLog *logrus.Logger
-
-func init() {
-	cfLog = logrus.StandardLogger()
-}
-
-func NodeFinderUseLogger(l *logrus.Logger) {
-	cfLog = l
-}
-
-type contactFinder struct {
-	findValue bool // true if we're using findValue
-	target    bits.Bitmap
-	node      *Node
-
-	grp *stop.Group
-
-	findValueMutex  *sync.Mutex
-	findValueResult []Contact
-
-	activeContactsMutex *sync.Mutex
-	activeContacts      []Contact
-
-	shortlistMutex *sync.Mutex
-	shortlist      []Contact
-	shortlistAdded map[bits.Bitmap]bool
-
-	closestContactMutex *sync.RWMutex
-	closestContact      *Contact
-	notGettingCloser    *atomic.Bool
-}
-
-func FindContacts(node *Node, target bits.Bitmap, findValue bool, parentGrp *stop.Group) ([]Contact, bool, error) {
-	cf := &contactFinder{
-		node:                node,
-		target:              target,
-		findValue:           findValue,
-		findValueMutex:      &sync.Mutex{},
-		activeContactsMutex: &sync.Mutex{},
-		shortlistMutex:      &sync.Mutex{},
-		shortlistAdded:      make(map[bits.Bitmap]bool),
-		grp:                 stop.New(parentGrp),
-		closestContactMutex: &sync.RWMutex{},
-		notGettingCloser:    atomic.NewBool(false),
-	}
-
-	return cf.Find()
-}
-
-func (cf *contactFinder) Stop() {
-	cf.grp.StopAndWait()
-}
-
-func (cf *contactFinder) Find() ([]Contact, bool, error) {
-	if cf.findValue {
-		cf.debug("starting iterativeFindValue")
-	} else {
-		cf.debug("starting iterativeFindNode")
-	}
-
-	cf.appendNewToShortlist(cf.node.rt.GetClosest(cf.target, alpha))
-	if len(cf.shortlist) == 0 {
-		return nil, false, errors.Err("[%s] find %s: no contacts in routing table", cf.node.id.HexShort(), cf.target.HexShort())
-	}
-
-	go cf.cycle(false)
-	timeout := 5 * time.Second
-CycleLoop:
-	for {
-		select {
-		case <-time.After(timeout):
-			go cf.cycle(false)
-		case <-cf.grp.Ch():
-			break CycleLoop
-		}
-	}
-
-	// TODO: what to do if we have less than K active contacts, shortlist is empty, but we have other contacts in our routing table whom we have not contacted. prolly contact them
-
-	var contacts []Contact
-	var found bool
-	if cf.findValue && len(cf.findValueResult) > 0 {
-		contacts = cf.findValueResult
-		found = true
-	} else {
-		contacts = cf.activeContacts
-		if len(contacts) > bucketSize {
-			contacts = contacts[:bucketSize]
-		}
-	}
-
-	cf.Stop()
-	return contacts, found, nil
-}
-
-// cycle does a single cycle of sending alpha probes and checking results against closestNode
-func (cf *contactFinder) cycle(bigCycle bool) {
-	cycleID := crypto.RandString(6)
-	if bigCycle {
-		cf.debug("LAUNCHING CYCLE %s, AND ITS A BIG CYCLE", cycleID)
-	} else {
-		cf.debug("LAUNCHING CYCLE %s", cycleID)
-	}
-	defer cf.debug("CYCLE %s DONE", cycleID)
-
-	cf.closestContactMutex.RLock()
-	closestContact := cf.closestContact
-	cf.closestContactMutex.RUnlock()
-
-	var wg sync.WaitGroup
-	ch := make(chan *Contact)
-
-	limit := alpha
-	if bigCycle {
-		limit = bucketSize
-	}
-
-	for i := 0; i < limit; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			ch <- cf.probe(cycleID)
-		}()
-	}
-
-	go func() {
-		wg.Wait()
-		close(ch)
-	}()
-
-	foundCloser := false
-	for {
-		c, more := <-ch
-		if !more {
-			break
-		}
-		if c != nil && (closestContact == nil || cf.target.Closer(c.ID, closestContact.ID)) {
-			if closestContact != nil {
-				cf.debug("|%s| best contact improved: %s -> %s", cycleID, closestContact.ID.HexShort(), c.ID.HexShort())
-			} else {
-				cf.debug("|%s| best contact starting at %s", cycleID, c.ID.HexShort())
-			}
-			foundCloser = true
-			closestContact = c
-		}
-	}
-
-	if cf.isSearchFinished() {
-		cf.grp.Stop()
-		return
-	}
-
-	if foundCloser {
-		cf.closestContactMutex.Lock()
-		// have to check again after locking in case other probes found a closer one in the meantime
-		if cf.closestContact == nil || cf.target.Closer(closestContact.ID, cf.closestContact.ID) {
-			cf.closestContact = closestContact
-		}
-		cf.closestContactMutex.Unlock()
-		go cf.cycle(false)
-	} else if !bigCycle {
-		cf.debug("|%s| no improvement, running big cycle", cycleID)
-		go cf.cycle(true)
-	} else {
-		// big cycle ran and there was no improvement, so we're done
-		cf.debug("|%s| big cycle ran, still no improvement", cycleID)
-		cf.notGettingCloser.Store(true)
-	}
-}
-
-// probe sends a single probe, updates the lists, and returns the closest contact it found
-func (cf *contactFinder) probe(cycleID string) *Contact {
-	maybeContact := cf.popFromShortlist()
-	if maybeContact == nil {
-		cf.debug("|%s| no contacts in shortlist, returning", cycleID)
-		return nil
-	}
-
-	c := *maybeContact
-
-	if c.ID.Equals(cf.node.id) {
-		return nil
-	}
-
-	cf.debug("|%s| probe %s: launching", cycleID, c.ID.HexShort())
-
-	req := Request{Arg: &cf.target}
-	if cf.findValue {
-		req.Method = findValueMethod
-	} else {
-		req.Method = findNodeMethod
-	}
-
-	var res *Response
-	resCh := cf.node.SendAsync(c, req)
-	select {
-	case res = <-resCh:
-	case <-cf.grp.Ch():
-		cf.debug("|%s| probe %s: canceled", cycleID, c.ID.HexShort())
-		return nil
-	}
-
-	if res == nil {
-		cf.debug("|%s| probe %s: req canceled or timed out", cycleID, c.ID.HexShort())
-		return nil
-	}
-
-	if cf.findValue && res.FindValueKey != "" {
-		cf.debug("|%s| probe %s: got value", cycleID, c.ID.HexShort())
-		cf.findValueMutex.Lock()
-		cf.findValueResult = res.Contacts
-		cf.findValueMutex.Unlock()
-		cf.grp.Stop()
-		return nil
-	}
-
-	cf.debug("|%s| probe %s: got %s", cycleID, c.ID.HexShort(), res.argsDebug())
-	cf.insertIntoActiveList(c)
-	cf.appendNewToShortlist(res.Contacts)
-
-	cf.activeContactsMutex.Lock()
-	contacts := cf.activeContacts
-	if len(contacts) > bucketSize {
-		contacts = contacts[:bucketSize]
-	}
-	contactsStr := ""
-	for _, c := range contacts {
-		contactsStr += c.ID.HexShort() + ", "
-	}
-	cf.activeContactsMutex.Unlock()
-
-	return cf.closest(res.Contacts...)
-}
-
-// appendNewToShortlist appends any new contacts to the shortlist and sorts it by distance
-// contacts that have already been added to the shortlist in the past are ignored
-func (cf *contactFinder) appendNewToShortlist(contacts []Contact) {
-	cf.shortlistMutex.Lock()
-	defer cf.shortlistMutex.Unlock()
-
-	for _, c := range contacts {
-		if _, ok := cf.shortlistAdded[c.ID]; !ok {
-			cf.shortlist = append(cf.shortlist, c)
-			cf.shortlistAdded[c.ID] = true
-		}
-	}
-
-	sortByDistance(cf.shortlist, cf.target)
-}
-
-// popFromShortlist pops the first contact off the shortlist and returns it
-func (cf *contactFinder) popFromShortlist() *Contact {
-	cf.shortlistMutex.Lock()
-	defer cf.shortlistMutex.Unlock()
-
-	if len(cf.shortlist) == 0 {
-		return nil
-	}
-
-	first := cf.shortlist[0]
-	cf.shortlist = cf.shortlist[1:]
-	return &first
-}
-
-// insertIntoActiveList inserts the contact into appropriate place in the list of active contacts (sorted by distance)
-func (cf *contactFinder) insertIntoActiveList(contact Contact) {
-	cf.activeContactsMutex.Lock()
-	defer cf.activeContactsMutex.Unlock()
-
-	inserted := false
-	for i, n := range cf.activeContacts {
-		if cf.target.Closer(contact.ID, n.ID) {
-			cf.activeContacts = append(cf.activeContacts[:i], append([]Contact{contact}, cf.activeContacts[i:]...)...)
-			inserted = true
-			break
-		}
-	}
-	if !inserted {
-		cf.activeContacts = append(cf.activeContacts, contact)
-	}
-}
-
-// isSearchFinished returns true if the search is done and should be stopped
-func (cf *contactFinder) isSearchFinished() bool {
-	if cf.findValue && len(cf.findValueResult) > 0 {
-		return true
-	}
-
-	select {
-	case <-cf.grp.Ch():
-		return true
-	default:
-	}
-
-	if cf.notGettingCloser.Load() {
-		return true
-	}
-
-	cf.activeContactsMutex.Lock()
-	defer cf.activeContactsMutex.Unlock()
-	return len(cf.activeContacts) >= bucketSize
-}
-
-func (cf *contactFinder) debug(format string, args ...interface{}) {
-	args = append([]interface{}{cf.node.id.HexShort()}, append([]interface{}{cf.target.HexShort()}, args...)...)
-	cfLog.Debugf("[%s] find %s: "+format, args...)
-}
-
-func (cf *contactFinder) closest(contacts ...Contact) *Contact {
-	if len(contacts) == 0 {
-		return nil
-	}
-	closest := contacts[0]
-	for _, c := range contacts {
-		if cf.target.Closer(c.ID, closest.ID) {
-			closest = c
-		}
-	}
-	return &closest
-}

dht/node_test.go

@@ -1,422 +0,0 @@
-package dht
-
-import (
-	"net"
-	"testing"
-	"time"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-	"github.com/lyoshenka/bencode"
-)
-
-func TestPing(t *testing.T) {
-	dhtNodeID := bits.Rand()
-	testNodeID := bits.Rand()
-
-	conn := newTestUDPConn("127.0.0.1:21217")
-
-	dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
-
-	err := dht.connect(conn)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer dht.Shutdown()
-
-	messageID := newMessageID()
-
-	data, err := bencode.EncodeBytes(map[string]interface{}{
-		headerTypeField:      requestType,
-		headerMessageIDField: messageID,
-		headerNodeIDField:    testNodeID.RawString(),
-		headerPayloadField:   "ping",
-		headerArgsField:      []string{},
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
-	timer := time.NewTimer(3 * time.Second)
-
-	select {
-	case <-timer.C:
-		t.Error("timeout")
-	case resp := <-conn.writes:
-		var response map[string]interface{}
-		err := bencode.DecodeBytes(resp.data, &response)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if len(response) != 4 {
-			t.Errorf("expected 4 response fields, got %d", len(response))
-		}
-
-		_, ok := response[headerTypeField]
-		if !ok {
-			t.Error("missing type field")
-		} else {
-			rType, ok := response[headerTypeField].(int64)
-			if !ok {
-				t.Error("type is not an integer")
-			} else if rType != responseType {
-				t.Error("unexpected response type")
-			}
-		}
-
-		_, ok = response[headerMessageIDField]
-		if !ok {
-			t.Error("missing message id field")
-		} else {
-			rMessageID, ok := response[headerMessageIDField].(string)
-			if !ok {
-				t.Error("message ID is not a string")
-			} else if rMessageID != string(messageID[:]) {
-				t.Error("unexpected message ID")
-			}
-		}
-
-		_, ok = response[headerNodeIDField]
-		if !ok {
-			t.Error("missing node id field")
-		} else {
-			rNodeID, ok := response[headerNodeIDField].(string)
-			if !ok {
-				t.Error("node ID is not a string")
-			} else if rNodeID != dhtNodeID.RawString() {
-				t.Error("unexpected node ID")
-			}
-		}
-
-		_, ok = response[headerPayloadField]
-		if !ok {
-			t.Error("missing payload field")
-		} else {
-			rNodeID, ok := response[headerPayloadField].(string)
-			if !ok {
-				t.Error("payload is not a string")
-			} else if rNodeID != pingSuccessResponse {
-				t.Error("did not pong")
-			}
-		}
-	}
-}
-
-func TestStore(t *testing.T) {
-	dhtNodeID := bits.Rand()
-	testNodeID := bits.Rand()
-
-	conn := newTestUDPConn("127.0.0.1:21217")
-
-	dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
-
-	err := dht.connect(conn)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer dht.Shutdown()
-
-	messageID := newMessageID()
-	blobHashToStore := bits.Rand()
-
-	storeRequest := Request{
-		ID:     messageID,
-		NodeID: testNodeID,
-		Method: storeMethod,
-		StoreArgs: &storeArgs{
-			BlobHash: blobHashToStore,
-			Value: storeArgsValue{
-				Token:  dht.node.tokens.Get(testNodeID, conn.addr),
-				LbryID: testNodeID,
-				Port:   9999,
-			},
-			NodeID: testNodeID,
-		},
-	}
-
-	_ = "64 " + // start message
-		"313A30 693065" + // type: 0
-		"313A31 3230   3A 6EB490B5788B63F0F7E6D92352024D0CBDEC2D3A" + // message id
-		"313A32 3438   3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
-		"313A33 35     3A 73746F7265" + // method
-		"313A34 6C" + // start args list
-		"3438 3A 3214D6C2F77FCB5E8D5FC07EDAFBA614F031CE8B2EAB49F924F8143F6DFBADE048D918710072FB98AB1B52B58F4E1468" + // block hash
-		"64" + // start value dict
-		"363A6C6272796964 3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // lbry id
-		"343A706F7274 69 33333333 65" + // port
-		"353A746F6B656E 3438 3A 17C2D8E1E48EF21567FE4AD5C8ED944B798D3B65AB58D0C9122AD6587D1B5FED472EA2CB12284CEFA1C21EFF302322BD" + // token
-		"65" + // end value dict
-		"3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
-		"693065" + // self store (integer)
-		"65" + // end args list
-		"65" // end message
-
-	data, err := bencode.EncodeBytes(storeRequest)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
-	timer := time.NewTimer(3 * time.Second)
-
-	var response map[string]interface{}
-	select {
-	case <-timer.C:
-		t.Fatal("timeout")
-	case resp := <-conn.writes:
-		err := bencode.DecodeBytes(resp.data, &response)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	verifyResponse(t, response, messageID, dhtNodeID.RawString())
-
-	_, ok := response[headerPayloadField]
-	if !ok {
-		t.Error("missing payload field")
-	} else {
-		rNodeID, ok := response[headerPayloadField].(string)
-		if !ok {
-			t.Error("payload is not a string")
-		} else if rNodeID != storeSuccessResponse {
-			t.Error("did not return OK")
-		}
-	}
-
-	if dht.node.store.CountStoredHashes() != 1 {
-		t.Error("dht store has wrong number of items")
-	}
-
-	items := dht.node.store.Get(blobHashToStore)
-	if len(items) != 1 {
-		t.Error("list created in store, but nothing in list")
-	}
-	if !items[0].ID.Equals(testNodeID) {
-		t.Error("wrong value stored")
-	}
-}
-
-func TestFindNode(t *testing.T) {
-	dhtNodeID := bits.Rand()
-	testNodeID := bits.Rand()
-
-	conn := newTestUDPConn("127.0.0.1:21217")
-
-	dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
-
-	err := dht.connect(conn)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer dht.Shutdown()
-
-	nodesToInsert := 3
-	var nodes []Contact
-	for i := 0; i < nodesToInsert; i++ {
-		n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
-		nodes = append(nodes, n)
-		dht.node.rt.Update(n)
-	}
-
-	messageID := newMessageID()
-	blobHashToFind := bits.Rand()
-
-	request := Request{
-		ID:     messageID,
-		NodeID: testNodeID,
-		Method: findNodeMethod,
-		Arg:    &blobHashToFind,
-	}
-
-	data, err := bencode.EncodeBytes(request)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
-	timer := time.NewTimer(3 * time.Second)
-
-	var response map[string]interface{}
-	select {
-	case <-timer.C:
-		t.Fatal("timeout")
-	case resp := <-conn.writes:
-		err := bencode.DecodeBytes(resp.data, &response)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	verifyResponse(t, response, messageID, dhtNodeID.RawString())
-
-	_, ok := response[headerPayloadField]
-	if !ok {
-		t.Fatal("missing payload field")
-	}
-
-	contacts, ok := response[headerPayloadField].([]interface{})
-	if !ok {
-		t.Fatal("payload is not a list")
-	}
-
-	verifyContacts(t, contacts, nodes)
-}
-
-func TestFindValueExisting(t *testing.T) {
-	dhtNodeID := bits.Rand()
-	testNodeID := bits.Rand()
-
-	conn := newTestUDPConn("127.0.0.1:21217")
-
-	dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
-
-	err := dht.connect(conn)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer dht.Shutdown()
-
-	nodesToInsert := 3
-	for i := 0; i < nodesToInsert; i++ {
-		n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
-		dht.node.rt.Update(n)
-	}
-
-	//data, _ := hex.DecodeString("64313a30693065313a3132303a7de8e57d34e316abbb5a8a8da50dcd1ad4c80e0f313a3234383a7ce1b831dec8689e44f80f547d2dea171f6a625e1a4ff6c6165e645f953103dabeb068a622203f859c6c64658fd3aa3b313a33393a66696e6456616c7565313a346c34383aa47624b8e7ee1e54df0c45e2eb858feb0b705bd2a78d8b739be31ba188f4bd6f56b371c51fecc5280d5fd26ba4168e966565")
-
-	messageID := newMessageID()
-	valueToFind := bits.Rand()
-
-	nodeToFind := Contact{ID: bits.Rand(), IP: net.ParseIP("1.2.3.4"), PeerPort: 1286}
-	dht.node.store.Upsert(valueToFind, nodeToFind)
-	dht.node.store.Upsert(valueToFind, nodeToFind)
-	dht.node.store.Upsert(valueToFind, nodeToFind)
-
-	request := Request{
-		ID:     messageID,
-		NodeID: testNodeID,
-		Method: findValueMethod,
-		Arg:    &valueToFind,
-	}
-
-	data, err := bencode.EncodeBytes(request)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
-	timer := time.NewTimer(3 * time.Second)
-
-	var response map[string]interface{}
-	select {
-	case <-timer.C:
-		t.Fatal("timeout")
-	case resp := <-conn.writes:
-		err := bencode.DecodeBytes(resp.data, &response)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	verifyResponse(t, response, messageID, dhtNodeID.RawString())
-
-	_, ok := response[headerPayloadField]
-	if !ok {
-		t.Fatal("missing payload field")
-	}
-
-	payload, ok := response[headerPayloadField].(map[string]interface{})
-	if !ok {
-		t.Fatal("payload is not a dictionary")
-	}
-
-	compactContacts, ok := payload[valueToFind.RawString()]
-	if !ok {
-		t.Fatal("payload is missing key for search value")
-	}
-
-	contacts, ok := compactContacts.([]interface{})
-	if !ok {
-		t.Fatal("search results are not a list")
-	}
-
-	verifyCompactContacts(t, contacts, []Contact{nodeToFind})
-}
-
-func TestFindValueFallbackToFindNode(t *testing.T) {
-	dhtNodeID := bits.Rand()
-	testNodeID := bits.Rand()
-
-	conn := newTestUDPConn("127.0.0.1:21217")
-
-	dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
-
-	err := dht.connect(conn)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer dht.Shutdown()
-
-	nodesToInsert := 3
-	var nodes []Contact
-	for i := 0; i < nodesToInsert; i++ {
-		n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
-		nodes = append(nodes, n)
-		dht.node.rt.Update(n)
-	}
-
-	messageID := newMessageID()
-	valueToFind := bits.Rand()
-
-	request := Request{
-		ID:     messageID,
-		NodeID: testNodeID,
-		Method: findValueMethod,
-		Arg:    &valueToFind,
-	}
-
-	data, err := bencode.EncodeBytes(request)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
-	timer := time.NewTimer(3 * time.Second)
-
-	var response map[string]interface{}
-	select {
-	case <-timer.C:
-		t.Fatal("timeout")
-	case resp := <-conn.writes:
-		err := bencode.DecodeBytes(resp.data, &response)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	verifyResponse(t, response, messageID, dhtNodeID.RawString())
-
-	_, ok := response[headerPayloadField]
-	if !ok {
-		t.Fatal("missing payload field")
-	}
-
-	payload, ok := response[headerPayloadField].(map[string]interface{})
-	if !ok {
-		t.Fatal("payload is not a dictionary")
-	}
-
-	contactsList, ok := payload[contactsField]
-	if !ok {
-		t.Fatal("payload is missing 'contacts' key")
-	}
-
-	contacts, ok := contactsList.([]interface{})
-	if !ok {
-		t.Fatal("'contacts' is not a list")
-	}
-
-	verifyContacts(t, contacts, nodes)
-}

dht/routing_table.go

@@ -1,463 +0,0 @@
-package dht
-
-import (
-	"encoding/json"
-	"fmt"
-	"net"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/stop"
-	"github.com/lbryio/reflector.go/dht/bits"
-)
-
-// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
-
-// TODO: use a tree with bucket splitting instead of a fixed bucket list. include jack's optimization (see link in commit mesg)
-// https://github.com/lbryio/lbry/pull/1211/commits/341b27b6d21ac027671d42458826d02735aaae41
-
-// peer is a contact with extra information
-type peer struct {
-	Contact      Contact
-	Distance     bits.Bitmap
-	LastActivity time.Time
-	// LastReplied time.Time
-	// LastRequested time.Time
-	// LastFailure time.Time
-	// SecondLastFailure time.Time
-	NumFailures int
-
-	//<lastPublished>,
-	//<originallyPublished>
-	//	<originalPublisherID>
-}
-
-func (p *peer) Touch() {
-	p.LastActivity = time.Now()
-	p.NumFailures = 0
-}
-
-// ActiveSince returns whether a peer has responded in the last `d` duration
-// this is used to check if the peer is "good", meaning that we believe the peer will respond to our requests
-func (p *peer) ActiveInLast(d time.Duration) bool {
-	return time.Since(p.LastActivity) < d
-}
-
-// IsBad returns whether a peer is "bad", meaning that it has failed to respond to multiple pings in a row
-func (p *peer) IsBad(maxFalures int) bool {
-	return p.NumFailures >= maxFalures
-}
-
-// Fail marks a peer as having failed to respond. It returns whether or not the peer should be removed from the routing table
-func (p *peer) Fail() {
-	p.NumFailures++
-}
-
-type bucket struct {
-	lock       *sync.RWMutex
-	peers      []peer
-	lastUpdate time.Time
-	Range      bits.Range // capitalized because `range` is a keyword
-}
-
-func newBucket(r bits.Range) *bucket {
-	return &bucket{
-		peers: make([]peer, 0, bucketSize),
-		lock:  &sync.RWMutex{},
-		Range: r,
-	}
-}
-
-// Len returns the number of peers in the bucket
-func (b bucket) Len() int {
-	b.lock.RLock()
-	defer b.lock.RUnlock()
-	return len(b.peers)
-}
-
-func (b bucket) Has(c Contact) bool {
-	b.lock.RLock()
-	defer b.lock.RUnlock()
-	for _, p := range b.peers {
-		if p.Contact.Equals(c, true) {
-			return true
-		}
-	}
-	return false
-}
-
-// Contacts returns a slice of the bucket's contacts
-func (b bucket) Contacts() []Contact {
-	b.lock.RLock()
-	defer b.lock.RUnlock()
-	contacts := make([]Contact, len(b.peers))
-	for i := range b.peers {
-		contacts[i] = b.peers[i].Contact
-	}
-	return contacts
-}
-
-// UpdatePeer marks a contact as having been successfully contacted. if insertIfNew and the contact is does not exist yet, it is inserted
-func (b *bucket) UpdatePeer(p peer, insertIfNew bool) error {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	if !b.Range.Contains(p.Distance) {
-		return errors.Err("this bucket range does not cover this peer")
-	}
-
-	peerIndex := find(p.Contact.ID, b.peers)
-	if peerIndex >= 0 {
-		b.lastUpdate = time.Now()
-		b.peers[peerIndex].Touch()
-		moveToBack(b.peers, peerIndex)
-	} else if insertIfNew {
-		hasRoom := true
-
-		if len(b.peers) >= bucketSize {
-			hasRoom = false
-			for i := range b.peers {
-				if b.peers[i].IsBad(maxPeerFails) {
-					// TODO: Ping contact first. Only remove if it does not respond
-					b.peers = append(b.peers[:i], b.peers[i+1:]...)
-					hasRoom = true
-					break
-				}
-			}
-		}
-
-		if hasRoom {
-			b.lastUpdate = time.Now()
-			p.Touch()
-			b.peers = append(b.peers, p)
-		}
-	}
-
-	return nil
-}
-
-// FailContact marks a contact as having failed, and removes it if it failed too many times
-func (b *bucket) FailContact(id bits.Bitmap) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-	i := find(id, b.peers)
-	if i >= 0 {
-		// BEP5 says not to remove the contact until the bucket is full and you try to insert
-		b.peers[i].Fail()
-	}
-}
-
-// find returns the contact in the bucket, or nil if the bucket does not contain the contact
-func find(id bits.Bitmap, peers []peer) int {
-	for i := range peers {
-		if peers[i].Contact.ID.Equals(id) {
-			return i
-		}
-	}
-	return -1
-}
-
-// NeedsRefresh returns true if bucket has not been updated in the last `refreshInterval`, false otherwise
-func (b *bucket) NeedsRefresh(refreshInterval time.Duration) bool {
-	b.lock.RLock()
-	defer b.lock.RUnlock()
-	return time.Since(b.lastUpdate) > refreshInterval
-}
-
-func (b *bucket) Split() (*bucket, *bucket) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	left := newBucket(b.Range.IntervalP(1, 2))
-	right := newBucket(b.Range.IntervalP(2, 2))
-	left.lastUpdate = b.lastUpdate
-	right.lastUpdate = b.lastUpdate
-
-	for _, p := range b.peers {
-		if left.Range.Contains(p.Distance) {
-			left.peers = append(left.peers, p)
-		} else {
-			right.peers = append(right.peers, p)
-		}
-	}
-
-	if len(b.peers) > 1 {
-		if len(left.peers) == 0 {
-			left, right = right.Split()
-			left.Range.Start = b.Range.Start
-		} else if len(right.peers) == 0 {
-			left, right = left.Split()
-			right.Range.End = b.Range.End
-		}
-	}
-
-	return left, right
-}
-
-type routingTable struct {
-	id      bits.Bitmap
-	buckets []*bucket
-	mu      *sync.RWMutex // this mutex is write-locked only when CHANGING THE NUMBER OF BUCKETS in the table
-}
-
-func newRoutingTable(id bits.Bitmap) *routingTable {
-	rt := routingTable{
-		id: id,
-		mu: &sync.RWMutex{},
-	}
-	rt.reset()
-	return &rt
-}
-
-func (rt *routingTable) reset() {
-	rt.mu.Lock()
-	defer rt.mu.Unlock()
-	rt.buckets = []*bucket{newBucket(bits.MaxRange())}
-}
-
-func (rt *routingTable) BucketInfo() string {
-	rt.mu.RLock()
-	defer rt.mu.RUnlock()
-
-	var bucketInfo []string
-	for i, b := range rt.buckets {
-		if b.Len() > 0 {
-			contacts := b.Contacts()
-			s := make([]string, len(contacts))
-			for j, c := range contacts {
-				s[j] = c.ID.HexShort()
-			}
-			bucketInfo = append(bucketInfo, fmt.Sprintf("bucket %d: (%d) %s", i, len(contacts), strings.Join(s, ", ")))
-		}
-	}
-	if len(bucketInfo) == 0 {
-		return "buckets are empty"
-	}
-	return strings.Join(bucketInfo, "\n")
-}
-
-// Update inserts or refreshes a contact
-func (rt *routingTable) Update(c Contact) {
-	rt.mu.Lock() // write lock, because updates may cause bucket splits
-	defer rt.mu.Unlock()
-
-	b := rt.bucketFor(c.ID)
-
-	if rt.shouldSplit(b, c) {
-		left, right := b.Split()
-
-		for i := range rt.buckets {
-			if rt.buckets[i].Range.Start.Equals(left.Range.Start) {
-				rt.buckets = append(rt.buckets[:i], append([]*bucket{left, right}, rt.buckets[i+1:]...)...)
-				break
-			}
-		}
-
-		if left.Range.Contains(c.ID) {
-			b = left
-		} else {
-			b = right
-		}
-	}
-
-	err := b.UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, true)
-	if err != nil {
-		log.Error(err)
-	}
-}
-
-// Fresh refreshes a contact if its already in the routing table
-func (rt *routingTable) Fresh(c Contact) {
-	rt.mu.RLock()
-	defer rt.mu.RUnlock()
-	err := rt.bucketFor(c.ID).UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, false)
-	if err != nil {
-		log.Error(err)
-	}
-}
-
-// FailContact marks a contact as having failed, and removes it if it failed too many times
-func (rt *routingTable) Fail(c Contact) {
-	rt.mu.RLock()
-	defer rt.mu.RUnlock()
-	rt.bucketFor(c.ID).FailContact(c.ID)
-}
-
-// GetClosest returns the closest `limit` contacts from the routing table.
-// This is a locking wrapper around getClosest()
-func (rt *routingTable) GetClosest(target bits.Bitmap, limit int) []Contact {
-	rt.mu.RLock()
-	defer rt.mu.RUnlock()
-	return rt.getClosest(target, limit)
-}
-
-// getClosest returns the closest `limit` contacts from the routing table
-func (rt *routingTable) getClosest(target bits.Bitmap, limit int) []Contact {
-	var contacts []Contact
-	for _, b := range rt.buckets {
-		contacts = append(contacts, b.Contacts()...)
-	}
-
-	sortByDistance(contacts, target)
-	if len(contacts) > limit {
-		contacts = contacts[:limit]
-	}
-
-	return contacts
-}
-
-// Count returns the number of contacts in the routing table
-func (rt *routingTable) Count() int {
-	rt.mu.RLock()
-	defer rt.mu.RUnlock()
-	count := 0
-	for _, bucket := range rt.buckets {
-		count += bucket.Len()
-	}
-	return count
-}
-
-// Len returns the number of buckets in the routing table
-func (rt *routingTable) Len() int {
-	rt.mu.RLock()
-	defer rt.mu.RUnlock()
-	return len(rt.buckets)
-}
-
-func (rt *routingTable) bucketFor(target bits.Bitmap) *bucket {
-	if rt.id.Equals(target) {
-		panic("routing table does not have a bucket for its own id")
-	}
-	distance := target.Xor(rt.id)
-	for _, b := range rt.buckets {
-		if b.Range.Contains(distance) {
-			return b
-		}
-	}
-	panic("target is not contained in any buckets")
-}
-
-func (rt *routingTable) shouldSplit(b *bucket, c Contact) bool {
-	if b.Has(c) {
-		return false
-	}
-	if b.Len() >= bucketSize {
-		if b.Range.Start.Equals(bits.Bitmap{}) { // this is the bucket covering our node id
-			return true
-		}
-		kClosest := rt.getClosest(rt.id, bucketSize)
-		kthClosest := kClosest[len(kClosest)-1]
-		if rt.id.Closer(c.ID, kthClosest.ID) {
-			return true
-		}
-	}
-	return false
-}
-
-//func (rt *routingTable) printBucketInfo() {
-//	fmt.Printf("there are %d contacts in %d buckets\n", rt.Count(), rt.Len())
-//	for i, b := range rt.buckets {
-//		fmt.Printf("bucket %d, %d contacts\n", i+1, len(b.peers))
-//		fmt.Printf("    start : %s\n", b.Range.Start.String())
-//		fmt.Printf("    stop  : %s\n", b.Range.End.String())
-//		fmt.Println("")
-//	}
-//}
-
-func (rt *routingTable) GetIDsForRefresh(refreshInterval time.Duration) []bits.Bitmap {
-	var bitmaps []bits.Bitmap
-	for i, bucket := range rt.buckets {
-		if bucket.NeedsRefresh(refreshInterval) {
-			bitmaps = append(bitmaps, bits.Rand().Prefix(i, false))
-		}
-	}
-	return bitmaps
-}
-
-const rtContactSep = "-"
-
-type rtSave struct {
-	ID       string   `json:"id"`
-	Contacts []string `json:"contacts"`
-}
-
-func (rt *routingTable) MarshalJSON() ([]byte, error) {
-	var data rtSave
-	data.ID = rt.id.Hex()
-	for _, b := range rt.buckets {
-		for _, c := range b.Contacts() {
-			data.Contacts = append(data.Contacts, strings.Join([]string{c.ID.Hex(), c.IP.String(), strconv.Itoa(c.Port)}, rtContactSep))
-		}
-	}
-	return json.Marshal(data)
-}
-
-func (rt *routingTable) UnmarshalJSON(b []byte) error {
-	var data rtSave
-	err := json.Unmarshal(b, &data)
-	if err != nil {
-		return err
-	}
-
-	rt.id, err = bits.FromHex(data.ID)
-	if err != nil {
-		return errors.Prefix("decoding ID", err)
-	}
-	rt.reset()
-
-	for _, s := range data.Contacts {
-		parts := strings.Split(s, rtContactSep)
-		if len(parts) != 3 {
-			return errors.Err("decoding contact %s: wrong number of parts", s)
-		}
-		var c Contact
-		c.ID, err = bits.FromHex(parts[0])
-		if err != nil {
-			return errors.Err("decoding contact %s: invalid ID: %s", s, err)
-		}
-		c.IP = net.ParseIP(parts[1])
-		if c.IP == nil {
-			return errors.Err("decoding contact %s: invalid IP", s)
-		}
-		c.Port, err = strconv.Atoi(parts[2])
-		if err != nil {
-			return errors.Err("decoding contact %s: invalid port: %s", s, err)
-		}
-		rt.Update(c)
-	}
-
-	return nil
-}
-
-// RoutingTableRefresh refreshes any buckets that need to be refreshed
-func RoutingTableRefresh(n *Node, refreshInterval time.Duration, parentGrp *stop.Group) {
-	done := stop.New()
-
-	for _, id := range n.rt.GetIDsForRefresh(refreshInterval) {
-		done.Add(1)
-		go func(id bits.Bitmap) {
-			defer done.Done()
-			_, _, err := FindContacts(n, id, false, parentGrp)
-			if err != nil {
-				log.Error("error finding contact during routing table refresh - ", err)
-			}
-		}(id)
-	}
-
-	done.Wait()
-	done.Stop()
-}
-
-func moveToBack(peers []peer, index int) {
-	if index < 0 || len(peers) <= index+1 {
-		return
-	}
-	p := peers[index]
-	for i := index; i < len(peers)-1; i++ {
-		peers[i] = peers[i+1]
-	}
-	peers[len(peers)-1] = p
-}

dht/routing_table_test.go

@@ -1,328 +0,0 @@
-package dht
-
-import (
-	"encoding/json"
-	"math/big"
-	"net"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-
-	"github.com/sebdah/goldie"
-)
-
-func TestBucket_Split(t *testing.T) {
-	rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
-	if len(rt.buckets) != 1 {
-		t.Errorf("there should only be one bucket so far")
-	}
-	if len(rt.buckets[0].peers) != 0 {
-		t.Errorf("there should be no contacts yet")
-	}
-
-	var tests = []struct {
-		name                  string
-		id                    bits.Bitmap
-		expectedBucketCount   int
-		expectedTotalContacts int
-	}{
-		//fill first bucket
-		{"b1-one", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100"), 1, 1},
-		{"b1-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200"), 1, 2},
-		{"b1-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300"), 1, 3},
-		{"b1-four", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400"), 1, 4},
-		{"b1-five", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000500"), 1, 5},
-		{"b1-six", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000600"), 1, 6},
-		{"b1-seven", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000700"), 1, 7},
-		{"b1-eight", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 1, 8},
-
-		// split off second bucket and fill it
-		{"b2-one", bits.FromHexP("001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 9},
-		{"b2-two", bits.FromHexP("002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 10},
-		{"b2-three", bits.FromHexP("003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 11},
-		{"b2-four", bits.FromHexP("004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 12},
-		{"b2-five", bits.FromHexP("005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 13},
-		{"b2-six", bits.FromHexP("006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 14},
-		{"b2-seven", bits.FromHexP("007000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
-
-		// at this point there are two buckets. the first has 7 contacts, the second has 8
-
-		// inserts into the second bucket should be skipped
-		{"dont-split", bits.FromHexP("009000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
-
-		// ... unless the ID is closer than the kth-closest contact
-		{"split-kth-closest", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 2, 16},
-
-		{"b3-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), 3, 17},
-		{"b3-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), 3, 18},
-	}
-
-	for i, testCase := range tests {
-		rt.Update(Contact{testCase.id, net.ParseIP("127.0.0.1"), 8000 + i, 0})
-
-		if len(rt.buckets) != testCase.expectedBucketCount {
-			t.Errorf("failed test case %s. there should be %d buckets, got %d", testCase.name, testCase.expectedBucketCount, len(rt.buckets))
-		}
-		if rt.Count() != testCase.expectedTotalContacts {
-			t.Errorf("failed test case %s. there should be %d contacts, got %d", testCase.name, testCase.expectedTotalContacts, rt.Count())
-		}
-	}
-
-	var testRanges = []struct {
-		id       bits.Bitmap
-		expected int
-	}{
-		{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0},
-		{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"), 0},
-		{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000410"), 1},
-		{bits.FromHexP("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007f0"), 1},
-		{bits.FromHexP("F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 2},
-		{bits.FromHexP("F00000000000000000000000000000000000000000000000000F00000000000000000000000000000000000000000000"), 2},
-		{bits.FromHexP("F0000000000000000000000000000000F0000000000000000000000000F0000000000000000000000000000000000000"), 2},
-	}
-
-	for _, tt := range testRanges {
-		bucket := bucketNumFor(rt, tt.id)
-		if bucket != tt.expected {
-			t.Errorf("bucketFor(%s, %s) => got %d, expected %d", tt.id.Hex(), rt.id.Hex(), bucket, tt.expected)
-		}
-	}
-}
-
-func bucketNumFor(rt *routingTable, target bits.Bitmap) int {
-	if rt.id.Equals(target) {
-		panic("routing table does not have a bucket for its own id")
-	}
-	distance := target.Xor(rt.id)
-	for i := range rt.buckets {
-		if rt.buckets[i].Range.Contains(distance) {
-			return i
-		}
-	}
-	panic("target is not contained in any buckets")
-}
-
-func TestBucket_Split_Continuous(t *testing.T) {
-	b := newBucket(bits.MaxRange())
-
-	left, right := b.Split()
-
-	if !left.Range.Start.Equals(b.Range.Start) {
-		t.Errorf("left bucket start does not align with original bucket start. got %s, expected %s", left.Range.Start, b.Range.Start)
-	}
-
-	if !right.Range.End.Equals(b.Range.End) {
-		t.Errorf("right bucket end does not align with original bucket end. got %s, expected %s", right.Range.End, b.Range.End)
-	}
-
-	leftEndNext := (&big.Int{}).Add(left.Range.End.Big(), big.NewInt(1))
-	if !bits.FromBigP(leftEndNext).Equals(right.Range.Start) {
-		t.Errorf("there's a gap between left bucket end and right bucket start. end is %s, start is %s", left.Range.End, right.Range.Start)
-	}
-}
-
-func TestBucket_Split_KthClosest_DoSplit(t *testing.T) {
-	rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
-
-	// add 4 low IDs
-	rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
-	rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
-	rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
-	rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
-
-	// add 4 high IDs
-	rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
-	rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
-	rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
-	rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
-
-	// split the bucket and fill the high bucket
-	rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
-	rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
-	rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
-	rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
-
-	// add a high ID. it should split because the high ID is closer than the Kth closest ID
-	rt.Update(Contact{bits.FromHexP("910000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
-
-	if len(rt.buckets) != 3 {
-		t.Errorf("expected 3 buckets, got %d", len(rt.buckets))
-	}
-	if rt.Count() != 13 {
-		t.Errorf("expected 13 contacts, got %d", rt.Count())
-	}
-}
-
-func TestBucket_Split_KthClosest_DontSplit(t *testing.T) {
-	rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
-
-	// add 4 low IDs
-	rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
-	rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
-	rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
-	rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
-
-	// add 4 high IDs
-	rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
-	rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
-	rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
-	rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
-
-	// split the bucket and fill the high bucket
-	rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
-	rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
-	rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
-	rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
-
-	// add a really high ID. this should not split because its not closer than the Kth closest ID
-	rt.Update(Contact{bits.FromHexP("ffff00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
-
-	if len(rt.buckets) != 2 {
-		t.Errorf("expected 2 buckets, got %d", len(rt.buckets))
-	}
-	if rt.Count() != 12 {
-		t.Errorf("expected 12 contacts, got %d", rt.Count())
-	}
-}
-
-func TestRoutingTable_GetClosest(t *testing.T) {
-	n1 := bits.FromHexP("FFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
-	n2 := bits.FromHexP("FFFFFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
-	n3 := bits.FromHexP("111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
-	rt := newRoutingTable(n1)
-	rt.Update(Contact{n2, net.ParseIP("127.0.0.1"), 8001, 0})
-	rt.Update(Contact{n3, net.ParseIP("127.0.0.1"), 8002, 0})
-
-	contacts := rt.GetClosest(bits.FromHexP("222222220000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 1)
-	if len(contacts) != 1 {
-		t.Fail()
-		return
-	}
-	if !contacts[0].ID.Equals(n3) {
-		t.Error(contacts[0])
-	}
-	contacts = rt.GetClosest(n2, 10)
-	if len(contacts) != 2 {
-		t.Error(len(contacts))
-		return
-	}
-	if !contacts[0].ID.Equals(n2) {
-		t.Error(contacts[0])
-	}
-	if !contacts[1].ID.Equals(n3) {
-		t.Error(contacts[1])
-	}
-}
-
-func TestRoutingTable_GetClosest_Empty(t *testing.T) {
-	n1 := bits.FromShortHexP("1")
-	rt := newRoutingTable(n1)
-
-	contacts := rt.GetClosest(bits.FromShortHexP("a"), 3)
-	if len(contacts) != 0 {
-		t.Error("there shouldn't be any contacts")
-		return
-	}
-}
-
-func TestRoutingTable_Refresh(t *testing.T) {
-	t.Skip("TODO: test routing table refreshing")
-}
-
-func TestRoutingTable_MoveToBack(t *testing.T) {
-	tt := map[string]struct {
-		data     []peer
-		index    int
-		expected []peer
-	}{
-		"simpleMove": {
-			data:     []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
-			index:    1,
-			expected: []peer{{NumFailures: 0}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 1}},
-		},
-		"moveFirst": {
-			data:     []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
-			index:    0,
-			expected: []peer{{NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 0}},
-		},
-		"moveLast": {
-			data:     []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
-			index:    3,
-			expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
-		},
-		"largeIndex": {
-			data:     []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
-			index:    27,
-			expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
-		},
-		"negativeIndex": {
-			data:     []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
-			index:    -12,
-			expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
-		},
-	}
-
-	for name, test := range tt {
-		moveToBack(test.data, test.index)
-		expected := make([]string, len(test.expected))
-		actual := make([]string, len(test.data))
-		for i := range actual {
-			actual[i] = strconv.Itoa(test.data[i].NumFailures)
-			expected[i] = strconv.Itoa(test.expected[i].NumFailures)
-		}
-
-		expJoin := strings.Join(expected, ",")
-		actJoin := strings.Join(actual, ",")
-
-		if actJoin != expJoin {
-			t.Errorf("%s failed: got %s; expected %s", name, actJoin, expJoin)
-		}
-	}
-}
-
-func TestRoutingTable_Save(t *testing.T) {
-	t.Skip("fix me")
-	id := bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41")
-	rt := newRoutingTable(id)
-
-	for i, b := range rt.buckets {
-		for j := 0; j < bucketSize; j++ {
-			toAdd := b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j)))
-			if toAdd.Cmp(b.Range.End) <= 0 {
-				rt.Update(Contact{
-					ID:   b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j))),
-					IP:   net.ParseIP("1.2.3." + strconv.Itoa(j)),
-					Port: 1 + i*bucketSize + j,
-				})
-			}
-		}
-	}
-
-	data, err := json.MarshalIndent(rt, "", "  ")
-	if err != nil {
-		t.Error(err)
-	}
-
-	goldie.Assert(t, t.Name(), data)
-}
-
-func TestRoutingTable_Load_ID(t *testing.T) {
-	t.Skip("fix me")
-	id := "1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"
-	data := []byte(`{"id": "` + id + `","contacts": []}`)
-
-	rt := routingTable{}
-	err := json.Unmarshal(data, &rt)
-	if err != nil {
-		t.Error(err)
-	}
-	if rt.id.Hex() != id {
-		t.Error("id mismatch")
-	}
-}
-
-func TestRoutingTable_Load_Contacts(t *testing.T) {
-	t.Skip("TODO")
-}

dht/rpc.go

@@ -1,187 +0,0 @@
-package dht
-
-import (
-	"context"
-	"net"
-	"net/http"
-	"strconv"
-	"sync"
-
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/reflector.go/dht/bits"
-
-	"github.com/gorilla/mux"
-	rpc2 "github.com/gorilla/rpc/v2"
-	"github.com/gorilla/rpc/v2/json"
-)
-
-type rpcReceiver struct {
-	dht *DHT
-}
-
-type RpcPingArgs struct {
-	Address string
-}
-
-func (rpc *rpcReceiver) Ping(r *http.Request, args *RpcPingArgs, result *string) error {
-	if args.Address == "" {
-		return errors.Err("no address given")
-	}
-
-	err := rpc.dht.Ping(args.Address)
-	if err != nil {
-		return err
-	}
-
-	*result = pingSuccessResponse
-	return nil
-}
-
-type RpcFindArgs struct {
-	Key    string
-	NodeID string
-	IP     string
-	Port   int
-}
-
-func (rpc *rpcReceiver) FindNode(r *http.Request, args *RpcFindArgs, result *[]Contact) error {
-	key, err := bits.FromHex(args.Key)
-	if err != nil {
-		return err
-	}
-
-	toQuery, err := bits.FromHex(args.NodeID)
-	if err != nil {
-		return err
-	}
-
-	c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
-	req := Request{Method: findNodeMethod, Arg: &key}
-
-	nodeResponse := rpc.dht.node.Send(c, req)
-	if nodeResponse != nil && nodeResponse.Contacts != nil {
-		*result = nodeResponse.Contacts
-	}
-	return nil
-}
-
-type RpcFindValueResult struct {
-	Contacts []Contact
-	Value    string
-}
-
-func (rpc *rpcReceiver) FindValue(r *http.Request, args *RpcFindArgs, result *RpcFindValueResult) error {
-	key, err := bits.FromHex(args.Key)
-	if err != nil {
-		return err
-	}
-	toQuery, err := bits.FromHex(args.NodeID)
-	if err != nil {
-		return err
-	}
-	c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
-	req := Request{Arg: &key, Method: findValueMethod}
-
-	nodeResponse := rpc.dht.node.Send(c, req)
-	if nodeResponse != nil && nodeResponse.FindValueKey != "" {
-		*result = RpcFindValueResult{Value: nodeResponse.FindValueKey}
-		return nil
-	}
-	if nodeResponse != nil && nodeResponse.Contacts != nil {
-		*result = RpcFindValueResult{Contacts: nodeResponse.Contacts}
-		return nil
-	}
-
-	return errors.Err("not sure what happened")
-}
-
-type RpcIterativeFindValueArgs struct {
-	Key string
-}
-
-type RpcIterativeFindValueResult struct {
-	Contacts   []Contact
-	FoundValue bool
-}
-
-func (rpc *rpcReceiver) IterativeFindValue(r *http.Request, args *RpcIterativeFindValueArgs, result *RpcIterativeFindValueResult) error {
-	key, err := bits.FromHex(args.Key)
-	if err != nil {
-		return err
-	}
-	foundContacts, found, err := FindContacts(rpc.dht.node, key, false, nil)
-	if err != nil {
-		return err
-	}
-	result.Contacts = foundContacts
-	result.FoundValue = found
-	return nil
-}
-
-type RpcBucketResponse struct {
-	Start       string
-	End         string
-	NumContacts int
-	Contacts    []Contact
-}
-
-type RpcRoutingTableResponse struct {
-	NodeID     string
-	NumBuckets int
-	Buckets    []RpcBucketResponse
-}
-
-func (rpc *rpcReceiver) GetRoutingTable(r *http.Request, args *struct{}, result *RpcRoutingTableResponse) error {
-	result.NodeID = rpc.dht.node.id.String()
-	result.NumBuckets = len(rpc.dht.node.rt.buckets)
-	for _, b := range rpc.dht.node.rt.buckets {
-		result.Buckets = append(result.Buckets, RpcBucketResponse{
-			Start:       b.Range.Start.String(),
-			End:         b.Range.End.String(),
-			NumContacts: b.Len(),
-			Contacts:    b.Contacts(),
-		})
-	}
-	return nil
-}
-
-func (rpc *rpcReceiver) AddKnownNode(r *http.Request, args *Contact, result *string) error {
-	rpc.dht.node.AddKnownNode(*args)
-	return nil
-}
-
-func (dht *DHT) runRPCServer(port int) {
-	addr := "0.0.0.0:" + strconv.Itoa(port)
-
-	s := rpc2.NewServer()
-	s.RegisterCodec(json.NewCodec(), "application/json")
-	s.RegisterCodec(json.NewCodec(), "application/json;charset=UTF-8")
-	err := s.RegisterService(&rpcReceiver{dht: dht}, "rpc")
-	if err != nil {
-		log.Error(errors.Prefix("registering rpc service", err))
-		return
-	}
-
-	handler := mux.NewRouter()
-	handler.Handle("/", s)
-	server := &http.Server{Addr: addr, Handler: handler}
-
-	wg := sync.WaitGroup{}
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		log.Printf("RPC server listening on %s", addr)
-		err := server.ListenAndServe()
-		if err != nil && err != http.ErrServerClosed {
-			log.Error(err)
-		}
-	}()
-
-	<-dht.grp.Ch()
-	err = server.Shutdown(context.Background())
-	if err != nil {
-		log.Error(errors.Prefix("shutting down rpc service", err))
-		return
-	}
-	wg.Wait()
-}

dht/store.go

@@ -1,62 +0,0 @@
-package dht
-
-import (
-	"sync"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-)
-
-// TODO: expire stored data after tExpire time
-
-type contactStore struct {
-	// map of blob hashes to (map of node IDs to bools)
-	hashes map[bits.Bitmap]map[bits.Bitmap]bool
-	// stores the peers themselves, so they can be updated in one place
-	contacts map[bits.Bitmap]Contact
-	lock     sync.RWMutex
-}
-
-func newStore() *contactStore {
-	return &contactStore{
-		hashes:   make(map[bits.Bitmap]map[bits.Bitmap]bool),
-		contacts: make(map[bits.Bitmap]Contact),
-	}
-}
-
-func (s *contactStore) Upsert(blobHash bits.Bitmap, contact Contact) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	if _, ok := s.hashes[blobHash]; !ok {
-		s.hashes[blobHash] = make(map[bits.Bitmap]bool)
-	}
-	s.hashes[blobHash][contact.ID] = true
-	s.contacts[contact.ID] = contact
-}
-
-func (s *contactStore) Get(blobHash bits.Bitmap) []Contact {
-	s.lock.RLock()
-	defer s.lock.RUnlock()
-
-	var contacts []Contact
-	if ids, ok := s.hashes[blobHash]; ok {
-		for id := range ids {
-			contact, ok := s.contacts[id]
-			if !ok {
-				panic("node id in IDs list, but not in nodeInfo")
-			}
-			contacts = append(contacts, contact)
-		}
-	}
-	return contacts
-}
-
-func (s *contactStore) RemoveTODO(contact Contact) {
-	// TODO: remove peer from everywhere
-}
-
-func (s *contactStore) CountStoredHashes() int {
-	s.lock.RLock()
-	defer s.lock.RUnlock()
-	return len(s.hashes)
-}

dht/testing.go

@@ -1,312 +0,0 @@
-package dht
-
-import (
-	"net"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/reflector.go/dht/bits"
-)
-
-var testingDHTIP = "127.0.0.1"
-var testingDHTFirstPort = 21000
-
-// TestingCreateNetwork initializes a testable DHT network with a specific number of nodes, with bootstrap and concurrent options.
-func TestingCreateNetwork(t *testing.T, numNodes int, bootstrap, concurrent bool) (*BootstrapNode, []*DHT) {
-	var bootstrapNode *BootstrapNode
-	var seeds []string
-
-	if bootstrap {
-		bootstrapAddress := testingDHTIP + ":" + strconv.Itoa(testingDHTFirstPort)
-		seeds = []string{bootstrapAddress}
-		bootstrapNode = NewBootstrapNode(bits.Rand(), 0, bootstrapDefaultRefreshDuration)
-		listener, err := net.ListenPacket(Network, bootstrapAddress)
-		if err != nil {
-			panic(err)
-		}
-
-		err = bootstrapNode.Connect(listener.(*net.UDPConn))
-		if err != nil {
-			t.Error("error connecting bootstrap node - ", err)
-		}
-	}
-
-	if numNodes < 1 {
-		return bootstrapNode, nil
-	}
-
-	firstPort := testingDHTFirstPort + 1
-	dhts := make([]*DHT, numNodes)
-
-	for i := 0; i < numNodes; i++ {
-		c := NewStandardConfig()
-		c.NodeID = bits.Rand().Hex()
-		c.Address = testingDHTIP + ":" + strconv.Itoa(firstPort+i)
-		c.SeedNodes = seeds
-		dht := New(c)
-
-		go func() {
-			err := dht.Start()
-			if err != nil {
-				t.Error("error starting dht - ", err)
-			}
-		}()
-		if !concurrent {
-			dht.WaitUntilJoined()
-		}
-		dhts[i] = dht
-	}
-
-	if concurrent {
-		for _, d := range dhts {
-			d.WaitUntilJoined()
-		}
-	}
-
-	return bootstrapNode, dhts
-}
-
-type timeoutErr struct {
-	error
-}
-
-func (t timeoutErr) Timeout() bool {
-	return true
-}
-
-func (t timeoutErr) Temporary() bool {
-	return true
-}
-
-// TODO: just use a normal net.Conn instead of this mock conn
-
-type testUDPPacket struct {
-	data []byte
-	addr *net.UDPAddr
-}
-
-type testUDPConn struct {
-	addr   *net.UDPAddr
-	toRead chan testUDPPacket
-	writes chan testUDPPacket
-
-	readDeadline time.Time
-}
-
-func newTestUDPConn(addr string) *testUDPConn {
-	parts := strings.Split(addr, ":")
-	if len(parts) != 2 {
-		panic("addr needs ip and port")
-	}
-	port, err := strconv.Atoi(parts[1])
-	if err != nil {
-		panic(err)
-	}
-	return &testUDPConn{
-		addr:   &net.UDPAddr{IP: net.IP(parts[0]), Port: port},
-		toRead: make(chan testUDPPacket),
-		writes: make(chan testUDPPacket),
-	}
-}
-
-func (t testUDPConn) ReadFromUDP(b []byte) (int, *net.UDPAddr, error) {
-	var timeoutCh <-chan time.Time
-	if !t.readDeadline.IsZero() {
-		timeoutCh = time.After(time.Until(t.readDeadline))
-	}
-
-	select {
-	case packet, ok := <-t.toRead:
-		if !ok {
-			return 0, nil, errors.Err("conn closed")
-		}
-		n := copy(b, packet.data)
-		return n, packet.addr, nil
-	case <-timeoutCh:
-		return 0, nil, timeoutErr{errors.Err("timeout")}
-	}
-}
-
-func (t testUDPConn) WriteToUDP(b []byte, addr *net.UDPAddr) (int, error) {
-	t.writes <- testUDPPacket{data: b, addr: addr}
-	return len(b), nil
-}
-
-func (t *testUDPConn) SetReadDeadline(tm time.Time) error {
-	t.readDeadline = tm
-	return nil
-}
-
-func (t *testUDPConn) SetWriteDeadline(tm time.Time) error {
-	return nil
-}
-
-func (t *testUDPConn) Close() error {
-	close(t.toRead)
-	t.writes = nil
-	return nil
-}
-
-func verifyResponse(t *testing.T, resp map[string]interface{}, id messageID, dhtNodeID string) {
-	if len(resp) != 4 {
-		t.Errorf("expected 4 response fields, got %d", len(resp))
-	}
-
-	_, ok := resp[headerTypeField]
-	if !ok {
-		t.Error("missing type field")
-	} else {
-		rType, ok := resp[headerTypeField].(int64)
-		if !ok {
-			t.Error("type is not an integer")
-		} else if rType != responseType {
-			t.Error("unexpected response type")
-		}
-	}
-
-	_, ok = resp[headerMessageIDField]
-	if !ok {
-		t.Error("missing message id field")
-	} else {
-		rMessageID, ok := resp[headerMessageIDField].(string)
-		if !ok {
-			t.Error("message ID is not a string")
-		} else if rMessageID != string(id[:]) {
-			t.Error("unexpected message ID")
-		}
-		if len(rMessageID) != messageIDLength {
-			t.Errorf("message ID should be %d chars long", messageIDLength)
-		}
-	}
-
-	_, ok = resp[headerNodeIDField]
-	if !ok {
-		t.Error("missing node id field")
-	} else {
-		rNodeID, ok := resp[headerNodeIDField].(string)
-		if !ok {
-			t.Error("node ID is not a string")
-		} else if rNodeID != dhtNodeID {
-			t.Error("unexpected node ID")
-		}
-		if len(rNodeID) != nodeIDLength {
-			t.Errorf("node ID should be %d chars long", nodeIDLength)
-		}
-	}
-}
-
-func verifyContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
-	if len(contacts) != len(nodes) {
-		t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
-		return
-	}
-
-	foundNodes := make(map[string]bool)
-
-	for _, c := range contacts {
-		contact, ok := c.([]interface{})
-		if !ok {
-			t.Error("contact is not a list")
-			return
-		}
-
-		if len(contact) != 3 {
-			t.Error("contact must be 3 items")
-			return
-		}
-
-		var currNode Contact
-		currNodeFound := false
-
-		id, ok := contact[0].(string)
-		if !ok {
-			t.Error("contact id is not a string")
-		} else {
-			if _, ok := foundNodes[id]; ok {
-				t.Errorf("contact %s appears multiple times", id)
-				continue
-			}
-			for _, n := range nodes {
-				if n.ID.RawString() == id {
-					currNode = n
-					currNodeFound = true
-					foundNodes[id] = true
-					break
-				}
-			}
-			if !currNodeFound {
-				t.Errorf("unexpected contact %s", id)
-				continue
-			}
-		}
-
-		ip, ok := contact[1].(string)
-		if !ok {
-			t.Error("contact IP is not a string")
-		} else if !currNode.IP.Equal(net.ParseIP(ip)) {
-			t.Errorf("contact IP mismatch. got %s; expected %s", ip, currNode.IP.String())
-		}
-
-		port, ok := contact[2].(int64)
-		if !ok {
-			t.Error("contact port is not an int")
-		} else if int(port) != currNode.Port {
-			t.Errorf("contact port mismatch. got %d; expected %d", port, currNode.Port)
-		}
-	}
-}
-
-func verifyCompactContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
-	if len(contacts) != len(nodes) {
-		t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
-		return
-	}
-
-	foundNodes := make(map[string]bool)
-
-	for _, c := range contacts {
-		compact, ok := c.(string)
-		if !ok {
-			t.Error("contact is not a string")
-			return
-		}
-
-		contact := Contact{}
-		err := contact.UnmarshalCompact([]byte(compact))
-		if err != nil {
-			t.Error(err)
-			return
-		}
-
-		var currNode Contact
-		currNodeFound := false
-
-		if _, ok := foundNodes[contact.ID.Hex()]; ok {
-			t.Errorf("contact %s appears multiple times", contact.ID.Hex())
-			continue
-		}
-		for _, n := range nodes {
-			if n.ID.Equals(contact.ID) {
-				currNode = n
-				currNodeFound = true
-				foundNodes[contact.ID.Hex()] = true
-				break
-			}
-		}
-		if !currNodeFound {
-			t.Errorf("unexpected contact %s", contact.ID.Hex())
-			continue
-		}
-
-		if !currNode.IP.Equal(contact.IP) {
-			t.Errorf("contact IP mismatch. got %s; expected %s", contact.IP.String(), currNode.IP.String())
-		}
-
-		if contact.Port != currNode.Port {
-			t.Errorf("contact port mismatch. got %d; expected %d", contact.Port, currNode.Port)
-		}
-	}
-}

dht/token_cache.go

@@ -1,71 +0,0 @@
-package dht
-
-import (
-	"sync"
-	"time"
-
-	"github.com/lbryio/reflector.go/dht/bits"
-
-	"github.com/lbryio/lbry.go/stop"
-)
-
-// TODO: this should be moved out of dht and into node, and it should be completely hidden inside node. dht should not need to know about tokens
-
-type tokenCacheEntry struct {
-	token      string
-	receivedAt time.Time
-}
-
-type tokenCache struct {
-	node       *Node
-	tokens     map[string]tokenCacheEntry
-	expiration time.Duration
-	lock       *sync.RWMutex
-}
-
-func newTokenCache(node *Node, expiration time.Duration) *tokenCache {
-	tc := &tokenCache{}
-	tc.node = node
-	tc.tokens = make(map[string]tokenCacheEntry)
-	tc.expiration = expiration
-	tc.lock = &sync.RWMutex{}
-	return tc
-}
-
-// TODO: if store fails, get new token. can happen if a node restarts but we have the token cached
-
-func (tc *tokenCache) Get(c Contact, hash bits.Bitmap, cancelCh stop.Chan) string {
-	tc.lock.RLock()
-	token, exists := tc.tokens[c.String()]
-	tc.lock.RUnlock()
-
-	if exists && time.Since(token.receivedAt) < tc.expiration {
-		return token.token
-	}
-
-	resCh := tc.node.SendAsync(c, Request{
-		Method: findValueMethod,
-		Arg:    &hash,
-	})
-
-	var res *Response
-
-	select {
-	case res = <-resCh:
-	case <-cancelCh:
-		return ""
-	}
-
-	if res == nil {
-		return ""
-	}
-
-	tc.lock.Lock()
-	tc.tokens[c.String()] = tokenCacheEntry{
-		token:      res.Token,
-		receivedAt: time.Now(),
-	}
-	tc.lock.Unlock()
-
-	return res.Token
-}

dht/token_manager.go

@@ -1,78 +0,0 @@
-package dht
-
-import (
-	"bytes"
-	"crypto/rand"
-	"crypto/sha256"
-	"net"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/lbryio/lbry.go/stop"
-	"github.com/lbryio/reflector.go/dht/bits"
-)
-
-type tokenManager struct {
-	secret     []byte
-	prevSecret []byte
-	lock       *sync.RWMutex
-	stop       *stop.Group
-}
-
-func (tm *tokenManager) Start(interval time.Duration) {
-	tm.secret = make([]byte, 64)
-	tm.prevSecret = make([]byte, 64)
-	tm.lock = &sync.RWMutex{}
-	tm.stop = stop.New()
-
-	tm.rotateSecret()
-
-	tm.stop.Add(1)
-	go func() {
-		defer tm.stop.Done()
-		tick := time.NewTicker(interval)
-		for {
-			select {
-			case <-tick.C:
-				tm.rotateSecret()
-			case <-tm.stop.Ch():
-				return
-			}
-		}
-	}()
-}
-
-func (tm *tokenManager) Stop() {
-	tm.stop.StopAndWait()
-}
-
-func (tm *tokenManager) Get(nodeID bits.Bitmap, addr *net.UDPAddr) string {
-	return genToken(tm.secret, nodeID, addr)
-}
-
-func (tm *tokenManager) Verify(token string, nodeID bits.Bitmap, addr *net.UDPAddr) bool {
-	return token == genToken(tm.secret, nodeID, addr) || token == genToken(tm.prevSecret, nodeID, addr)
-}
-
-func genToken(secret []byte, nodeID bits.Bitmap, addr *net.UDPAddr) string {
-	buf := bytes.Buffer{}
-	buf.Write(nodeID[:])
-	buf.Write(addr.IP)
-	buf.WriteString(strconv.Itoa(addr.Port))
-	buf.Write(secret)
-	t := sha256.Sum256(buf.Bytes())
-	return string(t[:])
-}
-
-func (tm *tokenManager) rotateSecret() {
-	tm.lock.Lock()
-	defer tm.lock.Unlock()
-
-	copy(tm.prevSecret, tm.secret)
-
-	_, err := rand.Read(tm.secret)
-	if err != nil {
-		panic(err)
-	}
-}

go.mod

@@ -0,0 +1,124 @@
+module github.com/lbryio/reflector.go
+
+go 1.20
+
+replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
+
+require (
+	github.com/aws/aws-sdk-go v1.45.24
+	github.com/bluele/gcache v0.0.2
+	github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7
+	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
+	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
+	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
+	github.com/davecgh/go-spew v1.1.1
+	github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
+	github.com/gin-gonic/gin v1.9.1
+	github.com/go-sql-driver/mysql v1.7.1
+	github.com/gogo/protobuf v1.3.2
+	github.com/golang/protobuf v1.5.3
+	github.com/google/gops v0.3.28
+	github.com/gorilla/mux v1.8.0
+	github.com/hashicorp/serf v0.10.1
+	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
+	github.com/johntdyer/slackrus v0.0.0-20230315191314-80bc92dee4fc
+	github.com/karrick/godirwalk v1.17.0
+	github.com/lbryio/chainquery v1.9.1-0.20230515181855-2fcba3115cfe
+	github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629
+	github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
+	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
+	github.com/prometheus/client_golang v1.16.0
+	github.com/quic-go/quic-go v0.39.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cast v1.5.1
+	github.com/spf13/cobra v1.7.0
+	github.com/stretchr/testify v1.8.4
+	github.com/volatiletech/null/v8 v8.1.2
+	go.uber.org/atomic v1.11.0
+	golang.org/x/sync v0.4.0
+)
+
+require (
+	github.com/armon/go-metrics v0.4.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
+	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
+	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
+	github.com/bytedance/sonic v1.9.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/friendsofgo/errors v0.9.2 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.14.0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/gofrs/uuid v4.2.0+incompatible // indirect
+	github.com/google/btree v1.0.0 // indirect
+	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
+	github.com/gorilla/rpc v1.2.0 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-msgpack v0.5.3 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/memberlist v0.5.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/johntdyer/slack-go v0.0.0-20230314151037-c5bf334f9b6e // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/miekg/dns v1.1.41 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/quic-go/qtls-go1-20 v0.3.4 // indirect
+	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/slack-go/slack v0.12.1 // indirect
+	github.com/spf13/afero v1.9.3 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.15.0 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/volatiletech/inflect v0.0.1 // indirect
+	github.com/volatiletech/randomize v0.0.1 // indirect
+	github.com/volatiletech/strmangle v0.0.4 // indirect
+	go.uber.org/mock v0.3.0 // indirect
+	golang.org/x/arch v0.3.0 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,843 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-metrics v0.4.0 h1:yCQqn7dwca4ITXb+CbubHmedzaQYHhNhrEXLYUeEe8Q=
+github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go v1.45.24 h1:TZx/CizkmCQn8Rtsb11iLYutEQVGK5PK9wAhwouELBo=
+github.com/aws/aws-sdk-go v1.45.24/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
+github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
+github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7 h1:7gNKWnX6OF+ERiXVw4I9RsHhZ52aumXdFE07nEx5v20=
+github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7/go.mod h1:M/KA3XJG5PJaApPiv4gWNsgcSJquOQTqumZNLyYE0KM=
+github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
+github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
+github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
+github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
+github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
+github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
+github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
+github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
+github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
+github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
+github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
+github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
+github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
+github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY=
+github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
+github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
+github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
+github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db h1:oZ4U9IqO8NS+61OmGTBi8vopzqTRxwQeogyBHdrhjbc=
+github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db/go.mod h1:Pk7/9x6tyChFTkahDvLBQMlvdsWvfC+yU8HTT5VD314=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/friendsofgo/errors v0.9.2 h1:X6NYxef4efCBdwI7BgS820zFaN7Cphrmb+Pljdzjtgk=
+github.com/friendsofgo/errors v0.9.2/go.mod h1:yCvFW5AkDIL9qn7suHVLiI/gH228n7PC4Pn44IGoTOI=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
+github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
+github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
+github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
+github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
+github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0=
+github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark=
+github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
+github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
+github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
+github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
+github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM=
+github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
+github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
+github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
+github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
+github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/johntdyer/slack-go v0.0.0-20230314151037-c5bf334f9b6e h1:5tRmeUw/tXT/DvaoloWTWwlyrEZrKA7pnrz/X+g9s34=
+github.com/johntdyer/slack-go v0.0.0-20230314151037-c5bf334f9b6e/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
+github.com/johntdyer/slackrus v0.0.0-20230315191314-80bc92dee4fc h1:enUIjGI+ljPLV2X3Mu3noR0P3m2NaIFGRsp96J8RBio=
+github.com/johntdyer/slackrus v0.0.0-20230315191314-80bc92dee4fc/go.mod h1:EM3NFHkhmCX05s6UvxWSJ8h/3mluH4tF6bYr9FXF1Cg=
+github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
+github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
+github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/lbryio/chainquery v1.9.1-0.20230515181855-2fcba3115cfe h1:xZ3ma924JghoG6mo0xSNTDujy9hR0mczqIm+GGeeux0=
+github.com/lbryio/chainquery v1.9.1-0.20230515181855-2fcba3115cfe/go.mod h1:GfIqGzrg0GA0+Wb0dWRgAtATtUGsYRHIpTQEIoapkKU=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629 h1:klpHPQ5iERUhczdITuKUpYuUZrWDGWb3zlAv3qYgc+o=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629/go.mod h1:JTkXBAVK8iHNcYmffbLzQ7IFKd/+/oBQGIwiG53bbqw=
+github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
+github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
+github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 h1:IhL9D2QfDWhLNDQpZ3Uiiw0gZEUYeLBS6uDqOd59G5o=
+github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
+github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
+github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
+github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
+github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
+github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
+github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
+github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
+github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
+github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
+github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
+github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
+github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
+github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg=
+github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
+github.com/quic-go/quic-go v0.39.0 h1:AgP40iThFMY0bj8jGxROhw3S0FMGa8ryqsmi9tBH3So=
+github.com/quic-go/quic-go v0.39.0/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc=
+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/slack-go/slack v0.12.1 h1:X97b9g2hnITDtNsNe5GkGx6O2/Sz/uC20ejRZN6QxOw=
+github.com/slack-go/slack v0.12.1/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
+github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk=
+github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
+github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
+github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
+github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
+github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
+github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
+github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/volatiletech/inflect v0.0.1 h1:2a6FcMQyhmPZcLa+uet3VJ8gLn/9svWhJxJYwvE8KsU=
+github.com/volatiletech/inflect v0.0.1/go.mod h1:IBti31tG6phkHitLlr5j7shC5SOo//x0AjDzaJU1PLA=
+github.com/volatiletech/null/v8 v8.1.2 h1:kiTiX1PpwvuugKwfvUNX/SU/5A2KGZMXfGD0DUHdKEI=
+github.com/volatiletech/null/v8 v8.1.2/go.mod h1:98DbwNoKEpRrYtGjWFctievIfm4n4MxG0A6EBUcoS5g=
+github.com/volatiletech/randomize v0.0.1 h1:eE5yajattWqTB2/eN8df4dw+8jwAzBtbdo5sbWC4nMk=
+github.com/volatiletech/randomize v0.0.1/go.mod h1:GN3U0QYqfZ9FOJ67bzax1cqZ5q2xuj2mXrXBjWaRTlY=
+github.com/volatiletech/strmangle v0.0.1/go.mod h1:F6RA6IkB5vq0yTG4GQ0UsbbRcl3ni9P76i+JrTBKFFg=
+github.com/volatiletech/strmangle v0.0.4 h1:CxrEPhobZL/PCZOTDSH1aq7s4Kv76hQpRoTVVlUOim4=
+github.com/volatiletech/strmangle v0.0.4/go.mod h1:ycDvbDkjDvhC0NUU8w3fWwl5JEMTV56vTKXzR3GeR+0=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
+go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
+golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o=
+golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
+golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
+gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

internal/metrics/metrics.go

@@ -0,0 +1,339 @@
+package metrics
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"syscall"
+	"time"
+
+	ee "github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+)
+
+type Server struct {
+	srv  *http.Server
+	stop *stop.Stopper
+}
+
+func NewServer(address string, path string) *Server {
+	h := http.NewServeMux()
+	h.Handle(path, promhttp.Handler())
+	return &Server{
+		srv: &http.Server{
+			Addr:    address,
+			Handler: h,
+			//https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/
+			//https://blog.cloudflare.com/exposing-go-on-the-internet/
+			ReadTimeout:  5 * time.Second,
+			WriteTimeout: 10 * time.Second,
+			IdleTimeout:  120 * time.Second,
+		},
+		stop: stop.New(),
+	}
+}
+
+func (s *Server) Start() {
+	s.stop.Add(1)
+	go func() {
+		defer s.stop.Done()
+		err := s.srv.ListenAndServe()
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
+			log.Error(err)
+		}
+	}()
+}
+
+func (s *Server) Shutdown() {
+	_ = s.srv.Shutdown(context.Background())
+	s.stop.StopAndWait()
+}
+
+const (
+	ns             = "reflector"
+	subsystemCache = "cache"
+	subsystemITTT  = "ittt"
+
+	labelDirection = "direction"
+	labelErrorType = "error_type"
+
+	DirectionUpload   = "upload"   // to reflector
+	DirectionDownload = "download" // from reflector
+
+	LabelCacheType = "cache_type"
+	LabelComponent = "component"
+	LabelSource    = "source"
+
+	errConnReset         = "conn_reset"
+	errReadConnReset     = "read_conn_reset"
+	errWriteConnReset    = "write_conn_reset"
+	errReadConnTimedOut  = "read_conn_timed_out"
+	errNoNetworkActivity = "no_network_activity"
+	errWriteConnTimedOut = "write_conn_timed_out"
+	errWriteBrokenPipe   = "write_broken_pipe"
+	errEPipe             = "e_pipe"
+	errETimedout         = "e_timedout"
+	errIOTimeout         = "io_timeout"
+	errUnexpectedEOF     = "unexpected_eof"
+	errUnexpectedEOFStr  = "unexpected_eof_str"
+	errJSONSyntax        = "json_syntax"
+	errBlobTooBig        = "blob_too_big"
+	errInvalidPeerJSON   = "invalid_peer_json"
+	errInvalidPeerData   = "invalid_peer_data"
+	errRequestTooLarge   = "request_too_large"
+	errDeadlineExceeded  = "deadline_exceeded"
+	errHashMismatch      = "hash_mismatch"
+	errProtectedBlob     = "protected_blob"
+	errInvalidBlobHash   = "invalid_blob_hash"
+	errZeroByteBlob      = "zero_byte_blob"
+	errInvalidCharacter  = "invalid_character"
+	errBlobNotFound      = "blob_not_found"
+	errNoErr             = "no_error"
+	errQuicProto         = "quic_protocol_violation"
+	errOther             = "other"
+)
+
+var (
+	ErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "error_total",
+		Help:      "Total number of errors",
+	}, []string{labelDirection, labelErrorType})
+
+	BlobDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "blob_download_total",
+		Help:      "Total number of blobs downloaded from reflector",
+	})
+	PeerDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "peer_download_total",
+		Help:      "Total number of blobs downloaded from reflector through tcp protocol",
+	})
+	Http3DownloadCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "http3_blob_download_total",
+		Help:      "Total number of blobs downloaded from reflector through QUIC protocol",
+	})
+	HttpDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "http_blob_download_total",
+		Help:      "Total number of blobs downloaded from reflector through HTTP protocol",
+	})
+
+	CacheHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: subsystemCache,
+		Name:      "hit_total",
+		Help:      "Total number of blobs retrieved from the cache storage",
+	}, []string{LabelCacheType, LabelComponent})
+	ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: subsystemITTT,
+		Name:      "this_hit_total",
+		Help:      "Total number of blobs retrieved from the this storage",
+	})
+	ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: subsystemITTT,
+		Name:      "that_hit_total",
+		Help:      "Total number of blobs retrieved from the that storage",
+	})
+	CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: subsystemCache,
+		Name:      "miss_total",
+		Help:      "Total number of blobs retrieved from origin rather than cache storage",
+	}, []string{LabelCacheType, LabelComponent})
+	CacheOriginRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: ns,
+		Subsystem: subsystemCache,
+		Name:      "origin_requests_total",
+		Help:      "How many Get requests are in flight from the cache to the origin",
+	}, []string{LabelCacheType, LabelComponent})
+	//during thundering-herd situations, the metric below should be a lot smaller than the metric above
+	CacheWaitingRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: ns,
+		Subsystem: subsystemCache,
+		Name:      "waiting_requests_total",
+		Help:      "How many cache requests are waiting for an in-flight origin request",
+	}, []string{LabelCacheType, LabelComponent})
+	CacheLRUEvictCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: subsystemCache,
+		Name:      "evict_total",
+		Help:      "Count of blobs evicted from cache",
+	}, []string{LabelCacheType, LabelComponent})
+	CacheRetrievalSpeed = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: ns,
+		Name:      "speed_mbps",
+		Help:      "Speed of blob retrieval from cache or from origin",
+	}, []string{LabelCacheType, LabelComponent, LabelSource})
+
+	BlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "blob_upload_total",
+		Help:      "Total number of blobs uploaded to reflector",
+	})
+	SDBlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "sdblob_upload_total",
+		Help:      "Total number of SD blobs (and therefore streams) uploaded to reflector",
+	})
+
+	MtrInBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "tcp_in_bytes",
+		Help:      "Total number of bytes downloaded through TCP",
+	})
+	MtrOutBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "tcp_out_bytes",
+		Help:      "Total number of bytes streamed out through TCP",
+	})
+	MtrInBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "udp_in_bytes",
+		Help:      "Total number of bytes downloaded through UDP",
+	})
+	MtrInBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "http_in_bytes",
+		Help:      "Total number of bytes downloaded through HTTP",
+	})
+	MtrOutBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "udp_out_bytes",
+		Help:      "Total number of bytes streamed out through UDP",
+	})
+	MtrOutBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "http_out_bytes",
+		Help:      "Total number of bytes streamed out through UDP",
+	})
+	MtrInBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "reflector_in_bytes",
+		Help:      "Total number of incoming bytes (from users)",
+	})
+	MtrOutBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "s3_out_bytes",
+		Help:      "Total number of outgoing bytes (to S3)",
+	})
+	MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "s3_in_bytes",
+		Help:      "Total number of incoming bytes (from S3-CF)",
+	})
+	Http3BlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: ns,
+		Name:      "http3_blob_request_queue_size",
+		Help:      "Blob requests of https queue size",
+	})
+	HttpBlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: ns,
+		Name:      "http_blob_request_queue_size",
+		Help:      "Blob requests queue size of the HTTP protocol",
+	})
+	RoutinesQueue = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: ns,
+		Name:      "routines",
+		Help:      "routines running by type",
+	}, []string{"package", "kind"})
+)
+
+func CacheLabels(name, component string) prometheus.Labels {
+	return prometheus.Labels{
+		LabelCacheType: name,
+		LabelComponent: component,
+	}
+}
+
+func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
+	if e == nil {
+		return
+	}
+
+	err := ee.Wrap(e, 0)
+	errType := errOther
+	if strings.Contains(err.Error(), "i/o timeout") {
+		errType = errIOTimeout
+	} else if errors.Is(e, syscall.ECONNRESET) {
+		// Looks like we're getting this when direction == "download", but read_conn_reset and
+		// write_conn_reset when its "upload"
+		errType = errConnReset
+	} else if errors.Is(e, context.DeadlineExceeded) {
+		errType = errDeadlineExceeded
+	} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
+		errType = errReadConnReset
+	} else if strings.Contains(err.Error(), "write: connection reset by peer") { // the other side closed the connection using TCP reset
+		errType = errWriteConnReset
+	} else if errors.Is(e, syscall.ETIMEDOUT) {
+		errType = errETimedout
+	} else if strings.Contains(err.Error(), "read: connection timed out") { // the other side closed the connection using TCP reset
+		//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
+		errType = errReadConnTimedOut
+	} else if strings.Contains(err.Error(), "NO_ERROR: No recent network activity") { // the other side closed the QUIC connection
+		//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
+		errType = errNoNetworkActivity
+	} else if strings.Contains(err.Error(), "write: connection timed out") {
+		errType = errWriteConnTimedOut
+	} else if errors.Is(e, io.ErrUnexpectedEOF) {
+		errType = errUnexpectedEOF
+	} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
+		errType = errUnexpectedEOFStr
+	} else if errors.Is(e, syscall.EPIPE) {
+		errType = errEPipe
+	} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
+		// I believe this is the same as EPipe when direction == "download", but not for upload
+		errType = errWriteBrokenPipe
+		//} else if errors.Is(e, reflector.ErrBlobTooBig) { # this creates a circular import
+		//	errType = errBlobTooBig
+	} else if strings.Contains(err.Error(), "blob must be at most") {
+		//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
+		errType = errBlobTooBig
+	} else if strings.Contains(err.Error(), "invalid json request") {
+		errType = errInvalidPeerJSON
+	} else if strings.Contains(err.Error(), "Invalid data") {
+		errType = errInvalidPeerData
+	} else if strings.Contains(err.Error(), "request is too large") {
+		errType = errRequestTooLarge
+	} else if strings.Contains(err.Error(), "Invalid blob hash length") {
+		errType = errInvalidBlobHash
+	} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
+		errType = errHashMismatch
+	} else if strings.Contains(err.Error(), "blob not found") {
+		errType = errBlobNotFound
+	} else if strings.Contains(err.Error(), "requested blob is protected") {
+		errType = errProtectedBlob
+	} else if strings.Contains(err.Error(), "0-byte blob received") {
+		errType = errZeroByteBlob
+	} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
+		errType = errQuicProto
+	} else if strings.Contains(err.Error(), "invalid character") {
+		errType = errInvalidCharacter
+	} else if _, ok := e.(*json.SyntaxError); ok {
+		errType = errJSONSyntax
+	} else if strings.Contains(err.Error(), "NO_ERROR") {
+		errType = errNoErr
+	} else {
+		log.Warnf("error '%s' for direction '%s' is not being tracked", err.TypeName(), direction)
+		shouldLog = true
+	}
+
+	ErrorCount.With(map[string]string{
+		labelDirection: direction,
+		labelErrorType: errType,
+	}).Inc()
+
+	return
+}

lite_db/db.go

@@ -0,0 +1,348 @@
+package lite_db
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	qt "github.com/lbryio/lbry.go/v2/extras/query"
+
+	"github.com/go-sql-driver/mysql"
+	_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
+	log "github.com/sirupsen/logrus"
+	"github.com/volatiletech/null/v8"
+)
+
+// SdBlob is a special blob that contains information on the rest of the blobs in the stream
+type SdBlob struct {
+	StreamName string `json:"stream_name"`
+	Blobs      []struct {
+		Length   int    `json:"length"`
+		BlobNum  int    `json:"blob_num"`
+		BlobHash string `json:"blob_hash,omitempty"`
+		IV       string `json:"iv"`
+	} `json:"blobs"`
+	StreamType        string `json:"stream_type"`
+	Key               string `json:"key"`
+	SuggestedFileName string `json:"suggested_file_name"`
+	StreamHash        string `json:"stream_hash"`
+}
+
+// SQL implements the DB interface
+type SQL struct {
+	conn *sql.DB
+
+	TrackAccessTime bool
+}
+
+func logQuery(query string, args ...interface{}) {
+	s, err := qt.InterpolateParams(query, args...)
+	if err != nil {
+		log.Errorln(err)
+	} else {
+		log.Debugln(s)
+	}
+}
+
+// Connect will create a connection to the database
+func (s *SQL) Connect(dsn string) error {
+	var err error
+	// interpolateParams is necessary. otherwise uploading a stream with thousands of blobs
+	// will hit MySQL's max_prepared_stmt_count limit because the prepared statements are all
+	// opened inside a transaction. closing them manually doesn't seem to help
+	dsn += "?parseTime=1&collation=utf8mb4_unicode_ci&interpolateParams=1"
+	s.conn, err = sql.Open("mysql", dsn)
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	s.conn.SetMaxIdleConns(12)
+
+	return errors.Err(s.conn.Ping())
+}
+
+// AddBlob adds a blob to the database.
+func (s *SQL) AddBlob(hash string, length int) error {
+	if s.conn == nil {
+		return errors.Err("not connected")
+	}
+
+	_, err := s.insertBlob(hash, length)
+	return err
+}
+
+func (s *SQL) insertBlob(hash string, length int) (int64, error) {
+	if length <= 0 {
+		return 0, errors.Err("length must be positive")
+	}
+	const isStored = true
+	now := time.Now()
+	args := []interface{}{hash, isStored, length, now}
+	blobID, err := s.exec(
+		"INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at=VALUES(last_accessed_at)",
+		args...,
+	)
+	if err != nil {
+		return 0, err
+	}
+
+	if blobID == 0 {
+		err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
+		if err != nil {
+			return 0, errors.Err(err)
+		}
+		if blobID == 0 {
+			return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
+		}
+	}
+
+	return blobID, nil
+}
+
+// HasBlob checks if the database contains the blob information.
+func (s *SQL) HasBlob(hash string) (bool, error) {
+	exists, err := s.HasBlobs([]string{hash})
+	if err != nil {
+		return false, err
+	}
+	return exists[hash], nil
+}
+
+// HasBlobs checks if the database contains the set of blobs and returns a bool map.
+func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
+	exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
+	_ = s.touch(streamsNeedingTouch)
+	return exists, err
+}
+
+func (s *SQL) touch(blobIDs []uint64) error {
+	if len(blobIDs) == 0 {
+		return nil
+	}
+
+	query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
+	args := make([]interface{}, len(blobIDs)+1)
+	args[0] = time.Now()
+	for i := range blobIDs {
+		args[i+1] = blobIDs[i]
+	}
+
+	startTime := time.Now()
+	_, err := s.exec(query, args...)
+	log.Debugf("blobs access query touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
+	return errors.Err(err)
+}
+
+func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
+	if s.conn == nil {
+		return nil, nil, errors.Err("not connected")
+	}
+
+	var (
+		hash           string
+		blobID         uint64
+		lastAccessedAt null.Time
+	)
+
+	var needsTouch []uint64
+	exists := make(map[string]bool)
+
+	touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
+	maxBatchSize := 10000
+	doneIndex := 0
+
+	for len(hashes) > doneIndex {
+		sliceEnd := doneIndex + maxBatchSize
+		if sliceEnd > len(hashes) {
+			sliceEnd = len(hashes)
+		}
+		log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
+		batch := hashes[doneIndex:sliceEnd]
+
+		// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
+
+		query := `SELECT hash, id, last_accessed_at
+FROM blob_ 
+WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
+		args := make([]interface{}, len(batch)+1)
+		args[0] = true
+		for i := range batch {
+			args[i+1] = batch[i]
+		}
+
+		logQuery(query, args...)
+
+		err := func() error {
+			startTime := time.Now()
+			rows, err := s.conn.Query(query, args...)
+			log.Debugf("hashes query took %s", time.Since(startTime))
+			if err != nil {
+				return errors.Err(err)
+			}
+			defer closeRows(rows)
+
+			for rows.Next() {
+				err := rows.Scan(&hash, &blobID, &lastAccessedAt)
+				if err != nil {
+					return errors.Err(err)
+				}
+				exists[hash] = true
+				if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
+					needsTouch = append(needsTouch, blobID)
+				}
+			}
+
+			err = rows.Err()
+			if err != nil {
+				return errors.Err(err)
+			}
+
+			doneIndex += len(batch)
+			return nil
+		}()
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	return exists, needsTouch, nil
+}
+
+// Delete will remove the blob from the db
+func (s *SQL) Delete(hash string) error {
+	_, err := s.exec("UPDATE blob_ set is_stored = ? WHERE hash = ?", 0, hash)
+	return errors.Err(err)
+}
+
+// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
+// but they are tracked so reflector knows what it is missing.
+func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int) error {
+	if s.conn == nil {
+		return errors.Err("not connected")
+	}
+
+	_, err := s.insertBlob(sdHash, sdBlobLength)
+	return err
+}
+
+// GetHashRange gets the smallest and biggest hashes in the db
+func (s *SQL) GetLRUBlobs(maxBlobs int) ([]string, error) {
+	if s.conn == nil {
+		return nil, errors.Err("not connected")
+	}
+
+	query := "SELECT hash from blob_ where is_stored = ? order by last_accessed_at limit ?"
+	const isStored = true
+	logQuery(query, isStored, maxBlobs)
+	rows, err := s.conn.Query(query, isStored, maxBlobs)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer closeRows(rows)
+	blobs := make([]string, 0, maxBlobs)
+	for rows.Next() {
+		var hash string
+		err := rows.Scan(&hash)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		blobs = append(blobs, hash)
+	}
+	return blobs, nil
+}
+
+func (s *SQL) AllBlobs() ([]string, error) {
+	if s.conn == nil {
+		return nil, errors.Err("not connected")
+	}
+
+	query := "SELECT hash from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
+	const isStored = true
+	logQuery(query, isStored)
+	rows, err := s.conn.Query(query, isStored)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer closeRows(rows)
+	totalBlobs, err := s.BlobsCount()
+	if err != nil {
+		return nil, err
+	}
+	blobs := make([]string, 0, totalBlobs)
+	for rows.Next() {
+		var hash string
+		err := rows.Scan(&hash)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		blobs = append(blobs, hash)
+	}
+	return blobs, nil
+}
+
+func (s *SQL) BlobsCount() (int, error) {
+	if s.conn == nil {
+		return 0, errors.Err("not connected")
+	}
+
+	query := "SELECT count(id) from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
+	const isStored = true
+	logQuery(query, isStored)
+	var count int
+	err := s.conn.QueryRow(query, isStored).Scan(&count)
+	return count, errors.Err(err)
+}
+
+func closeRows(rows *sql.Rows) {
+	if rows != nil {
+		err := rows.Close()
+		if err != nil {
+			log.Error("error closing rows: ", err)
+		}
+	}
+}
+
+func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
+	logQuery(query, args...)
+	attempt, maxAttempts := 0, 3
+Retry:
+	attempt++
+	result, err := s.conn.Exec(query, args...)
+	if isLockTimeoutError(err) {
+		if attempt <= maxAttempts {
+			//Error 1205: Lock wait timeout exceeded; try restarting transaction
+			goto Retry
+		}
+		err = errors.Prefix("Lock timeout for query "+query, err)
+	}
+
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+
+	lastID, err := result.LastInsertId()
+	return lastID, errors.Err(err)
+}
+
+func isLockTimeoutError(err error) bool {
+	e, ok := err.(*mysql.MySQLError)
+	return ok && e != nil && e.Number == 1205
+}
+
+/*  SQL schema
+
+in prod make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
+
+CREATE TABLE `blob_` (
+  `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+  `hash` char(96) NOT NULL,
+  `is_stored` tinyint(1) NOT NULL DEFAULT '0',
+  `length` bigint unsigned DEFAULT NULL,
+  `last_accessed_at` datetime DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `id` (`id`),
+  UNIQUE KEY `blob_hash_idx` (`hash`),
+  KEY `blob_last_accessed_idx` (`last_accessed_at`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+
+*/

main.go

@@ -1,7 +1,19 @@
 package main
 
-import "github.com/lbryio/reflector.go/cmd"
+import (
+	"math/rand"
+	"time"
+
+	"github.com/lbryio/reflector.go/cmd"
+
+	"github.com/google/gops/agent"
+	log "github.com/sirupsen/logrus"
+)
 
 func main() {
+	if err := agent.Listen(agent.Options{}); err != nil {
+		log.Fatal(err)
+	}
+	rand.Seed(time.Now().UnixNano())
 	cmd.Execute()
 }

meta/meta.go

@@ -1,3 +1,58 @@
 package meta
 
-var Version = ""
+import (
+	"fmt"
+	"strconv"
+	"time"
+)
+
+var (
+	name       = "prism-bin"
+	version    = "unknown"
+	commit     = "unknown"
+	commitLong = "unknown"
+	branch     = "unknown"
+	Time       = "unknown"
+	BuildTime  time.Time
+)
+
+// Name returns main application name
+func Name() string {
+	return name
+}
+
+// Version returns current application version
+func Version() string {
+	return version
+}
+
+// FullName returns current app version, commit and build time
+func FullName() string {
+	return fmt.Sprintf(
+		`Name: %v
+Version: %v
+branch: %v
+commit: %v
+commit long: %v
+build date: %v`, Name(), Version(), branch, commit, commitLong, BuildTime.String())
+}
+
+func init() {
+	if Time != "" {
+		t, err := strconv.Atoi(Time)
+		if err == nil {
+			BuildTime = time.Unix(int64(t), 0).UTC()
+		}
+	}
+}
+
+func VersionString() string {
+	var buildTime string
+	if BuildTime.IsZero() {
+		buildTime = "<now>"
+	} else {
+		buildTime = BuildTime.Format(time.RFC3339)
+	}
+
+	return fmt.Sprintf("version %s, built %s", version, buildTime)
+}

peer/server.go

@@ -1,316 +0,0 @@
-package peer
-
-import (
-	"bufio"
-	"crypto/sha512"
-	"encoding/hex"
-	"encoding/json"
-	"io"
-	"net"
-	"strings"
-	"time"
-
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/stop"
-	"github.com/lbryio/reflector.go/store"
-
-	"github.com/davecgh/go-spew/spew"
-	log "github.com/sirupsen/logrus"
-)
-
-const (
-	// DefaultPort is the port the peer server listens on if not passed in.
-	DefaultPort = 3333
-	// LbrycrdAddress to be used when paying for data. Not implemented yet.
-	LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
-)
-
-// Server is an instance of a peer server that houses the listener and store.
-type Server struct {
-	store  store.BlobStore
-	closed bool
-
-	grp *stop.Group
-}
-
-// NewServer returns an initialized Server pointer.
-func NewServer(store store.BlobStore) *Server {
-	return &Server{
-		store: store,
-		grp:   stop.New(),
-	}
-}
-
-// Shutdown gracefully shuts down the peer server.
-func (s *Server) Shutdown() {
-	log.Debug("shutting down peer server...")
-	s.grp.StopAndWait()
-	log.Debug("peer server stopped")
-}
-
-// Start starts the server listener to handle connections.
-func (s *Server) Start(address string) error {
-	log.Println("peer listening on " + address)
-	l, err := net.Listen("tcp4", address)
-	if err != nil {
-		return err
-	}
-
-	go s.listenForShutdown(l)
-	s.grp.Add(1)
-	go func() {
-		s.listenAndServe(l)
-		s.grp.Done()
-	}()
-
-	return nil
-}
-
-func (s *Server) listenForShutdown(listener net.Listener) {
-	<-s.grp.Ch()
-	s.closed = true
-	err := listener.Close()
-	if err != nil {
-		log.Error("error closing listener for peer server - ", err)
-	}
-}
-
-func (s *Server) listenAndServe(listener net.Listener) {
-	for {
-		conn, err := listener.Accept()
-		if err != nil {
-			if s.closed {
-				return
-			}
-			log.Error(err)
-		} else {
-			s.grp.Add(1)
-			go func() {
-				s.handleConnection(conn)
-				s.grp.Done()
-			}()
-		}
-	}
-}
-
-func (s *Server) handleConnection(conn net.Conn) {
-	defer func() {
-		if err := conn.Close(); err != nil {
-			log.Error(errors.Prefix("closing peer conn", err))
-		}
-	}()
-
-	timeoutDuration := 5 * time.Second
-
-	for {
-		var request []byte
-		var response []byte
-
-		err := conn.SetReadDeadline(time.Now().Add(timeoutDuration))
-		if err != nil {
-			log.Error(errors.FullTrace(err))
-		}
-
-		request, err = readNextRequest(conn)
-		if err != nil {
-			if err != io.EOF {
-				log.Errorln(err)
-			}
-			return
-		}
-
-		err = conn.SetReadDeadline(time.Time{})
-		if err != nil {
-			log.Error(errors.FullTrace(err))
-		}
-
-		if strings.Contains(string(request), `"requested_blobs"`) {
-			log.Debugln("received availability request")
-			response, err = s.handleAvailabilityRequest(request)
-		} else if strings.Contains(string(request), `"blob_data_payment_rate"`) {
-			log.Debugln("received rate negotiation request")
-			response, err = s.handlePaymentRateNegotiation(request)
-		} else if strings.Contains(string(request), `"requested_blob"`) {
-			log.Debugln("received blob request")
-			response, err = s.handleBlobRequest(request)
-		} else {
-			log.Errorln("invalid request")
-			spew.Dump(request)
-			return
-		}
-		if err != nil {
-			log.Error(err)
-			return
-		}
-
-		n, err := conn.Write(response)
-		if err != nil {
-			log.Errorln(err)
-			return
-		} else if n != len(response) {
-			log.Errorln(io.ErrShortWrite)
-			return
-		}
-	}
-}
-
-func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
-	var request availabilityRequest
-	err := json.Unmarshal(data, &request)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	availableBlobs := []string{}
-	for _, blobHash := range request.RequestedBlobs {
-		exists, err := s.store.Has(blobHash)
-		if err != nil {
-			return []byte{}, err
-		}
-		if exists {
-			availableBlobs = append(availableBlobs, blobHash)
-		}
-	}
-
-	return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs})
-}
-
-func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
-	var request paymentRateRequest
-	err := json.Unmarshal(data, &request)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	offerReply := paymentRateAccepted
-	if request.BlobDataPaymentRate < 0 {
-		offerReply = paymentRateTooLow
-	}
-
-	return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
-}
-
-func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
-	var request blobRequest
-	err := json.Unmarshal(data, &request)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	log.Println("Sending blob " + request.RequestedBlob[:8])
-
-	blob, err := s.store.Get(request.RequestedBlob)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
-		BlobHash: GetBlobHash(blob),
-		Length:   len(blob),
-	}})
-	if err != nil {
-		return []byte{}, err
-	}
-
-	return append(response, blob...), nil
-}
-
-func readNextRequest(conn net.Conn) ([]byte, error) {
-	request := make([]byte, 0)
-	eof := false
-	buf := bufio.NewReader(conn)
-
-	for {
-		chunk, err := buf.ReadBytes('}')
-		if err != nil {
-			if err != io.EOF {
-				log.Errorln("read error:", err)
-				return request, err
-			}
-			eof = true
-		}
-
-		//log.Debugln("got", len(chunk), "bytes.")
-		//spew.Dump(chunk)
-
-		if len(chunk) > 0 {
-			request = append(request, chunk...)
-
-			if len(request) > maxRequestSize {
-				return request, errRequestTooLarge
-			}
-
-			// yes, this is how the peer protocol knows when the request finishes
-			if isValidJSON(request) {
-				break
-			}
-		}
-
-		if eof {
-			break
-		}
-	}
-
-	//log.Debugln("total size:", len(request))
-	//if len(request) > 0 {
-	//	spew.Dump(request)
-	//}
-
-	if len(request) == 0 && eof {
-		return []byte{}, io.EOF
-	}
-
-	return request, nil
-}
-
-func isValidJSON(b []byte) bool {
-	var r json.RawMessage
-	return json.Unmarshal(b, &r) == nil
-}
-
-// GetBlobHash returns the sha512 hash hex encoded string of the blob byte slice.
-func GetBlobHash(blob []byte) string {
-	hashBytes := sha512.Sum384(blob)
-	return hex.EncodeToString(hashBytes[:])
-}
-
-const (
-	maxRequestSize      = 64 * (2 ^ 10) // 64kb
-	paymentRateAccepted = "RATE_ACCEPTED"
-	paymentRateTooLow   = "RATE_TOO_LOW"
-	//ToDo: paymentRateUnset is not used but exists in the protocol.
-	//paymentRateUnset    = "RATE_UNSET"
-)
-
-var errRequestTooLarge = errors.Base("request is too large")
-
-type availabilityRequest struct {
-	LbrycrdAddress bool     `json:"lbrycrd_address"`
-	RequestedBlobs []string `json:"requested_blobs"`
-}
-
-type availabilityResponse struct {
-	LbrycrdAddress string   `json:"lbrycrd_address"`
-	AvailableBlobs []string `json:"available_blobs"`
-}
-
-type paymentRateRequest struct {
-	BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
-}
-
-type paymentRateResponse struct {
-	BlobDataPaymentRate string `json:"blob_data_payment_rate"`
-}
-
-type blobRequest struct {
-	RequestedBlob string `json:"requested_blob"`
-}
-
-type incomingBlob struct {
-	Error    string `json:"error,omitempty"`
-	BlobHash string `json:"blob_hash"`
-	Length   int    `json:"length"`
-}
-type blobResponse struct {
-	IncomingBlob incomingBlob `json:"incoming_blob"`
-}

prism/prism.go

@@ -7,14 +7,14 @@ import (
 
 	"github.com/lbryio/reflector.go/cluster"
 	"github.com/lbryio/reflector.go/db"
-	"github.com/lbryio/reflector.go/dht"
-	"github.com/lbryio/reflector.go/dht/bits"
-	"github.com/lbryio/reflector.go/peer"
 	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/server/peer"
 	"github.com/lbryio/reflector.go/store"
 
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/stop"
+	"github.com/lbryio/lbry.go/v2/dht"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -79,7 +79,7 @@ func New(conf *Config) *Prism {
 		dht:       d,
 		cluster:   c,
 		peer:      peer.NewServer(conf.Blobs),
-		reflector: reflector.NewServer(conf.Blobs),
+		reflector: reflector.NewServer(conf.Blobs, conf.Blobs),
 
 		grp: stop.New(),
 	}

prism/prism_test.go

@@ -4,8 +4,9 @@ import (
 	"math/big"
 	"testing"
 
+	"github.com/lbryio/lbry.go/v2/dht/bits"
+
 	"github.com/davecgh/go-spew/spew"
-	"github.com/lbryio/reflector.go/dht/bits"
 )
 
 func TestAnnounceRange(t *testing.T) {

publish/mimetypes.go

@@ -0,0 +1,174 @@
+package publish
+
+import "strings"
+
+func guessMimeType(ext string) (string, string) {
+	if ext == "" {
+		return "application/octet-stream", "binary"
+	}
+
+	ext = strings.ToLower(strings.TrimLeft(strings.TrimSpace(ext), "."))
+
+	types := map[string]struct{ mime, t string }{
+		"a":        {"application/octet-stream", "binary"},
+		"ai":       {"application/postscript", "image"},
+		"aif":      {"audio/x-aiff", "audio"},
+		"aifc":     {"audio/x-aiff", "audio"},
+		"aiff":     {"audio/x-aiff", "audio"},
+		"au":       {"audio/basic", "audio"},
+		"avi":      {"video/x-msvideo", "video"},
+		"bat":      {"text/plain", "document"},
+		"bcpio":    {"application/x-bcpio", "binary"},
+		"bin":      {"application/octet-stream", "binary"},
+		"bmp":      {"image/bmp", "image"},
+		"c":        {"text/plain", "document"},
+		"cdf":      {"application/x-netcdf", "binary"},
+		"cpio":     {"application/x-cpio", "binary"},
+		"csh":      {"application/x-csh", "binary"},
+		"css":      {"text/css", "document"},
+		"csv":      {"text/csv", "document"},
+		"dll":      {"application/octet-stream", "binary"},
+		"doc":      {"application/msword", "document"},
+		"dot":      {"application/msword", "document"},
+		"dvi":      {"application/x-dvi", "binary"},
+		"eml":      {"message/rfc822", "document"},
+		"eps":      {"application/postscript", "document"},
+		"epub":     {"application/epub+zip", "document"},
+		"etx":      {"text/x-setext", "document"},
+		"exe":      {"application/octet-stream", "binary"},
+		"gif":      {"image/gif", "image"},
+		"gtar":     {"application/x-gtar", "binary"},
+		"h":        {"text/plain", "document"},
+		"hdf":      {"application/x-hdf", "binary"},
+		"htm":      {"text/html", "document"},
+		"html":     {"text/html", "document"},
+		"ico":      {"image/vnd.microsoft.icon", "image"},
+		"ief":      {"image/ief", "image"},
+		"iges":     {"model/iges", "model"},
+		"jpe":      {"image/jpeg", "image"},
+		"jpeg":     {"image/jpeg", "image"},
+		"jpg":      {"image/jpeg", "image"},
+		"js":       {"application/javascript", "document"},
+		"json":     {"application/json", "document"},
+		"ksh":      {"text/plain", "document"},
+		"latex":    {"application/x-latex", "binary"},
+		"m1v":      {"video/mpeg", "video"},
+		"m3u":      {"application/vnd.apple.mpegurl", "audio"},
+		"m3u8":     {"application/vnd.apple.mpegurl", "audio"},
+		"man":      {"application/x-troff-man", "document"},
+		"markdown": {"text/markdown", "document"},
+		"md":       {"text/markdown", "document"},
+		"me":       {"application/x-troff-me", "binary"},
+		"mht":      {"message/rfc822", "document"},
+		"mhtml":    {"message/rfc822", "document"},
+		"mif":      {"application/x-mif", "binary"},
+		"mov":      {"video/quicktime", "video"},
+		"movie":    {"video/x-sgi-movie", "video"},
+		"mp2":      {"audio/mpeg", "audio"},
+		"mp3":      {"audio/mpeg", "audio"},
+		"mp4":      {"video/mp4", "video"},
+		"mpa":      {"video/mpeg", "video"},
+		"mpe":      {"video/mpeg", "video"},
+		"mpeg":     {"video/mpeg", "video"},
+		"mpg":      {"video/mpeg", "video"},
+		"ms":       {"application/x-troff-ms", "binary"},
+		"nc":       {"application/x-netcdf", "binary"},
+		"nws":      {"message/rfc822", "document"},
+		"o":        {"application/octet-stream", "binary"},
+		"obj":      {"application/octet-stream", "model"},
+		"oda":      {"application/oda", "binary"},
+		"p12":      {"application/x-pkcs12", "binary"},
+		"p7c":      {"application/pkcs7-mime", "binary"},
+		"pbm":      {"image/x-portable-bitmap", "image"},
+		"pdf":      {"application/pdf", "document"},
+		"pfx":      {"application/x-pkcs12", "binary"},
+		"pgm":      {"image/x-portable-graymap", "image"},
+		"pl":       {"text/plain", "document"},
+		"png":      {"image/png", "image"},
+		"pnm":      {"image/x-portable-anymap", "image"},
+		"pot":      {"application/vnd.ms-powerpoint", "document"},
+		"ppa":      {"application/vnd.ms-powerpoint", "document"},
+		"ppm":      {"image/x-portable-pixmap", "image"},
+		"pps":      {"application/vnd.ms-powerpoint", "document"},
+		"ppt":      {"application/vnd.ms-powerpoint", "document"},
+		"ps":       {"application/postscript", "document"},
+		"pwz":      {"application/vnd.ms-powerpoint", "document"},
+		"py":       {"text/x-python", "document"},
+		"pyc":      {"application/x-python-code", "binary"},
+		"pyo":      {"application/x-python-code", "binary"},
+		"qt":       {"video/quicktime", "video"},
+		"ra":       {"audio/x-pn-realaudio", "audio"},
+		"ram":      {"application/x-pn-realaudio", "audio"},
+		"ras":      {"image/x-cmu-raster", "image"},
+		"rdf":      {"application/xml", "binary"},
+		"rgb":      {"image/x-rgb", "image"},
+		"roff":     {"application/x-troff", "binary"},
+		"rtx":      {"text/richtext", "document"},
+		"sgm":      {"text/x-sgml", "document"},
+		"sgml":     {"text/x-sgml", "document"},
+		"sh":       {"application/x-sh", "document"},
+		"shar":     {"application/x-shar", "binary"},
+		"snd":      {"audio/basic", "audio"},
+		"so":       {"application/octet-stream", "binary"},
+		"src":      {"application/x-wais-source", "binary"},
+		"stl":      {"model/stl", "model"},
+		"sv4cpio":  {"application/x-sv4cpio", "binary"},
+		"sv4crc":   {"application/x-sv4crc", "binary"},
+		"svg":      {"image/svg+xml", "image"},
+		"swf":      {"application/x-shockwave-flash", "binary"},
+		"t":        {"application/x-troff", "binary"},
+		"tar":      {"application/x-tar", "binary"},
+		"tcl":      {"application/x-tcl", "binary"},
+		"tex":      {"application/x-tex", "binary"},
+		"texi":     {"application/x-texinfo", "binary"},
+		"texinfo":  {"application/x-texinfo", "binary"},
+		"tif":      {"image/tiff", "image"},
+		"tiff":     {"image/tiff", "image"},
+		"tr":       {"application/x-troff", "binary"},
+		"tsv":      {"text/tab-separated-values", "document"},
+		"txt":      {"text/plain", "document"},
+		"ustar":    {"application/x-ustar", "binary"},
+		"vcf":      {"text/x-vcard", "document"},
+		"wav":      {"audio/x-wav", "audio"},
+		"webm":     {"video/webm", "video"},
+		"wiz":      {"application/msword", "document"},
+		"wsdl":     {"application/xml", "document"},
+		"xbm":      {"image/x-xbitmap", "image"},
+		"xlb":      {"application/vnd.ms-excel", "document"},
+		"xls":      {"application/vnd.ms-excel", "document"},
+		"xml":      {"text/xml", "document"},
+		"xpdl":     {"application/xml", "document"},
+		"xpm":      {"image/x-xpixmap", "image"},
+		"xsl":      {"application/xml", "document"},
+		"xwd":      {"image/x-xwindowdump", "image"},
+		"zip":      {"application/zip", "binary"},
+
+		// These are non-standard types, commonly found in the wild.
+		"cbr":  {"application/vnd.comicbook-rar", "document"},
+		"cbz":  {"application/vnd.comicbook+zip", "document"},
+		"flac": {"audio/flac", "audio"},
+		"lbry": {"application/x-ext-lbry", "document"},
+		"m4v":  {"video/m4v", "video"},
+		"mid":  {"audio/midi", "audio"},
+		"midi": {"audio/midi", "audio"},
+		"mkv":  {"video/x-matroska", "video"},
+		"mobi": {"application/x-mobipocket-ebook", "document"},
+		"oga":  {"audio/ogg", "audio"},
+		"ogv":  {"video/ogg", "video"},
+		"pct":  {"image/pict", "image"},
+		"pic":  {"image/pict", "image"},
+		"pict": {"image/pict", "image"},
+		"prc":  {"application/x-mobipocket-ebook", "document"},
+		"rtf":  {"application/rtf", "document"},
+		"xul":  {"text/xul", "document"},
+
+		// microsoft is special and has its own "standard"
+		// https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
+		"wmv": {"video/x-ms-wmv", "video"},
+	}
+
+	if data, ok := types[ext]; ok {
+		return data.mime, data.t
+	}
+	return "application/x-ext-" + ext, "binary"
+}

publish/publish.go

@@ -0,0 +1,291 @@
+package publish
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/lbryio/reflector.go/reflector"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/lbrycrd"
+	"github.com/lbryio/lbry.go/v2/stream"
+	pb "github.com/lbryio/types/v2/go"
+
+	"github.com/btcsuite/btcd/btcjson"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/txscript"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/golang/protobuf/proto"
+)
+
+/* TODO:
+import cert from wallet
+get all utxos from chainquery
+create transaction
+sign it with the channel
+track state of utxos across publishes from this channel so that we can just do one query to get utxos
+prioritize only confirmed utxos
+
+Handling all the issues we handle currently with lbrynet:
+	"Couldn't find private key for id",
+	"You already have a stream claim published under the name",
+	"Cannot publish using channel",
+	"txn-mempool-conflict",
+	"too-long-mempool-chain",
+	"Missing inputs",
+	"Not enough funds to cover this transaction",
+*/
+
+type Details struct {
+	Title       string
+	Description string
+	Author      string
+	Tags        []string
+	ReleaseTime int64
+}
+
+func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) {
+	if name == "" {
+		return nil, nil, errors.Err("name required")
+	}
+
+	//TODO: sign claim if publishing into channel
+
+	addr, err := btcutil.DecodeAddress(address, &lbrycrd.MainNetParams)
+	if errors.Is(err, btcutil.ErrUnknownAddressType) {
+		return nil, nil, errors.Err(`unknown address type. here's what you need to make this work:
+- deprecatedrpc=validateaddress" and "deprecatedrpc=signrawtransaction" in your lbrycrd.conf
+- github.com/btcsuite/btcd pinned to hash 306aecffea32
+- github.com/btcsuite/btcutil pinned to 4c204d697803
+- github.com/lbryio/lbry.go/v2 (make sure you have v2 at the end)`)
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	amount := 0.01
+	changeAddr := addr // TODO: fix this? or maybe its fine?
+	tx, err := baseTx(client, amount, changeAddr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	st, stPB, err := makeStream(path)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	stPB.Author = details.Author
+	stPB.ReleaseTime = details.ReleaseTime
+
+	claim := &pb.Claim{
+		Title:       details.Title,
+		Description: details.Description,
+		Type:        &pb.Claim_Stream{Stream: stPB},
+	}
+
+	err = addClaimToTx(tx, claim, name, amount, addr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// sign and send
+	signedTx, allInputsSigned, err := client.SignRawTransaction(tx)
+	if err != nil {
+		return nil, nil, err
+	}
+	if !allInputsSigned {
+		return nil, nil, errors.Err("not all inputs for the tx could be signed")
+	}
+
+	err = reflect(st, reflectorAddress)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	txid, err := client.SendRawTransaction(signedTx, false)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return signedTx, txid, nil
+}
+
+// TODO: lots of assumptions. hardcoded values need to be passed in or calculated
+func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
+	txFee := 0.0002 // TODO: estimate this better?
+
+	inputs, total, err := coinChooser(client, amount+txFee)
+	if err != nil {
+		return nil, err
+	}
+
+	change := total - amount - txFee
+
+	// create base raw tx
+	addresses := make(map[btcutil.Address]btcutil.Amount)
+	//changeAddr, err := client.GetNewAddress("")
+	changeAmount, err := btcutil.NewAmount(change)
+	if err != nil {
+		return nil, err
+	}
+	addresses[changeAddress] = changeAmount
+	lockTime := int64(0)
+	return client.CreateRawTransaction(inputs, addresses, &lockTime)
+}
+
+func coinChooser(client *lbrycrd.Client, amount float64) ([]btcjson.TransactionInput, float64, error) {
+	utxos, err := client.ListUnspentMin(1)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	sort.Slice(utxos, func(i, j int) bool { return utxos[i].Amount < utxos[j].Amount })
+
+	var utxo btcjson.ListUnspentResult
+	for _, u := range utxos {
+		if u.Spendable && u.Amount >= amount {
+			utxo = u
+			break
+		}
+	}
+	if utxo.TxID == "" {
+		return nil, 0, errors.Err("not enough utxos to create tx")
+	}
+
+	return []btcjson.TransactionInput{{Txid: utxo.TxID, Vout: utxo.Vout}}, utxo.Amount, nil
+}
+
+func addClaimToTx(tx *wire.MsgTx, claim *pb.Claim, name string, amount float64, claimAddress btcutil.Address) error {
+	claimBytes, err := proto.Marshal(claim)
+	if err != nil {
+		return err
+	}
+	claimBytes = append([]byte{0}, claimBytes...) // version 0 = no channel sig
+
+	amt, err := btcutil.NewAmount(amount)
+	if err != nil {
+		return err
+	}
+
+	script, err := getClaimPayoutScript(name, claimBytes, claimAddress)
+	if err != nil {
+		return err
+	}
+
+	tx.AddTxOut(wire.NewTxOut(int64(amt), script))
+	return nil
+}
+
+func Decode(client *lbrycrd.Client, tx *wire.MsgTx) (string, error) {
+	buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
+	if err := tx.Serialize(buf); err != nil {
+		return "", errors.Err(err)
+	}
+	//txHex := hex.EncodeToString(buf.Bytes())
+	//spew.Dump(txHex)
+	decoded, err := client.DecodeRawTransaction(buf.Bytes())
+	if err != nil {
+		return "", err
+	}
+
+	data, err := json.MarshalIndent(decoded, "", "  ")
+	return string(data), err
+}
+
+func reflect(st stream.Stream, reflectorAddress string) error {
+	// upload blobs to reflector
+	c := reflector.Client{}
+	err := c.Connect(reflectorAddress)
+	if err != nil {
+		return errors.Err(err)
+	}
+	for i, b := range st {
+		if i == 0 {
+			err = c.SendSDBlob(b)
+		} else {
+			err = c.SendBlob(b)
+		}
+		if err != nil {
+			return errors.Err(err)
+		}
+	}
+	return nil
+}
+
+func makeStream(path string) (stream.Stream, *pb.Stream, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, nil, errors.Err(err)
+	}
+	defer func() { _ = file.Close() }()
+	enc := stream.NewEncoder(file)
+
+	s, err := enc.Stream()
+	if err != nil {
+		return nil, nil, errors.Err(err)
+	}
+
+	streamProto := &pb.Stream{
+		Source: &pb.Source{
+			SdHash: enc.SDBlob().Hash(),
+			Name:   filepath.Base(file.Name()),
+			Size:   uint64(enc.SourceLen()),
+			Hash:   enc.SourceHash(),
+		},
+	}
+
+	mimeType, category := guessMimeType(filepath.Ext(file.Name()))
+	streamProto.Source.MediaType = mimeType
+
+	switch category {
+	case "video":
+		//t, err := streamVideoMetadata(path)
+		//if err != nil {
+		//	return nil, nil, err
+		//}
+		streamProto.Type = &pb.Stream_Video{}
+	case "audio":
+		streamProto.Type = &pb.Stream_Audio{}
+	case "image":
+		streamProto.Type = &pb.Stream_Image{}
+	}
+
+	return s, streamProto, nil
+}
+
+func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) {
+	//OP_CLAIM_NAME <name> <value> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
+
+	pkscript, err := txscript.PayToAddrScript(address)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	return txscript.NewScriptBuilder().
+		AddOp(txscript.OP_NOP6).  //OP_CLAIM_NAME
+		AddData([]byte(name)).    //<name>
+		AddData(value).           //<value>
+		AddOp(txscript.OP_2DROP). //OP_2DROP
+		AddOp(txscript.OP_DROP).  //OP_DROP
+		AddOps(pkscript).         //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
+		Script()
+}
+
+//func streamVideoMetadata(path string) (*pb.Stream_Video, error) {
+//	mi, err := mediainfo.GetMediaInfo(path)
+//	if err != nil {
+//		return nil, err
+//	}
+//	return &pb.Stream_Video{
+//		Video: &pb.Video{
+//			Duration: uint32(mi.General.Duration / 1000),
+//			Height:   uint32(mi.Video.Height),
+//			Width:    uint32(mi.Video.Width),
+//		},
+//	}, nil
+//}

publish/wallet.go

@@ -0,0 +1,59 @@
+package publish
+
+import (
+	"encoding/json"
+	"io"
+)
+
+func LoadWallet(r io.Reader) (WalletFile, error) {
+	var w WalletFile
+	err := json.NewDecoder(r).Decode(&w)
+	return w, err
+}
+
+type WalletFile struct {
+	Name        string      `json:"name"`
+	Version     int         `json:"version"`
+	Preferences WalletPrefs `json:"preferences"`
+	Accounts    []Account   `json:"accounts"`
+}
+
+type Account struct {
+	AddressGenerator AddressGenerator  `json:"address_generator"`
+	Certificates     map[string]string `json:"certificates"`
+	Encrypted        bool              `json:"encrypted"`
+	Ledger           string            `json:"ledger"`
+	ModifiedOn       float64           `json:"modified_on"`
+	Name             string            `json:"name"`
+	PrivateKey       string            `json:"private_key"`
+	PublicKey        string            `json:"public_key"`
+	Seed             string            `json:"seed"`
+}
+
+type AddressGenerator struct {
+	Name      string           `json:"name"`
+	Change    AddressGenParams `json:"change"` // should "change" and "receiving" be replaced with a map[string]AddressGenParams?
+	Receiving AddressGenParams `json:"receiving"`
+}
+
+type AddressGenParams struct {
+	Gap                   int `json:"gap"`
+	MaximumUsesPerAddress int `json:"maximum_uses_per_address"`
+}
+
+type WalletPrefs struct {
+	Shared struct {
+		Ts    float64 `json:"ts"`
+		Value struct {
+			Type  string `json:"type"`
+			Value struct {
+				AppWelcomeVersion int           `json:"app_welcome_version"`
+				Blocked           []interface{} `json:"blocked"`
+				Sharing3P         bool          `json:"sharing_3P"`
+				Subscriptions     []string      `json:"subscriptions"`
+				Tags              []string      `json:"tags"`
+			} `json:"value"`
+			Version string `json:"version"`
+		} `json:"value"`
+	} `json:"shared"`
+}

readme.md

@@ -1,25 +1,110 @@
 # Reflector
 
-A reflector cluster to accept LBRY content for hosting en masse, rehost the content, and make money on data fees (TODO). 
-This code includes Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
+Reflector is a central piece of software that providers LBRY with the following features:
+- Blobs reflection: when something is published, we capture the data and store it on our servers for quicker retrieval
+- Blobs distribution: when a piece of content is requested and the LBRY network doesn't have it, reflector will retrieve it from its storage and distribute it
+- Blobs caching: reflectors can be chained together in multiple regions or servers to form a chain of cached content. We call those "blobcaches". They are layered so that content distribution is favorable in all the regions we deploy it to
+
+There are a few other features embedded in reflector.go including publishing streams from Go, downloading or upload blobs, resolving content and more unfinished tools.
+
+This code includes a Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
 
 ## Installation
 
-coming soon
+- Install mysql 8 (5.7 might work too)
+- add a reflector user and database with password `reflector` with localhost access only
+- Create the tables as described [here](https://github.com/lbryio/reflector.go/blob/master/db/db.go#L735) (the link might not update as the code does so just look for the schema in that file)
+
+#### We do not support running reflector.go as a blob receiver, however if you want to run it as a private blobcache you may compile it yourself and run it as following:
+```bash
+./prism-bin reflector \
+--conf="none" \
+--disable-uploads=true \
+--use-db=false \
+--upstream-reflector="reflector.lbry.com" \
+--upstream-protocol="http" \
+--request-queue-size=200 \
+--disk-cache="2GB:/path/to/your/storage/:localdb" \
+```
+
+Create a systemd script if you want to run it automatically on startup or as a service.
 
 ## Usage
 
-coming soon
+Usage as reflector/blobcache:
+```bash
+Run reflector server
 
+Usage:
+  prism reflector [flags]
+
+Flags:
+      --disable-blocklist                 Disable blocklist watching/updating
+      --disable-uploads                   Disable uploads to this reflector server
+      --disk-cache string                 Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (default "100GB:/tmp/downloaded_blobs:localdb")
+  -h, --help                              help for reflector
+      --http-peer-port int                The port reflector will distribute content from over HTTP protocol (default 5569)
+      --http3-peer-port int               The port reflector will distribute content from over HTTP3 protocol (default 5568)
+      --mem-cache int                     enable in-memory cache with a max size of this many blobs
+      --metrics-port int                  The port reflector will use for prometheus metrics (default 2112)
+      --optional-disk-cache string        Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (this would get hit before the one specified in disk-cache)
+      --origin-endpoint string            HTTP edge endpoint for standard HTTP retrieval
+      --origin-endpoint-fallback string   HTTP edge endpoint for standard HTTP retrieval if first origin fails
+      --receiver-port int                 The port reflector will receive content from (default 5566)
+      --request-queue-size int            How many concurrent requests from downstream should be handled at once (the rest will wait) (default 200)
+      --tcp-peer-port int                 The port reflector will distribute content from for the TCP (LBRY) protocol (default 5567)
+      --upstream-protocol string          protocol used to fetch blobs from another upstream reflector server (tcp/http3/http) (default "http")
+      --upstream-reflector string         host:port of a reflector server where blobs are fetched from
+      --use-db                            Whether to connect to the reflector db or not (default true)
+
+Global Flags:
+      --conf string       Path to config. Use 'none' to disable (default "config.json")
+  -v, --verbose strings   Verbose logging for specific components
+```
+
+Other uses:
+
+```bash
+Prism is a single entry point application with multiple sub modules which can be leveraged individually or together
+
+Usage:
+  prism [command]
+
+Available Commands:
+  check-integrity check blobs integrity for a given path
+  cluster         Start(join) to or Start a new cluster
+  decode          Decode a claim value
+  dht             Run dht node
+  getstream       Get a stream from a reflector server
+  help            Help about any command
+  peer            Run peer server
+  populate-db     populate local database with blobs from a disk storage
+  publish         Publish a file
+  reflector       Run reflector server
+  resolve         Resolve a URL
+  send            Send a file to a reflector
+  sendblob        Send a random blob to a reflector server
+  start           Runs full prism application with cluster, dht, peer server, and reflector server.
+  test            Test things
+  upload          Upload blobs to S3
+  version         Print the version
+
+Flags:
+      --conf string       Path to config. Use 'none' to disable (default "config.json")
+  -h, --help              help for prism
+  -v, --verbose strings   Verbose logging for specific components
+```
 ## Running from Source
 
-This project requires [Go v1.10](https://golang.org/doc/install) or higher.
+This project requires [Go v1.20](https://golang.org/doc/install).
+
+On Ubuntu you can install it with `sudo snap install go --classic`
 
 ```
-go get -u github.com/lbryio/reflector.go
-cd "$(go env GOPATH)/src/github.com/lbryio/reflector.go"
+git clone git@github.com:lbryio/reflector.go.git
+cd reflector.go
 make
-./prism
+./dist/linux_amd64/prism-bin
 ```
 
 ## Contributing
@@ -32,9 +117,8 @@ This project is MIT licensed.
 
 ## Security
 
-We take security seriously. Please contact security@lbry.io regarding any security issues.
-Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
+We take security seriously. Please contact security@lbry.com regarding any security issues.
+Our PGP key is [here](https://lbry.com/faq/pgp-key) if you need it.
 
 ## Contact
-
-The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.io)
+The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777) (niko-at-lbry.com)

reflector/blocklist.go

@@ -0,0 +1,159 @@
+package reflector
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/store"
+	"github.com/lbryio/reflector.go/wallet"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const blocklistURL = "https://api.lbry.com/file/list_blocked"
+
+func (s *Server) enableBlocklist(b store.Blocklister) {
+	walletServers := []string{
+		"spv25.lbry.com:50001",
+		"spv26.lbry.com:50001",
+		"spv19.lbry.com:50001",
+		"spv14.lbry.com:50001",
+	}
+
+	updateBlocklist(b, walletServers, s.grp.Ch())
+	t := time.NewTicker(12 * time.Hour)
+	for {
+		select {
+		case <-s.grp.Ch():
+			return
+		case <-t.C:
+			updateBlocklist(b, walletServers, s.grp.Ch())
+		}
+	}
+}
+
+func updateBlocklist(b store.Blocklister, walletServers []string, stopper stop.Chan) {
+	log.Debugf("blocklist update starting")
+	values, err := blockedSdHashes(walletServers, stopper)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	for name, v := range values {
+		if v.Err != nil {
+			log.Error(errors.FullTrace(errors.Err("blocklist: %s: %s", name, v.Err)))
+			continue
+		}
+
+		err = b.Block(v.Value)
+		if err != nil {
+			log.Error(err)
+		}
+	}
+	log.Debugf("blocklist update done")
+}
+
+func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valOrErr, error) {
+	client := http.Client{Timeout: 1 * time.Second}
+	resp, err := client.Get(blocklistURL)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer func() {
+		err := resp.Body.Close()
+		if err != nil {
+			log.Errorln(errors.Err(err))
+		}
+	}()
+
+	var r struct {
+		Success bool   `json:"success"`
+		Error   string `json:"error"`
+		Data    struct {
+			Outpoints []string `json:"outpoints"`
+		} `json:"data"`
+	}
+
+	if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		return nil, errors.Err(err)
+	}
+
+	if !r.Success {
+		return nil, errors.Prefix("list_blocked API call", r.Error)
+	}
+
+	return sdHashesForOutpoints(walletServers, r.Data.Outpoints, stopper)
+}
+
+type valOrErr struct {
+	Value string
+	Err   error
+}
+
+// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
+func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan) (map[string]valOrErr, error) {
+	values := make(map[string]valOrErr)
+
+	node := wallet.NewNode()
+	err := node.Connect(walletServers, nil)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	done := make(chan bool)
+	metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Inc()
+	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Dec()
+		select {
+		case <-done:
+		case <-stopper:
+		}
+		node.Shutdown()
+	}()
+
+OutpointLoop:
+	for _, outpoint := range outpoints {
+		select {
+		case <-stopper:
+			break OutpointLoop
+		default:
+		}
+
+		parts := strings.Split(outpoint, ":")
+		if len(parts) != 2 {
+			values[outpoint] = valOrErr{Err: errors.Err("invalid outpoint format")}
+			continue
+		}
+
+		nout, err := strconv.Atoi(parts[1])
+		if err != nil {
+			values[outpoint] = valOrErr{Err: errors.Prefix("invalid nout", err)}
+			continue
+		}
+
+		claim, err := node.GetClaimInTx(parts[0], nout)
+		if err != nil {
+			values[outpoint] = valOrErr{Err: err}
+			continue
+		}
+
+		hash := hex.EncodeToString(claim.GetStream().GetSource().GetSdHash())
+		values[outpoint] = valOrErr{Value: hash, Err: nil}
+	}
+
+	select {
+	case done <- true:
+	default: // in case of race where stopper got stopped right after loop finished
+	}
+
+	return values, nil
+}

reflector/client.go

@@ -2,11 +2,11 @@ package reflector
 
 import (
 	"encoding/json"
+	"log"
 	"net"
-	"strconv"
 
-	"github.com/lbryio/lbry.go/errors"
-	log "github.com/sirupsen/logrus"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
 )
 
 // ErrBlobExists is a default error for when a blob already exists on the reflector server.
@@ -35,23 +35,36 @@ func (c *Client) Close() error {
 	return c.conn.Close()
 }
 
-// SendBlob sends a send blob request to the client.
-func (c *Client) SendBlob(blob []byte) error {
+// SendBlob sends a blob to the server.
+func (c *Client) SendBlob(blob stream.Blob) error {
+	return c.sendBlob(blob, false)
+}
+
+// SendSDBlob sends an SD blob request to the server.
+func (c *Client) SendSDBlob(blob stream.Blob) error {
+	return c.sendBlob(blob, true)
+}
+
+// sendBlob does the actual blob sending
+func (c *Client) sendBlob(blob stream.Blob, isSDBlob bool) error {
 	if !c.connected {
 		return errors.Err("not connected")
 	}
 
-	if len(blob) > maxBlobSize {
-		return errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
-	} else if len(blob) == 0 {
-		return errors.Err("blob is empty")
+	if err := blob.ValidForSend(); err != nil {
+		return errors.Err(err)
 	}
 
-	blobHash := BlobHash(blob)
-	sendRequest, err := json.Marshal(sendBlobRequest{
-		BlobSize: len(blob),
-		BlobHash: blobHash,
-	})
+	blobHash := blob.HashHex()
+	var req sendBlobRequest
+	if isSDBlob {
+		req.SdBlobSize = blob.Size()
+		req.SdBlobHash = blobHash
+	} else {
+		req.BlobSize = blob.Size()
+		req.BlobHash = blobHash
+	}
+	sendRequest, err := json.Marshal(req)
 	if err != nil {
 		return err
 	}
@@ -63,30 +76,51 @@ func (c *Client) SendBlob(blob []byte) error {
 
 	dec := json.NewDecoder(c.conn)
 
-	var sendResp sendBlobResponse
-	err = dec.Decode(&sendResp)
-	if err != nil {
-		return err
+	if isSDBlob {
+		var sendResp sendSdBlobResponse
+		err = dec.Decode(&sendResp)
+		if err != nil {
+			return err
+		}
+		if !sendResp.SendSdBlob {
+			return errors.Prefix(blobHash[:8], ErrBlobExists)
+		}
+		log.Println("Sending SD blob " + blobHash[:8])
+	} else {
+		var sendResp sendBlobResponse
+		err = dec.Decode(&sendResp)
+		if err != nil {
+			return err
+		}
+		if !sendResp.SendBlob {
+			return errors.Prefix(blobHash[:8], ErrBlobExists)
+		}
+		log.Println("Sending blob " + blobHash[:8])
 	}
 
-	if !sendResp.SendBlob {
-		return ErrBlobExists
-	}
-
-	log.Println("Sending blob " + blobHash[:8])
-
 	_, err = c.conn.Write(blob)
 	if err != nil {
 		return err
 	}
-	var transferResp blobTransferResponse
-	err = dec.Decode(&transferResp)
-	if err != nil {
-		return err
-	}
 
-	if !transferResp.ReceivedBlob {
-		return errors.Err("server did not received blob")
+	if isSDBlob {
+		var transferResp sdBlobTransferResponse
+		err = dec.Decode(&transferResp)
+		if err != nil {
+			return err
+		}
+		if !transferResp.ReceivedSdBlob {
+			return errors.Err("server did not received SD blob")
+		}
+	} else {
+		var transferResp blobTransferResponse
+		err = dec.Decode(&transferResp)
+		if err != nil {
+			return err
+		}
+		if !transferResp.ReceivedBlob {
+			return errors.Err("server did not received blob")
+		}
 	}
 
 	return nil
@@ -97,7 +131,7 @@ func (c *Client) doHandshake(version int) error {
 		return errors.Err("not connected")
 	}
 
-	handshake, err := json.Marshal(handshakeRequestResponse{Version: version})
+	handshake, err := json.Marshal(handshakeRequestResponse{Version: &version})
 	if err != nil {
 		return err
 	}
@@ -111,7 +145,9 @@ func (c *Client) doHandshake(version int) error {
 	err = json.NewDecoder(c.conn).Decode(&resp)
 	if err != nil {
 		return err
-	} else if resp.Version != version {
+	} else if resp.Version == nil {
+		return errors.Err("invalid handshake")
+	} else if *resp.Version != version {
 		return errors.Err("handshake version mismatch")
 	}
 

reflector/protected_content.go

@@ -0,0 +1,81 @@
+package reflector
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/bluele/gcache"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"golang.org/x/sync/singleflight"
+)
+
+const protectedListURL = "https://api.odysee.com/file/list_protected"
+
+type ProtectedContent struct {
+	SDHash  string `json:"sd_hash"`
+	ClaimID string `json:"claim_id"`
+}
+
+var protectedCache = gcache.New(10).Expiration(2 * time.Minute).Build()
+
+func GetProtectedContent() (interface{}, error) {
+	cachedVal, err := protectedCache.Get("protected")
+	if err == nil && cachedVal != nil {
+		return cachedVal.(map[string]bool), nil
+	}
+
+	method := "GET"
+	var r struct {
+		Success bool               `json:"success"`
+		Error   string             `json:"error"`
+		Data    []ProtectedContent `json:"data"`
+	}
+
+	client := &http.Client{}
+	req, err := http.NewRequest(method, protectedListURL, nil)
+
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer func() { _ = res.Body.Close() }()
+	if res.StatusCode != http.StatusOK {
+		return nil, errors.Err("unexpected status code %d", res.StatusCode)
+	}
+	if err = json.NewDecoder(res.Body).Decode(&r); err != nil {
+		return nil, errors.Err(err)
+	}
+
+	if !r.Success {
+		return nil, errors.Prefix("file/list_protected API call", r.Error)
+	}
+
+	protectedMap := make(map[string]bool, len(r.Data))
+	for _, pc := range r.Data {
+		protectedMap[pc.SDHash] = true
+	}
+	err = protectedCache.Set("protected", protectedMap)
+	if err != nil {
+		return protectedMap, errors.Err(err)
+	}
+	return protectedMap, nil
+}
+
+var sf = singleflight.Group{}
+
+func IsProtected(sdHash string) bool {
+	val, err, _ := sf.Do("protected", GetProtectedContent)
+	if err != nil {
+		return false
+	}
+	cachedMap, ok := val.(map[string]bool)
+	if !ok {
+		return false
+	}
+
+	return cachedMap[sdHash]
+}

reflector/server.go

@@ -7,13 +7,14 @@ import (
 	"encoding/json"
 	"io"
 	"net"
-	"strconv"
 	"time"
 
+	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/store"
 
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/stop"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/stream"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -27,27 +28,29 @@ const (
 	network          = "tcp4"
 	protocolVersion1 = 0
 	protocolVersion2 = 1
-	maxBlobSize      = 2 * 1024 * 1024
+	maxBlobSize      = stream.MaxBlobSize
 )
 
+var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
+
 // Server is and instance of the reflector server. It houses the blob store and listener.
 type Server struct {
 	Timeout time.Duration // timeout to read or write next message
 
-	StatLogger          *log.Logger   // logger to log stats
-	StatReportFrequency time.Duration // how often to log stats
+	EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
 
-	store store.BlobStore
-	grp   *stop.Group
-	stats *stats
+	underlyingStore store.BlobStore
+	outerStore      store.BlobStore
+	grp             *stop.Group
 }
 
 // NewServer returns an initialized reflector server pointer.
-func NewServer(store store.BlobStore) *Server {
+func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server {
 	return &Server{
-		Timeout: DefaultTimeout,
-		store:   store,
-		grp:     stop.New(),
+		Timeout:         DefaultTimeout,
+		underlyingStore: underlying,
+		outerStore:      outer,
+		grp:             stop.New(),
 	}
 }
 
@@ -58,16 +61,17 @@ func (s *Server) Shutdown() {
 	log.Println("reflector server stopped")
 }
 
-//Start starts the server to handle connections.
+// Start starts the server to handle connections.
 func (s *Server) Start(address string) error {
 	l, err := net.Listen(network, address)
 	if err != nil {
 		return errors.Err(err)
 	}
 	log.Println("reflector listening on " + address)
-
 	s.grp.Add(1)
+	metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Inc()
 	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Dec()
 		<-s.grp.Ch()
 		err := l.Close()
 		if err != nil {
@@ -77,14 +81,26 @@ func (s *Server) Start(address string) error {
 	}()
 
 	s.grp.Add(1)
+	metrics.RoutinesQueue.WithLabelValues("reflector", "start").Inc()
 	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "start").Dec()
 		s.listenAndServe(l)
 		s.grp.Done()
 	}()
 
-	s.stats = newStatLogger(s.StatLogger, s.StatReportFrequency, s.grp.Child())
-	if s.StatLogger != nil && s.StatReportFrequency > 0 {
-		s.stats.Start()
+	if s.EnableBlocklist {
+		if b, ok := s.underlyingStore.(store.Blocklister); ok {
+			s.grp.Add(1)
+			metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
+			go func() {
+				defer metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Dec()
+				s.enableBlocklist(b)
+				s.grp.Done()
+			}()
+		} else {
+			//s.Shutdown()
+			return errors.Err("blocklist is enabled but blob store does not support blocklisting")
+		}
 	}
 
 	return nil
@@ -100,7 +116,9 @@ func (s *Server) listenAndServe(listener net.Listener) {
 			log.Error(err)
 		} else {
 			s.grp.Add(1)
+			metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
 			go func() {
+				defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Dec()
 				s.handleConn(conn)
 				s.grp.Done()
 			}()
@@ -115,7 +133,9 @@ func (s *Server) handleConn(conn net.Conn) {
 		close(connNeedsClosing)
 	}()
 	s.grp.Add(1)
+	metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Inc()
 	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Dec()
 		defer s.grp.Done()
 		select {
 		case <-connNeedsClosing:
@@ -155,8 +175,13 @@ func (s *Server) handleConn(conn net.Conn) {
 }
 
 func (s *Server) doError(conn net.Conn, err error) error {
-	log.Errorln(errors.FullTrace(err))
-	s.stats.AddError(err)
+	if err == nil {
+		return nil
+	}
+	shouldLog := metrics.TrackError(metrics.DirectionUpload, err)
+	if shouldLog {
+		log.Errorln(errors.FullTrace(err))
+	}
 	if e2, ok := err.(*json.SyntaxError); ok {
 		log.Errorf("syntax error at byte offset %d", e2.Offset)
 	}
@@ -169,39 +194,46 @@ func (s *Server) doError(conn net.Conn, err error) error {
 }
 
 func (s *Server) receiveBlob(conn net.Conn) error {
-	var err error
-
 	blobSize, blobHash, isSdBlob, err := s.readBlobRequest(conn)
 	if err != nil {
 		return err
 	}
 
-	blobExists, err := s.store.Has(blobHash)
-	if err != nil {
-		return err
+	var wantsBlob bool
+	if bl, ok := s.underlyingStore.(store.Blocklister); ok {
+		wantsBlob, err = bl.Wants(blobHash)
+		if err != nil {
+			return err
+		}
+	} else {
+		blobExists, err := s.underlyingStore.Has(blobHash)
+		if err != nil {
+			return err
+		}
+		wantsBlob = !blobExists
 	}
 
 	var neededBlobs []string
 
-	if isSdBlob && blobExists {
-		if fsc, ok := s.store.(neededBlobChecker); ok {
-			neededBlobs, err = fsc.MissingBlobsForKnownStream(blobHash)
+	if isSdBlob && !wantsBlob {
+		if nbc, ok := s.underlyingStore.(neededBlobChecker); ok {
+			neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
 			if err != nil {
 				return err
 			}
 		} else {
-			// if we can't confirm that we have the full stream, we have to say that the sd blob is
-			// missing. if we say we have it, they wont try to send any content blobs
-			blobExists = false
+			// if we can't check for blobs in a stream, we have to say that the sd blob is
+			// missing. if we say we have the sd blob, they wont try to send any content blobs
+			wantsBlob = true
 		}
 	}
 
-	err = s.sendBlobResponse(conn, blobExists, isSdBlob, neededBlobs)
+	err = s.sendBlobResponse(conn, wantsBlob, isSdBlob, neededBlobs)
 	if err != nil {
 		return err
 	}
 
-	if blobExists {
+	if !wantsBlob {
 		return nil
 	}
 
@@ -227,17 +259,17 @@ func (s *Server) receiveBlob(conn net.Conn) error {
 	log.Debugln("Got blob " + blobHash[:8])
 
 	if isSdBlob {
-		err = s.store.PutSD(blobHash, blob)
+		err = s.outerStore.PutSD(blobHash, blob)
 	} else {
-		err = s.store.Put(blobHash, blob)
+		err = s.outerStore.Put(blobHash, blob)
 	}
 	if err != nil {
 		return err
 	}
-
-	s.stats.AddBlob()
+	metrics.MtrInBytesReflector.Add(float64(len(blob)))
+	metrics.BlobUploadCount.Inc()
 	if isSdBlob {
-		s.stats.AddStream()
+		metrics.SDBlobUploadCount.Inc()
 	}
 	return s.sendTransferResponse(conn, true, isSdBlob)
 }
@@ -247,7 +279,9 @@ func (s *Server) doHandshake(conn net.Conn) error {
 	err := s.read(conn, &handshake)
 	if err != nil {
 		return err
-	} else if handshake.Version != protocolVersion1 && handshake.Version != protocolVersion2 {
+	} else if handshake.Version == nil {
+		return errors.Err("handshake is missing protocol version")
+	} else if *handshake.Version != protocolVersion1 && *handshake.Version != protocolVersion2 {
 		return errors.Err("protocol version not supported")
 	}
 
@@ -282,7 +316,7 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
 		return blobSize, blobHash, isSdBlob, errors.Err("blob hash is empty")
 	}
 	if blobSize > maxBlobSize {
-		return blobSize, blobHash, isSdBlob, errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
+		return blobSize, blobHash, isSdBlob, errors.Err(ErrBlobTooBig)
 	}
 	if blobSize == 0 {
 		return blobSize, blobHash, isSdBlob, errors.Err("0-byte blob received")
@@ -291,14 +325,14 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
 	return blobSize, blobHash, isSdBlob, nil
 }
 
-func (s *Server) sendBlobResponse(conn net.Conn, blobExists, isSdBlob bool, neededBlobs []string) error {
+func (s *Server) sendBlobResponse(conn net.Conn, shouldSendBlob, isSdBlob bool, neededBlobs []string) error {
 	var response []byte
 	var err error
 
 	if isSdBlob {
-		response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: !blobExists, NeededBlobs: neededBlobs})
+		response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: shouldSendBlob, NeededBlobs: neededBlobs})
 	} else {
-		response, err = json.Marshal(sendBlobResponse{SendBlob: !blobExists})
+		response, err = json.Marshal(sendBlobResponse{SendBlob: shouldSendBlob})
 	}
 	if err != nil {
 		return err
@@ -329,7 +363,16 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
 		return errors.Err(err)
 	}
 
-	return errors.Err(json.NewDecoder(conn).Decode(v))
+	dec := json.NewDecoder(conn)
+	err = dec.Decode(v)
+	if err != nil {
+		data, _ := io.ReadAll(dec.Buffered())
+		if len(data) > 0 {
+			return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
+		}
+		return errors.Err(err)
+	}
+	return nil
 }
 
 func (s *Server) readRawBlob(conn net.Conn, blobSize int) ([]byte, error) {
@@ -365,17 +408,23 @@ func (s *Server) quitting() bool {
 	}
 }
 
+// BlobHash returns the sha512 hash hex encoded string of the blob byte slice.
 func BlobHash(blob []byte) string {
 	hashBytes := sha512.Sum384(blob)
 	return hex.EncodeToString(hashBytes[:])
 }
 
+func IsValidJSON(b []byte) bool {
+	var r json.RawMessage
+	return json.Unmarshal(b, &r) == nil
+}
+
 //type errorResponse struct {
 //	Error string `json:"error"`
 //}
 
 type handshakeRequestResponse struct {
-	Version int `json:"version"`
+	Version *int `json:"version"`
 }
 
 type sendBlobRequest struct {

reflector/server_test.go

@@ -2,19 +2,18 @@ package reflector
 
 import (
 	"crypto/rand"
+	"encoding/json"
 	"io"
+	"sort"
 	"strconv"
 	"testing"
 	"time"
 
-	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/reflector.go/store"
 
-	"encoding/json"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 
-	"sort"
-
-	"github.com/lbryio/reflector.go/dht/bits"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/phayes/freeport"
 )
 
@@ -24,7 +23,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
 		t.Fatal(err)
 	}
 
-	srv := NewServer(&store.MemoryBlobStore{})
+	srv := NewServer(store.NewMemStore(), store.NewMemStore())
 	err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
 	if err != nil {
 		t.Fatal(err)
@@ -121,7 +120,7 @@ func TestServer_Timeout(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	srv := NewServer(&store.MemoryBlobStore{})
+	srv := NewServer(store.NewMemStore(), store.NewMemStore())
 	srv.Timeout = testTimeout
 	err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
 	if err != nil {
@@ -146,8 +145,24 @@ func TestServer_Timeout(t *testing.T) {
 	}
 }
 
+//func TestServer_InvalidJSONHandshake(t *testing.T) {
+//	srv, port := startServerOnRandomPort(t)
+//	defer srv.Shutdown()
+//
+//	c := Client{}
+//	err := c.Connect(":" + strconv.Itoa(port))
+//	if err != nil {
+//		t.Fatal("error connecting client to server", err)
+//	}
+//
+//	_, err = c.conn.Write([]byte(`{"stuff":4,tf}"`))
+//	if err == nil {
+//		t.Error("expected an error")
+//	}
+//}
+
 type mockPartialStore struct {
-	store.MemoryBlobStore
+	*store.MemStore
 	missing []string
 }
 
@@ -167,7 +182,7 @@ func TestServer_PartialUpload(t *testing.T) {
 		missing[i] = bits.Rand().String()
 	}
 
-	st := store.BlobStore(&mockPartialStore{missing: missing})
+	st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(), missing: missing})
 	if _, ok := st.(neededBlobChecker); !ok {
 		t.Fatal("mock does not implement the relevant interface")
 	}
@@ -176,7 +191,7 @@ func TestServer_PartialUpload(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	srv := NewServer(st)
+	srv := NewServer(st, st)
 	err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
 	if err != nil {
 		t.Fatal(err)
@@ -226,18 +241,6 @@ func TestServer_PartialUpload(t *testing.T) {
 	}
 }
 
-//func MakeRandStream(size int) ([]byte, [][]byte) {
-//	blobs := make([][]byte, int(math.Ceil(float64(size)/maxBlobSize)))
-//	for i := 0; i < len(blobs); i++ {
-//		blobs[i] = randBlob(int(math.Min(maxBlobSize, float64(size))))
-//		size -= maxBlobSize
-//	}
-//
-//  //TODO: create SD blob for the stream
-//
-//	return nil, blobs
-//}
-
 func randBlob(size int) []byte {
 	//if size > maxBlobSize {
 	//	panic("blob size too big")

reflector/stats.go

@@ -1,93 +0,0 @@
-package reflector
-
-import (
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/lbryio/lbry.go/errors"
-	"github.com/lbryio/lbry.go/stop"
-
-	log "github.com/sirupsen/logrus"
-)
-
-type stats struct {
-	mu      *sync.Mutex
-	blobs   int
-	streams int
-	errors  map[string]int
-
-	logger  *log.Logger
-	logFreq time.Duration
-	grp     *stop.Group
-}
-
-func newStatLogger(logger *log.Logger, logFreq time.Duration, parentGrp *stop.Group) *stats {
-	return &stats{
-		mu:      &sync.Mutex{},
-		grp:     stop.New(parentGrp),
-		logger:  logger,
-		logFreq: logFreq,
-		errors:  make(map[string]int),
-	}
-}
-
-func (s *stats) Start() {
-	s.grp.Add(1)
-	go func() {
-		defer s.grp.Done()
-		s.runSlackLogger()
-	}()
-}
-
-func (s *stats) Shutdown() {
-	s.grp.StopAndWait()
-}
-
-func (s *stats) AddBlob() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.blobs += 1
-}
-func (s *stats) AddStream() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.streams += 1
-}
-func (s *stats) AddError(e error) {
-	if e == nil {
-		return
-	}
-	err := errors.Wrap(e, 0)
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.errors[err.TypeName()] += 1
-}
-
-func (s *stats) runSlackLogger() {
-	t := time.NewTicker(s.logFreq)
-	for {
-		select {
-		case <-s.grp.Ch():
-			return
-		case <-t.C:
-			s.log()
-		}
-	}
-}
-
-func (s *stats) log() {
-	s.mu.Lock()
-	blobs, streams := s.blobs, s.streams
-	s.blobs, s.streams = 0, 0
-	errStr := ""
-	for name, count := range s.errors {
-		errStr += fmt.Sprintf("%d %s, ", count, name)
-		delete(s.errors, name)
-	}
-	s.mu.Unlock()
-	s.logger.Printf(
-		"Stats: %d blobs, %d streams, errors: %s",
-		blobs, streams, errStr[:len(errStr)-2], // trim last comma and space
-	)
-}

reflector/uploader.go

@@ -0,0 +1,270 @@
+package reflector
+
+import (
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/lbryio/reflector.go/db"
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type increment int
+
+const (
+	sdInc increment = iota + 1
+	blobInc
+	errInc
+)
+
+type Summary struct {
+	Total, AlreadyStored, Sd, Blob, Err int
+}
+
+type Uploader struct {
+	db                     *db.SQL
+	store                  *store.DBBackedStore // could just be store.BlobStore interface
+	workers                int
+	skipExistsCheck        bool
+	deleteBlobsAfterUpload bool
+	stopper                *stop.Group
+	countChan              chan increment
+
+	count Summary
+}
+
+func NewUploader(db *db.SQL, store *store.DBBackedStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
+	return &Uploader{
+		db:                     db,
+		store:                  store,
+		workers:                workers,
+		skipExistsCheck:        skipExistsCheck,
+		deleteBlobsAfterUpload: deleteBlobsAfterUpload,
+		stopper:                stop.New(),
+		countChan:              make(chan increment),
+	}
+}
+
+func (u *Uploader) Stop() {
+	log.Infoln("stopping uploader")
+	u.stopper.StopAndWait()
+}
+
+func (u *Uploader) Upload(dirOrFilePath string) error {
+	paths, err := getPaths(dirOrFilePath)
+	if err != nil {
+		return err
+	}
+
+	u.count.Total = len(paths)
+
+	hashes := make([]string, len(paths))
+	for i, p := range paths {
+		hashes[i] = path.Base(p)
+	}
+
+	log.Debug("checking for existing blobs")
+
+	var exists map[string]bool
+	if !u.skipExistsCheck {
+		exists, err = u.db.HasBlobs(hashes, false)
+		if err != nil {
+			return err
+		}
+		u.count.AlreadyStored = len(exists)
+	}
+
+	log.Debugf("%d new blobs to upload", u.count.Total-u.count.AlreadyStored)
+
+	workerWG := sync.WaitGroup{}
+	pathChan := make(chan string)
+
+	for i := 0; i < u.workers; i++ {
+		workerWG.Add(1)
+		metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Inc()
+		go func(i int) {
+			defer metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Dec()
+			defer workerWG.Done()
+			defer func(i int) { log.Debugf("worker %d quitting", i) }(i)
+			u.worker(pathChan)
+		}(i)
+	}
+
+	countWG := sync.WaitGroup{}
+	countWG.Add(1)
+	metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Inc()
+	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Dec()
+		defer countWG.Done()
+		u.counter()
+	}()
+
+Upload:
+	for _, f := range paths {
+		if exists != nil && exists[path.Base(f)] {
+			continue
+		}
+
+		select {
+		case pathChan <- f:
+		case <-u.stopper.Ch():
+			break Upload
+		}
+	}
+
+	close(pathChan)
+	workerWG.Wait()
+	close(u.countChan)
+	countWG.Wait()
+	u.stopper.Stop()
+
+	log.Debugf(
+		"upload stats: %d blobs total, %d already stored, %d SD blobs uploaded, %d content blobs uploaded, %d errors",
+		u.count.Total, u.count.AlreadyStored, u.count.Sd, u.count.Blob, u.count.Err,
+	)
+	return nil
+}
+
+// worker reads paths from a channel,  uploads them, and optionally deletes them
+func (u *Uploader) worker(pathChan chan string) {
+	for {
+		select {
+		case <-u.stopper.Ch():
+			return
+		case filepath, ok := <-pathChan:
+			if !ok {
+				return
+			}
+
+			err := u.uploadBlob(filepath)
+			if err != nil {
+				log.Errorln(err)
+			} else if u.deleteBlobsAfterUpload {
+				err = os.Remove(filepath)
+				if err != nil {
+					log.Errorln(errors.Prefix("deleting blob", err))
+				}
+			}
+		}
+	}
+}
+
+// uploadBlob uploads a blob
+func (u *Uploader) uploadBlob(filepath string) (err error) {
+	defer func() {
+		if err != nil {
+			u.inc(errInc)
+		}
+	}()
+
+	blob, err := os.ReadFile(filepath)
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	hash := BlobHash(blob)
+	if hash != path.Base(filepath) {
+		return errors.Err("file name does not match hash (%s != %s), skipping", filepath, hash)
+	}
+
+	if IsValidJSON(blob) {
+		log.Debugf("uploading SD blob %s", hash)
+		err := u.store.PutSD(hash, blob)
+		if err != nil {
+			return errors.Prefix("uploading SD blob "+hash, err)
+		}
+		u.inc(sdInc)
+	} else {
+		log.Debugf("uploading blob %s", hash)
+		err = u.store.Put(hash, blob)
+		if err != nil {
+			return errors.Prefix("uploading blob "+hash, err)
+		}
+		u.inc(blobInc)
+	}
+
+	return nil
+}
+
+// counter updates the counts of how many sd blobs and content blobs were uploaded, and how many
+// errors were encountered. It occasionally prints the upload progress to debug.
+func (u *Uploader) counter() {
+	start := time.Now()
+
+	for {
+		select {
+		case <-u.stopper.Ch():
+			return
+		case incrementType, ok := <-u.countChan:
+			if !ok {
+				return
+			}
+			switch incrementType {
+			case sdInc:
+				u.count.Sd++
+			case blobInc:
+				u.count.Blob++
+			case errInc:
+				u.count.Err++
+			}
+		}
+		if (u.count.Sd+u.count.Blob)%50 == 0 {
+			log.Debugf("%d of %d done (%s elapsed, %.3fs per blob)", u.count.Sd+u.count.Blob, u.count.Total-u.count.AlreadyStored, time.Since(start).String(), time.Since(start).Seconds()/float64(u.count.Sd+u.count.Blob))
+		}
+	}
+}
+
+func (u *Uploader) GetSummary() Summary {
+	return u.count
+}
+
+func (u *Uploader) inc(t increment) {
+	select {
+	case u.countChan <- t:
+	case <-u.stopper.Ch():
+	}
+}
+
+// getPaths returns the paths for files to upload. it takes a path to a file or a dir. for a file,
+// it returns the full path to that file. for a dir, it returns the paths for all the files in the
+// dir
+func getPaths(dirOrFilePath string) ([]string, error) {
+	info, err := os.Stat(dirOrFilePath)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	if info.Mode().IsRegular() {
+		return []string{dirOrFilePath}, nil
+	}
+
+	f, err := os.Open(dirOrFilePath)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	files, err := f.Readdir(-1)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	err = f.Close()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	var filenames []string
+	for _, file := range files {
+		if !file.IsDir() {
+			filenames = append(filenames, dirOrFilePath+"/"+file.Name())
+		}
+	}
+
+	return filenames, nil
+}

scripts/lint.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+err=0
+trap 'err=1' ERR
+# All the .go files, excluding auto generated folders
+GO_FILES=$(find . -iname '*.go' -type f)
+(
+	go install golang.org/x/tools/cmd/goimports@latest                   # Used in build script for generated files
+#	go install golang.org/x/lint/golint@latest                           # Linter
+	go install github.com/jgautheron/gocyclo@latest                      # Check against high complexity
+	go install github.com/mdempsky/unconvert@latest                      # Identifies unnecessary type conversions
+	go install github.com/kisielk/errcheck@latest                        # Checks for unhandled errors
+	go install github.com/opennota/check/cmd/varcheck@latest             # Checks for unused vars
+	go install github.com/opennota/check/cmd/structcheck@latest          # Checks for unused fields in structs
+)
+echo "Running varcheck..." && varcheck $(go list ./...)
+echo "Running structcheck..." && structcheck $(go list ./...)
+# go vet is the official Go static analyzer
+echo "Running go vet..." && go vet $(go list ./...)
+# checks for unhandled errors
+echo "Running errcheck..." && errcheck $(go list ./...)
+# check for unnecessary conversions - ignore autogen code
+echo "Running unconvert..." && unconvert -v $(go list ./...)
+echo "Running gocyclo..." && gocyclo -ignore "_test" -avg -over 28 $GO_FILES
+#echo "Running golint..." && golint -set_exit_status $(go list ./...)
+test $err = 0 # Return non-zero if any command failed

server/http/routes.go

@@ -0,0 +1,105 @@
+package http
+
+import (
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/shared"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+func (s *Server) getBlob(c *gin.Context) {
+	waiter := &sync.WaitGroup{}
+	waiter.Add(1)
+	enqueue(&blobRequest{c: c, finished: waiter})
+	waiter.Wait()
+}
+
+func (s *Server) HandleGetBlob(c *gin.Context) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Errorf("Recovered from panic: %v", r)
+		}
+	}()
+	start := time.Now()
+	hash := c.Query("hash")
+	edgeToken := c.Query("edge_token")
+
+	if reflector.IsProtected(hash) && edgeToken != s.edgeToken {
+		_ = c.Error(errors.Err("requested blob is protected"))
+		c.String(http.StatusForbidden, "requested blob is protected")
+		return
+	}
+	if s.missesCache.Has(hash) {
+		serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
+		c.Header("Via", serialized)
+		if err != nil {
+			_ = c.Error(errors.Err(err))
+			c.String(http.StatusInternalServerError, err.Error())
+			return
+		}
+		c.AbortWithStatus(http.StatusNotFound)
+		return
+	}
+	blob, trace, err := s.store.Get(hash)
+	if err != nil {
+		serialized, serializeErr := trace.Serialize()
+		if serializeErr != nil {
+			_ = c.Error(errors.Prefix(serializeErr.Error(), err))
+			c.String(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err).Error())
+			return
+		}
+		c.Header("Via", serialized)
+
+		if errors.Is(err, store.ErrBlobNotFound) {
+			_ = s.missesCache.Set(hash, true)
+			c.AbortWithStatus(http.StatusNotFound)
+			return
+		}
+		_ = c.Error(err)
+		c.String(http.StatusInternalServerError, err.Error())
+		return
+	}
+	serialized, err := trace.Serialize()
+	if err != nil {
+		_ = c.Error(err)
+		c.String(http.StatusInternalServerError, err.Error())
+		return
+	}
+	metrics.MtrOutBytesHttp.Add(float64(len(blob)))
+	metrics.BlobDownloadCount.Inc()
+	metrics.HttpDownloadCount.Inc()
+	c.Header("Via", serialized)
+	c.Header("Content-Disposition", "filename="+hash)
+	c.Data(http.StatusOK, "application/octet-stream", blob)
+}
+
+func (s *Server) hasBlob(c *gin.Context) {
+	hash := c.Query("hash")
+	has, err := s.store.Has(hash)
+	if err != nil {
+		_ = c.Error(err)
+		c.String(http.StatusInternalServerError, err.Error())
+		return
+	}
+	if has {
+		c.Status(http.StatusNoContent)
+		return
+	}
+	c.Status(http.StatusNotFound)
+}
+
+func (s *Server) recoveryHandler(c *gin.Context, err interface{}) {
+	c.JSON(500, gin.H{
+		"title": "Error",
+		"err":   err,
+	})
+}

server/http/server.go

@@ -0,0 +1,82 @@
+package http
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+
+	"github.com/bluele/gcache"
+	nice "github.com/ekyoung/gin-nice-recovery"
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+// Server is an instance of a peer server that houses the listener and store.
+type Server struct {
+	store              store.BlobStore
+	grp                *stop.Group
+	concurrentRequests int
+	missesCache        gcache.Cache
+	edgeToken          string
+}
+
+// NewServer returns an initialized Server pointer.
+func NewServer(store store.BlobStore, requestQueueSize int, edgeToken string) *Server {
+	return &Server{
+		store:              store,
+		grp:                stop.New(),
+		concurrentRequests: requestQueueSize,
+		missesCache:        gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
+		edgeToken:          edgeToken,
+	}
+}
+
+// Shutdown gracefully shuts down the peer server.
+func (s *Server) Shutdown() {
+	log.Debug("shutting down HTTP server")
+	s.grp.StopAndWait()
+	log.Debug("HTTP server stopped")
+}
+
+// Start starts the server listener to handle connections.
+func (s *Server) Start(address string) error {
+	gin.SetMode(gin.ReleaseMode)
+	router := gin.New()
+	router.Use(gin.Logger())
+	// Install nice.Recovery, passing the handler to call after recovery
+	router.Use(nice.Recovery(s.recoveryHandler))
+	router.GET("/blob", s.getBlob)
+	router.HEAD("/blob", s.hasBlob)
+	srv := &http.Server{
+		Addr:    address,
+		Handler: router,
+	}
+	go s.listenForShutdown(srv)
+	go InitWorkers(s, s.concurrentRequests)
+	// Initializing the server in a goroutine so that
+	// it won't block the graceful shutdown handling below
+	s.grp.Add(1)
+	go func() {
+		defer s.grp.Done()
+		log.Println("HTTP server listening on " + address)
+		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Fatalf("listen: %s\n", err)
+		}
+	}()
+	return nil
+}
+
+func (s *Server) listenForShutdown(listener *http.Server) {
+	<-s.grp.Ch()
+	// The context is used to inform the server it has 5 seconds to finish
+	// the request it is currently handling
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := listener.Shutdown(ctx); err != nil {
+		log.Fatal("Server forced to shutdown:", err)
+	}
+}

server/http/worker.go

@@ -0,0 +1,46 @@
+package http
+
+import (
+	"sync"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+
+	"github.com/gin-gonic/gin"
+)
+
+type blobRequest struct {
+	c        *gin.Context
+	finished *sync.WaitGroup
+}
+
+var getReqCh = make(chan *blobRequest, 20000)
+
+func InitWorkers(server *Server, workers int) {
+	stopper := stop.New(server.grp)
+	for i := 0; i < workers; i++ {
+		metrics.RoutinesQueue.WithLabelValues("http", "worker").Inc()
+		go func(worker int) {
+			defer metrics.RoutinesQueue.WithLabelValues("http", "worker").Dec()
+			for {
+				select {
+				case <-stopper.Ch():
+				case r := <-getReqCh:
+					process(server, r)
+					metrics.HttpBlobReqQueue.Dec()
+				}
+			}
+		}(i)
+	}
+}
+
+func enqueue(b *blobRequest) {
+	metrics.HttpBlobReqQueue.Inc()
+	getReqCh <- b
+}
+
+func process(server *Server, r *blobRequest) {
+	server.HandleGetBlob(r.c)
+	r.finished.Done()
+}

server/http3/client.go

@@ -0,0 +1,141 @@
+package http3
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	"github.com/quic-go/quic-go/http3"
+	log "github.com/sirupsen/logrus"
+)
+
+// Client is an instance of a client connected to a server.
+type Client struct {
+	Timeout      time.Duration
+	conn         *http.Client
+	roundTripper *http3.RoundTripper
+	ServerAddr   string
+}
+
+// Close closes the connection with the client.
+func (c *Client) Close() error {
+	c.conn.CloseIdleConnections()
+	return c.roundTripper.Close()
+}
+
+// GetStream gets a stream
+func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
+	var sd stream.SDBlob
+
+	b, _, err := c.GetBlob(sdHash)
+	if err != nil {
+		return nil, err
+	}
+
+	err = sd.FromBlob(b)
+	if err != nil {
+		return nil, err
+	}
+
+	s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
+	s[0] = b
+
+	for i := 0; i < len(sd.BlobInfos)-1; i++ {
+		var trace shared.BlobTrace
+		s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
+		if err != nil {
+			return nil, err
+		}
+		log.Debug(trace.String())
+	}
+
+	return s, nil
+}
+
+// HasBlob checks if the blob is available
+func (c *Client) HasBlob(hash string) (bool, error) {
+	resp, err := c.conn.Get(fmt.Sprintf("https://%s/has/%s", c.ServerAddr, hash))
+	if err != nil {
+		return false, errors.Err(err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode == http.StatusOK {
+		return true, nil
+	}
+	if resp.StatusCode == http.StatusNotFound {
+		return false, nil
+	}
+	return false, errors.Err("non 200 status code returned: %d", resp.StatusCode)
+}
+
+// GetBlob gets a blob
+func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s?trace=true", c.ServerAddr, hash))
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode == http.StatusNotFound {
+		fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
+		return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(store.ErrBlobNotFound)
+	} else if resp.StatusCode != http.StatusOK {
+		return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err("non 200 status code returned: %d", resp.StatusCode)
+	}
+
+	tmp := getBuffer()
+	defer putBuffer(tmp)
+	serialized := resp.Header.Get("Via")
+	trace := shared.NewBlobTrace(time.Since(start), "http3")
+	if serialized != "" {
+		parsedTrace, err := shared.Deserialize(serialized)
+		if err != nil {
+			return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
+		}
+		trace = *parsedTrace
+	}
+	written, err := io.Copy(tmp, resp.Body)
+	if err != nil {
+		return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
+	}
+
+	blob := make([]byte, written)
+	copy(blob, tmp.Bytes())
+
+	metrics.MtrInBytesUdp.Add(float64(len(blob)))
+
+	return blob, trace.Stack(time.Since(start), "http3"), nil
+}
+
+// buffer pool to reduce GC
+// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
+var buffers = sync.Pool{
+	// New is called when a new instance is needed
+	New: func() interface{} {
+		buf := make([]byte, 0, stream.MaxBlobSize)
+		return bytes.NewBuffer(buf)
+	},
+}
+
+// getBuffer fetches a buffer from the pool
+func getBuffer() *bytes.Buffer {
+	return buffers.Get().(*bytes.Buffer)
+}
+
+// putBuffer returns a buffer to the pool
+func putBuffer(buf *bytes.Buffer) {
+	buf.Reset()
+	buffers.Put(buf)
+}

server/http3/server.go

@@ -0,0 +1,216 @@
+package http3
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+
+	"github.com/gorilla/mux"
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
+	log "github.com/sirupsen/logrus"
+)
+
+// Server is an instance of a peer server that houses the listener and store.
+type Server struct {
+	store              store.BlobStore
+	grp                *stop.Group
+	concurrentRequests int
+}
+
+// NewServer returns an initialized Server pointer.
+func NewServer(store store.BlobStore, requestQueueSize int) *Server {
+	return &Server{
+		store:              store,
+		grp:                stop.New(),
+		concurrentRequests: requestQueueSize,
+	}
+}
+
+// Shutdown gracefully shuts down the peer server.
+func (s *Server) Shutdown() {
+	log.Debug("shutting down http3 peer server")
+	s.grp.StopAndWait()
+	log.Debug("http3 peer server stopped")
+}
+
+func (s *Server) logError(e error) {
+	if e == nil {
+		return
+	}
+	shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
+	if shouldLog {
+		log.Errorln(errors.FullTrace(e))
+	}
+}
+
+type availabilityResponse struct {
+	LbrycrdAddress string `json:"lbrycrd_address"`
+	IsAvailable    bool   `json:"is_available"`
+}
+
+// Start starts the server listener to handle connections.
+func (s *Server) Start(address string) error {
+	log.Println("HTTP3 peer listening on " + address)
+	window500M := 500 * 1 << 20
+
+	quicConf := &quic.Config{
+		MaxStreamReceiveWindow:     uint64(window500M),
+		MaxConnectionReceiveWindow: uint64(window500M),
+		EnableDatagrams:            true,
+		HandshakeIdleTimeout:       4 * time.Second,
+		MaxIdleTimeout:             20 * time.Second,
+	}
+	r := mux.NewRouter()
+	r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
+		waiter := &sync.WaitGroup{}
+		waiter.Add(1)
+		enqueue(&blobRequest{request: r, reply: w, finished: waiter})
+		waiter.Wait()
+	})
+	r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		requestedBlob := vars["hash"]
+		blobExists, err := s.store.Has(requestedBlob)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			s.logError(err)
+			return
+		}
+		if !blobExists {
+			w.WriteHeader(http.StatusNotFound)
+		}
+		// LbrycrdAddress to be used when paying for data. Not implemented yet.
+		const LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
+		resp, err := json.Marshal(availabilityResponse{
+			LbrycrdAddress: LbrycrdAddress,
+			IsAvailable:    blobExists,
+		})
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			s.logError(err)
+			return
+		}
+		_, err = w.Write(resp)
+		if err != nil {
+			s.logError(err)
+		}
+	})
+	server := http3.Server{
+		Addr:       address,
+		Handler:    r,
+		TLSConfig:  generateTLSConfig(),
+		QuicConfig: quicConf,
+	}
+	go InitWorkers(s, s.concurrentRequests)
+	go s.listenForShutdown(&server)
+	s.grp.Add(1)
+	go func() {
+		s.listenAndServe(&server)
+		s.grp.Done()
+	}()
+
+	return nil
+}
+
+// Setup a bare-bones TLS config for the server
+func generateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{SerialNumber: big.NewInt(1)}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+		NextProtos:   []string{"http3-reflector-server"},
+	}
+}
+
+func (s *Server) listenAndServe(server *http3.Server) {
+	err := server.ListenAndServe()
+	if err != nil && err != quic.ErrServerClosed {
+		log.Errorln(errors.FullTrace(err))
+	}
+}
+
+func (s *Server) listenForShutdown(listener *http3.Server) {
+	<-s.grp.Ch()
+	err := listener.Close()
+	if err != nil {
+		log.Error("error closing listener for peer server - ", err)
+	}
+}
+
+func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	requestedBlob := vars["hash"]
+	traceParam := r.URL.Query().Get("trace")
+	var err error
+	wantsTrace := false
+	if traceParam != "" {
+		wantsTrace, err = strconv.ParseBool(traceParam)
+		if err != nil {
+			wantsTrace = false
+		}
+	}
+	if reflector.IsProtected(requestedBlob) {
+		http.Error(w, "requested blob is protected", http.StatusForbidden)
+		return
+	}
+	blob, trace, err := s.store.Get(requestedBlob)
+
+	if wantsTrace {
+		serialized, err := trace.Serialize()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusNotFound)
+			return
+		}
+		w.Header().Add("Via", serialized)
+		log.Debug(trace.String())
+	}
+	if err != nil {
+		if errors.Is(err, store.ErrBlobNotFound) {
+			http.Error(w, err.Error(), http.StatusNotFound)
+			return
+		}
+		fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
+		s.logError(err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	_, err = w.Write(blob)
+	if err != nil {
+		s.logError(err)
+	}
+	metrics.MtrOutBytesUdp.Add(float64(len(blob)))
+	metrics.BlobDownloadCount.Inc()
+	metrics.Http3DownloadCount.Inc()
+}

server/http3/store.go

@@ -0,0 +1,117 @@
+package http3
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/lbryio/reflector.go/shared"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
+)
+
+// Store is a blob store that gets blobs from a peer.
+// It satisfies the store.BlobStore interface but cannot put or delete blobs.
+type Store struct {
+	opts          StoreOpts
+	NotFoundCache *sync.Map
+}
+
+// StoreOpts allows to set options for a new Store.
+type StoreOpts struct {
+	Address string
+	Timeout time.Duration
+}
+
+// NewStore makes a new peer store.
+func NewStore(opts StoreOpts) *Store {
+	return &Store{opts: opts, NotFoundCache: &sync.Map{}}
+}
+
+func (p *Store) getClient() (*Client, error) {
+	var qconf quic.Config
+	window500M := 500 * 1 << 20
+	qconf.MaxStreamReceiveWindow = uint64(window500M)
+	qconf.MaxConnectionReceiveWindow = uint64(window500M)
+	qconf.EnableDatagrams = true
+	qconf.HandshakeIdleTimeout = 4 * time.Second
+	qconf.MaxIdleTimeout = 20 * time.Second
+	pool, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, err
+	}
+	roundTripper := &http3.RoundTripper{
+		TLSClientConfig: &tls.Config{
+			RootCAs:            pool,
+			InsecureSkipVerify: true,
+		},
+		QuicConfig: &qconf,
+	}
+	connection := &http.Client{
+		Transport: roundTripper,
+	}
+	c := &Client{
+		conn:         connection,
+		roundTripper: roundTripper,
+		ServerAddr:   p.opts.Address,
+	}
+	return c, errors.Prefix("connection error", err)
+}
+
+func (p *Store) Name() string { return "http3" }
+
+// Has asks the peer if they have a hash
+func (p *Store) Has(hash string) (bool, error) {
+	c, err := p.getClient()
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = c.Close() }()
+	return c.HasBlob(hash)
+}
+
+// Get downloads the blob from the peer
+func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	if lastChecked, ok := p.NotFoundCache.Load(hash); ok {
+		if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
+			return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound
+		}
+	}
+	c, err := p.getClient()
+	if err != nil && strings.Contains(err.Error(), "blob not found") {
+		p.NotFoundCache.Store(hash, time.Now())
+	}
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
+	}
+	defer func() { _ = c.Close() }()
+	return c.GetBlob(hash)
+}
+
+// Put is not supported
+func (p *Store) Put(hash string, blob stream.Blob) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// PutSD is not supported
+func (p *Store) PutSD(hash string, blob stream.Blob) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// Delete is not supported
+func (p *Store) Delete(hash string) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// Shutdown is not supported
+func (p *Store) Shutdown() {
+}

server/http3/worker.go

@@ -0,0 +1,46 @@
+package http3
+
+import (
+	"net/http"
+	"sync"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+)
+
+type blobRequest struct {
+	request  *http.Request
+	reply    http.ResponseWriter
+	finished *sync.WaitGroup
+}
+
+var getReqCh = make(chan *blobRequest, 20000)
+
+func InitWorkers(server *Server, workers int) {
+	stopper := stop.New(server.grp)
+	for i := 0; i < workers; i++ {
+		metrics.RoutinesQueue.WithLabelValues("http3", "worker").Inc()
+		go func(worker int) {
+			defer metrics.RoutinesQueue.WithLabelValues("http3", "worker").Dec()
+			for {
+				select {
+				case <-stopper.Ch():
+				case r := <-getReqCh:
+					metrics.Http3BlobReqQueue.Dec()
+					process(server, r)
+				}
+			}
+		}(i)
+	}
+}
+
+func enqueue(b *blobRequest) {
+	metrics.Http3BlobReqQueue.Inc()
+	getReqCh <- b
+}
+
+func process(server *Server, r *blobRequest) {
+	server.HandleGetBlob(r.reply, r.request)
+	r.finished.Done()
+}

server/peer/client.go

@@ -0,0 +1,207 @@
+package peer
+
+import (
+	"bufio"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"net"
+	"time"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// Client is an instance of a client connected to a server.
+type Client struct {
+	Timeout   time.Duration
+	conn      net.Conn
+	buf       *bufio.Reader
+	connected bool
+}
+
+// Connect connects to a specific clients and errors if it cannot be contacted.
+func (c *Client) Connect(address string) error {
+	var err error
+	if c.Timeout == 0 {
+		c.Timeout = 5 * time.Second
+	}
+	c.conn, err = net.Dial("tcp4", address)
+	if err != nil {
+		return err
+	}
+	c.connected = true
+	c.buf = bufio.NewReader(c.conn)
+	return nil
+}
+
+// Close closes the connection with the client.
+func (c *Client) Close() error {
+	c.connected = false
+	return c.conn.Close()
+}
+
+// GetStream gets a stream
+func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
+	if !c.connected {
+		return nil, errors.Err("not connected")
+	}
+
+	var sd stream.SDBlob
+
+	b, trace, err := c.GetBlob(sdHash)
+	if err != nil {
+		return nil, err
+	}
+	log.Debug(trace.String())
+
+	err = sd.FromBlob(b)
+	if err != nil {
+		return nil, err
+	}
+
+	s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
+	s[0] = b
+
+	for i := 0; i < len(sd.BlobInfos)-1; i++ {
+		s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
+		if err != nil {
+			return nil, err
+		}
+		log.Debug(trace.String())
+	}
+
+	return s, nil
+}
+
+// HasBlob checks if the blob is available
+func (c *Client) HasBlob(hash string) (bool, error) {
+	if !c.connected {
+		return false, errors.Err("not connected")
+	}
+
+	sendRequest, err := json.Marshal(availabilityRequest{
+		RequestedBlobs: []string{hash},
+	})
+	if err != nil {
+		return false, err
+	}
+
+	err = c.write(sendRequest)
+	if err != nil {
+		return false, err
+	}
+
+	var resp availabilityResponse
+	err = c.read(&resp)
+	if err != nil {
+		return false, err
+	}
+
+	for _, h := range resp.AvailableBlobs {
+		if h == hash {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// GetBlob gets a blob
+func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	if !c.connected {
+		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), errors.Err("not connected")
+	}
+
+	sendRequest, err := json.Marshal(blobRequest{
+		RequestedBlob: hash,
+	})
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
+	}
+
+	err = c.write(sendRequest)
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
+	}
+
+	var resp blobResponse
+	err = c.read(&resp)
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
+	}
+
+	trace := shared.NewBlobTrace(time.Since(start), "tcp")
+	if resp.RequestTrace != nil {
+		trace = *resp.RequestTrace
+	}
+	if resp.IncomingBlob.Error != "" {
+		return nil, trace, errors.Prefix(hash[:8], resp.IncomingBlob.Error)
+	}
+	if resp.IncomingBlob.BlobHash != hash {
+		return nil, trace.Stack(time.Since(start), "tcp"), errors.Prefix(hash[:8], "blob hash in response does not match requested hash")
+	}
+	if resp.IncomingBlob.Length <= 0 {
+		return nil, trace, errors.Prefix(hash[:8], "length reported as <= 0")
+	}
+
+	log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr())
+
+	blob, err := c.readRawBlob(resp.IncomingBlob.Length)
+	if err != nil {
+		return nil, (*resp.RequestTrace).Stack(time.Since(start), "tcp"), err
+	}
+	metrics.MtrInBytesTcp.Add(float64(len(blob)))
+	return blob, trace.Stack(time.Since(start), "tcp"), nil
+}
+
+func (c *Client) read(v interface{}) error {
+	err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	m, err := readNextMessage(c.buf)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("read %d bytes from %s", len(m), c.conn.RemoteAddr())
+
+	err = json.Unmarshal(m, v)
+	return errors.Err(err)
+}
+
+func (c *Client) readRawBlob(blobSize int) ([]byte, error) {
+	err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	blob := make([]byte, blobSize)
+	n, err := io.ReadFull(c.buf, blob)
+	log.Debugf("read %d bytes from %s", n, c.conn.RemoteAddr())
+	return blob, errors.Err(err)
+}
+
+func (c *Client) write(b []byte) error {
+	err := c.conn.SetWriteDeadline(time.Now().Add(c.Timeout))
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	log.Debugf("writing %d bytes to %s", len(b), c.conn.RemoteAddr())
+
+	n, err := c.conn.Write(b)
+	if err == nil && n != len(b) {
+		err = io.ErrShortWrite
+	}
+	return errors.Err(err)
+}

server/peer/server.go

@@ -0,0 +1,420 @@
+package peer
+
+import (
+	"bufio"
+	"encoding/hex"
+	"encoding/json"
+	ee "errors"
+	"io"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/shared"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	// DefaultPort is the port the peer server listens on if not passed in.
+	DefaultPort = 3333
+	// LbrycrdAddress to be used when paying for data. Not implemented yet.
+	LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
+)
+
+// Server is an instance of a peer server that houses the listener and store.
+type Server struct {
+	store  store.BlobStore
+	closed bool
+
+	grp *stop.Group
+}
+
+// NewServer returns an initialized Server pointer.
+func NewServer(store store.BlobStore) *Server {
+	return &Server{
+		store: store,
+		grp:   stop.New(),
+	}
+}
+
+// Shutdown gracefully shuts down the peer server.
+func (s *Server) Shutdown() {
+	log.Debug("shutting down peer server")
+	s.grp.StopAndWait()
+	log.Debug("peer server stopped")
+}
+
+// Start starts the server listener to handle connections.
+func (s *Server) Start(address string) error {
+	log.Println("peer listening on " + address)
+	l, err := net.Listen("tcp4", address)
+	if err != nil {
+		return err
+	}
+
+	go s.listenForShutdown(l)
+	s.grp.Add(1)
+	go func() {
+		s.listenAndServe(l)
+		s.grp.Done()
+	}()
+
+	return nil
+}
+
+func (s *Server) listenForShutdown(listener net.Listener) {
+	<-s.grp.Ch()
+	s.closed = true
+	err := listener.Close()
+	if err != nil {
+		log.Error("error closing listener for peer server - ", err)
+	}
+}
+
+func (s *Server) listenAndServe(listener net.Listener) {
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			if s.closed {
+				return
+			}
+			log.Error(errors.Prefix("accepting conn", err))
+		} else {
+			s.grp.Add(1)
+			metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Inc()
+			go func() {
+				defer metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Dec()
+				s.handleConnection(conn)
+				s.grp.Done()
+			}()
+		}
+	}
+}
+
+func (s *Server) handleConnection(conn net.Conn) {
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Error(errors.Prefix("closing peer conn", err))
+		}
+	}()
+
+	timeoutDuration := 1 * time.Minute
+	buf := bufio.NewReader(conn)
+
+	for {
+		var request []byte
+		var response []byte
+
+		err := conn.SetReadDeadline(time.Now().Add(timeoutDuration))
+		if err != nil {
+			log.Error(errors.FullTrace(err))
+		}
+
+		request, err = readNextMessage(buf)
+		if err != nil {
+			if err != io.EOF {
+				s.logError(err)
+			}
+			return
+		}
+
+		err = conn.SetReadDeadline(time.Time{})
+		if err != nil {
+			log.Error(errors.FullTrace(err))
+		}
+
+		response, err = s.handleCompositeRequest(request)
+		if err != nil {
+			log.Error(errors.FullTrace(err))
+			return
+		}
+
+		err = conn.SetWriteDeadline(time.Now().Add(timeoutDuration))
+		if err != nil {
+			log.Error(errors.FullTrace(err))
+		}
+
+		n, err := conn.Write(response)
+		if err != nil {
+			if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset
+				s.logError(err)
+			}
+			return
+		} else if n != len(response) {
+			log.Errorln(io.ErrShortWrite)
+			return
+		}
+
+		err = conn.SetWriteDeadline(time.Time{})
+		if err != nil {
+			log.Error(errors.FullTrace(err))
+		}
+	}
+}
+
+func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
+	var request availabilityRequest
+	err := json.Unmarshal(data, &request)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	availableBlobs := []string{}
+	for _, blobHash := range request.RequestedBlobs {
+		exists, err := s.store.Has(blobHash)
+		if err != nil {
+			return nil, err
+		}
+		if exists {
+			availableBlobs = append(availableBlobs, blobHash)
+		}
+	}
+
+	return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs})
+}
+
+//func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
+//	var request paymentRateRequest
+//	err := json.Unmarshal(data, &request)
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	offerReply := paymentRateAccepted
+//	if request.BlobDataPaymentRate < 0 {
+//		offerReply = paymentRateTooLow
+//	}
+//
+//	return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
+//}
+//
+//func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
+//	var request blobRequest
+//	err := json.Unmarshal(data, &request)
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	log.Debugln("Sending blob " + request.RequestedBlob[:8])
+//
+//	blob, err := s.store.Get(request.RequestedBlob)
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
+//		BlobHash: reflector.BlobHash(blob),
+//		Length:   len(blob),
+//	}})
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	return append(response, blob...), nil
+//}
+
+func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
+	var request compositeRequest
+	err := json.Unmarshal(data, &request)
+	if err != nil {
+		var je *json.SyntaxError
+		if ee.As(err, &je) {
+			return nil, errors.Err("invalid json request: offset %d in data %s", je.Offset, hex.EncodeToString(data))
+		}
+		return nil, errors.Err(err)
+	}
+
+	response := compositeResponse{
+		LbrycrdAddress: LbrycrdAddress,
+		AvailableBlobs: []string{},
+	}
+
+	if len(request.RequestedBlobs) > 0 {
+		for _, blobHash := range request.RequestedBlobs {
+			if reflector.IsProtected(blobHash) {
+				return nil, errors.Err("requested blob is protected")
+			}
+			exists, err := s.store.Has(blobHash)
+			if err != nil {
+				return nil, err
+			}
+			if exists {
+				response.AvailableBlobs = append(response.AvailableBlobs, blobHash)
+			}
+		}
+	}
+
+	if request.BlobDataPaymentRate != nil {
+		response.BlobDataPaymentRate = paymentRateAccepted
+		if *request.BlobDataPaymentRate < 0 {
+			response.BlobDataPaymentRate = paymentRateTooLow
+		}
+	}
+
+	var blob []byte
+	var trace shared.BlobTrace
+	if request.RequestedBlob != "" {
+		if len(request.RequestedBlob) != stream.BlobHashHexLength {
+			return nil, errors.Err("Invalid blob hash length")
+		}
+
+		log.Debugln("Sending blob " + request.RequestedBlob[:8])
+
+		blob, trace, err = s.store.Get(request.RequestedBlob)
+		log.Debug(trace.String())
+		if errors.Is(err, store.ErrBlobNotFound) {
+			response.IncomingBlob = &incomingBlob{
+				Error: err.Error(),
+			}
+		} else if err != nil {
+			return nil, err
+		} else {
+			response.IncomingBlob = &incomingBlob{
+				BlobHash: request.RequestedBlob,
+				Length:   len(blob),
+			}
+			metrics.MtrOutBytesTcp.Add(float64(len(blob)))
+			metrics.BlobDownloadCount.Inc()
+			metrics.PeerDownloadCount.Inc()
+		}
+	}
+
+	respData, err := json.Marshal(response)
+	if err != nil {
+		return nil, err
+	}
+
+	return append(respData, blob...), nil
+}
+
+func (s *Server) logError(e error) {
+	if e == nil {
+		return
+	}
+	shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
+	if shouldLog {
+		log.Errorln(errors.FullTrace(e))
+	}
+}
+
+func readNextMessage(buf *bufio.Reader) ([]byte, error) {
+	first_byte, err := buf.ReadByte()
+	if err != nil {
+		return nil, err
+	}
+	if first_byte != '{' {
+		// every request starts with '{'. Checking here disconnects earlier, so we don't wait until timeout
+		return nil, errInvalidData
+	}
+	msg := []byte("{")
+	eof := false
+
+	for {
+		chunk, err := buf.ReadBytes('}')
+		if err != nil {
+			if err != io.EOF {
+				//log.Errorln("readBytes error:", err) // logged by caller
+				return msg, err
+			}
+			eof = true
+		}
+
+		//log.Debugln("got", len(chunk), "bytes.")
+		//spew.Dump(chunk)
+
+		if len(chunk) > 0 {
+			msg = append(msg, chunk...)
+
+			if len(msg) > maxRequestSize {
+				return msg, errRequestTooLarge
+			} else if len(msg) > 0 && msg[0] != '{' {
+				return msg, errInvalidData
+			}
+
+			// yes, this is how the peer protocol knows when the request finishes
+			if reflector.IsValidJSON(msg) {
+				break
+			}
+		}
+
+		if eof {
+			break
+		}
+	}
+
+	//log.Debugln("total size:", len(request))
+	//if len(request) > 0 {
+	//	spew.Dump(request)
+	//}
+
+	if len(msg) == 0 && eof {
+		return nil, io.EOF
+	}
+
+	return msg, nil
+}
+
+const (
+	maxRequestSize      = 64 * (2 ^ 10) // 64kb
+	paymentRateAccepted = "RATE_ACCEPTED"
+	paymentRateTooLow   = "RATE_TOO_LOW"
+	//ToDo: paymentRateUnset is not used but exists in the protocol.
+	//paymentRateUnset    = "RATE_UNSET"
+)
+
+var errRequestTooLarge = errors.Base("request is too large")
+var errInvalidData = errors.Base("Invalid data")
+
+type availabilityRequest struct {
+	LbrycrdAddress bool     `json:"lbrycrd_address"`
+	RequestedBlobs []string `json:"requested_blobs"`
+}
+
+type availabilityResponse struct {
+	LbrycrdAddress string   `json:"lbrycrd_address"`
+	AvailableBlobs []string `json:"available_blobs"`
+}
+
+type paymentRateRequest struct {
+	BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
+}
+
+type paymentRateResponse struct {
+	BlobDataPaymentRate string `json:"blob_data_payment_rate"`
+}
+
+type blobRequest struct {
+	RequestedBlob string `json:"requested_blob"`
+}
+
+type incomingBlob struct {
+	Error    string `json:"error,omitempty"`
+	BlobHash string `json:"blob_hash"`
+	Length   int    `json:"length"`
+}
+type blobResponse struct {
+	IncomingBlob incomingBlob `json:"incoming_blob"`
+	RequestTrace *shared.BlobTrace
+}
+
+type compositeRequest struct {
+	LbrycrdAddress      bool     `json:"lbrycrd_address"`
+	RequestedBlobs      []string `json:"requested_blobs"`
+	BlobDataPaymentRate *float64 `json:"blob_data_payment_rate"`
+	RequestedBlob       string   `json:"requested_blob"`
+}
+
+type compositeResponse struct {
+	LbrycrdAddress      string        `json:"lbrycrd_address,omitempty"`
+	AvailableBlobs      []string      `json:"available_blobs"`
+	BlobDataPaymentRate string        `json:"blob_data_payment_rate,omitempty"`
+	IncomingBlob        *incomingBlob `json:"incoming_blob,omitempty"`
+}

server/peer/server_test.go

@@ -2,7 +2,10 @@ package peer
 
 import (
 	"bytes"
+	"io"
+	"net"
 	"testing"
+	"time"
 
 	"github.com/lbryio/reflector.go/store"
 )
@@ -34,7 +37,7 @@ var availabilityRequests = []pair{
 }
 
 func getServer(t *testing.T, withBlobs bool) *Server {
-	st := store.MemoryBlobStore{}
+	st := store.NewMemStore()
 	if withBlobs {
 		for k, v := range blobs {
 			err := st.Put(k, v)
@@ -43,7 +46,7 @@ func getServer(t *testing.T, withBlobs bool) *Server {
 			}
 		}
 	}
-	return NewServer(&st)
+	return NewServer(st)
 }
 
 func TestAvailabilityRequest_NoBlobs(t *testing.T) {
@@ -75,3 +78,62 @@ func TestAvailabilityRequest_WithBlobs(t *testing.T) {
 		}
 	}
 }
+
+func TestRequestFromConnection(t *testing.T) {
+	s := getServer(t, true)
+	err := s.Start("127.0.0.1:50505")
+	defer s.Shutdown()
+	if err != nil {
+		t.Error("error starting server", err)
+	}
+
+	for _, p := range availabilityRequests {
+		conn, err := net.Dial("tcp", "127.0.0.1:50505")
+		if err != nil {
+			t.Error("error opening connection", err)
+		}
+		defer func() { _ = conn.Close() }()
+
+		response := make([]byte, 8192)
+		_, err = conn.Write(p.request)
+		if err != nil {
+			t.Error("error writing", err)
+		}
+		_, err = conn.Read(response)
+		if err != nil {
+			t.Error("error reading", err)
+		}
+		if !bytes.Equal(response[:len(p.response)], p.response) {
+			t.Errorf("Response did not match expected response.\nExpected: %s\nGot: %s", string(p.response), string(response))
+		}
+	}
+}
+
+func TestInvalidData(t *testing.T) {
+	s := getServer(t, true)
+	err := s.Start("127.0.0.1:50503")
+	defer s.Shutdown()
+	if err != nil {
+		t.Error("error starting server", err)
+	}
+	conn, err := net.Dial("tcp", "127.0.0.1:50503")
+	if err != nil {
+		t.Error("error opening connection", err)
+	}
+	defer func() { _ = conn.Close() }()
+
+	response := make([]byte, 8192)
+	_, err = conn.Write([]byte("hello dear server, I would like blobs. Please"))
+	if err != nil {
+		t.Error("error writing", err)
+	}
+	err = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
+	if err != nil {
+		t.Error("error setting read deadline", err)
+	}
+	_, err = conn.Read(response)
+	if err != io.EOF {
+		t.Error("error reading", err)
+	}
+	println(response)
+}

server/peer/store.go

@@ -0,0 +1,82 @@
+package peer
+
+import (
+	"strings"
+	"time"
+
+	"github.com/lbryio/reflector.go/shared"
+	"github.com/lbryio/reflector.go/store"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+)
+
+// Store is a blob store that gets blobs from a peer.
+// It satisfies the store.BlobStore interface but cannot put or delete blobs.
+type Store struct {
+	opts StoreOpts
+}
+
+// StoreOpts allows to set options for a new Store.
+type StoreOpts struct {
+	Address string
+	Timeout time.Duration
+}
+
+// NewStore makes a new peer store.
+func NewStore(opts StoreOpts) *Store {
+	return &Store{opts: opts}
+}
+
+func (p *Store) getClient() (*Client, error) {
+	c := &Client{Timeout: p.opts.Timeout}
+	err := c.Connect(p.opts.Address)
+	return c, errors.Prefix("connection error", err)
+}
+
+func (p *Store) Name() string { return "peer" }
+
+// Has asks the peer if they have a hash
+func (p *Store) Has(hash string) (bool, error) {
+	c, err := p.getClient()
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = c.Close() }()
+	return c.HasBlob(hash)
+}
+
+// Get downloads the blob from the peer
+func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	c, err := p.getClient()
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
+	}
+	defer func() { _ = c.Close() }()
+	blob, trace, err := c.GetBlob(hash)
+	if err != nil && strings.Contains(err.Error(), "blob not found") {
+		return nil, trace, store.ErrBlobNotFound
+	}
+
+	return blob, trace, err
+}
+
+// Put is not supported
+func (p *Store) Put(hash string, blob stream.Blob) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// PutSD is not supported
+func (p *Store) PutSD(hash string, blob stream.Blob) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// Delete is not supported
+func (p *Store) Delete(hash string) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// Shutdown is not supported
+func (p *Store) Shutdown() {
+}

shared/errors.go

@@ -0,0 +1,6 @@
+package shared
+
+import "github.com/lbryio/lbry.go/v2/extras/errors"
+
+//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
+var ErrNotImplemented = errors.Base("this store does not implement this method")

shared/shared.go

@@ -0,0 +1,82 @@
+package shared
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+)
+
+type BlobStack struct {
+	Timing     time.Duration `json:"timing"`
+	OriginName string        `json:"origin_name"`
+	HostName   string        `json:"host_name"`
+}
+type BlobTrace struct {
+	Stacks []BlobStack `json:"stacks"`
+}
+
+var hostName *string
+
+func getHostName() string {
+	if hostName == nil {
+		hn, err := os.Hostname()
+		if err != nil {
+			hn = "unknown"
+		}
+		hostName = &hn
+	}
+	return *hostName
+}
+func (b *BlobTrace) Stack(timing time.Duration, originName string) BlobTrace {
+	b.Stacks = append(b.Stacks, BlobStack{
+		Timing:     timing,
+		OriginName: originName,
+		HostName:   getHostName(),
+	})
+	return *b
+}
+func (b *BlobTrace) Merge(otherTrance BlobTrace) BlobTrace {
+	b.Stacks = append(b.Stacks, otherTrance.Stacks...)
+	return *b
+}
+func NewBlobTrace(timing time.Duration, originName string) BlobTrace {
+	b := BlobTrace{}
+	b.Stacks = append(b.Stacks, BlobStack{
+		Timing:     timing,
+		OriginName: originName,
+		HostName:   getHostName(),
+	})
+	return b
+}
+
+func (b BlobTrace) String() string {
+	var fullTrace string
+	for i, stack := range b.Stacks {
+		delta := time.Duration(0)
+		if i > 0 {
+			delta = stack.Timing - b.Stacks[i-1].Timing
+		}
+		fullTrace += fmt.Sprintf("[%d](%s) origin: %s - timing: %s - delta: %s\n", i, stack.HostName, stack.OriginName, stack.Timing.String(), delta.String())
+	}
+	return fullTrace
+}
+
+func (b BlobTrace) Serialize() (string, error) {
+	t, err := json.Marshal(b)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+	return string(t), nil
+}
+
+func Deserialize(serializedData string) (*BlobTrace, error) {
+	var trace BlobTrace
+	err := json.Unmarshal([]byte(serializedData), &trace)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	return &trace, nil
+}

shared/shared_test.go

@@ -0,0 +1,36 @@
+package shared
+
+import (
+	"testing"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/util"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBlobTrace_Serialize(t *testing.T) {
+	hostName = util.PtrToString("test_machine")
+	stack := NewBlobTrace(10*time.Second, "test")
+	stack.Stack(20*time.Second, "test2")
+	stack.Stack(30*time.Second, "test3")
+	serialized, err := stack.Serialize()
+	assert.NoError(t, err)
+	t.Log(serialized)
+	expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\",\"host_name\":\"test_machine\"},{\"timing\":20000000000,\"origin_name\":\"test2\",\"host_name\":\"test_machine\"},{\"timing\":30000000000,\"origin_name\":\"test3\",\"host_name\":\"test_machine\"}]}"
+	assert.Equal(t, expected, serialized)
+}
+
+func TestBlobTrace_Deserialize(t *testing.T) {
+	hostName = util.PtrToString("test_machine")
+	serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
+	stack, err := Deserialize(serialized)
+	assert.NoError(t, err)
+	assert.Len(t, stack.Stacks, 3)
+	assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second)
+	assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second)
+	assert.Equal(t, stack.Stacks[2].Timing, 30*time.Second)
+	assert.Equal(t, stack.Stacks[0].OriginName, "test")
+	assert.Equal(t, stack.Stacks[1].OriginName, "test2")
+	assert.Equal(t, stack.Stacks[2].OriginName, "test3")
+}

store/atime_linux.go

@@ -0,0 +1,18 @@
+//go:build linux
+// +build linux
+
+package store
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func timespecToTime(ts syscall.Timespec) time.Time {
+	return time.Unix(ts.Sec, ts.Nsec)
+}
+
+func atime(fi os.FileInfo) time.Time {
+	return timespecToTime(fi.Sys().(*syscall.Stat_t).Atim)
+}

store/atime_universal.go

@@ -0,0 +1,12 @@
+// +build !linux
+
+package store
+
+import (
+	"os"
+	"time"
+)
+
+func atime(fi os.FileInfo) time.Time {
+	return fi.ModTime()
+}

store/caching.go

@@ -0,0 +1,107 @@
+package store
+
+import (
+	"time"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// CachingStore combines two stores, typically a local and a remote store, to improve performance.
+// Accessed blobs are stored in and retrieved from the cache. If they are not in the cache, they
+// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
+type CachingStore struct {
+	origin, cache BlobStore
+	component     string
+}
+
+// NewCachingStore makes a new caching disk store and returns a pointer to it.
+func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
+	return &CachingStore{
+		component: component,
+		origin:    WithSingleFlight(component, origin),
+		cache:     WithSingleFlight(component, cache),
+	}
+}
+
+const nameCaching = "caching"
+
+// Name is the cache type name
+func (c *CachingStore) Name() string { return nameCaching }
+
+// Has checks the cache and then the origin for a hash. It returns true if either store has it.
+func (c *CachingStore) Has(hash string) (bool, error) {
+	has, err := c.cache.Has(hash)
+	if has || err != nil {
+		return has, err
+	}
+	return c.origin.Has(hash)
+}
+
+// Get tries to get the blob from the cache first, falling back to the origin. If the blob comes
+// from the origin, it is also stored in the cache.
+func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	blob, trace, err := c.cache.Get(hash)
+	if err == nil || !errors.Is(err, ErrBlobNotFound) {
+		metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
+		rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
+		metrics.CacheRetrievalSpeed.With(map[string]string{
+			metrics.LabelCacheType: c.cache.Name(),
+			metrics.LabelComponent: c.component,
+			metrics.LabelSource:    "cache",
+		}).Set(rate)
+		return blob, trace.Stack(time.Since(start), c.Name()), err
+	}
+
+	metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
+
+	blob, trace, err = c.origin.Get(hash)
+	if err != nil {
+		return nil, trace.Stack(time.Since(start), c.Name()), err
+	}
+	// do not do this async unless you're prepared to deal with mayhem
+	err = c.cache.Put(hash, blob)
+	if err != nil {
+		log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
+	}
+	return blob, trace.Stack(time.Since(start), c.Name()), nil
+}
+
+// Put stores the blob in the origin and the cache
+func (c *CachingStore) Put(hash string, blob stream.Blob) error {
+	err := c.origin.Put(hash, blob)
+	if err != nil {
+		return err
+	}
+	return c.cache.Put(hash, blob)
+}
+
+// PutSD stores the sd blob in the origin and the cache
+func (c *CachingStore) PutSD(hash string, blob stream.Blob) error {
+	err := c.origin.PutSD(hash, blob)
+	if err != nil {
+		return err
+	}
+	return c.cache.PutSD(hash, blob)
+}
+
+// Delete deletes the blob from the origin and the cache
+func (c *CachingStore) Delete(hash string) error {
+	err := c.origin.Delete(hash)
+	if err != nil {
+		return err
+	}
+	return c.cache.Delete(hash)
+}
+
+// Shutdown shuts down the store gracefully
+func (c *CachingStore) Shutdown() {
+	c.origin.Shutdown()
+	c.cache.Shutdown()
+}

store/caching_test.go

@@ -0,0 +1,178 @@
+package store
+
+import (
+	"bytes"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/lbryio/reflector.go/shared"
+
+	"github.com/lbryio/lbry.go/v2/stream"
+)
+
+func TestCachingStore_Put(t *testing.T) {
+	origin := NewMemStore()
+	cache := NewMemStore()
+	s := NewCachingStore("test", origin, cache)
+
+	b := []byte("this is a blob of stuff")
+	hash := "hash"
+
+	err := s.Put(hash, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	has, err := origin.Has(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !has {
+		t.Errorf("failed to store blob in origin")
+	}
+
+	has, err = cache.Has(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !has {
+		t.Errorf("failed to store blob in cache")
+	}
+}
+
+func TestCachingStore_CacheMiss(t *testing.T) {
+	origin := NewMemStore()
+	cache := NewMemStore()
+	s := NewCachingStore("test", origin, cache)
+
+	b := []byte("this is a blob of stuff")
+	hash := "hash"
+	err := origin.Put(hash, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	res, stack, err := s.Get(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(b, res) {
+		t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
+	}
+	time.Sleep(10 * time.Millisecond) //storing to cache is done async so let's give it some time
+
+	has, err := cache.Has(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !has {
+		t.Errorf("Get() did not copy blob to cache")
+	}
+	t.Logf("stack: %s", stack.String())
+
+	res, stack, err = cache.Get(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(b, res) {
+		t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res))
+	}
+	t.Logf("stack: %s", stack.String())
+}
+
+func TestCachingStore_ThunderingHerd(t *testing.T) {
+	storeDelay := 100 * time.Millisecond
+	origin := NewSlowBlobStore(storeDelay)
+	cache := NewMemStore()
+	s := NewCachingStore("test", origin, cache)
+
+	b := []byte("this is a blob of stuff")
+	hash := "hash"
+	err := origin.Put(hash, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wg := &sync.WaitGroup{}
+
+	getNoErr := func() {
+		res, _, err := s.Get(hash)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(b, res) {
+			t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
+		}
+		wg.Done()
+	}
+
+	start := time.Now()
+
+	wg.Add(4)
+	go func() {
+		go getNoErr()
+		time.Sleep(10 * time.Millisecond)
+		go getNoErr()
+		time.Sleep(10 * time.Millisecond)
+		go getNoErr()
+		time.Sleep(10 * time.Millisecond)
+		go getNoErr()
+	}()
+
+	wg.Wait()
+	duration := time.Since(start)
+
+	// only the first getNoErr() should hit the origin. the rest should wait for the first request to return
+	// once the first returns, the others should return immediately
+	// therefore, if the delay much longer than 100ms, it means subsequent requests also went to the origin
+	expectedMaxDelay := storeDelay + 5*time.Millisecond // a bit of extra time to let requests finish
+	if duration > expectedMaxDelay {
+		t.Errorf("Expected delay of at most %s, got %s", expectedMaxDelay, duration)
+	}
+}
+
+// SlowBlobStore adds a delay to each request
+type SlowBlobStore struct {
+	mem   *MemStore
+	delay time.Duration
+}
+
+func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
+	return &SlowBlobStore{
+		mem:   NewMemStore(),
+		delay: delay,
+	}
+}
+func (s *SlowBlobStore) Name() string {
+	return "slow"
+}
+
+func (s *SlowBlobStore) Has(hash string) (bool, error) {
+	time.Sleep(s.delay)
+	return s.mem.Has(hash)
+}
+
+func (s *SlowBlobStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	time.Sleep(s.delay)
+	return s.mem.Get(hash)
+}
+
+func (s *SlowBlobStore) Put(hash string, blob stream.Blob) error {
+	time.Sleep(s.delay)
+	return s.mem.Put(hash, blob)
+}
+
+func (s *SlowBlobStore) PutSD(hash string, blob stream.Blob) error {
+	time.Sleep(s.delay)
+	return s.mem.PutSD(hash, blob)
+}
+
+func (s *SlowBlobStore) Delete(hash string) error {
+	time.Sleep(s.delay)
+	return s.mem.Delete(hash)
+}
+
+func (s *SlowBlobStore) Shutdown() {
+	return
+}

store/cloudfront_ro.go

@@ -0,0 +1,108 @@
+package store
+
+import (
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/meta"
+	"github.com/lbryio/reflector.go/shared"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// CloudFrontROStore reads from cloudfront. All writes panic.
+type CloudFrontROStore struct {
+	endpoint string // cloudflare endpoint
+}
+
+// NewCloudFrontROStore returns an initialized CloudFrontROStore store pointer.
+func NewCloudFrontROStore(endpoint string) *CloudFrontROStore {
+	return &CloudFrontROStore{endpoint: endpoint}
+}
+
+const nameCloudFrontRO = "cloudfront_ro"
+
+// Name is the cache type name
+func (c *CloudFrontROStore) Name() string { return nameCloudFrontRO }
+
+// Has checks if the hash is in the store.
+func (c *CloudFrontROStore) Has(hash string) (bool, error) {
+	status, body, err := c.cfRequest(http.MethodHead, hash)
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = body.Close() }()
+	switch status {
+	case http.StatusNotFound, http.StatusForbidden:
+		return false, nil
+	case http.StatusOK:
+		return true, nil
+	default:
+		return false, errors.Err("unexpected status %d", status)
+	}
+}
+
+// Get gets the blob from Cloudfront.
+func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	log.Debugf("Getting %s from S3", hash[:8])
+	start := time.Now()
+	defer func(t time.Time) {
+		log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
+	}(start)
+
+	status, body, err := c.cfRequest(http.MethodGet, hash)
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
+	}
+	defer func() { _ = body.Close() }()
+	switch status {
+	case http.StatusNotFound, http.StatusForbidden:
+		return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
+	case http.StatusOK:
+		b, err := io.ReadAll(body)
+		if err != nil {
+			return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
+		}
+		metrics.MtrInBytesS3.Add(float64(len(b)))
+		return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil
+	default:
+		return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status)
+	}
+}
+
+func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
+	url := c.endpoint + hash
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		return 0, nil, errors.Err(err)
+	}
+	req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return 0, nil, errors.Err(err)
+	}
+
+	return res.StatusCode, res.Body, nil
+}
+
+func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+func (c *CloudFrontROStore) Delete(_ string) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// Shutdown shuts down the store gracefully
+func (c *CloudFrontROStore) Shutdown() {
+}

store/cloudfront_rw.go

@@ -0,0 +1,62 @@
+package store
+
+import (
+	"time"
+
+	"github.com/lbryio/reflector.go/shared"
+
+	"github.com/lbryio/lbry.go/v2/stream"
+)
+
+// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3.
+type CloudFrontRWStore struct {
+	cf *ITTTStore
+	s3 *S3Store
+}
+
+// NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
+// NOTE: It panics if either argument is nil.
+func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore {
+	if cf == nil || s3 == nil {
+		panic("both stores must be set")
+	}
+	return &CloudFrontRWStore{cf: cf, s3: s3}
+}
+
+const nameCloudFrontRW = "cloudfront_rw"
+
+// Name is the cache type name
+func (c *CloudFrontRWStore) Name() string { return nameCloudFrontRW }
+
+// Has checks if the hash is in the store.
+func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
+	return c.cf.Has(hash)
+}
+
+// Get gets the blob from Cloudfront.
+func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	blob, trace, err := c.cf.Get(hash)
+	return blob, trace.Stack(time.Since(start), c.Name()), err
+}
+
+// Put stores the blob on S3
+func (c *CloudFrontRWStore) Put(hash string, blob stream.Blob) error {
+	return c.s3.Put(hash, blob)
+}
+
+// PutSD stores the sd blob on S3
+func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
+	return c.s3.PutSD(hash, blob)
+}
+
+// Delete deletes the blob from S3
+func (c *CloudFrontRWStore) Delete(hash string) error {
+	return c.s3.Delete(hash)
+}
+
+// Shutdown shuts down the store gracefully
+func (c *CloudFrontRWStore) Shutdown() {
+	c.s3.Shutdown()
+	c.cf.Shutdown()
+}

store/dbbacked.go

@@ -2,35 +2,67 @@ package store
 
 import (
 	"encoding/json"
+	"sync"
+	"time"
 
-	"github.com/lbryio/lbry.go/errors"
 	"github.com/lbryio/reflector.go/db"
+	"github.com/lbryio/reflector.go/shared"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	log "github.com/sirupsen/logrus"
 )
 
-// DBBackedS3Store is an instance of an S3 Store that is backed by a DB for what is stored.
-type DBBackedS3Store struct {
-	s3 *S3BlobStore
-	db *db.SQL
+// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
+type DBBackedStore struct {
+	blobs        BlobStore
+	db           *db.SQL
+	blockedMu    sync.RWMutex
+	blocked      map[string]bool
+	deleteOnMiss bool
 }
 
-// NewDBBackedS3Store returns an initialized store pointer.
-func NewDBBackedS3Store(s3 *S3BlobStore, db *db.SQL) *DBBackedS3Store {
-	return &DBBackedS3Store{s3: s3, db: db}
+// NewDBBackedStore returns an initialized store pointer.
+func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore {
+	return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss}
 }
 
-// Has returns T/F or Error ( if the DB errors ) if store contains the blob.
-func (d *DBBackedS3Store) Has(hash string) (bool, error) {
-	return d.db.HasBlob(hash)
+const nameDBBacked = "db-backed"
+
+// Name is the cache type name
+func (d *DBBackedStore) Name() string { return nameDBBacked }
+
+// Has returns true if the blob is in the store
+func (d *DBBackedStore) Has(hash string) (bool, error) {
+	return d.db.HasBlob(hash, false)
 }
 
-// Get returns the byte slice of the blob or an error.
-func (d *DBBackedS3Store) Get(hash string) ([]byte, error) {
-	return d.s3.Get(hash)
+// Get gets the blob
+func (d *DBBackedStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	has, err := d.db.HasBlob(hash, true)
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
+	}
+	if !has {
+		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), ErrBlobNotFound
+	}
+
+	b, stack, err := d.blobs.Get(hash)
+	if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) {
+		e2 := d.Delete(hash)
+		if e2 != nil {
+			log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
+		}
+	}
+
+	return b, stack.Stack(time.Since(start), d.Name()), err
 }
 
 // Put stores the blob in the S3 store and stores the blob information in the DB.
-func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
-	err := d.s3.Put(hash, blob)
+func (d *DBBackedStore) Put(hash string, blob stream.Blob) error {
+	err := d.blobs.Put(hash, blob)
 	if err != nil {
 		return err
 	}
@@ -40,17 +72,17 @@ func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
 
 // PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if
 // there is an error storing the blob information in the DB.
-func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
+func (d *DBBackedStore) PutSD(hash string, blob stream.Blob) error {
 	var blobContents db.SdBlob
 	err := json.Unmarshal(blob, &blobContents)
 	if err != nil {
-		return err
+		return errors.Err(err)
 	}
 	if blobContents.StreamHash == "" {
 		return errors.Err("sd blob is missing stream hash")
 	}
 
-	err = d.s3.PutSD(hash, blob)
+	err = d.blobs.PutSD(hash, blob)
 	if err != nil {
 		return err
 	}
@@ -58,9 +90,112 @@ func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
 	return d.db.AddSDBlob(hash, len(blob), blobContents)
 }
 
+func (d *DBBackedStore) Delete(hash string) error {
+	err := d.blobs.Delete(hash)
+	if err != nil {
+		return err
+	}
+
+	return d.db.Delete(hash)
+}
+
+// Block deletes the blob and prevents it from being uploaded in the future
+func (d *DBBackedStore) Block(hash string) error {
+	if blocked, err := d.isBlocked(hash); blocked || err != nil {
+		return err
+	}
+
+	log.Debugf("blocking %s", hash)
+
+	err := d.db.Block(hash)
+	if err != nil {
+		return err
+	}
+
+	//has, err := d.db.HasBlob(hash, false)
+	//if err != nil {
+	//	return err
+	//}
+	//
+	//if has {
+	//	err = d.blobs.Delete(hash)
+	//	if err != nil {
+	//		return err
+	//	}
+	//
+	//	err = d.db.Delete(hash)
+	//	if err != nil {
+	//		return err
+	//	}
+	//}
+
+	return d.markBlocked(hash)
+}
+
+// Wants returns false if the hash exists or is blocked, true otherwise
+func (d *DBBackedStore) Wants(hash string) (bool, error) {
+	blocked, err := d.isBlocked(hash)
+	if blocked || err != nil {
+		return false, err
+	}
+
+	has, err := d.Has(hash)
+	return !has, err
+}
+
 // MissingBlobsForKnownStream returns missing blobs for an existing stream
 // WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
 // like no blobs are missing
-func (d *DBBackedS3Store) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
+func (d *DBBackedStore) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
 	return d.db.MissingBlobsForKnownStream(sdHash)
 }
+
+func (d *DBBackedStore) markBlocked(hash string) error {
+	err := d.initBlocked()
+	if err != nil {
+		return err
+	}
+
+	d.blockedMu.Lock()
+	defer d.blockedMu.Unlock()
+
+	d.blocked[hash] = true
+	return nil
+}
+
+func (d *DBBackedStore) isBlocked(hash string) (bool, error) {
+	err := d.initBlocked()
+	if err != nil {
+		return false, err
+	}
+
+	d.blockedMu.RLock()
+	defer d.blockedMu.RUnlock()
+
+	return d.blocked[hash], nil
+}
+
+func (d *DBBackedStore) initBlocked() error {
+	// first check without blocking since this is the most likely scenario
+	if d.blocked != nil {
+		return nil
+	}
+
+	d.blockedMu.Lock()
+	defer d.blockedMu.Unlock()
+
+	// check again in case of race condition
+	if d.blocked != nil {
+		return nil
+	}
+
+	var err error
+	d.blocked, err = d.db.GetBlocked()
+
+	return err
+}
+
+// Shutdown shuts down the store gracefully
+func (d *DBBackedStore) Shutdown() {
+	d.blobs.Shutdown()
+}

store/disk.go

@@ -0,0 +1,146 @@
+package store
+
+import (
+	"os"
+	"path"
+	"time"
+
+	"github.com/lbryio/reflector.go/shared"
+	"github.com/lbryio/reflector.go/store/speedwalk"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+)
+
+// DiskStore stores blobs on a local disk
+type DiskStore struct {
+	// the location of blobs on disk
+	blobDir string
+	// store files in subdirectories based on the first N chars in the filename. 0 = don't create subdirectories.
+	prefixLength int
+
+	// true if initOnce ran, false otherwise
+	initialized bool
+}
+
+// NewDiskStore returns an initialized file disk store pointer.
+func NewDiskStore(dir string, prefixLength int) *DiskStore {
+	return &DiskStore{
+		blobDir:      dir,
+		prefixLength: prefixLength,
+	}
+}
+
+const nameDisk = "disk"
+
+// Name is the cache type name
+func (d *DiskStore) Name() string { return nameDisk }
+
+// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
+func (d *DiskStore) Has(hash string) (bool, error) {
+	err := d.initOnce()
+	if err != nil {
+		return false, err
+	}
+
+	_, err = os.Stat(d.path(hash))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, errors.Err(err)
+	}
+	return true, nil
+}
+
+// Get returns the blob or an error if the blob doesn't exist.
+func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	err := d.initOnce()
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
+	}
+
+	blob, err := os.ReadFile(d.path(hash))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
+		}
+		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
+	}
+	return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
+}
+
+// PutSD stores the sd blob on the disk
+func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
+	return d.Put(hash, blob)
+}
+
+// Delete deletes the blob from the store
+func (d *DiskStore) Delete(hash string) error {
+	err := d.initOnce()
+	if err != nil {
+		return err
+	}
+
+	has, err := d.Has(hash)
+	if err != nil {
+		return err
+	}
+	if !has {
+		return nil
+	}
+
+	err = os.Remove(d.path(hash))
+	return errors.Err(err)
+}
+
+// list returns the hashes of blobs that already exist in the blobDir
+func (d *DiskStore) list() ([]string, error) {
+	err := d.initOnce()
+	if err != nil {
+		return nil, err
+	}
+
+	return speedwalk.AllFiles(d.blobDir, true)
+}
+
+func (d *DiskStore) dir(hash string) string {
+	if d.prefixLength <= 0 || len(hash) < d.prefixLength {
+		return d.blobDir
+	}
+	return path.Join(d.blobDir, hash[:d.prefixLength])
+}
+func (d *DiskStore) tmpDir(hash string) string {
+	return path.Join(d.blobDir, "tmp")
+}
+func (d *DiskStore) path(hash string) string {
+	return path.Join(d.dir(hash), hash)
+}
+func (d *DiskStore) tmpPath(hash string) string {
+	return path.Join(d.tmpDir(hash), hash)
+}
+func (d *DiskStore) ensureDirExists(dir string) error {
+	return errors.Err(os.MkdirAll(dir, 0755))
+}
+
+func (d *DiskStore) initOnce() error {
+	if d.initialized {
+		return nil
+	}
+
+	err := d.ensureDirExists(d.blobDir)
+	if err != nil {
+		return err
+	}
+	err = d.ensureDirExists(path.Join(d.blobDir, "tmp"))
+	if err != nil {
+		return err
+	}
+	d.initialized = true
+	return nil
+}
+
+// Shutdown shuts down the store gracefully
+func (d *DiskStore) Shutdown() {
+}

store/disk_test.go

@@ -0,0 +1,44 @@
+package store
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDiskStore_Get(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "reflector_test_*")
+	require.NoError(t, err)
+	defer func() { _ = os.RemoveAll(tmpDir) }()
+	d := NewDiskStore(tmpDir, 2)
+
+	hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
+	data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
+
+	expectedPath := path.Join(tmpDir, hash[:2], hash)
+	err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
+	require.NoError(t, err)
+	err = os.WriteFile(expectedPath, data, os.ModePerm)
+	require.NoError(t, err)
+
+	blob, _, err := d.Get(hash)
+	assert.NoError(t, err)
+	assert.EqualValues(t, data, blob)
+}
+
+func TestDiskStore_GetNonexistentBlob(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "reflector_test_*")
+	require.NoError(t, err)
+	defer func() { _ = os.RemoveAll(tmpDir) }()
+	d := NewDiskStore(tmpDir, 2)
+
+	blob, _, err := d.Get("nonexistent")
+	assert.Nil(t, blob)
+	assert.True(t, errors.Is(err, ErrBlobNotFound))
+}

store/diskstore_put_darwin.go

@@ -0,0 +1,42 @@
+//go:build darwin
+// +build darwin
+
+package store
+
+import (
+	"bytes"
+	"io"
+	"os"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+)
+
+var openFileFlags = os.O_WRONLY | os.O_CREATE
+
+// Put stores the blob on disk
+func (d *DiskStore) Put(hash string, blob stream.Blob) error {
+	err := d.initOnce()
+	if err != nil {
+		return err
+	}
+
+	err = d.ensureDirExists(d.dir(hash))
+	if err != nil {
+		return err
+	}
+
+	// Open file with O_DIRECT
+	f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
+	if err != nil {
+		return errors.Err(err)
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, bytes.NewReader(blob))
+	if err != nil {
+		return errors.Err(err)
+	}
+	err = os.Rename(d.tmpPath(hash), d.path(hash))
+	return errors.Err(err)
+}