LBRYCommunity/reflector.go
1
0
Fork 0
Code Issues 7 Pull requests Projects Releases 10 Packages Wiki Activity

upgrade dependencies & introduce github ci #67

Merged
nikooo777 merged 2 commits from upgrades into master 2023-03-09 19:12:59 +01:00
Conversation 0 Commits 2 Files changed 37 +617 -560
37 changed files with 617 additions and 560 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
.github/workflows/go.yml vendored Normal file
View file

@ -0,0 +1,37 @@
name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.19.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Print changes since last version
run: git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline

62
.github/workflows/release.yml vendored Normal file
View file

@ -0,0 +1,62 @@
name: release
on:
push:
tags:
- "*.*.*"
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.19.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: Zip macos
run: zip -r reflector_darwin_amd64.zip ./dist/darwin_amd64
- name: Zip linux
run: zip -r reflector_linux_amd64.zip ./dist/linux_amd64
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Generate Changelog
run: git log $(git describe --tags --abbrev=0 @^)..@ --no-merges --oneline > ${{ github.workspace }}-CHANGELOG.txt
- name: upload to github releases
uses: softprops/action-gh-release@v1
with:
files: |
./reflector_linux_amd64.zip
./reflector_darwin_amd64.zip
body_path: ${{ github.workspace }}-CHANGELOG.txt
# - name: Login to DockerHub
# uses: docker/login-action@v2
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
# - name: Generate docker image
# run: make image
# - name: Docker push
# run: make publish_image

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
/vendor
/config.json*
/dist
/bin

9
.travis.yml
View file

@ -2,11 +2,8 @@ os: linux
dist: bionic
language: go
#env:
# - GO111MODULE=on
go:
- 1.17.x
- 1.19.x
cache:
directories:
@ -59,11 +56,11 @@ script:
# one last linter - ignore autogen code
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
# Finally, build the binary
- make
- make linux
deploy:
- provider: s3
local_dir: ./bin
local_dir: ./dist/linux_amd64
skip_cleanup: true
on:
repo: lbryio/reflector.go

2
Dockerfile
View file

@ -3,7 +3,7 @@ EXPOSE 8080
RUN mkdir /app
WORKDIR /app
COPY bin/prism-bin ./prism
COPY dist/linux_amd64/prism-bin ./prism
RUN chmod +x prism
ENTRYPOINT [ "/app/prism" ]

44
Makefile
View file

@ -1,25 +1,33 @@
version := $(shell git describe --dirty --always --long --abbrev=7)
commit := $(shell git rev-parse --short HEAD)
commit_long := $(shell git rev-parse HEAD)
branch := $(shell git rev-parse --abbrev-ref HEAD)
curTime := $(shell date +%s)
BINARY=prism-bin
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
BIN_DIR = ${DIR}/bin
IMPORT_PATH = github.com/lbryio/reflector.go
LDFLAGS="-X ${IMPORT_PATH}/meta.version=$(version) -X ${IMPORT_PATH}/meta.commit=$(commit) -X ${IMPORT_PATH}/meta.commitLong=$(commit_long) -X ${IMPORT_PATH}/meta.branch=$(branch) -X '${IMPORT_PATH}/meta.Time=$(curTime)'"
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
BIN_DIR = $(DIR)/dist
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
.PHONY: build clean test lint
.DEFAULT_GOAL: build
build:
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
clean:
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
.DEFAULT_GOAL := linux
.PHONY: test
test:
go test ./... -v -cover
go test -cover -v ./...
.PHONY: lint
lint:
go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
./scripts/lint.sh
.PHONY: linux
linux:
GOARCH=amd64 GOOS=linux go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/linux_amd64/${BINARY}
.PHONY: macos
macos:
GOARCH=amd64 GOOS=darwin go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/darwin_amd64/${BINARY}
.PHONY: image
image:
docker buildx build -t lbry/reflector:$(version) -t lbry/reflector:latest --platform linux/amd64 .

4
cluster/cluster.go
View file

@ -1,7 +1,7 @@
package cluster
import (
"io/ioutil"
"io"
baselog "log"
"sort"
"time"
@ -52,7 +52,7 @@ func (c *Cluster) Connect() error {
conf.MemberlistConfig.AdvertisePort = c.port
conf.NodeName = c.name
nullLogger := baselog.New(ioutil.Discard, "", 0)
nullLogger := baselog.New(io.Discard, "", 0)
conf.Logger = nullLogger
c.eventCh = make(chan serf.Event)

3
cmd/integrity.go
View file

@ -3,7 +3,6 @@ package cmd
import (
"crypto/sha512"
"encoding/hex"
"io/ioutil"
"os"
"path"
"runtime"
@ -72,7 +71,7 @@ func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int,
log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
}
blobPath := path.Join(diskStorePath, b[:2], b)
blob, err := ioutil.ReadFile(blobPath)
blob, err := os.ReadFile(blobPath)
if err != nil {
if os.IsNotExist(err) {
continue

3
cmd/root.go
View file

@ -2,7 +2,6 @@ package cmd
import (
"encoding/json"
"io/ioutil"
"os"
"strings"
@ -142,7 +141,7 @@ func argFuncs(funcs ...cobra.PositionalArgs) cobra.PositionalArgs {
func loadConfig(path string) (Config, error) {
var c Config
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return c, errors.Err("config file not found")

10
cmd/send.go
View file

@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/signal"
"path"
@ -39,13 +38,12 @@ func sendCmd(cmd *cobra.Command, args []string) {
reflectorAddress := args[0]
err := hackyReflector.Connect(reflectorAddress)
checkErr(err)
defer hackyReflector.Close()
defer func() { _ = hackyReflector.Close() }()
filePath := args[1]
file, err := os.Open(filePath)
checkErr(err)
defer file.Close()
defer func() { _ = file.Close() }()
sdCachePath := ""
sdCacheDir := mustGetFlagString(cmd, "sd-cache")
if sdCacheDir != "" {
@ -60,7 +58,7 @@ func sendCmd(cmd *cobra.Command, args []string) {
if sdCachePath != "" {
if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
sdBlob, err := ioutil.ReadFile(sdCachePath)
sdBlob, err := os.ReadFile(sdCachePath)
checkErr(err)
cachedSDBlob := &stream.SDBlob{}
err = cachedSDBlob.FromBlob(sdBlob)
@ -110,7 +108,7 @@ func sendCmd(cmd *cobra.Command, args []string) {
sd := enc.SDBlob()
//sd.StreamName = filepath.Base(filePath)
//sd.SuggestedFileName = filepath.Base(filePath)
err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666)
err = os.WriteFile(sdCachePath, sd.ToBlob(), 0666)
if err != nil {
fmt.Printf("error saving sd blob: %v\n", err)
fmt.Println(sd.ToJson())

2
cmd/sendblob.go
View file

@ -51,7 +51,7 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
file, err := os.Open(path)
checkErr(err)
defer file.Close()
defer func() { _ = file.Close() }()
s, err := stream.New(file)
checkErr(err)

2
cmd/version.go
View file

@ -18,5 +18,5 @@ func init() {
}
func versionCmd(cmd *cobra.Command, args []string) {
fmt.Println(meta.VersionString())
fmt.Println(meta.FullName())
}

2
db/db.go
View file

@ -17,7 +17,7 @@ import (
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null"
"github.com/volatiletech/null/v8"
"go.uber.org/atomic"
)

151
go.mod
View file

@ -1,121 +1,124 @@
module github.com/lbryio/reflector.go
go 1.17
go 1.19
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
require (
github.com/aws/aws-sdk-go v1.44.5
github.com/aws/aws-sdk-go v1.44.217
github.com/bluele/gcache v0.0.2
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
github.com/davecgh/go-spew v1.1.1
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
github.com/gin-gonic/gin v1.7.7
github.com/go-sql-driver/mysql v1.6.0
github.com/gogo/protobuf v1.2.1
github.com/golang/protobuf v1.5.2
github.com/google/gops v0.3.22
github.com/gin-gonic/gin v1.9.0
github.com/go-sql-driver/mysql v1.7.0
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.3
github.com/google/gops v0.3.27
github.com/gorilla/mux v1.8.0
github.com/hashicorp/serf v0.9.7
github.com/hashicorp/serf v0.10.1
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/johntdyer/slackrus v0.0.0-20211215141436-33e4a270affb
github.com/johntdyer/slackrus v0.0.0-20220912135606-861993969176
github.com/karrick/godirwalk v1.17.0
github.com/lbryio/chainquery v1.8.1
github.com/lbryio/lbry.go/v2 v2.7.2-0.20220321182539-d0aeb0c22b22
github.com/lbryio/chainquery v1.9.1-0.20230308044402-068c29e02865
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
github.com/lucas-clemente/quic-go v0.28.1
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/prometheus/client_golang v1.12.1
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cast v1.4.1
github.com/spf13/cobra v1.4.0
github.com/stretchr/testify v1.7.1
github.com/volatiletech/null v8.0.0+incompatible
go.uber.org/atomic v1.9.0
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
github.com/prometheus/client_golang v1.14.0
github.com/quic-go/quic-go v0.33.0
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cast v1.5.0
github.com/spf13/cobra v1.6.1
github.com/stretchr/testify v1.8.2
github.com/volatiletech/null/v8 v8.1.2
go.uber.org/atomic v1.10.0
golang.org/x/sync v0.1.0
)
require (
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
github.com/armon/go-metrics v0.4.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/bytedance/sonic v1.8.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/friendsofgo/errors v0.9.2 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-ini/ini v1.48.0 // indirect
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.11.2 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/goccy/go-json v0.10.0 // indirect
github.com/gofrs/uuid v4.2.0+incompatible // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/google/btree v1.0.0 // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/gorilla/rpc v1.2.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v0.5.3 // indirect
github.com/hashicorp/go-multierror v1.1.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.0 // indirect
github.com/hashicorp/golang-lru v0.5.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/memberlist v0.3.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/hashicorp/memberlist v0.5.0 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/lbryio/lbry.go v1.1.2 // indirect
github.com/leodido/go-urn v1.2.0 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
github.com/magiconair/properties v1.8.1 // indirect
github.com/marten-seemann/qpack v0.2.1 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/miekg/dns v1.1.41 // indirect
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nlopes/slack v0.6.0 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/pelletier/go-toml v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/onsi/ginkgo/v2 v2.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/qtls-go1-19 v0.2.1 // indirect
github.com/quic-go/qtls-go1-20 v0.1.1 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 // indirect
github.com/spf13/afero v1.1.2 // indirect
github.com/spf13/jwalterweatherman v1.0.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/slack-go/slack v0.12.1 // indirect
github.com/spf13/afero v1.9.3 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.7.1 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d // indirect
github.com/volatiletech/sqlboiler v3.4.0+incompatible // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
golang.org/x/mod v0.4.2 // indirect
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
golang.org/x/tools v0.1.1 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/ini.v1 v1.51.0 // indirect
github.com/spf13/viper v1.15.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.9 // indirect
github.com/volatiletech/inflect v0.0.1 // indirect
github.com/volatiletech/randomize v0.0.1 // indirect
github.com/volatiletech/strmangle v0.0.4 // indirect
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
golang.org/x/crypto v0.7.0 // indirect
golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

645
go.sum
View file

File diff suppressed because it is too large Load diff

4
lite_db/db.go
View file

@ -10,7 +10,7 @@ import (
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null"
"github.com/volatiletech/null/v8"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
@ -111,7 +111,7 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
s.touch(streamsNeedingTouch)
_ = s.touch(streamsNeedingTouch)
return exists, err
}

38
meta/meta.go
View file

@ -6,9 +6,36 @@ import (
"time"
)
var Version = ""
var Time = ""
var BuildTime time.Time
var (
name = "prism-bin"
version = "unknown"
commit = "unknown"
commitLong = "unknown"
branch = "unknown"
Time = "unknown"
BuildTime time.Time
)
// Name returns main application name
func Name() string {
return name
}
// Version returns current application version
func Version() string {
return version
}
// FullName returns current app version, commit and build time
func FullName() string {
return fmt.Sprintf(
`Name: %v
Version: %v
branch: %v
commit: %v
commit long: %v
build date: %v`, Name(), Version(), branch, commit, commitLong, BuildTime.String())
}
func init() {
if Time != "" {
@ -20,11 +47,6 @@ func init() {
}
func VersionString() string {
version := Version
if version == "" {
version = "<unset>"
}
var buildTime string
if BuildTime.IsZero() {
buildTime = "<now>"

3
publish/publish.go
View file

@ -222,8 +222,7 @@ func makeStream(path string) (stream.Stream, *pb.Stream, error) {
if err != nil {
return nil, nil, errors.Err(err)
}
defer file.Close()
defer func() { _ = file.Close() }()
enc := stream.NewEncoder(file)
s, err := enc.Stream()

6
readme.md
View file

@ -13,7 +13,7 @@ This code includes a Go implementations of the LBRY peer protocol, reflector pro
- Install mysql 8 (5.7 might work too)
- add a reflector user and database with password `reflector` with localhost access only
- Create the tables as described [here](https://github.com/lbryio/reflector.go/blob/ittt/db/db.go#L735) (the link might not update as the code does so just look for the schema in that file)
- Create the tables as described [here](https://github.com/lbryio/reflector.go/blob/master/db/db.go#L735) (the link might not update as the code does so just look for the schema in that file)
#### We do not support running reflector.go as a blob receiver, however if you want to run it as a private blobcache you may compile it yourself and run it as following:
```bash
@ -96,7 +96,7 @@ Flags:
```
## Running from Source
This project requires [Go v1.16](https://golang.org/doc/install).
This project requires [Go v1.19](https://golang.org/doc/install).
On Ubuntu you can install it with `sudo snap install go --classic`
@ -104,7 +104,7 @@ On Ubuntu you can install it with `sudo snap install go --classic`
git clone git@github.com:lbryio/reflector.go.git
cd reflector.go
make
./bin/prism-bin
./dist/linux_amd64/prism-bin
```
## Contributing

2
reflector/protected_content.go
View file

@ -42,7 +42,7 @@ func GetBlockedContent() (map[string]bool, error) {
if err != nil {
return nil, errors.Err(err)
}
defer res.Body.Close()
defer func() { _ = res.Body.Close() }()
if res.StatusCode != http.StatusOK {
return nil, errors.Err("unexpected status code %d", res.StatusCode)
}

3
reflector/server.go
View file

@ -6,7 +6,6 @@ import (
"encoding/hex"
"encoding/json"
"io"
"io/ioutil"
"net"
"time"
@ -367,7 +366,7 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
dec := json.NewDecoder(conn)
err = dec.Decode(v)
if err != nil {
data, _ := ioutil.ReadAll(dec.Buffered())
data, _ := io.ReadAll(dec.Buffered())
if len(data) > 0 {
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
}

3
reflector/uploader.go
View file

@ -1,7 +1,6 @@
package reflector
import (
"io/ioutil"
"os"
"path"
"sync"
@ -165,7 +164,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
}
}()
blob, err := ioutil.ReadFile(filepath)
blob, err := os.ReadFile(filepath)
if err != nil {
return errors.Err(err)
}

26
scripts/lint.sh Executable file
View file

@ -0,0 +1,26 @@
#!/usr/bin/env bash
err=0
trap 'err=1' ERR
# All the .go files, excluding auto generated folders
GO_FILES=$(find . -iname '*.go' -type f)
(
go install golang.org/x/tools/cmd/goimports@latest # Used in build script for generated files
# go install golang.org/x/lint/golint@latest # Linter
go install github.com/jgautheron/gocyclo@latest # Check against high complexity
go install github.com/mdempsky/unconvert@latest # Identifies unnecessary type conversions
go install github.com/kisielk/errcheck@latest # Checks for unhandled errors
go install github.com/opennota/check/cmd/varcheck@latest # Checks for unused vars
go install github.com/opennota/check/cmd/structcheck@latest # Checks for unused fields in structs
)
echo "Running varcheck..." && varcheck $(go list ./...)
echo "Running structcheck..." && structcheck $(go list ./...)
# go vet is the official Go static analyzer
echo "Running go vet..." && go vet $(go list ./...)
# checks for unhandled errors
echo "Running errcheck..." && errcheck $(go list ./...)
# check for unnecessary conversions - ignore autogen code
echo "Running unconvert..." && unconvert -v $(go list ./...)
echo "Running gocyclo..." && gocyclo -ignore "_test" -avg -over 28 $GO_FILES
#echo "Running golint..." && golint -set_exit_status $(go list ./...)
test $err = 0 # Return non-zero if any command failed

6
server/http3/client.go
View file

@ -16,7 +16,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lucas-clemente/quic-go/http3"
"github.com/quic-go/quic-go/http3"
log "github.com/sirupsen/logrus"
)
@ -69,7 +69,7 @@ func (c *Client) HasBlob(hash string) (bool, error) {
if err != nil {
return false, errors.Err(err)
}
defer resp.Body.Close()
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusOK {
return true, nil
}
@ -86,7 +86,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer resp.Body.Close()
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotFound {
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)

4
server/http3/server.go
View file

@ -22,8 +22,8 @@ import (
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/gorilla/mux"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
log "github.com/sirupsen/logrus"
)

8
server/http3/store.go
View file

@ -14,8 +14,8 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
)
// Store is a blob store that gets blobs from a peer.
@ -74,7 +74,7 @@ func (p *Store) Has(hash string) (bool, error) {
if err != nil {
return false, err
}
defer c.Close()
defer func() { _ = c.Close() }()
return c.HasBlob(hash)
}
@ -93,7 +93,7 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
defer func() { _ = c.Close() }()
return c.GetBlob(hash)
}

9
server/peer/server_test.go
View file

@ -92,7 +92,7 @@ func TestRequestFromConnection(t *testing.T) {
if err != nil {
t.Error("error opening connection", err)
}
defer conn.Close()
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write(p.request)
@ -120,14 +120,17 @@ func TestInvalidData(t *testing.T) {
if err != nil {
t.Error("error opening connection", err)
}
defer conn.Close()
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write([]byte("hello dear server, I would like blobs. Please"))
if err != nil {
t.Error("error writing", err)
}
conn.SetReadDeadline(time.Now().Add(5 * time.Second))
err = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
if err != nil {
t.Error("error setting read deadline", err)
}
_, err = conn.Read(response)
if err != io.EOF {
t.Error("error reading", err)

4
server/peer/store.go
View file

@ -42,7 +42,7 @@ func (p *Store) Has(hash string) (bool, error) {
if err != nil {
return false, err
}
defer c.Close()
defer func() { _ = c.Close() }()
return c.HasBlob(hash)
}
@ -53,7 +53,7 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
defer func() { _ = c.Close() }()
blob, trace, err := c.GetBlob(hash)
if err != nil && strings.Contains(err.Error(), "blob not found") {
return nil, trace, store.ErrBlobNotFound

3
store/atime_linux.go
View file

@ -1,3 +1,4 @@
//go:build linux
// +build linux
package store
@ -9,7 +10,7 @@ import (
)
func timespecToTime(ts syscall.Timespec) time.Time {
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
return time.Unix(ts.Sec, ts.Nsec)
}
func atime(fi os.FileInfo) time.Time {

10
store/cloudfront_ro.go
View file

@ -2,7 +2,6 @@ package store
import (
"io"
"io/ioutil"
"net/http"
"time"
@ -37,8 +36,7 @@ func (c *CloudFrontROStore) Has(hash string) (bool, error) {
if err != nil {
return false, err
}
defer body.Close()
defer func() { _ = body.Close() }()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return false, nil
@ -61,12 +59,12 @@ func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, err
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
}
defer body.Close()
defer func() { _ = body.Close() }()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
case http.StatusOK:
b, err := ioutil.ReadAll(body)
b, err := io.ReadAll(body)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
}
@ -83,7 +81,7 @@ func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser,
if err != nil {
return 0, nil, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version)
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
res, err := http.DefaultClient.Do(req)
if err != nil {

3
store/disk.go
View file

@ -1,7 +1,6 @@
package store
import (
"io/ioutil"
"os"
"path"
"time"
@ -62,7 +61,7 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
blob, err := ioutil.ReadFile(d.path(hash))
blob, err := os.ReadFile(d.path(hash))
if err != nil {
if os.IsNotExist(err) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)

11
store/disk_test.go
View file

@ -1,7 +1,6 @@
package store
import (
"io/ioutil"
"os"
"path"
"path/filepath"
@ -14,9 +13,9 @@ import (
)
func TestDiskStore_Get(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(tmpDir, 2)
hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
@ -25,7 +24,7 @@ func TestDiskStore_Get(t *testing.T) {
expectedPath := path.Join(tmpDir, hash[:2], hash)
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
require.NoError(t, err)
err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
err = os.WriteFile(expectedPath, data, os.ModePerm)
require.NoError(t, err)
blob, _, err := d.Get(hash)
@ -34,9 +33,9 @@ func TestDiskStore_Get(t *testing.T) {
}
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(tmpDir, 2)
blob, _, err := d.Get("nonexistent")

6
store/diskstore_put_linux.go
View file

@ -34,14 +34,12 @@ func (d *DiskStore) Put(hash string, blob stream.Blob) error {
if err != nil {
return errors.Err(err)
}
defer f.Close()
defer func() { _ = f.Close() }()
dio, err := directio.New(f)
if err != nil {
return errors.Err(err)
}
defer dio.Flush()
defer func() { _ = dio.Flush() }()
_, err = io.Copy(dio, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)

2
store/gcache.go
View file

@ -59,7 +59,7 @@ func NewGcacheStore(component string, store BlobStore, maxSize int, strategy Evi
}
go func() {
if lstr, ok := store.(lister); ok {
err := l.loadExisting(lstr, int(maxSize))
err := l.loadExisting(lstr, maxSize)
if err != nil {
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
}

5
store/gcache_test.go
View file

@ -2,7 +2,6 @@ package store
import (
"fmt"
"io/ioutil"
"os"
"reflect"
"testing"
@ -88,9 +87,9 @@ func TestGcacheStore_UnderlyingBlobMissing(t *testing.T) {
}
func TestGcacheStore_loadExisting(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(tmpDir, 2)
hash := "hash"

23
store/http.go
View file

@ -2,8 +2,8 @@ package store
import (
"bytes"
"context"
"io"
"io/ioutil"
"net"
"net/http"
"sync"
@ -46,7 +46,7 @@ func (n *HttpStore) Has(hash string) (bool, error) {
if err != nil {
return false, errors.Err(err)
}
defer res.Body.Close()
defer func() { _ = res.Body.Close() }()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
@ -55,7 +55,7 @@ func (n *HttpStore) Has(hash string) (bool, error) {
}
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
body, _ = io.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
@ -76,7 +76,7 @@ func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
defer res.Body.Close()
defer func() { _ = res.Body.Close() }()
tmp := getBuffer()
defer putBuffer(tmp)
serialized := res.Header.Get("Via")
@ -105,7 +105,7 @@ func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
}
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
body, _ = io.ReadAll(res.Body)
}
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
@ -143,14 +143,19 @@ func putBuffer(buf *bytes.Buffer) {
buffers.Put(buf)
}
func dialContext(ctx context.Context, network, address string) (net.Conn, error) {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}
return dialer.DialContext(ctx, network, address)
}
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
func getClient() *http.Client {
// Customize the Transport to have larger connection pool
defaultTransport := &http.Transport{
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
DialContext: dialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,

17
store/speedwalk/speedwalk.go
View file

@ -1,7 +1,8 @@
package speedwalk
import (
"io/ioutil"
"io/fs"
"os"
"path/filepath"
"runtime"
"sync"
@ -17,7 +18,19 @@ import (
// AllFiles recursively lists every file in every subdirectory of a given directory
// If basename is true, return the basename of each file. Otherwise return the full path starting at startDir.
func AllFiles(startDir string, basename bool) ([]string, error) {
items, err := ioutil.ReadDir(startDir)
entries, err := os.ReadDir(startDir)
if err != nil {
return nil, err
}
items := make([]fs.FileInfo, 0, len(entries))
for _, entry := range entries {
info, err := entry.Info()
if err != nil {
return nil, err
}
items = append(items, info)
}
if err != nil {
return nil, err
}