LBRYCommunity/reflector.go
1
0
Fork 0
Code Issues 7 Pull requests Projects Releases 10 Packages Wiki Activity

track approximate access time for blobs #44

Merged
lyoshenka merged 4 commits from track_access into master 2020-10-06 00:11:16 +02:00
Conversation 1 Commits 4 Files changed 18 +879 -61
18 changed files with 879 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cmd/getstream.go
View file

@ -3,6 +3,7 @@ package cmd
import (
"encoding/hex"
"os"
"time"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/store"
@ -49,7 +50,12 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
log.Fatal(err)
}
f, err := os.Create(wd + "/" + sd.SuggestedFileName)
filename := sd.SuggestedFileName
if filename == "" {
filename = "stream_" + time.Now().Format("20060102_150405")
}
f, err := os.Create(wd + "/" + filename)
if err != nil {
log.Fatal(err)
}

65
cmd/publish.go Normal file
View file

@ -0,0 +1,65 @@
package cmd
import (
"fmt"
"github.com/lbryio/reflector.go/publish"
"github.com/lbryio/lbry.go/v2/lbrycrd"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "publish FILE",
Short: "Publish a file",
Args: cobra.ExactArgs(1),
Run: publishCmd,
}
cmd.Flags().String("name", "", "Claim name")
cmd.Flags().String("title", "", "Title of the content")
cmd.Flags().String("description", "", "Description of the content")
cmd.Flags().String("author", "", "Content author")
cmd.Flags().String("tags", "", "Comma-separated list of tags")
cmd.Flags().Int64("release-time", 0, "original public release of content, seconds since UNIX epoch")
rootCmd.AddCommand(cmd)
}
func publishCmd(cmd *cobra.Command, args []string) {
var err error
claimName := mustGetFlagString(cmd, "name")
if claimName == "" {
log.Errorln("--name required")
return
}
path := args[0]
client, err := lbrycrd.NewWithDefaultURL(nil)
checkErr(err)
tx, txid, err := publish.Publish(
client,
path,
claimName,
"bSzpgkTnAoiT2YAhUShPpfpajPESfNXVTu",
publish.Details{
Title: mustGetFlagString(cmd, "title"),
Description: mustGetFlagString(cmd, "description"),
Author: mustGetFlagString(cmd, "author"),
Tags: nil,
ReleaseTime: mustGetFlagInt64(cmd, "release-time"),
},
"reflector.lbry.com:5566",
)
checkErr(err)
decoded, err := publish.Decode(client, tx)
checkErr(err)
fmt.Printf("TX: %s\n\n", decoded)
fmt.Printf("TXID: %s\n", txid.String())
}

7
cmd/reflector.go
View file

@ -25,6 +25,7 @@ var http3PeerPort int
var receiverPort int
var metricsPort int
var disableUploads bool
var disableBlocklist bool
var proxyAddress string
var proxyPort string
var proxyProtocol string
@ -47,12 +48,13 @@ func init() {
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics")
cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not")
rootCmd.AddCommand(cmd)
}
func reflectorCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector version %s, built %s", meta.Version, meta.BuildTime.Format(time.RFC3339))
log.Printf("reflector %s", meta.VersionString())
var blobStore store.BlobStore
if proxyAddress != "" {
@ -84,6 +86,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
if useDB {
db := new(db.SQL)
db.TrackAccessTime = true
err = db.Connect(globalConfig.DBConn)
if err != nil {
log.Fatal(err)
@ -93,7 +96,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
reflectorServer = reflector.NewServer(blobStore)
reflectorServer.Timeout = 3 * time.Minute
reflectorServer.EnableBlocklist = true
reflectorServer.EnableBlocklist = !disableBlocklist
err = reflectorServer.Start(":" + strconv.Itoa(receiverPort))
if err != nil {

16
cmd/root.go
View file

@ -4,6 +4,7 @@ import (
"encoding/json"
"io/ioutil"
"os"
"strings"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/extras/errors"
@ -68,8 +69,11 @@ func preRun(cmd *cobra.Command, args []string) {
debugLogger.SetOutput(os.Stderr)
if util.InSlice(verboseAll, verbose) {
logrus.Info("global verbose logging enabled")
logrus.SetLevel(logrus.DebugLevel)
verbose = []string{verboseDHT, verboseNodeFinder}
} else if len(verbose) > 0 {
logrus.Infof("verbose logging enabled for: %s", strings.Join(verbose, ", "))
}
for _, debugType := range verbose {
@ -147,3 +151,15 @@ func loadConfig(path string) (Config, error) {
err = json.Unmarshal(raw, &c)
return c, errors.Err(err)
}
func mustGetFlagString(cmd *cobra.Command, name string) string {
v, err := cmd.Flags().GetString(name)
checkErr(err)
return v
}
func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
v, err := cmd.Flags().GetInt64(name)
checkErr(err)
return v
}

33
cmd/sendblob.go
View file

@ -2,6 +2,8 @@ package cmd
import (
"crypto/rand"
"io/ioutil"
"os"
"github.com/lbryio/reflector.go/reflector"
@ -13,9 +15,9 @@ import (
func init() {
var cmd = &cobra.Command{
Use: "sendblob ADDRESS:PORT",
Use: "sendblob ADDRESS:PORT [PATH]",
Short: "Send a random blob to a reflector server",
Args: cobra.ExactArgs(1),
Args: cobra.RangeArgs(1, 2),
Run: sendBlobCmd,
}
rootCmd.AddCommand(cmd)
@ -23,6 +25,10 @@ func init() {
func sendBlobCmd(cmd *cobra.Command, args []string) {
addr := args[0]
var path string
if len(args) >= 2 {
path = args[1]
}
c := reflector.Client{}
err := c.Connect(addr)
@ -30,6 +36,7 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
log.Fatal("error connecting client to server: ", err)
}
if path == "" {
blob := make(stream.Blob, 1024)
_, err = rand.Read(blob)
if err != nil {
@ -40,4 +47,26 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
if err != nil {
log.Error(err)
}
return
}
file, err := os.Open(path)
checkErr(err)
data, err := ioutil.ReadAll(file)
checkErr(err)
s, err := stream.New(data)
checkErr(err)
sdBlob := &stream.SDBlob{}
err = sdBlob.FromBlob(s[0])
checkErr(err)
for i, b := range s {
if i == 0 {
err = c.SendSDBlob(b)
} else {
err = c.SendBlob(b)
}
checkErr(err)
}
}

2
cmd/test.go
View file

@ -27,7 +27,7 @@ func init() {
}
func testCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector version %s", meta.Version)
log.Printf("reflector %s", meta.VersionString())
memStore := store.NewMemoryBlobStore()

3
cmd/version.go
View file

@ -2,7 +2,6 @@ package cmd
import (
"fmt"
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/spf13/cobra"
@ -18,5 +17,5 @@ func init() {
}
func versionCmd(cmd *cobra.Command, args []string) {
fmt.Printf("version %s (built %s)\n", meta.Version, meta.BuildTime.Format(time.RFC3339))
fmt.Println(meta.VersionString())
}

81
db/db.go
View file

@ -32,6 +32,8 @@ type SdBlob struct {
// SQL implements the DB interface
type SQL struct {
conn *sql.DB
TrackAccessTime bool
}
func logQuery(query string, args ...interface{}) {
@ -75,9 +77,10 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
return 0, errors.Err("length must be positive")
}
args := []interface{}{hash, isStored, length}
blobID, err := s.exec(
"INSERT INTO blob_ (hash, is_stored, length) VALUES (?,?,?) ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
hash, isStored, length,
"INSERT INTO blob_ (hash, is_stored, length) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
args...,
)
if err != nil {
return 0, err
@ -97,9 +100,10 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
}
func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
args := []interface{}{hash, sdBlobID, time.Now()}
streamID, err := s.exec(
"INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (?,?)",
hash, sdBlobID,
"INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES ("+qt.Qs(len(args))+")",
args...,
)
if err != nil {
return 0, errors.Err(err)
@ -113,6 +117,13 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
if streamID == 0 {
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
}
if s.TrackAccessTime {
err := s.touch([]uint64{uint64(streamID)})
if err != nil {
return 0, errors.Err(err)
}
}
}
return streamID, nil
}
@ -128,12 +139,44 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
s.touch(streamsNeedingTouch)
return exists, err
}
func (s *SQL) touch(streamIDs []uint64) error {
if len(streamIDs) == 0 {
return nil
}
var hash string
query := "UPDATE stream SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(streamIDs)) + ")"
args := make([]interface{}, len(streamIDs)+1)
args[0] = time.Now()
for i := range streamIDs {
args[i+1] = streamIDs[i]
}
startTime := time.Now()
_, err := s.exec(query, args...)
log.Debugf("stream access query touched %d streams and took %s", len(streamIDs), time.Since(startTime))
return errors.Err(err)
}
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
if s.conn == nil {
return nil, nil, errors.Err("not connected")
}
var (
hash string
streamID uint64
lastAccessedAt time.Time
)
var needsTouch []uint64
exists := make(map[string]bool)
touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
maxBatchSize := 10000
doneIndex := 0
@ -145,7 +188,13 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
batch := hashes[doneIndex:sliceEnd]
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + qt.Qs(len(batch)) + ")"
// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
query := `SELECT b.hash, s.id, s.last_accessed_at
FROM blob_ b
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
WHERE b.is_stored = ? and b.hash IN (` + qt.Qs(len(batch)) + `)`
args := make([]interface{}, len(batch)+1)
args[0] = true
for i := range batch {
@ -164,11 +213,14 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
defer closeRows(rows)
for rows.Next() {
err := rows.Scan(&hash)
err := rows.Scan(&hash, &streamID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
if s.TrackAccessTime && lastAccessedAt.Before(touchDeadline) {
needsTouch = append(needsTouch, streamID)
}
}
err = rows.Err()
@ -180,11 +232,11 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
return nil
}()
if err != nil {
return nil, err
return nil, nil, err
}
}
return exists, nil
return exists, needsTouch, nil
}
// Delete will remove the blob from the db
@ -309,9 +361,10 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
return err
}
args := []interface{}{streamID, blobID, contentBlob.BlobNum}
_, err = s.exec(
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES (?,?,?)",
streamID, blobID, contentBlob.BlobNum,
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
args...,
)
if err != nil {
return errors.Err(err)
@ -482,9 +535,11 @@ CREATE TABLE stream (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
hash char(96) NOT NULL,
sd_blob_id BIGINT UNSIGNED NOT NULL,
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY stream_hash_idx (hash),
KEY stream_sd_blob_id_idx (sd_blob_id),
KEY last_accessed_at_idx (last_accessed_at),
FOREIGN KEY (sd_blob_id) REFERENCES blob_ (id) ON DELETE RESTRICT ON UPDATE CASCADE
);

12
go.mod
View file

@ -1,13 +1,15 @@
module github.com/lbryio/reflector.go
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
require (
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/aws/aws-sdk-go v1.16.11
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 // indirect
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
github.com/davecgh/go-spew v1.1.1
github.com/go-sql-driver/mysql v1.4.1
github.com/golang/protobuf v1.4.0
github.com/golang/protobuf v1.4.2
github.com/google/btree v1.0.0 // indirect
github.com/google/gops v0.3.7
github.com/gorilla/mux v1.7.4
@ -18,10 +20,11 @@ require (
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07
github.com/lbryio/chainquery v1.9.0
github.com/lbryio/lbry.go v1.1.2 // indirect
github.com/lbryio/go_mediainfo v0.0.0-20200109212001-4c7318fd92ad
github.com/lbryio/lbry.go v1.1.2
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec
github.com/lucas-clemente/quic-go v0.17.2
github.com/lucas-clemente/quic-go v0.18.0
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
github.com/prometheus/client_golang v0.9.2
github.com/sirupsen/logrus v1.4.2
@ -29,6 +32,7 @@ require (
github.com/spf13/cobra v0.0.3
github.com/spf13/pflag v1.0.3 // indirect
go.uber.org/atomic v1.5.1
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect
google.golang.org/appengine v1.6.2 // indirect

45
go.sum
View file

@ -26,6 +26,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0=
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
@ -61,6 +62,8 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
@ -76,11 +79,14 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -91,6 +97,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@ -195,11 +203,16 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/go_mediainfo v0.0.0-20200109212001-4c7318fd92ad h1:sfsYsbQXjA1uE51yFNtv4k79yMBzeGgAusK6KrLrcYs=
github.com/lbryio/go_mediainfo v0.0.0-20200109212001-4c7318fd92ad/go.mod h1:xJtOuuGAo32jLOmrdBWRCURfahqz6OvK/hdSLhmYA38=
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f h1:ovd2wPXzkT80vdP/FX5xcQeXu0i9RAo80SQ6qIsrAjM=
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY=
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
@ -210,6 +223,8 @@ github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiV
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lucas-clemente/quic-go v0.17.2 h1:4iQInIuNQkPNZmsy9rCnwuOzpH0qGnDo4jn0QfI/qE4=
github.com/lucas-clemente/quic-go v0.17.2/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
github.com/lucas-clemente/quic-go v0.18.0 h1:JhQDdqxdwdmGdKsKgXi1+coHRoGhvU6z0rNzOJqZ/4o=
github.com/lucas-clemente/quic-go v0.18.0/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg=
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
@ -220,8 +235,14 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/marten-seemann/qpack v0.1.0 h1:/0M7lkda/6mus9B8u34Asqm8ZhHAAt9Ho0vniNuVSVg=
github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
github.com/marten-seemann/qpack v0.2.0 h1:/r1rhZoOmgxVKBqPNnYilZBDEyw+6OUHCbBzA5jc2y0=
github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ=
github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc=
github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
github.com/marten-seemann/qtls-go1-15 v0.1.0 h1:i/YPXVxz8q9umso/5y474CNcHmTpA+5DH+mFPjx6PZg=
github.com/marten-seemann/qtls-go1-15 v0.1.0/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
@ -244,15 +265,20 @@ github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
@ -361,6 +387,7 @@ github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6Ut
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
@ -380,6 +407,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU=
golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -408,6 +437,9 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -419,6 +451,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -430,12 +463,19 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -478,11 +518,13 @@ google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoA
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@ -492,6 +534,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
@ -509,6 +552,8 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=

22
meta/meta.go
View file

@ -1,6 +1,7 @@
package meta
import (
"fmt"
"strconv"
"time"
)
@ -12,9 +13,24 @@ var BuildTime time.Time
func init() {
if Time != "" {
t, err := strconv.Atoi(Time)
if err != nil {
return
}
if err == nil {
BuildTime = time.Unix(int64(t), 0).UTC()
}
}
}
func VersionString() string {
version := Version
if version == "" {
version = "<unset>"
}
var buildTime string
if BuildTime.IsZero() {
buildTime = "<now>"
} else {
buildTime = BuildTime.Format(time.RFC3339)
}
return fmt.Sprintf("version %s, built %s", version, buildTime)
}

2
peer/client.go
View file

@ -153,7 +153,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, error) {
return nil, err
}
return stream.Blob(blob), nil
return blob, nil
}
func (c *Client) read(v interface{}) error {

2
peer/http3/server.go
View file

@ -163,7 +163,7 @@ func generateTLSConfig() *tls.Config {
func (s *Server) listenAndServe(server *http3.Server) {
err := server.ListenAndServe()
if err != nil {
if err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Errorln(errors.FullTrace(err))
}
}

174
publish/mimetypes.go Normal file
View file

@ -0,0 +1,174 @@
package publish
import "strings"
func guessMimeType(ext string) (string, string) {
if ext == "" {
return "application/octet-stream", "binary"
}
ext = strings.ToLower(strings.TrimLeft(strings.TrimSpace(ext), "."))
types := map[string]struct{ mime, t string }{
"a": {"application/octet-stream", "binary"},
"ai": {"application/postscript", "image"},
"aif": {"audio/x-aiff", "audio"},
"aifc": {"audio/x-aiff", "audio"},
"aiff": {"audio/x-aiff", "audio"},
"au": {"audio/basic", "audio"},
"avi": {"video/x-msvideo", "video"},
"bat": {"text/plain", "document"},
"bcpio": {"application/x-bcpio", "binary"},
"bin": {"application/octet-stream", "binary"},
"bmp": {"image/bmp", "image"},
"c": {"text/plain", "document"},
"cdf": {"application/x-netcdf", "binary"},
"cpio": {"application/x-cpio", "binary"},
"csh": {"application/x-csh", "binary"},
"css": {"text/css", "document"},
"csv": {"text/csv", "document"},
"dll": {"application/octet-stream", "binary"},
"doc": {"application/msword", "document"},
"dot": {"application/msword", "document"},
"dvi": {"application/x-dvi", "binary"},
"eml": {"message/rfc822", "document"},
"eps": {"application/postscript", "document"},
"epub": {"application/epub+zip", "document"},
"etx": {"text/x-setext", "document"},
"exe": {"application/octet-stream", "binary"},
"gif": {"image/gif", "image"},
"gtar": {"application/x-gtar", "binary"},
"h": {"text/plain", "document"},
"hdf": {"application/x-hdf", "binary"},
"htm": {"text/html", "document"},
"html": {"text/html", "document"},
"ico": {"image/vnd.microsoft.icon", "image"},
"ief": {"image/ief", "image"},
"iges": {"model/iges", "model"},
"jpe": {"image/jpeg", "image"},
"jpeg": {"image/jpeg", "image"},
"jpg": {"image/jpeg", "image"},
"js": {"application/javascript", "document"},
"json": {"application/json", "document"},
"ksh": {"text/plain", "document"},
"latex": {"application/x-latex", "binary"},
"m1v": {"video/mpeg", "video"},
"m3u": {"application/vnd.apple.mpegurl", "audio"},
"m3u8": {"application/vnd.apple.mpegurl", "audio"},
"man": {"application/x-troff-man", "document"},
"markdown": {"text/markdown", "document"},
"md": {"text/markdown", "document"},
"me": {"application/x-troff-me", "binary"},
"mht": {"message/rfc822", "document"},
"mhtml": {"message/rfc822", "document"},
"mif": {"application/x-mif", "binary"},
"mov": {"video/quicktime", "video"},
"movie": {"video/x-sgi-movie", "video"},
"mp2": {"audio/mpeg", "audio"},
"mp3": {"audio/mpeg", "audio"},
"mp4": {"video/mp4", "video"},
"mpa": {"video/mpeg", "video"},
"mpe": {"video/mpeg", "video"},
"mpeg": {"video/mpeg", "video"},
"mpg": {"video/mpeg", "video"},
"ms": {"application/x-troff-ms", "binary"},
"nc": {"application/x-netcdf", "binary"},
"nws": {"message/rfc822", "document"},
"o": {"application/octet-stream", "binary"},
"obj": {"application/octet-stream", "model"},
"oda": {"application/oda", "binary"},
"p12": {"application/x-pkcs12", "binary"},
"p7c": {"application/pkcs7-mime", "binary"},
"pbm": {"image/x-portable-bitmap", "image"},
"pdf": {"application/pdf", "document"},
"pfx": {"application/x-pkcs12", "binary"},
"pgm": {"image/x-portable-graymap", "image"},
"pl": {"text/plain", "document"},
"png": {"image/png", "image"},
"pnm": {"image/x-portable-anymap", "image"},
"pot": {"application/vnd.ms-powerpoint", "document"},
"ppa": {"application/vnd.ms-powerpoint", "document"},
"ppm": {"image/x-portable-pixmap", "image"},
"pps": {"application/vnd.ms-powerpoint", "document"},
"ppt": {"application/vnd.ms-powerpoint", "document"},
"ps": {"application/postscript", "document"},
"pwz": {"application/vnd.ms-powerpoint", "document"},
"py": {"text/x-python", "document"},
"pyc": {"application/x-python-code", "binary"},
"pyo": {"application/x-python-code", "binary"},
"qt": {"video/quicktime", "video"},
"ra": {"audio/x-pn-realaudio", "audio"},
"ram": {"application/x-pn-realaudio", "audio"},
"ras": {"image/x-cmu-raster", "image"},
"rdf": {"application/xml", "binary"},
"rgb": {"image/x-rgb", "image"},
"roff": {"application/x-troff", "binary"},
"rtx": {"text/richtext", "document"},
"sgm": {"text/x-sgml", "document"},
"sgml": {"text/x-sgml", "document"},
"sh": {"application/x-sh", "document"},
"shar": {"application/x-shar", "binary"},
"snd": {"audio/basic", "audio"},
"so": {"application/octet-stream", "binary"},
"src": {"application/x-wais-source", "binary"},
"stl": {"model/stl", "model"},
"sv4cpio": {"application/x-sv4cpio", "binary"},
"sv4crc": {"application/x-sv4crc", "binary"},
"svg": {"image/svg+xml", "image"},
"swf": {"application/x-shockwave-flash", "binary"},
"t": {"application/x-troff", "binary"},
"tar": {"application/x-tar", "binary"},
"tcl": {"application/x-tcl", "binary"},
"tex": {"application/x-tex", "binary"},
"texi": {"application/x-texinfo", "binary"},
"texinfo": {"application/x-texinfo", "binary"},
"tif": {"image/tiff", "image"},
"tiff": {"image/tiff", "image"},
"tr": {"application/x-troff", "binary"},
"tsv": {"text/tab-separated-values", "document"},
"txt": {"text/plain", "document"},
"ustar": {"application/x-ustar", "binary"},
"vcf": {"text/x-vcard", "document"},
"wav": {"audio/x-wav", "audio"},
"webm": {"video/webm", "video"},
"wiz": {"application/msword", "document"},
"wsdl": {"application/xml", "document"},
"xbm": {"image/x-xbitmap", "image"},
"xlb": {"application/vnd.ms-excel", "document"},
"xls": {"application/vnd.ms-excel", "document"},
"xml": {"text/xml", "document"},
"xpdl": {"application/xml", "document"},
"xpm": {"image/x-xpixmap", "image"},
"xsl": {"application/xml", "document"},
"xwd": {"image/x-xwindowdump", "image"},
"zip": {"application/zip", "binary"},
// These are non-standard types, commonly found in the wild.
"cbr": {"application/vnd.comicbook-rar", "document"},
"cbz": {"application/vnd.comicbook+zip", "document"},
"flac": {"audio/flac", "audio"},
"lbry": {"application/x-ext-lbry", "document"},
"m4v": {"video/m4v", "video"},
"mid": {"audio/midi", "audio"},
"midi": {"audio/midi", "audio"},
"mkv": {"video/x-matroska", "video"},
"mobi": {"application/x-mobipocket-ebook", "document"},
"oga": {"audio/ogg", "audio"},
"ogv": {"video/ogg", "video"},
"pct": {"image/pict", "image"},
"pic": {"image/pict", "image"},
"pict": {"image/pict", "image"},
"prc": {"application/x-mobipocket-ebook", "document"},
"rtf": {"application/rtf", "document"},
"xul": {"text/xul", "document"},
// microsoft is special and has its own "standard"
// https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
"wmv": {"video/x-ms-wmv", "video"},
}
if data, ok := types[ext]; ok {
return data.mime, data.t
}
return "application/x-ext-" + ext, "binary"
}

304
publish/publish.go Normal file
View file

@ -0,0 +1,304 @@
package publish
import (
"bytes"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"sort"
"github.com/lbryio/reflector.go/reflector"
mediainfo "github.com/lbryio/go_mediainfo"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/lbrycrd"
"github.com/lbryio/lbry.go/v2/stream"
pb "github.com/lbryio/types/v2/go"
"github.com/btcsuite/btcd/btcjson"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/txscript"
"github.com/btcsuite/btcd/wire"
"github.com/btcsuite/btcutil"
"github.com/golang/protobuf/proto"
"golang.org/x/crypto/sha3"
)
var TODO = `
import cert from wallet
get all utxos from chainquery
create transaction
sign it with the channel
track state of utxos across publishes from this channel so that we can just do one query to get utxos
prioritize only confirmed utxos
Handling all the issues we handle currently with lbrynet:
"Couldn't find private key for id",
"You already have a stream claim published under the name",
"Cannot publish using channel",
"txn-mempool-conflict",
"too-long-mempool-chain",
"Missing inputs",
"Not enough funds to cover this transaction",
}
`
func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) {
if name == "" {
return nil, nil, errors.Err("name required")
}
//TODO: sign claim if publishing into channel
addr, err := btcutil.DecodeAddress(address, &lbrycrd.MainNetParams)
if errors.Is(err, btcutil.ErrUnknownAddressType) {
return nil, nil, errors.Err(`unknown address type. here's what you need to make this work:
- deprecatedrpc=validateaddress" and "deprecatedrpc=signrawtransaction" in your lbrycrd.conf
- github.com/btcsuite/btcd pinned to hash 306aecffea32
- github.com/btcsuite/btcutil pinned to 4c204d697803
- github.com/lbryio/lbry.go/v2 (make sure you have v2 at the end)`)
}
if err != nil {
return nil, nil, err
}
amount := 0.01
changeAddr := addr // TODO: fix this? or maybe its fine?
tx, err := baseTx(client, amount, changeAddr)
if err != nil {
return nil, nil, err
}
claim, st, err := makeClaimAndStream(path, details)
if err != nil {
return nil, nil, err
}
err = addClaimToTx(tx, claim, name, amount, addr)
if err != nil {
return nil, nil, err
}
// sign and send
signedTx, allInputsSigned, err := client.SignRawTransaction(tx)
if err != nil {
return nil, nil, err
}
if !allInputsSigned {
return nil, nil, errors.Err("not all inputs for the tx could be signed")
}
err = reflect(st, reflectorAddress)
if err != nil {
return nil, nil, err
}
txid, err := client.SendRawTransaction(signedTx, false)
if err != nil {
return nil, nil, err
}
return signedTx, txid, nil
}
//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
txFee := 0.0002 // TODO: estimate this better?
inputs, total, err := coinChooser(client, amount+txFee)
if err != nil {
return nil, err
}
change := total - amount - txFee
// create base raw tx
addresses := make(map[btcutil.Address]btcutil.Amount)
//changeAddr, err := client.GetNewAddress("")
changeAmount, err := btcutil.NewAmount(change)
if err != nil {
return nil, err
}
addresses[changeAddress] = changeAmount
lockTime := int64(0)
return client.CreateRawTransaction(inputs, addresses, &lockTime)
}
func coinChooser(client *lbrycrd.Client, amount float64) ([]btcjson.TransactionInput, float64, error) {
utxos, err := client.ListUnspentMin(1)
if err != nil {
return nil, 0, err
}
sort.Slice(utxos, func(i, j int) bool { return utxos[i].Amount < utxos[j].Amount })
var utxo btcjson.ListUnspentResult
for _, u := range utxos {
if u.Spendable && u.Amount >= amount {
utxo = u
break
}
}
if utxo.TxID == "" {
return nil, 0, errors.Err("not enough utxos to create tx")
}
return []btcjson.TransactionInput{{Txid: utxo.TxID, Vout: utxo.Vout}}, utxo.Amount, nil
}
func addClaimToTx(tx *wire.MsgTx, claim *pb.Claim, name string, amount float64, claimAddress btcutil.Address) error {
claimBytes, err := proto.Marshal(claim)
if err != nil {
return err
}
claimBytes = append([]byte{0}, claimBytes...) // version 0 = no channel sig
amt, err := btcutil.NewAmount(amount)
if err != nil {
return err
}
script, err := getClaimPayoutScript(name, claimBytes, claimAddress)
if err != nil {
return err
}
tx.AddTxOut(wire.NewTxOut(int64(amt), script))
return nil
}
func Decode(client *lbrycrd.Client, tx *wire.MsgTx) (string, error) {
buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
if err := tx.Serialize(buf); err != nil {
return "", errors.Err(err)
}
//txHex := hex.EncodeToString(buf.Bytes())
//spew.Dump(txHex)
decoded, err := client.DecodeRawTransaction(buf.Bytes())
if err != nil {
return "", err
}
data, err := json.MarshalIndent(decoded, "", " ")
return string(data), err
}
func reflect(st stream.Stream, reflectorAddress string) error {
// upload blobs to reflector
c := reflector.Client{}
err := c.Connect(reflectorAddress)
if err != nil {
return errors.Err(err)
}
for i, b := range st {
if i == 0 {
err = c.SendSDBlob(b)
} else {
err = c.SendBlob(b)
}
if err != nil {
return errors.Err(err)
}
}
return nil
}
type Details struct {
Title string
Description string
Author string
Tags []string
ReleaseTime int64
}
func makeClaimAndStream(path string, details Details) (*pb.Claim, stream.Stream, error) {
file, err := os.Open(path)
if err != nil {
return nil, nil, errors.Err(err)
}
data, err := ioutil.ReadAll(file)
if err != nil {
return nil, nil, errors.Err(err)
}
s, err := stream.New(data)
if err != nil {
return nil, nil, errors.Err(err)
}
// make the claim
sdBlob := &stream.SDBlob{}
err = sdBlob.FromBlob(s[0])
if err != nil {
return nil, nil, errors.Err(err)
}
filehash := sha3.Sum384(data)
streamPB := &pb.Stream{
Author: details.Author,
ReleaseTime: details.ReleaseTime,
Source: &pb.Source{
SdHash: s[0].Hash(),
Name: filepath.Base(file.Name()),
Size: uint64(len(data)),
Hash: filehash[:],
},
}
mimeType, category := guessMimeType(filepath.Ext(file.Name()))
streamPB.Source.MediaType = mimeType
switch category {
case "video":
t, err := streamVideoMetadata(path)
if err != nil {
return nil, nil, err
}
streamPB.Type = t
case "audio":
streamPB.Type = &pb.Stream_Audio{}
case "image":
streamPB.Type = &pb.Stream_Image{}
}
claim := &pb.Claim{
Title: details.Title,
Description: details.Description,
Type: &pb.Claim_Stream{Stream: streamPB},
}
return claim, s, nil
}
func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) {
//OP_CLAIM_NAME <name> <value> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
pkscript, err := txscript.PayToAddrScript(address)
if err != nil {
return nil, errors.Err(err)
}
return txscript.NewScriptBuilder().
AddOp(txscript.OP_NOP6). //OP_CLAIM_NAME
AddData([]byte(name)). //<name>
AddData(value). //<value>
AddOp(txscript.OP_2DROP). //OP_2DROP
AddOp(txscript.OP_DROP). //OP_DROP
AddOps(pkscript). //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
Script()
}
func streamVideoMetadata(path string) (*pb.Stream_Video, error) {
mi, err := mediainfo.GetMediaInfo(path)
if err != nil {
return nil, err
}
return &pb.Stream_Video{
Video: &pb.Video{
Duration: uint32(mi.General.Duration / 1000),
Height: uint32(mi.Video.Height),
Width: uint32(mi.Video.Width),
},
}, nil
}

59
publish/wallet.go Normal file
View file

@ -0,0 +1,59 @@
package publish
import (
"encoding/json"
"io"
)
func LoadWallet(r io.Reader) (WalletFile, error) {
var w WalletFile
err := json.NewDecoder(r).Decode(&w)
return w, err
}
type WalletFile struct {
Name string `json:"name"`
Version int `json:"version"`
Preferences WalletPrefs `json:"preferences"`
Accounts []Account `json:"accounts"`
}
type Account struct {
AddressGenerator AddressGenerator `json:"address_generator"`
Certificates map[string]string `json:"certificates"`
Encrypted bool `json:"encrypted"`
Ledger string `json:"ledger"`
ModifiedOn float64 `json:"modified_on"`
Name string `json:"name"`
PrivateKey string `json:"private_key"`
PublicKey string `json:"public_key"`
Seed string `json:"seed"`
}
type AddressGenerator struct {
Name string `json:"name"`
Change AddressGenParams `json:"change"` // should "change" and "receiving" be replaced with a map[string]AddressGenParams?
Receiving AddressGenParams `json:"receiving"`
}
type AddressGenParams struct {
Gap int `json:"gap"`
MaximumUsesPerAddress int `json:"maximum_uses_per_address"`
}
type WalletPrefs struct {
Shared struct {
Ts float64 `json:"ts"`
Value struct {
Type string `json:"type"`
Value struct {
AppWelcomeVersion int `json:"app_welcome_version"`
Blocked []interface{} `json:"blocked"`
Sharing3P bool `json:"sharing_3P"`
Subscriptions []string `json:"subscriptions"`
Tags []string `json:"tags"`
} `json:"value"`
Version string `json:"version"`
} `json:"value"`
} `json:"shared"`
}

55
reflector/client.go
View file

@ -2,12 +2,11 @@ package reflector
import (
"encoding/json"
"log"
"net"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
// ErrBlobExists is a default error for when a blob already exists on the reflector server.
@ -36,8 +35,18 @@ func (c *Client) Close() error {
return c.conn.Close()
}
// SendBlob sends a send blob request to the client.
// SendBlob sends a blob to the server.
func (c *Client) SendBlob(blob stream.Blob) error {
return c.sendBlob(blob, false)
}
// SendSDBlob sends an SD blob request to the server.
func (c *Client) SendSDBlob(blob stream.Blob) error {
return c.sendBlob(blob, true)
}
// sendBlob does the actual blob sending
func (c *Client) sendBlob(blob stream.Blob, isSDBlob bool) error {
if !c.connected {
return errors.Err("not connected")
}
@ -47,10 +56,15 @@ func (c *Client) SendBlob(blob stream.Blob) error {
}
blobHash := blob.HashHex()
sendRequest, err := json.Marshal(sendBlobRequest{
BlobSize: blob.Size(),
BlobHash: blobHash,
})
var req sendBlobRequest
if isSDBlob {
req.SdBlobSize = blob.Size()
req.SdBlobHash = blobHash
} else {
req.BlobSize = blob.Size()
req.BlobHash = blobHash
}
sendRequest, err := json.Marshal(req)
if err != nil {
return err
}
@ -62,31 +76,52 @@ func (c *Client) SendBlob(blob stream.Blob) error {
dec := json.NewDecoder(c.conn)
if isSDBlob {
var sendResp sendSdBlobResponse
err = dec.Decode(&sendResp)
if err != nil {
return err
}
if !sendResp.SendSdBlob {
return errors.Prefix(blobHash[:8], ErrBlobExists)
}
log.Println("Sending SD blob " + blobHash[:8])
} else {
var sendResp sendBlobResponse
err = dec.Decode(&sendResp)
if err != nil {
return err
}
if !sendResp.SendBlob {
return errors.Prefix(blobHash[:8], ErrBlobExists)
}
log.Println("Sending blob " + blobHash[:8])
}
_, err = c.conn.Write(blob)
if err != nil {
return err
}
if isSDBlob {
var transferResp sdBlobTransferResponse
err = dec.Decode(&transferResp)
if err != nil {
return err
}
if !transferResp.ReceivedSdBlob {
return errors.Err("server did not received SD blob")
}
} else {
var transferResp blobTransferResponse
err = dec.Decode(&transferResp)
if err != nil {
return err
}
if !transferResp.ReceivedBlob {
return errors.Err("server did not received blob")
}
}
return nil
}

8
store/dbbacked.go
View file

@ -32,6 +32,14 @@ func (d *DBBackedStore) Has(hash string) (bool, error) {
// Get gets the blob
func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
has, err := d.db.HasBlob(hash)
if err != nil {
return nil, err
}
if !has {
return nil, ErrBlobNotFound
}
return d.blobs.Get(hash)
}