From 2cf4acdb595c07df7f7dca87e23b9dd2939e2684 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Fri, 2 Apr 2021 14:30:36 -0400 Subject: [PATCH] add 'send' command to efficiently send a file to reflector --- cmd/decode.go | 4 +- cmd/root.go | 6 ++ cmd/send.go | 160 +++++++++++++++++++++++++++++++++++++++++++++ cmd/sendblob.go | 6 +- go.mod | 12 ++-- go.sum | 30 +++------ publish/publish.go | 76 ++++++++++----------- wallet/client.go | 4 +- 8 files changed, 221 insertions(+), 77 deletions(-) create mode 100644 cmd/send.go diff --git a/cmd/decode.go b/cmd/decode.go index 4a54668..7cb677f 100644 --- a/cmd/decode.go +++ b/cmd/decode.go @@ -4,7 +4,7 @@ import ( "encoding/hex" "fmt" - "github.com/lbryio/lbry.go/v2/schema/claim" + "github.com/lbryio/lbry.go/v2/schema/stake" "github.com/davecgh/go-spew/spew" "github.com/golang/protobuf/jsonpb" @@ -23,7 +23,7 @@ func init() { } func decodeCmd(cmd *cobra.Command, args []string) { - c, err := claim.DecodeClaimHex(args[0], "") + c, err := stake.DecodeClaimHex(args[0], "") if err != nil { log.Fatal(err) } diff --git a/cmd/root.go b/cmd/root.go index ce28f8f..1e422a7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -163,3 +163,9 @@ func mustGetFlagInt64(cmd *cobra.Command, name string) int64 { checkErr(err) return v } + +func mustGetFlagBool(cmd *cobra.Command, name string) bool { + v, err := cmd.Flags().GetBool(name) + checkErr(err) + return v +} diff --git a/cmd/send.go b/cmd/send.go new file mode 100644 index 0000000..7021866 --- /dev/null +++ b/cmd/send.go @@ -0,0 +1,160 @@ +package cmd + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/signal" + "path" + "syscall" + + "github.com/lbryio/reflector.go/reflector" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/stream" + + "github.com/spf13/cobra" +) + +func init() { + var cmd = &cobra.Command{ + Use: "send ADDRESS:PORT PATH", + Short: "Send a file to a reflector", + Args: cobra.ExactArgs(2), + Run: sendCmd, + } + cmd.PersistentFlags().String("sd-cache", "", "path to dir where sd blobs will be cached") + rootCmd.AddCommand(cmd) +} + +// todo: if retrying a large file is slow, we can add the ability to seek ahead in the file so we're not +// re-uploading blobs that already exist + +var hackyReflector reflector.Client + +func sendCmd(cmd *cobra.Command, args []string) { + reflectorAddress := args[0] + err := hackyReflector.Connect(reflectorAddress) + checkErr(err) + defer hackyReflector.Close() + + filePath := args[1] + file, err := os.Open(filePath) + checkErr(err) + defer file.Close() + + sdCachePath := "" + sdCacheDir := mustGetFlagString(cmd, "sd-cache") + if sdCacheDir != "" { + if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) { + err = os.MkdirAll(sdCacheDir, 0777) + checkErr(err) + } + sdCachePath = path.Join(sdCacheDir, filePath+".sdblob") + } + + var enc *stream.Encoder + + if sdCachePath != "" { + if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) { + sdBlob, err := ioutil.ReadFile(sdCachePath) + checkErr(err) + cachedSDBlob := &stream.SDBlob{} + err = cachedSDBlob.FromBlob(sdBlob) + checkErr(err) + enc = stream.NewEncoderFromSD(file, cachedSDBlob) + } + } + if enc == nil { + enc = stream.NewEncoder(file) + } + + exitCode := 0 + + var killed bool + interruptChan := make(chan os.Signal, 1) + signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM) + go func() { + sig := <-interruptChan + fmt.Printf("caught %s, exiting...\n", sig.String()) + killed = true + exitCode = 1 + }() + + for { + if killed { + break + } + + b, err := enc.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + fmt.Printf("error reading next blob: %v\n", err) + exitCode = 1 + break + } + + err = hackyReflect(b, false) + if err != nil { + fmt.Printf("error reflecting blob %s: %v\n", b.HashHex()[:8], err) + exitCode = 1 + break + } + } + + sd := enc.SDBlob() + //sd.StreamName = filepath.Base(filePath) + //sd.SuggestedFileName = filepath.Base(filePath) + err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666) + if err != nil { + fmt.Printf("error saving sd blob: %v\n", err) + fmt.Println(sd.ToJson()) + exitCode = 1 + } + + if killed { + os.Exit(exitCode) + } + + if reflectorAddress != "" { + err = hackyReflect(sd.ToBlob(), true) + if err != nil { + fmt.Printf("error reflecting sd blob %s: %v\n", sd.HashHex()[:8], err) + exitCode = 1 + } + } + + ret := struct { + SDHash string `json:"sd_hash"` + SourceHash string `json:"source_hash"` + }{ + SDHash: sd.HashHex(), + SourceHash: hex.EncodeToString(enc.SourceHash()), + } + + j, err := json.MarshalIndent(ret, "", " ") + checkErr(err) + fmt.Println(string(j)) + os.Exit(exitCode) +} + +func hackyReflect(b stream.Blob, sd bool) error { + var err error + if sd { + err = hackyReflector.SendSDBlob(b) + } else { + err = hackyReflector.SendBlob(b) + } + + if errors.Is(err, reflector.ErrBlobExists) { + //fmt.Printf("%s already reflected\n", b.HashHex()[:8]) + return nil + } + + return err +} diff --git a/cmd/sendblob.go b/cmd/sendblob.go index 3cd9f05..f37f390 100644 --- a/cmd/sendblob.go +++ b/cmd/sendblob.go @@ -2,7 +2,6 @@ package cmd import ( "crypto/rand" - "io/ioutil" "os" "github.com/lbryio/reflector.go/reflector" @@ -52,9 +51,8 @@ func sendBlobCmd(cmd *cobra.Command, args []string) { file, err := os.Open(path) checkErr(err) - data, err := ioutil.ReadAll(file) - checkErr(err) - s, err := stream.New(data) + defer file.Close() + s, err := stream.New(file) checkErr(err) sdBlob := &stream.SDBlob{} diff --git a/go.mod b/go.mod index 9df120d..0911e81 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/lbryio/reflector.go replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 +//replace github.com/lbryio/lbry.go/v2 => ../lbry.go + require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/aws/aws-sdk-go v1.16.11 @@ -22,25 +24,23 @@ require ( github.com/karrick/godirwalk v1.16.1 github.com/lbryio/chainquery v1.9.0 github.com/lbryio/lbry.go v1.1.2 // indirect - github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 - github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec + github.com/lbryio/lbry.go/v2 v2.7.2-0.20210402181800-ec4d2cb37ce9 + github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 github.com/lucas-clemente/quic-go v0.18.1 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.2 github.com/sirupsen/logrus v1.4.2 - github.com/spf13/afero v1.4.1 + github.com/spf13/afero v1.4.1 // indirect github.com/spf13/cast v1.3.0 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.7.0 github.com/volatiletech/null v8.0.0+incompatible go.uber.org/atomic v1.5.1 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect google.golang.org/appengine v1.6.2 // indirect - gotest.tools v2.2.0+incompatible ) go 1.15 diff --git a/go.sum b/go.sum index 84b207a..a46d5f3 100644 --- a/go.sum +++ b/go.sum @@ -54,16 +54,15 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= +github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw= github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= @@ -81,7 +80,6 @@ github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4er github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= @@ -93,7 +91,6 @@ github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= @@ -150,8 +147,6 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce h1:xdsDDbiBDQTKASoGEZ+pEmF1OnWuu8AQ9I8iNbHNeno= @@ -163,7 +158,6 @@ github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0 github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= @@ -206,21 +200,21 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0= github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= -github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f h1:ovd2wPXzkT80vdP/FX5xcQeXu0i9RAo80SQ6qIsrAjM= github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c= github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY= github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4= +github.com/lbryio/lbry.go/v2 v2.7.2-0.20210402181800-ec4d2cb37ce9 h1:jjIfkL4LdLL/dngZixJTT5NneZbE1LPoWTpDNryuW9Q= +github.com/lbryio/lbry.go/v2 v2.7.2-0.20210402181800-ec4d2cb37ce9/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= -github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= -github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c= -github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= +github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M= +github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys= github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= @@ -350,7 +344,6 @@ github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbd github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spf13/afero v1.1.1 h1:Lt3ihYMlE+lreX1GS4Qw4ZsNpYQLxIXKBTEOXm3nt6I= github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= @@ -370,8 +363,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= @@ -433,7 +427,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko= golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= @@ -447,7 +440,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -470,7 +462,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= @@ -478,7 +469,6 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -538,7 +528,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -551,10 +540,11 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= diff --git a/publish/publish.go b/publish/publish.go index 417978e..9e585e8 100644 --- a/publish/publish.go +++ b/publish/publish.go @@ -3,7 +3,6 @@ package publish import ( "bytes" "encoding/json" - "io/ioutil" "os" "path/filepath" "sort" @@ -21,7 +20,6 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" "github.com/golang/protobuf/proto" - "golang.org/x/crypto/sha3" ) var TODO = ` @@ -43,6 +41,14 @@ var TODO = ` } ` +type Details struct { + Title string + Description string + Author string + Tags []string + ReleaseTime int64 +} + func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) { if name == "" { return nil, nil, errors.Err("name required") @@ -69,11 +75,20 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details return nil, nil, err } - claim, st, err := makeClaimAndStream(path, details) + st, stPB, err := makeStream(path) if err != nil { return nil, nil, err } + stPB.Author = details.Author + stPB.ReleaseTime = details.ReleaseTime + + claim := &pb.Claim{ + Title: details.Title, + Description: details.Description, + Type: &pb.Claim_Stream{Stream: stPB}, + } + err = addClaimToTx(tx, claim, name, amount, addr) if err != nil { return nil, nil, err @@ -203,50 +218,31 @@ func reflect(st stream.Stream, reflectorAddress string) error { return nil } -type Details struct { - Title string - Description string - Author string - Tags []string - ReleaseTime int64 -} - -func makeClaimAndStream(path string, details Details) (*pb.Claim, stream.Stream, error) { +func makeStream(path string) (stream.Stream, *pb.Stream, error) { file, err := os.Open(path) if err != nil { return nil, nil, errors.Err(err) } - data, err := ioutil.ReadAll(file) - if err != nil { - return nil, nil, errors.Err(err) - } - s, err := stream.New(data) + defer file.Close() + + enc := stream.NewEncoder(file) + + s, err := enc.Stream() if err != nil { return nil, nil, errors.Err(err) } - // make the claim - sdBlob := &stream.SDBlob{} - err = sdBlob.FromBlob(s[0]) - if err != nil { - return nil, nil, errors.Err(err) - } - - filehash := sha3.Sum384(data) - - streamPB := &pb.Stream{ - Author: details.Author, - ReleaseTime: details.ReleaseTime, + streamProto := &pb.Stream{ Source: &pb.Source{ - SdHash: s[0].Hash(), + SdHash: enc.SDBlob().Hash(), Name: filepath.Base(file.Name()), - Size: uint64(len(data)), - Hash: filehash[:], + Size: uint64(enc.SourceLen()), + Hash: enc.SourceHash(), }, } mimeType, category := guessMimeType(filepath.Ext(file.Name())) - streamPB.Source.MediaType = mimeType + streamProto.Source.MediaType = mimeType switch category { case "video": @@ -254,20 +250,14 @@ func makeClaimAndStream(path string, details Details) (*pb.Claim, stream.Stream, //if err != nil { // return nil, nil, err //} - streamPB.Type = &pb.Stream_Video{} + streamProto.Type = &pb.Stream_Video{} case "audio": - streamPB.Type = &pb.Stream_Audio{} + streamProto.Type = &pb.Stream_Audio{} case "image": - streamPB.Type = &pb.Stream_Image{} + streamProto.Type = &pb.Stream_Image{} } - claim := &pb.Claim{ - Title: details.Title, - Description: details.Description, - Type: &pb.Claim_Stream{Stream: streamPB}, - } - - return claim, s, nil + return s, streamProto, nil } func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) { diff --git a/wallet/client.go b/wallet/client.go index a4cb165..3dc0e8c 100644 --- a/wallet/client.go +++ b/wallet/client.go @@ -6,7 +6,7 @@ import ( "github.com/lbryio/chainquery/lbrycrd" "github.com/lbryio/lbry.go/v2/extras/errors" - "github.com/lbryio/lbry.go/v2/schema/claim" + "github.com/lbryio/lbry.go/v2/schema/stake" types "github.com/lbryio/types/v2/go" "github.com/btcsuite/btcutil" @@ -140,7 +140,7 @@ func (n *Node) GetClaimInTx(txid string, nout int) (*types.Claim, error) { return nil, errors.Err(err) } - ch, err := claim.DecodeClaimBytes(value, "") + ch, err := stake.DecodeClaimBytes(value, "") if err != nil { return nil, errors.Err(err) }