upgrade dependencies

adjust transaction summary fields

.travis.yml

@@ -1,8 +1,8 @@
 os: linux
-dist: xenial
+dist: bionic
 language: go
 go:
-  - 1.15.x
+  - 1.17.x
 
 env:
   global:

dht/config.go

@@ -17,7 +17,7 @@ const (
 
 	// TODO: all these constants should be defaults, and should be used to set values in the standard Config. then the code should use values in the config
 	// TODO: alternatively, have a global Config for constants. at least that way tests can modify the values
-	alpha           = 3             // this is the constant alpha in the spec
+	alpha           = 5             // this is the constant alpha in the spec
 	bucketSize      = 8             // this is the constant k in the spec
 	nodeIDLength    = bits.NumBytes // bytes. this is the constant B in the spec
 	messageIDLength = 20            // bytes.

dht/contact.go

@@ -2,6 +2,7 @@ package dht
 
 import (
 	"bytes"
+	"encoding/json"
 	"net"
 	"sort"
 	"strconv"
@@ -41,6 +42,20 @@ func (c Contact) String() string {
 	return str
 }
 
+func (c Contact) MarshalJSON() ([]byte, error) {
+	return json.Marshal(&struct {
+		ID       string
+		IP       string
+		Port     int
+		PeerPort int
+	}{
+		ID:       c.ID.Hex(),
+		IP:       c.IP.String(),
+		Port:     c.Port,
+		PeerPort: c.PeerPort,
+	})
+}
+
 // MarshalCompact returns a compact byteslice representation of the contact
 // NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
 func (c Contact) MarshalCompact() ([]byte, error) {

dht/message.go

@@ -40,6 +40,7 @@ const (
 	headerPayloadField   = "3"
 	headerArgsField      = "4"
 	contactsField        = "contacts"
+	pageField            = "p"
 	tokenField           = "token"
 	protocolVersionField = "protocolVersion"
 )
@@ -270,6 +271,7 @@ type Response struct {
 	FindValueKey    string
 	Token           string
 	ProtocolVersion int
+	Page            uint8
 }
 
 func (r Response) argsDebug() string {
@@ -390,10 +392,18 @@ func (r *Response) UnmarshalBencode(b []byte) error {
 
 	if contacts, ok := rawData[contactsField]; ok {
 		err = bencode.DecodeBytes(contacts, &r.Contacts)
+		delete(rawData, contactsField) // so it doesnt mess up findValue key finding below
 		if err != nil {
 			return err
 		}
-	} else {
+	}
+	if page, ok := rawData[pageField]; ok {
+		err = bencode.DecodeBytes(page, &r.Page)
+		delete(rawData, pageField) // so it doesnt mess up findValue key finding below
+		if err != nil {
+			return err
+		}
+	}
 	for k, v := range rawData {
 		r.FindValueKey = k
 		var compactContacts [][]byte
@@ -411,7 +421,6 @@ func (r *Response) UnmarshalBencode(b []byte) error {
 		}
 		break
 	}
-	}
 
 	return nil
 }

dht/rpc.go

@@ -102,6 +102,7 @@ type RpcIterativeFindValueArgs struct {
 type RpcIterativeFindValueResult struct {
 	Contacts   []Contact
 	FoundValue bool
+	Values     []Contact
 }
 
 func (rpc *rpcReceiver) IterativeFindValue(r *http.Request, args *RpcIterativeFindValueArgs, result *RpcIterativeFindValueResult) error {
@@ -109,12 +110,19 @@ func (rpc *rpcReceiver) IterativeFindValue(r *http.Request, args *RpcIterativeFi
 	if err != nil {
 		return err
 	}
-	foundContacts, found, err := FindContacts(rpc.dht.node, key, false, nil)
+	foundContacts, found, err := FindContacts(rpc.dht.node, key, true, nil)
 	if err != nil {
 		return err
 	}
 	result.Contacts = foundContacts
 	result.FoundValue = found
+	if found {
+		for _, contact := range foundContacts {
+			if contact.PeerPort > 0 {
+				result.Values = append(result.Values, contact)
+			}
+		}
+	}
 	return nil
 }
 

extras/api/server.go

@@ -17,6 +17,12 @@ import (
 // ResponseHeaders are returned with each response
 var ResponseHeaders map[string]string
 
+// CorsDomains Allowed domains for CORS Policy
+var CorsDomains []string
+
+// CorsAllowLocalhost if true localhost connections are always allowed
+var CorsAllowLocalhost bool
+
 // Log allows logging of events and errors
 var Log = func(*http.Request, *Response, error) {}
 
@@ -77,6 +83,32 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			w.Header().Set(key, value)
 		}
 	}
+	origin := r.Header.Get("origin")
+	for _, d := range CorsDomains {
+		if d == origin {
+			w.Header().Set("Access-Control-Allow-Origin", d)
+			vary := w.Header().Get("Vary")
+			if vary != "*" {
+				if vary != "" {
+					vary += ", "
+				}
+				vary += "Origin"
+			}
+			w.Header().Set("Vary", vary)
+		}
+	}
+
+	if CorsAllowLocalhost && strings.HasPrefix(origin, "http://localhost:") {
+		w.Header().Set("Access-Control-Allow-Origin", origin)
+		vary := w.Header().Get("Vary")
+		if vary != "*" {
+			if vary != "" {
+				vary += ", "
+			}
+			vary += "Origin"
+		}
+		w.Header().Set("Vary", vary)
+	}
 
 	// Stop here if its a preflighted OPTIONS request
 	if r.Method == "OPTIONS" {
@@ -89,6 +121,9 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if rsp.Error != nil {
 			ogErr := errors.Unwrap(rsp.Error)
 			if statusError, ok := ogErr.(StatusError); ok {
+				if statusError.Status == 0 {
+					statusError.Status = http.StatusInternalServerError
+				}
 				rsp.Status = statusError.Status
 			} else {
 				rsp.Status = http.StatusInternalServerError

extras/jsonrpc/daemon.go

@@ -15,16 +15,29 @@ import (
 	"github.com/mitchellh/mapstructure"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
-	"github.com/ybbus/jsonrpc"
+	"github.com/ybbus/jsonrpc/v2"
 )
 
 const DefaultPort = 5279
 
+const (
+	ErrorWalletNotLoaded     = "WalletNotLoadedError"
+	ErrorWalletAlreadyLoaded = "WalletAlreadyLoadedError"
+	ErrorWalletNotFound      = "WalletNotFoundError"
+	ErrorWalletAlreadyExists = "WalletAlreadyExistsError"
+)
+
 type Client struct {
 	conn    jsonrpc.RPCClient
 	address string
 }
 
+type Error struct {
+	Code    int
+	Name    string
+	Message string
+}
+
 func NewClient(address string) *Client {
 	d := Client{}
 
@@ -70,6 +83,15 @@ func Decode(data interface{}, targetStruct interface{}) error {
 	return nil
 }
 
+// WrapError adds error metadata from JSONRPC error response for clients to access
+func WrapError(rpcError *jsonrpc.RPCError) Error {
+	e := Error{Code: rpcError.Code, Message: rpcError.Message}
+	if d, ok := rpcError.Data.(map[string]interface{}); ok {
+		e.Name = d["name"].(string)
+	}
+	return e
+}
+
 func decodeNumber(data interface{}) (decimal.Decimal, error) {
 	var number string
 
@@ -106,6 +128,10 @@ func debugParams(params map[string]interface{}) string {
 	return strings.Join(s, " ")
 }
 
+func (e Error) Error() string {
+	return fmt.Sprintf("Error in daemon: %s", e.Message)
+}
+
 func (d *Client) callNoDecode(command string, params map[string]interface{}) (interface{}, error) {
 	log.Debugln("jsonrpc: " + command + " " + debugParams(params))
 	r, err := d.conn.Call(command, params)
@@ -114,7 +140,7 @@ func (d *Client) callNoDecode(command string, params map[string]interface{}) (in
 	}
 
 	if r.Error != nil {
-		return nil, errors.Err("Error in daemon: " + r.Error.Message)
+		return nil, WrapError(r.Error)
 	}
 
 	return r.Result, nil
@@ -138,6 +164,21 @@ func (d *Client) SetRPCTimeout(timeout time.Duration) {
 //				NEW SDK
 //============================================
 
+func (d *Client) AccountSend(accountID *string, amount, toAddress string) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	args := struct {
+		AccountID *string `json:"account_id"`
+		Amount    string  `json:"amount"`
+		Addresses string  `json:"addresses"`
+	}{
+		AccountID: accountID,
+		Amount:    amount,
+		Addresses: toAddress,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "account_send", structs.Map(args))
+}
+
 func (d *Client) AccountList(page uint64, pageSize uint64) (*AccountListResponse, error) {
 	response := new(AccountListResponse)
 	return response, d.call(response, "account_list", map[string]interface{}{
@@ -220,6 +261,13 @@ func (d *Client) AddressUnused(account *string) (*AddressUnusedResponse, error)
 	})
 }
 
+func (d *Client) TransactionShow(txid string) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	return response, d.call(response, "transaction_show", map[string]interface{}{
+		"txid": txid,
+	})
+}
+
 func (d *Client) ChannelList(account *string, page uint64, pageSize uint64, wid *string) (*ChannelListResponse, error) {
 	if page == 0 {
 		return nil, errors.Err("pages start from 1")
@@ -270,6 +318,7 @@ type ChannelCreateOptions struct {
 	CoverURL           *string  `json:"cover_url,omitempty"`
 	Featured           []string `json:"featured,omitempty"`
 	AccountID          *string  `json:"account_id,omitempty"`
+	FundingAccountIDs  []string `json:"funding_account_ids,omitempty"`
 }
 
 func (d *Client) ChannelCreate(name string, bid float64, options ChannelCreateOptions) (*TransactionSummary, error) {
@@ -529,24 +578,35 @@ func (d *Client) Resolve(urls string) (*ResolveResponse, error) {
 	})
 }
 
-func (d *Client) ClaimSearch(claimName, claimID, txid *string, nout *uint, page uint64, pageSize uint64) (*ClaimSearchResponse, error) {
+type ClaimSearchArgs struct {
-	response := new(ClaimSearchResponse)
-	args := struct {
 	ClaimID               *string  `json:"claim_id,omitempty"`
 	TXID                  *string  `json:"txid,omitempty"`
 	Nout                  *uint    `json:"nout,omitempty"`
 	Name                  *string  `json:"name,omitempty"`
-		IncludeProtobuf bool    `json:"include_protobuf"`
+	ClaimType             []string `json:"claim_type,omitempty"`
+	OrderBy               []string `json:"order_by,omitempty"`
+	LimitClaimsPerChannel *int     `json:"limit_claims_per_channel,omitempty"`
+	HasNoSource           *bool    `json:"has_no_source,omitempty"`
+	ReleaseTime           string   `json:"release_time,omitempty"`
+	ChannelIDs            []string `json:"channel_ids,omitempty"`
+	NoTotals              *bool    `json:"no_totals,omitempty"`
+	IncludeProtobuf       *bool    `json:"include_protobuf,omitempty"`
+	AnyTags               []string `json:"any_tags,omitempty"`
+
 	Page     uint64 `json:"page"`
 	PageSize uint64 `json:"page_size"`
-	}{
+}
-		ClaimID:         claimID,
+
-		TXID:            txid,
+func (d *Client) ClaimSearch(args ClaimSearchArgs) (*ClaimSearchResponse, error) {
-		Nout:            nout,
+	response := new(ClaimSearchResponse)
-		Name:            claimName,
+	if args.NoTotals == nil {
-		IncludeProtobuf: true,
+		nototals := true
-		Page:            page,
+		args.NoTotals = &nototals
-		PageSize:        pageSize,
+	}
+
+	if args.IncludeProtobuf == nil {
+		include := true
+		args.IncludeProtobuf = &include
 	}
 	structs.DefaultTagName = "json"
 	return response, d.call(response, "claim_search", structs.Map(args))

extras/jsonrpc/daemon_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/shopspring/decimal"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
@@ -54,6 +55,32 @@ func TestClient_AccountFund(t *testing.T) {
 	prettyPrint(*got)
 }
 
+func TestClient_AccountSend(t *testing.T) {
+	d := NewClient("")
+	accounts, err := d.AccountList(1, 20)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.NotEmpty(t, accounts.Items[1].ID) {
+		return
+	}
+	account := (accounts.Items)[1].ID
+
+	addressess, err := d.AddressList(&account, nil, 1, 20)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.NotEmpty(t, addressess.Items) {
+		return
+	}
+
+	got, err := d.AccountSend(&account, "0.01", string(addressess.Items[0].Address))
+	if !assert.NoError(t, err) {
+		return
+	}
+	prettyPrint(*got)
+}
+
 func TestClient_AccountList(t *testing.T) {
 	d := NewClient("")
 	got, err := d.AccountList(1, 20)
@@ -130,11 +157,11 @@ func TestClient_ChannelCreate(t *testing.T) {
 				State:   util.PtrToString("Ticino"),
 				City:    util.PtrToString("Lugano"),
 			}},
-			ThumbnailURL: util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+			ThumbnailURL: util.PtrToString("https://scrn.storni.info/2022-06-10_17-18-29-409175881.png"),
 		},
 		Email:      util.PtrToString("niko@lbry.com"),
 		WebsiteURL: util.PtrToString("https://lbry.com"),
-		CoverURL:   util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+		CoverURL:   util.PtrToString("https://scrn.storni.info/2022-06-10_17-18-29-409175881.png"),
 	})
 	if err != nil {
 		t.Error(err)
@@ -463,7 +490,14 @@ func TestClient_TxoSpendTest(t *testing.T) {
 
 func TestClient_ClaimSearch(t *testing.T) {
 	d := NewClient("")
-	got, err := d.ClaimSearch(nil, util.PtrToString(channelID), nil, nil, 1, 20)
+	got, err := d.ClaimSearch(ClaimSearchArgs{
+		ChannelIDs:  []string{channelID},
+		ReleaseTime: ">1633350820",
+		HasNoSource: util.PtrToBool(true),
+		OrderBy:     []string{"^release_time"},
+		Page:        1,
+		PageSize:    20,
+	})
 	if err != nil {
 		t.Error(err)
 		return
@@ -743,11 +777,15 @@ func TestClient_WalletList(t *testing.T) {
 	d := NewClient("")
 
 	id := "lbry#wallet#id:" + fmt.Sprintf("%d", rand.Int())
-	wList, err := d.WalletList(id, 1, 20)
+	_, err := d.WalletList(id, 1, 20)
 	if err == nil {
 		t.Fatalf("wallet %v was unexpectedly found", id)
 	}
-	if !strings.Contains(err.Error(), fmt.Sprintf("Couldn't find wallet: %v.", id)) {
+	derr, ok := err.(Error)
+	if !ok {
+		t.Fatalf("unknown error returned: %s", err)
+	}
+	if derr.Name != ErrorWalletNotLoaded {
 		t.Fatal(err)
 	}
 
@@ -756,7 +794,7 @@ func TestClient_WalletList(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	wList, err = d.WalletList(id, 1, 20)
+	wList, err := d.WalletList(id, 1, 20)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -791,3 +829,17 @@ func TestClient_WalletRemoveWalletAdd(t *testing.T) {
 		t.Fatalf("wallet ID mismatch, expected %q, got %q", wallet.ID, addedWallet.Name)
 	}
 }
+
+func TestClient_TransactionSummary(t *testing.T) {
+	d := NewClient("https://api.na-backend.odysee.com/api/v1/proxy")
+	r, err := d.TransactionShow("d104a1616c6af581e2046819de678f370d624e97cf176f95acaec4b183a42db6")
+	if err != nil {
+		t.Error(err)
+	}
+	if len(r.Outputs) != 2 {
+		t.Fatal("found wrong transaction")
+	}
+	if r.Outputs[0].Amount != "5.0" {
+		t.Error("found wrong lbc amount for transaction.")
+	}
+}

extras/jsonrpc/daemon_types.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os"
 	"reflect"
+	"strings"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
@@ -49,11 +50,10 @@ type File struct {
 	Height               int               `json:"height"`
 	IsFullyReflected     bool              `json:"is_fully_reflected"`
 	Key                  string            `json:"key"`
-	Metadata             *lbryschema.Claim `json:"protobuf"`
+	Value                *lbryschema.Claim `json:"protobuf"`
 	MimeType             string            `json:"mime_type"`
 	Nout                 int               `json:"nout"`
 	Outpoint             string            `json:"outpoint"`
-	Protobuf             string            `json:"protobuf"`
 	PurchaseReceipt      interface{}       `json:"purchase_receipt"`
 	ReflectorProgress    int               `json:"reflector_progress"`
 	SdHash               string            `json:"sd_hash"`
@@ -251,11 +251,11 @@ type Transaction struct {
 	NormalizedName     string            `json:"normalized_name"`
 	Nout               uint64            `json:"nout"`
 	PermanentUrl       string            `json:"permanent_url"`
+	SigningChannel     *Claim            `json:"signing_channel,omitempty"`
 	TimeStamp          uint64            `json:"time_stamp"`
-	Protobuf           string            `json:"protobuf,omitempty"`
 	Txid               string            `json:"txid"`
 	Type               string            `json:"type"`
-	Value              *lbryschema.Claim `json:"protobuf"`
+	Value              *lbryschema.Claim `json:"protobuf,omitempty"`
 }
 
 type TransactionSummary struct {
@@ -264,6 +264,7 @@ type TransactionSummary struct {
 	Inputs      []Transaction `json:"inputs"`
 	Outputs     []Transaction `json:"outputs"`
 	TotalFee    string        `json:"total_fee"`
+	TotalInput  string        `json:"total_input"`
 	TotalOutput string        `json:"total_output"`
 	Txid        string        `json:"txid"`
 }
@@ -320,6 +321,7 @@ type Claim struct {
 	Address                 string           `json:"address"`
 	Amount                  string           `json:"amount"`
 	CanonicalURL            string           `json:"canonical_url"`
+	ChannelID               string           `json:"channel_id"`
 	ClaimID                 string           `json:"claim_id"`
 	ClaimOp                 string           `json:"claim_op,omitempty"`
 	Confirmations           int              `json:"confirmations"`
@@ -368,7 +370,7 @@ type Meta struct {
 	TrendingMixed     float64 `json:"trending_mixed,omitempty"`
 }
 
-const reflectorURL = "http://blobs.lbry.io/"
+const coldStorageURL = "https://s3.wasabisys.com/blobs.lbry.com/"
 
 // GetStreamSizeByMagic uses "magic" to not just estimate, but actually return the exact size of a stream
 // It does so by fetching the sd blob and the last blob from our S3 bucket, decrypting and unpadding the last blob
@@ -378,7 +380,7 @@ func (c *Claim) GetStreamSizeByMagic() (streamSize uint64, e error) {
 	if c.Value.GetStream() == nil {
 		return 0, errors.Err("this claim is not a stream")
 	}
-	resp, err := http.Get(reflectorURL + hex.EncodeToString(c.Value.GetStream().Source.SdHash))
+	resp, err := http.Get(coldStorageURL + hex.EncodeToString(c.Value.GetStream().Source.SdHash))
 	if err != nil {
 		return 0, errors.Err(err)
 	}
@@ -401,7 +403,7 @@ func (c *Claim) GetStreamSizeByMagic() (streamSize uint64, e error) {
 		streamSize = uint64(stream.MaxBlobSize-1) * uint64(len(sdb.BlobInfos)-2)
 	}
 
-	resp2, err := http.Get(reflectorURL + hex.EncodeToString(lastBlobHash))
+	resp2, err := http.Get(coldStorageURL + hex.EncodeToString(lastBlobHash))
 	if err != nil {
 		return 0, errors.Err(err)
 	}
@@ -425,6 +427,33 @@ func (c *Claim) GetStreamSizeByMagic() (streamSize uint64, e error) {
 	return streamSize, nil
 }
 
+const (
+	ProtectedContentTag      = SpecialContentType("c:members-only")
+	PurchaseContentTag       = SpecialContentType("c:purchase:")
+	RentalContentTag         = SpecialContentType("c:rental:")
+	PreorderContentTag       = SpecialContentType("c:preorder:")
+	LegacyPurchaseContentTag = SpecialContentType("purchase:")
+	LegacyRentalContentTag   = SpecialContentType("rental:")
+	LegacyPreorderContentTag = SpecialContentType("preorder:")
+	ScheduledShowContentTag  = SpecialContentType("c:scheduled:show")
+	ScheduledHideContentTag  = SpecialContentType("c:scheduled:hide")
+	UnlistedContentTag       = SpecialContentType("c:unlisted")
+)
+
+type SpecialContentType string
+
+//IsContentSpecial returns true if the claim is of a special content type
+func (c *Claim) IsContentSpecial(specialTags ...SpecialContentType) bool {
+	for _, t := range c.Value.GetTags() {
+		for _, ct := range specialTags {
+			if strings.Contains(t, string(ct)) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 type StreamListResponse struct {
 	Items      []Claim `json:"items"`
 	Page       uint64  `json:"page"`

extras/lbryinc/client.go

@@ -10,11 +10,13 @@ import (
 	"net/url"
 	"time"
 
+	"golang.org/x/oauth2"
+
 	log "github.com/sirupsen/logrus"
 )
 
 const (
-	defaultServerAddress = "https://api.lbry.com"
+	defaultServerAddress = "https://api.odysee.tv"
 	timeout              = 5 * time.Second
 	headerForwardedFor   = "X-Forwarded-For"
 
@@ -26,6 +28,7 @@ const (
 // Client stores data about internal-apis call it is about to make.
 type Client struct {
 	AuthToken     string
+	OAuthToken    oauth2.TokenSource
 	Logger        *log.Logger
 	serverAddress string
 	extraHeaders  map[string]string
@@ -43,7 +46,34 @@ type ClientOpts struct {
 type APIResponse struct {
 	Success bool        `json:"success"`
 	Error   *string     `json:"error"`
-	Data    *ResponseData `json:"data"`
+	Data    interface{} `json:"data"`
+}
+
+type data struct {
+	obj   map[string]interface{}
+	array []interface{}
+}
+
+func (d data) IsObject() bool {
+	return d.obj != nil
+}
+
+func (d data) IsArray() bool {
+	return d.array != nil
+}
+
+func (d data) Object() (map[string]interface{}, error) {
+	if d.obj == nil {
+		return nil, errors.New("no object data found")
+	}
+	return d.obj, nil
+}
+
+func (d data) Array() ([]interface{}, error) {
+	if d.array == nil {
+		return nil, errors.New("no array data found")
+	}
+	return d.array, nil
 }
 
 // APIError wraps errors returned by LBRY API server to discern them from other kinds (like http errors).
@@ -56,7 +86,12 @@ func (e APIError) Error() string {
 }
 
 // ResponseData is a map containing parsed json response.
-type ResponseData map[string]interface{}
+type ResponseData interface {
+	IsObject() bool
+	IsArray() bool
+	Object() (map[string]interface{}, error)
+	Array() ([]interface{}, error)
+}
 
 func makeMethodPath(obj, method string) string {
 	return fmt.Sprintf("/%s/%s", obj, method)
@@ -83,13 +118,42 @@ func NewClient(authToken string, opts *ClientOpts) Client {
 	return c
 }
 
+// NewOauthClient returns a client instance for internal-apis. It requires Oauth Token Source to be provided
+// for authentication.
+func NewOauthClient(token oauth2.TokenSource, opts *ClientOpts) Client {
+	c := Client{
+		serverAddress: defaultServerAddress,
+		extraHeaders:  make(map[string]string),
+		OAuthToken:    token,
+		Logger:        log.StandardLogger(),
+	}
+	if opts != nil {
+		if opts.ServerAddress != "" {
+			c.serverAddress = opts.ServerAddress
+		}
+		if opts.RemoteIP != "" {
+			c.extraHeaders[headerForwardedFor] = opts.RemoteIP
+		}
+	}
+
+	return c
+}
+
 func (c Client) getEndpointURL(object, method string) string {
 	return fmt.Sprintf("%s%s", c.serverAddress, makeMethodPath(object, method))
 }
 
+func (c Client) getEndpointURLFromPath(path string) string {
+	return fmt.Sprintf("%s%s", c.serverAddress, path)
+}
+
 func (c Client) prepareParams(params map[string]interface{}) (string, error) {
 	form := url.Values{}
+	if c.AuthToken != "" {
 		form.Add("auth_token", c.AuthToken)
+	} else if c.OAuthToken == nil {
+		return "", errors.New("oauth token source must be supplied")
+	}
 	for k, v := range params {
 		if k == "auth_token" {
 			return "", errors.New("extra auth_token supplied in request params")
@@ -109,6 +173,16 @@ func (c Client) doCall(url string, payload string) ([]byte, error) {
 
 	req.Header.Add("Accept", "application/json")
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	if c.OAuthToken != nil {
+		t, err := c.OAuthToken.Token()
+		if err != nil {
+			return nil, err
+		}
+		if t.Type() != "Bearer" {
+			return nil, errors.New("internal-apis requires an oAuth token of type 'Bearer'")
+		}
+		t.SetAuthHeader(req)
+	}
 
 	for k, v := range c.extraHeaders {
 		req.Header.Set(k, v)
@@ -126,36 +200,70 @@ func (c Client) doCall(url string, payload string) ([]byte, error) {
 	return ioutil.ReadAll(r.Body)
 }
 
-// Call calls a remote internal-apis server, returning a response,
+// CallResource calls a remote internal-apis server resource, returning a response,
 // wrapped into standardized API Response struct.
-func (c Client) Call(object, method string, params map[string]interface{}) (ResponseData, error) {
+func (c Client) CallResource(object, method string, params map[string]interface{}) (ResponseData, error) {
-	var rd ResponseData
+	var d data
 	payload, err := c.prepareParams(params)
 	if err != nil {
-		return rd, err
+		return d, err
 	}
 
 	body, err := c.doCall(c.getEndpointURL(object, method), payload)
 	if err != nil {
-		return rd, err
+		return d, err
 	}
 	var ar APIResponse
 	err = json.Unmarshal(body, &ar)
 	if err != nil {
-		return rd, err
+		return d, err
 	}
 	if !ar.Success {
-		return rd, APIError{errors.New(*ar.Error)}
+		return d, APIError{errors.New(*ar.Error)}
 	}
-	return *ar.Data, err
+	if v, ok := ar.Data.([]interface{}); ok {
+		d.array = v
+	} else if v, ok := ar.Data.(map[string]interface{}); ok {
+		d.obj = v
+	}
+	return d, err
+}
+
+// Call calls a remote internal-apis server, returning a response,
+// wrapped into standardized API Response struct.
+func (c Client) Call(path string, params map[string]interface{}) (ResponseData, error) {
+	var d data
+	payload, err := c.prepareParams(params)
+	if err != nil {
+		return d, err
+	}
+
+	body, err := c.doCall(c.getEndpointURLFromPath(path), payload)
+	if err != nil {
+		return d, err
+	}
+	var ar APIResponse
+	err = json.Unmarshal(body, &ar)
+	if err != nil {
+		return d, err
+	}
+	if !ar.Success {
+		return d, APIError{errors.New(*ar.Error)}
+	}
+	if v, ok := ar.Data.([]interface{}); ok {
+		d.array = v
+	} else if v, ok := ar.Data.(map[string]interface{}); ok {
+		d.obj = v
+	}
+	return d, err
 }
 
 // UserMe returns user details for the user associated with the current auth_token.
 func (c Client) UserMe() (ResponseData, error) {
-	return c.Call(userObjectPath, userMeMethod, map[string]interface{}{})
+	return c.CallResource(userObjectPath, userMeMethod, map[string]interface{}{})
 }
 
 // UserHasVerifiedEmail calls has_verified_email method.
 func (c Client) UserHasVerifiedEmail() (ResponseData, error) {
-	return c.Call(userObjectPath, userHasVerifiedEmailMethod, map[string]interface{}{})
+	return c.CallResource(userObjectPath, userHasVerifiedEmailMethod, map[string]interface{}{})
 }

extras/lbryinc/client_test.go

@@ -6,6 +6,8 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"golang.org/x/oauth2"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -14,6 +16,14 @@ func launchDummyServer(lastReq **http.Request, path, response string, status int
 		if lastReq != nil {
 			*lastReq = &*r
 		}
+		authT := r.FormValue("auth_token")
+		if authT == "" {
+			accessT := r.Header.Get("Authorization")
+			if accessT == "" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+		}
 		if r.URL.Path != path {
 			fmt.Printf("path doesn't match: %v != %v", r.URL.Path, path)
 			w.WriteHeader(http.StatusNotFound)
@@ -32,7 +42,25 @@ func TestUserMe(t *testing.T) {
 	c := NewClient("realToken", &ClientOpts{ServerAddress: ts.URL})
 	r, err := c.UserMe()
 	assert.Nil(t, err)
-	assert.Equal(t, "user@lbry.tv", r["primary_email"])
+	robj, err := r.Object()
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, "user@lbry.tv", robj["primary_email"])
+}
+
+func TestListFiltered(t *testing.T) {
+	ts := launchDummyServer(nil, "/file/list_filtered", listFilteredResponse, http.StatusOK)
+	defer ts.Close()
+
+	c := NewClient("realToken", &ClientOpts{ServerAddress: ts.URL})
+	r, err := c.CallResource("file", "list_filtered", map[string]interface{}{"with_claim_id": "true"})
+	assert.Nil(t, err)
+	assert.True(t, r.IsArray())
+	_, err = r.Array()
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestUserHasVerifiedEmail(t *testing.T) {
@@ -42,8 +70,27 @@ func TestUserHasVerifiedEmail(t *testing.T) {
 	c := NewClient("realToken", &ClientOpts{ServerAddress: ts.URL})
 	r, err := c.UserHasVerifiedEmail()
 	assert.Nil(t, err)
-	assert.EqualValues(t, 12345, r["user_id"])
+	robj, err := r.Object()
-	assert.Equal(t, true, r["has_verified_email"])
+	if err != nil {
+		t.Error(err)
+	}
+	assert.EqualValues(t, 12345, robj["user_id"])
+	assert.Equal(t, true, robj["has_verified_email"])
+}
+
+func TestUserHasVerifiedEmailOAuth(t *testing.T) {
+	ts := launchDummyServer(nil, makeMethodPath(userObjectPath, userHasVerifiedEmailMethod), userHasVerifiedEmailResponse, http.StatusOK)
+	defer ts.Close()
+
+	c := NewOauthClient(oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "Test-Access-Token"}), &ClientOpts{ServerAddress: ts.URL})
+	r, err := c.UserHasVerifiedEmail()
+	assert.Nil(t, err)
+	robj, err := r.Object()
+	if err != nil {
+		t.Error(err)
+	}
+	assert.EqualValues(t, 12345, robj["user_id"])
+	assert.Equal(t, true, robj["has_verified_email"])
 }
 
 func TestRemoteIP(t *testing.T) {
@@ -61,7 +108,7 @@ func TestWrongToken(t *testing.T) {
 	c := NewClient("zcasdasc", nil)
 
 	r, err := c.UserHasVerifiedEmail()
-	assert.Nil(t, r)
+	assert.False(t, r.IsObject())
 	assert.EqualError(t, err, "api error: could not authenticate user")
 	assert.ErrorAs(t, err, &APIError{})
 }
@@ -70,7 +117,7 @@ func TestHTTPError(t *testing.T) {
 	c := NewClient("zcasdasc", &ClientOpts{ServerAddress: "http://lolcathost"})
 
 	r, err := c.UserHasVerifiedEmail()
-	assert.Nil(t, r)
+	assert.False(t, r.IsObject())
 	assert.EqualError(t, err, `Post "http://lolcathost/user/has_verified_email": dial tcp: lookup lolcathost: no such host`)
 }
 
@@ -81,7 +128,7 @@ func TestGatewayError(t *testing.T) {
 	c := NewClient("zcasdasc", &ClientOpts{ServerAddress: ts.URL})
 
 	r, err := c.UserHasVerifiedEmail()
-	assert.Nil(t, r)
+	assert.False(t, r.IsObject())
 	assert.EqualError(t, err, `server returned non-OK status: 502`)
 }
 
@@ -118,3 +165,18 @@ const userHasVerifiedEmailResponse = `{
 	  "has_verified_email": true
 	}
 }`
+
+const listFilteredResponse = `{
+  "success": true,
+  "error": null,
+  "data": [
+    {
+      "claim_id": "322ce77e9085d9da42279c790f7c9755b4916fca",
+      "outpoint": "20e04af21a569061ced7aa1801a43b4ed4839dfeb79919ea49a4059c7fe114c5:0"
+    },
+    {
+      "claim_id": "61496c567badcd98b82d9a700a8d56fd8a5fa8fb",
+      "outpoint": "657e4ec774524b326f9d3ecb9f468ea085bd1f3d450565f0330feca02e8fd25b:0"
+    }
+  ]
+}`

extras/util/slack.go

@@ -2,12 +2,15 @@ package util
 
 import (
 	"fmt"
+	"net"
+	"net/http"
 	"strings"
+	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
-	"github.com/nlopes/slack"
 	log "github.com/sirupsen/logrus"
+	"github.com/slack-go/slack"
 )
 
 var defaultChannel string
@@ -16,7 +19,18 @@ var slackApi *slack.Client
 
 // InitSlack Initializes a slack client with the given token and sets the default channel.
 func InitSlack(token string, channel string, username string) {
-	slackApi = slack.New(token)
+	c := &http.Client{
+		Transport: &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ResponseHeaderTimeout: 10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+	}
+	slackApi = slack.New(token, slack.OptionHTTPClient(c))
 	defaultChannel = channel
 	defaultUsername = username
 }
@@ -65,7 +79,13 @@ func sendToSlack(channel, username, message string) error {
 		err = errors.Err("no slack token provided")
 	} else {
 		log.Debugln("slack: " + channel + ": " + message)
+		for {
 			_, _, err = slackApi.PostMessage(channel, slack.MsgOptionText(message, false), slack.MsgOptionUsername(username))
+			if err != nil && strings.Contains(err.Error(), "timeout awaiting response headers") {
+				continue
+			}
+			break
+		}
 	}
 
 	if err != nil {

extras/util/strings.go

@@ -1,6 +1,11 @@
 package util
 
-import "strings"
+import (
+	"encoding/hex"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/unicode/norm"
+	"strings"
+)
 
 func StringSplitArg(stringToSplit, separator string) []interface{} {
 	split := strings.Split(stringToSplit, separator)
@@ -10,3 +15,39 @@ func StringSplitArg(stringToSplit, separator string) []interface{} {
 	}
 	return splitInterface
 }
+
+// NormalizeName Normalize names to remove weird characters and account to capitalization
+func NormalizeName(s string) string {
+	c := cases.Fold()
+	return c.String(norm.NFD.String(s))
+}
+
+// ReverseBytesInPlace reverse the bytes. thanks, Satoshi 😒
+func ReverseBytesInPlace(s []byte) {
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+}
+
+// TxIdToTxHash convert the txid to a hash for returning from the hub
+func TxIdToTxHash(txid string) []byte {
+	t, err := hex.DecodeString(txid)
+	if err != nil {
+		return nil
+	}
+
+	ReverseBytesInPlace(t)
+
+	return t
+}
+
+// TxHashToTxId convert the txHash from the response format back to an id
+func TxHashToTxId(txHash []byte) string {
+	t := make([]byte, len(txHash))
+	copy(t, txHash)
+
+	ReverseBytesInPlace(t)
+
+	return hex.EncodeToString(t)
+
+}

go.mod

@@ -1,54 +1,55 @@
+go 1.18
+
 module github.com/lbryio/lbry.go/v2
 
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
 
 require (
-	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/davecgh/go-spew v1.1.1
 	github.com/fatih/structs v1.1.0
-	github.com/go-errors/errors v1.0.1
+	github.com/go-errors/errors v1.4.2
-	github.com/go-ini/ini v1.48.0
+	github.com/go-ini/ini v1.67.0
-	github.com/go-ozzo/ozzo-validation v3.6.0+incompatible // indirect
+	github.com/golang/protobuf v1.5.2
-	github.com/golang/protobuf v1.3.2
+	github.com/gorilla/mux v1.8.0
-	github.com/google/go-cmp v0.3.1 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f // indirect
-	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/rpc v1.2.0
-	github.com/gorilla/websocket v1.4.1 // indirect
+	github.com/lbryio/ozzo-validation v3.0.3-0.20170512160344-202201e212ec+incompatible
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04
-	github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
 	github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5
-	github.com/mitchellh/mapstructure v1.1.2
+	github.com/mitchellh/mapstructure v1.5.0
-	github.com/nlopes/slack v0.6.0
+	github.com/sebdah/goldie v1.0.0
-	github.com/onsi/ginkgo v1.10.2 // indirect
+	github.com/sergi/go-diff v1.3.1
-	github.com/onsi/gomega v1.7.0 // indirect
+	github.com/shopspring/decimal v1.3.1
-	github.com/pkg/errors v0.8.1 // indirect
+	github.com/sirupsen/logrus v1.9.0
-	github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77
+	github.com/slack-go/slack v0.12.1
-	github.com/sergi/go-diff v1.0.0
+	github.com/spf13/cast v1.5.0
-	github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
+	github.com/stretchr/testify v1.8.2
-	github.com/sirupsen/logrus v1.4.2
+	github.com/ybbus/jsonrpc/v2 v2.1.7
-	github.com/smartystreets/assertions v1.0.1 // indirect
+	go.uber.org/atomic v1.10.0
-	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
+	golang.org/x/crypto v0.7.0
-	github.com/spf13/cast v1.3.0
+	golang.org/x/net v0.8.0
-	github.com/stretchr/testify v1.7.0
+	golang.org/x/oauth2 v0.6.0
-	github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d
+	golang.org/x/text v0.8.0
-	go.uber.org/atomic v1.4.0
+	golang.org/x/time v0.3.0
-	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc
+	google.golang.org/grpc v1.53.0
-	golang.org/x/net v0.0.0-20191009170851-d66e71096ffb
-	golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f // indirect
-	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0
-	google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 // indirect
-	google.golang.org/grpc v1.24.0
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
-	gopkg.in/ini.v1 v1.48.0 // indirect
 	gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79
-	gopkg.in/yaml.v2 v2.2.4 // indirect
 	gotest.tools v2.2.0+incompatible
 )
 
-go 1.15
+require (
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
+	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
+	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
+	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/onsi/gomega v1.7.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -1,5 +1,3 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
@@ -14,167 +12,139 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
-github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
-github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
-github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/go-ozzo/ozzo-validation v3.6.0+incompatible h1:msy24VGS42fKO9K1vLz82/GeYW1cILu7Nuuj1N3BBkE=
+github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
-github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
+github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
 github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
-github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
-github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
+github.com/lbryio/ozzo-validation v3.0.3-0.20170512160344-202201e212ec+incompatible h1:OH/jgRO/2lQ73n7PgtK/CvLZ0dwAVr5G5s635+YfUA4=
-github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
+github.com/lbryio/ozzo-validation v3.0.3-0.20170512160344-202201e212ec+incompatible/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
-github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M=
+github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 h1:IhL9D2QfDWhLNDQpZ3Uiiw0gZEUYeLBS6uDqOd59G5o=
-github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
-github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
-github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNlB5LGin1rljK7c49NLniGwJG2bg=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
-github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
+github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc=
-github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
-github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
-github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
+github.com/slack-go/slack v0.12.1 h1:X97b9g2hnITDtNsNe5GkGx6O2/Sz/uC20ejRZN6QxOw=
-github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/slack-go/slack v0.12.1/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
+github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/ybbus/jsonrpc/v2 v2.1.7 h1:QjoXuZhkXZ3oLBkrONBe2avzFkYeYLorpeA+d8175XQ=
+github.com/ybbus/jsonrpc/v2 v2.1.7/go.mod h1:rIuG1+ORoiqocf9xs/v+ecaAVeo3zcZHQgInyKFMeg0=
+go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
+go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
+golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
-golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
+golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
-golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
-golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
-google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
-gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

readme.md

@@ -2,6 +2,8 @@
 
 lbry.go is a set of tools and projects implemented in Golang. See each subfolder for more details
 
+**there are significant updates in the [v3 branch](https://github.com/lbryio/lbry.go/tree/v3). if you're starting a new project, strongly consider using that version instead**
+
 [![Build Status](https://travis-ci.org/lbryio/lbry.go.svg?branch=master)](https://travis-ci.org/lbryio/lbry.go)
 
 This project uses Go modules. Make sure you have Go 1.11+ installed.
@@ -34,7 +36,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
 ## Security
 
 We take security seriously. Please contact security@lbry.com regarding any issues you may encounter.
-Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
+Our PGP key is [here](https://lbry.com/faq/pgp-key) if you need it.
 
 ## Contact
 

schema/keys/keys.go

@@ -17,14 +17,6 @@ type publicKeyInfo struct {
 	PublicKey asn1.BitString
 }
 
-//This type provides compatibility with the btcec package
-type ecPrivateKey struct {
-	Version       int
-	PrivateKey    []byte
-	NamedCurveOID asn1.ObjectIdentifier `asn1:"optional,explicit,tag:0"`
-	PublicKey     asn1.BitString        `asn1:"optional,explicit,tag:1"`
-}
-
 func PublicKeyToDER(publicKey *btcec.PublicKey) ([]byte, error) {
 	var publicKeyBytes []byte
 	var publicKeyAlgorithm pkix.AlgorithmIdentifier
@@ -50,13 +42,48 @@ func PublicKeyToDER(publicKey *btcec.PublicKey) ([]byte, error) {
 
 }
 
+//This type provides compatibility with the btcec package
+type ecPrivateKey struct {
+	Version       int
+	PrivateKey    []byte
+	NamedCurveOID asn1.ObjectIdentifier `asn1:"optional,explicit,tag:0"`
+	PublicKey     asn1.BitString        `asn1:"optional,explicit,tag:1"`
+}
+
+func PrivateKeyToDER(key *btcec.PrivateKey) ([]byte, error) {
+	privateKey := make([]byte, (key.Curve.Params().N.BitLen()+7)/8)
+	oid := asn1.ObjectIdentifier{1, 3, 132, 0, 10}
+	return asn1.Marshal(ecPrivateKey{
+		Version:       1,
+		PrivateKey:    key.D.FillBytes(privateKey),
+		NamedCurveOID: oid,
+		PublicKey:     asn1.BitString{Bytes: elliptic.Marshal(key.Curve, key.X, key.Y)},
+	})
+}
+
 func GetPublicKeyFromBytes(pubKeyBytes []byte) (*btcec.PublicKey, error) {
+	if len(pubKeyBytes) == 33 {
+		return btcec.ParsePubKey(pubKeyBytes, btcec.S256())
+	}
 	PKInfo := publicKeyInfo{}
-	asn1.Unmarshal(pubKeyBytes, &PKInfo)
+	_, err := asn1.Unmarshal(pubKeyBytes, &PKInfo)
-	pubkeyBytes1 := []byte(PKInfo.PublicKey.Bytes)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	pubkeyBytes1 := PKInfo.PublicKey.Bytes
 	return btcec.ParsePubKey(pubkeyBytes1, btcec.S256())
 }
 
+func GetPrivateKeyFromBytes(privKeyBytes []byte) (*btcec.PrivateKey, *btcec.PublicKey, error) {
+	ecPK := ecPrivateKey{}
+	_, err := asn1.Unmarshal(privKeyBytes, &ecPK)
+	if err != nil {
+		return nil, nil, errors.Err(err)
+	}
+	priv, publ := btcec.PrivKeyFromBytes(btcec.S256(), ecPK.PrivateKey)
+	return priv, publ, nil
+}
+
 //Returns a btec.Private key object if provided a correct secp256k1 encoded pem.
 func ExtractKeyFromPem(pm string) (*btcec.PrivateKey, *btcec.PublicKey) {
 	byta := []byte(pm)
@@ -65,3 +92,17 @@ func ExtractKeyFromPem(pm string) (*btcec.PrivateKey, *btcec.PublicKey) {
 	asn1.Unmarshal(blck.Bytes, &ecp)
 	return btcec.PrivKeyFromBytes(btcec.S256(), ecp.PrivateKey)
 }
+
+type Signature struct {
+	btcec.Signature
+}
+
+func (s *Signature) LBRYSDKEncode() ([]byte, error) {
+	if s.R == nil || s.S == nil {
+		return nil, errors.Err("invalid signature, both S & R are nil")
+	}
+	rBytes := s.R.Bytes()
+	sBytes := s.S.Bytes()
+
+	return append(rBytes, sBytes...), nil
+}

schema/keys/keys_test.go

@@ -1,9 +1,12 @@
 package keys
 
 import (
+	"bytes"
 	"encoding/hex"
+	"encoding/pem"
 	"testing"
 
+	"github.com/btcsuite/btcd/btcec"
 	"gotest.tools/assert"
 )
 
@@ -36,3 +39,57 @@ func TestPublicKeyToDER(t *testing.T) {
 	}
 	assert.Assert(t, p1.IsEqual(p2), "The keys produced must be the same key!")
 }
+
+func TestPrivateKeyToDER(t *testing.T) {
+	private1, err := btcec.NewPrivateKey(btcec.S256())
+	if err != nil {
+		t.Fatal(err)
+	}
+	bytes, err := PrivateKeyToDER(private1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	private2, _, err := GetPrivateKeyFromBytes(bytes)
+	if err != nil {
+		t.Error(err)
+	}
+	if !private1.ToECDSA().Equal(private2.ToECDSA()) {
+		t.Error("private keys dont match")
+	}
+}
+
+func TestGetPrivateKeyFromBytes(t *testing.T) {
+	private, err := btcec.NewPrivateKey(btcec.S256())
+	if err != nil {
+		t.Fatal(err)
+	}
+	bytes, err := PrivateKeyToDER(private)
+	private2, _, err := GetPrivateKeyFromBytes(bytes)
+	if !private.ToECDSA().Equal(private2.ToECDSA()) {
+		t.Error("private keys dont match")
+	}
+}
+
+func TestEncodePEMAndBack(t *testing.T) {
+	private1, err := btcec.NewPrivateKey(btcec.S256())
+	if err != nil {
+		t.Fatal(err)
+	}
+	b := bytes.NewBuffer(nil)
+	derBytes, err := PrivateKeyToDER(private1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = pem.Encode(b, &pem.Block{Type: "PRIVATE KEY", Bytes: derBytes})
+	if err != nil {
+		t.Fatal(err)
+	}
+	println(string(b.Bytes()))
+	private2, _ := ExtractKeyFromPem(string(b.Bytes()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !private1.ToECDSA().Equal(private2.ToECDSA()) {
+		t.Error("private keys dont match")
+	}
+}

schema/stake/sign.go

@@ -6,11 +6,12 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/schema/address"
+	"github.com/lbryio/lbry.go/v2/schema/keys"
 
 	"github.com/btcsuite/btcd/btcec"
 )
 
-func Sign(privKey btcec.PrivateKey, channel StakeHelper, claim StakeHelper, k string) (*Signature, error) {
+func Sign(privKey btcec.PrivateKey, channel StakeHelper, claim StakeHelper, k string) (*keys.Signature, error) {
 	if channel.Claim.GetChannel() == nil {
 		return nil, errors.Err("claim as channel is not of type channel")
 	}
@@ -21,7 +22,7 @@ func Sign(privKey btcec.PrivateKey, channel StakeHelper, claim StakeHelper, k st
 	return claim.sign(privKey, channel, k)
 }
 
-func (c *StakeHelper) sign(privKey btcec.PrivateKey, channel StakeHelper, firstInputTxID string) (*Signature, error) {
+func (c *StakeHelper) sign(privKey btcec.PrivateKey, channel StakeHelper, firstInputTxID string) (*keys.Signature, error) {
 
 	txidBytes, err := hex.DecodeString(firstInputTxID)
 	if err != nil {
@@ -48,11 +49,11 @@ func (c *StakeHelper) sign(privKey btcec.PrivateKey, channel StakeHelper, firstI
 		return nil, errors.Err(err)
 	}
 
-	return &Signature{*sig}, nil
+	return &keys.Signature{*sig}, nil
 
 }
 
-func (c *StakeHelper) signV1(privKey btcec.PrivateKey, channel StakeHelper, claimAddress string) (*Signature, error) {
+func (c *StakeHelper) signV1(privKey btcec.PrivateKey, channel StakeHelper, claimAddress string) (*keys.Signature, error) {
 	metadataBytes, err := c.serializedNoSignature()
 	if err != nil {
 		return nil, errors.Err(err)
@@ -85,21 +86,7 @@ func (c *StakeHelper) signV1(privKey btcec.PrivateKey, channel StakeHelper, clai
 		return nil, errors.Err(err)
 	}
 
-	return &Signature{*sig}, nil
+	return &keys.Signature{Signature: *sig}, nil
-}
-
-type Signature struct {
-	btcec.Signature
-}
-
-func (s *Signature) LBRYSDKEncode() ([]byte, error) {
-	if s.R == nil || s.S == nil {
-		return nil, errors.Err("invalid signature, both S & R are nil")
-	}
-	rBytes := s.R.Bytes()
-	sBytes := s.S.Bytes()
-
-	return append(rBytes, sBytes...), nil
 }
 
 // rev reverses a byte slice. useful for switching endian-ness

scripts/release.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
-GO111MODULE=off go get github.com/caarlos0/svu
+go install github.com/caarlos0/svu@latest
 git tag `svu "$1"`
 git push --tags

stream/blob.go

@@ -35,7 +35,7 @@ func (b Blob) Hash() []byte {
 	return hashBytes[:]
 }
 
-// HashHex returns th blob hash as a hex string
+// HashHex returns the blob hash as a hex string
 func (b Blob) HashHex() string {
 	return hex.EncodeToString(b.Hash())
 }

stream/sdBlob.go

@@ -8,6 +8,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"strconv"
+	"strings"
 )
 
 const streamTypeLBRYFile = "lbryfile"
@@ -44,10 +45,39 @@ type SDBlob struct {
 	StreamHash        []byte     `json:"-"`
 }
 
+// Hash returns a hash of the SD blob data
+func (s SDBlob) Hash() []byte {
+	hashBytes := sha512.Sum384(s.ToBlob())
+	return hashBytes[:]
+}
+
+// HashHex returns the SD blob hash as a hex string
+func (s SDBlob) HashHex() string {
+	return hex.EncodeToString(s.Hash())
+}
+
+// ToJson returns the SD blob hash as JSON
+func (s SDBlob) ToJson() string {
+	j, err := json.MarshalIndent(s, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+	return string(j)
+}
+
 // ToBlob converts the SDBlob to a normal data Blob
-func (s SDBlob) ToBlob() (Blob, error) {
+func (s SDBlob) ToBlob() Blob {
-	b, err := json.Marshal(s)
+	jsonSD, err := json.Marshal(s)
-	return Blob(b), err
+	if err != nil {
+		panic(err)
+	}
+
+	// COMPATIBILITY HACK to make json output match python's json. this can be
+	// removed when we implement canonical JSON encoding
+	jsonSD = []byte(strings.Replace(string(jsonSD), ",", ", ", -1))
+	jsonSD = []byte(strings.Replace(string(jsonSD), ":", ": ", -1))
+
+	return jsonSD
 }
 
 // FromBlob unmarshals a data Blob that should contain SDBlob data
@@ -55,30 +85,6 @@ func (s *SDBlob) FromBlob(b Blob) error {
 	return json.Unmarshal(b, s)
 }
 
-func newSdBlob(blobs []Blob, key []byte, ivs [][]byte, streamName, suggestedFilename string) *SDBlob {
-	if len(ivs) != len(blobs)+1 { // +1 for terminating 0-length blob
-		panic("wrong number of IVs provided")
-	}
-
-	sd := &SDBlob{
-		StreamType:        streamTypeLBRYFile,
-		StreamName:        streamName,
-		SuggestedFileName: suggestedFilename,
-		Key:               key,
-	}
-
-	for i, b := range blobs {
-		sd.addBlob(b, ivs[i])
-	}
-
-	// terminating blob
-	sd.addBlob(Blob{}, ivs[len(ivs)-1])
-
-	sd.updateStreamHash()
-
-	return sd
-}
-
 // addBlob adds the blob's info to stream
 func (s *SDBlob) addBlob(b Blob, iv []byte) {
 	if len(iv) == 0 {

stream/stream.go

@@ -2,8 +2,10 @@ package stream
 
 import (
 	"bytes"
+	"crypto/sha512"
+	"hash"
+	"io"
 	"math"
-	"strings"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 )
@@ -13,66 +15,22 @@ type Stream []Blob
 // -1 to leave room for padding, since there must be at least one byte of pkcs7 padding
 const maxBlobDataSize = MaxBlobSize - 1
 
-// New creates a new Stream from a byte slice
+// New creates a new Stream from a stream of bytes.
-func New(data []byte) (Stream, error) {
+func New(src io.Reader) (Stream, error) {
-	key := randIV()
+	return NewEncoder(src).Stream()
-	ivs := make([][]byte, numContentBlobs(data)+1) // +1 for terminating 0-length blob
-	for i := range ivs {
-		ivs[i] = randIV()
-	}
-
-	return makeStream(data, key, ivs, "", "")
-}
-
-// Reconstruct creates a stream from the given data using predetermined IVs and key from the SD blob
-// NOTE: this will assume that all blobs except the last one are at max length. in theory this is not
-// required, but in practice this is always true. if this is false, streams may not match exactly
-func Reconstruct(data []byte, sdBlob SDBlob) (Stream, error) {
-	ivs := make([][]byte, len(sdBlob.BlobInfos))
-	for i := range ivs {
-		ivs[i] = sdBlob.BlobInfos[i].IV
-	}
-
-	return makeStream(data, sdBlob.Key, ivs, sdBlob.StreamName, sdBlob.SuggestedFileName)
-}
-
-func makeStream(data, key []byte, ivs [][]byte, streamName, suggestedFilename string) (Stream, error) {
-	var err error
-
-	numBlobs := numContentBlobs(data)
-	if len(ivs) != numBlobs+1 { // +1 for terminating 0-length blob
-		return nil, errors.Err("incorrect number of IVs provided")
-	}
-
-	s := make(Stream, numBlobs+1) // +1 for sd blob
-	for i := 0; i < numBlobs; i++ {
-		start := i * maxBlobDataSize
-		end := start + maxBlobDataSize
-		if end > len(data) {
-			end = len(data)
-		}
-		s[i+1], err = NewBlob(data[start:end], key, ivs[i])
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	sd := newSdBlob(s[1:], key, ivs, streamName, suggestedFilename)
-	jsonSD, err := sd.ToBlob()
-	if err != nil {
-		return nil, err
-	}
-
-	// COMPATIBILITY HACK to make json output match python's json. this can be
-	// removed when we implement canonical JSON encoding
-	jsonSD = []byte(strings.Replace(string(jsonSD), ",", ", ", -1))
-	jsonSD = []byte(strings.Replace(string(jsonSD), ":", ": ", -1))
-
-	s[0] = jsonSD
-	return s, nil
 }
 
+// Data returns the file data that a stream encapsulates.
+//
+// Deprecated: use Decode() instead. It's a more accurate name. Data() will be removed in the future.
 func (s Stream) Data() ([]byte, error) {
+	return s.Decode()
+}
+
+// Decode returns the file data that a stream encapsulates
+//
+// TODO: this should use io.Writer instead of returning bytes
+func (s Stream) Decode() ([]byte, error) {
 	if len(s) < 2 {
 		return nil, errors.Err("stream must be at least 2 blobs long") // sd blob and content blob
 	}
@@ -124,7 +82,161 @@ func (s Stream) Data() ([]byte, error) {
 	return file, nil
 }
 
-//numContentBlobs returns the number of content blobs required to store the data
+// Encoder reads bytes from a source and returns blobs of the stream
-func numContentBlobs(data []byte) int {
+type Encoder struct {
-	return int(math.Ceil(float64(len(data)) / float64(maxBlobDataSize)))
+	// source data to be encoded into a stream
+	src io.Reader
+	// preset IVs to use for encrypting blobs
+	ivs [][]byte
+	// an optionals hint about the total size of the source data
+	// encoder will use this to preallocate space for blobs
+	srcSizeHint int
+
+	// buffer for reading bytes from reader
+	buf []byte
+	// sd blob that gets built as stream is encoded
+	sd *SDBlob
+	// number of bytes read from src
+	srcLen int
+	// running hash bytes read from src
+	srcHash hash.Hash
+}
+
+// NewEncoder creates a new stream encoder
+func NewEncoder(src io.Reader) *Encoder {
+	return &Encoder{
+		src: src,
+
+		buf: make([]byte, maxBlobDataSize),
+		sd: &SDBlob{
+			StreamType: streamTypeLBRYFile,
+			Key:        randIV(),
+		},
+		srcHash: sha512.New384(),
+	}
+}
+
+// NewEncoderWithIVs creates a new encoder that uses preset cryptographic material
+func NewEncoderWithIVs(src io.Reader, key []byte, ivs [][]byte) *Encoder {
+	e := NewEncoder(src)
+	e.sd.Key = key
+	e.ivs = ivs
+	return e
+}
+
+// NewEncoderFromSD creates a new encoder that reuses cryptographic material from an sd blob
+// This can be used to reconstruct a stream exactly from a file
+// NOTE: this will assume that all blobs except the last one are at max length. in theory this is not
+// required, but in practice this is always true. if this is false, streams may not match exactly
+func NewEncoderFromSD(src io.Reader, sdBlob *SDBlob) *Encoder {
+	ivs := make([][]byte, len(sdBlob.BlobInfos))
+	for i := range ivs {
+		ivs[i] = sdBlob.BlobInfos[i].IV
+	}
+
+	e := NewEncoderWithIVs(src, sdBlob.Key, ivs)
+	e.sd.StreamName = sdBlob.StreamName
+	e.sd.SuggestedFileName = sdBlob.SuggestedFileName
+	return e
+}
+
+// TODO: consider making a NewPartialEncoder that also copies blobinfos from sdBlobs and seeks forward in the data
+// this would avoid re-creating blobs that were created in the past
+
+// Next reads the next chunk of data, encodes it into a blob, and adds it to the stream
+// When the source is fully consumed, Next() makes sure the stream is terminated (i.e. the sd blob
+// ends with an empty terminating blob) and returns io.EOF
+func (e *Encoder) Next() (Blob, error) {
+	n, err := e.src.Read(e.buf)
+	if err != nil {
+		if errors.Is(err, io.EOF) {
+			e.ensureTerminated()
+		}
+		return nil, err
+	}
+
+	e.srcLen += n
+	e.srcHash.Write(e.buf[:n])
+	iv := e.nextIV()
+
+	blob, err := NewBlob(e.buf[:n], e.sd.Key, iv)
+	if err != nil {
+		return nil, err
+	}
+
+	e.sd.addBlob(blob, iv)
+
+	return blob, nil
+}
+
+// Stream creates the whole stream in one call
+func (e *Encoder) Stream() (Stream, error) {
+	s := make(Stream, 1, 1+int(math.Ceil(float64(e.srcSizeHint)/maxBlobDataSize))) // len starts at 1 and cap is +1 to leave room for sd blob
+
+	for {
+		blob, err := e.Next()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return nil, err
+		}
+
+		s = append(s, blob)
+	}
+
+	s[0] = e.SDBlob().ToBlob()
+
+	if cap(s) > len(s) {
+		// size hint was too big. copy stream to smaller underlying array to free memory
+		// this might be premature optimization...
+		s = append(Stream(nil), s[:]...)
+	}
+
+	return s, nil
+}
+
+// SDBlob returns the sd blob so far
+func (e *Encoder) SDBlob() *SDBlob {
+	e.sd.updateStreamHash()
+	return e.sd
+}
+
+// SourceLen returns the number of bytes read from source
+func (e *Encoder) SourceLen() int {
+	return e.srcLen
+}
+
+// SourceLen returns a hash of the bytes read from source
+func (e *Encoder) SourceHash() []byte {
+	return e.srcHash.Sum(nil)
+}
+
+// SourceSizeHint sets a hint about the total size of the source
+// This helps allocate RAM more efficiently.
+// If the hint is wrong, it still works fine but there will be a small performance penalty.
+func (e *Encoder) SourceSizeHint(size int) *Encoder {
+	e.srcSizeHint = size
+	return e
+}
+
+func (e *Encoder) isTerminated() bool {
+	return len(e.sd.BlobInfos) >= 1 && e.sd.BlobInfos[len(e.sd.BlobInfos)-1].Length == 0
+}
+
+func (e *Encoder) ensureTerminated() {
+	if !e.isTerminated() {
+		e.sd.addBlob(Blob{}, e.nextIV())
+	}
+}
+
+// nextIV returns the next preset IV if there is one
+func (e *Encoder) nextIV() []byte {
+	if len(e.ivs) == 0 {
+		return randIV()
+	}
+
+	iv := e.ivs[0]
+	e.ivs = e.ivs[1:]
+	return iv
 }

stream/stream_test.go

@@ -2,13 +2,17 @@ package stream
 
 import (
 	"bytes"
+	"crypto/rand"
 	"crypto/sha256"
+	"crypto/sha512"
 	"encoding/hex"
+	"io"
 	"testing"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
 )
 
-func TestStreamToFile(t *testing.T) {
+var testdataBlobHashes = []string{
-	blobHashes := []string{
 	"1bf7d39c45d1a38ffa74bff179bf7f67d400ff57fa0b5a0308963f08d01712b3079530a8c188e8c89d9b390c6ee06f05", // sd hash
 	"a2f1841bb9c5f3b583ac3b8c07ee1a5bf9cc48923721c30d5ca6318615776c284e8936d72fa4db7fdda2e4e9598b1e6c",
 	"0c9675ad7f40f29dcd41883ed9cf7e145bbb13976d9b83ab9354f4f61a87f0f7771a56724c2aa7a5ab43c68d7942e5cb",
@@ -16,12 +20,13 @@ func TestStreamToFile(t *testing.T) {
 	"dcd2093f4a3eca9f6dd59d785d0bef068fee788481986aa894cf72ed4d992c0ff9d19d1743525de2f5c3c62f5ede1c58",
 }
 
-	stream := make(Stream, len(blobHashes))
+func TestStreamToFile(t *testing.T) {
-	for i, hash := range blobHashes {
+	stream := make(Stream, len(testdataBlobHashes))
+	for i, hash := range testdataBlobHashes {
 		stream[i] = testdata(t, hash)
 	}
 
-	data, err := stream.Data()
+	data, err := stream.Decode()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -33,6 +38,8 @@ func TestStreamToFile(t *testing.T) {
 		t.Errorf("file length mismatch. got %d, expected %d", actualLen, expectedLen)
 	}
 
+	expectedFileHash := sha512.Sum384(data)
+
 	expectedSha256 := unhex(t, "51e4d03bd6d69ea17d1be3ce01fdffa44ffe053f2dbce8d42a50283b2890fea2")
 	actualSha256 := sha256.Sum256(data)
 
@@ -46,22 +53,150 @@ func TestStreamToFile(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	newStream, err := Reconstruct(data, *sdBlob)
+	enc := NewEncoderFromSD(bytes.NewBuffer(data), sdBlob)
-	if err != nil {
+	newStream, err := enc.Stream()
-		t.Fatal(err)
+
+	if len(newStream) != len(testdataBlobHashes) {
+		t.Fatalf("stream length mismatch. got %d blobs, expected %d", len(newStream), len(testdataBlobHashes))
 	}
 
-	if len(newStream) != len(blobHashes) {
+	if enc.SourceLen() != expectedLen {
-		t.Fatalf("stream length mismatch. got %d blobs, expected %d", len(newStream), len(blobHashes))
+		t.Errorf("reconstructed file length mismatch. got %d, expected %d", enc.SourceLen(), expectedLen)
 	}
 
-	for i, hash := range blobHashes {
+	if !bytes.Equal(enc.SourceHash(), expectedFileHash[:]) {
+		t.Errorf("reconstructed file hash mismatch. got %s, expected %s", hex.EncodeToString(enc.SourceHash()), hex.EncodeToString(expectedFileHash[:]))
+	}
+
+	for i, hash := range testdataBlobHashes {
 		if newStream[i].HashHex() != hash {
 			t.Errorf("blob %d hash mismatch. got %s, expected %s", i, newStream[i].HashHex(), hash)
 		}
 	}
 }
 
+func TestMakeStream(t *testing.T) {
+	blobsToRead := 3
+	totalBlobs := blobsToRead + 3
+
+	data := make([]byte, ((totalBlobs-1)*maxBlobDataSize)+1000) // last blob is partial
+	_, err := rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	buf := bytes.NewBuffer(data)
+
+	enc := NewEncoder(buf)
+
+	stream := make(Stream, blobsToRead+1) // +1 for sd blob
+	for i := 1; i < blobsToRead+1; i++ {  // start at 1 to skip sd blob
+		stream[i], err = enc.Next()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	sdBlob := enc.SDBlob()
+
+	if len(sdBlob.BlobInfos) != blobsToRead {
+		t.Errorf("expected %d blobs in partial sdblob, got %d", blobsToRead, len(sdBlob.BlobInfos))
+	}
+	if enc.SourceLen() != maxBlobDataSize*blobsToRead {
+		t.Errorf("expected length of %d , got %d", maxBlobDataSize*blobsToRead, enc.SourceLen())
+	}
+
+	// now finish the stream, reusing key and IVs
+
+	buf = bytes.NewBuffer(data) // rewind to the beginning of the data
+
+	enc = NewEncoderFromSD(buf, sdBlob)
+
+	reconstructedStream, err := enc.Stream()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(reconstructedStream) != totalBlobs+1 { // +1 for the terminating blob at the end
+		t.Errorf("expected %d blobs in stream, got %d", totalBlobs+1, len(reconstructedStream))
+	}
+	if enc.SourceLen() != len(data) {
+		t.Errorf("expected length of %d , got %d", len(data), enc.SourceLen())
+	}
+
+	reconstructedSDBlob := enc.SDBlob()
+
+	for i := 0; i < len(sdBlob.BlobInfos); i++ {
+		if !bytes.Equal(sdBlob.BlobInfos[i].IV, reconstructedSDBlob.BlobInfos[i].IV) {
+			t.Errorf("blob info %d of reconstructed sd blobd does not match original sd blob", i)
+		}
+	}
+	for i := 1; i < len(stream); i++ { // start at 1 to skip sd blob
+		if !bytes.Equal(stream[i], reconstructedStream[i]) {
+			t.Errorf("blob %d of reconstructed stream does not match original stream", i)
+		}
+	}
+}
+
+func TestEmptyStream(t *testing.T) {
+	enc := NewEncoder(bytes.NewBuffer(nil))
+	_, err := enc.Next()
+	if !errors.Is(err, io.EOF) {
+		t.Errorf("expected io.EOF, got %v", err)
+	}
+	sd := enc.SDBlob()
+	if len(sd.BlobInfos) != 1 {
+		t.Errorf("expected 1 blobinfos in sd blob, got %d", len(sd.BlobInfos))
+	}
+	if sd.BlobInfos[0].Length != 0 {
+		t.Errorf("first and only blob to be the terminator blob")
+	}
+}
+
+func TestTermination(t *testing.T) {
+	b := make([]byte, 12)
+
+	enc := NewEncoder(bytes.NewBuffer(b))
+
+	_, err := enc.Next()
+	if err != nil {
+		t.Error(err)
+	}
+	if enc.isTerminated() {
+		t.Errorf("stream should not terminate until after EOF")
+	}
+
+	_, err = enc.Next()
+	if !errors.Is(err, io.EOF) {
+		t.Errorf("expected io.EOF, got %v", err)
+	}
+	if !enc.isTerminated() {
+		t.Errorf("stream should be terminated after EOF")
+	}
+
+	_, err = enc.Next()
+	if !errors.Is(err, io.EOF) {
+		t.Errorf("expected io.EOF on all subsequent reads, got %v", err)
+	}
+	sd := enc.SDBlob()
+	if len(sd.BlobInfos) != 2 {
+		t.Errorf("expected 2 blobinfos in sd blob, got %d", len(sd.BlobInfos))
+	}
+}
+
+func TestSizeHint(t *testing.T) {
+	b := make([]byte, 12)
+
+	newStream, err := NewEncoder(bytes.NewBuffer(b)).SourceSizeHint(5 * maxBlobDataSize).Stream()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if cap(newStream) != 2 { // 1 for sd blob, 1 for the 12 bytes of the actual stream
+		t.Fatalf("expected 2 blobs allocated, got %d", cap(newStream))
+	}
+}
+
 func TestNew(t *testing.T) {
 	t.Skip("TODO: test new stream creation and decryption")
 }

url/url.go

@@ -311,7 +311,7 @@ func parseModifier(modSeparator string, modValue string) (*UriModifier, error) {
 		if modSeparator == "#" {
 			claimId = modValue
 		} else if modSeparator == ":" {
-			claimSequence = parseInt(modValue, -1)
+			claimId = modValue
 		} else if modSeparator == "$" {
 			bidPosition = parseInt(modValue, -1)
 		}