From d63aba568d2fcbb9bbc4b054329d8794adbc3366 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 15 Aug 2019 23:34:25 -0400 Subject: [PATCH 01/20] Add transfer logic as post sync process step. Add authtoken for youtuber in test data setup to call transfer api Update e2e test to include scenario where we transfer channel. --- e2e/data_setup.sh | 3 +++ e2e/e2e.sh | 23 ++++++++++++++------- e2e/lbrycrd/docker/build.sh | 2 +- manager/manager.go | 4 ++++ manager/setup.go | 3 +++ manager/transfer.go | 41 +++++++++++++++++++++++++++++++++++++ manager/ytsync.go | 13 +++++++++++- sdk/api.go | 2 ++ 8 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 manager/transfer.go diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh index 2d46529..a1ad9a1 100755 --- a/e2e/data_setup.sh +++ b/e2e/data_setup.sh @@ -15,6 +15,9 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP" #Add youtuber to sync ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")' mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER" +#Insert an auth token for the youtuber to be used by ytsync +ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" #Add their youtube channel to be synced ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers) VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)" diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 027b5d0..96e738e 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -50,16 +50,25 @@ echo "successfully started..." #Data Setup for test ./data_setup.sh -# Execute the test! +# Execute the sync test! ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container -# Assert the status status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1') videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1') -if [[ $status != "synced" || $videoStatus != "published" ]]; then -docker-compose logs --tail="all" lbrycrd -docker-compose logs --tail="all" walletserver -docker-compose logs --tail="all" lbrynet -docker-compose logs --tail="all" internalapis +# Reset status for tranfer test +mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1" +# Trigger transfer api +echo "curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n1Ygra2pyD6cpESv9GtPM9kDkr4bPeu1Dc" +# Execute the transfer test! +./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container +transferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM youtube_data WHERE id=1') +if [[ $status != "synced" || $videoStatus != "published" || transferStatus != "1" ]]; then +echo "Channel Status: $status" +echo "Video Status: $videoStatus" +echo "Transfer Status: $transferStatus" +#docker-compose logs --tail="all" lbrycrd +#docker-compose logs --tail="all" walletserver +#docker-compose logs --tail="all" lbrynet +#docker-compose logs --tail="all" internalapis echo "List local /var/tmp" find /var/tmp exit 1; fi; \ No newline at end of file diff --git a/e2e/lbrycrd/docker/build.sh b/e2e/lbrycrd/docker/build.sh index 0487e72..12ad558 100755 --- a/e2e/lbrycrd/docker/build.sh +++ b/e2e/lbrycrd/docker/build.sh @@ -4,5 +4,5 @@ if [ $# -eq 0 ] echo "No docker tag argument supplied. Use './build.sh '" exit 1 fi -docker build --tag lbry/lbrycrd:$1 . +docker build --build-arg VERSION=$1 --tag lbry/lbrycrd:$1 . docker push lbry/lbrycrd:$1 \ No newline at end of file diff --git a/manager/manager.go b/manager/manager.go index fb72f1d..6a31f08 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -147,6 +147,8 @@ func (s *SyncManager) Start() error { AwsS3Bucket: s.awsS3Bucket, namer: namer.NewNamer(), Fee: channels[0].Fee, + publishAddress: channels[0].PublishAddress, + transferState: channels[0].TransferState, } shouldInterruptLoop = true } else { @@ -189,6 +191,8 @@ func (s *SyncManager) Start() error { AwsS3Bucket: s.awsS3Bucket, namer: namer.NewNamer(), Fee: c.Fee, + publishAddress: c.PublishAddress, + transferState: c.TransferState, }) if q != StatusFailed { continue queues diff --git a/manager/setup.go b/manager/setup.go index a669c50..4672d90 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -125,6 +125,9 @@ func (s *Sync) walletSetup() error { if s.claimAddress == "" { return errors.Err("found blank claim address") } + if s.transferState > 0 && s.publishAddress != "" { + s.claimAddress = s.publishAddress + } err = s.ensureEnoughUTXOs() if err != nil { diff --git a/manager/transfer.go b/manager/transfer.go new file mode 100644 index 0000000..1174051 --- /dev/null +++ b/manager/transfer.go @@ -0,0 +1,41 @@ +package manager + +import ( + "github.com/lbryio/lbry.go/extras/errors" + "github.com/lbryio/lbry.go/extras/jsonrpc" + "github.com/lbryio/lbry.go/extras/util" +) + +func TransferChannel(channel *Sync) error { + //Transfer channel + for _, video := range channel.syncedVideos { + //Todo - Wait for prior sync to see that the publish is confirmed in lbrycrd? + //Todo - We need to fix the ClaimSearch call in lbry.go for 38.5 lbrynet + c, err := channel.daemon.ClaimSearch(nil, &video.ClaimID, nil, nil) + if err != nil { + errors.Err(err) + } + if len(c.Claims) == 0 { + errors.Err("cannot transfer: no claim found for this video") + } else if len(c.Claims) > 1 { + errors.Err("cannot transfer: too many claims. claimID: %s", video.ClaimID) + } + + streamUpdateOptions := jsonrpc.StreamUpdateOptions{ + StreamCreateOptions: &jsonrpc.StreamCreateOptions{ + ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &channel.publishAddress}, + }, + Bid: util.PtrToString("0.009"), // Todo - Dont hardcode + } + + _, err = channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions) + if err != nil { + return err + } + // Todo - Post to remote db that video is transferred + } + + // Todo - Transfer Channel as last step and post back to remote db that channel is transferred. + + return nil +} diff --git a/manager/ytsync.go b/manager/ytsync.go index 39a93d8..2d1db77 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -88,6 +88,8 @@ type Sync struct { namer *namer.Namer walletMux *sync.RWMutex queue chan video + transferState int + publishAddress string } func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) { @@ -308,7 +310,16 @@ func (s *Sync) FullCycle() (e error) { return err } - return s.doSync() + err = s.doSync() + if err != nil { + return err + } + + if s.transferState == 1 && s.publishAddress != "" { // Channel needs transfer + return TransferChannel(s) + } + + return nil } func deleteSyncFolder(videoDirectory string) { diff --git a/sdk/api.go b/sdk/api.go index 952200b..e683201 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -45,6 +45,8 @@ type YoutubeChannel struct { DesiredChannelName string `json:"desired_channel_name"` Fee *Fee `json:"fee"` ChannelClaimID string `json:"channel_claim_id"` + TransferState int `json:"transfer_state"` + PublishAddress string `json:"publish_address"` } func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) { -- 2.45.2 From 9a5b6b4e56026b9c7ded7e875277ced10f073f9e Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Mon, 19 Aug 2019 22:18:09 -0400 Subject: [PATCH 02/20] Update to leverage lbrycrd v0.17.2.1, lbrynet v0.39.1, walletserver v0.39.1 --- e2e/docker-compose.yml | 8 ++-- e2e/lbrycrd/docker/start.sh | 57 +++++++++++++++++------------ e2e/walletserver/docker-compose.yml | 2 +- manager/setup.go | 6 +-- 4 files changed, 42 insertions(+), 31 deletions(-) diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 73e67b0..9416bd5 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -4,8 +4,8 @@ services: ## Lbrycrd ## ############# lbrycrd: - image: lbry/lbrycrd:v0.12.4.1 - restart: always + image: lbry/lbrycrd:v0.17.2.1 + restart: "no" ports: - "15201:29246" - "15200:29245" @@ -21,7 +21,7 @@ services: ## Wallet Server ## ################### walletserver: - image: lbry/wallet-server:v0.38.5 + image: lbry/wallet-server:v0.39.1 restart: always environment: - DB_DIRECTORY=/database @@ -47,7 +47,7 @@ services: ## Lbrynet ## ############# lbrynet: - image: lbry/lbrynet:v0.38.6 + image: lbry/lbrynet:v0.39.1 restart: always ports: - "15100:5279" diff --git a/e2e/lbrycrd/docker/start.sh b/e2e/lbrycrd/docker/start.sh index 89bbb76..0f25414 100644 --- a/e2e/lbrycrd/docker/start.sh +++ b/e2e/lbrycrd/docker/start.sh @@ -38,12 +38,15 @@ function set_config() { else echo "Creating a fresh config file from environment variables." ## Set config params - echo "port=${PORT=9246}" > $CONFIG_PATH - echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH - echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH + echo "port=${PORT=9246}" > $CONFIG_PATH + echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH + echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH - echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH - echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH + echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH + echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH + echo "deprecatedrpc=accounts" >> $CONFIG_PATH + echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH + echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH fi echo "Config: " cat $CONFIG_PATH @@ -77,15 +80,20 @@ case $RUN_MODE in ## Set config params ## TODO: Make this more automagic in the future. mkdir -p `dirname $CONFIG_PATH` - echo "rpcuser=lbry" > $CONFIG_PATH - echo "rpcpassword=lbry" >> $CONFIG_PATH - echo "rpcport=29245" >> $CONFIG_PATH - echo "rpcbind=0.0.0.0" >> $CONFIG_PATH - echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH - echo "regtest=1" >> $CONFIG_PATH - echo "txindex=1" >> $CONFIG_PATH - echo "server=1" >> $CONFIG_PATH - echo "printtoconsole=1" >> $CONFIG_PATH + echo "rpcuser=lbry" > $CONFIG_PATH + echo "rpcpassword=lbry" >> $CONFIG_PATH + echo "rpcport=29245" >> $CONFIG_PATH + echo "rpcbind=0.0.0.0" >> $CONFIG_PATH + echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH + echo "regtest=1" >> $CONFIG_PATH + echo "txindex=1" >> $CONFIG_PATH + echo "server=1" >> $CONFIG_PATH + echo "printtoconsole=1" >> $CONFIG_PATH + echo "deprecatedrpc=accounts" >> $CONFIG_PATH + echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH + echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH + echo "vbparams=segwit:0:999999999999" >> $CONFIG_PATH + echo "addresstype=legacy" >> $CONFIG_PATH #nohup advance &>/dev/null & lbrycrdd -conf=$CONFIG_PATH $1 @@ -94,15 +102,18 @@ case $RUN_MODE in ## Set config params ## TODO: Make this more automagic in the future. mkdir -p `dirname $CONFIG_PATH` - echo "rpcuser=lbry" > $CONFIG_PATH - echo "rpcpassword=lbry" >> $CONFIG_PATH - echo "rpcport=29245" >> $CONFIG_PATH - echo "rpcbind=0.0.0.0" >> $CONFIG_PATH - echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH - echo "testnet=1" >> $CONFIG_PATH - echo "txindex=1" >> $CONFIG_PATH - echo "server=1" >> $CONFIG_PATH - echo "printtoconsole=1" >> $CONFIG_PATH + echo "rpcuser=lbry" > $CONFIG_PATH + echo "rpcpassword=lbry" >> $CONFIG_PATH + echo "rpcport=29245" >> $CONFIG_PATH + echo "rpcbind=0.0.0.0" >> $CONFIG_PATH + echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH + echo "testnet=1" >> $CONFIG_PATH + echo "txindex=1" >> $CONFIG_PATH + echo "server=1" >> $CONFIG_PATH + echo "printtoconsole=1" >> $CONFIG_PATH + echo "deprecatedrpc=accounts" >> $CONFIG_PATH + echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH + echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH #nohup advance &>/dev/null & lbrycrdd -conf=$CONFIG_PATH $1 diff --git a/e2e/walletserver/docker-compose.yml b/e2e/walletserver/docker-compose.yml index a3104ba..034b0cc 100644 --- a/e2e/walletserver/docker-compose.yml +++ b/e2e/walletserver/docker-compose.yml @@ -8,7 +8,7 @@ services: ## Wallet Server ## ################### walletserver: - image: lbry/wallet-server:v0.38.5 + image: lbry/wallet-server:v0.38.6 restart: always networks: lbry-network: diff --git a/manager/setup.go b/manager/setup.go index 4672d90..c57b8c8 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -55,7 +55,7 @@ func (s *Sync) walletSetup() error { } else if balanceResp == nil { return errors.Err("no response") } - balance, err := strconv.ParseFloat((string)(*balanceResp), 64) + balance, err := strconv.ParseFloat(balanceResp.Total.String(), 64) if err != nil { return errors.Err(err) } @@ -188,7 +188,7 @@ func (s *Sync) ensureEnoughUTXOs() error { return errors.Err("no response") } - balanceAmount, err := strconv.ParseFloat((string)(*balance), 64) + balanceAmount, err := strconv.ParseFloat(balance.Total.String(), 64) if err != nil { return errors.Err(err) } @@ -312,7 +312,7 @@ func (s *Sync) ensureChannelOwnership() error { } else if balanceResp == nil { return errors.Err("no response") } - balance, err := decimal.NewFromString((string)(*balanceResp)) + balance, err := decimal.NewFromString(balanceResp.Total.String()) if err != nil { return errors.Err(err) } -- 2.45.2 From 45cea808edc16e710ec4b95e5d110b01fddde085 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 20 Aug 2019 01:06:51 -0400 Subject: [PATCH 03/20] Add marking video status when transfer of claim is complete. Add error handling when transfer of claim fails. --- e2e/e2e.sh | 4 ++-- manager/manager.go | 1 + manager/transfer.go | 23 +++++++++++++++++++--- manager/ytsync.go | 30 ++++++++++++++++++++++++++--- sdk/api.go | 47 ++++++++++++++++++++++++++++++--------------- 5 files changed, 81 insertions(+), 24 deletions(-) diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 96e738e..b0ed652 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -60,8 +60,10 @@ mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data echo "curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n1Ygra2pyD6cpESv9GtPM9kDkr4bPeu1Dc" # Execute the transfer test! ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container +# ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED transferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM youtube_data WHERE id=1') if [[ $status != "synced" || $videoStatus != "published" || transferStatus != "1" ]]; then +echo "~~!!!~~~FAILED~~~!!!~~" echo "Channel Status: $status" echo "Video Status: $videoStatus" echo "Transfer Status: $transferStatus" @@ -69,6 +71,4 @@ echo "Transfer Status: $transferStatus" #docker-compose logs --tail="all" walletserver #docker-compose logs --tail="all" lbrynet #docker-compose logs --tail="all" internalapis -echo "List local /var/tmp" -find /var/tmp exit 1; fi; \ No newline at end of file diff --git a/manager/manager.go b/manager/manager.go index 6a31f08..0ad6c38 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -97,6 +97,7 @@ const ( VideoStatusFailed = "failed" VideoStatusUpgradeFailed = "upgradefailed" VideoStatusUnpublished = "unpublished" + VideoStatusTranferFailed = "transferfailed" ) func (s *SyncManager) Start() error { diff --git a/manager/transfer.go b/manager/transfer.go index 1174051..9c51234 100644 --- a/manager/transfer.go +++ b/manager/transfer.go @@ -4,13 +4,13 @@ import ( "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/jsonrpc" "github.com/lbryio/lbry.go/extras/util" + "github.com/lbryio/ytsync/sdk" ) func TransferChannel(channel *Sync) error { //Transfer channel for _, video := range channel.syncedVideos { //Todo - Wait for prior sync to see that the publish is confirmed in lbrycrd? - //Todo - We need to fix the ClaimSearch call in lbry.go for 38.5 lbrynet c, err := channel.daemon.ClaimSearch(nil, &video.ClaimID, nil, nil) if err != nil { errors.Err(err) @@ -28,11 +28,28 @@ func TransferChannel(channel *Sync) error { Bid: util.PtrToString("0.009"), // Todo - Dont hardcode } + videoStatus := sdk.VideoStatus{ + ChannelID: channel.YoutubeChannelID, + VideoID: video.VideoID, + ClaimID: video.ClaimID, + ClaimName: video.ClaimName, + Status: VideoStatusPublished, + IsTransferred: util.PtrToBool(true), + } + _, err = channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions) if err != nil { - return err + videoStatus.FailureReason = err.Error() + videoStatus.Status = VideoStatusTranferFailed + videoStatus.IsTransferred = util.PtrToBool(false) + } + statusErr := channel.APIConfig.MarkVideoStatus(videoStatus) + if statusErr != nil { + return errors.Err(err) + } + if err != nil { + return errors.Err(err) } - // Todo - Post to remote db that video is transferred } // Todo - Transfer Channel as last step and post back to remote db that channel is transferred. diff --git a/manager/ytsync.go b/manager/ytsync.go index 2d1db77..1d2803f 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -511,7 +511,15 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int } fixed++ log.Debugf("updating %s in the database", videoID) - err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(claimSize)), claimMetadataVersion) + err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{ + ChannelID: s.YoutubeChannelID, + VideoID: videoID, + Status: VideoStatusPublished, + ClaimID: c.ClaimID, + ClaimName: c.Name, + Size: util.PtrToInt64(int64(claimSize)), + MetaDataVersion: claimMetadataVersion, + }) if err != nil { return count, fixed, 0, err } @@ -763,7 +771,15 @@ func (s *Sync) startWorker(workerNum int) { } else { s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize) } - err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0) + err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{ + ChannelID: s.YoutubeChannelID, + VideoID: v.ID(), + Status: videoStatus, + ClaimID: existingClaimID, + ClaimName: existingClaimName, + FailureReason: err.Error(), + Size: &existingClaimSize, + }) if err != nil { logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err)) } @@ -954,7 +970,15 @@ func (s *Sync) processVideo(v video) (err error) { } s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size()) - err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2) + err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{ + ChannelID: s.YoutubeChannelID, + VideoID: v.ID(), + Status: VideoStatusPublished, + ClaimID: summary.ClaimID, + ClaimName: summary.ClaimName, + Size: v.Size(), + MetaDataVersion: 2, + }) if err != nil { logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err)) } diff --git a/sdk/api.go b/sdk/api.go index e683201..d3b8559 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -236,32 +236,47 @@ func (a *APIConfig) DeleteVideos(videos []string) error { return errors.Err("invalid API response. Status code: %d", res.StatusCode) } -func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error { +type VideoStatus struct { + ChannelID string + VideoID string + Status string + ClaimID string + ClaimName string + FailureReason string + Size *int64 + MetaDataVersion uint + IsTransferred *bool +} + +func (a *APIConfig) MarkVideoStatus(status VideoStatus) error { endpoint := a.ApiURL + "/yt/video_status" - sanitizeFailureReason(&failureReason) + sanitizeFailureReason(&status.FailureReason) vals := url.Values{ - "youtube_channel_id": {channelID}, - "video_id": {videoID}, - "status": {status}, + "youtube_channel_id": {status.ChannelID}, + "video_id": {status.VideoID}, + "status": {status.Status}, "auth_token": {a.ApiToken}, } - if status == VideoStatusPublished || status == VideoStatusUpgradeFailed { - if claimID == "" || claimName == "" { - return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName) + if status.Status == VideoStatusPublished || status.Status == VideoStatusUpgradeFailed { + if status.ClaimID == "" || status.ClaimName == "" { + return errors.Err("claimID (%s) or claimName (%s) missing", status.ClaimID, status.ClaimName) } vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10)) - vals.Add("claim_id", claimID) - vals.Add("claim_name", claimName) - if metadataVersion > 0 { - vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion)) + vals.Add("claim_id", status.ClaimID) + vals.Add("claim_name", status.ClaimName) + if status.MetaDataVersion > 0 { + vals.Add("metadata_version", fmt.Sprintf("%d", status.MetaDataVersion)) } - if size != nil { - vals.Add("size", strconv.FormatInt(*size, 10)) + if status.Size != nil { + vals.Add("size", strconv.FormatInt(*status.Size, 10)) } } - if failureReason != "" { - vals.Add("failure_reason", failureReason) + if status.FailureReason != "" { + vals.Add("failure_reason", status.FailureReason) + } + if status.IsTransferred != nil { + vals.Add("transferred", strconv.FormatBool(*status.IsTransferred)) } res, _ := http.PostForm(endpoint, vals) defer res.Body.Close() -- 2.45.2 From eb2f6273e405b6a4cb08ad0a8e991b97f7a70d84 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 20 Aug 2019 23:38:36 -0400 Subject: [PATCH 04/20] Add transfer of channel after all video's have been transferred Add channel status call to notify external db that the channel has been transferred. --- manager/manager.go | 7 +++++++ manager/transfer.go | 34 +++++++++++++++++++++++++++++++--- sdk/api.go | 3 ++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/manager/manager.go b/manager/manager.go index 0ad6c38..aa956e1 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -100,6 +100,13 @@ const ( VideoStatusTranferFailed = "transferfailed" ) +const ( + TransferStateNotTouched = iota + TransferStatePending + TransferStateComplete + TransferStateFailed = -1 +) + func (s *SyncManager) Start() error { if logUtils.ShouldCleanOnStartup() { diff --git a/manager/transfer.go b/manager/transfer.go index 9c51234..8d9c4a9 100644 --- a/manager/transfer.go +++ b/manager/transfer.go @@ -8,7 +8,7 @@ import ( ) func TransferChannel(channel *Sync) error { - //Transfer channel + transferStatus := TransferStateComplete for _, video := range channel.syncedVideos { //Todo - Wait for prior sync to see that the publish is confirmed in lbrycrd? c, err := channel.daemon.ClaimSearch(nil, &video.ClaimID, nil, nil) @@ -39,6 +39,7 @@ func TransferChannel(channel *Sync) error { _, err = channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions) if err != nil { + transferStatus = TransferStateFailed videoStatus.FailureReason = err.Error() videoStatus.Status = VideoStatusTranferFailed videoStatus.IsTransferred = util.PtrToBool(false) @@ -51,8 +52,35 @@ func TransferChannel(channel *Sync) error { return errors.Err(err) } } - // Todo - Transfer Channel as last step and post back to remote db that channel is transferred. + //Transfer channel + if transferStatus < 0 { + return errors.Err("A video has failed to transfer for the channel...skipping channel transfer") + } + failureReason := "" + channelClaim, err := channel.daemon.ClaimSearch(nil, &channel.lbryChannelID, nil, nil) + if err != nil { + return errors.Err(err) + } + if channelClaim == nil { + return errors.Err("There is no channel claim for channel %s", channel.LbryChannelName) + } + updateOptions := jsonrpc.ChannelUpdateOptions{ + ChannelCreateOptions: jsonrpc.ChannelCreateOptions{ + ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ + ClaimAddress: &channel.publishAddress, + }, + }, + } + _, err = channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions) + if err != nil { + transferStatus = TransferStateFailed + failureReason = err.Error() + } - return nil + _, _, channelStatusErr := channel.APIConfig.SetChannelStatus(channel.lbryChannelID, VideoStatusPublished, failureReason, transferStatus) + if channelStatusErr != nil { + return errors.Err(err) + } + return errors.Err(err) } diff --git a/sdk/api.go b/sdk/api.go index d3b8559..837083e 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -133,7 +133,7 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error { } -func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string) (map[string]SyncedVideo, map[string]bool, error) { +func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string, transferState int) (map[string]SyncedVideo, map[string]bool, error) { type apiChannelStatusResponse struct { Success bool `json:"success"` Error null.String `json:"error"` @@ -148,6 +148,7 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea "auth_token": {a.ApiToken}, "sync_status": {status}, "failure_reason": {failureReason}, + "transferred": {strconv.Itoa(transferState)}, }) defer res.Body.Close() body, _ := ioutil.ReadAll(res.Body) -- 2.45.2 From 1b9ed266e082565c646d2110f54d4724d999db9b Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 21 Aug 2019 19:40:35 +0200 Subject: [PATCH 05/20] move channel status call to proper region fix shadowed errors refactor code to be more readable and consistent get video transfer state --- go.mod | 2 +- go.sum | 2 ++ manager/manager.go | 1 + manager/transfer.go | 28 +++++++++++----------------- manager/ytsync.go | 23 +++++++++++++++++------ sdk/api.go | 12 ++++++++---- 6 files changed, 40 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 3dd933a..b39bb77 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/hashicorp/serf v0.8.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v1.1.2 + github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/go.sum b/go.sum index f37b02d..2ebcb3b 100644 --- a/go.sum +++ b/go.sum @@ -177,6 +177,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= +github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 h1:3FKVv1m/J8mVQJTwIcF5BFv2E87y+nYqBqofJs6F720= +github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c= github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= diff --git a/manager/manager.go b/manager/manager.go index aa956e1..7fac0db 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -89,6 +89,7 @@ const ( StatusFinalized = "finalized" // no more changes allowed StatusAbandoned = "abandoned" // deleted on youtube or banned ) +const LatestMetadataVersion = 2 var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned} diff --git a/manager/transfer.go b/manager/transfer.go index 8d9c4a9..b23e5f3 100644 --- a/manager/transfer.go +++ b/manager/transfer.go @@ -7,18 +7,22 @@ import ( "github.com/lbryio/ytsync/sdk" ) -func TransferChannel(channel *Sync) error { - transferStatus := TransferStateComplete +func TransferChannelAndVideos(channel *Sync) error { + cleanTransfer := true for _, video := range channel.syncedVideos { + if !channel.syncedVideos[video.ClaimID].Published || channel.syncedVideos[video.ClaimID].Transferred || channel.syncedVideos[video.ClaimID].MetadataVersion != LatestMetadataVersion { + continue + } + //Todo - Wait for prior sync to see that the publish is confirmed in lbrycrd? c, err := channel.daemon.ClaimSearch(nil, &video.ClaimID, nil, nil) if err != nil { - errors.Err(err) + return errors.Err(err) } if len(c.Claims) == 0 { - errors.Err("cannot transfer: no claim found for this video") + return errors.Err("cannot transfer: no claim found for this video") } else if len(c.Claims) > 1 { - errors.Err("cannot transfer: too many claims. claimID: %s", video.ClaimID) + return errors.Err("cannot transfer: too many claims. claimID: %s", video.ClaimID) } streamUpdateOptions := jsonrpc.StreamUpdateOptions{ @@ -39,7 +43,7 @@ func TransferChannel(channel *Sync) error { _, err = channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions) if err != nil { - transferStatus = TransferStateFailed + cleanTransfer = false videoStatus.FailureReason = err.Error() videoStatus.Status = VideoStatusTranferFailed videoStatus.IsTransferred = util.PtrToBool(false) @@ -54,10 +58,9 @@ func TransferChannel(channel *Sync) error { } // Todo - Transfer Channel as last step and post back to remote db that channel is transferred. //Transfer channel - if transferStatus < 0 { + if !cleanTransfer { return errors.Err("A video has failed to transfer for the channel...skipping channel transfer") } - failureReason := "" channelClaim, err := channel.daemon.ClaimSearch(nil, &channel.lbryChannelID, nil, nil) if err != nil { return errors.Err(err) @@ -73,14 +76,5 @@ func TransferChannel(channel *Sync) error { }, } _, err = channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions) - if err != nil { - transferStatus = TransferStateFailed - failureReason = err.Error() - } - - _, _, channelStatusErr := channel.APIConfig.SetChannelStatus(channel.lbryChannelID, VideoStatusPublished, failureReason, transferStatus) - if channelStatusErr != nil { - return errors.Err(err) - } return errors.Err(err) } diff --git a/manager/ytsync.go b/manager/ytsync.go index 1d2803f..ec746a7 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -227,7 +227,7 @@ func (s *Sync) uploadWallet() error { } func (s *Sync) setStatusSyncing() error { - syncedVideos, claimNames, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSyncing, "") + syncedVideos, claimNames, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSyncing, "", nil) if err != nil { return err } @@ -315,8 +315,8 @@ func (s *Sync) FullCycle() (e error) { return err } - if s.transferState == 1 && s.publishAddress != "" { // Channel needs transfer - return TransferChannel(s) + if s.shouldTransfer() { + return TransferChannelAndVideos(s) } return nil @@ -331,8 +331,19 @@ func deleteSyncFolder(videoDirectory string) { _ = util.SendToSlack(err.Error()) } } - +func (s *Sync) shouldTransfer() bool { + return s.transferState == 1 && s.publishAddress != "" +} func (s *Sync) setChannelTerminationStatus(e *error) { + var transferState *int + + if s.shouldTransfer() { + if *e != nil { + transferState = util.PtrToInt(TransferStateComplete) + } else { + transferState = util.PtrToInt(TransferStateFailed) + } + } if *e != nil { //conditions for which a channel shouldn't be marked as failed noFailConditions := []string{ @@ -343,13 +354,13 @@ func (s *Sync) setChannelTerminationStatus(e *error) { return } failureReason := (*e).Error() - _, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusFailed, failureReason) + _, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusFailed, failureReason, transferState) if err != nil { msg := fmt.Sprintf("Failed setting failed state for channel %s", s.LbryChannelName) *e = errors.Prefix(msg+err.Error(), *e) } } else if !s.IsInterrupted() { - _, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSynced, "") + _, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSynced, "", transferState) if err != nil { *e = err } diff --git a/sdk/api.go b/sdk/api.go index 837083e..7795ac2 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -90,6 +90,7 @@ type SyncedVideo struct { ClaimID string `json:"claim_id"` Size int64 `json:"size"` MetadataVersion int8 `json:"metadata_version"` + Transferred bool `json:"transferred"` } func sanitizeFailureReason(s *string) { @@ -133,7 +134,7 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error { } -func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string, transferState int) (map[string]SyncedVideo, map[string]bool, error) { +func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string, transferState *int) (map[string]SyncedVideo, map[string]bool, error) { type apiChannelStatusResponse struct { Success bool `json:"success"` Error null.String `json:"error"` @@ -142,14 +143,17 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea endpoint := a.ApiURL + "/yt/channel_status" sanitizeFailureReason(&failureReason) - res, _ := http.PostForm(endpoint, url.Values{ + params := url.Values{ "channel_id": {channelID}, "sync_server": {a.HostName}, "auth_token": {a.ApiToken}, "sync_status": {status}, "failure_reason": {failureReason}, - "transferred": {strconv.Itoa(transferState)}, - }) + } + if transferState != nil { + params.Add("transferred", strconv.Itoa(*transferState)) + } + res, _ := http.PostForm(endpoint, params) defer res.Body.Close() body, _ := ioutil.ReadAll(res.Body) var response apiChannelStatusResponse -- 2.45.2 From 7a7de03c0f9c706c9b0caa06083b221720ae57f4 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 21 Aug 2019 20:01:24 +0200 Subject: [PATCH 06/20] adjust account balance minor fix --- manager/setup.go | 6 +++--- manager/ytsync.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/manager/setup.go b/manager/setup.go index c57b8c8..856081f 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -55,7 +55,7 @@ func (s *Sync) walletSetup() error { } else if balanceResp == nil { return errors.Err("no response") } - balance, err := strconv.ParseFloat(balanceResp.Total.String(), 64) + balance, err := strconv.ParseFloat(balanceResp.Available.String(), 64) if err != nil { return errors.Err(err) } @@ -188,7 +188,7 @@ func (s *Sync) ensureEnoughUTXOs() error { return errors.Err("no response") } - balanceAmount, err := strconv.ParseFloat(balance.Total.String(), 64) + balanceAmount, err := strconv.ParseFloat(balance.Available.String(), 64) if err != nil { return errors.Err(err) } @@ -312,7 +312,7 @@ func (s *Sync) ensureChannelOwnership() error { } else if balanceResp == nil { return errors.Err("no response") } - balance, err := decimal.NewFromString(balanceResp.Total.String()) + balance, err := decimal.NewFromString(balanceResp.Available.String()) if err != nil { return errors.Err(err) } diff --git a/manager/ytsync.go b/manager/ytsync.go index ec746a7..72b92ce 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -988,7 +988,7 @@ func (s *Sync) processVideo(v video) (err error) { ClaimID: summary.ClaimID, ClaimName: summary.ClaimName, Size: v.Size(), - MetaDataVersion: 2, + MetaDataVersion: LatestMetadataVersion, }) if err != nil { logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err)) -- 2.45.2 From c754562f966570fd5b7dc52d590e1851f1dda8c4 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Mon, 19 Aug 2019 22:18:09 -0400 Subject: [PATCH 07/20] Update to leverage lbrycrd v0.17.2.1, lbrynet v0.39.1, walletserver v0.39.1 --- go.mod | 2 +- go.sum | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index b39bb77..95fbfbf 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/hashicorp/serf v0.8.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 + github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/go.sum b/go.sum index 2ebcb3b..5f11304 100644 --- a/go.sum +++ b/go.sum @@ -177,10 +177,16 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= -github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 h1:3FKVv1m/J8mVQJTwIcF5BFv2E87y+nYqBqofJs6F720= -github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c= -github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035 h1:eGjqH+JB8+PR98NrgCOfhFXNe4jM9TWGeqXpAVtvaR4= +github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea h1:vFNLqaOFnFe5x0yOhGJJUmCCMfbSKjfh/KzILwJj5+Q= +github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.0.17 h1:Lt7xIguw6Q5LMKcEMfxD7IB7kUdP7xwrc0OxvBUl39E= +github.com/lbryio/lbry.go v1.0.17/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.0.18 h1:88ot2tNyMFM0g8xCxyruCra2c+e0huazcwihT3pX/Lc= +github.com/lbryio/lbry.go v1.0.18/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.1.0 h1:LDOjMROscAVbmjLYBh988E/UFqzunPik35kcRd/PUR4= +github.com/lbryio/lbry.go v1.1.0/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= -- 2.45.2 From 2c9c999c71e0ead48b2e752a8df2fb48be7e459a Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 20 Aug 2019 01:06:51 -0400 Subject: [PATCH 08/20] Add marking video status when transfer of claim is complete. Add error handling when transfer of claim fails. --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 95fbfbf..b39bb77 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/hashicorp/serf v0.8.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea + github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/go.sum b/go.sum index 5f11304..052e677 100644 --- a/go.sum +++ b/go.sum @@ -187,6 +187,8 @@ github.com/lbryio/lbry.go v1.0.18 h1:88ot2tNyMFM0g8xCxyruCra2c+e0huazcwihT3pX/Lc github.com/lbryio/lbry.go v1.0.18/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.1.0 h1:LDOjMROscAVbmjLYBh988E/UFqzunPik35kcRd/PUR4= github.com/lbryio/lbry.go v1.1.0/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 h1:3FKVv1m/J8mVQJTwIcF5BFv2E87y+nYqBqofJs6F720= +github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= -- 2.45.2 From 1fb1077a942ee78fa859d3813dcce26f0e046474 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 22 Aug 2019 00:09:33 -0400 Subject: [PATCH 09/20] Update curl call to show response Update channel status call to pass the correct name of the parameter --- e2e/e2e.sh | 2 +- sdk/api.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/e2e.sh b/e2e/e2e.sh index b0ed652..25e0988 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -57,7 +57,7 @@ videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELEC # Reset status for tranfer test mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1" # Trigger transfer api -echo "curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n1Ygra2pyD6cpESv9GtPM9kDkr4bPeu1Dc" +curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n1Ygra2pyD6cpESv9GtPM9kDkr4bPeu1Dc' # Execute the transfer test! ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container # ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED diff --git a/sdk/api.go b/sdk/api.go index 7795ac2..06ad84a 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -151,7 +151,7 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea "failure_reason": {failureReason}, } if transferState != nil { - params.Add("transferred", strconv.Itoa(*transferState)) + params.Add("transfer_state", strconv.Itoa(*transferState)) } res, _ := http.PostForm(endpoint, params) defer res.Body.Close() -- 2.45.2 From 4d16a1d6e8684c82b8e82e03d26ca27b946e25bc Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 23 Aug 2019 01:28:51 +0200 Subject: [PATCH 10/20] fix e2e tests update wallet server cleanup go.sum Fix broken balance logging Prevent failures once channel is transferred Wait for all UTXOs to confirm before transferring Fix various bugs --- e2e/docker-compose.yml | 4 +-- e2e/e2e.sh | 27 ++++++++++-------- go.sum | 10 ------- manager/setup.go | 6 +++- manager/transfer.go | 50 +++++++++++++++++++++++++++------ manager/ytsync.go | 64 ++++++++++++++++++++++++++---------------- 6 files changed, 105 insertions(+), 56 deletions(-) diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 9416bd5..0e053e2 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -21,7 +21,7 @@ services: ## Wallet Server ## ################### walletserver: - image: lbry/wallet-server:v0.39.1 + image: lbry/wallet-server:v0.39.2 restart: always environment: - DB_DIRECTORY=/database @@ -84,7 +84,7 @@ services: ## Internal APIs ## ################### internalapis: - image: lbry/internal-apis:master + image: lbry/internal-apis:transfers restart: "no" ports: - "15400:8080" diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 25e0988..f1f1af1 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -61,14 +61,19 @@ curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http: # Execute the transfer test! ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container # ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED -transferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM youtube_data WHERE id=1') -if [[ $status != "synced" || $videoStatus != "published" || transferStatus != "1" ]]; then -echo "~~!!!~~~FAILED~~~!!!~~" -echo "Channel Status: $status" -echo "Video Status: $videoStatus" -echo "Transfer Status: $transferStatus" -#docker-compose logs --tail="all" lbrycrd -#docker-compose logs --tail="all" walletserver -#docker-compose logs --tail="all" lbrynet -#docker-compose logs --tail="all" internalapis - exit 1; fi; \ No newline at end of file +channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1') +videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1') +if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" ]]; then + echo "~~!!!~~~FAILED~~~!!!~~" + echo "Channel Status: $status" + echo "Video Status: $videoStatus" + echo "Channel Transfer Status: $channelTransferStatus" + echo "Video Transfer Status: $videoTransferStatus" + #docker-compose logs --tail="all" lbrycrd + #docker-compose logs --tail="all" walletserver + #docker-compose logs --tail="all" lbrynet + #docker-compose logs --tail="all" internalapis + exit 1; +fi; + +#perhaps query lbrynet again (should be restarted) to see if the claim and the channel are actually on the right address \ No newline at end of file diff --git a/go.sum b/go.sum index 052e677..3eee78b 100644 --- a/go.sum +++ b/go.sum @@ -177,16 +177,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= -github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035 h1:eGjqH+JB8+PR98NrgCOfhFXNe4jM9TWGeqXpAVtvaR4= -github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea h1:vFNLqaOFnFe5x0yOhGJJUmCCMfbSKjfh/KzILwJj5+Q= -github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbry.go v1.0.17 h1:Lt7xIguw6Q5LMKcEMfxD7IB7kUdP7xwrc0OxvBUl39E= -github.com/lbryio/lbry.go v1.0.17/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbry.go v1.0.18 h1:88ot2tNyMFM0g8xCxyruCra2c+e0huazcwihT3pX/Lc= -github.com/lbryio/lbry.go v1.0.18/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbry.go v1.1.0 h1:LDOjMROscAVbmjLYBh988E/UFqzunPik35kcRd/PUR4= -github.com/lbryio/lbry.go v1.1.0/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 h1:3FKVv1m/J8mVQJTwIcF5BFv2E87y+nYqBqofJs6F720= github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= diff --git a/manager/setup.go b/manager/setup.go index 856081f..cb9db4f 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -197,7 +197,8 @@ func (s *Sync) ensureEnoughUTXOs() error { if desiredUTXOCount > maxUTXOs { desiredUTXOCount = maxUTXOs } - log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount) + availableBalance, _ := balance.Available.Float64() + log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount) broadcastFee := 0.1 prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false) @@ -268,6 +269,9 @@ func (s *Sync) ensureChannelOwnership() error { if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" { return nil } + if s.transferState == TransferStateComplete { + return nil + } channels, err := s.daemon.ChannelList(nil, 1, 50) if err != nil { return err diff --git a/manager/transfer.go b/manager/transfer.go index b23e5f3..f2d3514 100644 --- a/manager/transfer.go +++ b/manager/transfer.go @@ -5,12 +5,43 @@ import ( "github.com/lbryio/lbry.go/extras/jsonrpc" "github.com/lbryio/lbry.go/extras/util" "github.com/lbryio/ytsync/sdk" + log "github.com/sirupsen/logrus" ) +func waitConfirmations(s *Sync) error { + allConfirmed := false +waiting: + for !allConfirmed { + utxolist, err := s.daemon.UTXOList(nil) + if err != nil { + return err + } else if utxolist == nil { + return errors.Err("no response") + } + + for _, utxo := range *utxolist { + if utxo.Confirmations <= 0 { + err = s.waitForNewBlock() + if err != nil { + return err + } + continue waiting + } + } + allConfirmed = true + } + return nil +} + func TransferChannelAndVideos(channel *Sync) error { + err := waitConfirmations(channel) + if err != nil { + return err + } cleanTransfer := true for _, video := range channel.syncedVideos { - if !channel.syncedVideos[video.ClaimID].Published || channel.syncedVideos[video.ClaimID].Transferred || channel.syncedVideos[video.ClaimID].MetadataVersion != LatestMetadataVersion { + if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion { + log.Debugf("skipping video: %s", video.VideoID) continue } @@ -41,18 +72,19 @@ func TransferChannelAndVideos(channel *Sync) error { IsTransferred: util.PtrToBool(true), } - _, err = channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions) - if err != nil { + result, updateError := channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions) + if updateError != nil { cleanTransfer = false - videoStatus.FailureReason = err.Error() + videoStatus.FailureReason = updateError.Error() videoStatus.Status = VideoStatusTranferFailed videoStatus.IsTransferred = util.PtrToBool(false) } + log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked statusErr := channel.APIConfig.MarkVideoStatus(videoStatus) if statusErr != nil { - return errors.Err(err) + return errors.Err(statusErr) } - if err != nil { + if updateError != nil { return errors.Err(err) } } @@ -65,7 +97,7 @@ func TransferChannelAndVideos(channel *Sync) error { if err != nil { return errors.Err(err) } - if channelClaim == nil { + if channelClaim == nil || len(channelClaim.Claims) == 0 { return errors.Err("There is no channel claim for channel %s", channel.LbryChannelName) } updateOptions := jsonrpc.ChannelUpdateOptions{ @@ -75,6 +107,8 @@ func TransferChannelAndVideos(channel *Sync) error { }, }, } - _, err = channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions) + result, err := channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions) + log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked + return errors.Err(err) } diff --git a/manager/ytsync.go b/manager/ytsync.go index 72b92ce..3c13e5e 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -339,9 +339,9 @@ func (s *Sync) setChannelTerminationStatus(e *error) { if s.shouldTransfer() { if *e != nil { - transferState = util.PtrToInt(TransferStateComplete) - } else { transferState = util.PtrToInt(TransferStateFailed) + } else { + transferState = util.PtrToInt(TransferStateComplete) } } if *e != nil { @@ -538,6 +538,10 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int } idsToRemove := make([]string, 0, len(videoIDMap)) for vID, sv := range s.syncedVideos { + if sv.Transferred { + log.Infof("%s: claim was transferred, ignoring") + continue + } _, ok := videoIDMap[vID] if !ok && sv.Published { log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified", vID) @@ -547,10 +551,10 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 { err := s.Manager.apiConfig.DeleteVideos(idsToRemove) if err != nil { - return count, fixed, 0, err + return count, fixed, len(idsToRemove), err } } - return count, fixed, len(idsToRemove), nil + return count, fixed, 0, nil } func (s *Sync) getClaims() ([]jsonrpc.Claim, error) { @@ -567,15 +571,7 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) { return allClaims, nil } -func (s *Sync) doSync() error { - err := s.enableAddressReuse() - if err != nil { - return errors.Prefix("could not set address reuse policy", err) - } - err = s.walletSetup() - if err != nil { - return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err) - } +func (s *Sync) checkIntegrity() error { allClaims, err := s.getClaims() if err != nil { return err @@ -601,17 +597,6 @@ func (s *Sync) doSync() error { return errors.Prefix("error updating remote database", err) } - cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil) - if err != nil { - return errors.Prefix("error getting channel cert", err) - } - if cert != nil { - err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID) - if err != nil { - return errors.Prefix("error setting channel cert", err) - } - } - if nFixed > 0 || nRemoved > 0 { err := s.setStatusSyncing() if err != nil { @@ -638,6 +623,36 @@ func (s *Sync) doSync() error { if pubsOnWallet < pubsOnDB { logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID) } + return nil +} + +func (s *Sync) doSync() error { + err := s.enableAddressReuse() + if err != nil { + return errors.Prefix("could not set address reuse policy", err) + } + err = s.walletSetup() + if err != nil { + return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err) + } + + err = s.checkIntegrity() + if err != nil { + return err + } + + if s.transferState != TransferStateComplete { + cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil) + if err != nil { + return errors.Prefix("error getting channel cert", err) + } + if cert != nil { + err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID) + if err != nil { + return errors.Prefix("error setting channel cert", err) + } + } + } if s.StopOnError { log.Println("Will stop publishing if an error is detected") @@ -989,6 +1004,7 @@ func (s *Sync) processVideo(v video) (err error) { ClaimName: summary.ClaimName, Size: v.Size(), MetaDataVersion: LatestMetadataVersion, + IsTransferred: util.PtrToBool(s.shouldTransfer()), }) if err != nil { logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err)) -- 2.45.2 From cd6ddaaa859ee9ea444a09e7b557c12d21d17463 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 22 Aug 2019 21:51:14 -0400 Subject: [PATCH 11/20] Update lbrynet to match wallet server version. --- e2e/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 0e053e2..b25dea7 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -47,7 +47,7 @@ services: ## Lbrynet ## ############# lbrynet: - image: lbry/lbrynet:v0.39.1 + image: lbry/lbrynet:v0.39.2 restart: always ports: - "15100:5279" -- 2.45.2 From 419434db280bedb51b0e87fef6317404fd3d0a06 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 22 Aug 2019 22:10:36 -0400 Subject: [PATCH 12/20] Add success message when the e2e test passes. --- e2e/e2e.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/e2e.sh b/e2e/e2e.sh index f1f1af1..3eb3c20 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -74,6 +74,8 @@ if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStat #docker-compose logs --tail="all" lbrynet #docker-compose logs --tail="all" internalapis exit 1; + else + echo "SUCCESSSSSSSSSSSSS!" fi; #perhaps query lbrynet again (should be restarted) to see if the claim and the channel are actually on the right address \ No newline at end of file -- 2.45.2 From 2d6e53be3275477b1deab41a8129ecce796e9c71 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 22 Aug 2019 23:02:40 -0400 Subject: [PATCH 13/20] Added creation of supports for the e2e test --- e2e/e2e.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 3eb3c20..060d277 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -54,6 +54,11 @@ echo "successfully started..." ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1') videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1') +videoClaimID =$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1') +# Create Supports for published claim +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","${videoClaimID}",1.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","${videoClaimID}",2.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","${videoClaimID}",3.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 # Reset status for tranfer test mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1" # Trigger transfer api -- 2.45.2 From f7c80c2e5d71ce8ce28c889123c33ea45aecee80 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 27 Aug 2019 00:31:27 +0200 Subject: [PATCH 14/20] refactor transfer process start working on supports --- go.mod | 2 +- go.sum | 4 ++-- manager/transfer.go | 41 +++++++++++++++++++++++++++++------------ manager/ytsync.go | 6 +++++- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index b39bb77..93b8dc6 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/hashicorp/serf v0.8.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 + github.com/lbryio/lbry.go v0.0.0-20190826222547-3a3377d0e5ec github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/go.sum b/go.sum index 3eee78b..fa04109 100644 --- a/go.sum +++ b/go.sum @@ -177,8 +177,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= -github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 h1:3FKVv1m/J8mVQJTwIcF5BFv2E87y+nYqBqofJs6F720= -github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v0.0.0-20190826222547-3a3377d0e5ec h1:wusAaT6POFVJwkwNA25I48Nr1DYTcuCAqmmum+WCYP8= +github.com/lbryio/lbry.go v0.0.0-20190826222547-3a3377d0e5ec/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= diff --git a/manager/transfer.go b/manager/transfer.go index f2d3514..b230511 100644 --- a/manager/transfer.go +++ b/manager/transfer.go @@ -33,20 +33,34 @@ waiting: return nil } -func TransferChannelAndVideos(channel *Sync) error { - err := waitConfirmations(channel) +func abandonSupports(s *Sync) error { + totalPages := uint64(1) + var allSupports []jsonrpc.Claim + for page := uint64(1); page <= totalPages; page++ { + supports, err := s.daemon.SupportList(nil, page, 50) + if err != nil { + return errors.Prefix("cannot list claims", err) + } + allSupports = append(allSupports, (*supports).Items...) + totalPages = (*supports).TotalPages + } + return nil +} + +func transferVideos(s *Sync) error { + err := waitConfirmations(s) if err != nil { return err } cleanTransfer := true - for _, video := range channel.syncedVideos { + for _, video := range s.syncedVideos { if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion { log.Debugf("skipping video: %s", video.VideoID) continue } //Todo - Wait for prior sync to see that the publish is confirmed in lbrycrd? - c, err := channel.daemon.ClaimSearch(nil, &video.ClaimID, nil, nil) + c, err := s.daemon.ClaimSearch(nil, &video.ClaimID, nil, nil) if err != nil { return errors.Err(err) } @@ -58,13 +72,13 @@ func TransferChannelAndVideos(channel *Sync) error { streamUpdateOptions := jsonrpc.StreamUpdateOptions{ StreamCreateOptions: &jsonrpc.StreamCreateOptions{ - ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &channel.publishAddress}, + ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.publishAddress}, }, Bid: util.PtrToString("0.009"), // Todo - Dont hardcode } videoStatus := sdk.VideoStatus{ - ChannelID: channel.YoutubeChannelID, + ChannelID: s.YoutubeChannelID, VideoID: video.VideoID, ClaimID: video.ClaimID, ClaimName: video.ClaimName, @@ -72,7 +86,7 @@ func TransferChannelAndVideos(channel *Sync) error { IsTransferred: util.PtrToBool(true), } - result, updateError := channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions) + result, updateError := s.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions) if updateError != nil { cleanTransfer = false videoStatus.FailureReason = updateError.Error() @@ -80,7 +94,7 @@ func TransferChannelAndVideos(channel *Sync) error { videoStatus.IsTransferred = util.PtrToBool(false) } log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked - statusErr := channel.APIConfig.MarkVideoStatus(videoStatus) + statusErr := s.APIConfig.MarkVideoStatus(videoStatus) if statusErr != nil { return errors.Err(statusErr) } @@ -93,21 +107,24 @@ func TransferChannelAndVideos(channel *Sync) error { if !cleanTransfer { return errors.Err("A video has failed to transfer for the channel...skipping channel transfer") } - channelClaim, err := channel.daemon.ClaimSearch(nil, &channel.lbryChannelID, nil, nil) + return nil +} +func transferChannel(s *Sync) error { + channelClaim, err := s.daemon.ClaimSearch(nil, &s.lbryChannelID, nil, nil) if err != nil { return errors.Err(err) } if channelClaim == nil || len(channelClaim.Claims) == 0 { - return errors.Err("There is no channel claim for channel %s", channel.LbryChannelName) + return errors.Err("There is no channel claim for channel %s", s.LbryChannelName) } updateOptions := jsonrpc.ChannelUpdateOptions{ ChannelCreateOptions: jsonrpc.ChannelCreateOptions{ ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ - ClaimAddress: &channel.publishAddress, + ClaimAddress: &s.publishAddress, }, }, } - result, err := channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions) + result, err := s.daemon.ChannelUpdate(s.lbryChannelID, updateOptions) log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked return errors.Err(err) diff --git a/manager/ytsync.go b/manager/ytsync.go index 3c13e5e..980986a 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -316,7 +316,11 @@ func (s *Sync) FullCycle() (e error) { } if s.shouldTransfer() { - return TransferChannelAndVideos(s) + err = transferVideos(s) + if err != nil { + return err + } + return transferChannel(s) } return nil -- 2.45.2 From 4548c410829553b7fc9da8e1bbf79c1006415562 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Mon, 26 Aug 2019 21:02:50 -0400 Subject: [PATCH 15/20] Add chainquery to youtube sync repo --- e2e/chainquery/docker-compose.yml | 28 ++++++++ e2e/chainquery/docker/Dockerfile | 29 ++++++++ e2e/chainquery/docker/build.sh | 8 +++ e2e/chainquery/docker/healthcheck.sh | 2 + e2e/chainquery/docker/my.cnf | 9 +++ e2e/chainquery/docker/start.sh | 51 ++++++++++++++ e2e/chainqueryconfig.toml | 99 ++++++++++++++++++++++++++++ e2e/docker-compose.yml | 33 +++++++++- e2e/lbrynet/docker/Dockerfile | 2 +- 9 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 e2e/chainquery/docker-compose.yml create mode 100644 e2e/chainquery/docker/Dockerfile create mode 100755 e2e/chainquery/docker/build.sh create mode 100755 e2e/chainquery/docker/healthcheck.sh create mode 100644 e2e/chainquery/docker/my.cnf create mode 100755 e2e/chainquery/docker/start.sh create mode 100644 e2e/chainqueryconfig.toml diff --git a/e2e/chainquery/docker-compose.yml b/e2e/chainquery/docker-compose.yml new file mode 100644 index 0000000..452c8ff --- /dev/null +++ b/e2e/chainquery/docker-compose.yml @@ -0,0 +1,28 @@ +version: '3.4' + +services: + ########### + ## MYSQL ## + ########### + mysql: + image: mysql:5.7.23 + restart: "no" + ports: + - 3306:3306 + volumes: + - "../persist/chainquery/db:/var/lib/mysql" + ## This one may need to be tweaked based on where you run this docker-compose from. + - "../stuff/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf" + ################ + ## Chainquery ## + ################ + chainquery: + image: lbry/chainquery:v1.8.1 + restart: "no" + ports: + - 6300:6300 + depends_on: + - mysql + ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations. + volumes: + - "../persist/chainquery/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml" \ No newline at end of file diff --git a/e2e/chainquery/docker/Dockerfile b/e2e/chainquery/docker/Dockerfile new file mode 100644 index 0000000..efeb65f --- /dev/null +++ b/e2e/chainquery/docker/Dockerfile @@ -0,0 +1,29 @@ +## Get the latest source and extract it for the app container. +## Design choices, two RUN layers intended to keep builds faster, the zipped +FROM ubuntu:18.04 as prep +LABEL MAINTAINER="leopere [at] nixc [dot] us" +RUN apt-get update && \ + apt-get -y install unzip curl && \ + apt-get autoclean -y && \ + rm -rf /var/lib/apt/lists/* +WORKDIR / +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +COPY ./start.sh start +COPY ./healthcheck.sh healthcheck +ARG VERSION="master" +RUN curl -s -o /chainquery http://build.lbry.io/chainquery/branch-"${VERSION}"/chainquery && \ + chmod +x /chainquery + + +FROM ubuntu:18.04 as app +ARG VERSION="master" +ADD https://raw.githubusercontent.com/lbryio/chainquery/"${VERSION}"/config/default/chainqueryconfig.toml /etc/lbry/chainqueryconfig.toml.orig +RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery && \ + chown -R chainquery:chainquery /etc/lbry +COPY --from=prep ./healthcheck /chainquery /start /usr/bin/ +HEALTHCHECK --interval=1m --timeout=30s \ + CMD healthcheck +EXPOSE 6300 +USER chainquery +STOPSIGNAL SIGINT +CMD ["start"] \ No newline at end of file diff --git a/e2e/chainquery/docker/build.sh b/e2e/chainquery/docker/build.sh new file mode 100755 index 0000000..61a6dfe --- /dev/null +++ b/e2e/chainquery/docker/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash +if [ $# -eq 0 ] + then + echo "No docker tag argument supplied. Use './build.sh '" + exit 1 +fi +docker build --build-arg VERSION=$1 --tag lbry/chainquery:$1 . +docker push lbry/chainquery:$1 \ No newline at end of file diff --git a/e2e/chainquery/docker/healthcheck.sh b/e2e/chainquery/docker/healthcheck.sh new file mode 100755 index 0000000..6ea1b90 --- /dev/null +++ b/e2e/chainquery/docker/healthcheck.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +curl --fail http://localhost:6300/api/status || exit 1 \ No newline at end of file diff --git a/e2e/chainquery/docker/my.cnf b/e2e/chainquery/docker/my.cnf new file mode 100644 index 0000000..920ac23 --- /dev/null +++ b/e2e/chainquery/docker/my.cnf @@ -0,0 +1,9 @@ +# Default Homebrew MySQL server config +[mysqld] +# Only allow connections from localhost +innodb_log_file_size=5G +key_buffer_size=1G +innodb_flush_log_at_trx_commit = 0 +innodb_autoinc_lock_mode=2 +innodb_buffer_pool_size=1G +innodb_log_buffer_size=1G \ No newline at end of file diff --git a/e2e/chainquery/docker/start.sh b/e2e/chainquery/docker/start.sh new file mode 100755 index 0000000..2bd5d08 --- /dev/null +++ b/e2e/chainquery/docker/start.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +## Config setup + +## Setup Values +DEBUGMODE=$(echo "debugmode=$DEBUGMODE") +LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"") +MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") +APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") + +## Setup Defaults +DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false' +LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"' +MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' +APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' + +## Add setup value variable name to this list to get processed on container start +CONFIG_SETTINGS=( + DEBUGMODE + LBRYCRDURL + MYSQLDSN + APIMYSQLDSN +) + +function set_configs() { + ## Set configs on container start if not already set. + for i in "${!CONFIG_SETTINGS[@]}"; do + ## Indirect references http://tldp.org/LDP/abs/html/ivr.html + eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT" + eval TO_STRING=\$${CONFIG_SETTINGS[$i]} + ## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts. + sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/lbry/chainqueryconfig.toml + done + echo "Reading config for debugging." + cat /etc/lbry/chainqueryconfig.toml +} + +if [[ ! -f /etc/lbry/chainqueryconfig.toml ]]; then + echo "[INFO]: Did not find chainqueryconfig.toml" + echo " Installing default and configuring with provided environment variables if any." + ## Install fresh copy of config file. + echo "cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml" + cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml + ls -lAh /etc/lbry/ + set_configs +else + echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/lbry" +fi + +## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml +chainquery serve --configpath "/etc/lbry/" \ No newline at end of file diff --git a/e2e/chainqueryconfig.toml b/e2e/chainqueryconfig.toml new file mode 100644 index 0000000..300bdb5 --- /dev/null +++ b/e2e/chainqueryconfig.toml @@ -0,0 +1,99 @@ +#Debug mode outputs specific information to the console +#DEFAULT: false +#debugmode= + +#DebugQueryMode outputs SQL Boiler queries to the console. +#DEFAULT: false +#debugquerymode= + +#LBRYcrd URL is required for chainquery to query the blockchain +#DEFAULT: "rpc://lbry:lbry@localhost:9245" +lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245" + +#MySQL DSN is required for chainquery to store information. +#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery" +#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery" +mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery" + +#API MySQL DSN is required for chainquery to expose a SQL query service +#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery" +#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery" +#apimysqldsn= + +#API Host and Port is required for the API Server to bind and listen on. +#DEFAULT: "0.0.0.0:6300" +#apihostport= + +#Profile mode enables and disables the reporting of a profile for chainquery +#DEFAULT: false +#profilemode= + +#Daemon mode tells chainquery how hard it should work catch up processing the blockchain +#deamonmode=0 #BeastMode it continuously process block after block until caughtup. +#daemonmode=1 #SlowAndSteadyMode it will process block with a frequency of 1 block every 100ms +#daemonmode=2 #DelayMode it will process a block with a configured delay frequency (set via 'processingdelay') +#daemonmode=3 #DaemonMode it will process a block every iteration of the daemon. +#DEFAULT: 0 +#deamonmode= + +#Default client timeout is for communication with the api of chainquery +#DEFAULT: 20 #Measured in seconds +#defaultclienttimeout= + +#Processing delay is used to determine how frequently chainquery should process a block +# It is only used if Daemon mode is set to delay mode +#DEFAULT: 100 #Measured in milliseconds +#processingdelay= + +#Daemon delay is the frequency at which chainquery checks for work to do. +#DEFAULT: 1 #Measured in seconds +#daemondelay= + +#Profiling options - will output the time take for certain opertions related to the below category +#DEFAULT: false (for all 3 params) +#daemonprofile= +#lbrycrdprofile= +#mysqlprofile= + +#Slack Hook URL allows slack integration. All logging info level and above is posted to a slack channel. +#DEFAULT: "" +#slackhookurl= + +#Slack Channel is the channel that you want the messages to appear. Works together with the hook url. +#DEFAULT: "" +#slackchannel= + +#Slack Log Level tells chainquery what level of logging will be sent to the slack channel. It will log all levels below +# it as well. Panic=0,Fatal=1,Error=2,Warning=3,Info=4,Debug=5 +#DEFAULT: 0 +#slackloglevel= + +#The command that should be executed to trigger a self update of the software. For linux, for example, `.sh` +#DEFAULT: "" +#autoupdatecommand= + +#Twilio service of chainquery to send specifically important information to key users of the Chainquery install. +#DEFAULT: +##twiliosid="" +##twilioauthtoken="" +##smsrecipients=["",""] +##smsfromphonenumber="" +#twiliosid= +#twilioauthtoken= +#smsrecipients= +#smsfromphonenumber= + +#API Keys - Disallowed by default unless keys are entered. +#DEFAULT: [] +#apikeys= + +#Max Failures - Specifies the number of failures that can happen in processing a transaction. This is for parallel +#transaction processing which puts a transaction to the back of the processing queue if it fails. It can fail say if its +#source output to spend is not already processed. +#DEFAULT: 1000 +#maxfailures= + +#Block Chain Name - Specifies the chain params for parsing blocks, transactions, claims, and addresses. valid choices are +#lbrycrd_main, lbrycrd_testnet, and lbrycrd_regtest. +#DEFAULT: "lbrycrd_main" +blockchainname="lbrycrd_regtest" \ No newline at end of file diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index b25dea7..233c99d 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -100,4 +100,35 @@ services: - MYSQL_USER=lbry - MYSQL_PASS=lbry - MYSQL_DATABASE=lbry - entrypoint: wait-for-it mysql:3306 -- wait-for-it lbrynet:5279 -- ./latest serve \ No newline at end of file + entrypoint: wait-for-it mysql:3306 -- wait-for-it lbrynet:5279 -- ./latest serve + ###################### + ## MySQL Chainquery ## + ###################### + mysqlCQ: + image: mysql/mysql-server:5.7.27 + restart: "no" + ports: + - "15600:3306" + expose: + - "3306" + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - MYSQL_DATABASE=chainquery + - MYSQL_USER=lbry + - MYSQL_PASSWORD=lbry + - MYSQL_LOG_CONSOLE=true + volumes: + - "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf" + ################ + ## Chainquery ## + ################ + chainquery: + image: lbry/chainquery:v1.8.1 + restart: "no" + ports: + - 6300:6300 + depends_on: + - mysql + ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations. + volumes: + - ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml \ No newline at end of file diff --git a/e2e/lbrynet/docker/Dockerfile b/e2e/lbrynet/docker/Dockerfile index 258309f..35c752a 100644 --- a/e2e/lbrynet/docker/Dockerfile +++ b/e2e/lbrynet/docker/Dockerfile @@ -4,7 +4,7 @@ LABEL MAINTAINER="leopere [at] nixc [dot] us" RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it ## Add lbrynet -ARG VERSION="v0.38.6" +ARG VERSION="latest" RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL COPY start.sh /usr/bin/start -- 2.45.2 From e98061f2eb1cf9b94c68e8b6d95e0d8451f9c742 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 27 Aug 2019 01:19:14 -0400 Subject: [PATCH 16/20] Add chainquery to youtube sync repo to be used for e2e testing of support transfer Remove timeout for wait-for-it calls --- e2e/chainquery/docker/Dockerfile | 6 +++++- e2e/chainquery/docker/build.sh | 2 +- e2e/chainqueryconfig.toml | 2 +- e2e/docker-compose.yml | 10 ++++++---- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/e2e/chainquery/docker/Dockerfile b/e2e/chainquery/docker/Dockerfile index efeb65f..73b76dc 100644 --- a/e2e/chainquery/docker/Dockerfile +++ b/e2e/chainquery/docker/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:18.04 as prep LABEL MAINTAINER="leopere [at] nixc [dot] us" RUN apt-get update && \ - apt-get -y install unzip curl && \ + apt-get -y install unzip curl telnet wait-for-it && \ apt-get autoclean -y && \ rm -rf /var/lib/apt/lists/* WORKDIR / @@ -16,6 +16,10 @@ RUN curl -s -o /chainquery http://build.lbry.io/chainquery/branch-"${VERSION}"/c FROM ubuntu:18.04 as app +RUN apt-get update && \ + apt-get -y install telnet wait-for-it && \ + apt-get autoclean -y && \ + rm -rf /var/lib/apt/lists/* ARG VERSION="master" ADD https://raw.githubusercontent.com/lbryio/chainquery/"${VERSION}"/config/default/chainqueryconfig.toml /etc/lbry/chainqueryconfig.toml.orig RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery && \ diff --git a/e2e/chainquery/docker/build.sh b/e2e/chainquery/docker/build.sh index 61a6dfe..0935897 100755 --- a/e2e/chainquery/docker/build.sh +++ b/e2e/chainquery/docker/build.sh @@ -4,5 +4,5 @@ if [ $# -eq 0 ] echo "No docker tag argument supplied. Use './build.sh '" exit 1 fi -docker build --build-arg VERSION=$1 --tag lbry/chainquery:$1 . +docker build --no-cache --build-arg VERSION=$1 --tag lbry/chainquery:$1 . docker push lbry/chainquery:$1 \ No newline at end of file diff --git a/e2e/chainqueryconfig.toml b/e2e/chainqueryconfig.toml index 300bdb5..318a9ef 100644 --- a/e2e/chainqueryconfig.toml +++ b/e2e/chainqueryconfig.toml @@ -18,7 +18,7 @@ mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery" #API MySQL DSN is required for chainquery to expose a SQL query service #DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery" #SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery" -#apimysqldsn= +apimysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery" #API Host and Port is required for the API Server to bind and listen on. #DEFAULT: "0.0.0.0:6300" diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 233c99d..474649b 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -100,7 +100,7 @@ services: - MYSQL_USER=lbry - MYSQL_PASS=lbry - MYSQL_DATABASE=lbry - entrypoint: wait-for-it mysql:3306 -- wait-for-it lbrynet:5279 -- ./latest serve + entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrynet:5279 -- ./latest serve ###################### ## MySQL Chainquery ## ###################### @@ -123,12 +123,14 @@ services: ## Chainquery ## ################ chainquery: - image: lbry/chainquery:v1.8.1 + image: lbry/chainquery:master restart: "no" ports: - 6300:6300 depends_on: - - mysql + - lbrycrd + - mysqlCQ ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations. volumes: - - ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml \ No newline at end of file + - ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml + entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysqlCQ:3306 -- start \ No newline at end of file -- 2.45.2 From 577411cf5065fe862ca7dd3bb0f7fbb7a39a46a3 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 27 Aug 2019 19:49:51 +0200 Subject: [PATCH 17/20] fix e2e add support for supports (heh) --- e2e/e2e.sh | 10 ++++++---- manager/transfer.go | 17 +++++++++++++---- manager/ytsync.go | 8 ++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 060d277..560f856 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -54,11 +54,13 @@ echo "successfully started..." ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1') videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1') -videoClaimID =$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1') +videoClaimID=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1') # Create Supports for published claim -curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","${videoClaimID}",1.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 -curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","${videoClaimID}",2.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 -curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","${videoClaimID}",3.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","'${videoClaimID}'",1.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","'${videoClaimID}'",2.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","'${videoClaimID}'",3.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","'${videoClaimID}'",3.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 # Reset status for tranfer test mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1" # Trigger transfer api diff --git a/manager/transfer.go b/manager/transfer.go index b230511..60a4d1a 100644 --- a/manager/transfer.go +++ b/manager/transfer.go @@ -44,14 +44,23 @@ func abandonSupports(s *Sync) error { allSupports = append(allSupports, (*supports).Items...) totalPages = (*supports).TotalPages } + alreadyAbandoned := make(map[string]bool, len(allSupports)) + for _, support := range allSupports { + _, ok := alreadyAbandoned[support.ClaimID] + if ok { + continue + } + alreadyAbandoned[support.ClaimID] = true + summary, err := s.daemon.SupportAbandon(&support.ClaimID, nil, nil, nil, nil) + if err != nil { + return errors.Err(err) + } + log.Infof("Abandoned support of %s (%s total output) LBC for claim %s", support.Amount, summary.Outputs[0].Amount, support.ClaimID) + } return nil } func transferVideos(s *Sync) error { - err := waitConfirmations(s) - if err != nil { - return err - } cleanTransfer := true for _, video := range s.syncedVideos { if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion { diff --git a/manager/ytsync.go b/manager/ytsync.go index 980986a..8301daf 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -316,6 +316,14 @@ func (s *Sync) FullCycle() (e error) { } if s.shouldTransfer() { + err := waitConfirmations(s) + if err != nil { + return err + } + err = abandonSupports(s) + if err != nil { + return err + } err = transferVideos(s) if err != nil { return err -- 2.45.2 From bacb91e82a87fbc42c13c651dd15e7635f02aa0f Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Wed, 28 Aug 2019 00:10:54 -0400 Subject: [PATCH 18/20] Add supporty to create supports Update lbry.go dep to get lbrycrd commands --- .gitignore | 1 + e2e/docker-compose.yml | 7 +++---- e2e/e2e.sh | 15 +++++++++----- e2e/supporty/Makefile | 12 +++++++++++ e2e/supporty/supporty.go | 43 ++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 8 ++++---- manager/transfer.go | 4 ++-- util/util.go | 6 +++--- 9 files changed, 79 insertions(+), 19 deletions(-) create mode 100644 e2e/supporty/Makefile create mode 100644 e2e/supporty/supporty.go diff --git a/.gitignore b/.gitignore index d4c0eca..a0067ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ bin/ e2e/persist +e2e/supporty/supporty .env blobsfiles \ No newline at end of file diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 474649b..fae02ae 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -21,7 +21,7 @@ services: ## Wallet Server ## ################### walletserver: - image: lbry/wallet-server:v0.39.2 + image: lbry/wallet-server:v0.39.3 restart: always environment: - DB_DIRECTORY=/database @@ -47,7 +47,7 @@ services: ## Lbrynet ## ############# lbrynet: - image: lbry/lbrynet:v0.39.2 + image: lbry/lbrynet:v0.39.3 restart: always ports: - "15100:5279" @@ -93,14 +93,13 @@ services: depends_on: - mysql - lbrycrd - - lbrynet environment: - MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry - LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245 - MYSQL_USER=lbry - MYSQL_PASS=lbry - MYSQL_DATABASE=lbry - entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrynet:5279 -- ./latest serve + entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve ###################### ## MySQL Chainquery ## ###################### diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 560f856..e744f7c 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -4,6 +4,8 @@ set -e #Always compile ytsync make +#Always compile supporty +cd e2e/supporty && make && cd ../.. #OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details export LOCAL_TMP_DIR="/var/tmp:/var/tmp" @@ -55,11 +57,12 @@ echo "successfully started..." status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1') videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1') videoClaimID=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1') +videoClaimAddress=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2') # Create Supports for published claim -curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","'${videoClaimID}'",1.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 -curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","'${videoClaimID}'",2.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 -curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","'${videoClaimID}'",3.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 -curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"supportclaim","params":["@BeamerTest","'${videoClaimID}'",3.0]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 +./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 1.0 +./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 2.0 +./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 3.0 +./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 3.0 curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200 # Reset status for tranfer test mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1" @@ -70,12 +73,14 @@ curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http: # ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1') videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1') -if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" ]]; then +nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0') +if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "0" ]]; then echo "~~!!!~~~FAILED~~~!!!~~" echo "Channel Status: $status" echo "Video Status: $videoStatus" echo "Channel Transfer Status: $channelTransferStatus" echo "Video Transfer Status: $videoTransferStatus" + echo "Nr Unspent Supports: $nrUnspentSupports" #docker-compose logs --tail="all" lbrycrd #docker-compose logs --tail="all" walletserver #docker-compose logs --tail="all" lbrynet diff --git a/e2e/supporty/Makefile b/e2e/supporty/Makefile new file mode 100644 index 0000000..4c54d83 --- /dev/null +++ b/e2e/supporty/Makefile @@ -0,0 +1,12 @@ +BINARY=supporty + +DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) +BIN_DIR = ${DIR} + +.PHONY: build clean test lint +.DEFAULT_GOAL: build + + +build: + mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} supporty.go + chmod +x ${BIN_DIR}/${BINARY} \ No newline at end of file diff --git a/e2e/supporty/supporty.go b/e2e/supporty/supporty.go new file mode 100644 index 0000000..e6e7422 --- /dev/null +++ b/e2e/supporty/supporty.go @@ -0,0 +1,43 @@ +package main + +import ( + "os" + "strconv" + "strings" + + "github.com/lbryio/ytsync/util" + + "github.com/sirupsen/logrus" +) + +func main() { + if len(os.Args) != 6 { + logrus.Info(strings.Join(os.Args, ",")) + logrus.Fatal("Not enough arguments: name, claimID, address, blockchainName, claimAmount") + } + println("Supporty!") + lbrycrd, err := util.GetLbrycrdClient(os.Getenv("LBRYCRD_STRING")) + if err != nil { + logrus.Fatal(err) + } + if lbrycrd == nil { + logrus.Fatal("Lbrycrd Client is nil") + } + amount, err := strconv.ParseFloat(os.Args[5], 64) + if err != nil { + logrus.Error(err) + } + name := os.Args[1] + claimid := os.Args[2] + claimAddress := os.Args[3] + blockChainName := os.Args[4] + logrus.Infof("Supporting %s[%s] with %.2f LBC on chain %s at address %s", name, claimid, amount, blockChainName, claimAddress) + hash, err := lbrycrd.SupportClaim(name, claimid, claimAddress, blockChainName, amount) + if err != nil { + logrus.Error(err) + } + if hash == nil { + logrus.Fatal("Tx not created!") + } + logrus.Info("Tx: ", hash.String()) +} diff --git a/go.mod b/go.mod index 93b8dc6..8bb89c3 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/hashicorp/serf v0.8.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v0.0.0-20190826222547-3a3377d0e5ec + github.com/lbryio/lbry.go v0.0.0-20190828031244-ecbd404da042 github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/go.sum b/go.sum index fa04109..f1332ce 100644 --- a/go.sum +++ b/go.sum @@ -177,10 +177,10 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= -github.com/lbryio/lbry.go v0.0.0-20190826222547-3a3377d0e5ec h1:wusAaT6POFVJwkwNA25I48Nr1DYTcuCAqmmum+WCYP8= -github.com/lbryio/lbry.go v0.0.0-20190826222547-3a3377d0e5ec/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= -github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= +github.com/lbryio/lbry.go v0.0.0-20190828031244-ecbd404da042 h1:L1AZ4DOGEKKKqfufdfqUgOnvBwWHwjH6in9FbQCg2+4= +github.com/lbryio/lbry.go v0.0.0-20190828031244-ecbd404da042/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w= +github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ= +github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 h1:SpUbq2YBg3ncetkw8APUgn8nFF8dscKzzhyiWMM7XCc= diff --git a/manager/transfer.go b/manager/transfer.go index 60a4d1a..203eafa 100644 --- a/manager/transfer.go +++ b/manager/transfer.go @@ -102,7 +102,7 @@ func transferVideos(s *Sync) error { videoStatus.Status = VideoStatusTranferFailed videoStatus.IsTransferred = util.PtrToBool(false) } - log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked + log.Infof("TRANSFERRED %t", len(result.Outputs) != 0) statusErr := s.APIConfig.MarkVideoStatus(videoStatus) if statusErr != nil { return errors.Err(statusErr) @@ -134,7 +134,7 @@ func transferChannel(s *Sync) error { }, } result, err := s.daemon.ChannelUpdate(s.lbryChannelID, updateOptions) - log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked + log.Infof("TRANSFERRED %t", len(result.Outputs) != 0) return errors.Err(err) } diff --git a/util/util.go b/util/util.go index 3b5a081..d44de77 100644 --- a/util/util.go +++ b/util/util.go @@ -173,10 +173,10 @@ func CleanForStartup() error { if err != nil { return errors.Err(err) } - const minBlocksForUTXO = 110 + const minBlocksForUTXO = 200 if height < minBlocksForUTXO { - //Start reg test will some credits - txs, err := lbrycrd.Generate(uint32(110) - uint32(height)) + //Start reg test with some credits + txs, err := lbrycrd.Generate(uint32(minBlocksForUTXO) - uint32(height)) if err != nil { return errors.Err(err) } -- 2.45.2 From 0d8db26e968b93e6afe4f86b88a87293e26125ab Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 28 Aug 2019 15:14:32 +0200 Subject: [PATCH 19/20] use the right errors lib update dependencies --- blobs_reflector/reflect.go | 20 ++++++++------------ go.mod | 19 ++++++------------- go.sum | 38 ++++---------------------------------- thumbs/uploader.go | 2 +- 4 files changed, 19 insertions(+), 60 deletions(-) diff --git a/blobs_reflector/reflect.go b/blobs_reflector/reflect.go index b816beb..3dd5fe9 100644 --- a/blobs_reflector/reflect.go +++ b/blobs_reflector/reflect.go @@ -12,11 +12,12 @@ import ( "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/reflector" "github.com/lbryio/reflector.go/store" - "github.com/lbryio/ytsync/util" - log "github.com/sirupsen/logrus" + "github.com/lbryio/ytsync/util" ) +var dbHandle *db.SQL + func ReflectAndClean() error { err := reflectBlobs() if err != nil { @@ -53,7 +54,6 @@ func reflectBlobs() error { return errors.Prefix("cannot reflect blobs as the daemon is running", err) } - dbHandle := new(db.SQL) ex, err := os.Executable() if err != nil { return errors.Err(err) @@ -63,17 +63,13 @@ func reflectBlobs() error { if err != nil { return errors.Err(err) } - err = dbHandle.Connect(config.DBConn) - if err != nil { - return errors.Err(err) - } - defer func() { - err := dbHandle.CloseDB() + if dbHandle == nil { + dbHandle = new(db.SQL) + err = dbHandle.Connect(config.DBConn) if err != nil { - log.Errorf("failed to close db handle: %s", err.Error()) + return errors.Err(err) } - - }() + } st := store.NewDBBackedS3Store( store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName), dbHandle) diff --git a/go.mod b/go.mod index 8bb89c3..70850a1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/lbryio/ytsync require ( github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 github.com/Microsoft/go-winio v0.4.13 // indirect - github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf github.com/aws/aws-sdk-go v1.17.3 github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect @@ -11,24 +10,18 @@ require ( github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/memberlist v0.1.4 // indirect - github.com/hashicorp/serf v0.8.2 // indirect + github.com/go-ini/ini v1.41.0 // indirect github.com/kr/pretty v0.1.0 // indirect - github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v0.0.0-20190828031244-ecbd404da042 - github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 + github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303 + github.com/lbryio/reflector.go v1.0.6-0.20190828130752-949ea2f2d8f4 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 + github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 github.com/sirupsen/logrus v1.4.1 github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 - github.com/spf13/pflag v1.0.3 // indirect - golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect - golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect - golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect - golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 // indirect - golang.org/x/text v0.3.2 // indirect google.golang.org/api v0.3.2 google.golang.org/grpc v1.20.0 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + gopkg.in/yaml.v2 v2.2.2 // indirect ) diff --git a/go.sum b/go.sum index f1332ce..6ec593c 100644 --- a/go.sum +++ b/go.sum @@ -14,7 +14,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= @@ -27,7 +26,6 @@ github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/btcsuite/btcd v0.0.0-20180531025944-86fed781132a/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= -github.com/btcsuite/btcd v0.0.0-20190109040709-5bda5314ca95/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= @@ -39,9 +37,7 @@ github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+q github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= @@ -87,9 +83,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= @@ -110,22 +104,17 @@ github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36j github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -133,16 +122,13 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs= github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -152,7 +138,6 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ= @@ -165,7 +150,6 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -177,14 +161,14 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= -github.com/lbryio/lbry.go v0.0.0-20190828031244-ecbd404da042 h1:L1AZ4DOGEKKKqfufdfqUgOnvBwWHwjH6in9FbQCg2+4= -github.com/lbryio/lbry.go v0.0.0-20190828031244-ecbd404da042/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w= +github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303 h1:CyDDxUMREhAxPlgP+mgcArgkGJKtdXssj7CXk7o3h84= +github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w= github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ= github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= -github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 h1:SpUbq2YBg3ncetkw8APUgn8nFF8dscKzzhyiWMM7XCc= -github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY= +github.com/lbryio/reflector.go v1.0.6-0.20190828130752-949ea2f2d8f4 h1:3TKpf0WzAjDEcAqa9piXpMCChyBAYOSpC0Mdir9UYlY= +github.com/lbryio/reflector.go v1.0.6-0.20190828130752-949ea2f2d8f4/go.mod h1:IE1yfYwXb37AK6g97s6qOlgj9SW4z9sIiiB03BZT9pY= github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA= github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= @@ -198,7 +182,6 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCW github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -218,7 +201,6 @@ github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rR github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= @@ -272,7 +254,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 h1:UQHWkFUuJBy5rWN1DxosG/efssLu7u0fXXSTC2HHKfQ= github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -293,23 +274,18 @@ golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -319,7 +295,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -335,7 +310,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -348,7 +322,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuA golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190109165630-d30e00c24034/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -362,10 +335,8 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ= @@ -391,4 +362,3 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/thumbs/uploader.go b/thumbs/uploader.go index 4beff61..17a9693 100644 --- a/thumbs/uploader.go +++ b/thumbs/uploader.go @@ -5,7 +5,7 @@ import ( "net/http" "os" - "github.com/lbryio/errors.go" + "github.com/lbryio/lbry.go/extras/errors" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" -- 2.45.2 From 79ae630bc42b16e993424a97a64569a9c2c63b19 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 28 Aug 2019 15:18:09 +0200 Subject: [PATCH 20/20] update reflector lib --- go.mod | 2 +- go.sum | 15 ++------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 70850a1..6e6401c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/go-ini/ini v1.41.0 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303 - github.com/lbryio/reflector.go v1.0.6-0.20190828130752-949ea2f2d8f4 + github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/go.sum b/go.sum index 6ec593c..d2af9d2 100644 --- a/go.sum +++ b/go.sum @@ -25,13 +25,10 @@ github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/btcsuite/btcd v0.0.0-20180531025944-86fed781132a/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20180524032703-d4cc87b86016/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw= @@ -160,16 +157,14 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= -github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303 h1:CyDDxUMREhAxPlgP+mgcArgkGJKtdXssj7CXk7o3h84= github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w= github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ= github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= -github.com/lbryio/reflector.go v1.0.6-0.20190828130752-949ea2f2d8f4 h1:3TKpf0WzAjDEcAqa9piXpMCChyBAYOSpC0Mdir9UYlY= -github.com/lbryio/reflector.go v1.0.6-0.20190828130752-949ea2f2d8f4/go.mod h1:IE1yfYwXb37AK6g97s6qOlgj9SW4z9sIiiB03BZT9pY= -github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= +github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 h1:8k12lI18EEgMHkgGrgtRvJGl2Bq1vDSEbE4s+QaQ5Qc= +github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6/go.mod h1:Q1Cnuv5iLsB2rS4Vr0o0EJr4Gs/EgDXFH2V4WtoM5o8= github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA= github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU= @@ -194,8 +189,6 @@ github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nlopes/slack v0.2.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= -github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0= github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -239,7 +232,6 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/sirupsen/logrus v0.0.0-20180523074243-ea8897e79973/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= @@ -271,7 +263,6 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -341,14 +332,12 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64= -- 2.45.2