Fixed bugs around sending channel cert and enabling local testing with docker.

This commit is contained in:
Mark Beamer Jr 2019-07-30 23:32:02 -04:00 committed by Niko Storni
parent f26712b022
commit c84e1e6fb9
8 changed files with 211 additions and 49 deletions

4
.gitignore vendored
View file

@ -1,2 +1,4 @@
bin/
e2e/persist
e2e/persist
.env
blobsfiles

View file

@ -2,16 +2,18 @@ package blobs_reflector
import (
"encoding/json"
"io/ioutil"
"os"
"os/user"
"path/filepath"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/reflector.go/cmd"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/ytsync/util"
"github.com/mitchellh/go-ps"
"io/ioutil"
"os"
"os/user"
"path/filepath"
log "github.com/sirupsen/logrus"
)
@ -100,12 +102,7 @@ func cleanupLbrynet() error {
if running {
return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
}
usr, err := user.Current()
if err != nil {
log.Errorln(err.Error())
return errors.Err(err)
}
lbrynetDir := usr.HomeDir + "/.lbrynet/"
lbrynetDir := util.GetLBRYNetDir()
files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
if err != nil {
return errors.Err(err)
@ -116,7 +113,7 @@ func cleanupLbrynet() error {
return errors.Err(err)
}
}
blobsDir := lbrynetDir + "/blobfiles/"
blobsDir := util.GetBlobsDir()
err = os.RemoveAll(blobsDir)
if err != nil {
return errors.Err(err)
@ -129,6 +126,14 @@ func cleanupLbrynet() error {
}
func isLbrynetRunning() (bool, error) {
if util.IsUsingDocker() {
container, err := util.GetLBRYNetContainer(false)
if err != nil {
return false, err
}
return container != nil, nil
}
processes, err := ps.Processes()
if err != nil {
return true, errors.Err(err)

6
go.mod
View file

@ -3,10 +3,15 @@ module github.com/lbryio/ytsync
require (
cloud.google.com/go v0.37.4 // indirect
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
github.com/Microsoft/go-winio v0.4.13 // indirect
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
github.com/aws/aws-sdk-go v1.17.3
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/go-ini/ini v1.42.0 // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect
@ -23,6 +28,7 @@ require (
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/onsi/ginkgo v1.8.0 // indirect
github.com/onsi/gomega v1.5.0 // indirect
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
github.com/prometheus/common v0.3.0
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24

16
go.sum
View file

@ -6,6 +6,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/Microsoft/go-winio v0.4.13 h1:Hmi80lzZuI/CaYmlJp/b+FjZdRZhKu9c2mDVqKlLWVs=
github.com/Microsoft/go-winio v0.4.13/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
@ -56,6 +58,14 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
@ -73,6 +83,7 @@ github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
@ -189,8 +200,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY=
github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8 h1:oNQh5rAVygkVLLRYxLd098/smLHcMoKrn7ozfwFhx2Y=
github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
@ -239,6 +248,8 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
@ -364,6 +375,7 @@ golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

18
main.go
View file

@ -4,7 +4,6 @@ import (
"fmt"
"math/rand"
"os"
"os/user"
"time"
"github.com/lbryio/lbry.go/extras/util"
@ -13,7 +12,7 @@ import (
"github.com/spf13/cobra"
"github.com/lbryio/ytsync/manager"
logUtils "github.com/lbryio/ytsync/util"
ytUtils "github.com/lbryio/ytsync/util"
)
var Version string
@ -113,7 +112,6 @@ func ytSync(cmd *cobra.Command, args []string) {
apiURL := os.Getenv("LBRY_WEB_API")
apiToken := os.Getenv("LBRY_API_TOKEN")
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
blobsDir := os.Getenv("BLOBS_DIRECTORY")
lbrycrdString := os.Getenv("LBRYCRD_STRING")
awsS3ID := os.Getenv("AWS_S3_ID")
awsS3Secret := os.Getenv("AWS_S3_SECRET")
@ -150,14 +148,8 @@ func ytSync(cmd *cobra.Command, args []string) {
if lbrycrdString == "" {
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
}
if blobsDir == "" {
usr, err := user.Current()
if err != nil {
log.Errorln(err.Error())
return
}
blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
}
blobsDir := ytUtils.GetBlobsDir()
syncProperties := &sdk.SyncProperties{
SyncFrom: syncFrom,
@ -198,7 +190,7 @@ func ytSync(cmd *cobra.Command, args []string) {
)
err := sm.Start()
if err != nil {
logUtils.SendErrorToSlack(err.Error())
ytUtils.SendErrorToSlack(err.Error())
}
logUtils.SendInfoToSlack("Syncing process terminated!")
ytUtils.SendInfoToSlack("Syncing process terminated!")
}

View file

@ -1,6 +1,7 @@
package manager
import (
"context"
"fmt"
"io/ioutil"
"net/http"
@ -31,6 +32,8 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/mitchellh/go-ps"
log "github.com/sirupsen/logrus"
"google.golang.org/api/googleapi/transport"
@ -116,17 +119,9 @@ func (s *Sync) IsInterrupted() bool {
}
func (s *Sync) downloadWallet() error {
defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
defaultTempWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
key := aws.String("/wallets/" + s.YoutubeChannelID)
if os.Getenv("REGTEST") == "true" {
defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
defaultTempWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
key = aws.String("/regtest/" + s.YoutubeChannelID)
}
if _, err := os.Stat(defaultWalletDir); !os.IsNotExist(err) {
return errors.Err("default_wallet already exists")
defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
if err != nil {
return errors.Err(err)
}
creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
@ -168,6 +163,29 @@ func (s *Sync) downloadWallet() error {
return os.Rename(defaultTempWalletDir, defaultWalletDir)
}
func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
key = aws.String("/wallets/" + s.YoutubeChannelID)
if os.Getenv("REGTEST") == "true" {
defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
key = aws.String("/regtest/" + s.YoutubeChannelID)
}
walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
if walletPath != "" {
defaultWallet = walletPath + "/wallets/default_wallet"
tempWallet = walletPath + "/wallets/tmp_wallet"
}
if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
return "", "", nil, errors.Err("default_wallet already exists")
}
return
}
func (s *Sync) uploadWallet() error {
defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
key := aws.String("/wallets/" + s.YoutubeChannelID)
@ -272,13 +290,13 @@ func (s *Sync) FullCycle() (e error) {
defer deleteSyncFolder(s.videoDirectory)
log.Printf("Starting daemon")
err = startDaemonViaSystemd()
err = startDaemon()
if err != nil {
return err
}
log.Infoln("Waiting for daemon to finish starting...")
s.daemon = jsonrpc.NewClient("")
s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS"))
s.daemon.SetRPCTimeout(40 * time.Minute)
err = s.waitForDaemonStart()
@ -345,7 +363,7 @@ func (s *Sync) waitForDaemonStart() error {
func (s *Sync) stopAndUploadWallet(e *error) {
log.Printf("Stopping daemon")
shutdownErr := stopDaemonViaSystemd()
shutdownErr := stopDaemon()
if shutdownErr != nil {
logShutdownError(shutdownErr)
} else {
@ -553,6 +571,7 @@ func (s *Sync) doSync() error {
if err != nil {
return errors.Prefix("error getting channel cert", err)
}
println("lbryChannelID:", s.lbryChannelID)
err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
if err != nil {
return errors.Prefix("error setting channel cert", err)
@ -927,6 +946,58 @@ func (s *Sync) processVideo(v video) (err error) {
return nil
}
func startDaemon() error {
if logUtils.IsUsingDocker() {
return startDaemonViaDocker()
}
return startDaemonViaSystemd()
}
func stopDaemon() error {
if logUtils.IsUsingDocker() {
return stopDaemonViaDocker()
}
return stopDaemonViaSystemd()
}
func startDaemonViaDocker() error {
container, err := logUtils.GetLBRYNetContainer(true)
if err != nil {
return err
}
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
if err != nil {
return errors.Err(err)
}
return nil
}
func stopDaemonViaDocker() error {
container, err := logUtils.GetLBRYNetContainer(false)
if err != nil {
return err
}
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
err = cli.ContainerStop(context.Background(), container.ID, nil)
if err != nil {
return errors.Err(err)
}
return nil
}
func startDaemonViaSystemd() error {
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
if err != nil {

View file

@ -11,8 +11,6 @@ import (
"strings"
"time"
"github.com/lbryio/lbry.go/extras/api"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/null"
@ -103,24 +101,30 @@ func sanitizeFailureReason(s *string) {
func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
type apiSetChannelCertResponse struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data string `json:"data"`
}
endpoint := a.ApiURL + "/yt/channel_cert"
res, _ := http.PostForm(endpoint, url.Values{
"channel_id": {channelID},
"channel_cert": {certHex},
"auth_token": {a.ApiToken},
"channel_claim_id": {channelID},
"channel_cert": {certHex},
"auth_token": {a.ApiToken},
})
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response api.Response
var response apiSetChannelCertResponse
err := json.Unmarshal(body, &response)
if err != nil {
return errors.Err(err)
}
if response.Error != nil {
return errors.Err(response.Error)
if !response.Error.IsNull() {
return errors.Err(response.Error.String)
}
return nil

70
util/util.go Normal file
View file

@ -0,0 +1,70 @@
package util
import (
"context"
"fmt"
"os"
"os/user"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/prometheus/common/log"
)
func GetBlobsDir() string {
blobsDir := os.Getenv("BLOBS_DIRECTORY")
if blobsDir == "" {
usr, err := user.Current()
if err != nil {
log.Errorln(err.Error())
return ""
}
blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
}
return blobsDir
}
func GetLBRYNetDir() string {
lbrynetDir := os.Getenv("LBRYNET_DIR")
if lbrynetDir == "" {
usr, err := user.Current()
if err != nil {
log.Errorln(err.Error())
return ""
}
return usr.HomeDir + "/.lbrynet/"
}
return lbrynetDir
}
func GetLBRYNetContainer(all bool) (*types.Container, error) {
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
filters := filters.NewArgs()
filters.Add("name", "lbrynet")
containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: all, Filters: filters})
if err != nil {
panic(err)
}
for _, container := range containers {
fmt.Printf("%s %s\n", container.ID[:10], container.Image)
}
if len(containers) == 0 {
return nil, nil
}
if len(containers) > 1 {
return nil, errors.Err("more than one lbrynet container found")
}
return &containers[0], nil
}
func IsUsingDocker() bool {
return os.Getenv("LBRYNET_USE_DOCKER") == "true"
}