From e47775939510babe18a4815513b30b40949b339b Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sat, 10 Aug 2019 22:50:43 -0400 Subject: [PATCH] change to master tag of internal-apis docker image --- .travis.yml | 10 ++++++++-- e2e/data_setup.sh | 21 ++++++++++++++++++++ e2e/docker-compose.yml | 17 +++++++++++----- e2e/e2e.sh | 43 ++++++++++------------------------------- manager/manager.go | 4 ++-- manager/ytsync.go | 4 ++++ sdk/api.go | 4 ++-- sources/youtubeVideo.go | 6 +++++- thumbs/uploader.go | 2 +- util/util.go | 10 +++++++--- 10 files changed, 72 insertions(+), 49 deletions(-) create mode 100755 e2e/data_setup.sh diff --git a/.travis.yml b/.travis.yml index 38fc146..7af129e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,17 +19,23 @@ addons: apt: update: true packages: - - youtube-dl - ffmpeg - tree + - python-pip + +before_script: + - sudo pip install -U youtube-dl env: global: - GO111MODULE=on #GITHUB_TOKEN - secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI=" + script: - - ./e2e/e2e.sh + #- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko + - make + deploy: provider: script skip_cleanup: true diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh new file mode 100755 index 0000000..2d46529 --- /dev/null +++ b/e2e/data_setup.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +#Add a ytsync user +ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER" +#Insert an auth token for the user to be used by ytsync +ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" +#Give priveledges to ytsync user +ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP" + +#Add youtuber to sync +ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER" +#Add their youtube channel to be synced +ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers) +VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)" +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL" diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 4b28bba..6244f75 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -26,12 +26,14 @@ services: ## Wallet Server ## ################### walletserver: - image: lbry/wallet-server:master + image: lbry/wallet-server:v0.38.5 networks: - e2e restart: always + #volumes: + # - "./persist/.walletserver:/database" environment: - - DB_DIRECTORY=/tmp/ + - DB_DIRECTORY=/database - MAX_SEND=1000000000000000000000 - DAEMON_URL=http://lbry:lbry@lbrycrd:29245 - MAX_SUBS=1000000000000 @@ -57,7 +59,7 @@ services: image: lbry/lbrynet:v0.38.5 networks: - e2e - restart: "no" + restart: always ports: - "15100:5279" - "15101:5280" @@ -72,7 +74,12 @@ services: - "./persist/.lbrynet:/home/lbrynet" - ".:/etc/lbry" #Put your daemon_settings.yml here # /private/var/tmp for OSX and /var/tmp for Linux - - "${LOCAL_TMP_DIR}:/var/tmp" + - "${LOCAL_TMP_DIR}" +# - type: bind +# source: "${LOCAL_TMP_DIR}" +# target: "/tmp" +# bind: +# propagation: rshared #entrypoint: wait-for-it walletserver:50001 -- start ########### ## MySQL ## @@ -96,7 +103,7 @@ services: ## Internal APIs ## ################### internalapis: - image: lbry/internal-apis:latest + image: lbry/internal-apis:master networks: - e2e restart: "no" diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 9cd32cf..027b5d0 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -6,7 +6,7 @@ set -e make #OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details -export LOCAL_TMP_DIR="/var/tmp" +export LOCAL_TMP_DIR="/var/tmp:/var/tmp" #Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET touch -a .env && set -o allexport; source ./.env; set +o allexport @@ -28,21 +28,17 @@ export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum export TMP_DIR="/var/tmp" export UID -#ensure that docker can be run and managed from the user -USER=$(whoami) -sudo usermod -aG docker "$USER" - cd ./e2e docker-compose stop docker-compose rm -f echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin docker-compose pull if [[ -d persist ]]; then rm -rf persist; fi -rm -rf persist -mkdir persist -mkdir -p blobsfiles -mkdir persist/.lbrynet -chmod 777 -R ./persist +mkdir -m 0777 -p ./persist +mkdir -m 777 -p ./persist/.walletserver +mkdir -m 777 -p ./persist/.lbrynet +#sudo chown -Rv 999:999 ./persist/.walletserver +#sudo chown -Rv 1000:1000 ./persist/.lbrynet docker-compose up -d printf 'waiting for internal apis' until curl --output /dev/null --silent --head --fail http://localhost:15400; do @@ -51,38 +47,19 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do done echo "successfully started..." -# make sure we have permission to mess with the volumes -sudo find ./persist -type d -exec chmod 777 {} \; -sudo chown "$USER": -R ./persist - #Data Setup for test -#Add a ytsync user -ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")' -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER" -#Insert an auth token for the user to be used by ytsync -ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")' -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" -#Give priveledges to ytsync user -ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)' -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP" - -#Add youtuber to sync -ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")' -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER" -#Add their youtube channel to be synced -ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,google_id,google_token,created_at,source,total_videos,total_subscribers) -VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued',$GOOGLE_ID,'$GOOGLE_TOKEN','2019-08-01 00:00:00','sync',1,0)" -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL" - +./data_setup.sh # Execute the test! ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container # Assert the status status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1') videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1') -if [[ $status != "synced" && $videoStatus != "published" ]]; then +if [[ $status != "synced" || $videoStatus != "published" ]]; then docker-compose logs --tail="all" lbrycrd docker-compose logs --tail="all" walletserver docker-compose logs --tail="all" lbrynet docker-compose logs --tail="all" internalapis +echo "List local /var/tmp" +find /var/tmp exit 1; fi; \ No newline at end of file diff --git a/manager/manager.go b/manager/manager.go index ac6e016..39572ef 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -112,7 +112,7 @@ func (s *SyncManager) Start() error { for { err := s.checkUsedSpace() if err != nil { - return err + return errors.Err(err) } var syncs []Sync @@ -122,7 +122,7 @@ func (s *SyncManager) Start() error { if isSingleChannelSync { channels, err := s.apiConfig.FetchChannels("", s.syncProperties) if err != nil { - return err + return errors.Err(err) } if len(channels) != 1 { return errors.Err("Expected 1 channel, %d returned", len(channels)) diff --git a/manager/ytsync.go b/manager/ytsync.go index 71d8ca4..4ab4b3b 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -289,6 +289,10 @@ func (s *Sync) FullCycle() (e error) { if err != nil { return errors.Wrap(err, 0) } + err = os.Chmod(s.videoDirectory, 0766) + if err != nil { + return errors.Err(err) + } defer deleteSyncFolder(s.videoDirectory) log.Printf("Starting daemon") diff --git a/sdk/api.go b/sdk/api.go index 9fbdd3d..952200b 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -71,7 +71,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC var response apiJobsResponse err = json.Unmarshal(body, &response) if err != nil { - return nil, err + return nil, errors.Err(err) } if response.Data == nil { return nil, errors.Err(response.Error) @@ -152,7 +152,7 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea var response apiChannelStatusResponse err := json.Unmarshal(body, &response) if err != nil { - return nil, nil, err + return nil, nil, errors.Err(err) } if !response.Error.IsNull() { return nil, nil, errors.Err(response.Error.String) diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index aba0a6e..09551b3 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string { func (v *YoutubeVideo) download(useIPv6 bool) error { videoPath := v.getFullPath() - err := os.Mkdir(v.videoDir(), 0750) + err := os.Mkdir(v.videoDir(), 0777) if err != nil && !strings.Contains(err.Error(), "file exists") { return errors.Wrap(err, 0) } @@ -310,6 +310,10 @@ runcmd: if err != nil { return errors.Err(err) } + err = os.Chmod(v.getFullPath(), 0777) + if err != nil { + return errors.Err(err) + } videoSize := fi.Size() v.size = &videoSize return nil diff --git a/thumbs/uploader.go b/thumbs/uploader.go index 5d68557..4beff61 100644 --- a/thumbs/uploader.go +++ b/thumbs/uploader.go @@ -25,7 +25,7 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/" const ThumbnailEndpoint = "https://thumbnails.lbry.com/" func (u *thumbnailUploader) downloadThumbnail() error { - _ = os.Mkdir(thumbnailPath, 0750) + _ = os.Mkdir(thumbnailPath, 0777) img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name) if err != nil { return errors.Err(err) diff --git a/util/util.go b/util/util.go index 9280623..a546caf 100644 --- a/util/util.go +++ b/util/util.go @@ -166,7 +166,7 @@ func CleanForStartup() error { err = CleanupLbrynet() if err != nil { - return err + return errors.Err(err) } lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING")) @@ -187,7 +187,11 @@ func CleanForStartup() error { } defaultWalletDir := GetDefaultWalletPath() - return os.Remove(defaultWalletDir) + _, err = os.Stat(defaultWalletDir) + if os.IsNotExist(err) { + return nil + } + return errors.Err(os.Remove(defaultWalletDir)) } func CleanupLbrynet() error { @@ -215,7 +219,7 @@ func CleanupLbrynet() error { if err != nil { return errors.Err(err) } - err = os.Mkdir(blobsDir, 0755) + err = os.Mkdir(blobsDir, 0777) if err != nil { return errors.Err(err) }