change to master tag of internal-apis docker image

.travis.yml

@@ -19,17 +19,23 @@ addons:
   apt:
     update: true
     packages:
-      - youtube-dl
       - ffmpeg
       - tree
+      - python-pip
+
+before_script:
+  - sudo pip install -U youtube-dl
 
 env:
   global:
     - GO111MODULE=on
     #GITHUB_TOKEN
     - secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
+
 script:
-  - ./e2e/e2e.sh
+  #- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko
+  - make
+
 deploy:
   provider: script
   skip_cleanup: true

e2e/data_setup.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -e
+
+#Add a ytsync user
+ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
+#Insert an auth token for the user to be used by ytsync
+ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
+#Give priveledges to ytsync user
+ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
+
+#Add youtuber to sync
+ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
+#Add their youtube channel to be synced
+ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers)
+VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"

e2e/docker-compose.yml

@@ -26,12 +26,14 @@ services:
   ## Wallet Server ##
   ###################
   walletserver:
-    image: lbry/wallet-server:master
+    image: lbry/wallet-server:v0.38.5
     networks:
       - e2e
     restart: always
+    #volumes:
+    # - "./persist/.walletserver:/database"
     environment:
-      - DB_DIRECTORY=/tmp/
+      - DB_DIRECTORY=/database
       - MAX_SEND=1000000000000000000000
       - DAEMON_URL=http://lbry:lbry@lbrycrd:29245
       - MAX_SUBS=1000000000000
@@ -57,7 +59,7 @@ services:
     image: lbry/lbrynet:v0.38.5
     networks:
       - e2e
-    restart: "no"
+    restart: always
     ports:
       - "15100:5279"
       - "15101:5280"
@@ -72,7 +74,12 @@ services:
       - "./persist/.lbrynet:/home/lbrynet"
       - ".:/etc/lbry" #Put your daemon_settings.yml here
       # /private/var/tmp for OSX and /var/tmp for Linux
-      - "${LOCAL_TMP_DIR}:/var/tmp"
+      - "${LOCAL_TMP_DIR}"
+#      - type: bind
+#        source: "${LOCAL_TMP_DIR}"
+#        target: "/tmp"
+#        bind:
+#        propagation: rshared
     #entrypoint: wait-for-it walletserver:50001 -- start
   ###########
   ## MySQL ##
@@ -96,7 +103,7 @@ services:
   ## Internal APIs ##
   ###################
   internalapis:
-    image: lbry/internal-apis:latest
+    image: lbry/internal-apis:master
     networks:
       - e2e
     restart: "no"

e2e/e2e.sh

@@ -6,7 +6,7 @@ set -e
 make
 
 #OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details
-export LOCAL_TMP_DIR="/var/tmp"
+export LOCAL_TMP_DIR="/var/tmp:/var/tmp"
 
 #Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET
 touch -a .env && set -o allexport; source ./.env; set +o allexport
@@ -28,21 +28,17 @@ export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum
 export TMP_DIR="/var/tmp"
 export UID
 
-#ensure that docker can be run and managed from the user
-USER=$(whoami)
-sudo usermod -aG docker "$USER"
-
 cd ./e2e
 docker-compose stop
 docker-compose rm -f
 echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
 docker-compose pull
 if [[ -d persist ]]; then rm -rf persist; fi
-rm -rf persist
-mkdir persist
-mkdir -p blobsfiles
-mkdir persist/.lbrynet
-chmod 777 -R ./persist
+mkdir  -m 0777 -p ./persist
+mkdir -m 777 -p ./persist/.walletserver
+mkdir -m 777 -p ./persist/.lbrynet
+#sudo chown -Rv 999:999 ./persist/.walletserver
+#sudo chown -Rv 1000:1000 ./persist/.lbrynet
 docker-compose up -d
 printf 'waiting for internal apis'
 until curl --output /dev/null --silent --head --fail http://localhost:15400; do
@@ -51,38 +47,19 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do
 done
 echo "successfully started..."
 
-# make sure we have permission to mess with the volumes
-sudo find ./persist -type d -exec chmod 777 {} \;
-sudo chown "$USER": -R ./persist
-
 #Data Setup for test
-#Add a ytsync user
-ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
-mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
-#Insert an auth token for the user to be used by ytsync
-ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
-mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
-#Give priveledges to ytsync user
-ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
-mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
-
-#Add youtuber to sync
-ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
-mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
-#Add their youtube channel to be synced
-ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,google_id,google_token,created_at,source,total_videos,total_subscribers)
-VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued',$GOOGLE_ID,'$GOOGLE_TOKEN','2019-08-01 00:00:00','sync',1,0)"
-mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
-
+./data_setup.sh
 
 # Execute the test!
 ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
 # Assert the status
 status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
 videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
-if [[ $status != "synced" && $videoStatus != "published" ]]; then
+if [[ $status != "synced" || $videoStatus != "published" ]]; then
 docker-compose logs --tail="all" lbrycrd
 docker-compose logs --tail="all" walletserver
 docker-compose logs --tail="all" lbrynet
 docker-compose logs --tail="all" internalapis
+echo "List local /var/tmp"
+find /var/tmp
  exit 1; fi;

manager/manager.go

@@ -112,7 +112,7 @@ func (s *SyncManager) Start() error {
 	for {
 		err := s.checkUsedSpace()
 		if err != nil {
-			return err
+			return errors.Err(err)
 		}
 
 		var syncs []Sync
@@ -122,7 +122,7 @@ func (s *SyncManager) Start() error {
 		if isSingleChannelSync {
 			channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
 			if err != nil {
-				return err
+				return errors.Err(err)
 			}
 			if len(channels) != 1 {
 				return errors.Err("Expected 1 channel, %d returned", len(channels))

manager/ytsync.go

@@ -289,6 +289,10 @@ func (s *Sync) FullCycle() (e error) {
 	if err != nil {
 		return errors.Wrap(err, 0)
 	}
+	err = os.Chmod(s.videoDirectory, 0766)
+	if err != nil {
+		return errors.Err(err)
+	}
 
 	defer deleteSyncFolder(s.videoDirectory)
 	log.Printf("Starting daemon")

sdk/api.go

@@ -71,7 +71,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
 	var response apiJobsResponse
 	err = json.Unmarshal(body, &response)
 	if err != nil {
-		return nil, err
+		return nil, errors.Err(err)
 	}
 	if response.Data == nil {
 		return nil, errors.Err(response.Error)
@@ -152,7 +152,7 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
 	var response apiChannelStatusResponse
 	err := json.Unmarshal(body, &response)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Err(err)
 	}
 	if !response.Error.IsNull() {
 		return nil, nil, errors.Err(response.Error.String)

sources/youtubeVideo.go

@@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
 func (v *YoutubeVideo) download(useIPv6 bool) error {
 	videoPath := v.getFullPath()
 
-	err := os.Mkdir(v.videoDir(), 0750)
+	err := os.Mkdir(v.videoDir(), 0777)
 	if err != nil && !strings.Contains(err.Error(), "file exists") {
 		return errors.Wrap(err, 0)
 	}
@@ -310,6 +310,10 @@ runcmd:
 	if err != nil {
 		return errors.Err(err)
 	}
+	err = os.Chmod(v.getFullPath(), 0777)
+	if err != nil {
+		return errors.Err(err)
+	}
 	videoSize := fi.Size()
 	v.size = &videoSize
 	return nil

thumbs/uploader.go

@@ -25,7 +25,7 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/"
 const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
 
 func (u *thumbnailUploader) downloadThumbnail() error {
-	_ = os.Mkdir(thumbnailPath, 0750)
+	_ = os.Mkdir(thumbnailPath, 0777)
 	img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name)
 	if err != nil {
 		return errors.Err(err)

util/util.go

@@ -166,7 +166,7 @@ func CleanForStartup() error {
 
 	err = CleanupLbrynet()
 	if err != nil {
-		return err
+		return errors.Err(err)
 	}
 
 	lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
@@ -187,7 +187,11 @@ func CleanForStartup() error {
 	}
 
 	defaultWalletDir := GetDefaultWalletPath()
-	return os.Remove(defaultWalletDir)
+	_, err = os.Stat(defaultWalletDir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	return errors.Err(os.Remove(defaultWalletDir))
 }
 
 func CleanupLbrynet() error {
@@ -215,7 +219,7 @@ func CleanupLbrynet() error {
 	if err != nil {
 		return errors.Err(err)
 	}
-	err = os.Mkdir(blobsDir, 0755)
+	err = os.Mkdir(blobsDir, 0777)
 	if err != nil {
 		return errors.Err(err)
 	}