supply hostname for job listing

fix download race condition
adjusted error handling

cmd/selfsync.go

@@ -70,6 +70,10 @@ type APIYoutubeChannel struct {
 }
 
 func fetchChannels(status string) ([]APIYoutubeChannel, error) {
+	host, err := os.Hostname()
+	if err != nil {
+		return nil, errors.Err("could not detect system hostname")
+	}
 	url := APIURL + "/yt/jobs"
 	res, _ := http.PostForm(url, url2.Values{
 		"auth_token":  {APIToken},
@@ -77,11 +81,12 @@ func fetchChannels(status string) ([]APIYoutubeChannel, error) {
 		"min_videos":  {strconv.Itoa(1)},
 		"after":       {strconv.Itoa(int(syncFrom))},
 		"before":      {strconv.Itoa(int(syncUntil))},
+		"sync_server": {host},
 	})
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	var response APIJobsResponse
-	err := json.Unmarshal(body, &response)
+	err = json.Unmarshal(body, &response)
 	if err != nil {
 		return nil, err
 	}

ytsync/sources/youtubeVideo.go

@@ -82,7 +82,7 @@ func (v YoutubeVideo) getFilename() string {
 	if len(name) < 1 {
 		name = v.id
 	}
-	return v.dir + "/" + name + ".mp4"
+	return v.dir + "/" + v.id + "/" + name + ".mp4"
 }
 
 func (v YoutubeVideo) getAbbrevDescription() string {
@@ -97,7 +97,12 @@ func (v YoutubeVideo) getAbbrevDescription() string {
 func (v YoutubeVideo) download() error {
 	videoPath := v.getFilename()
 
-	_, err := os.Stat(videoPath)
+	err := os.Mkdir(v.dir+"/"+v.id, 0750)
+	if err != nil && !strings.Contains(err.Error(), "file exists") {
+		return errors.Wrap(err, 0)
+	}
+
+	_, err = os.Stat(videoPath)
 	if err != nil && !os.IsNotExist(err) {
 		return err
 	} else if err == nil {
@@ -112,7 +117,7 @@ func (v YoutubeVideo) download() error {
 	}
 
 	var downloadedFile *os.File
-	downloadedFile, err = os.Create(v.getFilename())
+	downloadedFile, err = os.Create(videoPath)
 	if err != nil {
 		return err
 	}

ytsync/ytsync.go

@@ -26,7 +26,7 @@ import (
 	"github.com/mitchellh/go-ps"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/api/googleapi/transport"
-	youtube "google.golang.org/api/youtube/v3"
+	"google.golang.org/api/youtube/v3"
 )
 
 const (
@@ -255,7 +255,6 @@ func (s *Sync) startWorker(workerNum int) {
 				log.Errorln(logMsg)
 				fatalErrors := []string{
 					":5279: read: connection reset by peer",
-					"net/http: request canceled (Client.Timeout exceeded while awaiting headers)",
 					"no space left on device",
 				}
 				if util.InSliceContains(err.Error(), fatalErrors) || s.StopOnError {
@@ -271,6 +270,7 @@ func (s *Sync) startWorker(workerNum int) {
 						"Error in daemon: Cannot publish empty file",
 						"Error extracting sts from embedded url response",
 						"Client.Timeout exceeded while awaiting headers)",
+						"video is bigger than 1GB, skipping for now",
 					}
 					if util.InSliceContains(err.Error(), errorsNoRetry) {
 						log.Println("This error should not be retried at all")