70 changed files with 1433 additions and 7144 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,10 +1 @@
 bin/
 e2e/persist
 e2e/supporty/supporty
 .env
 blobsfiles
 ytsync_docker
 e2e/config.json
 e2e/cookies.txt
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@ -9,8 +9,7 @@ builds:
    - amd64
  ldflags:
    - -X "{{ .Env.IMPORTPATH }}/meta.semVersion={{ .Tag }}" -X "{{ .Env.IMPORTPATH }}/meta.version={{ .Env.VERSIONSHORT }}" -X "{{ .Env.IMPORTPATH }}/meta.versionLong={{ .Env.VERSIONLONG }}" -X "{{ .Env.IMPORTPATH }}/meta.commitMsg={{ .Env.COMMITMSG }}"
-archives:
+archive:
  - id: zip
  name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
  replacements:
    linux: Linux
@ -26,3 +25,4 @@ changelog:
    exclude:
    - '^docs:'
    - '^test:'
--- a/.travis.yml
+++ b/.travis.yml
@ -1,40 +1,15 @@
 os: linux
-dist: bionic
+dist: xenial
 language: go
 go_import_path: github.com/lbryio/ytsync
 go:
-  - 1.17.x
+- 1.11.5
-
+- master
 install: true
 cache:
  directories:
    - $HOME/.cache/go-build
    - $HOME/gopath/pkg/mod
 services:
  - docker
 addons:
  apt:
    update: true
    packages:
      - ffmpeg
      - tree
      - python3-pip
 before_script:
  - sudo pip3 install -U yt-dlp
  - sudo add-apt-repository -y ppa:savoury1/ffmpeg4
 env:
  global:
    - GO111MODULE=on
    #GITHUB_TOKEN
    - secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
 script:
  #- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko
  - make
 deploy:
  provider: script
  skip_cleanup: true
--- a/0001-lbry-patch.patch
+++ b/0001-lbry-patch.patch
@ -1,111 +0,0 @@
 From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
 From: Niko Storni <niko@lbry.io>
 Date: Mon, 16 Dec 2019 00:13:36 +0100
 Subject: [PATCH] lbry-patch
 ---
 youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
 1 file changed, 35 insertions(+), 10 deletions(-)
 diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
 index b913d07a6..cd66a5b01 100644
 --- a/youtube_dl/extractor/youtube.py
 +++ b/youtube_dl/extractor/youtube.py
@@ -10,6 +10,7 @@ import random
 import re
 import time
 import traceback
 +import subprocess
 from .common import InfoExtractor, SearchInfoExtractor
 from ..jsinterp import JSInterpreter
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     _GEO_BYPASS = False
 +    _WGET_429_RATE_LIMIT = 8191
 +    _WGET_BINARY = "wget"
 +
     IE_NAME = 'youtube'
     _TESTS = [
         {
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         """ Return a string representation of a signature """
         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
 +    def _rate_limit_download(self, url, video_id, note=None):
 +        if note is None:
 +            self.report_download_webpage(video_id)
 +        elif note is not False:
 +            if video_id is None:
 +                self.to_screen('%s' % (note,))
 +            else:
 +                self.to_screen('%s: %s' % (video_id, note))
 +        source_address = self._downloader.params.get('source_address')
 +        return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
 +
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
             r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # Get video webpage
         url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
 -        video_webpage = self._download_webpage(url, video_id)
 +        video_webpage = self._rate_limit_download(url, video_id)
         # Attempt to extract SWF player URL
         mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
             })
             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
 -            video_info_webpage = self._download_webpage(
 +            video_info_webpage = self._rate_limit_download(
                 video_info_url, video_id,
 -                note='Refetching age-gated info webpage',
 -                errnote='unable to download video info webpage')
 +                note='Refetching age-gated info webpage')
             video_info = compat_parse_qs(video_info_webpage)
             pl_response = video_info.get('player_response', [None])[0]
             player_response = extract_player_response(pl_response, video_id)
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 # The general idea is to take a union of itags of both DASH manifests (for example
                 # video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
                 self.report_video_info_webpage_download(video_id)
 -                for el in ('embedded', 'detailpage', 'vevo', ''):
 +                for el in ('', 'embedded', 'detailpage', 'vevo'):
                     query = {
                         'video_id': video_id,
                         'ps': 'default',
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         query['el'] = el
                     if sts:
                         query['sts'] = sts
 -                    video_info_webpage = self._download_webpage(
 -                        '%s://www.youtube.com/get_video_info' % proto,
 -                        video_id, note=False,
 -                        errnote='unable to download video info webpage',
 -                        fatal=False, query=query)
 +
 +                    if el == '':
 +                        base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
 +                    else:
 +                        base_url = 'https://youtube.com/get_video_info'
 +
 +                        for q in query:
 +                            if q is None or q is "":
 +                                continue
 +                            if query[q] is None or query[q] is "":
 +                                continue
 +
 +                            base_url = base_url + "?{}={}".format(q, query[q])
 +
 +                    video_info_webpage = self._rate_limit_download(base_url, video_id)
 +
                     if not video_info_webpage:
                         continue
                     get_video_info = compat_parse_qs(video_info_webpage)
 -- 
 2.17.1
--- a/15
+++ b/15
@ -1,15 +0,0 @@
 The MIT License (MIT)
 Copyright (c) 2017-2020 LBRY Inc.
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
 "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
 distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
 following conditions:
 The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--- a/2
+++ b/2
@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/
 build:
-	mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
+	mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
 clean:
 	if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
--- a/README.md
+++ b/README.md
@ -8,17 +8,23 @@ With the support of said database, the tool is also able to keep all the channel
 # Requirements
- lbrynet SDK https://github.com/lbryio/lbry-sdk/releases (We strive to keep the latest release of ytsync compatible with the latest major release of the SDK)
+- lbrynet SDK https://github.com/lbryio/lbry/releases (We strive to keep the latest release of ytsync compatible with the latest major release of the SDK)
 - a lbrycrd node running (localhost or on a remote machine) with credits in it
 - internal-apis (you cannot run this one yourself)
 - python3-pip
 - yt-dlp (`pip3 install -U yt-dlp`)
 - ffmpeg (latest)
 # Setup
 - make sure daemon is stopped and can be controlled through `systemctl` (find example below)
 - extract the ytsync binary anywhere
- create and fill `config.json` using [this example](config.json.example)
+- add the environment variables necessary to the tool
  - export SLACK_TOKEN="a-token-to-spam-your-slack"
  - export SLACK_CHANNEL="youtube-status"
  - export YOUTUBE_API_KEY="youtube-api-key"
  - export LBRY_WEB_API="https://lbry-api-url-here"
  - export LBRY_API_TOKEN="internal-apis-token-for-ytsync-user"
  - export LBRYCRD_STRING="tcp://user:password@host:5429"
  - export AWS_S3_ID="THE-ID-LIES-HERE"
  - export AWS_S3_SECRET="THE-SECRET-LIES-HERE"
  - export AWS_S3_REGION="us-east-1"
  - export AWS_S3_BUCKET="ytsync-wallets"
 ## systemd script example
 `/etc/systemd/system/lbrynet.service`
@ -49,26 +55,23 @@ Usage:
 Flags:
      --after int                   Specify from when to pull jobs [Unix time](Default: 0)
-      --before int                  Specify until when to pull jobs [Unix time](Default: current Unix time) (default 1669311891)
+      --before int                  Specify until when to pull jobs [Unix time](Default: current Unix time) (default current timestamp)
      --channelID string            If specified, only this channel will be synced.
      --concurrent-jobs int         how many jobs to process concurrently (default 1)
  -h, --help                        help for ytsync
      --limit int                   limit the amount of channels to sync
-      --max-length int              Maximum video length to process (in hours) (default 2)
+      --max-length float            Maximum video length to process (in hours) (default 2)
      --max-size int                Maximum video size to process (in MB) (default 2048)
      --max-tries int               Number of times to try a publish that fails (default 3)
      --no-transfers                Skips the transferring process of videos, channels and supports
      --quick                       Look up only the last 50 videos from youtube
      --remove-db-unpublished       Remove videos from the database that are marked as published but aren't really published
      --run-once                    Whether the process should be stopped after one cycle or not
      --skip-space-check            Do not perform free space check on startup
      --status string               Specify which queue to pull from. Overrides --update
-      --status2 string              Specify which secondary queue to pull from.
+      --stop-on-error               If a publish fails, stop all publishing and exit
      --takeover-existing-channel   If channel exists and we don't own it, take over the channel
      --update                      Update previously synced channels instead of syncing new ones
      --upgrade-metadata            Upgrade videos if they're on the old metadata version
-      --videos-limit int            how many videos to process per channel (leave 0 for automatic detection)
+      --videos-limit int            how many videos to process per channel (default 1000)
 ```
 ## Running from Source
@ -85,17 +88,17 @@ Contributions to this project are welcome, encouraged, and compensated. For more
 ## Security
-We take security seriously. Please contact [security@lbry.io](mailto:security@lbry.io) regarding any security issues. Our PGP key is [here](https://lbry.com/faq/pgp-key) if you need it.
+We take security seriously. Please contact [security@lbry.io](mailto:security@lbry.io) regarding any security issues. Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
 ## Contact
-The primary contact for this project is [Niko Storni](https://github.com/nikooo777) (niko@lbry.com).
+The primary contact for this project is [Niko Storni](https://github.com/nikooo777) (niko@lbry.io).
 ## Additional Info and Links
- [https://lbry.com](https://lbry.com) - The live LBRY website
+- [https://lbry.io](https://lbry.io) - The live LBRY website
- [Discord Chat](https://chat.lbry.com) - A chat room for the LBRYians
+- [Discord Chat](https://chat.lbry.io) - A chat room for the LBRYians
- [Email us](mailto:hello@lbry.com) - LBRY Support email
+- [Email us](mailto:hello@lbry.io) - LBRY Support email
 - [Twitter](https://twitter.com/@lbryio) - LBRY Twitter page
 - [Facebook](https://www.facebook.com/lbryio/) - LBRY Facebook page
 - [Reddit](https://reddit.com/r/lbry) - LBRY Reddit page
--- a/blobs_reflector/reflect.go
+++ b/blobs_reflector/reflect.go
@ -1,93 +0,0 @@
 package blobs_reflector
 import (
 	"encoding/json"
 	"io/ioutil"
 	"os"
 	"os/user"
 	"path/filepath"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/reflector.go/cmd"
 	"github.com/lbryio/reflector.go/db"
 	"github.com/lbryio/reflector.go/reflector"
 	"github.com/lbryio/reflector.go/store"
 	"github.com/sirupsen/logrus"
 	"github.com/lbryio/ytsync/v5/util"
 )
 var dbHandle *db.SQL
 func ReflectAndClean() error {
 	err := reflectBlobs()
 	if err != nil {
 		return err
 	}
 	return util.CleanupLbrynet()
 }
 func loadConfig(path string) (cmd.Config, error) {
 	var c cmd.Config
 	raw, err := ioutil.ReadFile(path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return c, errors.Err("config file not found")
 		}
 		return c, err
 	}
 	err = json.Unmarshal(raw, &c)
 	return c, err
 }
 func reflectBlobs() error {
 	if util.IsBlobReflectionOff() {
 		return nil
 	}
 	logrus.Infoln("reflecting blobs...")
 	//make sure lbrynet is off
 	running, err := util.IsLbrynetRunning()
 	if err != nil {
 		return err
 	}
 	if running {
 		return errors.Prefix("cannot reflect blobs as the daemon is running", err)
 	}
 	logrus.SetLevel(logrus.InfoLevel)
 	defer logrus.SetLevel(logrus.DebugLevel)
 	ex, err := os.Executable()
 	if err != nil {
 		return errors.Err(err)
 	}
 	exPath := filepath.Dir(ex)
 	config, err := loadConfig(exPath + "/prism_config.json")
 	if err != nil {
 		return errors.Err(err)
 	}
 	if dbHandle == nil {
 		dbHandle = new(db.SQL)
 		err = dbHandle.Connect(config.DBConn)
 		if err != nil {
 			return errors.Err(err)
 		}
 	}
 	st := store.NewDBBackedStore(store.NewS3Store(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName), dbHandle, false)
 	uploadWorkers := 10
 	uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false, false)
 	usr, err := user.Current()
 	if err != nil {
 		return errors.Err(err)
 	}
 	blobsDir := usr.HomeDir + "/.lbrynet/blobfiles/"
 	err = uploader.Upload(blobsDir)
 	if err != nil {
 		return errors.Err(err)
 	}
 	if uploader.GetSummary().Err > 0 {
 		return errors.Err("not al blobs were reflected. Errors: %d", uploader.GetSummary().Err)
 	}
 	return nil
 }
--- a/config.json.example
+++ b/config.json.example
@ -1,35 +0,0 @@
 {
  "slack_token": "",
  "slack_channel": "ytsync-dev",
  "internal_apis_endpoint": "http://localhost:15400",
  "internal_apis_auth_token": "ytsyntoken",
  "lbrycrd_string": "tcp://lbry:lbry@localhost:15200",
  "wallet_s3_config": {
    "id": "",
    "secret": "",
    "region": "us-east-1",
    "bucket": "ytsync-wallets",
    "endpoint": ""
  },
  "blockchaindb_s3_config": {
    "id": "",
    "secret": "",
    "region": "us-east-1",
    "bucket": "blockchaindbs",
    "endpoint": ""
  },
  "thumbnails_s3_config": {
    "id": "",
    "secret": "",
    "region": "us-east-1",
    "bucket": "thumbnails.lbry.com",
    "endpoint": ""
  },
  "aws_thumbnails_s3_config": {
    "id": "",
    "secret": "",
    "region": "us-east-1",
    "bucket": "thumbnails.lbry.com",
    "endpoint": ""
  }
 }
--- a/configs/configs.go
+++ b/configs/configs.go
@ -1,75 +0,0 @@
 package configs
 import (
 	"os"
 	"regexp"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	log "github.com/sirupsen/logrus"
 	"github.com/tkanos/gonfig"
 )
 type S3Configs struct {
 	ID       string `json:"id"`
 	Secret   string `json:"secret"`
 	Region   string `json:"region"`
 	Bucket   string `json:"bucket"`
 	Endpoint string `json:"endpoint"`
 }
 type Configs struct {
 	SlackToken            string    `json:"slack_token"`
 	SlackChannel          string    `json:"slack_channel"`
 	InternalApisEndpoint  string    `json:"internal_apis_endpoint"`
 	InternalApisAuthToken string    `json:"internal_apis_auth_token"`
 	LbrycrdString         string    `json:"lbrycrd_string"`
 	WalletS3Config        S3Configs `json:"wallet_s3_config"`
 	BlockchaindbS3Config  S3Configs `json:"blockchaindb_s3_config"`
 	AWSThumbnailsS3Config S3Configs `json:"aws_thumbnails_s3_config"`
 	ThumbnailsS3Config    S3Configs `json:"thumbnails_s3_config"`
 }
 var Configuration *Configs
 func Init(configPath string) error {
 	if Configuration != nil {
 		return nil
 	}
 	c := Configs{}
 	err := gonfig.GetConf(configPath, &c)
 	if err != nil {
 		return errors.Err(err)
 	}
 	Configuration = &c
 	return nil
 }
 func (s *S3Configs) GetS3AWSConfig() *aws.Config {
 	return &aws.Config{
 		Credentials:      credentials.NewStaticCredentials(s.ID, s.Secret, ""),
 		Region:           &s.Region,
 		Endpoint:         &s.Endpoint,
 		S3ForcePathStyle: aws.Bool(true),
 	}
 }
 func (c *Configs) GetHostname() string {
 	var hostname string
 	var err error
 	hostname, err = os.Hostname()
 	if err != nil {
 		log.Error("could not detect system hostname")
 		hostname = "ytsync_unknown"
 	}
 	reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
 	if err == nil {
 		hostname = reg.ReplaceAllString(hostname, "_")
 	}
 	if len(hostname) > 30 {
 		hostname = hostname[0:30]
 	}
 	return hostname
 }
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@ -1,315 +0,0 @@
 package downloader
 import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"math"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"os/exec"
 	"path"
 	"strings"
 	"time"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/shared"
 	util2 "github.com/lbryio/ytsync/v5/util"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/sirupsen/logrus"
 )
 func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
 	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt", "--playlist-end", fmt.Sprintf("%d", maxVideos)}
 	ids, err := run(channelName, args, stopChan, pool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	videoIDs := make([]string, 0, maxVideos)
 	for i, v := range ids {
 		if v == "" {
 			continue
 		}
 		if i >= maxVideos {
 			break
 		}
 		videoIDs = append(videoIDs, v)
 	}
 	return videoIDs, nil
 }
 const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
 func GetVideoInformation(videoID string, stopChan stop.Chan, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
 	args := []string{
 		"--skip-download",
 		"--write-info-json",
 		fmt.Sprintf("https://www.youtube.com/watch?v=%s", videoID),
 		"--cookies",
 		"cookies.txt",
 		"-o",
 		path.Join(util2.GetVideoMetadataDir(), videoID),
 	}
 	_, err := run(videoID, args, stopChan, pool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	f, err := os.Open(path.Join(util2.GetVideoMetadataDir(), videoID+".info.json"))
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	// defer the closing of our jsonFile so that we can parse it later on
 	defer f.Close()
 	// read our opened jsonFile as a byte array.
 	byteValue, _ := ioutil.ReadAll(f)
 	var video *ytdl.YtdlVideo
 	err = json.Unmarshal(byteValue, &video)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	return video, nil
 }
 var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
 var errUploadTimeEmpty = errors.Base("upload time is empty")
 var errStatusParse = errors.Base("could not parse status, got number, need string")
 var errConnectionIssue = errors.Base("there was a connection issue with the api")
 func slack(format string, a ...interface{}) {
 	fmt.Printf(format+"\n", a...)
 	util.SendToSlack(format, a...)
 }
 func triggerScrape(videoID string, ip *net.TCPAddr) error {
 	//slack("Triggering scrape for %s", videoID)
 	u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
 	q := u.Query()
 	q.Set("twtimeline", "0")
 	q.Set("url", "https://www.youtube.com/watch?v="+videoID)
 	u.RawQuery = q.Encode()
 	//slack("GET %s", u.String())
 	client := getClient(ip)
 	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
 	if err != nil {
 		return errors.Err(err)
 	}
 	req.Header.Set("User-Agent", ChromeUA)
 	res, err := client.Do(req)
 	if err != nil {
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
 	var response struct {
 		Message  string `json:"message"`
 		Status   string `json:"status"`
 		VideoURL string `json:"video_url"`
 	}
 	err = json.NewDecoder(res.Body).Decode(&response)
 	if err != nil {
 		if strings.Contains(err.Error(), "cannot unmarshal number") {
 			return errors.Err(errStatusParse)
 		}
 		if strings.Contains(err.Error(), "no route to host") {
 			return errors.Err(errConnectionIssue)
 		}
 		return errors.Err(err)
 	}
 	switch response.Status {
 	case "removed_video":
 		return errors.Err("video previously removed from service")
 	case "no_video":
 		return errors.Err("they say 'video cannot be found'. wtf?")
 	default:
 		spew.Dump(response)
 	}
 	return nil
 	//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
 }
 func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploadDate string) (string, error) {
 	//slack("Getting upload time for %s", videoID)
 	release, err := config.GetReleasedDate(videoID)
 	if err != nil {
 		logrus.Error(err)
 	}
 	ytdlUploadDate, err := time.Parse("20060102", uploadDate)
 	if err != nil {
 		logrus.Error(err)
 	}
 	if release != nil {
 		//const sqlTimeFormat = "2006-01-02 15:04:05"
 		sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
 		if err == nil {
 			hoursDiff := math.Abs(sqlTime.Sub(ytdlUploadDate).Hours())
 			if hoursDiff > 48 {
 				logrus.Infof("upload day from APIs differs from the ytdl one by more than 2 days.")
 			} else {
 				return sqlTime.Format(releaseTimeFormat), nil
 			}
 		} else {
 			logrus.Error(err)
 		}
 	}
 	return ytdlUploadDate.Format(releaseTimeFormat), nil
 }
 func getClient(ip *net.TCPAddr) *http.Client {
 	if ip == nil {
 		return http.DefaultClient
 	}
 	return &http.Client{
 		Transport: &http.Transport{
 			Proxy: http.ProxyFromEnvironment,
 			DialContext: (&net.Dialer{
 				LocalAddr: ip,
 				Timeout:   30 * time.Second,
 				KeepAlive: 30 * time.Second,
 			}).DialContext,
 			MaxIdleConns:          100,
 			IdleConnTimeout:       90 * time.Second,
 			TLSHandshakeTimeout:   10 * time.Second,
 			ExpectContinueTimeout: 1 * time.Second,
 		},
 	}
 }
 const (
 	GoogleBotUA             = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
 	ChromeUA                = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
 	maxAttempts             = 3
 	extractionError         = "YouTube said: Unable to extract video data"
 	throttledError          = "HTTP Error 429"
 	AlternateThrottledError = "returned non-zero exit status 8"
 	youtubeDlError          = "exit status 1"
 	videoPremiereError      = "Premieres in"
 	liveEventError          = "This live event will begin in"
 )
 func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
 	var useragent []string
 	var lastError error
 	for attempts := 0; attempts < maxAttempts; attempts++ {
 		sourceAddress, err := getIPFromPool(use, stopChan, pool)
 		if err != nil {
 			return nil, err
 		}
 		argsForCommand := append(args, "--source-address", sourceAddress)
 		argsForCommand = append(argsForCommand, useragent...)
 		binary := "yt-dlp"
 		cmd := exec.Command(binary, argsForCommand...)
 		res, err := runCmd(cmd, stopChan)
 		pool.ReleaseIP(sourceAddress)
 		if err == nil {
 			return res, nil
 		}
 		lastError = err
 		if strings.Contains(err.Error(), youtubeDlError) {
 			if util.SubstringInSlice(err.Error(), shared.ErrorsNoRetry) {
 				break
 			}
 			if strings.Contains(err.Error(), extractionError) {
 				logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
 				useragent = nextUA(useragent)
 			}
 			if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
 				pool.SetThrottled(sourceAddress)
 				//we don't want throttle errors to count toward the max retries
 				attempts--
 			}
 		}
 	}
 	return nil, lastError
 }
 func nextUA(current []string) []string {
 	if len(current) == 0 {
 		return []string{"--user-agent", GoogleBotUA}
 	}
 	return []string{"--user-agent", ChromeUA}
 }
 func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
 	logrus.Infof("running yt-dlp cmd: %s", strings.Join(cmd.Args, " "))
 	var err error
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	err = cmd.Start()
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	outLog, err := ioutil.ReadAll(stdout)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	errorLog, err := ioutil.ReadAll(stderr)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	done := make(chan error, 1)
 	go func() {
 		done <- cmd.Wait()
 	}()
 	select {
 	case <-stopChan:
 		err := cmd.Process.Kill()
 		if err != nil {
 			return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
 		}
 		return nil, errors.Err("interrupted by user")
 	case err := <-done:
 		if err != nil {
 			//return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
 			return nil, errors.Prefix(string(errorLog), err)
 		}
 		return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
 	}
 }
 func getIPFromPool(use string, stopChan stop.Chan, pool *ip_manager.IPPool) (sourceAddress string, err error) {
 	for {
 		sourceAddress, err = pool.GetIP(use)
 		if err != nil {
 			if errors.Is(err, ip_manager.ErrAllThrottled) {
 				select {
 				case <-stopChan:
 					return "", errors.Err("interrupted by user")
 				default:
 					time.Sleep(ip_manager.IPCooldownPeriod)
 					continue
 				}
 			} else {
 				return "", err
 			}
 		}
 		break
 	}
 	return
 }
--- a/downloader/downloader_test.go
+++ b/downloader/downloader_test.go
@ -1,40 +0,0 @@
 package downloader
 import (
 	"testing"
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
 func TestGetPlaylistVideoIDs(t *testing.T) {
 	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50, nil, nil)
 	if err != nil {
 		logrus.Error(err)
 	}
 	for _, id := range videoIDs {
 		println(id)
 	}
 }
 func TestGetVideoInformation(t *testing.T) {
 	s := stop.New()
 	ip, err := ip_manager.GetIPPool(s)
 	assert.NoError(t, err)
 	video, err := GetVideoInformation("kDGOHNpRjzc", s.Ch(), ip)
 	assert.NoError(t, err)
 	assert.NotNil(t, video)
 	logrus.Info(video.ID)
 }
 func Test_getUploadTime(t *testing.T) {
 	configs := sdk.APIConfig{}
 	got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
 	assert.NoError(t, err)
 	t.Log(got)
 }
--- a/downloader/ytdl/Video.go
+++ b/downloader/ytdl/Video.go
@ -1,137 +0,0 @@
 package ytdl
 import (
 	"time"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/sirupsen/logrus"
 )
 type YtdlVideo struct {
 	ID                string      `json:"id"`
 	Title             string      `json:"title"`
 	Thumbnails        []Thumbnail `json:"thumbnails"`
 	Description       string      `json:"description"`
 	ChannelID         string      `json:"channel_id"`
 	Duration          int         `json:"duration"`
 	Categories        []string    `json:"categories"`
 	Tags              []string    `json:"tags"`
 	IsLive            bool        `json:"is_live"`
 	LiveStatus        string      `json:"live_status"`
 	ReleaseTimestamp  *int64      `json:"release_timestamp"`
 	uploadDateForReal *time.Time
 	Availability      string `json:"availability"`
 	ReleaseDate       string `json:"release_date"`
 	UploadDate        string `json:"upload_date"`
 	//WasLive           bool        `json:"was_live"`
 	//Formats              interface{}   `json:"formats"`
 	//Thumbnail            string        `json:"thumbnail"`
 	//Uploader             string        `json:"uploader"`
 	//UploaderID           string        `json:"uploader_id"`
 	//UploaderURL          string        `json:"uploader_url"`
 	//ChannelURL           string        `json:"channel_url"`
 	//ViewCount            int           `json:"view_count"`
 	//AverageRating        interface{}   `json:"average_rating"`
 	//AgeLimit             int           `json:"age_limit"`
 	//WebpageURL           string        `json:"webpage_url"`
 	//PlayableInEmbed      bool          `json:"playable_in_embed"`
 	//AutomaticCaptions    interface{}   `json:"automatic_captions"`
 	//Subtitles            interface{}   `json:"subtitles"`
 	//Chapters             interface{}   `json:"chapters"`
 	//LikeCount            int           `json:"like_count"`
 	//Channel              string        `json:"channel"`
 	//ChannelFollowerCount int           `json:"channel_follower_count"`
 	//OriginalURL          string        `json:"original_url"`
 	//WebpageURLBasename   string        `json:"webpage_url_basename"`
 	//WebpageURLDomain     string        `json:"webpage_url_domain"`
 	//Extractor            string        `json:"extractor"`
 	//ExtractorKey         string        `json:"extractor_key"`
 	//Playlist             interface{}   `json:"playlist"`
 	//PlaylistIndex        interface{}   `json:"playlist_index"`
 	//DisplayID            string        `json:"display_id"`
 	//Fulltitle            string        `json:"fulltitle"`
 	//DurationString       string        `json:"duration_string"`
 	//RequestedSubtitles   interface{}   `json:"requested_subtitles"`
 	//HasDrm               bool          `json:"__has_drm"`
 	//RequestedFormats     interface{}   `json:"requested_formats"`
 	//Format               string        `json:"format"`
 	//FormatID             string        `json:"format_id"`
 	//Ext                  string        `json:"ext"`
 	//Protocol             string        `json:"protocol"`
 	//Language             interface{}   `json:"language"`
 	//FormatNote           string        `json:"format_note"`
 	//FilesizeApprox       int           `json:"filesize_approx"`
 	//Tbr                  float64       `json:"tbr"`
 	//Width                int           `json:"width"`
 	//Height               int           `json:"height"`
 	//Resolution           string        `json:"resolution"`
 	//Fps                  int           `json:"fps"`
 	//DynamicRange         string        `json:"dynamic_range"`
 	//Vcodec               string        `json:"vcodec"`
 	//Vbr                  float64       `json:"vbr"`
 	//StretchedRatio       interface{}   `json:"stretched_ratio"`
 	//Acodec               string        `json:"acodec"`
 	//Abr                  float64       `json:"abr"`
 	//Asr                  int           `json:"asr"`
 	//Epoch                int           `json:"epoch"`
 	//Filename             string        `json:"filename"`
 	//Urls                 string        `json:"urls"`
 	//Type                 string        `json:"_type"`
 }
 type Thumbnail struct {
 	URL        string `json:"url"`
 	Preference int    `json:"preference"`
 	ID         string `json:"id"`
 	Height     int    `json:"height,omitempty"`
 	Width      int    `json:"width,omitempty"`
 	Resolution string `json:"resolution,omitempty"`
 }
 func (v *YtdlVideo) GetUploadTime() time.Time {
 	//priority list:
 	// release timestamp from yt
 	// release timestamp from morty
 	// release date from yt
 	// upload date from yt
 	if v.uploadDateForReal != nil {
 		return *v.uploadDateForReal
 	}
 	var ytdlReleaseTimestamp time.Time
 	if v.ReleaseTimestamp != nil && *v.ReleaseTimestamp > 0 {
 		ytdlReleaseTimestamp = time.Unix(*v.ReleaseTimestamp, 0).UTC()
 	}
 	//get morty timestamp
 	var mortyReleaseTimestamp time.Time
 	mortyRelease, err := sdk.GetAPIsConfigs().GetReleasedDate(v.ID)
 	if err != nil {
 		logrus.Error(err)
 	} else if mortyRelease != nil {
 		mortyReleaseTimestamp, err = time.ParseInLocation(time.RFC3339, mortyRelease.ReleaseTime, time.UTC)
 		if err != nil {
 			logrus.Error(err)
 		}
 	}
 	ytdlReleaseDate, err := time.Parse("20060102", v.ReleaseDate)
 	if err != nil {
 		logrus.Error(err)
 	}
 	ytdlUploadDate, err := time.Parse("20060102", v.UploadDate)
 	if err != nil {
 		logrus.Error(err)
 	}
 	if !ytdlReleaseTimestamp.IsZero() {
 		v.uploadDateForReal = &ytdlReleaseTimestamp
 	} else if !mortyReleaseTimestamp.IsZero() {
 		v.uploadDateForReal = &mortyReleaseTimestamp
 	} else if !ytdlReleaseDate.IsZero() {
 		v.uploadDateForReal = &ytdlReleaseDate
 	} else {
 		v.uploadDateForReal = &ytdlUploadDate
 	}
 	return *v.uploadDateForReal
 }
--- a/e2e/chainquery/docker-compose.yml
+++ b/e2e/chainquery/docker-compose.yml
@ -1,28 +0,0 @@
 version: '3.4'
 services:
  ###########
  ## MYSQL ##
  ###########
  mysql:
    image: mysql:5.7.23
    restart: "no"
    ports:
      - 3306:3306
    volumes:
      - "../persist/chainquery/db:/var/lib/mysql"
      ## This one may need to be tweaked based on where you run this docker-compose from.
      - "../stuff/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
  ################
  ## Chainquery ##
  ################
  chainquery:
    image: lbry/chainquery:v1.8.1
    restart: "no"
    ports:
      - 6300:6300
    depends_on:
      - mysql
    ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
    volumes:
      - "../persist/chainquery/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml"
--- a/e2e/chainquery/docker/Dockerfile
+++ b/e2e/chainquery/docker/Dockerfile
@ -1,33 +0,0 @@
 ## Get the latest source and extract it for the app container.
 ## Design choices, two RUN layers intended to keep builds faster, the zipped
 FROM ubuntu:18.04 as prep
 LABEL MAINTAINER="leopere [at] nixc [dot] us"
 RUN apt-get update && \
  apt-get -y install unzip curl telnet wait-for-it && \
  apt-get autoclean -y && \
  rm -rf /var/lib/apt/lists/*
 WORKDIR /
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 COPY ./start.sh start
 COPY ./healthcheck.sh healthcheck
 ARG VERSION="master"
 RUN curl -s -o /chainquery http://build.lbry.io/chainquery/branch-"${VERSION}"/chainquery && \
    chmod +x /chainquery
 FROM ubuntu:18.04 as app
 RUN apt-get update && \
  apt-get -y install telnet wait-for-it && \
  apt-get autoclean -y && \
  rm -rf /var/lib/apt/lists/*
 ARG VERSION="master"
 ADD https://raw.githubusercontent.com/lbryio/chainquery/"${VERSION}"/config/default/chainqueryconfig.toml /etc/lbry/chainqueryconfig.toml.orig
 RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery && \
  chown -R chainquery:chainquery /etc/lbry
 COPY --from=prep ./healthcheck /chainquery /start /usr/bin/
 HEALTHCHECK --interval=1m --timeout=30s \
  CMD healthcheck
 EXPOSE 6300
 USER chainquery
 STOPSIGNAL SIGINT
 CMD ["start"]
--- a/e2e/chainquery/docker/build.sh
+++ b/e2e/chainquery/docker/build.sh
@ -1,8 +0,0 @@
 #!/bin/bash
 if [ $# -eq 0 ]
  then
    echo "No docker tag argument supplied. Use './build.sh <tag>'"
    exit 1
 fi
 docker build --no-cache --build-arg VERSION=$1 --tag lbry/chainquery:$1 .
 docker push lbry/chainquery:$1
--- a/e2e/chainquery/docker/healthcheck.sh
+++ b/e2e/chainquery/docker/healthcheck.sh
@ -1,2 +0,0 @@
 #!/usr/bin/env bash
 curl --fail http://localhost:6300/api/status || exit 1
--- a/e2e/chainquery/docker/my.cnf
+++ b/e2e/chainquery/docker/my.cnf
@ -1,9 +0,0 @@
 # Default Homebrew MySQL server config
 [mysqld]
 # Only allow connections from localhost
 innodb_log_file_size=5G
 key_buffer_size=1G
 innodb_flush_log_at_trx_commit = 0
 innodb_autoinc_lock_mode=2
 innodb_buffer_pool_size=1G
 innodb_log_buffer_size=1G
--- a/e2e/chainquery/docker/start.sh
+++ b/e2e/chainquery/docker/start.sh
@ -1,51 +0,0 @@
 #!/usr/bin/env bash
 ## Config setup
 ## Setup Values
 DEBUGMODE=$(echo "debugmode=$DEBUGMODE")
 LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"")
 MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
 APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
 ## Setup Defaults
 DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false'
 LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"'
 MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
 APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
 ## Add setup value variable name to this list to get processed on container start
 CONFIG_SETTINGS=(
  DEBUGMODE
  LBRYCRDURL
  MYSQLDSN
  APIMYSQLDSN
 )
 function set_configs() {
  ## Set configs on container start if not already set.
  for i in "${!CONFIG_SETTINGS[@]}"; do
    ## Indirect references http://tldp.org/LDP/abs/html/ivr.html
    eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT"
    eval TO_STRING=\$${CONFIG_SETTINGS[$i]}
    ## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts.
    sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/lbry/chainqueryconfig.toml
  done
  echo "Reading config for debugging."
  cat /etc/lbry/chainqueryconfig.toml
 }
 if [[ ! -f /etc/lbry/chainqueryconfig.toml ]]; then
  echo "[INFO]: Did not find chainqueryconfig.toml"
  echo "        Installing default and configuring with provided environment variables if any."
  ## Install fresh copy of config file.
  echo "cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml"
  cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml
  ls -lAh /etc/lbry/
  set_configs
 else
  echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/lbry"
 fi
 ## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml
 chainquery serve --configpath "/etc/lbry/"
--- a/e2e/chainqueryconfig.toml
+++ b/e2e/chainqueryconfig.toml
@ -1,99 +0,0 @@
 #Debug mode outputs specific information to the console
 #DEFAULT: false
 #debugmode=
 #DebugQueryMode outputs SQL Boiler queries to the console.
 #DEFAULT: false
 #debugquerymode=
 #LBRYcrd URL is required for chainquery to query the blockchain
 #DEFAULT: "rpc://lbry:lbry@localhost:9245"
 lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
 #MySQL DSN is required for chainquery to store information.
 #DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
 #SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
 mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
 #API MySQL DSN is required for chainquery to expose a SQL query service
 #DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
 #SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
 apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
 #API Host and Port is required for the API Server to bind and listen on.
 #DEFAULT: "0.0.0.0:6300"
 #apihostport=
 #Profile mode enables and disables the reporting of a profile for chainquery
 #DEFAULT: false
 #profilemode=
 #Daemon mode tells chainquery how hard it should work catch up processing the blockchain
 #deamonmode=0 #BeastMode it continuously process block after block until caughtup.
 #daemonmode=1 #SlowAndSteadyMode it will process block with a frequency of 1 block every 100ms
 #daemonmode=2 #DelayMode it will process a block with a configured delay frequency (set via 'processingdelay')
 #daemonmode=3 #DaemonMode it will process a block every iteration of the daemon.
 #DEFAULT: 0
 #deamonmode=
 #Default client timeout is for communication with the api of chainquery
 #DEFAULT: 20 #Measured in seconds
 #defaultclienttimeout=
 #Processing delay is used to determine how frequently chainquery should process a block
 # It is only used if Daemon mode is set to delay mode
 #DEFAULT: 100 #Measured in milliseconds
 #processingdelay=
 #Daemon delay is the frequency at which chainquery checks for work to do.
 #DEFAULT: 1 #Measured in seconds
 #daemondelay=
 #Profiling options - will output the time take for certain opertions related to the below category
 #DEFAULT: false (for all 3 params)
 #daemonprofile=
 #lbrycrdprofile=
 #mysqlprofile=
 #Slack Hook URL allows slack integration. All logging info level and above is posted to a slack channel.
 #DEFAULT: ""
 #slackhookurl=
 #Slack Channel is the channel that you want the messages to appear. Works together with the hook url.
 #DEFAULT: ""
 #slackchannel=
 #Slack Log Level tells chainquery what level of logging will be sent to the slack channel. It will log all levels below
 # it as well. Panic=0,Fatal=1,Error=2,Warning=3,Info=4,Debug=5
 #DEFAULT: 0
 #slackloglevel=
 #The command that should be executed to trigger a self update of the software. For linux, for example, `<yourscript>.sh`
 #DEFAULT: ""
 #autoupdatecommand=
 #Twilio service of chainquery to send specifically important information to key users of the Chainquery install.
 #DEFAULT:
 ##twiliosid=""
 ##twilioauthtoken=""
 ##smsrecipients=["",""]
 ##smsfromphonenumber=""
 #twiliosid=
 #twilioauthtoken=
 #smsrecipients=
 #smsfromphonenumber=
 #API Keys - Disallowed by default unless keys are entered.
 #DEFAULT: []
 #apikeys=
 #Max Failures - Specifies the  number of failures that can happen in processing a transaction. This is for parallel
 #transaction processing which puts a transaction to the back of the processing queue if it fails. It can fail say if its
 #source output to spend is not already processed.
 #DEFAULT: 1000
 #maxfailures=
 #Block Chain Name - Specifies the chain params for parsing blocks, transactions, claims, and addresses. valid choices are
 #lbrycrd_main, lbrycrd_testnet, and lbrycrd_regtest.
 #DEFAULT: "lbrycrd_main"
 blockchainname="lbrycrd_regtest"
--- a/e2e/daemon_settings.yml
+++ b/e2e/daemon_settings.yml
@ -1,10 +0,0 @@
 blockchain_name: lbrycrd_regtest
 lbryum_servers:
 - walletserver:50001
 reflect_streams: false
 save_blobs: true
 save_files: false
 share_usage_data: false
 tcp_port: 3333
 udp_port: 4444
 use_upnp: false
--- a/e2e/data_setup.sh
+++ b/e2e/data_setup.sh
@ -1,24 +0,0 @@
 #!/usr/bin/env bash
 set -e
 #Add a ytsync user
 ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
 #Insert an auth token for the user to be used by ytsync
 ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
 #Give priveledges to ytsync user
 ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
 #Add youtuber to sync
 ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
 #Insert an auth token for the youtuber to be used by ytsync
 ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
 #Add their youtube channel to be synced
 ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit,reward_amount,reward_expiration)
 VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'$3',60,2048,0,'2019-08-01 00:00:00')"
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" --default-character-set=utf8 -P 15500 -e "$ADDYTCHANNEL"
--- a/e2e/docker-compose.yml
+++ b/e2e/docker-compose.yml
@ -1,141 +0,0 @@
 version: "3.5"
 services:
  #############
  ## Lbrycrd ##
  #############
  lbrycrd:
    image: lbry/lbrycrd:v0.17.3.2-deprecatedrpc
    restart: "no"
    ports:
      - "15201:29246"
      - "15200:29245"
    expose:
      - "29246"
      - "29245"
    ## host volumes for persistent data such as wallet private keys.
    volumes:
      - "./persist:/data"
    environment:
      - RUN_MODE=regtest
  ###################
  ## Wallet Server ##
  ###################
  walletserver:
    image: lbry/wallet-server:v0.101.1
    restart: always
    environment:
      - DB_DIRECTORY=/database
      - MAX_SEND=1000000000000000000000
      - DAEMON_URL=http://lbry:lbry@lbrycrd:29245
      - MAX_SUBS=1000000000000
      - BANDWIDTH_LIMIT=80000000000
      - SESSION_TIMEOUT=10000000000000000000000000
      - TCP_PORT=50001
      - ELASTIC_HOST=es01
    ports:
      - "15300:50001"
    expose:
      - "50001"
    depends_on:
      - lbrycrd
      - es01
    ulimits:
      nofile:
        soft: 90000
        hard: 90000
    #command: lbry.wallet.server.coin.LBC
    command: lbry.wallet.server.coin.LBCRegTest
  #############
  ## elasticsearch ##
  #############
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
    container_name: es01
    environment:
      - node.name=es01
      - discovery.type=single-node
      - indices.query.bool.max_clause_count=8196
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms4g -Xmx4g"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    ports:
      - "9200:9200"
    expose:
      - "9200"
  #############
  ## Lbrynet ##
  #############
  lbrynet:
    image: lbry/lbrynet:v0.99.0
    restart: always
    ports:
      - "15100:5279"
      - "15101:5280"
    expose:
      - "5279"
      - "5280"
    depends_on:
      - walletserver
    environment:
      - LBRY_STREAMING_SERVER=0.0.0.0:5280
      - LBRY_FEE_PER_NAME_CHAR=0
    volumes:
      - "./persist/.lbrynet:/home/lbrynet"
      - ".:/etc/lbry" #Put your daemon_settings.yml here
      # /private/var/tmp for OSX and /var/tmp for Linux
      - "${LOCAL_TMP_DIR}"
  ###########
  ## MySQL ##
  ###########
  mysql:
    image: mysql/mysql-server:5.7.33
    restart: "no"
    ports:
      - "15500:3306"
    expose:
      - "3306"
    environment:
      - MYSQL_ALLOW_EMPTY_PASSWORD=true
      - MYSQL_DATABASE=lbry
      - MYSQL_USER=lbry
      - MYSQL_PASSWORD=lbry
      - MYSQL_LOG_CONSOLE=true
    volumes:
      - "./init.sql:/docker-entrypoint-initdb.d/init.sql"
      - "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
  ###################
  ## Internal APIs ##
  ###################
  internalapis:
    image: odyseeteam/internal-apis:master
    restart: "no"
    ports:
      - "15400:8080"
    expose:
      - "8080"
    depends_on:
      - mysql
      - lbrycrd
    environment:
      - MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
      - LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
      - REPLICA_DSN=lbry:lbry@tcp(mysql:3306)/lbry
    entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
  ################
  ## Chainquery ##
  ################
  chainquery:
    image: odyseeteam/chainquery:master
    restart: "no"
    ports:
      - 6300:6300
    depends_on:
      - lbrycrd
      - mysql
    ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
    volumes:
      - ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
    entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
--- a/e2e/e2e.sh
+++ b/e2e/e2e.sh
@ -1,96 +0,0 @@
 #!/usr/bin/env bash
 set -e
 #Always compile ytsync
 make
 #Always compile supporty
 cd e2e/supporty && make && cd ../..
 #OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details
 export LOCAL_TMP_DIR="/var/tmp:/var/tmp"
 #Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET
 touch -a .env && set -o allexport; source ./.env; set +o allexport
 echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR"
 # Compose settings - docker only
 export LBRYNET_ADDRESS="http://localhost:15100"
 export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200" #required for supporty
 export LBRYNET_USE_DOCKER=true
 export REFLECT_BLOBS=false
 export CLEAN_ON_STARTUP=true
 export REGTEST=true
 # Local settings
 export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
 export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
 export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
 export TMP_DIR="/var/tmp"
 export CHAINNAME="lbrycrd_regtest"
 export UID
 cd ./e2e
 docker-compose stop
 docker-compose rm -f
 echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
 docker-compose pull
 if [[ -d persist ]]; then rm -rf persist; fi
 mkdir  -m 0777 -p ./persist
 mkdir -m 777 -p ./persist/.walletserver
 mkdir -m 777 -p ./persist/.lbrynet
 #sudo chown -Rv 999:999 ./persist/.walletserver
 #sudo chown -Rv 1000:1000 ./persist/.lbrynet
 docker-compose up -d
 printf 'waiting for internal apis'
 until curl --output /dev/null --silent --head --fail http://localhost:15400; do
    printf '.'
    sleep 1
 done
 echo "successfully started..."
 channelToSync="UCMn-zv1SE-2y6vyewscfFqw"
 channelName=@whatever"$(date +%s)"
 latestVideoID="yPJgjiMbmX0"
 #Data Setup for test
 ./data_setup.sh "$channelName" "$channelToSync" "$latestVideoID"
 # Execute the sync test!
 ./../bin/ytsync --channelID "$channelToSync" --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
 status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
 videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
 videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
 videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
 videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
 videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
 # Create Supports for published claim
 ./supporty/supporty "$channelName" "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
 ./supporty/supporty "$channelName" "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
 ./supporty/supporty "$channelName" "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
 ./supporty/supporty "$channelName" "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
 curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
 # Reset status for transfer test
 mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
 # Trigger transfer api
 curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
 # Execute the transfer test!
 ./../bin/ytsync --channelID $channelToSync --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
 # Check that the channel and the video are marked as transferred and that all supports are spent
 channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
 videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
 nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
 if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
    echo "~~!!!~~~FAILED~~~!!!~~"
    echo "Channel Status: $status"
    echo "Video Status: $videoStatus"
    echo "Channel Transfer Status: $channelTransferStatus"
    echo "Video Transfer Status: $videoTransferStatus"
    echo "Nr Unspent Supports: $nrUnspentSupports"
    #docker-compose logs --tail="all" lbrycrd
    #docker-compose logs --tail="all" walletserver
    #docker-compose logs --tail="all" lbrynet
    #docker-compose logs --tail="all" internalapis
    exit 1;
    else
      echo "SUCCESSSSSSSSSSSSS!"
 fi;
 docker-compose down
--- a/e2e/init.sql
+++ b/e2e/init.sql
@ -1,3 +0,0 @@
 CREATE DATABASE IF NOT EXISTS chainquery;
 GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
 FLUSH PRIVILEGES;
--- a/e2e/lbrycrd/docker-compose.yml
+++ b/e2e/lbrycrd/docker-compose.yml
@ -1,26 +0,0 @@
 version: "3"
 networks:
  lbry-network:
    external: true
 services:
  #############
  ## Lbrycrd ##
  #############
  lbrycrd:
    image: lbry/lbrycrd:v0.12.4.1
    restart: always
    networks:
      lbry-network:
        ipv4_address: 10.6.1.1
    ports:
      - "15201:29246"
      - "15200:29245"
    expose:
      - "29246"
      - "29245"
    ## host volumes for persistent data such as wallet private keys.
    volumes:
      - "../persist/data:/data"
    environment:
      - RUN_MODE=regtest
--- a/e2e/lbrycrd/docker/Dockerfile
+++ b/e2e/lbrycrd/docker/Dockerfile
@ -1,37 +0,0 @@
 FROM ubuntu:18.04 as prep
 LABEL MAINTAINER="leopere [at] nixc [dot] us"
 ## TODO: Implement version pinning. `apt-get install curl=<version>`
 RUN apt-get update && \
  apt-get -y install unzip curl build-essential && \
  apt-get autoclean -y && \
  rm -rf /var/lib/apt/lists/*
 WORKDIR /
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 COPY ./start.sh start
 COPY ./healthcheck.sh healthcheck
 COPY ./advance_blocks.sh advance
 COPY ./fix-permissions.c fix-permissions.c
 ## Add lbrycrd - Change the version below to create an image for a different tag/version
 ARG VERSION="v0.12.4.1"
 RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrycrd-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrycrd-linux.zip $URL
 RUN unzip ./lbrycrd-linux.zip && \
  gcc fix-permissions.c -o fix-permissions && \
  chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
 FROM ubuntu:18.04 as app
 COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
 RUN addgroup --gid 1000 lbrycrd && \
  adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
  mkdir /etc/lbry && \
  chown lbrycrd /etc/lbry && \
  chmod a+s /usr/bin/fix-permissions
 VOLUME ["/data"]
 WORKDIR /data
 ## TODO: Implement healthcheck.
 # HEALTHCHECK ["healthcheck"]
 EXPOSE 9246 9245 11337 29245
 USER lbrycrd
 CMD ["start"]
--- a/e2e/lbrycrd/docker/advance_blocks.sh
+++ b/e2e/lbrycrd/docker/advance_blocks.sh
@ -1,5 +0,0 @@
 #!/usr/bin/env bash
 while true; do
        lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 100 >> /tmp/output.log
        sleep 2
 done
--- a/e2e/lbrycrd/docker/build.sh
+++ b/e2e/lbrycrd/docker/build.sh
@ -1,8 +0,0 @@
 #!/bin/bash
 if [ $# -eq 0 ]
  then
    echo "No docker tag argument supplied. Use './build.sh <tag>'"
    exit 1
 fi
 docker build --build-arg VERSION=$1 --tag lbry/lbrycrd:$1 .
 docker push lbry/lbrycrd:$1
--- a/e2e/lbrycrd/docker/fix-permissions.c
+++ b/e2e/lbrycrd/docker/fix-permissions.c
@ -1,9 +0,0 @@
 #include <unistd.h>
 int main() {
  // This program needs to run with setuid == root
  // This needs to be in a compiled language because you cannot setuid bash scripts
  setuid(0);
  execle("/bin/bash", "bash", "-c",
         "/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/",
         (char*) NULL, (char*) NULL);
 }
--- a/e2e/lbrycrd/docker/healthcheck.sh
+++ b/e2e/lbrycrd/docker/healthcheck.sh
@ -1,4 +0,0 @@
 ## TODO: Implement a healthcheck for lbrycrd.
 curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;'  http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246
 ## OR
 lbrycrd-cli getinfo
--- a/e2e/lbrycrd/docker/start.sh
+++ b/e2e/lbrycrd/docker/start.sh
@ -1,127 +0,0 @@
 #!/usr/bin/env bash
 CONFIG_PATH=/etc/lbry/lbrycrd.conf
 function override_config_option() {
    # Remove existing config line from a config file
    #  and replace with environment fed value.
    # Does nothing if the variable does not exist.
    #  var     Name of ENV variable
    #  option  Name of config option
    #  config  Path of config file
    local var=$1 option=$2 config=$3
    if [[ -v $var ]]; then
        # Remove the existing config option:
        sed -i "/^$option\W*=/d" "$config"
        # Add the value from the environment:
        echo "$option=${!var}" >> "$config"
    fi
 }
 function set_config() {
  if [ -d "$CONFIG_PATH" ]; then
      echo "$CONFIG_PATH is a directory when it should be a file."
      exit 1
  elif [ -f "$CONFIG_PATH" ]; then
      echo "Merging the mounted config file with environment variables."
      local MERGED_CONFIG=/tmp/lbrycrd_merged.conf
      cat $CONFIG_PATH > $MERGED_CONFIG
      echo "" >> $MERGED_CONFIG
      override_config_option PORT port $MERGED_CONFIG
      override_config_option RPC_USER rpcuser $MERGED_CONFIG
      override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG
      override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
      override_config_option RPC_PORT rpcport $MERGED_CONFIG
      override_config_option RPC_BIND rpcbind $MERGED_CONFIG
      # Make the new merged config file the new CONFIG_PATH
      # This ensures that the original file the user mounted remains unmodified
      CONFIG_PATH=$MERGED_CONFIG
  else
      echo "Creating a fresh config file from environment variables."
      ## Set config params
      {
        echo "port=${PORT=9246}"
        echo "rpcuser=${RPC_USER=lbry}"
        echo "rpcpassword=${RPC_PASSWORD=lbry}"
        echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}"
        echo "rpcport=${RPC_PORT=9245}"
        echo "rpcbind=${RPC_BIND=0.0.0.0}"
        echo "deprecatedrpc=accounts"
        echo "deprecatedrpc=validateaddress"
        echo "deprecatedrpc=signrawtransaction"
      } >>  $CONFIG_PATH
  fi
  echo "Config: "
  cat $CONFIG_PATH
 }
 ## Ensure perms are correct prior to running main binary
 /usr/bin/fix-permissions
 ## You can optionally specify a run mode if you want to use lbry defined presets for compatibility.
 case $RUN_MODE in
  default )
    set_config
    lbrycrdd -server -conf=$CONFIG_PATH -printtoconsole
    ;;
  ## If it's a first run you need to do a full index including all transactions
  ## tx index creates an index of every single transaction in the block history if
  ## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs.
  ## This is generally specific to chainquery.
  reindex )
    ## Apply this RUN_MODE in the case you need to update a dataset.  NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run.
    set_config
    lbrycrdd -server -txindex -reindex -conf=$CONFIG_PATH -printtoconsole
    ;;
  chainquery )
    ## If your only goal is to run Chainquery against this instance of lbrycrd and you're starting a
    ## fresh local dataset use this run mode.
    set_config
    lbrycrdd -server -txindex -conf=$CONFIG_PATH -printtoconsole
    ;;
  regtest )
    ## Set config params
    ## TODO: Make this more automagic in the future.
    mkdir -p "$(dirname $CONFIG_PATH)"
    echo "rpcuser=lbry" >                       $CONFIG_PATH
    echo "rpcpassword=lbry" >>                  $CONFIG_PATH
    echo "rpcport=29245" >>                     $CONFIG_PATH
    echo "rpcbind=0.0.0.0" >>                   $CONFIG_PATH
    echo "rpcallowip=0.0.0.0/0" >>              $CONFIG_PATH
    echo "regtest=1" >>                         $CONFIG_PATH
    echo "txindex=1" >>                         $CONFIG_PATH
    echo "server=1" >>                          $CONFIG_PATH
    echo "printtoconsole=1" >>                  $CONFIG_PATH
    echo "deprecatedrpc=accounts" >>            $CONFIG_PATH
    echo "deprecatedrpc=validateaddress" >>     $CONFIG_PATH
    echo "deprecatedrpc=signrawtransaction" >>  $CONFIG_PATH
    echo "vbparams=segwit:0:999999999999" >>    $CONFIG_PATH
    echo "addresstype=legacy" >>                $CONFIG_PATH
    #nohup advance &>/dev/null &
    lbrycrdd -conf=$CONFIG_PATH $1
    ;;
  testnet )
    ## Set config params
    ## TODO: Make this more automagic in the future.
    mkdir -p "$(dirname $CONFIG_PATH)"
    echo "rpcuser=lbry" >                       $CONFIG_PATH
    echo "rpcpassword=lbry" >>                  $CONFIG_PATH
    echo "rpcport=29245" >>                     $CONFIG_PATH
    echo "rpcbind=0.0.0.0" >>                   $CONFIG_PATH
    echo "rpcallowip=0.0.0.0/0" >>              $CONFIG_PATH
    echo "testnet=1" >>                         $CONFIG_PATH
    echo "txindex=1" >>                         $CONFIG_PATH
    echo "server=1" >>                          $CONFIG_PATH
    echo "printtoconsole=1" >>                  $CONFIG_PATH
    echo "deprecatedrpc=accounts" >>            $CONFIG_PATH
    echo "deprecatedrpc=validateaddress" >>     $CONFIG_PATH
    echo "deprecatedrpc=signrawtransaction" >>  $CONFIG_PATH
    #nohup advance &>/dev/null &
    lbrycrdd -conf=$CONFIG_PATH $1
    ;;
  * )
    echo "Error, you must define a RUN_MODE environment variable."
    echo "Available options are testnet, regtest, chainquery, default, and reindex"
    ;;
 esac
--- a/e2e/lbrynet/docker-compose.yml
+++ b/e2e/lbrynet/docker-compose.yml
@ -1,23 +0,0 @@
 version: "3"
 networks:
  lbry-network:
    external: true
 services:
  #############
  ## Lbrynet ##
  #############
  lbrynet:
    image: lbry/lbrynet:v0.99.0
    restart: "no"
    networks:
      lbry-network:
        ipv4_address: 10.6.1.3
    ports:
      - "15100:5279"
      - "15101:5280"
    environment:
      - LBRY_STREAMING_SERVER=0.0.0.0:5280
    volumes:
      - "../persist/data/.lbrynet:/home/lbrynet"
      - "./settings:/etc/lbry" #Put your daemon_settings.yml here
--- a/e2e/lbrynet/docker/Dockerfile
+++ b/e2e/lbrynet/docker/Dockerfile
@ -1,27 +0,0 @@
 ## This base image is for running the latest lbrynet-daemon release.
 FROM ubuntu:18.04 as prep
 LABEL MAINTAINER="leopere [at] nixc [dot] us"
 RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it
 ## Add lbrynet
 ARG VERSION="latest"
 RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL
 COPY start.sh /usr/bin/start
 COPY checkmount.sh /usr/bin/checkmount
 RUN unzip /lbrynet.linux.zip -d /lbrynet/ && \
    mv /lbrynet/lbrynet /usr/bin && \
    chmod a+x /usr/bin/checkmount /usr/bin/start /usr/bin/lbrynet
 FROM ubuntu:18.04 as app
 COPY --from=prep /usr/bin/start /usr/bin/checkmount /usr/bin/lbrynet /usr/bin/
 RUN adduser lbrynet --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbrynet
 ## Daemon port [Intended for internal use]
 ## LBRYNET talks to peers on port 3333 [Intended for external use] this port is used to discover other lbrynet daemons with blobs.
 ## Expose 5566 Reflector port to listen on
 ## Expose 5279 Port the daemon API will listen on
 ## the lbryumx aka Wallet port [Intended for internal use]
 #EXPOSE 4444 3333 5566 5279 50001
 USER lbrynet
 ENTRYPOINT ["/usr/bin/checkmount"]
 CMD ["start"]
--- a/e2e/lbrynet/docker/build.sh
+++ b/e2e/lbrynet/docker/build.sh
@ -1,8 +0,0 @@
 #!/bin/bash
 if [ $# -eq 0 ]
  then
    echo "No docker tag argument supplied. Use './build.sh <tag>'"
    exit 1
 fi
 docker build --build-arg VERSION=$1 --tag lbry/lbrynet:$1 .
 docker push lbry/lbrynet:$1
--- a/e2e/lbrynet/docker/checkmount.sh
+++ b/e2e/lbrynet/docker/checkmount.sh
@ -1,12 +0,0 @@
 #!/bin/bash
 ## TODO: Make a bit more aware of the run mode of this appliance in case there is ever a test mode enabled in the start.sh
 mountpoint=/home/lbrynet
 if ! grep -qs ".* $mountpoint " /proc/mounts; then
    echo "$mountpoint not mounted, refusing to run."
    ## TODO: We should have documentation that this error references directly with a URL as to why it won't run without a volume.
    exit 1
 else
    bash -c "$*"
 fi
--- a/e2e/lbrynet/docker/start.sh
+++ b/e2e/lbrynet/docker/start.sh
@ -1,4 +0,0 @@
 #!/bin/bash
 lbrynet start \
  --api="${API_BIND_IP:-0.0.0.0}":"${API_PORT:-5279}" \
  --config="${CONFIG_PATH:-/etc/lbry/daemon_settings.yml}"
--- a/e2e/lbrynet/settings/daemon_settings.yml
+++ b/e2e/lbrynet/settings/daemon_settings.yml
@ -1,13 +0,0 @@
 #blockchain_name: lbrycrd_main
 #blockchain_name: lbrycrd_testnet
 blockchain_name: lbrycrd_regtest
 lbryum_servers:
 #  - spv1.lbry.com:50001 #Production Wallet Server
  - walletserver:50001
 save_blobs: true
 save_files: false
 reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
 share_usage_data: false
 tcp_port: 3333
 udp_port: 4444
 use_upnp: true
--- a/e2e/supporty/Makefile
+++ b/e2e/supporty/Makefile
@ -1,12 +0,0 @@
 BINARY=supporty
 DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
 BIN_DIR = ${DIR}
 .PHONY: build clean test lint
 .DEFAULT_GOAL: build
 build:
 	mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} supporty.go
 	chmod +x ${BIN_DIR}/${BINARY}
--- a/e2e/supporty/supporty.go
+++ b/e2e/supporty/supporty.go
@ -1,43 +0,0 @@
 package main
 import (
 	"os"
 	"strconv"
 	"strings"
 	"github.com/lbryio/ytsync/v5/util"
 	"github.com/sirupsen/logrus"
 )
 func main() {
 	if len(os.Args) != 6 {
 		logrus.Info(strings.Join(os.Args, ","))
 		logrus.Fatal("Not enough arguments: name, claimID, address, blockchainName, claimAmount")
 	}
 	println("Supporty!")
 	lbrycrd, err := util.GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
 	if err != nil {
 		logrus.Fatal(err)
 	}
 	if lbrycrd == nil {
 		logrus.Fatal("Lbrycrd Client is nil")
 	}
 	amount, err := strconv.ParseFloat(os.Args[5], 64)
 	if err != nil {
 		logrus.Error(err)
 	}
 	name := os.Args[1]
 	claimid := os.Args[2]
 	claimAddress := os.Args[3]
 	blockChainName := os.Args[4]
 	logrus.Infof("Supporting %s[%s] with %.2f LBC on chain %s at address %s", name, claimid, amount, blockChainName, claimAddress)
 	hash, err := lbrycrd.SupportClaim(name, claimid, claimAddress, blockChainName, amount)
 	if err != nil {
 		logrus.Error(err)
 	}
 	if hash == nil {
 		logrus.Fatal("Tx not created!")
 	}
 	logrus.Info("Tx: ", hash.String())
 }
--- a/e2e/walletserver/docker-compose.yml
+++ b/e2e/walletserver/docker-compose.yml
@ -1,35 +0,0 @@
 version: "3"
 networks:
  lbry-network:
    external: true
 services:
  ###################
  ## Wallet Server ##
  ###################
  walletserver:
    image: lbry/wallet-server:v0.73.1
    restart: always
    networks:
      lbry-network:
        ipv4_address: 10.6.1.2
    volumes:
      - "../persist/data/.walletserver/database:/database"
    environment:
      - DB_DIRECTORY=/database
      - MAX_SEND=1000000000000000000000
      - DAEMON_URL=http://lbry:lbry@lbrycrd:29245/
      - MAX_SUBS=1000000000000
      - BANDWIDTH_LIMIT=80000000000
      - SESSION_TIMEOUT=10000000000000000000000000
      - TCP_PORT=50001
    #network_mode: host
    #network_mode: bridge
    ports:
      - "50001:50001"
    expose:
      - "50001"
    ulimits:
      nofile: 90000
 #    command: lbry.wallet.server.coin.LBC
    command: lbry.wallet.server.coin.LBCRegTest
--- a/go.mod
+++ b/go.mod
@ -1,151 +1,44 @@
-go 1.17
+module github.com/lbryio/ytsync
 module github.com/lbryio/ytsync/v5
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
 //replace github.com/lbryio/lbry.go/v2 => /home/niko/go/src/github.com/lbryio/lbry.go/
 //replace github.com/lbryio/reflector.go => /home/niko/go/src/github.com/lbryio/reflector.go/
 require (
-	github.com/abadojack/whatlanggo v1.0.1
+	cloud.google.com/go v0.37.4 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
+	github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
-	github.com/aws/aws-sdk-go v1.44.6
+	github.com/PuerkitoBio/goquery v1.5.0 // indirect
-	github.com/davecgh/go-spew v1.1.1
+	github.com/aws/aws-sdk-go v1.17.3
-	github.com/docker/docker v20.10.17+incompatible
+	github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
-	github.com/lbryio/lbry.go/v2 v2.7.2-0.20220815204100-2adb8af5b68c
+	github.com/go-ini/ini v1.42.0 // indirect
-	github.com/lbryio/reflector.go v1.1.3-0.20220730181028-f5d30b1a6e79
+	github.com/golang/protobuf v1.3.1 // indirect
-	github.com/mitchellh/go-ps v1.0.0
+	github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect
-	github.com/prometheus/client_golang v1.12.1
+	github.com/gorilla/websocket v1.4.0 // indirect
-	github.com/shopspring/decimal v1.3.1
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.4.0
 	github.com/stretchr/testify v1.7.1
 	github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f
 	github.com/vbauerster/mpb/v7 v7.4.1
 	gopkg.in/vansante/go-ffprobe.v2 v2.0.3
 	gotest.tools v2.2.0+incompatible
 )
 require (
 	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bluele/gcache v0.0.2 // indirect
 	github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7 // indirect
 	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 // indirect
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d // indirect
 	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
 	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
 	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/gin-gonic/gin v1.7.7 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-ini/ini v1.48.0 // indirect
 	github.com/go-playground/locales v0.13.0 // indirect
 	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/go-playground/validator/v10 v10.4.1 // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/gofrs/uuid v3.2.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/rpc v1.2.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/memberlist v0.3.0 // indirect
 	github.com/hashicorp/serf v0.9.7 // indirect
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
-	github.com/johntdyer/slackrus v0.0.0-20211215141436-33e4a270affb // indirect
+	github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
-	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/lbryio/lbry.go v1.0.14
-	github.com/karrick/godirwalk v1.17.0 // indirect
+	github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
-	github.com/lbryio/chainquery v1.9.0 // indirect
+	github.com/mitchellh/mapstructure v1.1.2 // indirect
-	github.com/lbryio/lbry.go v1.1.2 // indirect
+	github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
-	github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 // indirect
+	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/leodido/go-urn v1.2.0 // indirect
+	github.com/onsi/gomega v1.5.0 // indirect
-	github.com/lucas-clemente/quic-go v0.28.1 // indirect
+	github.com/prometheus/common v0.3.0
-	github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
+	github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
-	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/sirupsen/logrus v1.4.1
-	github.com/marten-seemann/qpack v0.2.1 // indirect
+	github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
-	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
+	github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85
-	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
+	github.com/spf13/pflag v1.0.3 // indirect
 	github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/miekg/dns v1.1.41 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/onsi/ginkgo v1.16.4 // indirect
 	github.com/onsi/gomega v1.17.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/pelletier/go-toml v1.9.3 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/slack-go/slack v0.10.3 // indirect
 	github.com/spf13/afero v1.4.1 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.7.1 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/ugorji/go/codec v1.1.7 // indirect
 	github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d // indirect
 	github.com/volatiletech/null v8.0.0+incompatible // indirect
 	github.com/volatiletech/sqlboiler v3.4.0+incompatible // indirect
 	github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
-	go.uber.org/atomic v1.9.0 // indirect
+	go.opencensus.io v0.20.2 // indirect
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
-	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 // indirect
-	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
+	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	google.golang.org/api v0.3.2
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	google.golang.org/appengine v1.5.0 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/grpc v1.20.0 // indirect
-	golang.org/x/tools v0.1.5 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	gopkg.in/ini.v1 v1.42.0 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/yaml.v2 v2.2.2 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/ini.v1 v1.60.2 // indirect
 	gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	gotest.tools/v3 v3.2.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
--- a/ip_manager/throttle.go
+++ b/ip_manager/throttle.go
@ -1,227 +0,0 @@
 package ip_manager
 import (
 	"net"
 	"sort"
 	"sync"
 	"time"
 	"github.com/asaskevich/govalidator"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/ytsync/v5/util"
 	log "github.com/sirupsen/logrus"
 )
 const IPCooldownPeriod = 20 * time.Second
 const unbanTimeout = 48 * time.Hour
 var stopper = stop.New()
 type IPPool struct {
 	ips     []throttledIP
 	lock    *sync.RWMutex
 	stopGrp *stop.Group
 }
 type throttledIP struct {
 	IP           string
 	UsedForVideo string
 	LastUse      time.Time
 	Throttled    bool
 	InUse        bool
 }
 var ipPoolInstance *IPPool
 func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
 	if ipPoolInstance != nil {
 		return ipPoolInstance, nil
 	}
 	addrs, err := net.InterfaceAddrs()
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	var pool []throttledIP
 	for _, address := range addrs {
 		if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
 			if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
 				pool = append(pool, throttledIP{
 					IP:      ipnet.IP.String(),
 					LastUse: time.Now().Add(-5 * time.Minute),
 				})
 			} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
 				pool = append(pool, throttledIP{
 					IP:      ipnet.IP.String(),
 					LastUse: time.Now().Add(-5 * time.Minute),
 				})
 			}
 		}
 	}
 	ipPoolInstance = &IPPool{
 		ips:     pool,
 		lock:    &sync.RWMutex{},
 		stopGrp: stopGrp,
 	}
 	//ticker := time.NewTicker(10 * time.Second)
 	//go func() {
 	//	for {
 	//		select {
 	//		case <-stopGrp.Ch():
 	//			return
 	//		case <-ticker.C:
 	//			ipPoolInstance.lock.RLock()
 	//			for _, ip := range ipPoolInstance.ips {
 	//				log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
 	//			}
 	//			ipPoolInstance.lock.RUnlock()
 	//		}
 	//	}
 	//}()
 	return ipPoolInstance, nil
 }
 // AllThrottled checks whether the IPs provided are all throttled.
 // returns false if at least one IP is not throttled
 // Not thread safe, should use locking when called
 func AllThrottled(ips []throttledIP) bool {
 	for _, i := range ips {
 		if !i.Throttled {
 			return false
 		}
 	}
 	return true
 }
 // AllInUse checks whether the IPs provided are all currently in use.
 // returns false if at least one IP is not in use AND is not throttled
 // Not thread safe, should use locking when called
 func AllInUse(ips []throttledIP) bool {
 	for _, i := range ips {
 		if !i.InUse && !i.Throttled {
 			return false
 		}
 	}
 	return true
 }
 func (i *IPPool) ReleaseIP(ip string) {
 	i.lock.Lock()
 	defer i.lock.Unlock()
 	for j := range i.ips {
 		localIP := &i.ips[j]
 		if localIP.IP == ip {
 			localIP.InUse = false
 			localIP.LastUse = time.Now()
 			return
 		}
 	}
 	util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
 }
 func (i *IPPool) ReleaseAll() {
 	i.lock.Lock()
 	defer i.lock.Unlock()
 	for j := range i.ips {
 		if i.ips[j].Throttled {
 			continue
 		}
 		localIP := &i.ips[j]
 		localIP.InUse = false
 	}
 }
 func (i *IPPool) SetThrottled(ip string) {
 	i.lock.Lock()
 	defer i.lock.Unlock()
 	var tIP *throttledIP
 	for j, _ := range i.ips {
 		localIP := &i.ips[j]
 		if localIP.IP == ip {
 			if localIP.Throttled {
 				return
 			}
 			localIP.Throttled = true
 			tIP = localIP
 			break
 		}
 	}
 	util.SendErrorToSlack("%s set to throttled", ip)
 	stopper.Add(1)
 	go func(tIP *throttledIP) {
 		defer stopper.Done()
 		unbanTimer := time.NewTimer(unbanTimeout)
 		select {
 		case <-unbanTimer.C:
 			i.lock.Lock()
 			tIP.Throttled = false
 			i.lock.Unlock()
 			util.SendInfoToSlack("%s set back to not throttled", ip)
 		case <-i.stopGrp.Ch():
 			unbanTimer.Stop()
 		}
 	}(tIP)
 }
 var ErrAllInUse = errors.Base("all IPs are in use, try again")
 var ErrAllThrottled = errors.Base("all IPs are throttled")
 var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
 var ErrInterruptedByUser = errors.Base("interrupted by user")
 func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
 	i.lock.Lock()
 	defer i.lock.Unlock()
 	sort.Slice(i.ips, func(j, k int) bool {
 		return i.ips[j].LastUse.Before(i.ips[k].LastUse)
 	})
 	if !AllThrottled(i.ips) {
 		if AllInUse(i.ips) {
 			return nil, errors.Err(ErrAllInUse)
 		}
 		var nextIP *throttledIP
 		for j := range i.ips {
 			ip := &i.ips[j]
 			if ip.InUse || ip.Throttled {
 				continue
 			}
 			nextIP = ip
 			break
 		}
 		if nextIP == nil {
 			return nil, errors.Err(ErrResourceLock)
 		}
 		nextIP.InUse = true
 		nextIP.UsedForVideo = forVideo
 		return nextIP, nil
 	}
 	return nil, errors.Err(ErrAllThrottled)
 }
 func (i *IPPool) GetIP(forVideo string) (string, error) {
 	for {
 		ip, err := i.nextIP(forVideo)
 		if err != nil {
 			if errors.Is(err, ErrAllInUse) {
 				select {
 				case <-i.stopGrp.Ch():
 					return "", errors.Err(ErrInterruptedByUser)
 				default:
 					time.Sleep(5 * time.Second)
 					continue
 				}
 			} else if errors.Is(err, ErrAllThrottled) {
 				return "throttled", err
 			}
 			return "", err
 		}
 		if time.Since(ip.LastUse) < IPCooldownPeriod {
 			log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", ip.IP, (IPCooldownPeriod - time.Since(ip.LastUse)).Seconds())
 			time.Sleep(IPCooldownPeriod - time.Since(ip.LastUse))
 		}
 		return ip.IP, nil
 	}
 }
--- a/ip_manager/throttle_test.go
+++ b/ip_manager/throttle_test.go
@ -1,40 +0,0 @@
 package ip_manager
 import (
 	"testing"
 )
 func TestAll(t *testing.T) {
 	pool, err := GetIPPool()
 	if err != nil {
 		t.Fatal(err)
 	}
 	ip, err := pool.GetIP()
 	if err != nil {
 		t.Fatal(err)
 	}
 	t.Log(ip)
 	pool.ReleaseIP(ip)
 	ip2, err := pool.GetIP()
 	if err != nil {
 		t.Fatal(err)
 	}
 	if ip == ip2 && len(pool.ips) > 1 {
 		t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
 	}
 	t.Log(ip2)
 	pool.ReleaseIP(ip2)
 	for range pool.ips {
 		_, err = pool.GetIP()
 		if err != nil {
 			t.Fatal(err)
 		}
 	}
 	next, err := pool.nextIP()
 	if err != nil {
 		t.Logf("%s", err.Error())
 	} else {
 		t.Fatal(next)
 	}
 }
--- a/main.go
+++ b/main.go
@ -3,21 +3,16 @@ package main
 import (
 	"fmt"
 	"math/rand"
 	"net/http"
 	"os"
 	"os/user"
 	"time"
-	"github.com/lbryio/ytsync/v5/configs"
+	"github.com/lbryio/lbry.go/extras/util"
-	"github.com/lbryio/ytsync/v5/manager"
+	"github.com/lbryio/ytsync/sdk"
 	"github.com/lbryio/ytsync/v5/shared"
 	ytUtils "github.com/lbryio/ytsync/v5/util"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/lbryio/ytsync/manager"
 	log "github.com/sirupsen/logrus"
 )
 var Version string
@ -25,21 +20,30 @@ var Version string
 const defaultMaxTries = 3
 var (
-	cliFlags       shared.SyncFlags
+	stopOnError             bool
-	maxVideoLength int
+	maxTries                int
 	takeOverExistingChannel bool
 	refill                  int
 	limit                   int
 	skipSpaceCheck          bool
 	syncUpdate              bool
 	singleRun               bool
 	syncStatus              string
 	channelID               string
 	syncFrom                int64
 	syncUntil               int64
 	concurrentJobs          int
 	videosLimit             int
 	maxVideoSize            int
 	maxVideoLength          float64
 	removeDBUnpublished     bool
 	upgradeMetadata         bool
 )
 func main() {
 	rand.Seed(time.Now().UnixNano())
 	log.SetLevel(log.DebugLevel)
-	customFormatter := new(log.TextFormatter)
+
 	customFormatter.TimestampFormat = "2006-01-02 15:04:05"
 	customFormatter.FullTimestamp = true
 	log.SetFormatter(customFormatter)
 	http.Handle("/metrics", promhttp.Handler())
 	go func() {
 		log.Error(http.ListenAndServe(":2112", nil))
 	}()
 	cmd := &cobra.Command{
 		Use:   "ytsync",
 		Short: "Publish youtube channels into LBRY network automatically.",
@ -47,25 +51,23 @@ func main() {
 		Args:  cobra.RangeArgs(0, 0),
 	}
-	cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
+	cmd.Flags().BoolVar(&stopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
-	cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
+	cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
-	cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
+	cmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
-	cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
+	cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
-	cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
+	cmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
-	cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
+	cmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
-	cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
+	cmd.Flags().BoolVar(&singleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
-	cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
+	cmd.Flags().BoolVar(&removeDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
-	cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
+	cmd.Flags().BoolVar(&upgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
-	cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
+	cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
-	cmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
+	cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
-	cmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
+	cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
-	cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
+	cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
-	cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
+	cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
-	cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
+	cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
-	cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
+	cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
-	cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
+	cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)")
 	cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
 	cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
 	if err := cmd.Execute(); err != nil {
 		fmt.Println(err)
@ -74,62 +76,128 @@ func main() {
 }
 func ytSync(cmd *cobra.Command, args []string) {
-	err := configs.Init("./config.json")
+	var hostname string
-	if err != nil {
+	slackToken := os.Getenv("SLACK_TOKEN")
-		log.Fatalf("could not parse configuration file: %s", errors.FullTrace(err))
+	if slackToken == "" {
-	}
+		log.Error("A slack token was not present in env vars! Slack messages disabled!")
 	if configs.Configuration.SlackToken == "" {
 		log.Error("A slack token was not present in the config! Slack messages disabled!")
 	} else {
-		util.InitSlack(configs.Configuration.SlackToken, configs.Configuration.SlackChannel, configs.Configuration.GetHostname())
+		var err error
 		hostname, err = os.Hostname()
 		if err != nil {
 			log.Error("could not detect system hostname")
 			hostname = "ytsync-unknown"
 		}
 		util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
 	}
-	if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
+	if syncStatus != "" && !util.InSlice(syncStatus, manager.SyncStatuses) {
-		log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
+		log.Errorf("status must be one of the following: %v\n", manager.SyncStatuses)
 		return
 	}
-	if cliFlags.MaxTries < 1 {
+	if stopOnError && maxTries != defaultMaxTries {
 		log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
 		return
 	}
 	if maxTries < 1 {
 		log.Errorln("setting --max-tries less than 1 doesn't make sense")
 		return
 	}
-	if cliFlags.Limit < 0 {
+	if limit < 0 {
 		log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
 		return
 	}
 	cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
-	if configs.Configuration.InternalApisEndpoint == "" {
+	apiURL := os.Getenv("LBRY_WEB_API")
-		log.Errorln("An Internal APIs Endpoint was not defined")
+	apiToken := os.Getenv("LBRY_API_TOKEN")
 	youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
 	blobsDir := os.Getenv("BLOBS_DIRECTORY")
 	lbrycrdString := os.Getenv("LBRYCRD_STRING")
 	awsS3ID := os.Getenv("AWS_S3_ID")
 	awsS3Secret := os.Getenv("AWS_S3_SECRET")
 	awsS3Region := os.Getenv("AWS_S3_REGION")
 	awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
 	if apiURL == "" {
 		log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
 		return
 	}
-	if configs.Configuration.InternalApisAuthToken == "" {
+	if apiToken == "" {
-		log.Errorln("An Internal APIs auth token was not defined")
+		log.Errorln("An API Token was not defined. Please set the environment variable LBRY_API_TOKEN")
 		return
 	}
-	if configs.Configuration.WalletS3Config.ID == "" || configs.Configuration.WalletS3Config.Region == "" || configs.Configuration.WalletS3Config.Bucket == "" || configs.Configuration.WalletS3Config.Secret == "" || configs.Configuration.WalletS3Config.Endpoint == "" {
+	if youtubeAPIKey == "" {
-		log.Errorln("Wallet S3 configuration is incomplete")
+		log.Errorln("A Youtube API key was not defined. Please set the environment variable YOUTUBE_API_KEY")
 		return
 	}
-	if configs.Configuration.BlockchaindbS3Config.ID == "" || configs.Configuration.BlockchaindbS3Config.Region == "" || configs.Configuration.BlockchaindbS3Config.Bucket == "" || configs.Configuration.BlockchaindbS3Config.Secret == "" || configs.Configuration.BlockchaindbS3Config.Endpoint == "" {
+	if awsS3ID == "" {
-		log.Errorln("Blockchain DBs S3 configuration is incomplete")
+		log.Errorln("AWS S3 ID credentials were not defined. Please set the environment variable AWS_S3_ID")
 		return
 	}
-	if configs.Configuration.LbrycrdString == "" {
+	if awsS3Secret == "" {
-		log.Infoln("Using default (local) lbrycrd instance. Set lbrycrd_string if you want to use something else")
+		log.Errorln("AWS S3 Secret credentials were not defined. Please set the environment variable AWS_S3_SECRET")
 		return
 	}
-
+	if awsS3Region == "" {
-	blobsDir := ytUtils.GetBlobsDir()
+		log.Errorln("AWS S3 Region was not defined. Please set the environment variable AWS_S3_REGION")
-
+		return
-	sm := manager.NewSyncManager(
+	}
-		cliFlags,
+	if awsS3Bucket == "" {
-		blobsDir,
+		log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
-	)
+		return
-	err = sm.Start()
+	}
 	if lbrycrdString == "" {
 		log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
 	}
 	if blobsDir == "" {
 		usr, err := user.Current()
 		if err != nil {
-		ytUtils.SendErrorToSlack(errors.FullTrace(err))
+			log.Errorln(err.Error())
 			return
 		}
-	ytUtils.SendInfoToSlack("Syncing process terminated!")
+		blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
 	}
 	syncProperties := &sdk.SyncProperties{
 		SyncFrom:         syncFrom,
 		SyncUntil:        syncUntil,
 		YoutubeChannelID: channelID,
 	}
 	apiConfig := &sdk.APIConfig{
 		YoutubeAPIKey: youtubeAPIKey,
 		ApiURL:        apiURL,
 		ApiToken:      apiToken,
 		HostName:      hostname,
 	}
 	sm := manager.NewSyncManager(
 		stopOnError,
 		maxTries,
 		takeOverExistingChannel,
 		refill,
 		limit,
 		skipSpaceCheck,
 		syncUpdate,
 		concurrentJobs,
 		concurrentJobs,
 		blobsDir,
 		videosLimit,
 		maxVideoSize,
 		lbrycrdString,
 		awsS3ID,
 		awsS3Secret,
 		awsS3Region,
 		awsS3Bucket,
 		syncStatus,
 		singleRun,
 		syncProperties,
 		apiConfig,
 		maxVideoLength,
 		removeDBUnpublished,
 		upgradeMetadata,
 	)
 	err := sm.Start()
 	if err != nil {
 		manager.SendErrorToSlack(err.Error())
 	}
 	manager.SendInfoToSlack("Syncing process terminated!")
 }
--- a/manager/count.go
+++ b/manager/count.go
@ -0,0 +1,32 @@
 package manager
 import (
 	"net/http"
 	"github.com/lbryio/lbry.go/extras/errors"
 	"google.golang.org/api/googleapi/transport"
 	"google.golang.org/api/youtube/v3"
 )
 func (s *Sync) CountVideos() (uint64, error) {
 	client := &http.Client{
 		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
 	}
 	service, err := youtube.New(client)
 	if err != nil {
 		return 0, errors.Prefix("error creating YouTube service", err)
 	}
 	response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
 	if err != nil {
 		return 0, errors.Prefix("error getting channels", err)
 	}
 	if len(response.Items) < 1 {
 		return 0, errors.Err("youtube channel not found")
 	}
 	return response.Items[0].Statistics.VideoCount, nil
 }
--- a/manager/manager.go
+++ b/manager/manager.go
@ -3,185 +3,237 @@ package manager
 import (
 	"fmt"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
-	"github.com/lbryio/ytsync/v5/blobs_reflector"
+	"github.com/lbryio/ytsync/namer"
-	"github.com/lbryio/ytsync/v5/configs"
+	"github.com/lbryio/ytsync/sdk"
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/shared"
 	logUtils "github.com/lbryio/ytsync/v5/util"
-	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/extras/errors"
-	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/lbry.go/extras/util"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	log "github.com/sirupsen/logrus"
 )
 type SyncManager struct {
-	CliFlags   shared.SyncFlags
+	stopOnError             bool
-	ApiConfig  *sdk.APIConfig
+	maxTries                int
-	LbrycrdDsn string
+	takeOverExistingChannel bool
-
+	refill                  int
 	limit                   int
 	skipSpaceCheck          bool
 	syncUpdate              bool
 	concurrentJobs          int
 	concurrentVideos        int
 	blobsDir                string
-	channelsToSync []Sync
+	videosLimit             int
 	maxVideoSize            int
 	maxVideoLength          float64
 	lbrycrdString           string
 	awsS3ID                 string
 	awsS3Secret             string
 	awsS3Region             string
 	syncStatus              string
 	awsS3Bucket             string
 	singleRun               bool
 	syncProperties          *sdk.SyncProperties
 	apiConfig               *sdk.APIConfig
 	removeDBUnpublished     bool
 	upgradeMetadata         bool
 }
-func NewSyncManager(cliFlags shared.SyncFlags, blobsDir string) *SyncManager {
+func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool, refill int, limit int,
 	skipSpaceCheck bool, syncUpdate bool, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
 	maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
 	syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64, removeDBUnpublished bool, upgradeMetadata bool) *SyncManager {
 	return &SyncManager{
-		CliFlags:   cliFlags,
+		stopOnError:             stopOnError,
 		maxTries:                maxTries,
 		takeOverExistingChannel: takeOverExistingChannel,
 		refill:                  refill,
 		limit:                   limit,
 		skipSpaceCheck:          skipSpaceCheck,
 		syncUpdate:              syncUpdate,
 		concurrentJobs:          concurrentJobs,
 		concurrentVideos:        concurrentVideos,
 		blobsDir:                blobsDir,
-		LbrycrdDsn: configs.Configuration.LbrycrdString,
+		videosLimit:             videosLimit,
-		ApiConfig:  sdk.GetAPIsConfigs(),
+		maxVideoSize:            maxVideoSize,
 		maxVideoLength:          maxVideoLength,
 		lbrycrdString:           lbrycrdString,
 		awsS3ID:                 awsS3ID,
 		awsS3Secret:             awsS3Secret,
 		awsS3Region:             awsS3Region,
 		awsS3Bucket:             awsS3Bucket,
 		syncStatus:              syncStatus,
 		singleRun:               singleRun,
 		syncProperties:          syncProperties,
 		apiConfig:               apiConfig,
 		removeDBUnpublished:     removeDBUnpublished,
 		upgradeMetadata:         upgradeMetadata,
 	}
 }
-func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
+
-	s.channelsToSync = append(s.channelsToSync, Sync{
+const (
-		DbChannelData: channel,
+	StatusPending        = "pending"        // waiting for permission to sync
-		Manager:       s,
+	StatusPendingEmail   = "pendingemail"   // permission granted but missing email
-		namer:         namer.NewNamer(),
+	StatusQueued         = "queued"         // in sync queue. will be synced soon
-		hardVideoFailure: hardVideoFailure{
+	StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
-			lock: &sync.Mutex{},
+	StatusSyncing        = "syncing"        // syncing now
-		},
+	StatusSynced         = "synced"         // done
-	})
+	StatusFailed         = "failed"
-}
+	StatusFinalized      = "finalized" // no more changes allowed
 	StatusAbandoned      = "abandoned" // deleted on youtube or banned
 )
 var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
 const (
 	VideoStatusPublished     = "published"
 	VideoStatusFailed        = "failed"
 	VideoStatusUpgradeFailed = "upgradefailed"
 	VideoStatusUnpublished   = "unpublished"
 )
 func (s *SyncManager) Start() error {
-	if logUtils.ShouldCleanOnStartup() {
+
-		err := logUtils.CleanForStartup()
+	syncCount := 0
 	for {
 		err := s.checkUsedSpace()
 		if err != nil {
 			return err
 		}
 	}
-	var lastChannelProcessed string
+		var syncs []Sync
 	var secondLastChannelProcessed string
 	syncCount := 0
 	for {
 		s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
 		err := s.checkUsedSpace()
 		if err != nil {
 			return errors.Err(err)
 		}
 		shouldInterruptLoop := false
-		if s.CliFlags.IsSingleChannelSync() {
+		isSingleChannelSync := s.syncProperties.YoutubeChannelID != ""
-			channels, err := s.ApiConfig.FetchChannels("", &s.CliFlags)
+		if isSingleChannelSync {
 			channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
 			if err != nil {
-				return errors.Err(err)
+				return err
 			}
 			if len(channels) != 1 {
 				return errors.Err("Expected 1 channel, %d returned", len(channels))
 			}
-			s.enqueueChannel(&channels[0])
+			lbryChannelName := channels[0].DesiredChannelName
 			syncs = make([]Sync, 1)
 			syncs[0] = Sync{
 				APIConfig:               s.apiConfig,
 				YoutubeChannelID:        s.syncProperties.YoutubeChannelID,
 				LbryChannelName:         lbryChannelName,
 				lbryChannelID:           channels[0].ChannelClaimID,
 				StopOnError:             s.stopOnError,
 				MaxTries:                s.maxTries,
 				ConcurrentVideos:        s.concurrentVideos,
 				TakeOverExistingChannel: s.takeOverExistingChannel,
 				Refill:                  s.refill,
 				Manager:                 s,
 				LbrycrdString:           s.lbrycrdString,
 				AwsS3ID:                 s.awsS3ID,
 				AwsS3Secret:             s.awsS3Secret,
 				AwsS3Region:             s.awsS3Region,
 				AwsS3Bucket:             s.awsS3Bucket,
 				namer:                   namer.NewNamer(),
 				Fee:                     channels[0].Fee,
 			}
 			shouldInterruptLoop = true
 		} else {
 			var queuesToSync []string
-			if s.CliFlags.Status != "" {
+			//TODO: implement scrambling to avoid starvation of queues
-				queuesToSync = append(queuesToSync, shared.StatusSyncing, s.CliFlags.Status)
+			if s.syncStatus != "" {
-			} else if s.CliFlags.SyncUpdate {
+				queuesToSync = append(queuesToSync, s.syncStatus)
-				queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced)
+			} else if s.syncUpdate {
 				queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
 			} else {
-				queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusQueued)
+				queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
 			}
 			if s.CliFlags.SecondaryStatus != "" {
 				queuesToSync = append(queuesToSync, s.CliFlags.SecondaryStatus)
 			}
 		queues:
 			for _, q := range queuesToSync {
-				channels, err := s.ApiConfig.FetchChannels(q, &s.CliFlags)
+				channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
 				if err != nil {
 					return err
 				}
-				log.Infof("Currently processing the \"%s\" queue with %d channels", q, len(channels))
+				log.Infof("There are %d channels in the \"%s\" queue", len(channels), q)
-				for _, c := range channels {
+				if len(channels) > 0 {
-					s.enqueueChannel(&c)
+					c := channels[0]
-					queueAll := q == shared.StatusFailed || q == shared.StatusSyncing
+					syncs = append(syncs, Sync{
-					if !queueAll {
+						APIConfig:               s.apiConfig,
-						break queues
+						YoutubeChannelID:        c.ChannelId,
 						LbryChannelName:         c.DesiredChannelName,
 						lbryChannelID:           c.ChannelClaimID,
 						StopOnError:             s.stopOnError,
 						MaxTries:                s.maxTries,
 						ConcurrentVideos:        s.concurrentVideos,
 						TakeOverExistingChannel: s.takeOverExistingChannel,
 						Refill:                  s.refill,
 						Manager:                 s,
 						LbrycrdString:           s.lbrycrdString,
 						AwsS3ID:                 s.awsS3ID,
 						AwsS3Secret:             s.awsS3Secret,
 						AwsS3Region:             s.awsS3Region,
 						AwsS3Bucket:             s.awsS3Bucket,
 						namer:                   namer.NewNamer(),
 						Fee:                     c.Fee,
 					})
 					break
 				}
 			}
 				log.Infof("Drained the \"%s\" queue", q)
 		}
-		}
+		if len(syncs) == 0 {
 		if len(s.channelsToSync) == 0 {
 			log.Infoln("No channels to sync. Pausing 5 minutes!")
 			time.Sleep(5 * time.Minute)
 		}
-		for _, sync := range s.channelsToSync {
+		for _, sync := range syncs {
 			if lastChannelProcessed == sync.DbChannelData.ChannelId && secondLastChannelProcessed == lastChannelProcessed {
 				util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
 				stopTheLoops := errors.Err("Found channel %s running 3 times, set it to failed, and reprocess later", sync.DbChannelData.DesiredChannelName)
 				sync.setChannelTerminationStatus(&stopTheLoops)
 				continue
 			}
 			secondLastChannelProcessed = lastChannelProcessed
 			lastChannelProcessed = sync.DbChannelData.ChannelId
 			shouldNotCount := false
-			logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
+			SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
 			err := sync.FullCycle()
 			//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
 			ipPool, _ := ip_manager.GetIPPool(sync.grp)
 			if ipPool != nil {
 				ipPool.ReleaseAll()
 			}
 			if err != nil {
 				if strings.Contains(err.Error(), "quotaExceeded") {
 					logUtils.SleepUntilQuotaReset()
 				}
 				fatalErrors := []string{
 					"default_wallet already exists",
 					"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
 					"NotEnoughFunds",
 					"no space left on device",
-					"there was a problem uploading the wallet",
+					"failure uploading wallet",
 					"the channel in the wallet is different than the channel in the database",
 					"this channel does not belong to this wallet!",
 					"You already have a stream claim published under the name",
 				}
 				if util.SubstringInSlice(err.Error(), fatalErrors) {
 					return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
 				}
 				shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
 				if !shouldNotCount {
-					logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err))
+					SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
 				}
 			}
-			err = logUtils.CleanupMetadata()
+			SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
 			if err != nil {
 				log.Errorf("something went wrong while trying to clear out the video metadata directory: %s", errors.FullTrace(err))
 			}
 			err = blobs_reflector.ReflectAndClean()
 			if err != nil {
 				return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
 			}
 			logUtils.SendInfoToSlack("%s (%s) reached an end. Total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
 			if !shouldNotCount {
 				syncCount++
 			}
-			if sync.IsInterrupted() || (s.CliFlags.Limit != 0 && syncCount >= s.CliFlags.Limit) {
+			if sync.IsInterrupted() || (s.limit != 0 && syncCount >= s.limit) {
 				shouldInterruptLoop = true
 				break
 			}
 		}
-		if shouldInterruptLoop || s.CliFlags.SingleRun {
+		if shouldInterruptLoop || s.singleRun {
 			break
 		}
 	}
 	return nil
 }
-
+func (s *SyncManager) GetS3AWSConfig() aws.Config {
-func (s *SyncManager) checkUsedSpace() error {
+	return aws.Config{
-	usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir())
+		Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
-	if err != nil {
+		Region:      &s.awsS3Region,
 		return errors.Err(err)
 	}
-	if usedPctile >= 0.90 && !s.CliFlags.SkipSpaceCheck {
+}
 func (s *SyncManager) checkUsedSpace() error {
 	usedPctile, err := GetUsedSpace(s.blobsDir)
 	if err != nil {
 		return err
 	}
 	if usedPctile >= 0.90 && !s.skipSpaceCheck {
 		return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
 	}
 	log.Infof("disk usage: %.1f%%", usedPctile*100)
--- a/manager/s3_storage.go
+++ b/manager/s3_storage.go
@ -1,285 +0,0 @@
 package manager
 import (
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 	"github.com/lbryio/ytsync/v5/configs"
 	"github.com/lbryio/ytsync/v5/util"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	log "github.com/sirupsen/logrus"
 )
 func (s *Sync) getS3Downloader(config *aws.Config) (*s3manager.Downloader, error) {
 	s3Session, err := session.NewSession(config)
 	if err != nil {
 		return nil, errors.Prefix("error starting session", err)
 	}
 	downloader := s3manager.NewDownloader(s3Session)
 	return downloader, nil
 }
 func (s *Sync) getS3Uploader(config *aws.Config) (*s3manager.Uploader, error) {
 	s3Session, err := session.NewSession(config)
 	if err != nil {
 		return nil, errors.Prefix("error starting session", err)
 	}
 	uploader := s3manager.NewUploader(s3Session)
 	return uploader, nil
 }
 func (s *Sync) downloadWallet() error {
 	defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
 	if err != nil {
 		return errors.Err(err)
 	}
 	downloader, err := s.getS3Downloader(configs.Configuration.WalletS3Config.GetS3AWSConfig())
 	if err != nil {
 		return err
 	}
 	out, err := os.Create(defaultTempWalletDir)
 	if err != nil {
 		return errors.Prefix("error creating temp wallet", err)
 	}
 	defer out.Close()
 	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
 		Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
 		Key:    key,
 	})
 	if err != nil {
 		// Casting to the awserr.Error type will allow you to inspect the error
 		// code returned by the service in code. The error code can be used
 		// to switch on context specific functionality. In this case a context
 		// specific error message is printed to the user based on the bucket
 		// and key existing.
 		//
 		// For information on other S3 API error codes see:
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 		if aerr, ok := err.(awserr.Error); ok {
 			code := aerr.Code()
 			if code == s3.ErrCodeNoSuchKey {
 				return errors.Err("wallet not on S3")
 			}
 		}
 		return err
 	} else if bytesWritten == 0 {
 		return errors.Err("zero bytes written")
 	}
 	err = os.Rename(defaultTempWalletDir, defaultWalletDir)
 	if err != nil {
 		return errors.Prefix("error replacing temp wallet for default wallet", err)
 	}
 	return nil
 }
 func (s *Sync) downloadBlockchainDB() error {
 	if util.IsRegTest() {
 		return nil // tests fail if we re-use the same blockchain DB
 	}
 	defaultBDBPath, defaultTempBDBPath, key, err := s.getBlockchainDBPaths()
 	if err != nil {
 		return errors.Err(err)
 	}
 	files, err := filepath.Glob(defaultBDBPath + "*")
 	if err != nil {
 		return errors.Err(err)
 	}
 	for _, f := range files {
 		err = os.Remove(f)
 		if err != nil {
 			return errors.Err(err)
 		}
 	}
 	if s.DbChannelData.WipeDB {
 		return nil
 	}
 	downloader, err := s.getS3Downloader(configs.Configuration.BlockchaindbS3Config.GetS3AWSConfig())
 	if err != nil {
 		return errors.Err(err)
 	}
 	out, err := os.Create(defaultTempBDBPath)
 	if err != nil {
 		return errors.Prefix("error creating temp blockchain DB file", err)
 	}
 	defer out.Close()
 	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
 		Bucket: aws.String(configs.Configuration.BlockchaindbS3Config.Bucket),
 		Key:    key,
 	})
 	if err != nil {
 		// Casting to the awserr.Error type will allow you to inspect the error
 		// code returned by the service in code. The error code can be used
 		// to switch on context specific functionality. In this case a context
 		// specific error message is printed to the user based on the bucket
 		// and key existing.
 		//
 		// For information on other S3 API error codes see:
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 		if aerr, ok := err.(awserr.Error); ok {
 			code := aerr.Code()
 			if code == s3.ErrCodeNoSuchKey {
 				return nil // let ytsync sync the database by itself
 			}
 		}
 		return errors.Err(err)
 	} else if bytesWritten == 0 {
 		return errors.Err("zero bytes written")
 	}
 	blockchainDbDir := strings.Replace(defaultBDBPath, "blockchain.db", "", -1)
 	err = util.Untar(defaultTempBDBPath, blockchainDbDir)
 	if err != nil {
 		return errors.Prefix("error extracting blockchain.db files", err)
 	}
 	err = os.Remove(defaultTempBDBPath)
 	if err != nil {
 		return errors.Err(err)
 	}
 	log.Printf("blockchain.db data downloaded and extracted to %s", blockchainDbDir)
 	return nil
 }
 func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
 	defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
 	tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
 	key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
 	if util.IsRegTest() {
 		defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
 		tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
 		key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
 	}
 	lbryumDir := os.Getenv("LBRYUM_DIR")
 	if lbryumDir != "" {
 		defaultWallet = lbryumDir + "/wallets/default_wallet"
 		tempWallet = lbryumDir + "/wallets/tmp_wallet"
 	}
 	if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
 		return "", "", nil, errors.Err("default_wallet already exists")
 	}
 	return
 }
 func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
 	lbryumDir := os.Getenv("LBRYUM_DIR")
 	if lbryumDir == "" {
 		if util.IsRegTest() {
 			lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
 		} else {
 			lbryumDir = os.Getenv("HOME") + "/.lbryum"
 		}
 	}
 	defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
 	tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.tar"
 	key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId + ".tar")
 	if util.IsRegTest() {
 		defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
 		tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.tar"
 		key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId + ".tar")
 	}
 	return
 }
 func (s *Sync) uploadWallet() error {
 	defaultWalletDir := util.GetDefaultWalletPath()
 	key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
 	if util.IsRegTest() {
 		key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
 	}
 	if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
 		return errors.Err("default_wallet does not exist")
 	}
 	uploader, err := s.getS3Uploader(configs.Configuration.WalletS3Config.GetS3AWSConfig())
 	if err != nil {
 		return err
 	}
 	file, err := os.Open(defaultWalletDir)
 	if err != nil {
 		return err
 	}
 	defer file.Close()
 	start := time.Now()
 	for time.Since(start) < 30*time.Minute {
 		_, err = uploader.Upload(&s3manager.UploadInput{
 			Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
 			Key:    key,
 			Body:   file,
 		})
 		if err != nil {
 			time.Sleep(30 * time.Second)
 			continue
 		}
 		break
 	}
 	if err != nil {
 		return errors.Prefix("there was a problem uploading the wallet to S3", errors.Err(err))
 	}
 	log.Println("wallet uploaded to S3")
 	return os.Remove(defaultWalletDir)
 }
 func (s *Sync) uploadBlockchainDB() error {
 	defaultBDBDir, _, key, err := s.getBlockchainDBPaths()
 	if err != nil {
 		return errors.Err(err)
 	}
 	if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
 		return errors.Err("blockchain.db does not exist")
 	}
 	files, err := filepath.Glob(defaultBDBDir + "*")
 	if err != nil {
 		return errors.Err(err)
 	}
 	tarPath := strings.Replace(defaultBDBDir, "blockchain.db", "", -1) + s.DbChannelData.ChannelId + ".tar"
 	err = util.CreateTarball(tarPath, files)
 	if err != nil {
 		return err
 	}
 	uploader, err := s.getS3Uploader(configs.Configuration.BlockchaindbS3Config.GetS3AWSConfig())
 	if err != nil {
 		return err
 	}
 	file, err := os.Open(tarPath)
 	if err != nil {
 		return err
 	}
 	defer file.Close()
 	_, err = uploader.Upload(&s3manager.UploadInput{
 		Bucket: aws.String(configs.Configuration.BlockchaindbS3Config.Bucket),
 		Key:    key,
 		Body:   file,
 	})
 	if err != nil {
 		return err
 	}
 	log.Println("blockchain.db files uploaded to S3")
 	err = os.Remove(tarPath)
 	if err != nil {
 		return errors.Err(err)
 	}
 	return os.Remove(defaultBDBDir)
 }
--- a/manager/setup.go
+++ b/manager/setup.go
@ -3,41 +3,34 @@ package manager
 import (
 	"fmt"
 	"math"
 	"net/http"
 	"os"
 	"strconv"
 	"strings"
 	"time"
-	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/extras/errors"
-	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+	"github.com/lbryio/lbry.go/extras/jsonrpc"
-	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/lbry.go/extras/util"
-	"github.com/lbryio/ytsync/v5/shared"
+	"github.com/lbryio/lbry.go/lbrycrd"
 	"github.com/lbryio/ytsync/v5/timing"
 	logUtils "github.com/lbryio/ytsync/v5/util"
 	"github.com/lbryio/ytsync/v5/ytapi"
-	"github.com/lbryio/ytsync/v5/tags_manager"
+	"github.com/lbryio/ytsync/tagsManager"
-	"github.com/lbryio/ytsync/v5/thumbs"
+	"github.com/lbryio/ytsync/thumbs"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/api/googleapi/transport"
 	"google.golang.org/api/youtube/v3"
 )
 func (s *Sync) enableAddressReuse() error {
-	accountsResponse, err := s.daemon.AccountList(1, 50)
+	accountsResponse, err := s.daemon.AccountList()
 	if err != nil {
 		return errors.Err(err)
 	}
-	accounts := make([]jsonrpc.Account, 0, len(accountsResponse.Items))
+	accounts := accountsResponse.LBCMainnet
-	ledger := "lbc_mainnet"
+	if os.Getenv("REGTEST") == "true" {
-	if logUtils.IsRegTest() {
+		accounts = accountsResponse.LBCRegtest
 		ledger = "lbc_regtest"
 	}
 	for _, a := range accountsResponse.Items {
 		if *a.Ledger == ledger {
 			accounts = append(accounts, a)
 		}
 	}
 	for _, a := range accounts {
 		_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
 			ChangeMaxUses:    util.PtrToInt(1000),
@ -50,10 +43,6 @@ func (s *Sync) enableAddressReuse() error {
 	return nil
 }
 func (s *Sync) walletSetup() error {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("walletSetup").Add(time.Since(start))
 	}(start)
 	//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
 	s.walletMux.Lock()
 	defer s.walletMux.Unlock()
@ -68,13 +57,17 @@ func (s *Sync) walletSetup() error {
 	} else if balanceResp == nil {
 		return errors.Err("no response")
 	}
-	balance, err := strconv.ParseFloat(balanceResp.Available.String(), 64)
+	balance, err := strconv.ParseFloat((string)(*balanceResp), 64)
 	if err != nil {
 		return errors.Err(err)
 	}
 	log.Debugf("Starting balance is %.4f", balance)
-	videosOnYoutube := int(s.DbChannelData.TotalVideos)
+	n, err := s.CountVideos()
 	if err != nil {
 		return err
 	}
 	videosOnYoutube := int(n)
 	log.Debugf("Source channel has %d videos", videosOnYoutube)
 	if videosOnYoutube == 0 {
@ -97,32 +90,24 @@ func (s *Sync) walletSetup() error {
 	}
 	s.syncedVideosMux.RUnlock()
-	log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
+	log.Debugf("We already allocated credits for %d videos", publishedCount)
-	if videosOnYoutube > s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers) {
+	if videosOnYoutube > s.Manager.videosLimit {
-		videosOnYoutube = s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers)
+		videosOnYoutube = s.Manager.videosLimit
 	}
-	unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
+	unallocatedVideos := videosOnYoutube - publishedCount
-	if unallocatedVideos < 0 {
+	requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
-		unallocatedVideos = 0
+	if s.Manager.upgradeMetadata {
-	}
+		requiredBalance += float64(notUpgradedCount) * 0.001
 	channelFee := channelClaimAmount
 	channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
 	if channelAlreadyClaimed {
 		channelFee = 0.0
 	}
 	requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
 	if s.Manager.CliFlags.UpgradeMetadata {
 		requiredBalance += float64(notUpgradedCount) * estimatedMaxTxFee
 	}
 	refillAmount := 0.0
 	if balance < requiredBalance || balance < minimumAccountBalance {
-		refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
+		refillAmount = math.Max(requiredBalance-requiredBalance, minimumRefillAmount)
 	}
-	if s.Manager.CliFlags.Refill > 0 {
+	if s.Refill > 0 {
-		refillAmount += float64(s.Manager.CliFlags.Refill)
+		refillAmount += float64(s.Refill)
 	}
 	if refillAmount > 0 {
@ -130,25 +115,16 @@ func (s *Sync) walletSetup() error {
 		if err != nil {
 			return errors.Err(err)
 		}
 	} else if balance > requiredBalance {
 		extraLBC := balance - requiredBalance
 		if extraLBC > 5 {
 			sendBackAmount := extraLBC - 1
 			logUtils.SendInfoToSlack("channel %s has %.1f credits which is %.1f more than it requires (%.1f). We should send at least %.1f that back.", s.DbChannelData.ChannelId, balance, extraLBC, requiredBalance, sendBackAmount)
 		}
 	}
-	claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
+	claimAddress, err := s.daemon.AddressList(nil)
 	if err != nil {
 		return err
 	} else if claimAddress == nil {
-		return errors.Err("could not get an address")
+		return errors.Err("could not get unused address")
 	}
-	if s.DbChannelData.PublishAddress.Address == "" || !s.shouldTransfer() {
+	s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
-		s.DbChannelData.PublishAddress.Address = string(claimAddress.Items[0].Address)
+	if s.claimAddress == "" {
 		s.DbChannelData.PublishAddress.IsMine = true
 	}
 	if s.DbChannelData.PublishAddress.Address == "" {
 		return errors.Err("found blank claim address")
 	}
@ -160,47 +136,27 @@ func (s *Sync) walletSetup() error {
 	return nil
 }
-func (s *Sync) getDefaultAccount() (string, error) {
+func (s *Sync) ensureEnoughUTXOs() error {
-	start := time.Now()
+	accounts, err := s.daemon.AccountList()
 	defer func(start time.Time) {
 		timing.TimedComponent("getDefaultAccount").Add(time.Since(start))
 	}(start)
 	if s.defaultAccountID == "" {
 		accountsResponse, err := s.daemon.AccountList(1, 50)
 	if err != nil {
-			return "", errors.Err(err)
+		return errors.Err(err)
 	}
-		ledger := "lbc_mainnet"
+	accountsNet := (*accounts).LBCMainnet
-		if logUtils.IsRegTest() {
+	if os.Getenv("REGTEST") == "true" {
-			ledger = "lbc_regtest"
+		accountsNet = (*accounts).LBCRegtest
 	}
-		for _, a := range accountsResponse.Items {
+	defaultAccount := ""
-			if *a.Ledger == ledger {
+	for _, account := range accountsNet {
-				if a.IsDefault {
+		if account.IsDefault {
-					s.defaultAccountID = a.ID
+			defaultAccount = account.ID
 			break
 		}
 	}
 	if defaultAccount == "" {
 		return errors.Err("No default account found")
 	}
-		if s.defaultAccountID == "" {
+	utxolist, err := s.daemon.UTXOList(&defaultAccount)
 			return "", errors.Err("No default account found")
 		}
 	}
 	return s.defaultAccountID, nil
 }
 func (s *Sync) ensureEnoughUTXOs() error {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("ensureEnoughUTXOs").Add(time.Since(start))
 	}(start)
 	defaultAccount, err := s.getDefaultAccount()
 	if err != nil {
 		return err
 	}
 	utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
 	if err != nil {
 		return err
 	} else if utxolist == nil {
@ -210,19 +166,14 @@ func (s *Sync) ensureEnoughUTXOs() error {
 	target := 40
 	slack := int(float32(0.1) * float32(target))
 	count := 0
 	confirmedCount := 0
-	for _, utxo := range utxolist.Items {
+	for _, utxo := range *utxolist {
 		amount, _ := strconv.ParseFloat(utxo.Amount, 64)
-		if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
+		if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && amount != 0.0 {
 			if utxo.Confirmations > 0 {
 				confirmedCount++
 			}
 			count++
 		}
 	}
-	log.Infof("utxo count: %d (%d confirmed)", count, confirmedCount)
+	log.Infof("utxo count: %d", count)
 	UTXOWaitThreshold := 16
 	if count < target-slack {
 		balance, err := s.daemon.AccountBalance(&defaultAccount)
 		if err != nil {
@ -231,29 +182,16 @@ func (s *Sync) ensureEnoughUTXOs() error {
 			return errors.Err("no response")
 		}
-		balanceAmount, err := strconv.ParseFloat(balance.Available.String(), 64)
+		balanceAmount, err := strconv.ParseFloat((string)(*balance), 64)
 		if err != nil {
 			return errors.Err(err)
 		}
 		//this is dumb but sometimes the balance is negative and it breaks everything, so let's check again
 		if balanceAmount < 0 {
 			log.Infof("negative balance of %.2f found. Waiting to retry...", balanceAmount)
 			time.Sleep(10 * time.Second)
 			balanceAmount, err = strconv.ParseFloat(balance.Available.String(), 64)
 			if err != nil {
 				return errors.Err(err)
 			}
 		}
 		maxUTXOs := uint64(500)
 		desiredUTXOCount := uint64(math.Floor((balanceAmount) / 0.1))
 		if desiredUTXOCount > maxUTXOs {
 			desiredUTXOCount = maxUTXOs
 		}
-		if desiredUTXOCount < uint64(confirmedCount) {
+		log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
 			return nil
 		}
 		availableBalance, _ := balance.Available.Float64()
 		log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
 		broadcastFee := 0.1
 		prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
@ -262,13 +200,12 @@ func (s *Sync) ensureEnoughUTXOs() error {
 		} else if prefillTx == nil {
 			return errors.Err("no response")
 		}
-		if confirmedCount < UTXOWaitThreshold {
+
 		err = s.waitForNewBlock()
 		if err != nil {
 			return err
 		}
-		}
+	} else if !allUTXOsConfirmed(utxolist) {
 	} else if confirmedCount < UTXOWaitThreshold {
 		log.Println("Waiting for previous txns to confirm")
 		err := s.waitForNewBlock()
 		if err != nil {
@ -280,9 +217,6 @@ func (s *Sync) ensureEnoughUTXOs() error {
 }
 func (s *Sync) waitForNewBlock() error {
 	defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
 	log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
 	status, err := s.daemon.Status()
 	if err != nil {
 		return err
@ -295,91 +229,72 @@ func (s *Sync) waitForNewBlock() error {
 			return err
 		}
 	}
 	currentBlock := status.Wallet.Blocks
 	for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
 		if i%3 == 0 {
 			log.Printf("Waiting for new block (%d)...", currentBlock+1)
 		}
 		if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
 			err = s.GenerateRegtestBlock()
 			if err != nil {
 				return err
 			}
 		}
 		time.Sleep(10 * time.Second)
 		status, err = s.daemon.Status()
 		if err != nil {
 			return err
 		}
 	}
 	time.Sleep(5 * time.Second)
 	return nil
 }
 func (s *Sync) GenerateRegtestBlock() error {
 	lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
 	if err != nil {
 		return errors.Prefix("error getting lbrycrd client", err)
 	}
 	txs, err := lbrycrd.Generate(1)
 	if err != nil {
 		return errors.Prefix("error generating new block", err)
 	}
 	for _, tx := range txs {
 		log.Info("Generated tx: ", tx.String())
 	}
 	return nil
 }
 func (s *Sync) ensureChannelOwnership() error {
-	defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
+	if s.LbryChannelName == "" {
 	if s.DbChannelData.DesiredChannelName == "" {
 		return errors.Err("no channel name set")
 	}
-
+	//@TODO: get rid of this when imported channels are supported
-	channels, err := s.daemon.ChannelList(nil, 1, 500, nil)
+	if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
 		return nil
 	}
 	channels, err := s.daemon.ChannelList(nil, 1, 50)
 	if err != nil {
 		return err
 	} else if channels == nil {
 		return errors.Err("no channel response")
 	}
-
+	//special case for wallets we don't retain full control anymore
-	var channelToUse *jsonrpc.Transaction
+	if len((*channels).Items) > 1 {
-	if len((*channels).Items) > 0 {
+		// This wallet is probably not under our control anymore but we still want to publish to it
-		if s.DbChannelData.ChannelClaimID == "" {
+		// here we shall check if within all the channels there is one that was created by ytsync
 		SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
 		if s.lbryChannelID == "" {
 			return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
 		}
 		for _, c := range (*channels).Items {
-			log.Debugf("checking listed channel %s (%s)", c.ClaimID, c.Name)
+			if c.ClaimID != s.lbryChannelID {
-			if c.ClaimID != s.DbChannelData.ChannelClaimID {
+				if c.Name != s.LbryChannelName {
 				continue
 			}
 			if c.Name != s.DbChannelData.DesiredChannelName {
 					return errors.Err("the channel in the wallet is different than the channel in the database")
 				}
-			channelToUse = &c
+				return nil // we have the ytsync channel and both the claimID and the channelName from the database are correct
 			break
 			}
 		if channelToUse == nil {
 			return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.DbChannelData.ChannelClaimID, s.DbChannelData.DesiredChannelName)
 		}
-	} else if s.DbChannelData.TransferState == shared.TransferStateComplete {
+	}
-		return errors.Err("the channel was transferred but appears to have been abandoned!")
+	channelUsesOldMetadata := false
-	} else if s.DbChannelData.ChannelClaimID != "" {
+	if len((*channels).Items) == 1 {
-		return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.DbChannelData.ChannelClaimID)
+		channel := ((*channels).Items)[0]
 		if channel.Name == s.LbryChannelName {
 			channelUsesOldMetadata = channel.Value.GetThumbnail() == nil
 			//TODO: eventually get rid of this when the whole db is filled
 			if s.lbryChannelID == "" {
 				err = s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, channel.ClaimID)
 			} else if channel.ClaimID != s.lbryChannelID {
 				return errors.Err("the channel in the wallet is different than the channel in the database")
 			}
 			s.lbryChannelID = channel.ClaimID
 			if !channelUsesOldMetadata {
 				return err
 			}
 		} else {
 			return errors.Err("this channel does not belong to this wallet! Expected: %s, found: %s", s.LbryChannelName, channel.Name)
 		}
 	}
-	channelUsesOldMetadata := false
+	channelBidAmount := channelClaimAmount
 	if channelToUse != nil {
 		channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.GetLanguages()) == 0 && s.DbChannelData.Language != "")
 		if !channelUsesOldMetadata {
 			return nil
 		}
 	}
 	balanceResp, err := s.daemon.AccountBalance(nil)
 	if err != nil {
@ -387,43 +302,47 @@ func (s *Sync) ensureChannelOwnership() error {
 	} else if balanceResp == nil {
 		return errors.Err("no response")
 	}
-	balance, err := decimal.NewFromString(balanceResp.Available.String())
+	balance, err := decimal.NewFromString((string)(*balanceResp))
 	if err != nil {
 		return errors.Err(err)
 	}
-	if balance.LessThan(decimal.NewFromFloat(channelClaimAmount)) {
+	if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
-		err = s.addCredits(channelClaimAmount + estimatedMaxTxFee*3)
+		err = s.addCredits(channelBidAmount + 0.1)
 		if err != nil {
 			return err
 		}
 	}
 	client := &http.Client{
 		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
 	}
-	channelInfo, err := ytapi.ChannelInfo(s.DbChannelData.ChannelId)
+	service, err := youtube.New(client)
 	if err != nil {
-		if strings.Contains(err.Error(), "invalid character 'e' looking for beginning of value") {
+		return errors.Prefix("error creating YouTube service", err)
 			logUtils.SendInfoToSlack("failed to get channel data for %s. Waiting 1 minute to retry", s.DbChannelData.ChannelId)
 			time.Sleep(1 * time.Minute)
 			channelInfo, err = ytapi.ChannelInfo(s.DbChannelData.ChannelId)
 			if err != nil {
 				return err
 			}
 		} else {
 			return err
 		}
 	}
-	thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
+	response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
-	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId)
+	if err != nil {
 		return errors.Prefix("error getting channel details", err)
 	}
 	if len(response.Items) < 1 {
 		return errors.Err("youtube channel not found")
 	}
 	channelInfo := response.Items[0].Snippet
 	channelBranding := response.Items[0].BrandingSettings
 	thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
 	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
 	if err != nil {
 		return err
 	}
 	var bannerURL *string
-	if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
+	if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
-		bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
+		bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
 			"banner-"+s.DbChannelData.ChannelId,
 		)
 		if err != nil {
 			return err
 		}
@ -431,108 +350,81 @@ func (s *Sync) ensureChannelOwnership() error {
 	}
 	var languages []string = nil
-	if s.DbChannelData.Language != "" {
+	if channelInfo.DefaultLanguage != "" {
-		languages = []string{s.DbChannelData.Language}
+		languages = []string{channelInfo.DefaultLanguage}
 	}
 	var locations []jsonrpc.Location = nil
-	if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
+	if channelInfo.Country != "" {
-		locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
+		locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
 	}
 	var c *jsonrpc.TransactionSummary
 	var recoveredChannelClaimID string
 	claimCreateOptions := jsonrpc.ClaimCreateOptions{
-		Title:        &channelInfo.Microformat.MicroformatDataRenderer.Title,
+		Title:        &channelInfo.Title,
-		Description:  &channelInfo.Metadata.ChannelMetadataRenderer.Description,
+		Description:  &channelInfo.Description,
-		Tags:         tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId),
+		Tags:         tagsManager.GetTagsForChannel(s.YoutubeChannelID),
 		Languages:    languages,
 		Locations:    locations,
 		ThumbnailURL: &thumbnailURL,
 	}
 	if channelUsesOldMetadata {
-		da, err := s.getDefaultAccount()
+		c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
 		if err != nil {
 			return err
 		}
 		if s.DbChannelData.TransferState <= 1 {
 			c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
 			ClearTags:      util.PtrToBool(true),
 			ClearLocations: util.PtrToBool(true),
 			ClearLanguages: util.PtrToBool(true),
 			ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
 					AccountID: &da,
 					FundingAccountIDs: []string{
 						da,
 					},
 				ClaimCreateOptions: claimCreateOptions,
 				CoverURL:           bannerURL,
 			},
 		})
 	} else {
-			logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.DbChannelData.DesiredChannelName, s.DbChannelData.ChannelClaimID)
+		c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
 			return nil
 		}
 	} else {
 		c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelClaimAmount, jsonrpc.ChannelCreateOptions{
 			ClaimCreateOptions: claimCreateOptions,
 			CoverURL:           bannerURL,
 		})
 		if err != nil {
 			claimId, err2 := s.getChannelClaimIDForTimedOutCreation()
 			if err2 != nil {
 				err = errors.Prefix(err2.Error(), err)
 			} else {
 				recoveredChannelClaimID = claimId
 			}
 		}
 	}
 	if err != nil {
 		return err
 	}
-	if recoveredChannelClaimID != "" {
+	s.lbryChannelID = c.Outputs[0].ClaimID
-		s.DbChannelData.ChannelClaimID = recoveredChannelClaimID
+	return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
 	} else {
 		s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
 	}
 	return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
 }
-//getChannelClaimIDForTimedOutCreation is a raw function that returns the only channel that exists in the wallet
+func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool {
-// this is used because the SDK sucks and can't figure out when to return when creating a claim...
+	if utxolist == nil {
-func (s *Sync) getChannelClaimIDForTimedOutCreation() (string, error) {
+		return false
 	channels, err := s.daemon.ChannelList(nil, 1, 500, nil)
 	if err != nil {
 		return "", err
 	} else if channels == nil {
 		return "", errors.Err("no channel response")
 	}
 	if len((*channels).Items) != 1 {
 		return "", errors.Err("more than one channel found when trying to recover from SDK failure in creating the channel")
 	}
 	desiredChannel := (*channels).Items[0]
 	if desiredChannel.Name != s.DbChannelData.DesiredChannelName {
 		return "", errors.Err("the channel found in the wallet has a different name than the one we expected")
 	}
-	return desiredChannel.ClaimID, nil
+	if len(*utxolist) < 1 {
 		return false
 	}
 	for _, utxo := range *utxolist {
 		if utxo.Height == 0 {
 			return false
 		}
 	}
 	return true
 }
 func (s *Sync) addCredits(amountToAdd float64) error {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("addCredits").Add(time.Since(start))
 	}(start)
 	log.Printf("Adding %f credits", amountToAdd)
-	lbrycrdd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
+	var lbrycrdd *lbrycrd.Client
 	var err error
 	if s.LbrycrdString == "" {
 		lbrycrdd, err = lbrycrd.NewWithDefaultURL()
 		if err != nil {
 			return err
 		}
 	} else {
 		lbrycrdd, err = lbrycrd.New(s.LbrycrdString)
 		if err != nil {
 			return err
 		}
 	}
-	defaultAccount, err := s.getDefaultAccount()
+	addressResp, err := s.daemon.AddressUnused(nil)
 	if err != nil {
 		return err
 	}
 	addressResp, err := s.daemon.AddressUnused(&defaultAccount)
 	if err != nil {
 		return err
 	} else if addressResp == nil {
--- a/manager/transfer.go
+++ b/manager/transfer.go
@ -1,349 +0,0 @@
 package manager
 import (
 	"fmt"
 	"strconv"
 	"strings"
 	"time"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/timing"
 	log "github.com/sirupsen/logrus"
 )
 func waitConfirmations(s *Sync) error {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("waitConfirmations").Add(time.Since(start))
 	}(start)
 	defaultAccount, err := s.getDefaultAccount()
 	if err != nil {
 		return err
 	}
 	allConfirmed := false
 	waitCount := 0
 waiting:
 	for !allConfirmed && waitCount < 2 {
 		utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
 		if err != nil {
 			return err
 		} else if utxolist == nil {
 			return errors.Err("no response")
 		}
 		for _, utxo := range utxolist.Items {
 			if utxo.Confirmations <= 0 {
 				err = s.waitForNewBlock()
 				if err != nil {
 					return err
 				}
 				waitCount++
 				continue waiting
 			}
 		}
 		allConfirmed = true
 	}
 	return nil
 }
 type abandonResponse struct {
 	ClaimID string
 	Error   error
 	Amount  float64
 }
 func abandonSupports(s *Sync) (float64, error) {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("abandonSupports").Add(time.Since(start))
 	}(start)
 	totalPages := uint64(1)
 	var allSupports []jsonrpc.Claim
 	defaultAccount, err := s.getDefaultAccount()
 	if err != nil {
 		return 0, err
 	}
 	for page := uint64(1); page <= totalPages; page++ {
 		supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
 		if err != nil {
 			supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
 			if err != nil {
 				return 0, errors.Prefix("cannot list supports", err)
 			}
 		}
 		allSupports = append(allSupports, (*supports).Items...)
 		totalPages = (*supports).TotalPages
 	}
 	producerWG := &stop.Group{}
 	claimIDChan := make(chan string, len(allSupports))
 	abandonRspChan := make(chan abandonResponse, len(allSupports))
 	alreadyAbandoned := make(map[string]bool, len(allSupports))
 	producerWG.Add(1)
 	go func() {
 		defer producerWG.Done()
 		for _, support := range allSupports {
 			_, ok := alreadyAbandoned[support.ClaimID]
 			if ok {
 				continue
 			}
 			alreadyAbandoned[support.ClaimID] = true
 			claimIDChan <- support.ClaimID
 		}
 	}()
 	consumerWG := &stop.Group{}
 	//TODO: remove this once the SDK team fixes their RPC bugs....
 	s.daemon.SetRPCTimeout(60 * time.Second)
 	defer s.daemon.SetRPCTimeout(5 * time.Minute)
 	for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
 		consumerWG.Add(1)
 		go func() {
 			defer consumerWG.Done()
 		outer:
 			for {
 				claimID, more := <-claimIDChan
 				if !more {
 					return
 				} else {
 					summary, err := s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
 					if err != nil {
 						if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
 							log.Errorf("Support abandon for %s timed out, retrying...", claimID)
 							summary, err = s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
 							if err != nil {
 								//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
 								abandonRspChan <- abandonResponse{
 									ClaimID: claimID,
 									Error:   err,
 									Amount:  0, // this is likely wrong, but oh well... there is literally nothing I can do about it
 								}
 								continue
 							}
 						} else {
 							abandonRspChan <- abandonResponse{
 								ClaimID: claimID,
 								Error:   err,
 								Amount:  0,
 							}
 							continue
 						}
 					}
 					if summary == nil || len(*summary) < 1 {
 						abandonRspChan <- abandonResponse{
 							ClaimID: claimID,
 							Error:   errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
 							Amount:  0,
 						}
 						continue
 					}
 					var outputAmount float64
 					for _, tx := range *summary {
 						amount, err := strconv.ParseFloat(tx.Outputs[0].Amount, 64)
 						if err != nil {
 							abandonRspChan <- abandonResponse{
 								ClaimID: claimID,
 								Error:   errors.Err(err),
 								Amount:  0,
 							}
 							continue outer
 						}
 						outputAmount += amount
 					}
 					if err != nil {
 						abandonRspChan <- abandonResponse{
 							ClaimID: claimID,
 							Error:   errors.Err(err),
 							Amount:  0,
 						}
 						continue
 					}
 					log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
 					abandonRspChan <- abandonResponse{
 						ClaimID: claimID,
 						Error:   nil,
 						Amount:  outputAmount,
 					}
 					continue
 				}
 			}
 		}()
 	}
 	producerWG.Wait()
 	close(claimIDChan)
 	consumerWG.Wait()
 	close(abandonRspChan)
 	totalAbandoned := 0.0
 	for r := range abandonRspChan {
 		if r.Error != nil {
 			log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
 			continue
 		}
 		totalAbandoned += r.Amount
 	}
 	return totalAbandoned, nil
 }
 type updateInfo struct {
 	ClaimID             string
 	streamUpdateOptions *jsonrpc.StreamUpdateOptions
 	videoStatus         *shared.VideoStatus
 }
 func transferVideos(s *Sync) error {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("transferVideos").Add(time.Since(start))
 	}(start)
 	cleanTransfer := true
 	streamChan := make(chan updateInfo, s.Manager.CliFlags.ConcurrentJobs)
 	account, err := s.getDefaultAccount()
 	if err != nil {
 		return err
 	}
 	streams, err := s.daemon.StreamList(&account, 1, 30000)
 	if err != nil {
 		return errors.Err(err)
 	}
 	producerWG := &stop.Group{}
 	producerWG.Add(1)
 	go func() {
 		defer producerWG.Done()
 		for _, video := range s.syncedVideos {
 			if !video.Published || video.Transferred || video.MetadataVersion != shared.LatestMetadataVersion {
 				continue
 			}
 			var stream *jsonrpc.Claim = nil
 			for _, c := range streams.Items {
 				if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.DbChannelData.ChannelClaimID) {
 					continue
 				}
 				stream = &c
 				break
 			}
 			if stream == nil {
 				return
 			}
 			streamUpdateOptions := jsonrpc.StreamUpdateOptions{
 				StreamCreateOptions: &jsonrpc.StreamCreateOptions{
 					ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
 						ClaimAddress: &s.DbChannelData.PublishAddress.Address,
 						FundingAccountIDs: []string{
 							account,
 						},
 					},
 				},
 				Bid: util.PtrToString(fmt.Sprintf("%.5f", publishAmount/2.)),
 			}
 			videoStatus := shared.VideoStatus{
 				ChannelID:     s.DbChannelData.ChannelId,
 				VideoID:       video.VideoID,
 				ClaimID:       video.ClaimID,
 				ClaimName:     video.ClaimName,
 				Status:        shared.VideoStatusPublished,
 				IsTransferred: util.PtrToBool(true),
 			}
 			streamChan <- updateInfo{
 				ClaimID:             video.ClaimID,
 				streamUpdateOptions: &streamUpdateOptions,
 				videoStatus:         &videoStatus,
 			}
 		}
 	}()
 	consumerWG := &stop.Group{}
 	for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
 		consumerWG.Add(1)
 		go func(worker int) {
 			defer consumerWG.Done()
 			for {
 				ui, more := <-streamChan
 				if !more {
 					return
 				} else {
 					err := s.streamUpdate(&ui)
 					if err != nil {
 						cleanTransfer = false
 					}
 				}
 			}
 		}(i)
 	}
 	producerWG.Wait()
 	close(streamChan)
 	consumerWG.Wait()
 	if !cleanTransfer {
 		return errors.Err("A video has failed to transfer for the channel...skipping channel transfer")
 	}
 	return nil
 }
 func (s *Sync) streamUpdate(ui *updateInfo) error {
 	start := time.Now()
 	result, updateError := s.daemon.StreamUpdate(ui.ClaimID, *ui.streamUpdateOptions)
 	timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
 	if updateError != nil {
 		ui.videoStatus.FailureReason = updateError.Error()
 		ui.videoStatus.Status = shared.VideoStatusTransferFailed
 		ui.videoStatus.IsTransferred = util.PtrToBool(false)
 	} else {
 		ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
 	}
 	log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
 	statusErr := s.Manager.ApiConfig.MarkVideoStatus(*ui.videoStatus)
 	if statusErr != nil {
 		return errors.Prefix(statusErr.Error(), updateError)
 	}
 	return errors.Err(updateError)
 }
 func transferChannel(s *Sync) error {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("transferChannel").Add(time.Since(start))
 	}(start)
 	account, err := s.getDefaultAccount()
 	if err != nil {
 		return err
 	}
 	channelClaims, err := s.daemon.ChannelList(&account, 1, 50, nil)
 	if err != nil {
 		return errors.Err(err)
 	}
 	var channelClaim *jsonrpc.Transaction = nil
 	for _, c := range channelClaims.Items {
 		if c.ClaimID != s.DbChannelData.ChannelClaimID {
 			continue
 		}
 		channelClaim = &c
 		break
 	}
 	if channelClaim == nil {
 		return nil
 	}
 	updateOptions := jsonrpc.ChannelUpdateOptions{
 		Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
 		ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
 			ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
 				ClaimAddress: &s.DbChannelData.PublishAddress.Address,
 			},
 		},
 	}
 	result, err := s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, updateOptions)
 	if err != nil {
 		return errors.Err(err)
 	}
 	log.Infof("TRANSFERRED %t", len(result.Outputs) != 0)
 	return nil
 }
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@ -1,17 +0,0 @@
 package metrics
 import (
 	"github.com/lbryio/ytsync/v5/configs"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 var (
 	Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: "ytsync",
 		Subsystem: configs.Configuration.GetHostname(),
 		Name:      "duration",
 		Help:      "The durations of the individual modules",
 	}, []string{"path"})
 )
--- a/namer/names.go
+++ b/namer/names.go
@ -10,7 +10,7 @@ import (
 	"sync"
 )
-var claimNameRegexp = regexp.MustCompile(`[=&#:$@%？?;、\\"/<>%{}|｜^~\x60[\]\s]`)
+var titleRegexp = regexp.MustCompile(`[^a-zA-Z0-9]+`)
 type Namer struct {
 	mu    *sync.Mutex
@ -43,16 +43,9 @@ func (n *Namer) GetNextName(prefix string) string {
 	}
 	//if for some reasons the title can't be converted in a valid claim name (too short or not latin) then we use a hash
 	attempt = 1
 	if len(name) < 2 {
 		sum := md5.Sum([]byte(prefix))
 		for {
 		name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
 			if _, exists := n.names[name]; !exists {
 				break
 			}
 			attempt++
 		}
 	}
 	n.names[name] = true
@ -68,21 +61,18 @@ func getClaimNameFromTitle(title string, attempt int) string {
 	}
 	maxLen := 40 - len(suffix)
-	chunks := strings.Split(strings.ToLower(strings.Trim(claimNameRegexp.ReplaceAllString(title, "-"), "-")), "-")
+	chunks := strings.Split(strings.ToLower(strings.Trim(titleRegexp.ReplaceAllString(title, "-"), "-")), "-")
 	name := chunks[0]
 	if len(name) > maxLen {
-		return truncateUnicode(name, maxLen) + suffix
+		return name[:maxLen]
 	}
 	for _, chunk := range chunks[1:] {
 		if chunk == "" {
 			continue
 		}
 		tmpName := name + "-" + chunk
 		if len(tmpName) > maxLen {
 			if len(name) < 20 {
-				name = truncateUnicode(tmpName, maxLen-len(name))
+				name = tmpName[:maxLen]
 			}
 			break
 		}
@ -91,18 +81,3 @@ func getClaimNameFromTitle(title string, attempt int) string {
 	return name + suffix
 }
 func truncateUnicode(name string, limit int) string {
 	reNameBlacklist := regexp.MustCompile(`(&|>|<|\/|:|\n|\r)*`)
 	name = reNameBlacklist.ReplaceAllString(name, "")
 	result := name
 	chars := 0
 	for i := range name {
 		if chars >= limit {
 			result = name[:i]
 			break
 		}
 		chars++
 	}
 	return result
 }
--- a/namer/names_test.go
+++ b/namer/names_test.go
@ -1,28 +0,0 @@
 package namer
 import (
 	"testing"
 	"github.com/stretchr/testify/assert"
 )
 func Test_getClaimNameFromTitle(t *testing.T) {
 	name := getClaimNameFromTitle("СтопХам - \"В ожидании ответа\"", 0)
 	assert.Equal(t, "стопхам-в-ожидании", name)
 	name = getClaimNameFromTitle("SADB - \"A Weak Woman With a Strong Hood\"", 0)
 	assert.Equal(t, "sadb-a-weak-woman-with-a-strong-hood", name)
 	name = getClaimNameFromTitle("錢包整理術 5 Tips、哪種錢包最NG？｜有錢人默默在做的「錢包整理術」 ft.@SHIN LI", 0)
 	assert.Equal(t, "錢包整理術-5-tips-哪種錢包最ng", name)
 	name = getClaimNameFromTitle("اسرع-طريقة-لتختيم", 0)
 	assert.Equal(t, "اسرع-طريقة-لتختيم", name)
 	name = getClaimNameFromTitle("شكرا على 380 مشترك😍😍😍😍 لي يريد دعم ادا وصلنا المقطع 40 لايك وراح ادعم قناتين", 0)
 	assert.Equal(t, "شكرا-على-380-مشترك😍😍😍", name)
 	name = getClaimNameFromTitle("test-@", 0)
 	assert.Equal(t, "test", name)
 	name = getClaimNameFromTitle("『あなたはただの空の殻でした』", 0)
 	assert.Equal(t, "『あなたはただの空の殻でした』", name)
 	name = getClaimNameFromTitle("精靈樂章-這樣的夥伴沒問題嗎 幽暗隕石坑（夢魘） 王有無敵狀態...要會閃不然會被秒（無課）", 2)
 	assert.Equal(t, "精靈樂章-這樣的夥伴沒問題嗎-2", name)
 	name = getClaimNameFromTitle("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 50)
 	assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-50", name)
 }
--- a/sdk/api.go
+++ b/sdk/api.go
@ -11,12 +11,8 @@ import (
 	"strings"
 	"time"
-	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/extras/errors"
-	"github.com/lbryio/lbry.go/v2/extras/null"
+	"github.com/lbryio/lbry.go/extras/null"
 	"github.com/lbryio/ytsync/v5/configs"
 	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/util"
 	log "github.com/sirupsen/logrus"
 )
@ -26,57 +22,53 @@ const (
 )
 type APIConfig struct {
 	YoutubeAPIKey string
 	ApiURL        string
 	ApiToken      string
 	HostName      string
 }
-var instance *APIConfig
+type SyncProperties struct {
-
+	SyncFrom         int64
-func GetAPIsConfigs() *APIConfig {
+	SyncUntil        int64
-	if instance == nil {
+	YoutubeChannelID string
 		instance = &APIConfig{
 			ApiURL:   configs.Configuration.InternalApisEndpoint,
 			ApiToken: configs.Configuration.InternalApisAuthToken,
 			HostName: configs.Configuration.GetHostname(),
 		}
 	}
 	return instance
 }
-func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
+type Fee struct {
 	Amount   string `json:"amount"`
 	Address  string `json:"address"`
 	Currency string `json:"currency"`
 }
 type YoutubeChannel struct {
 	ChannelId          string `json:"channel_id"`
 	TotalVideos        uint   `json:"total_videos"`
 	DesiredChannelName string `json:"desired_channel_name"`
 	Fee                *Fee   `json:"fee"`
 	ChannelClaimID     string `json:"channel_claim_id"`
 }
 func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
 	type apiJobsResponse struct {
 		Success bool             `json:"success"`
 		Error   null.String      `json:"error"`
-		Data    []shared.YoutubeChannel `json:"data"`
+		Data    []YoutubeChannel `json:"data"`
 	}
 	endpoint := a.ApiURL + "/yt/jobs"
-	res, err := http.PostForm(endpoint, url.Values{
+	res, _ := http.PostForm(endpoint, url.Values{
 		"auth_token":  {a.ApiToken},
 		"sync_status": {status},
 		"min_videos":  {strconv.Itoa(1)},
-		"after":       {strconv.Itoa(int(cliFlags.SyncFrom))},
+		"after":       {strconv.Itoa(int(cp.SyncFrom))},
-		"before":      {strconv.Itoa(int(cliFlags.SyncUntil))},
+		"before":      {strconv.Itoa(int(cp.SyncUntil))},
 		"sync_server": {a.HostName},
-		"channel_id":  {cliFlags.ChannelID},
+		"channel_id":  {cp.YoutubeChannelID},
 	})
 	if err != nil {
 		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
 		time.Sleep(30 * time.Second)
 		return a.FetchChannels(status, cliFlags)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
 		return a.FetchChannels(status, cliFlags)
 	}
 	var response apiJobsResponse
-	err = json.Unmarshal(body, &response)
+	err := json.Unmarshal(body, &response)
 	if err != nil {
-		return nil, errors.Err(err)
+		return nil, err
 	}
 	if response.Data == nil {
 		return nil, errors.Err(response.Error)
@ -93,8 +85,6 @@ type SyncedVideo struct {
 	ClaimID         string `json:"claim_id"`
 	Size            int64  `json:"size"`
 	MetadataVersion int8   `json:"metadata_version"`
 	Transferred     bool   `json:"transferred"`
 	IsLbryFirst     bool   `json:"is_lbry_first"`
 }
 func sanitizeFailureReason(s *string) {
@ -105,47 +95,7 @@ func sanitizeFailureReason(s *string) {
 		*s = (*s)[:MaxReasonLength]
 	}
 }
-
+func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string) (map[string]SyncedVideo, map[string]bool, error) {
 func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
 	type apiSetChannelCertResponse struct {
 		Success bool        `json:"success"`
 		Error   null.String `json:"error"`
 		Data    string      `json:"data"`
 	}
 	endpoint := a.ApiURL + "/yt/channel_cert"
 	res, err := http.PostForm(endpoint, url.Values{
 		"channel_claim_id": {channelID},
 		"channel_cert":     {certHex},
 		"auth_token":       {a.ApiToken},
 	})
 	if err != nil {
 		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
 		time.Sleep(30 * time.Second)
 		return a.SetChannelCert(certHex, channelID)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
 		return a.SetChannelCert(certHex, channelID)
 	}
 	var response apiSetChannelCertResponse
 	err = json.Unmarshal(body, &response)
 	if err != nil {
 		return errors.Err(err)
 	}
 	if !response.Error.IsNull() {
 		return errors.Err(response.Error.String)
 	}
 	return nil
 }
 func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string, transferState *int) (map[string]SyncedVideo, map[string]bool, error) {
 	type apiChannelStatusResponse struct {
 		Success bool          `json:"success"`
 		Error   null.String   `json:"error"`
@ -154,34 +104,19 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
 	endpoint := a.ApiURL + "/yt/channel_status"
 	sanitizeFailureReason(&failureReason)
-	params := url.Values{
+	res, _ := http.PostForm(endpoint, url.Values{
 		"channel_id":     {channelID},
 		"sync_server":    {a.HostName},
 		"auth_token":     {a.ApiToken},
 		"sync_status":    {status},
 		"failure_reason": {failureReason},
-	}
+	})
 	if transferState != nil {
 		params.Add("transfer_state", strconv.Itoa(*transferState))
 	}
 	res, err := http.PostForm(endpoint, params)
 	if err != nil {
 		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
 		time.Sleep(30 * time.Second)
 		return a.SetChannelStatus(channelID, status, failureReason, transferState)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode >= http.StatusInternalServerError {
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
 		return a.SetChannelStatus(channelID, status, failureReason, transferState)
 	}
 	var response apiChannelStatusResponse
-	err = json.Unmarshal(body, &response)
+	err := json.Unmarshal(body, &response)
 	if err != nil {
-		return nil, nil, errors.Err(err)
+		return nil, nil, err
 	}
 	if !response.Error.IsNull() {
 		return nil, nil, errors.Err(response.Error.String)
@ -207,26 +142,15 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
 		Data    string      `json:"data"`
 	}
 	endpoint := a.ApiURL + "/yt/set_channel_claim_id"
-	res, err := http.PostForm(endpoint, url.Values{
+	res, _ := http.PostForm(endpoint, url.Values{
 		"channel_id":       {channelID},
 		"auth_token":       {a.ApiToken},
 		"channel_claim_id": {channelClaimID},
 	})
 	if err != nil {
 		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
 		time.Sleep(30 * time.Second)
 		return a.SetChannelClaimID(channelID, channelClaimID)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
 		return a.SetChannelClaimID(channelID, channelClaimID)
 	}
 	var response apiChannelStatusResponse
-	err = json.Unmarshal(body, &response)
+	err := json.Unmarshal(body, &response)
 	if err != nil {
 		return errors.Err(err)
 	}
@ -252,26 +176,15 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
 		"video_ids":  {videoIDs},
 		"auth_token": {a.ApiToken},
 	}
-	res, err := http.PostForm(endpoint, vals)
+	res, _ := http.PostForm(endpoint, vals)
 	if err != nil {
 		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
 		time.Sleep(30 * time.Second)
 		return a.DeleteVideos(videos)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
 		return a.DeleteVideos(videos)
 	}
 	var response struct {
 		Success bool        `json:"success"`
 		Error   null.String `json:"error"`
 		Data    null.String `json:"data"`
 	}
-	err = json.Unmarshal(body, &response)
+	err := json.Unmarshal(body, &response)
 	if err != nil {
 		return errors.Err(err)
 	}
@ -285,56 +198,42 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
 	return errors.Err("invalid API response. Status code: %d", res.StatusCode)
 }
-func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
+func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
 	endpoint := a.ApiURL + "/yt/video_status"
-	sanitizeFailureReason(&status.FailureReason)
+	sanitizeFailureReason(&failureReason)
 	vals := url.Values{
-		"youtube_channel_id": {status.ChannelID},
+		"youtube_channel_id": {channelID},
-		"video_id":           {status.VideoID},
+		"video_id":           {videoID},
-		"status":             {status.Status},
+		"status":             {status},
 		"auth_token":         {a.ApiToken},
 	}
-	if status.Status == VideoStatusPublished || status.Status == VideoStatusUpgradeFailed {
+	if status == VideoStatusPublished || status == VideoStatusUpgradeFailed {
-		if status.ClaimID == "" || status.ClaimName == "" {
+		if claimID == "" || claimName == "" {
-			return errors.Err("claimID (%s) or claimName (%s) missing", status.ClaimID, status.ClaimName)
+			return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName)
 		}
 		vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
-		vals.Add("claim_id", status.ClaimID)
+		vals.Add("claim_id", claimID)
-		vals.Add("claim_name", status.ClaimName)
+		vals.Add("claim_name", claimName)
-		if status.MetaDataVersion > 0 {
+		if metadataVersion > 0 {
-			vals.Add("metadata_version", fmt.Sprintf("%d", status.MetaDataVersion))
+			vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
 		}
-		if status.Size != nil {
+		if size != nil {
-			vals.Add("size", strconv.FormatInt(*status.Size, 10))
+			vals.Add("size", strconv.FormatInt(*size, 10))
 		}
 	}
-	if status.FailureReason != "" {
+	if failureReason != "" {
-		vals.Add("failure_reason", status.FailureReason)
+		vals.Add("failure_reason", failureReason)
 	}
 	if status.IsTransferred != nil {
 		vals.Add("transferred", strconv.FormatBool(*status.IsTransferred))
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
 		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
 		time.Sleep(30 * time.Second)
 		return a.MarkVideoStatus(status)
 	}
 	res, _ := http.PostForm(endpoint, vals)
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
 		return a.MarkVideoStatus(status)
 	}
 	var response struct {
 		Success bool        `json:"success"`
 		Error   null.String `json:"error"`
 		Data    null.String `json:"data"`
 	}
-	err = json.Unmarshal(body, &response)
+	err := json.Unmarshal(body, &response)
 	if err != nil {
 		return err
 	}
@ -346,96 +245,3 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
 	}
 	return errors.Err("invalid API response. Status code: %d", res.StatusCode)
 }
 func (a *APIConfig) VideoState(videoID string) (string, error) {
 	endpoint := a.ApiURL + "/yt/video_state"
 	vals := url.Values{
 		"video_id":   {videoID},
 		"auth_token": {a.ApiToken},
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
 		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
 		time.Sleep(30 * time.Second)
 		return a.VideoState(videoID)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode == http.StatusNotFound {
 		return "not_found", nil
 	}
 	if res.StatusCode != http.StatusOK {
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
 		return a.VideoState(videoID)
 	}
 	var response struct {
 		Success bool        `json:"success"`
 		Error   null.String `json:"error"`
 		Data    null.String `json:"data"`
 	}
 	err = json.Unmarshal(body, &response)
 	if err != nil {
 		return "", errors.Err(err)
 	}
 	if !response.Error.IsNull() {
 		return "", errors.Err(response.Error.String)
 	}
 	if !response.Data.IsNull() {
 		return response.Data.String, nil
 	}
 	return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
 }
 type VideoRelease struct {
 	ID            uint64 `json:"id"`
 	YoutubeDataID uint64 `json:"youtube_data_id"`
 	VideoID       string `json:"video_id"`
 	ReleaseTime   string `json:"release_time"`
 	CreatedAt     string `json:"created_at"`
 	UpdatedAt     string `json:"updated_at"`
 }
 func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
 	endpoint := a.ApiURL + "/yt/released"
 	vals := url.Values{
 		"video_id":   {videoID},
 		"auth_token": {a.ApiToken},
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
 		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
 		time.Sleep(30 * time.Second)
 		return a.GetReleasedDate(videoID)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode == http.StatusNotFound {
 		return nil, nil
 	}
 	if res.StatusCode != http.StatusOK {
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
 		return a.GetReleasedDate(videoID)
 	}
 	var response struct {
 		Success bool         `json:"success"`
 		Error   null.String  `json:"error"`
 		Data    VideoRelease `json:"data"`
 	}
 	err = json.Unmarshal(body, &response)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	if !response.Error.IsNull() {
 		return nil, errors.Err(response.Error.String)
 	}
 	if response.Data.ReleaseTime != "" {
 		return &response.Data, nil
 	}
 	return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
 }
--- a/shared/shared.go
+++ b/shared/shared.go
@ -1,221 +0,0 @@
 package shared
 import (
 	"encoding/json"
 	"time"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 )
 type Fee struct {
 	Amount   string `json:"amount"`
 	Address  string `json:"address"`
 	Currency string `json:"currency"`
 }
 type YoutubeChannel struct {
 	ChannelId          string         `json:"channel_id"`
 	TotalVideos        uint           `json:"total_videos"`
 	TotalSubscribers   uint           `json:"total_subscribers"`
 	DesiredChannelName string         `json:"desired_channel_name"`
 	Fee                *Fee           `json:"fee"`
 	ChannelClaimID     string         `json:"channel_claim_id"`
 	TransferState      int            `json:"transfer_state"`
 	PublishAddress     PublishAddress `json:"publish_address"`
 	PublicKey          string         `json:"public_key"`
 	LengthLimit        int            `json:"length_limit"`
 	SizeLimit          int            `json:"size_limit"`
 	LastUploadedVideo  string         `json:"last_uploaded_video"`
 	WipeDB             bool           `json:"wipe_db"`
 	Language           string         `json:"language"`
 }
 type PublishAddress struct {
 	Address string `json:"address"`
 	IsMine  bool   `json:"is_mine"`
 }
 func (p *PublishAddress) UnmarshalJSON(data []byte) error {
 	var s string
 	if err := json.Unmarshal(data, &s); err != nil {
 		return errors.Err(err)
 	}
 	p.Address = s
 	p.IsMine = false
 	return nil
 }
 var FatalErrors = []string{
 	":5279: read: connection reset by peer",
 	"no space left on device",
 	"NotEnoughFunds",
 	"Cannot publish using channel",
 	"cannot concatenate 'str' and 'NoneType' objects",
 	"more than 90% of the space has been used.",
 	"Couldn't find private key for id",
 	"You already have a stream claim published under the name",
 	"Missing inputs",
 }
 var ErrorsNoRetry = []string{
 	"Requested format is not available",
 	"non 200 status code received",
 	"This video contains content from",
 	"dont know which claim to update",
 	"uploader has not made this video available in your country",
 	"download error: AccessDenied: Access Denied",
 	"Playback on other websites has been disabled by the video owner",
 	"Error in daemon: Cannot publish empty file",
 	"Error extracting sts from embedded url response",
 	"Unable to extract signature tokens",
 	"Client.Timeout exceeded while awaiting headers",
 	"the video is too big to sync, skipping for now",
 	"video is too long to process",
 	"video is too short to process",
 	"no compatible format available for this video",
 	"Watch this video on YouTube.",
 	"have blocked it on copyright grounds",
 	"the video must be republished as we can't get the right size",
 	"HTTP Error 403",
 	"giving up after 0 fragment retries",
 	"Sorry about that",
 	"This video is not available",
 	"Video unavailable",
 	"requested format not available",
 	"interrupted by user",
 	"Sign in to confirm your age",
 	"This video is unavailable",
 	"video is a live stream and hasn't completed yet",
 	"Premieres in",
 	"Private video",
 	"This live event will begin in",
 	"This video has been removed by the uploader",
 	"Premiere will begin shortly",
 	"cannot unmarshal number 0.0",
 	"default youtube thumbnail found",
 	"livestream is likely bugged",
 }
 var WalletErrors = []string{
 	"Not enough funds to cover this transaction",
 	"failed: Not enough funds",
 	"Error in daemon: Insufficient funds, please deposit additional LBC",
 	//"Missing inputs",
 }
 var BlockchainErrors = []string{
 	"txn-mempool-conflict",
 	"too-long-mempool-chain",
 }
 var NeverRetryFailures = []string{
 	"Error extracting sts from embedded url response",
 	"Unable to extract signature tokens",
 	"the video is too big to sync, skipping for now",
 	"video is too long to process",
 	"video is too short to process",
 	"This video contains content from",
 	"no compatible format available for this video",
 	"Watch this video on YouTube.",
 	"have blocked it on copyright grounds",
 	"giving up after 0 fragment retries",
 	"Sign in to confirm your age",
 	"Playback on other websites has been disabled by the video owner",
 	"uploader has not made this video available in your country",
 	"This video has been removed by the uploader",
 	"Video unavailable",
 	"Video is not available - hardcoded fix",
 }
 type SyncFlags struct {
 	TakeOverExistingChannel bool
 	SkipSpaceCheck          bool
 	SyncUpdate              bool
 	SingleRun               bool
 	RemoveDBUnpublished     bool
 	UpgradeMetadata         bool
 	DisableTransfers        bool
 	QuickSync               bool
 	MaxTries                int
 	Refill                  int
 	Limit                   int
 	Status                  string
 	SecondaryStatus         string
 	ChannelID               string
 	SyncFrom                int64
 	SyncUntil               int64
 	ConcurrentJobs          int
 	VideosLimit             int
 	MaxVideoSize            int
 	MaxVideoLength          time.Duration
 }
 // VideosToSync dynamically figures out how many videos should be synced for a given subs count if nothing was otherwise specified
 func (f *SyncFlags) VideosToSync(totalSubscribers uint) int {
 	if f.VideosLimit > 0 {
 		return f.VideosLimit
 	}
 	defaultVideosToSync := map[int]int{
 		10000: 1000,
 		5000:  500,
 		1000:  400,
 		800:   250,
 		600:   200,
 		200:   80,
 		100:   20,
 		1:     10,
 	}
 	videosToSync := 0
 	for s, r := range defaultVideosToSync {
 		if int(totalSubscribers) >= s && r > videosToSync {
 			videosToSync = r
 		}
 	}
 	return videosToSync
 }
 func (f *SyncFlags) IsSingleChannelSync() bool {
 	return f.ChannelID != ""
 }
 type VideoStatus struct {
 	ChannelID       string
 	VideoID         string
 	Status          string
 	ClaimID         string
 	ClaimName       string
 	FailureReason   string
 	Size            *int64
 	MetaDataVersion uint
 	IsTransferred   *bool
 }
 const (
 	StatusPending        = "pending"        // waiting for permission to sync
 	StatusPendingEmail   = "pendingemail"   // permission granted but missing email
 	StatusQueued         = "queued"         // in sync queue. will be synced soon
 	StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
 	StatusSyncing        = "syncing"        // syncing now
 	StatusSynced         = "synced"         // done
 	StatusWipeDb         = "pendingdbwipe"  // in sync queue. lbryum database will be pruned
 	StatusFailed         = "failed"
 	StatusFinalized      = "finalized"     // no more changes allowed
 	StatusAbandoned      = "abandoned"     // deleted on youtube or banned
 	StatusAgeRestricted  = "agerestricted" // one or more videos are age restricted and should be reprocessed with special keys
 )
 var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb, StatusAgeRestricted}
 const LatestMetadataVersion = 2
 const (
 	VideoStatusPublished      = "published"
 	VideoStatusFailed         = "failed"
 	VideoStatusUpgradeFailed  = "upgradefailed"
 	VideoStatusUnpublished    = "unpublished"
 	VideoStatusTransferFailed = "transferfailed"
 )
 var VideoSyncStatuses = []string{VideoStatusPublished, VideoStatusFailed, VideoStatusUpgradeFailed, VideoStatusUnpublished, VideoStatusTransferFailed}
 const (
 	TransferStateNotTouched = iota
 	TransferStatePending
 	TransferStateComplete
 	TransferStateManual
 )
--- a/shared/shared_test.go
+++ b/shared/shared_test.go
@ -1,20 +0,0 @@
 package shared
 import (
 	"testing"
 	"gotest.tools/assert"
 )
 func TestSyncFlags_VideosToSync(t *testing.T) {
 	f := SyncFlags{}
 	assert.Equal(t, f.VideosToSync(0), 0)
 	assert.Equal(t, f.VideosToSync(1), 10)
 	assert.Equal(t, f.VideosToSync(5), 10)
 	assert.Equal(t, f.VideosToSync(10), 10)
 	assert.Equal(t, f.VideosToSync(101), 50)
 	assert.Equal(t, f.VideosToSync(500), 80)
 	assert.Equal(t, f.VideosToSync(21000), 1000)
 	f.VideosLimit = 1337
 	assert.Equal(t, f.VideosToSync(21), 1337)
 }
--- a/sources/shared.go
+++ b/sources/shared.go
@ -4,8 +4,8 @@ import (
 	"strings"
 	"sync"
-	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+	"github.com/lbryio/lbry.go/extras/jsonrpc"
-	"github.com/lbryio/ytsync/v5/namer"
+	"github.com/lbryio/ytsync/namer"
 )
 type SyncSummary struct {
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@ -1,43 +1,33 @@
 package sources
 import (
 	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"math"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
-	"github.com/lbryio/ytsync/v5/downloader"
+	"github.com/lbryio/lbry.go/extras/errors"
-	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+	"github.com/lbryio/lbry.go/extras/jsonrpc"
-	"github.com/lbryio/ytsync/v5/ip_manager"
+	"github.com/lbryio/lbry.go/extras/util"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/tags_manager"
 	"github.com/lbryio/ytsync/v5/thumbs"
 	"github.com/lbryio/ytsync/v5/timing"
 	logUtils "github.com/lbryio/ytsync/v5/util"
-	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/ytsync/namer"
-	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+	"github.com/lbryio/ytsync/sdk"
-	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/ytsync/tagsManager"
-	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/ytsync/thumbs"
-	"github.com/abadojack/whatlanggo"
+	"github.com/ChannelMeter/iso8601duration"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/nikooo777/ytdl"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
-	"github.com/vbauerster/mpb/v7"
+	"google.golang.org/api/youtube/v3"
 	"github.com/vbauerster/mpb/v7/decor"
 	"gopkg.in/vansante/go-ffprobe.v2"
 )
 type YoutubeVideo struct {
@ -47,22 +37,21 @@ type YoutubeVideo struct {
 	playlistPosition int64
 	size             *int64
 	maxVideoSize     int64
-	maxVideoLength   time.Duration
+	maxVideoLength   float64
 	publishedAt      time.Time
 	dir              string
-	youtubeInfo      *ytdl.YtdlVideo
+	youtubeInfo      *youtube.Video
 	youtubeChannelID string
 	tags             []string
 	awsConfig        aws.Config
 	thumbnailURL     string
 	lbryChannelID    string
 	mocked           bool
 	walletLock       *sync.RWMutex
 	stopGroup        *stop.Group
 	pool             *ip_manager.IPPool
 	progressBars     *mpb.Progress
 	progressBarWg    *sync.WaitGroup
 }
 const reflectorURL = "http://blobs.lbry.io/"
 var youtubeCategories = map[string]string{
 	"1":  "film & animation",
 	"2":  "autos & vehicles",
@ -98,33 +87,29 @@ var youtubeCategories = map[string]string{
 	"44": "trailers",
 }
-func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
+func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
-	// youtube-dl returns times in local timezone sometimes. this could break in the future
+	publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
 	// maybe we can file a PR to choose the timezone we want from youtube-dl
 	return &YoutubeVideo{
-		id:               videoData.ID,
+		id:               videoData.Id,
-		title:            videoData.Title,
+		title:            videoData.Snippet.Title,
-		description:      videoData.Description,
+		description:      videoData.Snippet.Description,
 		playlistPosition: playlistPosition,
-		publishedAt:      videoData.GetUploadTime(),
+		publishedAt:      publishedAt,
 		dir:              directory,
 		youtubeInfo:      videoData,
 		awsConfig:        awsConfig,
 		mocked:           false,
-		youtubeChannelID: videoData.ChannelID,
+		youtubeChannelID: videoData.Snippet.ChannelId,
-		stopGroup:        stopGroup,
+	}
 		pool:             pool,
 	}, nil
 }
-
+func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config) *YoutubeVideo {
 func NewMockedVideo(directory string, videoID string, youtubeChannelID string, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
 	return &YoutubeVideo{
 		id:               videoID,
 		playlistPosition: 0,
 		dir:              directory,
 		awsConfig:        awsConfig,
 		mocked:           true,
 		youtubeChannelID: youtubeChannelID,
 		stopGroup:        stopGroup,
 		pool:             pool,
 	}
 }
@ -175,47 +160,48 @@ func (v *YoutubeVideo) getFullPath() string {
 }
 func (v *YoutubeVideo) getAbbrevDescription() string {
-	maxLength := 6500
+	maxLines := 10
 	description := strings.TrimSpace(v.description)
 	additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
 	khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
 	if v.lbryChannelID == khanAcademyClaimID {
 		additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
 	}
-	if len(description) > maxLength {
+	if strings.Count(description, "\n") < maxLines {
 		description = description[:maxLength]
 	}
 		return description + "\n..." + additionalDescription
 	}
 	return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
 }
 func checkCookiesIntegrity() error {
 	fi, err := os.Stat("cookies.txt")
 	if err != nil {
 		return errors.Err(err)
 	}
 	if fi.Size() == 0 {
 		log.Errorf("cookies were cleared out. Attempting a restore from cookies-backup.txt")
 		input, err := ioutil.ReadFile("cookies-backup.txt")
 		if err != nil {
 			return errors.Err(err)
 		}
-		err = ioutil.WriteFile("cookies.txt", input, 0644)
+func (v *YoutubeVideo) fallbackDownload() error {
 	cmd := exec.Command("youtube-dl",
 		"--no-progress",
 		"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
 		"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
 		"--merge-output-format",
 		"mp4",
 		"https://www.youtube.com/watch?v="+v.ID())
 	log.Printf("Running command and waiting for it to finish...")
 	output, err := cmd.CombinedOutput()
 	log.Debugln(string(output))
 	if err != nil {
 		log.Printf("Command finished with error: %v", errors.Err(string(output)))
 		return errors.Err(err)
 	}
 	fi, err := os.Stat(v.getFullPath())
 	if err != nil {
 		return errors.Err(err)
 	}
-	}
+	videoSize := fi.Size()
 	v.size = &videoSize
 	return nil
 }
 func (v *YoutubeVideo) download() error {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("download").Add(time.Since(start))
 	}(start)
 	videoPath := v.getFullPath()
-	err := os.Mkdir(v.videoDir(), 0777)
+	err := os.Mkdir(v.videoDir(), 0750)
 	if err != nil && !strings.Contains(err.Error(), "file exists") {
 		return errors.Wrap(err, 0)
 	}
@ -227,427 +213,77 @@ func (v *YoutubeVideo) download() error {
 		log.Debugln(v.id + " already exists at " + videoPath)
 		return nil
 	}
 	qualities := []string{
 		"1080",
 		"720",
 		"480",
 		"360",
 	}
 	dur := time.Duration(v.youtubeInfo.Duration) * time.Second
 	if dur.Hours() > 1 { //for videos longer than 1 hour only sync up to 720p
 		qualities = []string{
 			"720",
 			"480",
 			"360",
 		}
 	}
-	metadataPath := path.Join(logUtils.GetVideoMetadataDir(), v.id+".info.json")
+	videoUrl := "https://www.youtube.com/watch?v=" + v.id
-	_, err = os.Stat(metadataPath)
+	videoInfo, err := ytdl.GetVideoInfo(videoUrl)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errors.Err("metadata information for video %s is missing! Why?", v.id)
 		}
 		return errors.Err(err)
 	}
 	metadata, err := parseVideoMetadata(metadataPath)
 	err = checkCookiesIntegrity()
 	if err != nil {
 		return err
 	}
 	ytdlArgs := []string{
 		"--no-progress",
 		"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
 		"--merge-output-format",
 		"mp4",
 		"--postprocessor-args",
 		"ffmpeg:-movflags faststart",
 		"--abort-on-unavailable-fragment",
 		"--fragment-retries",
 		"1",
 		"--cookies",
 		"cookies.txt",
 		"--extractor-args",
 		"youtube:player_client=android",
 		//"--concurrent-fragments",
 		//"2",
 		"--load-info-json",
 		metadataPath,
 	}
 	userAgent := []string{"--user-agent", downloader.ChromeUA}
 	if v.maxVideoSize > 0 {
 		ytdlArgs = append(ytdlArgs,
 			"--max-filesize",
 			fmt.Sprintf("%dM", v.maxVideoSize),
 		)
 	}
 	if v.maxVideoLength > 0 {
 		ytdlArgs = append(ytdlArgs,
 			"--match-filter",
 			fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())),
 		)
 	}
 	var sourceAddress string
 	for {
 		sourceAddress, err = v.pool.GetIP(v.id)
 		if err != nil {
 			if errors.Is(err, ip_manager.ErrAllThrottled) {
 				select {
 				case <-v.stopGroup.Ch():
 					return errors.Err("interrupted by user")
 				default:
 					time.Sleep(ip_manager.IPCooldownPeriod)
 					continue
 				}
 			} else {
 				return err
 			}
 		}
 		break
 	}
 	defer v.pool.ReleaseIP(sourceAddress)
 	ytdlArgs = append(ytdlArgs,
 		"--source-address",
 		sourceAddress,
 		fmt.Sprintf("https://www.youtube.com/watch?v=%s", v.id),
 	)
 	//speedThrottleRetries := 3
 	for i := 0; i < len(qualities); i++ {
 		quality := qualities[i]
 		argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=380][format_id!=251][format_id!=256][format_id!=327][format_id!=328]")
 		argsWithFilters = append(argsWithFilters, userAgent...)
 		//if speedThrottleRetries > 0 {
 		//	speedThrottleRetries--
 		//	argsWithFilters = append(argsWithFilters, "--throttled-rate", "180K")
 		//}
 		cmd := exec.Command("yt-dlp", argsWithFilters...)
 		log.Printf("Running command yt-dlp %s", strings.Join(argsWithFilters, " "))
 		stderr, err := cmd.StderrPipe()
 		if err != nil {
 			return errors.Err(err)
 		}
 		stdout, err := cmd.StdoutPipe()
 	if err != nil {
 		return errors.Err(err)
 	}
-		if err := cmd.Start(); err != nil {
+	codec := []string{"H.264"}
-			return errors.Err(err)
+	ext := []string{"mp4"}
 	//Filter requires a [] interface{}
 	codecFilter := make([]interface{}, len(codec))
 	for i, v := range codec {
 		codecFilter[i] = v
 	}
-		dlStopGrp := stop.New()
+	//Filter requires a [] interface{}
-
+	extFilter := make([]interface{}, len(ext))
-		ticker := time.NewTicker(400 * time.Millisecond)
+	for i, v := range ext {
-		go v.trackProgressBar(argsWithFilters, ticker, metadata, dlStopGrp, sourceAddress)
+		extFilter[i] = v
 		//ticker2 := time.NewTicker(10 * time.Second)
 		//v.monitorSlowDownload(ticker, dlStopGrp, sourceAddress, cmd)
 		errorLog, _ := ioutil.ReadAll(stderr)
 		outLog, _ := ioutil.ReadAll(stdout)
 		err = cmd.Wait()
 		//stop the progress bar
 		ticker.Stop()
 		dlStopGrp.Stop()
 		if err != nil {
 			if strings.Contains(err.Error(), "exit status 1") {
 				if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
 					v.pool.SetThrottled(sourceAddress)
 				} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
 					if i == (len(qualities) - 1) {
 						return errors.Err(string(errorLog))
 	}
 					continue //this bypasses the yt throttling IP redistribution... TODO: don't
 				} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
 					i-- //do not lower quality when trying a different user agent
 					userAgent = []string{downloader.GoogleBotUA}
 					log.Infof("trying different user agent for video %s", v.ID())
 					continue
 					//} else if strings.Contains(string(errorLog), "yt_dlp.utils.ThrottledDownload") {
 					//	log.Infof("throttled download speed for video %s. Retrying", v.ID())
 					//	i-- //do not lower quality when we're retrying a throttled download
 					//	continue
 				}
 				return errors.Err(string(errorLog))
 			}
 			return errors.Err(err)
 		}
 		log.Debugln(string(outLog))
-		if strings.Contains(string(outLog), "does not pass filter duration") {
+	formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
-			_ = v.delete("does not pass filter duration")
+	if len(formats) == 0 {
 		return errors.Err("no compatible format available for this video")
 	}
 	maxRetryAttempts := 5
 	isLengthLimitSet := v.maxVideoLength > 0.01
 	if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
 		return errors.Err("video is too long to process")
 	}
-		if strings.Contains(string(outLog), "File is larger than max-filesize") {
+
-			_ = v.delete("File is larger than max-filesize")
+	for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
-			return errors.Err("the video is too big to sync, skipping for now")
+		formatIndex := i
 		if i == maxRetryAttempts-1 {
 			formatIndex = len(formats) - 1
 		}
-		if string(errorLog) != "" {
+		var downloadedFile *os.File
-			if strings.Contains(string(errorLog), "HTTP Error 429") {
+		downloadedFile, err = os.Create(videoPath)
-				v.pool.SetThrottled(sourceAddress)
+		if err != nil {
 			return errors.Err(err)
 		}
-			log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
+		err = videoInfo.Download(formats[formatIndex], downloadedFile)
-			_ = v.delete("due to error")
+		_ = downloadedFile.Close()
-			return errors.Err(string(errorLog))
+		if err != nil {
 			//delete the video and ignore the error
 			_ = v.delete()
 			return errors.Err(err.Error())
 		}
 		fi, err := os.Stat(v.getFullPath())
 		if err != nil {
 			return errors.Err(err)
 		}
 		err = os.Chmod(v.getFullPath(), 0777)
 		if err != nil {
 			return errors.Err(err)
 		}
 		videoSize := fi.Size()
 		v.size = &videoSize
 		isVideoSizeLimitSet := v.maxVideoSize > 0
 		if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
 			//delete the video and ignore the error
 			_ = v.delete()
 			err = errors.Err("file is too big and there is no other format available")
 		} else {
 			break
 		}
 	return nil
 }
 func (v *YoutubeVideo) monitorSlowDownload(ticker *time.Ticker, stop *stop.Group, address string, cmd *exec.Cmd) {
 	count := 0
 	lastSize := int64(0)
 	for {
 		select {
 		case <-stop.Ch():
 			return
 		case <-ticker.C:
 			size, err := logUtils.DirSize(v.videoDir())
 			if err != nil {
 				log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
 				continue
 	}
-			delta := size - lastSize
+	return errors.Err(err)
 			avgSpeed := delta / 10
 			if avgSpeed < 200*1024 { //200 KB/s
 				count++
 			} else {
 				count--
 			}
 			if count > 3 {
 				err := cmd.Process.Signal(syscall.SIGKILL)
 				if err != nil {
 					log.Errorf("failure in killing slow download: %s", errors.Err(err))
 					return
 				}
 			}
 		}
 	}
 }
 func (v *YoutubeVideo) trackProgressBar(argsWithFilters []string, ticker *time.Ticker, metadata *ytMetadata, done *stop.Group, sourceAddress string) {
 	v.progressBarWg.Add(1)
 	go func() {
 		defer v.progressBarWg.Done()
 		//get size of the video before downloading
 		cmd := exec.Command("yt-dlp", append(argsWithFilters, "-s")...)
 		stdout, err := cmd.StdoutPipe()
 		if err != nil {
 			log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
 			return
 		}
 		if err := cmd.Start(); err != nil {
 			log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
 			return
 		}
 		outLog, _ := ioutil.ReadAll(stdout)
 		err = cmd.Wait()
 		output := string(outLog)
 		parts := strings.Split(output, ": ")
 		if len(parts) != 3 {
 			log.Errorf("couldn't parse audio and video parts from the output (%s)", output)
 			return
 		}
 		formats := strings.Split(parts[2], "+")
 		if len(formats) != 2 {
 			log.Errorf("couldn't parse formats from the output (%s)", output)
 			return
 		}
 		log.Debugf("'%s'", output)
 		videoFormat := formats[0]
 		audioFormat := strings.Replace(formats[1], "\n", "", -1)
 		videoSize := 0
 		audioSize := 0
 		if metadata != nil {
 			for _, f := range metadata.Formats {
 				if f.FormatID == videoFormat {
 					videoSize = f.Filesize
 				}
 				if f.FormatID == audioFormat {
 					audioSize = f.Filesize
 				}
 			}
 		}
 		log.Debugf("(%s) - videoSize: %d (%s), audiosize: %d (%s)", v.id, videoSize, videoFormat, audioSize, audioFormat)
 		bar := v.progressBars.AddBar(int64(videoSize+audioSize),
 			mpb.PrependDecorators(
 				decor.CountersKibiByte("% .2f / % .2f "),
 				// simple name decorator
 				decor.Name(fmt.Sprintf("id: %s src-ip: (%s)", v.id, sourceAddress)),
 				// decor.DSyncWidth bit enables column width synchronization
 				decor.Percentage(decor.WCSyncSpace),
 			),
 			mpb.AppendDecorators(
 				decor.EwmaETA(decor.ET_STYLE_GO, 90),
 				decor.Name(" ] "),
 				decor.EwmaSpeed(decor.UnitKiB, "% .2f ", 60),
 				decor.OnComplete(
 					// ETA decorator with ewma age of 60
 					decor.EwmaETA(decor.ET_STYLE_GO, 60), "done",
 				),
 			),
 			mpb.BarRemoveOnComplete(),
 		)
 		defer func() {
 			bar.Completed()
 			bar.Abort(true)
 		}()
 		origSize := int64(0)
 		lastUpdate := time.Now()
 		for {
 			select {
 			case <-done.Ch():
 				return
 			case <-ticker.C:
 				var err error
 				size, err := logUtils.DirSize(v.videoDir())
 				if err != nil {
 					log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
 					return
 				}
 				if size > origSize {
 					origSize = size
 					bar.SetCurrent(size)
 					if size > int64(videoSize+audioSize) {
 						bar.SetTotal(size+2048, false)
 					}
 					bar.DecoratorEwmaUpdate(time.Since(lastUpdate))
 					lastUpdate = time.Now()
 				}
 			}
 		}
 	}()
 }
 type ytMetadata struct {
 	ID      string `json:"id"`
 	Title   string `json:"title"`
 	Formats []struct {
 		Asr               int         `json:"asr"`
 		Filesize          int         `json:"filesize"`
 		FormatID          string      `json:"format_id"`
 		FormatNote        string      `json:"format_note"`
 		Fps               interface{} `json:"fps"`
 		Height            interface{} `json:"height"`
 		Quality           int         `json:"quality"`
 		Tbr               float64     `json:"tbr"`
 		URL               string      `json:"url"`
 		Width             interface{} `json:"width"`
 		Ext               string      `json:"ext"`
 		Vcodec            string      `json:"vcodec"`
 		Acodec            string      `json:"acodec"`
 		Abr               float64     `json:"abr,omitempty"`
 		DownloaderOptions struct {
 			HTTPChunkSize int `json:"http_chunk_size"`
 		} `json:"downloader_options,omitempty"`
 		Container   string `json:"container,omitempty"`
 		Format      string `json:"format"`
 		Protocol    string `json:"protocol"`
 		HTTPHeaders struct {
 			UserAgent      string `json:"User-Agent"`
 			AcceptCharset  string `json:"Accept-Charset"`
 			Accept         string `json:"Accept"`
 			AcceptEncoding string `json:"Accept-Encoding"`
 			AcceptLanguage string `json:"Accept-Language"`
 		} `json:"http_headers"`
 		Vbr float64 `json:"vbr,omitempty"`
 	} `json:"formats"`
 	Thumbnails []struct {
 		Height     int    `json:"height"`
 		URL        string `json:"url"`
 		Width      int    `json:"width"`
 		Resolution string `json:"resolution"`
 		ID         string `json:"id"`
 	} `json:"thumbnails"`
 	Description        string        `json:"description"`
 	UploadDate         string        `json:"upload_date"`
 	Uploader           string        `json:"uploader"`
 	UploaderID         string        `json:"uploader_id"`
 	UploaderURL        string        `json:"uploader_url"`
 	ChannelID          string        `json:"channel_id"`
 	ChannelURL         string        `json:"channel_url"`
 	Duration           int           `json:"duration"`
 	ViewCount          int           `json:"view_count"`
 	AverageRating      float64       `json:"average_rating"`
 	AgeLimit           int           `json:"age_limit"`
 	WebpageURL         string        `json:"webpage_url"`
 	Categories         []string      `json:"categories"`
 	Tags               []interface{} `json:"tags"`
 	IsLive             interface{}   `json:"is_live"`
 	LikeCount          int           `json:"like_count"`
 	DislikeCount       int           `json:"dislike_count"`
 	Channel            string        `json:"channel"`
 	Extractor          string        `json:"extractor"`
 	WebpageURLBasename string        `json:"webpage_url_basename"`
 	ExtractorKey       string        `json:"extractor_key"`
 	Playlist           interface{}   `json:"playlist"`
 	PlaylistIndex      interface{}   `json:"playlist_index"`
 	Thumbnail          string        `json:"thumbnail"`
 	DisplayID          string        `json:"display_id"`
 	Format             string        `json:"format"`
 	FormatID           string        `json:"format_id"`
 	Width              int           `json:"width"`
 	Height             int           `json:"height"`
 	Resolution         interface{}   `json:"resolution"`
 	Fps                int           `json:"fps"`
 	Vcodec             string        `json:"vcodec"`
 	Vbr                float64       `json:"vbr"`
 	StretchedRatio     interface{}   `json:"stretched_ratio"`
 	Acodec             string        `json:"acodec"`
 	Abr                float64       `json:"abr"`
 	Ext                string        `json:"ext"`
 	Fulltitle          string        `json:"fulltitle"`
 	Filename           string        `json:"_filename"`
 }
 func parseVideoMetadata(metadataPath string) (*ytMetadata, error) {
 	f, err := os.Open(metadataPath)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	// defer the closing of our jsonFile so that we can parse it later on
 	defer f.Close()
 	// read our opened jsonFile as a byte array.
 	byteValue, _ := ioutil.ReadAll(f)
 	// we initialize our Users array
 	var m ytMetadata
 	// we unmarshal our byteArray which contains our
 	// jsonFile's content into 'users' which we defined above
 	err = json.Unmarshal(byteValue, &m)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	return &m, nil
 }
 func (v *YoutubeVideo) videoDir() string {
-	return path.Join(v.dir, v.id)
+	return v.dir + "/" + v.id
 }
 func (v *YoutubeVideo) getDownloadedPath() (string, error) {
 	files, err := ioutil.ReadDir(v.videoDir())
 	log.Infoln(v.videoDir())
@ -662,20 +298,20 @@ func (v *YoutubeVideo) getDownloadedPath() (string, error) {
 			continue
 		}
 		if strings.Contains(v.getFullPath(), strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))) {
-			return path.Join(v.videoDir(), f.Name()), nil
+			return v.videoDir() + "/" + f.Name(), nil
 		}
 	}
 	return "", errors.Err("could not find any downloaded videos")
 }
-func (v *YoutubeVideo) delete(reason string) error {
+func (v *YoutubeVideo) delete() error {
 	videoPath, err := v.getDownloadedPath()
 	if err != nil {
 		log.Errorln(err)
 		return err
 	}
 	err = os.Remove(videoPath)
-	log.Debugf("%s deleted from disk for '%s' (%s)", v.id, reason, videoPath)
+	log.Debugf("%s deleted from disk (%s)", v.id, videoPath)
 	if err != nil {
 		err = errors.Prefix("delete error", err)
@ -687,20 +323,14 @@ func (v *YoutubeVideo) delete(reason string) error {
 }
 func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
-	thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
+	thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
-	if thumbnail.Width == 0 {
+	v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
 		return errors.Err("default youtube thumbnail found")
 	}
 	v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID())
 	return err
 }
 func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("publish").Add(time.Since(start))
 	}(start)
 	languages, locations, tags := v.getMetadata()
 	var fee *jsonrpc.Fee
 	if params.Fee != nil {
 		feeAmount, err := decimal.NewFromString(params.Fee.Amount)
@ -713,17 +343,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
 			FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
 		}
 	}
-	urlsRegex := regexp.MustCompile(`(?m) ?(f|ht)(tp)(s?)(://)(.*)[.|/](.*)`)
+
 	descriptionSample := urlsRegex.ReplaceAllString(v.description, "")
 	info := whatlanggo.Detect(descriptionSample)
 	info2 := whatlanggo.Detect(v.title)
 	if info.IsReliable() && info.Lang.Iso6391() != "" {
 		language := info.Lang.Iso6391()
 		languages = []string{language}
 	} else if info2.IsReliable() && info2.Lang.Iso6391() != "" {
 		language := info2.Lang.Iso6391()
 		languages = []string{language}
 	}
 	options := jsonrpc.StreamCreateOptions{
 		ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
 			Title:        &v.title,
@ -733,9 +353,6 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
 			ThumbnailURL: &v.thumbnailURL,
 			Tags:         tags,
 			Locations:    locations,
 			FundingAccountIDs: []string{
 				params.DefaultAccount,
 			},
 		},
 		Fee:         fee,
 		License:     util.PtrToString("Copyrighted (contact publisher)"),
@ -759,18 +376,15 @@ type SyncParams struct {
 	ChannelID      string
 	MaxVideoSize   int
 	Namer          *namer.Namer
-	MaxVideoLength time.Duration
+	MaxVideoLength float64
-	Fee            *shared.Fee
+	Fee            *sdk.Fee
 	DefaultAccount string
 }
-func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex, pbWg *sync.WaitGroup, pb *mpb.Progress) (*SyncSummary, error) {
+func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
-	v.maxVideoSize = int64(params.MaxVideoSize)
+	v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
 	v.maxVideoLength = params.MaxVideoLength
 	v.lbryChannelID = params.ChannelID
 	v.walletLock = walletLock
 	v.progressBars = pb
 	v.progressBarWg = pbWg
 	if reprocess && existingVideoData != nil && existingVideoData.Published {
 		summary, err := v.reprocess(daemon, params, existingVideoData)
 		return summary, errors.Prefix("upgrade failed", err)
@ -779,54 +393,17 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 }
 func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
-	var err error
+	err := v.download()
 	if v.youtubeInfo == nil {
 		return nil, errors.Err("Video is not available - hardcoded fix")
 	}
 	dur := time.Duration(v.youtubeInfo.Duration) * time.Second
 	minDuration := 7 * time.Second
 	if v.youtubeInfo.IsLive == true {
 		return nil, errors.Err("video is a live stream and hasn't completed yet")
 	}
 	if v.youtubeInfo.Availability != "public" {
 		return nil, errors.Err("video is not public")
 	}
 	if dur > v.maxVideoLength {
 		logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
 		return nil, errors.Err("video is too long to process")
 	}
 	if dur < minDuration {
 		logUtils.SendErrorToSlack("%s is %s long and the minimum is %s", v.id, dur.String(), minDuration.String())
 		return nil, errors.Err("video is too short to process")
 	}
 	buggedLivestream := v.youtubeInfo.LiveStatus == "post_live"
 	if buggedLivestream && dur >= 2*time.Hour {
 		return nil, errors.Err("livestream is likely bugged as it was recently published and has a length of %s which is more than 2 hours", dur.String())
 	}
 	for {
 		err = v.download()
 		if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
 			continue
 		} else if err != nil {
 			return nil, errors.Prefix("download error", err)
 		}
 		break
 	}
 	ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancelFn()
 	data, err := ffprobe.ProbeURL(ctx, v.getFullPath())
 	if err != nil {
-		log.Errorf("failure in probing downloaded video: %s", err.Error())
+		log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
-	} else {
+		fallBackErr := v.fallbackDownload()
-		if data.Format.Duration() < minDuration {
+		if fallBackErr != nil {
-			return nil, errors.Err("video is too short to process")
+			log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
 			return nil, errors.Prefix("download error", err) //return original error
 		}
 	}
 	log.Debugln("Downloaded " + v.id)
 	err = v.triggerThumbnailSave()
 	if err != nil {
 		return nil, errors.Prefix("thumbnail error", err)
@ -835,7 +412,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
 	summary, err := v.publish(daemon, params)
 	//delete the video in all cases (and ignore the error)
-	_ = v.delete("finished download and publish")
+	_ = v.delete()
 	return summary, errors.Prefix("publish error", err)
 }
@ -845,41 +422,31 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
 	locations = nil
 	tags = nil
 	if !v.mocked {
 		/*
 		if v.youtubeInfo.Snippet.DefaultLanguage != "" {
 				if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
 					v.youtubeInfo.Snippet.DefaultLanguage = "he"
 				}
 			languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
-			}*/
+		}
-		/*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil {
+		if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
 			locations = []jsonrpc.Location{{
 				Latitude:  util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
 				Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
 			}}
 		}*/
 		tags = v.youtubeInfo.Tags
 		}
-	tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
+		tags = v.youtubeInfo.Snippet.Tags
 	}
 	tags, err := tagsManager.SanitizeTags(tags, v.youtubeChannelID)
 	if err != nil {
 		log.Errorln(err.Error())
 	}
 	if !v.mocked {
-		for _, category := range v.youtubeInfo.Categories {
+		tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
 			tags = append(tags, youtubeCategories[category])
 		}
 	}
 	return languages, locations, tags
 }
 func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
-	c, err := daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
+	c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
 		ClaimID:  &existingVideoData.ClaimID,
 		Page:     1,
 		PageSize: 20,
 	})
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@ -897,8 +464,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		if v.mocked {
 			return nil, errors.Err("could not find thumbnail for mocked video")
 		}
-		thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
+		thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
-		thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID())
+		thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
 	} else {
 		thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
 	}
@ -909,14 +476,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 			videoSize = uint64(existingVideoData.Size)
 		} else {
 			log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
-			if !v.mocked {
+			//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
-				_, err = daemon.StreamAbandon(currentClaim.Txid, currentClaim.Nout, nil, true)
+			return nil, errors.Err("the video must be republished as we can't get the right size")
 				if err != nil {
 					return nil, errors.Err(err)
 				}
 				return v.downloadAndPublish(daemon, params)
 			}
 			return nil, errors.Prefix("the video must be republished as we can't get the right size and it doesn't exist on youtube anymore", err)
 		}
 	}
 	v.size = util.PtrToInt64(int64(videoSize))
@ -938,9 +499,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 			ThumbnailURL: &thumbnailURL,
 			Languages:    languages,
 			Locations:    locations,
 			FundingAccountIDs: []string{
 				params.DefaultAccount,
 			},
 		},
 		Author:    util.PtrToString(""),
 		License:   util.PtrToString("Copyrighted (contact publisher)"),
@ -948,18 +506,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		Height:    util.PtrToUint(720),
 		Width:     util.PtrToUint(1280),
 		Fee:       fee,
 		ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
 	}
 	v.walletLock.RLock()
 	defer v.walletLock.RUnlock()
 	if v.mocked {
 		start := time.Now()
 		pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
 			StreamCreateOptions: streamCreateOptions,
 			FileSize:            &videoSize,
 		})
 		timing.TimedComponent("StreamUpdate").Add(time.Since(start))
 		if err != nil {
 			return nil, err
 		}
@ -970,11 +525,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		}, nil
 	}
 	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	streamCreateOptions.ClaimCreateOptions.Title = &v.title
 	streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
-	streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration))
+	streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
 	streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
 	start := time.Now()
 	pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
 		ClearLanguages:      util.PtrToBool(true),
 		ClearLocations:      util.PtrToBool(true),
@ -982,7 +541,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		StreamCreateOptions: streamCreateOptions,
 		FileSize:            &videoSize,
 	})
 	timing.TimedComponent("StreamUpdate").Add(time.Since(start))
 	if err != nil {
 		return nil, err
 	}
--- a/sources/youtubeVideo_test.go
+++ b/sources/youtubeVideo_test.go
@ -1,53 +0,0 @@
 package sources
 import (
 	"regexp"
 	"testing"
 	"github.com/abadojack/whatlanggo"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
 func TestLanguageDetection(t *testing.T) {
 	description := `Om lättkränkta muslimer, och den bristande logiken i vad som anses vara att vanära profeten. Från Moderata riksdagspolitikern Hanif Balis podcast "God Ton", avsnitt 108, från oktober 2020, efter terrordådet där en fransk lärare fick huvudet avskuret efter att undervisat sin mångkulturella klass om frihet.`
 	info := whatlanggo.Detect(description)
 	logrus.Infof("confidence: %.2f", info.Confidence)
 	assert.True(t, info.IsReliable())
 	assert.True(t, info.Lang.Iso6391() != "")
 	assert.Equal(t, "sv", info.Lang.Iso6391())
 	description = `🥳週四直播 | 晚上來開個賽車🔰歡迎各位一起來玩! - PonPonLin蹦蹦林`
 	info = whatlanggo.Detect(description)
 	logrus.Infof("confidence: %.2f", info.Confidence)
 	assert.True(t, info.IsReliable())
 	assert.True(t, info.Lang.Iso6391() != "")
 	assert.Equal(t, "zh", info.Lang.Iso6391())
 	description = `成為這個頻道的會員並獲得獎勵：
 https://www.youtube.com/channel/UCOQFrooz-YGHjYb7s3-MrsQ/join
 _____________________________________________
 想聽我既音樂作品可以去下面LINK
 streetvoice 街聲：
 https://streetvoice.com/CTLam331/
 _____________________________________________
 想學結他、鋼琴
 有關音樂制作工作
 都可以搵我～
 大家快D訂閱喇
 不定期出片
 Website: http://ctlam331.wixsite.com/ctlamusic
 FB PAGE：https://www.facebook.com/ctlam331
 IG：ctlamusic`
 	urlsRegex := regexp.MustCompile(`(?m) ?(f|ht)(tp)(s?)(://)(.*)[.|/](.*)`)
 	descriptionSample := urlsRegex.ReplaceAllString(description, "")
 	info = whatlanggo.Detect(descriptionSample)
 	logrus.Infof("confidence: %.2f", info.Confidence)
 	assert.True(t, info.IsReliable())
 	assert.True(t, info.Lang.Iso6391() != "")
 	assert.Equal(t, "zh", info.Lang.Iso6391())
 }
--- a/tags_manager/tags_mapping.go
+++ b/tags_manager/tags_mapping.go
@ -1,4 +1,4 @@
-package tags_manager
+package tagsManager
 import (
 	"regexp"
@ -71,8 +71,6 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
 	return sanitizedTags, nil
 }
 const TagMaxLength = 50
 func normalizeTag(t string) (string, error) {
 	t = strings.ToLower(t)
 	multipleSpaces := regexp.MustCompile(`\s{2,}`)
@ -89,10 +87,7 @@ func normalizeTag(t string) (string, error) {
 	if weirdChars.MatchString(t) {
 		log.Debugf("tag '%s' has weird stuff in it, skipping\n", t)
 		return "", nil
-	}
+
 	if len(t) > TagMaxLength {
 		log.Debugf("tag '%s' is too long, skipping\n", t)
 		return "", nil
 	}
 	return t, nil
 }
--- a/tags_manager/tags_mapping_test.go
+++ b/tags_manager/tags_mapping_test.go
@ -1,4 +1,4 @@
-package tags_manager
+package tagsManager
 import (
 	"fmt"
--- a/thumbs/uploader.go
+++ b/thumbs/uploader.go
@ -1,20 +1,17 @@
 package thumbs
 import (
 	"google.golang.org/api/youtube/v3"
 	"io"
 	"net/http"
 	"os"
 	"strings"
-	"github.com/lbryio/ytsync/v5/configs"
+	"github.com/lbryio/errors.go"
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-	log "github.com/sirupsen/logrus"
+	"github.com/prometheus/common/log"
 )
 type thumbnailUploader struct {
@ -28,15 +25,13 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/"
 const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
 func (u *thumbnailUploader) downloadThumbnail() error {
-	_ = os.Mkdir(thumbnailPath, 0777)
+	_ = os.Mkdir(thumbnailPath, 0750)
 	img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name)
 	if err != nil {
 		return errors.Err(err)
 	}
 	defer img.Close()
-	if strings.HasPrefix(u.originalUrl, "//") {
+
 		u.originalUrl = "https:" + u.originalUrl
 	}
 	resp, err := http.Get(u.originalUrl)
 	if err != nil {
 		return errors.Err(err)
@ -70,10 +65,7 @@ func (u *thumbnailUploader) uploadThumbnail() error {
 		Key:    key,
 		Body:   thumb,
 		ACL:    aws.String("public-read"),
 		ContentType:  aws.String("image/jpeg"),
 		CacheControl: aws.String("public, max-age=2592000"),
 	})
 	u.mirroredUrl = ThumbnailEndpoint + u.name
 	return errors.Err(err)
 }
@ -84,11 +76,11 @@ func (u *thumbnailUploader) deleteTmpFile() {
 		log.Infof("failed to delete local thumbnail file: %s", err.Error())
 	}
 }
-func MirrorThumbnail(url string, name string) (string, error) {
+func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, error) {
 	tu := thumbnailUploader{
 		originalUrl: url,
 		name:        name,
-		s3Config:    *configs.Configuration.AWSThumbnailsS3Config.GetS3AWSConfig(),
+		s3Config:    s3Config,
 	}
 	err := tu.downloadThumbnail()
 	if err != nil {
@ -101,26 +93,18 @@ func MirrorThumbnail(url string, name string) (string, error) {
 		return "", err
 	}
 	//this is our own S3 storage
 	tu2 := thumbnailUploader{
 		originalUrl: url,
 		name:        name,
 		s3Config:    *configs.Configuration.ThumbnailsS3Config.GetS3AWSConfig(),
 	}
 	err = tu2.uploadThumbnail()
 	if err != nil {
 		return "", err
 	}
 	return tu.mirroredUrl, nil
 }
-func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
+func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
-	var bestWidth ytdl.Thumbnail
+	if thumbnails.Maxres != nil {
-	for _, thumbnail := range thumbnails {
+		return thumbnails.Maxres
-		if bestWidth.Width < thumbnail.Width {
+	} else if thumbnails.High != nil {
-			bestWidth = thumbnail
+		return thumbnails.High
 	} else if thumbnails.Medium != nil {
 		return thumbnails.Medium
 	} else if thumbnails.Standard != nil {
 		return thumbnails.Standard
 	}
-	}
+	return thumbnails.Default
 	return &bestWidth
 }
--- a/timing/timing.go
+++ b/timing/timing.go
@ -1,114 +0,0 @@
 package timing
 import (
 	"sync"
 	"sync/atomic"
 	"time"
 	"github.com/lbryio/ytsync/v5/metrics"
 	"github.com/sirupsen/logrus"
 )
 type Timing struct {
 	component    string
 	milliseconds int64
 	min          int64
 	max          int64
 	invocations  int32
 }
 var timings *sync.Map
 func TimedComponent(component string) *Timing {
 	if timings == nil {
 		timings = &sync.Map{}
 	}
 	stored, _ := timings.LoadOrStore(component, &Timing{
 		component:    component,
 		milliseconds: 0,
 		min:          int64(99999999),
 	})
 	t, _ := stored.(*Timing)
 	return t
 }
 func ClearTimings() {
 	if timings == nil {
 		return
 	}
 	timings.Range(func(key interface{}, value interface{}) bool {
 		timings.Delete(key)
 		return true
 	})
 }
 func Report() {
 	var totalTime time.Duration
 	timings.Range(func(key interface{}, value interface{}) bool {
 		totalTime += value.(*Timing).Get()
 		return true
 	})
 	timings.Range(func(key interface{}, value interface{}) bool {
 		component := key
 		componentRuntime := value.(*Timing).Get().String()
 		percentTime := float64(value.(*Timing).Get()) / float64(totalTime) * 100
 		invocations := value.(*Timing).Invocations()
 		avgTime := (time.Duration(int64(float64(value.(*Timing).Get()) / float64(value.(*Timing).Invocations())))).String()
 		minRuntime := value.(*Timing).Min().String()
 		maxRuntime := value.(*Timing).Max().String()
 		logrus.Printf("component %s ran for %s (%.2f%% of the total time) - invoked %d times with an average of %s per call, a minimum of %s and a maximum of %s",
 			component,
 			componentRuntime,
 			percentTime,
 			invocations,
 			avgTime,
 			minRuntime,
 			maxRuntime,
 		)
 		return true
 	})
 }
 func (t *Timing) Add(d time.Duration) {
 	metrics.Durations.WithLabelValues(t.component).Observe(d.Seconds())
 	atomic.AddInt64(&t.milliseconds, d.Milliseconds())
 	for {
 		oldMin := atomic.LoadInt64(&t.min)
 		if d.Milliseconds() < oldMin {
 			if atomic.CompareAndSwapInt64(&t.min, oldMin, d.Milliseconds()) {
 				break
 			}
 		} else {
 			break
 		}
 	}
 	for {
 		oldMax := atomic.LoadInt64(&t.max)
 		if d.Milliseconds() > oldMax {
 			if atomic.CompareAndSwapInt64(&t.max, oldMax, d.Milliseconds()) {
 				break
 			}
 		} else {
 			break
 		}
 	}
 	atomic.AddInt32(&t.invocations, 1)
 }
 func (t *Timing) Get() time.Duration {
 	ms := atomic.LoadInt64(&t.milliseconds)
 	return time.Duration(ms) * time.Millisecond
 }
 func (t *Timing) Invocations() int32 {
 	return atomic.LoadInt32(&t.invocations)
 }
 func (t *Timing) Min() time.Duration {
 	ms := atomic.LoadInt64(&t.min)
 	return time.Duration(ms) * time.Millisecond
 }
 func (t *Timing) Max() time.Duration {
 	ms := atomic.LoadInt64(&t.max)
 	return time.Duration(ms) * time.Millisecond
 }
--- a/util/archive.go
+++ b/util/archive.go
@ -1,109 +0,0 @@
 package util
 import (
 	"archive/tar"
 	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 )
 func CreateTarball(tarballFilePath string, filePaths []string) error {
 	file, err := os.Create(tarballFilePath)
 	if err != nil {
 		return errors.Err("Could not create tarball file '%s', got error '%s'", tarballFilePath, err.Error())
 	}
 	defer file.Close()
 	tarWriter := tar.NewWriter(file)
 	defer tarWriter.Close()
 	for _, filePath := range filePaths {
 		err := addFileToTarWriter(filePath, tarWriter)
 		if err != nil {
 			return errors.Err("Could not add file '%s', to tarball, got error '%s'", filePath, err.Error())
 		}
 	}
 	return nil
 }
 func addFileToTarWriter(filePath string, tarWriter *tar.Writer) error {
 	file, err := os.Open(filePath)
 	if err != nil {
 		return errors.Err("Could not open file '%s', got error '%s'", filePath, err.Error())
 	}
 	defer file.Close()
 	stat, err := file.Stat()
 	if err != nil {
 		return errors.Err("Could not get stat for file '%s', got error '%s'", filePath, err.Error())
 	}
 	header := &tar.Header{
 		Name:    stat.Name(),
 		Size:    stat.Size(),
 		Mode:    int64(stat.Mode()),
 		ModTime: stat.ModTime(),
 	}
 	err = tarWriter.WriteHeader(header)
 	if err != nil {
 		return errors.Err("Could not write header for file '%s', got error '%s'", filePath, err.Error())
 	}
 	_, err = io.Copy(tarWriter, file)
 	if err != nil {
 		return errors.Err("Could not copy the file '%s' data to the tarball, got error '%s'", filePath, err.Error())
 	}
 	return nil
 }
 func Untar(tarball, target string) error {
 	reader, err := os.Open(tarball)
 	if err != nil {
 		return errors.Err(err)
 	}
 	defer reader.Close()
 	tarReader := tar.NewReader(reader)
 	for {
 		header, err := tarReader.Next()
 		if err == io.EOF {
 			break
 		} else if err != nil {
 			return errors.Err(err)
 		}
 		path := filepath.Join(target, header.Name)
 		info := header.FileInfo()
 		if info.IsDir() {
 			if err = os.MkdirAll(path, info.Mode()); err != nil {
 				return errors.Err(err)
 			}
 			continue
 		}
 		err = extractFile(path, info, tarReader)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }
 func extractFile(path string, info fs.FileInfo, tarReader *tar.Reader) error {
 	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
 	if err != nil {
 		return errors.Err(err)
 	}
 	defer file.Close()
 	_, err = io.Copy(file, tarReader)
 	if err != nil {
 		return errors.Err(err)
 	}
 	return nil
 }
--- a/util/log_wrapper.go
+++ b/util/log_wrapper.go
@ -1,38 +0,0 @@
 package util
 import (
 	"fmt"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 	log "github.com/sirupsen/logrus"
 )
 // SendErrorToSlack Sends an error message to the default channel and to the process log.
 func SendErrorToSlack(format string, a ...interface{}) {
 	message := format
 	if len(a) > 0 {
 		message = fmt.Sprintf(format, a...)
 	}
 	log.Errorln(message)
 	log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
 	err := util.SendToSlack(":sos: ```" + message + "```")
 	log.SetLevel(log.DebugLevel)
 	if err != nil {
 		log.Errorln(err)
 	}
 }
 // SendInfoToSlack Sends an info message to the default channel and to the process log.
 func SendInfoToSlack(format string, a ...interface{}) {
 	message := format
 	if len(a) > 0 {
 		message = fmt.Sprintf(format, a...)
 	}
 	log.Infoln(message)
 	log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
 	err := util.SendToSlack(":information_source: " + message)
 	log.SetLevel(log.DebugLevel)
 	if err != nil {
 		log.Errorln(err)
 	}
 }
--- a/util/util.go
+++ b/util/util.go
@ -1,422 +0,0 @@
 package util
 import (
 	"context"
 	"os"
 	"os/exec"
 	"os/user"
 	"path/filepath"
 	"strconv"
 	"time"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/lbrycrd"
 	"github.com/lbryio/ytsync/v5/configs"
 	"github.com/lbryio/ytsync/v5/timing"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/mitchellh/go-ps"
 	log "github.com/sirupsen/logrus"
 )
 func GetBlobsDir() string {
 	blobsDir := os.Getenv("BLOBS_DIRECTORY")
 	if blobsDir == "" {
 		usr, err := user.Current()
 		if err != nil {
 			log.Error(err.Error())
 			return ""
 		}
 		blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
 	}
 	return blobsDir
 }
 func IsBlobReflectionOff() bool {
 	return os.Getenv("REFLECT_BLOBS") == "false"
 }
 func GetLBRYNetDir() string {
 	lbrynetDir := os.Getenv("LBRYNET_DIR")
 	if lbrynetDir == "" {
 		usr, err := user.Current()
 		if err != nil {
 			log.Errorln(err.Error())
 			return ""
 		}
 		return usr.HomeDir + "/.lbrynet/"
 	}
 	return lbrynetDir
 }
 func GetLbryumDir() string {
 	lbryumDir := os.Getenv("LBRYUM_DIR")
 	if lbryumDir == "" {
 		usr, err := user.Current()
 		if err != nil {
 			log.Errorln(err.Error())
 			return ""
 		}
 		return usr.HomeDir + "/.lbryum/"
 	}
 	return lbryumDir + "/"
 }
 const ALL = true
 const ONLINE = false
 func GetLBRYNetContainer(all bool) (*types.Container, error) {
 	return getDockerContainer("lbrynet", all)
 }
 func getDockerContainer(name string, all bool) (*types.Container, error) {
 	cli, err := client.NewEnvClient()
 	if err != nil {
 		panic(err)
 	}
 	filters := filters.NewArgs()
 	filters.Add("name", name)
 	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: all, Filters: filters})
 	if err != nil {
 		panic(err)
 	}
 	if len(containers) == 0 {
 		return nil, nil
 	}
 	if len(containers) > 1 {
 		return nil, errors.Err("more than one %s container found", name)
 	}
 	return &containers[0], nil
 }
 func IsUsingDocker() bool {
 	useDocker, err := strconv.ParseBool(os.Getenv("LBRYNET_USE_DOCKER"))
 	if err != nil {
 		return false
 	}
 	return useDocker
 }
 func IsRegTest() bool {
 	usesRegtest, err := strconv.ParseBool(os.Getenv("REGTEST"))
 	if err != nil {
 		return false
 	}
 	return usesRegtest
 }
 func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) {
 	chainName := os.Getenv("CHAINNAME")
 	chainParams, ok := lbrycrd.ChainParamsMap[chainName]
 	if !ok {
 		chainParams = lbrycrd.MainNetParams
 	}
 	var lbrycrdd *lbrycrd.Client
 	var err error
 	if lbrycrdString == "" {
 		lbrycrdd, err = lbrycrd.NewWithDefaultURL(&chainParams)
 		if err != nil {
 			return nil, err
 		}
 	} else {
 		lbrycrdd, err = lbrycrd.New(lbrycrdString, &chainParams)
 		if err != nil {
 			return nil, err
 		}
 	}
 	return lbrycrdd, nil
 }
 func ShouldCleanOnStartup() bool {
 	shouldClean, err := strconv.ParseBool(os.Getenv("CLEAN_ON_STARTUP"))
 	if err != nil {
 		return false
 	}
 	return shouldClean
 }
 func IsLbrynetRunning() (bool, error) {
 	if IsUsingDocker() {
 		container, err := GetLBRYNetContainer(ONLINE)
 		if err != nil {
 			return false, err
 		}
 		return container != nil, nil
 	}
 	processes, err := ps.Processes()
 	if err != nil {
 		return true, errors.Err(err)
 	}
 	var daemonProcessId = -1
 	for _, p := range processes {
 		if p.Executable() == "lbrynet" {
 			daemonProcessId = p.Pid()
 			break
 		}
 	}
 	running := daemonProcessId != -1
 	return running, nil
 }
 func CleanForStartup() error {
 	if !IsRegTest() {
 		return errors.Err("never cleanup wallet outside of regtest and with caution. this should only be done in local testing and requires regtest to be on")
 	}
 	running, err := IsLbrynetRunning()
 	if err != nil {
 		return err
 	}
 	if running {
 		err := StopDaemon()
 		if err != nil {
 			return err
 		}
 	}
 	err = CleanupLbrynet()
 	if err != nil {
 		return errors.Err(err)
 	}
 	lbrycrd, err := GetLbrycrdClient(configs.Configuration.LbrycrdString)
 	if err != nil {
 		return errors.Prefix("error getting lbrycrd client", err)
 	}
 	height, err := lbrycrd.GetBlockCount()
 	if err != nil {
 		return errors.Err(err)
 	}
 	const minBlocksForUTXO = 200
 	if height < minBlocksForUTXO {
 		//Start reg test with some credits
 		txs, err := lbrycrd.Generate(uint32(minBlocksForUTXO) - uint32(height))
 		if err != nil {
 			return errors.Err(err)
 		}
 		log.Debugf("REGTEST: Generated %d transactions to get some LBC!", len(txs))
 	}
 	defaultWalletDir := GetDefaultWalletPath()
 	_, err = os.Stat(defaultWalletDir)
 	if os.IsNotExist(err) {
 		return nil
 	}
 	return errors.Err(os.Remove(defaultWalletDir))
 }
 func CleanupLbrynet() error {
 	//make sure lbrynet is off
 	running, err := IsLbrynetRunning()
 	if err != nil {
 		return err
 	}
 	if running {
 		return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
 	}
 	lbrynetDir := GetLBRYNetDir()
 	files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
 	if err != nil {
 		return errors.Err(err)
 	}
 	for _, f := range files {
 		err = os.Remove(f)
 		if err != nil {
 			return errors.Err(err)
 		}
 	}
 	blobsDir := GetBlobsDir()
 	err = os.RemoveAll(blobsDir)
 	if err != nil {
 		return errors.Err(err)
 	}
 	err = os.Mkdir(blobsDir, 0777)
 	if err != nil {
 		return errors.Err(err)
 	}
 	lbryumDir := GetLbryumDir()
 	ledger := "lbc_mainnet"
 	if IsRegTest() {
 		ledger = "lbc_regtest"
 	}
 	lbryumDir = lbryumDir + ledger
 	files, err = filepath.Glob(lbryumDir + "/blockchain.db*")
 	if err != nil {
 		return errors.Err(err)
 	}
 	for _, f := range files {
 		err = os.Remove(f)
 		if err != nil {
 			return errors.Err(err)
 		}
 	}
 	return nil
 }
 var metadataDirInitialized = false
 func GetVideoMetadataDir() string {
 	dir := "./videos_metadata"
 	if !metadataDirInitialized {
 		metadataDirInitialized = true
 		_ = os.MkdirAll(dir, 0755)
 	}
 	return dir
 }
 func CleanupMetadata() error {
 	dir := GetVideoMetadataDir()
 	err := os.RemoveAll(dir)
 	if err != nil {
 		return errors.Err(err)
 	}
 	metadataDirInitialized = false
 	return nil
 }
 func SleepUntilQuotaReset() {
 	PST, _ := time.LoadLocation("America/Los_Angeles")
 	t := time.Now().In(PST)
 	n := time.Date(t.Year(), t.Month(), t.Day(), 24, 2, 0, 0, PST)
 	d := n.Sub(t)
 	if d < 0 {
 		n = n.Add(24 * time.Hour)
 		d = n.Sub(t)
 	}
 	log.Infof("gotta sleep %s until the quota resets", d.String())
 	time.Sleep(d)
 }
 func StartDaemon() error {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("startDaemon").Add(time.Since(start))
 	}(start)
 	if IsUsingDocker() {
 		return startDaemonViaDocker()
 	}
 	return startDaemonViaSystemd()
 }
 func StopDaemon() error {
 	start := time.Now()
 	defer func(start time.Time) {
 		timing.TimedComponent("stopDaemon").Add(time.Since(start))
 	}(start)
 	if IsUsingDocker() {
 		return stopDaemonViaDocker()
 	}
 	return stopDaemonViaSystemd()
 }
 func startDaemonViaDocker() error {
 	container, err := GetLBRYNetContainer(true)
 	if err != nil {
 		return err
 	}
 	cli, err := client.NewEnvClient()
 	if err != nil {
 		panic(err)
 	}
 	err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
 	if err != nil {
 		return errors.Err(err)
 	}
 	return nil
 }
 func stopDaemonViaDocker() error {
 	container, err := GetLBRYNetContainer(ONLINE)
 	if err != nil {
 		return err
 	}
 	cli, err := client.NewEnvClient()
 	if err != nil {
 		panic(err)
 	}
 	err = cli.ContainerStop(context.Background(), container.ID, nil)
 	if err != nil {
 		return errors.Err(err)
 	}
 	return nil
 }
 func startDaemonViaSystemd() error {
 	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
 	if err != nil {
 		return errors.Err(err)
 	}
 	return nil
 }
 func stopDaemonViaSystemd() error {
 	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
 	if err != nil {
 		return errors.Err(err)
 	}
 	return nil
 }
 func GetDefaultWalletPath() string {
 	defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
 	if IsRegTest() {
 		defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
 	}
 	walletPath := os.Getenv("LBRYUM_DIR")
 	if walletPath != "" {
 		defaultWalletDir = walletPath + "/wallets/default_wallet"
 	}
 	return defaultWalletDir
 }
 func GetBlockchainDBPath() string {
 	lbryumDir := os.Getenv("LBRYUM_DIR")
 	if lbryumDir == "" {
 		if IsRegTest() {
 			lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
 		} else {
 			lbryumDir = os.Getenv("HOME") + "/.lbryum"
 		}
 	}
 	defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db"
 	if IsRegTest() {
 		defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
 	}
 	return defaultDB
 }
 func GetBlockchainDirectoryName() string {
 	ledger := "lbc_mainnet"
 	if IsRegTest() {
 		ledger = "lbc_regtest"
 	}
 	return ledger
 }
 func DirSize(path string) (int64, error) {
 	var size int64
 	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 		if !info.IsDir() {
 			size += info.Size()
 		}
 		return err
 	})
 	return size, err
 }
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@ -1,388 +0,0 @@
 package ytapi
 import (
 	"bufio"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 	"github.com/lbryio/ytsync/v5/shared"
 	logUtils "github.com/lbryio/ytsync/v5/util"
 	"github.com/vbauerster/mpb/v7"
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 	"github.com/lbryio/ytsync/v5/downloader"
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/sources"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 	log "github.com/sirupsen/logrus"
 )
 type Video interface {
 	Size() *int64
 	ID() string
 	IDAndNum() string
 	PlaylistPosition() int
 	PublishedAt() time.Time
 	Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex, *sync.WaitGroup, *mpb.Progress) (*sources.SyncSummary, error)
 }
 type byPublishedAt []Video
 func (a byPublishedAt) Len() int           { return len(a) }
 func (a byPublishedAt) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
 type VideoParams struct {
 	VideoDir string
 	Stopper  *stop.Group
 	IPPool   *ip_manager.IPPool
 }
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
 	var videos []Video
 	if quickSync && maxVideos > 50 {
 		maxVideos = 50
 	}
 	allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	videoIDs := make([]string, 0, len(allVideos))
 	for _, video := range allVideos {
 		sv, ok := syncedVideos[video]
 		if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
 			continue
 		}
 		videoIDs = append(videoIDs, video)
 	}
 	log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
 	playlistMap := make(map[string]int64)
 	for i, videoID := range videoIDs {
 		playlistMap[videoID] = int64(i)
 	}
 	//this will ensure that we at least try to sync the video that was marked as last uploaded video in the database.
 	if lastUploadedVideo != "" {
 		_, ok := playlistMap[lastUploadedVideo]
 		if !ok {
 			playlistMap[lastUploadedVideo] = 0
 			videoIDs = append(videoIDs, lastUploadedVideo)
 		}
 	}
 	if len(videoIDs) < 1 {
 		if channelID == mostRecentlyFailedChannel {
 			return nil, errors.Err("playlist items not found")
 		}
 		mostRecentlyFailedChannel = channelID
 	}
 	vids, err := getVideos(channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
 	if err != nil {
 		return nil, err
 	}
 	for _, item := range vids {
 		positionInList := playlistMap[item.ID]
 		videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.Stopper, videoParams.IPPool)
 		if err != nil {
 			return nil, errors.Err(err)
 		}
 		videos = append(videos, videoToAdd)
 	}
 	for k, v := range syncedVideos {
 		newMetadataVersion := int8(2)
 		if !v.Published && v.MetadataVersion >= newMetadataVersion {
 			continue
 		}
 		if _, ok := playlistMap[k]; !ok {
 			videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.Stopper, videoParams.IPPool))
 		}
 	}
 	sort.Sort(byPublishedAt(videos))
 	return videos, nil
 }
 // CountVideosInChannel is unused for now... keeping it here just in case
 func CountVideosInChannel(channelID string) (int, error) {
 	url := "https://socialblade.com/youtube/channel/" + channelID
 	req, _ := http.NewRequest("GET", url, nil)
 	req.Header.Add("User-Agent", downloader.ChromeUA)
 	req.Header.Add("Accept", "*/*")
 	req.Header.Add("Host", "socialblade.com")
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return 0, errors.Err(err)
 	}
 	defer res.Body.Close()
 	var line string
 	scanner := bufio.NewScanner(res.Body)
 	for scanner.Scan() {
 		if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") {
 			line = scanner.Text()
 			break
 		}
 	}
 	if err := scanner.Err(); err != nil {
 		return 0, err
 	}
 	if line == "" {
 		return 0, errors.Err("upload count line not found")
 	}
 	matches := regexp.MustCompile(">([0-9]+)<").FindStringSubmatch(line)
 	if len(matches) != 2 {
 		return 0, errors.Err("upload count not found with regex")
 	}
 	num, err := strconv.Atoi(matches[1])
 	if err != nil {
 		return 0, errors.Err(err)
 	}
 	return num, nil
 }
 func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
 	url := "https://www.youtube.com/channel/" + channelID + "/about"
 	req, _ := http.NewRequest("GET", url, nil)
 	req.Header.Add("User-Agent", downloader.ChromeUA)
 	req.Header.Add("Accept", "*/*")
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	defer res.Body.Close()
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	pageBody := string(body)
 	dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26
 	if dataStartIndex == 25 {
 		dataStartIndex = strings.Index(pageBody, "var ytInitialData = ") + 20
 	}
 	dataEndIndex := strings.Index(pageBody, "]}}};") + 4
 	if dataEndIndex < dataStartIndex {
 		return nil, errors.Err("start index is lower than end index. cannot extract channel info!")
 	}
 	data := pageBody[dataStartIndex:dataEndIndex]
 	var decodedResponse YoutubeStatsResponse
 	err = json.Unmarshal([]byte(data), &decodedResponse)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	return &decodedResponse, nil
 }
 func getVideos(channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
 	config := sdk.GetAPIsConfigs()
 	var videos []*ytdl.YtdlVideo
 	for _, videoID := range videoIDs {
 		if len(videoID) < 5 {
 			continue
 		}
 		select {
 		case <-stopChan:
 			return videos, errors.Err("interrupted by user")
 		default:
 		}
 		state, err := config.VideoState(videoID)
 		if err != nil {
 			return nil, errors.Err(err)
 		}
 		if state == "published" {
 			continue
 		}
 		video, err := downloader.GetVideoInformation(videoID, stopChan, ipPool)
 		if err != nil {
 			errSDK := config.MarkVideoStatus(shared.VideoStatus{
 				ChannelID:     channelID,
 				VideoID:       videoID,
 				Status:        "failed",
 				FailureReason: err.Error(),
 			})
 			logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
 			if errSDK != nil {
 				return nil, errors.Err(errSDK)
 			}
 		} else {
 			videos = append(videos, video)
 		}
 	}
 	return videos, nil
 }
 type YoutubeStatsResponse struct {
 	Contents struct {
 		TwoColumnBrowseResultsRenderer struct {
 			Tabs []struct {
 				TabRenderer struct {
 					Title    string `json:"title"`
 					Selected bool   `json:"selected"`
 					Content  struct {
 						SectionListRenderer struct {
 							Contents []struct {
 								ItemSectionRenderer struct {
 									Contents []struct {
 										ChannelAboutFullMetadataRenderer struct {
 											Description struct {
 												SimpleText string `json:"simpleText"`
 											} `json:"description"`
 											ViewCountText struct {
 												SimpleText string `json:"simpleText"`
 											} `json:"viewCountText"`
 											JoinedDateText struct {
 												Runs []struct {
 													Text string `json:"text"`
 												} `json:"runs"`
 											} `json:"joinedDateText"`
 											CanonicalChannelURL        string `json:"canonicalChannelUrl"`
 											BypassBusinessEmailCaptcha bool   `json:"bypassBusinessEmailCaptcha"`
 											Title                      struct {
 												SimpleText string `json:"simpleText"`
 											} `json:"title"`
 											Avatar struct {
 												Thumbnails []struct {
 													URL    string `json:"url"`
 													Width  int    `json:"width"`
 													Height int    `json:"height"`
 												} `json:"thumbnails"`
 											} `json:"avatar"`
 											ShowDescription  bool `json:"showDescription"`
 											DescriptionLabel struct {
 												Runs []struct {
 													Text string `json:"text"`
 												} `json:"runs"`
 											} `json:"descriptionLabel"`
 											DetailsLabel struct {
 												Runs []struct {
 													Text string `json:"text"`
 												} `json:"runs"`
 											} `json:"detailsLabel"`
 											ChannelID string `json:"channelId"`
 										} `json:"channelAboutFullMetadataRenderer"`
 									} `json:"contents"`
 								} `json:"itemSectionRenderer"`
 							} `json:"contents"`
 						} `json:"sectionListRenderer"`
 					} `json:"content"`
 				} `json:"tabRenderer"`
 			} `json:"tabs"`
 		} `json:"twoColumnBrowseResultsRenderer"`
 	} `json:"contents"`
 	Header struct {
 		C4TabbedHeaderRenderer struct {
 			ChannelID string `json:"channelId"`
 			Title     string `json:"title"`
 			Avatar    struct {
 				Thumbnails []struct {
 					URL    string `json:"url"`
 					Width  int    `json:"width"`
 					Height int    `json:"height"`
 				} `json:"thumbnails"`
 			} `json:"avatar"`
 			Banner struct {
 				Thumbnails []struct {
 					URL    string `json:"url"`
 					Width  int    `json:"width"`
 					Height int    `json:"height"`
 				} `json:"thumbnails"`
 			} `json:"banner"`
 			VisitTracking struct {
 				RemarketingPing string `json:"remarketingPing"`
 			} `json:"visitTracking"`
 			SubscriberCountText struct {
 				SimpleText string `json:"simpleText"`
 			} `json:"subscriberCountText"`
 		} `json:"c4TabbedHeaderRenderer"`
 	} `json:"header"`
 	Metadata struct {
 		ChannelMetadataRenderer struct {
 			Title                string   `json:"title"`
 			Description          string   `json:"description"`
 			RssURL               string   `json:"rssUrl"`
 			ChannelConversionURL string   `json:"channelConversionUrl"`
 			ExternalID           string   `json:"externalId"`
 			Keywords             string   `json:"keywords"`
 			OwnerUrls            []string `json:"ownerUrls"`
 			Avatar               struct {
 				Thumbnails []struct {
 					URL    string `json:"url"`
 					Width  int    `json:"width"`
 					Height int    `json:"height"`
 				} `json:"thumbnails"`
 			} `json:"avatar"`
 			ChannelURL       string `json:"channelUrl"`
 			IsFamilySafe     bool   `json:"isFamilySafe"`
 			VanityChannelURL string `json:"vanityChannelUrl"`
 		} `json:"channelMetadataRenderer"`
 	} `json:"metadata"`
 	Topbar struct {
 		DesktopTopbarRenderer struct {
 			CountryCode string `json:"countryCode"`
 		} `json:"desktopTopbarRenderer"`
 	} `json:"topbar"`
 	Microformat struct {
 		MicroformatDataRenderer struct {
 			URLCanonical string `json:"urlCanonical"`
 			Title        string `json:"title"`
 			Description  string `json:"description"`
 			Thumbnail    struct {
 				Thumbnails []struct {
 					URL    string `json:"url"`
 					Width  int    `json:"width"`
 					Height int    `json:"height"`
 				} `json:"thumbnails"`
 			} `json:"thumbnail"`
 			SiteName           string   `json:"siteName"`
 			AppName            string   `json:"appName"`
 			AndroidPackage     string   `json:"androidPackage"`
 			IosAppStoreID      string   `json:"iosAppStoreId"`
 			IosAppArguments    string   `json:"iosAppArguments"`
 			OgType             string   `json:"ogType"`
 			URLApplinksWeb     string   `json:"urlApplinksWeb"`
 			URLApplinksIos     string   `json:"urlApplinksIos"`
 			URLApplinksAndroid string   `json:"urlApplinksAndroid"`
 			URLTwitterIos      string   `json:"urlTwitterIos"`
 			URLTwitterAndroid  string   `json:"urlTwitterAndroid"`
 			TwitterCardType    string   `json:"twitterCardType"`
 			TwitterSiteHandle  string   `json:"twitterSiteHandle"`
 			SchemaDotOrgType   string   `json:"schemaDotOrgType"`
 			Noindex            bool     `json:"noindex"`
 			Unlisted           bool     `json:"unlisted"`
 			FamilySafe         bool     `json:"familySafe"`
 			Tags               []string `json:"tags"`
 		} `json:"microformatDataRenderer"`
 	} `json:"microformat"`
 }
--- a/ytapi/ytapi_test.go
+++ b/ytapi/ytapi_test.go
@ -1,13 +0,0 @@
 package ytapi
 import (
 	"testing"
 	"github.com/stretchr/testify/assert"
 )
 func TestChannelInfo(t *testing.T) {
 	info, err := ChannelInfo("UCNQfQvFMPnInwsU_iGYArJQ")
 	assert.NoError(t, err)
 	assert.NotNil(t, info)
 }
		`@ -1,2 +0,0 @@`
			`#!/usr/bin/env bash`
			`curl --fail http://localhost:6300/api/status \|\| exit 1`