ytsync/manager/setup.go

package manager

import (
	"fmt"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/lbryio/lbry.go/extras/errors"
	"github.com/lbryio/lbry.go/extras/jsonrpc"
	"github.com/lbryio/lbry.go/extras/util"
	"github.com/lbryio/lbry.go/lbrycrd"

	"github.com/lbryio/ytsync/tagsManager"
	"github.com/lbryio/ytsync/thumbs"

	"github.com/shopspring/decimal"
	log "github.com/sirupsen/logrus"
	"google.golang.org/api/googleapi/transport"
	"google.golang.org/api/youtube/v3"
)

const minimumRefillAmount = 3

func (s *Sync) enableAddressReuse() error {
	accountsResponse, err := s.daemon.AccountList()
	if err != nil {
		return errors.Err(err)
	}
	accounts := accountsResponse.LBCMainnet
	if os.Getenv("REGTEST") == "true" {
		accounts = accountsResponse.LBCRegtest
	}
	for _, a := range accounts {
		_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
			ChangeMaxUses: 1000,
		})
		if err != nil {
			return errors.Err(err)
		}
	}
	return nil
}
func (s *Sync) walletSetup() error {
	//prevent unnecessary concurrent execution
	s.walletMux.Lock()
	defer s.walletMux.Unlock()
	err := s.ensureChannelOwnership()
	if err != nil {
		return err
	}

	balanceResp, err := s.daemon.AccountBalance(nil)
	if err != nil {
		return err
	} else if balanceResp == nil {
		return errors.Err("no response")
	}
	balance, err := strconv.ParseFloat((string)(*balanceResp), 64)
	if err != nil {
		return errors.Err(err)
	}
	log.Debugf("Starting balance is %.4f", balance)

	n, err := s.CountVideos()
	if err != nil {
		return err
	}
	numOnSource := int(n)

	log.Debugf("Source channel has %d videos", numOnSource)
	if numOnSource == 0 {
		return nil
	}

	s.syncedVideosMux.RLock()
	numPublished := len(s.syncedVideos) //should we only count published videos? Credits are allocated even for failed ones...
	s.syncedVideosMux.RUnlock()
	log.Debugf("We already allocated credits for %d videos", numPublished)

	if numOnSource-numPublished > s.Manager.videosLimit {
		numOnSource = s.Manager.videosLimit
	}

	minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
	if s.Manager.syncStatus == StatusPendingUpgrade {
		videosToUpgrade := 0
		for _, v := range s.syncedVideos {
			if v.Published && v.MetadataVersion < 2 {
				videosToUpgrade++
			}
		}
		minBalance += float64(videosToUpgrade) * 0.001
	}
	if numPublished > numOnSource && balance < minimumRefillAmount {
		SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
		minBalance = minimumRefillAmount
	}
	amountToAdd := minBalance - balance

	if s.Refill > 0 {
		if amountToAdd < 0 {
			amountToAdd = float64(s.Refill)
		} else {
			amountToAdd += float64(s.Refill)
		}
	}

	if amountToAdd > 0 {
		if amountToAdd < minimumRefillAmount {
			amountToAdd = minimumRefillAmount
		}
		err := s.addCredits(amountToAdd)
		if err != nil {
			return errors.Err(err)
		}
	}

	claimAddress, err := s.daemon.AddressList(nil)
	if err != nil {
		return err
	} else if claimAddress == nil {
		return errors.Err("could not get unused address")
	}
	s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
	if s.claimAddress == "" {
		return errors.Err("found blank claim address")
	}

	err = s.ensureEnoughUTXOs()
	if err != nil {
		return err
	}

	return nil
}

func (s *Sync) ensureEnoughUTXOs() error {
	accounts, err := s.daemon.AccountList()
	if err != nil {
		return errors.Err(err)
	}
	accountsNet := (*accounts).LBCMainnet
	if os.Getenv("REGTEST") == "true" {
		accountsNet = (*accounts).LBCRegtest
	}
	defaultAccount := ""
	for _, account := range accountsNet {
		if account.IsDefault {
			defaultAccount = account.ID
			break
		}
	}
	if defaultAccount == "" {
		return errors.Err("No default account found")
	}

	utxolist, err := s.daemon.UTXOList(&defaultAccount)
	if err != nil {
		return err
	} else if utxolist == nil {
		return errors.Err("no response")
	}

	target := 40
	slack := int(float32(0.1) * float32(target))
	count := 0

	for _, utxo := range *utxolist {
		amount, _ := strconv.ParseFloat(utxo.Amount, 64)
		if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && amount != 0.0 {
			count++
		}
	}
	log.Infof("utxo count: %d", count)
	if count < target-slack {
		balance, err := s.daemon.AccountBalance(&defaultAccount)
		if err != nil {
			return err
		} else if balance == nil {
			return errors.Err("no response")
		}

		balanceAmount, err := strconv.ParseFloat((string)(*balance), 64)
		if err != nil {
			return errors.Err(err)
		}
		broadcastFee := 0.01
		amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)

		log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance)

		prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target))
		if err != nil {
			return err
		} else if prefillTx == nil {
			return errors.Err("no response")
		}

		err = s.waitForNewBlock()
		if err != nil {
			return err
		}
	} else if !allUTXOsConfirmed(utxolist) {
		log.Println("Waiting for previous txns to confirm")
		err := s.waitForNewBlock()
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *Sync) waitForNewBlock() error {
	status, err := s.daemon.Status()
	if err != nil {
		return err
	}

	for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
		time.Sleep(5 * time.Second)
		status, err = s.daemon.Status()
		if err != nil {
			return err
		}
	}
	currentBlock := status.Wallet.Blocks
	for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
		if i%3 == 0 {
			log.Printf("Waiting for new block (%d)...", currentBlock+1)
		}
		time.Sleep(10 * time.Second)
		status, err = s.daemon.Status()
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Sync) ensureChannelOwnership() error {
	if s.LbryChannelName == "" {
		return errors.Err("no channel name set")
	}
	//@TODO: get rid of this when imported channels are supported
	if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
		return nil
	}
	channels, err := s.daemon.ChannelList(nil, 1, 50)
	if err != nil {
		return err
	} else if channels == nil {
		return errors.Err("no channel response")
	}
	//special case for wallets we don't retain full control anymore
	if len((*channels).Items) > 1 {
		// This wallet is probably not under our control anymore but we still want to publish to it
		// here we shall check if within all the channels there is one that was created by ytsync
		SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
		if s.lbryChannelID == "" {
			return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
		}
		for _, c := range (*channels).Items {
			if c.ClaimID != s.lbryChannelID {
				if c.Name != s.LbryChannelName {
					return errors.Err("the channel in the wallet is different than the channel in the database")
				}
				return nil // we have the ytsync channel and both the claimID and the channelName from the database are correct
			}
		}
	}
	channelUsesOldMetadata := false
	if len((*channels).Items) == 1 {
		channel := ((*channels).Items)[0]
		if channel.Name == s.LbryChannelName {
			channelUsesOldMetadata = channel.Value.GetThumbnail() == nil
			//TODO: eventually get rid of this when the whole db is filled
			if s.lbryChannelID == "" {
				err = s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, channel.ClaimID)
			} else if channel.ClaimID != s.lbryChannelID {
				return errors.Err("the channel in the wallet is different than the channel in the database")
			}
			s.lbryChannelID = channel.ClaimID
			if !channelUsesOldMetadata {
				return err
			}
		} else {
			return errors.Err("this channel does not belong to this wallet! Expected: %s, found: %s", s.LbryChannelName, channel.Name)
		}
	}

	channelBidAmount := channelClaimAmount

	balanceResp, err := s.daemon.AccountBalance(nil)
	if err != nil {
		return err
	} else if balanceResp == nil {
		return errors.Err("no response")
	}
	balance, err := decimal.NewFromString((string)(*balanceResp))
	if err != nil {
		return errors.Err(err)
	}

	if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
		err = s.addCredits(channelBidAmount + 0.1)
		if err != nil {
			return err
		}
	}
	client := &http.Client{
		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
	}

	service, err := youtube.New(client)
	if err != nil {
		return errors.Prefix("error creating YouTube service", err)
	}

	response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
	if err != nil {
		return errors.Prefix("error getting channel details", err)
	}

	if len(response.Items) < 1 {
		return errors.Err("youtube channel not found")
	}

	channelInfo := response.Items[0].Snippet
	channelBranding := response.Items[0].BrandingSettings

	thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
	if err != nil {
		return err
	}

	var bannerURL *string
	if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
		bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
		if err != nil {
			return err
		}
		bannerURL = &bURL
	}

	var languages []string = nil
	if channelInfo.DefaultLanguage != "" {
		languages = []string{channelInfo.DefaultLanguage}
	}
	var locations []jsonrpc.Location = nil
	if channelInfo.Country != "" {
		locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
	}
	var c *jsonrpc.TransactionSummary
	if channelUsesOldMetadata {
		c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
			ClearTags:      util.PtrToBool(true),
			ClearLocations: util.PtrToBool(true),
			ClearLanguages: util.PtrToBool(true),
			ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
				ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
					Title:        channelInfo.Title,
					Description:  channelInfo.Description,
					Tags:         tagsManager.GetTagsForChannel(s.YoutubeChannelID),
					Languages:    languages,
					Locations:    locations,
					ThumbnailURL: &thumbnailURL,
				},
				CoverURL: bannerURL,
			},
		})
	} else {
		c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
			ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
				Title:        channelInfo.Title,
				Description:  channelInfo.Description,
				Tags:         tagsManager.GetTagsForChannel(s.YoutubeChannelID),
				Languages:    languages,
				Locations:    locations,
				ThumbnailURL: &thumbnailURL,
			},
			CoverURL: bannerURL,
		})
	}

	if err != nil {
		return err
	}
	s.lbryChannelID = c.Outputs[0].ClaimID
	return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
}

func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool {
	if utxolist == nil {
		return false
	}

	if len(*utxolist) < 1 {
		return false
	}

	for _, utxo := range *utxolist {
		if utxo.Height == 0 {
			return false
		}
	}

	return true
}

func (s *Sync) addCredits(amountToAdd float64) error {
	log.Printf("Adding %f credits", amountToAdd)
	var lbrycrdd *lbrycrd.Client
	var err error
	if s.LbrycrdString == "" {
		lbrycrdd, err = lbrycrd.NewWithDefaultURL()
		if err != nil {
			return err
		}
	} else {
		lbrycrdd, err = lbrycrd.New(s.LbrycrdString)
		if err != nil {
			return err
		}
	}

	addressResp, err := s.daemon.AddressUnused(nil)
	if err != nil {
		return err
	} else if addressResp == nil {
		return errors.Err("no response")
	}
	address := string(*addressResp)

	_, err = lbrycrdd.SimpleSend(address, amountToAdd)
	if err != nil {
		return err
	}

	wait := 15 * time.Second
	log.Println("Waiting " + wait.String() + " for lbryum to let us know we have the new transaction")
	time.Sleep(wait)

	return nil
}