129 lines
2.9 KiB
Go
129 lines
2.9 KiB
Go
package main
|
|
|
|
import (
|
|
"os"
|
|
"sort"
|
|
"strconv"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"time"
|
|
|
|
"reflector-cleaner/atime"
|
|
|
|
"github.com/google/gops/agent"
|
|
"github.com/karrick/godirwalk"
|
|
"github.com/lbryio/lbry.go/v2/extras/errors"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
var blobsDir string
|
|
|
|
func main() {
|
|
if err := agent.Listen(agent.Options{}); err != nil {
|
|
panic(err)
|
|
}
|
|
if len(os.Args) != 2 {
|
|
panic("you must pass 1 argument: the path of the blobs directory")
|
|
}
|
|
blobsDir = os.Args[1]
|
|
thresholdStr := os.Getenv("DISK_THRESHOLD")
|
|
threshold := 0.90
|
|
var err error
|
|
if thresholdStr != "" {
|
|
threshold, err = strconv.ParseFloat(thresholdStr, 64)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
if _, err := os.Stat(blobsDir); os.IsNotExist(err) {
|
|
panic(errors.Err("directory doesn't exist: %s", blobsDir))
|
|
}
|
|
used, err := getUsedSpace()
|
|
if err != nil {
|
|
logrus.Errorln(err.Error())
|
|
return
|
|
}
|
|
logrus.Infof("disk usage: %.2f%%\n", used*100)
|
|
if used > threshold {
|
|
logrus.Infof("over %.2f%%, cleaning up", threshold*100)
|
|
err = WipeOldestBlobs()
|
|
if err != nil {
|
|
logrus.Errorln(err.Error())
|
|
return
|
|
}
|
|
used, err := getUsedSpace()
|
|
if err != nil {
|
|
logrus.Errorln(err.Error())
|
|
return
|
|
}
|
|
logrus.Infof("disk usage: %.2f%%\n", used*100)
|
|
logrus.Infoln("Done cleaning up")
|
|
}
|
|
}
|
|
|
|
// GetUsedSpace returns a value between 0 and 1, with 0 being completely empty and 1 being full, for the disk that holds the provided path
|
|
func getUsedSpace() (float64, error) {
|
|
var stat syscall.Statfs_t
|
|
err := syscall.Statfs(blobsDir, &stat)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
// Available blocks * size per block = available space in bytes
|
|
all := stat.Blocks * uint64(stat.Bsize)
|
|
free := stat.Bfree * uint64(stat.Bsize)
|
|
used := all - free
|
|
|
|
return float64(used) / float64(all), nil
|
|
}
|
|
|
|
func WipeOldestBlobs() (err error) {
|
|
type datedFile struct {
|
|
Atime time.Time
|
|
FullPath string
|
|
}
|
|
datedFiles := make([]datedFile, 0, 5000)
|
|
checkedBlobs := int32(0)
|
|
err = godirwalk.Walk(blobsDir, &godirwalk.Options{
|
|
Callback: func(osPathname string, de *godirwalk.Dirent) error {
|
|
if !de.IsDir() {
|
|
i := atomic.AddInt32(&checkedBlobs, 1)
|
|
if i%100 == 0 {
|
|
logrus.Infof("checked %d blobs", i)
|
|
}
|
|
if de.IsRegular() {
|
|
stat, err := os.Stat(osPathname)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
datedFiles = append(datedFiles, datedFile{
|
|
Atime: atime.Atime(stat),
|
|
FullPath: osPathname,
|
|
})
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
sort.Slice(datedFiles, func(i, j int) bool {
|
|
return datedFiles[i].Atime.Before(datedFiles[j].Atime)
|
|
})
|
|
//delete the first 5000 blobs
|
|
for i, df := range datedFiles {
|
|
if i >= 5000 {
|
|
break
|
|
}
|
|
if i%100 == 0 {
|
|
logrus.Infof("[%s] would delete %s", df.Atime.String(), df.FullPath)
|
|
}
|
|
//err = os.Remove(df.FullPath)
|
|
//if err != nil {
|
|
// return err
|
|
//}
|
|
}
|
|
return nil
|
|
}
|