make it fast!
This commit is contained in:
parent
3019f0864c
commit
363f33dadf
1 changed files with 76 additions and 39 deletions
115
main.go
115
main.go
|
@ -1,9 +1,14 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
@ -38,7 +43,7 @@ func main() {
|
|||
if _, err := os.Stat(blobsDir); os.IsNotExist(err) {
|
||||
panic(errors.Err("directory doesn't exist: %s", blobsDir))
|
||||
}
|
||||
used, err := getUsedSpace()
|
||||
used, _, err := getUsedSpace()
|
||||
if err != nil {
|
||||
logrus.Errorln(err.Error())
|
||||
return
|
||||
|
@ -51,29 +56,29 @@ func main() {
|
|||
logrus.Errorln(err.Error())
|
||||
return
|
||||
}
|
||||
used, err := getUsedSpace()
|
||||
usage, _, err := getUsedSpace()
|
||||
if err != nil {
|
||||
logrus.Errorln(err.Error())
|
||||
return
|
||||
}
|
||||
logrus.Infof("disk usage: %.2f%%\n", used*100)
|
||||
logrus.Infof("disk usage: %.2f%%\n", usage*100)
|
||||
logrus.Infoln("Done cleaning up")
|
||||
}
|
||||
}
|
||||
|
||||
// GetUsedSpace returns a value between 0 and 1, with 0 being completely empty and 1 being full, for the disk that holds the provided path
|
||||
func getUsedSpace() (float64, error) {
|
||||
func getUsedSpace() (usage float64, used uint64, err error) {
|
||||
var stat syscall.Statfs_t
|
||||
err := syscall.Statfs(blobsDir, &stat)
|
||||
err = syscall.Statfs(blobsDir, &stat)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return 0, 0, err
|
||||
}
|
||||
// Available blocks * size per block = available space in bytes
|
||||
all := stat.Blocks * uint64(stat.Bsize)
|
||||
free := stat.Bfree * uint64(stat.Bsize)
|
||||
used := all - free
|
||||
used = all - free
|
||||
|
||||
return float64(used) / float64(all), nil
|
||||
return float64(used) / float64(all), used, nil
|
||||
}
|
||||
|
||||
func WipeOldestBlobs() (err error) {
|
||||
|
@ -83,47 +88,79 @@ func WipeOldestBlobs() (err error) {
|
|||
}
|
||||
datedFiles := make([]datedFile, 0, 5000)
|
||||
checkedBlobs := int32(0)
|
||||
err = godirwalk.Walk(blobsDir, &godirwalk.Options{
|
||||
Callback: func(osPathname string, de *godirwalk.Dirent) error {
|
||||
if !de.IsDir() {
|
||||
i := atomic.AddInt32(&checkedBlobs, 1)
|
||||
if i%100 == 0 {
|
||||
logrus.Infof("checked %d blobs", i)
|
||||
}
|
||||
if de.IsRegular() {
|
||||
stat, err := os.Stat(osPathname)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
datedFiles = append(datedFiles, datedFile{
|
||||
Atime: atime.Atime(stat),
|
||||
FullPath: osPathname,
|
||||
})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc)
|
||||
})
|
||||
dirs, err := ioutil.ReadDir(blobsDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lock := &sync.Mutex{}
|
||||
wg := &sync.WaitGroup{}
|
||||
startTime := time.Now()
|
||||
usage, used, err := getUsedSpace()
|
||||
if err != nil {
|
||||
logrus.Errorln(err.Error())
|
||||
return
|
||||
}
|
||||
for i, dir := range dirs {
|
||||
if (i+1)%(runtime.NumCPU()-1) == 0 {
|
||||
wg.Wait()
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(dir string, lock *sync.Mutex, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
err = godirwalk.Walk(filepath.Join(blobsDir, dir), &godirwalk.Options{
|
||||
Callback: func(osPathname string, de *godirwalk.Dirent) error {
|
||||
if !de.IsDir() {
|
||||
i := atomic.AddInt32(&checkedBlobs, 1)
|
||||
if i%100 == 0 {
|
||||
logrus.Infof("checked %d blobs", i)
|
||||
}
|
||||
if i%100 == 0 {
|
||||
processedSize := float64(i * 2)
|
||||
speed := processedSize / time.Since(startTime).Seconds()
|
||||
eta, err := time.ParseDuration(fmt.Sprintf("%ds", int(float64(used)/1024./1024./speed)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logrus.Infof("%.2f GB checked (speed: %.2f MB/s) ETA: %s", processedSize/1024.0, speed, eta.String())
|
||||
}
|
||||
if de.IsRegular() {
|
||||
stat, err := os.Stat(osPathname)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lock.Lock()
|
||||
datedFiles = append(datedFiles, datedFile{
|
||||
Atime: atime.Atime(stat),
|
||||
FullPath: osPathname,
|
||||
})
|
||||
lock.Unlock()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc)
|
||||
})
|
||||
if err != nil {
|
||||
logrus.Errorf(errors.FullTrace(err))
|
||||
}
|
||||
}(dir.Name(), lock, wg)
|
||||
}
|
||||
wg.Wait()
|
||||
sort.Slice(datedFiles, func(i, j int) bool {
|
||||
return datedFiles[i].Atime.Before(datedFiles[j].Atime)
|
||||
})
|
||||
//delete the first 5000 blobs
|
||||
|
||||
spaceToFreeUp := ((float64(used) / usage) * 0.1) / 1024. / 1024.
|
||||
blobsToDelete := int(spaceToFreeUp / 2)
|
||||
logrus.Infof("spaceToFreeUp: %.2f MB - %d blobs", spaceToFreeUp, blobsToDelete)
|
||||
for i, df := range datedFiles {
|
||||
if i >= 5000 {
|
||||
if i >= blobsToDelete {
|
||||
break
|
||||
}
|
||||
if i%100 == 0 {
|
||||
logrus.Infof("[%s] would delete %s", df.Atime.String(), df.FullPath)
|
||||
err = os.Remove(df.FullPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
//err = os.Remove(df.FullPath)
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue