LBRYCommunity/reflector.go
1
0
Fork 0
Code Issues 7 Pull requests Projects Releases 10 Packages Wiki Activity

count errors too

Browse source
This commit is contained in:
Alex Grintsvayg 2018-05-18 09:37:26 -04:00
parent 39914c8ab8
commit 8e128ea851
1 changed files with 22 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
cmd/upload.go
View file

@ -74,11 +74,17 @@ func uploadCmd(cmd *cobra.Command, args []string) {
sdCount := 0 sdCount := 0
blobCount := 0 blobCount := 0
errCount := 0
workerWG := &sync.WaitGroup{} workerWG := &sync.WaitGroup{}
filenameChan := make(chan string) filenameChan := make(chan string)
counterWG := &sync.WaitGroup{} counterWG := &sync.WaitGroup{}
countChan := make(chan bool) countChan := make(chan int)
const (
sdInc = 1
blobInc = 2
errInc = 3
)
for i := 0; i < workers; i++ { for i := 0; i < workers; i++ {
go func(i int) { go func(i int) {
@ -105,6 +111,10 @@ func uploadCmd(cmd *cobra.Command, args []string) {
hash := peer.GetBlobHash(blob) hash := peer.GetBlobHash(blob)
if hash != filename { if hash != filename {
log.Errorf("worker %d: filename does not match hash (%s != %s), skipping", i, filename, hash) log.Errorf("worker %d: filename does not match hash (%s != %s), skipping", i, filename, hash)
select {
case countChan <- errInc:
case <-stopper.Chan():
}
continue continue
} }
@ -112,14 +122,14 @@ func uploadCmd(cmd *cobra.Command, args []string) {
log.Printf("worker %d: PUTTING SD BLOB %s", i, hash) log.Printf("worker %d: PUTTING SD BLOB %s", i, hash)
blobStore.PutSD(hash, blob) blobStore.PutSD(hash, blob)
select { select {
case countChan <- true: case countChan <- sdInc:
case <-stopper.Chan(): case <-stopper.Chan():
} }
} else { } else {
log.Printf("worker %d: putting %s", i, hash) log.Printf("worker %d: putting %s", i, hash)
blobStore.Put(hash, blob) blobStore.Put(hash, blob)
select { select {
case countChan <- false: case countChan <- blobInc:
case <-stopper.Chan(): case <-stopper.Chan():
} }
} }
@ -135,17 +145,21 @@ func uploadCmd(cmd *cobra.Command, args []string) {
select { select {
case <-stopper.Chan(): case <-stopper.Chan():
return return
case isSD, ok := <-countChan: case countType, ok := <-countChan:
if !ok { if !ok {
return return
} else if isSD { }
switch countType {
case sdInc:
sdCount++ sdCount++
} else { case blobInc:
blobCount++ blobCount++
case errInc:
errCount++
} }
} }
if (sdCount+blobCount)%50 == 0 { if (sdCount+blobCount)%50 == 0 {
log.Printf("%d of %d done (%s elapsed, %.2fs per blob)", sdCount+blobCount, totalCount-existsCount, time.Now().Sub(startTime).String(), time.Now().Sub(startTime).Seconds()/float64(sdCount+blobCount)) log.Printf("%d of %d done (%s elapsed, %.3fs per blob)", sdCount+blobCount, totalCount-existsCount, time.Now().Sub(startTime).String(), time.Now().Sub(startTime).Seconds()/float64(sdCount+blobCount))
} }
} }
}() }()
@ -175,6 +189,7 @@ Upload:
log.Printf("%d SD blobs uploaded", sdCount) log.Printf("%d SD blobs uploaded", sdCount)
log.Printf("%d content blobs uploaded", blobCount) log.Printf("%d content blobs uploaded", blobCount)
log.Printf("%d blobs already stored", existsCount) log.Printf("%d blobs already stored", existsCount)
log.Printf("%d errors encountered", errCount)
} }
func isJSON(data []byte) bool { func isJSON(data []byte) bool {