LBRYCommunity/reflector.go
1
0
Fork 0
Code Issues 7 Pull requests Projects Releases 10 Packages Wiki Activity

only touch blobs when you get them

Browse source
This commit is contained in:
Alex Grintsvayg 2021-01-05 11:36:33 -05:00
parent b33651ae26
commit 49714c02a6
No known key found for this signature in database
GPG key ID: AEB3F089F86A22B5
3 changed files with 34 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
db/db.go
View file

@ -111,6 +111,26 @@ func (s *SQL) AddBlobs(hash []string) error {
return nil
}
func (s *SQL) insertBlobs(hashes []string) error {
var (
q string
args []interface{}
)
dayAgo := time.Now().AddDate(0, 0, -1)
q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
for _, hash := range hashes {
q += "(?,?,?,?),"
args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
}
q = strings.TrimSuffix(q, ",")
_, err := s.exec(q, args...)
if err != nil {
return err
}
return nil
}
func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
if length <= 0 {
return 0, errors.Err("length must be positive")
@ -153,26 +173,6 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
return blobID, nil
}
func (s *SQL) insertBlobs(hashes []string) error {
var (
q string
args []interface{}
)
dayAgo := time.Now().AddDate(0, 0, -1)
q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
for _, hash := range hashes {
q += "(?,?,?,?),"
args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
}
q = strings.TrimSuffix(q, ",")
_, err := s.exec(q, args...)
if err != nil {
return err
}
return nil
}
func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
var (
q string
@ -212,8 +212,8 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
}
// HasBlob checks if the database contains the blob information.
func (s *SQL) HasBlob(hash string) (bool, error) {
exists, err := s.HasBlobs([]string{hash})
func (s *SQL) HasBlob(hash string, touch bool) (bool, error) {
exists, err := s.HasBlobs([]string{hash}, touch)
if err != nil {
return false, err
}
@ -221,15 +221,16 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
}
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
exists, idsNeedingTouch, err := s.hasBlobs(hashes)
go func() {
if touch {
if s.TrackAccess == TrackAccessBlobs {
s.touchBlobs(idsNeedingTouch)
} else if s.TrackAccess == TrackAccessStreams {
s.touchStreams(idsNeedingTouch)
}
}()
}
return exists, err
}

2
reflector/uploader.go
View file

@ -74,7 +74,7 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
var exists map[string]bool
if !u.skipExistsCheck {
exists, err = u.db.HasBlobs(hashes)
exists, err = u.db.HasBlobs(hashes, false)
if err != nil {
return err
}

16
store/dbbacked.go
View file

@ -32,12 +32,12 @@ func (d *DBBackedStore) Name() string { return nameDBBacked }
// Has returns true if the blob is in the store
func (d *DBBackedStore) Has(hash string) (bool, error) {
return d.db.HasBlob(hash)
return d.db.HasBlob(hash, false)
}
// Get gets the blob
func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
has, err := d.db.HasBlob(hash)
has, err := d.db.HasBlob(hash, true)
if err != nil {
return nil, err
}
@ -46,12 +46,10 @@ func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
}
b, err := d.blobs.Get(hash)
if d.deleteOnMiss {
if err != nil && errors.Is(err, ErrBlobNotFound) {
e2 := d.Delete(hash)
if e2 != nil {
log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
}
if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) {
e2 := d.Delete(hash)
if e2 != nil {
log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
}
}
@ -110,7 +108,7 @@ func (d *DBBackedStore) Block(hash string) error {
return err
}
has, err := d.db.HasBlob(hash)
has, err := d.db.HasBlob(hash, false)
if err != nil {
return err
}