dont overallocate ram when reading blobs from disk

ReadFile checks the file size and allocates a bit more space than we
expect we'll need. ReadAll uses Go's standard resizing algo, which
doubles the underlying array each time you hit the end. So ReadAll
ends up allocating 4MB for a full blob, while ReadFile allocates
slightly over 2MB.
This commit is contained in:
Alex Grintsvayg 2020-11-21 11:39:15 -05:00
parent 5df05cf46f
commit e70b9af3e4
No known key found for this signature in database
GPG key ID: AEB3F089F86A22B5
2 changed files with 48 additions and 5 deletions

View file

@ -58,17 +58,15 @@ func (d *DiskStore) Get(hash string) (stream.Blob, error) {
return nil, err
}
file, err := os.Open(d.path(hash))
blob, err := ioutil.ReadFile(d.path(hash))
if err != nil {
if os.IsNotExist(err) {
return nil, errors.Err(ErrBlobNotFound)
}
return nil, err
return nil, errors.Err(err)
}
defer file.Close()
blob, err := ioutil.ReadAll(file)
return blob, errors.Err(err)
return blob, nil
}
// Put stores the blob on disk

45
store/disk_test.go Normal file
View file

@ -0,0 +1,45 @@
package store
import (
"io/ioutil"
"os"
"path"
"path/filepath"
"testing"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDiskStore_Get(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
hash := "1234567890"
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
expectedPath := path.Join(tmpDir, hash[:2], hash)
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
require.NoError(t, err)
err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
require.NoError(t, err)
blob, err := d.Get(hash)
assert.NoError(t, err)
assert.EqualValues(t, data, blob)
}
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
blob, err := d.Get("nonexistent")
assert.Nil(t, blob)
assert.True(t, errors.Is(err, ErrBlobNotFound))
}