dont overallocate ram when reading blobs from disk
ReadFile checks the file size and allocates a bit more space than we expect we'll need. ReadAll uses Go's standard resizing algo, which doubles the underlying array each time you hit the end. So ReadAll ends up allocating 4MB for a full blob, while ReadFile allocates slightly over 2MB.
This commit is contained in:
parent
5df05cf46f
commit
e70b9af3e4
2 changed files with 48 additions and 5 deletions
store
|
@ -58,17 +58,15 @@ func (d *DiskStore) Get(hash string) (stream.Blob, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
file, err := os.Open(d.path(hash))
|
||||
blob, err := ioutil.ReadFile(d.path(hash))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
return nil, err
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
blob, err := ioutil.ReadAll(file)
|
||||
return blob, errors.Err(err)
|
||||
return blob, nil
|
||||
}
|
||||
|
||||
// Put stores the blob on disk
|
||||
|
|
45
store/disk_test.go
Normal file
45
store/disk_test.go
Normal file
|
@ -0,0 +1,45 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDiskStore_Get(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
d := NewDiskStore(tmpDir, 2)
|
||||
|
||||
hash := "1234567890"
|
||||
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
|
||||
|
||||
expectedPath := path.Join(tmpDir, hash[:2], hash)
|
||||
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
|
||||
require.NoError(t, err)
|
||||
err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
|
||||
require.NoError(t, err)
|
||||
|
||||
blob, err := d.Get(hash)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, data, blob)
|
||||
}
|
||||
|
||||
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
d := NewDiskStore(tmpDir, 2)
|
||||
|
||||
blob, err := d.Get("nonexistent")
|
||||
assert.Nil(t, blob)
|
||||
assert.True(t, errors.Is(err, ErrBlobNotFound))
|
||||
}
|
Loading…
Add table
Reference in a new issue