tests pass on converting stream to file

stream/blob.go

@@ -3,6 +3,7 @@ package stream
 import (
 	"bytes"
 	"crypto/aes"
+	"crypto/cipher"
 	"crypto/rand"
 	"crypto/sha512"
 	"encoding/hex"
@@ -12,7 +13,10 @@ import (
 	"github.com/lbryio/lbry.go/errors"
 )
 
-const MaxBlobSize = 2 * 1024 * 1024
+const MaxBlobSize = 2097152 // 2mb, or 2 * 2^20
+
+// -1 to leave room for padding, since there must be at least one byte of pkcs7 padding
+const maxBlobDataSize = MaxBlobSize - 1
 
 type Blob []byte
 
@@ -32,11 +36,6 @@ func (b Blob) Hash() []byte {
 	return hashBytes[:]
 }
 
-// HexHash returns th blob hash as a hex string
-func (b Blob) HashHex() string {
-	return hex.EncodeToString(b.Hash())
-}
-
 // ValidForSend returns true if the blob size is within the limits
 func (b Blob) ValidForSend() error {
 	if b.Size() > MaxBlobSize {
@@ -48,6 +47,86 @@ func (b Blob) ValidForSend() error {
 	return nil
 }
 
+func NewBlob(data, key, iv []byte) (Blob, error) {
+	if len(data) == 0 {
+		// this is here to match python behavior. in theory we could encrypt an empty blob
+		return nil, errors.Err("cannot encrypt empty slice")
+	}
+	blockCipher, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	if len(iv) != blockCipher.BlockSize() {
+		return nil, errors.Err("IV length must equal to block size")
+	}
+
+	cbc := cipher.NewCBCEncrypter(blockCipher, iv)
+	plaintext, err := pkcs7Pad(data, blockCipher.BlockSize())
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	ciphertext := make([]byte, len(plaintext))
+	cbc.CryptBlocks(ciphertext, plaintext)
+	return ciphertext, nil
+}
+
+func (b Blob) Plaintext(key, iv []byte) ([]byte, error) {
+	blockCipher, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	if len(iv) != blockCipher.BlockSize() {
+		return nil, errors.Err("IV length must equal to block size")
+	}
+
+	cbc := cipher.NewCBCDecrypter(blockCipher, iv)
+	plaintext := make([]byte, len(b))
+	cbc.CryptBlocks(plaintext, b)
+
+	plaintext, err = pkcs7Unpad(plaintext, blockCipher.BlockSize())
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	return plaintext, nil
+}
+
+// https://github.com/fullsailor/pkcs7/blob/master/pkcs7.go#L468
+func pkcs7Pad(data []byte, blockLen int) ([]byte, error) {
+	if blockLen < 1 {
+		return nil, errors.Err("invalid block length %d", blockLen)
+	}
+	padLen := blockLen - (len(data) % blockLen)
+	if padLen == 0 {
+		padLen = blockLen
+	}
+	pad := bytes.Repeat([]byte{byte(padLen)}, padLen)
+	return append(data, pad...), nil
+}
+
+func pkcs7Unpad(data []byte, blockLen int) ([]byte, error) {
+	if blockLen < 1 {
+		return nil, errors.Err("invalid block length %d", blockLen)
+	}
+	if len(data)%blockLen != 0 || len(data) == 0 {
+		return nil, errors.Err("invalid data length %d", len(data))
+	}
+
+	// the last byte is the length of padding
+	padLen := int(data[len(data)-1])
+
+	// check padding integrity, all bytes should be the same
+	pad := data[len(data)-padLen:]
+	for _, padbyte := range pad {
+		if padbyte != byte(padLen) {
+			return nil, errors.Err("invalid padding")
+		}
+	}
+
+	return data[:len(data)-padLen], nil
+}
+
 // BlobInfo is the stream descriptor info for a single blob in a stream
 // Encoding to and from JSON is customized to match existing behavior (see json.go in package)
 type BlobInfo struct {
@@ -161,6 +240,14 @@ func (s *SDBlob) computeStreamHash() []byte {
 	)
 }
 
+func (s SDBlob) fileSize() int {
+	size := 0
+	for _, bi := range s.BlobInfos {
+		size += bi.Length
+	}
+	return size
+}
+
 // streamHash calculates the stream hash, given the stream's fields and blobs
 func streamHash(hexStreamName, hexKey, hexSuggestedFileName string, blobInfos []BlobInfo) []byte {
 	blobSum := sha512.New384()
@@ -178,12 +265,12 @@ func streamHash(hexStreamName, hexKey, hexSuggestedFileName string, blobInfos []
 
 // randIV returns a random AES IV
 func randIV() []byte {
-	blob := make([]byte, aes.BlockSize)
-	_, err := rand.Read(blob)
+	iv := make([]byte, aes.BlockSize)
+	_, err := rand.Read(iv)
 	if err != nil {
-		panic("failed to make random blob")
+		panic("failed to make random iv")
 	}
-	return blob
+	return iv
 }
 
 // NullIV returns an IV of 0s

stream/blob_test.go

@@ -2,9 +2,12 @@ package stream
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -74,6 +77,187 @@ func TestSdBlob_UnmarshalJSON(t *testing.T) {
 	}
 }
 
+func Test_pkcs7Pad(t *testing.T) {
+	blockLen := 16
+	tests := map[string]struct {
+		data     []byte
+		expected []byte
+	}{
+		"empty": {
+			data:     []byte{},
+			expected: []byte{16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16},
+		},
+		"one": {
+			data:     []byte{0},
+			expected: []byte{0, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+		},
+		"seven": {
+			data:     []byte{1, 2, 3, 4, 5, 6, 7},
+			expected: []byte{1, 2, 3, 4, 5, 6, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9},
+		},
+		"fifteen": {
+			data:     []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+			expected: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
+		},
+		"sixteen": {
+			data:     []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+			expected: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16},
+		},
+		"twenty": {
+			data:     []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+			expected: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+		},
+	}
+
+	for name, tt := range tests {
+		actual, err := pkcs7Pad(tt.data, blockLen)
+		if err != nil {
+			t.Errorf("%s: %v", name, err)
+			continue
+		}
+
+		if !bytes.Equal(actual, tt.expected) {
+			t.Errorf("%s: got %s, expected %s", name, hex.EncodeToString(actual), hex.EncodeToString(tt.expected))
+		}
+
+		unpadded, err := pkcs7Unpad(actual, blockLen)
+		if err != nil {
+			t.Errorf("%s: unpad: %v", name, err)
+			continue
+		}
+
+		if !bytes.Equal(unpadded, tt.data) {
+			t.Errorf("%s: unpad: got %s, expected %s", name, hex.EncodeToString(unpadded), hex.EncodeToString(tt.data))
+		}
+	}
+}
+
+func testdata(t *testing.T, filename string) []byte {
+	data, err := ioutil.ReadFile(filepath.Join("testdata", filename))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return data
+}
+
+func TestBlob_Encrypt(t *testing.T) {
+	tests := map[string]struct {
+		key, iv, data, ciphertext string
+		err                       bool
+	}{
+		"no-data": {
+			key:        "efad181bb91c18e93a57178559a42f21",
+			iv:         "032cb97fa5292b3109a67239f7c626aa",
+			data:       "",
+			ciphertext: "33adfa34104bf90f0cc1d033104c0cdb",
+			err:        true,
+		},
+		"short-success": {
+			key:        "efad181bb91c18e93a57178559a42f21",
+			iv:         "032cb97fa5292b3109a67239f7c626aa",
+			data:       "abcdefg",
+			ciphertext: "33adfa34104bf90f0cc1d033104c0cdb",
+		},
+		"1024-bytes": {
+			key:        "efad181bb91c18e93a57178559a42f21",
+			iv:         "032cb97fa5292b3109a67239f7c626aa",
+			data:       "xbmmgjjqqzxbwolnawkhcgatpalewqjatfldazvofaiutyxbtooizfprzibwogcsgeisperxqoarovpobbsqcdizvsyhlvbzpoainpvytdgifecvbylratedcntobaksbpebhgzjwgdaayaluuhiormlfyoybxmepzimggvumeaokbppuoylwulczfcbwubmjfdgezrnjashclcswaxgxvyvcguegbqaoudjhnpfbrnezuhdiqawgmgwbitnhvhwieumyebcyhbanwvkyxwcwhrfehtovofygwewawfjfvndmqrtytpgsspwofpocwjqthofdaguyuvdzcsmzdhhfzzucalypvsyvjrrmbqpoyvbgcfkqlqvwqtjluqwbgunuyetogevyrbaxxtggmjydpqjlqgbrasqvclrvicowpnmsrkexbopepyuhopwtpmqihaggynihpikbypcbvsjogcpwpxkjsnruowgryphrwbovbmdnjfeuvrjrpdlqvacmqvzylpincrbhdtqtyuzqvnbnrnxwtmkqanwnmquonmdsqqfzllvoucqjlpzburgciikotssciipllkrkyxzofstrnnhhvfjtjdibzmzvzrcqhkrfwabwzwrzbmwqddadvveiazooeryjstfimlolypkpsflcfalcnceowxrchfawbxsegqnycgqgakggddgqazfppshlygbtsptnbwlwbnyybfoqbuajojhwthmuyrcikbpohdyjqynbbdeegqabwocjqpwsxsifothimhheeoukdkymcvtggdrcywukfvxqjzaafzhdqeewbietsfecshenoowntxhaoaolfksllkrlpatuofiwiohvrsqawodfgelgtsnyzgnxontcqiluanwywiivhzorbkljqcostysmwdgutyexaaqjimbqqejsmgktbwdskjtikrbkjzbakcvwiokeshgjmtewxtvhroeygpbbrbxzvmbyibbwtoqzqzkkgcuggbkhofuecdz",
+			ciphertext: "ff808d89979f9057196f9f74008d3561f310e5b8aaf771849d862ee7dad6a14f251c02d3f1d48b9554b706991d0a025e99092c4442e81970119c44dbf3e45aecd51062231336578ef37a732341f1b1b503ee855feeecf531d633075df9df9e6fca9428e2a181f854082f934c6f8523bc7167b3c2524a5d37cc4896da9bca1a02197760e3e407176ce74299db8e2969e433cdeb042a9773defde7ac87b4cd28cb53b6e41a53b6f160e5c24d1706b5917c0dba22c846e922a9054572ae9f190ca796cfd70ad5624e7dda5f4dce024a30a692c16779a38d6967d6d769893ffda4a832d8475449f7aa8de5f5c421b22e609433823f3c3036220a0ee4e1e5590d94cf86864521208ba8f2228f72527a69e7a5146745b1dacf93db720962ecb9c9c008e73f4ff8e31eff99dcef03bbc2c3b6e9c3f8e71a1df6efbf50c79f0fd66aa4a38ac31550d9e07887cb486229f6b9ac7b5f2d41ffd73c24dbbcb7642f49697756621dce838da68f4a0a0037b478b6404afac0f318f2056fee05ea964f10e5f4ce772434cd739b044bd51c58ceb174346cb73eebfc0d6bf14a0d0bc0dbcdd7b242981fb90bb3f93ef4f51a394eefe9638eb75844235c84297e02458fa37cedb5004f765cf5ad65951c210d7a4228e87e24c630482eae9670df5a0e4e1042ef2f909ac63eb41551e667ba994a1d36b85353b79e2919fdaa345e01641614fe424fee0c211ff698b8725e462d8f7ea590fdfb293600d2c526e634c0ad9bfe80d0c4845781ce635b1dce836dcc68bf1a9efbd6396d241a6c055368d1c8178be47af0617c32054ccb7dd1f52edada4e61484b6aa89916d44d7e3e67a563fee06120844d40a1359ee5cb1d54e4a94ed945acb84e006c4261d6831fd53c6ec802c67363435b60232dae8e262aa07693f8ec34d45894fbfa2d0be4a175574a8f633b8eb3063e6e01a563f4178f564d206e46d07e5a8a4ae8354d47b2d3355aa65cc43b3c748766c44147e3552000da76cbd185cd33c0663991ba6624aa250465d8755f8274b66abd6ebcc3005029e375d4e9a2703fdb8cdbe8bc3e70a52df3ad43c61be1993071ce15fa0340a0a901d106bbb4015d8effb89c814a311e3062804c6e6bc6d390869ed995d161ef67a6d4a1819f2aaa4903d80bbca29c2cb32dc8e1fa2330659b05186514dc65cda4b278146f689f9eb874e844537fdb3d110f4b2787934e4964500180c9682d510ffaf5bbf0c74791acedc26832ef9f4b34edcdc843efdf54c874ff119c327f49f5ac0c90b3ed362a3b34fbd79b17656a9dcf48273fd455c421c4b107cf667bf7f6ddf6c2284b7c62494dea6f5e4d10cd78afeafcebdef91211fd7c1ec2eb0801311fc92e04eda0400b7163d51e397281488827c5e4d3314eaab3ed1f4afcbb375e7567e61dc9c47f899b25c9ef4df0558828b36113e275a16d0a76fa9e8021571c661f36e7b7a009",
+		},
+		"2mb-Xes": {
+			key:        "efad181bb91c18e93a57178559a42f21",
+			iv:         "032cb97fa5292b3109a67239f7c626aa",
+			data:       strings.Repeat("x", 2*1024*1024-1),
+			ciphertext: strings.TrimSpace(string(testdata(t, "encoded-2mb-Xes-minus-one"))),
+		},
+	}
+
+	for testName, tt := range tests {
+		key := unhex(t, tt.key)
+		iv := unhex(t, tt.iv)
+		blob, err := NewBlob([]byte(tt.data), key, iv)
+		if err != nil {
+			if !tt.err {
+				t.Errorf("%s: %v", testName, err)
+			}
+		} else if tt.err {
+			t.Errorf("%s: expected an error but didn't get one", testName)
+		} else {
+			expected := unhex(t, tt.ciphertext)
+			if len(blob) != len(expected) {
+				t.Errorf("%s: length mismatch. got %d, expected %d", testName, len(blob), len(expected))
+			}
+			if !bytes.Equal(blob, expected) {
+				t.Errorf("%s: got %s, expected %s (len is %d)", testName,
+					hex.EncodeToString(blob)[4194270:],
+					tt.ciphertext[4194270:],
+					len(tt.ciphertext),
+				)
+			}
+		}
+	}
+}
+
+func TestBlob_Plaintext(t *testing.T) {
+	expected := unhex(t, "2d218ab43c66741d74211076c069f464811de7fe5767009faaa9982171cc57ef")
+	key := unhex(t, "b450f70bd285726e470428df6c6ff8d2")
+	iv := unhex(t, "0553e3eb17916333d3468286a30738f1")
+	blob := Blob(testdata(t, "a2f1841bb9c5f3b583ac3b8c07ee1a5bf9cc48923721c30d5ca6318615776c284e8936d72fa4db7fdda2e4e9598b1e6c"))
+	plaintext, err := blob.Plaintext(key, iv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	actual := sha256.Sum256(plaintext)
+	if !bytes.Equal(actual[:], expected) {
+		t.Errorf("hash mismatch. got %s, expected %s", hex.EncodeToString(actual[:]), hex.EncodeToString(expected))
+	}
+}
+
+func TestBlob_DecryptStream(t *testing.T) {
+	sdHash := "1bf7d39c45d1a38ffa74bff179bf7f67d400ff57fa0b5a0308963f08d01712b3079530a8c188e8c89d9b390c6ee06f05"
+	sdBlob := &SDBlob{}
+	err := json.Unmarshal(testdata(t, sdHash), sdBlob)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !sdBlob.IsValid() {
+		t.Fatal("sd blob does not appear to be valid")
+	}
+
+	var file []byte
+	for _, bi := range sdBlob.BlobInfos {
+		if bi.Length == 0 {
+			continue
+		}
+		blobHash := hex.EncodeToString(bi.BlobHash)
+		blob := Blob(testdata(t, blobHash))
+		plaintext, err := blob.Plaintext(sdBlob.Key, bi.IV)
+		if err != nil {
+			t.Fatal(err)
+		}
+		file = append(file, plaintext...)
+	}
+
+	expectedLen := 6990951
+	actualLen := len(file)
+
+	if actualLen != expectedLen {
+		t.Errorf("file length mismatch. got %d, expected %d", actualLen, expectedLen)
+	}
+
+	expectedSha256 := unhex(t, "51e4d03bd6d69ea17d1be3ce01fdffa44ffe053f2dbce8d42a50283b2890fea2")
+	actualSha256 := sha256.Sum256(file)
+
+	if !bytes.Equal(actualSha256[:], expectedSha256) {
+		t.Errorf("file hash mismatch. got %s, expected %s", hex.EncodeToString(actualSha256[:]), hex.EncodeToString(expectedSha256))
+	}
+}
+
+func TestNew(t *testing.T) {
+	t.Skip("TODO: test new stream creation and decryption")
+}
+
 func unhex(t *testing.T, s string) []byte {
 	r, err := hex.DecodeString(s)
 	if err != nil {

stream/stream.go

@@ -0,0 +1,76 @@
+package stream
+
+import (
+	"bytes"
+
+	"github.com/lbryio/lbry.go/errors"
+)
+
+type Stream []Blob
+
+func New(data []byte) (Stream, error) {
+	var err error
+
+	numBlobs := len(data) / maxBlobDataSize
+	if len(data)%maxBlobDataSize != 0 {
+		numBlobs++ // ++ for unfinished blob at the end
+	}
+
+	key := randIV()
+	ivs := make([][]byte, numBlobs)
+	for i := range ivs {
+		ivs[i] = randIV()
+	}
+
+	s := make(Stream, numBlobs+1) // +1 for sd blob
+	for i := 0; i < numBlobs; i++ {
+		start := i - 1*maxBlobDataSize
+		end := start + maxBlobDataSize
+		if end > len(data) {
+			end = len(data)
+		}
+		s[i+1], err = NewBlob(data[start:end], key, ivs[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	sd := newSdBlob(s[1:], key, ivs)
+	s[0], err = sd.ToBlob()
+	if err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+func (s Stream) Data() ([]byte, error) {
+	if len(s) < 2 {
+		return nil, errors.Err("stream must be at least 2 blobs long")
+	}
+
+	sdBlob := &SDBlob{}
+	err := sdBlob.FromBlob(s[0])
+	if err != nil {
+		return nil, err
+	}
+
+	if !sdBlob.IsValid() {
+		return nil, errors.Err("sd blob is not valid")
+	}
+
+	var file []byte
+	for i, b := range s[1:] {
+		if !bytes.Equal(b.Hash(), sdBlob.BlobInfos[i].BlobHash) {
+			return nil, errors.Err("blob hash doesn't match hash in blobInfo")
+		}
+
+		data, err := b.Plaintext(sdBlob.Key, sdBlob.BlobInfos[i].IV)
+		if err != nil {
+			return nil, err
+		}
+		file = append(file, data...)
+	}
+
+	return file, nil
+}

stream/testdata/1bf7d39c45d1a38ffa74bff179bf7f67d400ff57fa0b5a0308963f08d01712b3079530a8c188e8c89d9b390c6ee06f05

@@ -0,0 +1 @@
+{"stream_name": "43617375616c6c79204578706c61696e6564202d2054686520537065637472756d206f6620496e74656c6c6967656e63652e6d7034", "blobs": [{"length": 2097152, "blob_num": 0, "blob_hash": "a2f1841bb9c5f3b583ac3b8c07ee1a5bf9cc48923721c30d5ca6318615776c284e8936d72fa4db7fdda2e4e9598b1e6c", "iv": "0553e3eb17916333d3468286a30738f1"}, {"length": 2097152, "blob_num": 1, "blob_hash": "0c9675ad7f40f29dcd41883ed9cf7e145bbb13976d9b83ab9354f4f61a87f0f7771a56724c2aa7a5ab43c68d7942e5cb", "iv": "d4e94a3cc6dd5ef0f90d25cd73de5584"}, {"length": 2097152, "blob_num": 2, "blob_hash": "a4d07d442b9907036c75b6c92db316a8b8428733bf5ec976627a48a7c862bf84db33075d54125a7c0b297bd2dc445f1c", "iv": "1c0bf1dc467a2ca5e67774868aaae431"}, {"length": 699504, "blob_num": 3, "blob_hash": "dcd2093f4a3eca9f6dd59d785d0bef068fee788481986aa894cf72ed4d992c0ff9d19d1743525de2f5c3c62f5ede1c58", "iv": "14881d23e88e74d1c1715fff0f12b216"}, {"length": 0, "blob_num": 4, "iv": "8e7405e96eeeaf3e0720ca3e191fe38e"}], "stream_type": "lbryfile", "key": "b450f70bd285726e470428df6c6ff8d2", "suggested_file_name": "43617375616c6c79204578706c61696e6564202d2054686520537065637472756d206f6620496e74656c6c6967656e63652e6d7034", "stream_hash": "d756e860d8f49d03937c1a35a560636e792d97bee6f9660fc69e206cbcfe7f9297c5ede8428dd5f17d240e434eb557da"}