Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

Merge pull request #243 from lbryio/unicode-publish

Browse source 
Unicode publish
This commit is contained in:
Job Evers‐Meltzer 2016-11-08 08:45:26 -06:00 committed by GitHub
commit 906e6b238e
7 changed files with 109 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
lbrynet/conf.py
View file

@ -14,6 +14,8 @@ log = logging.getLogger(__name__)
LINUX = 1
DARWIN = 2
WINDOWS = 3
KB = 2**10
MB = 2**20
if sys.platform.startswith("darwin"):
@ -185,13 +187,13 @@ class AdjustableSettings(Setting):
class ApplicationSettings(Setting):
"""Settings that are constants and shouldn't be overriden"""
def __init__(self):
self.MAX_HANDSHAKE_SIZE = 2**16
self.MAX_REQUEST_SIZE = 2**16
self.MAX_BLOB_REQUEST_SIZE = 2**16
self.MAX_RESPONSE_INFO_SIZE = 2**16
self.MAX_HANDSHAKE_SIZE = 64*KB
self.MAX_REQUEST_SIZE = 64*KB
self.MAX_BLOB_REQUEST_SIZE = 64*KB
self.MAX_RESPONSE_INFO_SIZE = 64*KB
self.MAX_BLOB_INFOS_TO_REQUEST = 20
self.BLOBFILES_DIR = "blobfiles"
self.BLOB_SIZE = 2**21
self.BLOB_SIZE = 2*MB
self.LOG_FILE_NAME = "lbrynet.log"
self.LOG_POST_URL = "https://lbry.io/log-upload"
self.CRYPTSD_FILE_EXTENSION = ".cryptsd"

4
lbrynet/core/StreamCreator.py
View file

@ -65,12 +65,10 @@ class StreamCreator(object):
pass
def write(self, data):
from twisted.internet import reactor
self._write(data)
if self.stopped is False and self.streaming is False:
reactor.callLater(0, self.producer.resumeProducing)
def _write(self, data):
pass
pass

21
lbrynet/cryptstream/CryptStreamCreator.py
View file

@ -18,22 +18,23 @@ log = logging.getLogger(__name__)
class CryptStreamCreator(StreamCreator):
"""Create a new stream with blobs encrypted by a symmetric cipher.
Each blob is encrypted with the same key, but each blob has its own initialization vector
which is associated with the blob when the blob is associated with the stream."""
Each blob is encrypted with the same key, but each blob has its
own initialization vector which is associated with the blob when
the blob is associated with the stream.
"""
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
"""
@param blob_manager: Object that stores and provides access to blobs.
"""@param blob_manager: Object that stores and provides access to blobs.
@type blob_manager: BlobManager
@param name: the name of the stream, which will be presented to the user
@type name: string
@param key: the raw AES key which will be used to encrypt the blobs. If None, a random key will
be generated.
@param key: the raw AES key which will be used to encrypt the
blobs. If None, a random key will be generated.
@type key: string
@param iv_generator: a generator which yields initialization vectors for the blobs. Will be called
once for each blob.
@param iv_generator: a generator which yields initialization
vectors for the blobs. Will be called once for each blob.
@type iv_generator: a generator function which yields strings
@return: None
@ -72,10 +73,8 @@ class CryptStreamCreator(StreamCreator):
d.addCallback(self._blob_finished)
self.finished_deferreds.append(d)
log.debug("called close on final blob, returning from make_final_blob")
return d
def _write(self, data):
def close_blob(blob):
d = blob.close()
d.addCallback(self._blob_finished)
@ -94,4 +93,4 @@ class CryptStreamCreator(StreamCreator):
self.current_blob = None
def _get_blob_maker(self, iv, blob_creator):
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)

43
lbrynet/lbryfilemanager/EncryptedFileCreator.py
View file

@ -24,10 +24,7 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
key=None, iv_generator=None, suggested_file_name=None):
CryptStreamCreator.__init__(self, blob_manager, name, key, iv_generator)
self.lbry_file_manager = lbry_file_manager
if suggested_file_name is None:
self.suggested_file_name = name
else:
self.suggested_file_name = suggested_file_name
self.suggested_file_name = suggested_file_name or name
self.stream_hash = None
self.blob_infos = []
@ -37,9 +34,9 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
def _save_stream_info(self):
stream_info_manager = self.lbry_file_manager.stream_info_manager
d = stream_info_manager.save_stream(self.stream_hash, binascii.hexlify(self.name),
binascii.hexlify(self.key),
binascii.hexlify(self.suggested_file_name),
d = stream_info_manager.save_stream(self.stream_hash, hexlify(self.name),
hexlify(self.key),
hexlify(self.suggested_file_name),
self.blob_infos)
return d
@ -68,9 +65,9 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
def _make_stream_hash(self):
hashsum = get_lbry_hash_obj()
hashsum.update(binascii.hexlify(self.name))
hashsum.update(binascii.hexlify(self.key))
hashsum.update(binascii.hexlify(self.suggested_file_name))
hashsum.update(hexlify(self.name))
hashsum.update(hexlify(self.key))
hashsum.update(hexlify(self.suggested_file_name))
hashsum.update(self._get_blobs_hashsum())
self.stream_hash = hashsum.hexdigest()
@ -82,8 +79,7 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None,
iv_generator=None, suggested_file_name=None):
"""
Turn a plain file into an LBRY File.
"""Turn a plain file into an LBRY File.
An LBRY File is a collection of encrypted blobs of data and the metadata that binds them
together which, when decrypted and put back together according to the metadata, results
@ -113,8 +109,8 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
be generated.
@type key: string
@param iv_generator: a generator which yields initialization vectors for the blobs. Will be called
once for each blob.
@param iv_generator: a generator which yields initialization
vectors for the blobs. Will be called once for each blob.
@type iv_generator: a generator function which yields strings
@param suggested_file_name: what the file should be called when the LBRY File is saved to disk.
@ -130,7 +126,8 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
def make_stream_desc_file(stream_hash):
log.debug("creating the stream descriptor file")
descriptor_file_path = os.path.join(session.db_dir, file_name + settings.CRYPTSD_FILE_EXTENSION)
descriptor_file_path = os.path.join(
session.db_dir, file_name + settings.CRYPTSD_FILE_EXTENSION)
descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path)
d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True)
@ -141,8 +138,12 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
base_file_name = os.path.basename(file_name)
lbry_file_creator = EncryptedFileStreamCreator(session.blob_manager, lbry_file_manager, base_file_name,
key, iv_generator, suggested_file_name)
lbry_file_creator = EncryptedFileStreamCreator(
session.blob_manager,
lbry_file_manager,
base_file_name, key,
iv_generator,
suggested_file_name)
def start_stream():
file_sender = FileSender()
@ -155,3 +156,11 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
d = lbry_file_creator.setup()
d.addCallback(lambda _: start_stream())
return d
def hexlify(str_or_unicode):
if isinstance(str_or_unicode, unicode):
strng = str_or_unicode.encode('utf-8')
else:
strng = str_or_unicode
return binascii.hexlify(strng)

10
tests/mocks.py
View file

@ -8,6 +8,8 @@ from lbrynet.core import PTCWallet
from lbrynet.core import BlobAvailability
KB = 2**10
class Node(object):
def __init__(self, *args, **kwargs):
pass
@ -129,12 +131,12 @@ class GenFile(io.RawIOBase):
def readall(self):
return self.read()
def _generate_chunk(self, n=2**10):
output = self.pattern[self.last_offset:self.last_offset + n]
n_left = n - len(output)
def _generate_chunk(self, size=KB):
output = self.pattern[self.last_offset:self.last_offset + size]
n_left = size - len(output)
whole_patterns = n_left / len(self.pattern)
output += self.pattern * whole_patterns
self.last_offset = n - len(output)
self.last_offset = size - len(output)
output += self.pattern[:self.last_offset]
return output

0
tests/unit/lbryfilemanager/__init__.py Normal file
View file

59
tests/unit/lbryfilemanager/test_EncryptedFileCreator.py Normal file
View file

@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
import shutil
import tempfile
from Crypto.Cipher import AES
import mock
from twisted.trial import unittest
from lbrynet.core import BlobManager
from lbrynet.core import Session
from lbrynet.core.server import DHTHashAnnouncer
from lbrynet.lbryfilemanager import EncryptedFileCreator
from lbrynet.lbryfilemanager import EncryptedFileManager
from tests import mocks
MB = 2**20
def iv_generator():
while True:
yield '3' * AES.block_size
class CreateEncryptedFileTest(unittest.TestCase):
timeout = 5
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def create_file(self, filename):
session = mock.Mock(spec=Session.Session)(None, None)
hash_announcer = mock.Mock(spec=DHTHashAnnouncer.DHTHashAnnouncer)(None, None)
session.blob_manager = BlobManager.TempBlobManager(hash_announcer)
session.db_dir = self.tmp_dir
manager = mock.Mock(spec=EncryptedFileManager.EncryptedFileManager)()
handle = mocks.GenFile(3*MB, '1')
key = '2'*AES.block_size
return EncryptedFileCreator.create_lbry_file(
session, manager, filename, handle, key, iv_generator())
def test_can_create_file(self):
expected_stream_hash = ('41e6b247d923d191b154fb6f1b8529d6ddd6a73d65c357b1acb7'
'42dd83151fb66393a7709e9f346260a4f4db6de10c25')
filename = 'test.file'
d = self.create_file(filename)
d.addCallback(self.assertEqual, expected_stream_hash)
return d
def test_can_create_file_with_unicode_filename(self):
expected_stream_hash = ('d1da4258f3ce12edb91d7e8e160d091d3ab1432c2e55a6352dce0'
'2fd5adb86fe144e93e110075b5865fff8617776c6c0')
filename = u'☃.file'
d = self.create_file(filename)
d.addCallback(self.assertEqual, expected_stream_hash)
return d