Merge pull request #446 from lbryio/reseed

Script to Reseed a File
This commit is contained in:
Job Evers‐Meltzer 2017-02-21 06:11:32 -06:00 committed by GitHub
commit 998892ef0b
8 changed files with 236 additions and 1 deletions

View file

@ -242,6 +242,7 @@ class BlobFile(HashBlob):
self.readers += 1
return file_handle
except IOError:
log.exception('Failed to open %s', self.file_path)
self.close_read_handle(file_handle)
return None

View file

@ -64,6 +64,8 @@ class StreamCreator(object):
def _finished(self):
pass
# TODO: move the stream creation process to its own thread and
# remove the reactor from this process.
def write(self, data):
from twisted.internet import reactor
self._write(data)

View file

@ -77,6 +77,12 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
return d
# TODO: this should be run its own thread. Encrypting a large file can
# be very cpu intensive and there is no need to run that on the
# main reactor thread. The FileSender mechanism that is used is
# great when sending over the network, but this is all local so
# we can simply read the file from the disk without needing to
# involve reactor.
def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None,
iv_generator=None, suggested_file_name=None):
"""Turn a plain file into an LBRY File.
@ -146,6 +152,10 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
suggested_file_name)
def start_stream():
# TODO: Using FileSender isn't necessary, we can just read
# straight from the disk. The stream creation process
# should be in its own thread anyway so we don't need to
# worry about interacting with the twisted reactor
file_sender = FileSender()
d = file_sender.beginFileTransfer(file_handle, lbry_file_creator)
d.addCallback(lambda _: stop_file(lbry_file_creator))

View file

@ -41,6 +41,7 @@ class EncryptedFileManager(object):
def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
self.session = session
self.stream_info_manager = stream_info_manager
# TODO: why is sd_identifier part of the file manager?
self.sd_identifier = sd_identifier
self.lbry_files = []
self.sql_db = None

57
scripts/decrypt_blob.py Normal file
View file

@ -0,0 +1,57 @@
"""Decrypt a single blob"""
import argparse
import binascii
import logging
import os
import sys
from twisted.internet import defer
from twisted.internet import reactor
from lbrynet import conf
from lbrynet.cryptstream import CryptBlob
from lbrynet.core import HashBlob
from lbrynet.core import log_support
log = logging.getLogger('decrypt_blob')
def main():
conf.initialize_settings()
parser = argparse.ArgumentParser()
parser.add_argument('blob_file')
parser.add_argument('hex_key')
parser.add_argument('hex_iv')
parser.add_argument('output')
args = parser.parse_args()
log_support.configure_console()
d = run(args)
reactor.run()
@defer.inlineCallbacks
def run(args):
try:
yield decrypt_blob(args.blob_file, args.hex_key, args.hex_iv, args.output)
except Exception:
log.exception('Failed to decrypt blob')
finally:
reactor.callLater(0, reactor.stop)
@defer.inlineCallbacks
def decrypt_blob(blob_file, key, iv, output):
filename = os.path.abspath(blob_file)
length = os.path.getsize(filename)
directory, blob_hash = os.path.split(filename)
blob = HashBlob.BlobFile(directory, blob_hash, True, length)
decryptor = CryptBlob.StreamBlobDecryptor(
blob, binascii.unhexlify(key), binascii.unhexlify(iv), length)
with open(output, 'w') as f:
yield decryptor.decrypt(f.write)
if __name__ == '__main__':
sys.exit(main())

71
scripts/encrypt_blob.py Normal file
View file

@ -0,0 +1,71 @@
"""Encrypt a single file using the given key and iv"""
import argparse
import binascii
import logging
import os
import StringIO
import sys
from twisted.internet import defer
from twisted.internet import reactor
from lbrynet import conf
from lbrynet.cryptstream import CryptBlob
from lbrynet.core import HashBlob
from lbrynet.core import log_support
from lbrynet.core import cryptoutils
log = logging.getLogger('decrypt_blob')
def main():
conf.initialize_settings()
parser = argparse.ArgumentParser()
parser.add_argument('filename')
parser.add_argument('hex_key')
parser.add_argument('hex_iv')
args = parser.parse_args()
log_support.configure_console(level='DEBUG')
d = run(args)
reactor.run()
@defer.inlineCallbacks
def run(args):
try:
yield encrypt_blob(args.filename, args.hex_key, args.hex_iv)
except Exception:
log.exception('Failed to encrypt blob')
finally:
reactor.callLater(0, reactor.stop)
def encrypt_blob(filename, key, iv):
blob = Blob()
blob_maker = CryptBlob.CryptStreamBlobMaker(
binascii.unhexlify(key), binascii.unhexlify(iv), 0, blob)
with open(filename) as fin:
blob_maker.write(fin.read())
blob_maker.close()
class Blob(object):
def __init__(self):
self.data = StringIO.StringIO()
def write(self, data):
self.data.write(data)
def close(self):
hashsum = cryptoutils.get_lbry_hash_obj()
buffer = self.data.getvalue()
hashsum.update(buffer)
with open(hashsum.hexdigest(), 'w') as fout:
fout.write(buffer)
return defer.succeed(True)
if __name__ == '__main__':
sys.exit(main())

View file

@ -22,7 +22,6 @@ from lbrynet import conf
from lbrynet.core import Error
from lbrynet.core import Wallet
from lbrynet.core import BlobAvailability
from lbrynet.core import BlobManager
from lbrynet.core import HashAnnouncer
from lbrynet.core import PeerManager
from lbrynet.core import Session

94
scripts/reseed_file.py Normal file
View file

@ -0,0 +1,94 @@
"""Reseed a file.
Given a file and a matching sd_blob,
re-chunk and encrypt the file, adding
the new blobs to the manager.
"""
import argparse
import binascii
import logging
import json
import os
import sys
from twisted.internet import defer
from twisted.internet import reactor
from twisted.protocols import basic
from lbrynet import conf
from lbrynet.core import BlobManager
from lbrynet.core import HashAnnouncer
from lbrynet.core import log_support
from lbrynet.cryptstream import CryptStreamCreator
log = logging.getLogger('reseed_file')
def main():
conf.initialize_settings()
parser = argparse.ArgumentParser()
parser.add_argument('input_file')
parser.add_argument('sd_blob', help='a json file containing a key and the IVs')
args = parser.parse_args()
log_support.configure_console()
run(args)
reactor.run()
@defer.inlineCallbacks
def run(args):
try:
yield reseed_file(args.input_file, args.sd_blob)
except Exception as e:
log.exception('Failed to reseed')
finally:
reactor.stop()
@defer.inlineCallbacks
def reseed_file(input_file, sd_blob):
sd_blob = SdBlob.new_instance(sd_blob)
db_dir = conf.settings['data_dir']
blobfile_dir = os.path.join(db_dir, "blobfiles")
announcer = HashAnnouncer.DummyHashAnnouncer()
blob_manager = BlobManager.DiskBlobManager(announcer, blobfile_dir, db_dir)
yield blob_manager.setup()
creator = CryptStreamCreator.CryptStreamCreator(
blob_manager, None, sd_blob.key(), sd_blob.iv_generator())
file_sender = basic.FileSender()
with open(input_file) as f:
yield file_sender.beginFileTransfer(f, creator)
yield creator.stop()
for blob_info in sd_blob.blob_infos():
if 'blob_hash' not in blob_info:
# the last blob is always empty and without a hash
continue
blob = yield blob_manager.get_blob(blob_info['blob_hash'], True)
if not blob.verified:
print "Blob {} is not verified".format(blob)
class SdBlob(object):
def __init__(self, contents):
self.contents = contents
def key(self):
return binascii.unhexlify(self.contents['key'])
def iv_generator(self):
for blob_info in self.blob_infos():
yield binascii.unhexlify(blob_info['iv'])
def blob_infos(self):
return self.contents['blobs']
@classmethod
def new_instance(cls, filename):
with open(filename) as f:
return cls(json.load(f))
if __name__ == '__main__':
sys.exit(main())