forked from LBRYCommunity/lbry-sdk
Add script to reseed a file
This commit is contained in:
parent
9f4ccfe7c3
commit
90d3a0ef9c
8 changed files with 236 additions and 1 deletions
|
@ -243,6 +243,7 @@ class BlobFile(HashBlob):
|
|||
self.readers += 1
|
||||
return file_handle
|
||||
except IOError:
|
||||
log.exception('Failed to open %s', self.file_path)
|
||||
self.close_read_handle(file_handle)
|
||||
return None
|
||||
|
||||
|
|
|
@ -64,6 +64,8 @@ class StreamCreator(object):
|
|||
def _finished(self):
|
||||
pass
|
||||
|
||||
# TODO: move the stream creation process to its own thread and
|
||||
# remove the reactor from this process.
|
||||
def write(self, data):
|
||||
from twisted.internet import reactor
|
||||
self._write(data)
|
||||
|
|
|
@ -77,6 +77,12 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
|
|||
return d
|
||||
|
||||
|
||||
# TODO: this should be run its own thread. Encrypting a large file can
|
||||
# be very cpu intensive and there is no need to run that on the
|
||||
# main reactor thread. The FileSender mechanism that is used is
|
||||
# great when sending over the network, but this is all local so
|
||||
# we can simply read the file from the disk without needing to
|
||||
# involve reactor.
|
||||
def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None,
|
||||
iv_generator=None, suggested_file_name=None):
|
||||
"""Turn a plain file into an LBRY File.
|
||||
|
@ -146,6 +152,10 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
|||
suggested_file_name)
|
||||
|
||||
def start_stream():
|
||||
# TODO: Using FileSender isn't necessary, we can just read
|
||||
# straight from the disk. The stream creation process
|
||||
# should be in its own thread anyway so we don't need to
|
||||
# worry about interacting with the twisted reactor
|
||||
file_sender = FileSender()
|
||||
d = file_sender.beginFileTransfer(file_handle, lbry_file_creator)
|
||||
d.addCallback(lambda _: stop_file(lbry_file_creator))
|
||||
|
|
|
@ -41,6 +41,7 @@ class EncryptedFileManager(object):
|
|||
def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
|
||||
self.session = session
|
||||
self.stream_info_manager = stream_info_manager
|
||||
# TODO: why is sd_identifier part of the file manager?
|
||||
self.sd_identifier = sd_identifier
|
||||
self.lbry_files = []
|
||||
self.sql_db = None
|
||||
|
|
57
scripts/decrypt_blob.py
Normal file
57
scripts/decrypt_blob.py
Normal file
|
@ -0,0 +1,57 @@
|
|||
"""Decrypt a single blob"""
|
||||
import argparse
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import reactor
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.cryptstream import CryptBlob
|
||||
from lbrynet.core import HashBlob
|
||||
from lbrynet.core import log_support
|
||||
|
||||
|
||||
log = logging.getLogger('decrypt_blob')
|
||||
|
||||
|
||||
def main():
|
||||
conf.initialize_settings()
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('blob_file')
|
||||
parser.add_argument('hex_key')
|
||||
parser.add_argument('hex_iv')
|
||||
parser.add_argument('output')
|
||||
args = parser.parse_args()
|
||||
log_support.configure_console()
|
||||
|
||||
d = run(args)
|
||||
reactor.run()
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(args):
|
||||
try:
|
||||
yield decrypt_blob(args.blob_file, args.hex_key, args.hex_iv, args.output)
|
||||
except Exception:
|
||||
log.exception('Failed to decrypt blob')
|
||||
finally:
|
||||
reactor.callLater(0, reactor.stop)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def decrypt_blob(blob_file, key, iv, output):
|
||||
filename = os.path.abspath(blob_file)
|
||||
length = os.path.getsize(filename)
|
||||
directory, blob_hash = os.path.split(filename)
|
||||
blob = HashBlob.BlobFile(directory, blob_hash, True, length)
|
||||
decryptor = CryptBlob.StreamBlobDecryptor(
|
||||
blob, binascii.unhexlify(key), binascii.unhexlify(iv), length)
|
||||
with open(output, 'w') as f:
|
||||
yield decryptor.decrypt(f.write)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
71
scripts/encrypt_blob.py
Normal file
71
scripts/encrypt_blob.py
Normal file
|
@ -0,0 +1,71 @@
|
|||
"""Encrypt a single file using the given key and iv"""
|
||||
import argparse
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import StringIO
|
||||
import sys
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import reactor
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.cryptstream import CryptBlob
|
||||
from lbrynet.core import HashBlob
|
||||
from lbrynet.core import log_support
|
||||
from lbrynet.core import cryptoutils
|
||||
|
||||
|
||||
log = logging.getLogger('decrypt_blob')
|
||||
|
||||
|
||||
def main():
|
||||
conf.initialize_settings()
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('filename')
|
||||
parser.add_argument('hex_key')
|
||||
parser.add_argument('hex_iv')
|
||||
args = parser.parse_args()
|
||||
log_support.configure_console(level='DEBUG')
|
||||
|
||||
d = run(args)
|
||||
reactor.run()
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(args):
|
||||
try:
|
||||
yield encrypt_blob(args.filename, args.hex_key, args.hex_iv)
|
||||
except Exception:
|
||||
log.exception('Failed to encrypt blob')
|
||||
finally:
|
||||
reactor.callLater(0, reactor.stop)
|
||||
|
||||
|
||||
def encrypt_blob(filename, key, iv):
|
||||
blob = Blob()
|
||||
blob_maker = CryptBlob.CryptStreamBlobMaker(
|
||||
binascii.unhexlify(key), binascii.unhexlify(iv), 0, blob)
|
||||
with open(filename) as fin:
|
||||
blob_maker.write(fin.read())
|
||||
blob_maker.close()
|
||||
|
||||
|
||||
class Blob(object):
|
||||
def __init__(self):
|
||||
self.data = StringIO.StringIO()
|
||||
|
||||
def write(self, data):
|
||||
self.data.write(data)
|
||||
|
||||
def close(self):
|
||||
hashsum = cryptoutils.get_lbry_hash_obj()
|
||||
buffer = self.data.getvalue()
|
||||
hashsum.update(buffer)
|
||||
with open(hashsum.hexdigest(), 'w') as fout:
|
||||
fout.write(buffer)
|
||||
return defer.succeed(True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
|
@ -22,7 +22,6 @@ from lbrynet import conf
|
|||
from lbrynet.core import Error
|
||||
from lbrynet.core import Wallet
|
||||
from lbrynet.core import BlobAvailability
|
||||
from lbrynet.core import BlobManager
|
||||
from lbrynet.core import HashAnnouncer
|
||||
from lbrynet.core import PeerManager
|
||||
from lbrynet.core import Session
|
||||
|
|
94
scripts/reseed_file.py
Normal file
94
scripts/reseed_file.py
Normal file
|
@ -0,0 +1,94 @@
|
|||
"""Reseed a file.
|
||||
|
||||
Given a file and a matching sd_blob,
|
||||
re-chunk and encrypt the file, adding
|
||||
the new blobs to the manager.
|
||||
"""
|
||||
import argparse
|
||||
import binascii
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import reactor
|
||||
from twisted.protocols import basic
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import BlobManager
|
||||
from lbrynet.core import HashAnnouncer
|
||||
from lbrynet.core import log_support
|
||||
from lbrynet.cryptstream import CryptStreamCreator
|
||||
|
||||
|
||||
log = logging.getLogger('reseed_file')
|
||||
|
||||
|
||||
def main():
|
||||
conf.initialize_settings()
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('input_file')
|
||||
parser.add_argument('sd_blob', help='a json file containing a key and the IVs')
|
||||
args = parser.parse_args()
|
||||
log_support.configure_console()
|
||||
|
||||
run(args)
|
||||
reactor.run()
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(args):
|
||||
try:
|
||||
yield reseed_file(args.input_file, args.sd_blob)
|
||||
except Exception as e:
|
||||
log.exception('Failed to reseed')
|
||||
finally:
|
||||
reactor.stop()
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def reseed_file(input_file, sd_blob):
|
||||
sd_blob = SdBlob.new_instance(sd_blob)
|
||||
db_dir = conf.settings['data_dir']
|
||||
blobfile_dir = os.path.join(db_dir, "blobfiles")
|
||||
announcer = HashAnnouncer.DummyHashAnnouncer()
|
||||
blob_manager = BlobManager.DiskBlobManager(announcer, blobfile_dir, db_dir)
|
||||
yield blob_manager.setup()
|
||||
creator = CryptStreamCreator.CryptStreamCreator(
|
||||
blob_manager, None, sd_blob.key(), sd_blob.iv_generator())
|
||||
file_sender = basic.FileSender()
|
||||
with open(input_file) as f:
|
||||
yield file_sender.beginFileTransfer(f, creator)
|
||||
yield creator.stop()
|
||||
for blob_info in sd_blob.blob_infos():
|
||||
if 'blob_hash' not in blob_info:
|
||||
# the last blob is always empty and without a hash
|
||||
continue
|
||||
blob = yield blob_manager.get_blob(blob_info['blob_hash'], True)
|
||||
if not blob.verified:
|
||||
print "Blob {} is not verified".format(blob)
|
||||
|
||||
|
||||
class SdBlob(object):
|
||||
def __init__(self, contents):
|
||||
self.contents = contents
|
||||
|
||||
def key(self):
|
||||
return binascii.unhexlify(self.contents['key'])
|
||||
|
||||
def iv_generator(self):
|
||||
for blob_info in self.blob_infos():
|
||||
yield binascii.unhexlify(blob_info['iv'])
|
||||
|
||||
def blob_infos(self):
|
||||
return self.contents['blobs']
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, filename):
|
||||
with open(filename) as f:
|
||||
return cls(json.load(f))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
Loading…
Reference in a new issue