From b389e59452cce04baf557bcefbe1b41605e825f1 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 4 May 2018 22:12:43 -0300 Subject: [PATCH 01/18] replace pycrypto->cryptography on tests --- lbrynet/tests/functional/test_misc.py | 17 ++++----------- lbrynet/tests/functional/test_streamify.py | 4 ++-- lbrynet/tests/mocks.py | 21 ++++++++++++++----- .../tests/unit/cryptstream/test_cryptblob.py | 12 ++++++----- .../test_EncryptedFileCreator.py | 6 +++--- 5 files changed, 32 insertions(+), 28 deletions(-) diff --git a/lbrynet/tests/functional/test_misc.py b/lbrynet/tests/functional/test_misc.py index e806da5c2..34e0d80c9 100644 --- a/lbrynet/tests/functional/test_misc.py +++ b/lbrynet/tests/functional/test_misc.py @@ -7,8 +7,7 @@ import sys import random import unittest -from Crypto import Random -from Crypto.Hash import MD5 +from hashlib import md5 from lbrynet import conf from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager from lbrynet.core.Session import Session @@ -98,9 +97,6 @@ class LbryUploader(object): from twisted.internet import reactor self.reactor = reactor logging.debug("Starting the uploader") - Random.atfork() - r = random.Random() - r.seed("start_lbry_uploader") wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 1) @@ -191,10 +187,6 @@ def start_lbry_reuploader(sd_hash, kill_event, dead_event, logging.debug("Starting the uploader") - Random.atfork() - - r = random.Random() - r.seed("start_lbry_reuploader") wallet = FakeWallet() peer_port = 5553 + n @@ -297,7 +289,6 @@ def start_blob_uploader(blob_hash_queue, kill_event, dead_event, slow, is_genero logging.debug("Starting the uploader") - Random.atfork() wallet = FakeWallet() peer_manager = PeerManager() @@ -515,7 +506,7 @@ class TestTransfer(TestCase): def check_md5_sum(): f = open(os.path.join(db_dir, 'test_file')) - hashsum = MD5.new() + hashsum = md5() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "4ca2aafb4101c1e42235aad24fbb83be") @@ -688,7 +679,7 @@ class TestTransfer(TestCase): def check_md5_sum(): f = open(os.path.join(db_dir, 'test_file')) - hashsum = MD5.new() + hashsum = md5() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "4ca2aafb4101c1e42235aad24fbb83be") @@ -811,7 +802,7 @@ class TestTransfer(TestCase): def check_md5_sum(): f = open('test_file') - hashsum = MD5.new() + hashsum = md5() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "e5941d615f53312fd66638239c1f90d5") diff --git a/lbrynet/tests/functional/test_streamify.py b/lbrynet/tests/functional/test_streamify.py index c84630272..cda06758b 100644 --- a/lbrynet/tests/functional/test_streamify.py +++ b/lbrynet/tests/functional/test_streamify.py @@ -2,7 +2,7 @@ import os import shutil import tempfile -from Crypto.Hash import MD5 +from hashlib import md5 from twisted.trial.unittest import TestCase from twisted.internet import defer, threads @@ -127,7 +127,7 @@ class TestStreamify(TestCase): self.assertTrue(lbry_file.sd_hash, sd_hash) yield lbry_file.start() f = open('test_file') - hashsum = MD5.new() + hashsum = md5() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "68959747edc73df45e45db6379dd7b3b") diff --git a/lbrynet/tests/mocks.py b/lbrynet/tests/mocks.py index d2bce3730..7de4927c7 100644 --- a/lbrynet/tests/mocks.py +++ b/lbrynet/tests/mocks.py @@ -1,7 +1,10 @@ +import base64 import struct import io -from Crypto.PublicKey import RSA +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization from twisted.internet import defer, error from twisted.python.failure import Failure @@ -15,6 +18,12 @@ from lbrynet import conf from util import debug_kademlia_packet KB = 2**10 +PUBLIC_EXPOENT = 65537 # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html + + +def decode_rsa_key(pem_key): + decoded = base64.b64decode(''.join(pem_key.splitlines()[1:-1])) + return serialization.load_der_public_key(decoded, default_backend()) class FakeLBRYFile(object): @@ -137,9 +146,9 @@ class PointTraderKeyQueryHandler(object): if self.query_identifiers[0] in queries: new_encoded_pub_key = queries[self.query_identifiers[0]] try: - RSA.importKey(new_encoded_pub_key) + decode_rsa_key(new_encoded_pub_key) except (ValueError, TypeError, IndexError): - return defer.fail(Failure(ValueError("Client sent an invalid public key"))) + return defer.fail(Failure(ValueError("Client sent an invalid public key: {}".format(new_encoded_pub_key)))) self.public_key = new_encoded_pub_key self.wallet.set_public_key_for_peer(self.peer, self.public_key) fields = {'public_key': self.wallet.encoded_public_key} @@ -152,8 +161,10 @@ class PointTraderKeyQueryHandler(object): class Wallet(object): def __init__(self): - self.private_key = RSA.generate(1024) - self.encoded_public_key = self.private_key.publickey().exportKey() + self.private_key = rsa.generate_private_key(public_exponent=PUBLIC_EXPOENT, + key_size=1024, backend=default_backend()) + self.encoded_public_key = self.private_key.public_key().public_bytes(serialization.Encoding.PEM, + serialization.PublicFormat.PKCS1) self._config = None self.network = None self.wallet = None diff --git a/lbrynet/tests/unit/cryptstream/test_cryptblob.py b/lbrynet/tests/unit/cryptstream/test_cryptblob.py index 2378c5770..90719166e 100644 --- a/lbrynet/tests/unit/cryptstream/test_cryptblob.py +++ b/lbrynet/tests/unit/cryptstream/test_cryptblob.py @@ -5,11 +5,13 @@ from lbrynet.blob.blob_file import MAX_BLOB_SIZE from lbrynet.tests.mocks import mock_conf_settings -from Crypto import Random -from Crypto.Cipher import AES +from cryptography.hazmat.primitives.ciphers.algorithms import AES import random import string import StringIO +import os + +AES_BLOCK_SIZE_BYTES = AES.block_size / 8 class MocBlob(object): def __init__(self): @@ -44,8 +46,8 @@ class TestCryptBlob(unittest.TestCase): # max blob size is 2*2**20 -1 ( -1 due to required padding in the end ) blob = MocBlob() blob_num = 0 - key = Random.new().read(AES.block_size) - iv = Random.new().read(AES.block_size) + key = os.urandom(AES_BLOCK_SIZE_BYTES) + iv = os.urandom(AES_BLOCK_SIZE_BYTES) maker = CryptBlob.CryptStreamBlobMaker(key, iv, blob_num, blob) write_size = size_of_data string_to_encrypt = random_string(size_of_data) @@ -54,7 +56,7 @@ class TestCryptBlob(unittest.TestCase): done, num_bytes = maker.write(string_to_encrypt) yield maker.close() self.assertEqual(size_of_data, num_bytes) - expected_encrypted_blob_size = ((size_of_data / AES.block_size) + 1) * AES.block_size + expected_encrypted_blob_size = ((size_of_data / AES_BLOCK_SIZE_BYTES) + 1) * AES_BLOCK_SIZE_BYTES self.assertEqual(expected_encrypted_blob_size, len(blob.data)) if size_of_data < MAX_BLOB_SIZE-1: diff --git a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py index 07ad7e87f..6a4dcc8fd 100644 --- a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py +++ b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from Crypto.Cipher import AES +from cryptography.hazmat.primitives.ciphers.algorithms import AES import mock from twisted.trial import unittest from twisted.internet import defer @@ -18,7 +18,7 @@ MB = 2**20 def iv_generator(): while True: - yield '3' * AES.block_size + yield '3' * (AES.block_size / 8) class CreateEncryptedFileTest(unittest.TestCase): @@ -47,7 +47,7 @@ class CreateEncryptedFileTest(unittest.TestCase): @defer.inlineCallbacks def create_file(self, filename): handle = mocks.GenFile(3*MB, '1') - key = '2'*AES.block_size + key = '2' * (AES.block_size / 8) out = yield EncryptedFileCreator.create_lbry_file(self.session, self.file_manager, filename, handle, key, iv_generator()) defer.returnValue(out) From fab932abb66ae64b2ddace2eacd45b417be493b9 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 4 May 2018 23:07:05 -0300 Subject: [PATCH 02/18] replace pycrypto->cryptography --- lbrynet/cryptstream/CryptStreamCreator.py | 10 +-- .../pointtraderclient/pointtraderclient.py | 69 +++++++++++-------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/lbrynet/cryptstream/CryptStreamCreator.py b/lbrynet/cryptstream/CryptStreamCreator.py index e39a50c1d..a3042ac61 100644 --- a/lbrynet/cryptstream/CryptStreamCreator.py +++ b/lbrynet/cryptstream/CryptStreamCreator.py @@ -1,12 +1,12 @@ """ Utility for creating Crypt Streams, which are encrypted blobs and associated metadata. """ - +import os import logging + +from cryptography.hazmat.primitives.ciphers.algorithms import AES from twisted.internet import interfaces, defer from zope.interface import implements -from Crypto import Random -from Crypto.Cipher import AES from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker @@ -101,13 +101,13 @@ class CryptStreamCreator(object): @staticmethod def random_iv_generator(): while 1: - yield Random.new().read(AES.block_size) + yield os.urandom(AES.block_size / 8) def setup(self): """Create the symmetric key if it wasn't provided""" if self.key is None: - self.key = Random.new().read(AES.block_size) + self.key = os.urandom(AES.block_size / 8) return defer.succeed(True) diff --git a/lbrynet/pointtraderclient/pointtraderclient.py b/lbrynet/pointtraderclient/pointtraderclient.py index 4084ddc8a..030337185 100644 --- a/lbrynet/pointtraderclient/pointtraderclient.py +++ b/lbrynet/pointtraderclient/pointtraderclient.py @@ -1,16 +1,40 @@ +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization, hashes +from cryptography.hazmat.primitives.asymmetric import rsa, padding, utils + from lbrynet import conf from twisted.web.client import Agent, FileBodyProducer, Headers, ResponseDone from twisted.internet import threads, defer, protocol -from Crypto.Hash import SHA -from Crypto.PublicKey import RSA -from Crypto.Signature import PKCS1_PSS +from hashlib import sha1 from StringIO import StringIO import time import json import binascii +def gen_rsa_key(bits): + PUBLIC_EXPOENT = 65537 # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html + return rsa.generate_private_key(public_exponent=PUBLIC_EXPOENT, + key_size=4096, backend=default_backend()) + + +def sign(private_key, recipient_public_key=None, amount=None): + encoded_public_key = private_key.public_key().public_bytes(serialization.Encoding.PEM, + serialization.PublicFormat.PKCS1) + timestamp = time.time() + h = sha1() + h.update(encoded_public_key) + if amount and recipient_public_key: + h.update(recipient_public_key) + h.update(str(amount)) + h.update(str(timestamp)) + signature = private_key.sign(h.digest(), padding.PSS(mgf=padding.MGF1(hashes.SHA1()), + salt_length=padding.PSS.MAX_LENGTH), + utils.Prehashed(hashes.SHA1())) + return encoded_public_key, timestamp, binascii.hexlify(signature) + + class BeginningPrinter(protocol.Protocol): def __init__(self, finished): self.finished = finished @@ -64,7 +88,9 @@ def print_error(err): def register_new_account(private_key): data = {} - data['pub_key'] = private_key.publickey().exportKey() + encoded_public_key = private_key.public_key().public_bytes(serialization.Encoding.PEM, + serialization.PublicFormat.PKCS1) + data['pub_key'] = encoded_public_key def get_success_from_body(body): r = json.loads(body) @@ -79,15 +105,7 @@ def register_new_account(private_key): def send_points(private_key, recipient_public_key, amount): - encoded_public_key = private_key.publickey().exportKey() - timestamp = time.time() - h = SHA.new() - h.update(encoded_public_key) - h.update(recipient_public_key) - h.update(str(amount)) - h.update(str(timestamp)) - signer = PKCS1_PSS.new(private_key) - signature = binascii.hexlify(signer.sign(h)) + encoded_public_key, timestamp, signature = sign(private_key, recipient_public_key, amount) data = {} data['sender_pub_key'] = encoded_public_key @@ -110,13 +128,7 @@ def send_points(private_key, recipient_public_key, amount): def get_recent_transactions(private_key): - encoded_public_key = private_key.publickey().exportKey() - timestamp = time.time() - h = SHA.new() - h.update(encoded_public_key) - h.update(str(timestamp)) - signer = PKCS1_PSS.new(private_key) - signature = binascii.hexlify(signer.sign(h)) + encoded_public_key, timestamp, signature = sign(private_key) data = {} data['pub_key'] = encoded_public_key @@ -140,13 +152,7 @@ def get_recent_transactions(private_key): def get_balance(private_key): - encoded_public_key = private_key.publickey().exportKey() - timestamp = time.time() - h = SHA.new() - h.update(encoded_public_key) - h.update(str(timestamp)) - signer = PKCS1_PSS.new(private_key) - signature = binascii.hexlify(signer.sign(h)) + encoded_public_key, timestamp, signature = sign(private_key) data = {} data['pub_key'] = encoded_public_key @@ -203,13 +209,15 @@ def run_full_test(): return dl def do_transfer(unused, amount): - d = send_points(keys[0], keys[1].publickey().exportKey(), amount) + encoded_public_key = keys[1].public_key().public_bytes(serialization.Encoding.PEM, + serialization.PublicFormat.PKCS1) + d = send_points(keys[0], encoded_public_key, amount) return d - d1 = threads.deferToThread(RSA.generate, 4096) + d1 = threads.deferToThread(gen_rsa_key, 4096) d1.addCallback(save_key) d1.addCallback(register_new_account) - d2 = threads.deferToThread(RSA.generate, 4096) + d2 = threads.deferToThread(gen_rsa_key, 4096) d2.addCallback(save_key) d2.addCallback(register_new_account) dlist = defer.DeferredList([d1, d2]) @@ -222,6 +230,7 @@ def run_full_test(): if __name__ == "__main__": + conf.initialize_settings() from twisted.internet import reactor From 44b08ae6c9dc2c356df42213e2ff7f4db38651fb Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 4 May 2018 23:59:08 -0300 Subject: [PATCH 03/18] remove pycrypto --- requirements.txt | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 37361b412..c7ca316f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,6 @@ git+https://github.com/lbryio/lbryschema.git@v0.0.15#egg=lbryschema git+https://github.com/lbryio/lbryum.git@v3.2.1#egg=lbryum miniupnpc==1.9 pbkdf2==1.3 -pycrypto==2.6.1 pyyaml==3.12 PyGithub==1.34 qrcode==5.2.2 diff --git a/setup.py b/setup.py index 7027c6a32..de95e8fde 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ requires = [ 'lbryschema==0.0.15', 'lbryum==3.2.1', 'miniupnpc', - 'pycrypto', 'pyyaml', 'requests', 'txrequests', From 820789936f707c64d07745e7d9c4c7cb882dee4d Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 00:08:02 -0300 Subject: [PATCH 04/18] bump cryptography==2.2.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c7ca316f3..96b589b9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ Twisted==16.6.0 -cryptography==2.0.3 +cryptography==2.2.2 appdirs==1.4.3 argparse==1.2.1 docopt==0.6.2 From 1077d8f93719c2e314b59c2740662b46100e9635 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 00:09:24 -0300 Subject: [PATCH 05/18] update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49125aa34..a3f6bddde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ at anytime. * ### Changed + * changed cryptography version to 2.2.2 + * removed pycrypto dependency, replacing all calls to cryptography * several internal dht functions to use inlineCallbacks * `DHTHashAnnouncer` and `Node` manage functions to use `LoopingCall`s instead of scheduling with `callLater`. * `store` kademlia rpc method to block on the call finishing and to return storing peer information From d03fc80eac6d233ee8632e5b3cbf0ba13af7cc92 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 00:50:42 -0300 Subject: [PATCH 06/18] make analytics use treq --- lbrynet/analytics.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lbrynet/analytics.py b/lbrynet/analytics.py index debd99728..6cd1f1e69 100644 --- a/lbrynet/analytics.py +++ b/lbrynet/analytics.py @@ -1,8 +1,8 @@ import collections import logging + +import treq from twisted.internet import defer, task -from requests import auth -from txrequests import Session from lbrynet import conf from lbrynet.core import looping_call_manager, utils, system_info @@ -216,8 +216,8 @@ class Manager(object): class Api(object): - def __init__(self, session, url, write_key, enabled): - self.session = session + def __init__(self, cookies, url, write_key, enabled): + self.cookies = cookies self.url = url self._write_key = write_key self._enabled = enabled @@ -232,14 +232,17 @@ class Api(object): # timeout will have expired. # # by forcing the connection to close, we will disable the keep-alive. + + def update_cookies(response): + self.cookies.update(response.cookies()) + return response + assert endpoint[0] == '/' - headers = {"Connection": "close"} - return self.session.post( - self.url + endpoint, - json=data, - auth=auth.HTTPBasicAuth(self._write_key, ''), - headers=headers - ) + headers = {b"Connection": b"close"} + d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data, + headers=headers, cookies=self.cookies) + d.addCallback(update_cookies) + return d def track(self, event): """Send a single tracking event""" @@ -257,11 +260,10 @@ class Api(object): @classmethod def new_instance(cls, enabled=None): """Initialize an instance using values from the configuration""" - session = Session() if enabled is None: enabled = conf.settings['share_usage_data'] return cls( - session, + {}, conf.settings['ANALYTICS_ENDPOINT'], utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']), enabled, From 3982e150915a5e18500d9a97394714d4ee33bc35 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 01:16:26 -0300 Subject: [PATCH 07/18] download headers from s3 using treq --- lbrynet/core/Wallet.py | 53 +++++++++++------------------------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index 6770faca4..9e272a7d5 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -3,14 +3,12 @@ from collections import defaultdict, deque import datetime import logging from decimal import Decimal + +import treq from zope.interface import implements from twisted.internet import threads, reactor, defer, task from twisted.python.failure import Failure -from twisted.python.threadpool import ThreadPool -from twisted._threads._ithreads import AlreadyQuit from twisted.internet.error import ConnectionAborted -from txrequests import Session as _TxRequestsSession -from requests import Session as requestsSession from lbryum import wallet as lbryum_wallet from lbryum.network import Network @@ -36,29 +34,6 @@ from lbrynet.core.Error import DownloadCanceledError, RequestCanceledError log = logging.getLogger(__name__) -class TxRequestsSession(_TxRequestsSession): - # Session from txrequests would throw AlreadyQuit errors, this catches them - def __init__(self, pool=None, minthreads=1, maxthreads=4, **kwargs): - requestsSession.__init__(self, **kwargs) # pylint: disable=non-parent-init-called - self.ownPool = False - if pool is None: - self.ownPool = True - pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads) - # unclosed ThreadPool leads to reactor hangs at shutdown - # this is a problem in many situation, so better enforce pool stop here - - def stop_pool(): - try: - pool.stop() - except AlreadyQuit: - pass - - reactor.addSystemEventTrigger("after", "shutdown", stop_pool) - self.pool = pool - if self.ownPool: - pool.start() - - class ReservedPoints(object): def __init__(self, identifier, amount): self.identifier = identifier @@ -118,20 +93,18 @@ class Wallet(object): @defer.inlineCallbacks def fetch_headers_from_s3(self): - with TxRequestsSession() as s: - r = yield s.get(HEADERS_URL) - raw_headers = r.content - if not len(raw_headers) % HEADER_SIZE: # should be divisible by the header size - s3_height = (len(raw_headers) / HEADER_SIZE) - 1 - local_height = self.local_header_file_height() - if s3_height > local_height: - with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file: - headers_file.write(raw_headers) - log.info("fetched headers from s3 (s3 height: %i)", s3_height) - else: - log.warning("s3 is more out of date than we are") + response = yield treq.get(HEADERS_URL) + if not response.length % HEADER_SIZE: # should be divisible by the header size + s3_height = (response.length / HEADER_SIZE) - 1 + local_height = self.local_header_file_height() + if s3_height > local_height: + with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file: + yield treq.collect(response, headers_file.write) + log.info("fetched headers from s3 (s3 height: %i)", s3_height) else: - log.error("invalid size for headers from s3") + log.warning("s3 is more out of date than we are") + else: + log.error("invalid size for headers from s3") def local_header_file_height(self): headers_path = os.path.join(self.config.path, "blockchain_headers") From 0bf65836c7fe388d4782a335a4e5791ddb73c631 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 02:05:02 -0300 Subject: [PATCH 08/18] resume download from S3 instead of starting from scratch --- lbrynet/core/Wallet.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index 9e272a7d5..64a3223c6 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -93,13 +93,23 @@ class Wallet(object): @defer.inlineCallbacks def fetch_headers_from_s3(self): - response = yield treq.get(HEADERS_URL) - if not response.length % HEADER_SIZE: # should be divisible by the header size - s3_height = (response.length / HEADER_SIZE) - 1 + local_header_size = self.local_header_file_size() + resume_header = {"Range": "bytes={}-".format(local_header_size)} + response = yield treq.get(HEADERS_URL, headers=resume_header) + got_406 = response.code == 406 # our file is bigger + final_size_after_download = response.length + local_header_size + # should have something to download and a final length divisible by the header size + if not got_406 and final_size_after_download and not final_size_after_download % HEADER_SIZE: + s3_height = (final_size_after_download / HEADER_SIZE) - 1 local_height = self.local_header_file_height() if s3_height > local_height: - with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file: - yield treq.collect(response, headers_file.write) + if local_header_size: + log.info("Resuming download of %i bytes from s3", response.length) + with open(os.path.join(self.config.path, "blockchain_headers"), "a+b") as headers_file: + yield treq.collect(response, headers_file.write) + else: + with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file: + yield treq.collect(response, headers_file.write) log.info("fetched headers from s3 (s3 height: %i)", s3_height) else: log.warning("s3 is more out of date than we are") @@ -107,11 +117,15 @@ class Wallet(object): log.error("invalid size for headers from s3") def local_header_file_height(self): + return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0) + + def local_header_file_size(self): headers_path = os.path.join(self.config.path, "blockchain_headers") if os.path.isfile(headers_path): - return max((os.stat(headers_path).st_size / 112) - 1, 0) + return os.stat(headers_path).st_size return 0 + @defer.inlineCallbacks def get_remote_height(self, server, port): connected = defer.Deferred() @@ -136,7 +150,7 @@ class Wallet(object): try: remote_height = yield self.get_remote_height(server_url, port) log.info("%s:%i height: %i, local height: %s", server_url, port, remote_height, local_height) - if remote_height > local_height + s3_headers_depth: + if remote_height > (local_height + s3_headers_depth): defer.returnValue(True) except Exception as err: log.warning("error requesting remote height from %s:%i - %s", server_url, port, err) From 7f88dda0aebc39ca7c613cd1d955c36ccde147cb Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 02:18:03 -0300 Subject: [PATCH 09/18] use treq on loggly --- lbrynet/core/log_support.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index a7a46dfeb..9e0a635d1 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -6,8 +6,7 @@ import os import sys import traceback -from txrequests import Session -from requests.exceptions import ConnectionError +import treq from twisted.internet import defer import twisted.python.log @@ -35,13 +34,13 @@ TRACE = 5 class HTTPSHandler(logging.Handler): - def __init__(self, url, fqdn=False, localname=None, facility=None, session=None): + def __init__(self, url, fqdn=False, localname=None, facility=None, cookies=None): logging.Handler.__init__(self) self.url = url self.fqdn = fqdn self.localname = localname self.facility = facility - self.session = session if session is not None else Session() + self.cookies = cookies or {} def get_full_message(self, record): if record.exc_info: @@ -52,10 +51,8 @@ class HTTPSHandler(logging.Handler): @defer.inlineCallbacks def _emit(self, record): payload = self.format(record) - try: - yield self.session.post(self.url, data=payload) - except ConnectionError: - pass + response = yield treq.post(self.url, data=payload, cookies=self.cookies) + self.cookies.update(response.cookies()) def emit(self, record): return self._emit(record) From e170f3db3ed51dd5a1b610df44156a4c958ad627 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 02:20:21 -0300 Subject: [PATCH 10/18] remove txrequests dependency --- lbrynet/core/system_info.py | 2 +- requirements.txt | 1 - setup.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lbrynet/core/system_info.py b/lbrynet/core/system_info.py index 94a1f15f2..95cd74bc9 100644 --- a/lbrynet/core/system_info.py +++ b/lbrynet/core/system_info.py @@ -37,7 +37,7 @@ def get_platform(get_ip=True): "build": build_type.BUILD, # CI server sets this during build step } - # TODO: remove this from get_platform and add a get_external_ip function using txrequests + # TODO: remove this from get_platform and add a get_external_ip function using treq if get_ip: try: response = json.loads(urlopen("https://api.lbry.io/ip").read()) diff --git a/requirements.txt b/requirements.txt index 96b589b9b..bea7b8a65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,6 @@ pyyaml==3.12 PyGithub==1.34 qrcode==5.2.2 requests==2.9.1 -txrequests==0.9.5 service_identity==16.0.0 six>=1.9.0 slowaes==0.1a1 diff --git a/setup.py b/setup.py index de95e8fde..7cc30c751 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,6 @@ requires = [ 'miniupnpc', 'pyyaml', 'requests', - 'txrequests', 'txJSON-RPC', 'zope.interface', 'docopt' From 8bfc35753af906e9be63fce325f61307d4dc8d50 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 02:20:55 -0300 Subject: [PATCH 11/18] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3f6bddde..e448d2e46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ at anytime. * ### Changed + * changed txrequests for treq * changed cryptography version to 2.2.2 * removed pycrypto dependency, replacing all calls to cryptography * several internal dht functions to use inlineCallbacks From c90140f70124ba3fb4e528d359271aaf6f3d5048 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 02:33:37 -0300 Subject: [PATCH 12/18] make the exchange manager use treq instead of requests --- lbrynet/daemon/ExchangeRateManager.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lbrynet/daemon/ExchangeRateManager.py b/lbrynet/daemon/ExchangeRateManager.py index 6fce06380..69310ea8c 100644 --- a/lbrynet/daemon/ExchangeRateManager.py +++ b/lbrynet/daemon/ExchangeRateManager.py @@ -1,8 +1,9 @@ import time -import requests import logging import json -from twisted.internet import defer, threads + +import treq +from twisted.internet import defer from twisted.internet.task import LoopingCall from lbrynet.core.Error import InvalidExchangeRateResponse @@ -52,9 +53,10 @@ class MarketFeed(object): def is_online(self): return self._online + @defer.inlineCallbacks def _make_request(self): - r = requests.get(self.url, self.params, timeout=self.REQUESTS_TIMEOUT) - return r.text + response = yield treq.get(self.url, params=self.params, timeout=self.REQUESTS_TIMEOUT) + defer.returnValue((yield response.content())) def _handle_response(self, response): return NotImplementedError @@ -75,7 +77,7 @@ class MarketFeed(object): self._online = False def _update_price(self): - d = threads.deferToThread(self._make_request) + d = self._make_request() d.addCallback(self._handle_response) d.addCallback(self._subtract_fee) d.addCallback(self._save_price) From 3ee0f9756d505f0d9696435336eb9fb39a7288bf Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 5 May 2018 20:51:49 -0300 Subject: [PATCH 13/18] add treq as a dependency + pep8 --- lbrynet/tests/functional/test_misc.py | 1 - lbrynet/tests/mocks.py | 3 ++- requirements.txt | 1 + setup.py | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lbrynet/tests/functional/test_misc.py b/lbrynet/tests/functional/test_misc.py index 34e0d80c9..b134b6da2 100644 --- a/lbrynet/tests/functional/test_misc.py +++ b/lbrynet/tests/functional/test_misc.py @@ -4,7 +4,6 @@ import os import platform import shutil import sys -import random import unittest from hashlib import md5 diff --git a/lbrynet/tests/mocks.py b/lbrynet/tests/mocks.py index 7de4927c7..25ac42e80 100644 --- a/lbrynet/tests/mocks.py +++ b/lbrynet/tests/mocks.py @@ -148,7 +148,8 @@ class PointTraderKeyQueryHandler(object): try: decode_rsa_key(new_encoded_pub_key) except (ValueError, TypeError, IndexError): - return defer.fail(Failure(ValueError("Client sent an invalid public key: {}".format(new_encoded_pub_key)))) + value_error = ValueError("Client sent an invalid public key: {}".format(new_encoded_pub_key)) + return defer.fail(Failure(value_error)) self.public_key = new_encoded_pub_key self.wallet.set_public_key_for_peer(self.peer, self.public_key) fields = {'public_key': self.wallet.encoded_public_key} diff --git a/requirements.txt b/requirements.txt index bea7b8a65..37725800c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,4 @@ slowaes==0.1a1 txJSON-RPC==0.5 wsgiref==0.1.2 zope.interface==4.3.3 +treq==17.8.0 diff --git a/setup.py b/setup.py index 7cc30c751..d4e4c0f43 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ requires = [ 'requests', 'txJSON-RPC', 'zope.interface', + 'treq', 'docopt' ] From 805bfbd18e8c6f84fc242fd2bdff6eeb048ddd34 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 7 May 2018 11:49:54 -0300 Subject: [PATCH 14/18] remove point trader testing client --- lbrynet/pointtraderclient/__init__.py | 10 - .../pointtraderclient/pointtraderclient.py | 239 ------------------ 2 files changed, 249 deletions(-) delete mode 100644 lbrynet/pointtraderclient/__init__.py delete mode 100644 lbrynet/pointtraderclient/pointtraderclient.py diff --git a/lbrynet/pointtraderclient/__init__.py b/lbrynet/pointtraderclient/__init__.py deleted file mode 100644 index 4c5b43dde..000000000 --- a/lbrynet/pointtraderclient/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -A client library for sending and receiving payments on the point trader network. - -The point trader network is a simple payment system used solely for testing lbrynet-console. A user -creates a public key, registers it with the point trader server, and receives free points for -registering. The public key is used to spend points, and also used as an address to which points -are sent. To spend points, the public key signs a message containing the amount and the destination -public key and sends it to the point trader server. To check for payments, the recipient sends a -signed message asking the point trader server for its balance. -""" diff --git a/lbrynet/pointtraderclient/pointtraderclient.py b/lbrynet/pointtraderclient/pointtraderclient.py deleted file mode 100644 index 030337185..000000000 --- a/lbrynet/pointtraderclient/pointtraderclient.py +++ /dev/null @@ -1,239 +0,0 @@ -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization, hashes -from cryptography.hazmat.primitives.asymmetric import rsa, padding, utils - -from lbrynet import conf - -from twisted.web.client import Agent, FileBodyProducer, Headers, ResponseDone -from twisted.internet import threads, defer, protocol -from hashlib import sha1 -from StringIO import StringIO -import time -import json -import binascii - - -def gen_rsa_key(bits): - PUBLIC_EXPOENT = 65537 # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html - return rsa.generate_private_key(public_exponent=PUBLIC_EXPOENT, - key_size=4096, backend=default_backend()) - - -def sign(private_key, recipient_public_key=None, amount=None): - encoded_public_key = private_key.public_key().public_bytes(serialization.Encoding.PEM, - serialization.PublicFormat.PKCS1) - timestamp = time.time() - h = sha1() - h.update(encoded_public_key) - if amount and recipient_public_key: - h.update(recipient_public_key) - h.update(str(amount)) - h.update(str(timestamp)) - signature = private_key.sign(h.digest(), padding.PSS(mgf=padding.MGF1(hashes.SHA1()), - salt_length=padding.PSS.MAX_LENGTH), - utils.Prehashed(hashes.SHA1())) - return encoded_public_key, timestamp, binascii.hexlify(signature) - - -class BeginningPrinter(protocol.Protocol): - def __init__(self, finished): - self.finished = finished - self.data = "" - - def dataReceived(self, bytes): - self.data = self.data + bytes - - def connectionLost(self, reason): - if reason.check(ResponseDone) is not None: - self.finished.callback(str(self.data)) - else: - self.finished.errback(reason) - - -def read_body(response): - d = defer.Deferred() - response.deliverBody(BeginningPrinter(d)) - return d - - -def get_body(response): - if response.code != 200: - print "\n\n\n\nbad error code\n\n\n\n" - raise ValueError(response.phrase) - else: - return read_body(response) - - -def get_body_from_request(path, data): - - from twisted.internet import reactor - - jsondata = FileBodyProducer(StringIO(json.dumps(data))) - agent = Agent(reactor) - d = agent.request( - 'POST', conf.settings['pointtrader_server'] + path, - Headers({'Content-Type': ['application/json']}), jsondata) - d.addCallback(get_body) - return d - - -def print_response(response): - pass - - -def print_error(err): - print err.getTraceback() - return err - - -def register_new_account(private_key): - data = {} - encoded_public_key = private_key.public_key().public_bytes(serialization.Encoding.PEM, - serialization.PublicFormat.PKCS1) - data['pub_key'] = encoded_public_key - - def get_success_from_body(body): - r = json.loads(body) - if not 'success' in r or r['success'] is False: - return False - return True - - d = get_body_from_request('/register/', data) - - d.addCallback(get_success_from_body) - return d - - -def send_points(private_key, recipient_public_key, amount): - encoded_public_key, timestamp, signature = sign(private_key, recipient_public_key, amount) - - data = {} - data['sender_pub_key'] = encoded_public_key - data['recipient_pub_key'] = recipient_public_key - data['amount'] = amount - data['timestamp'] = timestamp - data['signature'] = signature - - def get_success_from_body(body): - r = json.loads(body) - if not 'success' in r or r['success'] is False: - return False - return True - - d = get_body_from_request('/send-points/', data) - - d.addCallback(get_success_from_body) - - return d - - -def get_recent_transactions(private_key): - encoded_public_key, timestamp, signature = sign(private_key) - - data = {} - data['pub_key'] = encoded_public_key - data['timestamp'] = timestamp - data['signature'] = signature - data['end_time'] = 0 - data['start_time'] = 120 - - def get_transactions_from_body(body): - r = json.loads(body) - if "transactions" not in r: - raise ValueError("Invalid response: no 'transactions' field") - else: - return r['transactions'] - - d = get_body_from_request('/get-transactions/', data) - - d.addCallback(get_transactions_from_body) - - return d - - -def get_balance(private_key): - encoded_public_key, timestamp, signature = sign(private_key) - - data = {} - data['pub_key'] = encoded_public_key - data['timestamp'] = timestamp - data['signature'] = signature - - def get_balance_from_body(body): - r = json.loads(body) - if not 'balance' in r: - raise ValueError("Invalid response: no 'balance' field") - else: - return float(r['balance']) - - d = get_body_from_request('/get-balance/', data) - - d.addCallback(get_balance_from_body) - - return d - - -def run_full_test(): - - keys = [] - - def save_key(private_key): - keys.append(private_key) - return private_key - - def check_balances_and_transactions(unused, bal1, bal2, num_transactions): - - def assert_balance_is(actual, expected): - assert abs(actual - expected) < .05 - print "correct balance. actual:", str(actual), "expected:", str(expected) - return True - - def assert_transaction_length_is(transactions, expected_length): - assert len(transactions) == expected_length - print "correct transaction length" - return True - - d1 = get_balance(keys[0]) - d1.addCallback(assert_balance_is, bal1) - - d2 = get_balance(keys[1]) - d2.addCallback(assert_balance_is, bal2) - - d3 = get_recent_transactions(keys[0]) - d3.addCallback(assert_transaction_length_is, num_transactions) - - d4 = get_recent_transactions(keys[1]) - d4.addCallback(assert_transaction_length_is, num_transactions) - - dl = defer.DeferredList([d1, d2, d3, d4]) - return dl - - def do_transfer(unused, amount): - encoded_public_key = keys[1].public_key().public_bytes(serialization.Encoding.PEM, - serialization.PublicFormat.PKCS1) - d = send_points(keys[0], encoded_public_key, amount) - return d - - d1 = threads.deferToThread(gen_rsa_key, 4096) - d1.addCallback(save_key) - d1.addCallback(register_new_account) - d2 = threads.deferToThread(gen_rsa_key, 4096) - d2.addCallback(save_key) - d2.addCallback(register_new_account) - dlist = defer.DeferredList([d1, d2]) - dlist.addCallback(check_balances_and_transactions, 1000, 1000, 0) - dlist.addCallback(do_transfer, 50) - dlist.addCallback(check_balances_and_transactions, 950, 1050, 1) - dlist.addCallback(do_transfer, 75) - dlist.addCallback(check_balances_and_transactions, 875, 1125, 2) - dlist.addErrback(print_error) - - -if __name__ == "__main__": - conf.initialize_settings() - - from twisted.internet import reactor - - reactor.callLater(1, run_full_test) - reactor.callLater(25, reactor.stop) - reactor.run() From fd04c607b27ca68d74fac2a72e8452cfc9a2f7df Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 7 May 2018 13:11:31 -0300 Subject: [PATCH 15/18] typos and fixes from code review --- lbrynet/core/Wallet.py | 4 +++- lbrynet/tests/mocks.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index 64a3223c6..577e6e3d4 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -98,8 +98,10 @@ class Wallet(object): response = yield treq.get(HEADERS_URL, headers=resume_header) got_406 = response.code == 406 # our file is bigger final_size_after_download = response.length + local_header_size + if got_406: + log.warning("s3 is more out of date than we are") # should have something to download and a final length divisible by the header size - if not got_406 and final_size_after_download and not final_size_after_download % HEADER_SIZE: + elif final_size_after_download and not final_size_after_download % HEADER_SIZE: s3_height = (final_size_after_download / HEADER_SIZE) - 1 local_height = self.local_header_file_height() if s3_height > local_height: diff --git a/lbrynet/tests/mocks.py b/lbrynet/tests/mocks.py index 25ac42e80..12770c188 100644 --- a/lbrynet/tests/mocks.py +++ b/lbrynet/tests/mocks.py @@ -18,7 +18,7 @@ from lbrynet import conf from util import debug_kademlia_packet KB = 2**10 -PUBLIC_EXPOENT = 65537 # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html +PUBLIC_EXPONENT = 65537 # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html def decode_rsa_key(pem_key): @@ -162,7 +162,7 @@ class PointTraderKeyQueryHandler(object): class Wallet(object): def __init__(self): - self.private_key = rsa.generate_private_key(public_exponent=PUBLIC_EXPOENT, + self.private_key = rsa.generate_private_key(public_exponent=PUBLIC_EXPONENT, key_size=1024, backend=default_backend()) self.encoded_public_key = self.private_key.public_key().public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.PKCS1) From 2299098884833ccf7017edb3b5d718305c93991c Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 7 May 2018 15:10:19 -0300 Subject: [PATCH 16/18] add integrity check for the headers file --- lbrynet/conf.py | 1 + lbrynet/core/Wallet.py | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 0be7a423e..9c596216e 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -236,6 +236,7 @@ FIXED_SETTINGS = { 'SLACK_WEBHOOK': ('nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5' 'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='), 'WALLET_TYPES': [LBRYUM_WALLET, LBRYCRD_WALLET], + 'HEADERS_FILE_SHA256_CHECKSUM': (366295, 'b0c8197153a33ccbc52fb81a279588b6015b68b7726f73f6a2b81f7e25bfe4b9') } ADJUSTABLE_SETTINGS = { diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index 577e6e3d4..00f1511a9 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -10,6 +10,7 @@ from twisted.internet import threads, reactor, defer, task from twisted.python.failure import Failure from twisted.internet.error import ConnectionAborted +from hashlib import sha256 from lbryum import wallet as lbryum_wallet from lbryum.network import Network from lbryum.simple_config import SimpleConfig @@ -127,7 +128,6 @@ class Wallet(object): return os.stat(headers_path).st_size return 0 - @defer.inlineCallbacks def get_remote_height(self, server, port): connected = defer.Deferred() @@ -143,6 +143,7 @@ class Wallet(object): from lbrynet import conf if conf.settings['blockchain_name'] != "lbrycrd_main": defer.returnValue(False) + self._check_header_file_integrity(conf) s3_headers_depth = conf.settings['s3_headers_depth'] if not s3_headers_depth: defer.returnValue(False) @@ -158,6 +159,27 @@ class Wallet(object): log.warning("error requesting remote height from %s:%i - %s", server_url, port, err) defer.returnValue(False) + def _check_header_file_integrity(self, conf): + # TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity + hashsum = sha256() + checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM'] + checksum_length_in_bytes = checksum_height * HEADER_SIZE + if self.local_header_file_size() < checksum_length_in_bytes: + return + headers_path = os.path.join(self.config.path, "blockchain_headers") + with open(headers_path, "rb") as headers_file: + hashsum.update(headers_file.read(checksum_length_in_bytes)) + current_checksum = hashsum.hexdigest() + if current_checksum != checksum: + msg = "Expected checksum {}, got {}".format(checksum, current_checksum) + log.warning("Wallet file corrupted, checksum mismatch. " + msg) + log.warning("Deleting header file so it can be downloaded again.") + os.unlink(headers_path) + elif (self.local_header_file_size() % HEADER_SIZE) != 0: + log.warning("Header file is good up to checkpoint height, but incomplete. Truncating to checkpoint.") + with open(headers_path, "rb+") as headers_file: + headers_file.truncate(checksum_length_in_bytes) + @defer.inlineCallbacks def start(self): should_download_headers = yield self.should_download_headers_from_s3() From d2e7ac5b6e39a91626793361655fc208399449d0 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 7 May 2018 15:10:27 -0300 Subject: [PATCH 17/18] update changelog for the integrity check and partial s3 downloads --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e448d2e46..28a256d4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ at anytime. * ### Changed + * check headers file integrity on startup, removing/truncating the file to force re-download when necessary + * support partial headers file download from S3 * changed txrequests for treq * changed cryptography version to 2.2.2 * removed pycrypto dependency, replacing all calls to cryptography From a0a7187f7d208a81f7d81efc8327e22cf37fda44 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 8 May 2018 16:09:06 -0300 Subject: [PATCH 18/18] add integrity checks after s3 download as well --- lbrynet/core/Wallet.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index 00f1511a9..8e7b4f429 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -113,7 +113,8 @@ class Wallet(object): else: with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file: yield treq.collect(response, headers_file.write) - log.info("fetched headers from s3 (s3 height: %i)", s3_height) + log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height) + self._check_header_file_integrity() else: log.warning("s3 is more out of date than we are") else: @@ -143,7 +144,7 @@ class Wallet(object): from lbrynet import conf if conf.settings['blockchain_name'] != "lbrycrd_main": defer.returnValue(False) - self._check_header_file_integrity(conf) + self._check_header_file_integrity() s3_headers_depth = conf.settings['s3_headers_depth'] if not s3_headers_depth: defer.returnValue(False) @@ -159,8 +160,11 @@ class Wallet(object): log.warning("error requesting remote height from %s:%i - %s", server_url, port, err) defer.returnValue(False) - def _check_header_file_integrity(self, conf): + def _check_header_file_integrity(self): # TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity + from lbrynet import conf + if conf.settings['blockchain_name'] != "lbrycrd_main": + return hashsum = sha256() checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM'] checksum_length_in_bytes = checksum_height * HEADER_SIZE