LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Merge #13210: Enable W191 indentation contains tabs and W291 trailing whitespace flake8 checks for Python files

Browse source 
0d31ef4762 Enable W191 and W291 flake8 checks. Remove trailing whitespace from Python files. Convert tabs to spaces. (John Bampton)

Pull request description:

Tree-SHA512: d062434310d6232469d7ca8e5f2ddb7db7e85cb2a299e609d98bacc318368e43e0777c9f4966df03d50f526bbe27207faa87a7464e62e14671194459a06ad969
This commit is contained in:
MarcoFalke 2018-05-10 20:59:36 -04:00
commit 81d0d565ab
No known key found for this signature in database
GPG key ID: D2EA4850E7528B25
13 changed files with 367 additions and 365 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
contrib/devtools/copyright_header.py
View file

@ -506,7 +506,7 @@ def file_has_hashbang(file_lines):
def insert_python_header(filename, file_lines, start_year, end_year): def insert_python_header(filename, file_lines, start_year, end_year):
if file_has_hashbang(file_lines): if file_has_hashbang(file_lines):
insert_idx = 1 insert_idx = 1
else: else:
insert_idx = 0 insert_idx = 0
header_lines = get_python_header_lines_to_insert(start_year, end_year) header_lines = get_python_header_lines_to_insert(start_year, end_year)
@ -571,7 +571,7 @@ def insert_cmd(argv):
if extension not in ['.h', '.cpp', '.cc', '.c', '.py']: if extension not in ['.h', '.cpp', '.cc', '.c', '.py']:
sys.exit("*** cannot insert for file extension %s" % extension) sys.exit("*** cannot insert for file extension %s" % extension)
if extension == '.py': if extension == '.py':
style = 'python' style = 'python'
else: else:
style = 'cpp' style = 'cpp'

4
contrib/devtools/lint-python.sh
View file

@ -61,6 +61,8 @@
# F823 local variable name … referenced before assignment # F823 local variable name … referenced before assignment
# F831 duplicate argument name in function definition # F831 duplicate argument name in function definition
# F841 local variable 'foo' is assigned to but never used # F841 local variable 'foo' is assigned to but never used
# W191 indentation contains tabs
# W291 trailing whitespace
# W292 no newline at end of file # W292 no newline at end of file
# W293 blank line contains whitespace # W293 blank line contains whitespace
# W504 line break after binary operator # W504 line break after binary operator
@ -71,4 +73,4 @@
# W605 invalid escape sequence "x" # W605 invalid escape sequence "x"
# W606 'async' and 'await' are reserved keywords starting with Python 3.7 # W606 'async' and 'await' are reserved keywords starting with Python 3.7
flake8 --ignore=B,C,E,F,I,N,W --select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,F401,E901,E902,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W292,W293,W504,W601,W602,W603,W604,W605,W606 . flake8 --ignore=B,C,E,F,I,N,W --select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,F401,E901,E902,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W191,W291,W292,W293,W504,W601,W602,W603,W604,W605,W606 .

2
contrib/devtools/security-check.py
View file

@ -150,7 +150,7 @@ def check_PE_DYNAMIC_BASE(executable):
def check_PE_HIGH_ENTROPY_VA(executable): def check_PE_HIGH_ENTROPY_VA(executable):
'''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR'''
(arch,bits) = get_PE_dll_characteristics(executable) (arch,bits) = get_PE_dll_characteristics(executable)
if arch == 'i386:x86-64': if arch == 'i386:x86-64':
reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA
else: # Unnecessary on 32-bit else: # Unnecessary on 32-bit
assert(arch == 'i386') assert(arch == 'i386')

16
contrib/devtools/test-security-check.py
View file

@ -32,15 +32,15 @@ class TestSecurityChecks(unittest.TestCase):
cc = 'gcc' cc = 'gcc'
write_testcode(source) write_testcode(source)
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']),
(1, executable+': failed PIE NX RELRO Canary')) (1, executable+': failed PIE NX RELRO Canary'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']),
(1, executable+': failed PIE RELRO Canary')) (1, executable+': failed PIE RELRO Canary'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']),
(1, executable+': failed PIE RELRO')) (1, executable+': failed PIE RELRO'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']),
(1, executable+': failed RELRO')) (1, executable+': failed RELRO'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']),
(0, '')) (0, ''))
def test_32bit_PE(self): def test_32bit_PE(self):
@ -49,11 +49,11 @@ class TestSecurityChecks(unittest.TestCase):
cc = 'i686-w64-mingw32-gcc' cc = 'i686-w64-mingw32-gcc'
write_testcode(source) write_testcode(source)
self.assertEqual(call_security_check(cc, source, executable, []), self.assertEqual(call_security_check(cc, source, executable, []),
(1, executable+': failed DYNAMIC_BASE NX')) (1, executable+': failed DYNAMIC_BASE NX'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']),
(1, executable+': failed DYNAMIC_BASE')) (1, executable+': failed DYNAMIC_BASE'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']),
(0, '')) (0, ''))
def test_64bit_PE(self): def test_64bit_PE(self):
source = 'test1.c' source = 'test1.c'

474
contrib/linearize/linearize-data.py
View file

@ -22,300 +22,300 @@ from binascii import hexlify, unhexlify
settings = {} settings = {}
def hex_switchEndian(s): def hex_switchEndian(s):
""" Switches the endianness of a hex string (in pairs of hex chars) """ """ Switches the endianness of a hex string (in pairs of hex chars) """
pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
return b''.join(pairList[::-1]).decode() return b''.join(pairList[::-1]).decode()
def uint32(x): def uint32(x):
return x & 0xffffffff return x & 0xffffffff
def bytereverse(x): def bytereverse(x):
return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
(((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) (((x) >> 8) & 0x0000ff00) | ((x) >> 24) ))
def bufreverse(in_buf): def bufreverse(in_buf):
out_words = [] out_words = []
for i in range(0, len(in_buf), 4): for i in range(0, len(in_buf), 4):
word = struct.unpack('@I', in_buf[i:i+4])[0] word = struct.unpack('@I', in_buf[i:i+4])[0]
out_words.append(struct.pack('@I', bytereverse(word))) out_words.append(struct.pack('@I', bytereverse(word)))
return b''.join(out_words) return b''.join(out_words)
def wordreverse(in_buf): def wordreverse(in_buf):
out_words = [] out_words = []
for i in range(0, len(in_buf), 4): for i in range(0, len(in_buf), 4):
out_words.append(in_buf[i:i+4]) out_words.append(in_buf[i:i+4])
out_words.reverse() out_words.reverse()
return b''.join(out_words) return b''.join(out_words)
def calc_hdr_hash(blk_hdr): def calc_hdr_hash(blk_hdr):
hash1 = hashlib.sha256() hash1 = hashlib.sha256()
hash1.update(blk_hdr) hash1.update(blk_hdr)
hash1_o = hash1.digest() hash1_o = hash1.digest()
hash2 = hashlib.sha256() hash2 = hashlib.sha256()
hash2.update(hash1_o) hash2.update(hash1_o)
hash2_o = hash2.digest() hash2_o = hash2.digest()
return hash2_o return hash2_o
def calc_hash_str(blk_hdr): def calc_hash_str(blk_hdr):
hash = calc_hdr_hash(blk_hdr) hash = calc_hdr_hash(blk_hdr)
hash = bufreverse(hash) hash = bufreverse(hash)
hash = wordreverse(hash) hash = wordreverse(hash)
hash_str = hexlify(hash).decode('utf-8') hash_str = hexlify(hash).decode('utf-8')
return hash_str return hash_str
def get_blk_dt(blk_hdr): def get_blk_dt(blk_hdr):
members = struct.unpack("<I", blk_hdr[68:68+4]) members = struct.unpack("<I", blk_hdr[68:68+4])
nTime = members[0] nTime = members[0]
dt = datetime.datetime.fromtimestamp(nTime) dt = datetime.datetime.fromtimestamp(nTime)
dt_ym = datetime.datetime(dt.year, dt.month, 1) dt_ym = datetime.datetime(dt.year, dt.month, 1)
return (dt_ym, nTime) return (dt_ym, nTime)
# When getting the list of block hashes, undo any byte reversals. # When getting the list of block hashes, undo any byte reversals.
def get_block_hashes(settings): def get_block_hashes(settings):
blkindex = [] blkindex = []
f = open(settings['hashlist'], "r") f = open(settings['hashlist'], "r")
for line in f: for line in f:
line = line.rstrip() line = line.rstrip()
if settings['rev_hash_bytes'] == 'true': if settings['rev_hash_bytes'] == 'true':
line = hex_switchEndian(line) line = hex_switchEndian(line)
blkindex.append(line) blkindex.append(line)
print("Read " + str(len(blkindex)) + " hashes") print("Read " + str(len(blkindex)) + " hashes")
return blkindex return blkindex
# The block map shouldn't give or receive byte-reversed hashes. # The block map shouldn't give or receive byte-reversed hashes.
def mkblockmap(blkindex): def mkblockmap(blkindex):
blkmap = {} blkmap = {}
for height,hash in enumerate(blkindex): for height,hash in enumerate(blkindex):
blkmap[hash] = height blkmap[hash] = height
return blkmap return blkmap
# Block header and extent on disk # Block header and extent on disk
BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size']) BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size'])
class BlockDataCopier: class BlockDataCopier:
def __init__(self, settings, blkindex, blkmap): def __init__(self, settings, blkindex, blkmap):
self.settings = settings self.settings = settings
self.blkindex = blkindex self.blkindex = blkindex
self.blkmap = blkmap self.blkmap = blkmap
self.inFn = 0 self.inFn = 0
self.inF = None self.inF = None
self.outFn = 0 self.outFn = 0
self.outsz = 0 self.outsz = 0
self.outF = None self.outF = None
self.outFname = None self.outFname = None
self.blkCountIn = 0 self.blkCountIn = 0
self.blkCountOut = 0 self.blkCountOut = 0
self.lastDate = datetime.datetime(2000, 1, 1) self.lastDate = datetime.datetime(2000, 1, 1)
self.highTS = 1408893517 - 315360000 self.highTS = 1408893517 - 315360000
self.timestampSplit = False self.timestampSplit = False
self.fileOutput = True self.fileOutput = True
self.setFileTime = False self.setFileTime = False
self.maxOutSz = settings['max_out_sz'] self.maxOutSz = settings['max_out_sz']
if 'output' in settings: if 'output' in settings:
self.fileOutput = False self.fileOutput = False
if settings['file_timestamp'] != 0: if settings['file_timestamp'] != 0:
self.setFileTime = True self.setFileTime = True
if settings['split_timestamp'] != 0: if settings['split_timestamp'] != 0:
self.timestampSplit = True self.timestampSplit = True
# Extents and cache for out-of-order blocks # Extents and cache for out-of-order blocks
self.blockExtents = {} self.blockExtents = {}
self.outOfOrderData = {} self.outOfOrderData = {}
self.outOfOrderSize = 0 # running total size for items in outOfOrderData self.outOfOrderSize = 0 # running total size for items in outOfOrderData
def writeBlock(self, inhdr, blk_hdr, rawblock): def writeBlock(self, inhdr, blk_hdr, rawblock):
blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock) blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock)
if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz): if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz):
self.outF.close() self.outF.close()
if self.setFileTime: if self.setFileTime:
os.utime(self.outFname, (int(time.time()), self.highTS)) os.utime(self.outFname, (int(time.time()), self.highTS))
self.outF = None self.outF = None
self.outFname = None self.outFname = None
self.outFn = self.outFn + 1 self.outFn = self.outFn + 1
self.outsz = 0 self.outsz = 0
(blkDate, blkTS) = get_blk_dt(blk_hdr) (blkDate, blkTS) = get_blk_dt(blk_hdr)
if self.timestampSplit and (blkDate > self.lastDate): if self.timestampSplit and (blkDate > self.lastDate):
print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str)
self.lastDate = blkDate self.lastDate = blkDate
if self.outF: if self.outF:
self.outF.close() self.outF.close()
if self.setFileTime: if self.setFileTime:
os.utime(self.outFname, (int(time.time()), self.highTS)) os.utime(self.outFname, (int(time.time()), self.highTS))
self.outF = None self.outF = None
self.outFname = None self.outFname = None
self.outFn = self.outFn + 1 self.outFn = self.outFn + 1
self.outsz = 0 self.outsz = 0
if not self.outF: if not self.outF:
if self.fileOutput: if self.fileOutput:
self.outFname = self.settings['output_file'] self.outFname = self.settings['output_file']
else: else:
self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn)
print("Output file " + self.outFname) print("Output file " + self.outFname)
self.outF = open(self.outFname, "wb") self.outF = open(self.outFname, "wb")
self.outF.write(inhdr) self.outF.write(inhdr)
self.outF.write(blk_hdr) self.outF.write(blk_hdr)
self.outF.write(rawblock) self.outF.write(rawblock)
self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock)
self.blkCountOut = self.blkCountOut + 1 self.blkCountOut = self.blkCountOut + 1
if blkTS > self.highTS: if blkTS > self.highTS:
self.highTS = blkTS self.highTS = blkTS
if (self.blkCountOut % 1000) == 0: if (self.blkCountOut % 1000) == 0:
print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' % print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' %
(self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) (self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex)))
def inFileName(self, fn): def inFileName(self, fn):
return os.path.join(self.settings['input'], "blk%05d.dat" % fn) return os.path.join(self.settings['input'], "blk%05d.dat" % fn)
def fetchBlock(self, extent): def fetchBlock(self, extent):
'''Fetch block contents from disk given extents''' '''Fetch block contents from disk given extents'''
with open(self.inFileName(extent.fn), "rb") as f: with open(self.inFileName(extent.fn), "rb") as f:
f.seek(extent.offset) f.seek(extent.offset)
return f.read(extent.size) return f.read(extent.size)
def copyOneBlock(self): def copyOneBlock(self):
'''Find the next block to be written in the input, and copy it to the output.''' '''Find the next block to be written in the input, and copy it to the output.'''
extent = self.blockExtents.pop(self.blkCountOut) extent = self.blockExtents.pop(self.blkCountOut)
if self.blkCountOut in self.outOfOrderData: if self.blkCountOut in self.outOfOrderData:
# If the data is cached, use it from memory and remove from the cache # If the data is cached, use it from memory and remove from the cache
rawblock = self.outOfOrderData.pop(self.blkCountOut) rawblock = self.outOfOrderData.pop(self.blkCountOut)
self.outOfOrderSize -= len(rawblock) self.outOfOrderSize -= len(rawblock)
else: # Otherwise look up data on disk else: # Otherwise look up data on disk
rawblock = self.fetchBlock(extent) rawblock = self.fetchBlock(extent)
self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) self.writeBlock(extent.inhdr, extent.blkhdr, rawblock)
def run(self): def run(self):
while self.blkCountOut < len(self.blkindex): while self.blkCountOut < len(self.blkindex):
if not self.inF: if not self.inF:
fname = self.inFileName(self.inFn) fname = self.inFileName(self.inFn)
print("Input file " + fname) print("Input file " + fname)
try: try:
self.inF = open(fname, "rb") self.inF = open(fname, "rb")
except IOError: except IOError:
print("Premature end of block data") print("Premature end of block data")
return return
inhdr = self.inF.read(8) inhdr = self.inF.read(8)
if (not inhdr or (inhdr[0] == "\0")): if (not inhdr or (inhdr[0] == "\0")):
self.inF.close() self.inF.close()
self.inF = None self.inF = None
self.inFn = self.inFn + 1 self.inFn = self.inFn + 1
continue continue
inMagic = inhdr[:4] inMagic = inhdr[:4]
if (inMagic != self.settings['netmagic']): if (inMagic != self.settings['netmagic']):
print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) print("Invalid magic: " + hexlify(inMagic).decode('utf-8'))
return return
inLenLE = inhdr[4:] inLenLE = inhdr[4:]
su = struct.unpack("<I", inLenLE) su = struct.unpack("<I", inLenLE)
inLen = su[0] - 80 # length without header inLen = su[0] - 80 # length without header
blk_hdr = self.inF.read(80) blk_hdr = self.inF.read(80)
inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen) inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen)
self.hash_str = calc_hash_str(blk_hdr) self.hash_str = calc_hash_str(blk_hdr)
if not self.hash_str in blkmap: if not self.hash_str in blkmap:
# Because blocks can be written to files out-of-order as of 0.10, the script # Because blocks can be written to files out-of-order as of 0.10, the script
# may encounter blocks it doesn't know about. Treat as debug output. # may encounter blocks it doesn't know about. Treat as debug output.
if settings['debug_output'] == 'true': if settings['debug_output'] == 'true':
print("Skipping unknown block " + self.hash_str) print("Skipping unknown block " + self.hash_str)
self.inF.seek(inLen, os.SEEK_CUR) self.inF.seek(inLen, os.SEEK_CUR)
continue continue
blkHeight = self.blkmap[self.hash_str] blkHeight = self.blkmap[self.hash_str]
self.blkCountIn += 1 self.blkCountIn += 1
if self.blkCountOut == blkHeight: if self.blkCountOut == blkHeight:
# If in-order block, just copy # If in-order block, just copy
rawblock = self.inF.read(inLen) rawblock = self.inF.read(inLen)
self.writeBlock(inhdr, blk_hdr, rawblock) self.writeBlock(inhdr, blk_hdr, rawblock)
# See if we can catch up to prior out-of-order blocks # See if we can catch up to prior out-of-order blocks
while self.blkCountOut in self.blockExtents: while self.blkCountOut in self.blockExtents:
self.copyOneBlock() self.copyOneBlock()
else: # If out-of-order, skip over block data for now else: # If out-of-order, skip over block data for now
self.blockExtents[blkHeight] = inExtent self.blockExtents[blkHeight] = inExtent
if self.outOfOrderSize < self.settings['out_of_order_cache_sz']: if self.outOfOrderSize < self.settings['out_of_order_cache_sz']:
# If there is space in the cache, read the data # If there is space in the cache, read the data
# Reading the data in file sequence instead of seeking and fetching it later is preferred, # Reading the data in file sequence instead of seeking and fetching it later is preferred,
# but we don't want to fill up memory # but we don't want to fill up memory
self.outOfOrderData[blkHeight] = self.inF.read(inLen) self.outOfOrderData[blkHeight] = self.inF.read(inLen)
self.outOfOrderSize += inLen self.outOfOrderSize += inLen
else: # If no space in cache, seek forward else: # If no space in cache, seek forward
self.inF.seek(inLen, os.SEEK_CUR) self.inF.seek(inLen, os.SEEK_CUR)
print("Done (%i blocks written)" % (self.blkCountOut)) print("Done (%i blocks written)" % (self.blkCountOut))
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) != 2: if len(sys.argv) != 2:
print("Usage: linearize-data.py CONFIG-FILE") print("Usage: linearize-data.py CONFIG-FILE")
sys.exit(1) sys.exit(1)
f = open(sys.argv[1]) f = open(sys.argv[1])
for line in f: for line in f:
# skip comment lines # skip comment lines
m = re.search('^\s*#', line) m = re.search('^\s*#', line)
if m: if m:
continue continue
# parse key=value lines # parse key=value lines
m = re.search('^(\w+)\s*=\s*(\S.*)$', line) m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
if m is None: if m is None:
continue continue
settings[m.group(1)] = m.group(2) settings[m.group(1)] = m.group(2)
f.close() f.close()
# Force hash byte format setting to be lowercase to make comparisons easier. # Force hash byte format setting to be lowercase to make comparisons easier.
# Also place upfront in case any settings need to know about it. # Also place upfront in case any settings need to know about it.
if 'rev_hash_bytes' not in settings: if 'rev_hash_bytes' not in settings:
settings['rev_hash_bytes'] = 'false' settings['rev_hash_bytes'] = 'false'
settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower()
if 'netmagic' not in settings: if 'netmagic' not in settings:
settings['netmagic'] = 'f9beb4d9' settings['netmagic'] = 'f9beb4d9'
if 'genesis' not in settings: if 'genesis' not in settings:
settings['genesis'] = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' settings['genesis'] = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'
if 'input' not in settings: if 'input' not in settings:
settings['input'] = 'input' settings['input'] = 'input'
if 'hashlist' not in settings: if 'hashlist' not in settings:
settings['hashlist'] = 'hashlist.txt' settings['hashlist'] = 'hashlist.txt'
if 'file_timestamp' not in settings: if 'file_timestamp' not in settings:
settings['file_timestamp'] = 0 settings['file_timestamp'] = 0
if 'split_timestamp' not in settings: if 'split_timestamp' not in settings:
settings['split_timestamp'] = 0 settings['split_timestamp'] = 0
if 'max_out_sz' not in settings: if 'max_out_sz' not in settings:
settings['max_out_sz'] = 1000 * 1000 * 1000 settings['max_out_sz'] = 1000 * 1000 * 1000
if 'out_of_order_cache_sz' not in settings: if 'out_of_order_cache_sz' not in settings:
settings['out_of_order_cache_sz'] = 100 * 1000 * 1000 settings['out_of_order_cache_sz'] = 100 * 1000 * 1000
if 'debug_output' not in settings: if 'debug_output' not in settings:
settings['debug_output'] = 'false' settings['debug_output'] = 'false'
settings['max_out_sz'] = int(settings['max_out_sz']) settings['max_out_sz'] = int(settings['max_out_sz'])
settings['split_timestamp'] = int(settings['split_timestamp']) settings['split_timestamp'] = int(settings['split_timestamp'])
settings['file_timestamp'] = int(settings['file_timestamp']) settings['file_timestamp'] = int(settings['file_timestamp'])
settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8')) settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8'))
settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz']) settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz'])
settings['debug_output'] = settings['debug_output'].lower() settings['debug_output'] = settings['debug_output'].lower()
if 'output_file' not in settings and 'output' not in settings: if 'output_file' not in settings and 'output' not in settings:
print("Missing output file / directory") print("Missing output file / directory")
sys.exit(1) sys.exit(1)
blkindex = get_block_hashes(settings) blkindex = get_block_hashes(settings)
blkmap = mkblockmap(blkindex) blkmap = mkblockmap(blkindex)
# Block hash map won't be byte-reversed. Neither should the genesis hash. # Block hash map won't be byte-reversed. Neither should the genesis hash.
if not settings['genesis'] in blkmap: if not settings['genesis'] in blkmap:
print("Genesis block not found in hashlist") print("Genesis block not found in hashlist")
else: else:
BlockDataCopier(settings, blkindex, blkmap).run() BlockDataCopier(settings, blkindex, blkmap).run()

214
contrib/linearize/linearize-hashes.py
View file

@ -22,135 +22,135 @@ import os.path
settings = {} settings = {}
def hex_switchEndian(s): def hex_switchEndian(s):
""" Switches the endianness of a hex string (in pairs of hex chars) """ """ Switches the endianness of a hex string (in pairs of hex chars) """
pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
return b''.join(pairList[::-1]).decode() return b''.join(pairList[::-1]).decode()
class BitcoinRPC: class BitcoinRPC:
def __init__(self, host, port, username, password): def __init__(self, host, port, username, password):
authpair = "%s:%s" % (username, password) authpair = "%s:%s" % (username, password)
authpair = authpair.encode('utf-8') authpair = authpair.encode('utf-8')
self.authhdr = b"Basic " + base64.b64encode(authpair) self.authhdr = b"Basic " + base64.b64encode(authpair)
self.conn = httplib.HTTPConnection(host, port=port, timeout=30) self.conn = httplib.HTTPConnection(host, port=port, timeout=30)
def execute(self, obj): def execute(self, obj):
try: try:
self.conn.request('POST', '/', json.dumps(obj), self.conn.request('POST', '/', json.dumps(obj),
{ 'Authorization' : self.authhdr, { 'Authorization' : self.authhdr,
'Content-type' : 'application/json' }) 'Content-type' : 'application/json' })
except ConnectionRefusedError: except ConnectionRefusedError:
print('RPC connection refused. Check RPC settings and the server status.', print('RPC connection refused. Check RPC settings and the server status.',
file=sys.stderr) file=sys.stderr)
return None return None
resp = self.conn.getresponse() resp = self.conn.getresponse()
if resp is None: if resp is None:
print("JSON-RPC: no response", file=sys.stderr) print("JSON-RPC: no response", file=sys.stderr)
return None return None
body = resp.read().decode('utf-8') body = resp.read().decode('utf-8')
resp_obj = json.loads(body) resp_obj = json.loads(body)
return resp_obj return resp_obj
@staticmethod @staticmethod
def build_request(idx, method, params): def build_request(idx, method, params):
obj = { 'version' : '1.1', obj = { 'version' : '1.1',
'method' : method, 'method' : method,
'id' : idx } 'id' : idx }
if params is None: if params is None:
obj['params'] = [] obj['params'] = []
else: else:
obj['params'] = params obj['params'] = params
return obj return obj
@staticmethod @staticmethod
def response_is_error(resp_obj): def response_is_error(resp_obj):
return 'error' in resp_obj and resp_obj['error'] is not None return 'error' in resp_obj and resp_obj['error'] is not None
def get_block_hashes(settings, max_blocks_per_call=10000): def get_block_hashes(settings, max_blocks_per_call=10000):
rpc = BitcoinRPC(settings['host'], settings['port'], rpc = BitcoinRPC(settings['host'], settings['port'],
settings['rpcuser'], settings['rpcpassword']) settings['rpcuser'], settings['rpcpassword'])
height = settings['min_height'] height = settings['min_height']
while height < settings['max_height']+1: while height < settings['max_height']+1:
num_blocks = min(settings['max_height']+1-height, max_blocks_per_call) num_blocks = min(settings['max_height']+1-height, max_blocks_per_call)
batch = [] batch = []
for x in range(num_blocks): for x in range(num_blocks):
batch.append(rpc.build_request(x, 'getblockhash', [height + x])) batch.append(rpc.build_request(x, 'getblockhash', [height + x]))
reply = rpc.execute(batch) reply = rpc.execute(batch)
if reply is None: if reply is None:
print('Cannot continue. Program will halt.') print('Cannot continue. Program will halt.')
return None return None
for x,resp_obj in enumerate(reply): for x,resp_obj in enumerate(reply):
if rpc.response_is_error(resp_obj): if rpc.response_is_error(resp_obj):
print('JSON-RPC: error at height', height+x, ': ', resp_obj['error'], file=sys.stderr) print('JSON-RPC: error at height', height+x, ': ', resp_obj['error'], file=sys.stderr)
sys.exit(1) sys.exit(1)
assert(resp_obj['id'] == x) # assume replies are in-sequence assert(resp_obj['id'] == x) # assume replies are in-sequence
if settings['rev_hash_bytes'] == 'true': if settings['rev_hash_bytes'] == 'true':
resp_obj['result'] = hex_switchEndian(resp_obj['result']) resp_obj['result'] = hex_switchEndian(resp_obj['result'])
print(resp_obj['result']) print(resp_obj['result'])
height += num_blocks height += num_blocks
def get_rpc_cookie(): def get_rpc_cookie():
# Open the cookie file # Open the cookie file
with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f: with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f:
combined = f.readline() combined = f.readline()
combined_split = combined.split(":") combined_split = combined.split(":")
settings['rpcuser'] = combined_split[0] settings['rpcuser'] = combined_split[0]
settings['rpcpassword'] = combined_split[1] settings['rpcpassword'] = combined_split[1]
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) != 2: if len(sys.argv) != 2:
print("Usage: linearize-hashes.py CONFIG-FILE") print("Usage: linearize-hashes.py CONFIG-FILE")
sys.exit(1) sys.exit(1)
f = open(sys.argv[1]) f = open(sys.argv[1])
for line in f: for line in f:
# skip comment lines # skip comment lines
m = re.search('^\s*#', line) m = re.search('^\s*#', line)
if m: if m:
continue continue
# parse key=value lines # parse key=value lines
m = re.search('^(\w+)\s*=\s*(\S.*)$', line) m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
if m is None: if m is None:
continue continue
settings[m.group(1)] = m.group(2) settings[m.group(1)] = m.group(2)
f.close() f.close()
if 'host' not in settings: if 'host' not in settings:
settings['host'] = '127.0.0.1' settings['host'] = '127.0.0.1'
if 'port' not in settings: if 'port' not in settings:
settings['port'] = 8332 settings['port'] = 8332
if 'min_height' not in settings: if 'min_height' not in settings:
settings['min_height'] = 0 settings['min_height'] = 0
if 'max_height' not in settings: if 'max_height' not in settings:
settings['max_height'] = 313000 settings['max_height'] = 313000
if 'rev_hash_bytes' not in settings: if 'rev_hash_bytes' not in settings:
settings['rev_hash_bytes'] = 'false' settings['rev_hash_bytes'] = 'false'
use_userpass = True use_userpass = True
use_datadir = False use_datadir = False
if 'rpcuser' not in settings or 'rpcpassword' not in settings: if 'rpcuser' not in settings or 'rpcpassword' not in settings:
use_userpass = False use_userpass = False
if 'datadir' in settings and not use_userpass: if 'datadir' in settings and not use_userpass:
use_datadir = True use_datadir = True
if not use_userpass and not use_datadir: if not use_userpass and not use_datadir:
print("Missing datadir or username and/or password in cfg file", file=sys.stderr) print("Missing datadir or username and/or password in cfg file", file=sys.stderr)
sys.exit(1) sys.exit(1)
settings['port'] = int(settings['port']) settings['port'] = int(settings['port'])
settings['min_height'] = int(settings['min_height']) settings['min_height'] = int(settings['min_height'])
settings['max_height'] = int(settings['max_height']) settings['max_height'] = int(settings['max_height'])
# Force hash byte format setting to be lowercase to make comparisons easier. # Force hash byte format setting to be lowercase to make comparisons easier.
settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower()
# Get the rpc user and pass from the cookie if the datadir is set # Get the rpc user and pass from the cookie if the datadir is set
if use_datadir: if use_datadir:
get_rpc_cookie() get_rpc_cookie()
get_block_hashes(settings) get_block_hashes(settings)

2
contrib/seeds/generate-seeds.py
View file

@ -11,7 +11,7 @@ argument:
nodes_main.txt nodes_main.txt
nodes_test.txt nodes_test.txt
These files must consist of lines in the format These files must consist of lines in the format
<ip> <ip>
<ip>:<port> <ip>:<port>

2
test/functional/feature_bip68_sequence.py
View file

@ -67,7 +67,7 @@ class BIP68Test(BitcoinTestFramework):
# If sequence locks were used, this would require 1 block for the # If sequence locks were used, this would require 1 block for the
# input to mature. # input to mature.
sequence_value = SEQUENCE_LOCKTIME_DISABLE_FLAG | 1 sequence_value = SEQUENCE_LOCKTIME_DISABLE_FLAG | 1
tx1.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=sequence_value)] tx1.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=sequence_value)]
tx1.vout = [CTxOut(value, CScript([b'a']))] tx1.vout = [CTxOut(value, CScript([b'a']))]
tx1_signed = self.nodes[0].signrawtransactionwithwallet(ToHex(tx1))["hex"] tx1_signed = self.nodes[0].signrawtransactionwithwallet(ToHex(tx1))["hex"]

2
test/functional/feature_maxuploadtarget.py
View file

@ -100,7 +100,7 @@ class MaxUploadTest(BitcoinTestFramework):
assert_equal(p2p_conns[0].block_receive_map[big_old_block], i+1) assert_equal(p2p_conns[0].block_receive_map[big_old_block], i+1)
assert_equal(len(self.nodes[0].getpeerinfo()), 3) assert_equal(len(self.nodes[0].getpeerinfo()), 3)
# At most a couple more tries should succeed (depending on how long # At most a couple more tries should succeed (depending on how long
# the test has been running so far). # the test has been running so far).
for i in range(3): for i in range(3):
p2p_conns[0].send_message(getdata_request) p2p_conns[0].send_message(getdata_request)

6
test/functional/feature_proxy.py
View file

@ -79,9 +79,9 @@ class ProxyTest(BitcoinTestFramework):
# Note: proxies are not used to connect to local nodes # Note: proxies are not used to connect to local nodes
# this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost
args = [ args = [
['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'],
['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'],
['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], ['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'],
[] []
] ]
if self.have_ipv6: if self.have_ipv6:

2
test/functional/p2p_feefilter.py
View file

@ -69,7 +69,7 @@ class FeeFilterTest(BitcoinTestFramework):
# Change tx fee rate to 10 sat/byte and test they are no longer received # Change tx fee rate to 10 sat/byte and test they are no longer received
node1.settxfee(Decimal("0.00010000")) node1.settxfee(Decimal("0.00010000"))
[node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)]
sync_mempools(self.nodes) # must be sure node 0 has received all txs sync_mempools(self.nodes) # must be sure node 0 has received all txs
# Send one transaction from node0 that should be received, so that we # Send one transaction from node0 that should be received, so that we
# we can sync the test on receipt (if node1's txs were relayed, they'd # we can sync the test on receipt (if node1's txs were relayed, they'd

2
test/functional/p2p_segwit.py
View file

@ -1659,7 +1659,7 @@ class SegWitTest(BitcoinTestFramework):
tx2.wit.vtxinwit.append(CTxInWitness()) tx2.wit.vtxinwit.append(CTxInWitness())
tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program ] tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program ]
total_value += tx.vout[i].nValue total_value += tx.vout[i].nValue
tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_toomany ] tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_toomany ]
tx2.vout.append(CTxOut(total_value, CScript([OP_TRUE]))) tx2.vout.append(CTxOut(total_value, CScript([OP_TRUE])))
tx2.rehash() tx2.rehash()

2
test/functional/wallet_dump.py
View file

@ -136,7 +136,7 @@ class WalletDumpTest(BitcoinTestFramework):
assert_equal(found_addr, test_addr_count) assert_equal(found_addr, test_addr_count)
assert_equal(found_script_addr, 2) assert_equal(found_script_addr, 2)
assert_equal(found_addr_chg, 90*2 + 50) # old reserve keys are marked as change now assert_equal(found_addr_chg, 90*2 + 50) # old reserve keys are marked as change now
assert_equal(found_addr_rsv, 90*2) assert_equal(found_addr_rsv, 90*2)
assert_equal(witness_addr_ret, witness_addr) assert_equal(witness_addr_ret, witness_addr)
# Overwriting should fail # Overwriting should fail