Make linearize scripts Python 3-compatible.
This commit is contained in:
Doug
Douglas Roark
parent
d5aa19813c
commit
3c8f63ba7c
3 changed files with 47 additions and 26 deletions
|
|
@ -1,5 +1,6 @@
|
|||
# Linearize
|
||||
Construct a linear, no-fork, best version of the Bitcoin blockchain.
|
||||
Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts
|
||||
run using Python 3 but are compatible with Python 2.
|
||||
|
||||
## Step 1: Download hash list
|
||||
|
||||
|
|
@ -9,7 +10,8 @@ Required configuration file settings for linearize-hashes:
|
|||
* RPC: `rpcuser`, `rpcpassword`
|
||||
|
||||
Optional config file setting for linearize-hashes:
|
||||
* RPC: `host`, `port` (Default: `127.0.0.1:8332`)
|
||||
* RPC: `host` (Default: `127.0.0.1`)
|
||||
* RPC: `port` (Default: `8332`)
|
||||
* Blockchain: `min_height`, `max_height`
|
||||
* `rev_hash_bytes`: If true, the written block hash list will be
|
||||
byte-reversed. (In other words, the hash returned by getblockhash will have its
|
||||
|
|
@ -17,6 +19,9 @@ bytes reversed.) False by default. Intended for generation of
|
|||
standalone hash lists but safe to use with linearize-data.py, which will output
|
||||
the same data no matter which byte format is chosen.
|
||||
|
||||
The `linearize-hashes` script requires a connection, local or remote, to a
|
||||
JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient.
|
||||
|
||||
## Step 2: Copy local block data
|
||||
|
||||
$ ./linearize-data.py linearize.cfg
|
||||
|
|
@ -24,13 +29,13 @@ the same data no matter which byte format is chosen.
|
|||
Required configuration file settings:
|
||||
* `output_file`: The file that will contain the final blockchain.
|
||||
or
|
||||
* `output`: Output directory for linearized blocks/blkNNNNN.dat output.
|
||||
* `output`: Output directory for linearized `blocks/blkNNNNN.dat` output.
|
||||
|
||||
Optional config file setting for linearize-data:
|
||||
* `file_timestamp`: Set each file's last-modified time to that of the most
|
||||
recent block in that file.
|
||||
* `genesis`: The hash of the genesis block in the blockchain.
|
||||
* `input: bitcoind blocks/ directory containing blkNNNNN.dat
|
||||
* `input`: bitcoind blocks/ directory containing blkNNNNN.dat
|
||||
* `hashlist`: text file containing list of block hashes created by
|
||||
linearize-hashes.py.
|
||||
* `max_out_sz`: Maximum size for files created by the `output_file` option.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# linearize-data.py: Construct a linear, no-fork version of the chain.
|
||||
#
|
||||
|
|
@ -8,29 +8,33 @@
|
|||
#
|
||||
|
||||
from __future__ import print_function, division
|
||||
try: # Python 3
|
||||
import http.client as httplib
|
||||
except ImportError: # Python 2
|
||||
import httplib
|
||||
import json
|
||||
import struct
|
||||
import re
|
||||
import os
|
||||
import os.path
|
||||
import base64
|
||||
import httplib
|
||||
import sys
|
||||
import hashlib
|
||||
import datetime
|
||||
import time
|
||||
from collections import namedtuple
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
settings = {}
|
||||
|
||||
##### Switch endian-ness #####
|
||||
def hex_switchEndian(s):
|
||||
""" Switches the endianness of a hex string (in pairs of hex chars) """
|
||||
pairList = [s[i]+s[i+1] for i in range(0,len(s),2)]
|
||||
return ''.join(pairList[::-1])
|
||||
pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
|
||||
return b''.join(pairList[::-1]).decode()
|
||||
|
||||
def uint32(x):
|
||||
return x & 0xffffffffL
|
||||
return x & 0xffffffff
|
||||
|
||||
def bytereverse(x):
|
||||
return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
|
||||
|
|
@ -41,14 +45,14 @@ def bufreverse(in_buf):
|
|||
for i in range(0, len(in_buf), 4):
|
||||
word = struct.unpack('@I', in_buf[i:i+4])[0]
|
||||
out_words.append(struct.pack('@I', bytereverse(word)))
|
||||
return ''.join(out_words)
|
||||
return b''.join(out_words)
|
||||
|
||||
def wordreverse(in_buf):
|
||||
out_words = []
|
||||
for i in range(0, len(in_buf), 4):
|
||||
out_words.append(in_buf[i:i+4])
|
||||
out_words.reverse()
|
||||
return ''.join(out_words)
|
||||
return b''.join(out_words)
|
||||
|
||||
def calc_hdr_hash(blk_hdr):
|
||||
hash1 = hashlib.sha256()
|
||||
|
|
@ -65,7 +69,7 @@ def calc_hash_str(blk_hdr):
|
|||
hash = calc_hdr_hash(blk_hdr)
|
||||
hash = bufreverse(hash)
|
||||
hash = wordreverse(hash)
|
||||
hash_str = hash.encode('hex')
|
||||
hash_str = hexlify(hash).decode('utf-8')
|
||||
return hash_str
|
||||
|
||||
def get_blk_dt(blk_hdr):
|
||||
|
|
@ -217,7 +221,7 @@ class BlockDataCopier:
|
|||
|
||||
inMagic = inhdr[:4]
|
||||
if (inMagic != self.settings['netmagic']):
|
||||
print("Invalid magic: " + inMagic.encode('hex'))
|
||||
print("Invalid magic: " + hexlify(inMagic).decode('utf-8'))
|
||||
return
|
||||
inLenLE = inhdr[4:]
|
||||
su = struct.unpack("<I", inLenLE)
|
||||
|
|
@ -294,14 +298,14 @@ if __name__ == '__main__':
|
|||
if 'split_timestamp' not in settings:
|
||||
settings['split_timestamp'] = 0
|
||||
if 'max_out_sz' not in settings:
|
||||
settings['max_out_sz'] = 1000L * 1000 * 1000
|
||||
settings['max_out_sz'] = 1000 * 1000 * 1000
|
||||
if 'out_of_order_cache_sz' not in settings:
|
||||
settings['out_of_order_cache_sz'] = 100 * 1000 * 1000
|
||||
|
||||
settings['max_out_sz'] = long(settings['max_out_sz'])
|
||||
settings['max_out_sz'] = int(settings['max_out_sz'])
|
||||
settings['split_timestamp'] = int(settings['split_timestamp'])
|
||||
settings['file_timestamp'] = int(settings['file_timestamp'])
|
||||
settings['netmagic'] = settings['netmagic'].decode('hex')
|
||||
settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8'))
|
||||
settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz'])
|
||||
|
||||
if 'output_file' not in settings and 'output' not in settings:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# linearize-hashes.py: List blocks in a linear, no-fork version of the chain.
|
||||
#
|
||||
|
|
@ -8,11 +8,14 @@
|
|||
#
|
||||
|
||||
from __future__ import print_function
|
||||
try: # Python 3
|
||||
import http.client as httplib
|
||||
except ImportError: # Python 2
|
||||
import httplib
|
||||
import json
|
||||
import struct
|
||||
import re
|
||||
import base64
|
||||
import httplib
|
||||
import sys
|
||||
|
||||
settings = {}
|
||||
|
|
@ -20,26 +23,32 @@ settings = {}
|
|||
##### Switch endian-ness #####
|
||||
def hex_switchEndian(s):
|
||||
""" Switches the endianness of a hex string (in pairs of hex chars) """
|
||||
pairList = [s[i]+s[i+1] for i in range(0,len(s),2)]
|
||||
return ''.join(pairList[::-1])
|
||||
pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
|
||||
return b''.join(pairList[::-1]).decode()
|
||||
|
||||
class BitcoinRPC:
|
||||
def __init__(self, host, port, username, password):
|
||||
authpair = "%s:%s" % (username, password)
|
||||
self.authhdr = "Basic %s" % (base64.b64encode(authpair))
|
||||
self.conn = httplib.HTTPConnection(host, port, False, 30)
|
||||
authpair = authpair.encode('utf-8')
|
||||
self.authhdr = b"Basic " + base64.b64encode(authpair)
|
||||
self.conn = httplib.HTTPConnection(host, port=port, timeout=30)
|
||||
|
||||
def execute(self, obj):
|
||||
self.conn.request('POST', '/', json.dumps(obj),
|
||||
{ 'Authorization' : self.authhdr,
|
||||
'Content-type' : 'application/json' })
|
||||
try:
|
||||
self.conn.request('POST', '/', json.dumps(obj),
|
||||
{ 'Authorization' : self.authhdr,
|
||||
'Content-type' : 'application/json' })
|
||||
except ConnectionRefusedError:
|
||||
print('RPC connection refused. Check RPC settings and the server status.',
|
||||
file=sys.stderr)
|
||||
return None
|
||||
|
||||
resp = self.conn.getresponse()
|
||||
if resp is None:
|
||||
print("JSON-RPC: no response", file=sys.stderr)
|
||||
return None
|
||||
|
||||
body = resp.read()
|
||||
body = resp.read().decode('utf-8')
|
||||
resp_obj = json.loads(body)
|
||||
return resp_obj
|
||||
|
||||
|
|
@ -70,6 +79,9 @@ def get_block_hashes(settings, max_blocks_per_call=10000):
|
|||
batch.append(rpc.build_request(x, 'getblockhash', [height + x]))
|
||||
|
||||
reply = rpc.execute(batch)
|
||||
if reply is None:
|
||||
print('Cannot continue. Program will halt.')
|
||||
return None
|
||||
|
||||
for x,resp_obj in enumerate(reply):
|
||||
if rpc.response_is_error(resp_obj):
|
||||
|
|
|
|||
Loading…
Reference in a new issue