Add p2p-fullblocktest.py

qa/pull-tester/rpc-tests.sh

@@ -36,6 +36,7 @@ testScripts=(
     'nodehandling.py'
     'reindex.py'
     'decodescript.py'
+    'p2p-fullblocktest.py'
 );
 testScriptsExt=(
     'bipdersig-p2p.py'

qa/rpc-tests/README.md

@@ -1,5 +1,5 @@
-Regression tests of RPC interface
+Regression tests
-=================================
+================
 
 ### [python-bitcoinrpc](https://github.com/jgarzik/python-bitcoinrpc)
 Git subtree of [https://github.com/jgarzik/python-bitcoinrpc](https://github.com/jgarzik/python-bitcoinrpc).
@@ -12,6 +12,28 @@ Base class for new regression tests.
 ### [test_framework/util.py](test_framework/util.py)
 Generally useful functions.
 
+### [test_framework/mininode.py](test_framework/mininode.py)
+Basic code to support p2p connectivity to a bitcoind.
+
+### [test_framework/comptool.py](test_framework/comptool.py)
+Framework for comparison-tool style, p2p tests.
+
+### [test_framework/script.py](test_framework/script.py)
+Utilities for manipulating transaction scripts (originally from python-bitcoinlib)
+
+### [test_framework/blockstore.py](test_framework/blockstore.py)
+Implements disk-backed block and tx storage.
+
+### [test_framework/key.py](test_framework/key.py)
+Wrapper around OpenSSL EC_Key (originally from python-bitcoinlib)
+
+### [test_framework/bignum.py](test_framework/bignum.py)
+Helpers for script.py
+
+### [test_framework/blocktools.py](test_framework/blocktools.py)
+Helper functions for creating blocks and transactions.
+
+
 Notes
 =====
 
@@ -49,3 +71,82 @@ to recover with:
 rm -rf cache
 killall bitcoind
 ```
+
+P2P test design notes
+---------------------
+
+## Mininode
+
+* ```mininode.py``` contains all the definitions for objects that pass
+over the network (```CBlock```, ```CTransaction```, etc, along with the network-level
+wrappers for them, ```msg_block```, ```msg_tx```, etc).
+
+* P2P tests have two threads.  One thread handles all network communication
+with the bitcoind(s) being tested (using python's asyncore package); the other
+implements the test logic.
+
+* ```NodeConn``` is the class used to connect to a bitcoind.  If you implement
+a callback class that derives from ```NodeConnCB``` and pass that to the
+```NodeConn``` object, your code will receive the appropriate callbacks when
+events of interest arrive.  NOTE: be sure to call
+```self.create_callback_map()``` in your derived classes' ```__init__```
+function, so that the correct mappings are set up between p2p messages and your
+callback functions.
+
+* You can pass the same handler to multiple ```NodeConn```'s if you like, or pass
+different ones to each -- whatever makes the most sense for your test.
+
+* Call ```NetworkThread.start()``` after all ```NodeConn``` objects are created to
+start the networking thread.  (Continue with the test logic in your existing
+thread.)
+
+* RPC calls are available in p2p tests.
+
+* Can be used to write free-form tests, where specific p2p-protocol behavior
+is tested.  Examples: ```p2p-accept-block.py```, ```maxblocksinflight.py```.
+
+## Comptool
+
+* Testing framework for writing tests that compare the block/tx acceptance
+behavior of a bitcoind against 1 or more other bitcoind instances, or against
+known outcomes, or both.
+
+* Set the ```num_nodes``` variable (defined in ```ComparisonTestFramework```) to start up
+1 or more nodes.  If using 1 node, then ```--testbinary``` can be used as a command line
+option to change the bitcoind binary used by the test.  If using 2 or more nodes,
+then ```--refbinary``` can be optionally used to change the bitcoind that will be used
+on nodes 2 and up.
+
+* Implement a (generator) function called ```get_tests()``` which yields ```TestInstance```s.
+Each ```TestInstance``` consists of:
+  - a list of ```[object, outcome, hash]``` entries
+    * ```object``` is a ```CBlock```, ```CTransaction```, or
+    ```CBlockHeader```.  ```CBlock```'s and ```CTransaction```'s are tested for
+    acceptance.  ```CBlockHeader```s can be used so that the test runner can deliver
+    complete headers-chains when requested from the bitcoind, to allow writing
+    tests where blocks can be delivered out of order but still processed by
+    headers-first bitcoind's.
+    * ```outcome``` is ```True```, ```False```, or ```None```.  If ```True```
+    or ```False```, the tip is compared with the expected tip -- either the
+    block passed in, or the hash specified as the optional 3rd entry.  If
+    ```None``` is specified, then the test will compare all the bitcoind's
+    being tested to see if they all agree on what the best tip is.
+    * ```hash``` is the block hash of the tip to compare against. Optional to
+    specify; if left out then the hash of the block passed in will be used as
+    the expected tip.  This allows for specifying an expected tip while testing
+    the handling of either invalid blocks or blocks delivered out of order,
+    which complete a longer chain.
+  - ```sync_every_block```: ```True/False```.  If ```False```, then all blocks
+    are inv'ed together, and the test runner waits until the node receives the
+    last one, and tests only the last block for tip acceptance using the
+    outcome and specified tip.  If ```True```, then each block is tested in
+    sequence and synced (this is slower when processing many blocks).
+  - ```sync_every_transaction```: ```True/False```.  Analogous to
+    ```sync_every_block```, except if the outcome on the last tx is "None",
+    then the contents of the entire mempool are compared across all bitcoind
+    connections.  If ```True``` or ```False```, then only the last tx's
+    acceptance is tested against the given outcome.
+
+* For examples of tests written in this framework, see
+  ```invalidblockrequest.py``` and ```p2p-fullblocktest.py```.
+

qa/rpc-tests/bipdersig-p2p.py

@@ -75,6 +75,7 @@ class BIP66Test(ComparisonTestFramework):
     def get_tests(self):
 
         self.coinbase_blocks = self.nodes[0].generate(2)
+        height = 3  # height of the next block to build
         self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0)
         self.nodeaddress = self.nodes[0].getnewaddress()
         self.last_block_time = time.time()
@@ -82,25 +83,27 @@ class BIP66Test(ComparisonTestFramework):
         ''' 98 more version 2 blocks '''
         test_blocks = []
         for i in xrange(98):
-            block = create_block(self.tip, create_coinbase(2), self.last_block_time + 1)
+            block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1)
             block.nVersion = 2
             block.rehash()
             block.solve()
             test_blocks.append([block, True])
             self.last_block_time += 1
             self.tip = block.sha256
+            height += 1
         yield TestInstance(test_blocks, sync_every_block=False)
 
         ''' Mine 749 version 3 blocks '''
         test_blocks = []
         for i in xrange(749):
-            block = create_block(self.tip, create_coinbase(2), self.last_block_time + 1)
+            block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1)
             block.nVersion = 3
             block.rehash()
             block.solve()
             test_blocks.append([block, True])
             self.last_block_time += 1
             self.tip = block.sha256
+            height += 1
         yield TestInstance(test_blocks, sync_every_block=False)
 
         ''' 
@@ -112,7 +115,7 @@ class BIP66Test(ComparisonTestFramework):
         unDERify(spendtx)
         spendtx.rehash()
 
-        block = create_block(self.tip, create_coinbase(2), self.last_block_time + 1)
+        block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1)
         block.nVersion = 3
         block.vtx.append(spendtx)
         block.hashMerkleRoot = block.calc_merkle_root()
@@ -121,6 +124,7 @@ class BIP66Test(ComparisonTestFramework):
 
         self.last_block_time += 1
         self.tip = block.sha256
+        height += 1
         yield TestInstance([[block, True]])
 
         ''' 
@@ -132,7 +136,7 @@ class BIP66Test(ComparisonTestFramework):
         unDERify(spendtx)
         spendtx.rehash()
 
-        block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1)
+        block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1)
         block.nVersion = 3
         block.vtx.append(spendtx)
         block.hashMerkleRoot = block.calc_merkle_root()
@@ -144,35 +148,38 @@ class BIP66Test(ComparisonTestFramework):
         ''' Mine 199 new version blocks on last valid tip '''
         test_blocks = []
         for i in xrange(199):
-            block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1)
+            block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1)
             block.nVersion = 3
             block.rehash()
             block.solve()
             test_blocks.append([block, True])
             self.last_block_time += 1
             self.tip = block.sha256
+            height += 1
         yield TestInstance(test_blocks, sync_every_block=False)
 
         ''' Mine 1 old version block '''
-        block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1)
+        block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1)
         block.nVersion = 2
         block.rehash()
         block.solve()
         self.last_block_time += 1
         self.tip = block.sha256
+        height += 1
         yield TestInstance([[block, True]])
 
         ''' Mine 1 new version block '''
-        block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1)
+        block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1)
         block.nVersion = 3
         block.rehash()
         block.solve()
         self.last_block_time += 1
         self.tip = block.sha256
+        height += 1
         yield TestInstance([[block, True]])
 
         ''' Mine 1 old version block, should be invalid '''
-        block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1)
+        block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1)
         block.nVersion = 2
         block.rehash()
         block.solve()

qa/rpc-tests/invalidblockrequest.py

@@ -46,12 +46,14 @@ class InvalidBlockRequestTest(ComparisonTestFramework):
         '''
         Create a new block with an anyone-can-spend coinbase
         '''
-        block = create_block(self.tip, create_coinbase(), self.block_time)
+        height = 1
+        block = create_block(self.tip, create_coinbase(height), self.block_time)
         self.block_time += 1
         block.solve()
         # Save the coinbase for later
         self.block1 = block
         self.tip = block.sha256
+        height += 1
         yield TestInstance([[block, True]])
 
         '''
@@ -59,11 +61,12 @@ class InvalidBlockRequestTest(ComparisonTestFramework):
         '''
         test = TestInstance(sync_every_block=False)
         for i in xrange(100):
-            block = create_block(self.tip, create_coinbase(), self.block_time)
+            block = create_block(self.tip, create_coinbase(height), self.block_time)
             block.solve()
             self.tip = block.sha256
             self.block_time += 1
             test.blocks_and_transactions.append([block, True])
+            height += 1
         yield test
 
         '''
@@ -73,7 +76,7 @@ class InvalidBlockRequestTest(ComparisonTestFramework):
         coinbase, spend of that spend).  Duplicate the 3rd transaction to 
         leave merkle root and blockheader unchanged but invalidate the block.
         '''
-        block2 = create_block(self.tip, create_coinbase(), self.block_time)
+        block2 = create_block(self.tip, create_coinbase(height), self.block_time)
         self.block_time += 1
 
         # chr(81) is OP_TRUE
@@ -95,11 +98,12 @@ class InvalidBlockRequestTest(ComparisonTestFramework):
 
         self.tip = block2.sha256
         yield TestInstance([[block2, False], [block2_orig, True]])
+        height += 1
 
         '''
         Make sure that a totally screwed up block is not valid.
         '''
-        block3 = create_block(self.tip, create_coinbase(), self.block_time)
+        block3 = create_block(self.tip, create_coinbase(height), self.block_time)
         self.block_time += 1
         block3.vtx[0].vout[0].nValue = 100*100000000 # Too high!
         block3.vtx[0].sha256=None

qa/rpc-tests/p2p-acceptblock.py

@@ -153,7 +153,7 @@ class AcceptBlockTest(BitcoinTestFramework):
         blocks_h2 = []  # the height 2 blocks on each node's chain
         block_time = time.time() + 1
         for i in xrange(2):
-            blocks_h2.append(create_block(tips[i], create_coinbase(), block_time))
+            blocks_h2.append(create_block(tips[i], create_coinbase(2), block_time))
             blocks_h2[i].solve()
             block_time += 1
         test_node.send_message(msg_block(blocks_h2[0]))
@@ -167,7 +167,7 @@ class AcceptBlockTest(BitcoinTestFramework):
         # 3. Send another block that builds on the original tip.
         blocks_h2f = []  # Blocks at height 2 that fork off the main chain
         for i in xrange(2):
-            blocks_h2f.append(create_block(tips[i], create_coinbase(), blocks_h2[i].nTime+1))
+            blocks_h2f.append(create_block(tips[i], create_coinbase(2), blocks_h2[i].nTime+1))
             blocks_h2f[i].solve()
         test_node.send_message(msg_block(blocks_h2f[0]))
         white_node.send_message(msg_block(blocks_h2f[1]))
@@ -186,7 +186,7 @@ class AcceptBlockTest(BitcoinTestFramework):
         # 4. Now send another block that builds on the forking chain.
         blocks_h3 = []
         for i in xrange(2):
-            blocks_h3.append(create_block(blocks_h2f[i].sha256, create_coinbase(), blocks_h2f[i].nTime+1))
+            blocks_h3.append(create_block(blocks_h2f[i].sha256, create_coinbase(3), blocks_h2f[i].nTime+1))
             blocks_h3[i].solve()
         test_node.send_message(msg_block(blocks_h3[0]))
         white_node.send_message(msg_block(blocks_h3[1]))
@@ -217,7 +217,7 @@ class AcceptBlockTest(BitcoinTestFramework):
         all_blocks = []   # node0's blocks
         for j in xrange(2):
             for i in xrange(288):
-                next_block = create_block(tips[j].sha256, create_coinbase(), tips[j].nTime+1)
+                next_block = create_block(tips[j].sha256, create_coinbase(i + 4), tips[j].nTime+1)
                 next_block.solve()
                 if j==0:
                     test_node.send_message(msg_block(next_block))

qa/rpc-tests/p2p-fullblocktest.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python2
+
+#
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+
+from test_framework.test_framework import ComparisonTestFramework
+from test_framework.util import *
+from test_framework.comptool import TestManager, TestInstance
+from test_framework.mininode import *
+from test_framework.blocktools import *
+import logging
+import copy
+import time
+import numbers
+from test_framework.key import CECKey
+from test_framework.script import CScript, CScriptOp, SignatureHash, SIGHASH_ALL, OP_TRUE
+
+class PreviousSpendableOutput(object):
+    def __init__(self, tx = CTransaction(), n = -1):
+        self.tx = tx
+        self.n = n  # the output we're spending
+
+'''
+This reimplements tests from the bitcoinj/FullBlockTestGenerator used
+by the pull-tester.
+
+We use the testing framework in which we expect a particular answer from
+each test.
+'''
+
+class FullBlockTest(ComparisonTestFramework):
+
+    ''' Can either run this test as 1 node with expected answers, or two and compare them. 
+        Change the "outcome" variable from each TestInstance object to only do the comparison. '''
+    def __init__(self):
+        self.num_nodes = 1
+        self.block_heights = {}
+        self.coinbase_key = CECKey()
+        self.coinbase_key.set_secretbytes(bytes("horsebattery"))
+        self.coinbase_pubkey = self.coinbase_key.get_pubkey()
+        self.block_time = int(time.time())+1
+        self.tip = None
+        self.blocks = {}
+
+    def run_test(self):
+        test = TestManager(self, self.options.tmpdir)
+        test.add_all_connections(self.nodes)
+        NetworkThread().start() # Start up network handling in another thread
+        test.run()
+
+    def add_transactions_to_block(self, block, tx_list):
+        [ tx.rehash() for tx in tx_list ]
+        block.vtx.extend(tx_list)
+        block.hashMerkleRoot = block.calc_merkle_root()
+        block.rehash()
+        return block
+    
+    # Create a block on top of self.tip, and advance self.tip to point to the new block
+    # if spend is specified, then 1 satoshi will be spent from that to an anyone-can-spend output,
+    # and rest will go to fees.
+    def next_block(self, number, spend=None, additional_coinbase_value=0, script=None):
+        if self.tip == None:
+            base_block_hash = self.genesis_hash
+        else:
+            base_block_hash = self.tip.sha256
+        # First create the coinbase
+        height = self.block_heights[base_block_hash] + 1
+        coinbase = create_coinbase(height, self.coinbase_pubkey)
+        coinbase.vout[0].nValue += additional_coinbase_value
+        if (spend != None):
+            coinbase.vout[0].nValue += spend.tx.vout[spend.n].nValue - 1 # all but one satoshi to fees
+        coinbase.rehash()
+        block = create_block(base_block_hash, coinbase, self.block_time)
+        if (spend != None):
+            tx = CTransaction()
+            tx.vin.append(CTxIn(COutPoint(spend.tx.sha256, spend.n), "", 0xffffffff))  # no signature yet
+            # This copies the java comparison tool testing behavior: the first
+            # txout has a garbage scriptPubKey, "to make sure we're not
+            # pre-verifying too much" (?)
+            tx.vout.append(CTxOut(0, CScript([random.randint(0,255), height & 255])))
+            if script == None:
+                tx.vout.append(CTxOut(1, CScript([OP_TRUE])))
+            else:
+                tx.vout.append(CTxOut(1, script))
+            # Now sign it if necessary
+            scriptSig = ""
+            scriptPubKey = bytearray(spend.tx.vout[spend.n].scriptPubKey)
+            if (scriptPubKey[0] == OP_TRUE):  # looks like an anyone-can-spend
+                scriptSig = CScript([OP_TRUE])
+            else:
+                # We have to actually sign it
+                (sighash, err) = SignatureHash(spend.tx.vout[spend.n].scriptPubKey, tx, 0, SIGHASH_ALL)
+                scriptSig = CScript([self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL]))])
+            tx.vin[0].scriptSig = scriptSig
+            # Now add the transaction to the block
+            block = self.add_transactions_to_block(block, [tx])
+        block.solve()
+        self.tip = block
+        self.block_heights[block.sha256] = height
+        self.block_time += 1
+        assert number not in self.blocks
+        self.blocks[number] = block
+        return block
+
+    def get_tests(self):
+        self.genesis_hash = int(self.nodes[0].getbestblockhash(), 16)
+        self.block_heights[self.genesis_hash] = 0
+        spendable_outputs = []
+
+        # save the current tip so it can be spent by a later block
+        def save_spendable_output():
+            spendable_outputs.append(self.tip)
+
+        # get an output that we previous marked as spendable
+        def get_spendable_output():
+            return PreviousSpendableOutput(spendable_outputs.pop(0).vtx[0], 0)
+
+        # returns a test case that asserts that the current tip was accepted
+        def accepted():
+            return TestInstance([[self.tip, True]])
+
+        # returns a test case that asserts that the current tip was rejected
+        def rejected():
+            return TestInstance([[self.tip, False]])
+       
+        # move the tip back to a previous block
+        def tip(number):
+            self.tip = self.blocks[number]
+
+        # creates a new block and advances the tip to that block
+        block = self.next_block
+
+
+        # Create a new block
+        block(0)
+        save_spendable_output()
+        yield accepted()
+
+
+        # Now we need that block to mature so we can spend the coinbase.
+        test = TestInstance(sync_every_block=False)
+        for i in range(100):
+            block(1000 + i)
+            test.blocks_and_transactions.append([self.tip, True])
+            save_spendable_output()
+        yield test
+
+
+        # Start by bulding a couple of blocks on top (which output is spent is in parentheses):
+        #     genesis -> b1 (0) -> b2 (1)
+        out0 = get_spendable_output()
+        block(1, spend=out0)
+        save_spendable_output()
+        yield accepted()
+
+        out1 = get_spendable_output()
+        block(2, spend=out1)
+        # Inv again, then deliver twice (shouldn't break anything).
+        yield accepted()
+
+
+        # so fork like this:
+        # 
+        #     genesis -> b1 (0) -> b2 (1)
+        #                      \-> b3 (1)
+        # 
+        # Nothing should happen at this point. We saw b2 first so it takes priority.
+        tip(1)
+        block(3, spend=out1)
+        # Deliver twice (should still not break anything)
+        yield rejected()
+
+
+        # Now we add another block to make the alternative chain longer.
+        # 
+        #     genesis -> b1 (0) -> b2 (1)
+        #                      \-> b3 (1) -> b4 (2)
+        out2 = get_spendable_output()
+        block(4, spend=out2)
+        yield accepted()
+
+
+        # ... and back to the first chain.
+        #     genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
+        #                      \-> b3 (1) -> b4 (2)
+        tip(2)
+        block(5, spend=out2)
+        save_spendable_output()
+        yield rejected()
+
+        out3 = get_spendable_output()
+        block(6, spend=out3)
+        yield accepted()
+
+
+        # Try to create a fork that double-spends
+        #     genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
+        #                                          \-> b7 (2) -> b8 (4)
+        #                      \-> b3 (1) -> b4 (2)
+        tip(5)
+        block(7, spend=out2)
+        yield rejected()
+
+        out4 = get_spendable_output()
+        block(8, spend=out4)
+        yield rejected()
+
+
+        # Try to create a block that has too much fee
+        #     genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
+        #                                                    \-> b9 (4)
+        #                      \-> b3 (1) -> b4 (2)
+        tip(6)
+        block(9, spend=out4, additional_coinbase_value=1)
+        yield rejected()
+
+        
+        # Create a fork that ends in a block with too much fee (the one that causes the reorg)
+        #     genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6  (3)
+        #                                          \-> b10 (3) -> b11 (4)
+        #                      \-> b3 (1) -> b4 (2)
+        tip(5)
+        block(10, spend=out3)
+        yield rejected()
+
+        block(11, spend=out4, additional_coinbase_value=1)
+        yield rejected()
+
+
+        # Try again, but with a valid fork first
+        #     genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6  (3)
+        #                                          \-> b12 (3) -> b13 (4) -> b14 (5)
+        #                                              (b12 added last)
+        #                      \-> b3 (1) -> b4 (2)
+        tip(5)
+        b12 = block(12, spend=out3)
+        save_spendable_output()
+        #yield TestInstance([[b12, False]])
+        b13 = block(13, spend=out4)
+        # Deliver the block header for b12, and the block b13.
+        # b13 should be accepted but the tip won't advance until b12 is delivered.
+        yield TestInstance([[CBlockHeader(b12), None], [b13, False]])
+
+        save_spendable_output()
+        out5 = get_spendable_output()
+        # b14 is invalid, but the node won't know that until it tries to connect
+        # Tip still can't advance because b12 is missing
+        block(14, spend=out5, additional_coinbase_value=1)
+        yield rejected()
+
+        yield TestInstance([[b12, True, b13.sha256]]) # New tip should be b13.
+
+        
+        # Test that a block with a lot of checksigs is okay
+        lots_of_checksigs = CScript([OP_CHECKSIG] * (1000000 / 50 - 1))
+        tip(13)
+        block(15, spend=out5, script=lots_of_checksigs)
+        yield accepted()
+
+
+        # Test that a block with too many checksigs is rejected
+        out6 = get_spendable_output()
+        too_many_checksigs = CScript([OP_CHECKSIG] * (1000000 / 50))
+        block(16, spend=out6, script=too_many_checksigs)
+        yield rejected()
+
+
+
+if __name__ == '__main__':
+    FullBlockTest().main()

qa/rpc-tests/script_test.py

@@ -124,10 +124,10 @@ def ParseScript(json_script):
     return parsed_script
             
 class TestBuilder(object):
-    def create_credit_tx(self, scriptPubKey):
+    def create_credit_tx(self, scriptPubKey, height):
         # self.tx1 is a coinbase transaction, modeled after the one created by script_tests.cpp
         # This allows us to reuse signatures created in the unit test framework.
-        self.tx1 = create_coinbase()                 # this has a bip34 scriptsig,
+        self.tx1 = create_coinbase(height)                 # this has a bip34 scriptsig,
         self.tx1.vin[0].scriptSig = CScript([0, 0])  # but this matches the unit tests
         self.tx1.vout[0].nValue = 0
         self.tx1.vout[0].scriptPubKey = scriptPubKey
@@ -168,7 +168,7 @@ class ScriptTest(ComparisonTestFramework):
 
         test = TestInstance(sync_every_block=False)
         test_build = TestBuilder()
-        test_build.create_credit_tx(scriptpubkey)
+        test_build.create_credit_tx(scriptpubkey, self.height)
         test_build.create_spend_tx(scriptsig)
         test_build.rehash()
 
@@ -176,16 +176,18 @@ class ScriptTest(ComparisonTestFramework):
         self.block_time += 1
         block.solve()
         self.tip = block.sha256
+        self.height += 1
         test.blocks_and_transactions = [[block, True]]
 
         for i in xrange(100):
-            block = create_block(self.tip, create_coinbase(), self.block_time)
+            block = create_block(self.tip, create_coinbase(self.height), self.block_time)
             self.block_time += 1
             block.solve()
             self.tip = block.sha256
+            self.height += 1
             test.blocks_and_transactions.append([block, True])
 
-        block = create_block(self.tip, create_coinbase(), self.block_time)
+        block = create_block(self.tip, create_coinbase(self.height), self.block_time)
         self.block_time += 1
         block.vtx.append(test_build.tx2)
         block.hashMerkleRoot = block.calc_merkle_root()
@@ -198,14 +200,16 @@ class ScriptTest(ComparisonTestFramework):
     def get_tests(self):
         self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0)
         self.block_time = 1333230000  # before the BIP16 switchover
+        self.height = 1
 
         '''
         Create a new block with an anyone-can-spend coinbase
         '''
-        block = create_block(self.tip, create_coinbase(), self.block_time)
+        block = create_block(self.tip, create_coinbase(self.height), self.block_time)
         self.block_time += 1
         block.solve()
         self.tip = block.sha256
+        self.height += 1
         yield TestInstance(objects=[[block, True]])
 
         '''
@@ -213,11 +217,12 @@ class ScriptTest(ComparisonTestFramework):
         '''
         test = TestInstance(objects=[], sync_every_block=False, sync_every_tx=False)
         for i in xrange(100):
-            b = create_block(self.tip, create_coinbase(), self.block_time)
+            b = create_block(self.tip, create_coinbase(self.height), self.block_time)
             b.solve()
             test.blocks_and_transactions.append([b, True])
             self.tip = b.sha256
             self.block_time += 1
+            self.height += 1
         yield test
  
         ''' Iterate through script tests. '''
@@ -229,6 +234,7 @@ class ScriptTest(ComparisonTestFramework):
                 self.nodes[1].invalidateblock(self.nodes[1].getblockhash(102))
 
             self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0)
+            self.height = 102
 
             [scriptsig, scriptpubkey, flags] = script_test[0:3]
             flags = ParseScriptFlags(flags)

qa/rpc-tests/test_framework/blockstore.py

@@ -10,6 +10,7 @@ class BlockStore(object):
     def __init__(self, datadir):
         self.blockDB = dbm.open(datadir + "/blocks", 'c')
         self.currentBlock = 0L
+        self.headers_map = dict()
     
     def close(self):
         self.blockDB.close()
@@ -26,24 +27,30 @@ class BlockStore(object):
         ret.calc_sha256()
         return ret
 
+    def get_header(self, blockhash):
+        try:
+            return self.headers_map[blockhash]
+        except KeyError:
+            return None
+
     # Note: this pulls full blocks out of the database just to retrieve
     # the headers -- perhaps we could keep a separate data structure
     # to avoid this overhead.
     def headers_for(self, locator, hash_stop, current_tip=None):
         if current_tip is None:
             current_tip = self.currentBlock
-        current_block = self.get(current_tip)
+        current_block_header = self.get_header(current_tip)
-        if current_block is None:
+        if current_block_header is None:
             return None
 
         response = msg_headers()
-        headersList = [ CBlockHeader(current_block) ]
+        headersList = [ current_block_header ]
         maxheaders = 2000
         while (headersList[0].sha256 not in locator.vHave):
             prevBlockHash = headersList[0].hashPrevBlock
-            prevBlock = self.get(prevBlockHash)
+            prevBlockHeader = self.get_header(prevBlockHash)
-            if prevBlock is not None:
+            if prevBlockHeader is not None:
-                headersList.insert(0, CBlockHeader(prevBlock))
+                headersList.insert(0, prevBlockHeader)
             else:
                 break
         headersList = headersList[:maxheaders] # truncate if we have too many
@@ -61,6 +68,10 @@ class BlockStore(object):
         except TypeError as e:
             print "Unexpected error: ", sys.exc_info()[0], e.args
         self.currentBlock = block.sha256
+        self.headers_map[block.sha256] = CBlockHeader(block)
+
+    def add_header(self, header):
+        self.headers_map[header.sha256] = header
 
     def get_blocks(self, inv):
         responses = []

qa/rpc-tests/test_framework/blocktools.py

@@ -5,7 +5,7 @@
 #
 
 from mininode import *
-from script import CScript, CScriptOp
+from script import CScript, CScriptOp, OP_TRUE, OP_CHECKSIG
 
 # Create a block (with regtest difficulty)
 def create_block(hashprev, coinbase, nTime=None):
@@ -37,19 +37,21 @@ def serialize_script_num(value):
         r[-1] |= 0x80
     return r
 
-counter=1
+# Create a coinbase transaction, assuming no miner fees.
-# Create an anyone-can-spend coinbase transaction, assuming no miner fees
+# If pubkey is passed in, the coinbase output will be a P2PK output;
-def create_coinbase(heightAdjust = 0):
+# otherwise an anyone-can-spend output.
-    global counter
+def create_coinbase(height, pubkey = None):
     coinbase = CTransaction()
     coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff), 
-                ser_string(serialize_script_num(counter+heightAdjust)), 0xffffffff))
+                ser_string(serialize_script_num(height)), 0xffffffff))
-    counter += 1
     coinbaseoutput = CTxOut()
     coinbaseoutput.nValue = 50*100000000
-    halvings = int((counter+heightAdjust)/150) # regtest
+    halvings = int(height/150) # regtest
     coinbaseoutput.nValue >>= halvings
-    coinbaseoutput.scriptPubKey = ""
+    if (pubkey != None):
+        coinbaseoutput.scriptPubKey = CScript([pubkey, OP_CHECKSIG])
+    else:
+        coinbaseoutput.scriptPubKey = CScript([OP_TRUE])
     coinbase.vout = [ coinbaseoutput ]
     coinbase.calc_sha256()
     return coinbase

qa/rpc-tests/test_framework/comptool.py

@@ -122,12 +122,19 @@ class TestNode(NodeConnCB):
 # Instances of these are generated by the test generator, and fed into the
 # comptool.
 #
-# "blocks_and_transactions" should be an array of [obj, True/False/None]:
+# "blocks_and_transactions" should be an array of
-#  - obj is either a CBlock or a CTransaction, and 
+#    [obj, True/False/None, hash/None]:
+#  - obj is either a CBlock, CBlockHeader, or a CTransaction, and
 #  - the second value indicates whether the object should be accepted
 #    into the blockchain or mempool (for tests where we expect a certain
 #    answer), or "None" if we don't expect a certain answer and are just
 #    comparing the behavior of the nodes being tested.
+#  - the third value is the hash to test the tip against (if None or omitted,
+#    use the hash of the block)
+#  - NOTE: if a block header, no test is performed; instead the header is
+#    just added to the block_store.  This is to facilitate block delivery
+#    when communicating with headers-first clients (when withholding an
+#    intermediate block).
 # sync_every_block: if True, then each block will be inv'ed, synced, and
 #    nodes will be tested based on the outcome for the block.  If False,
 #    then inv's accumulate until all blocks are processed (or max inv size
@@ -194,7 +201,6 @@ class TestManager(object):
         if not wait_until(blocks_requested, attempts=20*num_blocks):
             # print [ c.cb.block_request_map for c in self.connections ]
             raise AssertionError("Not all nodes requested block")
-        # --> Answer request (we did this inline!)
 
         # Send getheaders message
         [ c.cb.send_getheaders() for c in self.connections ]
@@ -217,7 +223,6 @@ class TestManager(object):
         if not wait_until(transaction_requested, attempts=20*num_events):
             # print [ c.cb.tx_request_map for c in self.connections ]
             raise AssertionError("Not all nodes requested transaction")
-        # --> Answer request (we did this inline!)
 
         # Get the mempool
         [ c.cb.send_mempool() for c in self.connections ]
@@ -272,29 +277,55 @@ class TestManager(object):
             # We use these variables to keep track of the last block
             # and last transaction in the tests, which are used
             # if we're not syncing on every block or every tx.
-            [ block, block_outcome ] = [ None, None ]
+            [ block, block_outcome, tip ] = [ None, None, None ]
             [ tx, tx_outcome ] = [ None, None ]
             invqueue = []
 
-            for b_or_t, outcome in test_instance.blocks_and_transactions:
+            for test_obj in test_instance.blocks_and_transactions:
+                b_or_t = test_obj[0]
+                outcome = test_obj[1]
                 # Determine if we're dealing with a block or tx
                 if isinstance(b_or_t, CBlock):  # Block test runner
                     block = b_or_t
                     block_outcome = outcome
+                    tip = block.sha256
+                    # each test_obj can have an optional third argument
+                    # to specify the tip we should compare with
+                    # (default is to use the block being tested)
+                    if len(test_obj) >= 3:
+                        tip = test_obj[2]
+
                     # Add to shared block_store, set as current block
+                    # If there was an open getdata request for the block
+                    # previously, and we didn't have an entry in the
+                    # block_store, then immediately deliver, because the
+                    # node wouldn't send another getdata request while
+                    # the earlier one is outstanding.
+                    first_block_with_hash = True
+                    if self.block_store.get(block.sha256) is not None:
+                        first_block_with_hash = False
                     with mininode_lock:
                         self.block_store.add_block(block)
                         for c in self.connections:
-                            c.cb.block_request_map[block.sha256] = False
+                            if first_block_with_hash and block.sha256 in c.cb.block_request_map and c.cb.block_request_map[block.sha256] == True:
+                                # There was a previous request for this block hash
+                                # Most likely, we delivered a header for this block
+                                # but never had the block to respond to the getdata
+                                c.send_message(msg_block(block))
+                            else:
+                                c.cb.block_request_map[block.sha256] = False
                     # Either send inv's to each node and sync, or add
                     # to invqueue for later inv'ing.
                     if (test_instance.sync_every_block):
                         [ c.cb.send_inv(block) for c in self.connections ]
                         self.sync_blocks(block.sha256, 1)
-                        if (not self.check_results(block.sha256, outcome)):
+                        if (not self.check_results(tip, outcome)):
                             raise AssertionError("Test failed at test %d" % test_number)
                     else:
                         invqueue.append(CInv(2, block.sha256))
+                elif isinstance(b_or_t, CBlockHeader):
+                    block_header = b_or_t
+                    self.block_store.add_header(block_header)
                 else:  # Tx test runner
                     assert(isinstance(b_or_t, CTransaction))
                     tx = b_or_t
@@ -322,9 +353,8 @@ class TestManager(object):
                 if len(invqueue) > 0:
                     [ c.send_message(msg_inv(invqueue)) for c in self.connections ]
                     invqueue = []
-                self.sync_blocks(block.sha256,
+                self.sync_blocks(block.sha256, len(test_instance.blocks_and_transactions))
-                                 len(test_instance.blocks_and_transactions))
+                if (not self.check_results(tip, block_outcome)):
-                if (not self.check_results(block.sha256, block_outcome)):
                     raise AssertionError("Block test failed at test %d" % test_number)
             if (not test_instance.sync_every_tx and tx is not None):
                 if len(invqueue) > 0:

qa/rpc-tests/test_framework/key.py

@@ -0,0 +1,215 @@
+# Copyright (c) 2011 Sam Rushing
+#
+# key.py - OpenSSL wrapper
+#
+# This file is modified from python-bitcoinlib.
+#
+
+"""ECC secp256k1 crypto routines
+
+WARNING: This module does not mlock() secrets; your private keys may end up on
+disk in swap! Use with caution!
+"""
+
+import ctypes
+import ctypes.util
+import hashlib
+import sys
+
+ssl = ctypes.cdll.LoadLibrary(ctypes.util.find_library ('ssl') or 'libeay32')
+
+ssl.BN_new.restype = ctypes.c_void_p
+ssl.BN_new.argtypes = []
+
+ssl.BN_bin2bn.restype = ctypes.c_void_p
+ssl.BN_bin2bn.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p]
+
+ssl.BN_CTX_free.restype = None
+ssl.BN_CTX_free.argtypes = [ctypes.c_void_p]
+
+ssl.BN_CTX_new.restype = ctypes.c_void_p
+ssl.BN_CTX_new.argtypes = []
+
+ssl.ECDH_compute_key.restype = ctypes.c_int
+ssl.ECDH_compute_key.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.ECDSA_sign.restype = ctypes.c_int
+ssl.ECDSA_sign.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.ECDSA_verify.restype = ctypes.c_int
+ssl.ECDSA_verify.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p]
+
+ssl.EC_KEY_free.restype = None
+ssl.EC_KEY_free.argtypes = [ctypes.c_void_p]
+
+ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p
+ssl.EC_KEY_new_by_curve_name.argtypes = [ctypes.c_int]
+
+ssl.EC_KEY_get0_group.restype = ctypes.c_void_p
+ssl.EC_KEY_get0_group.argtypes = [ctypes.c_void_p]
+
+ssl.EC_KEY_get0_public_key.restype = ctypes.c_void_p
+ssl.EC_KEY_get0_public_key.argtypes = [ctypes.c_void_p]
+
+ssl.EC_KEY_set_private_key.restype = ctypes.c_int
+ssl.EC_KEY_set_private_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.EC_KEY_set_conv_form.restype = None
+ssl.EC_KEY_set_conv_form.argtypes = [ctypes.c_void_p, ctypes.c_int]
+
+ssl.EC_KEY_set_public_key.restype = ctypes.c_int
+ssl.EC_KEY_set_public_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.i2o_ECPublicKey.restype = ctypes.c_void_p
+ssl.i2o_ECPublicKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.EC_POINT_new.restype = ctypes.c_void_p
+ssl.EC_POINT_new.argtypes = [ctypes.c_void_p]
+
+ssl.EC_POINT_free.restype = None
+ssl.EC_POINT_free.argtypes = [ctypes.c_void_p]
+
+ssl.EC_POINT_mul.restype = ctypes.c_int
+ssl.EC_POINT_mul.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+
+# this specifies the curve used with ECDSA.
+NID_secp256k1 = 714 # from openssl/obj_mac.h
+
+# Thx to Sam Devlin for the ctypes magic 64-bit fix.
+def _check_result(val, func, args):
+    if val == 0:
+        raise ValueError
+    else:
+        return ctypes.c_void_p (val)
+
+ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p
+ssl.EC_KEY_new_by_curve_name.errcheck = _check_result
+
+class CECKey(object):
+    """Wrapper around OpenSSL's EC_KEY"""
+
+    POINT_CONVERSION_COMPRESSED = 2
+    POINT_CONVERSION_UNCOMPRESSED = 4
+
+    def __init__(self):
+        self.k = ssl.EC_KEY_new_by_curve_name(NID_secp256k1)
+
+    def __del__(self):
+        if ssl:
+            ssl.EC_KEY_free(self.k)
+        self.k = None
+
+    def set_secretbytes(self, secret):
+        priv_key = ssl.BN_bin2bn(secret, 32, ssl.BN_new())
+        group = ssl.EC_KEY_get0_group(self.k)
+        pub_key = ssl.EC_POINT_new(group)
+        ctx = ssl.BN_CTX_new()
+        if not ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx):
+            raise ValueError("Could not derive public key from the supplied secret.")
+        ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx)
+        ssl.EC_KEY_set_private_key(self.k, priv_key)
+        ssl.EC_KEY_set_public_key(self.k, pub_key)
+        ssl.EC_POINT_free(pub_key)
+        ssl.BN_CTX_free(ctx)
+        return self.k
+
+    def set_privkey(self, key):
+        self.mb = ctypes.create_string_buffer(key)
+        return ssl.d2i_ECPrivateKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key))
+
+    def set_pubkey(self, key):
+        self.mb = ctypes.create_string_buffer(key)
+        return ssl.o2i_ECPublicKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key))
+
+    def get_privkey(self):
+        size = ssl.i2d_ECPrivateKey(self.k, 0)
+        mb_pri = ctypes.create_string_buffer(size)
+        ssl.i2d_ECPrivateKey(self.k, ctypes.byref(ctypes.pointer(mb_pri)))
+        return mb_pri.raw
+
+    def get_pubkey(self):
+        size = ssl.i2o_ECPublicKey(self.k, 0)
+        mb = ctypes.create_string_buffer(size)
+        ssl.i2o_ECPublicKey(self.k, ctypes.byref(ctypes.pointer(mb)))
+        return mb.raw
+
+    def get_raw_ecdh_key(self, other_pubkey):
+        ecdh_keybuffer = ctypes.create_string_buffer(32)
+        r = ssl.ECDH_compute_key(ctypes.pointer(ecdh_keybuffer), 32,
+                                 ssl.EC_KEY_get0_public_key(other_pubkey.k),
+                                 self.k, 0)
+        if r != 32:
+            raise Exception('CKey.get_ecdh_key(): ECDH_compute_key() failed')
+        return ecdh_keybuffer.raw
+
+    def get_ecdh_key(self, other_pubkey, kdf=lambda k: hashlib.sha256(k).digest()):
+        # FIXME: be warned it's not clear what the kdf should be as a default
+        r = self.get_raw_ecdh_key(other_pubkey)
+        return kdf(r)
+
+    def sign(self, hash):
+        # FIXME: need unit tests for below cases
+        if not isinstance(hash, bytes):
+            raise TypeError('Hash must be bytes instance; got %r' % hash.__class__)
+        if len(hash) != 32:
+            raise ValueError('Hash must be exactly 32 bytes long')
+
+        sig_size0 = ctypes.c_uint32()
+        sig_size0.value = ssl.ECDSA_size(self.k)
+        mb_sig = ctypes.create_string_buffer(sig_size0.value)
+        result = ssl.ECDSA_sign(0, hash, len(hash), mb_sig, ctypes.byref(sig_size0), self.k)
+        assert 1 == result
+        return mb_sig.raw[:sig_size0.value]
+
+    def verify(self, hash, sig):
+        """Verify a DER signature"""
+        return ssl.ECDSA_verify(0, hash, len(hash), sig, len(sig), self.k) == 1
+
+    def set_compressed(self, compressed):
+        if compressed:
+            form = self.POINT_CONVERSION_COMPRESSED
+        else:
+            form = self.POINT_CONVERSION_UNCOMPRESSED
+        ssl.EC_KEY_set_conv_form(self.k, form)
+
+
+class CPubKey(bytes):
+    """An encapsulated public key
+
+    Attributes:
+
+    is_valid      - Corresponds to CPubKey.IsValid()
+    is_fullyvalid - Corresponds to CPubKey.IsFullyValid()
+    is_compressed - Corresponds to CPubKey.IsCompressed()
+    """
+
+    def __new__(cls, buf, _cec_key=None):
+        self = super(CPubKey, cls).__new__(cls, buf)
+        if _cec_key is None:
+            _cec_key = CECKey()
+        self._cec_key = _cec_key
+        self.is_fullyvalid = _cec_key.set_pubkey(self) != 0
+        return self
+
+    @property
+    def is_valid(self):
+        return len(self) > 0
+
+    @property
+    def is_compressed(self):
+        return len(self) == 33
+
+    def verify(self, hash, sig):
+        return self._cec_key.verify(hash, sig)
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        # Always have represent as b'<secret>' so test cases don't have to
+        # change for py2/3
+        if sys.version > '3':
+            return '%s(%s)' % (self.__class__.__name__, super(CPubKey, self).__repr__())
+        else:
+            return '%s(b%s)' % (self.__class__.__name__, super(CPubKey, self).__repr__())
+

qa/rpc-tests/test_framework/script.py

@@ -27,7 +27,7 @@ if sys.version > '3':
 import copy
 import struct
 
-import test_framework.bignum
+from test_framework.bignum import bn2vch
 
 MAX_SCRIPT_SIZE = 10000
 MAX_SCRIPT_ELEMENT_SIZE = 520
@@ -664,7 +664,7 @@ class CScript(bytes):
             elif other == -1:
                 other = bytes(bchr(OP_1NEGATE))
             else:
-                other = CScriptOp.encode_op_pushdata(bignum.bn2vch(other))
+                other = CScriptOp.encode_op_pushdata(bn2vch(other))
         elif isinstance(other, (bytes, bytearray)):
             other = CScriptOp.encode_op_pushdata(other)
         return other