Merge #8141: Continuing port of java comparison tool

ff2dcf2 Tests: Edit bloated varint test and add option for 'barely expensive' tests (mrbandrews)
12c5a16 Catch exceptions from non-canonical encoding and print only to log (mrbandrews)
291f8aa Continuing port of java comptool (mrbandrews)
8c9e681 Tests: Rework blockstore to avoid re-serialization. (mrbandrews)

qa/rpc-tests/test_framework/blockstore.py

@@ -13,20 +13,31 @@ class BlockStore(object):
         self.blockDB = dbm.ndbm.open(datadir + "/blocks", 'c')
         self.currentBlock = 0
         self.headers_map = dict()
-    
+
     def close(self):
         self.blockDB.close()
 
+    def erase(self, blockhash):
+        del self.blockDB[repr(blockhash)]
+
+    # lookup an entry and return the item as raw bytes
     def get(self, blockhash):
-        serialized_block = None
+        value = None
         try:
-            serialized_block = self.blockDB[repr(blockhash)]
+            value = self.blockDB[repr(blockhash)]
         except KeyError:
             return None
-        f = BytesIO(serialized_block)
+        return value
-        ret = CBlock()
+
-        ret.deserialize(f)
+    # lookup an entry and return it as a CBlock
-        ret.calc_sha256()
+    def get_block(self, blockhash):
+        ret = None
+        serialized_block = self.get(blockhash)
+        if serialized_block is not None:
+            f = BytesIO(serialized_block)
+            ret = CBlock()
+            ret.deserialize(f)
+            ret.calc_sha256()
         return ret
 
     def get_header(self, blockhash):
@@ -75,13 +86,16 @@ class BlockStore(object):
     def add_header(self, header):
         self.headers_map[header.sha256] = header
 
+    # lookup the hashes in "inv", and return p2p messages for delivering
+    # blocks found.
     def get_blocks(self, inv):
         responses = []
         for i in inv:
             if (i.type == 2): # MSG_BLOCK
-                block = self.get(i.hash)
+                data = self.get(i.hash)
-                if block is not None:
+                if data is not None:
-                    responses.append(msg_block(block))
+                    # Use msg_generic to avoid re-serialization
+                    responses.append(msg_generic(b"block", data))
         return responses
 
     def get_locator(self, current_tip=None):
@@ -90,11 +104,11 @@ class BlockStore(object):
         r = []
         counter = 0
         step = 1
-        lastBlock = self.get(current_tip)
+        lastBlock = self.get_block(current_tip)
         while lastBlock is not None:
             r.append(lastBlock.hashPrevBlock)
             for i in range(step):
-                lastBlock = self.get(lastBlock.hashPrevBlock)
+                lastBlock = self.get_block(lastBlock.hashPrevBlock)
                 if lastBlock is None:
                     break
             counter += 1
@@ -111,16 +125,23 @@ class TxStore(object):
     def close(self):
         self.txDB.close()
 
+    # lookup an entry and return the item as raw bytes
     def get(self, txhash):
-        serialized_tx = None
+        value = None
         try:
-            serialized_tx = self.txDB[repr(txhash)]
+            value = self.txDB[repr(txhash)]
         except KeyError:
             return None
-        f = BytesIO(serialized_tx)
+        return value
-        ret = CTransaction()
+
-        ret.deserialize(f)
+    def get_transaction(self, txhash):
-        ret.calc_sha256()
+        ret = None
+        serialized_tx = self.get(txhash)
+        if serialized_tx is not None:
+            f = BytesIO(serialized_tx)
+            ret = CTransaction()
+            ret.deserialize(f)
+            ret.calc_sha256()
         return ret
 
     def add_transaction(self, tx):
@@ -136,5 +157,5 @@ class TxStore(object):
             if (i.type == 1): # MSG_TX
                 tx = self.get(i.hash)
                 if tx is not None:
-                    responses.append(msg_tx(tx))
+                    responses.append(msg_generic(b"tx", tx))
         return responses

qa/rpc-tests/test_framework/blocktools.py

@@ -56,12 +56,27 @@ def create_coinbase(height, pubkey = None):
     coinbase.calc_sha256()
     return coinbase
 
-# Create a transaction with an anyone-can-spend output, that spends the
+# Create a transaction.
-# nth output of prevtx.
+# If the scriptPubKey is not specified, make it anyone-can-spend.
-def create_transaction(prevtx, n, sig, value):
+def create_transaction(prevtx, n, sig, value, scriptPubKey=CScript()):
     tx = CTransaction()
     assert(n < len(prevtx.vout))
     tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), sig, 0xffffffff))
-    tx.vout.append(CTxOut(value, b""))
+    tx.vout.append(CTxOut(value, scriptPubKey))
     tx.calc_sha256()
     return tx
+
+def get_legacy_sigopcount_block(block, fAccurate=True):
+    count = 0
+    for tx in block.vtx:
+        count += get_legacy_sigopcount_tx(tx, fAccurate)
+    return count
+
+def get_legacy_sigopcount_tx(tx, fAccurate=True):
+    count = 0
+    for i in tx.vout:
+        count += i.scriptPubKey.GetSigOpCount(fAccurate)
+    for j in tx.vin:
+        # scriptSig might be of type bytes, so convert to CScript for the moment
+        count += CScript(j.scriptSig).GetSigOpCount(fAccurate)
+    return count

qa/rpc-tests/test_framework/mininode.py

@@ -836,6 +836,18 @@ class msg_block(object):
     def __repr__(self):
         return "msg_block(block=%s)" % (repr(self.block))
 
+# for cases where a user needs tighter control over what is sent over the wire
+# note that the user must supply the name of the command, and the data
+class msg_generic(object):
+    def __init__(self, command, data=None):
+        self.command = command
+        self.data = data
+
+    def serialize(self):
+        return self.data
+
+    def __repr__(self):
+        return "msg_generic()"
 
 class msg_getaddr(object):
     command = b"getaddr"

src/main.cpp

@@ -5609,6 +5609,11 @@ bool ProcessMessages(CNode* pfrom)
                 // Allow exceptions from over-long size
                 LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what());
             }
+            else if (strstr(e.what(), "non-canonical ReadCompactSize()"))
+            {
+                // Allow exceptions from non-canonical encoding
+                LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what());
+            }
             else
             {
                 PrintExceptionContinue(&e, "ProcessMessages()");