Switch blocks to a constant-space Merkle root/branch algorithm.
This switches the Merkle tree logic for blocks to one that runs in constant (small) space. The old code is moved to tests, and a new test is added that for various combinations of block sizes, transaction positions to compute a branch for, and mutations: * Verifies that the old code and new code agree for the Merkle root. * Verifies that the old code and new code agree for the Merkle branch. * Verifies that the computed Merkle branch is valid. * Verifies that mutations don't change the Merkle root. * Verifies that mutations are correctly detected.
This commit is contained in:
Pieter Wuille
parent
ee60e5625b
commit
eece63fa72
12 changed files with 182 additions and 75 deletions
|
|
@ -57,6 +57,7 @@ BITCOIN_TESTS =\
|
|||
test/dbwrapper_tests.cpp \
|
||||
test/main_tests.cpp \
|
||||
test/mempool_tests.cpp \
|
||||
test/merkle_tests.cpp \
|
||||
test/miner_tests.cpp \
|
||||
test/mruset_tests.cpp \
|
||||
test/multisig_tests.cpp \
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include "chainparams.h"
|
||||
#include "consensus/merkle.h"
|
||||
|
||||
#include "tinyformat.h"
|
||||
#include "util.h"
|
||||
|
|
@ -32,7 +33,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesi
|
|||
genesis.nVersion = nVersion;
|
||||
genesis.vtx.push_back(txNew);
|
||||
genesis.hashPrevBlock.SetNull();
|
||||
genesis.hashMerkleRoot = genesis.ComputeMerkleRoot();
|
||||
genesis.hashMerkleRoot = BlockMerkleRoot(genesis);
|
||||
return genesis;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -150,3 +150,23 @@ uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector<uint2
|
|||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
uint256 BlockMerkleRoot(const CBlock& block, bool* mutated)
|
||||
{
|
||||
std::vector<uint256> leaves;
|
||||
leaves.resize(block.vtx.size());
|
||||
for (size_t s = 0; s < block.vtx.size(); s++) {
|
||||
leaves[s] = block.vtx[s].GetHash();
|
||||
}
|
||||
return ComputeMerkleRoot(leaves, mutated);
|
||||
}
|
||||
|
||||
std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position)
|
||||
{
|
||||
std::vector<uint256> leaves;
|
||||
leaves.resize(block.vtx.size());
|
||||
for (size_t s = 0; s < block.vtx.size(); s++) {
|
||||
leaves[s] = block.vtx[s].GetHash();
|
||||
}
|
||||
return ComputeMerkleBranch(leaves, position);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,10 +8,25 @@
|
|||
#include <stdint.h>
|
||||
#include <vector>
|
||||
|
||||
#include "primitives/transaction.h"
|
||||
#include "primitives/block.h"
|
||||
#include "uint256.h"
|
||||
|
||||
uint256 ComputeMerkleRoot(const std::vector<uint256>& leaves, bool* mutated = NULL);
|
||||
std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position);
|
||||
uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector<uint256>& branch, uint32_t position);
|
||||
|
||||
/*
|
||||
* Compute the Merkle root of the transactions in a block.
|
||||
* *mutated is set to true if a duplicated subtree was found.
|
||||
*/
|
||||
uint256 BlockMerkleRoot(const CBlock& block, bool* mutated = NULL);
|
||||
|
||||
/*
|
||||
* Compute the Merkle branch for the tree of transactions in a block, for a
|
||||
* given position.
|
||||
* This can be verified using ComputeMerkleRootFromBranch.
|
||||
*/
|
||||
std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
#include "checkpoints.h"
|
||||
#include "checkqueue.h"
|
||||
#include "consensus/consensus.h"
|
||||
#include "consensus/merkle.h"
|
||||
#include "consensus/validation.h"
|
||||
#include "hash.h"
|
||||
#include "init.h"
|
||||
|
|
@ -2876,7 +2877,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo
|
|||
// Check the merkle root.
|
||||
if (fCheckMerkleRoot) {
|
||||
bool mutated;
|
||||
uint256 hashMerkleRoot2 = block.ComputeMerkleRoot(&mutated);
|
||||
uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated);
|
||||
if (block.hashMerkleRoot != hashMerkleRoot2)
|
||||
return state.DoS(100, error("CheckBlock(): hashMerkleRoot mismatch"),
|
||||
REJECT_INVALID, "bad-txnmrklroot", true);
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include "chainparams.h"
|
||||
#include "coins.h"
|
||||
#include "consensus/consensus.h"
|
||||
#include "consensus/merkle.h"
|
||||
#include "consensus/validation.h"
|
||||
#include "hash.h"
|
||||
#include "main.h"
|
||||
|
|
@ -373,7 +374,7 @@ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned
|
|||
assert(txCoinbase.vin[0].scriptSig.size() <= 100);
|
||||
|
||||
pblock->vtx[0] = txCoinbase;
|
||||
pblock->hashMerkleRoot = pblock->ComputeMerkleRoot();
|
||||
pblock->hashMerkleRoot = BlockMerkleRoot(*pblock);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
|
|
|||
|
|
@ -15,69 +15,6 @@ uint256 CBlockHeader::GetHash() const
|
|||
return SerializeHash(*this);
|
||||
}
|
||||
|
||||
uint256 CBlock::ComputeMerkleRoot(bool* fMutated) const
|
||||
{
|
||||
/* WARNING! If you're reading this because you're learning about crypto
|
||||
and/or designing a new system that will use merkle trees, keep in mind
|
||||
that the following merkle tree algorithm has a serious flaw related to
|
||||
duplicate txids, resulting in a vulnerability (CVE-2012-2459).
|
||||
|
||||
The reason is that if the number of hashes in the list at a given time
|
||||
is odd, the last one is duplicated before computing the next level (which
|
||||
is unusual in Merkle trees). This results in certain sequences of
|
||||
transactions leading to the same merkle root. For example, these two
|
||||
trees:
|
||||
|
||||
A A
|
||||
/ \ / \
|
||||
B C B C
|
||||
/ \ | / \ / \
|
||||
D E F D E F F
|
||||
/ \ / \ / \ / \ / \ / \ / \
|
||||
1 2 3 4 5 6 1 2 3 4 5 6 5 6
|
||||
|
||||
for transaction lists [1,2,3,4,5,6] and [1,2,3,4,5,6,5,6] (where 5 and
|
||||
6 are repeated) result in the same root hash A (because the hash of both
|
||||
of (F) and (F,F) is C).
|
||||
|
||||
The vulnerability results from being able to send a block with such a
|
||||
transaction list, with the same merkle root, and the same block hash as
|
||||
the original without duplication, resulting in failed validation. If the
|
||||
receiving node proceeds to mark that block as permanently invalid
|
||||
however, it will fail to accept further unmodified (and thus potentially
|
||||
valid) versions of the same block. We defend against this by detecting
|
||||
the case where we would hash two identical hashes at the end of the list
|
||||
together, and treating that identically to the block having an invalid
|
||||
merkle root. Assuming no double-SHA256 collisions, this will detect all
|
||||
known ways of changing the transactions without affecting the merkle
|
||||
root.
|
||||
*/
|
||||
std::vector<uint256> vMerkleTree;
|
||||
vMerkleTree.reserve(vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes.
|
||||
for (std::vector<CTransaction>::const_iterator it(vtx.begin()); it != vtx.end(); ++it)
|
||||
vMerkleTree.push_back(it->GetHash());
|
||||
int j = 0;
|
||||
bool mutated = false;
|
||||
for (int nSize = vtx.size(); nSize > 1; nSize = (nSize + 1) / 2)
|
||||
{
|
||||
for (int i = 0; i < nSize; i += 2)
|
||||
{
|
||||
int i2 = std::min(i+1, nSize-1);
|
||||
if (i2 == i + 1 && i2 + 1 == nSize && vMerkleTree[j+i] == vMerkleTree[j+i2]) {
|
||||
// Two identical hashes at the end of the list at a particular level.
|
||||
mutated = true;
|
||||
}
|
||||
vMerkleTree.push_back(Hash(BEGIN(vMerkleTree[j+i]), END(vMerkleTree[j+i]),
|
||||
BEGIN(vMerkleTree[j+i2]), END(vMerkleTree[j+i2])));
|
||||
}
|
||||
j += nSize;
|
||||
}
|
||||
if (fMutated) {
|
||||
*fMutated = mutated;
|
||||
}
|
||||
return (vMerkleTree.empty() ? uint256() : vMerkleTree.back());
|
||||
}
|
||||
|
||||
std::string CBlock::ToString() const
|
||||
{
|
||||
std::stringstream s;
|
||||
|
|
|
|||
|
|
@ -118,12 +118,6 @@ public:
|
|||
return block;
|
||||
}
|
||||
|
||||
// Build the merkle tree for this block and return the merkle root.
|
||||
// If non-NULL, *mutated is set to whether mutation was detected in the merkle
|
||||
// tree (a duplication of transactions in the block leading to an identical
|
||||
// merkle root).
|
||||
uint256 ComputeMerkleRoot(bool* mutated = NULL) const;
|
||||
|
||||
std::string ToString() const;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -72,5 +72,4 @@ BOOST_AUTO_TEST_CASE(test_combiner_all)
|
|||
Test.disconnect(&ReturnTrue);
|
||||
BOOST_CHECK(Test());
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_SUITE_END()
|
||||
|
|
|
|||
136
src/test/merkle_tests.cpp
Normal file
136
src/test/merkle_tests.cpp
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
// Copyright (c) 2015 The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include "consensus/merkle.h"
|
||||
#include "test/test_bitcoin.h"
|
||||
#include "random.h"
|
||||
|
||||
#include <boost/test/unit_test.hpp>
|
||||
|
||||
BOOST_FIXTURE_TEST_SUITE(merkle_tests, TestingSetup)
|
||||
|
||||
// Older version of the merkle root computation code, for comparison.
|
||||
static uint256 BlockBuildMerkleTree(const CBlock& block, bool* fMutated, std::vector<uint256>& vMerkleTree)
|
||||
{
|
||||
vMerkleTree.clear();
|
||||
vMerkleTree.reserve(block.vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes.
|
||||
for (std::vector<CTransaction>::const_iterator it(block.vtx.begin()); it != block.vtx.end(); ++it)
|
||||
vMerkleTree.push_back(it->GetHash());
|
||||
int j = 0;
|
||||
bool mutated = false;
|
||||
for (int nSize = block.vtx.size(); nSize > 1; nSize = (nSize + 1) / 2)
|
||||
{
|
||||
for (int i = 0; i < nSize; i += 2)
|
||||
{
|
||||
int i2 = std::min(i+1, nSize-1);
|
||||
if (i2 == i + 1 && i2 + 1 == nSize && vMerkleTree[j+i] == vMerkleTree[j+i2]) {
|
||||
// Two identical hashes at the end of the list at a particular level.
|
||||
mutated = true;
|
||||
}
|
||||
vMerkleTree.push_back(Hash(vMerkleTree[j+i].begin(), vMerkleTree[j+i].end(),
|
||||
vMerkleTree[j+i2].begin(), vMerkleTree[j+i2].end()));
|
||||
}
|
||||
j += nSize;
|
||||
}
|
||||
if (fMutated) {
|
||||
*fMutated = mutated;
|
||||
}
|
||||
return (vMerkleTree.empty() ? uint256() : vMerkleTree.back());
|
||||
}
|
||||
|
||||
// Older version of the merkle branch computation code, for comparison.
|
||||
static std::vector<uint256> BlockGetMerkleBranch(const CBlock& block, const std::vector<uint256>& vMerkleTree, int nIndex)
|
||||
{
|
||||
std::vector<uint256> vMerkleBranch;
|
||||
int j = 0;
|
||||
for (int nSize = block.vtx.size(); nSize > 1; nSize = (nSize + 1) / 2)
|
||||
{
|
||||
int i = std::min(nIndex^1, nSize-1);
|
||||
vMerkleBranch.push_back(vMerkleTree[j+i]);
|
||||
nIndex >>= 1;
|
||||
j += nSize;
|
||||
}
|
||||
return vMerkleBranch;
|
||||
}
|
||||
|
||||
static inline int ctz(uint32_t i) {
|
||||
if (i == 0) return 0;
|
||||
int j = 0;
|
||||
while (!(i & 1)) {
|
||||
j++;
|
||||
i >>= 1;
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(merkle_test)
|
||||
{
|
||||
for (int i = 0; i < 32; i++) {
|
||||
// Try 32 block sizes: all sizes from 0 to 16 inclusive, and then 15 random sizes.
|
||||
int ntx = (i <= 16) ? i : 17 + (insecure_rand() % 4000);
|
||||
// Try up to 3 mutations.
|
||||
for (int mutate = 0; mutate <= 3; mutate++) {
|
||||
int duplicate1 = mutate >= 1 ? 1 << ctz(ntx) : 0; // The last how many transactions to duplicate first.
|
||||
if (duplicate1 >= ntx) break; // Duplication of the entire tree results in a different root (it adds a level).
|
||||
int ntx1 = ntx + duplicate1; // The resulting number of transactions after the first duplication.
|
||||
int duplicate2 = mutate >= 2 ? 1 << ctz(ntx1) : 0; // Likewise for the second mutation.
|
||||
if (duplicate2 >= ntx1) break;
|
||||
int ntx2 = ntx1 + duplicate2;
|
||||
int duplicate3 = mutate >= 3 ? 1 << ctz(ntx2) : 0; // And for the the third mutation.
|
||||
if (duplicate3 >= ntx2) break;
|
||||
int ntx3 = ntx2 + duplicate3;
|
||||
// Build a block with ntx different transactions.
|
||||
CBlock block;
|
||||
block.vtx.resize(ntx);
|
||||
for (int j = 0; j < ntx; j++) {
|
||||
CMutableTransaction mtx;
|
||||
mtx.nLockTime = j;
|
||||
block.vtx[j] = mtx;
|
||||
}
|
||||
// Compute the root of the block before mutating it.
|
||||
bool unmutatedMutated = false;
|
||||
uint256 unmutatedRoot = BlockMerkleRoot(block, &unmutatedMutated);
|
||||
BOOST_CHECK(unmutatedMutated == false);
|
||||
// Optionally mutate by duplicating the last transactions, resulting in the same merkle root.
|
||||
block.vtx.resize(ntx3);
|
||||
for (int j = 0; j < duplicate1; j++) {
|
||||
block.vtx[ntx + j] = block.vtx[ntx + j - duplicate1];
|
||||
}
|
||||
for (int j = 0; j < duplicate2; j++) {
|
||||
block.vtx[ntx1 + j] = block.vtx[ntx1 + j - duplicate2];
|
||||
}
|
||||
for (int j = 0; j < duplicate3; j++) {
|
||||
block.vtx[ntx2 + j] = block.vtx[ntx2 + j - duplicate3];
|
||||
}
|
||||
// Compute the merkle root and merkle tree using the old mechanism.
|
||||
bool oldMutated = false;
|
||||
std::vector<uint256> merkleTree;
|
||||
uint256 oldRoot = BlockBuildMerkleTree(block, &oldMutated, merkleTree);
|
||||
// Compute the merkle root using the new mechanism.
|
||||
bool newMutated = false;
|
||||
uint256 newRoot = BlockMerkleRoot(block, &newMutated);
|
||||
BOOST_CHECK(oldRoot == newRoot);
|
||||
BOOST_CHECK(newRoot == unmutatedRoot);
|
||||
BOOST_CHECK((newRoot == uint256()) == (ntx == 0));
|
||||
BOOST_CHECK(oldMutated == newMutated);
|
||||
BOOST_CHECK(newMutated == !!mutate);
|
||||
// If no mutation was done (once for every ntx value), try up to 16 branches.
|
||||
if (mutate == 0) {
|
||||
for (int loop = 0; loop < std::min(ntx, 16); loop++) {
|
||||
// If ntx <= 16, try all branches. Otherise, try 16 random ones.
|
||||
int mtx = loop;
|
||||
if (ntx > 16) {
|
||||
mtx = insecure_rand() % ntx;
|
||||
}
|
||||
std::vector<uint256> newBranch = BlockMerkleBranch(block, mtx);
|
||||
std::vector<uint256> oldBranch = BlockGetMerkleBranch(block, merkleTree, mtx);
|
||||
BOOST_CHECK(oldBranch == newBranch);
|
||||
BOOST_CHECK(ComputeMerkleRootFromBranch(block.vtx[mtx].GetHash(), newBranch, mtx) == oldRoot);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_SUITE_END()
|
||||
|
|
@ -5,6 +5,7 @@
|
|||
#include "chainparams.h"
|
||||
#include "coins.h"
|
||||
#include "consensus/consensus.h"
|
||||
#include "consensus/merkle.h"
|
||||
#include "consensus/validation.h"
|
||||
#include "main.h"
|
||||
#include "miner.h"
|
||||
|
|
@ -93,7 +94,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
|
|||
pblock->vtx[0] = CTransaction(txCoinbase);
|
||||
if (txFirst.size() < 2)
|
||||
txFirst.push_back(new CTransaction(pblock->vtx[0]));
|
||||
pblock->hashMerkleRoot = pblock->ComputeMerkleRoot();
|
||||
pblock->hashMerkleRoot = BlockMerkleRoot(*pblock);
|
||||
pblock->nNonce = blockinfo[i].nonce;
|
||||
CValidationState state;
|
||||
BOOST_CHECK(ProcessNewBlock(state, chainparams, NULL, pblock, true, NULL));
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include "consensus/merkle.h"
|
||||
#include "merkleblock.h"
|
||||
#include "serialize.h"
|
||||
#include "streams.h"
|
||||
|
|
@ -48,7 +49,7 @@ BOOST_AUTO_TEST_CASE(pmt_test1)
|
|||
}
|
||||
|
||||
// calculate actual merkle root and height
|
||||
uint256 merkleRoot1 = block.ComputeMerkleRoot();
|
||||
uint256 merkleRoot1 = BlockMerkleRoot(block);
|
||||
std::vector<uint256> vTxid(nTx, uint256());
|
||||
for (unsigned int j=0; j<nTx; j++)
|
||||
vTxid[j] = block.vtx[j].GetHash();
|
||||
|
|
|
|||
Loading…
Reference in a new issue