LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Merge pull request #6777

Browse source 
dcd8e27 Refer to obfuscate_key via pointer in peripheral CLevelDB classes (James O'Beirne)
1488506 Add tests for gettxoutsetinfo, CLevelDBBatch, CLevelDBIterator (James O'Beirne)
0fdf8c8 Handle obfuscation in CLevelDBIterator (James O'Beirne)
3499ce1 Encapsulate CLevelDB iterators cleanly (Pieter Wuille)
This commit is contained in:
Wladimir J. van der Laan 2015-10-13 12:22:40 +02:00
commit 9caaf6ed22
No known key found for this signature in database
GPG key ID: 74810B012346C9A6
6 changed files with 245 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
qa/pull-tester/rpc-tests.py
View file

@ -38,7 +38,7 @@ for i in range(1,len(sys.argv)):
buildDir = BUILDDIR
os.environ["BITCOIND"] = buildDir + '/src/bitcoind' + EXEEXT
os.environ["BITCOINCLI"] = buildDir + '/src/bitcoin-cli' + EXEEXT
#Disable Windows tests by default
if EXEEXT == ".exe" and "-win" not in opts:
print "Win tests currently disabled. Use -win option to enable"
@ -67,6 +67,7 @@ testScripts = [
'reindex.py',
'decodescript.py',
'p2p-fullblocktest.py',
'blockchain.py',
]
testScriptsExt = [
'bipdersig-p2p.py',
@ -98,10 +99,10 @@ if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
rpcTestDir = buildDir + '/qa/rpc-tests/'
#Run Tests
for i in range(len(testScripts)):
if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts
if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts
or testScripts[i] in opts or re.sub(".py$", "", testScripts[i]) in opts ):
print "Running testscript " + testScripts[i] + "..."
subprocess.call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
print "Running testscript " + testScripts[i] + "..."
subprocess.call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
#exit if help is called so we print just one set of instructions
p = re.compile(" -h| --help")
if p.match(passOn):
@ -109,9 +110,9 @@ if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
#Run Extended Tests
for i in range(len(testScriptsExt)):
if ('-extended' in opts or testScriptsExt[i] in opts
if ('-extended' in opts or testScriptsExt[i] in opts
or re.sub(".py$", "", testScriptsExt[i]) in opts):
print "Running 2nd level testscript " + testScriptsExt[i] + "..."
subprocess.call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
print "Running 2nd level testscript " + testScriptsExt[i] + "..."
subprocess.call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
else:
print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled"

52
qa/rpc-tests/blockchain.py Executable file
View file

@ -0,0 +1,52 @@
#!/usr/bin/env python2
# Copyright (c) 2014 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Test RPC calls related to blockchain state.
#
import decimal
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
initialize_chain,
assert_equal,
start_nodes,
connect_nodes_bi,
)
class BlockchainTest(BitcoinTestFramework):
"""
Test blockchain-related RPC calls:
- gettxoutsetinfo
"""
def setup_chain(self):
print("Initializing test directory " + self.options.tmpdir)
initialize_chain(self.options.tmpdir)
def setup_network(self, split=False):
self.nodes = start_nodes(2, self.options.tmpdir)
connect_nodes_bi(self.nodes, 0, 1)
self.is_network_split = False
self.sync_all()
def run_test(self):
node = self.nodes[0]
res = node.gettxoutsetinfo()
assert_equal(res[u'total_amount'], decimal.Decimal('8725.00000000'))
assert_equal(res[u'transactions'], 200)
assert_equal(res[u'height'], 200)
assert_equal(res[u'txouts'], 200)
assert_equal(res[u'bytes_serialized'], 13000),
assert_equal(len(res[u'bestblock']), 64)
assert_equal(len(res[u'hash_serialized']), 64)
if __name__ == '__main__':
BlockchainTest().main()

9
src/leveldbwrapper.cpp
View file

@ -131,7 +131,7 @@ std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const
bool CLevelDBWrapper::IsEmpty()
{
boost::scoped_ptr<leveldb::Iterator> it(NewIterator());
boost::scoped_ptr<CLevelDBIterator> it(NewIterator());
it->SeekToFirst();
return !(it->Valid());
}
@ -145,3 +145,10 @@ std::string CLevelDBWrapper::GetObfuscateKeyHex() const
{
return HexStr(obfuscate_key);
}
CLevelDBIterator::~CLevelDBIterator() { delete piter; }
bool CLevelDBIterator::Valid() { return piter->Valid(); }
void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); }
void CLevelDBIterator::SeekToLast() { piter->SeekToLast(); }
void CLevelDBIterator::Next() { piter->Next(); }
void CLevelDBIterator::Prev() { piter->Prev(); }

82
src/leveldbwrapper.h
View file

@ -32,13 +32,13 @@ class CLevelDBBatch
private:
leveldb::WriteBatch batch;
const std::vector<unsigned char> obfuscate_key;
const std::vector<unsigned char> *obfuscate_key;
public:
/**
* @param[in] obfuscate_key If passed, XOR data with this key.
*/
CLevelDBBatch(const std::vector<unsigned char>& obfuscate_key) : obfuscate_key(obfuscate_key) { };
CLevelDBBatch(const std::vector<unsigned char> *obfuscate_key) : obfuscate_key(obfuscate_key) { };
template <typename K, typename V>
void Write(const K& key, const V& value)
@ -51,7 +51,7 @@ public:
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
ssValue.reserve(ssValue.GetSerializeSize(value));
ssValue << value;
ssValue.Xor(obfuscate_key);
ssValue.Xor(*obfuscate_key);
leveldb::Slice slValue(&ssValue[0], ssValue.size());
batch.Put(slKey, slValue);
@ -68,7 +68,72 @@ public:
batch.Delete(slKey);
}
};
class CLevelDBIterator
{
private:
leveldb::Iterator *piter;
const std::vector<unsigned char> *obfuscate_key;
public:
/**
* @param[in] piterIn The original leveldb iterator.
* @param[in] obfuscate_key If passed, XOR data with this key.
*/
CLevelDBIterator(leveldb::Iterator *piterIn, const std::vector<unsigned char>* obfuscate_key) :
piter(piterIn), obfuscate_key(obfuscate_key) { };
~CLevelDBIterator();
bool Valid();
void SeekToFirst();
void SeekToLast();
template<typename K> void Seek(const K& key) {
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(ssKey.GetSerializeSize(key));
ssKey << key;
leveldb::Slice slKey(&ssKey[0], ssKey.size());
piter->Seek(slKey);
}
void Next();
void Prev();
template<typename K> bool GetKey(K& key) {
leveldb::Slice slKey = piter->key();
try {
CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION);
ssKey >> key;
} catch(std::exception &e) {
return false;
}
return true;
}
unsigned int GetKeySize() {
return piter->key().size();
}
template<typename V> bool GetValue(V& value) {
leveldb::Slice slValue = piter->value();
try {
CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION);
ssValue.Xor(*obfuscate_key);
ssValue >> value;
} catch(std::exception &e) {
return false;
}
return true;
}
unsigned int GetValueSize() {
return piter->value().size();
}
};
class CLevelDBWrapper
{
private:
@ -145,7 +210,7 @@ public:
template <typename K, typename V>
bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error)
{
CLevelDBBatch batch(obfuscate_key);
CLevelDBBatch batch(&obfuscate_key);
batch.Write(key, value);
return WriteBatch(batch, fSync);
}
@ -172,7 +237,7 @@ public:
template <typename K>
bool Erase(const K& key, bool fSync = false) throw(leveldb_error)
{
CLevelDBBatch batch(obfuscate_key);
CLevelDBBatch batch(&obfuscate_key);
batch.Erase(key);
return WriteBatch(batch, fSync);
}
@ -187,14 +252,13 @@ public:
bool Sync() throw(leveldb_error)
{
CLevelDBBatch batch(obfuscate_key);
CLevelDBBatch batch(&obfuscate_key);
return WriteBatch(batch, true);
}
// not exactly clean encapsulation, but it's easiest for now
leveldb::Iterator* NewIterator()
CLevelDBIterator *NewIterator()
{
return pdb->NewIterator(iteroptions);
return new CLevelDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key);
}
/**

81
src/test/leveldbwrapper_tests.cpp
View file

@ -46,7 +46,86 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper)
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
}
}
// Test batch operations
BOOST_AUTO_TEST_CASE(leveldbwrapper_batch)
{
// Perform tests both obfuscated and non-obfuscated.
for (int i = 0; i < 2; i++) {
bool obfuscate = (bool)i;
path ph = temp_directory_path() / unique_path();
CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate);
char key = 'i';
uint256 in = GetRandHash();
char key2 = 'j';
uint256 in2 = GetRandHash();
char key3 = 'k';
uint256 in3 = GetRandHash();
uint256 res;
CLevelDBBatch batch(&dbw.GetObfuscateKey());
batch.Write(key, in);
batch.Write(key2, in2);
batch.Write(key3, in3);
// Remove key3 before it's even been written
batch.Erase(key3);
dbw.WriteBatch(batch);
BOOST_CHECK(dbw.Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
BOOST_CHECK(dbw.Read(key2, res));
BOOST_CHECK_EQUAL(res.ToString(), in2.ToString());
// key3 never should've been written
BOOST_CHECK(dbw.Read(key3, res) == false);
}
}
BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator)
{
// Perform tests both obfuscated and non-obfuscated.
for (int i = 0; i < 2; i++) {
bool obfuscate = (bool)i;
path ph = temp_directory_path() / unique_path();
CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate);
// The two keys are intentionally chosen for ordering
char key = 'j';
uint256 in = GetRandHash();
BOOST_CHECK(dbw.Write(key, in));
char key2 = 'k';
uint256 in2 = GetRandHash();
BOOST_CHECK(dbw.Write(key2, in2));
boost::scoped_ptr<CLevelDBIterator> it(const_cast<CLevelDBWrapper*>(&dbw)->NewIterator());
// Be sure to seek past the obfuscation key (if it exists)
it->Seek(key);
char key_res;
uint256 val_res;
it->GetKey(key_res);
it->GetValue(val_res);
BOOST_CHECK_EQUAL(key_res, key);
BOOST_CHECK_EQUAL(val_res.ToString(), in.ToString());
it->Next();
it->GetKey(key_res);
it->GetValue(val_res);
BOOST_CHECK_EQUAL(key_res, key2);
BOOST_CHECK_EQUAL(val_res.ToString(), in2.ToString());
it->Next();
BOOST_CHECK_EQUAL(it->Valid(), false);
}
}
// Test that we do not obfuscation if there is existing data.
BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
{

67
src/txdb.cpp
View file

@ -49,7 +49,7 @@ uint256 CCoinsViewDB::GetBestBlock() const {
}
bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
CLevelDBBatch batch(db.GetObfuscateKey());
CLevelDBBatch batch(&db.GetObfuscateKey());
size_t count = 0;
size_t changed = 0;
for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
@ -98,8 +98,8 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
/* It seems that there are no "const iterators" for LevelDB. Since we
only need read operations on it, use a const-cast to get around
that restriction. */
boost::scoped_ptr<leveldb::Iterator> pcursor(const_cast<CLevelDBWrapper*>(&db)->NewIterator());
pcursor->SeekToFirst();
boost::scoped_ptr<CLevelDBIterator> pcursor(const_cast<CLevelDBWrapper*>(&db)->NewIterator());
pcursor->Seek('c');
CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION);
stats.hashBlock = GetBestBlock();
@ -107,22 +107,10 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
CAmount nTotalAmount = 0;
while (pcursor->Valid()) {
boost::this_thread::interruption_point();
try {
leveldb::Slice slKey = pcursor->key();
CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION);
char chType;
ssKey >> chType;
if (chType == DB_COINS) {
leveldb::Slice slValue = pcursor->value();
CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
CCoins coins;
ssValue >> coins;
uint256 txhash;
ssKey >> txhash;
ss << txhash;
ss << VARINT(coins.nVersion);
ss << (coins.fCoinBase ? 'c' : 'n');
ss << VARINT(coins.nHeight);
std::pair<char, uint256> key;
CCoins coins;
if (pcursor->GetKey(key) && key.first == 'c') {
if (pcursor->GetValue(coins)) {
stats.nTransactions++;
for (unsigned int i=0; i<coins.vout.size(); i++) {
const CTxOut &out = coins.vout[i];
@ -133,13 +121,15 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
nTotalAmount += out.nValue;
}
}
stats.nSerializedSize += 32 + slValue.size();
stats.nSerializedSize += 32 + pcursor->GetKeySize();
ss << VARINT(0);
} else {
return error("CCoinsViewDB::GetStats() : unable to read value");
}
pcursor->Next();
} catch (const std::exception& e) {
return error("%s: Deserialize or I/O error - %s", __func__, e.what());
} else {
break;
}
pcursor->Next();
}
{
LOCK(cs_main);
@ -151,7 +141,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
}
bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
CLevelDBBatch batch(GetObfuscateKey());
CLevelDBBatch batch(&GetObfuscateKey());
for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second);
}
@ -167,7 +157,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) {
}
bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) {
CLevelDBBatch batch(GetObfuscateKey());
CLevelDBBatch batch(&GetObfuscateKey());
for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++)
batch.Write(make_pair(DB_TXINDEX, it->first), it->second);
return WriteBatch(batch);
@ -187,26 +177,17 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
bool CBlockTreeDB::LoadBlockIndexGuts()
{
boost::scoped_ptr<leveldb::Iterator> pcursor(NewIterator());
boost::scoped_ptr<CLevelDBIterator> pcursor(NewIterator());
CDataStream ssKeySet(SER_DISK, CLIENT_VERSION);
ssKeySet << make_pair(DB_BLOCK_INDEX, uint256());
pcursor->Seek(ssKeySet.str());
pcursor->Seek(make_pair('b', uint256()));
// Load mapBlockIndex
while (pcursor->Valid()) {
boost::this_thread::interruption_point();
try {
leveldb::Slice slKey = pcursor->key();
CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION);
char chType;
ssKey >> chType;
if (chType == DB_BLOCK_INDEX) {
leveldb::Slice slValue = pcursor->value();
CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
CDiskBlockIndex diskindex;
ssValue >> diskindex;
std::pair<char, uint256> key;
if (pcursor->GetKey(key) && key.first == 'b') {
CDiskBlockIndex diskindex;
if (pcursor->GetValue(diskindex)) {
// Construct block index object
CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash());
pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev);
@ -227,10 +208,10 @@ bool CBlockTreeDB::LoadBlockIndexGuts()
pcursor->Next();
} else {
break; // if shutdown requested or finished loading block index
return error("LoadBlockIndex() : failed to read value");
}
} catch (const std::exception& e) {
return error("%s: Deserialize or I/O error - %s", __func__, e.what());
} else {
break;
}
}