LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Add chainstate obfuscation to avoid spurious antivirus detection

Browse source 
Adds an `obfuscate` parameter to `CLevelDBWrapper` and makes use of it
for all new chainstate stores built via `CCoinsViewDB`. Also adds an
`Xor` method to `CDataStream`.

Thanks to @sipa @laanwj @pstratem @dexX7 @KyrosKrane @gmaxwell.
This commit is contained in:
James O'Beirne 2015-09-07 15:22:23 -07:00
parent 3932ff50c5
commit 42cb388167
7 changed files with 338 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Makefile.test.include
View file

@ -53,6 +53,7 @@ BITCOIN_TESTS =\
test/hash_tests.cpp \
test/key_tests.cpp \
test/limitedmap_tests.cpp \
test/leveldbwrapper_tests.cpp \
test/main_tests.cpp \
test/mempool_tests.cpp \
test/miner_tests.cpp \
@ -72,6 +73,7 @@ BITCOIN_TESTS =\
test/sighash_tests.cpp \
test/sigopcount_tests.cpp \
test/skiplist_tests.cpp \
test/streams_tests.cpp \
test/test_bitcoin.cpp \
test/test_bitcoin.h \
test/timedata_tests.cpp \

60
src/leveldbwrapper.cpp
View file

@ -5,6 +5,7 @@
#include "leveldbwrapper.h"
#include "util.h"
#include "random.h"
#include <boost/filesystem.hpp>
@ -12,6 +13,7 @@
#include <leveldb/env.h>
#include <leveldb/filter_policy.h>
#include <memenv.h>
#include <stdint.h>
void HandleError(const leveldb::Status& status) throw(leveldb_error)
{
@ -43,7 +45,7 @@ static leveldb::Options GetOptions(size_t nCacheSize)
return options;
}
CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe)
CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
{
penv = NULL;
readoptions.verify_checksums = true;
@ -67,6 +69,25 @@ CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCa
leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb);
HandleError(status);
LogPrintf("Opened LevelDB successfully\n");
// The base-case obfuscation key, which is a noop.
obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
if (!key_exists && obfuscate && IsEmpty()) {
// Initialize non-degenerate obfuscation if it won't upset
// existing, non-obfuscated data.
std::vector<unsigned char> new_key = CreateObfuscateKey();
// Write `new_key` so we don't obfuscate the key with itself
Write(OBFUSCATE_KEY_KEY, new_key);
obfuscate_key = new_key;
LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), GetObfuscateKeyHex());
}
LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex());
}
CLevelDBWrapper::~CLevelDBWrapper()
@ -87,3 +108,40 @@ bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) throw(leveldb
HandleError(status);
return true;
}
// Prefixed with null character to avoid collisions with other keys
//
// We must use a string constructor which specifies length so that we copy
// past the null-terminator.
const std::string CLevelDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
const unsigned int CLevelDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
/**
* Returns a string (consisting of 8 random bytes) suitable for use as an
* obfuscating XOR key.
*/
std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const
{
unsigned char buff[OBFUSCATE_KEY_NUM_BYTES];
GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES);
return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]);
}
bool CLevelDBWrapper::IsEmpty()
{
boost::scoped_ptr<leveldb::Iterator> it(NewIterator());
it->SeekToFirst();
return !(it->Valid());
}
const std::vector<unsigned char>& CLevelDBWrapper::GetObfuscateKey() const
{
return obfuscate_key;
}
std::string CLevelDBWrapper::GetObfuscateKeyHex() const
{
return HexStr(obfuscate_key);
}

53
src/leveldbwrapper.h
View file

@ -9,6 +9,7 @@
#include "serialize.h"
#include "streams.h"
#include "util.h"
#include "utilstrencodings.h"
#include "version.h"
#include <boost/filesystem/path.hpp>
@ -31,8 +32,14 @@ class CLevelDBBatch
private:
leveldb::WriteBatch batch;
const std::vector<unsigned char> obfuscate_key;
public:
/**
* @param[in] obfuscate_key If passed, XOR data with this key.
*/
CLevelDBBatch(const std::vector<unsigned char>& obfuscate_key) : obfuscate_key(obfuscate_key) { };
template <typename K, typename V>
void Write(const K& key, const V& value)
{
@ -44,6 +51,7 @@ public:
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
ssValue.reserve(ssValue.GetSerializeSize(value));
ssValue << value;
ssValue.Xor(obfuscate_key);
leveldb::Slice slValue(&ssValue[0], ssValue.size());
batch.Put(slKey, slValue);
@ -85,8 +93,27 @@ private:
//! the database itself
leveldb::DB* pdb;
//! a key used for optional XOR-obfuscation of the database
std::vector<unsigned char> obfuscate_key;
//! the key under which the obfuscation key is stored
static const std::string OBFUSCATE_KEY_KEY;
//! the length of the obfuscate key in number of bytes
static const unsigned int OBFUSCATE_KEY_NUM_BYTES;
std::vector<unsigned char> CreateObfuscateKey() const;
public:
CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false);
/**
* @param[in] path Location in the filesystem where leveldb data will be stored.
* @param[in] nCacheSize Configures various leveldb cache settings.
* @param[in] fMemory If true, use leveldb's memory environment.
* @param[in] fWipe If true, remove all existing data.
* @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR
* with a zero'd byte array.
*/
CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
~CLevelDBWrapper();
template <typename K, typename V>
@ -107,6 +134,7 @@ public:
}
try {
CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION);
ssValue.Xor(obfuscate_key);
ssValue >> value;
} catch (const std::exception&) {
return false;
@ -117,7 +145,7 @@ public:
template <typename K, typename V>
bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error)
{
CLevelDBBatch batch;
CLevelDBBatch batch(obfuscate_key);
batch.Write(key, value);
return WriteBatch(batch, fSync);
}
@ -144,7 +172,7 @@ public:
template <typename K>
bool Erase(const K& key, bool fSync = false) throw(leveldb_error)
{
CLevelDBBatch batch;
CLevelDBBatch batch(obfuscate_key);
batch.Erase(key);
return WriteBatch(batch, fSync);
}
@ -159,7 +187,7 @@ public:
bool Sync() throw(leveldb_error)
{
CLevelDBBatch batch;
CLevelDBBatch batch(obfuscate_key);
return WriteBatch(batch, true);
}
@ -168,6 +196,23 @@ public:
{
return pdb->NewIterator(iteroptions);
}
/**
* Return true if the database managed by this class contains no entries.
*/
bool IsEmpty();
/**
* Accessor for obfuscate_key.
*/
const std::vector<unsigned char>& GetObfuscateKey() const;
/**
* Return the obfuscate_key as a hex-formatted string.
*/
std::string GetObfuscateKeyHex() const;
};
#endif // BITCOIN_LEVELDBWRAPPER_H

23
src/streams.h
View file

@ -296,6 +296,29 @@ public:
data.insert(data.end(), begin(), end());
clear();
}
/**
* XOR the contents of this stream with a certain key.
*
* @param[in] key The key used to XOR the data in this stream.
*/
void Xor(const std::vector<unsigned char>& key)
{
if (key.size() == 0) {
return;
}
for (size_type i = 0, j = 0; i != size(); i++) {
vch[i] ^= key[j++];
// This potentially acts on very many bytes of data, so it's
// important that we calculate `j`, i.e. the `key` index in this
// way instead of doing a %, which would effectively be a division
// for each byte Xor'd -- much slower than need be.
if (j == key.size())
j = 0;
}
}
};

128
src/test/leveldbwrapper_tests.cpp Normal file
View file

@ -0,0 +1,128 @@
// Copyright (c) 2012-2013 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "leveldbwrapper.h"
#include "uint256.h"
#include "random.h"
#include "test/test_bitcoin.h"
#include <boost/assign/std/vector.hpp> // for 'operator+=()'
#include <boost/assert.hpp>
#include <boost/test/unit_test.hpp>
using namespace std;
using namespace boost::assign; // bring 'operator+=()' into scope
using namespace boost::filesystem;
// Test if a string consists entirely of null characters
bool is_null_key(const vector<unsigned char>& key) {
bool isnull = true;
for (unsigned int i = 0; i < key.size(); i++)
isnull &= (key[i] == '\x00');
return isnull;
}
BOOST_FIXTURE_TEST_SUITE(leveldbwrapper_tests, BasicTestingSetup)
BOOST_AUTO_TEST_CASE(leveldbwrapper)
{
// Perform tests both obfuscated and non-obfuscated.
for (int i = 0; i < 2; i++) {
bool obfuscate = (bool)i;
path ph = temp_directory_path() / unique_path();
CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate);
char key = 'k';
uint256 in = GetRandHash();
uint256 res;
// Ensure that we're doing real obfuscation when obfuscate=true
BOOST_CHECK(obfuscate != is_null_key(dbw.GetObfuscateKey()));
BOOST_CHECK(dbw.Write(key, in));
BOOST_CHECK(dbw.Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
}
}
// Test that we do not obfuscation if there is existing data.
BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
{
// We're going to share this path between two wrappers
path ph = temp_directory_path() / unique_path();
create_directories(ph);
// Set up a non-obfuscated wrapper to write some initial data.
CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false);
char key = 'k';
uint256 in = GetRandHash();
uint256 res;
BOOST_CHECK(dbw->Write(key, in));
BOOST_CHECK(dbw->Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
// Call the destructor to free leveldb LOCK
delete dbw;
// Now, set up another wrapper that wants to obfuscate the same directory
CLevelDBWrapper odbw(ph, (1 << 10), false, false, true);
// Check that the key/val we wrote with unobfuscated wrapper exists and
// is readable.
uint256 res2;
BOOST_CHECK(odbw.Read(key, res2));
BOOST_CHECK_EQUAL(res2.ToString(), in.ToString());
BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data
BOOST_CHECK(is_null_key(odbw.GetObfuscateKey())); // The key should be an empty string
uint256 in2 = GetRandHash();
uint256 res3;
// Check that we can write successfully
BOOST_CHECK(odbw.Write(key, in2));
BOOST_CHECK(odbw.Read(key, res3));
BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString());
}
// Ensure that we start obfuscating during a reindex.
BOOST_AUTO_TEST_CASE(existing_data_reindex)
{
// We're going to share this path between two wrappers
path ph = temp_directory_path() / unique_path();
create_directories(ph);
// Set up a non-obfuscated wrapper to write some initial data.
CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false);
char key = 'k';
uint256 in = GetRandHash();
uint256 res;
BOOST_CHECK(dbw->Write(key, in));
BOOST_CHECK(dbw->Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
// Call the destructor to free leveldb LOCK
delete dbw;
// Simulate a -reindex by wiping the existing data store
CLevelDBWrapper odbw(ph, (1 << 10), false, true, true);
// Check that the key/val we wrote with unobfuscated wrapper doesn't exist
uint256 res2;
BOOST_CHECK(!odbw.Read(key, res2));
BOOST_CHECK(!is_null_key(odbw.GetObfuscateKey()));
uint256 in2 = GetRandHash();
uint256 res3;
// Check that we can write successfully
BOOST_CHECK(odbw.Write(key, in2));
BOOST_CHECK(odbw.Read(key, res3));
BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString());
}
BOOST_AUTO_TEST_SUITE_END()

67
src/test/streams_tests.cpp Normal file
View file

@ -0,0 +1,67 @@
// Copyright (c) 2012-2013 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "streams.h"
#include "support/allocators/zeroafterfree.h"
#include "test/test_bitcoin.h"
#include <boost/assign/std/vector.hpp> // for 'operator+=()'
#include <boost/assert.hpp>
#include <boost/test/unit_test.hpp>
using namespace std;
using namespace boost::assign; // bring 'operator+=()' into scope
BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup)
BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
{
std::vector<char> in;
std::vector<char> expected_xor;
std::vector<unsigned char> key;
CDataStream ds(in, 0, 0);
// Degenerate case
key += '\x00','\x00';
ds.Xor(key);
BOOST_CHECK_EQUAL(
std::string(expected_xor.begin(), expected_xor.end()),
std::string(ds.begin(), ds.end()));
in += '\x0f','\xf0';
expected_xor += '\xf0','\x0f';
// Single character key
ds.clear();
ds.insert(ds.begin(), in.begin(), in.end());
key.clear();
key += '\xff';
ds.Xor(key);
BOOST_CHECK_EQUAL(
std::string(expected_xor.begin(), expected_xor.end()),
std::string(ds.begin(), ds.end()));
// Multi character key
in.clear();
expected_xor.clear();
in += '\xf0','\x0f';
expected_xor += '\x0f','\x00';
ds.clear();
ds.insert(ds.begin(), in.begin(), in.end());
key.clear();
key += '\xff','\x0f';
ds.Xor(key);
BOOST_CHECK_EQUAL(
std::string(expected_xor.begin(), expected_xor.end()),
std::string(ds.begin(), ds.end()));
}
BOOST_AUTO_TEST_SUITE_END()

27
src/txdb.cpp
View file

@ -29,18 +29,8 @@ static const char DB_REINDEX_FLAG = 'R';
static const char DB_LAST_BLOCK = 'l';
void static BatchWriteCoins(CLevelDBBatch &batch, const uint256 &hash, const CCoins &coins) {
if (coins.IsPruned())
batch.Erase(make_pair(DB_COINS, hash));
else
batch.Write(make_pair(DB_COINS, hash), coins);
}
void static BatchWriteHashBestChain(CLevelDBBatch &batch, const uint256 &hash) {
batch.Write(DB_BEST_BLOCK, hash);
}
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe) {
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true)
{
}
bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const {
@ -59,12 +49,15 @@ uint256 CCoinsViewDB::GetBestBlock() const {
}
bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
CLevelDBBatch batch;
CLevelDBBatch batch(db.GetObfuscateKey());
size_t count = 0;
size_t changed = 0;
for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
if (it->second.flags & CCoinsCacheEntry::DIRTY) {
BatchWriteCoins(batch, it->first, it->second.coins);
if (it->second.coins.IsPruned())
batch.Erase(make_pair(DB_COINS, it->first));
else
batch.Write(make_pair(DB_COINS, it->first), it->second.coins);
changed++;
}
count++;
@ -72,7 +65,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
mapCoins.erase(itOld);
}
if (!hashBlock.IsNull())
BatchWriteHashBestChain(batch, hashBlock);
batch.Write(DB_BEST_BLOCK, hashBlock);
LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
return db.WriteBatch(batch);
@ -158,7 +151,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
}
bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
CLevelDBBatch batch;
CLevelDBBatch batch(GetObfuscateKey());
for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second);
}
@ -174,7 +167,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) {
}
bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) {
CLevelDBBatch batch;
CLevelDBBatch batch(GetObfuscateKey());
for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++)
batch.Write(make_pair(DB_TXINDEX, it->first), it->second);
return WriteBatch(batch);