remove leveldb, made tests pass

error on Ctrl+C during reindex

src/Makefile.am

@@ -19,7 +19,7 @@ else
 LIBUNIVALUE = $(UNIVALUE_LIBS)
 endif
 
-BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(ICU_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(ICU_CFLAGS)
+BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(ICU_CPPFLAGS) $(BOOST_CPPFLAGS) $(CRYPTO_CFLAGS) $(ICU_CFLAGS)
 
 BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include
 BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS)
@@ -122,8 +122,6 @@ BITCOIN_CORE_H = \
   fs.h \
   httprpc.h \
   httpserver.h \
-  index/base.h \
-  index/txindex.h \
   indirectmap.h \
   init.h \
   interfaces/handler.h \
@@ -133,7 +131,6 @@ BITCOIN_CORE_H = \
   key_io.h \
   keystore.h \
   lbry.h \
-  dbwrapper.h \
   limitedmap.h \
   logging.h \
   memusage.h \
@@ -235,10 +232,7 @@ libbitcoin_server_a_SOURCES = \
   consensus/tx_verify.cpp \
   httprpc.cpp \
   httpserver.cpp \
-  index/base.cpp \
-  index/txindex.cpp \
   init.cpp \
-  dbwrapper.cpp \
   lbry.cpp \
   merkleblock.cpp \
   miner.cpp \
@@ -417,7 +411,7 @@ claimtrie_libclaimtrie_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
 claimtrie_libclaimtrie_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 claimtrie_libclaimtrie_a_CFLAGS = $(PIE_FLAGS)
 claimtrie_libclaimtrie_a_SOURCES = \
-  claimtrie/sqlite/sqlite3.c \
+  sqlite/sqlite3.c \
   claimtrie/data.cpp \
   claimtrie/forks.cpp \
   claimtrie/hashes.cpp \
@@ -488,8 +482,6 @@ lbrycrdd_LDADD = \
   $(LIBBITCOIN_ZMQ) \
   $(LIBBITCOIN_CONSENSUS) \
   $(LIBBITCOIN_CRYPTO) \
-  $(LIBLEVELDB) \
-  $(LIBLEVELDB_SSE42) \
   $(LIBMEMENV) \
   $(LIBSECP256K1)
 
@@ -587,7 +579,6 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in:  $(am__configure_deps)
 clean-local:
 	-$(MAKE) -C secp256k1 clean
 	-$(MAKE) -C univalue clean
-	-rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno
 	-rm -f config.h
 	-rm -rf test/__pycache__
 
@@ -612,10 +603,6 @@ endif
 	@test -f $(PROTOC)
 	$(AM_V_GEN) $(PROTOC) --cpp_out=$(@D) --proto_path=$(<D) $<
 
-if EMBEDDED_LEVELDB
-include Makefile.leveldb.include
-endif
-
 if ENABLE_TESTS
 include Makefile.test.include
 endif

src/Makefile.bench.include

@@ -41,8 +41,6 @@ bench_bench_bitcoin_LDADD = \
   $(LIBBITCOIN_UTIL) \
   $(LIBBITCOIN_CONSENSUS) \
   $(LIBBITCOIN_CRYPTO) \
-  $(LIBLEVELDB) \
-  $(LIBLEVELDB_SSE42) \
   $(LIBMEMENV) \
   $(LIBSECP256K1) \
   $(LIBUNIVALUE)

src/Makefile.leveldb.include

@@ -1,149 +0,0 @@
-# Copyright (c) 2016 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-LIBLEVELDB_INT = leveldb/libleveldb.a
-LIBMEMENV_INT  = leveldb/libmemenv.a
-LIBLEVELDB_SSE42_INT  = leveldb/libleveldb_sse42.a
-
-EXTRA_LIBRARIES += $(LIBLEVELDB_INT)
-EXTRA_LIBRARIES += $(LIBMEMENV_INT)
-EXTRA_LIBRARIES += $(LIBLEVELDB_SSE42_INT)
-
-LIBLEVELDB += $(LIBLEVELDB_INT)
-LIBMEMENV += $(LIBMEMENV_INT)
-LIBLEVELDB_SSE42 = $(LIBLEVELDB_SSE42_INT)
-
-LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include
-LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv
-
-LEVELDB_CPPFLAGS_INT =
-LEVELDB_CPPFLAGS_INT += -I$(srcdir)/leveldb
-LEVELDB_CPPFLAGS_INT += $(LEVELDB_TARGET_FLAGS)
-LEVELDB_CPPFLAGS_INT += -DLEVELDB_ATOMIC_PRESENT
-LEVELDB_CPPFLAGS_INT += -D__STDC_LIMIT_MACROS
-
-if TARGET_WINDOWS
-LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1
-else
-LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX
-endif
-
-leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS)
-leveldb_libleveldb_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
-
-leveldb_libleveldb_a_SOURCES=
-leveldb_libleveldb_a_SOURCES += leveldb/port/atomic_pointer.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_example.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/win/stdint.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/port.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/thread_annotations.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/db.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/options.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/comparator.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/filter_policy.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/slice.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table_builder.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/env.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/c.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/iterator.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/cache.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/dumpfile.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/write_batch.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/status.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_format.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch_internal.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/filename.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/builder.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/skiplist.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/snapshot.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/block.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/merger.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/format.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/iterator_wrapper.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix_test_helper.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/arena.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/random.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/posix_logger.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/hash.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/coding.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/testutil.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/mutexlock.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/logging.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/testharness.h
-
-leveldb_libleveldb_a_SOURCES += leveldb/db/builder.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/c.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/dumpfile.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/filename.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/repair.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/block.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/format.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/iterator.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/merger.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/table_builder.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/table.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/arena.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/bloom.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/cache.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/coding.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/comparator.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/env.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/filter_policy.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/hash.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/logging.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/options.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/status.cc
-
-if TARGET_WINDOWS
-leveldb_libleveldb_a_SOURCES += leveldb/util/env_win.cc
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.cc
-else
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.cc
-endif
-
-leveldb_libmemenv_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS)
-leveldb_libmemenv_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS)
-leveldb_libmemenv_a_SOURCES =  leveldb/helpers/memenv/memenv.cc
-leveldb_libmemenv_a_SOURCES += leveldb/helpers/memenv/memenv.h
-
-leveldb_libleveldb_sse42_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS)
-leveldb_libleveldb_sse42_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS)
-if ENABLE_HWCRC32
-leveldb_libleveldb_sse42_a_CPPFLAGS += -DLEVELDB_PLATFORM_POSIX_SSE
-leveldb_libleveldb_sse42_a_CXXFLAGS += $(SSE42_CXXFLAGS)
-endif
-leveldb_libleveldb_sse42_a_SOURCES =  leveldb/port/port_posix_sse.cc

src/Makefile.qt.include

@@ -408,7 +408,7 @@ endif
 if ENABLE_ZMQ
 qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
 endif
-qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \
+qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBMEMENV) \
   $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(ICU_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
   $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
 qt_lbrycrd_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)

src/Makefile.qttest.include

@@ -62,8 +62,8 @@ endif
 if ENABLE_ZMQ
 qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
 endif
-qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \
+qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
-  $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
+  $(LIBMEMENV) $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
   $(QR_LIBS) $(PROTOBUF_LIBS) $(ICU_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
   $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
 qt_test_test_lbrycrd_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)

src/Makefile.test.include

@@ -56,7 +56,6 @@ BITCOIN_TESTS =\
   test/key_io_tests.cpp \
   test/key_tests.cpp \
   test/limitedmap_tests.cpp \
-  test/dbwrapper_tests.cpp \
   test/main_tests.cpp \
   test/mempool_tests.cpp \
   test/merkle_tests.cpp \
@@ -95,7 +94,6 @@ BITCOIN_TESTS =\
   test/timedata_tests.cpp \
   test/torcontrol_tests.cpp \
   test/transaction_tests.cpp \
-  test/txindex_tests.cpp \
   test/txvalidation_tests.cpp \
   test/txvalidationcache_tests.cpp \
   test/uint256_tests.cpp \
@@ -126,7 +124,7 @@ test_test_lbrycrd_LDADD += $(LIBBITCOIN_WALLET)
 endif
 
 test_test_lbrycrd_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
-  $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
+  $(LIBMEMENV) $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
 test_test_lbrycrd_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 
 test_test_lbrycrd_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS)

src/chain.h

@@ -376,73 +376,6 @@ int64_t GetBlockProofEquivalentTime(const CBlockIndex& to, const CBlockIndex& fr
 const CBlockIndex* LastCommonAncestor(const CBlockIndex* pa, const CBlockIndex* pb);
 
 
-/** Used to marshal pointers into hashes for db storage. */
-class CDiskBlockIndex : public CBlockIndex
-{
-public:
-    uint256 hashPrev;
-
-    CDiskBlockIndex() {
-        hashPrev = uint256();
-    }
-
-    explicit CDiskBlockIndex(const CBlockIndex* pindex) : CBlockIndex(*pindex) {
-        hashPrev = (pprev ? pprev->GetBlockHash() : uint256());
-    }
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        int _nVersion = s.GetVersion();
-        if (!(s.GetType() & SER_GETHASH))
-            READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));
-
-        READWRITE(VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED));
-        READWRITE(VARINT(nStatus));
-        READWRITE(VARINT(nTx));
-        if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO))
-            READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
-        if (nStatus & BLOCK_HAVE_DATA)
-            READWRITE(VARINT(nDataPos));
-        if (nStatus & BLOCK_HAVE_UNDO)
-            READWRITE(VARINT(nUndoPos));
-
-        // block header
-        READWRITE(this->nVersion);
-        READWRITE(hashPrev);
-        READWRITE(hashMerkleRoot);
-        READWRITE(hashClaimTrie);
-        READWRITE(nTime);
-        READWRITE(nBits);
-        READWRITE(nNonce);
-    }
-
-    uint256 GetBlockHash() const
-    {
-        CBlockHeader block;
-        block.nVersion        = nVersion;
-        block.hashPrevBlock   = hashPrev;
-        block.hashMerkleRoot  = hashMerkleRoot;
-        block.hashClaimTrie   = hashClaimTrie;
-        block.nTime           = nTime;
-        block.nBits           = nBits;
-        block.nNonce          = nNonce;
-        return block.GetHash();
-    }
-
-    std::string ToString() const
-    {
-        std::string str = "CDiskBlockIndex(";
-        str += CBlockIndex::ToString();
-        str += strprintf("\n                hashBlock=%s, hashClaimTrie=%s, hashPrev=%s)",
-            GetBlockHash().ToString(),
-            hashClaimTrie.ToString(),
-            hashPrev.ToString());
-        return str;
-    }
-};
-
 /** An in-memory indexed chain of blocks. */
 class CChain {
 private:

src/claimtrie/CMakeLists.txt

@@ -15,7 +15,7 @@ set(CLAIMTRIE_SRC
     trie.cpp
     txoutpoint.cpp
     uints.cpp
-    sqlite/sqlite3.c
+    ../sqlite/sqlite3.c
 )
 
 if(BIND)

src/claimtrie/trie.cpp

@@ -2,6 +2,7 @@
 #include <forks.h>
 #include <hashes.h>
 #include <log.h>
+#include <sqlite.h>
 #include <trie.h>
 
 #include <algorithm>
@@ -47,7 +48,7 @@ void applyPragmas(sqlite::database& db, std::size_t cache)
     db << "PRAGMA case_sensitive_like=true";
 }
 
-CClaimTrie::CClaimTrie(bool fWipe, int height,
+CClaimTrie::CClaimTrie(int64_t cacheBytes, bool fWipe, int height,
                        const std::string& dataDir,
                        int nNormalizedNameForkHeight,
                        int64_t nOriginalClaimExpirationTime,

src/claimtrie/trie.h

@@ -27,7 +27,7 @@ public:
     CClaimTrie() = delete;
     CClaimTrie(CClaimTrie&&) = delete;
     CClaimTrie(const CClaimTrie&) = delete;
-    CClaimTrie(bool fWipe, int height = 0,
+    CClaimTrie(int64_t cacheBytes, bool fWipe, int height = 0,
                const std::string& dataDir = ".",
                int nNormalizedNameForkHeight = 1,
                int64_t nOriginalClaimExpirationTime = 1,

src/coins.cpp

@@ -10,7 +10,7 @@
 bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; }
 uint256 CCoinsView::GetBestBlock() const { return uint256(); }
 std::vector<uint256> CCoinsView::GetHeadBlocks() const { return std::vector<uint256>(); }
-bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; }
+bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync) { return false; }
 CCoinsViewCursor *CCoinsView::Cursor() const { return nullptr; }
 
 bool CCoinsView::HaveCoin(const COutPoint &outpoint) const
@@ -25,7 +25,7 @@ bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->
 uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); }
 std::vector<uint256> CCoinsViewBacked::GetHeadBlocks() const { return base->GetHeadBlocks(); }
 void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; }
-bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); }
+bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync) { return base->BatchWrite(mapCoins, hashBlock, sync); }
 CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); }
 size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); }
 
@@ -142,7 +142,7 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) {
     hashBlock = hashBlockIn;
 }
 
-bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) {
+bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn, bool sync) {
     for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); it = mapCoins.erase(it)) {
         // Ignore non-dirty entries (optimization).
         if (!(it->second.flags & CCoinsCacheEntry::DIRTY)) {
@@ -200,8 +200,8 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn
     return true;
 }
 
-bool CCoinsViewCache::Flush() {
+bool CCoinsViewCache::Flush(bool sync) {
-    bool fOk = base->BatchWrite(cacheCoins, hashBlock);
+    bool fOk = base->BatchWrite(cacheCoins, hashBlock, sync);
     cacheCoins.clear();
     cachedCoinsUsage = 0;
     return fOk;

src/coins.h

@@ -124,21 +124,19 @@ typedef std::unordered_map<COutPoint, CCoinsCacheEntry, SaltedOutpointHasher> CC
 /** Cursor for iterating over CoinsView state */
 class CCoinsViewCursor
 {
+    uint256 hashBlock;
 public:
     CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {}
-    virtual ~CCoinsViewCursor() {}
+    virtual ~CCoinsViewCursor() noexcept {}
 
     virtual bool GetKey(COutPoint &key) const = 0;
     virtual bool GetValue(Coin &coin) const = 0;
-    virtual unsigned int GetValueSize() const = 0;
 
     virtual bool Valid() const = 0;
     virtual void Next() = 0;
 
     //! Get best block at the time this cursor was created
     const uint256 &GetBestBlock() const { return hashBlock; }
-private:
-    uint256 hashBlock;
 };
 
 /** Abstract view on the open txout dataset. */
@@ -165,7 +163,7 @@ public:
 
     //! Do a bulk modification (multiple Coin changes + BestBlock change).
     //! The passed mapCoins can be modified.
-    virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock);
+    virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync);
 
     //! Get a cursor to iterate over the whole state
     virtual CCoinsViewCursor *Cursor() const;
@@ -191,7 +189,7 @@ public:
     uint256 GetBestBlock() const override;
     std::vector<uint256> GetHeadBlocks() const override;
     void SetBackend(CCoinsView &viewIn);
-    bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override;
+    bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync) override;
     CCoinsViewCursor *Cursor() const override;
     size_t EstimateSize() const override;
 };
@@ -224,7 +222,7 @@ public:
     bool HaveCoin(const COutPoint &outpoint) const override;
     uint256 GetBestBlock() const override;
     void SetBestBlock(const uint256 &hashBlock);
-    bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override;
+    bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync) override;
     CCoinsViewCursor* Cursor() const override {
         throw std::logic_error("CCoinsViewCache cursor iteration not supported.");
     }
@@ -266,7 +264,7 @@ public:
      * Failure to call this method before destruction will cause the changes to be forgotten.
      * If false is returned, the state of this cache (and its backing view) will be undefined.
      */
-    bool Flush();
+    bool Flush(bool sync = false);
 
     /**
      * Removes the UTXO with the given outpoint from the cache, if it is

src/dbwrapper.cpp

@@ -1,267 +0,0 @@
-// Copyright (c) 2012-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#include <dbwrapper.h>
-
-#include <memory>
-#include <random.h>
-
-#include <leveldb/cache.h>
-#include <leveldb/env.h>
-#include <leveldb/filter_policy.h>
-#include <memenv.h>
-#include <stdint.h>
-#include <algorithm>
-
-class CBitcoinLevelDBLogger : public leveldb::Logger {
-public:
-    // This code is adapted from posix_logger.h, which is why it is using vsprintf.
-    // Please do not do this in normal code
-    void Logv(const char * format, va_list ap) override {
-            if (!LogAcceptCategory(BCLog::LEVELDB)) {
-                return;
-            }
-            char buffer[500];
-            for (int iter = 0; iter < 2; iter++) {
-                char* base;
-                int bufsize;
-                if (iter == 0) {
-                    bufsize = sizeof(buffer);
-                    base = buffer;
-                }
-                else {
-                    bufsize = 30000;
-                    base = new char[bufsize];
-                }
-                char* p = base;
-                char* limit = base + bufsize;
-
-                // Print the message
-                if (p < limit) {
-                    va_list backup_ap;
-                    va_copy(backup_ap, ap);
-                    // Do not use vsnprintf elsewhere in bitcoin source code, see above.
-                    p += vsnprintf(p, limit - p, format, backup_ap);
-                    va_end(backup_ap);
-                }
-
-                // Truncate to available space if necessary
-                if (p >= limit) {
-                    if (iter == 0) {
-                        continue;       // Try again with larger buffer
-                    }
-                    else {
-                        p = limit - 1;
-                    }
-                }
-
-                // Add newline if necessary
-                if (p == base || p[-1] != '\n') {
-                    *p++ = '\n';
-                }
-
-                assert(p <= limit);
-                base[std::min(bufsize - 1, (int)(p - base))] = '\0';
-                LogPrintf("leveldb: %s", base);  /* Continued */
-                if (base != buffer) {
-                    delete[] base;
-                }
-                break;
-            }
-    }
-};
-
-static void SetMaxOpenFiles(leveldb::Options *options) {
-    // On most platforms the default setting of max_open_files (which is 1000)
-    // is optimal. On Windows using a large file count is OK because the handles
-    // do not interfere with select() loops. On 64-bit Unix hosts this value is
-    // also OK, because up to that amount LevelDB will use an mmap
-    // implementation that does not use extra file descriptors (the fds are
-    // closed after being mmaped).
-    //
-    // Increasing the value beyond the nmap count is dangerous because LevelDB will
-    // fall back to a non-mmap implementation when the file count is too large (thus contending select()).
-    // On 32-bit Unix host we should decrease the value because the handles use
-    // up real fds, and we want to avoid fd exhaustion issues.
-    //
-    // See PR #12495 for further discussion.
-
-    int default_open_files = 400;
-#ifndef WIN32
-    if (sizeof(void*) < 8) {
-        options->max_open_files = 64;
-    }
-#endif
-    LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
-             options->max_open_files, default_open_files);
-}
-
-static leveldb::Options GetOptions(size_t nCacheSize)
-{
-    leveldb::Options options;
-    auto write_cache = std::min(nCacheSize / 4, size_t(32 * 1024 * 1024)); // cap write_cache
-    options.block_cache = leveldb::NewLRUCache(nCacheSize - write_cache * 2);
-    options.write_buffer_size = write_cache; // up to two write buffers may be held in memory simultaneously
-    options.filter_policy = leveldb::NewBloomFilterPolicy(12);
-    options.compression = leveldb::kNoCompression;
-    options.info_log = new CBitcoinLevelDBLogger();
-    if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
-        // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
-        // on corruption in later versions.
-        options.paranoid_checks = true;
-    }
-    SetMaxOpenFiles(&options);
-    return options;
-}
-
-CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
-        : m_name(fs::basename(path)), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION)
-{
-    penv = nullptr;
-    readoptions.verify_checksums = true;
-    iteroptions.verify_checksums = true;
-    iteroptions.fill_cache = false;
-    syncoptions.sync = true;
-    options = GetOptions(nCacheSize);
-    options.create_if_missing = true;
-    if (fMemory) {
-        penv = leveldb::NewMemEnv(leveldb::Env::Default());
-        options.env = penv;
-    } else {
-        if (fWipe) {
-            LogPrintf("Wiping LevelDB in %s\n", path.string());
-            leveldb::Status result = leveldb::DestroyDB(path.string(), options);
-            dbwrapper_private::HandleError(result);
-        }
-        TryCreateDirectories(path);
-        LogPrintf("Opening LevelDB in %s\n", path.string());
-    }
-    leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb);
-    dbwrapper_private::HandleError(status);
-    LogPrintf("Opened LevelDB successfully\n");
-
-    if (gArgs.GetBoolArg("-forcecompactdb", false)) {
-        LogPrintf("Starting database compaction of %s\n", path.string());
-        pdb->CompactRange(nullptr, nullptr);
-        LogPrintf("Finished database compaction of %s\n", path.string());
-    }
-
-    // The base-case obfuscation key, which is a noop.
-    obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
-
-    bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
-
-    if (!key_exists && obfuscate && IsEmpty()) {
-        // Initialize non-degenerate obfuscation if it won't upset
-        // existing, non-obfuscated data.
-        std::vector<unsigned char> new_key = CreateObfuscateKey();
-
-        // Write `new_key` so we don't obfuscate the key with itself
-        Write(OBFUSCATE_KEY_KEY, new_key);
-        obfuscate_key = new_key;
-
-        LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), HexStr(obfuscate_key));
-    }
-
-    LogPrintf("Using obfuscation key for %s: %s\n", path.string(), HexStr(obfuscate_key));
-}
-
-CDBWrapper::~CDBWrapper()
-{
-    delete pdb;
-    pdb = nullptr;
-    delete options.filter_policy;
-    options.filter_policy = nullptr;
-    delete options.info_log;
-    options.info_log = nullptr;
-    delete options.block_cache;
-    options.block_cache = nullptr;
-    delete penv;
-    options.env = nullptr;
-}
-
-bool CDBWrapper::Sync() {
-    CDBBatch batch(*this);
-    return WriteBatch(batch, true);
-}
-
-bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
-{
-    if (!pdb)
-        return false;
-
-    const bool log_memory = LogAcceptCategory(BCLog::LEVELDB);
-    double mem_before = 0;
-    if (log_memory) {
-        mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
-    }
-    leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
-    dbwrapper_private::HandleError(status);
-    if (log_memory) {
-        double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
-        LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
-                 m_name, mem_before, mem_after);
-    }
-    return true;
-}
-
-size_t CDBWrapper::DynamicMemoryUsage() const {
-    std::string memory;
-    if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory)) {
-        LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
-        return 0;
-    }
-    return stoul(memory);
-}
-
-// Prefixed with null character to avoid collisions with other keys
-//
-// We must use a string constructor which specifies length so that we copy
-// past the null-terminator.
-const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
-
-const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
-
-/**
- * Returns a string (consisting of 8 random bytes) suitable for use as an
- * obfuscating XOR key.
- */
-std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
-{
-    unsigned char buff[OBFUSCATE_KEY_NUM_BYTES];
-    GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES);
-    return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]);
-
-}
-
-bool CDBWrapper::IsEmpty()
-{
-    std::unique_ptr<CDBIterator> it(NewIterator());
-    it->SeekToFirst();
-    return !(it->Valid());
-}
-
-CDBIterator::~CDBIterator() { delete piter; }
-bool CDBIterator::Valid() const { return piter->Valid(); }
-void CDBIterator::SeekToFirst() { piter->SeekToFirst(); }
-void CDBIterator::Next() { piter->Next(); }
-
-namespace dbwrapper_private {
-
-void HandleError(const leveldb::Status& status)
-{
-    if (status.ok())
-        return;
-    const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
-    LogPrintf("%s\n", errmsg);
-    LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
-    throw dbwrapper_error(errmsg);
-}
-
-const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
-{
-    return w.obfuscate_key;
-}
-
-} // namespace dbwrapper_private

src/dbwrapper.h

@@ -1,350 +0,0 @@
-// Copyright (c) 2012-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#ifndef BITCOIN_DBWRAPPER_H
-#define BITCOIN_DBWRAPPER_H
-
-#include <clientversion.h>
-#include <fs.h>
-#include <serialize.h>
-#include <streams.h>
-#include <util.h>
-#include <utilstrencodings.h>
-#include <version.h>
-
-#include <leveldb/db.h>
-#include <leveldb/write_batch.h>
-
-class dbwrapper_error : public std::runtime_error
-{
-public:
-    explicit dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {}
-};
-
-class CDBWrapper;
-
-/** These should be considered an implementation detail of the specific database.
- */
-namespace dbwrapper_private {
-
-/** Handle database error by throwing dbwrapper_error exception.
- */
-    void HandleError(const leveldb::Status& status);
-
-/** Work around circular dependency, as well as for testing in dbwrapper_tests.
- * Database obfuscation should be considered an implementation detail of the
- * specific database.
- */
-    const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
-
-};
-
-class CDBIterator
-{
-private:
-    const CDBWrapper &parent;
-    leveldb::Iterator *piter;
-
-public:
-
-    /**
-     * @param[in] _parent          Parent CDBWrapper instance.
-     * @param[in] _piter           The original leveldb iterator.
-     */
-    CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) :
-            parent(_parent), piter(_piter) { };
-    ~CDBIterator();
-
-    bool Valid() const;
-
-    void SeekToFirst();
-
-    template<typename K> void Seek(const K& key) {
-        CDataStream ssKey(SER_DISK, CLIENT_VERSION);
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-        piter->Seek(slKey);
-    }
-
-    void Next();
-
-    template<typename K> bool GetKey(K& key) {
-        leveldb::Slice slKey = piter->key();
-        try {
-            CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION);
-            ssKey >> key;
-        } catch (const std::exception&) {
-            return false;
-        }
-        return true;
-    }
-
-    template<typename V> bool GetValue(V& value) {
-        leveldb::Slice slValue = piter->value();
-        try {
-            CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION);
-            ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
-            ssValue >> value;
-        } catch (const std::exception&) {
-            return false;
-        }
-        return true;
-    }
-
-    unsigned int GetValueSize() {
-        return piter->value().size();
-    }
-
-};
-
-class CDBBatch;
-
-class CDBWrapper
-{
-    friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
-private:
-    //! custom environment this database is using (may be nullptr in case of default environment)
-    leveldb::Env* penv;
-
-    //! database options used
-    leveldb::Options options;
-
-    //! options used when reading from the database
-    leveldb::ReadOptions readoptions;
-
-    //! options used when iterating over values of the database
-    leveldb::ReadOptions iteroptions;
-
-    //! options used when writing to the database
-    leveldb::WriteOptions writeoptions;
-
-    //! options used when sync writing to the database
-    leveldb::WriteOptions syncoptions;
-
-    //! the database itself
-    leveldb::DB* pdb;
-
-    //! the name of this database
-    std::string m_name;
-
-    //! a key used for optional XOR-obfuscation of the database
-    std::vector<unsigned char> obfuscate_key;
-
-    //! the key under which the obfuscation key is stored
-    static const std::string OBFUSCATE_KEY_KEY;
-
-    //! the length of the obfuscate key in number of bytes
-    static const unsigned int OBFUSCATE_KEY_NUM_BYTES;
-
-    std::vector<unsigned char> CreateObfuscateKey() const;
-
-public:
-    mutable CDataStream ssKey, ssValue;
-
-    /**
-     * @param[in] path        Location in the filesystem where leveldb data will be stored.
-     * @param[in] nCacheSize  Configures various leveldb cache settings.
-     * @param[in] fMemory     If true, use leveldb's memory environment.
-     * @param[in] fWipe       If true, remove all existing data.
-     * @param[in] obfuscate   If true, store data obfuscated via simple XOR. If false, XOR
-     *                        with a zero'd byte array.
-     */
-    CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
-    virtual ~CDBWrapper();
-
-    CDBWrapper(const CDBWrapper&) = delete;
-    /* CDBWrapper& operator=(const CDBWrapper&) = delete; */
-
-    template <typename K, typename V>
-    bool Read(const K& key, V& value) const
-    {
-        assert(ssKey.empty());
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-
-        std::string strValue;
-        leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
-        ssKey.clear();
-        if (!status.ok()) {
-            if (status.IsNotFound())
-                return false;
-            LogPrintf("LevelDB read failure: %s\n", status.ToString());
-            dbwrapper_private::HandleError(status);
-        }
-        try {
-            CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION);
-            ssValue.Xor(obfuscate_key);
-            ssValue >> value;
-        } catch (const std::exception&) {
-            return false;
-        }
-        return true;
-    }
-
-    template <typename K, typename V>
-    bool Write(const K& key, const V& value, bool fSync = false);
-
-    template <typename K>
-    bool Exists(const K& key) const
-    {
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-
-        std::string strValue;
-        leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
-        ssKey.clear();
-        if (!status.ok()) {
-            if (status.IsNotFound())
-                return false;
-            LogPrintf("LevelDB read failure: %s\n", status.ToString());
-            dbwrapper_private::HandleError(status);
-        }
-        return true;
-    }
-
-    template <typename K>
-    bool Erase(const K& key, bool fSync = false);
-
-    bool WriteBatch(CDBBatch& batch, bool fSync = false);
-
-    // Get an estimate of LevelDB memory usage (in bytes).
-    size_t DynamicMemoryUsage() const;
-
-    // not available for LevelDB; provide for compatibility with BDB
-    bool Flush()
-    {
-        return true;
-    }
-
-    bool Sync();
-
-    CDBIterator *NewIterator()
-    {
-        return new CDBIterator(*this, pdb->NewIterator(iteroptions));
-    }
-
-    /**
-     * Return true if the database managed by this class contains no entries.
-     */
-    bool IsEmpty();
-
-    template<typename K>
-    size_t EstimateSize(const K& key_begin, const K& key_end) const
-    {
-        CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
-        ssKey1.reserve(ssKey.capacity());
-        ssKey2.reserve(ssKey.capacity());
-        ssKey1 << key_begin;
-        ssKey2 << key_end;
-        leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
-        leveldb::Slice slKey2(ssKey2.data(), ssKey2.size());
-        uint64_t size = 0;
-        leveldb::Range range(slKey1, slKey2);
-        pdb->GetApproximateSizes(&range, 1, &size);
-        return size;
-    }
-
-    /**
-     * Compact a certain range of keys in the database.
-     */
-    template<typename K>
-    void CompactRange(const K& key_begin, const K& key_end) const
-    {
-        CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
-        ssKey1.reserve(ssKey.capacity());
-        ssKey2.reserve(ssKey.capacity());
-        ssKey1 << key_begin;
-        ssKey2 << key_end;
-        leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
-        leveldb::Slice slKey2(ssKey2.data(), ssKey2.size());
-        pdb->CompactRange(&slKey1, &slKey2);
-    }
-
-};
-
-/** Batch of changes queued to be written to a CDBWrapper */
-class CDBBatch
-{
-    friend class CDBWrapper;
-    const CDBWrapper &parent;
-    leveldb::WriteBatch batch;
-
-    size_t size_estimate;
-    CDataStream ssKey, ssValue;
-
-public:
-    /**
-     * @param[in] _parent   CDBWrapper that this batch is to be submitted to
-     */
-    explicit CDBBatch(const CDBWrapper &_parent) : parent(_parent), size_estimate(0),
-        ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) {
-        ssKey.reserve(parent.ssKey.capacity());
-        ssValue.reserve(parent.ssValue.capacity());
-    };
-
-    void Clear()
-    {
-        batch.Clear();
-        size_estimate = 0;
-    }
-
-    template <typename K, typename V>
-    void Write(const K& key, const V& value)
-    {
-        assert(ssKey.empty());
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-
-        assert(ssValue.empty());
-        ssValue << value;
-        ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
-        leveldb::Slice slValue(ssValue.data(), ssValue.size());
-
-        batch.Put(slKey, slValue);
-        // LevelDB serializes writes as:
-        // - byte: header
-        // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
-        // - byte[]: key
-        // - varint: value length
-        // - byte[]: value
-        // The formula below assumes the key and value are both less than 16k.
-        size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
-        ssKey.clear();
-        ssValue.clear();
-    }
-
-    template <typename K>
-    void Erase(const K& key)
-    {
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-
-        batch.Delete(slKey);
-        // LevelDB serializes erases as:
-        // - byte: header
-        // - varint: key length
-        // - byte[]: key
-        // The formula below assumes the key is less than 16kB.
-        size_estimate += 2 + (slKey.size() > 127) + slKey.size();
-        ssKey.clear();
-    }
-
-    size_t SizeEstimate() const { return size_estimate; }
-};
-
-template<typename K>
-bool CDBWrapper::Erase(const K &key, bool fSync) {
-    CDBBatch batch(*this);
-    batch.Erase(key);
-    return WriteBatch(batch, fSync);
-}
-
-template<typename K, typename V>
-bool CDBWrapper::Write(const K &key, const V &value, bool fSync) {
-    CDBBatch batch(*this);
-    batch.Write(key, value);
-    return WriteBatch(batch, fSync);
-}
-
-#endif // BITCOIN_DBWRAPPER_H

src/index/base.cpp

@@ -1,278 +0,0 @@
-// Copyright (c) 2017-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#include <chainparams.h>
-#include <index/base.h>
-#include <shutdown.h>
-#include <tinyformat.h>
-#include <ui_interface.h>
-#include <util.h>
-#include <validation.h>
-#include <warnings.h>
-
-constexpr char DB_BEST_BLOCK = 'B';
-
-constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds
-constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds
-
-template<typename... Args>
-static void FatalError(const char* fmt, const Args&... args)
-{
-    std::string strMessage = tfm::format(fmt, args...);
-    SetMiscWarning(strMessage);
-    LogPrintf("*** %s\n", strMessage);
-    uiInterface.ThreadSafeMessageBox(
-        "Error: A fatal internal error occurred, see debug.log for details",
-        "", CClientUIInterface::MSG_ERROR);
-    StartShutdown();
-}
-
-BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) :
-    CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate)
-{}
-
-bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const
-{
-    bool success = Read(DB_BEST_BLOCK, locator);
-    if (!success) {
-        locator.SetNull();
-    }
-    return success;
-}
-
-bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator)
-{
-    return Write(DB_BEST_BLOCK, locator);
-}
-
-BaseIndex::~BaseIndex()
-{
-    Interrupt();
-    Stop();
-}
-
-bool BaseIndex::Init()
-{
-    CBlockLocator locator;
-    if (!GetDB().ReadBestBlock(locator)) {
-        locator.SetNull();
-    }
-
-    LOCK(cs_main);
-    m_best_block_index = FindForkInGlobalIndex(chainActive, locator);
-    m_synced = m_best_block_index.load() == chainActive.Tip();
-    return true;
-}
-
-static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev)
-{
-    AssertLockHeld(cs_main);
-
-    if (!pindex_prev) {
-        return chainActive.Genesis();
-    }
-
-    const CBlockIndex* pindex = chainActive.Next(pindex_prev);
-    if (pindex) {
-        return pindex;
-    }
-
-    return chainActive.Next(chainActive.FindFork(pindex_prev));
-}
-
-void BaseIndex::ThreadSync()
-{
-    const CBlockIndex* pindex = m_best_block_index.load();
-    if (!m_synced) {
-        auto& consensus_params = Params().GetConsensus();
-
-        int64_t last_log_time = 0;
-        int64_t last_locator_write_time = 0;
-        while (true) {
-            if (m_interrupt) {
-                WriteBestBlock(pindex);
-                return;
-            }
-
-            {
-                LOCK(cs_main);
-                const CBlockIndex* pindex_next = NextSyncBlock(pindex);
-                if (!pindex_next) {
-                    WriteBestBlock(pindex);
-                    m_best_block_index = pindex;
-                    m_synced = true;
-                    break;
-                }
-                pindex = pindex_next;
-            }
-
-            int64_t current_time = GetTime();
-            if (last_log_time + SYNC_LOG_INTERVAL < current_time) {
-                LogPrintf("Syncing %s with block chain from height %d\n",
-                          GetName(), pindex->nHeight);
-                last_log_time = current_time;
-            }
-
-            if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
-                WriteBestBlock(pindex);
-                last_locator_write_time = current_time;
-            }
-
-            CBlock block;
-            if (!ReadBlockFromDisk(block, pindex, consensus_params)) {
-                FatalError("%s: Failed to read block %s from disk",
-                           __func__, pindex->GetBlockHash().ToString());
-                return;
-            }
-            if (!WriteBlock(block, pindex)) {
-                FatalError("%s: Failed to write block %s to index database",
-                           __func__, pindex->GetBlockHash().ToString());
-                return;
-            }
-        }
-    }
-
-    if (pindex) {
-        LogPrintf("%s is enabled at height %d\n", GetName(), pindex->nHeight);
-    } else {
-        LogPrintf("%s is enabled\n", GetName());
-    }
-}
-
-bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index)
-{
-    LOCK(cs_main);
-    if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) {
-        return error("%s: Failed to write locator to disk", __func__);
-    }
-    return true;
-}
-
-void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
-                               const std::vector<CTransactionRef>& txn_conflicted)
-{
-    if (!m_synced) {
-        return;
-    }
-
-    const CBlockIndex* best_block_index = m_best_block_index.load();
-    if (!best_block_index) {
-        if (pindex->nHeight != 0) {
-            FatalError("%s: First block connected is not the genesis block (height=%d)",
-                       __func__, pindex->nHeight);
-            return;
-        }
-    } else {
-        // Ensure block connects to an ancestor of the current best block. This should be the case
-        // most of the time, but may not be immediately after the sync thread catches up and sets
-        // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are
-        // in the ValidationInterface queue backlog even after the sync thread has caught up to the
-        // new chain tip. In this unlikely event, log a warning and let the queue clear.
-        if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) {
-            LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */
-                      "known best chain (tip=%s); not updating index\n",
-                      __func__, pindex->GetBlockHash().ToString(),
-                      best_block_index->GetBlockHash().ToString());
-            return;
-        }
-    }
-
-    if (WriteBlock(*block, pindex)) {
-        m_best_block_index = pindex;
-    } else {
-        FatalError("%s: Failed to write block %s to index",
-                   __func__, pindex->GetBlockHash().ToString());
-        return;
-    }
-}
-
-void BaseIndex::ChainStateFlushed(const CBlockLocator& locator)
-{
-    if (!m_synced) {
-        return;
-    }
-
-    const uint256& locator_tip_hash = locator.vHave.front();
-    const CBlockIndex* locator_tip_index;
-    {
-        LOCK(cs_main);
-        locator_tip_index = LookupBlockIndex(locator_tip_hash);
-    }
-
-    if (!locator_tip_index) {
-        FatalError("%s: First block (hash=%s) in locator was not found",
-                   __func__, locator_tip_hash.ToString());
-        return;
-    }
-
-    // This checks that ChainStateFlushed callbacks are received after BlockConnected. The check may fail
-    // immediately after the sync thread catches up and sets m_synced. Consider the case where
-    // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue
-    // backlog even after the sync thread has caught up to the new chain tip. In this unlikely
-    // event, log a warning and let the queue clear.
-    const CBlockIndex* best_block_index = m_best_block_index.load();
-    if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) {
-        LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */
-                  "chain (tip=%s); not writing index locator\n",
-                  __func__, locator_tip_hash.ToString(),
-                  best_block_index->GetBlockHash().ToString());
-        return;
-    }
-
-    if (!GetDB().WriteBestBlock(locator)) {
-        error("%s: Failed to write locator to disk", __func__);
-    }
-}
-
-bool BaseIndex::BlockUntilSyncedToCurrentChain()
-{
-    AssertLockNotHeld(cs_main);
-
-    if (!m_synced) {
-        return false;
-    }
-
-    {
-        // Skip the queue-draining stuff if we know we're caught up with
-        // chainActive.Tip().
-        LOCK(cs_main);
-        const CBlockIndex* chain_tip = chainActive.Tip();
-        const CBlockIndex* best_block_index = m_best_block_index.load();
-        if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) {
-            return true;
-        }
-    }
-
-    LogPrintf("%s: %s is catching up on block notifications\n", __func__, GetName());
-    SyncWithValidationInterfaceQueue();
-    return true;
-}
-
-void BaseIndex::Interrupt()
-{
-    m_interrupt();
-}
-
-void BaseIndex::Start()
-{
-    // Need to register this ValidationInterface before running Init(), so that
-    // callbacks are not missed if Init sets m_synced to true.
-    RegisterValidationInterface(this);
-    if (!Init()) {
-        FatalError("%s: %s failed to initialize", __func__, GetName());
-        return;
-    }
-
-    m_thread_sync = std::thread(&TraceThread<std::function<void()>>, GetName(),
-                                std::bind(&BaseIndex::ThreadSync, this));
-}
-
-void BaseIndex::Stop()
-{
-    UnregisterValidationInterface(this);
-
-    if (m_thread_sync.joinable()) {
-        m_thread_sync.join();
-    }
-}

src/index/base.h

@@ -1,100 +0,0 @@
-// Copyright (c) 2017-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#ifndef BITCOIN_INDEX_BASE_H
-#define BITCOIN_INDEX_BASE_H
-
-#include <dbwrapper.h>
-#include <primitives/block.h>
-#include <primitives/transaction.h>
-#include <threadinterrupt.h>
-#include <uint256.h>
-#include <validationinterface.h>
-
-class CBlockIndex;
-
-/**
- * Base class for indices of blockchain data. This implements
- * CValidationInterface and ensures blocks are indexed sequentially according
- * to their position in the active chain.
- */
-class BaseIndex : public CValidationInterface
-{
-protected:
-    class DB : public CDBWrapper
-    {
-    public:
-        DB(const fs::path& path, size_t n_cache_size,
-           bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false);
-
-        ~DB() override {}
-
-        /// Read block locator of the chain that the txindex is in sync with.
-        bool ReadBestBlock(CBlockLocator& locator) const;
-
-        /// Write block locator of the chain that the txindex is in sync with.
-        bool WriteBestBlock(const CBlockLocator& locator);
-    };
-
-private:
-    /// Whether the index is in sync with the main chain. The flag is flipped
-    /// from false to true once, after which point this starts processing
-    /// ValidationInterface notifications to stay in sync.
-    std::atomic<bool> m_synced{false};
-
-    /// The last block in the chain that the index is in sync with.
-    std::atomic<const CBlockIndex*> m_best_block_index{nullptr};
-
-    std::thread m_thread_sync;
-    CThreadInterrupt m_interrupt;
-
-    /// Sync the index with the block index starting from the current best block.
-    /// Intended to be run in its own thread, m_thread_sync, and can be
-    /// interrupted with m_interrupt. Once the index gets in sync, the m_synced
-    /// flag is set and the BlockConnected ValidationInterface callback takes
-    /// over and the sync thread exits.
-    void ThreadSync();
-
-    /// Write the current chain block locator to the DB.
-    bool WriteBestBlock(const CBlockIndex* block_index);
-
-protected:
-    void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
-                        const std::vector<CTransactionRef>& txn_conflicted) override;
-
-    void ChainStateFlushed(const CBlockLocator& locator) override;
-
-    /// Initialize internal state from the database and block index.
-    virtual bool Init();
-
-    /// Write update index entries for a newly connected block.
-    virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; }
-
-    virtual DB& GetDB() const = 0;
-
-    /// Get the name of the index for display in logs.
-    virtual const char* GetName() const = 0;
-
-public:
-    /// Destructor interrupts sync thread if running and blocks until it exits.
-    virtual ~BaseIndex();
-
-    /// Blocks the current thread until the index is caught up to the current
-    /// state of the block chain. This only blocks if the index has gotten in
-    /// sync once and only needs to process blocks in the ValidationInterface
-    /// queue. If the index is catching up from far behind, this method does
-    /// not block and immediately returns false.
-    bool BlockUntilSyncedToCurrentChain();
-
-    void Interrupt();
-
-    /// Start initializes the sync state and registers the instance as a
-    /// ValidationInterface so that it stays in sync with blockchain updates.
-    void Start();
-
-    /// Stops the instance from staying in sync with blockchain updates.
-    void Stop();
-};
-
-#endif // BITCOIN_INDEX_BASE_H

src/index/txindex.cpp

@@ -1,262 +0,0 @@
-// Copyright (c) 2017-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#include <index/txindex.h>
-#include <shutdown.h>
-#include <ui_interface.h>
-#include <util.h>
-#include <validation.h>
-
-#include <boost/thread.hpp>
-
-constexpr char DB_BEST_BLOCK = 'B';
-constexpr char DB_TXINDEX = 't';
-constexpr char DB_TXINDEX_BLOCK = 'T';
-
-std::unique_ptr<TxIndex> g_txindex;
-
-/**
- * Access to the txindex database (indexes/txindex/)
- *
- * The database stores a block locator of the chain the database is synced to
- * so that the TxIndex can efficiently determine the point it last stopped at.
- * A locator is used instead of a simple hash of the chain tip because blocks
- * and block index entries may not be flushed to disk until after this database
- * is updated.
- */
-class TxIndex::DB : public BaseIndex::DB
-{
-public:
-    explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
-    ~DB() override {}
-
-    /// Read the disk location of the transaction data with the given hash. Returns false if the
-    /// transaction hash is not indexed.
-    bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
-
-    /// Write a batch of transaction positions to the DB.
-    bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
-
-    /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not
-    /// been upgraded yet to the new database.
-    bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
-};
-
-TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
-    BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
-{}
-
-bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
-{
-    return Read(std::make_pair(DB_TXINDEX, txid), pos);
-}
-
-bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
-{
-    CDBBatch batch(*this);
-    for (const auto& tuple : v_pos) {
-        batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
-    }
-    return WriteBatch(batch);
-}
-
-/*
- * Safely persist a transfer of data from the old txindex database to the new one, and compact the
- * range of keys updated. This is used internally by MigrateData.
- */
-static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb,
-                                         CDBBatch& batch_newdb, CDBBatch& batch_olddb,
-                                         const std::pair<unsigned char, uint256>& begin_key,
-                                         const std::pair<unsigned char, uint256>& end_key)
-{
-    // Sync new DB changes to disk before deleting from old DB.
-    newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
-    olddb.WriteBatch(batch_olddb);
-    olddb.CompactRange(begin_key, end_key);
-
-    batch_newdb.Clear();
-    batch_olddb.Clear();
-}
-
-bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
-{
-    // The prior implementation of txindex was always in sync with block index
-    // and presence was indicated with a boolean DB flag. If the flag is set,
-    // this means the txindex from a previous version is valid and in sync with
-    // the chain tip. The first step of the migration is to unset the flag and
-    // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
-    // index entries are copied over in batches to the new database. Finally,
-    // DB_TXINDEX_BLOCK is erased from the old database and the block hash is
-    // written to the new database.
-    //
-    // Unsetting the boolean flag ensures that if the node is downgraded to a
-    // previous version, it will not see a corrupted, partially migrated index
-    // -- it will see that the txindex is disabled. When the node is upgraded
-    // again, the migration will pick up where it left off and sync to the block
-    // with hash DB_TXINDEX_BLOCK.
-    bool f_legacy_flag = false;
-    block_tree_db.ReadFlag("txindex", f_legacy_flag);
-    if (f_legacy_flag) {
-        if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
-            return error("%s: cannot write block indicator", __func__);
-        }
-        if (!block_tree_db.WriteFlag("txindex", false)) {
-            return error("%s: cannot write block index db flag", __func__);
-        }
-    }
-
-    CBlockLocator locator;
-    if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
-        return true;
-    }
-
-    int64_t count = 0;
-    LogPrintf("Upgrading txindex database... [0%%]\n");
-    uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true);
-    int report_done = 0;
-    const size_t batch_size = 1 << 24; // 16 MiB
-
-    CDBBatch batch_newdb(*this);
-    CDBBatch batch_olddb(block_tree_db);
-
-    std::pair<unsigned char, uint256> key;
-    std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
-    std::pair<unsigned char, uint256> prev_key = begin_key;
-
-    bool interrupted = false;
-    std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
-    for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
-        boost::this_thread::interruption_point();
-        if (ShutdownRequested()) {
-            interrupted = true;
-            break;
-        }
-
-        if (!cursor->GetKey(key)) {
-            return error("%s: cannot get key from valid cursor", __func__);
-        }
-        if (key.first != DB_TXINDEX) {
-            break;
-        }
-
-        // Log progress every 10%.
-        if (++count % 256 == 0) {
-            // Since txids are uniformly random and traversed in increasing order, the high 16 bits
-            // of the hash can be used to estimate the current progress.
-            const uint256& txid = key.second;
-            uint32_t high_nibble =
-                (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
-                (static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
-            int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
-
-            uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true);
-            if (report_done < percentage_done/10) {
-                LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
-                report_done = percentage_done/10;
-            }
-        }
-
-        CDiskTxPos value;
-        if (!cursor->GetValue(value)) {
-            return error("%s: cannot parse txindex record", __func__);
-        }
-        batch_newdb.Write(key, value);
-        batch_olddb.Erase(key);
-
-        if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
-            // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
-            // because LevelDB iterators are guaranteed to provide a consistent view of the
-            // underlying data, like a lightweight snapshot.
-            WriteTxIndexMigrationBatches(*this, block_tree_db,
-                                         batch_newdb, batch_olddb,
-                                         prev_key, key);
-            prev_key = key;
-        }
-    }
-
-    // If these final DB batches complete the migration, write the best block
-    // hash marker to the new database and delete from the old one. This signals
-    // that the former is fully caught up to that point in the blockchain and
-    // that all txindex entries have been removed from the latter.
-    if (!interrupted) {
-        batch_olddb.Erase(DB_TXINDEX_BLOCK);
-        batch_newdb.Write(DB_BEST_BLOCK, locator);
-    }
-
-    WriteTxIndexMigrationBatches(*this, block_tree_db,
-                                 batch_newdb, batch_olddb,
-                                 begin_key, key);
-
-    if (interrupted) {
-        LogPrintf("[CANCELLED].\n");
-        return false;
-    }
-
-    uiInterface.ShowProgress("", 100, false);
-
-    LogPrintf("[DONE].\n");
-    return true;
-}
-
-TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
-    : m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
-{}
-
-TxIndex::~TxIndex() {}
-
-bool TxIndex::Init()
-{
-    LOCK(cs_main);
-
-    // Attempt to migrate txindex from the old database to the new one. Even if
-    // chain_tip is null, the node could be reindexing and we still want to
-    // delete txindex records in the old database.
-    if (!m_db->MigrateData(*pblocktree, chainActive.GetLocator())) {
-        return false;
-    }
-
-    return BaseIndex::Init();
-}
-
-bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
-{
-    CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
-    std::vector<std::pair<uint256, CDiskTxPos>> vPos;
-    vPos.reserve(block.vtx.size());
-    for (const auto& tx : block.vtx) {
-        vPos.emplace_back(tx->GetHash(), pos);
-        pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION);
-    }
-    return m_db->WriteTxs(vPos);
-}
-
-BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
-
-bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
-{
-    CDiskTxPos postx;
-    if (!m_db->ReadTxPos(tx_hash, postx)) {
-        return false;
-    }
-
-    CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
-    if (file.IsNull()) {
-        return error("%s: OpenBlockFile failed", __func__);
-    }
-    CBlockHeader header;
-    try {
-        file >> header;
-        if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) {
-            return error("%s: fseek(...) failed", __func__);
-        }
-        file >> tx;
-    } catch (const std::exception& e) {
-        return error("%s: Deserialize or I/O error - %s", __func__, e.what());
-    }
-    if (tx->GetHash() != tx_hash) {
-        return error("%s: txid mismatch", __func__);
-    }
-    block_hash = header.GetHash();
-    return true;
-}

src/index/txindex.h

@@ -1,79 +0,0 @@
-// Copyright (c) 2017-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#ifndef BITCOIN_INDEX_TXINDEX_H
-#define BITCOIN_INDEX_TXINDEX_H
-
-#include <chain.h>
-#include <index/base.h>
-#include <txdb.h>
-
-struct CDiskTxPos : public CDiskBlockPos
-{
-    unsigned int nTxOffset; // after header
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITEAS(CDiskBlockPos, *this);
-        READWRITE(VARINT(nTxOffset));
-    }
-
-    CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
-    }
-
-    CDiskTxPos() {
-        SetNull();
-    }
-
-    void SetNull() {
-        CDiskBlockPos::SetNull();
-        nTxOffset = 0;
-    }
-};
-
-/**
- * TxIndex is used to look up transactions included in the blockchain by hash.
- * The index is written to a LevelDB database and records the filesystem
- * location of each transaction by transaction hash.
- */
-class TxIndex final : public BaseIndex
-{
-protected:
-    class DB;
-
-private:
-    const std::unique_ptr<DB> m_db;
-
-protected:
-    /// Override base class init to migrate from old database.
-    bool Init() override;
-
-    bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
-
-    BaseIndex::DB& GetDB() const override;
-
-    const char* GetName() const override { return "txindex"; }
-
-public:
-    /// Constructs the index, which becomes available to be queried.
-    explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
-
-    // Destructor is declared because this class contains a unique_ptr to an incomplete type.
-    virtual ~TxIndex() override;
-
-    /// Look up a transaction by hash.
-    ///
-    /// @param[in]   tx_hash  The hash of the transaction to be returned.
-    /// @param[out]  block_hash  The hash of the block the transaction is found in.
-    /// @param[out]  tx  The transaction itself.
-    /// @return  true if transaction is found, false otherwise
-    bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const;
-};
-
-/// The global transaction index, used in GetTransaction. May be null.
-extern std::unique_ptr<TxIndex> g_txindex;
-
-#endif // BITCOIN_INDEX_TXINDEX_H

src/init.cpp

@@ -15,12 +15,12 @@
 #include <chainparams.h>
 #include <checkpoints.h>
 #include <claimtrie/forks.h>
+#include <clientversion.h>
 #include <compat/sanity.h>
 #include <consensus/validation.h>
 #include <fs.h>
 #include <httpserver.h>
 #include <httprpc.h>
-#include <index/txindex.h>
 #include <key.h>
 #include <lbry.h>
 #include <validation.h>
@@ -46,11 +46,10 @@
 #include <uint256.h>
 #include <util.h>
 #include <utilmoneystr.h>
+#include <utilstrencodings.h>
 #include <validationinterface.h>
 #include <warnings.h>
 #include <walletinitinterface.h>
-#include <stdint.h>
-#include <stdio.h>
 
 #ifndef WIN32
 #include <signal.h>
@@ -181,9 +180,6 @@ void Interrupt()
     InterruptMapPort();
     if (g_connman)
         g_connman->Interrupt();
-    if (g_txindex) {
-        g_txindex->Interrupt();
-    }
 }
 
 void Shutdown()
@@ -212,7 +208,6 @@ void Shutdown()
     // using the other before destroying them.
     if (peerLogic) UnregisterValidationInterface(peerLogic.get());
     if (g_connman) g_connman->Stop();
-    if (g_txindex) g_txindex->Stop();
 
     StopTorControl();
 
@@ -225,7 +220,6 @@ void Shutdown()
     // destruct and reset all to nullptr.
     peerLogic.reset();
     g_connman.reset();
-    g_txindex.reset();
 
     if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) {
         DumpMempool();
@@ -353,7 +347,7 @@ void SetupServerArgs()
 
     // Hidden Options
     std::vector<std::string> hidden_args = {"-rpcssl", "-benchmark", "-h", "-help", "-socks", "-tor", "-debugnet", "-whitelistalwaysrelay",
-        "-prematurewitness", "-walletprematurewitness", "-promiscuousmempoolflags", "-blockminsize", "-dbcrashratio", "-forcecompactdb", "-usehd",
+        "-prematurewitness", "-walletprematurewitness", "-promiscuousmempoolflags", "-blockminsize", "-forcecompactdb", "-usehd",
         // GUI args. These will be overwritten by SetupUIArgs for the GUI
         "-allowselfsignedrootcertificates", "-choosedatadir", "-lang=<lang>", "-min", "-resetguisettings", "-rootcertificates=<file>", "-splash", "-uiplatform"};
 
@@ -369,7 +363,6 @@ void SetupServerArgs()
     gArgs.AddArg("-blocksonly", strprintf("Whether to operate in a blocks only mode (default: %u)", DEFAULT_BLOCKSONLY), true, OptionsCategory::OPTIONS);
     gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS);
-    gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), true, OptionsCategory::OPTIONS);
     gArgs.AddArg("-dbcache=<n>", strprintf("Set database cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS);
@@ -387,7 +380,7 @@ void SetupServerArgs()
 #else
     hidden_args.emplace_back("-pid");
 #endif
-    gArgs.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. "
+    gArgs.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -rescan. "
             "Warning: Reverting this setting requires re-downloading the entire blockchain. "
             "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", false, OptionsCategory::OPTIONS);
@@ -397,7 +390,6 @@ void SetupServerArgs()
 #else
     hidden_args.emplace_back("-sysperms");
 #endif
-    gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-bantime=<n>", strprintf("Number of seconds to keep misbehaving peers from reconnecting (default: %u)", DEFAULT_MISBEHAVING_BANTIME), false, OptionsCategory::CONNECTION);
@@ -625,7 +617,7 @@ static void CleanupBlockRevFiles()
     // start removing block files.
     int nContigCounter = 0;
     for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
-        if (atoi(item.first) == nContigCounter) {
+        if (std::atoi(item.first.c_str()) == nContigCounter) {
             nContigCounter++;
             continue;
         }
@@ -933,12 +925,6 @@ bool AppInitParameterInteraction()
         return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", "").c_str()));
     }
 
-    // if using block pruning, then disallow txindex
-    if (gArgs.GetArg("-prune", 0)) {
-        if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX))
-            return InitError(_("Prune mode is incompatible with -txindex."));
-    }
-
     // -bind and -whitebind can't be set when not listening
     size_t nUserBind = gArgs.GetArgs("-bind").size() + gArgs.GetArgs("-whitebind").size();
     if (nUserBind != 0 && !gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) {
@@ -1416,22 +1402,24 @@ bool AppInitMain()
     int64_t nTotalCache = (gArgs.GetArg("-dbcache", nDefaultDbCache) << 20);
     nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
     nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache
-    int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20);
+
+    // we're going to chop the cache into three pieces:
+    // the coin cache, the block cache, the claimtrie cache
+    // however, we want the claimtrie cache to be larger than the others
+
+    int64_t nBlockTreeDBCache = std::min(nTotalCache / 4, nMaxBlockDBCache << 20);
+    int64_t nCoinDBCache = std::min(nTotalCache / 8, nMaxCoinsDBCache << 20);
+    int64_t nClaimtrieCache = nTotalCache / 4;
     nTotalCache -= nBlockTreeDBCache;
-    int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
-    nTotalCache -= nTxIndexCache;
-    int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
-    nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
     nTotalCache -= nCoinDBCache;
+    nTotalCache -= nClaimtrieCache;
     nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
     std::cout << "nTotalCache: " << nTotalCache << ", nCoinCacheUsage: " << nCoinCacheUsage << ", nCoinDBCache: " << nCoinDBCache << std::endl;
     int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000;
     LogPrintf("Cache configuration:\n");
-    LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
+    LogPrintf("* Using %.1fMiB for block index database cache\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
-    if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
+    LogPrintf("* Using %.1fMiB for chain state database cache\n", nCoinDBCache * (1.0 / 1024 / 1024));
-        LogPrintf("* Using %.1fMiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024));
+    LogPrintf("* Using %.1fMiB for claimtrie database cache\n", nClaimtrieCache * (1.0 / 1024 / 1024));
-    }
-    LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
     LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
 
     bool fLoaded = false;
@@ -1501,19 +1489,12 @@ bool AppInitMain()
                 pcoinsdbview.reset(new CCoinsViewDB(nCoinDBCache, false, fReset || fReindexChainState));
                 pcoinscatcher.reset(new CCoinsViewErrorCatcher(pcoinsdbview.get()));
 
-                // If necessary, upgrade from older database format.
-                // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate
-                if (!pcoinsdbview->Upgrade()) {
-                    strLoadError = _("Error upgrading chainstate database");
-                    break;
-                }
-
                 if (g_logger->Enabled() && LogAcceptCategory(BCLog::CLAIMS))
                     CLogPrint::global().setLogger(g_logger);
 
                 delete pclaimTrie;
                 auto& consensus = chainparams.GetConsensus();
-                pclaimTrie = new CClaimTrie(fReindex || fReindexChainState, 0,
+                pclaimTrie = new CClaimTrie(nClaimtrieCache, fReindex || fReindexChainState, 0,
                                             GetDataDir().string(),
                                             consensus.nNormalizedNameForkHeight,
                                             consensus.nOriginalClaimExpirationTime,
@@ -1625,12 +1606,6 @@ bool AppInitMain()
         ::feeEstimator.Read(est_filein);
     fFeeEstimatesInitialized = true;
 
-    // ********************************************************* Step 8: start indexers
-    if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
-        g_txindex = MakeUnique<TxIndex>(nTxIndexCache, false, fReindex);
-        g_txindex->Start();
-    }
-
     // ********************************************************* Step 9: load wallet
     if (!g_wallet_init_interface.Open()) return false;
 

src/leveldb/.gitignore

@@ -1,13 +0,0 @@
-build_config.mk
-*.a
-*.o
-*.dylib*
-*.so
-*.so.*
-*_test
-db_bench
-leveldbutil
-Release
-Debug
-Benchmark
-vs2010.*

src/leveldb/.travis.yml

@@ -1,13 +0,0 @@
-language: cpp
-compiler:
-- clang
-- gcc
-os:
-- linux
-- osx
-sudo: false
-before_install:
-- echo $LANG
-- echo $LC_ALL
-script:
-- make -j 4 check

src/leveldb/AUTHORS

@@ -1,12 +0,0 @@
-# Names should be added to this file like so:
-# Name or Organization <email address>
-
-Google Inc.
-
-# Initial version authors:
-Jeffrey Dean <jeff@google.com>
-Sanjay Ghemawat <sanjay@google.com>
-
-# Partial list of contributors:
-Kevin Regan <kevin.d.regan@gmail.com>
-Johan Bilien <jobi@litl.com>

src/leveldb/CONTRIBUTING.md

@@ -1,36 +0,0 @@
-# Contributing
-
-We'd love to accept your code patches! However, before we can take them, we
-have to jump a couple of legal hurdles.
-
-## Contributor License Agreements
-
-Please fill out either the individual or corporate Contributor License
-Agreement as appropriate.
-
-* If you are an individual writing original source code and you're sure you
-own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual).
-* If you work for a company that wants to allow you to contribute your work,
-then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate).
-
-Follow either of the two links above to access the appropriate CLA and
-instructions for how to sign and return it.
-
-## Submitting a Patch
-
-1. Sign the contributors license agreement above.
-2. Decide which code you want to submit. A submission should be a set of changes
-that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues).
-Please don't mix more than one logical change per submission, because it makes
-the history hard to follow. If you want to make a change
-(e.g. add a sample or feature) that doesn't have a corresponding issue in the
-issue tracker, please create one.
-3. **Submitting**: When you are ready to submit, send us a Pull Request. Be
-sure to include the issue number you fixed and the name you used to sign
-the CLA.
-
-## Writing Code ##
-
-If your contribution contains code, please make sure that it follows 
-[the style guide](http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml).
-Otherwise we will have to ask you to make changes, and that's no fun for anyone.

src/leveldb/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

src/leveldb/Makefile

@@ -1,424 +0,0 @@
-# Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#-----------------------------------------------
-# Uncomment exactly one of the lines labelled (A), (B), and (C) below
-# to switch between compilation modes.
-
-# (A) Production use (optimized mode)
-OPT ?= -O2 -DNDEBUG
-# (B) Debug mode, w/ full line-level debugging symbols
-# OPT ?= -g2
-# (C) Profiling mode: opt, but w/debugging symbols
-# OPT ?= -O2 -g2 -DNDEBUG
-#-----------------------------------------------
-
-# detect what platform we're building on
-$(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \
-    ./build_detect_platform build_config.mk ./)
-# this file is generated by the previous line to set build flags and sources
-include build_config.mk
-
-TESTS = \
-	db/autocompact_test \
-	db/c_test \
-	db/corruption_test \
-	db/db_test \
-	db/dbformat_test \
-	db/fault_injection_test \
-	db/filename_test \
-	db/log_test \
-	db/recovery_test \
-	db/skiplist_test \
-	db/version_edit_test \
-	db/version_set_test \
-	db/write_batch_test \
-	helpers/memenv/memenv_test \
-	issues/issue178_test \
-	issues/issue200_test \
-	table/filter_block_test \
-	table/table_test \
-	util/arena_test \
-	util/bloom_test \
-	util/cache_test \
-	util/coding_test \
-	util/crc32c_test \
-	util/env_posix_test \
-	util/env_test \
-	util/hash_test
-
-UTILS = \
-	db/db_bench \
-	db/leveldbutil
-
-# Put the object files in a subdirectory, but the application at the top of the object dir.
-PROGNAMES := $(notdir $(TESTS) $(UTILS))
-
-# On Linux may need libkyotocabinet-dev for dependency.
-BENCHMARKS = \
-	doc/bench/db_bench_sqlite3 \
-	doc/bench/db_bench_tree_db
-
-CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
-CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT)
-
-LDFLAGS += $(PLATFORM_LDFLAGS)
-LIBS += $(PLATFORM_LIBS)
-
-SIMULATOR_OUTDIR=out-ios-x86
-DEVICE_OUTDIR=out-ios-arm
-
-ifeq ($(PLATFORM), IOS)
-# Note: iOS should probably be using libtool, not ar.
-AR=xcrun ar
-SIMULATORSDK=$(shell xcrun -sdk iphonesimulator --show-sdk-path)
-DEVICESDK=$(shell xcrun -sdk iphoneos --show-sdk-path)
-DEVICE_CFLAGS = -isysroot "$(DEVICESDK)" -arch armv6 -arch armv7 -arch armv7s -arch arm64
-SIMULATOR_CFLAGS = -isysroot "$(SIMULATORSDK)" -arch i686 -arch x86_64
-STATIC_OUTDIR=out-ios-universal
-else
-STATIC_OUTDIR=out-static
-SHARED_OUTDIR=out-shared
-STATIC_PROGRAMS := $(addprefix $(STATIC_OUTDIR)/, $(PROGNAMES))
-SHARED_PROGRAMS := $(addprefix $(SHARED_OUTDIR)/, db_bench)
-endif
-
-STATIC_LIBOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(SOURCES:.cc=.o))
-STATIC_MEMENVOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
-
-DEVICE_LIBOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(SOURCES:.cc=.o))
-DEVICE_MEMENVOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
-
-SIMULATOR_LIBOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(SOURCES:.cc=.o))
-SIMULATOR_MEMENVOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
-
-SHARED_LIBOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(SOURCES:.cc=.o))
-SHARED_MEMENVOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
-
-TESTUTIL := $(STATIC_OUTDIR)/util/testutil.o
-TESTHARNESS := $(STATIC_OUTDIR)/util/testharness.o $(TESTUTIL)
-
-STATIC_TESTOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(TESTS)))
-STATIC_UTILOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(UTILS)))
-STATIC_ALLOBJS := $(STATIC_LIBOBJECTS) $(STATIC_MEMENVOBJECTS) $(STATIC_TESTOBJS) $(STATIC_UTILOBJS) $(TESTHARNESS)
-DEVICE_ALLOBJS := $(DEVICE_LIBOBJECTS) $(DEVICE_MEMENVOBJECTS)
-SIMULATOR_ALLOBJS := $(SIMULATOR_LIBOBJECTS) $(SIMULATOR_MEMENVOBJECTS)
-
-default: all
-
-# Should we build shared libraries?
-ifneq ($(PLATFORM_SHARED_EXT),)
-
-# Many leveldb test apps use non-exported API's. Only build a subset for testing.
-SHARED_ALLOBJS := $(SHARED_LIBOBJECTS) $(SHARED_MEMENVOBJECTS) $(TESTHARNESS)
-
-ifneq ($(PLATFORM_SHARED_VERSIONED),true)
-SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT)
-SHARED_LIB2 = $(SHARED_LIB1)
-SHARED_LIB3 = $(SHARED_LIB1)
-SHARED_LIBS = $(SHARED_LIB1)
-SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a
-else
-# Update db.h if you change these.
-SHARED_VERSION_MAJOR = 1
-SHARED_VERSION_MINOR = 20
-SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT)
-SHARED_LIB2 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR)
-SHARED_LIB3 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR).$(SHARED_VERSION_MINOR)
-SHARED_LIBS = $(SHARED_OUTDIR)/$(SHARED_LIB1) $(SHARED_OUTDIR)/$(SHARED_LIB2) $(SHARED_OUTDIR)/$(SHARED_LIB3)
-$(SHARED_OUTDIR)/$(SHARED_LIB1): $(SHARED_OUTDIR)/$(SHARED_LIB3)
-	ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB1)
-$(SHARED_OUTDIR)/$(SHARED_LIB2): $(SHARED_OUTDIR)/$(SHARED_LIB3)
-	ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB2)
-SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a
-endif
-
-$(SHARED_OUTDIR)/$(SHARED_LIB3): $(SHARED_LIBOBJECTS)
-	$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED_LIB2) $(SHARED_LIBOBJECTS) -o $(SHARED_OUTDIR)/$(SHARED_LIB3) $(LIBS)
-
-endif  # PLATFORM_SHARED_EXT
-
-all: $(SHARED_LIBS) $(SHARED_PROGRAMS) $(STATIC_OUTDIR)/libleveldb.a $(STATIC_OUTDIR)/libmemenv.a $(STATIC_PROGRAMS)
-
-check: $(STATIC_PROGRAMS)
-	for t in $(notdir $(TESTS)); do echo "***** Running $$t"; $(STATIC_OUTDIR)/$$t || exit 1; done
-
-clean:
-	-rm -rf out-static out-shared out-ios-x86 out-ios-arm out-ios-universal
-	-rm -f build_config.mk
-	-rm -rf ios-x86 ios-arm
-
-$(STATIC_OUTDIR):
-	mkdir $@
-
-$(STATIC_OUTDIR)/db: | $(STATIC_OUTDIR)
-	mkdir $@
-
-$(STATIC_OUTDIR)/helpers/memenv: | $(STATIC_OUTDIR)
-	mkdir -p $@
-
-$(STATIC_OUTDIR)/port: | $(STATIC_OUTDIR)
-	mkdir $@
-
-$(STATIC_OUTDIR)/table: | $(STATIC_OUTDIR)
-	mkdir $@
-
-$(STATIC_OUTDIR)/util: | $(STATIC_OUTDIR)
-	mkdir $@
-
-.PHONY: STATIC_OBJDIRS
-STATIC_OBJDIRS: \
-	$(STATIC_OUTDIR)/db \
-	$(STATIC_OUTDIR)/port \
-	$(STATIC_OUTDIR)/table \
-	$(STATIC_OUTDIR)/util \
-	$(STATIC_OUTDIR)/helpers/memenv
-
-$(SHARED_OUTDIR):
-	mkdir $@
-
-$(SHARED_OUTDIR)/db: | $(SHARED_OUTDIR)
-	mkdir $@
-
-$(SHARED_OUTDIR)/helpers/memenv: | $(SHARED_OUTDIR)
-	mkdir -p $@
-
-$(SHARED_OUTDIR)/port: | $(SHARED_OUTDIR)
-	mkdir $@
-
-$(SHARED_OUTDIR)/table: | $(SHARED_OUTDIR)
-	mkdir $@
-
-$(SHARED_OUTDIR)/util: | $(SHARED_OUTDIR)
-	mkdir $@
-
-.PHONY: SHARED_OBJDIRS
-SHARED_OBJDIRS: \
-	$(SHARED_OUTDIR)/db \
-	$(SHARED_OUTDIR)/port \
-	$(SHARED_OUTDIR)/table \
-	$(SHARED_OUTDIR)/util \
-	$(SHARED_OUTDIR)/helpers/memenv
-
-$(DEVICE_OUTDIR):
-	mkdir $@
-
-$(DEVICE_OUTDIR)/db: | $(DEVICE_OUTDIR)
-	mkdir $@
-
-$(DEVICE_OUTDIR)/helpers/memenv: | $(DEVICE_OUTDIR)
-	mkdir -p $@
-
-$(DEVICE_OUTDIR)/port: | $(DEVICE_OUTDIR)
-	mkdir $@
-
-$(DEVICE_OUTDIR)/table: | $(DEVICE_OUTDIR)
-	mkdir $@
-
-$(DEVICE_OUTDIR)/util: | $(DEVICE_OUTDIR)
-	mkdir $@
-
-.PHONY: DEVICE_OBJDIRS
-DEVICE_OBJDIRS: \
-	$(DEVICE_OUTDIR)/db \
-	$(DEVICE_OUTDIR)/port \
-	$(DEVICE_OUTDIR)/table \
-	$(DEVICE_OUTDIR)/util \
-	$(DEVICE_OUTDIR)/helpers/memenv
-
-$(SIMULATOR_OUTDIR):
-	mkdir $@
-
-$(SIMULATOR_OUTDIR)/db: | $(SIMULATOR_OUTDIR)
-	mkdir $@
-
-$(SIMULATOR_OUTDIR)/helpers/memenv: | $(SIMULATOR_OUTDIR)
-	mkdir -p $@
-
-$(SIMULATOR_OUTDIR)/port: | $(SIMULATOR_OUTDIR)
-	mkdir $@
-
-$(SIMULATOR_OUTDIR)/table: | $(SIMULATOR_OUTDIR)
-	mkdir $@
-
-$(SIMULATOR_OUTDIR)/util: | $(SIMULATOR_OUTDIR)
-	mkdir $@
-
-.PHONY: SIMULATOR_OBJDIRS
-SIMULATOR_OBJDIRS: \
-	$(SIMULATOR_OUTDIR)/db \
-	$(SIMULATOR_OUTDIR)/port \
-	$(SIMULATOR_OUTDIR)/table \
-	$(SIMULATOR_OUTDIR)/util \
-	$(SIMULATOR_OUTDIR)/helpers/memenv
-
-$(STATIC_ALLOBJS): | STATIC_OBJDIRS
-$(DEVICE_ALLOBJS): | DEVICE_OBJDIRS
-$(SIMULATOR_ALLOBJS): | SIMULATOR_OBJDIRS
-$(SHARED_ALLOBJS): | SHARED_OBJDIRS
-
-ifeq ($(PLATFORM), IOS)
-$(DEVICE_OUTDIR)/libleveldb.a: $(DEVICE_LIBOBJECTS)
-	rm -f $@
-	$(AR) -rs $@ $(DEVICE_LIBOBJECTS)
-
-$(SIMULATOR_OUTDIR)/libleveldb.a: $(SIMULATOR_LIBOBJECTS)
-	rm -f $@
-	$(AR) -rs $@ $(SIMULATOR_LIBOBJECTS)
-
-$(DEVICE_OUTDIR)/libmemenv.a: $(DEVICE_MEMENVOBJECTS)
-	rm -f $@
-	$(AR) -rs $@ $(DEVICE_MEMENVOBJECTS)
-
-$(SIMULATOR_OUTDIR)/libmemenv.a: $(SIMULATOR_MEMENVOBJECTS)
-	rm -f $@
-	$(AR) -rs $@ $(SIMULATOR_MEMENVOBJECTS)
-
-# For iOS, create universal object libraries to be used on both the simulator and
-# a device.
-$(STATIC_OUTDIR)/libleveldb.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a
-	lipo -create $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a -output $@
-
-$(STATIC_OUTDIR)/libmemenv.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a
-	lipo -create $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a -output $@
-else
-$(STATIC_OUTDIR)/libleveldb.a:$(STATIC_LIBOBJECTS)
-	rm -f $@
-	$(AR) -rs $@ $(STATIC_LIBOBJECTS)
-
-$(STATIC_OUTDIR)/libmemenv.a:$(STATIC_MEMENVOBJECTS)
-	rm -f $@
-	$(AR) -rs $@ $(STATIC_MEMENVOBJECTS)
-endif
-
-$(SHARED_MEMENVLIB):$(SHARED_MEMENVOBJECTS)
-	rm -f $@
-	$(AR) -rs $@ $(SHARED_MEMENVOBJECTS)
-
-$(STATIC_OUTDIR)/db_bench:db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/db_bench_sqlite3:doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS)
-
-$(STATIC_OUTDIR)/db_bench_tree_db:doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS)
-
-$(STATIC_OUTDIR)/leveldbutil:db/leveldbutil.cc $(STATIC_LIBOBJECTS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/leveldbutil.cc $(STATIC_LIBOBJECTS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/arena_test:util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/autocompact_test:db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/bloom_test:util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/c_test:$(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/cache_test:util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/coding_test:util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/corruption_test:db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/crc32c_test:util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/db_test:db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/dbformat_test:db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/env_posix_test:util/env_posix_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_posix_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/env_test:util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/fault_injection_test:db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/filename_test:db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/filter_block_test:table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/hash_test:util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/issue178_test:issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/issue200_test:issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/log_test:db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/recovery_test:db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/table_test:table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/skiplist_test:db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/version_edit_test:db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/version_set_test:db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/write_batch_test:db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(STATIC_OUTDIR)/memenv_test:$(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS)
-	$(XCRUN) $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) -o $@ $(LIBS)
-
-$(SHARED_OUTDIR)/db_bench:$(SHARED_OUTDIR)/db/db_bench.o $(SHARED_LIBS) $(TESTUTIL)
-	$(XCRUN) $(CXX) $(LDFLAGS) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SHARED_OUTDIR)/db/db_bench.o $(TESTUTIL) $(SHARED_OUTDIR)/$(SHARED_LIB3) -o $@ $(LIBS)
-
-.PHONY: run-shared
-run-shared: $(SHARED_OUTDIR)/db_bench
-	LD_LIBRARY_PATH=$(SHARED_OUTDIR) $(SHARED_OUTDIR)/db_bench
-
-$(SIMULATOR_OUTDIR)/%.o: %.cc
-	xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@
-
-$(DEVICE_OUTDIR)/%.o: %.cc
-	xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) $(DEVICE_CFLAGS) -c $< -o $@
-
-$(SIMULATOR_OUTDIR)/%.o: %.c
-	xcrun -sdk iphonesimulator $(CC) $(CFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@
-
-$(DEVICE_OUTDIR)/%.o: %.c
-	xcrun -sdk iphoneos $(CC) $(CFLAGS) $(DEVICE_CFLAGS) -c $< -o $@
-
-$(STATIC_OUTDIR)/%.o: %.cc
-	$(CXX) $(CXXFLAGS) -c $< -o $@
-
-$(STATIC_OUTDIR)/%.o: %.c
-	$(CC) $(CFLAGS) -c $< -o $@
-
-$(SHARED_OUTDIR)/%.o: %.cc
-	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
-
-$(SHARED_OUTDIR)/%.o: %.c
-	$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
-
-$(STATIC_OUTDIR)/port/port_posix_sse.o: port/port_posix_sse.cc
-	$(CXX) $(CXXFLAGS) $(PLATFORM_SSEFLAGS) -c $< -o $@
-
-$(SHARED_OUTDIR)/port/port_posix_sse.o: port/port_posix_sse.cc
-	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(PLATFORM_SSEFLAGS) -c $< -o $@

src/leveldb/NEWS

@@ -1,17 +0,0 @@
-Release 1.2 2011-05-16
-----------------------
-
-Fixes for larger databases (tested up to one billion 100-byte entries,
-i.e., ~100GB).
-
-(1) Place hard limit on number of level-0 files.  This fixes errors
-of the form "too many open files".
-
-(2) Fixed memtable management.  Before the fix, a heavy write burst
-could cause unbounded memory usage.
-
-A fix for a logging bug where the reader would incorrectly complain
-about corruption.
-
-Allow public access to WriteBatch contents so that users can easily
-wrap a DB.

src/leveldb/README.md

@@ -1,174 +0,0 @@
-**LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.**
-
-[![Build Status](https://travis-ci.org/google/leveldb.svg?branch=master)](https://travis-ci.org/google/leveldb)
-
-Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
-
-# Features
-  * Keys and values are arbitrary byte arrays.
-  * Data is stored sorted by key.
-  * Callers can provide a custom comparison function to override the sort order.
-  * The basic operations are `Put(key,value)`, `Get(key)`, `Delete(key)`.
-  * Multiple changes can be made in one atomic batch.
-  * Users can create a transient snapshot to get a consistent view of data.
-  * Forward and backward iteration is supported over the data.
-  * Data is automatically compressed using the [Snappy compression library](http://google.github.io/snappy/).
-  * External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions.
-
-# Documentation
-  [LevelDB library documentation](https://github.com/google/leveldb/blob/master/doc/index.md) is online and bundled with the source code.
-
-
-# Limitations
-  * This is not a SQL database.  It does not have a relational data model, it does not support SQL queries, and it has no support for indexes.
-  * Only a single process (possibly multi-threaded) can access a particular database at a time.
-  * There is no client-server support builtin to the library.  An application that needs such support will have to wrap their own server around the library.
-
-# Contributing to the leveldb Project
-The leveldb project welcomes contributions. leveldb's primary goal is to be
-a reliable and fast key/value store. Changes that are in line with the
-features/limitations outlined above, and meet the requirements below,
-will be considered.
-
-Contribution requirements:
-
-1. **POSIX only**. We _generally_ will only accept changes that are both
-   compiled, and tested on a POSIX platform - usually Linux. Very small
-   changes will sometimes be accepted, but consider that more of an
-   exception than the rule.
-
-2. **Stable API**. We strive very hard to maintain a stable API. Changes that
-   require changes for projects using leveldb _might_ be rejected without
-   sufficient benefit to the project.
-
-3. **Tests**: All changes must be accompanied by a new (or changed) test, or
-   a sufficient explanation as to why a new (or changed) test is not required.
-
-## Submitting a Pull Request
-Before any pull request will be accepted the author must first sign a
-Contributor License Agreement (CLA) at https://cla.developers.google.com/.
-
-In order to keep the commit timeline linear
-[squash](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits)
-your changes down to a single commit and [rebase](https://git-scm.com/docs/git-rebase)
-on google/leveldb/master. This keeps the commit timeline linear and more easily sync'ed
-with the internal repository at Google. More information at GitHub's
-[About Git rebase](https://help.github.com/articles/about-git-rebase/) page.
-
-# Performance
-
-Here is a performance report (with explanations) from the run of the
-included db_bench program.  The results are somewhat noisy, but should
-be enough to get a ballpark performance estimate.
-
-## Setup
-
-We use a database with a million entries.  Each entry has a 16 byte
-key, and a 100 byte value.  Values used by the benchmark compress to
-about half their original size.
-
-    LevelDB:    version 1.1
-    Date:       Sun May  1 12:11:26 2011
-    CPU:        4 x Intel(R) Core(TM)2 Quad CPU    Q6600  @ 2.40GHz
-    CPUCache:   4096 KB
-    Keys:       16 bytes each
-    Values:     100 bytes each (50 bytes after compression)
-    Entries:    1000000
-    Raw Size:   110.6 MB (estimated)
-    File Size:  62.9 MB (estimated)
-
-## Write performance
-
-The "fill" benchmarks create a brand new database, in either
-sequential, or random order.  The "fillsync" benchmark flushes data
-from the operating system to the disk after every operation; the other
-write operations leave the data sitting in the operating system buffer
-cache for a while.  The "overwrite" benchmark does random writes that
-update existing keys in the database.
-
-    fillseq      :       1.765 micros/op;   62.7 MB/s
-    fillsync     :     268.409 micros/op;    0.4 MB/s (10000 ops)
-    fillrandom   :       2.460 micros/op;   45.0 MB/s
-    overwrite    :       2.380 micros/op;   46.5 MB/s
-
-Each "op" above corresponds to a write of a single key/value pair.
-I.e., a random write benchmark goes at approximately 400,000 writes per second.
-
-Each "fillsync" operation costs much less (0.3 millisecond)
-than a disk seek (typically 10 milliseconds).  We suspect that this is
-because the hard disk itself is buffering the update in its memory and
-responding before the data has been written to the platter.  This may
-or may not be safe based on whether or not the hard disk has enough
-power to save its memory in the event of a power failure.
-
-## Read performance
-
-We list the performance of reading sequentially in both the forward
-and reverse direction, and also the performance of a random lookup.
-Note that the database created by the benchmark is quite small.
-Therefore the report characterizes the performance of leveldb when the
-working set fits in memory.  The cost of reading a piece of data that
-is not present in the operating system buffer cache will be dominated
-by the one or two disk seeks needed to fetch the data from disk.
-Write performance will be mostly unaffected by whether or not the
-working set fits in memory.
-
-    readrandom  : 16.677 micros/op;  (approximately 60,000 reads per second)
-    readseq     :  0.476 micros/op;  232.3 MB/s
-    readreverse :  0.724 micros/op;  152.9 MB/s
-
-LevelDB compacts its underlying storage data in the background to
-improve read performance.  The results listed above were done
-immediately after a lot of random writes.  The results after
-compactions (which are usually triggered automatically) are better.
-
-    readrandom  : 11.602 micros/op;  (approximately 85,000 reads per second)
-    readseq     :  0.423 micros/op;  261.8 MB/s
-    readreverse :  0.663 micros/op;  166.9 MB/s
-
-Some of the high cost of reads comes from repeated decompression of blocks
-read from disk.  If we supply enough cache to the leveldb so it can hold the
-uncompressed blocks in memory, the read performance improves again:
-
-    readrandom  : 9.775 micros/op;  (approximately 100,000 reads per second before compaction)
-    readrandom  : 5.215 micros/op;  (approximately 190,000 reads per second after compaction)
-
-## Repository contents
-
-See [doc/index.md](doc/index.md) for more explanation. See
-[doc/impl.md](doc/impl.md) for a brief overview of the implementation.
-
-The public interface is in include/*.h.  Callers should not include or
-rely on the details of any other header files in this package.  Those
-internal APIs may be changed without warning.
-
-Guide to header files:
-
-* **include/db.h**: Main interface to the DB: Start here
-
-* **include/options.h**: Control over the behavior of an entire database,
-and also control over the behavior of individual reads and writes.
-
-* **include/comparator.h**: Abstraction for user-specified comparison function.
-If you want just bytewise comparison of keys, you can use the default
-comparator, but clients can write their own comparator implementations if they
-want custom ordering (e.g. to handle different character encodings, etc.)
-
-* **include/iterator.h**: Interface for iterating over data. You can get
-an iterator from a DB object.
-
-* **include/write_batch.h**: Interface for atomically applying multiple
-updates to a database.
-
-* **include/slice.h**: A simple module for maintaining a pointer and a
-length into some other byte array.
-
-* **include/status.h**: Status is returned from many of the public interfaces
-and is used to report success and various kinds of errors.
-
-* **include/env.h**:
-Abstraction of the OS environment.  A posix implementation of this interface is
-in util/env_posix.cc
-
-* **include/table.h, include/table_builder.h**: Lower-level modules that most
-clients probably won't use directly

src/leveldb/TODO

@@ -1,14 +0,0 @@
-ss
-- Stats
-
-db
-- Maybe implement DB::BulkDeleteForRange(start_key, end_key)
-  that would blow away files whose ranges are entirely contained
-  within [start_key..end_key]?  For Chrome, deletion of obsolete
-  object stores, etc. can be done in the background anyway, so
-  probably not that important.
-- There have been requests for MultiGet.
-
-After a range is completely deleted, what gets rid of the
-corresponding files if we do no future changes to that range.  Make
-the conditions for triggering compactions fire in more situations?

src/leveldb/WINDOWS.md

@@ -1,39 +0,0 @@
-# Building LevelDB On Windows
-
-## Prereqs 
-
-Install the [Windows Software Development Kit version 7.1](http://www.microsoft.com/downloads/dlx/en-us/listdetailsview.aspx?FamilyID=6b6c21d2-2006-4afa-9702-529fa782d63b).
-
-Download and extract the [Snappy source distribution](http://snappy.googlecode.com/files/snappy-1.0.5.tar.gz)
-
-1. Open the "Windows SDK 7.1 Command Prompt" :
-   Start Menu -> "Microsoft Windows SDK v7.1" > "Windows SDK 7.1 Command Prompt"
-2. Change the directory to the leveldb project
-
-## Building the Static lib 
-
-* 32 bit Version 
-
-        setenv /x86
-        msbuild.exe /p:Configuration=Release /p:Platform=Win32 /p:Snappy=..\snappy-1.0.5
-
-* 64 bit Version 
-
-        setenv /x64
-        msbuild.exe /p:Configuration=Release /p:Platform=x64 /p:Snappy=..\snappy-1.0.5
-
-
-## Building and Running the Benchmark app
-
-* 32 bit Version 
-
-	    setenv /x86
-	    msbuild.exe /p:Configuration=Benchmark /p:Platform=Win32 /p:Snappy=..\snappy-1.0.5
-		Benchmark\leveldb.exe
-
-* 64 bit Version 
-
-	    setenv /x64
-	    msbuild.exe /p:Configuration=Benchmark /p:Platform=x64 /p:Snappy=..\snappy-1.0.5
-	    x64\Benchmark\leveldb.exe
-

src/leveldb/build_detect_platform

@@ -1,259 +0,0 @@
-#!/bin/sh
-#
-# Detects OS we're compiling on and outputs a file specified by the first
-# argument, which in turn gets read while processing Makefile.
-#
-# The output will set the following variables:
-#   CC                          C Compiler path
-#   CXX                         C++ Compiler path
-#   PLATFORM_LDFLAGS            Linker flags
-#   PLATFORM_LIBS               Libraries flags
-#   PLATFORM_SHARED_EXT         Extension for shared libraries
-#   PLATFORM_SHARED_LDFLAGS     Flags for building shared library
-#                               This flag is embedded just before the name
-#                               of the shared library without intervening spaces
-#   PLATFORM_SHARED_CFLAGS      Flags for compiling objects for shared library
-#   PLATFORM_CCFLAGS            C compiler flags
-#   PLATFORM_CXXFLAGS           C++ compiler flags.  Will contain:
-#   PLATFORM_SHARED_VERSIONED   Set to 'true' if platform supports versioned
-#                               shared libraries, empty otherwise.
-#
-# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
-#
-#       -DLEVELDB_ATOMIC_PRESENT     if <atomic> is present
-#       -DLEVELDB_PLATFORM_POSIX     for Posix-based platforms
-#       -DSNAPPY                     if the Snappy library is present
-#
-
-OUTPUT=$1
-PREFIX=$2
-if test -z "$OUTPUT" || test -z "$PREFIX"; then
-  echo "usage: $0 <output-filename> <directory_prefix>" >&2
-  exit 1
-fi
-
-# Delete existing output, if it exists
-rm -f $OUTPUT
-touch $OUTPUT
-
-if test -z "$CC"; then
-    CC=cc
-fi
-
-if test -z "$CXX"; then
-    CXX=g++
-fi
-
-if test -z "$TMPDIR"; then
-    TMPDIR=/tmp
-fi
-
-# Detect OS
-if test -z "$TARGET_OS"; then
-    TARGET_OS=`uname -s`
-fi
-
-COMMON_FLAGS=
-CROSS_COMPILE=
-PLATFORM_CCFLAGS=
-PLATFORM_CXXFLAGS=
-PLATFORM_LDFLAGS=
-PLATFORM_LIBS=
-PLATFORM_SHARED_EXT="so"
-PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
-PLATFORM_SHARED_CFLAGS="-fPIC"
-PLATFORM_SHARED_VERSIONED=true
-PLATFORM_SSEFLAGS=
-
-MEMCMP_FLAG=
-if [ "$CXX" = "g++" ]; then
-    # Use libc's memcmp instead of GCC's memcmp.  This results in ~40%
-    # performance improvement on readrandom under gcc 4.4.3 on Linux/x86.
-    MEMCMP_FLAG="-fno-builtin-memcmp"
-fi
-
-case "$TARGET_OS" in
-    CYGWIN_*)
-        PLATFORM=OS_LINUX
-        COMMON_FLAGS="$MEMCMP_FLAG -lpthread -DOS_LINUX -DCYGWIN"
-        PLATFORM_LDFLAGS="-lpthread"
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        ;;
-    Darwin)
-        PLATFORM=OS_MACOSX
-        COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
-        PLATFORM_SHARED_EXT=dylib
-        [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
-        PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name $INSTALL_PATH/"
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        ;;
-    Linux)
-        PLATFORM=OS_LINUX
-        COMMON_FLAGS="$MEMCMP_FLAG -pthread -DOS_LINUX"
-        PLATFORM_LDFLAGS="-pthread"
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        ;;
-    SunOS)
-        PLATFORM=OS_SOLARIS
-        COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS"
-        PLATFORM_LIBS="-lpthread -lrt"
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        ;;
-    FreeBSD)
-        PLATFORM=OS_FREEBSD
-        COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD"
-        PLATFORM_LIBS="-lpthread"
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        ;;
-    GNU/kFreeBSD)
-        PLATFORM=OS_KFREEBSD
-        COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_KFREEBSD"
-        PLATFORM_LIBS="-lpthread"
-        PORT_FILE=port/port_posix.cc
-        ;;
-    NetBSD)
-        PLATFORM=OS_NETBSD
-        COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD"
-        PLATFORM_LIBS="-lpthread -lgcc_s"
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        ;;
-    OpenBSD)
-        PLATFORM=OS_OPENBSD
-        COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_OPENBSD"
-        PLATFORM_LDFLAGS="-pthread"
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        ;;
-    DragonFly)
-        PLATFORM=OS_DRAGONFLYBSD
-        COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD"
-        PLATFORM_LIBS="-lpthread"
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        ;;
-    OS_ANDROID_CROSSCOMPILE)
-        PLATFORM=OS_ANDROID
-        COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
-        PLATFORM_LDFLAGS=""  # All pthread features are in the Android C library
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        CROSS_COMPILE=true
-        ;;
-    HP-UX)
-        PLATFORM=OS_HPUX
-        COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_HPUX"
-        PLATFORM_LDFLAGS="-pthread"
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        # man ld: +h internal_name
-        PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl,"
-        ;;
-    IOS)
-        PLATFORM=IOS
-        COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
-        [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
-        PORT_FILE=port/port_posix.cc
-        PORT_SSE_FILE=port/port_posix_sse.cc
-        PLATFORM_SHARED_EXT=
-        PLATFORM_SHARED_LDFLAGS=
-        PLATFORM_SHARED_CFLAGS=
-        PLATFORM_SHARED_VERSIONED=
-        ;;
-    OS_WINDOWS_CROSSCOMPILE | NATIVE_WINDOWS)
-        PLATFORM=OS_WINDOWS
-        COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_WINDOWS -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1"
-        PLATFORM_SOURCES="util/env_win.cc"
-        PLATFORM_LIBS="-lshlwapi"
-        PORT_FILE=port/port_win.cc
-        CROSS_COMPILE=true
-        ;;
-    *)
-        echo "Unknown platform!" >&2
-        exit 1
-esac
-
-# We want to make a list of all cc files within util, db, table, and helpers
-# except for the test and benchmark files. By default, find will output a list
-# of all files matching either rule, so we need to append -print to make the
-# prune take effect.
-DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
-
-set -f # temporarily disable globbing so that our patterns aren't expanded
-PRUNE_TEST="-name *test*.cc -prune"
-PRUNE_BENCH="-name *_bench.cc -prune"
-PRUNE_TOOL="-name leveldbutil.cc -prune"
-PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
-
-set +f # re-enable globbing
-
-# The sources consist of the portable files, plus the platform-specific port
-# file.
-echo "SOURCES=$PORTABLE_FILES $PORT_FILE $PORT_SSE_FILE" >> $OUTPUT
-echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT
-
-if [ "$CROSS_COMPILE" = "true" ]; then
-    # Cross-compiling; do not try any compilation tests.
-    true
-else
-    CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$"
-
-    # If -std=c++0x works, use <atomic> as fallback for when memory barriers
-    # are not available.
-    $CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null  <<EOF
-      #include <atomic>
-      int main() {}
-EOF
-    if [ "$?" = 0 ]; then
-        COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_ATOMIC_PRESENT"
-        PLATFORM_CXXFLAGS="-std=c++0x"
-    else
-        COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
-    fi
-
-    # Test whether tcmalloc is available
-    $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null  <<EOF
-      int main() {}
-EOF
-    if [ "$?" = 0 ]; then
-        PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
-    fi
-
-    rm -f $CXXOUTPUT 2>/dev/null
-
-    # Test if gcc SSE 4.2 is supported
-    $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -msse4.2 2>/dev/null  <<EOF
-      int main() {}
-EOF
-    if [ "$?" = 0 ]; then
-        PLATFORM_SSEFLAGS="-msse4.2"
-    fi
-
-    rm -f $CXXOUTPUT 2>/dev/null
-fi
-
-# Use the SSE 4.2 CRC32C intrinsics iff runtime checks indicate compiler supports them.
-if [ -n "$PLATFORM_SSEFLAGS" ]; then
-    PLATFORM_SSEFLAGS="$PLATFORM_SSEFLAGS -DLEVELDB_PLATFORM_POSIX_SSE"
-fi
-
-PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
-PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
-
-echo "CC=$CC" >> $OUTPUT
-echo "CXX=$CXX" >> $OUTPUT
-echo "PLATFORM=$PLATFORM" >> $OUTPUT
-echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT
-echo "PLATFORM_LIBS=$PLATFORM_LIBS" >> $OUTPUT
-echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT
-echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT
-echo "PLATFORM_SSEFLAGS=$PLATFORM_SSEFLAGS" >> $OUTPUT
-echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
-echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
-echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
-echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT

src/leveldb/db/autocompact_test.cc

@@ -1,118 +0,0 @@
-// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/db.h"
-#include "db/db_impl.h"
-#include "leveldb/cache.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-class AutoCompactTest {
- public:
-  std::string dbname_;
-  Cache* tiny_cache_;
-  Options options_;
-  DB* db_;
-
-  AutoCompactTest() {
-    dbname_ = test::TmpDir() + "/autocompact_test";
-    tiny_cache_ = NewLRUCache(100);
-    options_.block_cache = tiny_cache_;
-    DestroyDB(dbname_, options_);
-    options_.create_if_missing = true;
-    options_.compression = kNoCompression;
-    ASSERT_OK(DB::Open(options_, dbname_, &db_));
-  }
-
-  ~AutoCompactTest() {
-    delete db_;
-    DestroyDB(dbname_, Options());
-    delete tiny_cache_;
-  }
-
-  std::string Key(int i) {
-    char buf[100];
-    snprintf(buf, sizeof(buf), "key%06d", i);
-    return std::string(buf);
-  }
-
-  uint64_t Size(const Slice& start, const Slice& limit) {
-    Range r(start, limit);
-    uint64_t size;
-    db_->GetApproximateSizes(&r, 1, &size);
-    return size;
-  }
-
-  void DoReads(int n);
-};
-
-static const int kValueSize = 200 * 1024;
-static const int kTotalSize = 100 * 1024 * 1024;
-static const int kCount = kTotalSize / kValueSize;
-
-// Read through the first n keys repeatedly and check that they get
-// compacted (verified by checking the size of the key space).
-void AutoCompactTest::DoReads(int n) {
-  std::string value(kValueSize, 'x');
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-
-  // Fill database
-  for (int i = 0; i < kCount; i++) {
-    ASSERT_OK(db_->Put(WriteOptions(), Key(i), value));
-  }
-  ASSERT_OK(dbi->TEST_CompactMemTable());
-
-  // Delete everything
-  for (int i = 0; i < kCount; i++) {
-    ASSERT_OK(db_->Delete(WriteOptions(), Key(i)));
-  }
-  ASSERT_OK(dbi->TEST_CompactMemTable());
-
-  // Get initial measurement of the space we will be reading.
-  const int64_t initial_size = Size(Key(0), Key(n));
-  const int64_t initial_other_size = Size(Key(n), Key(kCount));
-
-  // Read until size drops significantly.
-  std::string limit_key = Key(n);
-  for (int read = 0; true; read++) {
-    ASSERT_LT(read, 100) << "Taking too long to compact";
-    Iterator* iter = db_->NewIterator(ReadOptions());
-    for (iter->SeekToFirst();
-         iter->Valid() && iter->key().ToString() < limit_key;
-         iter->Next()) {
-      // Drop data
-    }
-    delete iter;
-    // Wait a little bit to allow any triggered compactions to complete.
-    Env::Default()->SleepForMicroseconds(1000000);
-    uint64_t size = Size(Key(0), Key(n));
-    fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n",
-            read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0);
-    if (size <= initial_size/10) {
-      break;
-    }
-  }
-
-  // Verify that the size of the key space not touched by the reads
-  // is pretty much unchanged.
-  const int64_t final_other_size = Size(Key(n), Key(kCount));
-  ASSERT_LE(final_other_size, initial_other_size + 1048576);
-  ASSERT_GE(final_other_size, initial_other_size/5 - 1048576);
-}
-
-TEST(AutoCompactTest, ReadAll) {
-  DoReads(kCount);
-}
-
-TEST(AutoCompactTest, ReadHalf) {
-  DoReads(kCount/2);
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/builder.cc

@@ -1,88 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/builder.h"
-
-#include "db/filename.h"
-#include "db/dbformat.h"
-#include "db/table_cache.h"
-#include "db/version_edit.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-
-namespace leveldb {
-
-Status BuildTable(const std::string& dbname,
-                  Env* env,
-                  const Options& options,
-                  TableCache* table_cache,
-                  Iterator* iter,
-                  FileMetaData* meta) {
-  Status s;
-  meta->file_size = 0;
-  iter->SeekToFirst();
-
-  std::string fname = TableFileName(dbname, meta->number);
-  if (iter->Valid()) {
-    WritableFile* file;
-    s = env->NewWritableFile(fname, &file);
-    if (!s.ok()) {
-      return s;
-    }
-
-    TableBuilder* builder = new TableBuilder(options, file);
-    meta->smallest.DecodeFrom(iter->key());
-    for (; iter->Valid(); iter->Next()) {
-      Slice key = iter->key();
-      meta->largest.DecodeFrom(key);
-      builder->Add(key, iter->value());
-    }
-
-    // Finish and check for builder errors
-    if (s.ok()) {
-      s = builder->Finish();
-      if (s.ok()) {
-        meta->file_size = builder->FileSize();
-        assert(meta->file_size > 0);
-      }
-    } else {
-      builder->Abandon();
-    }
-    delete builder;
-
-    // Finish and check for file errors
-    if (s.ok()) {
-      s = file->Sync();
-    }
-    if (s.ok()) {
-      s = file->Close();
-    }
-    delete file;
-    file = NULL;
-
-    if (s.ok()) {
-      // Verify that the table is usable
-      Iterator* it = table_cache->NewIterator(ReadOptions(),
-                                              meta->number,
-                                              meta->file_size);
-      s = it->status();
-      delete it;
-    }
-  }
-
-  // Check for input iterator errors
-  if (!iter->status().ok()) {
-    s = iter->status();
-  }
-
-  if (s.ok() && meta->file_size > 0) {
-    // Keep it
-  } else {
-    env->DeleteFile(fname);
-  }
-  return s;
-}
-
-}  // namespace leveldb

src/leveldb/db/builder.h

@@ -1,34 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_BUILDER_H_
-#define STORAGE_LEVELDB_DB_BUILDER_H_
-
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-struct Options;
-struct FileMetaData;
-
-class Env;
-class Iterator;
-class TableCache;
-class VersionEdit;
-
-// Build a Table file from the contents of *iter.  The generated file
-// will be named according to meta->number.  On success, the rest of
-// *meta will be filled with metadata about the generated table.
-// If no data is present in *iter, meta->file_size will be set to
-// zero, and no Table file will be produced.
-extern Status BuildTable(const std::string& dbname,
-                         Env* env,
-                         const Options& options,
-                         TableCache* table_cache,
-                         Iterator* iter,
-                         FileMetaData* meta);
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_BUILDER_H_

src/leveldb/db/c.cc

@@ -1,595 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/c.h"
-
-#include <stdlib.h>
-#include <unistd.h>
-#include "leveldb/cache.h"
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/filter_policy.h"
-#include "leveldb/iterator.h"
-#include "leveldb/options.h"
-#include "leveldb/status.h"
-#include "leveldb/write_batch.h"
-
-using leveldb::Cache;
-using leveldb::Comparator;
-using leveldb::CompressionType;
-using leveldb::DB;
-using leveldb::Env;
-using leveldb::FileLock;
-using leveldb::FilterPolicy;
-using leveldb::Iterator;
-using leveldb::kMajorVersion;
-using leveldb::kMinorVersion;
-using leveldb::Logger;
-using leveldb::NewBloomFilterPolicy;
-using leveldb::NewLRUCache;
-using leveldb::Options;
-using leveldb::RandomAccessFile;
-using leveldb::Range;
-using leveldb::ReadOptions;
-using leveldb::SequentialFile;
-using leveldb::Slice;
-using leveldb::Snapshot;
-using leveldb::Status;
-using leveldb::WritableFile;
-using leveldb::WriteBatch;
-using leveldb::WriteOptions;
-
-extern "C" {
-
-struct leveldb_t              { DB*               rep; };
-struct leveldb_iterator_t     { Iterator*         rep; };
-struct leveldb_writebatch_t   { WriteBatch        rep; };
-struct leveldb_snapshot_t     { const Snapshot*   rep; };
-struct leveldb_readoptions_t  { ReadOptions       rep; };
-struct leveldb_writeoptions_t { WriteOptions      rep; };
-struct leveldb_options_t      { Options           rep; };
-struct leveldb_cache_t        { Cache*            rep; };
-struct leveldb_seqfile_t      { SequentialFile*   rep; };
-struct leveldb_randomfile_t   { RandomAccessFile* rep; };
-struct leveldb_writablefile_t { WritableFile*     rep; };
-struct leveldb_logger_t       { Logger*           rep; };
-struct leveldb_filelock_t     { FileLock*         rep; };
-
-struct leveldb_comparator_t : public Comparator {
-  void* state_;
-  void (*destructor_)(void*);
-  int (*compare_)(
-      void*,
-      const char* a, size_t alen,
-      const char* b, size_t blen);
-  const char* (*name_)(void*);
-
-  virtual ~leveldb_comparator_t() {
-    (*destructor_)(state_);
-  }
-
-  virtual int Compare(const Slice& a, const Slice& b) const {
-    return (*compare_)(state_, a.data(), a.size(), b.data(), b.size());
-  }
-
-  virtual const char* Name() const {
-    return (*name_)(state_);
-  }
-
-  // No-ops since the C binding does not support key shortening methods.
-  virtual void FindShortestSeparator(std::string*, const Slice&) const { }
-  virtual void FindShortSuccessor(std::string* key) const { }
-};
-
-struct leveldb_filterpolicy_t : public FilterPolicy {
-  void* state_;
-  void (*destructor_)(void*);
-  const char* (*name_)(void*);
-  char* (*create_)(
-      void*,
-      const char* const* key_array, const size_t* key_length_array,
-      int num_keys,
-      size_t* filter_length);
-  unsigned char (*key_match_)(
-      void*,
-      const char* key, size_t length,
-      const char* filter, size_t filter_length);
-
-  virtual ~leveldb_filterpolicy_t() {
-    (*destructor_)(state_);
-  }
-
-  virtual const char* Name() const {
-    return (*name_)(state_);
-  }
-
-  virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
-    std::vector<const char*> key_pointers(n);
-    std::vector<size_t> key_sizes(n);
-    for (int i = 0; i < n; i++) {
-      key_pointers[i] = keys[i].data();
-      key_sizes[i] = keys[i].size();
-    }
-    size_t len;
-    char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len);
-    dst->append(filter, len);
-    free(filter);
-  }
-
-  virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const {
-    return (*key_match_)(state_, key.data(), key.size(),
-                         filter.data(), filter.size());
-  }
-};
-
-struct leveldb_env_t {
-  Env* rep;
-  bool is_default;
-};
-
-static bool SaveError(char** errptr, const Status& s) {
-  assert(errptr != NULL);
-  if (s.ok()) {
-    return false;
-  } else if (*errptr == NULL) {
-    *errptr = strdup(s.ToString().c_str());
-  } else {
-    // TODO(sanjay): Merge with existing error?
-    free(*errptr);
-    *errptr = strdup(s.ToString().c_str());
-  }
-  return true;
-}
-
-static char* CopyString(const std::string& str) {
-  char* result = reinterpret_cast<char*>(malloc(sizeof(char) * str.size()));
-  memcpy(result, str.data(), sizeof(char) * str.size());
-  return result;
-}
-
-leveldb_t* leveldb_open(
-    const leveldb_options_t* options,
-    const char* name,
-    char** errptr) {
-  DB* db;
-  if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) {
-    return NULL;
-  }
-  leveldb_t* result = new leveldb_t;
-  result->rep = db;
-  return result;
-}
-
-void leveldb_close(leveldb_t* db) {
-  delete db->rep;
-  delete db;
-}
-
-void leveldb_put(
-    leveldb_t* db,
-    const leveldb_writeoptions_t* options,
-    const char* key, size_t keylen,
-    const char* val, size_t vallen,
-    char** errptr) {
-  SaveError(errptr,
-            db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
-}
-
-void leveldb_delete(
-    leveldb_t* db,
-    const leveldb_writeoptions_t* options,
-    const char* key, size_t keylen,
-    char** errptr) {
-  SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen)));
-}
-
-
-void leveldb_write(
-    leveldb_t* db,
-    const leveldb_writeoptions_t* options,
-    leveldb_writebatch_t* batch,
-    char** errptr) {
-  SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
-}
-
-char* leveldb_get(
-    leveldb_t* db,
-    const leveldb_readoptions_t* options,
-    const char* key, size_t keylen,
-    size_t* vallen,
-    char** errptr) {
-  char* result = NULL;
-  std::string tmp;
-  Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
-  if (s.ok()) {
-    *vallen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vallen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
-leveldb_iterator_t* leveldb_create_iterator(
-    leveldb_t* db,
-    const leveldb_readoptions_t* options) {
-  leveldb_iterator_t* result = new leveldb_iterator_t;
-  result->rep = db->rep->NewIterator(options->rep);
-  return result;
-}
-
-const leveldb_snapshot_t* leveldb_create_snapshot(
-    leveldb_t* db) {
-  leveldb_snapshot_t* result = new leveldb_snapshot_t;
-  result->rep = db->rep->GetSnapshot();
-  return result;
-}
-
-void leveldb_release_snapshot(
-    leveldb_t* db,
-    const leveldb_snapshot_t* snapshot) {
-  db->rep->ReleaseSnapshot(snapshot->rep);
-  delete snapshot;
-}
-
-char* leveldb_property_value(
-    leveldb_t* db,
-    const char* propname) {
-  std::string tmp;
-  if (db->rep->GetProperty(Slice(propname), &tmp)) {
-    // We use strdup() since we expect human readable output.
-    return strdup(tmp.c_str());
-  } else {
-    return NULL;
-  }
-}
-
-void leveldb_approximate_sizes(
-    leveldb_t* db,
-    int num_ranges,
-    const char* const* range_start_key, const size_t* range_start_key_len,
-    const char* const* range_limit_key, const size_t* range_limit_key_len,
-    uint64_t* sizes) {
-  Range* ranges = new Range[num_ranges];
-  for (int i = 0; i < num_ranges; i++) {
-    ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]);
-    ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]);
-  }
-  db->rep->GetApproximateSizes(ranges, num_ranges, sizes);
-  delete[] ranges;
-}
-
-void leveldb_compact_range(
-    leveldb_t* db,
-    const char* start_key, size_t start_key_len,
-    const char* limit_key, size_t limit_key_len) {
-  Slice a, b;
-  db->rep->CompactRange(
-      // Pass NULL Slice if corresponding "const char*" is NULL
-      (start_key ? (a = Slice(start_key, start_key_len), &a) : NULL),
-      (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : NULL));
-}
-
-void leveldb_destroy_db(
-    const leveldb_options_t* options,
-    const char* name,
-    char** errptr) {
-  SaveError(errptr, DestroyDB(name, options->rep));
-}
-
-void leveldb_repair_db(
-    const leveldb_options_t* options,
-    const char* name,
-    char** errptr) {
-  SaveError(errptr, RepairDB(name, options->rep));
-}
-
-void leveldb_iter_destroy(leveldb_iterator_t* iter) {
-  delete iter->rep;
-  delete iter;
-}
-
-unsigned char leveldb_iter_valid(const leveldb_iterator_t* iter) {
-  return iter->rep->Valid();
-}
-
-void leveldb_iter_seek_to_first(leveldb_iterator_t* iter) {
-  iter->rep->SeekToFirst();
-}
-
-void leveldb_iter_seek_to_last(leveldb_iterator_t* iter) {
-  iter->rep->SeekToLast();
-}
-
-void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) {
-  iter->rep->Seek(Slice(k, klen));
-}
-
-void leveldb_iter_next(leveldb_iterator_t* iter) {
-  iter->rep->Next();
-}
-
-void leveldb_iter_prev(leveldb_iterator_t* iter) {
-  iter->rep->Prev();
-}
-
-const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) {
-  Slice s = iter->rep->key();
-  *klen = s.size();
-  return s.data();
-}
-
-const char* leveldb_iter_value(const leveldb_iterator_t* iter, size_t* vlen) {
-  Slice s = iter->rep->value();
-  *vlen = s.size();
-  return s.data();
-}
-
-void leveldb_iter_get_error(const leveldb_iterator_t* iter, char** errptr) {
-  SaveError(errptr, iter->rep->status());
-}
-
-leveldb_writebatch_t* leveldb_writebatch_create() {
-  return new leveldb_writebatch_t;
-}
-
-void leveldb_writebatch_destroy(leveldb_writebatch_t* b) {
-  delete b;
-}
-
-void leveldb_writebatch_clear(leveldb_writebatch_t* b) {
-  b->rep.Clear();
-}
-
-void leveldb_writebatch_put(
-    leveldb_writebatch_t* b,
-    const char* key, size_t klen,
-    const char* val, size_t vlen) {
-  b->rep.Put(Slice(key, klen), Slice(val, vlen));
-}
-
-void leveldb_writebatch_delete(
-    leveldb_writebatch_t* b,
-    const char* key, size_t klen) {
-  b->rep.Delete(Slice(key, klen));
-}
-
-void leveldb_writebatch_iterate(
-    leveldb_writebatch_t* b,
-    void* state,
-    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
-    void (*deleted)(void*, const char* k, size_t klen)) {
-  class H : public WriteBatch::Handler {
-   public:
-    void* state_;
-    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-    void (*deleted_)(void*, const char* k, size_t klen);
-    virtual void Put(const Slice& key, const Slice& value) {
-      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-    }
-    virtual void Delete(const Slice& key) {
-      (*deleted_)(state_, key.data(), key.size());
-    }
-  };
-  H handler;
-  handler.state_ = state;
-  handler.put_ = put;
-  handler.deleted_ = deleted;
-  b->rep.Iterate(&handler);
-}
-
-leveldb_options_t* leveldb_options_create() {
-  return new leveldb_options_t;
-}
-
-void leveldb_options_destroy(leveldb_options_t* options) {
-  delete options;
-}
-
-void leveldb_options_set_comparator(
-    leveldb_options_t* opt,
-    leveldb_comparator_t* cmp) {
-  opt->rep.comparator = cmp;
-}
-
-void leveldb_options_set_filter_policy(
-    leveldb_options_t* opt,
-    leveldb_filterpolicy_t* policy) {
-  opt->rep.filter_policy = policy;
-}
-
-void leveldb_options_set_create_if_missing(
-    leveldb_options_t* opt, unsigned char v) {
-  opt->rep.create_if_missing = v;
-}
-
-void leveldb_options_set_error_if_exists(
-    leveldb_options_t* opt, unsigned char v) {
-  opt->rep.error_if_exists = v;
-}
-
-void leveldb_options_set_paranoid_checks(
-    leveldb_options_t* opt, unsigned char v) {
-  opt->rep.paranoid_checks = v;
-}
-
-void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) {
-  opt->rep.env = (env ? env->rep : NULL);
-}
-
-void leveldb_options_set_info_log(leveldb_options_t* opt, leveldb_logger_t* l) {
-  opt->rep.info_log = (l ? l->rep : NULL);
-}
-
-void leveldb_options_set_write_buffer_size(leveldb_options_t* opt, size_t s) {
-  opt->rep.write_buffer_size = s;
-}
-
-void leveldb_options_set_max_open_files(leveldb_options_t* opt, int n) {
-  opt->rep.max_open_files = n;
-}
-
-void leveldb_options_set_cache(leveldb_options_t* opt, leveldb_cache_t* c) {
-  opt->rep.block_cache = c->rep;
-}
-
-void leveldb_options_set_block_size(leveldb_options_t* opt, size_t s) {
-  opt->rep.block_size = s;
-}
-
-void leveldb_options_set_block_restart_interval(leveldb_options_t* opt, int n) {
-  opt->rep.block_restart_interval = n;
-}
-
-void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
-  opt->rep.compression = static_cast<CompressionType>(t);
-}
-
-leveldb_comparator_t* leveldb_comparator_create(
-    void* state,
-    void (*destructor)(void*),
-    int (*compare)(
-        void*,
-        const char* a, size_t alen,
-        const char* b, size_t blen),
-    const char* (*name)(void*)) {
-  leveldb_comparator_t* result = new leveldb_comparator_t;
-  result->state_ = state;
-  result->destructor_ = destructor;
-  result->compare_ = compare;
-  result->name_ = name;
-  return result;
-}
-
-void leveldb_comparator_destroy(leveldb_comparator_t* cmp) {
-  delete cmp;
-}
-
-leveldb_filterpolicy_t* leveldb_filterpolicy_create(
-    void* state,
-    void (*destructor)(void*),
-    char* (*create_filter)(
-        void*,
-        const char* const* key_array, const size_t* key_length_array,
-        int num_keys,
-        size_t* filter_length),
-    unsigned char (*key_may_match)(
-        void*,
-        const char* key, size_t length,
-        const char* filter, size_t filter_length),
-    const char* (*name)(void*)) {
-  leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t;
-  result->state_ = state;
-  result->destructor_ = destructor;
-  result->create_ = create_filter;
-  result->key_match_ = key_may_match;
-  result->name_ = name;
-  return result;
-}
-
-void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t* filter) {
-  delete filter;
-}
-
-leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
-  // Make a leveldb_filterpolicy_t, but override all of its methods so
-  // they delegate to a NewBloomFilterPolicy() instead of user
-  // supplied C functions.
-  struct Wrapper : public leveldb_filterpolicy_t {
-    const FilterPolicy* rep_;
-    ~Wrapper() { delete rep_; }
-    const char* Name() const { return rep_->Name(); }
-    void CreateFilter(const Slice* keys, int n, std::string* dst) const {
-      return rep_->CreateFilter(keys, n, dst);
-    }
-    bool KeyMayMatch(const Slice& key, const Slice& filter) const {
-      return rep_->KeyMayMatch(key, filter);
-    }
-    static void DoNothing(void*) { }
-  };
-  Wrapper* wrapper = new Wrapper;
-  wrapper->rep_ = NewBloomFilterPolicy(bits_per_key);
-  wrapper->state_ = NULL;
-  wrapper->destructor_ = &Wrapper::DoNothing;
-  return wrapper;
-}
-
-leveldb_readoptions_t* leveldb_readoptions_create() {
-  return new leveldb_readoptions_t;
-}
-
-void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) {
-  delete opt;
-}
-
-void leveldb_readoptions_set_verify_checksums(
-    leveldb_readoptions_t* opt,
-    unsigned char v) {
-  opt->rep.verify_checksums = v;
-}
-
-void leveldb_readoptions_set_fill_cache(
-    leveldb_readoptions_t* opt, unsigned char v) {
-  opt->rep.fill_cache = v;
-}
-
-void leveldb_readoptions_set_snapshot(
-    leveldb_readoptions_t* opt,
-    const leveldb_snapshot_t* snap) {
-  opt->rep.snapshot = (snap ? snap->rep : NULL);
-}
-
-leveldb_writeoptions_t* leveldb_writeoptions_create() {
-  return new leveldb_writeoptions_t;
-}
-
-void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) {
-  delete opt;
-}
-
-void leveldb_writeoptions_set_sync(
-    leveldb_writeoptions_t* opt, unsigned char v) {
-  opt->rep.sync = v;
-}
-
-leveldb_cache_t* leveldb_cache_create_lru(size_t capacity) {
-  leveldb_cache_t* c = new leveldb_cache_t;
-  c->rep = NewLRUCache(capacity);
-  return c;
-}
-
-void leveldb_cache_destroy(leveldb_cache_t* cache) {
-  delete cache->rep;
-  delete cache;
-}
-
-leveldb_env_t* leveldb_create_default_env() {
-  leveldb_env_t* result = new leveldb_env_t;
-  result->rep = Env::Default();
-  result->is_default = true;
-  return result;
-}
-
-void leveldb_env_destroy(leveldb_env_t* env) {
-  if (!env->is_default) delete env->rep;
-  delete env;
-}
-
-void leveldb_free(void* ptr) {
-  free(ptr);
-}
-
-int leveldb_major_version() {
-  return kMajorVersion;
-}
-
-int leveldb_minor_version() {
-  return kMinorVersion;
-}
-
-}  // end extern "C"

src/leveldb/db/c_test.c

@@ -1,390 +0,0 @@
-/* Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-   Use of this source code is governed by a BSD-style license that can be
-   found in the LICENSE file. See the AUTHORS file for names of contributors. */
-
-#include "leveldb/c.h"
-
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-const char* phase = "";
-static char dbname[200];
-
-static void StartPhase(const char* name) {
-  fprintf(stderr, "=== Test %s\n", name);
-  phase = name;
-}
-
-static const char* GetTempDir(void) {
-    const char* ret = getenv("TEST_TMPDIR");
-    if (ret == NULL || ret[0] == '\0')
-        ret = "/tmp";
-    return ret;
-}
-
-#define CheckNoError(err)                                               \
-  if ((err) != NULL) {                                                  \
-    fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \
-    abort();                                                            \
-  }
-
-#define CheckCondition(cond)                                            \
-  if (!(cond)) {                                                        \
-    fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \
-    abort();                                                            \
-  }
-
-static void CheckEqual(const char* expected, const char* v, size_t n) {
-  if (expected == NULL && v == NULL) {
-    // ok
-  } else if (expected != NULL && v != NULL && n == strlen(expected) &&
-             memcmp(expected, v, n) == 0) {
-    // ok
-    return;
-  } else {
-    fprintf(stderr, "%s: expected '%s', got '%s'\n",
-            phase,
-            (expected ? expected : "(null)"),
-            (v ? v : "(null"));
-    abort();
-  }
-}
-
-static void Free(char** ptr) {
-  if (*ptr) {
-    free(*ptr);
-    *ptr = NULL;
-  }
-}
-
-static void CheckGet(
-    leveldb_t* db,
-    const leveldb_readoptions_t* options,
-    const char* key,
-    const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  char* val;
-  val = leveldb_get(db, options, key, strlen(key), &val_len, &err);
-  CheckNoError(err);
-  CheckEqual(expected, val, val_len);
-  Free(&val);
-}
-
-static void CheckIter(leveldb_iterator_t* iter,
-                      const char* key, const char* val) {
-  size_t len;
-  const char* str;
-  str = leveldb_iter_key(iter, &len);
-  CheckEqual(key, str, len);
-  str = leveldb_iter_value(iter, &len);
-  CheckEqual(val, str, len);
-}
-
-// Callback from leveldb_writebatch_iterate()
-static void CheckPut(void* ptr,
-                     const char* k, size_t klen,
-                     const char* v, size_t vlen) {
-  int* state = (int*) ptr;
-  CheckCondition(*state < 2);
-  switch (*state) {
-    case 0:
-      CheckEqual("bar", k, klen);
-      CheckEqual("b", v, vlen);
-      break;
-    case 1:
-      CheckEqual("box", k, klen);
-      CheckEqual("c", v, vlen);
-      break;
-  }
-  (*state)++;
-}
-
-// Callback from leveldb_writebatch_iterate()
-static void CheckDel(void* ptr, const char* k, size_t klen) {
-  int* state = (int*) ptr;
-  CheckCondition(*state == 2);
-  CheckEqual("bar", k, klen);
-  (*state)++;
-}
-
-static void CmpDestroy(void* arg) { }
-
-static int CmpCompare(void* arg, const char* a, size_t alen,
-                      const char* b, size_t blen) {
-  int n = (alen < blen) ? alen : blen;
-  int r = memcmp(a, b, n);
-  if (r == 0) {
-    if (alen < blen) r = -1;
-    else if (alen > blen) r = +1;
-  }
-  return r;
-}
-
-static const char* CmpName(void* arg) {
-  return "foo";
-}
-
-// Custom filter policy
-static unsigned char fake_filter_result = 1;
-static void FilterDestroy(void* arg) { }
-static const char* FilterName(void* arg) {
-  return "TestFilter";
-}
-static char* FilterCreate(
-    void* arg,
-    const char* const* key_array, const size_t* key_length_array,
-    int num_keys,
-    size_t* filter_length) {
-  *filter_length = 4;
-  char* result = malloc(4);
-  memcpy(result, "fake", 4);
-  return result;
-}
-unsigned char FilterKeyMatch(
-    void* arg,
-    const char* key, size_t length,
-    const char* filter, size_t filter_length) {
-  CheckCondition(filter_length == 4);
-  CheckCondition(memcmp(filter, "fake", 4) == 0);
-  return fake_filter_result;
-}
-
-int main(int argc, char** argv) {
-  leveldb_t* db;
-  leveldb_comparator_t* cmp;
-  leveldb_cache_t* cache;
-  leveldb_env_t* env;
-  leveldb_options_t* options;
-  leveldb_readoptions_t* roptions;
-  leveldb_writeoptions_t* woptions;
-  char* err = NULL;
-  int run = -1;
-
-  CheckCondition(leveldb_major_version() >= 1);
-  CheckCondition(leveldb_minor_version() >= 1);
-
-  snprintf(dbname, sizeof(dbname),
-           "%s/leveldb_c_test-%d",
-           GetTempDir(),
-           ((int) geteuid()));
-
-  StartPhase("create_objects");
-  cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
-  env = leveldb_create_default_env();
-  cache = leveldb_cache_create_lru(100000);
-
-  options = leveldb_options_create();
-  leveldb_options_set_comparator(options, cmp);
-  leveldb_options_set_error_if_exists(options, 1);
-  leveldb_options_set_cache(options, cache);
-  leveldb_options_set_env(options, env);
-  leveldb_options_set_info_log(options, NULL);
-  leveldb_options_set_write_buffer_size(options, 100000);
-  leveldb_options_set_paranoid_checks(options, 1);
-  leveldb_options_set_max_open_files(options, 10);
-  leveldb_options_set_block_size(options, 1024);
-  leveldb_options_set_block_restart_interval(options, 8);
-  leveldb_options_set_compression(options, leveldb_no_compression);
-
-  roptions = leveldb_readoptions_create();
-  leveldb_readoptions_set_verify_checksums(roptions, 1);
-  leveldb_readoptions_set_fill_cache(roptions, 0);
-
-  woptions = leveldb_writeoptions_create();
-  leveldb_writeoptions_set_sync(woptions, 1);
-
-  StartPhase("destroy");
-  leveldb_destroy_db(options, dbname, &err);
-  Free(&err);
-
-  StartPhase("open_error");
-  db = leveldb_open(options, dbname, &err);
-  CheckCondition(err != NULL);
-  Free(&err);
-
-  StartPhase("leveldb_free");
-  db = leveldb_open(options, dbname, &err);
-  CheckCondition(err != NULL);
-  leveldb_free(err);
-  err = NULL;
-
-  StartPhase("open");
-  leveldb_options_set_create_if_missing(options, 1);
-  db = leveldb_open(options, dbname, &err);
-  CheckNoError(err);
-  CheckGet(db, roptions, "foo", NULL);
-
-  StartPhase("put");
-  leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
-  CheckNoError(err);
-  CheckGet(db, roptions, "foo", "hello");
-
-  StartPhase("compactall");
-  leveldb_compact_range(db, NULL, 0, NULL, 0);
-  CheckGet(db, roptions, "foo", "hello");
-
-  StartPhase("compactrange");
-  leveldb_compact_range(db, "a", 1, "z", 1);
-  CheckGet(db, roptions, "foo", "hello");
-
-  StartPhase("writebatch");
-  {
-    leveldb_writebatch_t* wb = leveldb_writebatch_create();
-    leveldb_writebatch_put(wb, "foo", 3, "a", 1);
-    leveldb_writebatch_clear(wb);
-    leveldb_writebatch_put(wb, "bar", 3, "b", 1);
-    leveldb_writebatch_put(wb, "box", 3, "c", 1);
-    leveldb_writebatch_delete(wb, "bar", 3);
-    leveldb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "foo", "hello");
-    CheckGet(db, roptions, "bar", NULL);
-    CheckGet(db, roptions, "box", "c");
-    int pos = 0;
-    leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
-    CheckCondition(pos == 3);
-    leveldb_writebatch_destroy(wb);
-  }
-
-  StartPhase("iter");
-  {
-    leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
-    CheckCondition(!leveldb_iter_valid(iter));
-    leveldb_iter_seek_to_first(iter);
-    CheckCondition(leveldb_iter_valid(iter));
-    CheckIter(iter, "box", "c");
-    leveldb_iter_next(iter);
-    CheckIter(iter, "foo", "hello");
-    leveldb_iter_prev(iter);
-    CheckIter(iter, "box", "c");
-    leveldb_iter_prev(iter);
-    CheckCondition(!leveldb_iter_valid(iter));
-    leveldb_iter_seek_to_last(iter);
-    CheckIter(iter, "foo", "hello");
-    leveldb_iter_seek(iter, "b", 1);
-    CheckIter(iter, "box", "c");
-    leveldb_iter_get_error(iter, &err);
-    CheckNoError(err);
-    leveldb_iter_destroy(iter);
-  }
-
-  StartPhase("approximate_sizes");
-  {
-    int i;
-    int n = 20000;
-    char keybuf[100];
-    char valbuf[100];
-    uint64_t sizes[2];
-    const char* start[2] = { "a", "k00000000000000010000" };
-    size_t start_len[2] = { 1, 21 };
-    const char* limit[2] = { "k00000000000000010000", "z" };
-    size_t limit_len[2] = { 21, 1 };
-    leveldb_writeoptions_set_sync(woptions, 0);
-    for (i = 0; i < n; i++) {
-      snprintf(keybuf, sizeof(keybuf), "k%020d", i);
-      snprintf(valbuf, sizeof(valbuf), "v%020d", i);
-      leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
-                  &err);
-      CheckNoError(err);
-    }
-    leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
-    CheckCondition(sizes[0] > 0);
-    CheckCondition(sizes[1] > 0);
-  }
-
-  StartPhase("property");
-  {
-    char* prop = leveldb_property_value(db, "nosuchprop");
-    CheckCondition(prop == NULL);
-    prop = leveldb_property_value(db, "leveldb.stats");
-    CheckCondition(prop != NULL);
-    Free(&prop);
-  }
-
-  StartPhase("snapshot");
-  {
-    const leveldb_snapshot_t* snap;
-    snap = leveldb_create_snapshot(db);
-    leveldb_delete(db, woptions, "foo", 3, &err);
-    CheckNoError(err);
-    leveldb_readoptions_set_snapshot(roptions, snap);
-    CheckGet(db, roptions, "foo", "hello");
-    leveldb_readoptions_set_snapshot(roptions, NULL);
-    CheckGet(db, roptions, "foo", NULL);
-    leveldb_release_snapshot(db, snap);
-  }
-
-  StartPhase("repair");
-  {
-    leveldb_close(db);
-    leveldb_options_set_create_if_missing(options, 0);
-    leveldb_options_set_error_if_exists(options, 0);
-    leveldb_repair_db(options, dbname, &err);
-    CheckNoError(err);
-    db = leveldb_open(options, dbname, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "foo", NULL);
-    CheckGet(db, roptions, "bar", NULL);
-    CheckGet(db, roptions, "box", "c");
-    leveldb_options_set_create_if_missing(options, 1);
-    leveldb_options_set_error_if_exists(options, 1);
-  }
-
-  StartPhase("filter");
-  for (run = 0; run < 2; run++) {
-    // First run uses custom filter, second run uses bloom filter
-    CheckNoError(err);
-    leveldb_filterpolicy_t* policy;
-    if (run == 0) {
-      policy = leveldb_filterpolicy_create(
-          NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName);
-    } else {
-      policy = leveldb_filterpolicy_create_bloom(10);
-    }
-
-    // Create new database
-    leveldb_close(db);
-    leveldb_destroy_db(options, dbname, &err);
-    leveldb_options_set_filter_policy(options, policy);
-    db = leveldb_open(options, dbname, &err);
-    CheckNoError(err);
-    leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err);
-    CheckNoError(err);
-    leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
-    CheckNoError(err);
-    leveldb_compact_range(db, NULL, 0, NULL, 0);
-
-    fake_filter_result = 1;
-    CheckGet(db, roptions, "foo", "foovalue");
-    CheckGet(db, roptions, "bar", "barvalue");
-    if (phase == 0) {
-      // Must not find value when custom filter returns false
-      fake_filter_result = 0;
-      CheckGet(db, roptions, "foo", NULL);
-      CheckGet(db, roptions, "bar", NULL);
-      fake_filter_result = 1;
-
-      CheckGet(db, roptions, "foo", "foovalue");
-      CheckGet(db, roptions, "bar", "barvalue");
-    }
-    leveldb_options_set_filter_policy(options, NULL);
-    leveldb_filterpolicy_destroy(policy);
-  }
-
-  StartPhase("cleanup");
-  leveldb_close(db);
-  leveldb_options_destroy(options);
-  leveldb_readoptions_destroy(roptions);
-  leveldb_writeoptions_destroy(woptions);
-  leveldb_cache_destroy(cache);
-  leveldb_comparator_destroy(cmp);
-  leveldb_env_destroy(env);
-
-  fprintf(stderr, "PASS\n");
-  return 0;
-}

src/leveldb/db/corruption_test.cc

@@ -1,374 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/db.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include "leveldb/cache.h"
-#include "leveldb/env.h"
-#include "leveldb/table.h"
-#include "leveldb/write_batch.h"
-#include "db/db_impl.h"
-#include "db/filename.h"
-#include "db/log_format.h"
-#include "db/version_set.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-static const int kValueSize = 1000;
-
-class CorruptionTest {
- public:
-  test::ErrorEnv env_;
-  std::string dbname_;
-  Cache* tiny_cache_;
-  Options options_;
-  DB* db_;
-
-  CorruptionTest() {
-    tiny_cache_ = NewLRUCache(100);
-    options_.env = &env_;
-    options_.block_cache = tiny_cache_;
-    dbname_ = test::TmpDir() + "/corruption_test";
-    DestroyDB(dbname_, options_);
-
-    db_ = NULL;
-    options_.create_if_missing = true;
-    Reopen();
-    options_.create_if_missing = false;
-  }
-
-  ~CorruptionTest() {
-     delete db_;
-     DestroyDB(dbname_, Options());
-     delete tiny_cache_;
-  }
-
-  Status TryReopen() {
-    delete db_;
-    db_ = NULL;
-    return DB::Open(options_, dbname_, &db_);
-  }
-
-  void Reopen() {
-    ASSERT_OK(TryReopen());
-  }
-
-  void RepairDB() {
-    delete db_;
-    db_ = NULL;
-    ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
-  }
-
-  void Build(int n) {
-    std::string key_space, value_space;
-    WriteBatch batch;
-    for (int i = 0; i < n; i++) {
-      //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
-      Slice key = Key(i, &key_space);
-      batch.Clear();
-      batch.Put(key, Value(i, &value_space));
-      WriteOptions options;
-      // Corrupt() doesn't work without this sync on windows; stat reports 0 for
-      // the file size.
-      if (i == n - 1) {
-        options.sync = true;
-      }
-      ASSERT_OK(db_->Write(options, &batch));
-    }
-  }
-
-  void Check(int min_expected, int max_expected) {
-    int next_expected = 0;
-    int missed = 0;
-    int bad_keys = 0;
-    int bad_values = 0;
-    int correct = 0;
-    std::string value_space;
-    Iterator* iter = db_->NewIterator(ReadOptions());
-    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-      uint64_t key;
-      Slice in(iter->key());
-      if (in == "" || in == "~") {
-        // Ignore boundary keys.
-        continue;
-      }
-      if (!ConsumeDecimalNumber(&in, &key) ||
-          !in.empty() ||
-          key < next_expected) {
-        bad_keys++;
-        continue;
-      }
-      missed += (key - next_expected);
-      next_expected = key + 1;
-      if (iter->value() != Value(key, &value_space)) {
-        bad_values++;
-      } else {
-        correct++;
-      }
-    }
-    delete iter;
-
-    fprintf(stderr,
-            "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n",
-            min_expected, max_expected, correct, bad_keys, bad_values, missed);
-    ASSERT_LE(min_expected, correct);
-    ASSERT_GE(max_expected, correct);
-  }
-
-  void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
-    // Pick file to corrupt
-    std::vector<std::string> filenames;
-    ASSERT_OK(env_.GetChildren(dbname_, &filenames));
-    uint64_t number;
-    FileType type;
-    std::string fname;
-    int picked_number = -1;
-    for (size_t i = 0; i < filenames.size(); i++) {
-      if (ParseFileName(filenames[i], &number, &type) &&
-          type == filetype &&
-          int(number) > picked_number) {  // Pick latest file
-        fname = dbname_ + "/" + filenames[i];
-        picked_number = number;
-      }
-    }
-    ASSERT_TRUE(!fname.empty()) << filetype;
-
-    struct stat sbuf;
-    if (stat(fname.c_str(), &sbuf) != 0) {
-      const char* msg = strerror(errno);
-      ASSERT_TRUE(false) << fname << ": " << msg;
-    }
-
-    if (offset < 0) {
-      // Relative to end of file; make it absolute
-      if (-offset > sbuf.st_size) {
-        offset = 0;
-      } else {
-        offset = sbuf.st_size + offset;
-      }
-    }
-    if (offset > sbuf.st_size) {
-      offset = sbuf.st_size;
-    }
-    if (offset + bytes_to_corrupt > sbuf.st_size) {
-      bytes_to_corrupt = sbuf.st_size - offset;
-    }
-
-    // Do it
-    std::string contents;
-    Status s = ReadFileToString(Env::Default(), fname, &contents);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    for (int i = 0; i < bytes_to_corrupt; i++) {
-      contents[i + offset] ^= 0x80;
-    }
-    s = WriteStringToFile(Env::Default(), contents, fname);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-  }
-
-  int Property(const std::string& name) {
-    std::string property;
-    int result;
-    if (db_->GetProperty(name, &property) &&
-        sscanf(property.c_str(), "%d", &result) == 1) {
-      return result;
-    } else {
-      return -1;
-    }
-  }
-
-  // Return the ith key
-  Slice Key(int i, std::string* storage) {
-    char buf[100];
-    snprintf(buf, sizeof(buf), "%016d", i);
-    storage->assign(buf, strlen(buf));
-    return Slice(*storage);
-  }
-
-  // Return the value to associate with the specified key
-  Slice Value(int k, std::string* storage) {
-    Random r(k);
-    return test::RandomString(&r, kValueSize, storage);
-  }
-};
-
-TEST(CorruptionTest, Recovery) {
-  Build(100);
-  Check(100, 100);
-  Corrupt(kLogFile, 19, 1);      // WriteBatch tag for first record
-  Corrupt(kLogFile, log::kBlockSize + 1000, 1);  // Somewhere in second block
-  Reopen();
-
-  // The 64 records in the first two log blocks are completely lost.
-  Check(36, 36);
-}
-
-TEST(CorruptionTest, RecoverWriteError) {
-  env_.writable_file_error_ = true;
-  Status s = TryReopen();
-  ASSERT_TRUE(!s.ok());
-}
-
-TEST(CorruptionTest, NewFileErrorDuringWrite) {
-  // Do enough writing to force minor compaction
-  env_.writable_file_error_ = true;
-  const int num = 3 + (Options().write_buffer_size / kValueSize);
-  std::string value_storage;
-  Status s;
-  for (int i = 0; s.ok() && i < num; i++) {
-    WriteBatch batch;
-    batch.Put("a", Value(100, &value_storage));
-    s = db_->Write(WriteOptions(), &batch);
-  }
-  ASSERT_TRUE(!s.ok());
-  ASSERT_GE(env_.num_writable_file_errors_, 1);
-  env_.writable_file_error_ = false;
-  Reopen();
-}
-
-TEST(CorruptionTest, TableFile) {
-  Build(100);
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
-  dbi->TEST_CompactRange(0, NULL, NULL);
-  dbi->TEST_CompactRange(1, NULL, NULL);
-
-  Corrupt(kTableFile, 100, 1);
-  Check(90, 99);
-}
-
-TEST(CorruptionTest, TableFileRepair) {
-  options_.block_size = 2 * kValueSize;  // Limit scope of corruption
-  options_.paranoid_checks = true;
-  Reopen();
-  Build(100);
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
-  dbi->TEST_CompactRange(0, NULL, NULL);
-  dbi->TEST_CompactRange(1, NULL, NULL);
-
-  Corrupt(kTableFile, 100, 1);
-  RepairDB();
-  Reopen();
-  Check(95, 99);
-}
-
-TEST(CorruptionTest, TableFileIndexData) {
-  Build(10000);  // Enough to build multiple Tables
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
-
-  Corrupt(kTableFile, -2000, 500);
-  Reopen();
-  Check(5000, 9999);
-}
-
-TEST(CorruptionTest, MissingDescriptor) {
-  Build(1000);
-  RepairDB();
-  Reopen();
-  Check(1000, 1000);
-}
-
-TEST(CorruptionTest, SequenceNumberRecovery) {
-  ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
-  ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
-  ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3"));
-  ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4"));
-  ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5"));
-  RepairDB();
-  Reopen();
-  std::string v;
-  ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
-  ASSERT_EQ("v5", v);
-  // Write something.  If sequence number was not recovered properly,
-  // it will be hidden by an earlier write.
-  ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6"));
-  ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
-  ASSERT_EQ("v6", v);
-  Reopen();
-  ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
-  ASSERT_EQ("v6", v);
-}
-
-TEST(CorruptionTest, CorruptedDescriptor) {
-  ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
-  dbi->TEST_CompactRange(0, NULL, NULL);
-
-  Corrupt(kDescriptorFile, 0, 1000);
-  Status s = TryReopen();
-  ASSERT_TRUE(!s.ok());
-
-  RepairDB();
-  Reopen();
-  std::string v;
-  ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
-  ASSERT_EQ("hello", v);
-}
-
-TEST(CorruptionTest, CompactionInputError) {
-  Build(10);
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
-  const int last = config::kMaxMemCompactLevel;
-  ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
-
-  Corrupt(kTableFile, 100, 1);
-  Check(5, 9);
-
-  // Force compactions by writing lots of values
-  Build(10000);
-  Check(10000, 10000);
-}
-
-TEST(CorruptionTest, CompactionInputErrorParanoid) {
-  options_.paranoid_checks = true;
-  options_.write_buffer_size = 512 << 10;
-  Reopen();
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-
-  // Make multiple inputs so we need to compact.
-  for (int i = 0; i < 2; i++) {
-    Build(10);
-    dbi->TEST_CompactMemTable();
-    Corrupt(kTableFile, 100, 1);
-    env_.SleepForMicroseconds(100000);
-  }
-  dbi->CompactRange(NULL, NULL);
-
-  // Write must fail because of corrupted table
-  std::string tmp1, tmp2;
-  Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2));
-  ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
-}
-
-TEST(CorruptionTest, UnrelatedKeys) {
-  Build(10);
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
-  Corrupt(kTableFile, 100, 1);
-
-  std::string tmp1, tmp2;
-  ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
-  std::string v;
-  ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
-  ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
-  dbi->TEST_CompactMemTable();
-  ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
-  ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/db_impl.h

@@ -1,211 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_
-#define STORAGE_LEVELDB_DB_DB_IMPL_H_
-
-#include <deque>
-#include <set>
-#include "db/dbformat.h"
-#include "db/log_writer.h"
-#include "db/snapshot.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "port/port.h"
-#include "port/thread_annotations.h"
-
-namespace leveldb {
-
-class MemTable;
-class TableCache;
-class Version;
-class VersionEdit;
-class VersionSet;
-
-class DBImpl : public DB {
- public:
-  DBImpl(const Options& options, const std::string& dbname);
-  virtual ~DBImpl();
-
-  // Implementations of the DB interface
-  virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
-  virtual Status Delete(const WriteOptions&, const Slice& key);
-  virtual Status Write(const WriteOptions& options, WriteBatch* updates);
-  virtual Status Get(const ReadOptions& options,
-                     const Slice& key,
-                     std::string* value);
-  virtual Iterator* NewIterator(const ReadOptions&);
-  virtual const Snapshot* GetSnapshot();
-  virtual void ReleaseSnapshot(const Snapshot* snapshot);
-  virtual bool GetProperty(const Slice& property, std::string* value);
-  virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
-  virtual void CompactRange(const Slice* begin, const Slice* end);
-
-  // Extra methods (for testing) that are not in the public DB interface
-
-  // Compact any files in the named level that overlap [*begin,*end]
-  void TEST_CompactRange(int level, const Slice* begin, const Slice* end);
-
-  // Force current memtable contents to be compacted.
-  Status TEST_CompactMemTable();
-
-  // Return an internal iterator over the current state of the database.
-  // The keys of this iterator are internal keys (see format.h).
-  // The returned iterator should be deleted when no longer needed.
-  Iterator* TEST_NewInternalIterator();
-
-  // Return the maximum overlapping data (in bytes) at next level for any
-  // file at a level >= 1.
-  int64_t TEST_MaxNextLevelOverlappingBytes();
-
-  // Record a sample of bytes read at the specified internal key.
-  // Samples are taken approximately once every config::kReadBytesPeriod
-  // bytes.
-  void RecordReadSample(Slice key);
-
- private:
-  friend class DB;
-  struct CompactionState;
-  struct Writer;
-
-  Iterator* NewInternalIterator(const ReadOptions&,
-                                SequenceNumber* latest_snapshot,
-                                uint32_t* seed);
-
-  Status NewDB();
-
-  // Recover the descriptor from persistent storage.  May do a significant
-  // amount of work to recover recently logged updates.  Any changes to
-  // be made to the descriptor are added to *edit.
-  Status Recover(VersionEdit* edit, bool* save_manifest)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
-  void MaybeIgnoreError(Status* s) const;
-
-  // Delete any unneeded files and stale in-memory entries.
-  void DeleteObsoleteFiles();
-
-  // Compact the in-memory write buffer to disk.  Switches to a new
-  // log-file/memtable and writes a new descriptor iff successful.
-  // Errors are recorded in bg_error_.
-  void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
-  Status RecoverLogFile(uint64_t log_number, bool last_log, bool* save_manifest,
-                        VersionEdit* edit, SequenceNumber* max_sequence)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
-  Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
-  Status MakeRoomForWrite(bool force /* compact even if there is room? */)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-  WriteBatch* BuildBatchGroup(Writer** last_writer);
-
-  void RecordBackgroundError(const Status& s);
-
-  void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-  static void BGWork(void* db);
-  void BackgroundCall();
-  void  BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-  void CleanupCompaction(CompactionState* compact)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-  Status DoCompactionWork(CompactionState* compact)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
-  Status OpenCompactionOutputFile(CompactionState* compact);
-  Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input);
-  Status InstallCompactionResults(CompactionState* compact)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
-  // Constant after construction
-  Env* const env_;
-  const InternalKeyComparator internal_comparator_;
-  const InternalFilterPolicy internal_filter_policy_;
-  const Options options_;  // options_.comparator == &internal_comparator_
-  bool owns_info_log_;
-  bool owns_cache_;
-  const std::string dbname_;
-
-  // table_cache_ provides its own synchronization
-  TableCache* table_cache_;
-
-  // Lock over the persistent DB state.  Non-NULL iff successfully acquired.
-  FileLock* db_lock_;
-
-  // State below is protected by mutex_
-  port::Mutex mutex_;
-  port::AtomicPointer shutting_down_;
-  port::CondVar bg_cv_;          // Signalled when background work finishes
-  MemTable* mem_;
-  MemTable* imm_;                // Memtable being compacted
-  port::AtomicPointer has_imm_;  // So bg thread can detect non-NULL imm_
-  WritableFile* logfile_;
-  uint64_t logfile_number_;
-  log::Writer* log_;
-  uint32_t seed_;                // For sampling.
-
-  // Queue of writers.
-  std::deque<Writer*> writers_;
-  WriteBatch* tmp_batch_;
-
-  SnapshotList snapshots_;
-
-  // Set of table files to protect from deletion because they are
-  // part of ongoing compactions.
-  std::set<uint64_t> pending_outputs_;
-
-  // Has a background compaction been scheduled or is running?
-  bool bg_compaction_scheduled_;
-
-  // Information for a manual compaction
-  struct ManualCompaction {
-    int level;
-    bool done;
-    const InternalKey* begin;   // NULL means beginning of key range
-    const InternalKey* end;     // NULL means end of key range
-    InternalKey tmp_storage;    // Used to keep track of compaction progress
-  };
-  ManualCompaction* manual_compaction_;
-
-  VersionSet* versions_;
-
-  // Have we encountered a background error in paranoid mode?
-  Status bg_error_;
-
-  // Per level compaction stats.  stats_[level] stores the stats for
-  // compactions that produced data for the specified "level".
-  struct CompactionStats {
-    int64_t micros;
-    int64_t bytes_read;
-    int64_t bytes_written;
-
-    CompactionStats() : micros(0), bytes_read(0), bytes_written(0) { }
-
-    void Add(const CompactionStats& c) {
-      this->micros += c.micros;
-      this->bytes_read += c.bytes_read;
-      this->bytes_written += c.bytes_written;
-    }
-  };
-  CompactionStats stats_[config::kNumLevels];
-
-  // No copying allowed
-  DBImpl(const DBImpl&);
-  void operator=(const DBImpl&);
-
-  const Comparator* user_comparator() const {
-    return internal_comparator_.user_comparator();
-  }
-};
-
-// Sanitize db options.  The caller should delete result.info_log if
-// it is not equal to src.info_log.
-extern Options SanitizeOptions(const std::string& db,
-                               const InternalKeyComparator* icmp,
-                               const InternalFilterPolicy* ipolicy,
-                               const Options& src);
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_DB_IMPL_H_

src/leveldb/db/db_iter.cc

@@ -1,317 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/db_iter.h"
-
-#include "db/filename.h"
-#include "db/db_impl.h"
-#include "db/dbformat.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-#include "port/port.h"
-#include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
-
-namespace leveldb {
-
-#if 0
-static void DumpInternalIter(Iterator* iter) {
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    ParsedInternalKey k;
-    if (!ParseInternalKey(iter->key(), &k)) {
-      fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str());
-    } else {
-      fprintf(stderr, "@ '%s'\n", k.DebugString().c_str());
-    }
-  }
-}
-#endif
-
-namespace {
-
-// Memtables and sstables that make the DB representation contain
-// (userkey,seq,type) => uservalue entries.  DBIter
-// combines multiple entries for the same userkey found in the DB
-// representation into a single entry while accounting for sequence
-// numbers, deletion markers, overwrites, etc.
-class DBIter: public Iterator {
- public:
-  // Which direction is the iterator currently moving?
-  // (1) When moving forward, the internal iterator is positioned at
-  //     the exact entry that yields this->key(), this->value()
-  // (2) When moving backwards, the internal iterator is positioned
-  //     just before all entries whose user key == this->key().
-  enum Direction {
-    kForward,
-    kReverse
-  };
-
-  DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s,
-         uint32_t seed)
-      : db_(db),
-        user_comparator_(cmp),
-        iter_(iter),
-        sequence_(s),
-        direction_(kForward),
-        valid_(false),
-        rnd_(seed),
-        bytes_counter_(RandomPeriod()) {
-  }
-  virtual ~DBIter() {
-    delete iter_;
-  }
-  virtual bool Valid() const { return valid_; }
-  virtual Slice key() const {
-    assert(valid_);
-    return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_;
-  }
-  virtual Slice value() const {
-    assert(valid_);
-    return (direction_ == kForward) ? iter_->value() : saved_value_;
-  }
-  virtual Status status() const {
-    if (status_.ok()) {
-      return iter_->status();
-    } else {
-      return status_;
-    }
-  }
-
-  virtual void Next();
-  virtual void Prev();
-  virtual void Seek(const Slice& target);
-  virtual void SeekToFirst();
-  virtual void SeekToLast();
-
- private:
-  void FindNextUserEntry(bool skipping, std::string* skip);
-  void FindPrevUserEntry();
-  bool ParseKey(ParsedInternalKey* key);
-
-  inline void SaveKey(const Slice& k, std::string* dst) {
-    dst->assign(k.data(), k.size());
-  }
-
-  inline void ClearSavedValue() {
-    if (saved_value_.capacity() > 1048576) {
-      std::string empty;
-      swap(empty, saved_value_);
-    } else {
-      saved_value_.clear();
-    }
-  }
-
-  // Pick next gap with average value of config::kReadBytesPeriod.
-  ssize_t RandomPeriod() {
-    return rnd_.Uniform(2*config::kReadBytesPeriod);
-  }
-
-  DBImpl* db_;
-  const Comparator* const user_comparator_;
-  Iterator* const iter_;
-  SequenceNumber const sequence_;
-
-  Status status_;
-  std::string saved_key_;     // == current key when direction_==kReverse
-  std::string saved_value_;   // == current raw value when direction_==kReverse
-  Direction direction_;
-  bool valid_;
-
-  Random rnd_;
-  ssize_t bytes_counter_;
-
-  // No copying allowed
-  DBIter(const DBIter&);
-  void operator=(const DBIter&);
-};
-
-inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
-  Slice k = iter_->key();
-  ssize_t n = k.size() + iter_->value().size();
-  bytes_counter_ -= n;
-  while (bytes_counter_ < 0) {
-    bytes_counter_ += RandomPeriod();
-    db_->RecordReadSample(k);
-  }
-  if (!ParseInternalKey(k, ikey)) {
-    status_ = Status::Corruption("corrupted internal key in DBIter");
-    return false;
-  } else {
-    return true;
-  }
-}
-
-void DBIter::Next() {
-  assert(valid_);
-
-  if (direction_ == kReverse) {  // Switch directions?
-    direction_ = kForward;
-    // iter_ is pointing just before the entries for this->key(),
-    // so advance into the range of entries for this->key() and then
-    // use the normal skipping code below.
-    if (!iter_->Valid()) {
-      iter_->SeekToFirst();
-    } else {
-      iter_->Next();
-    }
-    if (!iter_->Valid()) {
-      valid_ = false;
-      saved_key_.clear();
-      return;
-    }
-    // saved_key_ already contains the key to skip past.
-  } else {
-    // Store in saved_key_ the current key so we skip it below.
-    SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
-  }
-
-  FindNextUserEntry(true, &saved_key_);
-}
-
-void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
-  // Loop until we hit an acceptable entry to yield
-  assert(iter_->Valid());
-  assert(direction_ == kForward);
-  do {
-    ParsedInternalKey ikey;
-    if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
-      switch (ikey.type) {
-        case kTypeDeletion:
-          // Arrange to skip all upcoming entries for this key since
-          // they are hidden by this deletion.
-          SaveKey(ikey.user_key, skip);
-          skipping = true;
-          break;
-        case kTypeValue:
-          if (skipping &&
-              user_comparator_->Compare(ikey.user_key, *skip) <= 0) {
-            // Entry hidden
-          } else {
-            valid_ = true;
-            saved_key_.clear();
-            return;
-          }
-          break;
-      }
-    }
-    iter_->Next();
-  } while (iter_->Valid());
-  saved_key_.clear();
-  valid_ = false;
-}
-
-void DBIter::Prev() {
-  assert(valid_);
-
-  if (direction_ == kForward) {  // Switch directions?
-    // iter_ is pointing at the current entry.  Scan backwards until
-    // the key changes so we can use the normal reverse scanning code.
-    assert(iter_->Valid());  // Otherwise valid_ would have been false
-    SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
-    while (true) {
-      iter_->Prev();
-      if (!iter_->Valid()) {
-        valid_ = false;
-        saved_key_.clear();
-        ClearSavedValue();
-        return;
-      }
-      if (user_comparator_->Compare(ExtractUserKey(iter_->key()),
-                                    saved_key_) < 0) {
-        break;
-      }
-    }
-    direction_ = kReverse;
-  }
-
-  FindPrevUserEntry();
-}
-
-void DBIter::FindPrevUserEntry() {
-  assert(direction_ == kReverse);
-
-  ValueType value_type = kTypeDeletion;
-  if (iter_->Valid()) {
-    do {
-      ParsedInternalKey ikey;
-      if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
-        if ((value_type != kTypeDeletion) &&
-            user_comparator_->Compare(ikey.user_key, saved_key_) < 0) {
-          // We encountered a non-deleted value in entries for previous keys,
-          break;
-        }
-        value_type = ikey.type;
-        if (value_type == kTypeDeletion) {
-          saved_key_.clear();
-          ClearSavedValue();
-        } else {
-          Slice raw_value = iter_->value();
-          if (saved_value_.capacity() > raw_value.size() + 1048576) {
-            std::string empty;
-            swap(empty, saved_value_);
-          }
-          SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
-          saved_value_.assign(raw_value.data(), raw_value.size());
-        }
-      }
-      iter_->Prev();
-    } while (iter_->Valid());
-  }
-
-  if (value_type == kTypeDeletion) {
-    // End
-    valid_ = false;
-    saved_key_.clear();
-    ClearSavedValue();
-    direction_ = kForward;
-  } else {
-    valid_ = true;
-  }
-}
-
-void DBIter::Seek(const Slice& target) {
-  direction_ = kForward;
-  ClearSavedValue();
-  saved_key_.clear();
-  AppendInternalKey(
-      &saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek));
-  iter_->Seek(saved_key_);
-  if (iter_->Valid()) {
-    FindNextUserEntry(false, &saved_key_ /* temporary storage */);
-  } else {
-    valid_ = false;
-  }
-}
-
-void DBIter::SeekToFirst() {
-  direction_ = kForward;
-  ClearSavedValue();
-  iter_->SeekToFirst();
-  if (iter_->Valid()) {
-    FindNextUserEntry(false, &saved_key_ /* temporary storage */);
-  } else {
-    valid_ = false;
-  }
-}
-
-void DBIter::SeekToLast() {
-  direction_ = kReverse;
-  ClearSavedValue();
-  iter_->SeekToLast();
-  FindPrevUserEntry();
-}
-
-}  // anonymous namespace
-
-Iterator* NewDBIterator(
-    DBImpl* db,
-    const Comparator* user_key_comparator,
-    Iterator* internal_iter,
-    SequenceNumber sequence,
-    uint32_t seed) {
-  return new DBIter(db, user_key_comparator, internal_iter, sequence, seed);
-}
-
-}  // namespace leveldb

src/leveldb/db/db_iter.h

@@ -1,28 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_DB_ITER_H_
-#define STORAGE_LEVELDB_DB_DB_ITER_H_
-
-#include <stdint.h>
-#include "leveldb/db.h"
-#include "db/dbformat.h"
-
-namespace leveldb {
-
-class DBImpl;
-
-// Return a new iterator that converts internal keys (yielded by
-// "*internal_iter") that were live at the specified "sequence" number
-// into appropriate user keys.
-extern Iterator* NewDBIterator(
-    DBImpl* db,
-    const Comparator* user_key_comparator,
-    Iterator* internal_iter,
-    SequenceNumber sequence,
-    uint32_t seed);
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_DB_ITER_H_

src/leveldb/db/dbformat.cc

@@ -1,140 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include "db/dbformat.h"
-#include "port/port.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {
-  assert(seq <= kMaxSequenceNumber);
-  assert(t <= kValueTypeForSeek);
-  return (seq << 8) | t;
-}
-
-void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
-  result->append(key.user_key.data(), key.user_key.size());
-  PutFixed64(result, PackSequenceAndType(key.sequence, key.type));
-}
-
-std::string ParsedInternalKey::DebugString() const {
-  char buf[50];
-  snprintf(buf, sizeof(buf), "' @ %llu : %d",
-           (unsigned long long) sequence,
-           int(type));
-  std::string result = "'";
-  result += EscapeString(user_key.ToString());
-  result += buf;
-  return result;
-}
-
-std::string InternalKey::DebugString() const {
-  std::string result;
-  ParsedInternalKey parsed;
-  if (ParseInternalKey(rep_, &parsed)) {
-    result = parsed.DebugString();
-  } else {
-    result = "(bad)";
-    result.append(EscapeString(rep_));
-  }
-  return result;
-}
-
-const char* InternalKeyComparator::Name() const {
-  return "leveldb.InternalKeyComparator";
-}
-
-int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
-  // Order by:
-  //    increasing user key (according to user-supplied comparator)
-  //    decreasing sequence number
-  //    decreasing type (though sequence# should be enough to disambiguate)
-  int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
-  if (r == 0) {
-    const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
-    const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
-    if (anum > bnum) {
-      r = -1;
-    } else if (anum < bnum) {
-      r = +1;
-    }
-  }
-  return r;
-}
-
-void InternalKeyComparator::FindShortestSeparator(
-      std::string* start,
-      const Slice& limit) const {
-  // Attempt to shorten the user portion of the key
-  Slice user_start = ExtractUserKey(*start);
-  Slice user_limit = ExtractUserKey(limit);
-  std::string tmp(user_start.data(), user_start.size());
-  user_comparator_->FindShortestSeparator(&tmp, user_limit);
-  if (tmp.size() < user_start.size() &&
-      user_comparator_->Compare(user_start, tmp) < 0) {
-    // User key has become shorter physically, but larger logically.
-    // Tack on the earliest possible number to the shortened user key.
-    PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
-    assert(this->Compare(*start, tmp) < 0);
-    assert(this->Compare(tmp, limit) < 0);
-    start->swap(tmp);
-  }
-}
-
-void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
-  Slice user_key = ExtractUserKey(*key);
-  std::string tmp(user_key.data(), user_key.size());
-  user_comparator_->FindShortSuccessor(&tmp);
-  if (tmp.size() < user_key.size() &&
-      user_comparator_->Compare(user_key, tmp) < 0) {
-    // User key has become shorter physically, but larger logically.
-    // Tack on the earliest possible number to the shortened user key.
-    PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
-    assert(this->Compare(*key, tmp) < 0);
-    key->swap(tmp);
-  }
-}
-
-const char* InternalFilterPolicy::Name() const {
-  return user_policy_->Name();
-}
-
-void InternalFilterPolicy::CreateFilter(const Slice* keys, int n,
-                                        std::string* dst) const {
-  // We rely on the fact that the code in table.cc does not mind us
-  // adjusting keys[].
-  Slice* mkey = const_cast<Slice*>(keys);
-  for (int i = 0; i < n; i++) {
-    mkey[i] = ExtractUserKey(keys[i]);
-    // TODO(sanjay): Suppress dups?
-  }
-  user_policy_->CreateFilter(keys, n, dst);
-}
-
-bool InternalFilterPolicy::KeyMayMatch(const Slice& key, const Slice& f) const {
-  return user_policy_->KeyMayMatch(ExtractUserKey(key), f);
-}
-
-LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
-  size_t usize = user_key.size();
-  size_t needed = usize + 13;  // A conservative estimate
-  char* dst;
-  if (needed <= sizeof(space_)) {
-    dst = space_;
-  } else {
-    dst = new char[needed];
-  }
-  start_ = dst;
-  dst = EncodeVarint32(dst, usize + 8);
-  kstart_ = dst;
-  memcpy(dst, user_key.data(), usize);
-  dst += usize;
-  EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek));
-  dst += 8;
-  end_ = dst;
-}
-
-}  // namespace leveldb

src/leveldb/db/dbformat.h

@@ -1,230 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_
-#define STORAGE_LEVELDB_DB_DBFORMAT_H_
-
-#include <stdio.h>
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/filter_policy.h"
-#include "leveldb/slice.h"
-#include "leveldb/table_builder.h"
-#include "util/coding.h"
-#include "util/logging.h"
-
-namespace leveldb {
-
-// Grouping of constants.  We may want to make some of these
-// parameters set via options.
-namespace config {
-static const int kNumLevels = 7;
-
-// Level-0 compaction is started when we hit this many files.
-static const int kL0_CompactionTrigger = 4;
-
-// Soft limit on number of level-0 files.  We slow down writes at this point.
-static const int kL0_SlowdownWritesTrigger = 8;
-
-// Maximum number of level-0 files.  We stop writes at this point.
-static const int kL0_StopWritesTrigger = 12;
-
-// Maximum level to which a new compacted memtable is pushed if it
-// does not create overlap.  We try to push to level 2 to avoid the
-// relatively expensive level 0=>1 compactions and to avoid some
-// expensive manifest file operations.  We do not push all the way to
-// the largest level since that can generate a lot of wasted disk
-// space if the same key space is being repeatedly overwritten.
-static const int kMaxMemCompactLevel = 2;
-
-// Approximate gap in bytes between samples of data read during iteration.
-static const int kReadBytesPeriod = 1048576;
-
-}  // namespace config
-
-class InternalKey;
-
-// Value types encoded as the last component of internal keys.
-// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
-// data structures.
-enum ValueType {
-  kTypeDeletion = 0x0,
-  kTypeValue = 0x1
-};
-// kValueTypeForSeek defines the ValueType that should be passed when
-// constructing a ParsedInternalKey object for seeking to a particular
-// sequence number (since we sort sequence numbers in decreasing order
-// and the value type is embedded as the low 8 bits in the sequence
-// number in internal keys, we need to use the highest-numbered
-// ValueType, not the lowest).
-static const ValueType kValueTypeForSeek = kTypeValue;
-
-typedef uint64_t SequenceNumber;
-
-// We leave eight bits empty at the bottom so a type and sequence#
-// can be packed together into 64-bits.
-static const SequenceNumber kMaxSequenceNumber =
-    ((0x1ull << 56) - 1);
-
-struct ParsedInternalKey {
-  Slice user_key;
-  SequenceNumber sequence;
-  ValueType type;
-
-  ParsedInternalKey() { }  // Intentionally left uninitialized (for speed)
-  ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
-      : user_key(u), sequence(seq), type(t) { }
-  std::string DebugString() const;
-};
-
-// Return the length of the encoding of "key".
-inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
-  return key.user_key.size() + 8;
-}
-
-// Append the serialization of "key" to *result.
-extern void AppendInternalKey(std::string* result,
-                              const ParsedInternalKey& key);
-
-// Attempt to parse an internal key from "internal_key".  On success,
-// stores the parsed data in "*result", and returns true.
-//
-// On error, returns false, leaves "*result" in an undefined state.
-extern bool ParseInternalKey(const Slice& internal_key,
-                             ParsedInternalKey* result);
-
-// Returns the user key portion of an internal key.
-inline Slice ExtractUserKey(const Slice& internal_key) {
-  assert(internal_key.size() >= 8);
-  return Slice(internal_key.data(), internal_key.size() - 8);
-}
-
-inline ValueType ExtractValueType(const Slice& internal_key) {
-  assert(internal_key.size() >= 8);
-  const size_t n = internal_key.size();
-  uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
-  unsigned char c = num & 0xff;
-  return static_cast<ValueType>(c);
-}
-
-// A comparator for internal keys that uses a specified comparator for
-// the user key portion and breaks ties by decreasing sequence number.
-class InternalKeyComparator : public Comparator {
- private:
-  const Comparator* user_comparator_;
- public:
-  explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) { }
-  virtual const char* Name() const;
-  virtual int Compare(const Slice& a, const Slice& b) const;
-  virtual void FindShortestSeparator(
-      std::string* start,
-      const Slice& limit) const;
-  virtual void FindShortSuccessor(std::string* key) const;
-
-  const Comparator* user_comparator() const { return user_comparator_; }
-
-  int Compare(const InternalKey& a, const InternalKey& b) const;
-};
-
-// Filter policy wrapper that converts from internal keys to user keys
-class InternalFilterPolicy : public FilterPolicy {
- private:
-  const FilterPolicy* const user_policy_;
- public:
-  explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) { }
-  virtual const char* Name() const;
-  virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const;
-  virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const;
-};
-
-// Modules in this directory should keep internal keys wrapped inside
-// the following class instead of plain strings so that we do not
-// incorrectly use string comparisons instead of an InternalKeyComparator.
-class InternalKey {
- private:
-  std::string rep_;
- public:
-  InternalKey() { }   // Leave rep_ as empty to indicate it is invalid
-  InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) {
-    AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t));
-  }
-
-  void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); }
-  Slice Encode() const {
-    assert(!rep_.empty());
-    return rep_;
-  }
-
-  Slice user_key() const { return ExtractUserKey(rep_); }
-
-  void SetFrom(const ParsedInternalKey& p) {
-    rep_.clear();
-    AppendInternalKey(&rep_, p);
-  }
-
-  void Clear() { rep_.clear(); }
-
-  std::string DebugString() const;
-};
-
-inline int InternalKeyComparator::Compare(
-    const InternalKey& a, const InternalKey& b) const {
-  return Compare(a.Encode(), b.Encode());
-}
-
-inline bool ParseInternalKey(const Slice& internal_key,
-                             ParsedInternalKey* result) {
-  const size_t n = internal_key.size();
-  if (n < 8) return false;
-  uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
-  unsigned char c = num & 0xff;
-  result->sequence = num >> 8;
-  result->type = static_cast<ValueType>(c);
-  result->user_key = Slice(internal_key.data(), n - 8);
-  return (c <= static_cast<unsigned char>(kTypeValue));
-}
-
-// A helper class useful for DBImpl::Get()
-class LookupKey {
- public:
-  // Initialize *this for looking up user_key at a snapshot with
-  // the specified sequence number.
-  LookupKey(const Slice& user_key, SequenceNumber sequence);
-
-  ~LookupKey();
-
-  // Return a key suitable for lookup in a MemTable.
-  Slice memtable_key() const { return Slice(start_, end_ - start_); }
-
-  // Return an internal key (suitable for passing to an internal iterator)
-  Slice internal_key() const { return Slice(kstart_, end_ - kstart_); }
-
-  // Return the user key
-  Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); }
-
- private:
-  // We construct a char array of the form:
-  //    klength  varint32               <-- start_
-  //    userkey  char[klength]          <-- kstart_
-  //    tag      uint64
-  //                                    <-- end_
-  // The array is a suitable MemTable key.
-  // The suffix starting with "userkey" can be used as an InternalKey.
-  const char* start_;
-  const char* kstart_;
-  const char* end_;
-  char space_[200];      // Avoid allocation for short keys
-
-  // No copying allowed
-  LookupKey(const LookupKey&);
-  void operator=(const LookupKey&);
-};
-
-inline LookupKey::~LookupKey() {
-  if (start_ != space_) delete[] start_;
-}
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_DBFORMAT_H_

src/leveldb/db/dbformat_test.cc

@@ -1,112 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/dbformat.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-static std::string IKey(const std::string& user_key,
-                        uint64_t seq,
-                        ValueType vt) {
-  std::string encoded;
-  AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
-  return encoded;
-}
-
-static std::string Shorten(const std::string& s, const std::string& l) {
-  std::string result = s;
-  InternalKeyComparator(BytewiseComparator()).FindShortestSeparator(&result, l);
-  return result;
-}
-
-static std::string ShortSuccessor(const std::string& s) {
-  std::string result = s;
-  InternalKeyComparator(BytewiseComparator()).FindShortSuccessor(&result);
-  return result;
-}
-
-static void TestKey(const std::string& key,
-                    uint64_t seq,
-                    ValueType vt) {
-  std::string encoded = IKey(key, seq, vt);
-
-  Slice in(encoded);
-  ParsedInternalKey decoded("", 0, kTypeValue);
-
-  ASSERT_TRUE(ParseInternalKey(in, &decoded));
-  ASSERT_EQ(key, decoded.user_key.ToString());
-  ASSERT_EQ(seq, decoded.sequence);
-  ASSERT_EQ(vt, decoded.type);
-
-  ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded));
-}
-
-class FormatTest { };
-
-TEST(FormatTest, InternalKey_EncodeDecode) {
-  const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" };
-  const uint64_t seq[] = {
-    1, 2, 3,
-    (1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1,
-    (1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1,
-    (1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1
-  };
-  for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
-    for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
-      TestKey(keys[k], seq[s], kTypeValue);
-      TestKey("hello", 1, kTypeDeletion);
-    }
-  }
-}
-
-TEST(FormatTest, InternalKeyShortSeparator) {
-  // When user keys are same
-  ASSERT_EQ(IKey("foo", 100, kTypeValue),
-            Shorten(IKey("foo", 100, kTypeValue),
-                    IKey("foo", 99, kTypeValue)));
-  ASSERT_EQ(IKey("foo", 100, kTypeValue),
-            Shorten(IKey("foo", 100, kTypeValue),
-                    IKey("foo", 101, kTypeValue)));
-  ASSERT_EQ(IKey("foo", 100, kTypeValue),
-            Shorten(IKey("foo", 100, kTypeValue),
-                    IKey("foo", 100, kTypeValue)));
-  ASSERT_EQ(IKey("foo", 100, kTypeValue),
-            Shorten(IKey("foo", 100, kTypeValue),
-                    IKey("foo", 100, kTypeDeletion)));
-
-  // When user keys are misordered
-  ASSERT_EQ(IKey("foo", 100, kTypeValue),
-            Shorten(IKey("foo", 100, kTypeValue),
-                    IKey("bar", 99, kTypeValue)));
-
-  // When user keys are different, but correctly ordered
-  ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
-            Shorten(IKey("foo", 100, kTypeValue),
-                    IKey("hello", 200, kTypeValue)));
-
-  // When start user key is prefix of limit user key
-  ASSERT_EQ(IKey("foo", 100, kTypeValue),
-            Shorten(IKey("foo", 100, kTypeValue),
-                    IKey("foobar", 200, kTypeValue)));
-
-  // When limit user key is prefix of start user key
-  ASSERT_EQ(IKey("foobar", 100, kTypeValue),
-            Shorten(IKey("foobar", 100, kTypeValue),
-                    IKey("foo", 200, kTypeValue)));
-}
-
-TEST(FormatTest, InternalKeyShortestSuccessor) {
-  ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
-            ShortSuccessor(IKey("foo", 100, kTypeValue)));
-  ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue),
-            ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/dumpfile.cc

@@ -1,225 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include "db/dbformat.h"
-#include "db/filename.h"
-#include "db/log_reader.h"
-#include "db/version_edit.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-#include "leveldb/options.h"
-#include "leveldb/status.h"
-#include "leveldb/table.h"
-#include "leveldb/write_batch.h"
-#include "util/logging.h"
-
-namespace leveldb {
-
-namespace {
-
-bool GuessType(const std::string& fname, FileType* type) {
-  size_t pos = fname.rfind('/');
-  std::string basename;
-  if (pos == std::string::npos) {
-    basename = fname;
-  } else {
-    basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1);
-  }
-  uint64_t ignored;
-  return ParseFileName(basename, &ignored, type);
-}
-
-// Notified when log reader encounters corruption.
-class CorruptionReporter : public log::Reader::Reporter {
- public:
-  WritableFile* dst_;
-  virtual void Corruption(size_t bytes, const Status& status) {
-    std::string r = "corruption: ";
-    AppendNumberTo(&r, bytes);
-    r += " bytes; ";
-    r += status.ToString();
-    r.push_back('\n');
-    dst_->Append(r);
-  }
-};
-
-// Print contents of a log file. (*func)() is called on every record.
-Status PrintLogContents(Env* env, const std::string& fname,
-                        void (*func)(uint64_t, Slice, WritableFile*),
-                        WritableFile* dst) {
-  SequentialFile* file;
-  Status s = env->NewSequentialFile(fname, &file);
-  if (!s.ok()) {
-    return s;
-  }
-  CorruptionReporter reporter;
-  reporter.dst_ = dst;
-  log::Reader reader(file, &reporter, true, 0);
-  Slice record;
-  std::string scratch;
-  while (reader.ReadRecord(&record, &scratch)) {
-    (*func)(reader.LastRecordOffset(), record, dst);
-  }
-  delete file;
-  return Status::OK();
-}
-
-// Called on every item found in a WriteBatch.
-class WriteBatchItemPrinter : public WriteBatch::Handler {
- public:
-  WritableFile* dst_;
-  virtual void Put(const Slice& key, const Slice& value) {
-    std::string r = "  put '";
-    AppendEscapedStringTo(&r, key);
-    r += "' '";
-    AppendEscapedStringTo(&r, value);
-    r += "'\n";
-    dst_->Append(r);
-  }
-  virtual void Delete(const Slice& key) {
-    std::string r = "  del '";
-    AppendEscapedStringTo(&r, key);
-    r += "'\n";
-    dst_->Append(r);
-  }
-};
-
-
-// Called on every log record (each one of which is a WriteBatch)
-// found in a kLogFile.
-static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) {
-  std::string r = "--- offset ";
-  AppendNumberTo(&r, pos);
-  r += "; ";
-  if (record.size() < 12) {
-    r += "log record length ";
-    AppendNumberTo(&r, record.size());
-    r += " is too small\n";
-    dst->Append(r);
-    return;
-  }
-  WriteBatch batch;
-  WriteBatchInternal::SetContents(&batch, record);
-  r += "sequence ";
-  AppendNumberTo(&r, WriteBatchInternal::Sequence(&batch));
-  r.push_back('\n');
-  dst->Append(r);
-  WriteBatchItemPrinter batch_item_printer;
-  batch_item_printer.dst_ = dst;
-  Status s = batch.Iterate(&batch_item_printer);
-  if (!s.ok()) {
-    dst->Append("  error: " + s.ToString() + "\n");
-  }
-}
-
-Status DumpLog(Env* env, const std::string& fname, WritableFile* dst) {
-  return PrintLogContents(env, fname, WriteBatchPrinter, dst);
-}
-
-// Called on every log record (each one of which is a WriteBatch)
-// found in a kDescriptorFile.
-static void VersionEditPrinter(uint64_t pos, Slice record, WritableFile* dst) {
-  std::string r = "--- offset ";
-  AppendNumberTo(&r, pos);
-  r += "; ";
-  VersionEdit edit;
-  Status s = edit.DecodeFrom(record);
-  if (!s.ok()) {
-    r += s.ToString();
-    r.push_back('\n');
-  } else {
-    r += edit.DebugString();
-  }
-  dst->Append(r);
-}
-
-Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) {
-  return PrintLogContents(env, fname, VersionEditPrinter, dst);
-}
-
-Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) {
-  uint64_t file_size;
-  RandomAccessFile* file = NULL;
-  Table* table = NULL;
-  Status s = env->GetFileSize(fname, &file_size);
-  if (s.ok()) {
-    s = env->NewRandomAccessFile(fname, &file);
-  }
-  if (s.ok()) {
-    // We use the default comparator, which may or may not match the
-    // comparator used in this database. However this should not cause
-    // problems since we only use Table operations that do not require
-    // any comparisons.  In particular, we do not call Seek or Prev.
-    s = Table::Open(Options(), file, file_size, &table);
-  }
-  if (!s.ok()) {
-    delete table;
-    delete file;
-    return s;
-  }
-
-  ReadOptions ro;
-  ro.fill_cache = false;
-  Iterator* iter = table->NewIterator(ro);
-  std::string r;
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    r.clear();
-    ParsedInternalKey key;
-    if (!ParseInternalKey(iter->key(), &key)) {
-      r = "badkey '";
-      AppendEscapedStringTo(&r, iter->key());
-      r += "' => '";
-      AppendEscapedStringTo(&r, iter->value());
-      r += "'\n";
-      dst->Append(r);
-    } else {
-      r = "'";
-      AppendEscapedStringTo(&r, key.user_key);
-      r += "' @ ";
-      AppendNumberTo(&r, key.sequence);
-      r += " : ";
-      if (key.type == kTypeDeletion) {
-        r += "del";
-      } else if (key.type == kTypeValue) {
-        r += "val";
-      } else {
-        AppendNumberTo(&r, key.type);
-      }
-      r += " => '";
-      AppendEscapedStringTo(&r, iter->value());
-      r += "'\n";
-      dst->Append(r);
-    }
-  }
-  s = iter->status();
-  if (!s.ok()) {
-    dst->Append("iterator error: " + s.ToString() + "\n");
-  }
-
-  delete iter;
-  delete table;
-  delete file;
-  return Status::OK();
-}
-
-}  // namespace
-
-Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) {
-  FileType ftype;
-  if (!GuessType(fname, &ftype)) {
-    return Status::InvalidArgument(fname + ": unknown file type");
-  }
-  switch (ftype) {
-    case kLogFile:         return DumpLog(env, fname, dst);
-    case kDescriptorFile:  return DumpDescriptor(env, fname, dst);
-    case kTableFile:       return DumpTable(env, fname, dst);
-    default:
-      break;
-  }
-  return Status::InvalidArgument(fname + ": not a dump-able file type");
-}
-
-}  // namespace leveldb

src/leveldb/db/fault_injection_test.cc

@@ -1,554 +0,0 @@
-// Copyright 2014 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-// This test uses a custom Env to keep track of the state of a filesystem as of
-// the last "sync". It then checks for data loss errors by purposely dropping
-// file data (or entire files) not protected by a "sync".
-
-#include "leveldb/db.h"
-
-#include <map>
-#include <set>
-#include "db/db_impl.h"
-#include "db/filename.h"
-#include "db/log_format.h"
-#include "db/version_set.h"
-#include "leveldb/cache.h"
-#include "leveldb/env.h"
-#include "leveldb/table.h"
-#include "leveldb/write_batch.h"
-#include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-static const int kValueSize = 1000;
-static const int kMaxNumValues = 2000;
-static const size_t kNumIterations = 3;
-
-class FaultInjectionTestEnv;
-
-namespace {
-
-// Assume a filename, and not a directory name like "/foo/bar/"
-static std::string GetDirName(const std::string filename) {
-  size_t found = filename.find_last_of("/\\");
-  if (found == std::string::npos) {
-    return "";
-  } else {
-    return filename.substr(0, found);
-  }
-}
-
-Status SyncDir(const std::string& dir) {
-  // As this is a test it isn't required to *actually* sync this directory.
-  return Status::OK();
-}
-
-// A basic file truncation function suitable for this test.
-Status Truncate(const std::string& filename, uint64_t length) {
-  leveldb::Env* env = leveldb::Env::Default();
-
-  SequentialFile* orig_file;
-  Status s = env->NewSequentialFile(filename, &orig_file);
-  if (!s.ok())
-    return s;
-
-  char* scratch = new char[length];
-  leveldb::Slice result;
-  s = orig_file->Read(length, &result, scratch);
-  delete orig_file;
-  if (s.ok()) {
-    std::string tmp_name = GetDirName(filename) + "/truncate.tmp";
-    WritableFile* tmp_file;
-    s = env->NewWritableFile(tmp_name, &tmp_file);
-    if (s.ok()) {
-      s = tmp_file->Append(result);
-      delete tmp_file;
-      if (s.ok()) {
-        s = env->RenameFile(tmp_name, filename);
-      } else {
-        env->DeleteFile(tmp_name);
-      }
-    }
-  }
-
-  delete[] scratch;
-
-  return s;
-}
-
-struct FileState {
-  std::string filename_;
-  ssize_t pos_;
-  ssize_t pos_at_last_sync_;
-  ssize_t pos_at_last_flush_;
-
-  FileState(const std::string& filename)
-      : filename_(filename),
-        pos_(-1),
-        pos_at_last_sync_(-1),
-        pos_at_last_flush_(-1) { }
-
-  FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {}
-
-  bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; }
-
-  Status DropUnsyncedData() const;
-};
-
-}  // anonymous namespace
-
-// A wrapper around WritableFile which informs another Env whenever this file
-// is written to or sync'ed.
-class TestWritableFile : public WritableFile {
- public:
-  TestWritableFile(const FileState& state,
-                   WritableFile* f,
-                   FaultInjectionTestEnv* env);
-  virtual ~TestWritableFile();
-  virtual Status Append(const Slice& data);
-  virtual Status Close();
-  virtual Status Flush();
-  virtual Status Sync();
-
- private:
-  FileState state_;
-  WritableFile* target_;
-  bool writable_file_opened_;
-  FaultInjectionTestEnv* env_;
-
-  Status SyncParent();
-};
-
-class FaultInjectionTestEnv : public EnvWrapper {
- public:
-  FaultInjectionTestEnv() : EnvWrapper(Env::Default()), filesystem_active_(true) {}
-  virtual ~FaultInjectionTestEnv() { }
-  virtual Status NewWritableFile(const std::string& fname,
-                                 WritableFile** result);
-  virtual Status NewAppendableFile(const std::string& fname,
-                                   WritableFile** result);
-  virtual Status DeleteFile(const std::string& f);
-  virtual Status RenameFile(const std::string& s, const std::string& t);
-
-  void WritableFileClosed(const FileState& state);
-  Status DropUnsyncedFileData();
-  Status DeleteFilesCreatedAfterLastDirSync();
-  void DirWasSynced();
-  bool IsFileCreatedSinceLastDirSync(const std::string& filename);
-  void ResetState();
-  void UntrackFile(const std::string& f);
-  // Setting the filesystem to inactive is the test equivalent to simulating a
-  // system reset. Setting to inactive will freeze our saved filesystem state so
-  // that it will stop being recorded. It can then be reset back to the state at
-  // the time of the reset.
-  bool IsFilesystemActive() const { return filesystem_active_; }
-  void SetFilesystemActive(bool active) { filesystem_active_ = active; }
-
- private:
-  port::Mutex mutex_;
-  std::map<std::string, FileState> db_file_state_;
-  std::set<std::string> new_files_since_last_dir_sync_;
-  bool filesystem_active_;  // Record flushes, syncs, writes
-};
-
-TestWritableFile::TestWritableFile(const FileState& state,
-                                   WritableFile* f,
-                                   FaultInjectionTestEnv* env)
-    : state_(state),
-      target_(f),
-      writable_file_opened_(true),
-      env_(env) {
-  assert(f != NULL);
-}
-
-TestWritableFile::~TestWritableFile() {
-  if (writable_file_opened_) {
-    Close();
-  }
-  delete target_;
-}
-
-Status TestWritableFile::Append(const Slice& data) {
-  Status s = target_->Append(data);
-  if (s.ok() && env_->IsFilesystemActive()) {
-    state_.pos_ += data.size();
-  }
-  return s;
-}
-
-Status TestWritableFile::Close() {
-  writable_file_opened_ = false;
-  Status s = target_->Close();
-  if (s.ok()) {
-    env_->WritableFileClosed(state_);
-  }
-  return s;
-}
-
-Status TestWritableFile::Flush() {
-  Status s = target_->Flush();
-  if (s.ok() && env_->IsFilesystemActive()) {
-    state_.pos_at_last_flush_ = state_.pos_;
-  }
-  return s;
-}
-
-Status TestWritableFile::SyncParent() {
-  Status s = SyncDir(GetDirName(state_.filename_));
-  if (s.ok()) {
-    env_->DirWasSynced();
-  }
-  return s;
-}
-
-Status TestWritableFile::Sync() {
-  if (!env_->IsFilesystemActive()) {
-    return Status::OK();
-  }
-  // Ensure new files referred to by the manifest are in the filesystem.
-  Status s = target_->Sync();
-  if (s.ok()) {
-    state_.pos_at_last_sync_ = state_.pos_;
-  }
-  if (env_->IsFileCreatedSinceLastDirSync(state_.filename_)) {
-    Status ps = SyncParent();
-    if (s.ok() && !ps.ok()) {
-      s = ps;
-    }
-  }
-  return s;
-}
-
-Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname,
-                                              WritableFile** result) {
-  WritableFile* actual_writable_file;
-  Status s = target()->NewWritableFile(fname, &actual_writable_file);
-  if (s.ok()) {
-    FileState state(fname);
-    state.pos_ = 0;
-    *result = new TestWritableFile(state, actual_writable_file, this);
-    // NewWritableFile doesn't append to files, so if the same file is
-    // opened again then it will be truncated - so forget our saved
-    // state.
-    UntrackFile(fname);
-    MutexLock l(&mutex_);
-    new_files_since_last_dir_sync_.insert(fname);
-  }
-  return s;
-}
-
-Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname,
-                                                WritableFile** result) {
-  WritableFile* actual_writable_file;
-  Status s = target()->NewAppendableFile(fname, &actual_writable_file);
-  if (s.ok()) {
-    FileState state(fname);
-    state.pos_ = 0;
-    {
-      MutexLock l(&mutex_);
-      if (db_file_state_.count(fname) == 0) {
-        new_files_since_last_dir_sync_.insert(fname);
-      } else {
-        state = db_file_state_[fname];
-      }
-    }
-    *result = new TestWritableFile(state, actual_writable_file, this);
-  }
-  return s;
-}
-
-Status FaultInjectionTestEnv::DropUnsyncedFileData() {
-  Status s;
-  MutexLock l(&mutex_);
-  for (std::map<std::string, FileState>::const_iterator it =
-           db_file_state_.begin();
-       s.ok() && it != db_file_state_.end(); ++it) {
-    const FileState& state = it->second;
-    if (!state.IsFullySynced()) {
-      s = state.DropUnsyncedData();
-    }
-  }
-  return s;
-}
-
-void FaultInjectionTestEnv::DirWasSynced() {
-  MutexLock l(&mutex_);
-  new_files_since_last_dir_sync_.clear();
-}
-
-bool FaultInjectionTestEnv::IsFileCreatedSinceLastDirSync(
-    const std::string& filename) {
-  MutexLock l(&mutex_);
-  return new_files_since_last_dir_sync_.find(filename) !=
-         new_files_since_last_dir_sync_.end();
-}
-
-void FaultInjectionTestEnv::UntrackFile(const std::string& f) {
-  MutexLock l(&mutex_);
-  db_file_state_.erase(f);
-  new_files_since_last_dir_sync_.erase(f);
-}
-
-Status FaultInjectionTestEnv::DeleteFile(const std::string& f) {
-  Status s = EnvWrapper::DeleteFile(f);
-  ASSERT_OK(s);
-  if (s.ok()) {
-    UntrackFile(f);
-  }
-  return s;
-}
-
-Status FaultInjectionTestEnv::RenameFile(const std::string& s,
-                                         const std::string& t) {
-  Status ret = EnvWrapper::RenameFile(s, t);
-
-  if (ret.ok()) {
-    MutexLock l(&mutex_);
-    if (db_file_state_.find(s) != db_file_state_.end()) {
-      db_file_state_[t] = db_file_state_[s];
-      db_file_state_.erase(s);
-    }
-
-    if (new_files_since_last_dir_sync_.erase(s) != 0) {
-      assert(new_files_since_last_dir_sync_.find(t) ==
-             new_files_since_last_dir_sync_.end());
-      new_files_since_last_dir_sync_.insert(t);
-    }
-  }
-
-  return ret;
-}
-
-void FaultInjectionTestEnv::ResetState() {
-  // Since we are not destroying the database, the existing files
-  // should keep their recorded synced/flushed state. Therefore
-  // we do not reset db_file_state_ and new_files_since_last_dir_sync_.
-  MutexLock l(&mutex_);
-  SetFilesystemActive(true);
-}
-
-Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() {
-  // Because DeleteFile access this container make a copy to avoid deadlock
-  mutex_.Lock();
-  std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(),
-                                  new_files_since_last_dir_sync_.end());
-  mutex_.Unlock();
-  Status s;
-  std::set<std::string>::const_iterator it;
-  for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) {
-    s = DeleteFile(*it);
-  }
-  return s;
-}
-
-void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
-  MutexLock l(&mutex_);
-  db_file_state_[state.filename_] = state;
-}
-
-Status FileState::DropUnsyncedData() const {
-  ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
-  return Truncate(filename_, sync_pos);
-}
-
-class FaultInjectionTest {
- public:
-  enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR };
-  enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES };
-
-  FaultInjectionTestEnv* env_;
-  std::string dbname_;
-  Cache* tiny_cache_;
-  Options options_;
-  DB* db_;
-
-  FaultInjectionTest()
-      : env_(new FaultInjectionTestEnv),
-        tiny_cache_(NewLRUCache(100)),
-        db_(NULL) {
-    dbname_ = test::TmpDir() + "/fault_test";
-    DestroyDB(dbname_, Options());  // Destroy any db from earlier run
-    options_.reuse_logs = true;
-    options_.env = env_;
-    options_.paranoid_checks = true;
-    options_.block_cache = tiny_cache_;
-    options_.create_if_missing = true;
-  }
-
-  ~FaultInjectionTest() {
-    CloseDB();
-    DestroyDB(dbname_, Options());
-    delete tiny_cache_;
-    delete env_;
-  }
-
-  void ReuseLogs(bool reuse) {
-    options_.reuse_logs = reuse;
-  }
-
-  void Build(int start_idx, int num_vals) {
-    std::string key_space, value_space;
-    WriteBatch batch;
-    for (int i = start_idx; i < start_idx + num_vals; i++) {
-      Slice key = Key(i, &key_space);
-      batch.Clear();
-      batch.Put(key, Value(i, &value_space));
-      WriteOptions options;
-      ASSERT_OK(db_->Write(options, &batch));
-    }
-  }
-
-  Status ReadValue(int i, std::string* val) const {
-    std::string key_space, value_space;
-    Slice key = Key(i, &key_space);
-    Value(i, &value_space);
-    ReadOptions options;
-    return db_->Get(options, key, val);
-  }
-
-  Status Verify(int start_idx, int num_vals,
-                ExpectedVerifResult expected) const {
-    std::string val;
-    std::string value_space;
-    Status s;
-    for (int i = start_idx; i < start_idx + num_vals && s.ok(); i++) {
-      Value(i, &value_space);
-      s = ReadValue(i, &val);
-      if (expected == VAL_EXPECT_NO_ERROR) {
-        if (s.ok()) {
-          ASSERT_EQ(value_space, val);
-        }
-      } else if (s.ok()) {
-        fprintf(stderr, "Expected an error at %d, but was OK\n", i);
-        s = Status::IOError(dbname_, "Expected value error:");
-      } else {
-        s = Status::OK();  // An expected error
-      }
-    }
-    return s;
-  }
-
-  // Return the ith key
-  Slice Key(int i, std::string* storage) const {
-    char buf[100];
-    snprintf(buf, sizeof(buf), "%016d", i);
-    storage->assign(buf, strlen(buf));
-    return Slice(*storage);
-  }
-
-  // Return the value to associate with the specified key
-  Slice Value(int k, std::string* storage) const {
-    Random r(k);
-    return test::RandomString(&r, kValueSize, storage);
-  }
-
-  Status OpenDB() {
-    delete db_;
-    db_ = NULL;
-    env_->ResetState();
-    return DB::Open(options_, dbname_, &db_);
-  }
-
-  void CloseDB() {
-    delete db_;
-    db_ = NULL;
-  }
-
-  void DeleteAllData() {
-    Iterator* iter = db_->NewIterator(ReadOptions());
-    WriteOptions options;
-    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-      ASSERT_OK(db_->Delete(WriteOptions(), iter->key()));
-    }
-
-    delete iter;
-  }
-
-  void ResetDBState(ResetMethod reset_method) {
-    switch (reset_method) {
-      case RESET_DROP_UNSYNCED_DATA:
-        ASSERT_OK(env_->DropUnsyncedFileData());
-        break;
-      case RESET_DELETE_UNSYNCED_FILES:
-        ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync());
-        break;
-      default:
-        assert(false);
-    }
-  }
-
-  void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) {
-    DeleteAllData();
-    Build(0, num_pre_sync);
-    db_->CompactRange(NULL, NULL);
-    Build(num_pre_sync, num_post_sync);
-  }
-
-  void PartialCompactTestReopenWithFault(ResetMethod reset_method,
-                                         int num_pre_sync,
-                                         int num_post_sync) {
-    env_->SetFilesystemActive(false);
-    CloseDB();
-    ResetDBState(reset_method);
-    ASSERT_OK(OpenDB());
-    ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR));
-    ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR));
-  }
-
-  void NoWriteTestPreFault() {
-  }
-
-  void NoWriteTestReopenWithFault(ResetMethod reset_method) {
-    CloseDB();
-    ResetDBState(reset_method);
-    ASSERT_OK(OpenDB());
-  }
-
-  void DoTest() {
-    Random rnd(0);
-    ASSERT_OK(OpenDB());
-    for (size_t idx = 0; idx < kNumIterations; idx++) {
-      int num_pre_sync = rnd.Uniform(kMaxNumValues);
-      int num_post_sync = rnd.Uniform(kMaxNumValues);
-
-      PartialCompactTestPreFault(num_pre_sync, num_post_sync);
-      PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA,
-                                        num_pre_sync,
-                                        num_post_sync);
-
-      NoWriteTestPreFault();
-      NoWriteTestReopenWithFault(RESET_DROP_UNSYNCED_DATA);
-
-      PartialCompactTestPreFault(num_pre_sync, num_post_sync);
-      // No new files created so we expect all values since no files will be
-      // dropped.
-      PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES,
-                                        num_pre_sync + num_post_sync,
-                                        0);
-
-      NoWriteTestPreFault();
-      NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES);
-    }
-  }
-};
-
-TEST(FaultInjectionTest, FaultTestNoLogReuse) {
-  ReuseLogs(false);
-  DoTest();
-}
-
-TEST(FaultInjectionTest, FaultTestWithLogReuse) {
-  ReuseLogs(true);
-  DoTest();
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/filename.cc

@@ -1,144 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <ctype.h>
-#include <stdio.h>
-#include "db/filename.h"
-#include "db/dbformat.h"
-#include "leveldb/env.h"
-#include "util/logging.h"
-
-namespace leveldb {
-
-// A utility routine: write "data" to the named file and Sync() it.
-extern Status WriteStringToFileSync(Env* env, const Slice& data,
-                                    const std::string& fname);
-
-static std::string MakeFileName(const std::string& name, uint64_t number,
-                                const char* suffix) {
-  char buf[100];
-  snprintf(buf, sizeof(buf), "/%06llu.%s",
-           static_cast<unsigned long long>(number),
-           suffix);
-  return name + buf;
-}
-
-std::string LogFileName(const std::string& name, uint64_t number) {
-  assert(number > 0);
-  return MakeFileName(name, number, "log");
-}
-
-std::string TableFileName(const std::string& name, uint64_t number) {
-  assert(number > 0);
-  return MakeFileName(name, number, "ldb");
-}
-
-std::string SSTTableFileName(const std::string& name, uint64_t number) {
-  assert(number > 0);
-  return MakeFileName(name, number, "sst");
-}
-
-std::string DescriptorFileName(const std::string& dbname, uint64_t number) {
-  assert(number > 0);
-  char buf[100];
-  snprintf(buf, sizeof(buf), "/MANIFEST-%06llu",
-           static_cast<unsigned long long>(number));
-  return dbname + buf;
-}
-
-std::string CurrentFileName(const std::string& dbname) {
-  return dbname + "/CURRENT";
-}
-
-std::string LockFileName(const std::string& dbname) {
-  return dbname + "/LOCK";
-}
-
-std::string TempFileName(const std::string& dbname, uint64_t number) {
-  assert(number > 0);
-  return MakeFileName(dbname, number, "dbtmp");
-}
-
-std::string InfoLogFileName(const std::string& dbname) {
-  return dbname + "/LOG";
-}
-
-// Return the name of the old info log file for "dbname".
-std::string OldInfoLogFileName(const std::string& dbname) {
-  return dbname + "/LOG.old";
-}
-
-
-// Owned filenames have the form:
-//    dbname/CURRENT
-//    dbname/LOCK
-//    dbname/LOG
-//    dbname/LOG.old
-//    dbname/MANIFEST-[0-9]+
-//    dbname/[0-9]+.(log|sst|ldb)
-bool ParseFileName(const std::string& fname,
-                   uint64_t* number,
-                   FileType* type) {
-  Slice rest(fname);
-  if (rest == "CURRENT") {
-    *number = 0;
-    *type = kCurrentFile;
-  } else if (rest == "LOCK") {
-    *number = 0;
-    *type = kDBLockFile;
-  } else if (rest == "LOG" || rest == "LOG.old") {
-    *number = 0;
-    *type = kInfoLogFile;
-  } else if (rest.starts_with("MANIFEST-")) {
-    rest.remove_prefix(strlen("MANIFEST-"));
-    uint64_t num;
-    if (!ConsumeDecimalNumber(&rest, &num)) {
-      return false;
-    }
-    if (!rest.empty()) {
-      return false;
-    }
-    *type = kDescriptorFile;
-    *number = num;
-  } else {
-    // Avoid strtoull() to keep filename format independent of the
-    // current locale
-    uint64_t num;
-    if (!ConsumeDecimalNumber(&rest, &num)) {
-      return false;
-    }
-    Slice suffix = rest;
-    if (suffix == Slice(".log")) {
-      *type = kLogFile;
-    } else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) {
-      *type = kTableFile;
-    } else if (suffix == Slice(".dbtmp")) {
-      *type = kTempFile;
-    } else {
-      return false;
-    }
-    *number = num;
-  }
-  return true;
-}
-
-Status SetCurrentFile(Env* env, const std::string& dbname,
-                      uint64_t descriptor_number) {
-  // Remove leading "dbname/" and add newline to manifest file name
-  std::string manifest = DescriptorFileName(dbname, descriptor_number);
-  Slice contents = manifest;
-  assert(contents.starts_with(dbname + "/"));
-  contents.remove_prefix(dbname.size() + 1);
-  std::string tmp = TempFileName(dbname, descriptor_number);
-  Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp);
-  if (s.ok()) {
-    s = env->RenameFile(tmp, CurrentFileName(dbname));
-  }
-  if (!s.ok()) {
-    env->DeleteFile(tmp);
-  }
-  return s;
-}
-
-}  // namespace leveldb

src/leveldb/db/filename.h

@@ -1,85 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// File names used by DB code
-
-#ifndef STORAGE_LEVELDB_DB_FILENAME_H_
-#define STORAGE_LEVELDB_DB_FILENAME_H_
-
-#include <stdint.h>
-#include <string>
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-#include "port/port.h"
-
-namespace leveldb {
-
-class Env;
-
-enum FileType {
-  kLogFile,
-  kDBLockFile,
-  kTableFile,
-  kDescriptorFile,
-  kCurrentFile,
-  kTempFile,
-  kInfoLogFile  // Either the current one, or an old one
-};
-
-// Return the name of the log file with the specified number
-// in the db named by "dbname".  The result will be prefixed with
-// "dbname".
-extern std::string LogFileName(const std::string& dbname, uint64_t number);
-
-// Return the name of the sstable with the specified number
-// in the db named by "dbname".  The result will be prefixed with
-// "dbname".
-extern std::string TableFileName(const std::string& dbname, uint64_t number);
-
-// Return the legacy file name for an sstable with the specified number
-// in the db named by "dbname". The result will be prefixed with
-// "dbname".
-extern std::string SSTTableFileName(const std::string& dbname, uint64_t number);
-
-// Return the name of the descriptor file for the db named by
-// "dbname" and the specified incarnation number.  The result will be
-// prefixed with "dbname".
-extern std::string DescriptorFileName(const std::string& dbname,
-                                      uint64_t number);
-
-// Return the name of the current file.  This file contains the name
-// of the current manifest file.  The result will be prefixed with
-// "dbname".
-extern std::string CurrentFileName(const std::string& dbname);
-
-// Return the name of the lock file for the db named by
-// "dbname".  The result will be prefixed with "dbname".
-extern std::string LockFileName(const std::string& dbname);
-
-// Return the name of a temporary file owned by the db named "dbname".
-// The result will be prefixed with "dbname".
-extern std::string TempFileName(const std::string& dbname, uint64_t number);
-
-// Return the name of the info log file for "dbname".
-extern std::string InfoLogFileName(const std::string& dbname);
-
-// Return the name of the old info log file for "dbname".
-extern std::string OldInfoLogFileName(const std::string& dbname);
-
-// If filename is a leveldb file, store the type of the file in *type.
-// The number encoded in the filename is stored in *number.  If the
-// filename was successfully parsed, returns true.  Else return false.
-extern bool ParseFileName(const std::string& filename,
-                          uint64_t* number,
-                          FileType* type);
-
-// Make the CURRENT file point to the descriptor file with the
-// specified number.
-extern Status SetCurrentFile(Env* env, const std::string& dbname,
-                             uint64_t descriptor_number);
-
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_FILENAME_H_

src/leveldb/db/filename_test.cc

@@ -1,123 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/filename.h"
-
-#include "db/dbformat.h"
-#include "port/port.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-class FileNameTest { };
-
-TEST(FileNameTest, Parse) {
-  Slice db;
-  FileType type;
-  uint64_t number;
-
-  // Successful parses
-  static struct {
-    const char* fname;
-    uint64_t number;
-    FileType type;
-  } cases[] = {
-    { "100.log",            100,   kLogFile },
-    { "0.log",              0,     kLogFile },
-    { "0.sst",              0,     kTableFile },
-    { "0.ldb",              0,     kTableFile },
-    { "CURRENT",            0,     kCurrentFile },
-    { "LOCK",               0,     kDBLockFile },
-    { "MANIFEST-2",         2,     kDescriptorFile },
-    { "MANIFEST-7",         7,     kDescriptorFile },
-    { "LOG",                0,     kInfoLogFile },
-    { "LOG.old",            0,     kInfoLogFile },
-    { "18446744073709551615.log", 18446744073709551615ull, kLogFile },
-  };
-  for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
-    std::string f = cases[i].fname;
-    ASSERT_TRUE(ParseFileName(f, &number, &type)) << f;
-    ASSERT_EQ(cases[i].type, type) << f;
-    ASSERT_EQ(cases[i].number, number) << f;
-  }
-
-  // Errors
-  static const char* errors[] = {
-    "",
-    "foo",
-    "foo-dx-100.log",
-    ".log",
-    "",
-    "manifest",
-    "CURREN",
-    "CURRENTX",
-    "MANIFES",
-    "MANIFEST",
-    "MANIFEST-",
-    "XMANIFEST-3",
-    "MANIFEST-3x",
-    "LOC",
-    "LOCKx",
-    "LO",
-    "LOGx",
-    "18446744073709551616.log",
-    "184467440737095516150.log",
-    "100",
-    "100.",
-    "100.lop"
-  };
-  for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
-    std::string f = errors[i];
-    ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
-  }
-}
-
-TEST(FileNameTest, Construction) {
-  uint64_t number;
-  FileType type;
-  std::string fname;
-
-  fname = CurrentFileName("foo");
-  ASSERT_EQ("foo/", std::string(fname.data(), 4));
-  ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
-  ASSERT_EQ(0, number);
-  ASSERT_EQ(kCurrentFile, type);
-
-  fname = LockFileName("foo");
-  ASSERT_EQ("foo/", std::string(fname.data(), 4));
-  ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
-  ASSERT_EQ(0, number);
-  ASSERT_EQ(kDBLockFile, type);
-
-  fname = LogFileName("foo", 192);
-  ASSERT_EQ("foo/", std::string(fname.data(), 4));
-  ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
-  ASSERT_EQ(192, number);
-  ASSERT_EQ(kLogFile, type);
-
-  fname = TableFileName("bar", 200);
-  ASSERT_EQ("bar/", std::string(fname.data(), 4));
-  ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
-  ASSERT_EQ(200, number);
-  ASSERT_EQ(kTableFile, type);
-
-  fname = DescriptorFileName("bar", 100);
-  ASSERT_EQ("bar/", std::string(fname.data(), 4));
-  ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
-  ASSERT_EQ(100, number);
-  ASSERT_EQ(kDescriptorFile, type);
-
-  fname = TempFileName("tmp", 999);
-  ASSERT_EQ("tmp/", std::string(fname.data(), 4));
-  ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
-  ASSERT_EQ(999, number);
-  ASSERT_EQ(kTempFile, type);
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/leveldbutil.cc

@@ -1,65 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include "leveldb/dumpfile.h"
-#include "leveldb/env.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-namespace {
-
-class StdoutPrinter : public WritableFile {
- public:
-  virtual Status Append(const Slice& data) {
-    fwrite(data.data(), 1, data.size(), stdout);
-    return Status::OK();
-  }
-  virtual Status Close() { return Status::OK(); }
-  virtual Status Flush() { return Status::OK(); }
-  virtual Status Sync() { return Status::OK(); }
-  virtual std::string GetName() const { return "[stdout]"; }
-};
-
-bool HandleDumpCommand(Env* env, char** files, int num) {
-  StdoutPrinter printer;
-  bool ok = true;
-  for (int i = 0; i < num; i++) {
-    Status s = DumpFile(env, files[i], &printer);
-    if (!s.ok()) {
-      fprintf(stderr, "%s\n", s.ToString().c_str());
-      ok = false;
-    }
-  }
-  return ok;
-}
-
-}  // namespace
-}  // namespace leveldb
-
-static void Usage() {
-  fprintf(
-      stderr,
-      "Usage: leveldbutil command...\n"
-      "   dump files...         -- dump contents of specified files\n"
-      );
-}
-
-int main(int argc, char** argv) {
-  leveldb::Env* env = leveldb::Env::Default();
-  bool ok = true;
-  if (argc < 2) {
-    Usage();
-    ok = false;
-  } else {
-    std::string command = argv[1];
-    if (command == "dump") {
-      ok = leveldb::HandleDumpCommand(env, argv+2, argc-2);
-    } else {
-      Usage();
-      ok = false;
-    }
-  }
-  return (ok ? 0 : 1);
-}

src/leveldb/db/log_format.h

@@ -1,35 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Log format information shared by reader and writer.
-// See ../doc/log_format.md for more detail.
-
-#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_
-#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_
-
-namespace leveldb {
-namespace log {
-
-enum RecordType {
-  // Zero is reserved for preallocated files
-  kZeroType = 0,
-
-  kFullType = 1,
-
-  // For fragments
-  kFirstType = 2,
-  kMiddleType = 3,
-  kLastType = 4
-};
-static const int kMaxRecordType = kLastType;
-
-static const int kBlockSize = 32768;
-
-// Header is checksum (4 bytes), length (2 bytes), type (1 byte).
-static const int kHeaderSize = 4 + 2 + 1;
-
-}  // namespace log
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_LOG_FORMAT_H_

src/leveldb/db/log_reader.cc

@@ -1,284 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/log_reader.h"
-
-#include <stdio.h>
-#include "leveldb/env.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-
-namespace leveldb {
-namespace log {
-
-Reader::Reporter::~Reporter() {
-}
-
-Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
-               uint64_t initial_offset)
-    : file_(file),
-      reporter_(reporter),
-      checksum_(checksum),
-      backing_store_(new char[kBlockSize]),
-      buffer_(),
-      eof_(false),
-      last_record_offset_(0),
-      end_of_buffer_offset_(0),
-      initial_offset_(initial_offset),
-      resyncing_(initial_offset > 0) {
-}
-
-Reader::~Reader() {
-  delete[] backing_store_;
-}
-
-bool Reader::SkipToInitialBlock() {
-  size_t offset_in_block = initial_offset_ % kBlockSize;
-  uint64_t block_start_location = initial_offset_ - offset_in_block;
-
-  // Don't search a block if we'd be in the trailer
-  if (offset_in_block > kBlockSize - 6) {
-    offset_in_block = 0;
-    block_start_location += kBlockSize;
-  }
-
-  end_of_buffer_offset_ = block_start_location;
-
-  // Skip to start of first block that can contain the initial record
-  if (block_start_location > 0) {
-    Status skip_status = file_->Skip(block_start_location);
-    if (!skip_status.ok()) {
-      ReportDrop(block_start_location, skip_status);
-      return false;
-    }
-  }
-
-  return true;
-}
-
-bool Reader::ReadRecord(Slice* record, std::string* scratch) {
-  if (last_record_offset_ < initial_offset_) {
-    if (!SkipToInitialBlock()) {
-      return false;
-    }
-  }
-
-  scratch->clear();
-  record->clear();
-  bool in_fragmented_record = false;
-  // Record offset of the logical record that we're reading
-  // 0 is a dummy value to make compilers happy
-  uint64_t prospective_record_offset = 0;
-
-  Slice fragment;
-  while (true) {
-    const unsigned int record_type = ReadPhysicalRecord(&fragment);
-
-    // ReadPhysicalRecord may have only had an empty trailer remaining in its
-    // internal buffer. Calculate the offset of the next physical record now
-    // that it has returned, properly accounting for its header size.
-    uint64_t physical_record_offset =
-        end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size();
-
-    if (resyncing_) {
-      if (record_type == kMiddleType) {
-        continue;
-      } else if (record_type == kLastType) {
-        resyncing_ = false;
-        continue;
-      } else {
-        resyncing_ = false;
-      }
-    }
-
-    switch (record_type) {
-      case kFullType:
-        if (in_fragmented_record) {
-          // Handle bug in earlier versions of log::Writer where
-          // it could emit an empty kFirstType record at the tail end
-          // of a block followed by a kFullType or kFirstType record
-          // at the beginning of the next block.
-          if (scratch->empty()) {
-            in_fragmented_record = false;
-          } else {
-            ReportCorruption(scratch->size(), "partial record without end(1)");
-          }
-        }
-        prospective_record_offset = physical_record_offset;
-        scratch->clear();
-        *record = fragment;
-        last_record_offset_ = prospective_record_offset;
-        return true;
-
-      case kFirstType:
-        if (in_fragmented_record) {
-          // Handle bug in earlier versions of log::Writer where
-          // it could emit an empty kFirstType record at the tail end
-          // of a block followed by a kFullType or kFirstType record
-          // at the beginning of the next block.
-          if (scratch->empty()) {
-            in_fragmented_record = false;
-          } else {
-            ReportCorruption(scratch->size(), "partial record without end(2)");
-          }
-        }
-        prospective_record_offset = physical_record_offset;
-        scratch->assign(fragment.data(), fragment.size());
-        in_fragmented_record = true;
-        break;
-
-      case kMiddleType:
-        if (!in_fragmented_record) {
-          ReportCorruption(fragment.size(),
-                           "missing start of fragmented record(1)");
-        } else {
-          scratch->append(fragment.data(), fragment.size());
-        }
-        break;
-
-      case kLastType:
-        if (!in_fragmented_record) {
-          ReportCorruption(fragment.size(),
-                           "missing start of fragmented record(2)");
-        } else {
-          scratch->append(fragment.data(), fragment.size());
-          *record = Slice(*scratch);
-          last_record_offset_ = prospective_record_offset;
-          return true;
-        }
-        break;
-
-      case kEof:
-        if (in_fragmented_record) {
-          // This can be caused by the writer dying immediately after
-          // writing a physical record but before completing the next; don't
-          // treat it as a corruption, just ignore the entire logical record.
-          scratch->clear();
-        }
-        return false;
-
-      case kBadRecord:
-        if (in_fragmented_record) {
-          ReportCorruption(scratch->size(), "error in middle of record");
-          in_fragmented_record = false;
-          scratch->clear();
-        }
-        break;
-
-      default: {
-        char buf[40];
-        snprintf(buf, sizeof(buf), "unknown record type %u", record_type);
-        ReportCorruption(
-            (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
-            buf);
-        in_fragmented_record = false;
-        scratch->clear();
-        break;
-      }
-    }
-  }
-  return false;
-}
-
-uint64_t Reader::LastRecordOffset() {
-  return last_record_offset_;
-}
-
-void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
-  ReportDrop(bytes, Status::Corruption(reason, file_->GetName()));
-}
-
-void Reader::ReportDrop(uint64_t bytes, const Status& reason) {
-  if (reporter_ != NULL &&
-      end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
-    reporter_->Corruption(static_cast<size_t>(bytes), reason);
-  }
-}
-
-unsigned int Reader::ReadPhysicalRecord(Slice* result) {
-  while (true) {
-    if (buffer_.size() < kHeaderSize) {
-      if (!eof_) {
-        // Last read was a full read, so this is a trailer to skip
-        buffer_.clear();
-        Status status = file_->Read(kBlockSize, &buffer_, backing_store_);
-        end_of_buffer_offset_ += buffer_.size();
-        if (!status.ok()) {
-          buffer_.clear();
-          ReportDrop(kBlockSize, status);
-          eof_ = true;
-          return kEof;
-        } else if (buffer_.size() < kBlockSize) {
-          eof_ = true;
-        }
-        continue;
-      } else {
-        // Note that if buffer_ is non-empty, we have a truncated header at the
-        // end of the file, which can be caused by the writer crashing in the
-        // middle of writing the header. Instead of considering this an error,
-        // just report EOF.
-        buffer_.clear();
-        return kEof;
-      }
-    }
-
-    // Parse the header
-    const char* header = buffer_.data();
-    const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
-    const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
-    const unsigned int type = header[6];
-    const uint32_t length = a | (b << 8);
-    if (kHeaderSize + length > buffer_.size()) {
-      size_t drop_size = buffer_.size();
-      buffer_.clear();
-      if (!eof_) {
-        ReportCorruption(drop_size, "bad record length");
-        return kBadRecord;
-      }
-      // If the end of the file has been reached without reading |length| bytes
-      // of payload, assume the writer died in the middle of writing the record.
-      // Don't report a corruption.
-      return kEof;
-    }
-
-    if (type == kZeroType && length == 0) {
-      // Skip zero length record without reporting any drops since
-      // such records are produced by the mmap based writing code in
-      // env_posix.cc that preallocates file regions.
-      buffer_.clear();
-      return kBadRecord;
-    }
-
-    // Check crc
-    if (checksum_) {
-      uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header));
-      uint32_t actual_crc = crc32c::Value(header + 6, 1 + length);
-      if (actual_crc != expected_crc) {
-        // Drop the rest of the buffer since "length" itself may have
-        // been corrupted and if we trust it, we could find some
-        // fragment of a real log record that just happens to look
-        // like a valid log record.
-        size_t drop_size = buffer_.size();
-        buffer_.clear();
-        ReportCorruption(drop_size, "checksum mismatch");
-        return kBadRecord;
-      }
-    }
-
-    buffer_.remove_prefix(kHeaderSize + length);
-
-    // Skip physical record that started before initial_offset_
-    if (end_of_buffer_offset_ - buffer_.size() - kHeaderSize - length <
-        initial_offset_) {
-      result->clear();
-      return kBadRecord;
-    }
-
-    *result = Slice(header + kHeaderSize, length);
-    return type;
-  }
-}
-
-}  // namespace log
-}  // namespace leveldb

src/leveldb/db/log_reader.h

@@ -1,113 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_LOG_READER_H_
-#define STORAGE_LEVELDB_DB_LOG_READER_H_
-
-#include <stdint.h>
-
-#include "db/log_format.h"
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class SequentialFile;
-
-namespace log {
-
-class Reader {
- public:
-  // Interface for reporting errors.
-  class Reporter {
-   public:
-    virtual ~Reporter();
-
-    // Some corruption was detected.  "size" is the approximate number
-    // of bytes dropped due to the corruption.
-    virtual void Corruption(size_t bytes, const Status& status) = 0;
-  };
-
-  // Create a reader that will return log records from "*file".
-  // "*file" must remain live while this Reader is in use.
-  //
-  // If "reporter" is non-NULL, it is notified whenever some data is
-  // dropped due to a detected corruption.  "*reporter" must remain
-  // live while this Reader is in use.
-  //
-  // If "checksum" is true, verify checksums if available.
-  //
-  // The Reader will start reading at the first record located at physical
-  // position >= initial_offset within the file.
-  Reader(SequentialFile* file, Reporter* reporter, bool checksum,
-         uint64_t initial_offset);
-
-  ~Reader();
-
-  // Read the next record into *record.  Returns true if read
-  // successfully, false if we hit end of the input.  May use
-  // "*scratch" as temporary storage.  The contents filled in *record
-  // will only be valid until the next mutating operation on this
-  // reader or the next mutation to *scratch.
-  bool ReadRecord(Slice* record, std::string* scratch);
-
-  // Returns the physical offset of the last record returned by ReadRecord.
-  //
-  // Undefined before the first call to ReadRecord.
-  uint64_t LastRecordOffset();
-
- private:
-  SequentialFile* const file_;
-  Reporter* const reporter_;
-  bool const checksum_;
-  char* const backing_store_;
-  Slice buffer_;
-  bool eof_;   // Last Read() indicated EOF by returning < kBlockSize
-
-  // Offset of the last record returned by ReadRecord.
-  uint64_t last_record_offset_;
-  // Offset of the first location past the end of buffer_.
-  uint64_t end_of_buffer_offset_;
-
-  // Offset at which to start looking for the first record to return
-  uint64_t const initial_offset_;
-
-  // True if we are resynchronizing after a seek (initial_offset_ > 0). In
-  // particular, a run of kMiddleType and kLastType records can be silently
-  // skipped in this mode
-  bool resyncing_;
-
-  // Extend record types with the following special values
-  enum {
-    kEof = kMaxRecordType + 1,
-    // Returned whenever we find an invalid physical record.
-    // Currently there are three situations in which this happens:
-    // * The record has an invalid CRC (ReadPhysicalRecord reports a drop)
-    // * The record is a 0-length record (No drop is reported)
-    // * The record is below constructor's initial_offset (No drop is reported)
-    kBadRecord = kMaxRecordType + 2
-  };
-
-  // Skips all blocks that are completely before "initial_offset_".
-  //
-  // Returns true on success. Handles reporting.
-  bool SkipToInitialBlock();
-
-  // Return type, or one of the preceding special values
-  unsigned int ReadPhysicalRecord(Slice* result);
-
-  // Reports dropped bytes to the reporter.
-  // buffer_ must be updated to remove the dropped bytes prior to invocation.
-  void ReportCorruption(uint64_t bytes, const char* reason);
-  void ReportDrop(uint64_t bytes, const Status& reason);
-
-  // No copying allowed
-  Reader(const Reader&);
-  void operator=(const Reader&);
-};
-
-}  // namespace log
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_LOG_READER_H_

src/leveldb/db/log_test.cc

@@ -1,591 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "leveldb/env.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-#include "util/random.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-namespace log {
-
-// Construct a string of the specified length made out of the supplied
-// partial string.
-static std::string BigString(const std::string& partial_string, size_t n) {
-  std::string result;
-  while (result.size() < n) {
-    result.append(partial_string);
-  }
-  result.resize(n);
-  return result;
-}
-
-// Construct a string from a number
-static std::string NumberString(int n) {
-  char buf[50];
-  snprintf(buf, sizeof(buf), "%d.", n);
-  return std::string(buf);
-}
-
-// Return a skewed potentially long string
-static std::string RandomSkewedString(int i, Random* rnd) {
-  return BigString(NumberString(i), rnd->Skewed(17));
-}
-
-class LogTest {
- private:
-  class StringDest : public WritableFile {
-   public:
-    std::string contents_;
-
-    virtual Status Close() { return Status::OK(); }
-    virtual Status Flush() { return Status::OK(); }
-    virtual Status Sync() { return Status::OK(); }
-    virtual Status Append(const Slice& slice) {
-      contents_.append(slice.data(), slice.size());
-      return Status::OK();
-    }
-  };
-
-  class StringSource : public SequentialFile {
-   public:
-    Slice contents_;
-    bool force_error_;
-    bool returned_partial_;
-    StringSource() : force_error_(false), returned_partial_(false) { }
-
-    virtual Status Read(size_t n, Slice* result, char* scratch) {
-      ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error";
-
-      if (force_error_) {
-        force_error_ = false;
-        returned_partial_ = true;
-        return Status::Corruption("read error");
-      }
-
-      if (contents_.size() < n) {
-        n = contents_.size();
-        returned_partial_ = true;
-      }
-      *result = Slice(contents_.data(), n);
-      contents_.remove_prefix(n);
-      return Status::OK();
-    }
-
-    virtual Status Skip(uint64_t n) {
-      if (n > contents_.size()) {
-        contents_.clear();
-        return Status::NotFound("in-memory file skipped past end");
-      }
-
-      contents_.remove_prefix(n);
-
-      return Status::OK();
-    }
-  };
-
-  class ReportCollector : public Reader::Reporter {
-   public:
-    size_t dropped_bytes_;
-    std::string message_;
-
-    ReportCollector() : dropped_bytes_(0) { }
-    virtual void Corruption(size_t bytes, const Status& status) {
-      dropped_bytes_ += bytes;
-      message_.append(status.ToString());
-    }
-  };
-
-  StringDest dest_;
-  StringSource source_;
-  ReportCollector report_;
-  bool reading_;
-  Writer* writer_;
-  Reader* reader_;
-
-  // Record metadata for testing initial offset functionality
-  static size_t initial_offset_record_sizes_[];
-  static uint64_t initial_offset_last_record_offsets_[];
-  static int num_initial_offset_records_;
-
- public:
-  LogTest() : reading_(false),
-              writer_(new Writer(&dest_)),
-              reader_(new Reader(&source_, &report_, true/*checksum*/,
-                      0/*initial_offset*/)) {
-  }
-
-  ~LogTest() {
-    delete writer_;
-    delete reader_;
-  }
-
-  void ReopenForAppend() {
-    delete writer_;
-    writer_ = new Writer(&dest_, dest_.contents_.size());
-  }
-
-  void Write(const std::string& msg) {
-    ASSERT_TRUE(!reading_) << "Write() after starting to read";
-    writer_->AddRecord(Slice(msg));
-  }
-
-  size_t WrittenBytes() const {
-    return dest_.contents_.size();
-  }
-
-  std::string Read() {
-    if (!reading_) {
-      reading_ = true;
-      source_.contents_ = Slice(dest_.contents_);
-    }
-    std::string scratch;
-    Slice record;
-    if (reader_->ReadRecord(&record, &scratch)) {
-      return record.ToString();
-    } else {
-      return "EOF";
-    }
-  }
-
-  void IncrementByte(int offset, int delta) {
-    dest_.contents_[offset] += delta;
-  }
-
-  void SetByte(int offset, char new_byte) {
-    dest_.contents_[offset] = new_byte;
-  }
-
-  void ShrinkSize(int bytes) {
-    dest_.contents_.resize(dest_.contents_.size() - bytes);
-  }
-
-  void FixChecksum(int header_offset, int len) {
-    // Compute crc of type/len/data
-    uint32_t crc = crc32c::Value(&dest_.contents_[header_offset+6], 1 + len);
-    crc = crc32c::Mask(crc);
-    EncodeFixed32(&dest_.contents_[header_offset], crc);
-  }
-
-  void ForceError() {
-    source_.force_error_ = true;
-  }
-
-  size_t DroppedBytes() const {
-    return report_.dropped_bytes_;
-  }
-
-  std::string ReportMessage() const {
-    return report_.message_;
-  }
-
-  // Returns OK iff recorded error message contains "msg"
-  std::string MatchError(const std::string& msg) const {
-    if (report_.message_.find(msg) == std::string::npos) {
-      return report_.message_;
-    } else {
-      return "OK";
-    }
-  }
-
-  void WriteInitialOffsetLog() {
-    for (int i = 0; i < num_initial_offset_records_; i++) {
-      std::string record(initial_offset_record_sizes_[i],
-                         static_cast<char>('a' + i));
-      Write(record);
-    }
-  }
-
-  void StartReadingAt(uint64_t initial_offset) {
-    delete reader_;
-    reader_ = new Reader(&source_, &report_, true/*checksum*/, initial_offset);
-  }
-
-  void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
-    WriteInitialOffsetLog();
-    reading_ = true;
-    source_.contents_ = Slice(dest_.contents_);
-    Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
-                                       WrittenBytes() + offset_past_end);
-    Slice record;
-    std::string scratch;
-    ASSERT_TRUE(!offset_reader->ReadRecord(&record, &scratch));
-    delete offset_reader;
-  }
-
-  void CheckInitialOffsetRecord(uint64_t initial_offset,
-                                int expected_record_offset) {
-    WriteInitialOffsetLog();
-    reading_ = true;
-    source_.contents_ = Slice(dest_.contents_);
-    Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
-                                       initial_offset);
-
-    // Read all records from expected_record_offset through the last one.
-    ASSERT_LT(expected_record_offset, num_initial_offset_records_);
-    for (; expected_record_offset < num_initial_offset_records_;
-         ++expected_record_offset) {
-      Slice record;
-      std::string scratch;
-      ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
-      ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
-                record.size());
-      ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
-                offset_reader->LastRecordOffset());
-      ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
-    }
-    delete offset_reader;
-  }
-};
-
-size_t LogTest::initial_offset_record_sizes_[] =
-    {10000,  // Two sizable records in first block
-     10000,
-     2 * log::kBlockSize - 1000,  // Span three blocks
-     1,
-     13716,  // Consume all but two bytes of block 3.
-     log::kBlockSize - kHeaderSize, // Consume the entirety of block 4.
-    };
-
-uint64_t LogTest::initial_offset_last_record_offsets_[] =
-    {0,
-     kHeaderSize + 10000,
-     2 * (kHeaderSize + 10000),
-     2 * (kHeaderSize + 10000) +
-         (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
-     2 * (kHeaderSize + 10000) +
-         (2 * log::kBlockSize - 1000) + 3 * kHeaderSize
-         + kHeaderSize + 1,
-     3 * log::kBlockSize,
-    };
-
-// LogTest::initial_offset_last_record_offsets_ must be defined before this.
-int LogTest::num_initial_offset_records_ =
-    sizeof(LogTest::initial_offset_last_record_offsets_)/sizeof(uint64_t);
-
-TEST(LogTest, Empty) {
-  ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, ReadWrite) {
-  Write("foo");
-  Write("bar");
-  Write("");
-  Write("xxxx");
-  ASSERT_EQ("foo", Read());
-  ASSERT_EQ("bar", Read());
-  ASSERT_EQ("", Read());
-  ASSERT_EQ("xxxx", Read());
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ("EOF", Read());  // Make sure reads at eof work
-}
-
-TEST(LogTest, ManyBlocks) {
-  for (int i = 0; i < 100000; i++) {
-    Write(NumberString(i));
-  }
-  for (int i = 0; i < 100000; i++) {
-    ASSERT_EQ(NumberString(i), Read());
-  }
-  ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, Fragmentation) {
-  Write("small");
-  Write(BigString("medium", 50000));
-  Write(BigString("large", 100000));
-  ASSERT_EQ("small", Read());
-  ASSERT_EQ(BigString("medium", 50000), Read());
-  ASSERT_EQ(BigString("large", 100000), Read());
-  ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, MarginalTrailer) {
-  // Make a trailer that is exactly the same length as an empty record.
-  const int n = kBlockSize - 2*kHeaderSize;
-  Write(BigString("foo", n));
-  ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
-  Write("");
-  Write("bar");
-  ASSERT_EQ(BigString("foo", n), Read());
-  ASSERT_EQ("", Read());
-  ASSERT_EQ("bar", Read());
-  ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, MarginalTrailer2) {
-  // Make a trailer that is exactly the same length as an empty record.
-  const int n = kBlockSize - 2*kHeaderSize;
-  Write(BigString("foo", n));
-  ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
-  Write("bar");
-  ASSERT_EQ(BigString("foo", n), Read());
-  ASSERT_EQ("bar", Read());
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ(0, DroppedBytes());
-  ASSERT_EQ("", ReportMessage());
-}
-
-TEST(LogTest, ShortTrailer) {
-  const int n = kBlockSize - 2*kHeaderSize + 4;
-  Write(BigString("foo", n));
-  ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
-  Write("");
-  Write("bar");
-  ASSERT_EQ(BigString("foo", n), Read());
-  ASSERT_EQ("", Read());
-  ASSERT_EQ("bar", Read());
-  ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, AlignedEof) {
-  const int n = kBlockSize - 2*kHeaderSize + 4;
-  Write(BigString("foo", n));
-  ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
-  ASSERT_EQ(BigString("foo", n), Read());
-  ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, OpenForAppend) {
-  Write("hello");
-  ReopenForAppend();
-  Write("world");
-  ASSERT_EQ("hello", Read());
-  ASSERT_EQ("world", Read());
-  ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, RandomRead) {
-  const int N = 500;
-  Random write_rnd(301);
-  for (int i = 0; i < N; i++) {
-    Write(RandomSkewedString(i, &write_rnd));
-  }
-  Random read_rnd(301);
-  for (int i = 0; i < N; i++) {
-    ASSERT_EQ(RandomSkewedString(i, &read_rnd), Read());
-  }
-  ASSERT_EQ("EOF", Read());
-}
-
-// Tests of all the error paths in log_reader.cc follow:
-
-TEST(LogTest, ReadError) {
-  Write("foo");
-  ForceError();
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ(kBlockSize, DroppedBytes());
-  ASSERT_EQ("OK", MatchError("read error"));
-}
-
-TEST(LogTest, BadRecordType) {
-  Write("foo");
-  // Type is stored in header[6]
-  IncrementByte(6, 100);
-  FixChecksum(0, 3);
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ(3, DroppedBytes());
-  ASSERT_EQ("OK", MatchError("unknown record type"));
-}
-
-TEST(LogTest, TruncatedTrailingRecordIsIgnored) {
-  Write("foo");
-  ShrinkSize(4);   // Drop all payload as well as a header byte
-  ASSERT_EQ("EOF", Read());
-  // Truncated last record is ignored, not treated as an error.
-  ASSERT_EQ(0, DroppedBytes());
-  ASSERT_EQ("", ReportMessage());
-}
-
-TEST(LogTest, BadLength) {
-  const int kPayloadSize = kBlockSize - kHeaderSize;
-  Write(BigString("bar", kPayloadSize));
-  Write("foo");
-  // Least significant size byte is stored in header[4].
-  IncrementByte(4, 1);
-  ASSERT_EQ("foo", Read());
-  ASSERT_EQ(kBlockSize, DroppedBytes());
-  ASSERT_EQ("OK", MatchError("bad record length"));
-}
-
-TEST(LogTest, BadLengthAtEndIsIgnored) {
-  Write("foo");
-  ShrinkSize(1);
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ(0, DroppedBytes());
-  ASSERT_EQ("", ReportMessage());
-}
-
-TEST(LogTest, ChecksumMismatch) {
-  Write("foo");
-  IncrementByte(0, 10);
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ(10, DroppedBytes());
-  ASSERT_EQ("OK", MatchError("checksum mismatch"));
-}
-
-TEST(LogTest, UnexpectedMiddleType) {
-  Write("foo");
-  SetByte(6, kMiddleType);
-  FixChecksum(0, 3);
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ(3, DroppedBytes());
-  ASSERT_EQ("OK", MatchError("missing start"));
-}
-
-TEST(LogTest, UnexpectedLastType) {
-  Write("foo");
-  SetByte(6, kLastType);
-  FixChecksum(0, 3);
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ(3, DroppedBytes());
-  ASSERT_EQ("OK", MatchError("missing start"));
-}
-
-TEST(LogTest, UnexpectedFullType) {
-  Write("foo");
-  Write("bar");
-  SetByte(6, kFirstType);
-  FixChecksum(0, 3);
-  ASSERT_EQ("bar", Read());
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ(3, DroppedBytes());
-  ASSERT_EQ("OK", MatchError("partial record without end"));
-}
-
-TEST(LogTest, UnexpectedFirstType) {
-  Write("foo");
-  Write(BigString("bar", 100000));
-  SetByte(6, kFirstType);
-  FixChecksum(0, 3);
-  ASSERT_EQ(BigString("bar", 100000), Read());
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ(3, DroppedBytes());
-  ASSERT_EQ("OK", MatchError("partial record without end"));
-}
-
-TEST(LogTest, MissingLastIsIgnored) {
-  Write(BigString("bar", kBlockSize));
-  // Remove the LAST block, including header.
-  ShrinkSize(14);
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ("", ReportMessage());
-  ASSERT_EQ(0, DroppedBytes());
-}
-
-TEST(LogTest, PartialLastIsIgnored) {
-  Write(BigString("bar", kBlockSize));
-  // Cause a bad record length in the LAST block.
-  ShrinkSize(1);
-  ASSERT_EQ("EOF", Read());
-  ASSERT_EQ("", ReportMessage());
-  ASSERT_EQ(0, DroppedBytes());
-}
-
-TEST(LogTest, SkipIntoMultiRecord) {
-  // Consider a fragmented record:
-  //    first(R1), middle(R1), last(R1), first(R2)
-  // If initial_offset points to a record after first(R1) but before first(R2)
-  // incomplete fragment errors are not actual errors, and must be suppressed
-  // until a new first or full record is encountered.
-  Write(BigString("foo", 3*kBlockSize));
-  Write("correct");
-  StartReadingAt(kBlockSize);
-
-  ASSERT_EQ("correct", Read());
-  ASSERT_EQ("", ReportMessage());
-  ASSERT_EQ(0, DroppedBytes());
-  ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, ErrorJoinsRecords) {
-  // Consider two fragmented records:
-  //    first(R1) last(R1) first(R2) last(R2)
-  // where the middle two fragments disappear.  We do not want
-  // first(R1),last(R2) to get joined and returned as a valid record.
-
-  // Write records that span two blocks
-  Write(BigString("foo", kBlockSize));
-  Write(BigString("bar", kBlockSize));
-  Write("correct");
-
-  // Wipe the middle block
-  for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
-    SetByte(offset, 'x');
-  }
-
-  ASSERT_EQ("correct", Read());
-  ASSERT_EQ("EOF", Read());
-  const size_t dropped = DroppedBytes();
-  ASSERT_LE(dropped, 2*kBlockSize + 100);
-  ASSERT_GE(dropped, 2*kBlockSize);
-}
-
-TEST(LogTest, ReadStart) {
-  CheckInitialOffsetRecord(0, 0);
-}
-
-TEST(LogTest, ReadSecondOneOff) {
-  CheckInitialOffsetRecord(1, 1);
-}
-
-TEST(LogTest, ReadSecondTenThousand) {
-  CheckInitialOffsetRecord(10000, 1);
-}
-
-TEST(LogTest, ReadSecondStart) {
-  CheckInitialOffsetRecord(10007, 1);
-}
-
-TEST(LogTest, ReadThirdOneOff) {
-  CheckInitialOffsetRecord(10008, 2);
-}
-
-TEST(LogTest, ReadThirdStart) {
-  CheckInitialOffsetRecord(20014, 2);
-}
-
-TEST(LogTest, ReadFourthOneOff) {
-  CheckInitialOffsetRecord(20015, 3);
-}
-
-TEST(LogTest, ReadFourthFirstBlockTrailer) {
-  CheckInitialOffsetRecord(log::kBlockSize - 4, 3);
-}
-
-TEST(LogTest, ReadFourthMiddleBlock) {
-  CheckInitialOffsetRecord(log::kBlockSize + 1, 3);
-}
-
-TEST(LogTest, ReadFourthLastBlock) {
-  CheckInitialOffsetRecord(2 * log::kBlockSize + 1, 3);
-}
-
-TEST(LogTest, ReadFourthStart) {
-  CheckInitialOffsetRecord(
-      2 * (kHeaderSize + 1000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
-      3);
-}
-
-TEST(LogTest, ReadInitialOffsetIntoBlockPadding) {
-  CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5);
-}
-
-TEST(LogTest, ReadEnd) {
-  CheckOffsetPastEndReturnsNoRecords(0);
-}
-
-TEST(LogTest, ReadPastEnd) {
-  CheckOffsetPastEndReturnsNoRecords(5);
-}
-
-}  // namespace log
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/log_writer.cc

@@ -1,112 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/log_writer.h"
-
-#include <stdint.h>
-#include "leveldb/env.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-
-namespace leveldb {
-namespace log {
-
-static void InitTypeCrc(uint32_t* type_crc) {
-  for (int i = 0; i <= kMaxRecordType; i++) {
-    char t = static_cast<char>(i);
-    type_crc[i] = crc32c::Value(&t, 1);
-  }
-}
-
-Writer::Writer(WritableFile* dest)
-    : dest_(dest),
-      block_offset_(0) {
-  InitTypeCrc(type_crc_);
-}
-
-Writer::Writer(WritableFile* dest, uint64_t dest_length)
-    : dest_(dest), block_offset_(dest_length % kBlockSize) {
-  InitTypeCrc(type_crc_);
-}
-
-Writer::~Writer() {
-}
-
-Status Writer::AddRecord(const Slice& slice) {
-  const char* ptr = slice.data();
-  size_t left = slice.size();
-
-  // Fragment the record if necessary and emit it.  Note that if slice
-  // is empty, we still want to iterate once to emit a single
-  // zero-length record
-  Status s;
-  bool begin = true;
-  do {
-    const int leftover = kBlockSize - block_offset_;
-    assert(leftover >= 0);
-    if (leftover < kHeaderSize) {
-      // Switch to a new block
-      if (leftover > 0) {
-        // Fill the trailer (literal below relies on kHeaderSize being 7)
-        assert(kHeaderSize == 7);
-        dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover));
-      }
-      block_offset_ = 0;
-    }
-
-    // Invariant: we never leave < kHeaderSize bytes in a block.
-    assert(kBlockSize - block_offset_ - kHeaderSize >= 0);
-
-    const size_t avail = kBlockSize - block_offset_ - kHeaderSize;
-    const size_t fragment_length = (left < avail) ? left : avail;
-
-    RecordType type;
-    const bool end = (left == fragment_length);
-    if (begin && end) {
-      type = kFullType;
-    } else if (begin) {
-      type = kFirstType;
-    } else if (end) {
-      type = kLastType;
-    } else {
-      type = kMiddleType;
-    }
-
-    s = EmitPhysicalRecord(type, ptr, fragment_length);
-    ptr += fragment_length;
-    left -= fragment_length;
-    begin = false;
-  } while (s.ok() && left > 0);
-  return s;
-}
-
-Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
-  assert(n <= 0xffff);  // Must fit in two bytes
-  assert(block_offset_ + kHeaderSize + n <= kBlockSize);
-
-  // Format the header
-  char buf[kHeaderSize];
-  buf[4] = static_cast<char>(n & 0xff);
-  buf[5] = static_cast<char>(n >> 8);
-  buf[6] = static_cast<char>(t);
-
-  // Compute the crc of the record type and the payload.
-  uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n);
-  crc = crc32c::Mask(crc);                 // Adjust for storage
-  EncodeFixed32(buf, crc);
-
-  // Write the header and the payload
-  Status s = dest_->Append(Slice(buf, kHeaderSize));
-  if (s.ok()) {
-    s = dest_->Append(Slice(ptr, n));
-    if (s.ok()) {
-      s = dest_->Flush();
-    }
-  }
-  block_offset_ += kHeaderSize + n;
-  return s;
-}
-
-}  // namespace log
-}  // namespace leveldb

src/leveldb/db/log_writer.h

@@ -1,54 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_
-#define STORAGE_LEVELDB_DB_LOG_WRITER_H_
-
-#include <stdint.h>
-#include "db/log_format.h"
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class WritableFile;
-
-namespace log {
-
-class Writer {
- public:
-  // Create a writer that will append data to "*dest".
-  // "*dest" must be initially empty.
-  // "*dest" must remain live while this Writer is in use.
-  explicit Writer(WritableFile* dest);
-
-  // Create a writer that will append data to "*dest".
-  // "*dest" must have initial length "dest_length".
-  // "*dest" must remain live while this Writer is in use.
-  Writer(WritableFile* dest, uint64_t dest_length);
-
-  ~Writer();
-
-  Status AddRecord(const Slice& slice);
-
- private:
-  WritableFile* dest_;
-  int block_offset_;       // Current offset in block
-
-  // crc32c values for all supported record types.  These are
-  // pre-computed to reduce the overhead of computing the crc of the
-  // record type stored in the header.
-  uint32_t type_crc_[kMaxRecordType + 1];
-
-  Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
-
-  // No copying allowed
-  Writer(const Writer&);
-  void operator=(const Writer&);
-};
-
-}  // namespace log
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_LOG_WRITER_H_

src/leveldb/db/memtable.cc

@@ -1,145 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/memtable.h"
-#include "db/dbformat.h"
-#include "leveldb/comparator.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-static Slice GetLengthPrefixedSlice(const char* data) {
-  uint32_t len;
-  const char* p = data;
-  p = GetVarint32Ptr(p, p + 5, &len);  // +5: we assume "p" is not corrupted
-  return Slice(p, len);
-}
-
-MemTable::MemTable(const InternalKeyComparator& cmp)
-    : comparator_(cmp),
-      refs_(0),
-      table_(comparator_, &arena_) {
-}
-
-MemTable::~MemTable() {
-  assert(refs_ == 0);
-}
-
-size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); }
-
-int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
-    const {
-  // Internal keys are encoded as length-prefixed strings.
-  Slice a = GetLengthPrefixedSlice(aptr);
-  Slice b = GetLengthPrefixedSlice(bptr);
-  return comparator.Compare(a, b);
-}
-
-// Encode a suitable internal key target for "target" and return it.
-// Uses *scratch as scratch space, and the returned pointer will point
-// into this scratch space.
-static const char* EncodeKey(std::string* scratch, const Slice& target) {
-  scratch->clear();
-  PutVarint32(scratch, target.size());
-  scratch->append(target.data(), target.size());
-  return scratch->data();
-}
-
-class MemTableIterator: public Iterator {
- public:
-  explicit MemTableIterator(MemTable::Table* table) : iter_(table) { }
-
-  virtual bool Valid() const { return iter_.Valid(); }
-  virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); }
-  virtual void SeekToFirst() { iter_.SeekToFirst(); }
-  virtual void SeekToLast() { iter_.SeekToLast(); }
-  virtual void Next() { iter_.Next(); }
-  virtual void Prev() { iter_.Prev(); }
-  virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); }
-  virtual Slice value() const {
-    Slice key_slice = GetLengthPrefixedSlice(iter_.key());
-    return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
-  }
-
-  virtual Status status() const { return Status::OK(); }
-
- private:
-  MemTable::Table::Iterator iter_;
-  std::string tmp_;       // For passing to EncodeKey
-
-  // No copying allowed
-  MemTableIterator(const MemTableIterator&);
-  void operator=(const MemTableIterator&);
-};
-
-Iterator* MemTable::NewIterator() {
-  return new MemTableIterator(&table_);
-}
-
-void MemTable::Add(SequenceNumber s, ValueType type,
-                   const Slice& key,
-                   const Slice& value) {
-  // Format of an entry is concatenation of:
-  //  key_size     : varint32 of internal_key.size()
-  //  key bytes    : char[internal_key.size()]
-  //  value_size   : varint32 of value.size()
-  //  value bytes  : char[value.size()]
-  size_t key_size = key.size();
-  size_t val_size = value.size();
-  size_t internal_key_size = key_size + 8;
-  const size_t encoded_len =
-      VarintLength(internal_key_size) + internal_key_size +
-      VarintLength(val_size) + val_size;
-  char* buf = arena_.Allocate(encoded_len);
-  char* p = EncodeVarint32(buf, internal_key_size);
-  memcpy(p, key.data(), key_size);
-  p += key_size;
-  EncodeFixed64(p, (s << 8) | type);
-  p += 8;
-  p = EncodeVarint32(p, val_size);
-  memcpy(p, value.data(), val_size);
-  assert(p + val_size == buf + encoded_len);
-  table_.Insert(buf);
-}
-
-bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
-  Slice memkey = key.memtable_key();
-  Table::Iterator iter(&table_);
-  iter.Seek(memkey.data());
-  if (iter.Valid()) {
-    // entry format is:
-    //    klength  varint32
-    //    userkey  char[klength]
-    //    tag      uint64
-    //    vlength  varint32
-    //    value    char[vlength]
-    // Check that it belongs to same user key.  We do not check the
-    // sequence number since the Seek() call above should have skipped
-    // all entries with overly large sequence numbers.
-    const char* entry = iter.key();
-    uint32_t key_length;
-    const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
-    if (comparator_.comparator.user_comparator()->Compare(
-            Slice(key_ptr, key_length - 8),
-            key.user_key()) == 0) {
-      // Correct user key
-      const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
-      switch (static_cast<ValueType>(tag & 0xff)) {
-        case kTypeValue: {
-          Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
-          value->assign(v.data(), v.size());
-          return true;
-        }
-        case kTypeDeletion:
-          *s = Status::NotFound(Slice());
-          return true;
-      }
-    }
-  }
-  return false;
-}
-
-}  // namespace leveldb

src/leveldb/db/memtable.h

@@ -1,88 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_
-#define STORAGE_LEVELDB_DB_MEMTABLE_H_
-
-#include <string>
-#include "leveldb/db.h"
-#include "db/dbformat.h"
-#include "db/skiplist.h"
-#include "util/arena.h"
-
-namespace leveldb {
-
-class InternalKeyComparator;
-class Mutex;
-class MemTableIterator;
-
-class MemTable {
- public:
-  // MemTables are reference counted.  The initial reference count
-  // is zero and the caller must call Ref() at least once.
-  explicit MemTable(const InternalKeyComparator& comparator);
-
-  // Increase reference count.
-  void Ref() { ++refs_; }
-
-  // Drop reference count.  Delete if no more references exist.
-  void Unref() {
-    --refs_;
-    assert(refs_ >= 0);
-    if (refs_ <= 0) {
-      delete this;
-    }
-  }
-
-  // Returns an estimate of the number of bytes of data in use by this
-  // data structure. It is safe to call when MemTable is being modified.
-  size_t ApproximateMemoryUsage();
-
-  // Return an iterator that yields the contents of the memtable.
-  //
-  // The caller must ensure that the underlying MemTable remains live
-  // while the returned iterator is live.  The keys returned by this
-  // iterator are internal keys encoded by AppendInternalKey in the
-  // db/format.{h,cc} module.
-  Iterator* NewIterator();
-
-  // Add an entry into memtable that maps key to value at the
-  // specified sequence number and with the specified type.
-  // Typically value will be empty if type==kTypeDeletion.
-  void Add(SequenceNumber seq, ValueType type,
-           const Slice& key,
-           const Slice& value);
-
-  // If memtable contains a value for key, store it in *value and return true.
-  // If memtable contains a deletion for key, store a NotFound() error
-  // in *status and return true.
-  // Else, return false.
-  bool Get(const LookupKey& key, std::string* value, Status* s);
-
- private:
-  ~MemTable();  // Private since only Unref() should be used to delete it
-
-  struct KeyComparator {
-    const InternalKeyComparator comparator;
-    explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
-    int operator()(const char* a, const char* b) const;
-  };
-  friend class MemTableIterator;
-  friend class MemTableBackwardIterator;
-
-  typedef SkipList<const char*, KeyComparator> Table;
-
-  KeyComparator comparator_;
-  int refs_;
-  Arena arena_;
-  Table table_;
-
-  // No copying allowed
-  MemTable(const MemTable&);
-  void operator=(const MemTable&);
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_MEMTABLE_H_

src/leveldb/db/recovery_test.cc

@@ -1,324 +0,0 @@
-// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/db_impl.h"
-#include "db/filename.h"
-#include "db/version_set.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/write_batch.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-class RecoveryTest {
- public:
-  RecoveryTest() : env_(Env::Default()), db_(NULL) {
-    dbname_ = test::TmpDir() + "/recovery_test";
-    DestroyDB(dbname_, Options());
-    Open();
-  }
-
-  ~RecoveryTest() {
-    Close();
-    DestroyDB(dbname_, Options());
-  }
-
-  DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); }
-  Env* env() const { return env_; }
-
-  bool CanAppend() {
-    WritableFile* tmp;
-    Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp);
-    delete tmp;
-    if (s.IsNotSupportedError()) {
-      return false;
-    } else {
-      return true;
-    }
-  }
-
-  void Close() {
-    delete db_;
-    db_ = NULL;
-  }
-
-  void Open(Options* options = NULL) {
-    Close();
-    Options opts;
-    if (options != NULL) {
-      opts = *options;
-    } else {
-      opts.reuse_logs = true;  // TODO(sanjay): test both ways
-      opts.create_if_missing = true;
-    }
-    if (opts.env == NULL) {
-      opts.env = env_;
-    }
-    ASSERT_OK(DB::Open(opts, dbname_, &db_));
-    ASSERT_EQ(1, NumLogs());
-  }
-
-  Status Put(const std::string& k, const std::string& v) {
-    return db_->Put(WriteOptions(), k, v);
-  }
-
-  std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
-    std::string result;
-    Status s = db_->Get(ReadOptions(), k, &result);
-    if (s.IsNotFound()) {
-      result = "NOT_FOUND";
-    } else if (!s.ok()) {
-      result = s.ToString();
-    }
-    return result;
-  }
-
-  std::string ManifestFileName() {
-    std::string current;
-    ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), &current));
-    size_t len = current.size();
-    if (len > 0 && current[len-1] == '\n') {
-      current.resize(len - 1);
-    }
-    return dbname_ + "/" + current;
-  }
-
-  std::string LogName(uint64_t number) {
-    return LogFileName(dbname_, number);
-  }
-
-  size_t DeleteLogFiles() {
-    std::vector<uint64_t> logs = GetFiles(kLogFile);
-    for (size_t i = 0; i < logs.size(); i++) {
-      ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]);
-    }
-    return logs.size();
-  }
-
-  uint64_t FirstLogFile() {
-    return GetFiles(kLogFile)[0];
-  }
-
-  std::vector<uint64_t> GetFiles(FileType t) {
-    std::vector<std::string> filenames;
-    ASSERT_OK(env_->GetChildren(dbname_, &filenames));
-    std::vector<uint64_t> result;
-    for (size_t i = 0; i < filenames.size(); i++) {
-      uint64_t number;
-      FileType type;
-      if (ParseFileName(filenames[i], &number, &type) && type == t) {
-        result.push_back(number);
-      }
-    }
-    return result;
-  }
-
-  int NumLogs() {
-    return GetFiles(kLogFile).size();
-  }
-
-  int NumTables() {
-    return GetFiles(kTableFile).size();
-  }
-
-  uint64_t FileSize(const std::string& fname) {
-    uint64_t result;
-    ASSERT_OK(env_->GetFileSize(fname, &result)) << fname;
-    return result;
-  }
-
-  void CompactMemTable() {
-    dbfull()->TEST_CompactMemTable();
-  }
-
-  // Directly construct a log file that sets key to val.
-  void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) {
-    std::string fname = LogFileName(dbname_, lognum);
-    WritableFile* file;
-    ASSERT_OK(env_->NewWritableFile(fname, &file));
-    log::Writer writer(file);
-    WriteBatch batch;
-    batch.Put(key, val);
-    WriteBatchInternal::SetSequence(&batch, seq);
-    ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
-    ASSERT_OK(file->Flush());
-    delete file;
-  }
-
- private:
-  std::string dbname_;
-  Env* env_;
-  DB* db_;
-};
-
-TEST(RecoveryTest, ManifestReused) {
-  if (!CanAppend()) {
-    fprintf(stderr, "skipping test because env does not support appending\n");
-    return;
-  }
-  ASSERT_OK(Put("foo", "bar"));
-  Close();
-  std::string old_manifest = ManifestFileName();
-  Open();
-  ASSERT_EQ(old_manifest, ManifestFileName());
-  ASSERT_EQ("bar", Get("foo"));
-  Open();
-  ASSERT_EQ(old_manifest, ManifestFileName());
-  ASSERT_EQ("bar", Get("foo"));
-}
-
-TEST(RecoveryTest, LargeManifestCompacted) {
-  if (!CanAppend()) {
-    fprintf(stderr, "skipping test because env does not support appending\n");
-    return;
-  }
-  ASSERT_OK(Put("foo", "bar"));
-  Close();
-  std::string old_manifest = ManifestFileName();
-
-  // Pad with zeroes to make manifest file very big.
-  {
-    uint64_t len = FileSize(old_manifest);
-    WritableFile* file;
-    ASSERT_OK(env()->NewAppendableFile(old_manifest, &file));
-    std::string zeroes(3*1048576 - static_cast<size_t>(len), 0);
-    ASSERT_OK(file->Append(zeroes));
-    ASSERT_OK(file->Flush());
-    delete file;
-  }
-
-  Open();
-  std::string new_manifest = ManifestFileName();
-  ASSERT_NE(old_manifest, new_manifest);
-  ASSERT_GT(10000, FileSize(new_manifest));
-  ASSERT_EQ("bar", Get("foo"));
-
-  Open();
-  ASSERT_EQ(new_manifest, ManifestFileName());
-  ASSERT_EQ("bar", Get("foo"));
-}
-
-TEST(RecoveryTest, NoLogFiles) {
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_EQ(1, DeleteLogFiles());
-  Open();
-  ASSERT_EQ("NOT_FOUND", Get("foo"));
-  Open();
-  ASSERT_EQ("NOT_FOUND", Get("foo"));
-}
-
-TEST(RecoveryTest, LogFileReuse) {
-  if (!CanAppend()) {
-    fprintf(stderr, "skipping test because env does not support appending\n");
-    return;
-  }
-  for (int i = 0; i < 2; i++) {
-    ASSERT_OK(Put("foo", "bar"));
-    if (i == 0) {
-      // Compact to ensure current log is empty
-      CompactMemTable();
-    }
-    Close();
-    ASSERT_EQ(1, NumLogs());
-    uint64_t number = FirstLogFile();
-    if (i == 0) {
-      ASSERT_EQ(0, FileSize(LogName(number)));
-    } else {
-      ASSERT_LT(0, FileSize(LogName(number)));
-    }
-    Open();
-    ASSERT_EQ(1, NumLogs());
-    ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
-    ASSERT_EQ("bar", Get("foo"));
-    Open();
-    ASSERT_EQ(1, NumLogs());
-    ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
-    ASSERT_EQ("bar", Get("foo"));
-  }
-}
-
-TEST(RecoveryTest, MultipleMemTables) {
-  // Make a large log.
-  const int kNum = 1000;
-  for (int i = 0; i < kNum; i++) {
-    char buf[100];
-    snprintf(buf, sizeof(buf), "%050d", i);
-    ASSERT_OK(Put(buf, buf));
-  }
-  ASSERT_EQ(0, NumTables());
-  Close();
-  ASSERT_EQ(0, NumTables());
-  ASSERT_EQ(1, NumLogs());
-  uint64_t old_log_file = FirstLogFile();
-
-  // Force creation of multiple memtables by reducing the write buffer size.
-  Options opt;
-  opt.reuse_logs = true;
-  opt.write_buffer_size = (kNum*100) / 2;
-  Open(&opt);
-  ASSERT_LE(2, NumTables());
-  ASSERT_EQ(1, NumLogs());
-  ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log";
-  for (int i = 0; i < kNum; i++) {
-    char buf[100];
-    snprintf(buf, sizeof(buf), "%050d", i);
-    ASSERT_EQ(buf, Get(buf));
-  }
-}
-
-TEST(RecoveryTest, MultipleLogFiles) {
-  ASSERT_OK(Put("foo", "bar"));
-  Close();
-  ASSERT_EQ(1, NumLogs());
-
-  // Make a bunch of uncompacted log files.
-  uint64_t old_log = FirstLogFile();
-  MakeLogFile(old_log+1, 1000, "hello", "world");
-  MakeLogFile(old_log+2, 1001, "hi", "there");
-  MakeLogFile(old_log+3, 1002, "foo", "bar2");
-
-  // Recover and check that all log files were processed.
-  Open();
-  ASSERT_LE(1, NumTables());
-  ASSERT_EQ(1, NumLogs());
-  uint64_t new_log = FirstLogFile();
-  ASSERT_LE(old_log+3, new_log);
-  ASSERT_EQ("bar2", Get("foo"));
-  ASSERT_EQ("world", Get("hello"));
-  ASSERT_EQ("there", Get("hi"));
-
-  // Test that previous recovery produced recoverable state.
-  Open();
-  ASSERT_LE(1, NumTables());
-  ASSERT_EQ(1, NumLogs());
-  if (CanAppend()) {
-    ASSERT_EQ(new_log, FirstLogFile());
-  }
-  ASSERT_EQ("bar2", Get("foo"));
-  ASSERT_EQ("world", Get("hello"));
-  ASSERT_EQ("there", Get("hi"));
-
-  // Check that introducing an older log file does not cause it to be re-read.
-  Close();
-  MakeLogFile(old_log+1, 2000, "hello", "stale write");
-  Open();
-  ASSERT_LE(1, NumTables());
-  ASSERT_EQ(1, NumLogs());
-  if (CanAppend()) {
-    ASSERT_EQ(new_log, FirstLogFile());
-  }
-  ASSERT_EQ("bar2", Get("foo"));
-  ASSERT_EQ("world", Get("hello"));
-  ASSERT_EQ("there", Get("hi"));
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/repair.cc

@@ -1,461 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// We recover the contents of the descriptor from the other files we find.
-// (1) Any log files are first converted to tables
-// (2) We scan every table to compute
-//     (a) smallest/largest for the table
-//     (b) largest sequence number in the table
-// (3) We generate descriptor contents:
-//      - log number is set to zero
-//      - next-file-number is set to 1 + largest file number we found
-//      - last-sequence-number is set to largest sequence# found across
-//        all tables (see 2c)
-//      - compaction pointers are cleared
-//      - every table file is added at level 0
-//
-// Possible optimization 1:
-//   (a) Compute total size and use to pick appropriate max-level M
-//   (b) Sort tables by largest sequence# in the table
-//   (c) For each table: if it overlaps earlier table, place in level-0,
-//       else place in level-M.
-// Possible optimization 2:
-//   Store per-table metadata (smallest, largest, largest-seq#, ...)
-//   in the table's meta section to speed up ScanTable.
-
-#include "db/builder.h"
-#include "db/db_impl.h"
-#include "db/dbformat.h"
-#include "db/filename.h"
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "db/memtable.h"
-#include "db/table_cache.h"
-#include "db/version_edit.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-
-namespace leveldb {
-
-namespace {
-
-class Repairer {
- public:
-  Repairer(const std::string& dbname, const Options& options)
-      : dbname_(dbname),
-        env_(options.env),
-        icmp_(options.comparator),
-        ipolicy_(options.filter_policy),
-        options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
-        owns_info_log_(options_.info_log != options.info_log),
-        owns_cache_(options_.block_cache != options.block_cache),
-        next_file_number_(1) {
-    // TableCache can be small since we expect each table to be opened once.
-    table_cache_ = new TableCache(dbname_, &options_, 10);
-  }
-
-  ~Repairer() {
-    delete table_cache_;
-    if (owns_info_log_) {
-      delete options_.info_log;
-    }
-    if (owns_cache_) {
-      delete options_.block_cache;
-    }
-  }
-
-  Status Run() {
-    Status status = FindFiles();
-    if (status.ok()) {
-      ConvertLogFilesToTables();
-      ExtractMetaData();
-      status = WriteDescriptor();
-    }
-    if (status.ok()) {
-      unsigned long long bytes = 0;
-      for (size_t i = 0; i < tables_.size(); i++) {
-        bytes += tables_[i].meta.file_size;
-      }
-      Log(options_.info_log,
-          "**** Repaired leveldb %s; "
-          "recovered %d files; %llu bytes. "
-          "Some data may have been lost. "
-          "****",
-          dbname_.c_str(),
-          static_cast<int>(tables_.size()),
-          bytes);
-    }
-    return status;
-  }
-
- private:
-  struct TableInfo {
-    FileMetaData meta;
-    SequenceNumber max_sequence;
-  };
-
-  std::string const dbname_;
-  Env* const env_;
-  InternalKeyComparator const icmp_;
-  InternalFilterPolicy const ipolicy_;
-  Options const options_;
-  bool owns_info_log_;
-  bool owns_cache_;
-  TableCache* table_cache_;
-  VersionEdit edit_;
-
-  std::vector<std::string> manifests_;
-  std::vector<uint64_t> table_numbers_;
-  std::vector<uint64_t> logs_;
-  std::vector<TableInfo> tables_;
-  uint64_t next_file_number_;
-
-  Status FindFiles() {
-    std::vector<std::string> filenames;
-    Status status = env_->GetChildren(dbname_, &filenames);
-    if (!status.ok()) {
-      return status;
-    }
-    if (filenames.empty()) {
-      return Status::IOError(dbname_, "repair found no files");
-    }
-
-    uint64_t number;
-    FileType type;
-    for (size_t i = 0; i < filenames.size(); i++) {
-      if (ParseFileName(filenames[i], &number, &type)) {
-        if (type == kDescriptorFile) {
-          manifests_.push_back(filenames[i]);
-        } else {
-          if (number + 1 > next_file_number_) {
-            next_file_number_ = number + 1;
-          }
-          if (type == kLogFile) {
-            logs_.push_back(number);
-          } else if (type == kTableFile) {
-            table_numbers_.push_back(number);
-          } else {
-            // Ignore other files
-          }
-        }
-      }
-    }
-    return status;
-  }
-
-  void ConvertLogFilesToTables() {
-    for (size_t i = 0; i < logs_.size(); i++) {
-      std::string logname = LogFileName(dbname_, logs_[i]);
-      Status status = ConvertLogToTable(logs_[i]);
-      if (!status.ok()) {
-        Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
-            (unsigned long long) logs_[i],
-            status.ToString().c_str());
-      }
-      ArchiveFile(logname);
-    }
-  }
-
-  Status ConvertLogToTable(uint64_t log) {
-    struct LogReporter : public log::Reader::Reporter {
-      Env* env;
-      Logger* info_log;
-      uint64_t lognum;
-      virtual void Corruption(size_t bytes, const Status& s) {
-        // We print error messages for corruption, but continue repairing.
-        Log(info_log, "Log #%llu: dropping %d bytes; %s",
-            (unsigned long long) lognum,
-            static_cast<int>(bytes),
-            s.ToString().c_str());
-      }
-    };
-
-    // Open the log file
-    std::string logname = LogFileName(dbname_, log);
-    SequentialFile* lfile;
-    Status status = env_->NewSequentialFile(logname, &lfile);
-    if (!status.ok()) {
-      return status;
-    }
-
-    // Create the log reader.
-    LogReporter reporter;
-    reporter.env = env_;
-    reporter.info_log = options_.info_log;
-    reporter.lognum = log;
-    // We intentionally make log::Reader do checksumming so that
-    // corruptions cause entire commits to be skipped instead of
-    // propagating bad information (like overly large sequence
-    // numbers).
-    log::Reader reader(lfile, &reporter, false/*do not checksum*/,
-                       0/*initial_offset*/);
-
-    // Read all the records and add to a memtable
-    std::string scratch;
-    Slice record;
-    WriteBatch batch;
-    MemTable* mem = new MemTable(icmp_);
-    mem->Ref();
-    int counter = 0;
-    while (reader.ReadRecord(&record, &scratch)) {
-      if (record.size() < 12) {
-        reporter.Corruption(
-            record.size(), Status::Corruption("log record too small", logname));
-        continue;
-      }
-      WriteBatchInternal::SetContents(&batch, record);
-      status = WriteBatchInternal::InsertInto(&batch, mem);
-      if (status.ok()) {
-        counter += WriteBatchInternal::Count(&batch);
-      } else {
-        Log(options_.info_log, "Log #%llu: ignoring %s",
-            (unsigned long long) log,
-            status.ToString().c_str());
-        status = Status::OK();  // Keep going with rest of file
-      }
-    }
-    delete lfile;
-
-    // Do not record a version edit for this conversion to a Table
-    // since ExtractMetaData() will also generate edits.
-    FileMetaData meta;
-    meta.number = next_file_number_++;
-    Iterator* iter = mem->NewIterator();
-    status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
-    delete iter;
-    mem->Unref();
-    mem = NULL;
-    if (status.ok()) {
-      if (meta.file_size > 0) {
-        table_numbers_.push_back(meta.number);
-      }
-    }
-    Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
-        (unsigned long long) log,
-        counter,
-        (unsigned long long) meta.number,
-        status.ToString().c_str());
-    return status;
-  }
-
-  void ExtractMetaData() {
-    for (size_t i = 0; i < table_numbers_.size(); i++) {
-      ScanTable(table_numbers_[i]);
-    }
-  }
-
-  Iterator* NewTableIterator(const FileMetaData& meta) {
-    // Same as compaction iterators: if paranoid_checks are on, turn
-    // on checksum verification.
-    ReadOptions r;
-    r.verify_checksums = options_.paranoid_checks;
-    return table_cache_->NewIterator(r, meta.number, meta.file_size);
-  }
-
-  void ScanTable(uint64_t number) {
-    TableInfo t;
-    t.meta.number = number;
-    std::string fname = TableFileName(dbname_, number);
-    Status status = env_->GetFileSize(fname, &t.meta.file_size);
-    if (!status.ok()) {
-      // Try alternate file name.
-      fname = SSTTableFileName(dbname_, number);
-      Status s2 = env_->GetFileSize(fname, &t.meta.file_size);
-      if (s2.ok()) {
-        status = Status::OK();
-      }
-    }
-    if (!status.ok()) {
-      ArchiveFile(TableFileName(dbname_, number));
-      ArchiveFile(SSTTableFileName(dbname_, number));
-      Log(options_.info_log, "Table #%llu: dropped: %s",
-          (unsigned long long) t.meta.number,
-          status.ToString().c_str());
-      return;
-    }
-
-    // Extract metadata by scanning through table.
-    int counter = 0;
-    Iterator* iter = NewTableIterator(t.meta);
-    bool empty = true;
-    ParsedInternalKey parsed;
-    t.max_sequence = 0;
-    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-      Slice key = iter->key();
-      if (!ParseInternalKey(key, &parsed)) {
-        Log(options_.info_log, "Table #%llu: unparsable key %s",
-            (unsigned long long) t.meta.number,
-            EscapeString(key).c_str());
-        continue;
-      }
-
-      counter++;
-      if (empty) {
-        empty = false;
-        t.meta.smallest.DecodeFrom(key);
-      }
-      t.meta.largest.DecodeFrom(key);
-      if (parsed.sequence > t.max_sequence) {
-        t.max_sequence = parsed.sequence;
-      }
-    }
-    if (!iter->status().ok()) {
-      status = iter->status();
-    }
-    delete iter;
-    Log(options_.info_log, "Table #%llu: %d entries %s",
-        (unsigned long long) t.meta.number,
-        counter,
-        status.ToString().c_str());
-
-    if (status.ok()) {
-      tables_.push_back(t);
-    } else {
-      RepairTable(fname, t);  // RepairTable archives input file.
-    }
-  }
-
-  void RepairTable(const std::string& src, TableInfo t) {
-    // We will copy src contents to a new table and then rename the
-    // new table over the source.
-
-    // Create builder.
-    std::string copy = TableFileName(dbname_, next_file_number_++);
-    WritableFile* file;
-    Status s = env_->NewWritableFile(copy, &file);
-    if (!s.ok()) {
-      return;
-    }
-    TableBuilder* builder = new TableBuilder(options_, file);
-
-    // Copy data.
-    Iterator* iter = NewTableIterator(t.meta);
-    int counter = 0;
-    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-      builder->Add(iter->key(), iter->value());
-      counter++;
-    }
-    delete iter;
-
-    ArchiveFile(src);
-    if (counter == 0) {
-      builder->Abandon();  // Nothing to save
-    } else {
-      s = builder->Finish();
-      if (s.ok()) {
-        t.meta.file_size = builder->FileSize();
-      }
-    }
-    delete builder;
-    builder = NULL;
-
-    if (s.ok()) {
-      s = file->Close();
-    }
-    delete file;
-    file = NULL;
-
-    if (counter > 0 && s.ok()) {
-      std::string orig = TableFileName(dbname_, t.meta.number);
-      s = env_->RenameFile(copy, orig);
-      if (s.ok()) {
-        Log(options_.info_log, "Table #%llu: %d entries repaired",
-            (unsigned long long) t.meta.number, counter);
-        tables_.push_back(t);
-      }
-    }
-    if (!s.ok()) {
-      env_->DeleteFile(copy);
-    }
-  }
-
-  Status WriteDescriptor() {
-    std::string tmp = TempFileName(dbname_, 1);
-    WritableFile* file;
-    Status status = env_->NewWritableFile(tmp, &file);
-    if (!status.ok()) {
-      return status;
-    }
-
-    SequenceNumber max_sequence = 0;
-    for (size_t i = 0; i < tables_.size(); i++) {
-      if (max_sequence < tables_[i].max_sequence) {
-        max_sequence = tables_[i].max_sequence;
-      }
-    }
-
-    edit_.SetComparatorName(icmp_.user_comparator()->Name());
-    edit_.SetLogNumber(0);
-    edit_.SetNextFile(next_file_number_);
-    edit_.SetLastSequence(max_sequence);
-
-    for (size_t i = 0; i < tables_.size(); i++) {
-      // TODO(opt): separate out into multiple levels
-      const TableInfo& t = tables_[i];
-      edit_.AddFile(0, t.meta.number, t.meta.file_size,
-                    t.meta.smallest, t.meta.largest);
-    }
-
-    //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
-    {
-      log::Writer log(file);
-      std::string record;
-      edit_.EncodeTo(&record);
-      status = log.AddRecord(record);
-    }
-    if (status.ok()) {
-      status = file->Close();
-    }
-    delete file;
-    file = NULL;
-
-    if (!status.ok()) {
-      env_->DeleteFile(tmp);
-    } else {
-      // Discard older manifests
-      for (size_t i = 0; i < manifests_.size(); i++) {
-        ArchiveFile(dbname_ + "/" + manifests_[i]);
-      }
-
-      // Install new manifest
-      status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1));
-      if (status.ok()) {
-        status = SetCurrentFile(env_, dbname_, 1);
-      } else {
-        env_->DeleteFile(tmp);
-      }
-    }
-    return status;
-  }
-
-  void ArchiveFile(const std::string& fname) {
-    // Move into another directory.  E.g., for
-    //    dir/foo
-    // rename to
-    //    dir/lost/foo
-    const char* slash = strrchr(fname.c_str(), '/');
-    std::string new_dir;
-    if (slash != NULL) {
-      new_dir.assign(fname.data(), slash - fname.data());
-    }
-    new_dir.append("/lost");
-    env_->CreateDir(new_dir);  // Ignore error
-    std::string new_file = new_dir;
-    new_file.append("/");
-    new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
-    Status s = env_->RenameFile(fname, new_file);
-    Log(options_.info_log, "Archiving %s: %s\n",
-        fname.c_str(), s.ToString().c_str());
-  }
-};
-}  // namespace
-
-Status RepairDB(const std::string& dbname, const Options& options) {
-  Repairer repairer(dbname, options);
-  return repairer.Run();
-}
-
-}  // namespace leveldb

src/leveldb/db/skiplist.h

@@ -1,384 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_
-#define STORAGE_LEVELDB_DB_SKIPLIST_H_
-
-// Thread safety
-// -------------
-//
-// Writes require external synchronization, most likely a mutex.
-// Reads require a guarantee that the SkipList will not be destroyed
-// while the read is in progress.  Apart from that, reads progress
-// without any internal locking or synchronization.
-//
-// Invariants:
-//
-// (1) Allocated nodes are never deleted until the SkipList is
-// destroyed.  This is trivially guaranteed by the code since we
-// never delete any skip list nodes.
-//
-// (2) The contents of a Node except for the next/prev pointers are
-// immutable after the Node has been linked into the SkipList.
-// Only Insert() modifies the list, and it is careful to initialize
-// a node and use release-stores to publish the nodes in one or
-// more lists.
-//
-// ... prev vs. next pointer ordering ...
-
-#include <assert.h>
-#include <stdlib.h>
-#include "port/port.h"
-#include "util/arena.h"
-#include "util/random.h"
-
-namespace leveldb {
-
-class Arena;
-
-template<typename Key, class Comparator>
-class SkipList {
- private:
-  struct Node;
-
- public:
-  // Create a new SkipList object that will use "cmp" for comparing keys,
-  // and will allocate memory using "*arena".  Objects allocated in the arena
-  // must remain allocated for the lifetime of the skiplist object.
-  explicit SkipList(Comparator cmp, Arena* arena);
-
-  // Insert key into the list.
-  // REQUIRES: nothing that compares equal to key is currently in the list.
-  void Insert(const Key& key);
-
-  // Returns true iff an entry that compares equal to key is in the list.
-  bool Contains(const Key& key) const;
-
-  // Iteration over the contents of a skip list
-  class Iterator {
-   public:
-    // Initialize an iterator over the specified list.
-    // The returned iterator is not valid.
-    explicit Iterator(const SkipList* list);
-
-    // Returns true iff the iterator is positioned at a valid node.
-    bool Valid() const;
-
-    // Returns the key at the current position.
-    // REQUIRES: Valid()
-    const Key& key() const;
-
-    // Advances to the next position.
-    // REQUIRES: Valid()
-    void Next();
-
-    // Advances to the previous position.
-    // REQUIRES: Valid()
-    void Prev();
-
-    // Advance to the first entry with a key >= target
-    void Seek(const Key& target);
-
-    // Position at the first entry in list.
-    // Final state of iterator is Valid() iff list is not empty.
-    void SeekToFirst();
-
-    // Position at the last entry in list.
-    // Final state of iterator is Valid() iff list is not empty.
-    void SeekToLast();
-
-   private:
-    const SkipList* list_;
-    Node* node_;
-    // Intentionally copyable
-  };
-
- private:
-  enum { kMaxHeight = 12 };
-
-  // Immutable after construction
-  Comparator const compare_;
-  Arena* const arena_;    // Arena used for allocations of nodes
-
-  Node* const head_;
-
-  // Modified only by Insert().  Read racily by readers, but stale
-  // values are ok.
-  port::AtomicPointer max_height_;   // Height of the entire list
-
-  inline int GetMaxHeight() const {
-    return static_cast<int>(
-        reinterpret_cast<intptr_t>(max_height_.NoBarrier_Load()));
-  }
-
-  // Read/written only by Insert().
-  Random rnd_;
-
-  Node* NewNode(const Key& key, int height);
-  int RandomHeight();
-  bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); }
-
-  // Return true if key is greater than the data stored in "n"
-  bool KeyIsAfterNode(const Key& key, Node* n) const;
-
-  // Return the earliest node that comes at or after key.
-  // Return NULL if there is no such node.
-  //
-  // If prev is non-NULL, fills prev[level] with pointer to previous
-  // node at "level" for every level in [0..max_height_-1].
-  Node* FindGreaterOrEqual(const Key& key, Node** prev) const;
-
-  // Return the latest node with a key < key.
-  // Return head_ if there is no such node.
-  Node* FindLessThan(const Key& key) const;
-
-  // Return the last node in the list.
-  // Return head_ if list is empty.
-  Node* FindLast() const;
-
-  // No copying allowed
-  SkipList(const SkipList&);
-  void operator=(const SkipList&);
-};
-
-// Implementation details follow
-template<typename Key, class Comparator>
-struct SkipList<Key,Comparator>::Node {
-  explicit Node(const Key& k) : key(k) { }
-
-  Key const key;
-
-  // Accessors/mutators for links.  Wrapped in methods so we can
-  // add the appropriate barriers as necessary.
-  Node* Next(int n) {
-    assert(n >= 0);
-    // Use an 'acquire load' so that we observe a fully initialized
-    // version of the returned Node.
-    return reinterpret_cast<Node*>(next_[n].Acquire_Load());
-  }
-  void SetNext(int n, Node* x) {
-    assert(n >= 0);
-    // Use a 'release store' so that anybody who reads through this
-    // pointer observes a fully initialized version of the inserted node.
-    next_[n].Release_Store(x);
-  }
-
-  // No-barrier variants that can be safely used in a few locations.
-  Node* NoBarrier_Next(int n) {
-    assert(n >= 0);
-    return reinterpret_cast<Node*>(next_[n].NoBarrier_Load());
-  }
-  void NoBarrier_SetNext(int n, Node* x) {
-    assert(n >= 0);
-    next_[n].NoBarrier_Store(x);
-  }
-
- private:
-  // Array of length equal to the node height.  next_[0] is lowest level link.
-  port::AtomicPointer next_[1];
-};
-
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node*
-SkipList<Key,Comparator>::NewNode(const Key& key, int height) {
-  char* mem = arena_->AllocateAligned(
-      sizeof(Node) + sizeof(port::AtomicPointer) * (height - 1));
-  return new (mem) Node(key);
-}
-
-template<typename Key, class Comparator>
-inline SkipList<Key,Comparator>::Iterator::Iterator(const SkipList* list) {
-  list_ = list;
-  node_ = NULL;
-}
-
-template<typename Key, class Comparator>
-inline bool SkipList<Key,Comparator>::Iterator::Valid() const {
-  return node_ != NULL;
-}
-
-template<typename Key, class Comparator>
-inline const Key& SkipList<Key,Comparator>::Iterator::key() const {
-  assert(Valid());
-  return node_->key;
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::Next() {
-  assert(Valid());
-  node_ = node_->Next(0);
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::Prev() {
-  // Instead of using explicit "prev" links, we just search for the
-  // last node that falls before key.
-  assert(Valid());
-  node_ = list_->FindLessThan(node_->key);
-  if (node_ == list_->head_) {
-    node_ = NULL;
-  }
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::Seek(const Key& target) {
-  node_ = list_->FindGreaterOrEqual(target, NULL);
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::SeekToFirst() {
-  node_ = list_->head_->Next(0);
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::SeekToLast() {
-  node_ = list_->FindLast();
-  if (node_ == list_->head_) {
-    node_ = NULL;
-  }
-}
-
-template<typename Key, class Comparator>
-int SkipList<Key,Comparator>::RandomHeight() {
-  // Increase height with probability 1 in kBranching
-  static const unsigned int kBranching = 4;
-  int height = 1;
-  while (height < kMaxHeight && ((rnd_.Next() % kBranching) == 0)) {
-    height++;
-  }
-  assert(height > 0);
-  assert(height <= kMaxHeight);
-  return height;
-}
-
-template<typename Key, class Comparator>
-bool SkipList<Key,Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
-  // NULL n is considered infinite
-  return (n != NULL) && (compare_(n->key, key) < 0);
-}
-
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOrEqual(const Key& key, Node** prev)
-    const {
-  Node* x = head_;
-  int level = GetMaxHeight() - 1;
-  while (true) {
-    Node* next = x->Next(level);
-    if (KeyIsAfterNode(key, next)) {
-      // Keep searching in this list
-      x = next;
-    } else {
-      if (prev != NULL) prev[level] = x;
-      if (level == 0) {
-        return next;
-      } else {
-        // Switch to next list
-        level--;
-      }
-    }
-  }
-}
-
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node*
-SkipList<Key,Comparator>::FindLessThan(const Key& key) const {
-  Node* x = head_;
-  int level = GetMaxHeight() - 1;
-  while (true) {
-    assert(x == head_ || compare_(x->key, key) < 0);
-    Node* next = x->Next(level);
-    if (next == NULL || compare_(next->key, key) >= 0) {
-      if (level == 0) {
-        return x;
-      } else {
-        // Switch to next list
-        level--;
-      }
-    } else {
-      x = next;
-    }
-  }
-}
-
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast()
-    const {
-  Node* x = head_;
-  int level = GetMaxHeight() - 1;
-  while (true) {
-    Node* next = x->Next(level);
-    if (next == NULL) {
-      if (level == 0) {
-        return x;
-      } else {
-        // Switch to next list
-        level--;
-      }
-    } else {
-      x = next;
-    }
-  }
-}
-
-template<typename Key, class Comparator>
-SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena)
-    : compare_(cmp),
-      arena_(arena),
-      head_(NewNode(0 /* any key will do */, kMaxHeight)),
-      max_height_(reinterpret_cast<void*>(1)),
-      rnd_(0xdeadbeef) {
-  for (int i = 0; i < kMaxHeight; i++) {
-    head_->SetNext(i, NULL);
-  }
-}
-
-template<typename Key, class Comparator>
-void SkipList<Key,Comparator>::Insert(const Key& key) {
-  // TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual()
-  // here since Insert() is externally synchronized.
-  Node* prev[kMaxHeight];
-  Node* x = FindGreaterOrEqual(key, prev);
-
-  // Our data structure does not allow duplicate insertion
-  assert(x == NULL || !Equal(key, x->key));
-
-  int height = RandomHeight();
-  if (height > GetMaxHeight()) {
-    for (int i = GetMaxHeight(); i < height; i++) {
-      prev[i] = head_;
-    }
-    //fprintf(stderr, "Change height from %d to %d\n", max_height_, height);
-
-    // It is ok to mutate max_height_ without any synchronization
-    // with concurrent readers.  A concurrent reader that observes
-    // the new value of max_height_ will see either the old value of
-    // new level pointers from head_ (NULL), or a new value set in
-    // the loop below.  In the former case the reader will
-    // immediately drop to the next level since NULL sorts after all
-    // keys.  In the latter case the reader will use the new node.
-    max_height_.NoBarrier_Store(reinterpret_cast<void*>(height));
-  }
-
-  x = NewNode(key, height);
-  for (int i = 0; i < height; i++) {
-    // NoBarrier_SetNext() suffices since we will add a barrier when
-    // we publish a pointer to "x" in prev[i].
-    x->NoBarrier_SetNext(i, prev[i]->NoBarrier_Next(i));
-    prev[i]->SetNext(i, x);
-  }
-}
-
-template<typename Key, class Comparator>
-bool SkipList<Key,Comparator>::Contains(const Key& key) const {
-  Node* x = FindGreaterOrEqual(key, NULL);
-  if (x != NULL && Equal(key, x->key)) {
-    return true;
-  } else {
-    return false;
-  }
-}
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_SKIPLIST_H_

src/leveldb/db/skiplist_test.cc

@@ -1,378 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/skiplist.h"
-#include <set>
-#include "leveldb/env.h"
-#include "util/arena.h"
-#include "util/hash.h"
-#include "util/random.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-typedef uint64_t Key;
-
-struct Comparator {
-  int operator()(const Key& a, const Key& b) const {
-    if (a < b) {
-      return -1;
-    } else if (a > b) {
-      return +1;
-    } else {
-      return 0;
-    }
-  }
-};
-
-class SkipTest { };
-
-TEST(SkipTest, Empty) {
-  Arena arena;
-  Comparator cmp;
-  SkipList<Key, Comparator> list(cmp, &arena);
-  ASSERT_TRUE(!list.Contains(10));
-
-  SkipList<Key, Comparator>::Iterator iter(&list);
-  ASSERT_TRUE(!iter.Valid());
-  iter.SeekToFirst();
-  ASSERT_TRUE(!iter.Valid());
-  iter.Seek(100);
-  ASSERT_TRUE(!iter.Valid());
-  iter.SeekToLast();
-  ASSERT_TRUE(!iter.Valid());
-}
-
-TEST(SkipTest, InsertAndLookup) {
-  const int N = 2000;
-  const int R = 5000;
-  Random rnd(1000);
-  std::set<Key> keys;
-  Arena arena;
-  Comparator cmp;
-  SkipList<Key, Comparator> list(cmp, &arena);
-  for (int i = 0; i < N; i++) {
-    Key key = rnd.Next() % R;
-    if (keys.insert(key).second) {
-      list.Insert(key);
-    }
-  }
-
-  for (int i = 0; i < R; i++) {
-    if (list.Contains(i)) {
-      ASSERT_EQ(keys.count(i), 1);
-    } else {
-      ASSERT_EQ(keys.count(i), 0);
-    }
-  }
-
-  // Simple iterator tests
-  {
-    SkipList<Key, Comparator>::Iterator iter(&list);
-    ASSERT_TRUE(!iter.Valid());
-
-    iter.Seek(0);
-    ASSERT_TRUE(iter.Valid());
-    ASSERT_EQ(*(keys.begin()), iter.key());
-
-    iter.SeekToFirst();
-    ASSERT_TRUE(iter.Valid());
-    ASSERT_EQ(*(keys.begin()), iter.key());
-
-    iter.SeekToLast();
-    ASSERT_TRUE(iter.Valid());
-    ASSERT_EQ(*(keys.rbegin()), iter.key());
-  }
-
-  // Forward iteration test
-  for (int i = 0; i < R; i++) {
-    SkipList<Key, Comparator>::Iterator iter(&list);
-    iter.Seek(i);
-
-    // Compare against model iterator
-    std::set<Key>::iterator model_iter = keys.lower_bound(i);
-    for (int j = 0; j < 3; j++) {
-      if (model_iter == keys.end()) {
-        ASSERT_TRUE(!iter.Valid());
-        break;
-      } else {
-        ASSERT_TRUE(iter.Valid());
-        ASSERT_EQ(*model_iter, iter.key());
-        ++model_iter;
-        iter.Next();
-      }
-    }
-  }
-
-  // Backward iteration test
-  {
-    SkipList<Key, Comparator>::Iterator iter(&list);
-    iter.SeekToLast();
-
-    // Compare against model iterator
-    for (std::set<Key>::reverse_iterator model_iter = keys.rbegin();
-         model_iter != keys.rend();
-         ++model_iter) {
-      ASSERT_TRUE(iter.Valid());
-      ASSERT_EQ(*model_iter, iter.key());
-      iter.Prev();
-    }
-    ASSERT_TRUE(!iter.Valid());
-  }
-}
-
-// We want to make sure that with a single writer and multiple
-// concurrent readers (with no synchronization other than when a
-// reader's iterator is created), the reader always observes all the
-// data that was present in the skip list when the iterator was
-// constructor.  Because insertions are happening concurrently, we may
-// also observe new values that were inserted since the iterator was
-// constructed, but we should never miss any values that were present
-// at iterator construction time.
-//
-// We generate multi-part keys:
-//     <key,gen,hash>
-// where:
-//     key is in range [0..K-1]
-//     gen is a generation number for key
-//     hash is hash(key,gen)
-//
-// The insertion code picks a random key, sets gen to be 1 + the last
-// generation number inserted for that key, and sets hash to Hash(key,gen).
-//
-// At the beginning of a read, we snapshot the last inserted
-// generation number for each key.  We then iterate, including random
-// calls to Next() and Seek().  For every key we encounter, we
-// check that it is either expected given the initial snapshot or has
-// been concurrently added since the iterator started.
-class ConcurrentTest {
- private:
-  static const uint32_t K = 4;
-
-  static uint64_t key(Key key) { return (key >> 40); }
-  static uint64_t gen(Key key) { return (key >> 8) & 0xffffffffu; }
-  static uint64_t hash(Key key) { return key & 0xff; }
-
-  static uint64_t HashNumbers(uint64_t k, uint64_t g) {
-    uint64_t data[2] = { k, g };
-    return Hash(reinterpret_cast<char*>(data), sizeof(data), 0);
-  }
-
-  static Key MakeKey(uint64_t k, uint64_t g) {
-    assert(sizeof(Key) == sizeof(uint64_t));
-    assert(k <= K);  // We sometimes pass K to seek to the end of the skiplist
-    assert(g <= 0xffffffffu);
-    return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff));
-  }
-
-  static bool IsValidKey(Key k) {
-    return hash(k) == (HashNumbers(key(k), gen(k)) & 0xff);
-  }
-
-  static Key RandomTarget(Random* rnd) {
-    switch (rnd->Next() % 10) {
-      case 0:
-        // Seek to beginning
-        return MakeKey(0, 0);
-      case 1:
-        // Seek to end
-        return MakeKey(K, 0);
-      default:
-        // Seek to middle
-        return MakeKey(rnd->Next() % K, 0);
-    }
-  }
-
-  // Per-key generation
-  struct State {
-    port::AtomicPointer generation[K];
-    void Set(int k, intptr_t v) {
-      generation[k].Release_Store(reinterpret_cast<void*>(v));
-    }
-    intptr_t Get(int k) {
-      return reinterpret_cast<intptr_t>(generation[k].Acquire_Load());
-    }
-
-    State() {
-      for (int k = 0; k < K; k++) {
-        Set(k, 0);
-      }
-    }
-  };
-
-  // Current state of the test
-  State current_;
-
-  Arena arena_;
-
-  // SkipList is not protected by mu_.  We just use a single writer
-  // thread to modify it.
-  SkipList<Key, Comparator> list_;
-
- public:
-  ConcurrentTest() : list_(Comparator(), &arena_) { }
-
-  // REQUIRES: External synchronization
-  void WriteStep(Random* rnd) {
-    const uint32_t k = rnd->Next() % K;
-    const intptr_t g = current_.Get(k) + 1;
-    const Key key = MakeKey(k, g);
-    list_.Insert(key);
-    current_.Set(k, g);
-  }
-
-  void ReadStep(Random* rnd) {
-    // Remember the initial committed state of the skiplist.
-    State initial_state;
-    for (int k = 0; k < K; k++) {
-      initial_state.Set(k, current_.Get(k));
-    }
-
-    Key pos = RandomTarget(rnd);
-    SkipList<Key, Comparator>::Iterator iter(&list_);
-    iter.Seek(pos);
-    while (true) {
-      Key current;
-      if (!iter.Valid()) {
-        current = MakeKey(K, 0);
-      } else {
-        current = iter.key();
-        ASSERT_TRUE(IsValidKey(current)) << current;
-      }
-      ASSERT_LE(pos, current) << "should not go backwards";
-
-      // Verify that everything in [pos,current) was not present in
-      // initial_state.
-      while (pos < current) {
-        ASSERT_LT(key(pos), K) << pos;
-
-        // Note that generation 0 is never inserted, so it is ok if
-        // <*,0,*> is missing.
-        ASSERT_TRUE((gen(pos) == 0) ||
-                    (gen(pos) > static_cast<Key>(initial_state.Get(key(pos))))
-                    ) << "key: " << key(pos)
-                      << "; gen: " << gen(pos)
-                      << "; initgen: "
-                      << initial_state.Get(key(pos));
-
-        // Advance to next key in the valid key space
-        if (key(pos) < key(current)) {
-          pos = MakeKey(key(pos) + 1, 0);
-        } else {
-          pos = MakeKey(key(pos), gen(pos) + 1);
-        }
-      }
-
-      if (!iter.Valid()) {
-        break;
-      }
-
-      if (rnd->Next() % 2) {
-        iter.Next();
-        pos = MakeKey(key(pos), gen(pos) + 1);
-      } else {
-        Key new_target = RandomTarget(rnd);
-        if (new_target > pos) {
-          pos = new_target;
-          iter.Seek(new_target);
-        }
-      }
-    }
-  }
-};
-const uint32_t ConcurrentTest::K;
-
-// Simple test that does single-threaded testing of the ConcurrentTest
-// scaffolding.
-TEST(SkipTest, ConcurrentWithoutThreads) {
-  ConcurrentTest test;
-  Random rnd(test::RandomSeed());
-  for (int i = 0; i < 10000; i++) {
-    test.ReadStep(&rnd);
-    test.WriteStep(&rnd);
-  }
-}
-
-class TestState {
- public:
-  ConcurrentTest t_;
-  int seed_;
-  port::AtomicPointer quit_flag_;
-
-  enum ReaderState {
-    STARTING,
-    RUNNING,
-    DONE
-  };
-
-  explicit TestState(int s)
-      : seed_(s),
-        quit_flag_(NULL),
-        state_(STARTING),
-        state_cv_(&mu_) {}
-
-  void Wait(ReaderState s) {
-    mu_.Lock();
-    while (state_ != s) {
-      state_cv_.Wait();
-    }
-    mu_.Unlock();
-  }
-
-  void Change(ReaderState s) {
-    mu_.Lock();
-    state_ = s;
-    state_cv_.Signal();
-    mu_.Unlock();
-  }
-
- private:
-  port::Mutex mu_;
-  ReaderState state_;
-  port::CondVar state_cv_;
-};
-
-static void ConcurrentReader(void* arg) {
-  TestState* state = reinterpret_cast<TestState*>(arg);
-  Random rnd(state->seed_);
-  int64_t reads = 0;
-  state->Change(TestState::RUNNING);
-  while (!state->quit_flag_.Acquire_Load()) {
-    state->t_.ReadStep(&rnd);
-    ++reads;
-  }
-  state->Change(TestState::DONE);
-}
-
-static void RunConcurrent(int run) {
-  const int seed = test::RandomSeed() + (run * 100);
-  Random rnd(seed);
-  const int N = 1000;
-  const int kSize = 1000;
-  for (int i = 0; i < N; i++) {
-    if ((i % 100) == 0) {
-      fprintf(stderr, "Run %d of %d\n", i, N);
-    }
-    TestState state(seed + 1);
-    Env::Default()->Schedule(ConcurrentReader, &state);
-    state.Wait(TestState::RUNNING);
-    for (int i = 0; i < kSize; i++) {
-      state.t_.WriteStep(&rnd);
-    }
-    state.quit_flag_.Release_Store(&state);  // Any non-NULL arg will do
-    state.Wait(TestState::DONE);
-  }
-}
-
-TEST(SkipTest, Concurrent1) { RunConcurrent(1); }
-TEST(SkipTest, Concurrent2) { RunConcurrent(2); }
-TEST(SkipTest, Concurrent3) { RunConcurrent(3); }
-TEST(SkipTest, Concurrent4) { RunConcurrent(4); }
-TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/snapshot.h

@@ -1,67 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_
-#define STORAGE_LEVELDB_DB_SNAPSHOT_H_
-
-#include "db/dbformat.h"
-#include "leveldb/db.h"
-
-namespace leveldb {
-
-class SnapshotList;
-
-// Snapshots are kept in a doubly-linked list in the DB.
-// Each SnapshotImpl corresponds to a particular sequence number.
-class SnapshotImpl : public Snapshot {
- public:
-  SequenceNumber number_;  // const after creation
-
- private:
-  friend class SnapshotList;
-
-  // SnapshotImpl is kept in a doubly-linked circular list
-  SnapshotImpl* prev_;
-  SnapshotImpl* next_;
-
-  SnapshotList* list_;                 // just for sanity checks
-};
-
-class SnapshotList {
- public:
-  SnapshotList() {
-    list_.prev_ = &list_;
-    list_.next_ = &list_;
-  }
-
-  bool empty() const { return list_.next_ == &list_; }
-  SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; }
-  SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; }
-
-  const SnapshotImpl* New(SequenceNumber seq) {
-    SnapshotImpl* s = new SnapshotImpl;
-    s->number_ = seq;
-    s->list_ = this;
-    s->next_ = &list_;
-    s->prev_ = list_.prev_;
-    s->prev_->next_ = s;
-    s->next_->prev_ = s;
-    return s;
-  }
-
-  void Delete(const SnapshotImpl* s) {
-    assert(s->list_ == this);
-    s->prev_->next_ = s->next_;
-    s->next_->prev_ = s->prev_;
-    delete s;
-  }
-
- private:
-  // Dummy head of doubly-linked list of snapshots
-  SnapshotImpl list_;
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_SNAPSHOT_H_

src/leveldb/db/table_cache.cc

@@ -1,127 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/table_cache.h"
-
-#include "db/filename.h"
-#include "leveldb/env.h"
-#include "leveldb/table.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-struct TableAndFile {
-  RandomAccessFile* file;
-  Table* table;
-};
-
-static void DeleteEntry(const Slice& key, void* value) {
-  TableAndFile* tf = reinterpret_cast<TableAndFile*>(value);
-  delete tf->table;
-  delete tf->file;
-  delete tf;
-}
-
-static void UnrefEntry(void* arg1, void* arg2) {
-  Cache* cache = reinterpret_cast<Cache*>(arg1);
-  Cache::Handle* h = reinterpret_cast<Cache::Handle*>(arg2);
-  cache->Release(h);
-}
-
-TableCache::TableCache(const std::string& dbname,
-                       const Options* options,
-                       int entries)
-    : env_(options->env),
-      dbname_(dbname),
-      options_(options),
-      cache_(NewLRUCache(entries)) {
-}
-
-TableCache::~TableCache() {
-  delete cache_;
-}
-
-Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
-                             Cache::Handle** handle) {
-  Status s;
-  char buf[sizeof(file_number)];
-  EncodeFixed64(buf, file_number);
-  Slice key(buf, sizeof(buf));
-  *handle = cache_->Lookup(key);
-  if (*handle == NULL) {
-    std::string fname = TableFileName(dbname_, file_number);
-    RandomAccessFile* file = NULL;
-    Table* table = NULL;
-    s = env_->NewRandomAccessFile(fname, &file);
-    if (!s.ok()) {
-      std::string old_fname = SSTTableFileName(dbname_, file_number);
-      if (env_->NewRandomAccessFile(old_fname, &file).ok()) {
-        s = Status::OK();
-      }
-    }
-    if (s.ok()) {
-      s = Table::Open(*options_, file, file_size, &table);
-    }
-
-    if (!s.ok()) {
-      assert(table == NULL);
-      delete file;
-      // We do not cache error results so that if the error is transient,
-      // or somebody repairs the file, we recover automatically.
-    } else {
-      TableAndFile* tf = new TableAndFile;
-      tf->file = file;
-      tf->table = table;
-      *handle = cache_->Insert(key, tf, 1, &DeleteEntry);
-    }
-  }
-  return s;
-}
-
-Iterator* TableCache::NewIterator(const ReadOptions& options,
-                                  uint64_t file_number,
-                                  uint64_t file_size,
-                                  Table** tableptr) {
-  if (tableptr != NULL) {
-    *tableptr = NULL;
-  }
-
-  Cache::Handle* handle = NULL;
-  Status s = FindTable(file_number, file_size, &handle);
-  if (!s.ok()) {
-    return NewErrorIterator(s);
-  }
-
-  Table* table = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
-  Iterator* result = table->NewIterator(options);
-  result->RegisterCleanup(&UnrefEntry, cache_, handle);
-  if (tableptr != NULL) {
-    *tableptr = table;
-  }
-  return result;
-}
-
-Status TableCache::Get(const ReadOptions& options,
-                       uint64_t file_number,
-                       uint64_t file_size,
-                       const Slice& k,
-                       void* arg,
-                       void (*saver)(void*, const Slice&, const Slice&)) {
-  Cache::Handle* handle = NULL;
-  Status s = FindTable(file_number, file_size, &handle);
-  if (s.ok()) {
-    Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
-    s = t->InternalGet(options, k, arg, saver);
-    cache_->Release(handle);
-  }
-  return s;
-}
-
-void TableCache::Evict(uint64_t file_number) {
-  char buf[sizeof(file_number)];
-  EncodeFixed64(buf, file_number);
-  cache_->Erase(Slice(buf, sizeof(buf)));
-}
-
-}  // namespace leveldb

src/leveldb/db/table_cache.h

@@ -1,61 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Thread-safe (provides internal synchronization)
-
-#ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_
-#define STORAGE_LEVELDB_DB_TABLE_CACHE_H_
-
-#include <string>
-#include <stdint.h>
-#include "db/dbformat.h"
-#include "leveldb/cache.h"
-#include "leveldb/table.h"
-#include "port/port.h"
-
-namespace leveldb {
-
-class Env;
-
-class TableCache {
- public:
-  TableCache(const std::string& dbname, const Options* options, int entries);
-  ~TableCache();
-
-  // Return an iterator for the specified file number (the corresponding
-  // file length must be exactly "file_size" bytes).  If "tableptr" is
-  // non-NULL, also sets "*tableptr" to point to the Table object
-  // underlying the returned iterator, or NULL if no Table object underlies
-  // the returned iterator.  The returned "*tableptr" object is owned by
-  // the cache and should not be deleted, and is valid for as long as the
-  // returned iterator is live.
-  Iterator* NewIterator(const ReadOptions& options,
-                        uint64_t file_number,
-                        uint64_t file_size,
-                        Table** tableptr = NULL);
-
-  // If a seek to internal key "k" in specified file finds an entry,
-  // call (*handle_result)(arg, found_key, found_value).
-  Status Get(const ReadOptions& options,
-             uint64_t file_number,
-             uint64_t file_size,
-             const Slice& k,
-             void* arg,
-             void (*handle_result)(void*, const Slice&, const Slice&));
-
-  // Evict any entry for the specified file number
-  void Evict(uint64_t file_number);
-
- private:
-  Env* const env_;
-  const std::string dbname_;
-  const Options* options_;
-  Cache* cache_;
-
-  Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_TABLE_CACHE_H_

src/leveldb/db/version_edit.cc

@@ -1,266 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/version_edit.h"
-
-#include "db/version_set.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-// Tag numbers for serialized VersionEdit.  These numbers are written to
-// disk and should not be changed.
-enum Tag {
-  kComparator           = 1,
-  kLogNumber            = 2,
-  kNextFileNumber       = 3,
-  kLastSequence         = 4,
-  kCompactPointer       = 5,
-  kDeletedFile          = 6,
-  kNewFile              = 7,
-  // 8 was used for large value refs
-  kPrevLogNumber        = 9
-};
-
-void VersionEdit::Clear() {
-  comparator_.clear();
-  log_number_ = 0;
-  prev_log_number_ = 0;
-  last_sequence_ = 0;
-  next_file_number_ = 0;
-  has_comparator_ = false;
-  has_log_number_ = false;
-  has_prev_log_number_ = false;
-  has_next_file_number_ = false;
-  has_last_sequence_ = false;
-  deleted_files_.clear();
-  new_files_.clear();
-}
-
-void VersionEdit::EncodeTo(std::string* dst) const {
-  if (has_comparator_) {
-    PutVarint32(dst, kComparator);
-    PutLengthPrefixedSlice(dst, comparator_);
-  }
-  if (has_log_number_) {
-    PutVarint32(dst, kLogNumber);
-    PutVarint64(dst, log_number_);
-  }
-  if (has_prev_log_number_) {
-    PutVarint32(dst, kPrevLogNumber);
-    PutVarint64(dst, prev_log_number_);
-  }
-  if (has_next_file_number_) {
-    PutVarint32(dst, kNextFileNumber);
-    PutVarint64(dst, next_file_number_);
-  }
-  if (has_last_sequence_) {
-    PutVarint32(dst, kLastSequence);
-    PutVarint64(dst, last_sequence_);
-  }
-
-  for (size_t i = 0; i < compact_pointers_.size(); i++) {
-    PutVarint32(dst, kCompactPointer);
-    PutVarint32(dst, compact_pointers_[i].first);  // level
-    PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode());
-  }
-
-  for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
-       iter != deleted_files_.end();
-       ++iter) {
-    PutVarint32(dst, kDeletedFile);
-    PutVarint32(dst, iter->first);   // level
-    PutVarint64(dst, iter->second);  // file number
-  }
-
-  for (size_t i = 0; i < new_files_.size(); i++) {
-    const FileMetaData& f = new_files_[i].second;
-    PutVarint32(dst, kNewFile);
-    PutVarint32(dst, new_files_[i].first);  // level
-    PutVarint64(dst, f.number);
-    PutVarint64(dst, f.file_size);
-    PutLengthPrefixedSlice(dst, f.smallest.Encode());
-    PutLengthPrefixedSlice(dst, f.largest.Encode());
-  }
-}
-
-static bool GetInternalKey(Slice* input, InternalKey* dst) {
-  Slice str;
-  if (GetLengthPrefixedSlice(input, &str)) {
-    dst->DecodeFrom(str);
-    return true;
-  } else {
-    return false;
-  }
-}
-
-static bool GetLevel(Slice* input, int* level) {
-  uint32_t v;
-  if (GetVarint32(input, &v) &&
-      v < config::kNumLevels) {
-    *level = v;
-    return true;
-  } else {
-    return false;
-  }
-}
-
-Status VersionEdit::DecodeFrom(const Slice& src) {
-  Clear();
-  Slice input = src;
-  const char* msg = NULL;
-  uint32_t tag;
-
-  // Temporary storage for parsing
-  int level;
-  uint64_t number;
-  FileMetaData f;
-  Slice str;
-  InternalKey key;
-
-  while (msg == NULL && GetVarint32(&input, &tag)) {
-    switch (tag) {
-      case kComparator:
-        if (GetLengthPrefixedSlice(&input, &str)) {
-          comparator_ = str.ToString();
-          has_comparator_ = true;
-        } else {
-          msg = "comparator name";
-        }
-        break;
-
-      case kLogNumber:
-        if (GetVarint64(&input, &log_number_)) {
-          has_log_number_ = true;
-        } else {
-          msg = "log number";
-        }
-        break;
-
-      case kPrevLogNumber:
-        if (GetVarint64(&input, &prev_log_number_)) {
-          has_prev_log_number_ = true;
-        } else {
-          msg = "previous log number";
-        }
-        break;
-
-      case kNextFileNumber:
-        if (GetVarint64(&input, &next_file_number_)) {
-          has_next_file_number_ = true;
-        } else {
-          msg = "next file number";
-        }
-        break;
-
-      case kLastSequence:
-        if (GetVarint64(&input, &last_sequence_)) {
-          has_last_sequence_ = true;
-        } else {
-          msg = "last sequence number";
-        }
-        break;
-
-      case kCompactPointer:
-        if (GetLevel(&input, &level) &&
-            GetInternalKey(&input, &key)) {
-          compact_pointers_.push_back(std::make_pair(level, key));
-        } else {
-          msg = "compaction pointer";
-        }
-        break;
-
-      case kDeletedFile:
-        if (GetLevel(&input, &level) &&
-            GetVarint64(&input, &number)) {
-          deleted_files_.insert(std::make_pair(level, number));
-        } else {
-          msg = "deleted file";
-        }
-        break;
-
-      case kNewFile:
-        if (GetLevel(&input, &level) &&
-            GetVarint64(&input, &f.number) &&
-            GetVarint64(&input, &f.file_size) &&
-            GetInternalKey(&input, &f.smallest) &&
-            GetInternalKey(&input, &f.largest)) {
-          new_files_.push_back(std::make_pair(level, f));
-        } else {
-          msg = "new-file entry";
-        }
-        break;
-
-      default:
-        msg = "unknown tag";
-        break;
-    }
-  }
-
-  if (msg == NULL && !input.empty()) {
-    msg = "invalid tag";
-  }
-
-  Status result;
-  if (msg != NULL) {
-    result = Status::Corruption("VersionEdit", msg);
-  }
-  return result;
-}
-
-std::string VersionEdit::DebugString() const {
-  std::string r;
-  r.append("VersionEdit {");
-  if (has_comparator_) {
-    r.append("\n  Comparator: ");
-    r.append(comparator_);
-  }
-  if (has_log_number_) {
-    r.append("\n  LogNumber: ");
-    AppendNumberTo(&r, log_number_);
-  }
-  if (has_prev_log_number_) {
-    r.append("\n  PrevLogNumber: ");
-    AppendNumberTo(&r, prev_log_number_);
-  }
-  if (has_next_file_number_) {
-    r.append("\n  NextFile: ");
-    AppendNumberTo(&r, next_file_number_);
-  }
-  if (has_last_sequence_) {
-    r.append("\n  LastSeq: ");
-    AppendNumberTo(&r, last_sequence_);
-  }
-  for (size_t i = 0; i < compact_pointers_.size(); i++) {
-    r.append("\n  CompactPointer: ");
-    AppendNumberTo(&r, compact_pointers_[i].first);
-    r.append(" ");
-    r.append(compact_pointers_[i].second.DebugString());
-  }
-  for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
-       iter != deleted_files_.end();
-       ++iter) {
-    r.append("\n  DeleteFile: ");
-    AppendNumberTo(&r, iter->first);
-    r.append(" ");
-    AppendNumberTo(&r, iter->second);
-  }
-  for (size_t i = 0; i < new_files_.size(); i++) {
-    const FileMetaData& f = new_files_[i].second;
-    r.append("\n  AddFile: ");
-    AppendNumberTo(&r, new_files_[i].first);
-    r.append(" ");
-    AppendNumberTo(&r, f.number);
-    r.append(" ");
-    AppendNumberTo(&r, f.file_size);
-    r.append(" ");
-    r.append(f.smallest.DebugString());
-    r.append(" .. ");
-    r.append(f.largest.DebugString());
-  }
-  r.append("\n}\n");
-  return r;
-}
-
-}  // namespace leveldb

src/leveldb/db/version_edit.h

@@ -1,107 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_
-#define STORAGE_LEVELDB_DB_VERSION_EDIT_H_
-
-#include <set>
-#include <utility>
-#include <vector>
-#include "db/dbformat.h"
-
-namespace leveldb {
-
-class VersionSet;
-
-struct FileMetaData {
-  int refs;
-  int allowed_seeks;          // Seeks allowed until compaction
-  uint64_t number;
-  uint64_t file_size;         // File size in bytes
-  InternalKey smallest;       // Smallest internal key served by table
-  InternalKey largest;        // Largest internal key served by table
-
-  FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { }
-};
-
-class VersionEdit {
- public:
-  VersionEdit() { Clear(); }
-  ~VersionEdit() { }
-
-  void Clear();
-
-  void SetComparatorName(const Slice& name) {
-    has_comparator_ = true;
-    comparator_ = name.ToString();
-  }
-  void SetLogNumber(uint64_t num) {
-    has_log_number_ = true;
-    log_number_ = num;
-  }
-  void SetPrevLogNumber(uint64_t num) {
-    has_prev_log_number_ = true;
-    prev_log_number_ = num;
-  }
-  void SetNextFile(uint64_t num) {
-    has_next_file_number_ = true;
-    next_file_number_ = num;
-  }
-  void SetLastSequence(SequenceNumber seq) {
-    has_last_sequence_ = true;
-    last_sequence_ = seq;
-  }
-  void SetCompactPointer(int level, const InternalKey& key) {
-    compact_pointers_.push_back(std::make_pair(level, key));
-  }
-
-  // Add the specified file at the specified number.
-  // REQUIRES: This version has not been saved (see VersionSet::SaveTo)
-  // REQUIRES: "smallest" and "largest" are smallest and largest keys in file
-  void AddFile(int level, uint64_t file,
-               uint64_t file_size,
-               const InternalKey& smallest,
-               const InternalKey& largest) {
-    FileMetaData f;
-    f.number = file;
-    f.file_size = file_size;
-    f.smallest = smallest;
-    f.largest = largest;
-    new_files_.push_back(std::make_pair(level, f));
-  }
-
-  // Delete the specified "file" from the specified "level".
-  void DeleteFile(int level, uint64_t file) {
-    deleted_files_.insert(std::make_pair(level, file));
-  }
-
-  void EncodeTo(std::string* dst) const;
-  Status DecodeFrom(const Slice& src);
-
-  std::string DebugString() const;
-
- private:
-  friend class VersionSet;
-
-  typedef std::set< std::pair<int, uint64_t> > DeletedFileSet;
-
-  std::string comparator_;
-  uint64_t log_number_;
-  uint64_t prev_log_number_;
-  uint64_t next_file_number_;
-  SequenceNumber last_sequence_;
-  bool has_comparator_;
-  bool has_log_number_;
-  bool has_prev_log_number_;
-  bool has_next_file_number_;
-  bool has_last_sequence_;
-
-  std::vector< std::pair<int, InternalKey> > compact_pointers_;
-  DeletedFileSet deleted_files_;
-  std::vector< std::pair<int, FileMetaData> > new_files_;
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_VERSION_EDIT_H_

src/leveldb/db/version_edit_test.cc

@@ -1,46 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/version_edit.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-static void TestEncodeDecode(const VersionEdit& edit) {
-  std::string encoded, encoded2;
-  edit.EncodeTo(&encoded);
-  VersionEdit parsed;
-  Status s = parsed.DecodeFrom(encoded);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  parsed.EncodeTo(&encoded2);
-  ASSERT_EQ(encoded, encoded2);
-}
-
-class VersionEditTest { };
-
-TEST(VersionEditTest, EncodeDecode) {
-  static const uint64_t kBig = 1ull << 50;
-
-  VersionEdit edit;
-  for (int i = 0; i < 4; i++) {
-    TestEncodeDecode(edit);
-    edit.AddFile(3, kBig + 300 + i, kBig + 400 + i,
-                 InternalKey("foo", kBig + 500 + i, kTypeValue),
-                 InternalKey("zoo", kBig + 600 + i, kTypeDeletion));
-    edit.DeleteFile(4, kBig + 700 + i);
-    edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue));
-  }
-
-  edit.SetComparatorName("foo");
-  edit.SetLogNumber(kBig + 100);
-  edit.SetNextFile(kBig + 200);
-  edit.SetLastSequence(kBig + 1000);
-  TestEncodeDecode(edit);
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/version_set.h

@@ -1,398 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// The representation of a DBImpl consists of a set of Versions.  The
-// newest version is called "current".  Older versions may be kept
-// around to provide a consistent view to live iterators.
-//
-// Each Version keeps track of a set of Table files per level.  The
-// entire set of versions is maintained in a VersionSet.
-//
-// Version,VersionSet are thread-compatible, but require external
-// synchronization on all accesses.
-
-#ifndef STORAGE_LEVELDB_DB_VERSION_SET_H_
-#define STORAGE_LEVELDB_DB_VERSION_SET_H_
-
-#include <map>
-#include <set>
-#include <vector>
-#include "db/dbformat.h"
-#include "db/version_edit.h"
-#include "port/port.h"
-#include "port/thread_annotations.h"
-
-namespace leveldb {
-
-namespace log { class Writer; }
-
-class Compaction;
-class Iterator;
-class MemTable;
-class TableBuilder;
-class TableCache;
-class Version;
-class VersionSet;
-class WritableFile;
-
-// Return the smallest index i such that files[i]->largest >= key.
-// Return files.size() if there is no such file.
-// REQUIRES: "files" contains a sorted list of non-overlapping files.
-extern int FindFile(const InternalKeyComparator& icmp,
-                    const std::vector<FileMetaData*>& files,
-                    const Slice& key);
-
-// Returns true iff some file in "files" overlaps the user key range
-// [*smallest,*largest].
-// smallest==NULL represents a key smaller than all keys in the DB.
-// largest==NULL represents a key largest than all keys in the DB.
-// REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges
-//           in sorted order.
-extern bool SomeFileOverlapsRange(
-    const InternalKeyComparator& icmp,
-    bool disjoint_sorted_files,
-    const std::vector<FileMetaData*>& files,
-    const Slice* smallest_user_key,
-    const Slice* largest_user_key);
-
-class Version {
- public:
-  // Append to *iters a sequence of iterators that will
-  // yield the contents of this Version when merged together.
-  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
-
-  // Lookup the value for key.  If found, store it in *val and
-  // return OK.  Else return a non-OK status.  Fills *stats.
-  // REQUIRES: lock is not held
-  struct GetStats {
-    FileMetaData* seek_file;
-    int seek_file_level;
-  };
-  Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
-             GetStats* stats);
-
-  // Adds "stats" into the current state.  Returns true if a new
-  // compaction may need to be triggered, false otherwise.
-  // REQUIRES: lock is held
-  bool UpdateStats(const GetStats& stats);
-
-  // Record a sample of bytes read at the specified internal key.
-  // Samples are taken approximately once every config::kReadBytesPeriod
-  // bytes.  Returns true if a new compaction may need to be triggered.
-  // REQUIRES: lock is held
-  bool RecordReadSample(Slice key);
-
-  // Reference count management (so Versions do not disappear out from
-  // under live iterators)
-  void Ref();
-  void Unref();
-
-  void GetOverlappingInputs(
-      int level,
-      const InternalKey* begin,         // NULL means before all keys
-      const InternalKey* end,           // NULL means after all keys
-      std::vector<FileMetaData*>* inputs);
-
-  // Returns true iff some file in the specified level overlaps
-  // some part of [*smallest_user_key,*largest_user_key].
-  // smallest_user_key==NULL represents a key smaller than all keys in the DB.
-  // largest_user_key==NULL represents a key largest than all keys in the DB.
-  bool OverlapInLevel(int level,
-                      const Slice* smallest_user_key,
-                      const Slice* largest_user_key);
-
-  // Return the level at which we should place a new memtable compaction
-  // result that covers the range [smallest_user_key,largest_user_key].
-  int PickLevelForMemTableOutput(const Slice& smallest_user_key,
-                                 const Slice& largest_user_key);
-
-  int NumFiles(int level) const { return files_[level].size(); }
-
-  // Return a human readable string that describes this version's contents.
-  std::string DebugString() const;
-
- private:
-  friend class Compaction;
-  friend class VersionSet;
-
-  class LevelFileNumIterator;
-  Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const;
-
-  // Call func(arg, level, f) for every file that overlaps user_key in
-  // order from newest to oldest.  If an invocation of func returns
-  // false, makes no more calls.
-  //
-  // REQUIRES: user portion of internal_key == user_key.
-  void ForEachOverlapping(Slice user_key, Slice internal_key,
-                          void* arg,
-                          bool (*func)(void*, int, FileMetaData*));
-
-  VersionSet* vset_;            // VersionSet to which this Version belongs
-  Version* next_;               // Next version in linked list
-  Version* prev_;               // Previous version in linked list
-  int refs_;                    // Number of live refs to this version
-
-  // List of files per level
-  std::vector<FileMetaData*> files_[config::kNumLevels];
-
-  // Next file to compact based on seek stats.
-  FileMetaData* file_to_compact_;
-  int file_to_compact_level_;
-
-  // Level that should be compacted next and its compaction score.
-  // Score < 1 means compaction is not strictly needed.  These fields
-  // are initialized by Finalize().
-  double compaction_score_;
-  int compaction_level_;
-
-  explicit Version(VersionSet* vset)
-      : vset_(vset), next_(this), prev_(this), refs_(0),
-        file_to_compact_(NULL),
-        file_to_compact_level_(-1),
-        compaction_score_(-1),
-        compaction_level_(-1) {
-  }
-
-  ~Version();
-
-  // No copying allowed
-  Version(const Version&);
-  void operator=(const Version&);
-};
-
-class VersionSet {
- public:
-  VersionSet(const std::string& dbname,
-             const Options* options,
-             TableCache* table_cache,
-             const InternalKeyComparator*);
-  ~VersionSet();
-
-  // Apply *edit to the current version to form a new descriptor that
-  // is both saved to persistent state and installed as the new
-  // current version.  Will release *mu while actually writing to the file.
-  // REQUIRES: *mu is held on entry.
-  // REQUIRES: no other thread concurrently calls LogAndApply()
-  Status LogAndApply(VersionEdit* edit, port::Mutex* mu)
-      EXCLUSIVE_LOCKS_REQUIRED(mu);
-
-  // Recover the last saved descriptor from persistent storage.
-  Status Recover(bool *save_manifest);
-
-  // Return the current version.
-  Version* current() const { return current_; }
-
-  // Return the current manifest file number
-  uint64_t ManifestFileNumber() const { return manifest_file_number_; }
-
-  // Allocate and return a new file number
-  uint64_t NewFileNumber() { return next_file_number_++; }
-
-  // Arrange to reuse "file_number" unless a newer file number has
-  // already been allocated.
-  // REQUIRES: "file_number" was returned by a call to NewFileNumber().
-  void ReuseFileNumber(uint64_t file_number) {
-    if (next_file_number_ == file_number + 1) {
-      next_file_number_ = file_number;
-    }
-  }
-
-  // Return the number of Table files at the specified level.
-  int NumLevelFiles(int level) const;
-
-  // Return the combined file size of all files at the specified level.
-  int64_t NumLevelBytes(int level) const;
-
-  // Return the last sequence number.
-  uint64_t LastSequence() const { return last_sequence_; }
-
-  // Set the last sequence number to s.
-  void SetLastSequence(uint64_t s) {
-    assert(s >= last_sequence_);
-    last_sequence_ = s;
-  }
-
-  // Mark the specified file number as used.
-  void MarkFileNumberUsed(uint64_t number);
-
-  // Return the current log file number.
-  uint64_t LogNumber() const { return log_number_; }
-
-  // Return the log file number for the log file that is currently
-  // being compacted, or zero if there is no such log file.
-  uint64_t PrevLogNumber() const { return prev_log_number_; }
-
-  // Pick level and inputs for a new compaction.
-  // Returns NULL if there is no compaction to be done.
-  // Otherwise returns a pointer to a heap-allocated object that
-  // describes the compaction.  Caller should delete the result.
-  Compaction* PickCompaction();
-
-  // Return a compaction object for compacting the range [begin,end] in
-  // the specified level.  Returns NULL if there is nothing in that
-  // level that overlaps the specified range.  Caller should delete
-  // the result.
-  Compaction* CompactRange(
-      int level,
-      const InternalKey* begin,
-      const InternalKey* end);
-
-  // Return the maximum overlapping data (in bytes) at next level for any
-  // file at a level >= 1.
-  int64_t MaxNextLevelOverlappingBytes();
-
-  // Create an iterator that reads over the compaction inputs for "*c".
-  // The caller should delete the iterator when no longer needed.
-  Iterator* MakeInputIterator(Compaction* c);
-
-  // Returns true iff some level needs a compaction.
-  bool NeedsCompaction() const {
-    Version* v = current_;
-    return (v->compaction_score_ >= 1) || (v->file_to_compact_ != NULL);
-  }
-
-  // Add all files listed in any live version to *live.
-  // May also mutate some internal state.
-  void AddLiveFiles(std::set<uint64_t>* live);
-
-  // Return the approximate offset in the database of the data for
-  // "key" as of version "v".
-  uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key);
-
-  // Return a human-readable short (single-line) summary of the number
-  // of files per level.  Uses *scratch as backing store.
-  struct LevelSummaryStorage {
-    char buffer[100];
-  };
-  const char* LevelSummary(LevelSummaryStorage* scratch) const;
-
- private:
-  class Builder;
-
-  friend class Compaction;
-  friend class Version;
-
-  bool ReuseManifest(const std::string& dscname, const std::string& dscbase);
-
-  void Finalize(Version* v);
-
-  void GetRange(const std::vector<FileMetaData*>& inputs,
-                InternalKey* smallest,
-                InternalKey* largest);
-
-  void GetRange2(const std::vector<FileMetaData*>& inputs1,
-                 const std::vector<FileMetaData*>& inputs2,
-                 InternalKey* smallest,
-                 InternalKey* largest);
-
-  void SetupOtherInputs(Compaction* c);
-
-  // Save current contents to *log
-  Status WriteSnapshot(log::Writer* log);
-
-  void AppendVersion(Version* v);
-
-  Env* const env_;
-  const std::string dbname_;
-  const Options* const options_;
-  TableCache* const table_cache_;
-  const InternalKeyComparator icmp_;
-  uint64_t next_file_number_;
-  uint64_t manifest_file_number_;
-  uint64_t last_sequence_;
-  uint64_t log_number_;
-  uint64_t prev_log_number_;  // 0 or backing store for memtable being compacted
-
-  // Opened lazily
-  WritableFile* descriptor_file_;
-  log::Writer* descriptor_log_;
-  Version dummy_versions_;  // Head of circular doubly-linked list of versions.
-  Version* current_;        // == dummy_versions_.prev_
-
-  // Per-level key at which the next compaction at that level should start.
-  // Either an empty string, or a valid InternalKey.
-  std::string compact_pointer_[config::kNumLevels];
-
-  // No copying allowed
-  VersionSet(const VersionSet&);
-  void operator=(const VersionSet&);
-};
-
-// A Compaction encapsulates information about a compaction.
-class Compaction {
- public:
-  ~Compaction();
-
-  // Return the level that is being compacted.  Inputs from "level"
-  // and "level+1" will be merged to produce a set of "level+1" files.
-  int level() const { return level_; }
-
-  // Return the object that holds the edits to the descriptor done
-  // by this compaction.
-  VersionEdit* edit() { return &edit_; }
-
-  // "which" must be either 0 or 1
-  int num_input_files(int which) const { return inputs_[which].size(); }
-
-  // Return the ith input file at "level()+which" ("which" must be 0 or 1).
-  FileMetaData* input(int which, int i) const { return inputs_[which][i]; }
-
-  // Maximum size of files to build during this compaction.
-  uint64_t MaxOutputFileSize() const { return max_output_file_size_; }
-
-  // Is this a trivial compaction that can be implemented by just
-  // moving a single input file to the next level (no merging or splitting)
-  bool IsTrivialMove() const;
-
-  // Add all inputs to this compaction as delete operations to *edit.
-  void AddInputDeletions(VersionEdit* edit);
-
-  // Returns true if the information we have available guarantees that
-  // the compaction is producing data in "level+1" for which no data exists
-  // in levels greater than "level+1".
-  bool IsBaseLevelForKey(const Slice& user_key);
-
-  // Returns true iff we should stop building the current output
-  // before processing "internal_key".
-  bool ShouldStopBefore(const Slice& internal_key);
-
-  // Release the input version for the compaction, once the compaction
-  // is successful.
-  void ReleaseInputs();
-
- private:
-  friend class Version;
-  friend class VersionSet;
-
-  Compaction(const Options* options, int level);
-
-  int level_;
-  uint64_t max_output_file_size_;
-  Version* input_version_;
-  VersionEdit edit_;
-
-  // Each compaction reads inputs from "level_" and "level_+1"
-  std::vector<FileMetaData*> inputs_[2];      // The two sets of inputs
-
-  // State used to check for number of overlapping grandparent files
-  // (parent == level_ + 1, grandparent == level_ + 2)
-  std::vector<FileMetaData*> grandparents_;
-  size_t grandparent_index_;  // Index in grandparent_starts_
-  bool seen_key_;             // Some output key has been seen
-  int64_t overlapped_bytes_;  // Bytes of overlap between current output
-                              // and grandparent files
-
-  // State for implementing IsBaseLevelForKey
-
-  // level_ptrs_ holds indices into input_version_->levels_: our state
-  // is that we are positioned at one of the file ranges for each
-  // higher level than the ones involved in this compaction (i.e. for
-  // all L >= level_ + 2).
-  size_t level_ptrs_[config::kNumLevels];
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_DB_VERSION_SET_H_

src/leveldb/db/version_set_test.cc

@@ -1,179 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/version_set.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-class FindFileTest {
- public:
-  std::vector<FileMetaData*> files_;
-  bool disjoint_sorted_files_;
-
-  FindFileTest() : disjoint_sorted_files_(true) { }
-
-  ~FindFileTest() {
-    for (int i = 0; i < files_.size(); i++) {
-      delete files_[i];
-    }
-  }
-
-  void Add(const char* smallest, const char* largest,
-           SequenceNumber smallest_seq = 100,
-           SequenceNumber largest_seq = 100) {
-    FileMetaData* f = new FileMetaData;
-    f->number = files_.size() + 1;
-    f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
-    f->largest = InternalKey(largest, largest_seq, kTypeValue);
-    files_.push_back(f);
-  }
-
-  int Find(const char* key) {
-    InternalKey target(key, 100, kTypeValue);
-    InternalKeyComparator cmp(BytewiseComparator());
-    return FindFile(cmp, files_, target.Encode());
-  }
-
-  bool Overlaps(const char* smallest, const char* largest) {
-    InternalKeyComparator cmp(BytewiseComparator());
-    Slice s(smallest != NULL ? smallest : "");
-    Slice l(largest != NULL ? largest : "");
-    return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, files_,
-                                 (smallest != NULL ? &s : NULL),
-                                 (largest != NULL ? &l : NULL));
-  }
-};
-
-TEST(FindFileTest, Empty) {
-  ASSERT_EQ(0, Find("foo"));
-  ASSERT_TRUE(! Overlaps("a", "z"));
-  ASSERT_TRUE(! Overlaps(NULL, "z"));
-  ASSERT_TRUE(! Overlaps("a", NULL));
-  ASSERT_TRUE(! Overlaps(NULL, NULL));
-}
-
-TEST(FindFileTest, Single) {
-  Add("p", "q");
-  ASSERT_EQ(0, Find("a"));
-  ASSERT_EQ(0, Find("p"));
-  ASSERT_EQ(0, Find("p1"));
-  ASSERT_EQ(0, Find("q"));
-  ASSERT_EQ(1, Find("q1"));
-  ASSERT_EQ(1, Find("z"));
-
-  ASSERT_TRUE(! Overlaps("a", "b"));
-  ASSERT_TRUE(! Overlaps("z1", "z2"));
-  ASSERT_TRUE(Overlaps("a", "p"));
-  ASSERT_TRUE(Overlaps("a", "q"));
-  ASSERT_TRUE(Overlaps("a", "z"));
-  ASSERT_TRUE(Overlaps("p", "p1"));
-  ASSERT_TRUE(Overlaps("p", "q"));
-  ASSERT_TRUE(Overlaps("p", "z"));
-  ASSERT_TRUE(Overlaps("p1", "p2"));
-  ASSERT_TRUE(Overlaps("p1", "z"));
-  ASSERT_TRUE(Overlaps("q", "q"));
-  ASSERT_TRUE(Overlaps("q", "q1"));
-
-  ASSERT_TRUE(! Overlaps(NULL, "j"));
-  ASSERT_TRUE(! Overlaps("r", NULL));
-  ASSERT_TRUE(Overlaps(NULL, "p"));
-  ASSERT_TRUE(Overlaps(NULL, "p1"));
-  ASSERT_TRUE(Overlaps("q", NULL));
-  ASSERT_TRUE(Overlaps(NULL, NULL));
-}
-
-
-TEST(FindFileTest, Multiple) {
-  Add("150", "200");
-  Add("200", "250");
-  Add("300", "350");
-  Add("400", "450");
-  ASSERT_EQ(0, Find("100"));
-  ASSERT_EQ(0, Find("150"));
-  ASSERT_EQ(0, Find("151"));
-  ASSERT_EQ(0, Find("199"));
-  ASSERT_EQ(0, Find("200"));
-  ASSERT_EQ(1, Find("201"));
-  ASSERT_EQ(1, Find("249"));
-  ASSERT_EQ(1, Find("250"));
-  ASSERT_EQ(2, Find("251"));
-  ASSERT_EQ(2, Find("299"));
-  ASSERT_EQ(2, Find("300"));
-  ASSERT_EQ(2, Find("349"));
-  ASSERT_EQ(2, Find("350"));
-  ASSERT_EQ(3, Find("351"));
-  ASSERT_EQ(3, Find("400"));
-  ASSERT_EQ(3, Find("450"));
-  ASSERT_EQ(4, Find("451"));
-
-  ASSERT_TRUE(! Overlaps("100", "149"));
-  ASSERT_TRUE(! Overlaps("251", "299"));
-  ASSERT_TRUE(! Overlaps("451", "500"));
-  ASSERT_TRUE(! Overlaps("351", "399"));
-
-  ASSERT_TRUE(Overlaps("100", "150"));
-  ASSERT_TRUE(Overlaps("100", "200"));
-  ASSERT_TRUE(Overlaps("100", "300"));
-  ASSERT_TRUE(Overlaps("100", "400"));
-  ASSERT_TRUE(Overlaps("100", "500"));
-  ASSERT_TRUE(Overlaps("375", "400"));
-  ASSERT_TRUE(Overlaps("450", "450"));
-  ASSERT_TRUE(Overlaps("450", "500"));
-}
-
-TEST(FindFileTest, MultipleNullBoundaries) {
-  Add("150", "200");
-  Add("200", "250");
-  Add("300", "350");
-  Add("400", "450");
-  ASSERT_TRUE(! Overlaps(NULL, "149"));
-  ASSERT_TRUE(! Overlaps("451", NULL));
-  ASSERT_TRUE(Overlaps(NULL, NULL));
-  ASSERT_TRUE(Overlaps(NULL, "150"));
-  ASSERT_TRUE(Overlaps(NULL, "199"));
-  ASSERT_TRUE(Overlaps(NULL, "200"));
-  ASSERT_TRUE(Overlaps(NULL, "201"));
-  ASSERT_TRUE(Overlaps(NULL, "400"));
-  ASSERT_TRUE(Overlaps(NULL, "800"));
-  ASSERT_TRUE(Overlaps("100", NULL));
-  ASSERT_TRUE(Overlaps("200", NULL));
-  ASSERT_TRUE(Overlaps("449", NULL));
-  ASSERT_TRUE(Overlaps("450", NULL));
-}
-
-TEST(FindFileTest, OverlapSequenceChecks) {
-  Add("200", "200", 5000, 3000);
-  ASSERT_TRUE(! Overlaps("199", "199"));
-  ASSERT_TRUE(! Overlaps("201", "300"));
-  ASSERT_TRUE(Overlaps("200", "200"));
-  ASSERT_TRUE(Overlaps("190", "200"));
-  ASSERT_TRUE(Overlaps("200", "210"));
-}
-
-TEST(FindFileTest, OverlappingFiles) {
-  Add("150", "600");
-  Add("400", "500");
-  disjoint_sorted_files_ = false;
-  ASSERT_TRUE(! Overlaps("100", "149"));
-  ASSERT_TRUE(! Overlaps("601", "700"));
-  ASSERT_TRUE(Overlaps("100", "150"));
-  ASSERT_TRUE(Overlaps("100", "200"));
-  ASSERT_TRUE(Overlaps("100", "300"));
-  ASSERT_TRUE(Overlaps("100", "400"));
-  ASSERT_TRUE(Overlaps("100", "500"));
-  ASSERT_TRUE(Overlaps("375", "400"));
-  ASSERT_TRUE(Overlaps("450", "450"));
-  ASSERT_TRUE(Overlaps("450", "500"));
-  ASSERT_TRUE(Overlaps("450", "700"));
-  ASSERT_TRUE(Overlaps("600", "700"));
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/db/write_batch.cc

@@ -1,147 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// WriteBatch::rep_ :=
-//    sequence: fixed64
-//    count: fixed32
-//    data: record[count]
-// record :=
-//    kTypeValue varstring varstring         |
-//    kTypeDeletion varstring
-// varstring :=
-//    len: varint32
-//    data: uint8[len]
-
-#include "leveldb/write_batch.h"
-
-#include "leveldb/db.h"
-#include "db/dbformat.h"
-#include "db/memtable.h"
-#include "db/write_batch_internal.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
-static const size_t kHeader = 12;
-
-WriteBatch::WriteBatch() {
-  Clear();
-}
-
-WriteBatch::~WriteBatch() { }
-
-WriteBatch::Handler::~Handler() { }
-
-void WriteBatch::Clear() {
-  rep_.clear();
-  rep_.resize(kHeader);
-}
-
-Status WriteBatch::Iterate(Handler* handler) const {
-  Slice input(rep_);
-  if (input.size() < kHeader) {
-    return Status::Corruption("malformed WriteBatch (too small)");
-  }
-
-  input.remove_prefix(kHeader);
-  Slice key, value;
-  int found = 0;
-  while (!input.empty()) {
-    found++;
-    char tag = input[0];
-    input.remove_prefix(1);
-    switch (tag) {
-      case kTypeValue:
-        if (GetLengthPrefixedSlice(&input, &key) &&
-            GetLengthPrefixedSlice(&input, &value)) {
-          handler->Put(key, value);
-        } else {
-          return Status::Corruption("bad WriteBatch Put");
-        }
-        break;
-      case kTypeDeletion:
-        if (GetLengthPrefixedSlice(&input, &key)) {
-          handler->Delete(key);
-        } else {
-          return Status::Corruption("bad WriteBatch Delete");
-        }
-        break;
-      default:
-        return Status::Corruption("unknown WriteBatch tag");
-    }
-  }
-  if (found != WriteBatchInternal::Count(this)) {
-    return Status::Corruption("WriteBatch has wrong count");
-  } else {
-    return Status::OK();
-  }
-}
-
-int WriteBatchInternal::Count(const WriteBatch* b) {
-  return DecodeFixed32(b->rep_.data() + 8);
-}
-
-void WriteBatchInternal::SetCount(WriteBatch* b, int n) {
-  EncodeFixed32(&b->rep_[8], n);
-}
-
-SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) {
-  return SequenceNumber(DecodeFixed64(b->rep_.data()));
-}
-
-void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) {
-  EncodeFixed64(&b->rep_[0], seq);
-}
-
-void WriteBatch::Put(const Slice& key, const Slice& value) {
-  WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
-  rep_.push_back(static_cast<char>(kTypeValue));
-  PutLengthPrefixedSlice(&rep_, key);
-  PutLengthPrefixedSlice(&rep_, value);
-}
-
-void WriteBatch::Delete(const Slice& key) {
-  WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
-  rep_.push_back(static_cast<char>(kTypeDeletion));
-  PutLengthPrefixedSlice(&rep_, key);
-}
-
-namespace {
-class MemTableInserter : public WriteBatch::Handler {
- public:
-  SequenceNumber sequence_;
-  MemTable* mem_;
-
-  virtual void Put(const Slice& key, const Slice& value) {
-    mem_->Add(sequence_, kTypeValue, key, value);
-    sequence_++;
-  }
-  virtual void Delete(const Slice& key) {
-    mem_->Add(sequence_, kTypeDeletion, key, Slice());
-    sequence_++;
-  }
-};
-}  // namespace
-
-Status WriteBatchInternal::InsertInto(const WriteBatch* b,
-                                      MemTable* memtable) {
-  MemTableInserter inserter;
-  inserter.sequence_ = WriteBatchInternal::Sequence(b);
-  inserter.mem_ = memtable;
-  return b->Iterate(&inserter);
-}
-
-void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
-  assert(contents.size() >= kHeader);
-  b->rep_.assign(contents.data(), contents.size());
-}
-
-void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) {
-  SetCount(dst, Count(dst) + Count(src));
-  assert(src->rep_.size() >= kHeader);
-  dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader);
-}
-
-}  // namespace leveldb

src/leveldb/db/write_batch_internal.h

@@ -1,50 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
-#define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
-
-#include "db/dbformat.h"
-#include "leveldb/write_batch.h"
-
-namespace leveldb {
-
-class MemTable;
-
-// WriteBatchInternal provides static methods for manipulating a
-// WriteBatch that we don't want in the public WriteBatch interface.
-class WriteBatchInternal {
- public:
-  // Return the number of entries in the batch.
-  static int Count(const WriteBatch* batch);
-
-  // Set the count for the number of entries in the batch.
-  static void SetCount(WriteBatch* batch, int n);
-
-  // Return the sequence number for the start of this batch.
-  static SequenceNumber Sequence(const WriteBatch* batch);
-
-  // Store the specified number as the sequence number for the start of
-  // this batch.
-  static void SetSequence(WriteBatch* batch, SequenceNumber seq);
-
-  static Slice Contents(const WriteBatch* batch) {
-    return Slice(batch->rep_);
-  }
-
-  static size_t ByteSize(const WriteBatch* batch) {
-    return batch->rep_.size();
-  }
-
-  static void SetContents(WriteBatch* batch, const Slice& contents);
-
-  static Status InsertInto(const WriteBatch* batch, MemTable* memtable);
-
-  static void Append(WriteBatch* dst, const WriteBatch* src);
-};
-
-}  // namespace leveldb
-
-
-#endif  // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_

src/leveldb/db/write_batch_test.cc

@@ -1,120 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/db.h"
-
-#include "db/memtable.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/env.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-static std::string PrintContents(WriteBatch* b) {
-  InternalKeyComparator cmp(BytewiseComparator());
-  MemTable* mem = new MemTable(cmp);
-  mem->Ref();
-  std::string state;
-  Status s = WriteBatchInternal::InsertInto(b, mem);
-  int count = 0;
-  Iterator* iter = mem->NewIterator();
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    ParsedInternalKey ikey;
-    ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey));
-    switch (ikey.type) {
-      case kTypeValue:
-        state.append("Put(");
-        state.append(ikey.user_key.ToString());
-        state.append(", ");
-        state.append(iter->value().ToString());
-        state.append(")");
-        count++;
-        break;
-      case kTypeDeletion:
-        state.append("Delete(");
-        state.append(ikey.user_key.ToString());
-        state.append(")");
-        count++;
-        break;
-    }
-    state.append("@");
-    state.append(NumberToString(ikey.sequence));
-  }
-  delete iter;
-  if (!s.ok()) {
-    state.append("ParseError()");
-  } else if (count != WriteBatchInternal::Count(b)) {
-    state.append("CountMismatch()");
-  }
-  mem->Unref();
-  return state;
-}
-
-class WriteBatchTest { };
-
-TEST(WriteBatchTest, Empty) {
-  WriteBatch batch;
-  ASSERT_EQ("", PrintContents(&batch));
-  ASSERT_EQ(0, WriteBatchInternal::Count(&batch));
-}
-
-TEST(WriteBatchTest, Multiple) {
-  WriteBatch batch;
-  batch.Put(Slice("foo"), Slice("bar"));
-  batch.Delete(Slice("box"));
-  batch.Put(Slice("baz"), Slice("boo"));
-  WriteBatchInternal::SetSequence(&batch, 100);
-  ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch));
-  ASSERT_EQ(3, WriteBatchInternal::Count(&batch));
-  ASSERT_EQ("Put(baz, boo)@102"
-            "Delete(box)@101"
-            "Put(foo, bar)@100",
-            PrintContents(&batch));
-}
-
-TEST(WriteBatchTest, Corruption) {
-  WriteBatch batch;
-  batch.Put(Slice("foo"), Slice("bar"));
-  batch.Delete(Slice("box"));
-  WriteBatchInternal::SetSequence(&batch, 200);
-  Slice contents = WriteBatchInternal::Contents(&batch);
-  WriteBatchInternal::SetContents(&batch,
-                                  Slice(contents.data(),contents.size()-1));
-  ASSERT_EQ("Put(foo, bar)@200"
-            "ParseError()",
-            PrintContents(&batch));
-}
-
-TEST(WriteBatchTest, Append) {
-  WriteBatch b1, b2;
-  WriteBatchInternal::SetSequence(&b1, 200);
-  WriteBatchInternal::SetSequence(&b2, 300);
-  WriteBatchInternal::Append(&b1, &b2);
-  ASSERT_EQ("",
-            PrintContents(&b1));
-  b2.Put("a", "va");
-  WriteBatchInternal::Append(&b1, &b2);
-  ASSERT_EQ("Put(a, va)@200",
-            PrintContents(&b1));
-  b2.Clear();
-  b2.Put("b", "vb");
-  WriteBatchInternal::Append(&b1, &b2);
-  ASSERT_EQ("Put(a, va)@200"
-            "Put(b, vb)@201",
-            PrintContents(&b1));
-  b2.Delete("foo");
-  WriteBatchInternal::Append(&b1, &b2);
-  ASSERT_EQ("Put(a, va)@200"
-            "Put(b, vb)@202"
-            "Put(b, vb)@201"
-            "Delete(foo)@203",
-            PrintContents(&b1));
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/doc/bench/db_bench_sqlite3.cc

@@ -1,718 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <sqlite3.h>
-#include "util/histogram.h"
-#include "util/random.h"
-#include "util/testutil.h"
-
-// Comma-separated list of operations to run in the specified order
-//   Actual benchmarks:
-//
-//   fillseq       -- write N values in sequential key order in async mode
-//   fillseqsync   -- write N/100 values in sequential key order in sync mode
-//   fillseqbatch  -- batch write N values in sequential key order in async mode
-//   fillrandom    -- write N values in random key order in async mode
-//   fillrandsync  -- write N/100 values in random key order in sync mode
-//   fillrandbatch -- batch write N values in sequential key order in async mode
-//   overwrite     -- overwrite N values in random key order in async mode
-//   fillrand100K  -- write N/1000 100K values in random order in async mode
-//   fillseq100K   -- write N/1000 100K values in sequential order in async mode
-//   readseq       -- read N times sequentially
-//   readrandom    -- read N times in random order
-//   readrand100K  -- read N/1000 100K values in sequential order in async mode
-static const char* FLAGS_benchmarks =
-    "fillseq,"
-    "fillseqsync,"
-    "fillseqbatch,"
-    "fillrandom,"
-    "fillrandsync,"
-    "fillrandbatch,"
-    "overwrite,"
-    "overwritebatch,"
-    "readrandom,"
-    "readseq,"
-    "fillrand100K,"
-    "fillseq100K,"
-    "readseq,"
-    "readrand100K,"
-    ;
-
-// Number of key/values to place in database
-static int FLAGS_num = 1000000;
-
-// Number of read operations to do.  If negative, do FLAGS_num reads.
-static int FLAGS_reads = -1;
-
-// Size of each value
-static int FLAGS_value_size = 100;
-
-// Print histogram of operation timings
-static bool FLAGS_histogram = false;
-
-// Arrange to generate values that shrink to this fraction of
-// their original size after compression
-static double FLAGS_compression_ratio = 0.5;
-
-// Page size. Default 1 KB.
-static int FLAGS_page_size = 1024;
-
-// Number of pages.
-// Default cache size = FLAGS_page_size * FLAGS_num_pages = 4 MB.
-static int FLAGS_num_pages = 4096;
-
-// If true, do not destroy the existing database.  If you set this
-// flag and also specify a benchmark that wants a fresh database, that
-// benchmark will fail.
-static bool FLAGS_use_existing_db = false;
-
-// If true, we allow batch writes to occur
-static bool FLAGS_transaction = true;
-
-// If true, we enable Write-Ahead Logging
-static bool FLAGS_WAL_enabled = true;
-
-// Use the db with the following name.
-static const char* FLAGS_db = NULL;
-
-inline
-static void ExecErrorCheck(int status, char *err_msg) {
-  if (status != SQLITE_OK) {
-    fprintf(stderr, "SQL error: %s\n", err_msg);
-    sqlite3_free(err_msg);
-    exit(1);
-  }
-}
-
-inline
-static void StepErrorCheck(int status) {
-  if (status != SQLITE_DONE) {
-    fprintf(stderr, "SQL step error: status = %d\n", status);
-    exit(1);
-  }
-}
-
-inline
-static void ErrorCheck(int status) {
-  if (status != SQLITE_OK) {
-    fprintf(stderr, "sqlite3 error: status = %d\n", status);
-    exit(1);
-  }
-}
-
-inline
-static void WalCheckpoint(sqlite3* db_) {
-  // Flush all writes to disk
-  if (FLAGS_WAL_enabled) {
-    sqlite3_wal_checkpoint_v2(db_, NULL, SQLITE_CHECKPOINT_FULL, NULL, NULL);
-  }
-}
-
-namespace leveldb {
-
-// Helper for quickly generating random data.
-namespace {
-class RandomGenerator {
- private:
-  std::string data_;
-  int pos_;
-
- public:
-  RandomGenerator() {
-    // We use a limited amount of data over and over again and ensure
-    // that it is larger than the compression window (32KB), and also
-    // large enough to serve all typical value sizes we want to write.
-    Random rnd(301);
-    std::string piece;
-    while (data_.size() < 1048576) {
-      // Add a short fragment that is as compressible as specified
-      // by FLAGS_compression_ratio.
-      test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
-      data_.append(piece);
-    }
-    pos_ = 0;
-  }
-
-  Slice Generate(int len) {
-    if (pos_ + len > data_.size()) {
-      pos_ = 0;
-      assert(len < data_.size());
-    }
-    pos_ += len;
-    return Slice(data_.data() + pos_ - len, len);
-  }
-};
-
-static Slice TrimSpace(Slice s) {
-  int start = 0;
-  while (start < s.size() && isspace(s[start])) {
-    start++;
-  }
-  int limit = s.size();
-  while (limit > start && isspace(s[limit-1])) {
-    limit--;
-  }
-  return Slice(s.data() + start, limit - start);
-}
-
-}  // namespace
-
-class Benchmark {
- private:
-  sqlite3* db_;
-  int db_num_;
-  int num_;
-  int reads_;
-  double start_;
-  double last_op_finish_;
-  int64_t bytes_;
-  std::string message_;
-  Histogram hist_;
-  RandomGenerator gen_;
-  Random rand_;
-
-  // State kept for progress messages
-  int done_;
-  int next_report_;     // When to report next
-
-  void PrintHeader() {
-    const int kKeySize = 16;
-    PrintEnvironment();
-    fprintf(stdout, "Keys:       %d bytes each\n", kKeySize);
-    fprintf(stdout, "Values:     %d bytes each\n", FLAGS_value_size);
-    fprintf(stdout, "Entries:    %d\n", num_);
-    fprintf(stdout, "RawSize:    %.1f MB (estimated)\n",
-            ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
-             / 1048576.0));
-    PrintWarnings();
-    fprintf(stdout, "------------------------------------------------\n");
-  }
-
-  void PrintWarnings() {
-#if defined(__GNUC__) && !defined(__OPTIMIZE__)
-    fprintf(stdout,
-            "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
-            );
-#endif
-#ifndef NDEBUG
-    fprintf(stdout,
-            "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
-#endif
-  }
-
-  void PrintEnvironment() {
-    fprintf(stderr, "SQLite:     version %s\n", SQLITE_VERSION);
-
-#if defined(__linux)
-    time_t now = time(NULL);
-    fprintf(stderr, "Date:       %s", ctime(&now));  // ctime() adds newline
-
-    FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
-    if (cpuinfo != NULL) {
-      char line[1000];
-      int num_cpus = 0;
-      std::string cpu_type;
-      std::string cache_size;
-      while (fgets(line, sizeof(line), cpuinfo) != NULL) {
-        const char* sep = strchr(line, ':');
-        if (sep == NULL) {
-          continue;
-        }
-        Slice key = TrimSpace(Slice(line, sep - 1 - line));
-        Slice val = TrimSpace(Slice(sep + 1));
-        if (key == "model name") {
-          ++num_cpus;
-          cpu_type = val.ToString();
-        } else if (key == "cache size") {
-          cache_size = val.ToString();
-        }
-      }
-      fclose(cpuinfo);
-      fprintf(stderr, "CPU:        %d * %s\n", num_cpus, cpu_type.c_str());
-      fprintf(stderr, "CPUCache:   %s\n", cache_size.c_str());
-    }
-#endif
-  }
-
-  void Start() {
-    start_ = Env::Default()->NowMicros() * 1e-6;
-    bytes_ = 0;
-    message_.clear();
-    last_op_finish_ = start_;
-    hist_.Clear();
-    done_ = 0;
-    next_report_ = 100;
-  }
-
-  void FinishedSingleOp() {
-    if (FLAGS_histogram) {
-      double now = Env::Default()->NowMicros() * 1e-6;
-      double micros = (now - last_op_finish_) * 1e6;
-      hist_.Add(micros);
-      if (micros > 20000) {
-        fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
-        fflush(stderr);
-      }
-      last_op_finish_ = now;
-    }
-
-    done_++;
-    if (done_ >= next_report_) {
-      if      (next_report_ < 1000)   next_report_ += 100;
-      else if (next_report_ < 5000)   next_report_ += 500;
-      else if (next_report_ < 10000)  next_report_ += 1000;
-      else if (next_report_ < 50000)  next_report_ += 5000;
-      else if (next_report_ < 100000) next_report_ += 10000;
-      else if (next_report_ < 500000) next_report_ += 50000;
-      else                            next_report_ += 100000;
-      fprintf(stderr, "... finished %d ops%30s\r", done_, "");
-      fflush(stderr);
-    }
-  }
-
-  void Stop(const Slice& name) {
-    double finish = Env::Default()->NowMicros() * 1e-6;
-
-    // Pretend at least one op was done in case we are running a benchmark
-    // that does not call FinishedSingleOp().
-    if (done_ < 1) done_ = 1;
-
-    if (bytes_ > 0) {
-      char rate[100];
-      snprintf(rate, sizeof(rate), "%6.1f MB/s",
-               (bytes_ / 1048576.0) / (finish - start_));
-      if (!message_.empty()) {
-        message_  = std::string(rate) + " " + message_;
-      } else {
-        message_ = rate;
-      }
-    }
-
-    fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
-            name.ToString().c_str(),
-            (finish - start_) * 1e6 / done_,
-            (message_.empty() ? "" : " "),
-            message_.c_str());
-    if (FLAGS_histogram) {
-      fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
-    }
-    fflush(stdout);
-  }
-
- public:
-  enum Order {
-    SEQUENTIAL,
-    RANDOM
-  };
-  enum DBState {
-    FRESH,
-    EXISTING
-  };
-
-  Benchmark()
-  : db_(NULL),
-    db_num_(0),
-    num_(FLAGS_num),
-    reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
-    bytes_(0),
-    rand_(301) {
-    std::vector<std::string> files;
-    std::string test_dir;
-    Env::Default()->GetTestDirectory(&test_dir);
-    Env::Default()->GetChildren(test_dir, &files);
-    if (!FLAGS_use_existing_db) {
-      for (int i = 0; i < files.size(); i++) {
-        if (Slice(files[i]).starts_with("dbbench_sqlite3")) {
-          std::string file_name(test_dir);
-          file_name += "/";
-          file_name += files[i];
-          Env::Default()->DeleteFile(file_name.c_str());
-        }
-      }
-    }
-  }
-
-  ~Benchmark() {
-    int status = sqlite3_close(db_);
-    ErrorCheck(status);
-  }
-
-  void Run() {
-    PrintHeader();
-    Open();
-
-    const char* benchmarks = FLAGS_benchmarks;
-    while (benchmarks != NULL) {
-      const char* sep = strchr(benchmarks, ',');
-      Slice name;
-      if (sep == NULL) {
-        name = benchmarks;
-        benchmarks = NULL;
-      } else {
-        name = Slice(benchmarks, sep - benchmarks);
-        benchmarks = sep + 1;
-      }
-
-      bytes_ = 0;
-      Start();
-
-      bool known = true;
-      bool write_sync = false;
-      if (name == Slice("fillseq")) {
-        Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1);
-        WalCheckpoint(db_);
-      } else if (name == Slice("fillseqbatch")) {
-        Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1000);
-        WalCheckpoint(db_);
-      } else if (name == Slice("fillrandom")) {
-        Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1);
-        WalCheckpoint(db_);
-      } else if (name == Slice("fillrandbatch")) {
-        Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1000);
-        WalCheckpoint(db_);
-      } else if (name == Slice("overwrite")) {
-        Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1);
-        WalCheckpoint(db_);
-      } else if (name == Slice("overwritebatch")) {
-        Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1000);
-        WalCheckpoint(db_);
-      } else if (name == Slice("fillrandsync")) {
-        write_sync = true;
-        Write(write_sync, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1);
-        WalCheckpoint(db_);
-      } else if (name == Slice("fillseqsync")) {
-        write_sync = true;
-        Write(write_sync, SEQUENTIAL, FRESH, num_ / 100, FLAGS_value_size, 1);
-        WalCheckpoint(db_);
-      } else if (name == Slice("fillrand100K")) {
-        Write(write_sync, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1);
-        WalCheckpoint(db_);
-      } else if (name == Slice("fillseq100K")) {
-        Write(write_sync, SEQUENTIAL, FRESH, num_ / 1000, 100 * 1000, 1);
-        WalCheckpoint(db_);
-      } else if (name == Slice("readseq")) {
-        ReadSequential();
-      } else if (name == Slice("readrandom")) {
-        Read(RANDOM, 1);
-      } else if (name == Slice("readrand100K")) {
-        int n = reads_;
-        reads_ /= 1000;
-        Read(RANDOM, 1);
-        reads_ = n;
-      } else {
-        known = false;
-        if (name != Slice()) {  // No error message for empty name
-          fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
-        }
-      }
-      if (known) {
-        Stop(name);
-      }
-    }
-  }
-
-  void Open() {
-    assert(db_ == NULL);
-
-    int status;
-    char file_name[100];
-    char* err_msg = NULL;
-    db_num_++;
-
-    // Open database
-    std::string tmp_dir;
-    Env::Default()->GetTestDirectory(&tmp_dir);
-    snprintf(file_name, sizeof(file_name),
-             "%s/dbbench_sqlite3-%d.db",
-             tmp_dir.c_str(),
-             db_num_);
-    status = sqlite3_open(file_name, &db_);
-    if (status) {
-      fprintf(stderr, "open error: %s\n", sqlite3_errmsg(db_));
-      exit(1);
-    }
-
-    // Change SQLite cache size
-    char cache_size[100];
-    snprintf(cache_size, sizeof(cache_size), "PRAGMA cache_size = %d",
-             FLAGS_num_pages);
-    status = sqlite3_exec(db_, cache_size, NULL, NULL, &err_msg);
-    ExecErrorCheck(status, err_msg);
-
-    // FLAGS_page_size is defaulted to 1024
-    if (FLAGS_page_size != 1024) {
-      char page_size[100];
-      snprintf(page_size, sizeof(page_size), "PRAGMA page_size = %d",
-               FLAGS_page_size);
-      status = sqlite3_exec(db_, page_size, NULL, NULL, &err_msg);
-      ExecErrorCheck(status, err_msg);
-    }
-
-    // Change journal mode to WAL if WAL enabled flag is on
-    if (FLAGS_WAL_enabled) {
-      std::string WAL_stmt = "PRAGMA journal_mode = WAL";
-
-      // LevelDB's default cache size is a combined 4 MB
-      std::string WAL_checkpoint = "PRAGMA wal_autocheckpoint = 4096";
-      status = sqlite3_exec(db_, WAL_stmt.c_str(), NULL, NULL, &err_msg);
-      ExecErrorCheck(status, err_msg);
-      status = sqlite3_exec(db_, WAL_checkpoint.c_str(), NULL, NULL, &err_msg);
-      ExecErrorCheck(status, err_msg);
-    }
-
-    // Change locking mode to exclusive and create tables/index for database
-    std::string locking_stmt = "PRAGMA locking_mode = EXCLUSIVE";
-    std::string create_stmt =
-          "CREATE TABLE test (key blob, value blob, PRIMARY KEY(key))";
-    std::string stmt_array[] = { locking_stmt, create_stmt };
-    int stmt_array_length = sizeof(stmt_array) / sizeof(std::string);
-    for (int i = 0; i < stmt_array_length; i++) {
-      status = sqlite3_exec(db_, stmt_array[i].c_str(), NULL, NULL, &err_msg);
-      ExecErrorCheck(status, err_msg);
-    }
-  }
-
-  void Write(bool write_sync, Order order, DBState state,
-             int num_entries, int value_size, int entries_per_batch) {
-    // Create new database if state == FRESH
-    if (state == FRESH) {
-      if (FLAGS_use_existing_db) {
-        message_ = "skipping (--use_existing_db is true)";
-        return;
-      }
-      sqlite3_close(db_);
-      db_ = NULL;
-      Open();
-      Start();
-    }
-
-    if (num_entries != num_) {
-      char msg[100];
-      snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
-      message_ = msg;
-    }
-
-    char* err_msg = NULL;
-    int status;
-
-    sqlite3_stmt *replace_stmt, *begin_trans_stmt, *end_trans_stmt;
-    std::string replace_str = "REPLACE INTO test (key, value) VALUES (?, ?)";
-    std::string begin_trans_str = "BEGIN TRANSACTION;";
-    std::string end_trans_str = "END TRANSACTION;";
-
-    // Check for synchronous flag in options
-    std::string sync_stmt = (write_sync) ? "PRAGMA synchronous = FULL" :
-                                           "PRAGMA synchronous = OFF";
-    status = sqlite3_exec(db_, sync_stmt.c_str(), NULL, NULL, &err_msg);
-    ExecErrorCheck(status, err_msg);
-
-    // Preparing sqlite3 statements
-    status = sqlite3_prepare_v2(db_, replace_str.c_str(), -1,
-                                &replace_stmt, NULL);
-    ErrorCheck(status);
-    status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
-                                &begin_trans_stmt, NULL);
-    ErrorCheck(status);
-    status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1,
-                                &end_trans_stmt, NULL);
-    ErrorCheck(status);
-
-    bool transaction = (entries_per_batch > 1);
-    for (int i = 0; i < num_entries; i += entries_per_batch) {
-      // Begin write transaction
-      if (FLAGS_transaction && transaction) {
-        status = sqlite3_step(begin_trans_stmt);
-        StepErrorCheck(status);
-        status = sqlite3_reset(begin_trans_stmt);
-        ErrorCheck(status);
-      }
-
-      // Create and execute SQL statements
-      for (int j = 0; j < entries_per_batch; j++) {
-        const char* value = gen_.Generate(value_size).data();
-
-        // Create values for key-value pair
-        const int k = (order == SEQUENTIAL) ? i + j :
-                      (rand_.Next() % num_entries);
-        char key[100];
-        snprintf(key, sizeof(key), "%016d", k);
-
-        // Bind KV values into replace_stmt
-        status = sqlite3_bind_blob(replace_stmt, 1, key, 16, SQLITE_STATIC);
-        ErrorCheck(status);
-        status = sqlite3_bind_blob(replace_stmt, 2, value,
-                                   value_size, SQLITE_STATIC);
-        ErrorCheck(status);
-
-        // Execute replace_stmt
-        bytes_ += value_size + strlen(key);
-        status = sqlite3_step(replace_stmt);
-        StepErrorCheck(status);
-
-        // Reset SQLite statement for another use
-        status = sqlite3_clear_bindings(replace_stmt);
-        ErrorCheck(status);
-        status = sqlite3_reset(replace_stmt);
-        ErrorCheck(status);
-
-        FinishedSingleOp();
-      }
-
-      // End write transaction
-      if (FLAGS_transaction && transaction) {
-        status = sqlite3_step(end_trans_stmt);
-        StepErrorCheck(status);
-        status = sqlite3_reset(end_trans_stmt);
-        ErrorCheck(status);
-      }
-    }
-
-    status = sqlite3_finalize(replace_stmt);
-    ErrorCheck(status);
-    status = sqlite3_finalize(begin_trans_stmt);
-    ErrorCheck(status);
-    status = sqlite3_finalize(end_trans_stmt);
-    ErrorCheck(status);
-  }
-
-  void Read(Order order, int entries_per_batch) {
-    int status;
-    sqlite3_stmt *read_stmt, *begin_trans_stmt, *end_trans_stmt;
-
-    std::string read_str = "SELECT * FROM test WHERE key = ?";
-    std::string begin_trans_str = "BEGIN TRANSACTION;";
-    std::string end_trans_str = "END TRANSACTION;";
-
-    // Preparing sqlite3 statements
-    status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
-                                &begin_trans_stmt, NULL);
-    ErrorCheck(status);
-    status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1,
-                                &end_trans_stmt, NULL);
-    ErrorCheck(status);
-    status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &read_stmt, NULL);
-    ErrorCheck(status);
-
-    bool transaction = (entries_per_batch > 1);
-    for (int i = 0; i < reads_; i += entries_per_batch) {
-      // Begin read transaction
-      if (FLAGS_transaction && transaction) {
-        status = sqlite3_step(begin_trans_stmt);
-        StepErrorCheck(status);
-        status = sqlite3_reset(begin_trans_stmt);
-        ErrorCheck(status);
-      }
-
-      // Create and execute SQL statements
-      for (int j = 0; j < entries_per_batch; j++) {
-        // Create key value
-        char key[100];
-        int k = (order == SEQUENTIAL) ? i + j : (rand_.Next() % reads_);
-        snprintf(key, sizeof(key), "%016d", k);
-
-        // Bind key value into read_stmt
-        status = sqlite3_bind_blob(read_stmt, 1, key, 16, SQLITE_STATIC);
-        ErrorCheck(status);
-
-        // Execute read statement
-        while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {}
-        StepErrorCheck(status);
-
-        // Reset SQLite statement for another use
-        status = sqlite3_clear_bindings(read_stmt);
-        ErrorCheck(status);
-        status = sqlite3_reset(read_stmt);
-        ErrorCheck(status);
-        FinishedSingleOp();
-      }
-
-      // End read transaction
-      if (FLAGS_transaction && transaction) {
-        status = sqlite3_step(end_trans_stmt);
-        StepErrorCheck(status);
-        status = sqlite3_reset(end_trans_stmt);
-        ErrorCheck(status);
-      }
-    }
-
-    status = sqlite3_finalize(read_stmt);
-    ErrorCheck(status);
-    status = sqlite3_finalize(begin_trans_stmt);
-    ErrorCheck(status);
-    status = sqlite3_finalize(end_trans_stmt);
-    ErrorCheck(status);
-  }
-
-  void ReadSequential() {
-    int status;
-    sqlite3_stmt *pStmt;
-    std::string read_str = "SELECT * FROM test ORDER BY key";
-
-    status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &pStmt, NULL);
-    ErrorCheck(status);
-    for (int i = 0; i < reads_ && SQLITE_ROW == sqlite3_step(pStmt); i++) {
-      bytes_ += sqlite3_column_bytes(pStmt, 1) + sqlite3_column_bytes(pStmt, 2);
-      FinishedSingleOp();
-    }
-
-    status = sqlite3_finalize(pStmt);
-    ErrorCheck(status);
-  }
-
-};
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  std::string default_db_path;
-  for (int i = 1; i < argc; i++) {
-    double d;
-    int n;
-    char junk;
-    if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
-      FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
-    } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
-               (n == 0 || n == 1)) {
-      FLAGS_histogram = n;
-    } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
-      FLAGS_compression_ratio = d;
-    } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
-               (n == 0 || n == 1)) {
-      FLAGS_use_existing_db = n;
-    } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
-      FLAGS_num = n;
-    } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
-      FLAGS_reads = n;
-    } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
-      FLAGS_value_size = n;
-    } else if (leveldb::Slice(argv[i]) == leveldb::Slice("--no_transaction")) {
-      FLAGS_transaction = false;
-    } else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) {
-      FLAGS_page_size = n;
-    } else if (sscanf(argv[i], "--num_pages=%d%c", &n, &junk) == 1) {
-      FLAGS_num_pages = n;
-    } else if (sscanf(argv[i], "--WAL_enabled=%d%c", &n, &junk) == 1 &&
-               (n == 0 || n == 1)) {
-      FLAGS_WAL_enabled = n;
-    } else if (strncmp(argv[i], "--db=", 5) == 0) {
-      FLAGS_db = argv[i] + 5;
-    } else {
-      fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
-      exit(1);
-    }
-  }
-
-  // Choose a location for the test database if none given with --db=<path>
-  if (FLAGS_db == NULL) {
-      leveldb::Env::Default()->GetTestDirectory(&default_db_path);
-      default_db_path += "/dbbench";
-      FLAGS_db = default_db_path.c_str();
-  }
-
-  leveldb::Benchmark benchmark;
-  benchmark.Run();
-  return 0;
-}

src/leveldb/doc/bench/db_bench_tree_db.cc

@@ -1,528 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <kcpolydb.h>
-#include "util/histogram.h"
-#include "util/random.h"
-#include "util/testutil.h"
-
-// Comma-separated list of operations to run in the specified order
-//   Actual benchmarks:
-//
-//   fillseq       -- write N values in sequential key order in async mode
-//   fillrandom    -- write N values in random key order in async mode
-//   overwrite     -- overwrite N values in random key order in async mode
-//   fillseqsync   -- write N/100 values in sequential key order in sync mode
-//   fillrandsync  -- write N/100 values in random key order in sync mode
-//   fillrand100K  -- write N/1000 100K values in random order in async mode
-//   fillseq100K   -- write N/1000 100K values in seq order in async mode
-//   readseq       -- read N times sequentially
-//   readseq100K   -- read N/1000 100K values in sequential order in async mode
-//   readrand100K  -- read N/1000 100K values in sequential order in async mode
-//   readrandom    -- read N times in random order
-static const char* FLAGS_benchmarks =
-    "fillseq,"
-    "fillseqsync,"
-    "fillrandsync,"
-    "fillrandom,"
-    "overwrite,"
-    "readrandom,"
-    "readseq,"
-    "fillrand100K,"
-    "fillseq100K,"
-    "readseq100K,"
-    "readrand100K,"
-    ;
-
-// Number of key/values to place in database
-static int FLAGS_num = 1000000;
-
-// Number of read operations to do.  If negative, do FLAGS_num reads.
-static int FLAGS_reads = -1;
-
-// Size of each value
-static int FLAGS_value_size = 100;
-
-// Arrange to generate values that shrink to this fraction of
-// their original size after compression
-static double FLAGS_compression_ratio = 0.5;
-
-// Print histogram of operation timings
-static bool FLAGS_histogram = false;
-
-// Cache size. Default 4 MB
-static int FLAGS_cache_size = 4194304;
-
-// Page size. Default 1 KB
-static int FLAGS_page_size = 1024;
-
-// If true, do not destroy the existing database.  If you set this
-// flag and also specify a benchmark that wants a fresh database, that
-// benchmark will fail.
-static bool FLAGS_use_existing_db = false;
-
-// Compression flag. If true, compression is on. If false, compression
-// is off.
-static bool FLAGS_compression = true;
-
-// Use the db with the following name.
-static const char* FLAGS_db = NULL;
-
-inline
-static void DBSynchronize(kyotocabinet::TreeDB* db_)
-{
-  // Synchronize will flush writes to disk
-  if (!db_->synchronize()) {
-    fprintf(stderr, "synchronize error: %s\n", db_->error().name());
-  }
-}
-
-namespace leveldb {
-
-// Helper for quickly generating random data.
-namespace {
-class RandomGenerator {
- private:
-  std::string data_;
-  int pos_;
-
- public:
-  RandomGenerator() {
-    // We use a limited amount of data over and over again and ensure
-    // that it is larger than the compression window (32KB), and also
-    // large enough to serve all typical value sizes we want to write.
-    Random rnd(301);
-    std::string piece;
-    while (data_.size() < 1048576) {
-      // Add a short fragment that is as compressible as specified
-      // by FLAGS_compression_ratio.
-      test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
-      data_.append(piece);
-    }
-    pos_ = 0;
-  }
-
-  Slice Generate(int len) {
-    if (pos_ + len > data_.size()) {
-      pos_ = 0;
-      assert(len < data_.size());
-    }
-    pos_ += len;
-    return Slice(data_.data() + pos_ - len, len);
-  }
-};
-
-static Slice TrimSpace(Slice s) {
-  int start = 0;
-  while (start < s.size() && isspace(s[start])) {
-    start++;
-  }
-  int limit = s.size();
-  while (limit > start && isspace(s[limit-1])) {
-    limit--;
-  }
-  return Slice(s.data() + start, limit - start);
-}
-
-}  // namespace
-
-class Benchmark {
- private:
-  kyotocabinet::TreeDB* db_;
-  int db_num_;
-  int num_;
-  int reads_;
-  double start_;
-  double last_op_finish_;
-  int64_t bytes_;
-  std::string message_;
-  Histogram hist_;
-  RandomGenerator gen_;
-  Random rand_;
-  kyotocabinet::LZOCompressor<kyotocabinet::LZO::RAW> comp_;
-
-  // State kept for progress messages
-  int done_;
-  int next_report_;     // When to report next
-
-  void PrintHeader() {
-    const int kKeySize = 16;
-    PrintEnvironment();
-    fprintf(stdout, "Keys:       %d bytes each\n", kKeySize);
-    fprintf(stdout, "Values:     %d bytes each (%d bytes after compression)\n",
-            FLAGS_value_size,
-            static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
-    fprintf(stdout, "Entries:    %d\n", num_);
-    fprintf(stdout, "RawSize:    %.1f MB (estimated)\n",
-            ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
-             / 1048576.0));
-    fprintf(stdout, "FileSize:   %.1f MB (estimated)\n",
-            (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_)
-             / 1048576.0));
-    PrintWarnings();
-    fprintf(stdout, "------------------------------------------------\n");
-  }
-
-  void PrintWarnings() {
-#if defined(__GNUC__) && !defined(__OPTIMIZE__)
-    fprintf(stdout,
-            "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
-            );
-#endif
-#ifndef NDEBUG
-    fprintf(stdout,
-            "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
-#endif
-  }
-
-  void PrintEnvironment() {
-    fprintf(stderr, "Kyoto Cabinet:    version %s, lib ver %d, lib rev %d\n",
-            kyotocabinet::VERSION, kyotocabinet::LIBVER, kyotocabinet::LIBREV);
-
-#if defined(__linux)
-    time_t now = time(NULL);
-    fprintf(stderr, "Date:           %s", ctime(&now));  // ctime() adds newline
-
-    FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
-    if (cpuinfo != NULL) {
-      char line[1000];
-      int num_cpus = 0;
-      std::string cpu_type;
-      std::string cache_size;
-      while (fgets(line, sizeof(line), cpuinfo) != NULL) {
-        const char* sep = strchr(line, ':');
-        if (sep == NULL) {
-          continue;
-        }
-        Slice key = TrimSpace(Slice(line, sep - 1 - line));
-        Slice val = TrimSpace(Slice(sep + 1));
-        if (key == "model name") {
-          ++num_cpus;
-          cpu_type = val.ToString();
-        } else if (key == "cache size") {
-          cache_size = val.ToString();
-        }
-      }
-      fclose(cpuinfo);
-      fprintf(stderr, "CPU:            %d * %s\n", num_cpus, cpu_type.c_str());
-      fprintf(stderr, "CPUCache:       %s\n", cache_size.c_str());
-    }
-#endif
-  }
-
-  void Start() {
-    start_ = Env::Default()->NowMicros() * 1e-6;
-    bytes_ = 0;
-    message_.clear();
-    last_op_finish_ = start_;
-    hist_.Clear();
-    done_ = 0;
-    next_report_ = 100;
-  }
-
-  void FinishedSingleOp() {
-    if (FLAGS_histogram) {
-      double now = Env::Default()->NowMicros() * 1e-6;
-      double micros = (now - last_op_finish_) * 1e6;
-      hist_.Add(micros);
-      if (micros > 20000) {
-        fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
-        fflush(stderr);
-      }
-      last_op_finish_ = now;
-    }
-
-    done_++;
-    if (done_ >= next_report_) {
-      if      (next_report_ < 1000)   next_report_ += 100;
-      else if (next_report_ < 5000)   next_report_ += 500;
-      else if (next_report_ < 10000)  next_report_ += 1000;
-      else if (next_report_ < 50000)  next_report_ += 5000;
-      else if (next_report_ < 100000) next_report_ += 10000;
-      else if (next_report_ < 500000) next_report_ += 50000;
-      else                            next_report_ += 100000;
-      fprintf(stderr, "... finished %d ops%30s\r", done_, "");
-      fflush(stderr);
-    }
-  }
-
-  void Stop(const Slice& name) {
-    double finish = Env::Default()->NowMicros() * 1e-6;
-
-    // Pretend at least one op was done in case we are running a benchmark
-    // that does not call FinishedSingleOp().
-    if (done_ < 1) done_ = 1;
-
-    if (bytes_ > 0) {
-      char rate[100];
-      snprintf(rate, sizeof(rate), "%6.1f MB/s",
-               (bytes_ / 1048576.0) / (finish - start_));
-      if (!message_.empty()) {
-        message_  = std::string(rate) + " " + message_;
-      } else {
-        message_ = rate;
-      }
-    }
-
-    fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
-            name.ToString().c_str(),
-            (finish - start_) * 1e6 / done_,
-            (message_.empty() ? "" : " "),
-            message_.c_str());
-    if (FLAGS_histogram) {
-      fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
-    }
-    fflush(stdout);
-  }
-
- public:
-  enum Order {
-    SEQUENTIAL,
-    RANDOM
-  };
-  enum DBState {
-    FRESH,
-    EXISTING
-  };
-
-  Benchmark()
-  : db_(NULL),
-    num_(FLAGS_num),
-    reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
-    bytes_(0),
-    rand_(301) {
-    std::vector<std::string> files;
-    std::string test_dir;
-    Env::Default()->GetTestDirectory(&test_dir);
-    Env::Default()->GetChildren(test_dir.c_str(), &files);
-    if (!FLAGS_use_existing_db) {
-      for (int i = 0; i < files.size(); i++) {
-        if (Slice(files[i]).starts_with("dbbench_polyDB")) {
-          std::string file_name(test_dir);
-          file_name += "/";
-          file_name += files[i];
-          Env::Default()->DeleteFile(file_name.c_str());
-        }
-      }
-    }
-  }
-
-  ~Benchmark() {
-    if (!db_->close()) {
-      fprintf(stderr, "close error: %s\n", db_->error().name());
-    }
-  }
-
-  void Run() {
-    PrintHeader();
-    Open(false);
-
-    const char* benchmarks = FLAGS_benchmarks;
-    while (benchmarks != NULL) {
-      const char* sep = strchr(benchmarks, ',');
-      Slice name;
-      if (sep == NULL) {
-        name = benchmarks;
-        benchmarks = NULL;
-      } else {
-        name = Slice(benchmarks, sep - benchmarks);
-        benchmarks = sep + 1;
-      }
-
-      Start();
-
-      bool known = true;
-      bool write_sync = false;
-      if (name == Slice("fillseq")) {
-        Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1);
-        DBSynchronize(db_);
-      } else if (name == Slice("fillrandom")) {
-        Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1);
-        DBSynchronize(db_);
-      } else if (name == Slice("overwrite")) {
-        Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1);
-        DBSynchronize(db_);
-      } else if (name == Slice("fillrandsync")) {
-        write_sync = true;
-        Write(write_sync, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1);
-        DBSynchronize(db_);
-      } else if (name == Slice("fillseqsync")) {
-        write_sync = true;
-        Write(write_sync, SEQUENTIAL, FRESH, num_ / 100, FLAGS_value_size, 1);
-        DBSynchronize(db_);
-      } else if (name == Slice("fillrand100K")) {
-        Write(write_sync, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1);
-        DBSynchronize(db_);
-      } else if (name == Slice("fillseq100K")) {
-        Write(write_sync, SEQUENTIAL, FRESH, num_ / 1000, 100 * 1000, 1);
-        DBSynchronize(db_);
-      } else if (name == Slice("readseq")) {
-        ReadSequential();
-      } else if (name == Slice("readrandom")) {
-        ReadRandom();
-      } else if (name == Slice("readrand100K")) {
-        int n = reads_;
-        reads_ /= 1000;
-        ReadRandom();
-        reads_ = n;
-      } else if (name == Slice("readseq100K")) {
-        int n = reads_;
-        reads_ /= 1000;
-        ReadSequential();
-        reads_ = n;
-      } else {
-        known = false;
-        if (name != Slice()) {  // No error message for empty name
-          fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
-        }
-      }
-      if (known) {
-        Stop(name);
-      }
-    }
-  }
-
- private:
-    void Open(bool sync) {
-    assert(db_ == NULL);
-
-    // Initialize db_
-    db_ = new kyotocabinet::TreeDB();
-    char file_name[100];
-    db_num_++;
-    std::string test_dir;
-    Env::Default()->GetTestDirectory(&test_dir);
-    snprintf(file_name, sizeof(file_name),
-             "%s/dbbench_polyDB-%d.kct",
-             test_dir.c_str(),
-             db_num_);
-
-    // Create tuning options and open the database
-    int open_options = kyotocabinet::PolyDB::OWRITER |
-                       kyotocabinet::PolyDB::OCREATE;
-    int tune_options = kyotocabinet::TreeDB::TSMALL |
-        kyotocabinet::TreeDB::TLINEAR;
-    if (FLAGS_compression) {
-      tune_options |= kyotocabinet::TreeDB::TCOMPRESS;
-      db_->tune_compressor(&comp_);
-    }
-    db_->tune_options(tune_options);
-    db_->tune_page_cache(FLAGS_cache_size);
-    db_->tune_page(FLAGS_page_size);
-    db_->tune_map(256LL<<20);
-    if (sync) {
-      open_options |= kyotocabinet::PolyDB::OAUTOSYNC;
-    }
-    if (!db_->open(file_name, open_options)) {
-      fprintf(stderr, "open error: %s\n", db_->error().name());
-    }
-  }
-
-  void Write(bool sync, Order order, DBState state,
-             int num_entries, int value_size, int entries_per_batch) {
-    // Create new database if state == FRESH
-    if (state == FRESH) {
-      if (FLAGS_use_existing_db) {
-        message_ = "skipping (--use_existing_db is true)";
-        return;
-      }
-      delete db_;
-      db_ = NULL;
-      Open(sync);
-      Start();  // Do not count time taken to destroy/open
-    }
-
-    if (num_entries != num_) {
-      char msg[100];
-      snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
-      message_ = msg;
-    }
-
-    // Write to database
-    for (int i = 0; i < num_entries; i++)
-    {
-      const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % num_entries);
-      char key[100];
-      snprintf(key, sizeof(key), "%016d", k);
-      bytes_ += value_size + strlen(key);
-      std::string cpp_key = key;
-      if (!db_->set(cpp_key, gen_.Generate(value_size).ToString())) {
-        fprintf(stderr, "set error: %s\n", db_->error().name());
-      }
-      FinishedSingleOp();
-    }
-  }
-
-  void ReadSequential() {
-    kyotocabinet::DB::Cursor* cur = db_->cursor();
-    cur->jump();
-    std::string ckey, cvalue;
-    while (cur->get(&ckey, &cvalue, true)) {
-      bytes_ += ckey.size() + cvalue.size();
-      FinishedSingleOp();
-    }
-    delete cur;
-  }
-
-  void ReadRandom() {
-    std::string value;
-    for (int i = 0; i < reads_; i++) {
-      char key[100];
-      const int k = rand_.Next() % reads_;
-      snprintf(key, sizeof(key), "%016d", k);
-      db_->get(key, &value);
-      FinishedSingleOp();
-    }
-  }
-};
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  std::string default_db_path;
-  for (int i = 1; i < argc; i++) {
-    double d;
-    int n;
-    char junk;
-    if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
-      FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
-    } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
-      FLAGS_compression_ratio = d;
-    } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
-               (n == 0 || n == 1)) {
-      FLAGS_histogram = n;
-    } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
-      FLAGS_num = n;
-    } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
-      FLAGS_reads = n;
-    } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
-      FLAGS_value_size = n;
-    } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
-      FLAGS_cache_size = n;
-    } else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) {
-      FLAGS_page_size = n;
-    } else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1 &&
-               (n == 0 || n == 1)) {
-      FLAGS_compression = (n == 1) ? true : false;
-    } else if (strncmp(argv[i], "--db=", 5) == 0) {
-      FLAGS_db = argv[i] + 5;
-    } else {
-      fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
-      exit(1);
-    }
-  }
-
-  // Choose a location for the test database if none given with --db=<path>
-  if (FLAGS_db == NULL) {
-      leveldb::Env::Default()->GetTestDirectory(&default_db_path);
-      default_db_path += "/dbbench";
-      FLAGS_db = default_db_path.c_str();
-  }
-
-  leveldb::Benchmark benchmark;
-  benchmark.Run();
-  return 0;
-}

src/leveldb/doc/benchmark.html

@@ -1,459 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<title>LevelDB Benchmarks</title>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-<style>
-body {
-  font-family:Helvetica,sans-serif;
-  padding:20px;
-}
-
-h2 {
-  padding-top:30px;
-}
-
-table.bn {
-  width:800px;
-  border-collapse:collapse;
-  border:0;
-  padding:0;
-}
-
-table.bnbase {
-  width:650px;
-}
-
-table.bn td {
-  padding:2px 0;
-}
-
-table.bn td.c1 {
-  font-weight:bold;
-  width:150px;
-}
-
-table.bn td.c1 div.e {
-  float:right;
-  font-weight:normal;
-}
-
-table.bn td.c2 {
-  width:150px;
-  text-align:right;
-  padding:2px;
-}
-
-table.bn td.c3 {
-  width:350px;
-}
-
-table.bn td.c4 {
-  width:150px;
-  font-size:small;
-  padding-left:4px;
-}
-
-/* chart bars */
-div.bldb {
-  background-color:#0255df;
-}
-
-div.bkct {
-  background-color:#df5555;
-}
-
-div.bsql {
-  background-color:#aadf55;
-}
-
-.code {
-  font-family:monospace;
-  font-size:large;
-}
-
-.todo {
-  color: red;
-}
-
-</style>
-</head>
-<body>
-<h1>LevelDB Benchmarks</h1>
-<p>Google, July 2011</p>
-<hr>
-
-<p>In order to test LevelDB's performance, we benchmark it against other well-established database implementations. We compare LevelDB (revision 39) against <a href="http://www.sqlite.org/">SQLite3</a> (version 3.7.6.3) and <a href="http://fallabs.com/kyotocabinet/spex.html">Kyoto Cabinet's</a> (version 1.2.67) TreeDB (a B+Tree based key-value store). We would like to acknowledge Scott Hess and Mikio Hirabayashi for their suggestions and contributions to the SQLite3 and Kyoto Cabinet benchmarks, respectively.</p>
-
-<p>Benchmarks were all performed on a six-core Intel(R) Xeon(R) CPU X5650 @ 2.67GHz, with 12288 KB of total L3 cache and 12 GB of DDR3 RAM at 1333 MHz. (Note that LevelDB uses at most two CPUs since the benchmarks are single threaded: one to run the benchmark, and one for background compactions.) We ran the benchmarks on two machines (with identical processors), one with an Ext3 file system and one with an Ext4 file system. The machine with the Ext3 file system has a SATA Hitachi HDS721050CLA362 hard drive. The machine with the Ext4 file system has a SATA Samsung HD502HJ hard drive. Both hard drives spin at 7200 RPM and have hard drive write-caching enabled (using `hdparm -W 1 [device]`). The numbers reported below are the median of three measurements.</p>
-
-<h4>Benchmark Source Code</h4>
-<p>We wrote benchmark tools for SQLite and Kyoto TreeDB based on LevelDB's <span class="code">db_bench</span>. The code for each of the benchmarks resides here:</p>
-<ul>
-	<li> <b>LevelDB:</b> <a href="http://code.google.com/p/leveldb/source/browse/trunk/db/db_bench.cc">db/db_bench.cc</a>.</li>
-	<li> <b>SQLite:</b> <a href="http://code.google.com/p/leveldb/source/browse/#svn%2Ftrunk%2Fdoc%2Fbench%2Fdb_bench_sqlite3.cc">doc/bench/db_bench_sqlite3.cc</a>.</li>
-	<li> <b>Kyoto TreeDB:</b> <a href="http://code.google.com/p/leveldb/source/browse/#svn%2Ftrunk%2Fdoc%2Fbench%2Fdb_bench_tree_db.cc">doc/bench/db_bench_tree_db.cc</a>.</li>
-</ul>
-
-<h4>Custom Build Specifications</h4>
-<ul>
-<li>LevelDB: LevelDB was compiled with the <a href="http://code.google.com/p/google-perftools">tcmalloc</a> library and the <a href="http://code.google.com/p/snappy/">Snappy</a> compression library (revision 33).  Assertions were disabled.</li>
-<li>TreeDB: TreeDB was compiled using the <a href="http://www.oberhumer.com/opensource/lzo/">LZO</a> compression library (version 2.03). Furthermore, we enabled the TSMALL and TLINEAR options when opening the database in order to reduce the footprint of each record.</li>
-<li>SQLite: We tuned SQLite's performance, by setting its locking mode to exclusive.  We also enabled SQLite's <a href="http://www.sqlite.org/draft/wal.html">write-ahead logging</a>.</li>
-</ul>
-
-<h2>1. Baseline Performance</h2>
-<p>This section gives the baseline performance of all the
-databases.  Following sections show how performance changes as various
-parameters are varied.  For the baseline:</p>
-<ul>
-	<li> Each database is allowed 4 MB of cache memory.</li>
-        <li> Databases are opened in <em>asynchronous</em> write mode.
-             (LevelDB's sync option, TreeDB's OAUTOSYNC option, and
-             SQLite3's synchronous options are all turned off).  I.e.,
-             every write is pushed to the operating system, but the
-             benchmark does not wait for the write to reach the disk.</li>
-	<li> Keys are 16 bytes each.</li>
-        <li> Value are 100 bytes each (with enough redundancy so that
-             a simple compressor shrinks them to 50% of their original
-             size).</li>
-	<li> Sequential reads/writes traverse the key space in increasing order.</li>
-	<li> Random reads/writes traverse the key space in random order.</li>
-</ul>
-
-<h3>A. Sequential Reads</h3>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">4,030,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">1,010,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:95px">&nbsp;</div></td>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">383,000 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:33px">&nbsp;</div></td>
-</table>
-<h3>B. Random Reads</h3>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">129,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:298px">&nbsp;</div></td>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">151,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:350px">&nbsp;</div></td>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">134,000 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:310px">&nbsp;</div></td>
-</table>
-<h3>C. Sequential Writes</h3>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">779,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">342,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:154px">&nbsp;</div></td>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">48,600 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:22px">&nbsp;</div></td>
-</table>
-<h3>D. Random Writes</h3>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">164,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">88,500 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:188px">&nbsp;</div></td>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">9,860 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:21px">&nbsp;</div></td>
-</table>
-
-<p>LevelDB outperforms both SQLite3 and TreeDB in sequential and random write operations and sequential read operations. Kyoto Cabinet has the fastest random read operations.</p>
-
-<h2>2. Write Performance under Different Configurations</h2>
-<h3>A. Large Values </h3>
-<p>For this benchmark, we start with an empty database, and write 100,000 byte values (~50% compressible). To keep the benchmark running time reasonable, we stop after writing 1000 values.</p>
-<h4>Sequential Writes</h4>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">1,100 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:234px">&nbsp;</div></td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">1,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:224px">&nbsp;</div></td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">1,600 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:350px">&nbsp;</div></td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">480 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:105px">&nbsp;</div></td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">1,100 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:240px">&nbsp;</div></td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">1,600 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:350px">&nbsp;</div></td></tr>
-</table>
-<p>LevelDB doesn't perform as well with large values of 100,000 bytes each. This is because LevelDB writes keys and values at least twice: first time to the transaction log, and second time (during a compaction) to a sorted file.
-With larger values, LevelDB's per-operation efficiency is swamped by the
-cost of extra copies of large values.</p>
-<h3>B. Batch Writes</h3>
-<p>A batch write is a set of writes that are applied atomically to the underlying database. A single batch of N writes may be significantly faster than N individual writes. The following benchmark writes one thousand batches where each batch contains one thousand 100-byte values. TreeDB does not support batch writes and is omitted from this benchmark.</p>
-<h4>Sequential Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">840,000 entries/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(1.08x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">124,000 entries/sec</td>
-    <td class="c3"><div class="bsql" style="width:52px">&nbsp;</div></td>
-    <td class="c4">(2.55x baseline)</td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">221,000 entries/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(1.35x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">22,000 entries/sec</td>
-    <td class="c3"><div class="bsql" style="width:34px">&nbsp;</div></td>
-    <td class="c4">(2.23x baseline)</td></tr>
-</table>
-
-<p>Because of the way LevelDB persistent storage is organized, batches of
-random writes are not much slower (only a factor of 4x) than batches
-of sequential writes.</p>
-
-<h3>C. Synchronous Writes</h3>
-<p>In the following benchmark, we enable the synchronous writing modes
-of all of the databases.  Since this change significantly slows down the
-benchmark, we stop after 10,000 writes. For synchronous write tests, we've
-disabled hard drive write-caching (using `hdparm -W 0 [device]`).</p>
-<ul>
-    <li>For LevelDB, we set WriteOptions.sync = true.</li>
-    <li>In TreeDB, we enabled TreeDB's OAUTOSYNC option.</li>
-    <li>For SQLite3, we set "PRAGMA synchronous = FULL".</li>
-</ul>
-<h4>Sequential Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">100 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(0.003x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">7 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:27px">&nbsp;</div></td>
-    <td class="c4">(0.0004x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">88 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:315px">&nbsp;</div></td>
-    <td class="c4">(0.002x baseline)</td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">100 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(0.015x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">8 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:29px">&nbsp;</div></td>
-    <td class="c4">(0.001x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">88 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:314px">&nbsp;</div></td>
-    <td class="c4">(0.009x baseline)</td></tr>
-</table>
-
-<p>Also see the <code>ext4</code> performance numbers below
-since synchronous writes behave significantly differently
-on <code>ext3</code> and <code>ext4</code>.</p>
-
-<h3>D. Turning Compression Off</h3>
-
-<p>In the baseline measurements, LevelDB and TreeDB were using
-light-weight compression
-(<a href="http://code.google.com/p/snappy/">Snappy</a> for LevelDB,
-and <a href="http://www.oberhumer.com/opensource/lzo/">LZO</a> for
-TreeDB). SQLite3, by default does not use compression.  The
-experiments below show what happens when compression is disabled in
-all of the databases (the SQLite3 numbers are just a copy of
-its baseline measurements):</p>
-
-<h4>Sequential Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">594,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(0.76x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">485,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:239px">&nbsp;</div></td>
-    <td class="c4">(1.42x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">48,600 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:29px">&nbsp;</div></td>
-    <td class="c4">(1.00x baseline)</td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">135,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:296px">&nbsp;</div></td>
-    <td class="c4">(0.82x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">159,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(1.80x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">9,860 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:22px">&nbsp;</div></td>
-    <td class="c4">(1.00x baseline)</td></tr>
-</table>
-
-<p>LevelDB's write performance is better with compression than without
-since compression decreases the amount of data that has to be written
-to disk.  Therefore LevelDB users can leave compression enabled in
-most scenarios without having worry about a tradeoff between space
-usage and performance.  TreeDB's performance on the other hand is
-better without compression than with compression.  Presumably this is
-because TreeDB's compression library (LZO) is more expensive than
-LevelDB's compression library (Snappy).<p>
-
-<h3>E. Using More Memory</h3>
-<p>We increased the overall cache size for each database to 128 MB. For LevelDB, we partitioned 128 MB into a 120 MB write buffer and 8 MB of cache (up from 2 MB of write buffer and 2 MB of cache). For SQLite3, we kept the page size at 1024 bytes, but increased the number of pages to 131,072 (up from 4096). For TreeDB, we also kept the page size at 1024 bytes, but increased the cache size to 128 MB (up from 4 MB).</p>
-<h4>Sequential Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">812,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(1.04x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">321,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:138px">&nbsp;</div></td>
-    <td class="c4">(0.94x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">48,500 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:21px">&nbsp;</div></td>
-    <td class="c4">(1.00x baseline)</td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">355,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(2.16x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">284,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:280px">&nbsp;</div></td>
-    <td class="c4">(3.21x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">9,670 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:10px">&nbsp;</div></td>
-    <td class="c4">(0.98x baseline)</td></tr>
-</table>
-
-<p>SQLite's performance does not change substantially when compared to
-the baseline, but the random write performance for both LevelDB and
-TreeDB increases significantly.  LevelDB's performance improves
-because a larger write buffer reduces the need to merge sorted files
-(since it creates a smaller number of larger sorted files).  TreeDB's
-performance goes up because the entire database is available in memory
-for fast in-place updates.</p>
-
-  <h2>3. Read Performance under Different Configurations</h2>
-<h3>A. Larger Caches</h3>
-<p>We increased the overall memory usage to 128 MB for each database.
-For LevelDB, we allocated 8 MB to LevelDB's write buffer and 120 MB
-to LevelDB's cache. The other databases don't differentiate between a
-write buffer and a cache, so we simply set their cache size to 128
-MB.</p>
-<h4>Sequential Reads</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">5,210,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(1.29x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">1,070,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:72px">&nbsp;</div></td>
-    <td class="c4">(1.06x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">609,000 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:41px">&nbsp;</div></td>
-    <td class="c4">(1.59x baseline)</td></tr>
-</table>
-
-<h4>Random Reads</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">190,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:144px">&nbsp;</div></td>
-    <td class="c4">(1.47x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">463,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(3.07x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">186,000 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:141px">&nbsp;</div></td>
-    <td class="c4">(1.39x baseline)</td></tr>
-</table>
-
-<p>As expected, the read performance of all of the databases increases
-when the caches are enlarged.  In particular, TreeDB seems to make
-very effective use of a cache that is large enough to hold the entire
-database.</p>
-
-<h3>B. No Compression Reads </h3>
-<p>For this benchmark, we populated a database with 1 million entries consisting of 16 byte keys and 100 byte values. We compiled LevelDB and Kyoto Cabinet without compression support, so results that are read out from the database are already uncompressed. We've listed the SQLite3 baseline read performance as a point of comparison.</p>
-<h4>Sequential Reads</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">4,880,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(1.21x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">1,230,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:88px">&nbsp;</div></td>
-    <td class="c4">(3.60x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">383,000 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:27px">&nbsp;</div></td>
-    <td class="c4">(1.00x baseline)</td></tr>
-</table>
-<h4>Random Reads</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
-    <td class="c2">149,000 ops/sec</td>
-    <td class="c3"><div class="bldb" style="width:300px">&nbsp;</div></td>
-    <td class="c4">(1.16x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
-    <td class="c2">175,000 ops/sec</td>
-    <td class="c3"><div class="bkct" style="width:350px">&nbsp;</div></td>
-    <td class="c4">(1.16x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
-    <td class="c2">134,000 ops/sec</td>
-    <td class="c3"><div class="bsql" style="width:268px">&nbsp;</div></td>
-    <td class="c4">(1.00x baseline)</td></tr>
-</table>
-
-<p>Performance of both LevelDB and TreeDB improves a small amount when
-compression is disabled.  Note however that under different workloads,
-performance may very well be better with compression if it allows more
-of the working set to fit in memory.</p>
-
-<h2>Note about Ext4 Filesystems</h2>
-<p>The preceding numbers are for an ext3 file system. Synchronous writes are much slower under <a href="http://en.wikipedia.org/wiki/Ext4">ext4</a> (LevelDB drops to ~31 writes / second and TreeDB drops to ~5 writes / second; SQLite3's synchronous writes do not noticeably drop) due to ext4's different handling of <span class="code">fsync</span> / <span class="code">msync</span> calls. Even LevelDB's asynchronous write performance drops somewhat since it spreads its storage across multiple files and issues <span class="code">fsync</span> calls when switching to a new file.</p>
-
-<h2>Acknowledgements</h2>
-<p>Jeff Dean and Sanjay Ghemawat wrote LevelDB. Kevin Tseng wrote and compiled these benchmarks. Mikio Hirabayashi, Scott Hess, and Gabor Cselle provided help and advice.</p>
-</body>
-</html>

src/leveldb/doc/impl.md

@@ -1,170 +0,0 @@
-## Files
-
-The implementation of leveldb is similar in spirit to the representation of a
-single [Bigtable tablet (section 5.3)](http://research.google.com/archive/bigtable.html).
-However the organization of the files that make up the representation is
-somewhat different and is explained below.
-
-Each database is represented by a set of files stored in a directory. There are
-several different types of files as documented below:
-
-### Log files
-
-A log file (*.log) stores a sequence of recent updates. Each update is appended
-to the current log file. When the log file reaches a pre-determined size
-(approximately 4MB by default), it is converted to a sorted table (see below)
-and a new log file is created for future updates.
-
-A copy of the current log file is kept in an in-memory structure (the
-`memtable`). This copy is consulted on every read so that read operations
-reflect all logged updates.
-
-## Sorted tables
-
-A sorted table (*.ldb) stores a sequence of entries sorted by key. Each entry is
-either a value for the key, or a deletion marker for the key. (Deletion markers
-are kept around to hide obsolete values present in older sorted tables).
-
-The set of sorted tables are organized into a sequence of levels. The sorted
-table generated from a log file is placed in a special **young** level (also
-called level-0). When the number of young files exceeds a certain threshold
-(currently four), all of the young files are merged together with all of the
-overlapping level-1 files to produce a sequence of new level-1 files (we create
-a new level-1 file for every 2MB of data.)
-
-Files in the young level may contain overlapping keys. However files in other
-levels have distinct non-overlapping key ranges. Consider level number L where
-L >= 1. When the combined size of files in level-L exceeds (10^L) MB (i.e., 10MB
-for level-1, 100MB for level-2, ...), one file in level-L, and all of the
-overlapping files in level-(L+1) are merged to form a set of new files for
-level-(L+1). These merges have the effect of gradually migrating new updates
-from the young level to the largest level using only bulk reads and writes
-(i.e., minimizing expensive seeks).
-
-### Manifest
-
-A MANIFEST file lists the set of sorted tables that make up each level, the
-corresponding key ranges, and other important metadata. A new MANIFEST file
-(with a new number embedded in the file name) is created whenever the database
-is reopened. The MANIFEST file is formatted as a log, and changes made to the
-serving state (as files are added or removed) are appended to this log.
-
-### Current
-
-CURRENT is a simple text file that contains the name of the latest MANIFEST
-file.
-
-### Info logs
-
-Informational messages are printed to files named LOG and LOG.old.
-
-### Others
-
-Other files used for miscellaneous purposes may also be present (LOCK, *.dbtmp).
-
-## Level 0
-
-When the log file grows above a certain size (1MB by default):
-Create a brand new memtable and log file and direct future updates here
-In the background:
-Write the contents of the previous memtable to an sstable
-Discard the memtable
-Delete the old log file and the old memtable
-Add the new sstable to the young (level-0) level.
-
-## Compactions
-
-When the size of level L exceeds its limit, we compact it in a background
-thread. The compaction picks a file from level L and all overlapping files from
-the next level L+1. Note that if a level-L file overlaps only part of a
-level-(L+1) file, the entire file at level-(L+1) is used as an input to the
-compaction and will be discarded after the compaction.  Aside: because level-0
-is special (files in it may overlap each other), we treat compactions from
-level-0 to level-1 specially: a level-0 compaction may pick more than one
-level-0 file in case some of these files overlap each other.
-
-A compaction merges the contents of the picked files to produce a sequence of
-level-(L+1) files. We switch to producing a new level-(L+1) file after the
-current output file has reached the target file size (2MB). We also switch to a
-new output file when the key range of the current output file has grown enough
-to overlap more than ten level-(L+2) files.  This last rule ensures that a later
-compaction of a level-(L+1) file will not pick up too much data from
-level-(L+2).
-
-The old files are discarded and the new files are added to the serving state.
-
-Compactions for a particular level rotate through the key space. In more detail,
-for each level L, we remember the ending key of the last compaction at level L.
-The next compaction for level L will pick the first file that starts after this
-key (wrapping around to the beginning of the key space if there is no such
-file).
-
-Compactions drop overwritten values. They also drop deletion markers if there
-are no higher numbered levels that contain a file whose range overlaps the
-current key.
-
-### Timing
-
-Level-0 compactions will read up to four 1MB files from level-0, and at worst
-all the level-1 files (10MB). I.e., we will read 14MB and write 14MB.
-
-Other than the special level-0 compactions, we will pick one 2MB file from level
-L. In the worst case, this will overlap ~ 12 files from level L+1 (10 because
-level-(L+1) is ten times the size of level-L, and another two at the boundaries
-since the file ranges at level-L will usually not be aligned with the file
-ranges at level-L+1). The compaction will therefore read 26MB and write 26MB.
-Assuming a disk IO rate of 100MB/s (ballpark range for modern drives), the worst
-compaction cost will be approximately 0.5 second.
-
-If we throttle the background writing to something small, say 10% of the full
-100MB/s speed, a compaction may take up to 5 seconds. If the user is writing at
-10MB/s, we might build up lots of level-0 files (~50 to hold the 5*10MB). This
-may significantly increase the cost of reads due to the overhead of merging more
-files together on every read.
-
-Solution 1: To reduce this problem, we might want to increase the log switching
-threshold when the number of level-0 files is large. Though the downside is that
-the larger this threshold, the more memory we will need to hold the
-corresponding memtable.
-
-Solution 2: We might want to decrease write rate artificially when the number of
-level-0 files goes up.
-
-Solution 3: We work on reducing the cost of very wide merges. Perhaps most of
-the level-0 files will have their blocks sitting uncompressed in the cache and
-we will only need to worry about the O(N) complexity in the merging iterator.
-
-### Number of files
-
-Instead of always making 2MB files, we could make larger files for larger levels
-to reduce the total file count, though at the expense of more bursty
-compactions.  Alternatively, we could shard the set of files into multiple
-directories.
-
-An experiment on an ext3 filesystem on Feb 04, 2011 shows the following timings
-to do 100K file opens in directories with varying number of files:
-
-
-| Files in directory | Microseconds to open a file |
-|-------------------:|----------------------------:|
-|               1000 |                           9 |
-|              10000 |                          10 |
-|             100000 |                          16 |
-
-So maybe even the sharding is not necessary on modern filesystems?
-
-## Recovery
-
-* Read CURRENT to find name of the latest committed MANIFEST
-* Read the named MANIFEST file
-* Clean up stale files
-* We could open all sstables here, but it is probably better to be lazy...
-* Convert log chunk to a new level-0 sstable
-* Start directing new writes to a new log file with recovered sequence#
-
-## Garbage collection of files
-
-`DeleteObsoleteFiles()` is called at the end of every compaction and at the end
-of recovery. It finds the names of all files in the database. It deletes all log
-files that are not the current log file. It deletes all table files that are not
-referenced from some level and are not the output of an active compaction.

src/leveldb/doc/index.md

@@ -1,523 +0,0 @@
-leveldb
-=======
-
-_Jeff Dean, Sanjay Ghemawat_
-
-The leveldb library provides a persistent key value store. Keys and values are
-arbitrary byte arrays.  The keys are ordered within the key value store
-according to a user-specified comparator function.
-
-## Opening A Database
-
-A leveldb database has a name which corresponds to a file system directory. All
-of the contents of database are stored in this directory. The following example
-shows how to open a database, creating it if necessary:
-
-```c++
-#include <cassert>
-#include "leveldb/db.h"
-
-leveldb::DB* db;
-leveldb::Options options;
-options.create_if_missing = true;
-leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
-assert(status.ok());
-...
-```
-
-If you want to raise an error if the database already exists, add the following
-line before the `leveldb::DB::Open` call:
-
-```c++
-options.error_if_exists = true;
-```
-
-## Status
-
-You may have noticed the `leveldb::Status` type above. Values of this type are
-returned by most functions in leveldb that may encounter an error. You can check
-if such a result is ok, and also print an associated error message:
-
-```c++
-leveldb::Status s = ...;
-if (!s.ok()) cerr << s.ToString() << endl;
-```
-
-## Closing A Database
-
-When you are done with a database, just delete the database object. Example:
-
-```c++
-... open the db as described above ...
-... do something with db ...
-delete db;
-```
-
-## Reads And Writes
-
-The database provides Put, Delete, and Get methods to modify/query the database.
-For example, the following code moves the value stored under key1 to key2.
-
-```c++
-std::string value;
-leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
-if (s.ok()) s = db->Put(leveldb::WriteOptions(), key2, value);
-if (s.ok()) s = db->Delete(leveldb::WriteOptions(), key1);
-```
-
-## Atomic Updates
-
-Note that if the process dies after the Put of key2 but before the delete of
-key1, the same value may be left stored under multiple keys. Such problems can
-be avoided by using the `WriteBatch` class to atomically apply a set of updates:
-
-```c++
-#include "leveldb/write_batch.h"
-...
-std::string value;
-leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
-if (s.ok()) {
-  leveldb::WriteBatch batch;
-  batch.Delete(key1);
-  batch.Put(key2, value);
-  s = db->Write(leveldb::WriteOptions(), &batch);
-}
-```
-
-The `WriteBatch` holds a sequence of edits to be made to the database, and these
-edits within the batch are applied in order. Note that we called Delete before
-Put so that if key1 is identical to key2, we do not end up erroneously dropping
-the value entirely.
-
-Apart from its atomicity benefits, `WriteBatch` may also be used to speed up
-bulk updates by placing lots of individual mutations into the same batch.
-
-## Synchronous Writes
-
-By default, each write to leveldb is asynchronous: it returns after pushing the
-write from the process into the operating system. The transfer from operating
-system memory to the underlying persistent storage happens asynchronously. The
-sync flag can be turned on for a particular write to make the write operation
-not return until the data being written has been pushed all the way to
-persistent storage. (On Posix systems, this is implemented by calling either
-`fsync(...)` or `fdatasync(...)` or `msync(..., MS_SYNC)` before the write
-operation returns.)
-
-```c++
-leveldb::WriteOptions write_options;
-write_options.sync = true;
-db->Put(write_options, ...);
-```
-
-Asynchronous writes are often more than a thousand times as fast as synchronous
-writes. The downside of asynchronous writes is that a crash of the machine may
-cause the last few updates to be lost. Note that a crash of just the writing
-process (i.e., not a reboot) will not cause any loss since even when sync is
-false, an update is pushed from the process memory into the operating system
-before it is considered done.
-
-Asynchronous writes can often be used safely. For example, when loading a large
-amount of data into the database you can handle lost updates by restarting the
-bulk load after a crash. A hybrid scheme is also possible where every Nth write
-is synchronous, and in the event of a crash, the bulk load is restarted just
-after the last synchronous write finished by the previous run. (The synchronous
-write can update a marker that describes where to restart on a crash.)
-
-`WriteBatch` provides an alternative to asynchronous writes. Multiple updates
-may be placed in the same WriteBatch and applied together using a synchronous
-write (i.e., `write_options.sync` is set to true). The extra cost of the
-synchronous write will be amortized across all of the writes in the batch.
-
-## Concurrency
-
-A database may only be opened by one process at a time. The leveldb
-implementation acquires a lock from the operating system to prevent misuse.
-Within a single process, the same `leveldb::DB` object may be safely shared by
-multiple concurrent threads. I.e., different threads may write into or fetch
-iterators or call Get on the same database without any external synchronization
-(the leveldb implementation will automatically do the required synchronization).
-However other objects (like Iterator and `WriteBatch`) may require external
-synchronization. If two threads share such an object, they must protect access
-to it using their own locking protocol. More details are available in the public
-header files.
-
-## Iteration
-
-The following example demonstrates how to print all key,value pairs in a
-database.
-
-```c++
-leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions());
-for (it->SeekToFirst(); it->Valid(); it->Next()) {
-  cout << it->key().ToString() << ": "  << it->value().ToString() << endl;
-}
-assert(it->status().ok());  // Check for any errors found during the scan
-delete it;
-```
-
-The following variation shows how to process just the keys in the range
-[start,limit):
-
-```c++
-for (it->Seek(start);
-   it->Valid() && it->key().ToString() < limit;
-   it->Next()) {
-  ...
-}
-```
-
-You can also process entries in reverse order. (Caveat: reverse iteration may be
-somewhat slower than forward iteration.)
-
-```c++
-for (it->SeekToLast(); it->Valid(); it->Prev()) {
-  ...
-}
-```
-
-## Snapshots
-
-Snapshots provide consistent read-only views over the entire state of the
-key-value store.  `ReadOptions::snapshot` may be non-NULL to indicate that a
-read should operate on a particular version of the DB state. If
-`ReadOptions::snapshot` is NULL, the read will operate on an implicit snapshot
-of the current state.
-
-Snapshots are created by the `DB::GetSnapshot()` method:
-
-```c++
-leveldb::ReadOptions options;
-options.snapshot = db->GetSnapshot();
-... apply some updates to db ...
-leveldb::Iterator* iter = db->NewIterator(options);
-... read using iter to view the state when the snapshot was created ...
-delete iter;
-db->ReleaseSnapshot(options.snapshot);
-```
-
-Note that when a snapshot is no longer needed, it should be released using the
-`DB::ReleaseSnapshot` interface. This allows the implementation to get rid of
-state that was being maintained just to support reading as of that snapshot.
-
-## Slice
-
-The return value of the `it->key()` and `it->value()` calls above are instances
-of the `leveldb::Slice` type. Slice is a simple structure that contains a length
-and a pointer to an external byte array. Returning a Slice is a cheaper
-alternative to returning a `std::string` since we do not need to copy
-potentially large keys and values. In addition, leveldb methods do not return
-null-terminated C-style strings since leveldb keys and values are allowed to
-contain `'\0'` bytes.
-
-C++ strings and null-terminated C-style strings can be easily converted to a
-Slice:
-
-```c++
-leveldb::Slice s1 = "hello";
-
-std::string str("world");
-leveldb::Slice s2 = str;
-```
-
-A Slice can be easily converted back to a C++ string:
-
-```c++
-std::string str = s1.ToString();
-assert(str == std::string("hello"));
-```
-
-Be careful when using Slices since it is up to the caller to ensure that the
-external byte array into which the Slice points remains live while the Slice is
-in use. For example, the following is buggy:
-
-```c++
-leveldb::Slice slice;
-if (...) {
-  std::string str = ...;
-  slice = str;
-}
-Use(slice);
-```
-
-When the if statement goes out of scope, str will be destroyed and the backing
-storage for slice will disappear.
-
-## Comparators
-
-The preceding examples used the default ordering function for key, which orders
-bytes lexicographically. You can however supply a custom comparator when opening
-a database.  For example, suppose each database key consists of two numbers and
-we should sort by the first number, breaking ties by the second number. First,
-define a proper subclass of `leveldb::Comparator` that expresses these rules:
-
-```c++
-class TwoPartComparator : public leveldb::Comparator {
- public:
-  // Three-way comparison function:
-  //   if a < b: negative result
-  //   if a > b: positive result
-  //   else: zero result
-  int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const {
-    int a1, a2, b1, b2;
-    ParseKey(a, &a1, &a2);
-    ParseKey(b, &b1, &b2);
-    if (a1 < b1) return -1;
-    if (a1 > b1) return +1;
-    if (a2 < b2) return -1;
-    if (a2 > b2) return +1;
-    return 0;
-  }
-
-  // Ignore the following methods for now:
-  const char* Name() const { return "TwoPartComparator"; }
-  void FindShortestSeparator(std::string*, const leveldb::Slice&) const {}
-  void FindShortSuccessor(std::string*) const {}
-};
-```
-
-Now create a database using this custom comparator:
-
-```c++
-TwoPartComparator cmp;
-leveldb::DB* db;
-leveldb::Options options;
-options.create_if_missing = true;
-options.comparator = &cmp;
-leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
-...
-```
-
-### Backwards compatibility
-
-The result of the comparator's Name method is attached to the database when it
-is created, and is checked on every subsequent database open. If the name
-changes, the `leveldb::DB::Open` call will fail. Therefore, change the name if
-and only if the new key format and comparison function are incompatible with
-existing databases, and it is ok to discard the contents of all existing
-databases.
-
-You can however still gradually evolve your key format over time with a little
-bit of pre-planning. For example, you could store a version number at the end of
-each key (one byte should suffice for most uses). When you wish to switch to a
-new key format (e.g., adding an optional third part to the keys processed by
-`TwoPartComparator`), (a) keep the same comparator name (b) increment the
-version number for new keys (c) change the comparator function so it uses the
-version numbers found in the keys to decide how to interpret them.
-
-## Performance
-
-Performance can be tuned by changing the default values of the types defined in
-`include/leveldb/options.h`.
-
-### Block size
-
-leveldb groups adjacent keys together into the same block and such a block is
-the unit of transfer to and from persistent storage. The default block size is
-approximately 4096 uncompressed bytes.  Applications that mostly do bulk scans
-over the contents of the database may wish to increase this size. Applications
-that do a lot of point reads of small values may wish to switch to a smaller
-block size if performance measurements indicate an improvement. There isn't much
-benefit in using blocks smaller than one kilobyte, or larger than a few
-megabytes. Also note that compression will be more effective with larger block
-sizes.
-
-### Compression
-
-Each block is individually compressed before being written to persistent
-storage. Compression is on by default since the default compression method is
-very fast, and is automatically disabled for uncompressible data. In rare cases,
-applications may want to disable compression entirely, but should only do so if
-benchmarks show a performance improvement:
-
-```c++
-leveldb::Options options;
-options.compression = leveldb::kNoCompression;
-... leveldb::DB::Open(options, name, ...) ....
-```
-
-### Cache
-
-The contents of the database are stored in a set of files in the filesystem and
-each file stores a sequence of compressed blocks. If options.cache is non-NULL,
-it is used to cache frequently used uncompressed block contents.
-
-```c++
-#include "leveldb/cache.h"
-
-leveldb::Options options;
-options.cache = leveldb::NewLRUCache(100 * 1048576);  // 100MB cache
-leveldb::DB* db;
-leveldb::DB::Open(options, name, &db);
-... use the db ...
-delete db
-delete options.cache;
-```
-
-Note that the cache holds uncompressed data, and therefore it should be sized
-according to application level data sizes, without any reduction from
-compression. (Caching of compressed blocks is left to the operating system
-buffer cache, or any custom Env implementation provided by the client.)
-
-When performing a bulk read, the application may wish to disable caching so that
-the data processed by the bulk read does not end up displacing most of the
-cached contents. A per-iterator option can be used to achieve this:
-
-```c++
-leveldb::ReadOptions options;
-options.fill_cache = false;
-leveldb::Iterator* it = db->NewIterator(options);
-for (it->SeekToFirst(); it->Valid(); it->Next()) {
-  ...
-}
-```
-
-### Key Layout
-
-Note that the unit of disk transfer and caching is a block. Adjacent keys
-(according to the database sort order) will usually be placed in the same block.
-Therefore the application can improve its performance by placing keys that are
-accessed together near each other and placing infrequently used keys in a
-separate region of the key space.
-
-For example, suppose we are implementing a simple file system on top of leveldb.
-The types of entries we might wish to store are:
-
-    filename -> permission-bits, length, list of file_block_ids
-    file_block_id -> data
-
-We might want to prefix filename keys with one letter (say '/') and the
-`file_block_id` keys with a different letter (say '0') so that scans over just
-the metadata do not force us to fetch and cache bulky file contents.
-
-### Filters
-
-Because of the way leveldb data is organized on disk, a single `Get()` call may
-involve multiple reads from disk. The optional FilterPolicy mechanism can be
-used to reduce the number of disk reads substantially.
-
-```c++
-leveldb::Options options;
-options.filter_policy = NewBloomFilterPolicy(10);
-leveldb::DB* db;
-leveldb::DB::Open(options, "/tmp/testdb", &db);
-... use the database ...
-delete db;
-delete options.filter_policy;
-```
-
-The preceding code associates a Bloom filter based filtering policy with the
-database.  Bloom filter based filtering relies on keeping some number of bits of
-data in memory per key (in this case 10 bits per key since that is the argument
-we passed to `NewBloomFilterPolicy`). This filter will reduce the number of
-unnecessary disk reads needed for Get() calls by a factor of approximately
-a 100. Increasing the bits per key will lead to a larger reduction at the cost
-of more memory usage. We recommend that applications whose working set does not
-fit in memory and that do a lot of random reads set a filter policy.
-
-If you are using a custom comparator, you should ensure that the filter policy
-you are using is compatible with your comparator. For example, consider a
-comparator that ignores trailing spaces when comparing keys.
-`NewBloomFilterPolicy` must not be used with such a comparator. Instead, the
-application should provide a custom filter policy that also ignores trailing
-spaces. For example:
-
-```c++
-class CustomFilterPolicy : public leveldb::FilterPolicy {
- private:
-  FilterPolicy* builtin_policy_;
-
- public:
-  CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) {}
-  ~CustomFilterPolicy() { delete builtin_policy_; }
-
-  const char* Name() const { return "IgnoreTrailingSpacesFilter"; }
-
-  void CreateFilter(const Slice* keys, int n, std::string* dst) const {
-    // Use builtin bloom filter code after removing trailing spaces
-    std::vector<Slice> trimmed(n);
-    for (int i = 0; i < n; i++) {
-      trimmed[i] = RemoveTrailingSpaces(keys[i]);
-    }
-    return builtin_policy_->CreateFilter(&trimmed[i], n, dst);
-  }
-};
-```
-
-Advanced applications may provide a filter policy that does not use a bloom
-filter but uses some other mechanism for summarizing a set of keys. See
-`leveldb/filter_policy.h` for detail.
-
-## Checksums
-
-leveldb associates checksums with all data it stores in the file system. There
-are two separate controls provided over how aggressively these checksums are
-verified:
-
-`ReadOptions::verify_checksums` may be set to true to force checksum
-verification of all data that is read from the file system on behalf of a
-particular read.  By default, no such verification is done.
-
-`Options::paranoid_checks` may be set to true before opening a database to make
-the database implementation raise an error as soon as it detects an internal
-corruption. Depending on which portion of the database has been corrupted, the
-error may be raised when the database is opened, or later by another database
-operation. By default, paranoid checking is off so that the database can be used
-even if parts of its persistent storage have been corrupted.
-
-If a database is corrupted (perhaps it cannot be opened when paranoid checking
-is turned on), the `leveldb::RepairDB` function may be used to recover as much
-of the data as possible
-
-## Approximate Sizes
-
-The `GetApproximateSizes` method can used to get the approximate number of bytes
-of file system space used by one or more key ranges.
-
-```c++
-leveldb::Range ranges[2];
-ranges[0] = leveldb::Range("a", "c");
-ranges[1] = leveldb::Range("x", "z");
-uint64_t sizes[2];
-leveldb::Status s = db->GetApproximateSizes(ranges, 2, sizes);
-```
-
-The preceding call will set `sizes[0]` to the approximate number of bytes of
-file system space used by the key range `[a..c)` and `sizes[1]` to the
-approximate number of bytes used by the key range `[x..z)`.
-
-## Environment
-
-All file operations (and other operating system calls) issued by the leveldb
-implementation are routed through a `leveldb::Env` object. Sophisticated clients
-may wish to provide their own Env implementation to get better control.
-For example, an application may introduce artificial delays in the file IO
-paths to limit the impact of leveldb on other activities in the system.
-
-```c++
-class SlowEnv : public leveldb::Env {
-  ... implementation of the Env interface ...
-};
-
-SlowEnv env;
-leveldb::Options options;
-options.env = &env;
-Status s = leveldb::DB::Open(options, ...);
-```
-
-## Porting
-
-leveldb may be ported to a new platform by providing platform specific
-implementations of the types/methods/functions exported by
-`leveldb/port/port.h`.  See `leveldb/port/port_example.h` for more details.
-
-In addition, the new platform may need a new default `leveldb::Env`
-implementation.  See `leveldb/util/env_posix.h` for an example.
-
-## Other Information
-
-Details about the leveldb implementation may be found in the following
-documents:
-
-1. [Implementation notes](impl.md)
-2. [Format of an immutable Table file](table_format.md)
-3. [Format of a log file](log_format.md)

src/leveldb/doc/log_format.md

@@ -1,75 +0,0 @@
-leveldb Log format
-==================
-The log file contents are a sequence of 32KB blocks.  The only exception is that
-the tail of the file may contain a partial block.
-
-Each block consists of a sequence of records:
-
-    block := record* trailer?
-    record :=
-      checksum: uint32     // crc32c of type and data[] ; little-endian
-      length: uint16       // little-endian
-      type: uint8          // One of FULL, FIRST, MIDDLE, LAST
-      data: uint8[length]
-
-A record never starts within the last six bytes of a block (since it won't fit).
-Any leftover bytes here form the trailer, which must consist entirely of zero
-bytes and must be skipped by readers.
-
-Aside: if exactly seven bytes are left in the current block, and a new non-zero
-length record is added, the writer must emit a FIRST record (which contains zero
-bytes of user data) to fill up the trailing seven bytes of the block and then
-emit all of the user data in subsequent blocks.
-
-More types may be added in the future.  Some Readers may skip record types they
-do not understand, others may report that some data was skipped.
-
-    FULL == 1
-    FIRST == 2
-    MIDDLE == 3
-    LAST == 4
-
-The FULL record contains the contents of an entire user record.
-
-FIRST, MIDDLE, LAST are types used for user records that have been split into
-multiple fragments (typically because of block boundaries).  FIRST is the type
-of the first fragment of a user record, LAST is the type of the last fragment of
-a user record, and MIDDLE is the type of all interior fragments of a user
-record.
-
-Example: consider a sequence of user records:
-
-    A: length 1000
-    B: length 97270
-    C: length 8000
-
-**A** will be stored as a FULL record in the first block.
-
-**B** will be split into three fragments: first fragment occupies the rest of
-the first block, second fragment occupies the entirety of the second block, and
-the third fragment occupies a prefix of the third block.  This will leave six
-bytes free in the third block, which will be left empty as the trailer.
-
-**C** will be stored as a FULL record in the fourth block.
-
-----
-
-## Some benefits over the recordio format:
-
-1. We do not need any heuristics for resyncing - just go to next block boundary
-   and scan.  If there is a corruption, skip to the next block.  As a
-   side-benefit, we do not get confused when part of the contents of one log
-   file are embedded as a record inside another log file.
-
-2. Splitting at approximate boundaries (e.g., for mapreduce) is simple: find the
-   next block boundary and skip records until we hit a FULL or FIRST record.
-
-3. We do not need extra buffering for large records.
-
-## Some downsides compared to recordio format:
-
-1. No packing of tiny records.  This could be fixed by adding a new record type,
-   so it is a shortcoming of the current implementation, not necessarily the
-   format.
-
-2. No compression.  Again, this could be fixed by adding new record types.

src/leveldb/doc/table_format.md

@@ -1,107 +0,0 @@
-leveldb File format
-===================
-
-    <beginning_of_file>
-    [data block 1]
-    [data block 2]
-    ...
-    [data block N]
-    [meta block 1]
-    ...
-    [meta block K]
-    [metaindex block]
-    [index block]
-    [Footer]        (fixed size; starts at file_size - sizeof(Footer))
-    <end_of_file>
-
-The file contains internal pointers.  Each such pointer is called
-a BlockHandle and contains the following information:
-
-    offset:   varint64
-    size:     varint64
-
-See [varints](https://developers.google.com/protocol-buffers/docs/encoding#varints)
-for an explanation of varint64 format.
-
-1.  The sequence of key/value pairs in the file are stored in sorted
-order and partitioned into a sequence of data blocks.  These blocks
-come one after another at the beginning of the file.  Each data block
-is formatted according to the code in `block_builder.cc`, and then
-optionally compressed.
-
-2. After the data blocks we store a bunch of meta blocks.  The
-supported meta block types are described below.  More meta block types
-may be added in the future.  Each meta block is again formatted using
-`block_builder.cc` and then optionally compressed.
-
-3. A "metaindex" block.  It contains one entry for every other meta
-block where the key is the name of the meta block and the value is a
-BlockHandle pointing to that meta block.
-
-4. An "index" block.  This block contains one entry per data block,
-where the key is a string >= last key in that data block and before
-the first key in the successive data block.  The value is the
-BlockHandle for the data block.
-
-5. At the very end of the file is a fixed length footer that contains
-the BlockHandle of the metaindex and index blocks as well as a magic number.
-
-        metaindex_handle: char[p];     // Block handle for metaindex
-        index_handle:     char[q];     // Block handle for index
-        padding:          char[40-p-q];// zeroed bytes to make fixed length
-                                       // (40==2*BlockHandle::kMaxEncodedLength)
-        magic:            fixed64;     // == 0xdb4775248b80fb57 (little-endian)
-
-## "filter" Meta Block
-
-If a `FilterPolicy` was specified when the database was opened, a
-filter block is stored in each table.  The "metaindex" block contains
-an entry that maps from `filter.<N>` to the BlockHandle for the filter
-block where `<N>` is the string returned by the filter policy's
-`Name()` method.
-
-The filter block stores a sequence of filters, where filter i contains
-the output of `FilterPolicy::CreateFilter()` on all keys that are stored
-in a block whose file offset falls within the range
-
-    [ i*base ... (i+1)*base-1 ]
-
-Currently, "base" is 2KB.  So for example, if blocks X and Y start in
-the range `[ 0KB .. 2KB-1 ]`, all of the keys in X and Y will be
-converted to a filter by calling `FilterPolicy::CreateFilter()`, and the
-resulting filter will be stored as the first filter in the filter
-block.
-
-The filter block is formatted as follows:
-
-    [filter 0]
-    [filter 1]
-    [filter 2]
-    ...
-    [filter N-1]
-
-    [offset of filter 0]                  : 4 bytes
-    [offset of filter 1]                  : 4 bytes
-    [offset of filter 2]                  : 4 bytes
-    ...
-    [offset of filter N-1]                : 4 bytes
-
-    [offset of beginning of offset array] : 4 bytes
-    lg(base)                              : 1 byte
-
-The offset array at the end of the filter block allows efficient
-mapping from a data block offset to the corresponding filter.
-
-## "stats" Meta Block
-
-This meta block contains a bunch of stats.  The key is the name
-of the statistic.  The value contains the statistic.
-
-TODO(postrelease): record following stats.
-
-    data size
-    index size
-    key size (uncompressed)
-    value size (uncompressed)
-    number of entries
-    number of data blocks

src/leveldb/helpers/memenv/memenv.cc

@@ -1,401 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "helpers/memenv/memenv.h"
-
-#include "leveldb/env.h"
-#include "leveldb/status.h"
-#include "port/port.h"
-#include "util/mutexlock.h"
-#include <map>
-#include <string.h>
-#include <string>
-#include <vector>
-
-namespace leveldb {
-
-namespace {
-
-class FileState {
- public:
-  // FileStates are reference counted. The initial reference count is zero
-  // and the caller must call Ref() at least once.
-  FileState() : refs_(0), size_(0) {}
-
-  // Increase the reference count.
-  void Ref() {
-    MutexLock lock(&refs_mutex_);
-    ++refs_;
-  }
-
-  // Decrease the reference count. Delete if this is the last reference.
-  void Unref() {
-    bool do_delete = false;
-
-    {
-      MutexLock lock(&refs_mutex_);
-      --refs_;
-      assert(refs_ >= 0);
-      if (refs_ <= 0) {
-        do_delete = true;
-      }
-    }
-
-    if (do_delete) {
-      delete this;
-    }
-  }
-
-  uint64_t Size() const { return size_; }
-
-  Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
-    if (offset > size_) {
-      return Status::IOError("Offset greater than file size.");
-    }
-    const uint64_t available = size_ - offset;
-    if (n > available) {
-      n = static_cast<size_t>(available);
-    }
-    if (n == 0) {
-      *result = Slice();
-      return Status::OK();
-    }
-
-    assert(offset / kBlockSize <= SIZE_MAX);
-    size_t block = static_cast<size_t>(offset / kBlockSize);
-    size_t block_offset = offset % kBlockSize;
-
-    if (n <= kBlockSize - block_offset) {
-      // The requested bytes are all in the first block.
-      *result = Slice(blocks_[block] + block_offset, n);
-      return Status::OK();
-    }
-
-    size_t bytes_to_copy = n;
-    char* dst = scratch;
-
-    while (bytes_to_copy > 0) {
-      size_t avail = kBlockSize - block_offset;
-      if (avail > bytes_to_copy) {
-        avail = bytes_to_copy;
-      }
-      memcpy(dst, blocks_[block] + block_offset, avail);
-
-      bytes_to_copy -= avail;
-      dst += avail;
-      block++;
-      block_offset = 0;
-    }
-
-    *result = Slice(scratch, n);
-    return Status::OK();
-  }
-
-  Status Append(const Slice& data) {
-    const char* src = data.data();
-    size_t src_len = data.size();
-
-    while (src_len > 0) {
-      size_t avail;
-      size_t offset = size_ % kBlockSize;
-
-      if (offset != 0) {
-        // There is some room in the last block.
-        avail = kBlockSize - offset;
-      } else {
-        // No room in the last block; push new one.
-        blocks_.push_back(new char[kBlockSize]);
-        avail = kBlockSize;
-      }
-
-      if (avail > src_len) {
-        avail = src_len;
-      }
-      memcpy(blocks_.back() + offset, src, avail);
-      src_len -= avail;
-      src += avail;
-      size_ += avail;
-    }
-
-    return Status::OK();
-  }
-
- private:
-  // Private since only Unref() should be used to delete it.
-  ~FileState() {
-    for (std::vector<char*>::iterator i = blocks_.begin(); i != blocks_.end();
-         ++i) {
-      delete [] *i;
-    }
-  }
-
-  // No copying allowed.
-  FileState(const FileState&);
-  void operator=(const FileState&);
-
-  port::Mutex refs_mutex_;
-  int refs_;  // Protected by refs_mutex_;
-
-  // The following fields are not protected by any mutex. They are only mutable
-  // while the file is being written, and concurrent access is not allowed
-  // to writable files.
-  std::vector<char*> blocks_;
-  uint64_t size_;
-
-  enum { kBlockSize = 8 * 1024 };
-};
-
-class SequentialFileImpl : public SequentialFile {
- public:
-  explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) {
-    file_->Ref();
-  }
-
-  ~SequentialFileImpl() {
-    file_->Unref();
-  }
-
-  virtual Status Read(size_t n, Slice* result, char* scratch) {
-    Status s = file_->Read(pos_, n, result, scratch);
-    if (s.ok()) {
-      pos_ += result->size();
-    }
-    return s;
-  }
-
-  virtual Status Skip(uint64_t n) {
-    if (pos_ > file_->Size()) {
-      return Status::IOError("pos_ > file_->Size()");
-    }
-    const uint64_t available = file_->Size() - pos_;
-    if (n > available) {
-      n = available;
-    }
-    pos_ += n;
-    return Status::OK();
-  }
-
-  virtual std::string GetName() const { return "[memenv]"; }
- private:
-  FileState* file_;
-  uint64_t pos_;
-};
-
-class RandomAccessFileImpl : public RandomAccessFile {
- public:
-  explicit RandomAccessFileImpl(FileState* file) : file_(file) {
-    file_->Ref();
-  }
-
-  ~RandomAccessFileImpl() {
-    file_->Unref();
-  }
-
-  virtual Status Read(uint64_t offset, size_t n, Slice* result,
-                      char* scratch) const {
-    return file_->Read(offset, n, result, scratch);
-  }
-
-  virtual std::string GetName() const { return "[memenv]"; }
- private:
-  FileState* file_;
-};
-
-class WritableFileImpl : public WritableFile {
- public:
-  WritableFileImpl(FileState* file) : file_(file) {
-    file_->Ref();
-  }
-
-  ~WritableFileImpl() {
-    file_->Unref();
-  }
-
-  virtual Status Append(const Slice& data) {
-    return file_->Append(data);
-  }
-
-  virtual Status Close() { return Status::OK(); }
-  virtual Status Flush() { return Status::OK(); }
-  virtual Status Sync() { return Status::OK(); }
-
-  virtual std::string GetName() const { return "[memenv]"; }
- private:
-  FileState* file_;
-};
-
-class NoOpLogger : public Logger {
- public:
-  virtual void Logv(const char* format, va_list ap) { }
-};
-
-class InMemoryEnv : public EnvWrapper {
- public:
-  explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
-
-  virtual ~InMemoryEnv() {
-    for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
-      i->second->Unref();
-    }
-  }
-
-  // Partial implementation of the Env interface.
-  virtual Status NewSequentialFile(const std::string& fname,
-                                   SequentialFile** result) {
-    MutexLock lock(&mutex_);
-    if (file_map_.find(fname) == file_map_.end()) {
-      *result = NULL;
-      return Status::IOError(fname, "File not found");
-    }
-
-    *result = new SequentialFileImpl(file_map_[fname]);
-    return Status::OK();
-  }
-
-  virtual Status NewRandomAccessFile(const std::string& fname,
-                                     RandomAccessFile** result) {
-    MutexLock lock(&mutex_);
-    if (file_map_.find(fname) == file_map_.end()) {
-      *result = NULL;
-      return Status::IOError(fname, "File not found");
-    }
-
-    *result = new RandomAccessFileImpl(file_map_[fname]);
-    return Status::OK();
-  }
-
-  virtual Status NewWritableFile(const std::string& fname,
-                                 WritableFile** result) {
-    MutexLock lock(&mutex_);
-    if (file_map_.find(fname) != file_map_.end()) {
-      DeleteFileInternal(fname);
-    }
-
-    FileState* file = new FileState();
-    file->Ref();
-    file_map_[fname] = file;
-
-    *result = new WritableFileImpl(file);
-    return Status::OK();
-  }
-
-  virtual Status NewAppendableFile(const std::string& fname,
-                                   WritableFile** result) {
-    MutexLock lock(&mutex_);
-    FileState** sptr = &file_map_[fname];
-    FileState* file = *sptr;
-    if (file == NULL) {
-      file = new FileState();
-      file->Ref();
-    }
-    *result = new WritableFileImpl(file);
-    return Status::OK();
-  }
-
-  virtual bool FileExists(const std::string& fname) {
-    MutexLock lock(&mutex_);
-    return file_map_.find(fname) != file_map_.end();
-  }
-
-  virtual Status GetChildren(const std::string& dir,
-                             std::vector<std::string>* result) {
-    MutexLock lock(&mutex_);
-    result->clear();
-
-    for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
-      const std::string& filename = i->first;
-
-      if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' &&
-          Slice(filename).starts_with(Slice(dir))) {
-        result->push_back(filename.substr(dir.size() + 1));
-      }
-    }
-
-    return Status::OK();
-  }
-
-  void DeleteFileInternal(const std::string& fname) {
-    if (file_map_.find(fname) == file_map_.end()) {
-      return;
-    }
-
-    file_map_[fname]->Unref();
-    file_map_.erase(fname);
-  }
-
-  virtual Status DeleteFile(const std::string& fname) {
-    MutexLock lock(&mutex_);
-    if (file_map_.find(fname) == file_map_.end()) {
-      return Status::IOError(fname, "File not found");
-    }
-
-    DeleteFileInternal(fname);
-    return Status::OK();
-  }
-
-  virtual Status CreateDir(const std::string& dirname) {
-    return Status::OK();
-  }
-
-  virtual Status DeleteDir(const std::string& dirname) {
-    return Status::OK();
-  }
-
-  virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) {
-    MutexLock lock(&mutex_);
-    if (file_map_.find(fname) == file_map_.end()) {
-      return Status::IOError(fname, "File not found");
-    }
-
-    *file_size = file_map_[fname]->Size();
-    return Status::OK();
-  }
-
-  virtual Status RenameFile(const std::string& src,
-                            const std::string& target) {
-    MutexLock lock(&mutex_);
-    if (file_map_.find(src) == file_map_.end()) {
-      return Status::IOError(src, "File not found");
-    }
-
-    DeleteFileInternal(target);
-    file_map_[target] = file_map_[src];
-    file_map_.erase(src);
-    return Status::OK();
-  }
-
-  virtual Status LockFile(const std::string& fname, FileLock** lock) {
-    *lock = new FileLock;
-    return Status::OK();
-  }
-
-  virtual Status UnlockFile(FileLock* lock) {
-    delete lock;
-    return Status::OK();
-  }
-
-  virtual Status GetTestDirectory(std::string* path) {
-    *path = "/test";
-    return Status::OK();
-  }
-
-  virtual Status NewLogger(const std::string& fname, Logger** result) {
-    *result = new NoOpLogger;
-    return Status::OK();
-  }
-
- private:
-  // Map from filenames to FileState objects, representing a simple file system.
-  typedef std::map<std::string, FileState*> FileSystem;
-  port::Mutex mutex_;
-  FileSystem file_map_;  // Protected by mutex_.
-};
-
-}  // namespace
-
-Env* NewMemEnv(Env* base_env) {
-  return new InMemoryEnv(base_env);
-}
-
-}  // namespace leveldb

src/leveldb/helpers/memenv/memenv.h

@@ -1,20 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
-#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
-
-namespace leveldb {
-
-class Env;
-
-// Returns a new environment that stores its data in memory and delegates
-// all non-file-storage tasks to base_env. The caller must delete the result
-// when it is no longer needed.
-// *base_env must remain live while the result is in use.
-Env* NewMemEnv(Env* base_env);
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_

src/leveldb/helpers/memenv/memenv_test.cc

@@ -1,241 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "helpers/memenv/memenv.h"
-
-#include "db/db_impl.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "util/testharness.h"
-#include <string>
-#include <vector>
-
-namespace leveldb {
-
-class MemEnvTest {
- public:
-  Env* env_;
-
-  MemEnvTest()
-      : env_(NewMemEnv(Env::Default())) {
-  }
-  ~MemEnvTest() {
-    delete env_;
-  }
-};
-
-TEST(MemEnvTest, Basics) {
-  uint64_t file_size;
-  WritableFile* writable_file;
-  std::vector<std::string> children;
-
-  ASSERT_OK(env_->CreateDir("/dir"));
-
-  // Check that the directory is empty.
-  ASSERT_TRUE(!env_->FileExists("/dir/non_existent"));
-  ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok());
-  ASSERT_OK(env_->GetChildren("/dir", &children));
-  ASSERT_EQ(0, children.size());
-
-  // Create a file.
-  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
-  ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
-  ASSERT_EQ(0, file_size);
-  delete writable_file;
-
-  // Check that the file exists.
-  ASSERT_TRUE(env_->FileExists("/dir/f"));
-  ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
-  ASSERT_EQ(0, file_size);
-  ASSERT_OK(env_->GetChildren("/dir", &children));
-  ASSERT_EQ(1, children.size());
-  ASSERT_EQ("f", children[0]);
-
-  // Write to the file.
-  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
-  ASSERT_OK(writable_file->Append("abc"));
-  delete writable_file;
-
-  // Check that append works.
-  ASSERT_OK(env_->NewAppendableFile("/dir/f", &writable_file));
-  ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
-  ASSERT_EQ(3, file_size);
-  ASSERT_OK(writable_file->Append("hello"));
-  delete writable_file;
-
-  // Check for expected size.
-  ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
-  ASSERT_EQ(8, file_size);
-
-  // Check that renaming works.
-  ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok());
-  ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g"));
-  ASSERT_TRUE(!env_->FileExists("/dir/f"));
-  ASSERT_TRUE(env_->FileExists("/dir/g"));
-  ASSERT_OK(env_->GetFileSize("/dir/g", &file_size));
-  ASSERT_EQ(8, file_size);
-
-  // Check that opening non-existent file fails.
-  SequentialFile* seq_file;
-  RandomAccessFile* rand_file;
-  ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok());
-  ASSERT_TRUE(!seq_file);
-  ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok());
-  ASSERT_TRUE(!rand_file);
-
-  // Check that deleting works.
-  ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok());
-  ASSERT_OK(env_->DeleteFile("/dir/g"));
-  ASSERT_TRUE(!env_->FileExists("/dir/g"));
-  ASSERT_OK(env_->GetChildren("/dir", &children));
-  ASSERT_EQ(0, children.size());
-  ASSERT_OK(env_->DeleteDir("/dir"));
-}
-
-TEST(MemEnvTest, ReadWrite) {
-  WritableFile* writable_file;
-  SequentialFile* seq_file;
-  RandomAccessFile* rand_file;
-  Slice result;
-  char scratch[100];
-
-  ASSERT_OK(env_->CreateDir("/dir"));
-
-  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
-  ASSERT_OK(writable_file->Append("hello "));
-  ASSERT_OK(writable_file->Append("world"));
-  delete writable_file;
-
-  // Read sequentially.
-  ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
-  ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
-  ASSERT_EQ(0, result.compare("hello"));
-  ASSERT_OK(seq_file->Skip(1));
-  ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
-  ASSERT_EQ(0, result.compare("world"));
-  ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
-  ASSERT_EQ(0, result.size());
-  ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
-  ASSERT_OK(seq_file->Read(1000, &result, scratch));
-  ASSERT_EQ(0, result.size());
-  delete seq_file;
-
-  // Random reads.
-  ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
-  ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
-  ASSERT_EQ(0, result.compare("world"));
-  ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
-  ASSERT_EQ(0, result.compare("hello"));
-  ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
-  ASSERT_EQ(0, result.compare("d"));
-
-  // Too high offset.
-  ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok());
-  delete rand_file;
-}
-
-TEST(MemEnvTest, Locks) {
-  FileLock* lock;
-
-  // These are no-ops, but we test they return success.
-  ASSERT_OK(env_->LockFile("some file", &lock));
-  ASSERT_OK(env_->UnlockFile(lock));
-}
-
-TEST(MemEnvTest, Misc) {
-  std::string test_dir;
-  ASSERT_OK(env_->GetTestDirectory(&test_dir));
-  ASSERT_TRUE(!test_dir.empty());
-
-  WritableFile* writable_file;
-  ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file));
-
-  // These are no-ops, but we test they return success.
-  ASSERT_OK(writable_file->Sync());
-  ASSERT_OK(writable_file->Flush());
-  ASSERT_OK(writable_file->Close());
-  delete writable_file;
-}
-
-TEST(MemEnvTest, LargeWrite) {
-  const size_t kWriteSize = 300 * 1024;
-  char* scratch = new char[kWriteSize * 2];
-
-  std::string write_data;
-  for (size_t i = 0; i < kWriteSize; ++i) {
-    write_data.append(1, static_cast<char>(i));
-  }
-
-  WritableFile* writable_file;
-  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
-  ASSERT_OK(writable_file->Append("foo"));
-  ASSERT_OK(writable_file->Append(write_data));
-  delete writable_file;
-
-  SequentialFile* seq_file;
-  Slice result;
-  ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
-  ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
-  ASSERT_EQ(0, result.compare("foo"));
-
-  size_t read = 0;
-  std::string read_data;
-  while (read < kWriteSize) {
-    ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch));
-    read_data.append(result.data(), result.size());
-    read += result.size();
-  }
-  ASSERT_TRUE(write_data == read_data);
-  delete seq_file;
-  delete [] scratch;
-}
-
-TEST(MemEnvTest, DBTest) {
-  Options options;
-  options.create_if_missing = true;
-  options.env = env_;
-  DB* db;
-
-  const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")};
-  const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")};
-
-  ASSERT_OK(DB::Open(options, "/dir/db", &db));
-  for (size_t i = 0; i < 3; ++i) {
-    ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i]));
-  }
-
-  for (size_t i = 0; i < 3; ++i) {
-    std::string res;
-    ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
-    ASSERT_TRUE(res == vals[i]);
-  }
-
-  Iterator* iterator = db->NewIterator(ReadOptions());
-  iterator->SeekToFirst();
-  for (size_t i = 0; i < 3; ++i) {
-    ASSERT_TRUE(iterator->Valid());
-    ASSERT_TRUE(keys[i] == iterator->key());
-    ASSERT_TRUE(vals[i] == iterator->value());
-    iterator->Next();
-  }
-  ASSERT_TRUE(!iterator->Valid());
-  delete iterator;
-
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db);
-  ASSERT_OK(dbi->TEST_CompactMemTable());
-
-  for (size_t i = 0; i < 3; ++i) {
-    std::string res;
-    ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
-    ASSERT_TRUE(res == vals[i]);
-  }
-
-  delete db;
-}
-
-}  // namespace leveldb
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

src/leveldb/include/leveldb/c.h

@@ -1,290 +0,0 @@
-/* Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-  Use of this source code is governed by a BSD-style license that can be
-  found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-  C bindings for leveldb.  May be useful as a stable ABI that can be
-  used by programs that keep leveldb in a shared library, or for
-  a JNI api.
-
-  Does not support:
-  . getters for the option types
-  . custom comparators that implement key shortening
-  . custom iter, db, env, cache implementations using just the C bindings
-
-  Some conventions:
-
-  (1) We expose just opaque struct pointers and functions to clients.
-  This allows us to change internal representations without having to
-  recompile clients.
-
-  (2) For simplicity, there is no equivalent to the Slice type.  Instead,
-  the caller has to pass the pointer and length as separate
-  arguments.
-
-  (3) Errors are represented by a null-terminated c string.  NULL
-  means no error.  All operations that can raise an error are passed
-  a "char** errptr" as the last argument.  One of the following must
-  be true on entry:
-     *errptr == NULL
-     *errptr points to a malloc()ed null-terminated error message
-       (On Windows, *errptr must have been malloc()-ed by this library.)
-  On success, a leveldb routine leaves *errptr unchanged.
-  On failure, leveldb frees the old value of *errptr and
-  set *errptr to a malloc()ed error message.
-
-  (4) Bools have the type unsigned char (0 == false; rest == true)
-
-  (5) All of the pointer arguments must be non-NULL.
-*/
-
-#ifndef STORAGE_LEVELDB_INCLUDE_C_H_
-#define STORAGE_LEVELDB_INCLUDE_C_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdarg.h>
-#include <stddef.h>
-#include <stdint.h>
-
-/* Exported types */
-
-typedef struct leveldb_t               leveldb_t;
-typedef struct leveldb_cache_t         leveldb_cache_t;
-typedef struct leveldb_comparator_t    leveldb_comparator_t;
-typedef struct leveldb_env_t           leveldb_env_t;
-typedef struct leveldb_filelock_t      leveldb_filelock_t;
-typedef struct leveldb_filterpolicy_t  leveldb_filterpolicy_t;
-typedef struct leveldb_iterator_t      leveldb_iterator_t;
-typedef struct leveldb_logger_t        leveldb_logger_t;
-typedef struct leveldb_options_t       leveldb_options_t;
-typedef struct leveldb_randomfile_t    leveldb_randomfile_t;
-typedef struct leveldb_readoptions_t   leveldb_readoptions_t;
-typedef struct leveldb_seqfile_t       leveldb_seqfile_t;
-typedef struct leveldb_snapshot_t      leveldb_snapshot_t;
-typedef struct leveldb_writablefile_t  leveldb_writablefile_t;
-typedef struct leveldb_writebatch_t    leveldb_writebatch_t;
-typedef struct leveldb_writeoptions_t  leveldb_writeoptions_t;
-
-/* DB operations */
-
-extern leveldb_t* leveldb_open(
-    const leveldb_options_t* options,
-    const char* name,
-    char** errptr);
-
-extern void leveldb_close(leveldb_t* db);
-
-extern void leveldb_put(
-    leveldb_t* db,
-    const leveldb_writeoptions_t* options,
-    const char* key, size_t keylen,
-    const char* val, size_t vallen,
-    char** errptr);
-
-extern void leveldb_delete(
-    leveldb_t* db,
-    const leveldb_writeoptions_t* options,
-    const char* key, size_t keylen,
-    char** errptr);
-
-extern void leveldb_write(
-    leveldb_t* db,
-    const leveldb_writeoptions_t* options,
-    leveldb_writebatch_t* batch,
-    char** errptr);
-
-/* Returns NULL if not found.  A malloc()ed array otherwise.
-   Stores the length of the array in *vallen. */
-extern char* leveldb_get(
-    leveldb_t* db,
-    const leveldb_readoptions_t* options,
-    const char* key, size_t keylen,
-    size_t* vallen,
-    char** errptr);
-
-extern leveldb_iterator_t* leveldb_create_iterator(
-    leveldb_t* db,
-    const leveldb_readoptions_t* options);
-
-extern const leveldb_snapshot_t* leveldb_create_snapshot(
-    leveldb_t* db);
-
-extern void leveldb_release_snapshot(
-    leveldb_t* db,
-    const leveldb_snapshot_t* snapshot);
-
-/* Returns NULL if property name is unknown.
-   Else returns a pointer to a malloc()-ed null-terminated value. */
-extern char* leveldb_property_value(
-    leveldb_t* db,
-    const char* propname);
-
-extern void leveldb_approximate_sizes(
-    leveldb_t* db,
-    int num_ranges,
-    const char* const* range_start_key, const size_t* range_start_key_len,
-    const char* const* range_limit_key, const size_t* range_limit_key_len,
-    uint64_t* sizes);
-
-extern void leveldb_compact_range(
-    leveldb_t* db,
-    const char* start_key, size_t start_key_len,
-    const char* limit_key, size_t limit_key_len);
-
-/* Management operations */
-
-extern void leveldb_destroy_db(
-    const leveldb_options_t* options,
-    const char* name,
-    char** errptr);
-
-extern void leveldb_repair_db(
-    const leveldb_options_t* options,
-    const char* name,
-    char** errptr);
-
-/* Iterator */
-
-extern void leveldb_iter_destroy(leveldb_iterator_t*);
-extern unsigned char leveldb_iter_valid(const leveldb_iterator_t*);
-extern void leveldb_iter_seek_to_first(leveldb_iterator_t*);
-extern void leveldb_iter_seek_to_last(leveldb_iterator_t*);
-extern void leveldb_iter_seek(leveldb_iterator_t*, const char* k, size_t klen);
-extern void leveldb_iter_next(leveldb_iterator_t*);
-extern void leveldb_iter_prev(leveldb_iterator_t*);
-extern const char* leveldb_iter_key(const leveldb_iterator_t*, size_t* klen);
-extern const char* leveldb_iter_value(const leveldb_iterator_t*, size_t* vlen);
-extern void leveldb_iter_get_error(const leveldb_iterator_t*, char** errptr);
-
-/* Write batch */
-
-extern leveldb_writebatch_t* leveldb_writebatch_create();
-extern void leveldb_writebatch_destroy(leveldb_writebatch_t*);
-extern void leveldb_writebatch_clear(leveldb_writebatch_t*);
-extern void leveldb_writebatch_put(
-    leveldb_writebatch_t*,
-    const char* key, size_t klen,
-    const char* val, size_t vlen);
-extern void leveldb_writebatch_delete(
-    leveldb_writebatch_t*,
-    const char* key, size_t klen);
-extern void leveldb_writebatch_iterate(
-    leveldb_writebatch_t*,
-    void* state,
-    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
-    void (*deleted)(void*, const char* k, size_t klen));
-
-/* Options */
-
-extern leveldb_options_t* leveldb_options_create();
-extern void leveldb_options_destroy(leveldb_options_t*);
-extern void leveldb_options_set_comparator(
-    leveldb_options_t*,
-    leveldb_comparator_t*);
-extern void leveldb_options_set_filter_policy(
-    leveldb_options_t*,
-    leveldb_filterpolicy_t*);
-extern void leveldb_options_set_create_if_missing(
-    leveldb_options_t*, unsigned char);
-extern void leveldb_options_set_error_if_exists(
-    leveldb_options_t*, unsigned char);
-extern void leveldb_options_set_paranoid_checks(
-    leveldb_options_t*, unsigned char);
-extern void leveldb_options_set_env(leveldb_options_t*, leveldb_env_t*);
-extern void leveldb_options_set_info_log(leveldb_options_t*, leveldb_logger_t*);
-extern void leveldb_options_set_write_buffer_size(leveldb_options_t*, size_t);
-extern void leveldb_options_set_max_open_files(leveldb_options_t*, int);
-extern void leveldb_options_set_cache(leveldb_options_t*, leveldb_cache_t*);
-extern void leveldb_options_set_block_size(leveldb_options_t*, size_t);
-extern void leveldb_options_set_block_restart_interval(leveldb_options_t*, int);
-
-enum {
-  leveldb_no_compression = 0,
-  leveldb_snappy_compression = 1
-};
-extern void leveldb_options_set_compression(leveldb_options_t*, int);
-
-/* Comparator */
-
-extern leveldb_comparator_t* leveldb_comparator_create(
-    void* state,
-    void (*destructor)(void*),
-    int (*compare)(
-        void*,
-        const char* a, size_t alen,
-        const char* b, size_t blen),
-    const char* (*name)(void*));
-extern void leveldb_comparator_destroy(leveldb_comparator_t*);
-
-/* Filter policy */
-
-extern leveldb_filterpolicy_t* leveldb_filterpolicy_create(
-    void* state,
-    void (*destructor)(void*),
-    char* (*create_filter)(
-        void*,
-        const char* const* key_array, const size_t* key_length_array,
-        int num_keys,
-        size_t* filter_length),
-    unsigned char (*key_may_match)(
-        void*,
-        const char* key, size_t length,
-        const char* filter, size_t filter_length),
-    const char* (*name)(void*));
-extern void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t*);
-
-extern leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(
-    int bits_per_key);
-
-/* Read options */
-
-extern leveldb_readoptions_t* leveldb_readoptions_create();
-extern void leveldb_readoptions_destroy(leveldb_readoptions_t*);
-extern void leveldb_readoptions_set_verify_checksums(
-    leveldb_readoptions_t*,
-    unsigned char);
-extern void leveldb_readoptions_set_fill_cache(
-    leveldb_readoptions_t*, unsigned char);
-extern void leveldb_readoptions_set_snapshot(
-    leveldb_readoptions_t*,
-    const leveldb_snapshot_t*);
-
-/* Write options */
-
-extern leveldb_writeoptions_t* leveldb_writeoptions_create();
-extern void leveldb_writeoptions_destroy(leveldb_writeoptions_t*);
-extern void leveldb_writeoptions_set_sync(
-    leveldb_writeoptions_t*, unsigned char);
-
-/* Cache */
-
-extern leveldb_cache_t* leveldb_cache_create_lru(size_t capacity);
-extern void leveldb_cache_destroy(leveldb_cache_t* cache);
-
-/* Env */
-
-extern leveldb_env_t* leveldb_create_default_env();
-extern void leveldb_env_destroy(leveldb_env_t*);
-
-/* Utility */
-
-/* Calls free(ptr).
-   REQUIRES: ptr was malloc()-ed and returned by one of the routines
-   in this file.  Note that in certain cases (typically on Windows), you
-   may need to call this routine instead of free(ptr) to dispose of
-   malloc()-ed memory returned by this library. */
-extern void leveldb_free(void* ptr);
-
-/* Return the major version number for this release. */
-extern int leveldb_major_version();
-
-/* Return the minor version number for this release. */
-extern int leveldb_minor_version();
-
-#ifdef __cplusplus
-}  /* end extern "C" */
-#endif
-
-#endif  /* STORAGE_LEVELDB_INCLUDE_C_H_ */

src/leveldb/include/leveldb/cache.h

@@ -1,110 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Cache is an interface that maps keys to values.  It has internal
-// synchronization and may be safely accessed concurrently from
-// multiple threads.  It may automatically evict entries to make room
-// for new entries.  Values have a specified charge against the cache
-// capacity.  For example, a cache where the values are variable
-// length strings, may use the length of the string as the charge for
-// the string.
-//
-// A builtin cache implementation with a least-recently-used eviction
-// policy is provided.  Clients may use their own implementations if
-// they want something more sophisticated (like scan-resistance, a
-// custom eviction policy, variable cache sizing, etc.)
-
-#ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_
-#define STORAGE_LEVELDB_INCLUDE_CACHE_H_
-
-#include <stdint.h>
-#include "leveldb/slice.h"
-
-namespace leveldb {
-
-class Cache;
-
-// Create a new cache with a fixed size capacity.  This implementation
-// of Cache uses a least-recently-used eviction policy.
-extern Cache* NewLRUCache(size_t capacity);
-
-class Cache {
- public:
-  Cache() { }
-
-  // Destroys all existing entries by calling the "deleter"
-  // function that was passed to the constructor.
-  virtual ~Cache();
-
-  // Opaque handle to an entry stored in the cache.
-  struct Handle { };
-
-  // Insert a mapping from key->value into the cache and assign it
-  // the specified charge against the total cache capacity.
-  //
-  // Returns a handle that corresponds to the mapping.  The caller
-  // must call this->Release(handle) when the returned mapping is no
-  // longer needed.
-  //
-  // When the inserted entry is no longer needed, the key and
-  // value will be passed to "deleter".
-  virtual Handle* Insert(const Slice& key, void* value, size_t charge,
-                         void (*deleter)(const Slice& key, void* value)) = 0;
-
-  // If the cache has no mapping for "key", returns NULL.
-  //
-  // Else return a handle that corresponds to the mapping.  The caller
-  // must call this->Release(handle) when the returned mapping is no
-  // longer needed.
-  virtual Handle* Lookup(const Slice& key) = 0;
-
-  // Release a mapping returned by a previous Lookup().
-  // REQUIRES: handle must not have been released yet.
-  // REQUIRES: handle must have been returned by a method on *this.
-  virtual void Release(Handle* handle) = 0;
-
-  // Return the value encapsulated in a handle returned by a
-  // successful Lookup().
-  // REQUIRES: handle must not have been released yet.
-  // REQUIRES: handle must have been returned by a method on *this.
-  virtual void* Value(Handle* handle) = 0;
-
-  // If the cache contains entry for key, erase it.  Note that the
-  // underlying entry will be kept around until all existing handles
-  // to it have been released.
-  virtual void Erase(const Slice& key) = 0;
-
-  // Return a new numeric id.  May be used by multiple clients who are
-  // sharing the same cache to partition the key space.  Typically the
-  // client will allocate a new id at startup and prepend the id to
-  // its cache keys.
-  virtual uint64_t NewId() = 0;
-
-  // Remove all cache entries that are not actively in use.  Memory-constrained
-  // applications may wish to call this method to reduce memory usage.
-  // Default implementation of Prune() does nothing.  Subclasses are strongly
-  // encouraged to override the default implementation.  A future release of
-  // leveldb may change Prune() to a pure abstract method.
-  virtual void Prune() {}
-
-  // Return an estimate of the combined charges of all elements stored in the
-  // cache.
-  virtual size_t TotalCharge() const = 0;
-
- private:
-  void LRU_Remove(Handle* e);
-  void LRU_Append(Handle* e);
-  void Unref(Handle* e);
-
-  struct Rep;
-  Rep* rep_;
-
-  // No copying allowed
-  Cache(const Cache&);
-  void operator=(const Cache&);
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_CACHE_H_

src/leveldb/include/leveldb/comparator.h

@@ -1,63 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
-#define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
-
-#include <string>
-
-namespace leveldb {
-
-class Slice;
-
-// A Comparator object provides a total order across slices that are
-// used as keys in an sstable or a database.  A Comparator implementation
-// must be thread-safe since leveldb may invoke its methods concurrently
-// from multiple threads.
-class Comparator {
- public:
-  virtual ~Comparator();
-
-  // Three-way comparison.  Returns value:
-  //   < 0 iff "a" < "b",
-  //   == 0 iff "a" == "b",
-  //   > 0 iff "a" > "b"
-  virtual int Compare(const Slice& a, const Slice& b) const = 0;
-
-  // The name of the comparator.  Used to check for comparator
-  // mismatches (i.e., a DB created with one comparator is
-  // accessed using a different comparator.
-  //
-  // The client of this package should switch to a new name whenever
-  // the comparator implementation changes in a way that will cause
-  // the relative ordering of any two keys to change.
-  //
-  // Names starting with "leveldb." are reserved and should not be used
-  // by any clients of this package.
-  virtual const char* Name() const = 0;
-
-  // Advanced functions: these are used to reduce the space requirements
-  // for internal data structures like index blocks.
-
-  // If *start < limit, changes *start to a short string in [start,limit).
-  // Simple comparator implementations may return with *start unchanged,
-  // i.e., an implementation of this method that does nothing is correct.
-  virtual void FindShortestSeparator(
-      std::string* start,
-      const Slice& limit) const = 0;
-
-  // Changes *key to a short string >= *key.
-  // Simple comparator implementations may return with *key unchanged,
-  // i.e., an implementation of this method that does nothing is correct.
-  virtual void FindShortSuccessor(std::string* key) const = 0;
-};
-
-// Return a builtin comparator that uses lexicographic byte-wise
-// ordering.  The result remains the property of this module and
-// must not be deleted.
-extern const Comparator* BytewiseComparator();
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_

src/leveldb/include/leveldb/db.h

@@ -1,163 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_DB_H_
-#define STORAGE_LEVELDB_INCLUDE_DB_H_
-
-#include <stdint.h>
-#include <stdio.h>
-#include "leveldb/iterator.h"
-#include "leveldb/options.h"
-
-namespace leveldb {
-
-// Update Makefile if you change these
-static const int kMajorVersion = 1;
-static const int kMinorVersion = 20;
-
-struct Options;
-struct ReadOptions;
-struct WriteOptions;
-class WriteBatch;
-
-// Abstract handle to particular state of a DB.
-// A Snapshot is an immutable object and can therefore be safely
-// accessed from multiple threads without any external synchronization.
-class Snapshot {
- protected:
-  virtual ~Snapshot();
-};
-
-// A range of keys
-struct Range {
-  Slice start;          // Included in the range
-  Slice limit;          // Not included in the range
-
-  Range() { }
-  Range(const Slice& s, const Slice& l) : start(s), limit(l) { }
-};
-
-// A DB is a persistent ordered map from keys to values.
-// A DB is safe for concurrent access from multiple threads without
-// any external synchronization.
-class DB {
- public:
-  // Open the database with the specified "name".
-  // Stores a pointer to a heap-allocated database in *dbptr and returns
-  // OK on success.
-  // Stores NULL in *dbptr and returns a non-OK status on error.
-  // Caller should delete *dbptr when it is no longer needed.
-  static Status Open(const Options& options,
-                     const std::string& name,
-                     DB** dbptr);
-
-  DB() { }
-  virtual ~DB();
-
-  // Set the database entry for "key" to "value".  Returns OK on success,
-  // and a non-OK status on error.
-  // Note: consider setting options.sync = true.
-  virtual Status Put(const WriteOptions& options,
-                     const Slice& key,
-                     const Slice& value) = 0;
-
-  // Remove the database entry (if any) for "key".  Returns OK on
-  // success, and a non-OK status on error.  It is not an error if "key"
-  // did not exist in the database.
-  // Note: consider setting options.sync = true.
-  virtual Status Delete(const WriteOptions& options, const Slice& key) = 0;
-
-  // Apply the specified updates to the database.
-  // Returns OK on success, non-OK on failure.
-  // Note: consider setting options.sync = true.
-  virtual Status Write(const WriteOptions& options, WriteBatch* updates) = 0;
-
-  // If the database contains an entry for "key" store the
-  // corresponding value in *value and return OK.
-  //
-  // If there is no entry for "key" leave *value unchanged and return
-  // a status for which Status::IsNotFound() returns true.
-  //
-  // May return some other Status on an error.
-  virtual Status Get(const ReadOptions& options,
-                     const Slice& key, std::string* value) = 0;
-
-  // Return a heap-allocated iterator over the contents of the database.
-  // The result of NewIterator() is initially invalid (caller must
-  // call one of the Seek methods on the iterator before using it).
-  //
-  // Caller should delete the iterator when it is no longer needed.
-  // The returned iterator should be deleted before this db is deleted.
-  virtual Iterator* NewIterator(const ReadOptions& options) = 0;
-
-  // Return a handle to the current DB state.  Iterators created with
-  // this handle will all observe a stable snapshot of the current DB
-  // state.  The caller must call ReleaseSnapshot(result) when the
-  // snapshot is no longer needed.
-  virtual const Snapshot* GetSnapshot() = 0;
-
-  // Release a previously acquired snapshot.  The caller must not
-  // use "snapshot" after this call.
-  virtual void ReleaseSnapshot(const Snapshot* snapshot) = 0;
-
-  // DB implementations can export properties about their state
-  // via this method.  If "property" is a valid property understood by this
-  // DB implementation, fills "*value" with its current value and returns
-  // true.  Otherwise returns false.
-  //
-  //
-  // Valid property names include:
-  //
-  //  "leveldb.num-files-at-level<N>" - return the number of files at level <N>,
-  //     where <N> is an ASCII representation of a level number (e.g. "0").
-  //  "leveldb.stats" - returns a multi-line string that describes statistics
-  //     about the internal operation of the DB.
-  //  "leveldb.sstables" - returns a multi-line string that describes all
-  //     of the sstables that make up the db contents.
-  //  "leveldb.approximate-memory-usage" - returns the approximate number of
-  //     bytes of memory in use by the DB.
-  virtual bool GetProperty(const Slice& property, std::string* value) = 0;
-
-  // For each i in [0,n-1], store in "sizes[i]", the approximate
-  // file system space used by keys in "[range[i].start .. range[i].limit)".
-  //
-  // Note that the returned sizes measure file system space usage, so
-  // if the user data compresses by a factor of ten, the returned
-  // sizes will be one-tenth the size of the corresponding user data size.
-  //
-  // The results may not include the sizes of recently written data.
-  virtual void GetApproximateSizes(const Range* range, int n,
-                                   uint64_t* sizes) = 0;
-
-  // Compact the underlying storage for the key range [*begin,*end].
-  // In particular, deleted and overwritten versions are discarded,
-  // and the data is rearranged to reduce the cost of operations
-  // needed to access the data.  This operation should typically only
-  // be invoked by users who understand the underlying implementation.
-  //
-  // begin==NULL is treated as a key before all keys in the database.
-  // end==NULL is treated as a key after all keys in the database.
-  // Therefore the following call will compact the entire database:
-  //    db->CompactRange(NULL, NULL);
-  virtual void CompactRange(const Slice* begin, const Slice* end) = 0;
-
- private:
-  // No copying allowed
-  DB(const DB&);
-  void operator=(const DB&);
-};
-
-// Destroy the contents of the specified database.
-// Be very careful using this method.
-Status DestroyDB(const std::string& name, const Options& options);
-
-// If a DB cannot be opened, you may attempt to call this method to
-// resurrect as much of the contents of the database as possible.
-// Some data may be lost, so be careful when calling this function
-// on a database that contains important information.
-Status RepairDB(const std::string& dbname, const Options& options);
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_DB_H_

src/leveldb/include/leveldb/dumpfile.h

@@ -1,25 +0,0 @@
-// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
-#define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
-
-#include <string>
-#include "leveldb/env.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-// Dump the contents of the file named by fname in text format to
-// *dst.  Makes a sequence of dst->Append() calls; each call is passed
-// the newline-terminated text corresponding to a single item found
-// in the file.
-//
-// Returns a non-OK result if fname does not name a leveldb storage
-// file, or if the file cannot be read.
-Status DumpFile(Env* env, const std::string& fname, WritableFile* dst);
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_

src/leveldb/include/leveldb/env.h

@@ -1,363 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// An Env is an interface used by the leveldb implementation to access
-// operating system functionality like the filesystem etc.  Callers
-// may wish to provide a custom Env object when opening a database to
-// get fine gain control; e.g., to rate limit file system operations.
-//
-// All Env implementations are safe for concurrent access from
-// multiple threads without any external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_ENV_H_
-#define STORAGE_LEVELDB_INCLUDE_ENV_H_
-
-#include <string>
-#include <vector>
-#include <stdarg.h>
-#include <stdint.h>
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class FileLock;
-class Logger;
-class RandomAccessFile;
-class SequentialFile;
-class Slice;
-class WritableFile;
-
-class Env {
- public:
-  Env() { }
-  virtual ~Env();
-
-  // Return a default environment suitable for the current operating
-  // system.  Sophisticated users may wish to provide their own Env
-  // implementation instead of relying on this default environment.
-  //
-  // The result of Default() belongs to leveldb and must never be deleted.
-  static Env* Default();
-
-  // Create a brand new sequentially-readable file with the specified name.
-  // On success, stores a pointer to the new file in *result and returns OK.
-  // On failure stores NULL in *result and returns non-OK.  If the file does
-  // not exist, returns a non-OK status.
-  //
-  // The returned file will only be accessed by one thread at a time.
-  virtual Status NewSequentialFile(const std::string& fname,
-                                   SequentialFile** result) = 0;
-
-  // Create a brand new random access read-only file with the
-  // specified name.  On success, stores a pointer to the new file in
-  // *result and returns OK.  On failure stores NULL in *result and
-  // returns non-OK.  If the file does not exist, returns a non-OK
-  // status.
-  //
-  // The returned file may be concurrently accessed by multiple threads.
-  virtual Status NewRandomAccessFile(const std::string& fname,
-                                     RandomAccessFile** result) = 0;
-
-  // Create an object that writes to a new file with the specified
-  // name.  Deletes any existing file with the same name and creates a
-  // new file.  On success, stores a pointer to the new file in
-  // *result and returns OK.  On failure stores NULL in *result and
-  // returns non-OK.
-  //
-  // The returned file will only be accessed by one thread at a time.
-  virtual Status NewWritableFile(const std::string& fname,
-                                 WritableFile** result) = 0;
-
-  // Create an object that either appends to an existing file, or
-  // writes to a new file (if the file does not exist to begin with).
-  // On success, stores a pointer to the new file in *result and
-  // returns OK.  On failure stores NULL in *result and returns
-  // non-OK.
-  //
-  // The returned file will only be accessed by one thread at a time.
-  //
-  // May return an IsNotSupportedError error if this Env does
-  // not allow appending to an existing file.  Users of Env (including
-  // the leveldb implementation) must be prepared to deal with
-  // an Env that does not support appending.
-  virtual Status NewAppendableFile(const std::string& fname,
-                                   WritableFile** result);
-
-  // Returns true iff the named file exists.
-  virtual bool FileExists(const std::string& fname) = 0;
-
-  // Store in *result the names of the children of the specified directory.
-  // The names are relative to "dir".
-  // Original contents of *results are dropped.
-  virtual Status GetChildren(const std::string& dir,
-                             std::vector<std::string>* result) = 0;
-
-  // Delete the named file.
-  virtual Status DeleteFile(const std::string& fname) = 0;
-
-  // Create the specified directory.
-  virtual Status CreateDir(const std::string& dirname) = 0;
-
-  // Delete the specified directory.
-  virtual Status DeleteDir(const std::string& dirname) = 0;
-
-  // Store the size of fname in *file_size.
-  virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) = 0;
-
-  // Rename file src to target.
-  virtual Status RenameFile(const std::string& src,
-                            const std::string& target) = 0;
-
-  // Lock the specified file.  Used to prevent concurrent access to
-  // the same db by multiple processes.  On failure, stores NULL in
-  // *lock and returns non-OK.
-  //
-  // On success, stores a pointer to the object that represents the
-  // acquired lock in *lock and returns OK.  The caller should call
-  // UnlockFile(*lock) to release the lock.  If the process exits,
-  // the lock will be automatically released.
-  //
-  // If somebody else already holds the lock, finishes immediately
-  // with a failure.  I.e., this call does not wait for existing locks
-  // to go away.
-  //
-  // May create the named file if it does not already exist.
-  virtual Status LockFile(const std::string& fname, FileLock** lock) = 0;
-
-  // Release the lock acquired by a previous successful call to LockFile.
-  // REQUIRES: lock was returned by a successful LockFile() call
-  // REQUIRES: lock has not already been unlocked.
-  virtual Status UnlockFile(FileLock* lock) = 0;
-
-  // Arrange to run "(*function)(arg)" once in a background thread.
-  //
-  // "function" may run in an unspecified thread.  Multiple functions
-  // added to the same Env may run concurrently in different threads.
-  // I.e., the caller may not assume that background work items are
-  // serialized.
-  virtual void Schedule(
-      void (*function)(void* arg),
-      void* arg) = 0;
-
-  // Start a new thread, invoking "function(arg)" within the new thread.
-  // When "function(arg)" returns, the thread will be destroyed.
-  virtual void StartThread(void (*function)(void* arg), void* arg) = 0;
-
-  // *path is set to a temporary directory that can be used for testing. It may
-  // or many not have just been created. The directory may or may not differ
-  // between runs of the same process, but subsequent calls will return the
-  // same directory.
-  virtual Status GetTestDirectory(std::string* path) = 0;
-
-  // Create and return a log file for storing informational messages.
-  virtual Status NewLogger(const std::string& fname, Logger** result) = 0;
-
-  // Returns the number of micro-seconds since some fixed point in time. Only
-  // useful for computing deltas of time.
-  virtual uint64_t NowMicros() = 0;
-
-  // Sleep/delay the thread for the prescribed number of micro-seconds.
-  virtual void SleepForMicroseconds(int micros) = 0;
-
- private:
-  // No copying allowed
-  Env(const Env&);
-  void operator=(const Env&);
-};
-
-// A file abstraction for reading sequentially through a file
-class SequentialFile {
- public:
-  SequentialFile() { }
-  virtual ~SequentialFile();
-
-  // Read up to "n" bytes from the file.  "scratch[0..n-1]" may be
-  // written by this routine.  Sets "*result" to the data that was
-  // read (including if fewer than "n" bytes were successfully read).
-  // May set "*result" to point at data in "scratch[0..n-1]", so
-  // "scratch[0..n-1]" must be live when "*result" is used.
-  // If an error was encountered, returns a non-OK status.
-  //
-  // REQUIRES: External synchronization
-  virtual Status Read(size_t n, Slice* result, char* scratch) = 0;
-
-  // Skip "n" bytes from the file. This is guaranteed to be no
-  // slower that reading the same data, but may be faster.
-  //
-  // If end of file is reached, skipping will stop at the end of the
-  // file, and Skip will return OK.
-  //
-  // REQUIRES: External synchronization
-  virtual Status Skip(uint64_t n) = 0;
-
-  // Get a name for the file, only for error reporting
-  virtual std::string GetName() const = 0;
-
- private:
-  // No copying allowed
-  SequentialFile(const SequentialFile&);
-  void operator=(const SequentialFile&);
-};
-
-// A file abstraction for randomly reading the contents of a file.
-class RandomAccessFile {
- public:
-  RandomAccessFile() { }
-  virtual ~RandomAccessFile();
-
-  // Read up to "n" bytes from the file starting at "offset".
-  // "scratch[0..n-1]" may be written by this routine.  Sets "*result"
-  // to the data that was read (including if fewer than "n" bytes were
-  // successfully read).  May set "*result" to point at data in
-  // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
-  // "*result" is used.  If an error was encountered, returns a non-OK
-  // status.
-  //
-  // Safe for concurrent use by multiple threads.
-  virtual Status Read(uint64_t offset, size_t n, Slice* result,
-                      char* scratch) const = 0;
-
-  // Get a name for the file, only for error reporting
-  virtual std::string GetName() const = 0;
-
-  virtual char* AllocateScratch(std::size_t size) const { return new char[size]; };
-  virtual void DeallocateScratch(char* pointer) const { delete[] pointer; };
-
- private:
-  // No copying allowed
-  RandomAccessFile(const RandomAccessFile&);
-  void operator=(const RandomAccessFile&);
-};
-
-// A file abstraction for sequential writing.  The implementation
-// must provide buffering since callers may append small fragments
-// at a time to the file.
-class WritableFile {
- public:
-  WritableFile() { }
-  virtual ~WritableFile();
-
-  virtual Status Append(const Slice& data) = 0;
-  virtual Status Close() = 0;
-  virtual Status Flush() = 0;
-  virtual Status Sync() = 0;
-
-  // Get a name for the file, only for error reporting
-  virtual std::string GetName() const = 0;
-
- private:
-  // No copying allowed
-  WritableFile(const WritableFile&);
-  void operator=(const WritableFile&);
-};
-
-// An interface for writing log messages.
-class Logger {
- public:
-  Logger() { }
-  virtual ~Logger();
-
-  // Write an entry to the log file with the specified format.
-  virtual void Logv(const char* format, va_list ap) = 0;
-
- private:
-  // No copying allowed
-  Logger(const Logger&);
-  void operator=(const Logger&);
-};
-
-
-// Identifies a locked file.
-class FileLock {
- public:
-  FileLock() { }
-  virtual ~FileLock();
- private:
-  // No copying allowed
-  FileLock(const FileLock&);
-  void operator=(const FileLock&);
-};
-
-// Log the specified data to *info_log if info_log is non-NULL.
-extern void Log(Logger* info_log, const char* format, ...)
-#   if defined(__GNUC__) || defined(__clang__)
-    __attribute__((__format__ (__printf__, 2, 3)))
-#   endif
-    ;
-
-// A utility routine: write "data" to the named file.
-extern Status WriteStringToFile(Env* env, const Slice& data,
-                                const std::string& fname);
-
-// A utility routine: read contents of named file into *data
-extern Status ReadFileToString(Env* env, const std::string& fname,
-                               std::string* data);
-
-// An implementation of Env that forwards all calls to another Env.
-// May be useful to clients who wish to override just part of the
-// functionality of another Env.
-class EnvWrapper : public Env {
- public:
-  // Initialize an EnvWrapper that delegates all calls to *t
-  explicit EnvWrapper(Env* t) : target_(t) { }
-  virtual ~EnvWrapper();
-
-  // Return the target to which this Env forwards all calls
-  Env* target() const { return target_; }
-
-  // The following text is boilerplate that forwards all methods to target()
-  Status NewSequentialFile(const std::string& f, SequentialFile** r) {
-    return target_->NewSequentialFile(f, r);
-  }
-  Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) {
-    return target_->NewRandomAccessFile(f, r);
-  }
-  Status NewWritableFile(const std::string& f, WritableFile** r) {
-    return target_->NewWritableFile(f, r);
-  }
-  Status NewAppendableFile(const std::string& f, WritableFile** r) {
-    return target_->NewAppendableFile(f, r);
-  }
-  bool FileExists(const std::string& f) { return target_->FileExists(f); }
-  Status GetChildren(const std::string& dir, std::vector<std::string>* r) {
-    return target_->GetChildren(dir, r);
-  }
-  Status DeleteFile(const std::string& f) { return target_->DeleteFile(f); }
-  Status CreateDir(const std::string& d) { return target_->CreateDir(d); }
-  Status DeleteDir(const std::string& d) { return target_->DeleteDir(d); }
-  Status GetFileSize(const std::string& f, uint64_t* s) {
-    return target_->GetFileSize(f, s);
-  }
-  Status RenameFile(const std::string& s, const std::string& t) {
-    return target_->RenameFile(s, t);
-  }
-  Status LockFile(const std::string& f, FileLock** l) {
-    return target_->LockFile(f, l);
-  }
-  Status UnlockFile(FileLock* l) { return target_->UnlockFile(l); }
-  void Schedule(void (*f)(void*), void* a) {
-    return target_->Schedule(f, a);
-  }
-  void StartThread(void (*f)(void*), void* a) {
-    return target_->StartThread(f, a);
-  }
-  virtual Status GetTestDirectory(std::string* path) {
-    return target_->GetTestDirectory(path);
-  }
-  virtual Status NewLogger(const std::string& fname, Logger** result) {
-    return target_->NewLogger(fname, result);
-  }
-  uint64_t NowMicros() {
-    return target_->NowMicros();
-  }
-  void SleepForMicroseconds(int micros) {
-    target_->SleepForMicroseconds(micros);
-  }
- private:
-  Env* target_;
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_ENV_H_

src/leveldb/include/leveldb/filter_policy.h

@@ -1,70 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A database can be configured with a custom FilterPolicy object.
-// This object is responsible for creating a small filter from a set
-// of keys.  These filters are stored in leveldb and are consulted
-// automatically by leveldb to decide whether or not to read some
-// information from disk. In many cases, a filter can cut down the
-// number of disk seeks form a handful to a single disk seek per
-// DB::Get() call.
-//
-// Most people will want to use the builtin bloom filter support (see
-// NewBloomFilterPolicy() below).
-
-#ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
-#define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
-
-#include <string>
-
-namespace leveldb {
-
-class Slice;
-
-class FilterPolicy {
- public:
-  virtual ~FilterPolicy();
-
-  // Return the name of this policy.  Note that if the filter encoding
-  // changes in an incompatible way, the name returned by this method
-  // must be changed.  Otherwise, old incompatible filters may be
-  // passed to methods of this type.
-  virtual const char* Name() const = 0;
-
-  // keys[0,n-1] contains a list of keys (potentially with duplicates)
-  // that are ordered according to the user supplied comparator.
-  // Append a filter that summarizes keys[0,n-1] to *dst.
-  //
-  // Warning: do not change the initial contents of *dst.  Instead,
-  // append the newly constructed filter to *dst.
-  virtual void CreateFilter(const Slice* keys, int n, std::string* dst)
-      const = 0;
-
-  // "filter" contains the data appended by a preceding call to
-  // CreateFilter() on this class.  This method must return true if
-  // the key was in the list of keys passed to CreateFilter().
-  // This method may return true or false if the key was not on the
-  // list, but it should aim to return false with a high probability.
-  virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const = 0;
-};
-
-// Return a new filter policy that uses a bloom filter with approximately
-// the specified number of bits per key.  A good value for bits_per_key
-// is 10, which yields a filter with ~ 1% false positive rate.
-//
-// Callers must delete the result after any database that is using the
-// result has been closed.
-//
-// Note: if you are using a custom comparator that ignores some parts
-// of the keys being compared, you must not use NewBloomFilterPolicy()
-// and must provide your own FilterPolicy that also ignores the
-// corresponding parts of the keys.  For example, if the comparator
-// ignores trailing spaces, it would be incorrect to use a
-// FilterPolicy (like NewBloomFilterPolicy) that does not ignore
-// trailing spaces in keys.
-extern const FilterPolicy* NewBloomFilterPolicy(int bits_per_key);
-
-}
-
-#endif  // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_

src/leveldb/include/leveldb/iterator.h

@@ -1,100 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// An iterator yields a sequence of key/value pairs from a source.
-// The following class defines the interface.  Multiple implementations
-// are provided by this library.  In particular, iterators are provided
-// to access the contents of a Table or a DB.
-//
-// Multiple threads can invoke const methods on an Iterator without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Iterator must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
-#define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
-
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class Iterator {
- public:
-  Iterator();
-  virtual ~Iterator();
-
-  // An iterator is either positioned at a key/value pair, or
-  // not valid.  This method returns true iff the iterator is valid.
-  virtual bool Valid() const = 0;
-
-  // Position at the first key in the source.  The iterator is Valid()
-  // after this call iff the source is not empty.
-  virtual void SeekToFirst() = 0;
-
-  // Position at the last key in the source.  The iterator is
-  // Valid() after this call iff the source is not empty.
-  virtual void SeekToLast() = 0;
-
-  // Position at the first key in the source that is at or past target.
-  // The iterator is Valid() after this call iff the source contains
-  // an entry that comes at or past target.
-  virtual void Seek(const Slice& target) = 0;
-
-  // Moves to the next entry in the source.  After this call, Valid() is
-  // true iff the iterator was not positioned at the last entry in the source.
-  // REQUIRES: Valid()
-  virtual void Next() = 0;
-
-  // Moves to the previous entry in the source.  After this call, Valid() is
-  // true iff the iterator was not positioned at the first entry in source.
-  // REQUIRES: Valid()
-  virtual void Prev() = 0;
-
-  // Return the key for the current entry.  The underlying storage for
-  // the returned slice is valid only until the next modification of
-  // the iterator.
-  // REQUIRES: Valid()
-  virtual Slice key() const = 0;
-
-  // Return the value for the current entry.  The underlying storage for
-  // the returned slice is valid only until the next modification of
-  // the iterator.
-  // REQUIRES: Valid()
-  virtual Slice value() const = 0;
-
-  // If an error has occurred, return it.  Else return an ok status.
-  virtual Status status() const = 0;
-
-  // Clients are allowed to register function/arg1/arg2 triples that
-  // will be invoked when this iterator is destroyed.
-  //
-  // Note that unlike all of the preceding methods, this method is
-  // not abstract and therefore clients should not override it.
-  typedef void (*CleanupFunction)(void* arg1, void* arg2);
-  void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
-
- private:
-  struct Cleanup {
-    CleanupFunction function;
-    void* arg1;
-    void* arg2;
-    Cleanup* next;
-  };
-  Cleanup cleanup_;
-
-  // No copying allowed
-  Iterator(const Iterator&);
-  void operator=(const Iterator&);
-};
-
-// Return an empty iterator (yields nothing).
-extern Iterator* NewEmptyIterator();
-
-// Return an empty iterator with the specified status.
-extern Iterator* NewErrorIterator(const Status& status);
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_

src/leveldb/include/leveldb/options.h

@@ -1,213 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
-#define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
-
-#include <stddef.h>
-
-namespace leveldb {
-
-class Cache;
-class Comparator;
-class Env;
-class FilterPolicy;
-class Logger;
-class Snapshot;
-
-// DB contents are stored in a set of blocks, each of which holds a
-// sequence of key,value pairs.  Each block may be compressed before
-// being stored in a file.  The following enum describes which
-// compression method (if any) is used to compress a block.
-enum CompressionType {
-  // NOTE: do not change the values of existing entries, as these are
-  // part of the persistent format on disk.
-  kNoCompression     = 0x0,
-  kSnappyCompression = 0x1
-};
-
-// Options to control the behavior of a database (passed to DB::Open)
-struct Options {
-  // -------------------
-  // Parameters that affect behavior
-
-  // Comparator used to define the order of keys in the table.
-  // Default: a comparator that uses lexicographic byte-wise ordering
-  //
-  // REQUIRES: The client must ensure that the comparator supplied
-  // here has the same name and orders keys *exactly* the same as the
-  // comparator provided to previous open calls on the same DB.
-  const Comparator* comparator;
-
-  // If true, the database will be created if it is missing.
-  // Default: false
-  bool create_if_missing;
-
-  // If true, an error is raised if the database already exists.
-  // Default: false
-  bool error_if_exists;
-
-  // If true, the implementation will do aggressive checking of the
-  // data it is processing and will stop early if it detects any
-  // errors.  This may have unforeseen ramifications: for example, a
-  // corruption of one DB entry may cause a large number of entries to
-  // become unreadable or for the entire DB to become unopenable.
-  // Default: false
-  bool paranoid_checks;
-
-  // Use the specified object to interact with the environment,
-  // e.g. to read/write files, schedule background work, etc.
-  // Default: Env::Default()
-  Env* env;
-
-  // Any internal progress/error information generated by the db will
-  // be written to info_log if it is non-NULL, or to a file stored
-  // in the same directory as the DB contents if info_log is NULL.
-  // Default: NULL
-  Logger* info_log;
-
-  // -------------------
-  // Parameters that affect performance
-
-  // Amount of data to build up in memory (backed by an unsorted log
-  // on disk) before converting to a sorted on-disk file.
-  //
-  // Larger values increase performance, especially during bulk loads.
-  // Up to two write buffers may be held in memory at the same time,
-  // so you may wish to adjust this parameter to control memory usage.
-  // Also, a larger write buffer will result in a longer recovery time
-  // the next time the database is opened.
-  //
-  // Default: 4MB
-  size_t write_buffer_size;
-
-  // Number of open files that can be used by the DB.  You may need to
-  // increase this if your database has a large working set (budget
-  // one open file per 2MB of working set).
-  //
-  // Default: 1000
-  int max_open_files;
-
-  // Control over blocks (user data is stored in a set of blocks, and
-  // a block is the unit of reading from disk).
-
-  // If non-NULL, use the specified cache for blocks.
-  // If NULL, leveldb will automatically create and use an 8MB internal cache.
-  // Default: NULL
-  Cache* block_cache;
-
-  // Approximate size of user data packed per block.  Note that the
-  // block size specified here corresponds to uncompressed data.  The
-  // actual size of the unit read from disk may be smaller if
-  // compression is enabled.  This parameter can be changed dynamically.
-  //
-  // Default: 4K
-  size_t block_size;
-
-  // Number of keys between restart points for delta encoding of keys.
-  // This parameter can be changed dynamically.  Most clients should
-  // leave this parameter alone.
-  //
-  // Default: 16
-  int block_restart_interval;
-
-  // Leveldb will write up to this amount of bytes to a file before
-  // switching to a new one.
-  // Most clients should leave this parameter alone.  However if your
-  // filesystem is more efficient with larger files, you could
-  // consider increasing the value.  The downside will be longer
-  // compactions and hence longer latency/performance hiccups.
-  // Another reason to increase this parameter might be when you are
-  // initially populating a large database.
-  //
-  // Default: 2MB
-  size_t max_file_size;
-
-  // Compress blocks using the specified compression algorithm.  This
-  // parameter can be changed dynamically.
-  //
-  // Default: kSnappyCompression, which gives lightweight but fast
-  // compression.
-  //
-  // Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz:
-  //    ~200-500MB/s compression
-  //    ~400-800MB/s decompression
-  // Note that these speeds are significantly faster than most
-  // persistent storage speeds, and therefore it is typically never
-  // worth switching to kNoCompression.  Even if the input data is
-  // incompressible, the kSnappyCompression implementation will
-  // efficiently detect that and will switch to uncompressed mode.
-  CompressionType compression;
-
-  // EXPERIMENTAL: If true, append to existing MANIFEST and log files
-  // when a database is opened.  This can significantly speed up open.
-  //
-  // Default: currently false, but may become true later.
-  bool reuse_logs;
-
-  // If non-NULL, use the specified filter policy to reduce disk reads.
-  // Many applications will benefit from passing the result of
-  // NewBloomFilterPolicy() here.
-  //
-  // Default: NULL
-  const FilterPolicy* filter_policy;
-
-  // Create an Options object with default values for all fields.
-  Options();
-};
-
-// Options that control read operations
-struct ReadOptions {
-  // If true, all data read from underlying storage will be
-  // verified against corresponding checksums.
-  // Default: false
-  bool verify_checksums;
-
-  // Should the data read for this iteration be cached in memory?
-  // Callers may wish to set this field to false for bulk scans.
-  // Default: true
-  bool fill_cache;
-
-  // If "snapshot" is non-NULL, read as of the supplied snapshot
-  // (which must belong to the DB that is being read and which must
-  // not have been released).  If "snapshot" is NULL, use an implicit
-  // snapshot of the state at the beginning of this read operation.
-  // Default: NULL
-  const Snapshot* snapshot;
-
-  ReadOptions()
-      : verify_checksums(false),
-        fill_cache(true),
-        snapshot(NULL) {
-  }
-};
-
-// Options that control write operations
-struct WriteOptions {
-  // If true, the write will be flushed from the operating system
-  // buffer cache (by calling WritableFile::Sync()) before the write
-  // is considered complete.  If this flag is true, writes will be
-  // slower.
-  //
-  // If this flag is false, and the machine crashes, some recent
-  // writes may be lost.  Note that if it is just the process that
-  // crashes (i.e., the machine does not reboot), no writes will be
-  // lost even if sync==false.
-  //
-  // In other words, a DB write with sync==false has similar
-  // crash semantics as the "write()" system call.  A DB write
-  // with sync==true has similar crash semantics to a "write()"
-  // system call followed by "fsync()".
-  //
-  // Default: false
-  bool sync;
-
-  WriteOptions()
-      : sync(false) {
-  }
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_OPTIONS_H_

src/leveldb/include/leveldb/slice.h

@@ -1,109 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Slice is a simple structure containing a pointer into some external
-// storage and a size.  The user of a Slice must ensure that the slice
-// is not used after the corresponding external storage has been
-// deallocated.
-//
-// Multiple threads can invoke const methods on a Slice without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Slice must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_
-#define STORAGE_LEVELDB_INCLUDE_SLICE_H_
-
-#include <assert.h>
-#include <stddef.h>
-#include <string.h>
-#include <string>
-
-namespace leveldb {
-
-class Slice {
- public:
-  // Create an empty slice.
-  Slice() : data_(""), size_(0) { }
-
-  // Create a slice that refers to d[0,n-1].
-  Slice(const char* d, size_t n) : data_(d), size_(n) { }
-
-  // Create a slice that refers to the contents of "s"
-  Slice(const std::string& s) : data_(s.data()), size_(s.size()) { }
-
-  // Create a slice that refers to s[0,strlen(s)-1]
-  Slice(const char* s) : data_(s), size_(strlen(s)) { }
-
-  // Return a pointer to the beginning of the referenced data
-  const char* data() const { return data_; }
-
-  // Return the length (in bytes) of the referenced data
-  size_t size() const { return size_; }
-
-  // Return true iff the length of the referenced data is zero
-  bool empty() const { return size_ == 0; }
-
-  // Return the ith byte in the referenced data.
-  // REQUIRES: n < size()
-  char operator[](size_t n) const {
-    assert(n < size());
-    return data_[n];
-  }
-
-  // Change this slice to refer to an empty array
-  void clear() { data_ = ""; size_ = 0; }
-
-  // Drop the first "n" bytes from this slice.
-  void remove_prefix(size_t n) {
-    assert(n <= size());
-    data_ += n;
-    size_ -= n;
-  }
-
-  // Return a string that contains the copy of the referenced data.
-  std::string ToString() const { return std::string(data_, size_); }
-
-  // Three-way comparison.  Returns value:
-  //   <  0 iff "*this" <  "b",
-  //   == 0 iff "*this" == "b",
-  //   >  0 iff "*this" >  "b"
-  int compare(const Slice& b) const;
-
-  // Return true iff "x" is a prefix of "*this"
-  bool starts_with(const Slice& x) const {
-    return ((size_ >= x.size_) &&
-            (memcmp(data_, x.data_, x.size_) == 0));
-  }
-
- private:
-  const char* data_;
-  size_t size_;
-
-  // Intentionally copyable
-};
-
-inline bool operator==(const Slice& x, const Slice& y) {
-  return ((x.size() == y.size()) &&
-          (memcmp(x.data(), y.data(), x.size()) == 0));
-}
-
-inline bool operator!=(const Slice& x, const Slice& y) {
-  return !(x == y);
-}
-
-inline int Slice::compare(const Slice& b) const {
-  const size_t min_len = (size_ < b.size_) ? size_ : b.size_;
-  int r = memcmp(data_, b.data_, min_len);
-  if (r == 0) {
-    if (size_ < b.size_) r = -1;
-    else if (size_ > b.size_) r = +1;
-  }
-  return r;
-}
-
-}  // namespace leveldb
-
-
-#endif  // STORAGE_LEVELDB_INCLUDE_SLICE_H_

src/leveldb/include/leveldb/status.h

@@ -1,112 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Status encapsulates the result of an operation.  It may indicate success,
-// or it may indicate an error with an associated error message.
-//
-// Multiple threads can invoke const methods on a Status without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Status must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_
-#define STORAGE_LEVELDB_INCLUDE_STATUS_H_
-
-#include <string>
-#include "leveldb/slice.h"
-
-namespace leveldb {
-
-class Status {
- public:
-  // Create a success status.
-  Status() : state_(NULL) { }
-  ~Status() { delete[] state_; }
-
-  // Copy the specified status.
-  Status(const Status& s);
-  void operator=(const Status& s);
-
-  // Return a success status.
-  static Status OK() { return Status(); }
-
-  // Return error status of an appropriate type.
-  static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) {
-    return Status(kNotFound, msg, msg2);
-  }
-  static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) {
-    return Status(kCorruption, msg, msg2);
-  }
-  static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) {
-    return Status(kNotSupported, msg, msg2);
-  }
-  static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) {
-    return Status(kInvalidArgument, msg, msg2);
-  }
-  static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) {
-    return Status(kIOError, msg, msg2);
-  }
-
-  // Returns true iff the status indicates success.
-  bool ok() const { return (state_ == NULL); }
-
-  // Returns true iff the status indicates a NotFound error.
-  bool IsNotFound() const { return code() == kNotFound; }
-
-  // Returns true iff the status indicates a Corruption error.
-  bool IsCorruption() const { return code() == kCorruption; }
-
-  // Returns true iff the status indicates an IOError.
-  bool IsIOError() const { return code() == kIOError; }
-
-  // Returns true iff the status indicates a NotSupportedError.
-  bool IsNotSupportedError() const { return code() == kNotSupported; }
-
-  // Returns true iff the status indicates an InvalidArgument.
-  bool IsInvalidArgument() const { return code() == kInvalidArgument; }
-
-  // Return a string representation of this status suitable for printing.
-  // Returns the string "OK" for success.
-  std::string ToString() const;
-
- private:
-  // OK status has a NULL state_.  Otherwise, state_ is a new[] array
-  // of the following form:
-  //    state_[0..3] == length of message
-  //    state_[4]    == code
-  //    state_[5..]  == message
-  const char* state_;
-
-  enum Code {
-    kOk = 0,
-    kNotFound = 1,
-    kCorruption = 2,
-    kNotSupported = 3,
-    kInvalidArgument = 4,
-    kIOError = 5
-  };
-
-  Code code() const {
-    return (state_ == NULL) ? kOk : static_cast<Code>(state_[4]);
-  }
-
-  Status(Code code, const Slice& msg, const Slice& msg2);
-  static const char* CopyState(const char* s);
-};
-
-inline Status::Status(const Status& s) {
-  state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-}
-inline void Status::operator=(const Status& s) {
-  // The following condition catches both aliasing (when this == &s),
-  // and the common case where both s and *this are ok.
-  if (state_ != s.state_) {
-    delete[] state_;
-    state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-  }
-}
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_STATUS_H_

src/leveldb/include/leveldb/table.h

@@ -1,85 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_H_
-#define STORAGE_LEVELDB_INCLUDE_TABLE_H_
-
-#include <stdint.h>
-#include "leveldb/iterator.h"
-
-namespace leveldb {
-
-class Block;
-class BlockHandle;
-class Footer;
-struct Options;
-class RandomAccessFile;
-struct ReadOptions;
-class TableCache;
-
-// A Table is a sorted map from strings to strings.  Tables are
-// immutable and persistent.  A Table may be safely accessed from
-// multiple threads without external synchronization.
-class Table {
- public:
-  // Attempt to open the table that is stored in bytes [0..file_size)
-  // of "file", and read the metadata entries necessary to allow
-  // retrieving data from the table.
-  //
-  // If successful, returns ok and sets "*table" to the newly opened
-  // table.  The client should delete "*table" when no longer needed.
-  // If there was an error while initializing the table, sets "*table"
-  // to NULL and returns a non-ok status.  Does not take ownership of
-  // "*source", but the client must ensure that "source" remains live
-  // for the duration of the returned table's lifetime.
-  //
-  // *file must remain live while this Table is in use.
-  static Status Open(const Options& options,
-                     RandomAccessFile* file,
-                     uint64_t file_size,
-                     Table** table);
-
-  ~Table();
-
-  // Returns a new iterator over the table contents.
-  // The result of NewIterator() is initially invalid (caller must
-  // call one of the Seek methods on the iterator before using it).
-  Iterator* NewIterator(const ReadOptions&) const;
-
-  // Given a key, return an approximate byte offset in the file where
-  // the data for that key begins (or would begin if the key were
-  // present in the file).  The returned value is in terms of file
-  // bytes, and so includes effects like compression of the underlying data.
-  // E.g., the approximate offset of the last key in the table will
-  // be close to the file length.
-  uint64_t ApproximateOffsetOf(const Slice& key) const;
-
- private:
-  struct Rep;
-  Rep* rep_;
-
-  explicit Table(Rep* rep) { rep_ = rep; }
-  static Iterator* BlockReader(void*, const ReadOptions&, const Slice&);
-
-  // Calls (*handle_result)(arg, ...) with the entry found after a call
-  // to Seek(key).  May not make such a call if filter policy says
-  // that key is not present.
-  friend class TableCache;
-  Status InternalGet(
-      const ReadOptions&, const Slice& key,
-      void* arg,
-      void (*handle_result)(void* arg, const Slice& k, const Slice& v));
-
-
-  void ReadMeta(const Footer& footer);
-  void ReadFilter(const Slice& filter_handle_value);
-
-  // No copying allowed
-  Table(const Table&);
-  void operator=(const Table&);
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_TABLE_H_

src/leveldb/include/leveldb/table_builder.h

@@ -1,92 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// TableBuilder provides the interface used to build a Table
-// (an immutable and sorted map from keys to values).
-//
-// Multiple threads can invoke const methods on a TableBuilder without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same TableBuilder must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
-#define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
-
-#include <stdint.h>
-#include "leveldb/options.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class BlockBuilder;
-class BlockHandle;
-class WritableFile;
-
-class TableBuilder {
- public:
-  // Create a builder that will store the contents of the table it is
-  // building in *file.  Does not close the file.  It is up to the
-  // caller to close the file after calling Finish().
-  TableBuilder(const Options& options, WritableFile* file);
-
-  // REQUIRES: Either Finish() or Abandon() has been called.
-  ~TableBuilder();
-
-  // Change the options used by this builder.  Note: only some of the
-  // option fields can be changed after construction.  If a field is
-  // not allowed to change dynamically and its value in the structure
-  // passed to the constructor is different from its value in the
-  // structure passed to this method, this method will return an error
-  // without changing any fields.
-  Status ChangeOptions(const Options& options);
-
-  // Add key,value to the table being constructed.
-  // REQUIRES: key is after any previously added key according to comparator.
-  // REQUIRES: Finish(), Abandon() have not been called
-  void Add(const Slice& key, const Slice& value);
-
-  // Advanced operation: flush any buffered key/value pairs to file.
-  // Can be used to ensure that two adjacent entries never live in
-  // the same data block.  Most clients should not need to use this method.
-  // REQUIRES: Finish(), Abandon() have not been called
-  void Flush();
-
-  // Return non-ok iff some error has been detected.
-  Status status() const;
-
-  // Finish building the table.  Stops using the file passed to the
-  // constructor after this function returns.
-  // REQUIRES: Finish(), Abandon() have not been called
-  Status Finish();
-
-  // Indicate that the contents of this builder should be abandoned.  Stops
-  // using the file passed to the constructor after this function returns.
-  // If the caller is not going to call Finish(), it must call Abandon()
-  // before destroying this builder.
-  // REQUIRES: Finish(), Abandon() have not been called
-  void Abandon();
-
-  // Number of calls to Add() so far.
-  uint64_t NumEntries() const;
-
-  // Size of the file generated so far.  If invoked after a successful
-  // Finish() call, returns the size of the final generated file.
-  uint64_t FileSize() const;
-
- private:
-  bool ok() const { return status().ok(); }
-  void WriteBlock(BlockBuilder* block, BlockHandle* handle);
-  void WriteRawBlock(const Slice& data, CompressionType, BlockHandle* handle);
-
-  struct Rep;
-  Rep* rep_;
-
-  // No copying allowed
-  TableBuilder(const TableBuilder&);
-  void operator=(const TableBuilder&);
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_

src/leveldb/include/leveldb/write_batch.h

@@ -1,64 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// WriteBatch holds a collection of updates to apply atomically to a DB.
-//
-// The updates are applied in the order in which they are added
-// to the WriteBatch.  For example, the value of "key" will be "v3"
-// after the following batch is written:
-//
-//    batch.Put("key", "v1");
-//    batch.Delete("key");
-//    batch.Put("key", "v2");
-//    batch.Put("key", "v3");
-//
-// Multiple threads can invoke const methods on a WriteBatch without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same WriteBatch must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
-#define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
-
-#include <string>
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class Slice;
-
-class WriteBatch {
- public:
-  WriteBatch();
-  ~WriteBatch();
-
-  // Store the mapping "key->value" in the database.
-  void Put(const Slice& key, const Slice& value);
-
-  // If the database contains a mapping for "key", erase it.  Else do nothing.
-  void Delete(const Slice& key);
-
-  // Clear all updates buffered in this batch.
-  void Clear();
-
-  // Support for iterating over the contents of a batch.
-  class Handler {
-   public:
-    virtual ~Handler();
-    virtual void Put(const Slice& key, const Slice& value) = 0;
-    virtual void Delete(const Slice& key) = 0;
-  };
-  Status Iterate(Handler* handler) const;
-
- private:
-  friend class WriteBatchInternal;
-
-  std::string rep_;  // See comment in write_batch.cc for the format of rep_
-
-  // Intentionally copyable
-};
-
-}  // namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_

src/leveldb/issues/issue178_test.cc

@@ -1,92 +0,0 @@
-// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-// Test for issue 178: a manual compaction causes deleted data to reappear.
-#include <iostream>
-#include <sstream>
-#include <cstdlib>
-
-#include "leveldb/db.h"
-#include "leveldb/write_batch.h"
-#include "util/testharness.h"
-
-namespace {
-
-const int kNumKeys = 1100000;
-
-std::string Key1(int i) {
-  char buf[100];
-  snprintf(buf, sizeof(buf), "my_key_%d", i);
-  return buf;
-}
-
-std::string Key2(int i) {
-  return Key1(i) + "_xxx";
-}
-
-class Issue178 { };
-
-TEST(Issue178, Test) {
-  // Get rid of any state from an old run.
-  std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test";
-  DestroyDB(dbpath, leveldb::Options());
-
-  // Open database.  Disable compression since it affects the creation
-  // of layers and the code below is trying to test against a very
-  // specific scenario.
-  leveldb::DB* db;
-  leveldb::Options db_options;
-  db_options.create_if_missing = true;
-  db_options.compression = leveldb::kNoCompression;
-  ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db));
-
-  // create first key range
-  leveldb::WriteBatch batch;
-  for (size_t i = 0; i < kNumKeys; i++) {
-    batch.Put(Key1(i), "value for range 1 key");
-  }
-  ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));
-
-  // create second key range
-  batch.Clear();
-  for (size_t i = 0; i < kNumKeys; i++) {
-    batch.Put(Key2(i), "value for range 2 key");
-  }
-  ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));
-
-  // delete second key range
-  batch.Clear();
-  for (size_t i = 0; i < kNumKeys; i++) {
-    batch.Delete(Key2(i));
-  }
-  ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));
-
-  // compact database
-  std::string start_key = Key1(0);
-  std::string end_key = Key1(kNumKeys - 1);
-  leveldb::Slice least(start_key.data(), start_key.size());
-  leveldb::Slice greatest(end_key.data(), end_key.size());
-
-  // commenting out the line below causes the example to work correctly
-  db->CompactRange(&least, &greatest);
-
-  // count the keys
-  leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions());
-  size_t num_keys = 0;
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    num_keys++;
-  }
-  delete iter;
-  ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys";
-
-  // close database
-  delete db;
-  DestroyDB(dbpath, leveldb::Options());
-}
-
-}  // anonymous namespace
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}