From de05c9ecbb3ed60b39e281518846c508d07255e2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 13 Jul 2016 18:38:04 -0400 Subject: [PATCH 1/3] db: add options to configure block index database There was a previous assumption that blockindex would be quite small. With addressindex and spentindex enabled the blockindex is much larger and the amount of cache allocated for it should also increase. Furthermore, enabling compression should decrease the amount of disk space required and less data to write/read. The default leveldb max_open_files is set to 1000, for the blockindex the default is set to 1000 with compression. The 64 value that is current is kept for the utxo database and does not enable compression. Two additional options are added here to be able to configure the values for leveldb and the block index: - `-dbmaxopenfiles` A number of files for leveldb to keep open - `-dbcompression` Boolean 0 or 1 to enable snappy leveldb compression --- src/dbwrapper.cpp | 10 +++++----- src/dbwrapper.h | 16 +++++++++------- src/init.cpp | 21 ++++++++++++++++++--- src/main.h | 2 ++ src/txdb.cpp | 4 ++-- src/txdb.h | 2 +- 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 1907e2fa7..5f1f175b8 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -29,14 +29,14 @@ void HandleError(const leveldb::Status& status) throw(dbwrapper_error) throw dbwrapper_error("Unknown database error"); } -static leveldb::Options GetOptions(size_t nCacheSize) +static leveldb::Options GetOptions(size_t nCacheSize, bool compression, int maxOpenFiles) { leveldb::Options options; options.block_cache = leveldb::NewLRUCache(nCacheSize / 2); options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously options.filter_policy = leveldb::NewBloomFilterPolicy(10); - options.compression = leveldb::kNoCompression; - options.max_open_files = 64; + options.compression = compression ? leveldb::kSnappyCompression : leveldb::kNoCompression; + options.max_open_files = maxOpenFiles; if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) { // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error // on corruption in later versions. @@ -45,14 +45,14 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } -CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) +CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate, bool compression, int maxOpenFiles) { penv = NULL; readoptions.verify_checksums = true; iteroptions.verify_checksums = true; iteroptions.fill_cache = false; syncoptions.sync = true; - options = GetOptions(nCacheSize); + options = GetOptions(nCacheSize, compression, maxOpenFiles); options.create_if_missing = true; if (fMemory) { penv = leveldb::NewMemEnv(leveldb::Env::Default()); diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 5e7313f7e..2acd5fc05 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -169,14 +169,16 @@ private: public: /** - * @param[in] path Location in the filesystem where leveldb data will be stored. - * @param[in] nCacheSize Configures various leveldb cache settings. - * @param[in] fMemory If true, use leveldb's memory environment. - * @param[in] fWipe If true, remove all existing data. - * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR - * with a zero'd byte array. + * @param[in] path Location in the filesystem where leveldb data will be stored. + * @param[in] nCacheSize Configures various leveldb cache settings. + * @param[in] fMemory If true, use leveldb's memory environment. + * @param[in] fWipe If true, remove all existing data. + * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR + * with a zero'd byte array. + * @param[in] compression Enable snappy compression for the database + * @param[in] maxOpenFiles The maximum number of open files for the database */ - CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); + CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false, bool compression = false, int maxOpenFiles = 64); ~CDBWrapper(); template diff --git a/src/init.cpp b/src/init.cpp index 389350092..4d952ada0 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1294,18 +1294,33 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) } } + // block tree db settings + int dbMaxOpenFiles = GetArg("-dbmaxopenfiles", DEFAULT_DB_MAX_OPEN_FILES); + bool dbCompression = GetBoolArg("-dbcompression", DEFAULT_DB_COMPRESSION); + + LogPrintf("Block index database configuration:\n"); + LogPrintf("* Using %d max open files\n", dbMaxOpenFiles); + LogPrintf("* Compression is %s\n", dbCompression ? "enabled" : "disabled"); + // cache size calculations int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache int64_t nBlockTreeDBCache = nTotalCache / 8; - if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) - nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB + if (GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX) || GetBoolArg("-spentindex", DEFAULT_SPENTINDEX)) { + // enable 3/4 of the cache if addressindex and/or spentindex is enabled + nBlockTreeDBCache = nTotalCache * 3 / 4; + } else { + if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB + } + } nTotalCache -= nBlockTreeDBCache; int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nTotalCache -= nCoinDBCache; nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache LogPrintf("Cache configuration:\n"); + LogPrintf("* Max cache setting possible %.1fMiB\n", nMaxDbCache); LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024)); @@ -1326,7 +1341,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) delete pcoinscatcher; delete pblocktree; - pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex); + pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex, dbCompression, dbMaxOpenFiles); pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex); pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); pcoinsTip = new CCoinsViewCache(pcoinscatcher); diff --git a/src/main.h b/src/main.h index 3f8a3fb32..1361a75f0 100644 --- a/src/main.h +++ b/src/main.h @@ -115,6 +115,8 @@ static const bool DEFAULT_TXINDEX = false; static const bool DEFAULT_ADDRESSINDEX = false; static const bool DEFAULT_TIMESTAMPINDEX = false; static const bool DEFAULT_SPENTINDEX = false; +static const unsigned int DEFAULT_DB_MAX_OPEN_FILES = 1000; +static const bool DEFAULT_DB_COMPRESSION = true; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; diff --git a/src/txdb.cpp b/src/txdb.cpp index 48150198f..f204989f1 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -33,7 +33,7 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; -CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) +CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true, false, 64) { } @@ -75,7 +75,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return db.WriteBatch(batch); } -CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { +CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe, bool compression, int maxOpenFiles) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe, false, compression, maxOpenFiles) { } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { diff --git a/src/txdb.h b/src/txdb.h index 14d501278..b2a99e0b3 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -54,7 +54,7 @@ public: class CBlockTreeDB : public CDBWrapper { public: - CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); + CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool compression = true, int maxOpenFiles = 1000); private: CBlockTreeDB(const CBlockTreeDB&); void operator=(const CBlockTreeDB&); From 1b36e2c62686fa88dc6a0f1f4917ab61913f0766 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Sat, 9 Jul 2016 02:46:52 +0000 Subject: [PATCH 2/3] rpc: minimize locking in getrawtransaction since there is i/o that can happen when retrieving extended input and output information, limiting the duration of the lock in getrawtransaction can help improve concurrency --- src/rpcrawtransaction.cpp | 92 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index ee39fbefd..64050694e 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -59,8 +59,10 @@ void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fInclud out.push_back(Pair("addresses", a)); } -void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) +void TxToJSONExpanded(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, + int nHeight = 0, int nConfirmations = 0, int nBlockTime = 0) { + uint256 txid = tx.GetHash(); entry.push_back(Pair("txid", txid.GetHex())); entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); @@ -121,6 +123,63 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) } entry.push_back(Pair("vout", vout)); + if (!hashBlock.IsNull()) { + entry.push_back(Pair("blockhash", hashBlock.GetHex())); + + if (nConfirmations > 0) { + entry.push_back(Pair("height", nHeight)); + entry.push_back(Pair("confirmations", nConfirmations)); + entry.push_back(Pair("time", nBlockTime)); + entry.push_back(Pair("blocktime", nBlockTime)); + } else { + entry.push_back(Pair("height", -1)); + entry.push_back(Pair("confirmations", 0)); + } + } + +} + +void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) +{ + + uint256 txid = tx.GetHash(); + entry.push_back(Pair("txid", txid.GetHex())); + entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); + entry.push_back(Pair("version", tx.nVersion)); + entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); + + UniValue vin(UniValue::VARR); + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + UniValue in(UniValue::VOBJ); + if (tx.IsCoinBase()) + in.push_back(Pair("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); + else { + in.push_back(Pair("txid", txin.prevout.hash.GetHex())); + in.push_back(Pair("vout", (int64_t)txin.prevout.n)); + UniValue o(UniValue::VOBJ); + o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true))); + o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); + in.push_back(Pair("scriptSig", o)); + } + in.push_back(Pair("sequence", (int64_t)txin.nSequence)); + vin.push_back(in); + } + entry.push_back(Pair("vin", vin)); + + UniValue vout(UniValue::VARR); + for (unsigned int i = 0; i < tx.vout.size(); i++) { + const CTxOut& txout = tx.vout[i]; + UniValue out(UniValue::VOBJ); + out.push_back(Pair("value", ValueFromAmount(txout.nValue))); + out.push_back(Pair("valueSat", txout.nValue)); + out.push_back(Pair("n", (int64_t)i)); + UniValue o(UniValue::VOBJ); + ScriptPubKeyToJSON(txout.scriptPubKey, o, true); + out.push_back(Pair("scriptPubKey", o)); + vout.push_back(out); + } + entry.push_back(Pair("vout", vout)); + if (!hashBlock.IsNull()) { entry.push_back(Pair("blockhash", hashBlock.GetHex())); BlockMap::iterator mi = mapBlockIndex.find(hashBlock); @@ -206,8 +265,6 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) + HelpExampleRpc("getrawtransaction", "\"mytxid\", 1") ); - LOCK(cs_main); - uint256 hash = ParseHashV(params[0], "parameter 1"); bool fVerbose = false; @@ -215,9 +272,31 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) fVerbose = (params[1].get_int() != 0); CTransaction tx; + uint256 hashBlock; - if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction"); + int nHeight = 0; + int nConfirmations = 0; + int nBlockTime = 0; + + { + LOCK(cs_main); + if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction"); + + BlockMap::iterator mi = mapBlockIndex.find(hashBlock); + if (mi != mapBlockIndex.end() && (*mi).second) { + CBlockIndex* pindex = (*mi).second; + if (chainActive.Contains(pindex)) { + nHeight = pindex->nHeight; + nConfirmations = 1 + chainActive.Height() - pindex->nHeight; + nBlockTime = pindex->GetBlockTime(); + } else { + nHeight = -1; + nConfirmations = 0; + nBlockTime = pindex->GetBlockTime(); + } + } + } string strHex = EncodeHexTx(tx); @@ -226,7 +305,8 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) UniValue result(UniValue::VOBJ); result.push_back(Pair("hex", strHex)); - TxToJSON(tx, hashBlock, result); + TxToJSONExpanded(tx, hashBlock, result, nHeight, nConfirmations, nBlockTime); + return result; } From 69ea12c6897465362c211dab15a4c93e4f8a7db7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 14 Jul 2016 16:30:31 -0400 Subject: [PATCH 3/3] tests: test dbwrapper options compression and maxopenfiles --- src/test/dbwrapper_tests.cpp | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index e39931587..466a0f4de 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -47,6 +47,49 @@ BOOST_AUTO_TEST_CASE(dbwrapper) } } +BOOST_AUTO_TEST_CASE(dbwrapper_compression) +{ + // Perform tests both with compression and without + for (int i = 0; i < 2; i++) { + bool compression = (bool)i; + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false, compression); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + } +} + +BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_64) +{ + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 64); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); +} + +BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_1000) +{ + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 1000); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); +} + // Test batch operations BOOST_AUTO_TEST_CASE(dbwrapper_batch) {