dash/blackcoin-more
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #27 from braydonf/0.12.1-bitcore-dboptions

Browse Source 
Add options to configure block index database
This commit is contained in:
Chris Kleeschulte
2016-07-22 09:49:18 -04:00
committed by GitHub
parent 0c15767527 69ea12c689
commit 796a2742c1
8 changed files with 166 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/dbwrapper.cpp
View File

@@ -29,14 +29,14 @@ void HandleError(const leveldb::Status& status) throw(dbwrapper_error)
throw dbwrapper_error("Unknown database error");
}
static leveldb::Options GetOptions(size_t nCacheSize)
static leveldb::Options GetOptions(size_t nCacheSize, bool compression, int maxOpenFiles)
{
leveldb::Options options;
options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
options.filter_policy = leveldb::NewBloomFilterPolicy(10);
options.compression = leveldb::kNoCompression;
options.max_open_files = 64;
options.compression = compression ? leveldb::kSnappyCompression : leveldb::kNoCompression;
options.max_open_files = maxOpenFiles;
if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
// LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
// on corruption in later versions.
@@ -45,14 +45,14 @@ static leveldb::Options GetOptions(size_t nCacheSize)
return options;
}
CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate, bool compression, int maxOpenFiles)
{
penv = NULL;
readoptions.verify_checksums = true;
iteroptions.verify_checksums = true;
iteroptions.fill_cache = false;
syncoptions.sync = true;
options = GetOptions(nCacheSize);
options = GetOptions(nCacheSize, compression, maxOpenFiles);
options.create_if_missing = true;
if (fMemory) {
penv = leveldb::NewMemEnv(leveldb::Env::Default());

16
src/dbwrapper.h
View File

@@ -169,14 +169,16 @@ private:
public:
/**
* @param[in] path Location in the filesystem where leveldb data will be stored.
* @param[in] nCacheSize Configures various leveldb cache settings.
* @param[in] fMemory If true, use leveldb's memory environment.
* @param[in] fWipe If true, remove all existing data.
* @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR
* with a zero'd byte array.
* @param[in] path Location in the filesystem where leveldb data will be stored.
* @param[in] nCacheSize Configures various leveldb cache settings.
* @param[in] fMemory If true, use leveldb's memory environment.
* @param[in] fWipe If true, remove all existing data.
* @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR
* with a zero'd byte array.
* @param[in] compression Enable snappy compression for the database
* @param[in] maxOpenFiles The maximum number of open files for the database
*/
CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false, bool compression = false, int maxOpenFiles = 64);
~CDBWrapper();
template <typename K, typename V>

21
src/init.cpp
View File

@@ -1294,18 +1294,33 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
}
}
// block tree db settings
int dbMaxOpenFiles = GetArg("-dbmaxopenfiles", DEFAULT_DB_MAX_OPEN_FILES);
bool dbCompression = GetBoolArg("-dbcompression", DEFAULT_DB_COMPRESSION);
LogPrintf("Block index database configuration:\n");
LogPrintf("* Using %d max open files\n", dbMaxOpenFiles);
LogPrintf("* Compression is %s\n", dbCompression ? "enabled" : "disabled");
// cache size calculations
int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20);
nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache
int64_t nBlockTreeDBCache = nTotalCache / 8;
if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX))
nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB
if (GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX) || GetBoolArg("-spentindex", DEFAULT_SPENTINDEX)) {
// enable 3/4 of the cache if addressindex and/or spentindex is enabled
nBlockTreeDBCache = nTotalCache * 3 / 4;
} else {
if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB
}
}
nTotalCache -= nBlockTreeDBCache;
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
nTotalCache -= nCoinDBCache;
nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
LogPrintf("Cache configuration:\n");
LogPrintf("* Max cache setting possible %.1fMiB\n", nMaxDbCache);
LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024));
@@ -1326,7 +1341,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
delete pcoinscatcher;
delete pblocktree;
pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex);
pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex, dbCompression, dbMaxOpenFiles);
pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
pcoinsTip = new CCoinsViewCache(pcoinscatcher);

2
src/main.h
View File

@@ -115,6 +115,8 @@ static const bool DEFAULT_TXINDEX = false;
static const bool DEFAULT_ADDRESSINDEX = false;
static const bool DEFAULT_TIMESTAMPINDEX = false;
static const bool DEFAULT_SPENTINDEX = false;
static const unsigned int DEFAULT_DB_MAX_OPEN_FILES = 1000;
static const bool DEFAULT_DB_COMPRESSION = true;
static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100;
static const bool DEFAULT_TESTSAFEMODE = false;

92
src/rpcrawtransaction.cpp
View File

@@ -59,8 +59,10 @@ void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fInclud
out.push_back(Pair("addresses", a));
}
void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry)
void TxToJSONExpanded(const CTransaction& tx, const uint256 hashBlock, UniValue& entry,
int nHeight = 0, int nConfirmations = 0, int nBlockTime = 0)
{
uint256 txid = tx.GetHash();
entry.push_back(Pair("txid", txid.GetHex()));
entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)));
@@ -121,6 +123,63 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry)
}
entry.push_back(Pair("vout", vout));
if (!hashBlock.IsNull()) {
entry.push_back(Pair("blockhash", hashBlock.GetHex()));
if (nConfirmations > 0) {
entry.push_back(Pair("height", nHeight));
entry.push_back(Pair("confirmations", nConfirmations));
entry.push_back(Pair("time", nBlockTime));
entry.push_back(Pair("blocktime", nBlockTime));
} else {
entry.push_back(Pair("height", -1));
entry.push_back(Pair("confirmations", 0));
}
}
}
void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry)
{
uint256 txid = tx.GetHash();
entry.push_back(Pair("txid", txid.GetHex()));
entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)));
entry.push_back(Pair("version", tx.nVersion));
entry.push_back(Pair("locktime", (int64_t)tx.nLockTime));
UniValue vin(UniValue::VARR);
BOOST_FOREACH(const CTxIn& txin, tx.vin) {
UniValue in(UniValue::VOBJ);
if (tx.IsCoinBase())
in.push_back(Pair("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())));
else {
in.push_back(Pair("txid", txin.prevout.hash.GetHex()));
in.push_back(Pair("vout", (int64_t)txin.prevout.n));
UniValue o(UniValue::VOBJ);
o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true)));
o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())));
in.push_back(Pair("scriptSig", o));
}
in.push_back(Pair("sequence", (int64_t)txin.nSequence));
vin.push_back(in);
}
entry.push_back(Pair("vin", vin));
UniValue vout(UniValue::VARR);
for (unsigned int i = 0; i < tx.vout.size(); i++) {
const CTxOut& txout = tx.vout[i];
UniValue out(UniValue::VOBJ);
out.push_back(Pair("value", ValueFromAmount(txout.nValue)));
out.push_back(Pair("valueSat", txout.nValue));
out.push_back(Pair("n", (int64_t)i));
UniValue o(UniValue::VOBJ);
ScriptPubKeyToJSON(txout.scriptPubKey, o, true);
out.push_back(Pair("scriptPubKey", o));
vout.push_back(out);
}
entry.push_back(Pair("vout", vout));
if (!hashBlock.IsNull()) {
entry.push_back(Pair("blockhash", hashBlock.GetHex()));
BlockMap::iterator mi = mapBlockIndex.find(hashBlock);
@@ -206,8 +265,6 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp)
+ HelpExampleRpc("getrawtransaction", "\"mytxid\", 1")
);
LOCK(cs_main);
uint256 hash = ParseHashV(params[0], "parameter 1");
bool fVerbose = false;
@@ -215,9 +272,31 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp)
fVerbose = (params[1].get_int() != 0);
CTransaction tx;
uint256 hashBlock;
if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true))
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction");
int nHeight = 0;
int nConfirmations = 0;
int nBlockTime = 0;
{
LOCK(cs_main);
if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true))
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction");
BlockMap::iterator mi = mapBlockIndex.find(hashBlock);
if (mi != mapBlockIndex.end() && (*mi).second) {
CBlockIndex* pindex = (*mi).second;
if (chainActive.Contains(pindex)) {
nHeight = pindex->nHeight;
nConfirmations = 1 + chainActive.Height() - pindex->nHeight;
nBlockTime = pindex->GetBlockTime();
} else {
nHeight = -1;
nConfirmations = 0;
nBlockTime = pindex->GetBlockTime();
}
}
}
string strHex = EncodeHexTx(tx);
@@ -226,7 +305,8 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp)
UniValue result(UniValue::VOBJ);
result.push_back(Pair("hex", strHex));
TxToJSON(tx, hashBlock, result);
TxToJSONExpanded(tx, hashBlock, result, nHeight, nConfirmations, nBlockTime);
return result;
}

43
src/test/dbwrapper_tests.cpp
View File

@@ -47,6 +47,49 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
}
}
BOOST_AUTO_TEST_CASE(dbwrapper_compression)
{
// Perform tests both with compression and without
for (int i = 0; i < 2; i++) {
bool compression = (bool)i;
path ph = temp_directory_path() / unique_path();
CDBWrapper dbw(ph, (1 << 20), true, false, false, compression);
char key = 'k';
uint256 in = GetRandHash();
uint256 res;
BOOST_CHECK(dbw.Write(key, in));
BOOST_CHECK(dbw.Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
}
}
BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_64)
{
path ph = temp_directory_path() / unique_path();
CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 64);
char key = 'k';
uint256 in = GetRandHash();
uint256 res;
BOOST_CHECK(dbw.Write(key, in));
BOOST_CHECK(dbw.Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
}
BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_1000)
{
path ph = temp_directory_path() / unique_path();
CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 1000);
char key = 'k';
uint256 in = GetRandHash();
uint256 res;
BOOST_CHECK(dbw.Write(key, in));
BOOST_CHECK(dbw.Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
}
// Test batch operations
BOOST_AUTO_TEST_CASE(dbwrapper_batch)
{

4
src/txdb.cpp
View File

@@ -33,7 +33,7 @@ static const char DB_REINDEX_FLAG = 'R';
static const char DB_LAST_BLOCK = 'l';
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true)
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true, false, 64)
{
}
@@ -75,7 +75,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
return db.WriteBatch(batch);
}
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe, bool compression, int maxOpenFiles) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe, false, compression, maxOpenFiles) {
}
bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) {

2
src/txdb.h
View File

@@ -54,7 +54,7 @@ public:
class CBlockTreeDB : public CDBWrapper
{
public:
CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);
CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool compression = true, int maxOpenFiles = 1000);
private:
CBlockTreeDB(const CBlockTreeDB&);
void operator=(const CBlockTreeDB&);