mirror of https://github.com/facebook/rocksdb.git
Dbid feature
Summary: Create a new type of file on startup if it doesn't already exist called DBID. This will store a unique number generated from boost library's uuid header file. The use-case is to identify the case of a db losing all its data and coming back up either empty or from an image(backup/live replica's recovery) the key point to note is that DBID is not stored in a backup or db snapshot It's preferable to use Boost for uuid because: 1) A non-standard way of generating uuid is not good 2) /proc/sys/kernel/random/uuid generates a uuid but only on linux environments and the solution would not be clean 3) c++ doesn't have any direct way to get a uuid 4) Boost is a very good library that was already having linkage in rocksdb from third-party Note: I had to update the TOOLCHAIN_REV in build files to get latest verison of boost from third-party as the older version had a bug. I had to put Wno-uninitialized in Makefile because boost-1.51 has an unitialized variable and rocksdb would not comiple otherwise. Latet open-source for boost is 1.54 but is not there in third-party. I have notified the concerned people in fbcode about it. @kailiu : While releasing to third-party, an additional dependency will need to be created for boost in TARGETS file. I can help identify. Test Plan: Expand db_test to test 2 cases 1) Restarting db with Id file present - verify that no change to Id 2)Restarting db with Id file deleted - verify that a different Id is there after reopen Also run make all check Reviewers: dhruba, haobo, kailiu, sdong Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D13587
This commit is contained in:
Mayank Agarwal
parent
ae8e0770b4
commit
9b50106f9a
|
|
@ -134,14 +134,6 @@ $PWD/build_tools/build_detect_version
|
|||
# of all files matching either rule, so we need to append -print to make the
|
||||
# prune take effect.
|
||||
DIRS="util db table utilities"
|
||||
if test "$USE_THRIFT"; then
|
||||
DIRS="$DIRS thrift/server_utils.cpp thrift/gen-cpp "
|
||||
THRIFTSERVER=leveldb_server
|
||||
fi
|
||||
|
||||
if test "$USE_SCRIBE"; then
|
||||
DIRS="$DIRS scribe "
|
||||
fi
|
||||
|
||||
set -f # temporarily disable globbing so that our patterns arent expanded
|
||||
PRUNE_TEST="-name *test*.cc -prune"
|
||||
|
|
@ -227,14 +219,6 @@ if test "$USE_HDFS"; then
|
|||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $HDFS_LDFLAGS"
|
||||
fi
|
||||
|
||||
# shall we build thrift server or scribe logger
|
||||
if test "$USE_THRIFT" || test "$USE_SCRIBE" ; then
|
||||
THRIFT_CCFLAGS=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp -I/usr/include -std=gnu++0x"
|
||||
THRIFT_LDFLAGS=" -lexample -lserver -lthrift_base -ltransport -lthrift_exception -lutil -L./thrift/libs "
|
||||
COMMON_FLAGS="$COMMON_FLAGS $THRIFT_CCFLAGS"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $THRIFT_LDFLAGS"
|
||||
fi
|
||||
|
||||
# if Intel SSE instruction set is supported, set USE_SSE=" -msse -msse4.2 "
|
||||
COMMON_FLAGS="$COMMON_FLAGS $USE_SSE"
|
||||
|
||||
|
|
@ -254,5 +238,4 @@ echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
|
|||
echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
|
||||
echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
|
||||
echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT
|
||||
echo "THRIFTSERVER=$THRIFTSERVER" >> $OUTPUT
|
||||
echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> $OUTPUT
|
||||
|
|
|
|||
|
|
@ -4,21 +4,16 @@
|
|||
# fbcode settings. It uses the latest g++ compiler and also
|
||||
# uses jemalloc
|
||||
|
||||
TOOLCHAIN_REV=f365dbeae46a30414a2874a6f45e73e10f1caf7d
|
||||
TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
|
||||
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
|
||||
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
|
||||
TOOL_JEMALLOC=jemalloc-3.0.0/2f45f3a
|
||||
TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
|
||||
GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1
|
||||
|
||||
# location of snappy headers and libraries
|
||||
SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
|
||||
SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
|
||||
|
||||
# location of boost headers and libraries
|
||||
THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/include -std=gnu++0x"
|
||||
THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
|
||||
THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/lib"
|
||||
|
||||
# location of libevent
|
||||
LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
|
||||
LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
|
||||
|
|
@ -27,7 +22,7 @@ LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
|
|||
export USE_SSE=" -msse -msse4.2 "
|
||||
|
||||
CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang $CLANG_INCLUDES"
|
||||
CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
|
||||
CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $LIBEVENT_INCLUDE"
|
||||
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
|
||||
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
|
||||
|
||||
|
|
@ -37,8 +32,8 @@ CXXFLAGS="$CFLAGS -nostdinc++ -std=gnu++0x"
|
|||
CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
|
||||
|
||||
EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
|
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
|
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
|
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
|
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $LIBEVENT_LIBS"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2"
|
||||
EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin"
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@
|
|||
# fbcode settings. It uses the latest g++ compiler and also
|
||||
# uses jemalloc
|
||||
|
||||
TOOLCHAIN_REV=83eb773e262fa705eaebbf3de40db71d53fabf2e
|
||||
TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
|
||||
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
|
||||
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
|
||||
TOOL_JEMALLOC=jemalloc-3.3.1/2f45f3a
|
||||
TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
|
||||
|
||||
# location of libhdfs libraries
|
||||
if test "$USE_HDFS"; then
|
||||
|
|
@ -27,11 +27,6 @@ SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
|
|||
ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include"
|
||||
ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a"
|
||||
|
||||
# location of boost headers and libraries
|
||||
THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/include"
|
||||
THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
|
||||
THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/lib"
|
||||
|
||||
# location of libevent
|
||||
LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
|
||||
LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
|
||||
|
|
@ -40,7 +35,7 @@ LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
|
|||
export USE_SSE=" -msse -msse4.2 "
|
||||
|
||||
CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/gcc"
|
||||
CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
|
||||
CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $LIBEVENT_INCLUDE"
|
||||
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
|
||||
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
|
||||
|
||||
|
|
@ -48,8 +43,8 @@ CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/bin/gold -m64 -mtune=g
|
|||
CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
|
||||
|
||||
EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
|
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
|
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
|
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
|
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $LIBEVENT_LIBS"
|
||||
|
||||
PLATFORM_LDFLAGS="-L$TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/lib -L$TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
|
||||
|
||||
|
|
|
|||
|
|
@ -509,6 +509,7 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
|
|||
break;
|
||||
case kCurrentFile:
|
||||
case kDBLockFile:
|
||||
case kIdentityFile:
|
||||
case kMetaDatabase:
|
||||
keep = true;
|
||||
break;
|
||||
|
|
@ -639,6 +640,13 @@ Status DBImpl::Recover(VersionEdit* edit, MemTable* external_table,
|
|||
dbname_, "exists (error_if_exists is true)");
|
||||
}
|
||||
}
|
||||
// Check for the IDENTITY file and create it if not there
|
||||
if (!env_->FileExists(IdentityFileName(dbname_))) {
|
||||
s = SetIdentityFile(env_, dbname_);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Status s = versions_->Recover();
|
||||
|
|
|
|||
|
|
@ -1630,12 +1630,12 @@ TEST(DBTest, ManifestRollOver) {
|
|||
ASSERT_OK(Put("manifest_key1", std::string(1000, '1')));
|
||||
ASSERT_OK(Put("manifest_key2", std::string(1000, '2')));
|
||||
ASSERT_OK(Put("manifest_key3", std::string(1000, '3')));
|
||||
uint64_t manifest_before_fulsh =
|
||||
uint64_t manifest_before_flush =
|
||||
dbfull()->TEST_Current_Manifest_FileNo();
|
||||
dbfull()->Flush(FlushOptions()); // This should trigger LogAndApply.
|
||||
uint64_t manifest_after_flush =
|
||||
dbfull()->TEST_Current_Manifest_FileNo();
|
||||
ASSERT_GT(manifest_after_flush, manifest_before_fulsh);
|
||||
ASSERT_GT(manifest_after_flush, manifest_before_flush);
|
||||
Reopen(&options);
|
||||
ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(),
|
||||
manifest_after_flush);
|
||||
|
|
@ -1647,6 +1647,36 @@ TEST(DBTest, ManifestRollOver) {
|
|||
} while (ChangeCompactOptions());
|
||||
}
|
||||
|
||||
TEST(DBTest, IdentityAcrossRestarts) {
|
||||
do {
|
||||
std::string idfilename = IdentityFileName(dbname_);
|
||||
unique_ptr<SequentialFile> idfile;
|
||||
const EnvOptions soptions;
|
||||
ASSERT_OK(env_->NewSequentialFile(idfilename, &idfile, soptions));
|
||||
char buffer1[100];
|
||||
Slice id1;
|
||||
ASSERT_OK(idfile->Read(100, &id1, buffer1));
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Reopen(&options);
|
||||
char buffer2[100];
|
||||
Slice id2;
|
||||
ASSERT_OK(env_->NewSequentialFile(idfilename, &idfile, soptions));
|
||||
ASSERT_OK(idfile->Read(100, &id2, buffer2));
|
||||
// id1 should match id2 because identity was not regenerated
|
||||
ASSERT_EQ(id1.ToString(), id2.ToString());
|
||||
|
||||
ASSERT_OK(env_->DeleteFile(idfilename));
|
||||
Reopen(&options);
|
||||
char buffer3[100];
|
||||
Slice id3;
|
||||
ASSERT_OK(env_->NewSequentialFile(idfilename, &idfile, soptions));
|
||||
ASSERT_OK(idfile->Read(100, &id3, buffer3));
|
||||
// id1 should NOT match id2 because identity was regenerated
|
||||
ASSERT_NE(id1.ToString(0), id3.ToString());
|
||||
} while (ChangeCompactOptions());
|
||||
}
|
||||
|
||||
TEST(DBTest, RecoverWithLargeLog) {
|
||||
do {
|
||||
{
|
||||
|
|
|
|||
|
|
@ -14,12 +14,11 @@
|
|||
#include "db/write_batch_internal.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
#include "boost/lexical_cast.hpp"
|
||||
#include "rocksdb/env.h"
|
||||
#include <vector>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <stdlib.h>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
|
|
@ -62,7 +61,7 @@ class DeleteFileTest {
|
|||
options.sync = false;
|
||||
ReadOptions roptions;
|
||||
for (int i = startkey; i < (numkeys + startkey) ; i++) {
|
||||
std::string temp = boost::lexical_cast<std::string>(i);
|
||||
std::string temp = std::to_string(i);
|
||||
Slice key(temp);
|
||||
Slice value(temp);
|
||||
ASSERT_OK(db_->Put(options, key, value));
|
||||
|
|
|
|||
|
|
@ -127,7 +127,12 @@ std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
|
|||
return dbname + buf;
|
||||
}
|
||||
|
||||
std::string IdentityFileName(const std::string& dbname) {
|
||||
return dbname + "/IDENTITY";
|
||||
}
|
||||
|
||||
// Owned filenames have the form:
|
||||
// dbname/IDENTITY
|
||||
// dbname/CURRENT
|
||||
// dbname/LOCK
|
||||
// dbname/LOG
|
||||
|
|
@ -143,7 +148,10 @@ bool ParseFileName(const std::string& fname,
|
|||
if (fname.length() > 1 && fname[0] == '/') {
|
||||
rest.remove_prefix(1);
|
||||
}
|
||||
if (rest == "CURRENT") {
|
||||
if (rest == "IDENTITY") {
|
||||
*number = 0;
|
||||
*type = kIdentityFile;
|
||||
} else if (rest == "CURRENT") {
|
||||
*number = 0;
|
||||
*type = kCurrentFile;
|
||||
} else if (rest == "LOCK") {
|
||||
|
|
@ -223,4 +231,18 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
|
|||
return s;
|
||||
}
|
||||
|
||||
Status SetIdentityFile(Env* env, const std::string& dbname) {
|
||||
std::string id = env->GenerateUniqueId();
|
||||
assert(!id.empty());
|
||||
std::string tmp = TempFileName(dbname, id.size());
|
||||
Status s = WriteStringToFileSync(env, id, tmp);
|
||||
if (s.ok()) {
|
||||
s = env->RenameFile(tmp, IdentityFileName(dbname));
|
||||
}
|
||||
if (!s.ok()) {
|
||||
env->DeleteFile(tmp);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ enum FileType {
|
|||
kCurrentFile,
|
||||
kTempFile,
|
||||
kInfoLogFile, // Either the current one, or an old one
|
||||
kMetaDatabase
|
||||
kMetaDatabase,
|
||||
kIdentityFile
|
||||
};
|
||||
|
||||
// Return the name of the log file with the specified number
|
||||
|
|
@ -82,6 +83,11 @@ extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
|
|||
extern std::string MetaDatabaseName(const std::string& dbname,
|
||||
uint64_t number);
|
||||
|
||||
// Return the name of the Identity file which stores a unique number for the db
|
||||
// that will get regenerated if the db loses all its data and is recreated fresh
|
||||
// either from a backup-image or empty
|
||||
extern std::string IdentityFileName(const std::string& dbname);
|
||||
|
||||
// If filename is a rocksdb file, store the type of the file in *type.
|
||||
// The number encoded in the filename is stored in *number. If the
|
||||
// filename was successfully parsed, returns true. Else return false.
|
||||
|
|
@ -94,5 +100,7 @@ extern bool ParseFileName(const std::string& filename,
|
|||
extern Status SetCurrentFile(Env* env, const std::string& dbname,
|
||||
uint64_t descriptor_number);
|
||||
|
||||
// Make the IDENTITY file for the db
|
||||
extern Status SetIdentityFile(Env* env, const std::string& dbname);
|
||||
|
||||
} // namespace rocksdb
|
||||
|
|
|
|||
|
|
@ -233,6 +233,9 @@ class Env {
|
|||
// Converts seconds-since-Jan-01-1970 to a printable string
|
||||
virtual std::string TimeToString(uint64_t time) = 0;
|
||||
|
||||
// Generates a unique id that can be used to identify a db
|
||||
virtual std::string GenerateUniqueId();
|
||||
|
||||
private:
|
||||
// No copying allowed
|
||||
Env(const Env&);
|
||||
|
|
|
|||
|
|
@ -1386,6 +1386,25 @@ void PosixEnv::StartThread(void (*function)(void* arg), void* arg) {
|
|||
|
||||
} // namespace
|
||||
|
||||
std::string Env::GenerateUniqueId() {
|
||||
std::string uuid_file = "/proc/sys/kernel/random/uuid";
|
||||
if (FileExists(uuid_file)) {
|
||||
std::string uuid;
|
||||
Status s = ReadFileToString(this, uuid_file, &uuid);
|
||||
if (s.ok()) {
|
||||
return uuid;
|
||||
}
|
||||
}
|
||||
// Could not read uuid_file - generate uuid using "nanos-random"
|
||||
Random64 r(time(nullptr));
|
||||
uint64_t random_uuid_portion =
|
||||
r.Uniform(std::numeric_limits<uint64_t>::max());
|
||||
uint64_t nanos_uuid_portion = NowNanos();
|
||||
char uuid2[200];
|
||||
snprintf(uuid2, 200, "%lx-%lx", nanos_uuid_portion, random_uuid_portion);
|
||||
return uuid2;
|
||||
}
|
||||
|
||||
Env* Env::Default() {
|
||||
static PosixEnv default_env;
|
||||
return &default_env;
|
||||
|
|
|
|||
Loading…
Reference in New Issue