mirror of https://github.com/facebook/rocksdb.git
Update dump_tool and undump_tool to accept Options
Summary: Refactor dump_tool and undump_tool so that it's possible to use them with customized options for example setting a specific comparator similar to what Dragon is doing with the LdbTool https://phabricator.fb.com/diffusion/FBCODE/browse/master/dragon/tools/Ldb.cpp Test Plan: compiles used it to dump / undump a dragon shard Reviewers: sdong, yhchiang, igor Reviewed By: igor Subscribers: dhruba, adsharma Differential Revision: https://reviews.facebook.net/D47853
This commit is contained in:
Islam AbdelRahman
parent
eb5b637fb0
commit
9babaeed16
|
|
@ -0,0 +1,45 @@
|
|||
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#pragma once
|
||||
#ifndef ROCKSDB_LITE
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
struct DumpOptions {
|
||||
// Database that will be dumped
|
||||
std::string db_path;
|
||||
// File location that will contain dump output
|
||||
std::string dump_location;
|
||||
// Dont include db information header in the dump
|
||||
bool anonymous = false;
|
||||
};
|
||||
|
||||
class DbDumpTool {
|
||||
public:
|
||||
bool Run(const DumpOptions& dump_options,
|
||||
rocksdb::Options options = rocksdb::Options());
|
||||
};
|
||||
|
||||
struct UndumpOptions {
|
||||
// Database that we will load the dumped file into
|
||||
std::string db_path;
|
||||
// File location of the dumped file that will be loaded
|
||||
std::string dump_location;
|
||||
// Compact the db after loading the dumped file
|
||||
bool compact_db = false;
|
||||
};
|
||||
|
||||
class DbUndumpTool {
|
||||
public:
|
||||
bool Run(const UndumpOptions& undump_options,
|
||||
rocksdb::Options options = rocksdb::Options());
|
||||
};
|
||||
} // namespace rocksdb
|
||||
#endif // ROCKSDB_LITE
|
||||
1
src.mk
1
src.mk
|
|
@ -76,6 +76,7 @@ LIB_SOURCES = \
|
|||
table/plain_table_reader.cc \
|
||||
table/table_properties.cc \
|
||||
table/two_level_iterator.cc \
|
||||
tools/dump/db_dump_tool.cc \
|
||||
util/arena.cc \
|
||||
util/auto_roll_logger.cc \
|
||||
util/bloom.cc \
|
||||
|
|
|
|||
|
|
@ -0,0 +1,261 @@
|
|||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
|
||||
#ifndef __STDC_FORMAT_MACROS
|
||||
#define __STDC_FORMAT_MACROS
|
||||
#endif
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <iostream>
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/db_dump_tool.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
bool DbDumpTool::Run(const DumpOptions& dump_options,
|
||||
rocksdb::Options options) {
|
||||
rocksdb::DB* dbptr;
|
||||
rocksdb::Status status;
|
||||
std::unique_ptr<rocksdb::WritableFile> dumpfile;
|
||||
char hostname[1024];
|
||||
int64_t timesec;
|
||||
std::string abspath;
|
||||
char json[4096];
|
||||
|
||||
static const char* magicstr = "ROCKDUMP";
|
||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
|
||||
|
||||
rocksdb::Env* env = rocksdb::Env::Default();
|
||||
|
||||
// Open the database
|
||||
options.create_if_missing = false;
|
||||
status = rocksdb::DB::OpenForReadOnly(options, dump_options.db_path, &dbptr);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to open database '" << dump_options.db_path
|
||||
<< "' for reading: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
const std::unique_ptr<rocksdb::DB> db(dbptr);
|
||||
|
||||
status = env->NewWritableFile(dump_options.dump_location, &dumpfile,
|
||||
rocksdb::EnvOptions());
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to open dump file '" << dump_options.dump_location
|
||||
<< "' for writing: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
rocksdb::Slice magicslice(magicstr, 8);
|
||||
status = dumpfile->Append(magicslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
rocksdb::Slice versionslice(versionstr, 8);
|
||||
status = dumpfile->Append(versionslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (dump_options.anonymous) {
|
||||
snprintf(json, sizeof(json), "{}");
|
||||
} else {
|
||||
status = env->GetHostName(hostname, sizeof(hostname));
|
||||
status = env->GetCurrentTime(×ec);
|
||||
status = env->GetAbsolutePath(dump_options.db_path, &abspath);
|
||||
snprintf(json, sizeof(json),
|
||||
"{ \"database-path\": \"%s\", \"hostname\": \"%s\", "
|
||||
"\"creation-time\": %" PRIi64 " }",
|
||||
abspath.c_str(), hostname, timesec);
|
||||
}
|
||||
|
||||
rocksdb::Slice infoslice(json, strlen(json));
|
||||
char infosize[4];
|
||||
rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size());
|
||||
rocksdb::Slice infosizeslice(infosize, 4);
|
||||
status = dumpfile->Append(infosizeslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
status = dumpfile->Append(infoslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
const std::unique_ptr<rocksdb::Iterator> it(
|
||||
db->NewIterator(rocksdb::ReadOptions()));
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
char keysize[4];
|
||||
rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size());
|
||||
rocksdb::Slice keysizeslice(keysize, 4);
|
||||
status = dumpfile->Append(keysizeslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
status = dumpfile->Append(it->key());
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
char valsize[4];
|
||||
rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size());
|
||||
rocksdb::Slice valsizeslice(valsize, 4);
|
||||
status = dumpfile->Append(valsizeslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
status = dumpfile->Append(it->value());
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (!it->status().ok()) {
|
||||
std::cerr << "Database iteration failed: " << status.ToString()
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DbUndumpTool::Run(const UndumpOptions& undump_options,
|
||||
rocksdb::Options options) {
|
||||
rocksdb::DB* dbptr;
|
||||
rocksdb::Status status;
|
||||
rocksdb::Env* env;
|
||||
std::unique_ptr<rocksdb::SequentialFile> dumpfile;
|
||||
rocksdb::Slice slice;
|
||||
char scratch8[8];
|
||||
|
||||
static const char* magicstr = "ROCKDUMP";
|
||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
|
||||
|
||||
env = rocksdb::Env::Default();
|
||||
|
||||
status = env->NewSequentialFile(undump_options.dump_location, &dumpfile,
|
||||
rocksdb::EnvOptions());
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to open dump file '" << undump_options.dump_location
|
||||
<< "' for reading: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
status = dumpfile->Read(8, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 8 ||
|
||||
memcmp(slice.data(), magicstr, 8) != 0) {
|
||||
std::cerr << "File '" << undump_options.dump_location
|
||||
<< "' is not a recognizable dump file." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
status = dumpfile->Read(8, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 8 ||
|
||||
memcmp(slice.data(), versionstr, 8) != 0) {
|
||||
std::cerr << "File '" << undump_options.dump_location
|
||||
<< "' version not recognized." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 4) {
|
||||
std::cerr << "Unable to read info blob size." << std::endl;
|
||||
return false;
|
||||
}
|
||||
uint32_t infosize = rocksdb::DecodeFixed32(slice.data());
|
||||
status = dumpfile->Skip(infosize);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
options.create_if_missing = true;
|
||||
status = rocksdb::DB::Open(options, undump_options.db_path, &dbptr);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to open database '" << undump_options.db_path
|
||||
<< "' for writing: " << status.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
const std::unique_ptr<rocksdb::DB> db(dbptr);
|
||||
|
||||
uint32_t last_keysize = 64;
|
||||
size_t last_valsize = 1 << 20;
|
||||
std::unique_ptr<char[]> keyscratch(new char[last_keysize]);
|
||||
std::unique_ptr<char[]> valscratch(new char[last_valsize]);
|
||||
|
||||
while (1) {
|
||||
uint32_t keysize, valsize;
|
||||
rocksdb::Slice keyslice;
|
||||
rocksdb::Slice valslice;
|
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 4) break;
|
||||
keysize = rocksdb::DecodeFixed32(slice.data());
|
||||
if (keysize > last_keysize) {
|
||||
while (keysize > last_keysize) last_keysize *= 2;
|
||||
keyscratch = std::unique_ptr<char[]>(new char[last_keysize]);
|
||||
}
|
||||
|
||||
status = dumpfile->Read(keysize, &keyslice, keyscratch.get());
|
||||
if (!status.ok() || keyslice.size() != keysize) {
|
||||
std::cerr << "Key read failure: "
|
||||
<< (status.ok() ? "insufficient data" : status.ToString())
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 4) {
|
||||
std::cerr << "Unable to read value size: "
|
||||
<< (status.ok() ? "insufficient data" : status.ToString())
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
valsize = rocksdb::DecodeFixed32(slice.data());
|
||||
if (valsize > last_valsize) {
|
||||
while (valsize > last_valsize) last_valsize *= 2;
|
||||
valscratch = std::unique_ptr<char[]>(new char[last_valsize]);
|
||||
}
|
||||
|
||||
status = dumpfile->Read(valsize, &valslice, valscratch.get());
|
||||
if (!status.ok() || valslice.size() != valsize) {
|
||||
std::cerr << "Unable to read value: "
|
||||
<< (status.ok() ? "insufficient data" : status.ToString())
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
status = db->Put(rocksdb::WriteOptions(), keyslice, valslice);
|
||||
if (!status.ok()) {
|
||||
fprintf(stderr, "Unable to write database entry\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (undump_options.compact_db) {
|
||||
status = db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr);
|
||||
if (!status.ok()) {
|
||||
fprintf(stderr,
|
||||
"Unable to compact the database after loading the dumped file\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace rocksdb
|
||||
#endif // ROCKSDB_LITE
|
||||
|
|
@ -1,154 +1,63 @@
|
|||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#ifndef GFLAGS
|
||||
#if !(defined GFLAGS) || defined(ROCKSDB_LITE)
|
||||
|
||||
#include <cstdio>
|
||||
int main() {
|
||||
#ifndef GFLAGS
|
||||
fprintf(stderr, "Please install gflags to run rocksdb tools\n");
|
||||
#endif
|
||||
#ifdef ROCKSDB_LITE
|
||||
fprintf(stderr, "DbDumpTool is not supported in ROCKSDB_LITE\n");
|
||||
#endif
|
||||
return 1;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#ifndef __STDC_FORMAT_MACROS
|
||||
#define __STDC_FORMAT_MACROS
|
||||
#endif
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <gflags/gflags.h>
|
||||
#include <iostream>
|
||||
#include "rocksdb/convenience.h"
|
||||
#include "rocksdb/db_dump_tool.h"
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
DEFINE_bool(anonymous, false, "Output an empty information blob.");
|
||||
|
||||
void usage(const char* name) {
|
||||
std::cout << "usage: " << name << " [--anonymous] <db> <dumpfile>"
|
||||
<< std::endl;
|
||||
}
|
||||
DEFINE_string(db_path, "", "Path to the db that will be dumped");
|
||||
DEFINE_string(dump_location, "", "Path to where the dump file location");
|
||||
DEFINE_bool(anonymous, false,
|
||||
"Remove information like db path, creation time from dumped file");
|
||||
DEFINE_string(db_options, "",
|
||||
"Options string used to open the database that will be dumped");
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
rocksdb::DB* dbptr;
|
||||
rocksdb::Options options;
|
||||
rocksdb::Status status;
|
||||
std::unique_ptr<rocksdb::WritableFile> dumpfile;
|
||||
char hostname[1024];
|
||||
int64_t timesec;
|
||||
std::string abspath;
|
||||
char json[4096];
|
||||
|
||||
GFLAGS::ParseCommandLineFlags(&argc, &argv, true);
|
||||
|
||||
static const char* magicstr = "ROCKDUMP";
|
||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
|
||||
|
||||
if (argc != 3) {
|
||||
usage(argv[0]);
|
||||
exit(1);
|
||||
if (FLAGS_db_path == "" || FLAGS_dump_location == "") {
|
||||
fprintf(stderr, "Please set --db_path and --dump_location\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
rocksdb::Env* env = rocksdb::Env::Default();
|
||||
rocksdb::DumpOptions dump_options;
|
||||
dump_options.db_path = FLAGS_db_path;
|
||||
dump_options.dump_location = FLAGS_dump_location;
|
||||
dump_options.anonymous = FLAGS_anonymous;
|
||||
|
||||
// Open the database
|
||||
options.create_if_missing = false;
|
||||
status = rocksdb::DB::OpenForReadOnly(options, argv[1], &dbptr);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to open database '" << argv[1]
|
||||
<< "' for reading: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
const std::unique_ptr<rocksdb::DB> db(dbptr);
|
||||
|
||||
status = env->NewWritableFile(argv[2], &dumpfile, rocksdb::EnvOptions());
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to open dump file '" << argv[2]
|
||||
<< "' for writing: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
rocksdb::Slice magicslice(magicstr, 8);
|
||||
status = dumpfile->Append(magicslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
rocksdb::Slice versionslice(versionstr, 8);
|
||||
status = dumpfile->Append(versionslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (FLAGS_anonymous) {
|
||||
snprintf(json, sizeof(json), "{}");
|
||||
} else {
|
||||
status = env->GetHostName(hostname, sizeof(hostname));
|
||||
status = env->GetCurrentTime(×ec);
|
||||
status = env->GetAbsolutePath(argv[1], &abspath);
|
||||
snprintf(json, sizeof(json),
|
||||
"{ \"database-path\": \"%s\", \"hostname\": \"%s\", "
|
||||
"\"creation-time\": %" PRIi64 " }",
|
||||
abspath.c_str(), hostname, timesec);
|
||||
}
|
||||
|
||||
rocksdb::Slice infoslice(json, strlen(json));
|
||||
char infosize[4];
|
||||
rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size());
|
||||
rocksdb::Slice infosizeslice(infosize, 4);
|
||||
status = dumpfile->Append(infosizeslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
status = dumpfile->Append(infoslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
const std::unique_ptr<rocksdb::Iterator> it(
|
||||
db->NewIterator(rocksdb::ReadOptions()));
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
char keysize[4];
|
||||
rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size());
|
||||
rocksdb::Slice keysizeslice(keysize, 4);
|
||||
status = dumpfile->Append(keysizeslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
rocksdb::Options db_options;
|
||||
if (FLAGS_db_options != "") {
|
||||
rocksdb::Options parsed_options;
|
||||
rocksdb::Status s = rocksdb::GetOptionsFromString(
|
||||
db_options, FLAGS_db_options, &parsed_options);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "Cannot parse provided db_options\n");
|
||||
return 1;
|
||||
}
|
||||
status = dumpfile->Append(it->key());
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
char valsize[4];
|
||||
rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size());
|
||||
rocksdb::Slice valsizeslice(valsize, 4);
|
||||
status = dumpfile->Append(valsizeslice);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
status = dumpfile->Append(it->value());
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Append failed: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
if (!it->status().ok()) {
|
||||
std::cerr << "Database iteration failed: " << status.ToString()
|
||||
<< std::endl;
|
||||
exit(1);
|
||||
db_options = parsed_options;
|
||||
}
|
||||
|
||||
rocksdb::DbDumpTool tool;
|
||||
if (!tool.Run(dump_options, db_options)) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // GFLAGS
|
||||
#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE)
|
||||
|
|
|
|||
|
|
@ -1,136 +1,62 @@
|
|||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#if !(defined GFLAGS) || defined(ROCKSDB_LITE)
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
void usage(const char *name) {
|
||||
std::cout << "usage: " << name << " <dumpfile> <rocksdb>" << std::endl;
|
||||
#include <cstdio>
|
||||
int main() {
|
||||
#ifndef GFLAGS
|
||||
fprintf(stderr, "Please install gflags to run rocksdb tools\n");
|
||||
#endif
|
||||
#ifdef ROCKSDB_LITE
|
||||
fprintf(stderr, "DbUndumpTool is not supported in ROCKSDB_LITE\n");
|
||||
#endif
|
||||
return 1;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
#include "rocksdb/convenience.h"
|
||||
#include "rocksdb/db_dump_tool.h"
|
||||
|
||||
DEFINE_string(dump_location, "", "Path to the dump file that will be loaded");
|
||||
DEFINE_string(db_path, "", "Path to the db that we will undump the file into");
|
||||
DEFINE_bool(compact, false, "Compact the db after loading the dumped file");
|
||||
DEFINE_string(db_options, "",
|
||||
"Options string used to open the database that will be loaded");
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
rocksdb::DB *dbptr;
|
||||
rocksdb::Options options;
|
||||
rocksdb::Status status;
|
||||
rocksdb::Env *env;
|
||||
std::unique_ptr<rocksdb::SequentialFile> dumpfile;
|
||||
rocksdb::Slice slice;
|
||||
char scratch8[8];
|
||||
GFLAGS::ParseCommandLineFlags(&argc, &argv, true);
|
||||
|
||||
static const char *magicstr = "ROCKDUMP";
|
||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
|
||||
|
||||
if (argc != 3) {
|
||||
usage(argv[0]);
|
||||
exit(1);
|
||||
if (FLAGS_db_path == "" || FLAGS_dump_location == "") {
|
||||
fprintf(stderr, "Please set --db_path and --dump_location\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
env = rocksdb::Env::Default();
|
||||
rocksdb::UndumpOptions undump_options;
|
||||
undump_options.db_path = FLAGS_db_path;
|
||||
undump_options.dump_location = FLAGS_dump_location;
|
||||
undump_options.compact_db = FLAGS_compact;
|
||||
|
||||
status = env->NewSequentialFile(argv[1], &dumpfile, rocksdb::EnvOptions());
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to open dump file '" << argv[1]
|
||||
<< "' for reading: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
status = dumpfile->Read(8, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 8 ||
|
||||
memcmp(slice.data(), magicstr, 8) != 0) {
|
||||
std::cerr << "File '" << argv[1] << "' is not a recognizable dump file."
|
||||
<< std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
status = dumpfile->Read(8, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 8 ||
|
||||
memcmp(slice.data(), versionstr, 8) != 0) {
|
||||
std::cerr << "File '" << argv[1] << "' version not recognized."
|
||||
<< std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 4) {
|
||||
std::cerr << "Unable to read info blob size." << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
uint32_t infosize = rocksdb::DecodeFixed32(slice.data());
|
||||
status = dumpfile->Skip(infosize);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
options.create_if_missing = true;
|
||||
status = rocksdb::DB::Open(options, argv[2], &dbptr);
|
||||
if (!status.ok()) {
|
||||
std::cerr << "Unable to open database '" << argv[2]
|
||||
<< "' for writing: " << status.ToString() << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
const std::unique_ptr<rocksdb::DB> db(dbptr);
|
||||
|
||||
uint32_t last_keysize = 64;
|
||||
size_t last_valsize = 1 << 20;
|
||||
std::unique_ptr<char[]> keyscratch(new char[last_keysize]);
|
||||
std::unique_ptr<char[]> valscratch(new char[last_valsize]);
|
||||
|
||||
while (1) {
|
||||
uint32_t keysize, valsize;
|
||||
rocksdb::Slice keyslice;
|
||||
rocksdb::Slice valslice;
|
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 4) break;
|
||||
keysize = rocksdb::DecodeFixed32(slice.data());
|
||||
if (keysize > last_keysize) {
|
||||
while (keysize > last_keysize) last_keysize *= 2;
|
||||
keyscratch = std::unique_ptr<char[]>(new char[last_keysize]);
|
||||
}
|
||||
|
||||
status = dumpfile->Read(keysize, &keyslice, keyscratch.get());
|
||||
if (!status.ok() || keyslice.size() != keysize) {
|
||||
std::cerr << "Key read failure: "
|
||||
<< (status.ok() ? "insufficient data" : status.ToString())
|
||||
<< std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8);
|
||||
if (!status.ok() || slice.size() != 4) {
|
||||
std::cerr << "Unable to read value size: "
|
||||
<< (status.ok() ? "insufficient data" : status.ToString())
|
||||
<< std::endl;
|
||||
exit(1);
|
||||
}
|
||||
valsize = rocksdb::DecodeFixed32(slice.data());
|
||||
if (valsize > last_valsize) {
|
||||
while (valsize > last_valsize) last_valsize *= 2;
|
||||
valscratch = std::unique_ptr<char[]>(new char[last_valsize]);
|
||||
}
|
||||
|
||||
status = dumpfile->Read(valsize, &valslice, valscratch.get());
|
||||
if (!status.ok() || valslice.size() != valsize) {
|
||||
std::cerr << "Unable to read value: "
|
||||
<< (status.ok() ? "insufficient data" : status.ToString())
|
||||
<< std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
status = db->Put(rocksdb::WriteOptions(), keyslice, valslice);
|
||||
if (!status.ok()) {
|
||||
fprintf(stderr, "Unable to write database entry\n");
|
||||
exit(1);
|
||||
rocksdb::Options db_options;
|
||||
if (FLAGS_db_options != "") {
|
||||
rocksdb::Options parsed_options;
|
||||
rocksdb::Status s = rocksdb::GetOptionsFromString(
|
||||
db_options, FLAGS_db_options, &parsed_options);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "Cannot parse provided db_options\n");
|
||||
return 1;
|
||||
}
|
||||
db_options = parsed_options;
|
||||
}
|
||||
|
||||
rocksdb::DbUndumpTool tool;
|
||||
if (!tool.Run(undump_options, db_options)) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,6 @@ TESTDIR=`mktemp -d /tmp/rocksdb-dump-test.XXXXX`
|
|||
DUMPFILE="tools/sample-dump.dmp"
|
||||
|
||||
# Verify that the sample dump file is undumpable and then redumpable.
|
||||
./rocksdb_undump $DUMPFILE $TESTDIR/db
|
||||
./rocksdb_dump --anonymous $TESTDIR/db $TESTDIR/dump
|
||||
./rocksdb_undump --dump_location=$DUMPFILE --db_path=$TESTDIR/db
|
||||
./rocksdb_dump --anonymous --db_path=$TESTDIR/db --dump_location=$TESTDIR/dump
|
||||
cmp $DUMPFILE $TESTDIR/dump
|
||||
|
|
|
|||
Loading…
Reference in New Issue