dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enhance the ldb tool to support ttl databases 

Summary: ldb works with raw data from the database and needs to be aware of ttl-database to work with it meaningfully. '-ttl' option now tells it that. Also added onto the ldb_test.py test. This option may be specified alongwith put, get, scan or dump. There is no support to provide a ttl-value and it uses default forever because there is no use-case for this currently.

Test Plan: make ldb_test; python tools/ldb_test.py

Reviewers: dhruba, sheki, haobo, vamsi

Reviewed By: sheki

CC: leveldb

Differential Revision: https://reviews.facebook.net/D10797
This commit is contained in:
Mayank Agarwal 2013-05-13 19:11:56 -07:00
parent 8c9411c637
commit 8a48410f09
4 changed files with 65 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
tools/ldb_test.py
View File

@ -43,7 +43,7 @@ class LDBTestCase(unittest.TestCase):
def dbParam(self, dbName):
return "--db=%s" % os.path.join(self.TMP_DIR, dbName)
def assertRunOKFull(self, params, expectedOutput):
def assertRunOKFull(self, params, expectedOutput, unexpected=False):
"""
All command-line params must be specified.
Allows full flexibility in testing; for example: missing db param.
@ -52,7 +52,10 @@ class LDBTestCase(unittest.TestCase):
output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" %
params, shell=True)
self.assertEquals(output.strip(), expectedOutput.strip());
if not unexpected:
self.assertEqual(output.strip(), expectedOutput.strip())
else:
self.assertNotEqual(output.strip(), expectedOutput.strip())
def assertRunFAILFull(self, params):
"""
@ -70,13 +73,13 @@ class LDBTestCase(unittest.TestCase):
"Exception should have been raised for command with params: %s" %
params)
def assertRunOK(self, params, expectedOutput):
def assertRunOK(self, params, expectedOutput, unexpected=False):
"""
Uses the default test db.
"""
self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params),
expectedOutput)
expectedOutput, unexpected)
def assertRunFAIL(self, params):
"""
@ -123,7 +126,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("scan", "x2 : y2\nx3 : y3")
self.assertRunOK("delete NonExistentKey", "OK")
# It is wierd that GET and SCAN raise exception for
# It is weird that GET and SCAN raise exception for
# non-existent key, while delete does not
def dumpDb(self, params, dumpFile):
@ -142,6 +145,16 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAIL("batchput k1")
self.assertRunFAIL("batchput k1 v1 k2")
def testInvalidCmdLines(self):
print "Running testInvalidCmdLines..."
# db not specified
self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
# No param called he
self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing")
# max_keys is not applicable for put
self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing")
# hex has invalid boolean value
def testHexPutGet(self):
print "Running testHexPutGet..."
self.assertRunOK("put a1 b1 --create_if_missing", "OK")
@ -171,6 +184,18 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("delete --hex 0x6133", "OK")
self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4")
def testTtlPutGet(self):
print "Running testTtlPutGet..."
self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK")
self.assertRunOK("scan ", "a1 : b1", True)
self.assertRunOK("dump --ttl ", "a1 ==> b1", True)
self.assertRunOK("scan --hex --ttl", "0x6131 : 0x6231")
self.assertRunOK("get a1", "b1", True)
self.assertRunOK("get --ttl a1", "b1")
self.assertRunOK("put a3 b3 --create_if_missing", "OK")
# fails because timstamp's length is greater than value's
self.assertRunFAIL("get --ttl a3")
def testInvalidCmdLines(self):
print "Running testInvalidCmdLines..."
# db not specified

30
util/ldb_cmd.cc
View File

@ -33,6 +33,7 @@ const string LDBCommand::ARG_DB = "db";
const string LDBCommand::ARG_HEX = "hex";
const string LDBCommand::ARG_KEY_HEX = "key_hex";
const string LDBCommand::ARG_VALUE_HEX = "value_hex";
const string LDBCommand::ARG_TTL = "ttl";
const string LDBCommand::ARG_FROM = "from";
const string LDBCommand::ARG_TO = "to";
const string LDBCommand::ARG_MAX_KEYS = "max_keys";
@ -538,9 +539,9 @@ const string DBDumperCommand::ARG_STATS = "stats";
DBDumperCommand::DBDumperCommand(const vector<string>& params,
const map<string, string>& options, const vector<string>& flags) :
LDBCommand(options, flags, true,
BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
ARG_FROM, ARG_TO, ARG_MAX_KEYS,
ARG_COUNT_ONLY, ARG_STATS})),
BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
ARG_VALUE_HEX, ARG_FROM, ARG_TO,
ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_STATS})),
null_from_(true),
null_to_(true),
max_keys_(-1),
@ -568,7 +569,7 @@ DBDumperCommand::DBDumperCommand(const vector<string>& params,
" has an invalid value");
} catch(const out_of_range&) {
exec_state_ = LDBCommandExecuteResult::FAILED(ARG_MAX_KEYS +
" has a valuei out-of-range");
" has a value out-of-range");
}
}
@ -636,8 +637,8 @@ void DBDumperCommand::DoCommand() {
++count;
if (!count_only_) {
string str = PrintKeyValue(iter->key().ToString(),
iter->value().ToString(),
is_key_hex_, is_value_hex_);
iter->value().ToString(),
is_key_hex_, is_value_hex_);
fprintf(stdout, "%s\n", str.c_str());
}
}
@ -908,8 +909,9 @@ void WALDumperCommand::DoCommand() {
GetCommand::GetCommand(const vector<string>& params,
const map<string, string>& options, const vector<string>& flags) :
LDBCommand(options, flags, true,
BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
LDBCommand(options, flags, true, BuildCmdLineOptions({ARG_TTL, ARG_HEX,
ARG_KEY_HEX,
ARG_VALUE_HEX})) {
if (params.size() != 1) {
exec_state_ = LDBCommandExecuteResult::FAILED(
@ -996,7 +998,7 @@ void ApproxSizeCommand::DoCommand() {
BatchPutCommand::BatchPutCommand(const vector<string>& params,
const map<string, string>& options, const vector<string>& flags) :
LDBCommand(options, flags, false,
BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
ARG_CREATE_IF_MISSING})) {
if (params.size() < 2) {
@ -1048,8 +1050,9 @@ Options BatchPutCommand::PrepareOptionsForOpenDB() {
ScanCommand::ScanCommand(const vector<string>& params,
const map<string, string>& options, const vector<string>& flags) :
LDBCommand(options, flags, true,
BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
ARG_FROM, ARG_TO, ARG_MAX_KEYS})),
BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
ARG_VALUE_HEX, ARG_FROM, ARG_TO,
ARG_MAX_KEYS})),
start_key_specified_(false),
end_key_specified_(false),
max_keys_scanned_(-1) {
@ -1158,7 +1161,7 @@ void DeleteCommand::DoCommand() {
PutCommand::PutCommand(const vector<string>& params,
const map<string, string>& options, const vector<string>& flags) :
LDBCommand(options, flags, false,
BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
ARG_CREATE_IF_MISSING})) {
if (params.size() != 2) {
@ -1209,7 +1212,8 @@ const char* DBQuerierCommand::DELETE_CMD = "delete";
DBQuerierCommand::DBQuerierCommand(const vector<string>& params,
const map<string, string>& options, const vector<string>& flags) :
LDBCommand(options, flags, false,
BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
ARG_VALUE_HEX})) {
}

18
util/ldb_cmd.h
View File

@ -12,14 +12,14 @@
#include <algorithm>
#include <stdio.h>
#include "leveldb/db.h"
#include "db/version_set.h"
#include "leveldb/env.h"
#include "leveldb/options.h"
#include "leveldb/iterator.h"
#include "leveldb/slice.h"
#include "db/version_set.h"
#include "util/logging.h"
#include "util/ldb_cmd_execute_result.h"
#include "utilities/utility_db.h"
using std::string;
using std::map;
@ -36,6 +36,7 @@ public:
static const string ARG_HEX;
static const string ARG_KEY_HEX;
static const string ARG_VALUE_HEX;
static const string ARG_TTL;
static const string ARG_FROM;
static const string ARG_TO;
static const string ARG_MAX_KEYS;
@ -157,6 +158,9 @@ protected:
/** If true, the value is input/output as hex in get/put/scan/delete etc. */
bool is_value_hex_;
/** If true, the value is treated as timestamp suffixed */
bool is_db_ttl_;
/**
* Map of options passed on the command-line.
*/
@ -179,6 +183,7 @@ protected:
is_read_only_(is_read_only),
is_key_hex_(false),
is_value_hex_(false),
is_db_ttl_(false),
option_map_(options),
flags_(flags),
valid_cmd_line_options_(valid_cmd_line_options) {
@ -190,6 +195,7 @@ protected:
is_key_hex_ = IsKeyHex(options, flags);
is_value_hex_ = IsValueHex(options, flags);
is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
}
void OpenDB() {
@ -199,7 +205,13 @@ protected:
}
// Open the DB.
Status st;
if (is_read_only_) {
if (is_db_ttl_) {
if (is_read_only_) {
st = UtilityDB::OpenTtlDB(opt, db_path_, &db_, 0, true);
} else {
st = UtilityDB::OpenTtlDB(opt, db_path_, &db_);
}
} else if (is_read_only_) {
st = DB::OpenForReadOnly(opt, db_path_, &db_);
} else {
st = DB::Open(opt, db_path_, &db_);

3
util/ldb_tool.cc
View File

@ -18,6 +18,9 @@ public:
ret.append("\n");
ret.append("The following optional parameters control if keys/values are "
"input/output as hex or as plain strings:\n");
ret.append(" --" + LDBCommand::ARG_TTL +
" with 'put','get','scan','dump','query','batchput'"
" : DB supports ttl and value is internally timestamp-suffixed\n");
ret.append(" --" + LDBCommand::ARG_KEY_HEX +
" : Keys are input/output as hex\n");
ret.append(" --" + LDBCommand::ARG_VALUE_HEX +