mirror of https://github.com/facebook/rocksdb.git
Wide Column support in ldb (#11754)
Summary: wide_columns can now be pretty-printed in the following commands - `./ldb dump_wal` - `./ldb dump` - `./ldb idump` - `./ldb dump_live_files` - `./ldb scan` - `./sst_dump --command=scan` There are opportunities to refactor to reduce some nearly identical code. This PR is initial change to add wide column support in `ldb` and `sst_dump` tool. More PRs to come for the refactor. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11754 Test Plan: **New Tests added** - `WideColumnsHelperTest::DumpWideColumns` - `WideColumnsHelperTest::DumpSliceAsWideColumns` **Changes added to existing tests** - `ExternalSSTFileTest::BasicMixed` added to cover mixed case (This test should have been added in https://github.com/facebook/rocksdb/issues/11688). This test does not verify the ldb or sst_dump output. This test was used to create test SST files having some rows with wide columns and some without and the generated SST files were used to manually test sst_dump_tool. - `createSST()` in `sst_dump_test` now takes `wide_column_one_in` to add wide column value in SST **dump_wal** ``` ./ldb dump_wal --walfile=/tmp/rocksdbtest-226125/db_wide_basic_test_2675429_2308393776696827948/000004.log --print_value --header ``` ``` Sequence,Count,ByteSize,Physical Offset,Key(s) : value 1,1,59,0,PUT_ENTITY(0) : 0x:0x68656C6C6F 0x617474725F6E616D6531:0x666F6F 0x617474725F6E616D6532:0x626172 2,1,34,42,PUT_ENTITY(0) : 0x617474725F6F6E65:0x74776F 0x617474725F7468726565:0x666F7572 3,1,17,7d,PUT(0) : 0x7468697264 : 0x62617A ``` **idump** ``` ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ idump ``` ``` 'first' seq:1, type:22 => :hello attr_name1:foo attr_name2:bar 'second' seq:2, type:22 => attr_one:two attr_three:four 'third' seq:3, type:1 => baz Internal keys in range: 3 ``` **SST Dump from dump_live_files** ``` ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ compact ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ dump_live_files ``` ``` ... ============================== SST Files ============================== /tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/000013.sst level:1 ------------------------------ Process /tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/000013.sst Sst file format: block-based 'first' seq:0, type:22 => :hello attr_name1:foo attr_name2:bar 'second' seq:0, type:22 => attr_one:two attr_three:four 'third' seq:0, type:1 => baz ... ``` **dump** ``` ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ dump ``` ``` first ==> :hello attr_name1:foo attr_name2:bar second ==> attr_one:two attr_three:four third ==> baz Keys in range: 3 ``` **scan** ``` ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ scan ``` ``` first : :hello attr_name1:foo attr_name2:bar second : attr_one:two attr_three:four third : baz ``` **sst_dump** ``` ./sst_dump --file=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/000013.sst --command=scan ``` ``` options.env is 0x7ff54b296000 Process /tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/000013.sst Sst file format: block-based from [] to [] 'first' seq:0, type:22 => :hello attr_name1:foo attr_name2:bar 'second' seq:0, type:22 => attr_one:two attr_three:four 'third' seq:0, type:1 => baz ``` Reviewed By: ltamasi Differential Revision: D48837999 Pulled By: jaykorean fbshipit-source-id: b0280f0589d2b9716bb9b50530ffcabb397d140f
This commit is contained in:
Jay Huh
Facebook GitHub Bot
parent
c073c2edde
commit
ea9a5b2914
|
|
@ -714,6 +714,7 @@ set(SOURCES
|
|||
db/wal_manager.cc
|
||||
db/wide/wide_column_serialization.cc
|
||||
db/wide/wide_columns.cc
|
||||
db/wide/wide_columns_helper.cc
|
||||
db/write_batch.cc
|
||||
db/write_batch_base.cc
|
||||
db/write_controller.cc
|
||||
|
|
@ -1357,6 +1358,7 @@ if(WITH_TESTS)
|
|||
db/wal_edit_test.cc
|
||||
db/wide/db_wide_basic_test.cc
|
||||
db/wide/wide_column_serialization_test.cc
|
||||
db/wide/wide_columns_helper_test.cc
|
||||
db/write_batch_test.cc
|
||||
db/write_callback_test.cc
|
||||
db/write_controller_test.cc
|
||||
|
|
|
|||
3
Makefile
3
Makefile
|
|
@ -1984,6 +1984,9 @@ cache_reservation_manager_test: $(OBJ_DIR)/cache/cache_reservation_manager_test.
|
|||
wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_test.o $(TEST_LIBRARY) $(LIBRARY)
|
||||
$(AM_LINK)
|
||||
|
||||
wide_columns_helper_test: $(OBJ_DIR)/db/wide/wide_columns_helper_test.o $(TEST_LIBRARY) $(LIBRARY)
|
||||
$(AM_LINK)
|
||||
|
||||
#-------------------------------------------------
|
||||
# make install related stuff
|
||||
PREFIX ?= /usr/local
|
||||
|
|
|
|||
7
TARGETS
7
TARGETS
|
|
@ -103,6 +103,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
|
|||
"db/wal_manager.cc",
|
||||
"db/wide/wide_column_serialization.cc",
|
||||
"db/wide/wide_columns.cc",
|
||||
"db/wide/wide_columns_helper.cc",
|
||||
"db/write_batch.cc",
|
||||
"db/write_batch_base.cc",
|
||||
"db/write_controller.cc",
|
||||
|
|
@ -5552,6 +5553,12 @@ cpp_unittest_wrapper(name="wide_column_serialization_test",
|
|||
extra_compiler_flags=[])
|
||||
|
||||
|
||||
cpp_unittest_wrapper(name="wide_columns_helper_test",
|
||||
srcs=["db/wide/wide_columns_helper_test.cc"],
|
||||
deps=[":rocksdb_test_lib"],
|
||||
extra_compiler_flags=[])
|
||||
|
||||
|
||||
cpp_unittest_wrapper(name="work_queue_test",
|
||||
srcs=["util/work_queue_test.cc"],
|
||||
deps=[":rocksdb_test_lib"],
|
||||
|
|
|
|||
|
|
@ -588,6 +588,63 @@ TEST_F(ExternalSSTFileTest, BasicWideColumn) {
|
|||
kRangeDelSkipConfigs));
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileTest, BasicMixed) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
|
||||
SstFileWriter sst_file_writer(EnvOptions(), options);
|
||||
|
||||
// Current file size should be 0 after sst_file_writer init and before open
|
||||
// a file.
|
||||
ASSERT_EQ(sst_file_writer.FileSize(), 0);
|
||||
|
||||
std::string file = sst_files_dir_ + "mixed_file.sst";
|
||||
ASSERT_OK(sst_file_writer.Open(file));
|
||||
for (int k = 0; k < 100; k++) {
|
||||
if (k % 5 == 0) {
|
||||
std::string val1 = Key(k) + "_attr_1_val";
|
||||
std::string val2 = Key(k) + "_attr_2_val";
|
||||
WideColumns columns{{"attr_1", val1}, {"attr_2", val2}};
|
||||
ASSERT_OK(sst_file_writer.PutEntity(Key(k), columns));
|
||||
} else {
|
||||
ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
|
||||
}
|
||||
}
|
||||
ExternalSstFileInfo file_info;
|
||||
ASSERT_OK(sst_file_writer.Finish(&file_info));
|
||||
|
||||
// Current file size should be non-zero after success write.
|
||||
ASSERT_GT(sst_file_writer.FileSize(), 0);
|
||||
|
||||
ASSERT_EQ(file_info.file_path, file);
|
||||
ASSERT_EQ(file_info.num_entries, 100);
|
||||
ASSERT_EQ(file_info.smallest_key, Key(0));
|
||||
ASSERT_EQ(file_info.largest_key, Key(99));
|
||||
ASSERT_EQ(file_info.num_range_del_entries, 0);
|
||||
ASSERT_EQ(file_info.smallest_range_del_key, "");
|
||||
ASSERT_EQ(file_info.largest_range_del_key, "");
|
||||
|
||||
DestroyAndReopen(options);
|
||||
// Add file using file path
|
||||
ASSERT_OK(DeprecatedAddFile({file}));
|
||||
ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
|
||||
for (int k = 0; k < 10; k++) {
|
||||
if (k % 5 == 0) {
|
||||
PinnableWideColumns result;
|
||||
ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(),
|
||||
Key(k), &result));
|
||||
std::string val1 = Key(k) + "_attr_1_val";
|
||||
std::string val2 = Key(k) + "_attr_2_val";
|
||||
WideColumns expected_columns{{"attr_1", val1}, {"attr_2", val2}};
|
||||
ASSERT_EQ(result.columns(), expected_columns);
|
||||
} else {
|
||||
ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
|
||||
}
|
||||
}
|
||||
} while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction |
|
||||
kRangeDelSkipConfigs));
|
||||
}
|
||||
|
||||
class SstFileWriterCollector : public TablePropertiesCollector {
|
||||
public:
|
||||
explicit SstFileWriterCollector(const std::string prefix) : prefix_(prefix) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include "db/wide/wide_columns_helper.h"
|
||||
|
||||
#include "db/wide/wide_column_serialization.h"
|
||||
|
||||
namespace ROCKSDB_NAMESPACE {
|
||||
void WideColumnsHelper::DumpWideColumns(const WideColumns& columns,
|
||||
std::ostream& os, bool hex) {
|
||||
if (columns.empty()) {
|
||||
return;
|
||||
}
|
||||
if (hex) {
|
||||
os << std::hex;
|
||||
}
|
||||
auto it = columns.begin();
|
||||
os << *it;
|
||||
for (++it; it != columns.end(); ++it) {
|
||||
os << ' ' << *it;
|
||||
}
|
||||
}
|
||||
Status WideColumnsHelper::DumpSliceAsWideColumns(const Slice& value,
|
||||
std::ostream& oss, bool hex) {
|
||||
WideColumns columns;
|
||||
Slice value_copy = value;
|
||||
const Status s = WideColumnSerialization::Deserialize(value_copy, columns);
|
||||
if (s.ok()) {
|
||||
DumpWideColumns(columns, oss, hex);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#pragma once
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
|
||||
#include "rocksdb/rocksdb_namespace.h"
|
||||
#include "rocksdb/wide_columns.h"
|
||||
|
||||
namespace ROCKSDB_NAMESPACE {
|
||||
class WideColumnsHelper {
|
||||
public:
|
||||
static void DumpWideColumns(const WideColumns& columns, std::ostream& oss,
|
||||
bool hex);
|
||||
static Status DumpSliceAsWideColumns(const Slice& value, std::ostream& oss,
|
||||
bool hex);
|
||||
};
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include "db/wide/wide_columns_helper.h"
|
||||
|
||||
#include "db/wide/wide_column_serialization.h"
|
||||
#include "test_util/testharness.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace ROCKSDB_NAMESPACE {
|
||||
|
||||
TEST(WideColumnsHelperTest, DumpWideColumns) {
|
||||
WideColumns columns{{"foo", "bar"}, {"hello", "world"}};
|
||||
std::ostringstream oss;
|
||||
WideColumnsHelper::DumpWideColumns(columns, oss, false /* hex */);
|
||||
EXPECT_EQ("foo:bar hello:world", oss.str());
|
||||
}
|
||||
|
||||
TEST(WideColumnsHelperTest, DumpSliceAsWideColumns) {
|
||||
WideColumns columns{{"foo", "bar"}, {"hello", "world"}};
|
||||
std::string output;
|
||||
ASSERT_OK(WideColumnSerialization::Serialize(columns, output));
|
||||
Slice input(output);
|
||||
|
||||
std::ostringstream oss;
|
||||
ASSERT_OK(
|
||||
WideColumnsHelper::DumpSliceAsWideColumns(input, oss, false /* hex */));
|
||||
|
||||
EXPECT_EQ("foo:bar hello:world", oss.str());
|
||||
}
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
|
|
@ -37,6 +37,7 @@
|
|||
|
||||
#include "db/db_impl/db_impl.h"
|
||||
#include "db/version_set.h"
|
||||
#include "db/wide/wide_columns_helper.h"
|
||||
#include "db_stress_tool/db_stress_env_wrapper.h"
|
||||
#include "db_stress_tool/db_stress_listener.h"
|
||||
#include "db_stress_tool/db_stress_shared_state.h"
|
||||
|
|
@ -629,13 +630,7 @@ inline std::string WideColumnsToHex(const WideColumns& columns) {
|
|||
|
||||
std::ostringstream oss;
|
||||
|
||||
oss << std::hex;
|
||||
|
||||
auto it = columns.begin();
|
||||
oss << *it;
|
||||
for (++it; it != columns.end(); ++it) {
|
||||
oss << ' ' << *it;
|
||||
}
|
||||
WideColumnsHelper::DumpWideColumns(columns, oss, true);
|
||||
|
||||
return oss.str();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,8 +74,19 @@ inline bool operator!=(const WideColumn& lhs, const WideColumn& rhs) {
|
|||
inline std::ostream& operator<<(std::ostream& os, const WideColumn& column) {
|
||||
const bool hex =
|
||||
(os.flags() & std::ios_base::basefield) == std::ios_base::hex;
|
||||
os << column.name().ToString(hex) << ':' << column.value().ToString(hex);
|
||||
|
||||
if (!column.name().empty()) {
|
||||
if (hex) {
|
||||
os << "0x";
|
||||
}
|
||||
os << column.name().ToString(hex);
|
||||
}
|
||||
os << ':';
|
||||
if (!column.value().empty()) {
|
||||
if (hex) {
|
||||
os << "0x";
|
||||
}
|
||||
os << column.value().ToString(hex);
|
||||
}
|
||||
return os;
|
||||
}
|
||||
|
||||
|
|
|
|||
2
src.mk
2
src.mk
|
|
@ -94,6 +94,7 @@ LIB_SOURCES = \
|
|||
db/wal_manager.cc \
|
||||
db/wide/wide_column_serialization.cc \
|
||||
db/wide/wide_columns.cc \
|
||||
db/wide/wide_columns_helper.cc \
|
||||
db/write_batch.cc \
|
||||
db/write_batch_base.cc \
|
||||
db/write_controller.cc \
|
||||
|
|
@ -528,6 +529,7 @@ TEST_MAIN_SOURCES = \
|
|||
db/wal_manager_test.cc \
|
||||
db/wide/db_wide_basic_test.cc \
|
||||
db/wide/wide_column_serialization_test.cc \
|
||||
db/wide/wide_columns_helper_test.cc \
|
||||
db/write_batch_test.cc \
|
||||
db/write_callback_test.cc \
|
||||
db/write_controller_test.cc \
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@
|
|||
|
||||
#include "db/blob/blob_index.h"
|
||||
#include "db/memtable.h"
|
||||
#include "db/wide/wide_column_serialization.h"
|
||||
#include "db/wide/wide_columns_helper.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "options/cf_options.h"
|
||||
#include "port/port.h"
|
||||
|
|
@ -489,9 +491,23 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num,
|
|||
|
||||
if (print_kv) {
|
||||
if (!decode_blob_index_ || ikey.type != kTypeBlobIndex) {
|
||||
fprintf(stdout, "%s => %s\n",
|
||||
ikey.DebugString(true, output_hex_).c_str(),
|
||||
value.ToString(output_hex_).c_str());
|
||||
if (ikey.type == kTypeWideColumnEntity) {
|
||||
std::ostringstream oss;
|
||||
const Status s = WideColumnsHelper::DumpSliceAsWideColumns(
|
||||
iter->value(), oss, output_hex_);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "%s => error deserializing wide columns\n",
|
||||
ikey.DebugString(true, output_hex_).c_str());
|
||||
continue;
|
||||
}
|
||||
fprintf(stdout, "%s => %s\n",
|
||||
ikey.DebugString(true, output_hex_).c_str(),
|
||||
oss.str().c_str());
|
||||
} else {
|
||||
fprintf(stdout, "%s => %s\n",
|
||||
ikey.DebugString(true, output_hex_).c_str(),
|
||||
value.ToString(output_hex_).c_str());
|
||||
}
|
||||
} else {
|
||||
BlobIndex blob_index;
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@
|
|||
#include "db/dbformat.h"
|
||||
#include "db/log_reader.h"
|
||||
#include "db/version_util.h"
|
||||
#include "db/wide/wide_column_serialization.h"
|
||||
#include "db/wide/wide_columns_helper.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "file/filename.h"
|
||||
#include "rocksdb/cache.h"
|
||||
|
|
@ -1884,8 +1886,20 @@ void InternalDumpCommand::DoCommand() {
|
|||
std::string key = ikey.DebugString(is_key_hex_);
|
||||
Slice value(key_version.value);
|
||||
if (!decode_blob_index_ || value_type != kTypeBlobIndex) {
|
||||
fprintf(stdout, "%s => %s\n", key.c_str(),
|
||||
value.ToString(is_value_hex_).c_str());
|
||||
if (value_type == kTypeWideColumnEntity) {
|
||||
std::ostringstream oss;
|
||||
const Status s = WideColumnsHelper::DumpSliceAsWideColumns(
|
||||
value, oss, is_value_hex_);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "%s => error deserializing wide columns\n",
|
||||
key.c_str());
|
||||
} else {
|
||||
fprintf(stdout, "%s => %s\n", key.c_str(), oss.str().c_str());
|
||||
}
|
||||
} else {
|
||||
fprintf(stdout, "%s => %s\n", key.c_str(),
|
||||
value.ToString(is_value_hex_).c_str());
|
||||
}
|
||||
} else {
|
||||
BlobIndex blob_index;
|
||||
|
||||
|
|
@ -2185,10 +2199,31 @@ void DBDumperCommand::DoDumpCommand() {
|
|||
if (is_db_ttl_ && timestamp_) {
|
||||
fprintf(stdout, "%s ", TimeToHumanString(rawtime).c_str());
|
||||
}
|
||||
std::string str =
|
||||
PrintKeyValue(iter->key().ToString(), iter->value().ToString(),
|
||||
is_key_hex_, is_value_hex_);
|
||||
fprintf(stdout, "%s\n", str.c_str());
|
||||
// (TODO) TTL Iterator does not support wide columns yet.
|
||||
if (is_db_ttl_ || iter->columns().empty() ||
|
||||
(iter->columns().size() == 1 &&
|
||||
iter->columns().front().name() == kDefaultWideColumnName)) {
|
||||
std::string str =
|
||||
PrintKeyValue(iter->key().ToString(), iter->value().ToString(),
|
||||
is_key_hex_, is_value_hex_);
|
||||
fprintf(stdout, "%s\n", str.c_str());
|
||||
} else {
|
||||
/*
|
||||
// Sample plaintext output (first column is kDefaultWideColumnName)
|
||||
key_1 ==> :foo attr_name1:bar attr_name2:baz
|
||||
|
||||
// Sample hex output (first column is kDefaultWideColumnName)
|
||||
0x6669727374 ==> :0x68656C6C6F 0x617474725F6E616D6531:0x666F6F
|
||||
*/
|
||||
|
||||
std::ostringstream oss;
|
||||
WideColumnsHelper::DumpWideColumns(iter->columns(), oss, is_value_hex_);
|
||||
std::string str = PrintKeyValue(
|
||||
iter->key().ToString(), oss.str().c_str(), is_key_hex_,
|
||||
false); // is_value_hex_ is already honored in oss. avoid
|
||||
// double-hexing it.
|
||||
fprintf(stdout, "%s\n", str.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2529,6 +2564,16 @@ class InMemoryHandler : public WriteBatch::Handler {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Status PutEntityCF(uint32_t cf, const Slice& key,
|
||||
const Slice& value) override {
|
||||
row_ << "PUT_ENTITY(" << cf << ") : ";
|
||||
std::string k = LDBCommand::StringToHex(key.ToString());
|
||||
if (print_values_) {
|
||||
return WideColumnsHelper::DumpSliceAsWideColumns(value, row_, true);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override {
|
||||
row_ << "MERGE(" << cf << ") : ";
|
||||
commonPutMerge(key, value);
|
||||
|
|
@ -3042,7 +3087,10 @@ void ScanCommand::DoCommand() {
|
|||
if (no_value_) {
|
||||
fprintf(stdout, "%.*s\n", static_cast<int>(key_slice.size()),
|
||||
key_slice.data());
|
||||
} else {
|
||||
// (TODO) TTL Iterator does not support wide columns yet.
|
||||
} else if (is_db_ttl_ || it->columns().empty() ||
|
||||
(it->columns().size() == 1 &&
|
||||
it->columns().front().name() == kDefaultWideColumnName)) {
|
||||
Slice val_slice = it->value();
|
||||
std::string formatted_value;
|
||||
if (is_value_hex_) {
|
||||
|
|
@ -3052,6 +3100,20 @@ void ScanCommand::DoCommand() {
|
|||
fprintf(stdout, "%.*s : %.*s\n", static_cast<int>(key_slice.size()),
|
||||
key_slice.data(), static_cast<int>(val_slice.size()),
|
||||
val_slice.data());
|
||||
} else {
|
||||
/*
|
||||
// Sample plaintext output (first column is kDefaultWideColumnName)
|
||||
key_1 : :foo attr_name1:bar attr_name2:baz
|
||||
|
||||
// Sample hex output (first column is kDefaultWideColumnName)
|
||||
0x6669727374 : :0x68656C6C6F 0x617474725F6E616D6531:0x666F6F
|
||||
*/
|
||||
|
||||
std::ostringstream oss;
|
||||
WideColumnsHelper::DumpWideColumns(it->columns(), oss, is_value_hex_);
|
||||
fprintf(stdout, "%.*s : %.*s\n", static_cast<int>(key_slice.size()),
|
||||
key_slice.data(), static_cast<int>(oss.str().length()),
|
||||
oss.str().c_str());
|
||||
}
|
||||
|
||||
num_keys_scanned++;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "db/wide/wide_column_serialization.h"
|
||||
#include "file/random_access_file_reader.h"
|
||||
#include "port/stack_trace.h"
|
||||
#include "rocksdb/convenience.h"
|
||||
|
|
@ -24,10 +25,11 @@ namespace ROCKSDB_NAMESPACE {
|
|||
const uint32_t kOptLength = 1024;
|
||||
|
||||
namespace {
|
||||
static std::string MakeKey(int i) {
|
||||
static std::string MakeKey(int i,
|
||||
ValueType value_type = ValueType::kTypeValue) {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "k_%04d", i);
|
||||
InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
|
||||
InternalKey key(std::string(buf), 0, value_type);
|
||||
return key.Encode().ToString();
|
||||
}
|
||||
|
||||
|
|
@ -44,6 +46,16 @@ static std::string MakeValue(int i) {
|
|||
return key.Encode().ToString();
|
||||
}
|
||||
|
||||
static std::string MakeWideColumn(int i) {
|
||||
std::string val = MakeValue(i);
|
||||
std::string val1 = "attr_1_val_" + val;
|
||||
std::string val2 = "attr_2_val_" + val;
|
||||
WideColumns columns{{"attr_1", val1}, {"attr_2", val2}};
|
||||
std::string entity;
|
||||
EXPECT_OK(WideColumnSerialization::Serialize(columns, entity));
|
||||
return entity;
|
||||
}
|
||||
|
||||
void cleanup(const Options& opts, const std::string& file_name) {
|
||||
Env* env = opts.env;
|
||||
ASSERT_OK(env->DeleteFile(file_name));
|
||||
|
|
@ -94,7 +106,8 @@ class SSTDumpToolTest : public testing::Test {
|
|||
snprintf(usage[2], kOptLength, "--file=%s", file_path.c_str());
|
||||
}
|
||||
|
||||
void createSST(const Options& opts, const std::string& file_name) {
|
||||
void createSST(const Options& opts, const std::string& file_name,
|
||||
uint32_t wide_column_one_in = 0) {
|
||||
Env* test_env = opts.env;
|
||||
FileOptions file_options(opts);
|
||||
ReadOptions read_options;
|
||||
|
|
@ -123,7 +136,12 @@ class SSTDumpToolTest : public testing::Test {
|
|||
const char* comparator_name = ikc.user_comparator()->Name();
|
||||
if (strcmp(comparator_name, ReverseBytewiseComparator()->Name()) == 0) {
|
||||
for (int32_t i = num_keys; i >= 0; i--) {
|
||||
tb->Add(MakeKey(i), MakeValue(i));
|
||||
if (wide_column_one_in == 0 || i % wide_column_one_in != 0) {
|
||||
tb->Add(MakeKey(i), MakeValue(i));
|
||||
} else {
|
||||
tb->Add(MakeKey(i, ValueType::kTypeWideColumnEntity),
|
||||
MakeWideColumn(i));
|
||||
}
|
||||
}
|
||||
} else if (strcmp(comparator_name,
|
||||
test::BytewiseComparatorWithU64TsWrapper()->Name()) ==
|
||||
|
|
@ -133,7 +151,12 @@ class SSTDumpToolTest : public testing::Test {
|
|||
}
|
||||
} else {
|
||||
for (uint32_t i = 0; i < num_keys; i++) {
|
||||
tb->Add(MakeKey(i), MakeValue(i));
|
||||
if (wide_column_one_in == 0 || i % wide_column_one_in != 0) {
|
||||
tb->Add(MakeKey(i), MakeValue(i));
|
||||
} else {
|
||||
tb->Add(MakeKey(i, ValueType::kTypeWideColumnEntity),
|
||||
MakeWideColumn(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
ASSERT_OK(tb->Finish());
|
||||
|
|
@ -164,7 +187,7 @@ TEST_F(SSTDumpToolTest, EmptyFilter) {
|
|||
Options opts;
|
||||
opts.env = env();
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(opts, file_path);
|
||||
createSST(opts, file_path, 10);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=raw", usage);
|
||||
|
|
@ -212,7 +235,7 @@ TEST_F(SSTDumpToolTest, SstDumpComparatorWithU64Ts) {
|
|||
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
|
||||
std::string file_path =
|
||||
MakeFilePath("rocksdb_sst_comparator_with_u64_ts.sst");
|
||||
createSST(opts, file_path);
|
||||
createSST(opts, file_path, 10);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=raw", usage);
|
||||
|
|
@ -234,7 +257,7 @@ TEST_F(SSTDumpToolTest, FilterBlock) {
|
|||
ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, true));
|
||||
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(opts, file_path);
|
||||
createSST(opts, file_path, 10);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=raw", usage);
|
||||
|
|
@ -300,7 +323,7 @@ TEST_F(SSTDumpToolTest, CompressedSizes) {
|
|||
ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
|
||||
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(opts, file_path);
|
||||
createSST(opts, file_path, 10);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=recompress", usage);
|
||||
|
|
@ -426,7 +449,7 @@ TEST_F(SSTDumpToolTest, RawOutput) {
|
|||
Options opts;
|
||||
opts.env = env();
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(opts, file_path);
|
||||
createSST(opts, file_path, 10);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=raw", usage);
|
||||
|
|
|
|||
Loading…
Reference in New Issue