From ea9a5b2914e09b9105436e7d6f932bc4434d02c5 Mon Sep 17 00:00:00 2001 From: Jay Huh Date: Wed, 30 Aug 2023 12:45:52 -0700 Subject: [PATCH] Wide Column support in ldb (#11754) Summary: wide_columns can now be pretty-printed in the following commands - `./ldb dump_wal` - `./ldb dump` - `./ldb idump` - `./ldb dump_live_files` - `./ldb scan` - `./sst_dump --command=scan` There are opportunities to refactor to reduce some nearly identical code. This PR is initial change to add wide column support in `ldb` and `sst_dump` tool. More PRs to come for the refactor. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11754 Test Plan: **New Tests added** - `WideColumnsHelperTest::DumpWideColumns` - `WideColumnsHelperTest::DumpSliceAsWideColumns` **Changes added to existing tests** - `ExternalSSTFileTest::BasicMixed` added to cover mixed case (This test should have been added in https://github.com/facebook/rocksdb/issues/11688). This test does not verify the ldb or sst_dump output. This test was used to create test SST files having some rows with wide columns and some without and the generated SST files were used to manually test sst_dump_tool. - `createSST()` in `sst_dump_test` now takes `wide_column_one_in` to add wide column value in SST **dump_wal** ``` ./ldb dump_wal --walfile=/tmp/rocksdbtest-226125/db_wide_basic_test_2675429_2308393776696827948/000004.log --print_value --header ``` ``` Sequence,Count,ByteSize,Physical Offset,Key(s) : value 1,1,59,0,PUT_ENTITY(0) : 0x:0x68656C6C6F 0x617474725F6E616D6531:0x666F6F 0x617474725F6E616D6532:0x626172 2,1,34,42,PUT_ENTITY(0) : 0x617474725F6F6E65:0x74776F 0x617474725F7468726565:0x666F7572 3,1,17,7d,PUT(0) : 0x7468697264 : 0x62617A ``` **idump** ``` ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ idump ``` ``` 'first' seq:1, type:22 => :hello attr_name1:foo attr_name2:bar 'second' seq:2, type:22 => attr_one:two attr_three:four 'third' seq:3, type:1 => baz Internal keys in range: 3 ``` **SST Dump from dump_live_files** ``` ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ compact ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ dump_live_files ``` ``` ... ============================== SST Files ============================== /tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/000013.sst level:1 ------------------------------ Process /tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/000013.sst Sst file format: block-based 'first' seq:0, type:22 => :hello attr_name1:foo attr_name2:bar 'second' seq:0, type:22 => attr_one:two attr_three:four 'third' seq:0, type:1 => baz ... ``` **dump** ``` ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ dump ``` ``` first ==> :hello attr_name1:foo attr_name2:bar second ==> attr_one:two attr_three:four third ==> baz Keys in range: 3 ``` **scan** ``` ./ldb --db=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/ scan ``` ``` first : :hello attr_name1:foo attr_name2:bar second : attr_one:two attr_three:four third : baz ``` **sst_dump** ``` ./sst_dump --file=/tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/000013.sst --command=scan ``` ``` options.env is 0x7ff54b296000 Process /tmp/rocksdbtest-226125/db_wide_basic_test_3481961_2308393776696827948/000013.sst Sst file format: block-based from [] to [] 'first' seq:0, type:22 => :hello attr_name1:foo attr_name2:bar 'second' seq:0, type:22 => attr_one:two attr_three:four 'third' seq:0, type:1 => baz ``` Reviewed By: ltamasi Differential Revision: D48837999 Pulled By: jaykorean fbshipit-source-id: b0280f0589d2b9716bb9b50530ffcabb397d140f --- CMakeLists.txt | 2 + Makefile | 3 ++ TARGETS | 7 +++ db/external_sst_file_test.cc | 57 ++++++++++++++++++++++ db/wide/wide_columns_helper.cc | 36 ++++++++++++++ db/wide/wide_columns_helper.h | 21 ++++++++ db/wide/wide_columns_helper_test.cc | 39 +++++++++++++++ db_stress_tool/db_stress_common.h | 9 +--- include/rocksdb/wide_columns.h | 15 +++++- src.mk | 2 + table/sst_file_dumper.cc | 22 +++++++-- tools/ldb_cmd.cc | 76 ++++++++++++++++++++++++++--- tools/sst_dump_test.cc | 43 ++++++++++++---- 13 files changed, 303 insertions(+), 29 deletions(-) create mode 100644 db/wide/wide_columns_helper.cc create mode 100644 db/wide/wide_columns_helper.h create mode 100644 db/wide/wide_columns_helper_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e30f6631e..2be25e2c01 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -714,6 +714,7 @@ set(SOURCES db/wal_manager.cc db/wide/wide_column_serialization.cc db/wide/wide_columns.cc + db/wide/wide_columns_helper.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc @@ -1357,6 +1358,7 @@ if(WITH_TESTS) db/wal_edit_test.cc db/wide/db_wide_basic_test.cc db/wide/wide_column_serialization_test.cc + db/wide/wide_columns_helper_test.cc db/write_batch_test.cc db/write_callback_test.cc db/write_controller_test.cc diff --git a/Makefile b/Makefile index 41ca0ee28f..08ad7e48ab 100644 --- a/Makefile +++ b/Makefile @@ -1984,6 +1984,9 @@ cache_reservation_manager_test: $(OBJ_DIR)/cache/cache_reservation_manager_test. wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) +wide_columns_helper_test: $(OBJ_DIR)/db/wide/wide_columns_helper_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + #------------------------------------------------- # make install related stuff PREFIX ?= /usr/local diff --git a/TARGETS b/TARGETS index 5125fcf54a..ad0da61008 100644 --- a/TARGETS +++ b/TARGETS @@ -103,6 +103,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[ "db/wal_manager.cc", "db/wide/wide_column_serialization.cc", "db/wide/wide_columns.cc", + "db/wide/wide_columns_helper.cc", "db/write_batch.cc", "db/write_batch_base.cc", "db/write_controller.cc", @@ -5552,6 +5553,12 @@ cpp_unittest_wrapper(name="wide_column_serialization_test", extra_compiler_flags=[]) +cpp_unittest_wrapper(name="wide_columns_helper_test", + srcs=["db/wide/wide_columns_helper_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + cpp_unittest_wrapper(name="work_queue_test", srcs=["util/work_queue_test.cc"], deps=[":rocksdb_test_lib"], diff --git a/db/external_sst_file_test.cc b/db/external_sst_file_test.cc index 4507b16c29..9a0b7e6592 100644 --- a/db/external_sst_file_test.cc +++ b/db/external_sst_file_test.cc @@ -588,6 +588,63 @@ TEST_F(ExternalSSTFileTest, BasicWideColumn) { kRangeDelSkipConfigs)); } +TEST_F(ExternalSSTFileTest, BasicMixed) { + do { + Options options = CurrentOptions(); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // Current file size should be 0 after sst_file_writer init and before open + // a file. + ASSERT_EQ(sst_file_writer.FileSize(), 0); + + std::string file = sst_files_dir_ + "mixed_file.sst"; + ASSERT_OK(sst_file_writer.Open(file)); + for (int k = 0; k < 100; k++) { + if (k % 5 == 0) { + std::string val1 = Key(k) + "_attr_1_val"; + std::string val2 = Key(k) + "_attr_2_val"; + WideColumns columns{{"attr_1", val1}, {"attr_2", val2}}; + ASSERT_OK(sst_file_writer.PutEntity(Key(k), columns)); + } else { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + } + ExternalSstFileInfo file_info; + ASSERT_OK(sst_file_writer.Finish(&file_info)); + + // Current file size should be non-zero after success write. + ASSERT_GT(sst_file_writer.FileSize(), 0); + + ASSERT_EQ(file_info.file_path, file); + ASSERT_EQ(file_info.num_entries, 100); + ASSERT_EQ(file_info.smallest_key, Key(0)); + ASSERT_EQ(file_info.largest_key, Key(99)); + ASSERT_EQ(file_info.num_range_del_entries, 0); + ASSERT_EQ(file_info.smallest_range_del_key, ""); + ASSERT_EQ(file_info.largest_range_del_key, ""); + + DestroyAndReopen(options); + // Add file using file path + ASSERT_OK(DeprecatedAddFile({file})); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + for (int k = 0; k < 10; k++) { + if (k % 5 == 0) { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + Key(k), &result)); + std::string val1 = Key(k) + "_attr_1_val"; + std::string val2 = Key(k) + "_attr_2_val"; + WideColumns expected_columns{{"attr_1", val1}, {"attr_2", val2}}; + ASSERT_EQ(result.columns(), expected_columns); + } else { + ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); + } + } + } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction | + kRangeDelSkipConfigs)); +} + class SstFileWriterCollector : public TablePropertiesCollector { public: explicit SstFileWriterCollector(const std::string prefix) : prefix_(prefix) { diff --git a/db/wide/wide_columns_helper.cc b/db/wide/wide_columns_helper.cc new file mode 100644 index 0000000000..80a6b5c43b --- /dev/null +++ b/db/wide/wide_columns_helper.cc @@ -0,0 +1,36 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/wide/wide_columns_helper.h" + +#include "db/wide/wide_column_serialization.h" + +namespace ROCKSDB_NAMESPACE { +void WideColumnsHelper::DumpWideColumns(const WideColumns& columns, + std::ostream& os, bool hex) { + if (columns.empty()) { + return; + } + if (hex) { + os << std::hex; + } + auto it = columns.begin(); + os << *it; + for (++it; it != columns.end(); ++it) { + os << ' ' << *it; + } +} +Status WideColumnsHelper::DumpSliceAsWideColumns(const Slice& value, + std::ostream& oss, bool hex) { + WideColumns columns; + Slice value_copy = value; + const Status s = WideColumnSerialization::Deserialize(value_copy, columns); + if (s.ok()) { + DumpWideColumns(columns, oss, hex); + } + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/wide/wide_columns_helper.h b/db/wide/wide_columns_helper.h new file mode 100644 index 0000000000..83c8548a39 --- /dev/null +++ b/db/wide/wide_columns_helper.h @@ -0,0 +1,21 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/wide_columns.h" + +namespace ROCKSDB_NAMESPACE { +class WideColumnsHelper { + public: + static void DumpWideColumns(const WideColumns& columns, std::ostream& oss, + bool hex); + static Status DumpSliceAsWideColumns(const Slice& value, std::ostream& oss, + bool hex); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/db/wide/wide_columns_helper_test.cc b/db/wide/wide_columns_helper_test.cc new file mode 100644 index 0000000000..482bba5310 --- /dev/null +++ b/db/wide/wide_columns_helper_test.cc @@ -0,0 +1,39 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/wide/wide_columns_helper.h" + +#include "db/wide/wide_column_serialization.h" +#include "test_util/testharness.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +TEST(WideColumnsHelperTest, DumpWideColumns) { + WideColumns columns{{"foo", "bar"}, {"hello", "world"}}; + std::ostringstream oss; + WideColumnsHelper::DumpWideColumns(columns, oss, false /* hex */); + EXPECT_EQ("foo:bar hello:world", oss.str()); +} + +TEST(WideColumnsHelperTest, DumpSliceAsWideColumns) { + WideColumns columns{{"foo", "bar"}, {"hello", "world"}}; + std::string output; + ASSERT_OK(WideColumnSerialization::Serialize(columns, output)); + Slice input(output); + + std::ostringstream oss; + ASSERT_OK( + WideColumnsHelper::DumpSliceAsWideColumns(input, oss, false /* hex */)); + + EXPECT_EQ("foo:bar hello:world", oss.str()); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index 32e77d40c6..4d011434f0 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -37,6 +37,7 @@ #include "db/db_impl/db_impl.h" #include "db/version_set.h" +#include "db/wide/wide_columns_helper.h" #include "db_stress_tool/db_stress_env_wrapper.h" #include "db_stress_tool/db_stress_listener.h" #include "db_stress_tool/db_stress_shared_state.h" @@ -629,13 +630,7 @@ inline std::string WideColumnsToHex(const WideColumns& columns) { std::ostringstream oss; - oss << std::hex; - - auto it = columns.begin(); - oss << *it; - for (++it; it != columns.end(); ++it) { - oss << ' ' << *it; - } + WideColumnsHelper::DumpWideColumns(columns, oss, true); return oss.str(); } diff --git a/include/rocksdb/wide_columns.h b/include/rocksdb/wide_columns.h index 5af3f51de8..1da66b73c7 100644 --- a/include/rocksdb/wide_columns.h +++ b/include/rocksdb/wide_columns.h @@ -74,8 +74,19 @@ inline bool operator!=(const WideColumn& lhs, const WideColumn& rhs) { inline std::ostream& operator<<(std::ostream& os, const WideColumn& column) { const bool hex = (os.flags() & std::ios_base::basefield) == std::ios_base::hex; - os << column.name().ToString(hex) << ':' << column.value().ToString(hex); - + if (!column.name().empty()) { + if (hex) { + os << "0x"; + } + os << column.name().ToString(hex); + } + os << ':'; + if (!column.value().empty()) { + if (hex) { + os << "0x"; + } + os << column.value().ToString(hex); + } return os; } diff --git a/src.mk b/src.mk index 7d2663b995..2992f5c4a6 100644 --- a/src.mk +++ b/src.mk @@ -94,6 +94,7 @@ LIB_SOURCES = \ db/wal_manager.cc \ db/wide/wide_column_serialization.cc \ db/wide/wide_columns.cc \ + db/wide/wide_columns_helper.cc \ db/write_batch.cc \ db/write_batch_base.cc \ db/write_controller.cc \ @@ -528,6 +529,7 @@ TEST_MAIN_SOURCES = \ db/wal_manager_test.cc \ db/wide/db_wide_basic_test.cc \ db/wide/wide_column_serialization_test.cc \ + db/wide/wide_columns_helper_test.cc \ db/write_batch_test.cc \ db/write_callback_test.cc \ db/write_controller_test.cc \ diff --git a/table/sst_file_dumper.cc b/table/sst_file_dumper.cc index 4a9fd38079..2334b3ee27 100644 --- a/table/sst_file_dumper.cc +++ b/table/sst_file_dumper.cc @@ -16,6 +16,8 @@ #include "db/blob/blob_index.h" #include "db/memtable.h" +#include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "db/write_batch_internal.h" #include "options/cf_options.h" #include "port/port.h" @@ -489,9 +491,23 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, if (print_kv) { if (!decode_blob_index_ || ikey.type != kTypeBlobIndex) { - fprintf(stdout, "%s => %s\n", - ikey.DebugString(true, output_hex_).c_str(), - value.ToString(output_hex_).c_str()); + if (ikey.type == kTypeWideColumnEntity) { + std::ostringstream oss; + const Status s = WideColumnsHelper::DumpSliceAsWideColumns( + iter->value(), oss, output_hex_); + if (!s.ok()) { + fprintf(stderr, "%s => error deserializing wide columns\n", + ikey.DebugString(true, output_hex_).c_str()); + continue; + } + fprintf(stdout, "%s => %s\n", + ikey.DebugString(true, output_hex_).c_str(), + oss.str().c_str()); + } else { + fprintf(stdout, "%s => %s\n", + ikey.DebugString(true, output_hex_).c_str(), + value.ToString(output_hex_).c_str()); + } } else { BlobIndex blob_index; diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index 263cbe472c..be48e02040 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -22,6 +22,8 @@ #include "db/dbformat.h" #include "db/log_reader.h" #include "db/version_util.h" +#include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "db/write_batch_internal.h" #include "file/filename.h" #include "rocksdb/cache.h" @@ -1884,8 +1886,20 @@ void InternalDumpCommand::DoCommand() { std::string key = ikey.DebugString(is_key_hex_); Slice value(key_version.value); if (!decode_blob_index_ || value_type != kTypeBlobIndex) { - fprintf(stdout, "%s => %s\n", key.c_str(), - value.ToString(is_value_hex_).c_str()); + if (value_type == kTypeWideColumnEntity) { + std::ostringstream oss; + const Status s = WideColumnsHelper::DumpSliceAsWideColumns( + value, oss, is_value_hex_); + if (!s.ok()) { + fprintf(stderr, "%s => error deserializing wide columns\n", + key.c_str()); + } else { + fprintf(stdout, "%s => %s\n", key.c_str(), oss.str().c_str()); + } + } else { + fprintf(stdout, "%s => %s\n", key.c_str(), + value.ToString(is_value_hex_).c_str()); + } } else { BlobIndex blob_index; @@ -2185,10 +2199,31 @@ void DBDumperCommand::DoDumpCommand() { if (is_db_ttl_ && timestamp_) { fprintf(stdout, "%s ", TimeToHumanString(rawtime).c_str()); } - std::string str = - PrintKeyValue(iter->key().ToString(), iter->value().ToString(), - is_key_hex_, is_value_hex_); - fprintf(stdout, "%s\n", str.c_str()); + // (TODO) TTL Iterator does not support wide columns yet. + if (is_db_ttl_ || iter->columns().empty() || + (iter->columns().size() == 1 && + iter->columns().front().name() == kDefaultWideColumnName)) { + std::string str = + PrintKeyValue(iter->key().ToString(), iter->value().ToString(), + is_key_hex_, is_value_hex_); + fprintf(stdout, "%s\n", str.c_str()); + } else { + /* + // Sample plaintext output (first column is kDefaultWideColumnName) + key_1 ==> :foo attr_name1:bar attr_name2:baz + + // Sample hex output (first column is kDefaultWideColumnName) + 0x6669727374 ==> :0x68656C6C6F 0x617474725F6E616D6531:0x666F6F + */ + + std::ostringstream oss; + WideColumnsHelper::DumpWideColumns(iter->columns(), oss, is_value_hex_); + std::string str = PrintKeyValue( + iter->key().ToString(), oss.str().c_str(), is_key_hex_, + false); // is_value_hex_ is already honored in oss. avoid + // double-hexing it. + fprintf(stdout, "%s\n", str.c_str()); + } } } @@ -2529,6 +2564,16 @@ class InMemoryHandler : public WriteBatch::Handler { return Status::OK(); } + Status PutEntityCF(uint32_t cf, const Slice& key, + const Slice& value) override { + row_ << "PUT_ENTITY(" << cf << ") : "; + std::string k = LDBCommand::StringToHex(key.ToString()); + if (print_values_) { + return WideColumnsHelper::DumpSliceAsWideColumns(value, row_, true); + } + return Status::OK(); + } + Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override { row_ << "MERGE(" << cf << ") : "; commonPutMerge(key, value); @@ -3042,7 +3087,10 @@ void ScanCommand::DoCommand() { if (no_value_) { fprintf(stdout, "%.*s\n", static_cast(key_slice.size()), key_slice.data()); - } else { + // (TODO) TTL Iterator does not support wide columns yet. + } else if (is_db_ttl_ || it->columns().empty() || + (it->columns().size() == 1 && + it->columns().front().name() == kDefaultWideColumnName)) { Slice val_slice = it->value(); std::string formatted_value; if (is_value_hex_) { @@ -3052,6 +3100,20 @@ void ScanCommand::DoCommand() { fprintf(stdout, "%.*s : %.*s\n", static_cast(key_slice.size()), key_slice.data(), static_cast(val_slice.size()), val_slice.data()); + } else { + /* + // Sample plaintext output (first column is kDefaultWideColumnName) + key_1 : :foo attr_name1:bar attr_name2:baz + + // Sample hex output (first column is kDefaultWideColumnName) + 0x6669727374 : :0x68656C6C6F 0x617474725F6E616D6531:0x666F6F + */ + + std::ostringstream oss; + WideColumnsHelper::DumpWideColumns(it->columns(), oss, is_value_hex_); + fprintf(stdout, "%.*s : %.*s\n", static_cast(key_slice.size()), + key_slice.data(), static_cast(oss.str().length()), + oss.str().c_str()); } num_keys_scanned++; diff --git a/tools/sst_dump_test.cc b/tools/sst_dump_test.cc index 481c4b7220..f0b71bf8ea 100644 --- a/tools/sst_dump_test.cc +++ b/tools/sst_dump_test.cc @@ -9,6 +9,7 @@ #include +#include "db/wide/wide_column_serialization.h" #include "file/random_access_file_reader.h" #include "port/stack_trace.h" #include "rocksdb/convenience.h" @@ -24,10 +25,11 @@ namespace ROCKSDB_NAMESPACE { const uint32_t kOptLength = 1024; namespace { -static std::string MakeKey(int i) { +static std::string MakeKey(int i, + ValueType value_type = ValueType::kTypeValue) { char buf[100]; snprintf(buf, sizeof(buf), "k_%04d", i); - InternalKey key(std::string(buf), 0, ValueType::kTypeValue); + InternalKey key(std::string(buf), 0, value_type); return key.Encode().ToString(); } @@ -44,6 +46,16 @@ static std::string MakeValue(int i) { return key.Encode().ToString(); } +static std::string MakeWideColumn(int i) { + std::string val = MakeValue(i); + std::string val1 = "attr_1_val_" + val; + std::string val2 = "attr_2_val_" + val; + WideColumns columns{{"attr_1", val1}, {"attr_2", val2}}; + std::string entity; + EXPECT_OK(WideColumnSerialization::Serialize(columns, entity)); + return entity; +} + void cleanup(const Options& opts, const std::string& file_name) { Env* env = opts.env; ASSERT_OK(env->DeleteFile(file_name)); @@ -94,7 +106,8 @@ class SSTDumpToolTest : public testing::Test { snprintf(usage[2], kOptLength, "--file=%s", file_path.c_str()); } - void createSST(const Options& opts, const std::string& file_name) { + void createSST(const Options& opts, const std::string& file_name, + uint32_t wide_column_one_in = 0) { Env* test_env = opts.env; FileOptions file_options(opts); ReadOptions read_options; @@ -123,7 +136,12 @@ class SSTDumpToolTest : public testing::Test { const char* comparator_name = ikc.user_comparator()->Name(); if (strcmp(comparator_name, ReverseBytewiseComparator()->Name()) == 0) { for (int32_t i = num_keys; i >= 0; i--) { - tb->Add(MakeKey(i), MakeValue(i)); + if (wide_column_one_in == 0 || i % wide_column_one_in != 0) { + tb->Add(MakeKey(i), MakeValue(i)); + } else { + tb->Add(MakeKey(i, ValueType::kTypeWideColumnEntity), + MakeWideColumn(i)); + } } } else if (strcmp(comparator_name, test::BytewiseComparatorWithU64TsWrapper()->Name()) == @@ -133,7 +151,12 @@ class SSTDumpToolTest : public testing::Test { } } else { for (uint32_t i = 0; i < num_keys; i++) { - tb->Add(MakeKey(i), MakeValue(i)); + if (wide_column_one_in == 0 || i % wide_column_one_in != 0) { + tb->Add(MakeKey(i), MakeValue(i)); + } else { + tb->Add(MakeKey(i, ValueType::kTypeWideColumnEntity), + MakeWideColumn(i)); + } } } ASSERT_OK(tb->Finish()); @@ -164,7 +187,7 @@ TEST_F(SSTDumpToolTest, EmptyFilter) { Options opts; opts.env = env(); std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=raw", usage); @@ -212,7 +235,7 @@ TEST_F(SSTDumpToolTest, SstDumpComparatorWithU64Ts) { opts.table_factory.reset(new BlockBasedTableFactory(table_opts)); std::string file_path = MakeFilePath("rocksdb_sst_comparator_with_u64_ts.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=raw", usage); @@ -234,7 +257,7 @@ TEST_F(SSTDumpToolTest, FilterBlock) { ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, true)); opts.table_factory.reset(new BlockBasedTableFactory(table_opts)); std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=raw", usage); @@ -300,7 +323,7 @@ TEST_F(SSTDumpToolTest, CompressedSizes) { ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false)); opts.table_factory.reset(new BlockBasedTableFactory(table_opts)); std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=recompress", usage); @@ -426,7 +449,7 @@ TEST_F(SSTDumpToolTest, RawOutput) { Options opts; opts.env = env(); std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=raw", usage);