dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Embed column family name in SST file 

Summary:
Added the column family name to the properties block. This property
is omitted only if the property is unavailable, such as when RepairDB()
writes SST files.

In a next diff, I will change RepairDB to use this new property for
deciding to which column family an existing SST file belongs. If this
property is missing, it will add it to the "unknown" column family (same
as its existing behavior).

Test Plan:
New unit test:

  $ ./db_table_properties_test --gtest_filter=DBTablePropertiesTest.GetColumnFamilyNameProperty

Reviewers: IslamAbdelRahman, yhchiang, sdong

Reviewed By: sdong

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D55605
This commit is contained in:
Andrew Kryczka 2016-04-06 23:10:32 -07:00
parent ab4c62332e
commit 2391ef7214
29 changed files with 248 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
db/builder.cc
View File

@ -41,13 +41,17 @@ TableBuilder* NewTableBuilder(
const InternalKeyComparator& internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* file,
const CompressionType compression_type,
uint32_t column_family_id, const std::string& column_family_name,
WritableFileWriter* file, const CompressionType compression_type,
const CompressionOptions& compression_opts, const bool skip_filters) {
assert((column_family_id ==
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
column_family_name.empty());
return ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, internal_comparator,
int_tbl_prop_collector_factories, compression_type,
compression_opts, skip_filters),
compression_opts, skip_filters,
column_family_name),
column_family_id, file);
}
@ -58,12 +62,16 @@ Status BuildTable(
const InternalKeyComparator& internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories,
uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
uint32_t column_family_id, const std::string& column_family_name,
std::vector<SequenceNumber> snapshots,
SequenceNumber earliest_write_conflict_snapshot,
const CompressionType compression,
const CompressionOptions& compression_opts, bool paranoid_file_checks,
InternalStats* internal_stats, const Env::IOPriority io_priority,
TableProperties* table_properties, int level) {
assert((column_family_id ==
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
column_family_name.empty());
// Reports the IOStats for flush for every following bytes.
const size_t kReportFlushIOStatsEvery = 1048576;
Status s;
@ -87,7 +95,8 @@ Status BuildTable(
builder = NewTableBuilder(
ioptions, internal_comparator, int_tbl_prop_collector_factories,
column_family_id, file_writer.get(), compression, compression_opts);
column_family_id, column_family_name, file_writer.get(), compression,
compression_opts);
}
MergeHelper merge(env, internal_comparator.user_comparator(),

13
db/builder.h
View File

@ -33,13 +33,16 @@ class WritableFileWriter;
class InternalStats;
class InternalIterator;
// @param column_family_name Name of the column family that is also identified
// by column_family_id, or empty string if unknown. It must outlive the
// TableBuilder returned by this function.
TableBuilder* NewTableBuilder(
const ImmutableCFOptions& options,
const InternalKeyComparator& internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* file,
const CompressionType compression_type,
uint32_t column_family_id, const std::string& column_family_name,
WritableFileWriter* file, const CompressionType compression_type,
const CompressionOptions& compression_opts,
const bool skip_filters = false);
@ -48,6 +51,9 @@ TableBuilder* NewTableBuilder(
// *meta will be filled with metadata about the generated table.
// If no data is present in *iter, meta->file_size will be set to
// zero, and no Table file will be produced.
//
// @param column_family_name Name of the column family that is also identified
// by column_family_id, or empty string if unknown.
extern Status BuildTable(
const std::string& dbname, Env* env, const ImmutableCFOptions& options,
const EnvOptions& env_options, TableCache* table_cache,
@ -55,7 +61,8 @@ extern Status BuildTable(
const InternalKeyComparator& internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories,
uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
uint32_t column_family_id, const std::string& column_family_name,
std::vector<SequenceNumber> snapshots,
SequenceNumber earliest_write_conflict_snapshot,
const CompressionType compression,
const CompressionOptions& compression_opts, bool paranoid_file_checks,

2
db/compaction_job.cc
View File

@ -1019,7 +1019,7 @@ Status CompactionJob::OpenCompactionOutputFile(
cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
sub_compact->builder.reset(NewTableBuilder(
*cfd->ioptions(), cfd->internal_comparator(),
cfd->int_tbl_prop_collector_factories(), cfd->GetID(),
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
cfd->ioptions()->compression_opts, skip_filters));
LogFlush(db_options_.info_log);

4
db/db_impl.cc
View File

@ -1452,8 +1452,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
s = BuildTable(
dbname_, env_, *cfd->ioptions(), env_options_, cfd->table_cache(),
iter.get(), &meta, cfd->internal_comparator(),
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), snapshot_seqs,
earliest_write_conflict_snapshot,
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
snapshot_seqs, earliest_write_conflict_snapshot,
GetCompressionFlush(*cfd->ioptions()),
cfd->ioptions()->compression_opts, paranoid_file_checks,
cfd->internal_stats(), Env::IO_HIGH, &info.table_properties);

28
db/db_table_properties_test.cc
View File

@ -215,6 +215,34 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) {
TestGetPropertiesOfTablesInRange(std::move(ranges));
}
}
TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) {
std::string kExtraCfName = "pikachu";
CreateAndReopenWithCF({kExtraCfName}, CurrentOptions());
// Create one table per CF, then verify it was created with the column family
// name property.
for (int cf = 0; cf < 2; ++cf) {
Put(cf, "key", "val");
Flush(cf);
TablePropertiesCollection fname_to_props;
ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props));
ASSERT_EQ(1U, fname_to_props.size());
std::string expected_cf_name;
if (cf > 0) {
expected_cf_name = kExtraCfName;
} else {
expected_cf_name = kDefaultColumnFamilyName;
}
ASSERT_EQ(expected_cf_name,
fname_to_props.begin()->second->column_family_name);
ASSERT_EQ(cf, static_cast<uint32_t>(
fname_to_props.begin()->second->column_family_id));
}
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

5
db/flush_job.cc
View File

@ -238,8 +238,9 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(),
cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
existing_snapshots_, earliest_write_conflict_snapshot_,
output_compression_, cfd_->ioptions()->compression_opts,
cfd_->GetName(), existing_snapshots_,
earliest_write_conflict_snapshot_, output_compression_,
cfd_->ioptions()->compression_opts,
mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
Env::IO_HIGH, &table_properties_, 0 /* level */);
info.table_properties = table_properties_;

27
db/plain_table_db_test.cc
View File

@ -263,7 +263,9 @@ class TestPlainTableReader : public PlainTableReader {
const TableProperties* table_properties,
unique_ptr<RandomAccessFileReader>&& file,
const ImmutableCFOptions& ioptions,
bool* expect_bloom_not_match, bool store_index_in_file)
bool* expect_bloom_not_match, bool store_index_in_file,
uint32_t column_family_id,
const std::string& column_family_name)
: PlainTableReader(ioptions, std::move(file), env_options, icomparator,
encoding_type, file_size, table_properties),
expect_bloom_not_match_(expect_bloom_not_match) {
@ -276,6 +278,8 @@ class TestPlainTableReader : public PlainTableReader {
EXPECT_TRUE(s.ok());
TableProperties* props = const_cast<TableProperties*>(table_properties);
EXPECT_EQ(column_family_id, static_cast<uint32_t>(props->column_family_id));
EXPECT_EQ(column_family_name, props->column_family_name);
if (store_index_in_file) {
auto bloom_version_ptr = props->user_collected_properties.find(
PlainTablePropertyNames::kBloomVersion);
@ -308,13 +312,17 @@ extern const uint64_t kPlainTableMagicNumber;
class TestPlainTableFactory : public PlainTableFactory {
public:
explicit TestPlainTableFactory(bool* expect_bloom_not_match,
const PlainTableOptions& options)
const PlainTableOptions& options,
uint32_t column_family_id,
std::string column_family_name)
: PlainTableFactory(options),
bloom_bits_per_key_(options.bloom_bits_per_key),
hash_table_ratio_(options.hash_table_ratio),
index_sparseness_(options.index_sparseness),
store_index_in_file_(options.store_index_in_file),
expect_bloom_not_match_(expect_bloom_not_match) {}
expect_bloom_not_match_(expect_bloom_not_match),
column_family_id_(column_family_id),
column_family_name_(std::move(column_family_name)) {}
Status NewTableReader(const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file,
@ -354,7 +362,7 @@ class TestPlainTableFactory : public PlainTableFactory {
table_reader_options.internal_comparator, encoding_type, file_size,
bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, props,
std::move(file), table_reader_options.ioptions, expect_bloom_not_match_,
store_index_in_file_));
store_index_in_file_, column_family_id_, column_family_name_));
*table = std::move(new_reader);
return s;
@ -366,6 +374,8 @@ class TestPlainTableFactory : public PlainTableFactory {
size_t index_sparseness_;
bool store_index_in_file_;
bool* expect_bloom_not_match_;
const uint32_t column_family_id_;
const std::string column_family_name_;
};
TEST_P(PlainTableDBTest, Flush) {
@ -492,7 +502,8 @@ TEST_P(PlainTableDBTest, Flush2) {
plain_table_options.encoding_type = encoding_type;
plain_table_options.store_index_in_file = store_index_in_file;
options.table_factory.reset(new TestPlainTableFactory(
&expect_bloom_not_match, plain_table_options));
&expect_bloom_not_match, plain_table_options,
0 /* column_family_id */, kDefaultColumnFamilyName));
DestroyAndReopen(&options);
ASSERT_OK(Put("0000000000000bar", "b"));
@ -561,7 +572,8 @@ TEST_P(PlainTableDBTest, Iterator) {
plain_table_options.encoding_type = encoding_type;
options.table_factory.reset(new TestPlainTableFactory(
&expect_bloom_not_match, plain_table_options));
&expect_bloom_not_match, plain_table_options,
0 /* column_family_id */, kDefaultColumnFamilyName));
} else {
PlainTableOptions plain_table_options;
plain_table_options.user_key_len = 16;
@ -572,7 +584,8 @@ TEST_P(PlainTableDBTest, Iterator) {
plain_table_options.encoding_type = encoding_type;
options.table_factory.reset(new TestPlainTableFactory(
&expect_bloom_not_match, plain_table_options));
&expect_bloom_not_match, plain_table_options,
0 /* column_family_id */, kDefaultColumnFamilyName));
}
DestroyAndReopen(&options);

7
db/repair.cc
View File

@ -294,9 +294,10 @@ class Repairer {
status = BuildTable(
dbname_, env_, ioptions_, env_options_, table_cache_, iter.get(),
&meta, icmp_, &int_tbl_prop_collector_factories_,
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, {},
kMaxSequenceNumber, kNoCompression, CompressionOptions(), false,
nullptr);
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
std::string() /* column_family_name */, {}, kMaxSequenceNumber,
kNoCompression, CompressionOptions(), false,
nullptr /* internal_stats */);
}
delete mem->Unref();
delete cf_mems_default;

5
db/table_properties_collector_test.cc
View File

@ -36,6 +36,7 @@ class TablePropertiesTest : public testing::Test,
// Utilities test functions
namespace {
static const uint32_t kTestColumnFamilyId = 66;
static const std::string kTestColumnFamilyName = "test_column_fam";
void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
const InternalKeyComparator& internal_comparator,
@ -48,8 +49,8 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
builder->reset(NewTableBuilder(
ioptions, internal_comparator, int_tbl_prop_collector_factories,
kTestColumnFamilyId /* column_family_id */, writable->get(),
options.compression, options.compression_opts));
kTestColumnFamilyId, kTestColumnFamilyName,
writable->get(), options.compression, options.compression_opts));
}
} // namespace

92
include/rocksdb/table_properties.h
View File

@ -27,47 +27,6 @@ namespace rocksdb {
// }
typedef std::map<std::string, std::string> UserCollectedProperties;
// TableProperties contains a bunch of read-only properties of its associated
// table.
struct TableProperties {
public:
// the total size of all data blocks.
uint64_t data_size = 0;
// the size of index block.
uint64_t index_size = 0;
// the size of filter block.
uint64_t filter_size = 0;
// total raw key size
uint64_t raw_key_size = 0;
// total raw value size
uint64_t raw_value_size = 0;
// the number of blocks in this table
uint64_t num_data_blocks = 0;
// the number of entries in this table
uint64_t num_entries = 0;
// format version, reserved for backward compatibility
uint64_t format_version = 0;
// If 0, key is variable length. Otherwise number of bytes for each key.
uint64_t fixed_key_len = 0;
// The name of the filter policy used in this table.
// If no filter policy is used, `filter_policy_name` will be an empty string.
std::string filter_policy_name;
// user collected properties
UserCollectedProperties user_collected_properties;
UserCollectedProperties readable_properties;
// convert this object to a human readable form
// @prop_delim: delimiter for each property.
std::string ToString(const std::string& prop_delim = "; ",
const std::string& kv_delim = "=") const;
// Aggregate the numerical member variables of the specified
// TableProperties.
void Add(const TableProperties& tp);
};
// table properties' human-readable names in the property block.
struct TablePropertiesNames {
static const std::string kDataSize;
@ -80,6 +39,8 @@ struct TablePropertiesNames {
static const std::string kFormatVersion;
static const std::string kFixedKeyLen;
static const std::string kFilterPolicy;
static const std::string kColumnFamilyName;
static const std::string kColumnFamilyId;
};
extern const std::string kPropertiesBlock;
@ -158,6 +119,55 @@ class TablePropertiesCollectorFactory {
virtual const char* Name() const = 0;
};
// TableProperties contains a bunch of read-only properties of its associated
// table.
struct TableProperties {
public:
// the total size of all data blocks.
uint64_t data_size = 0;
// the size of index block.
uint64_t index_size = 0;
// the size of filter block.
uint64_t filter_size = 0;
// total raw key size
uint64_t raw_key_size = 0;
// total raw value size
uint64_t raw_value_size = 0;
// the number of blocks in this table
uint64_t num_data_blocks = 0;
// the number of entries in this table
uint64_t num_entries = 0;
// format version, reserved for backward compatibility
uint64_t format_version = 0;
// If 0, key is variable length. Otherwise number of bytes for each key.
uint64_t fixed_key_len = 0;
// ID of column family for this SST file, corresponding to the CF identified
// by column_family_name.
uint64_t column_family_id =
rocksdb::TablePropertiesCollectorFactory::Context::kUnknownColumnFamily;
// Name of the column family with which this SST file is associated.
// If column family is unknown, `column_family_name` will be an empty string.
std::string column_family_name;
// The name of the filter policy used in this table.
// If no filter policy is used, `filter_policy_name` will be an empty string.
std::string filter_policy_name;
// user collected properties
UserCollectedProperties user_collected_properties;
UserCollectedProperties readable_properties;
// convert this object to a human readable form
// @prop_delim: delimiter for each property.
std::string ToString(const std::string& prop_delim = "; ",
const std::string& kv_delim = "=") const;
// Aggregate the numerical member variables of the specified
// TableProperties.
void Add(const TableProperties& tp);
};
// Extra properties
// Below is a list of non-basic properties that are collected by database
// itself. Especially some properties regarding to the internal keys (which

19
table/block_based_table_builder.cc
View File

@ -472,6 +472,8 @@ struct BlockBasedTableBuilder::Rep {
std::string compressed_output;
std::unique_ptr<FlushBlockPolicy> flush_block_policy;
uint32_t column_family_id;
const std::string& column_family_name;
std::vector<std::unique_ptr<IntTblPropCollector>> table_properties_collectors;
@ -480,9 +482,10 @@ struct BlockBasedTableBuilder::Rep {
const InternalKeyComparator& icomparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* f,
uint32_t _column_family_id, WritableFileWriter* f,
const CompressionType _compression_type,
const CompressionOptions& _compression_opts, const bool skip_filters)
const CompressionOptions& _compression_opts, const bool skip_filters,
const std::string& _column_family_name)
: ioptions(_ioptions),
table_options(table_opt),
internal_comparator(icomparator),
@ -500,7 +503,9 @@ struct BlockBasedTableBuilder::Rep {
_ioptions, table_options)),
flush_block_policy(
table_options.flush_block_policy_factory->NewFlushBlockPolicy(
table_options, data_block)) {
table_options, data_block)),
column_family_id(_column_family_id),
column_family_name(_column_family_name) {
for (auto& collector_factories : *int_tbl_prop_collector_factories) {
table_properties_collectors.emplace_back(
collector_factories->CreateIntTblPropCollector(column_family_id));
@ -520,7 +525,8 @@ BlockBasedTableBuilder::BlockBasedTableBuilder(
int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* file,
const CompressionType compression_type,
const CompressionOptions& compression_opts, const bool skip_filters) {
const CompressionOptions& compression_opts, const bool skip_filters,
const std::string& column_family_name) {
BlockBasedTableOptions sanitized_table_options(table_options);
if (sanitized_table_options.format_version == 0 &&
sanitized_table_options.checksum != kCRC32c) {
@ -534,7 +540,8 @@ BlockBasedTableBuilder::BlockBasedTableBuilder(
rep_ = new Rep(ioptions, sanitized_table_options, internal_comparator,
int_tbl_prop_collector_factories, column_family_id, file,
compression_type, compression_opts, skip_filters);
compression_type, compression_opts, skip_filters,
column_family_name);
if (rep_->filter_block != nullptr) {
rep_->filter_block->StartBlock(0);
@ -790,6 +797,8 @@ Status BlockBasedTableBuilder::Finish() {
// Write properties block.
{
PropertyBlockBuilder property_block_builder;
r->props.column_family_id = r->column_family_id;
r->props.column_family_name = r->column_family_name;
r->props.filter_policy_name = r->table_options.filter_policy != nullptr ?
r->table_options.filter_policy->Name() : "";
r->props.index_size =

3
table/block_based_table_builder.h
View File

@ -42,7 +42,8 @@ class BlockBasedTableBuilder : public TableBuilder {
int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* file,
const CompressionType compression_type,
const CompressionOptions& compression_opts, const bool skip_filters);
const CompressionOptions& compression_opts, const bool skip_filters,
const std::string& column_family_name);
// REQUIRES: Either Finish() or Abandon() has been called.
~BlockBasedTableBuilder();

3
table/block_based_table_factory.cc
View File

@ -76,7 +76,8 @@ TableBuilder* BlockBasedTableFactory::NewTableBuilder(
table_builder_options.int_tbl_prop_collector_factories, column_family_id,
file, table_builder_options.compression_type,
table_builder_options.compression_opts,
table_builder_options.skip_filters);
table_builder_options.skip_filters,
table_builder_options.column_family_name);
return table_builder;
}

5
table/cuckoo_table_builder.cc
View File

@ -52,7 +52,8 @@ CuckooTableBuilder::CuckooTableBuilder(
uint32_t max_num_hash_table, uint32_t max_search_depth,
const Comparator* user_comparator, uint32_t cuckoo_block_size,
bool use_module_hash, bool identity_as_first_hash,
uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t))
uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t),
uint32_t column_family_id, const std::string& column_family_name)
: num_hash_func_(2),
file_(file),
max_hash_table_ratio_(max_hash_table_ratio),
@ -76,6 +77,8 @@ CuckooTableBuilder::CuckooTableBuilder(
properties_.num_data_blocks = 1;
properties_.index_size = 0;
properties_.filter_size = 0;
properties_.column_family_id = column_family_id;
properties_.column_family_name = column_family_name;
}
void CuckooTableBuilder::Add(const Slice& key, const Slice& value) {

4
table/cuckoo_table_builder.h
View File

@ -27,7 +27,9 @@ class CuckooTableBuilder: public TableBuilder {
uint32_t cuckoo_block_size, bool use_module_hash,
bool identity_as_first_hash,
uint64_t (*get_slice_hash)(const Slice&, uint32_t,
uint64_t));
uint64_t),
uint32_t column_family_id,
const std::string& column_family_name);
// REQUIRES: Either Finish() or Abandon() has been called.
~CuckooTableBuilder() {}

39
table/cuckoo_table_builder_test.cc
View File

@ -89,6 +89,8 @@ class CuckooBuilderTest : public testing::Test {
ASSERT_EQ(props->data_size, expected_unused_bucket.size() *
(expected_table_size + expected_cuckoo_block_size - 1));
ASSERT_EQ(props->raw_key_size, keys.size()*props->fixed_key_len);
ASSERT_EQ(props->column_family_id, 0);
ASSERT_EQ(props->column_family_name, kDefaultColumnFamilyName);
delete props;
// Check contents of the bucket.
@ -148,7 +150,8 @@ TEST_F(CuckooBuilderTest, SuccessWithEmptyFile) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, 4, 100,
BytewiseComparator(), 1, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
ASSERT_EQ(0UL, builder.FileSize());
ASSERT_OK(builder.Finish());
@ -183,7 +186,8 @@ TEST_F(CuckooBuilderTest, WriteSuccessNoCollisionFullKey) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
100, BytewiseComparator(), 1, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t i = 0; i < user_keys.size(); i++) {
builder.Add(Slice(keys[i]), Slice(values[i]));
@ -230,7 +234,8 @@ TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionFullKey) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
100, BytewiseComparator(), 1, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t i = 0; i < user_keys.size(); i++) {
builder.Add(Slice(keys[i]), Slice(values[i]));
@ -276,9 +281,10 @@ TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionAndCuckooBlock) {
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
unique_ptr<WritableFileWriter> file_writer(
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
100, BytewiseComparator(), cuckoo_block_size,
false, false, GetSliceHash);
CuckooTableBuilder builder(
file_writer.get(), kHashTableRatio, num_hash_fun, 100,
BytewiseComparator(), cuckoo_block_size, false, false, GetSliceHash,
0 /* column_family_id */, kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t i = 0; i < user_keys.size(); i++) {
builder.Add(Slice(keys[i]), Slice(values[i]));
@ -330,7 +336,8 @@ TEST_F(CuckooBuilderTest, WithCollisionPathFullKey) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
100, BytewiseComparator(), 1, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t i = 0; i < user_keys.size(); i++) {
builder.Add(Slice(keys[i]), Slice(values[i]));
@ -379,7 +386,8 @@ TEST_F(CuckooBuilderTest, WithCollisionPathFullKeyAndCuckooBlock) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
100, BytewiseComparator(), 2, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t i = 0; i < user_keys.size(); i++) {
builder.Add(Slice(keys[i]), Slice(values[i]));
@ -421,7 +429,8 @@ TEST_F(CuckooBuilderTest, WriteSuccessNoCollisionUserKey) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
100, BytewiseComparator(), 1, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t i = 0; i < user_keys.size(); i++) {
builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i]));
@ -464,7 +473,8 @@ TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionUserKey) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
100, BytewiseComparator(), 1, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t i = 0; i < user_keys.size(); i++) {
builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i]));
@ -509,7 +519,8 @@ TEST_F(CuckooBuilderTest, WithCollisionPathUserKey) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
2, BytewiseComparator(), 1, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t i = 0; i < user_keys.size(); i++) {
builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i]));
@ -553,7 +564,8 @@ TEST_F(CuckooBuilderTest, FailWhenCollisionPathTooLong) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
2, BytewiseComparator(), 1, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t i = 0; i < user_keys.size(); i++) {
builder.Add(Slice(GetInternalKey(user_keys[i], false)), Slice("value"));
@ -580,7 +592,8 @@ TEST_F(CuckooBuilderTest, FailWhenSameKeyInserted) {
new WritableFileWriter(std::move(writable_file), EnvOptions()));
CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun,
100, BytewiseComparator(), 1, false, false,
GetSliceHash);
GetSliceHash, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
builder.Add(Slice(GetInternalKey(user_key, false)), Slice("value1"));

3
table/cuckoo_table_factory.cc
View File

@ -38,7 +38,8 @@ TableBuilder* CuckooTableFactory::NewTableBuilder(
table_options_.max_search_depth,
table_builder_options.internal_comparator.user_comparator(),
table_options_.cuckoo_block_size, table_options_.use_module_hash,
table_options_.identity_as_first_hash, nullptr);
table_options_.identity_as_first_hash, nullptr /* get_slice_hash */,
column_family_id, table_builder_options.column_family_name);
}
std::string CuckooTableFactory::GetPrintableTableOptions() const {

12
table/cuckoo_table_reader_test.cc
View File

@ -98,8 +98,9 @@ class CuckooReaderTest : public testing::Test {
unique_ptr<WritableFileWriter> file_writer(
new WritableFileWriter(std::move(writable_file), env_options));
CuckooTableBuilder builder(file_writer.get(), 0.9, kNumHashFunc, 100, ucomp,
2, false, false, GetSliceHash);
CuckooTableBuilder builder(
file_writer.get(), 0.9, kNumHashFunc, 100, ucomp, 2, false, false,
GetSliceHash, 0 /* column_family_id */, kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint32_t key_idx = 0; key_idx < num_items; ++key_idx) {
builder.Add(Slice(keys[key_idx]), Slice(values[key_idx]));
@ -405,9 +406,10 @@ void WriteFile(const std::vector<std::string>& keys,
ASSERT_OK(env->NewWritableFile(fname, &writable_file, env_options));
unique_ptr<WritableFileWriter> file_writer(
new WritableFileWriter(std::move(writable_file), env_options));
CuckooTableBuilder builder(file_writer.get(), hash_ratio, 64, 1000,
test::Uint64Comparator(), 5, false,
FLAGS_identity_as_first_hash, nullptr);
CuckooTableBuilder builder(
file_writer.get(), hash_ratio, 64, 1000, test::Uint64Comparator(), 5,
false, FLAGS_identity_as_first_hash, nullptr, 0 /* column_family_id */,
kDefaultColumnFamilyName);
ASSERT_OK(builder.status());
for (uint64_t key_idx = 0; key_idx < num; ++key_idx) {
// Value is just a part of key.

11
table/meta_blocks.cc
View File

@ -69,11 +69,15 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) {
Add(TablePropertiesNames::kFilterSize, props.filter_size);
Add(TablePropertiesNames::kFormatVersion, props.format_version);
Add(TablePropertiesNames::kFixedKeyLen, props.fixed_key_len);
Add(TablePropertiesNames::kColumnFamilyId, props.column_family_id);
if (!props.filter_policy_name.empty()) {
Add(TablePropertiesNames::kFilterPolicy,
props.filter_policy_name);
}
if (!props.column_family_name.empty()) {
Add(TablePropertiesNames::kColumnFamilyName, props.column_family_name);
}
}
Slice PropertyBlockBuilder::Finish() {
@ -171,7 +175,10 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
{TablePropertiesNames::kFormatVersion,
&new_table_properties->format_version},
{TablePropertiesNames::kFixedKeyLen,
&new_table_properties->fixed_key_len}, };
&new_table_properties->fixed_key_len},
{TablePropertiesNames::kColumnFamilyId,
&new_table_properties->column_family_id},
};
std::string last_key;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
@ -203,6 +210,8 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
*(pos->second) = val;
} else if (key == TablePropertiesNames::kFilterPolicy) {
new_table_properties->filter_policy_name = raw_val.ToString();
} else if (key == TablePropertiesNames::kColumnFamilyName) {
new_table_properties->column_family_name = raw_val.ToString();
} else {
// handle user-collected properties
new_table_properties->user_collected_properties.insert(

7
table/plain_table_builder.cc
View File

@ -62,8 +62,9 @@ PlainTableBuilder::PlainTableBuilder(
int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* file, uint32_t user_key_len,
EncodingType encoding_type, size_t index_sparseness,
uint32_t bloom_bits_per_key, uint32_t num_probes, size_t huge_page_tlb_size,
double hash_table_ratio, bool store_index_in_file)
uint32_t bloom_bits_per_key, const std::string& column_family_name,
uint32_t num_probes, size_t huge_page_tlb_size, double hash_table_ratio,
bool store_index_in_file)
: ioptions_(ioptions),
bloom_block_(num_probes),
file_(file),
@ -94,6 +95,8 @@ PlainTableBuilder::PlainTableBuilder(
// To support roll-back to previous version, now still use version 0 for
// plain encoding.
properties_.format_version = (encoding_type == kPlain) ? 0 : 1;
properties_.column_family_id = column_family_id;
properties_.column_family_name = column_family_name;
if (ioptions_.prefix_extractor) {
properties_.user_collected_properties

10
table/plain_table_builder.h
View File

@ -6,15 +6,16 @@
#pragma once
#ifndef ROCKSDB_LITE
#include <stdint.h>
#include <string>
#include <vector>
#include "rocksdb/options.h"
#include "rocksdb/status.h"
#include "table/table_builder.h"
#include "table/plain_table_key_coding.h"
#include "rocksdb/table.h"
#include "rocksdb/table_properties.h"
#include "table/bloom_block.h"
#include "table/plain_table_index.h"
#include "table/plain_table_key_coding.h"
#include "table/table_builder.h"
namespace rocksdb {
@ -37,8 +38,9 @@ class PlainTableBuilder: public TableBuilder {
uint32_t column_family_id, WritableFileWriter* file,
uint32_t user_key_size, EncodingType encoding_type,
size_t index_sparseness, uint32_t bloom_bits_per_key,
uint32_t num_probes = 6, size_t huge_page_tlb_size = 0,
double hash_table_ratio = 0, bool store_index_in_file = false);
const std::string& column_family_name, uint32_t num_probes = 6,
size_t huge_page_tlb_size = 0, double hash_table_ratio = 0,
bool store_index_in_file = false);
// REQUIRES: Either Finish() or Abandon() has been called.
~PlainTableBuilder();

3
table/plain_table_factory.cc
View File

@ -37,7 +37,8 @@ TableBuilder* PlainTableFactory::NewTableBuilder(
table_builder_options.ioptions,
table_builder_options.int_tbl_prop_collector_factories, column_family_id,
file, table_options_.user_key_len, table_options_.encoding_type,
table_options_.index_sparseness, table_options_.bloom_bits_per_key, 6,
table_options_.index_sparseness, table_options_.bloom_bits_per_key,
table_builder_options.column_family_name, 6,
table_options_.huge_page_tlb_size, table_options_.hash_table_ratio,
table_options_.store_index_in_file);
}

4
table/sst_file_writer.cc
View File

@ -83,6 +83,7 @@ struct SstFileWriter::Rep {
ImmutableCFOptions ioptions;
InternalKeyComparator internal_comparator;
ExternalSstFileInfo file_info;
std::string column_family_name;
};
SstFileWriter::SstFileWriter(const EnvOptions& env_options,
@ -114,7 +115,8 @@ Status SstFileWriter::Open(const std::string& file_path) {
TableBuilderOptions table_builder_options(
r->ioptions, r->internal_comparator, &int_tbl_prop_collector_factories,
compression_type, r->ioptions.compression_opts, false);
compression_type, r->ioptions.compression_opts, false /* skip_filters */,
r->column_family_name);
r->file_writer.reset(
new WritableFileWriter(std::move(sst_file), r->env_options));
r->builder.reset(r->ioptions.table_factory->NewTableBuilder(

10
table/table_builder.h
View File

@ -52,21 +52,23 @@ struct TableBuilderOptions {
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
_int_tbl_prop_collector_factories,
CompressionType _compression_type,
const CompressionOptions& _compression_opts, bool _skip_filters)
const CompressionOptions& _compression_opts, bool _skip_filters,
const std::string& _column_family_name)
: ioptions(_ioptions),
internal_comparator(_internal_comparator),
int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories),
compression_type(_compression_type),
compression_opts(_compression_opts),
skip_filters(_skip_filters) {}
skip_filters(_skip_filters),
column_family_name(_column_family_name) {}
const ImmutableCFOptions& ioptions;
const InternalKeyComparator& internal_comparator;
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories;
CompressionType compression_type;
const CompressionOptions& compression_opts;
// This is only used for BlockBasedTableBuilder
bool skip_filters = false;
bool skip_filters; // only used by BlockBasedTableBuilder
const std::string& column_family_name;
};
// TableBuilder provides the interface used to build a Table

4
table/table_properties.cc
View File

@ -108,6 +108,10 @@ const std::string TablePropertiesNames::kFormatVersion =
"rocksdb.format.version";
const std::string TablePropertiesNames::kFixedKeyLen =
"rocksdb.fixed.key.length";
const std::string TablePropertiesNames::kColumnFamilyId =
"rocksdb.column.family.id";
const std::string TablePropertiesNames::kColumnFamilyName =
"rocksdb.column.family.name";
extern const std::string kPropertiesBlock = "rocksdb.properties";
// Old property block name for backward compatibility

5
table/table_reader_bench.cc
View File

@ -98,8 +98,9 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
tb = opts.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
CompressionType::kNoCompression,
CompressionOptions(), false),
0, file_writer.get());
CompressionOptions(), false /* skip_filters */,
kDefaultColumnFamilyName),
0 /* column_family_id */, file_writer.get());
} else {
s = DB::Open(opts, dbname, &db);
ASSERT_OK(s);

12
table/table_test.cc
View File

@ -270,10 +270,12 @@ class TableConstructor: public Constructor {
unique_ptr<TableBuilder> builder;
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories;
std::string column_family_name;
builder.reset(ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, internal_comparator,
&int_tbl_prop_collector_factories,
options.compression, CompressionOptions(), false),
TableBuilderOptions(
ioptions, internal_comparator, &int_tbl_prop_collector_factories,
options.compression, CompressionOptions(), false /* skip_filters */,
column_family_name),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer_.get()));
@ -1941,9 +1943,11 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) {
InternalKeyComparator ikc(options.comparator);
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories;
std::string column_family_name;
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
kNoCompression, CompressionOptions(), false),
kNoCompression, CompressionOptions(),
false /* skip_filters */, column_family_name),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));

3
tools/sst_dump_test.cc
View File

@ -58,10 +58,11 @@ void createSST(const std::string& file_name,
int_tbl_prop_collector_factories;
unique_ptr<WritableFileWriter> file_writer(
new WritableFileWriter(std::move(file), EnvOptions()));
std::string column_family_name;
tb.reset(opts.table_factory->NewTableBuilder(
TableBuilderOptions(imoptions, ikc, &int_tbl_prop_collector_factories,
CompressionType::kNoCompression, CompressionOptions(),
false),
false /* skip_filters */, column_family_name),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));

4
tools/sst_dump_tool.cc
View File

@ -203,8 +203,10 @@ int SstFileReader::ShowAllCompressionSizes(size_t block_size) {
i = (i == kLZ4HCCompression) ? kZSTDNotFinalCompression
: CompressionType(i + 1)) {
CompressionOptions compress_opt;
std::string column_family_name;
TableBuilderOptions tb_opts(imoptions, ikc, &block_based_table_factories, i,
compress_opt, false);
compress_opt, false /* skip_filters */,
column_family_name);
uint64_t file_size = CalculateCompressedTableSize(tb_opts, block_size);
fprintf(stdout, "Compression: %s", compress_type.find(i)->second);
fprintf(stdout, " Size: %" PRIu64 "\n", file_size);