mirror of https://github.com/facebook/rocksdb.git
Ran clang-format on db/ directory (#10910)
Summary: Ran `find ./db/ -type f | xargs clang-format -i`. Excluded minor changes it tried to make on db/db_impl/. Everything else it changed was directly under db/ directory. Included minor manual touchups mentioned in PR commit history. Pull Request resolved: https://github.com/facebook/rocksdb/pull/10910 Reviewed By: riversand963 Differential Revision: D40880683 Pulled By: ajkr fbshipit-source-id: cfe26cda05b3fb9a72e3cb82c286e21d8c5c4174
This commit is contained in:
parent
ff9ad2c39b
commit
5cf6ab6f31
|
@ -8,6 +8,7 @@
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
#include "db/arena_wrapped_db_iter.h"
|
#include "db/arena_wrapped_db_iter.h"
|
||||||
|
|
||||||
#include "memory/arena.h"
|
#include "memory/arena.h"
|
||||||
#include "rocksdb/env.h"
|
#include "rocksdb/env.h"
|
||||||
#include "rocksdb/iterator.h"
|
#include "rocksdb/iterator.h"
|
||||||
|
|
|
@ -9,7 +9,9 @@
|
||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "db/db_impl/db_impl.h"
|
#include "db/db_impl/db_impl.h"
|
||||||
#include "db/db_iter.h"
|
#include "db/db_iter.h"
|
||||||
#include "db/range_del_aggregator.h"
|
#include "db/range_del_aggregator.h"
|
||||||
|
|
|
@ -281,7 +281,8 @@ Status BuildTable(
|
||||||
meta->fd.file_size = file_size;
|
meta->fd.file_size = file_size;
|
||||||
meta->marked_for_compaction = builder->NeedCompact();
|
meta->marked_for_compaction = builder->NeedCompact();
|
||||||
assert(meta->fd.GetFileSize() > 0);
|
assert(meta->fd.GetFileSize() > 0);
|
||||||
tp = builder->GetTableProperties(); // refresh now that builder is finished
|
tp = builder
|
||||||
|
->GetTableProperties(); // refresh now that builder is finished
|
||||||
if (memtable_payload_bytes != nullptr &&
|
if (memtable_payload_bytes != nullptr &&
|
||||||
memtable_garbage_bytes != nullptr) {
|
memtable_garbage_bytes != nullptr) {
|
||||||
const CompactionIterationStats& ci_stats = c_iter.iter_stats();
|
const CompactionIterationStats& ci_stats = c_iter.iter_stats();
|
||||||
|
|
175
db/c_test.c
175
db/c_test.c
|
@ -98,21 +98,15 @@ static void Free(char** ptr) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckValue(
|
static void CheckValue(char* err, const char* expected, char** actual,
|
||||||
char* err,
|
|
||||||
const char* expected,
|
|
||||||
char** actual,
|
|
||||||
size_t actual_length) {
|
size_t actual_length) {
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
CheckEqual(expected, *actual, actual_length);
|
CheckEqual(expected, *actual, actual_length);
|
||||||
Free(actual);
|
Free(actual);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckGet(
|
static void CheckGet(rocksdb_t* db, const rocksdb_readoptions_t* options,
|
||||||
rocksdb_t* db,
|
const char* key, const char* expected) {
|
||||||
const rocksdb_readoptions_t* options,
|
|
||||||
const char* key,
|
|
||||||
const char* expected) {
|
|
||||||
char* err = NULL;
|
char* err = NULL;
|
||||||
size_t val_len;
|
size_t val_len;
|
||||||
char* val;
|
char* val;
|
||||||
|
@ -122,11 +116,8 @@ static void CheckGet(
|
||||||
Free(&val);
|
Free(&val);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckGetCF(
|
static void CheckGetCF(rocksdb_t* db, const rocksdb_readoptions_t* options,
|
||||||
rocksdb_t* db,
|
rocksdb_column_family_handle_t* handle, const char* key,
|
||||||
const rocksdb_readoptions_t* options,
|
|
||||||
rocksdb_column_family_handle_t* handle,
|
|
||||||
const char* key,
|
|
||||||
const char* expected) {
|
const char* expected) {
|
||||||
char* err = NULL;
|
char* err = NULL;
|
||||||
size_t val_len;
|
size_t val_len;
|
||||||
|
@ -174,8 +165,8 @@ static void CheckMultiGetValues(size_t num_keys, char** values,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckIter(rocksdb_iterator_t* iter,
|
static void CheckIter(rocksdb_iterator_t* iter, const char* key,
|
||||||
const char* key, const char* val) {
|
const char* val) {
|
||||||
size_t len;
|
size_t len;
|
||||||
const char* str;
|
const char* str;
|
||||||
str = rocksdb_iter_key(iter, &len);
|
str = rocksdb_iter_key(iter, &len);
|
||||||
|
@ -185,9 +176,8 @@ static void CheckIter(rocksdb_iterator_t* iter,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Callback from rocksdb_writebatch_iterate()
|
// Callback from rocksdb_writebatch_iterate()
|
||||||
static void CheckPut(void* ptr,
|
static void CheckPut(void* ptr, const char* k, size_t klen, const char* v,
|
||||||
const char* k, size_t klen,
|
size_t vlen) {
|
||||||
const char* v, size_t vlen) {
|
|
||||||
int* state = (int*)ptr;
|
int* state = (int*)ptr;
|
||||||
CheckCondition(*state < 2);
|
CheckCondition(*state < 2);
|
||||||
switch (*state) {
|
switch (*state) {
|
||||||
|
@ -213,14 +203,16 @@ static void CheckDel(void* ptr, const char* k, size_t klen) {
|
||||||
|
|
||||||
static void CmpDestroy(void* arg) { (void)arg; }
|
static void CmpDestroy(void* arg) { (void)arg; }
|
||||||
|
|
||||||
static int CmpCompare(void* arg, const char* a, size_t alen,
|
static int CmpCompare(void* arg, const char* a, size_t alen, const char* b,
|
||||||
const char* b, size_t blen) {
|
size_t blen) {
|
||||||
(void)arg;
|
(void)arg;
|
||||||
size_t n = (alen < blen) ? alen : blen;
|
size_t n = (alen < blen) ? alen : blen;
|
||||||
int r = memcmp(a, b, n);
|
int r = memcmp(a, b, n);
|
||||||
if (r == 0) {
|
if (r == 0) {
|
||||||
if (alen < blen) r = -1;
|
if (alen < blen)
|
||||||
else if (alen > blen) r = +1;
|
r = -1;
|
||||||
|
else if (alen > blen)
|
||||||
|
r = +1;
|
||||||
}
|
}
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
@ -405,11 +397,9 @@ static const char* MergeOperatorName(void* arg) {
|
||||||
return "TestMergeOperator";
|
return "TestMergeOperator";
|
||||||
}
|
}
|
||||||
static char* MergeOperatorFullMerge(
|
static char* MergeOperatorFullMerge(
|
||||||
void* arg,
|
void* arg, const char* key, size_t key_length, const char* existing_value,
|
||||||
const char* key, size_t key_length,
|
size_t existing_value_length, const char* const* operands_list,
|
||||||
const char* existing_value, size_t existing_value_length,
|
const size_t* operands_list_length, int num_operands,
|
||||||
const char* const* operands_list, const size_t* operands_list_length,
|
|
||||||
int num_operands,
|
|
||||||
unsigned char* success, size_t* new_value_length) {
|
unsigned char* success, size_t* new_value_length) {
|
||||||
(void)arg;
|
(void)arg;
|
||||||
(void)key;
|
(void)key;
|
||||||
|
@ -425,12 +415,12 @@ static char* MergeOperatorFullMerge(
|
||||||
memcpy(result, "fake", 4);
|
memcpy(result, "fake", 4);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
static char* MergeOperatorPartialMerge(
|
static char* MergeOperatorPartialMerge(void* arg, const char* key,
|
||||||
void* arg,
|
size_t key_length,
|
||||||
const char* key, size_t key_length,
|
const char* const* operands_list,
|
||||||
const char* const* operands_list, const size_t* operands_list_length,
|
const size_t* operands_list_length,
|
||||||
int num_operands,
|
int num_operands, unsigned char* success,
|
||||||
unsigned char* success, size_t* new_value_length) {
|
size_t* new_value_length) {
|
||||||
(void)arg;
|
(void)arg;
|
||||||
(void)key;
|
(void)key;
|
||||||
(void)key_length;
|
(void)key_length;
|
||||||
|
@ -444,10 +434,8 @@ static char* MergeOperatorPartialMerge(
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckTxnGet(
|
static void CheckTxnGet(rocksdb_transaction_t* txn,
|
||||||
rocksdb_transaction_t* txn,
|
const rocksdb_readoptions_t* options, const char* key,
|
||||||
const rocksdb_readoptions_t* options,
|
|
||||||
const char* key,
|
|
||||||
const char* expected) {
|
const char* expected) {
|
||||||
char* err = NULL;
|
char* err = NULL;
|
||||||
size_t val_len;
|
size_t val_len;
|
||||||
|
@ -502,10 +490,8 @@ static void CheckTxnPinGetCF(rocksdb_transaction_t* txn,
|
||||||
rocksdb_pinnableslice_destroy(p);
|
rocksdb_pinnableslice_destroy(p);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckTxnDBGet(
|
static void CheckTxnDBGet(rocksdb_transactiondb_t* txn_db,
|
||||||
rocksdb_transactiondb_t* txn_db,
|
const rocksdb_readoptions_t* options, const char* key,
|
||||||
const rocksdb_readoptions_t* options,
|
|
||||||
const char* key,
|
|
||||||
const char* expected) {
|
const char* expected) {
|
||||||
char* err = NULL;
|
char* err = NULL;
|
||||||
size_t val_len;
|
size_t val_len;
|
||||||
|
@ -649,30 +635,20 @@ int main(int argc, char** argv) {
|
||||||
char* err = NULL;
|
char* err = NULL;
|
||||||
int run = -1;
|
int run = -1;
|
||||||
|
|
||||||
snprintf(dbname, sizeof(dbname),
|
snprintf(dbname, sizeof(dbname), "%s/rocksdb_c_test-%d", GetTempDir(),
|
||||||
"%s/rocksdb_c_test-%d",
|
|
||||||
GetTempDir(),
|
|
||||||
((int)geteuid()));
|
((int)geteuid()));
|
||||||
|
|
||||||
snprintf(dbbackupname, sizeof(dbbackupname),
|
snprintf(dbbackupname, sizeof(dbbackupname), "%s/rocksdb_c_test-%d-backup",
|
||||||
"%s/rocksdb_c_test-%d-backup",
|
GetTempDir(), ((int)geteuid()));
|
||||||
GetTempDir(),
|
|
||||||
((int) geteuid()));
|
|
||||||
|
|
||||||
snprintf(dbcheckpointname, sizeof(dbcheckpointname),
|
snprintf(dbcheckpointname, sizeof(dbcheckpointname),
|
||||||
"%s/rocksdb_c_test-%d-checkpoint",
|
"%s/rocksdb_c_test-%d-checkpoint", GetTempDir(), ((int)geteuid()));
|
||||||
GetTempDir(),
|
|
||||||
((int) geteuid()));
|
|
||||||
|
|
||||||
snprintf(sstfilename, sizeof(sstfilename),
|
snprintf(sstfilename, sizeof(sstfilename), "%s/rocksdb_c_test-%d-sst",
|
||||||
"%s/rocksdb_c_test-%d-sst",
|
GetTempDir(), ((int)geteuid()));
|
||||||
GetTempDir(),
|
|
||||||
((int)geteuid()));
|
|
||||||
|
|
||||||
snprintf(dbpathname, sizeof(dbpathname),
|
snprintf(dbpathname, sizeof(dbpathname), "%s/rocksdb_c_test-%d-dbpath",
|
||||||
"%s/rocksdb_c_test-%d-dbpath",
|
GetTempDir(), ((int)geteuid()));
|
||||||
GetTempDir(),
|
|
||||||
((int) geteuid()));
|
|
||||||
|
|
||||||
StartPhase("create_objects");
|
StartPhase("create_objects");
|
||||||
cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
|
cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
|
||||||
|
@ -746,7 +722,8 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_destroy_db(options, dbbackupname, &err);
|
rocksdb_destroy_db(options, dbbackupname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_backup_engine_t *be = rocksdb_backup_engine_open(options, dbbackupname, &err);
|
rocksdb_backup_engine_t* be =
|
||||||
|
rocksdb_backup_engine_open(options, dbbackupname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_backup_engine_create_new_backup(be, db, &err);
|
rocksdb_backup_engine_create_new_backup(be, db, &err);
|
||||||
|
@ -759,7 +736,8 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_backup_engine_create_new_backup(be, db, &err);
|
rocksdb_backup_engine_create_new_backup(be, db, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
const rocksdb_backup_engine_info_t* bei = rocksdb_backup_engine_get_backup_info(be);
|
const rocksdb_backup_engine_info_t* bei =
|
||||||
|
rocksdb_backup_engine_get_backup_info(be);
|
||||||
CheckCondition(rocksdb_backup_engine_info_count(bei) > 1);
|
CheckCondition(rocksdb_backup_engine_info_count(bei) > 1);
|
||||||
rocksdb_backup_engine_info_destroy(bei);
|
rocksdb_backup_engine_info_destroy(bei);
|
||||||
|
|
||||||
|
@ -778,9 +756,11 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_destroy_db(options, dbname, &err);
|
rocksdb_destroy_db(options, dbname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_restore_options_t *restore_options = rocksdb_restore_options_create();
|
rocksdb_restore_options_t* restore_options =
|
||||||
|
rocksdb_restore_options_create();
|
||||||
rocksdb_restore_options_set_keep_log_files(restore_options, 0);
|
rocksdb_restore_options_set_keep_log_files(restore_options, 0);
|
||||||
rocksdb_backup_engine_restore_db_from_latest_backup(be, dbname, dbname, restore_options, &err);
|
rocksdb_backup_engine_restore_db_from_latest_backup(be, dbname, dbname,
|
||||||
|
restore_options, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
rocksdb_restore_options_destroy(restore_options);
|
rocksdb_restore_options_destroy(restore_options);
|
||||||
|
|
||||||
|
@ -799,7 +779,8 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_destroy_db(options, dbcheckpointname, &err);
|
rocksdb_destroy_db(options, dbcheckpointname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_checkpoint_t* checkpoint = rocksdb_checkpoint_object_create(db, &err);
|
rocksdb_checkpoint_t* checkpoint =
|
||||||
|
rocksdb_checkpoint_object_create(db, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err);
|
rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err);
|
||||||
|
@ -1041,13 +1022,17 @@ int main(int argc, char** argv) {
|
||||||
CheckCondition(count == 3);
|
CheckCondition(count == 3);
|
||||||
size_t size;
|
size_t size;
|
||||||
char* value;
|
char* value;
|
||||||
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "box", 3, &size, &err);
|
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "box", 3, &size,
|
||||||
|
&err);
|
||||||
CheckValue(err, "c", &value, size);
|
CheckValue(err, "c", &value, size);
|
||||||
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "bar", 3, &size, &err);
|
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "bar", 3, &size,
|
||||||
|
&err);
|
||||||
CheckValue(err, NULL, &value, size);
|
CheckValue(err, NULL, &value, size);
|
||||||
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "foo", 3, &size, &err);
|
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions,
|
||||||
|
"foo", 3, &size, &err);
|
||||||
CheckValue(err, "hello", &value, size);
|
CheckValue(err, "hello", &value, size);
|
||||||
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "box", 3, &size, &err);
|
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions,
|
||||||
|
"box", 3, &size, &err);
|
||||||
CheckValue(err, "c", &value, size);
|
CheckValue(err, "c", &value, size);
|
||||||
rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
|
rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
@ -1162,7 +1147,8 @@ int main(int argc, char** argv) {
|
||||||
size_t vals_sizes[3];
|
size_t vals_sizes[3];
|
||||||
char* errs[3];
|
char* errs[3];
|
||||||
const char* expected[3] = {"c", "hello", NULL};
|
const char* expected[3] = {"c", "hello", NULL};
|
||||||
rocksdb_multi_get(db, roptions, 3, keys, keys_sizes, vals, vals_sizes, errs);
|
rocksdb_multi_get(db, roptions, 3, keys, keys_sizes, vals, vals_sizes,
|
||||||
|
errs);
|
||||||
CheckMultiGetValues(3, vals, vals_sizes, errs, expected);
|
CheckMultiGetValues(3, vals, vals_sizes, errs, expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1393,8 +1379,8 @@ int main(int argc, char** argv) {
|
||||||
factory);
|
factory);
|
||||||
db = CheckCompaction(db, options_with_filter_factory, roptions, woptions);
|
db = CheckCompaction(db, options_with_filter_factory, roptions, woptions);
|
||||||
|
|
||||||
rocksdb_options_set_compaction_filter_factory(
|
rocksdb_options_set_compaction_filter_factory(options_with_filter_factory,
|
||||||
options_with_filter_factory, NULL);
|
NULL);
|
||||||
rocksdb_options_destroy(options_with_filter_factory);
|
rocksdb_options_destroy(options_with_filter_factory);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1449,7 +1435,8 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_close(db);
|
rocksdb_close(db);
|
||||||
|
|
||||||
size_t cflen;
|
size_t cflen;
|
||||||
char** column_fams = rocksdb_list_column_families(db_options, dbname, &cflen, &err);
|
char** column_fams =
|
||||||
|
rocksdb_list_column_families(db_options, dbname, &cflen, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
CheckEqual("default", column_fams[0], 7);
|
CheckEqual("default", column_fams[0], 7);
|
||||||
CheckEqual("cf1", column_fams[1], 3);
|
CheckEqual("cf1", column_fams[1], 3);
|
||||||
|
@ -1465,7 +1452,8 @@ int main(int argc, char** argv) {
|
||||||
LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names,
|
LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names,
|
||||||
NULL);
|
NULL);
|
||||||
|
|
||||||
db = rocksdb_open_column_families(db_options, dbname, 2, cf_names, cf_opts, handles, &err);
|
db = rocksdb_open_column_families(db_options, dbname, 2, cf_names, cf_opts,
|
||||||
|
handles, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_put_cf(db, woptions, handles[1], "foo", 3, "hello", 5, &err);
|
rocksdb_put_cf(db, woptions, handles[1], "foo", 3, "hello", 5, &err);
|
||||||
|
@ -1486,8 +1474,7 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create();
|
rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create();
|
||||||
rocksdb_flushoptions_set_wait(flush_options, 1);
|
rocksdb_flushoptions_set_wait(flush_options, 1);
|
||||||
rocksdb_flush_cf(db, flush_options, handles[1], &err);
|
rocksdb_flush_cf(db, flush_options, handles[1], &err);
|
||||||
CheckNoError(err)
|
CheckNoError(err) rocksdb_flushoptions_destroy(flush_options);
|
||||||
rocksdb_flushoptions_destroy(flush_options);
|
|
||||||
|
|
||||||
CheckGetCF(db, roptions, handles[1], "foo", "hello");
|
CheckGetCF(db, roptions, handles[1], "foo", "hello");
|
||||||
CheckPinGetCF(db, roptions, handles[1], "foo", "hello");
|
CheckPinGetCF(db, roptions, handles[1], "foo", "hello");
|
||||||
|
@ -1525,12 +1512,14 @@ int main(int argc, char** argv) {
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
const char* keys[3] = {"box", "box", "barfooxx"};
|
const char* keys[3] = {"box", "box", "barfooxx"};
|
||||||
const rocksdb_column_family_handle_t* get_handles[3] = { handles[0], handles[1], handles[1] };
|
const rocksdb_column_family_handle_t* get_handles[3] = {
|
||||||
|
handles[0], handles[1], handles[1]};
|
||||||
const size_t keys_sizes[3] = {3, 3, 8};
|
const size_t keys_sizes[3] = {3, 3, 8};
|
||||||
char* vals[3];
|
char* vals[3];
|
||||||
size_t vals_sizes[3];
|
size_t vals_sizes[3];
|
||||||
char* errs[3];
|
char* errs[3];
|
||||||
rocksdb_multi_get_cf(db, roptions, get_handles, 3, keys, keys_sizes, vals, vals_sizes, errs);
|
rocksdb_multi_get_cf(db, roptions, get_handles, 3, keys, keys_sizes, vals,
|
||||||
|
vals_sizes, errs);
|
||||||
|
|
||||||
int i;
|
int i;
|
||||||
for (i = 0; i < 3; i++) {
|
for (i = 0; i < 3; i++) {
|
||||||
|
@ -1592,7 +1581,8 @@ int main(int argc, char** argv) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rocksdb_iterator_t* iter = rocksdb_create_iterator_cf(db, roptions, handles[1]);
|
rocksdb_iterator_t* iter =
|
||||||
|
rocksdb_create_iterator_cf(db, roptions, handles[1]);
|
||||||
CheckCondition(!rocksdb_iter_valid(iter));
|
CheckCondition(!rocksdb_iter_valid(iter));
|
||||||
rocksdb_iter_seek_to_first(iter);
|
rocksdb_iter_seek_to_first(iter);
|
||||||
CheckCondition(rocksdb_iter_valid(iter));
|
CheckCondition(rocksdb_iter_valid(iter));
|
||||||
|
@ -1605,9 +1595,11 @@ int main(int argc, char** argv) {
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
rocksdb_iter_destroy(iter);
|
rocksdb_iter_destroy(iter);
|
||||||
|
|
||||||
rocksdb_column_family_handle_t* iters_cf_handles[2] = { handles[0], handles[1] };
|
rocksdb_column_family_handle_t* iters_cf_handles[2] = {handles[0],
|
||||||
|
handles[1]};
|
||||||
rocksdb_iterator_t* iters_handles[2];
|
rocksdb_iterator_t* iters_handles[2];
|
||||||
rocksdb_create_iterators(db, roptions, iters_cf_handles, iters_handles, 2, &err);
|
rocksdb_create_iterators(db, roptions, iters_cf_handles, iters_handles, 2,
|
||||||
|
&err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
iter = iters_handles[0];
|
iter = iters_handles[0];
|
||||||
|
@ -1652,7 +1644,8 @@ int main(int argc, char** argv) {
|
||||||
{
|
{
|
||||||
// Create new database
|
// Create new database
|
||||||
rocksdb_options_set_allow_mmap_reads(options, 1);
|
rocksdb_options_set_allow_mmap_reads(options, 1);
|
||||||
rocksdb_options_set_prefix_extractor(options, rocksdb_slicetransform_create_fixed_prefix(3));
|
rocksdb_options_set_prefix_extractor(
|
||||||
|
options, rocksdb_slicetransform_create_fixed_prefix(3));
|
||||||
rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
|
rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
|
||||||
rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16);
|
rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16);
|
||||||
rocksdb_options_set_allow_concurrent_memtable_write(options, 0);
|
rocksdb_options_set_allow_concurrent_memtable_write(options, 0);
|
||||||
|
@ -1747,7 +1740,8 @@ int main(int argc, char** argv) {
|
||||||
// amount of memory used within memtables should grow
|
// amount of memory used within memtables should grow
|
||||||
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >=
|
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >=
|
||||||
rocksdb_approximate_memory_usage_get_mem_table_total(mu1));
|
rocksdb_approximate_memory_usage_get_mem_table_total(mu1));
|
||||||
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >=
|
CheckCondition(
|
||||||
|
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >=
|
||||||
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1));
|
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1));
|
||||||
|
|
||||||
rocksdb_memory_consumers_destroy(consumers);
|
rocksdb_memory_consumers_destroy(consumers);
|
||||||
|
@ -2839,10 +2833,14 @@ int main(int argc, char** argv) {
|
||||||
db = rocksdb_open(options, dbname, &err);
|
db = rocksdb_open(options, dbname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_put(db, woptions, "a", 1, "0", 1, &err); CheckNoError(err);
|
rocksdb_put(db, woptions, "a", 1, "0", 1, &err);
|
||||||
rocksdb_put(db, woptions, "foo", 3, "bar", 3, &err); CheckNoError(err);
|
CheckNoError(err);
|
||||||
rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err); CheckNoError(err);
|
rocksdb_put(db, woptions, "foo", 3, "bar", 3, &err);
|
||||||
rocksdb_put(db, woptions, "g1", 2, "0", 1, &err); CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err);
|
||||||
|
CheckNoError(err);
|
||||||
|
rocksdb_put(db, woptions, "g1", 2, "0", 1, &err);
|
||||||
|
CheckNoError(err);
|
||||||
|
|
||||||
// testing basic case with no iterate_upper_bound and no prefix_extractor
|
// testing basic case with no iterate_upper_bound and no prefix_extractor
|
||||||
{
|
{
|
||||||
|
@ -3021,7 +3019,8 @@ int main(int argc, char** argv) {
|
||||||
|
|
||||||
// iterate
|
// iterate
|
||||||
rocksdb_transaction_put(txn, "bar", 3, "hi", 2, &err);
|
rocksdb_transaction_put(txn, "bar", 3, "hi", 2, &err);
|
||||||
rocksdb_iterator_t* iter = rocksdb_transaction_create_iterator(txn, roptions);
|
rocksdb_iterator_t* iter =
|
||||||
|
rocksdb_transaction_create_iterator(txn, roptions);
|
||||||
CheckCondition(!rocksdb_iter_valid(iter));
|
CheckCondition(!rocksdb_iter_valid(iter));
|
||||||
rocksdb_iter_seek_to_first(iter);
|
rocksdb_iter_seek_to_first(iter);
|
||||||
CheckCondition(rocksdb_iter_valid(iter));
|
CheckCondition(rocksdb_iter_valid(iter));
|
||||||
|
|
|
@ -192,8 +192,7 @@ Status CheckCFPathsSupported(const DBOptions& db_options,
|
||||||
return Status::NotSupported(
|
return Status::NotSupported(
|
||||||
"More than one CF paths are only supported in "
|
"More than one CF paths are only supported in "
|
||||||
"universal and level compaction styles. ");
|
"universal and level compaction styles. ");
|
||||||
} else if (cf_options.cf_paths.empty() &&
|
} else if (cf_options.cf_paths.empty() && db_options.db_paths.size() > 1) {
|
||||||
db_options.db_paths.size() > 1) {
|
|
||||||
return Status::NotSupported(
|
return Status::NotSupported(
|
||||||
"More than one DB paths are only supported in "
|
"More than one DB paths are only supported in "
|
||||||
"universal and level compaction styles. ");
|
"universal and level compaction styles. ");
|
||||||
|
@ -205,7 +204,7 @@ Status CheckCFPathsSupported(const DBOptions& db_options,
|
||||||
namespace {
|
namespace {
|
||||||
const uint64_t kDefaultTtl = 0xfffffffffffffffe;
|
const uint64_t kDefaultTtl = 0xfffffffffffffffe;
|
||||||
const uint64_t kDefaultPeriodicCompSecs = 0xfffffffffffffffe;
|
const uint64_t kDefaultPeriodicCompSecs = 0xfffffffffffffffe;
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
|
ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
|
||||||
const ColumnFamilyOptions& src) {
|
const ColumnFamilyOptions& src) {
|
||||||
|
@ -353,7 +352,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
|
||||||
// were not deleted yet, when we open the DB we will find these .trash files
|
// were not deleted yet, when we open the DB we will find these .trash files
|
||||||
// and schedule them to be deleted (or delete immediately if SstFileManager
|
// and schedule them to be deleted (or delete immediately if SstFileManager
|
||||||
// was not used)
|
// was not used)
|
||||||
auto sfm = static_cast<SstFileManagerImpl*>(db_options.sst_file_manager.get());
|
auto sfm =
|
||||||
|
static_cast<SstFileManagerImpl*>(db_options.sst_file_manager.get());
|
||||||
for (size_t i = 0; i < result.cf_paths.size(); i++) {
|
for (size_t i = 0; i < result.cf_paths.size(); i++) {
|
||||||
DeleteScheduler::CleanupDirectory(db_options.env, sfm,
|
DeleteScheduler::CleanupDirectory(db_options.env, sfm,
|
||||||
result.cf_paths[i].path)
|
result.cf_paths[i].path)
|
||||||
|
@ -610,8 +610,8 @@ ColumnFamilyData::ColumnFamilyData(
|
||||||
compaction_picker_.reset(
|
compaction_picker_.reset(
|
||||||
new FIFOCompactionPicker(ioptions_, &internal_comparator_));
|
new FIFOCompactionPicker(ioptions_, &internal_comparator_));
|
||||||
} else if (ioptions_.compaction_style == kCompactionStyleNone) {
|
} else if (ioptions_.compaction_style == kCompactionStyleNone) {
|
||||||
compaction_picker_.reset(new NullCompactionPicker(
|
compaction_picker_.reset(
|
||||||
ioptions_, &internal_comparator_));
|
new NullCompactionPicker(ioptions_, &internal_comparator_));
|
||||||
ROCKS_LOG_WARN(ioptions_.logger,
|
ROCKS_LOG_WARN(ioptions_.logger,
|
||||||
"Column family %s does not use any background compaction. "
|
"Column family %s does not use any background compaction. "
|
||||||
"Compactions can only be done via CompactFiles\n",
|
"Compactions can only be done via CompactFiles\n",
|
||||||
|
@ -878,7 +878,7 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
|
||||||
return static_cast<int>(res);
|
return static_cast<int>(res);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause>
|
std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause>
|
||||||
ColumnFamilyData::GetWriteStallConditionAndCause(
|
ColumnFamilyData::GetWriteStallConditionAndCause(
|
||||||
|
@ -1012,7 +1012,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||||
mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
|
mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
|
||||||
(compaction_needed_bytes -
|
(compaction_needed_bytes -
|
||||||
mutable_cf_options.soft_pending_compaction_bytes_limit) >
|
mutable_cf_options.soft_pending_compaction_bytes_limit) >
|
||||||
3 * (mutable_cf_options.hard_pending_compaction_bytes_limit -
|
3 *
|
||||||
|
(mutable_cf_options.hard_pending_compaction_bytes_limit -
|
||||||
mutable_cf_options.soft_pending_compaction_bytes_limit) /
|
mutable_cf_options.soft_pending_compaction_bytes_limit) /
|
||||||
4;
|
4;
|
||||||
|
|
||||||
|
@ -1305,8 +1306,8 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ColumnFamilyData::InstallSuperVersion(
|
void ColumnFamilyData::InstallSuperVersion(SuperVersionContext* sv_context,
|
||||||
SuperVersionContext* sv_context, InstrumentedMutex* db_mutex) {
|
InstrumentedMutex* db_mutex) {
|
||||||
db_mutex->AssertHeld();
|
db_mutex->AssertHeld();
|
||||||
return InstallSuperVersion(sv_context, mutable_cf_options_);
|
return InstallSuperVersion(sv_context, mutable_cf_options_);
|
||||||
}
|
}
|
||||||
|
@ -1483,8 +1484,8 @@ Env::WriteLifeTimeHint ColumnFamilyData::CalculateSSTWriteHint(int level) {
|
||||||
// than base_level.
|
// than base_level.
|
||||||
return Env::WLTH_MEDIUM;
|
return Env::WLTH_MEDIUM;
|
||||||
}
|
}
|
||||||
return static_cast<Env::WriteLifeTimeHint>(level - base_level +
|
return static_cast<Env::WriteLifeTimeHint>(
|
||||||
static_cast<int>(Env::WLTH_MEDIUM));
|
level - base_level + static_cast<int>(Env::WLTH_MEDIUM));
|
||||||
}
|
}
|
||||||
|
|
||||||
Status ColumnFamilyData::AddDirectories(
|
Status ColumnFamilyData::AddDirectories(
|
||||||
|
@ -1580,8 +1581,8 @@ ColumnFamilyData* ColumnFamilySet::GetColumnFamily(uint32_t id) const {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ColumnFamilyData* ColumnFamilySet::GetColumnFamily(const std::string& name)
|
ColumnFamilyData* ColumnFamilySet::GetColumnFamily(
|
||||||
const {
|
const std::string& name) const {
|
||||||
auto cfd_iter = column_families_.find(name);
|
auto cfd_iter = column_families_.find(name);
|
||||||
if (cfd_iter != column_families_.end()) {
|
if (cfd_iter != column_families_.end()) {
|
||||||
auto cfd = GetColumnFamily(cfd_iter->second);
|
auto cfd = GetColumnFamily(cfd_iter->second);
|
||||||
|
|
|
@ -163,8 +163,8 @@ extern const double kIncSlowdownRatio;
|
||||||
class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
|
class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
|
||||||
public:
|
public:
|
||||||
// create while holding the mutex
|
// create while holding the mutex
|
||||||
ColumnFamilyHandleImpl(
|
ColumnFamilyHandleImpl(ColumnFamilyData* cfd, DBImpl* db,
|
||||||
ColumnFamilyData* cfd, DBImpl* db, InstrumentedMutex* mutex);
|
InstrumentedMutex* mutex);
|
||||||
// destroy without mutex
|
// destroy without mutex
|
||||||
virtual ~ColumnFamilyHandleImpl();
|
virtual ~ColumnFamilyHandleImpl();
|
||||||
virtual ColumnFamilyData* cfd() const { return cfd_; }
|
virtual ColumnFamilyData* cfd() const { return cfd_; }
|
||||||
|
@ -189,7 +189,8 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
|
||||||
class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
|
class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
|
||||||
public:
|
public:
|
||||||
ColumnFamilyHandleInternal()
|
ColumnFamilyHandleInternal()
|
||||||
: ColumnFamilyHandleImpl(nullptr, nullptr, nullptr), internal_cfd_(nullptr) {}
|
: ColumnFamilyHandleImpl(nullptr, nullptr, nullptr),
|
||||||
|
internal_cfd_(nullptr) {}
|
||||||
|
|
||||||
void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
|
void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
|
||||||
virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
|
virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
|
||||||
|
@ -656,8 +657,7 @@ class ColumnFamilySet {
|
||||||
// ColumnFamilySet supports iteration
|
// ColumnFamilySet supports iteration
|
||||||
class iterator {
|
class iterator {
|
||||||
public:
|
public:
|
||||||
explicit iterator(ColumnFamilyData* cfd)
|
explicit iterator(ColumnFamilyData* cfd) : current_(cfd) {}
|
||||||
: current_(cfd) {}
|
|
||||||
// NOTE: minimum operators for for-loop iteration
|
// NOTE: minimum operators for for-loop iteration
|
||||||
iterator& operator++() {
|
iterator& operator++() {
|
||||||
current_ = current_->next_;
|
current_ = current_->next_;
|
||||||
|
|
|
@ -39,9 +39,7 @@ class EnvCounter : public SpecialEnv {
|
||||||
public:
|
public:
|
||||||
explicit EnvCounter(Env* base)
|
explicit EnvCounter(Env* base)
|
||||||
: SpecialEnv(base), num_new_writable_file_(0) {}
|
: SpecialEnv(base), num_new_writable_file_(0) {}
|
||||||
int GetNumberOfNewWritableFileCalls() {
|
int GetNumberOfNewWritableFileCalls() { return num_new_writable_file_; }
|
||||||
return num_new_writable_file_;
|
|
||||||
}
|
|
||||||
Status NewWritableFile(const std::string& f, std::unique_ptr<WritableFile>* r,
|
Status NewWritableFile(const std::string& f, std::unique_ptr<WritableFile>* r,
|
||||||
const EnvOptions& soptions) override {
|
const EnvOptions& soptions) override {
|
||||||
++num_new_writable_file_;
|
++num_new_writable_file_;
|
||||||
|
@ -206,15 +204,12 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
}
|
}
|
||||||
#endif // !ROCKSDB_LITE
|
#endif // !ROCKSDB_LITE
|
||||||
|
|
||||||
|
|
||||||
void Open(std::vector<std::string> cf,
|
void Open(std::vector<std::string> cf,
|
||||||
std::vector<ColumnFamilyOptions> options = {}) {
|
std::vector<ColumnFamilyOptions> options = {}) {
|
||||||
ASSERT_OK(TryOpen(cf, options));
|
ASSERT_OK(TryOpen(cf, options));
|
||||||
}
|
}
|
||||||
|
|
||||||
void Open() {
|
void Open() { Open({"default"}); }
|
||||||
Open({"default"});
|
|
||||||
}
|
|
||||||
|
|
||||||
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
|
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
|
||||||
|
|
||||||
|
@ -335,9 +330,7 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf]));
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf]));
|
||||||
}
|
}
|
||||||
|
|
||||||
void WaitForCompaction() {
|
void WaitForCompaction() { ASSERT_OK(dbfull()->TEST_WaitForCompact()); }
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t MaxTotalInMemoryState() {
|
uint64_t MaxTotalInMemoryState() {
|
||||||
return dbfull()->TEST_MaxTotalInMemoryState();
|
return dbfull()->TEST_MaxTotalInMemoryState();
|
||||||
|
@ -354,9 +347,7 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
Status Merge(int cf, const std::string& key, const std::string& value) {
|
Status Merge(int cf, const std::string& key, const std::string& value) {
|
||||||
return db_->Merge(WriteOptions(), handles_[cf], Slice(key), Slice(value));
|
return db_->Merge(WriteOptions(), handles_[cf], Slice(key), Slice(value));
|
||||||
}
|
}
|
||||||
Status Flush(int cf) {
|
Status Flush(int cf) { return db_->Flush(FlushOptions(), handles_[cf]); }
|
||||||
return db_->Flush(FlushOptions(), handles_[cf]);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string Get(int cf, const std::string& key) {
|
std::string Get(int cf, const std::string& key) {
|
||||||
ReadOptions options;
|
ReadOptions options;
|
||||||
|
@ -521,8 +512,8 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
return static_cast<int>(files.size());
|
return static_cast<int>(files.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
void RecalculateWriteStallConditions(ColumnFamilyData* cfd,
|
void RecalculateWriteStallConditions(
|
||||||
const MutableCFOptions& mutable_cf_options) {
|
ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options) {
|
||||||
// add lock to avoid race condition between
|
// add lock to avoid race condition between
|
||||||
// `RecalculateWriteStallConditions` which writes to CFStats and
|
// `RecalculateWriteStallConditions` which writes to CFStats and
|
||||||
// background `DBImpl::DumpStats()` threads which read CFStats
|
// background `DBImpl::DumpStats()` threads which read CFStats
|
||||||
|
@ -970,8 +961,7 @@ TEST_P(ColumnFamilyTest, FlushTest) {
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < 3; ++i) {
|
for (int i = 0; i < 3; ++i) {
|
||||||
uint64_t max_total_in_memory_state =
|
uint64_t max_total_in_memory_state = MaxTotalInMemoryState();
|
||||||
MaxTotalInMemoryState();
|
|
||||||
ASSERT_OK(Flush(i));
|
ASSERT_OK(Flush(i));
|
||||||
AssertMaxTotalInMemoryState(max_total_in_memory_state);
|
AssertMaxTotalInMemoryState(max_total_in_memory_state);
|
||||||
}
|
}
|
||||||
|
@ -2123,7 +2113,6 @@ TEST_P(ColumnFamilyTest, ReadOnlyDBTest) {
|
||||||
ASSERT_EQ("bla", Get(1, "foo"));
|
ASSERT_EQ("bla", Get(1, "foo"));
|
||||||
ASSERT_EQ("blablablabla", Get(2, "foo"));
|
ASSERT_EQ("blablablabla", Get(2, "foo"));
|
||||||
|
|
||||||
|
|
||||||
// test newiterators
|
// test newiterators
|
||||||
{
|
{
|
||||||
std::vector<Iterator*> iterators;
|
std::vector<Iterator*> iterators;
|
||||||
|
@ -2488,7 +2477,7 @@ void DropSingleColumnFamily(ColumnFamilyTest* cf_test, int cf_id,
|
||||||
}
|
}
|
||||||
test_stage = kChildThreadFinishDroppingColumnFamily;
|
test_stage = kChildThreadFinishDroppingColumnFamily;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(ColumnFamilyTest, CreateAndDropRace) {
|
TEST_P(ColumnFamilyTest, CreateAndDropRace) {
|
||||||
const int kCfCount = 5;
|
const int kCfCount = 5;
|
||||||
|
|
|
@ -348,9 +348,7 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SetDB(DB* db) {
|
void SetDB(DB* db) { db_ = db; }
|
||||||
db_ = db;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char* Name() const override { return "FilterWithGet"; }
|
const char* Name() const override { return "FilterWithGet"; }
|
||||||
|
|
||||||
|
@ -358,7 +356,6 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
|
||||||
DB* db_;
|
DB* db_;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
std::shared_ptr<FilterWithGet> cf(new FilterWithGet());
|
std::shared_ptr<FilterWithGet> cf(new FilterWithGet());
|
||||||
|
|
||||||
Options options;
|
Options options;
|
||||||
|
@ -385,7 +382,6 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
|
||||||
db->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), {fname}, 0));
|
db->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), {fname}, 0));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
delete db;
|
delete db;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -400,8 +396,7 @@ TEST_F(CompactFilesTest, SentinelCompressionType) {
|
||||||
}
|
}
|
||||||
// Check that passing `CompressionType::kDisableCompressionOption` to
|
// Check that passing `CompressionType::kDisableCompressionOption` to
|
||||||
// `CompactFiles` causes it to use the column family compression options.
|
// `CompactFiles` causes it to use the column family compression options.
|
||||||
for (auto compaction_style :
|
for (auto compaction_style : {CompactionStyle::kCompactionStyleLevel,
|
||||||
{CompactionStyle::kCompactionStyleLevel,
|
|
||||||
CompactionStyle::kCompactionStyleUniversal,
|
CompactionStyle::kCompactionStyleUniversal,
|
||||||
CompactionStyle::kCompactionStyleNone}) {
|
CompactionStyle::kCompactionStyleNone}) {
|
||||||
ASSERT_OK(DestroyDB(db_name_, Options()));
|
ASSERT_OK(DestroyDB(db_name_, Options()));
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
#include "util/string_util.h"
|
#include "util/string_util.h"
|
||||||
#include "utilities/merge_operators.h"
|
#include "utilities/merge_operators.h"
|
||||||
|
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
|
@ -249,7 +248,7 @@ class TwoStrComparator : public Comparator {
|
||||||
|
|
||||||
void FindShortSuccessor(std::string* /*key*/) const override {}
|
void FindShortSuccessor(std::string* /*key*/) const override {}
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class ComparatorDBTest
|
class ComparatorDBTest
|
||||||
: public testing::Test,
|
: public testing::Test,
|
||||||
|
@ -470,7 +469,7 @@ void VerifySuccessor(const Slice& s, const Slice& t) {
|
||||||
ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(t, s));
|
ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(t, s));
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(ComparatorDBTest, IsSameLengthImmediateSuccessor) {
|
TEST_P(ComparatorDBTest, IsSameLengthImmediateSuccessor) {
|
||||||
{
|
{
|
||||||
|
|
|
@ -26,8 +26,7 @@ Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
|
||||||
}
|
}
|
||||||
|
|
||||||
Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family,
|
Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family,
|
||||||
const RangePtr* ranges, size_t n,
|
const RangePtr* ranges, size_t n, bool include_end) {
|
||||||
bool include_end) {
|
|
||||||
return (static_cast_with_check<DBImpl>(db->GetRootDB()))
|
return (static_cast_with_check<DBImpl>(db->GetRootDB()))
|
||||||
->DeleteFilesInRanges(column_family, ranges, n, include_end);
|
->DeleteFilesInRanges(column_family, ranges, n, include_end);
|
||||||
}
|
}
|
||||||
|
@ -47,9 +46,8 @@ Status VerifySstFileChecksum(const Options& options,
|
||||||
InternalKeyComparator internal_comparator(options.comparator);
|
InternalKeyComparator internal_comparator(options.comparator);
|
||||||
ImmutableOptions ioptions(options);
|
ImmutableOptions ioptions(options);
|
||||||
|
|
||||||
Status s = ioptions.fs->NewRandomAccessFile(file_path,
|
Status s = ioptions.fs->NewRandomAccessFile(
|
||||||
FileOptions(env_options),
|
file_path, FileOptions(env_options), &file, nullptr);
|
||||||
&file, nullptr);
|
|
||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
s = ioptions.fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr);
|
s = ioptions.fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr);
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -65,7 +65,7 @@ class ErrorEnv : public EnvWrapper {
|
||||||
return target()->NewWritableFile(fname, result, soptions);
|
return target()->NewWritableFile(fname, result, soptions);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
class CorruptionTest : public testing::Test {
|
class CorruptionTest : public testing::Test {
|
||||||
public:
|
public:
|
||||||
std::shared_ptr<Env> env_guard_;
|
std::shared_ptr<Env> env_guard_;
|
||||||
|
@ -138,9 +138,7 @@ class CorruptionTest : public testing::Test {
|
||||||
return DB::Open(opt, dbname_, &db_);
|
return DB::Open(opt, dbname_, &db_);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Reopen(Options* options = nullptr) {
|
void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); }
|
||||||
ASSERT_OK(TryReopen(options));
|
|
||||||
}
|
|
||||||
|
|
||||||
void RepairDB() {
|
void RepairDB() {
|
||||||
delete db_;
|
delete db_;
|
||||||
|
@ -183,8 +181,7 @@ class CorruptionTest : public testing::Test {
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
uint64_t key;
|
uint64_t key;
|
||||||
Slice in(iter->key());
|
Slice in(iter->key());
|
||||||
if (!ConsumeDecimalNumber(&in, &key) ||
|
if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
|
||||||
!in.empty() ||
|
|
||||||
key < next_expected) {
|
key < next_expected) {
|
||||||
bad_keys++;
|
bad_keys++;
|
||||||
continue;
|
continue;
|
||||||
|
@ -200,7 +197,8 @@ class CorruptionTest : public testing::Test {
|
||||||
iter->status().PermitUncheckedError();
|
iter->status().PermitUncheckedError();
|
||||||
delete iter;
|
delete iter;
|
||||||
|
|
||||||
fprintf(stderr,
|
fprintf(
|
||||||
|
stderr,
|
||||||
"expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%llu\n",
|
"expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%llu\n",
|
||||||
min_expected, max_expected, correct, bad_keys, bad_values,
|
min_expected, max_expected, correct, bad_keys, bad_values,
|
||||||
static_cast<unsigned long long>(missed));
|
static_cast<unsigned long long>(missed));
|
||||||
|
@ -217,8 +215,7 @@ class CorruptionTest : public testing::Test {
|
||||||
std::string fname;
|
std::string fname;
|
||||||
int picked_number = -1;
|
int picked_number = -1;
|
||||||
for (size_t i = 0; i < filenames.size(); i++) {
|
for (size_t i = 0; i < filenames.size(); i++) {
|
||||||
if (ParseFileName(filenames[i], &number, &type) &&
|
if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
|
||||||
type == filetype &&
|
|
||||||
static_cast<int>(number) > picked_number) { // Pick latest file
|
static_cast<int>(number) > picked_number) { // Pick latest file
|
||||||
fname = dbname_ + "/" + filenames[i];
|
fname = dbname_ + "/" + filenames[i];
|
||||||
picked_number = static_cast<int>(number);
|
picked_number = static_cast<int>(number);
|
||||||
|
@ -244,7 +241,6 @@ class CorruptionTest : public testing::Test {
|
||||||
FAIL() << "no file found at level";
|
FAIL() << "no file found at level";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
int Property(const std::string& name) {
|
int Property(const std::string& name) {
|
||||||
std::string property;
|
std::string property;
|
||||||
int result;
|
int result;
|
||||||
|
|
|
@ -77,9 +77,7 @@ class CuckooTableDBTest : public testing::Test {
|
||||||
return db_->Put(WriteOptions(), k, v);
|
return db_->Put(WriteOptions(), k, v);
|
||||||
}
|
}
|
||||||
|
|
||||||
Status Delete(const std::string& k) {
|
Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); }
|
||||||
return db_->Delete(WriteOptions(), k);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string Get(const std::string& k) {
|
std::string Get(const std::string& k) {
|
||||||
ReadOptions options;
|
ReadOptions options;
|
||||||
|
@ -313,23 +311,21 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) {
|
||||||
// Write some keys using plain table.
|
// Write some keys using plain table.
|
||||||
std::shared_ptr<TableFactory> block_based_factory(
|
std::shared_ptr<TableFactory> block_based_factory(
|
||||||
NewBlockBasedTableFactory());
|
NewBlockBasedTableFactory());
|
||||||
std::shared_ptr<TableFactory> plain_table_factory(
|
std::shared_ptr<TableFactory> plain_table_factory(NewPlainTableFactory());
|
||||||
NewPlainTableFactory());
|
std::shared_ptr<TableFactory> cuckoo_table_factory(NewCuckooTableFactory());
|
||||||
std::shared_ptr<TableFactory> cuckoo_table_factory(
|
|
||||||
NewCuckooTableFactory());
|
|
||||||
options.create_if_missing = false;
|
options.create_if_missing = false;
|
||||||
options.table_factory.reset(NewAdaptiveTableFactory(
|
options.table_factory.reset(
|
||||||
plain_table_factory, block_based_factory, plain_table_factory,
|
NewAdaptiveTableFactory(plain_table_factory, block_based_factory,
|
||||||
cuckoo_table_factory));
|
plain_table_factory, cuckoo_table_factory));
|
||||||
Reopen(&options);
|
Reopen(&options);
|
||||||
ASSERT_OK(Put("key4", "v4"));
|
ASSERT_OK(Put("key4", "v4"));
|
||||||
ASSERT_OK(Put("key1", "v5"));
|
ASSERT_OK(Put("key1", "v5"));
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
|
|
||||||
// Write some keys using block based table.
|
// Write some keys using block based table.
|
||||||
options.table_factory.reset(NewAdaptiveTableFactory(
|
options.table_factory.reset(
|
||||||
block_based_factory, block_based_factory, plain_table_factory,
|
NewAdaptiveTableFactory(block_based_factory, block_based_factory,
|
||||||
cuckoo_table_factory));
|
plain_table_factory, cuckoo_table_factory));
|
||||||
Reopen(&options);
|
Reopen(&options);
|
||||||
ASSERT_OK(Put("key5", "v6"));
|
ASSERT_OK(Put("key5", "v6"));
|
||||||
ASSERT_OK(Put("key2", "v7"));
|
ASSERT_OK(Put("key2", "v7"));
|
||||||
|
|
|
@ -2914,7 +2914,7 @@ class TableFileListener : public EventListener {
|
||||||
InstrumentedMutex mutex_;
|
InstrumentedMutex mutex_;
|
||||||
std::unordered_map<std::string, std::vector<std::string>> cf_to_paths_;
|
std::unordered_map<std::string, std::vector<std::string>> cf_to_paths_;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBasicTest, LastSstFileNotInManifest) {
|
TEST_F(DBBasicTest, LastSstFileNotInManifest) {
|
||||||
// If the last sst file is not tracked in MANIFEST,
|
// If the last sst file is not tracked in MANIFEST,
|
||||||
|
|
|
@ -414,7 +414,7 @@ class ReadOnlyCacheWrapper : public CacheWrapper {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBlockCacheTest, TestWithSameCompressed) {
|
TEST_F(DBBlockCacheTest, TestWithSameCompressed) {
|
||||||
auto table_options = GetTableOptions();
|
auto table_options = GetTableOptions();
|
||||||
|
@ -1973,7 +1973,7 @@ struct CacheKeyDecoder {
|
||||||
DownwardInvolution(decoded_session_counter));
|
DownwardInvolution(decoded_session_counter));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(CacheKeyTest, Encodings) {
|
TEST_F(CacheKeyTest, Encodings) {
|
||||||
// This test primarily verifies this claim from cache_key.cc:
|
// This test primarily verifies this claim from cache_key.cc:
|
||||||
|
|
|
@ -43,7 +43,7 @@ const std::string kStandard128Ribbon =
|
||||||
test::Standard128RibbonFilterPolicy::kClassName();
|
test::Standard128RibbonFilterPolicy::kClassName();
|
||||||
const std::string kAutoBloom = BloomFilterPolicy::kClassName();
|
const std::string kAutoBloom = BloomFilterPolicy::kClassName();
|
||||||
const std::string kAutoRibbon = RibbonFilterPolicy::kClassName();
|
const std::string kAutoRibbon = RibbonFilterPolicy::kClassName();
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// DB tests related to bloom filter.
|
// DB tests related to bloom filter.
|
||||||
|
|
||||||
|
@ -622,7 +622,7 @@ class AlwaysTrueFilterPolicy : public ReadOnlyBuiltinFilterPolicy {
|
||||||
bool skip_;
|
bool skip_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
|
TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
|
||||||
constexpr int maxKey = 10;
|
constexpr int maxKey = 10;
|
||||||
|
@ -767,10 +767,10 @@ INSTANTIATE_TEST_CASE_P(
|
||||||
|
|
||||||
INSTANTIATE_TEST_CASE_P(
|
INSTANTIATE_TEST_CASE_P(
|
||||||
FormatLatest, DBBloomFilterTestWithParam,
|
FormatLatest, DBBloomFilterTestWithParam,
|
||||||
::testing::Values(
|
::testing::Values(std::make_tuple(kAutoBloom, true, kLatestFormatVersion),
|
||||||
std::make_tuple(kAutoBloom, true, kLatestFormatVersion),
|
|
||||||
std::make_tuple(kAutoBloom, false, kLatestFormatVersion),
|
std::make_tuple(kAutoBloom, false, kLatestFormatVersion),
|
||||||
std::make_tuple(kAutoRibbon, false, kLatestFormatVersion)));
|
std::make_tuple(kAutoRibbon, false,
|
||||||
|
kLatestFormatVersion)));
|
||||||
#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
||||||
|
|
||||||
TEST_F(DBBloomFilterTest, BloomFilterRate) {
|
TEST_F(DBBloomFilterTest, BloomFilterRate) {
|
||||||
|
@ -840,7 +840,7 @@ std::vector<CompatibilityConfig> kCompatibilityConfigs = {
|
||||||
BlockBasedTableOptions().format_version},
|
BlockBasedTableOptions().format_version},
|
||||||
{kCompatibilityRibbonPolicy, true, BlockBasedTableOptions().format_version},
|
{kCompatibilityRibbonPolicy, true, BlockBasedTableOptions().format_version},
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
|
TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
|
@ -1678,7 +1678,7 @@ class TestingContextCustomFilterPolicy
|
||||||
private:
|
private:
|
||||||
mutable std::string test_report_;
|
mutable std::string test_report_;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
|
TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
|
||||||
auto policy = std::make_shared<TestingContextCustomFilterPolicy>(15, 8, 5);
|
auto policy = std::make_shared<TestingContextCustomFilterPolicy>(15, 8, 5);
|
||||||
|
@ -2186,16 +2186,14 @@ INSTANTIATE_TEST_CASE_P(DBBloomFilterTestVaryPrefixAndFormatVer,
|
||||||
std::make_tuple(false, 2),
|
std::make_tuple(false, 2),
|
||||||
std::make_tuple(false, 3),
|
std::make_tuple(false, 3),
|
||||||
std::make_tuple(false, 4),
|
std::make_tuple(false, 4),
|
||||||
std::make_tuple(false, 5),
|
std::make_tuple(false, 5), std::make_tuple(true, 2),
|
||||||
std::make_tuple(true, 2),
|
std::make_tuple(true, 3), std::make_tuple(true, 4),
|
||||||
std::make_tuple(true, 3),
|
|
||||||
std::make_tuple(true, 4),
|
|
||||||
std::make_tuple(true, 5)));
|
std::make_tuple(true, 5)));
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
namespace {
|
namespace {
|
||||||
static const std::string kPlainTable = "test_PlainTableBloom";
|
static const std::string kPlainTable = "test_PlainTableBloom";
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class BloomStatsTestWithParam
|
class BloomStatsTestWithParam
|
||||||
: public DBBloomFilterTest,
|
: public DBBloomFilterTest,
|
||||||
|
@ -2408,7 +2406,7 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
|
||||||
dbtest->Flush();
|
dbtest->Flush();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBloomFilterTest, PrefixScan) {
|
TEST_F(DBBloomFilterTest, PrefixScan) {
|
||||||
while (ChangeFilterOptions()) {
|
while (ChangeFilterOptions()) {
|
||||||
|
@ -3169,7 +3167,7 @@ std::pair<uint64_t, uint64_t> CheckedAndUseful(uint64_t checked,
|
||||||
uint64_t useful) {
|
uint64_t useful) {
|
||||||
return {checked, useful};
|
return {checked, useful};
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// This uses a prefix_extractor + comparator combination that violates
|
// This uses a prefix_extractor + comparator combination that violates
|
||||||
// one of the old obsolete, unnecessary axioms of prefix extraction:
|
// one of the old obsolete, unnecessary axioms of prefix extraction:
|
||||||
|
@ -3377,7 +3375,7 @@ class NonIdempotentFixed4Transform : public SliceTransform {
|
||||||
|
|
||||||
bool InDomain(const Slice& src) const override { return src.size() >= 5; }
|
bool InDomain(const Slice& src) const override { return src.size() >= 5; }
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// This uses a prefix_extractor + comparator combination that violates
|
// This uses a prefix_extractor + comparator combination that violates
|
||||||
// two of the old obsolete, unnecessary axioms of prefix extraction:
|
// two of the old obsolete, unnecessary axioms of prefix extraction:
|
||||||
|
|
|
@ -255,9 +255,8 @@ Options DeletionTriggerOptions(Options options) {
|
||||||
return options;
|
return options;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool HaveOverlappingKeyRanges(
|
bool HaveOverlappingKeyRanges(const Comparator* c, const SstFileMetaData& a,
|
||||||
const Comparator* c,
|
const SstFileMetaData& b) {
|
||||||
const SstFileMetaData& a, const SstFileMetaData& b) {
|
|
||||||
if (c->CompareWithoutTimestamp(a.smallestkey, b.smallestkey) >= 0) {
|
if (c->CompareWithoutTimestamp(a.smallestkey, b.smallestkey) >= 0) {
|
||||||
if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) {
|
if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) {
|
||||||
// b.smallestkey <= a.smallestkey <= b.largestkey
|
// b.smallestkey <= a.smallestkey <= b.largestkey
|
||||||
|
@ -282,18 +281,15 @@ bool HaveOverlappingKeyRanges(
|
||||||
// Identifies all files between level "min_level" and "max_level"
|
// Identifies all files between level "min_level" and "max_level"
|
||||||
// which has overlapping key range with "input_file_meta".
|
// which has overlapping key range with "input_file_meta".
|
||||||
void GetOverlappingFileNumbersForLevelCompaction(
|
void GetOverlappingFileNumbersForLevelCompaction(
|
||||||
const ColumnFamilyMetaData& cf_meta,
|
const ColumnFamilyMetaData& cf_meta, const Comparator* comparator,
|
||||||
const Comparator* comparator,
|
int min_level, int max_level, const SstFileMetaData* input_file_meta,
|
||||||
int min_level, int max_level,
|
|
||||||
const SstFileMetaData* input_file_meta,
|
|
||||||
std::set<std::string>* overlapping_file_names) {
|
std::set<std::string>* overlapping_file_names) {
|
||||||
std::set<const SstFileMetaData*> overlapping_files;
|
std::set<const SstFileMetaData*> overlapping_files;
|
||||||
overlapping_files.insert(input_file_meta);
|
overlapping_files.insert(input_file_meta);
|
||||||
for (int m = min_level; m <= max_level; ++m) {
|
for (int m = min_level; m <= max_level; ++m) {
|
||||||
for (auto& file : cf_meta.levels[m].files) {
|
for (auto& file : cf_meta.levels[m].files) {
|
||||||
for (auto* included_file : overlapping_files) {
|
for (auto* included_file : overlapping_files) {
|
||||||
if (HaveOverlappingKeyRanges(
|
if (HaveOverlappingKeyRanges(comparator, *included_file, file)) {
|
||||||
comparator, *included_file, file)) {
|
|
||||||
overlapping_files.insert(&file);
|
overlapping_files.insert(&file);
|
||||||
overlapping_file_names->insert(file.name);
|
overlapping_file_names->insert(file.name);
|
||||||
break;
|
break;
|
||||||
|
@ -316,12 +312,9 @@ void VerifyCompactionResult(
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
const SstFileMetaData* PickFileRandomly(
|
const SstFileMetaData* PickFileRandomly(const ColumnFamilyMetaData& cf_meta,
|
||||||
const ColumnFamilyMetaData& cf_meta,
|
Random* rand, int* level = nullptr) {
|
||||||
Random* rand,
|
auto file_id = rand->Uniform(static_cast<int>(cf_meta.file_count)) + 1;
|
||||||
int* level = nullptr) {
|
|
||||||
auto file_id = rand->Uniform(static_cast<int>(
|
|
||||||
cf_meta.file_count)) + 1;
|
|
||||||
for (auto& level_meta : cf_meta.levels) {
|
for (auto& level_meta : cf_meta.levels) {
|
||||||
if (file_id <= level_meta.files.size()) {
|
if (file_id <= level_meta.files.size()) {
|
||||||
if (level != nullptr) {
|
if (level != nullptr) {
|
||||||
|
@ -747,7 +740,6 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
|
TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
|
||||||
const int kNumKeysPerFile = 100;
|
const int kNumKeysPerFile = 100;
|
||||||
|
|
||||||
|
@ -1254,14 +1246,8 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveNonOverlappingFiles) {
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
// non overlapping ranges
|
// non overlapping ranges
|
||||||
std::vector<std::pair<int32_t, int32_t>> ranges = {
|
std::vector<std::pair<int32_t, int32_t>> ranges = {
|
||||||
{100, 199},
|
{100, 199}, {300, 399}, {0, 99}, {200, 299},
|
||||||
{300, 399},
|
{600, 699}, {400, 499}, {500, 550}, {551, 599},
|
||||||
{0, 99},
|
|
||||||
{200, 299},
|
|
||||||
{600, 699},
|
|
||||||
{400, 499},
|
|
||||||
{500, 550},
|
|
||||||
{551, 599},
|
|
||||||
};
|
};
|
||||||
int32_t value_size = 10 * 1024; // 10 KB
|
int32_t value_size = 10 * 1024; // 10 KB
|
||||||
|
|
||||||
|
@ -1310,7 +1296,8 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveNonOverlappingFiles) {
|
||||||
{200, 299},
|
{200, 299},
|
||||||
{600, 699},
|
{600, 699},
|
||||||
{400, 499},
|
{400, 499},
|
||||||
{500, 560}, // this range overlap with the next one
|
{500, 560}, // this range overlap with the next
|
||||||
|
// one
|
||||||
{551, 599},
|
{551, 599},
|
||||||
};
|
};
|
||||||
for (size_t i = 0; i < ranges.size(); i++) {
|
for (size_t i = 0; i < ranges.size(); i++) {
|
||||||
|
@ -2736,7 +2723,6 @@ TEST_P(DBCompactionTestWithParam, ManualCompaction) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) {
|
TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760);
|
options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760);
|
||||||
|
@ -2873,12 +2859,11 @@ TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) {
|
||||||
auto file_meta = PickFileRandomly(cf_meta, &rnd, &level);
|
auto file_meta = PickFileRandomly(cf_meta, &rnd, &level);
|
||||||
compaction_input_file_names.push_back(file_meta->name);
|
compaction_input_file_names.push_back(file_meta->name);
|
||||||
GetOverlappingFileNumbersForLevelCompaction(
|
GetOverlappingFileNumbersForLevelCompaction(
|
||||||
cf_meta, options.comparator, level, output_level,
|
cf_meta, options.comparator, level, output_level, file_meta,
|
||||||
file_meta, &overlapping_file_names);
|
&overlapping_file_names);
|
||||||
}
|
}
|
||||||
|
|
||||||
ASSERT_OK(dbfull()->CompactFiles(
|
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
|
||||||
CompactionOptions(), handles_[1],
|
|
||||||
compaction_input_file_names,
|
compaction_input_file_names,
|
||||||
output_level));
|
output_level));
|
||||||
|
|
||||||
|
@ -2903,8 +2888,7 @@ TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) {
|
||||||
options.write_buffer_size = kKeysPerBuffer * kKvSize;
|
options.write_buffer_size = kKeysPerBuffer * kKvSize;
|
||||||
options.max_write_buffer_number = 2;
|
options.max_write_buffer_number = 2;
|
||||||
options.target_file_size_base =
|
options.target_file_size_base =
|
||||||
options.write_buffer_size *
|
options.write_buffer_size * (options.max_write_buffer_number - 1);
|
||||||
(options.max_write_buffer_number - 1);
|
|
||||||
options.level0_file_num_compaction_trigger = kNumL1Files;
|
options.level0_file_num_compaction_trigger = kNumL1Files;
|
||||||
options.max_bytes_for_level_base =
|
options.max_bytes_for_level_base =
|
||||||
options.level0_file_num_compaction_trigger *
|
options.level0_file_num_compaction_trigger *
|
||||||
|
@ -2924,8 +2908,7 @@ TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) {
|
||||||
|
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
|
||||||
const int kNumInsertedKeys =
|
const int kNumInsertedKeys = options.level0_file_num_compaction_trigger *
|
||||||
options.level0_file_num_compaction_trigger *
|
|
||||||
(options.max_write_buffer_number - 1) *
|
(options.max_write_buffer_number - 1) *
|
||||||
kKeysPerBuffer;
|
kKeysPerBuffer;
|
||||||
|
|
||||||
|
@ -3625,8 +3608,7 @@ TEST_F(DBCompactionTest, CompactFilesPendingL0Bug) {
|
||||||
ASSERT_EQ(kNumL0Files, cf_meta.levels[0].files.size());
|
ASSERT_EQ(kNumL0Files, cf_meta.levels[0].files.size());
|
||||||
std::vector<std::string> input_filenames;
|
std::vector<std::string> input_filenames;
|
||||||
input_filenames.push_back(cf_meta.levels[0].files.front().name);
|
input_filenames.push_back(cf_meta.levels[0].files.front().name);
|
||||||
ASSERT_OK(dbfull()
|
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), input_filenames,
|
||||||
->CompactFiles(CompactionOptions(), input_filenames,
|
|
||||||
0 /* output_level */));
|
0 /* output_level */));
|
||||||
TEST_SYNC_POINT("DBCompactionTest::CompactFilesPendingL0Bug:ManualCompacted");
|
TEST_SYNC_POINT("DBCompactionTest::CompactFilesPendingL0Bug:ManualCompacted");
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
||||||
|
@ -4953,7 +4935,8 @@ TEST_F(DBCompactionTest, CompactRangeSkipFlushAfterDelay) {
|
||||||
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
||||||
ASSERT_OK(dbfull()->Flush(flush_opts));
|
ASSERT_OK(dbfull()->Flush(flush_opts));
|
||||||
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
||||||
TEST_SYNC_POINT("DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush");
|
TEST_SYNC_POINT(
|
||||||
|
"DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush");
|
||||||
manual_compaction_thread.join();
|
manual_compaction_thread.join();
|
||||||
|
|
||||||
// If CompactRange's flush was skipped, the final Put above will still be
|
// If CompactRange's flush was skipped, the final Put above will still be
|
||||||
|
@ -5248,8 +5231,8 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
|
||||||
std::shared_ptr<ConcurrentTaskLimiter> unique_limiter(
|
std::shared_ptr<ConcurrentTaskLimiter> unique_limiter(
|
||||||
NewConcurrentTaskLimiter("unique_limiter", -1));
|
NewConcurrentTaskLimiter("unique_limiter", -1));
|
||||||
|
|
||||||
const char* cf_names[] = {"default", "0", "1", "2", "3", "4", "5",
|
const char* cf_names[] = {"default", "0", "1", "2", "3", "4", "5", "6", "7",
|
||||||
"6", "7", "8", "9", "a", "b", "c", "d", "e", "f" };
|
"8", "9", "a", "b", "c", "d", "e", "f"};
|
||||||
const unsigned int cf_count = sizeof cf_names / sizeof cf_names[0];
|
const unsigned int cf_count = sizeof cf_names / sizeof cf_names[0];
|
||||||
|
|
||||||
std::unordered_map<std::string, CompactionLimiter*> cf_to_limiter;
|
std::unordered_map<std::string, CompactionLimiter*> cf_to_limiter;
|
||||||
|
@ -5292,9 +5275,8 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
|
||||||
CreateColumnFamilies({cf_names[cf]}, option_vector[cf]);
|
CreateColumnFamilies({cf_names[cf]}, option_vector[cf]);
|
||||||
}
|
}
|
||||||
|
|
||||||
ReopenWithColumnFamilies(std::vector<std::string>(cf_names,
|
ReopenWithColumnFamilies(
|
||||||
cf_names + cf_count),
|
std::vector<std::string>(cf_names, cf_names + cf_count), option_vector);
|
||||||
option_vector);
|
|
||||||
|
|
||||||
port::Mutex mutex;
|
port::Mutex mutex;
|
||||||
|
|
||||||
|
@ -5434,9 +5416,7 @@ TEST_P(DBCompactionDirectIOTest, DirectIO) {
|
||||||
});
|
});
|
||||||
if (options.use_direct_io_for_flush_and_compaction) {
|
if (options.use_direct_io_for_flush_and_compaction) {
|
||||||
SyncPoint::GetInstance()->SetCallBack(
|
SyncPoint::GetInstance()->SetCallBack(
|
||||||
"SanitizeOptions:direct_io", [&](void* /*arg*/) {
|
"SanitizeOptions:direct_io", [&](void* /*arg*/) { readahead = true; });
|
||||||
readahead = true;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
SyncPoint::GetInstance()->EnableProcessing();
|
SyncPoint::GetInstance()->EnableProcessing();
|
||||||
CreateAndReopenWithCF({"pikachu"}, options);
|
CreateAndReopenWithCF({"pikachu"}, options);
|
||||||
|
|
|
@ -65,8 +65,7 @@ Status DBImpl::FlushForGetLiveFiles() {
|
||||||
}
|
}
|
||||||
|
|
||||||
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
||||||
uint64_t* manifest_file_size,
|
uint64_t* manifest_file_size, bool flush_memtable) {
|
||||||
bool flush_memtable) {
|
|
||||||
*manifest_file_size = 0;
|
*manifest_file_size = 0;
|
||||||
|
|
||||||
mutex_.Lock();
|
mutex_.Lock();
|
||||||
|
|
|
@ -1822,8 +1822,8 @@ TEST_F(DBFlushTest, ManualFlushFailsInReadOnlyMode) {
|
||||||
ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable());
|
ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable());
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
uint64_t num_bg_errors;
|
uint64_t num_bg_errors;
|
||||||
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBackgroundErrors,
|
ASSERT_TRUE(
|
||||||
&num_bg_errors));
|
db_->GetIntProperty(DB::Properties::kBackgroundErrors, &num_bg_errors));
|
||||||
ASSERT_GT(num_bg_errors, 0);
|
ASSERT_GT(num_bg_errors, 0);
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
#include "db/db_info_dumper.h"
|
#include "db/db_info_dumper.h"
|
||||||
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <cinttypes>
|
#include <cinttypes>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
|
@ -392,7 +392,7 @@ struct ReferenceIterator {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// Use an internal iterator that sometimes returns errors and sometimes
|
// Use an internal iterator that sometimes returns errors and sometimes
|
||||||
// adds/removes entries on the fly. Do random operations on a DBIter and
|
// adds/removes entries on the fly. Do random operations on a DBIter and
|
||||||
|
@ -482,12 +482,11 @@ TEST_F(DBIteratorStressTest, StressTest) {
|
||||||
std::cout << "entries:";
|
std::cout << "entries:";
|
||||||
for (size_t i = 0; i < data.entries.size(); ++i) {
|
for (size_t i = 0; i < data.entries.size(); ++i) {
|
||||||
Entry& e = data.entries[i];
|
Entry& e = data.entries[i];
|
||||||
std::cout
|
std::cout << "\n idx " << i << ": \"" << e.key << "\": \""
|
||||||
<< "\n idx " << i << ": \"" << e.key << "\": \""
|
|
||||||
<< e.value << "\" seq: " << e.sequence << " type: "
|
<< e.value << "\" seq: " << e.sequence << " type: "
|
||||||
<< (e.type == kTypeValue
|
<< (e.type == kTypeValue ? "val"
|
||||||
? "val"
|
: e.type == kTypeDeletion ? "del"
|
||||||
: e.type == kTypeDeletion ? "del" : "merge");
|
: "merge");
|
||||||
}
|
}
|
||||||
std::cout << std::endl;
|
std::cout << std::endl;
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,12 +3,13 @@
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
#include <string>
|
|
||||||
#include <vector>
|
|
||||||
#include <algorithm>
|
|
||||||
#include <utility>
|
|
||||||
|
|
||||||
#include "db/db_iter.h"
|
#include "db/db_iter.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
#include "db/dbformat.h"
|
#include "db/dbformat.h"
|
||||||
#include "rocksdb/comparator.h"
|
#include "rocksdb/comparator.h"
|
||||||
#include "rocksdb/options.h"
|
#include "rocksdb/options.h"
|
||||||
|
@ -429,7 +430,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
|
||||||
db_iter->SeekToLast();
|
db_iter->SeekToLast();
|
||||||
|
|
||||||
ASSERT_TRUE(db_iter->Valid());
|
ASSERT_TRUE(db_iter->Valid());
|
||||||
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_key_skipped_count), 1);
|
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_key_skipped_count),
|
||||||
|
1);
|
||||||
ASSERT_EQ(db_iter->key().ToString(), "b");
|
ASSERT_EQ(db_iter->key().ToString(), "b");
|
||||||
|
|
||||||
SetPerfLevel(kDisable);
|
SetPerfLevel(kDisable);
|
||||||
|
@ -557,7 +559,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
|
||||||
db_iter->SeekToLast();
|
db_iter->SeekToLast();
|
||||||
|
|
||||||
ASSERT_TRUE(db_iter->Valid());
|
ASSERT_TRUE(db_iter->Valid());
|
||||||
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
|
ASSERT_EQ(
|
||||||
|
static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
|
||||||
ASSERT_EQ(db_iter->key().ToString(), "b");
|
ASSERT_EQ(db_iter->key().ToString(), "b");
|
||||||
|
|
||||||
SetPerfLevel(kDisable);
|
SetPerfLevel(kDisable);
|
||||||
|
@ -3013,7 +3016,6 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace8) {
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_F(DBIteratorTest, SeekPrefixTombstones) {
|
TEST_F(DBIteratorTest, SeekPrefixTombstones) {
|
||||||
ReadOptions ro;
|
ReadOptions ro;
|
||||||
Options options;
|
Options options;
|
||||||
|
|
|
@ -236,7 +236,7 @@ namespace {
|
||||||
std::string MakeLongKey(size_t length, char c) {
|
std::string MakeLongKey(size_t length, char c) {
|
||||||
return std::string(length, c);
|
return std::string(length, c);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(DBIteratorTest, IterLongKeys) {
|
TEST_P(DBIteratorTest, IterLongKeys) {
|
||||||
ASSERT_OK(Put(MakeLongKey(20, 0), "0"));
|
ASSERT_OK(Put(MakeLongKey(20, 0), "0"));
|
||||||
|
@ -1037,7 +1037,8 @@ TEST_P(DBIteratorTest, DBIteratorBoundTest) {
|
||||||
iter->Next();
|
iter->Next();
|
||||||
|
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 2);
|
ASSERT_EQ(
|
||||||
|
static_cast<int>(get_perf_context()->internal_delete_skipped_count), 2);
|
||||||
|
|
||||||
// now testing with iterate_bound
|
// now testing with iterate_bound
|
||||||
Slice prefix("c");
|
Slice prefix("c");
|
||||||
|
@ -1060,7 +1061,8 @@ TEST_P(DBIteratorTest, DBIteratorBoundTest) {
|
||||||
// even though the key is deleted
|
// even though the key is deleted
|
||||||
// hence internal_delete_skipped_count should be 0
|
// hence internal_delete_skipped_count should be 0
|
||||||
ASSERT_TRUE(!iter->Valid());
|
ASSERT_TRUE(!iter->Valid());
|
||||||
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
|
ASSERT_EQ(
|
||||||
|
static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2180,8 +2182,8 @@ TEST_P(DBIteratorTest, IteratorWithLocalStatistics) {
|
||||||
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV), (uint64_t)total_prev);
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV), (uint64_t)total_prev);
|
||||||
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV_FOUND),
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV_FOUND),
|
||||||
(uint64_t)total_prev_found);
|
(uint64_t)total_prev_found);
|
||||||
ASSERT_EQ(TestGetTickerCount(options, ITER_BYTES_READ), (uint64_t)total_bytes);
|
ASSERT_EQ(TestGetTickerCount(options, ITER_BYTES_READ),
|
||||||
|
(uint64_t)total_bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_P(DBIteratorTest, ReadAhead) {
|
TEST_P(DBIteratorTest, ReadAhead) {
|
||||||
|
@ -2310,8 +2312,8 @@ TEST_P(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
|
||||||
EXPECT_EQ(get_perf_context()->internal_merge_count, 0);
|
EXPECT_EQ(get_perf_context()->internal_merge_count, 0);
|
||||||
EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2);
|
EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2);
|
||||||
EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2);
|
EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2);
|
||||||
EXPECT_EQ(1, options.statistics->getTickerCount(
|
EXPECT_EQ(1,
|
||||||
NUMBER_OF_RESEEKS_IN_ITERATION));
|
options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_P(DBIteratorTest, Refresh) {
|
TEST_P(DBIteratorTest, Refresh) {
|
||||||
|
|
|
@ -55,14 +55,13 @@ SequenceNumber ReadRecords(std::unique_ptr<TransactionLogIterator>& iter,
|
||||||
return res.sequence;
|
return res.sequence;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExpectRecords(
|
void ExpectRecords(const int expected_no_records,
|
||||||
const int expected_no_records,
|
|
||||||
std::unique_ptr<TransactionLogIterator>& iter) {
|
std::unique_ptr<TransactionLogIterator>& iter) {
|
||||||
int num_records;
|
int num_records;
|
||||||
ReadRecords(iter, num_records);
|
ReadRecords(iter, num_records);
|
||||||
ASSERT_EQ(num_records, expected_no_records);
|
ASSERT_EQ(num_records, expected_no_records);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTestXactLogIterator, TransactionLogIterator) {
|
TEST_F(DBTestXactLogIterator, TransactionLogIterator) {
|
||||||
do {
|
do {
|
||||||
|
@ -97,8 +96,7 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
|
||||||
static const char* sync_points[LOG_ITERATOR_RACE_TEST_COUNT][4] = {
|
static const char* sync_points[LOG_ITERATOR_RACE_TEST_COUNT][4] = {
|
||||||
{"WalManager::GetSortedWalFiles:1", "WalManager::PurgeObsoleteFiles:1",
|
{"WalManager::GetSortedWalFiles:1", "WalManager::PurgeObsoleteFiles:1",
|
||||||
"WalManager::PurgeObsoleteFiles:2", "WalManager::GetSortedWalFiles:2"},
|
"WalManager::PurgeObsoleteFiles:2", "WalManager::GetSortedWalFiles:2"},
|
||||||
{"WalManager::GetSortedWalsOfType:1",
|
{"WalManager::GetSortedWalsOfType:1", "WalManager::PurgeObsoleteFiles:1",
|
||||||
"WalManager::PurgeObsoleteFiles:1",
|
|
||||||
"WalManager::PurgeObsoleteFiles:2",
|
"WalManager::PurgeObsoleteFiles:2",
|
||||||
"WalManager::GetSortedWalsOfType:2"}};
|
"WalManager::GetSortedWalsOfType:2"}};
|
||||||
for (int test = 0; test < LOG_ITERATOR_RACE_TEST_COUNT; ++test) {
|
for (int test = 0; test < LOG_ITERATOR_RACE_TEST_COUNT; ++test) {
|
||||||
|
|
|
@ -224,8 +224,8 @@ TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamilies) {
|
||||||
// Now cf_path_0_ in cache_ has been properly decreased and cf_path_0_'s entry
|
// Now cf_path_0_ in cache_ has been properly decreased and cf_path_0_'s entry
|
||||||
// is dropped from cache
|
// is dropped from cache
|
||||||
ASSERT_EQ(0, cache_->Size());
|
ASSERT_EQ(0, cache_->Size());
|
||||||
ASSERT_OK(DestroyDB(dbname_, options,
|
ASSERT_OK(
|
||||||
{{"cf1", cf_options}, {"cf2", cf_options}}));
|
DestroyDB(dbname_, options, {{"cf1", cf_options}, {"cf2", cf_options}}));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) {
|
TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) {
|
||||||
|
@ -313,8 +313,8 @@ TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) {
|
||||||
delete db;
|
delete db;
|
||||||
ASSERT_EQ(0, cache_->Size());
|
ASSERT_EQ(0, cache_->Size());
|
||||||
}
|
}
|
||||||
ASSERT_OK(DestroyDB(dbname_, options,
|
ASSERT_OK(
|
||||||
{{"cf1", cf_options}, {"cf2", cf_options}}));
|
DestroyDB(dbname_, options, {{"cf1", cf_options}, {"cf2", cf_options}}));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(DBLogicalBlockSizeCacheTest, DestroyColumnFamilyHandle) {
|
TEST_F(DBLogicalBlockSizeCacheTest, DestroyColumnFamilyHandle) {
|
||||||
|
|
|
@ -39,7 +39,7 @@ class LimitedStringAppendMergeOp : public StringAppendTESTOperator {
|
||||||
private:
|
private:
|
||||||
size_t limit_ = 0;
|
size_t limit_ = 0;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class DBMergeOperandTest : public DBTestBase {
|
class DBMergeOperandTest : public DBTestBase {
|
||||||
public:
|
public:
|
||||||
|
|
|
@ -84,8 +84,7 @@ TEST_F(DBMergeOperatorTest, LimitMergeOperands) {
|
||||||
Options options;
|
Options options;
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
// Use only the latest two merge operands.
|
// Use only the latest two merge operands.
|
||||||
options.merge_operator =
|
options.merge_operator = std::make_shared<LimitedStringAppendMergeOp>(2, ',');
|
||||||
std::make_shared<LimitedStringAppendMergeOp>(2, ',');
|
|
||||||
options.env = env_;
|
options.env = env_;
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
// All K1 values are in memtable.
|
// All K1 values are in memtable.
|
||||||
|
@ -203,7 +202,6 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) {
|
||||||
VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}});
|
VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class MergeOperatorPinningTest : public DBMergeOperatorTest,
|
class MergeOperatorPinningTest : public DBMergeOperatorTest,
|
||||||
public testing::WithParamInterface<bool> {
|
public testing::WithParamInterface<bool> {
|
||||||
public:
|
public:
|
||||||
|
|
|
@ -627,7 +627,6 @@ TEST_F(DBOptionsTest, SetBackgroundFlushThreads) {
|
||||||
ASSERT_EQ(3, dbfull()->TEST_BGFlushesAllowed());
|
ASSERT_EQ(3, dbfull()->TEST_BGFlushesAllowed());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_F(DBOptionsTest, SetBackgroundJobs) {
|
TEST_F(DBOptionsTest, SetBackgroundJobs) {
|
||||||
Options options;
|
Options options;
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
|
@ -691,7 +690,8 @@ TEST_F(DBOptionsTest, SetDelayedWriteRateOption) {
|
||||||
options.delayed_write_rate = 2 * 1024U * 1024U;
|
options.delayed_write_rate = 2 * 1024U * 1024U;
|
||||||
options.env = env_;
|
options.env = env_;
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
ASSERT_EQ(2 * 1024U * 1024U, dbfull()->TEST_write_controler().max_delayed_write_rate());
|
ASSERT_EQ(2 * 1024U * 1024U,
|
||||||
|
dbfull()->TEST_write_controler().max_delayed_write_rate());
|
||||||
|
|
||||||
ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}}));
|
ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}}));
|
||||||
ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate());
|
ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate());
|
||||||
|
|
|
@ -270,7 +270,8 @@ void GetExpectedTableProperties(
|
||||||
const int kDeletionCount = kTableCount * kDeletionsPerTable;
|
const int kDeletionCount = kTableCount * kDeletionsPerTable;
|
||||||
const int kMergeCount = kTableCount * kMergeOperandsPerTable;
|
const int kMergeCount = kTableCount * kMergeOperandsPerTable;
|
||||||
const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable;
|
const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable;
|
||||||
const int kKeyCount = kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
|
const int kKeyCount =
|
||||||
|
kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
|
||||||
const int kAvgSuccessorSize = kKeySize / 5;
|
const int kAvgSuccessorSize = kKeySize / 5;
|
||||||
const int kEncodingSavePerKey = kKeySize / 4;
|
const int kEncodingSavePerKey = kKeySize / 4;
|
||||||
expected_tp->raw_key_size = kKeyCount * (kKeySize + 8);
|
expected_tp->raw_key_size = kKeyCount * (kKeySize + 8);
|
||||||
|
@ -281,7 +282,8 @@ void GetExpectedTableProperties(
|
||||||
expected_tp->num_merge_operands = kMergeCount;
|
expected_tp->num_merge_operands = kMergeCount;
|
||||||
expected_tp->num_range_deletions = kRangeDeletionCount;
|
expected_tp->num_range_deletions = kRangeDeletionCount;
|
||||||
expected_tp->num_data_blocks =
|
expected_tp->num_data_blocks =
|
||||||
kTableCount * (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
|
kTableCount *
|
||||||
|
(kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
|
||||||
kBlockSize;
|
kBlockSize;
|
||||||
expected_tp->data_size =
|
expected_tp->data_size =
|
||||||
kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize));
|
kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize));
|
||||||
|
@ -1120,7 +1122,8 @@ class CountingUserTblPropCollector : public TablePropertiesCollector {
|
||||||
std::string encoded;
|
std::string encoded;
|
||||||
PutVarint32(&encoded, count_);
|
PutVarint32(&encoded, count_);
|
||||||
*properties = UserCollectedProperties{
|
*properties = UserCollectedProperties{
|
||||||
{"CountingUserTblPropCollector", message_}, {"Count", encoded},
|
{"CountingUserTblPropCollector", message_},
|
||||||
|
{"Count", encoded},
|
||||||
};
|
};
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
|
@ -2122,7 +2125,7 @@ std::string PopMetaIndexKey(InternalIterator* meta_iter) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBPropertiesTest, TableMetaIndexKeys) {
|
TEST_F(DBPropertiesTest, TableMetaIndexKeys) {
|
||||||
// This is to detect unexpected churn in metaindex block keys. This is more
|
// This is to detect unexpected churn in metaindex block keys. This is more
|
||||||
|
|
|
@ -238,7 +238,8 @@ TEST_F(DBRangeDelTest, SentinelsOmittedFromOutputFile) {
|
||||||
const Snapshot* snapshot = db_->GetSnapshot();
|
const Snapshot* snapshot = db_->GetSnapshot();
|
||||||
|
|
||||||
// gaps between ranges creates sentinels in our internal representation
|
// gaps between ranges creates sentinels in our internal representation
|
||||||
std::vector<std::pair<std::string, std::string>> range_dels = {{"a", "b"}, {"c", "d"}, {"e", "f"}};
|
std::vector<std::pair<std::string, std::string>> range_dels = {
|
||||||
|
{"a", "b"}, {"c", "d"}, {"e", "f"}};
|
||||||
for (const auto& range_del : range_dels) {
|
for (const auto& range_del : range_dels) {
|
||||||
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
|
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
|
||||||
range_del.first, range_del.second));
|
range_del.first, range_del.second));
|
||||||
|
@ -567,8 +568,8 @@ TEST_F(DBRangeDelTest, PutDeleteRangeMergeFlush) {
|
||||||
std::string val;
|
std::string val;
|
||||||
PutFixed64(&val, 1);
|
PutFixed64(&val, 1);
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), "key", val));
|
ASSERT_OK(db_->Put(WriteOptions(), "key", val));
|
||||||
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
|
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "key",
|
||||||
"key", "key_"));
|
"key_"));
|
||||||
ASSERT_OK(db_->Merge(WriteOptions(), "key", val));
|
ASSERT_OK(db_->Merge(WriteOptions(), "key", val));
|
||||||
ASSERT_OK(db_->Flush(FlushOptions()));
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
||||||
|
|
||||||
|
@ -1649,7 +1650,8 @@ TEST_F(DBRangeDelTest, RangeTombstoneWrittenToMinimalSsts) {
|
||||||
const auto& table_props = name_and_table_props.second;
|
const auto& table_props = name_and_table_props.second;
|
||||||
// The range tombstone should only be output to the second L1 SST.
|
// The range tombstone should only be output to the second L1 SST.
|
||||||
if (name.size() >= l1_metadata[1].name.size() &&
|
if (name.size() >= l1_metadata[1].name.size() &&
|
||||||
name.substr(name.size() - l1_metadata[1].name.size()).compare(l1_metadata[1].name) == 0) {
|
name.substr(name.size() - l1_metadata[1].name.size())
|
||||||
|
.compare(l1_metadata[1].name) == 0) {
|
||||||
ASSERT_EQ(1, table_props->num_range_deletions);
|
ASSERT_EQ(1, table_props->num_range_deletions);
|
||||||
++num_range_deletions;
|
++num_range_deletions;
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -499,7 +499,7 @@ class TraceFileEnv : public EnvWrapper {
|
||||||
private:
|
private:
|
||||||
std::atomic<int> files_closed_{0};
|
std::atomic<int> files_closed_{0};
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBSecondaryTest, SecondaryCloseFiles) {
|
TEST_F(DBSecondaryTest, SecondaryCloseFiles) {
|
||||||
Options options;
|
Options options;
|
||||||
|
|
|
@ -80,14 +80,16 @@ TEST_F(DBStatisticsTest, CompressionStatsTest) {
|
||||||
ASSERT_OK(Put(Key(i), rnd.RandomString(128) + std::string(128, 'a')));
|
ASSERT_OK(Put(Key(i), rnd.RandomString(128) + std::string(128, 'a')));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED)
|
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED) -
|
||||||
- currentCompressions, 0);
|
currentCompressions,
|
||||||
|
0);
|
||||||
|
|
||||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||||
auto r = Get(Key(i));
|
auto r = Get(Key(i));
|
||||||
}
|
}
|
||||||
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED)
|
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED) -
|
||||||
- currentDecompressions, 0);
|
currentDecompressions,
|
||||||
|
0);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) {
|
TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) {
|
||||||
|
|
|
@ -52,7 +52,7 @@ void VerifyTableProperties(DB* db, uint64_t expected_entries_size) {
|
||||||
|
|
||||||
VerifySstUniqueIds(props);
|
VerifySstUniqueIds(props);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class DBTablePropertiesTest : public DBTestBase,
|
class DBTablePropertiesTest : public DBTestBase,
|
||||||
public testing::WithParamInterface<std::string> {
|
public testing::WithParamInterface<std::string> {
|
||||||
|
@ -240,7 +240,6 @@ TablePropertiesCollection
|
||||||
DBTablePropertiesTest::TestGetPropertiesOfTablesInRange(
|
DBTablePropertiesTest::TestGetPropertiesOfTablesInRange(
|
||||||
std::vector<Range> ranges, std::size_t* num_properties,
|
std::vector<Range> ranges, std::size_t* num_properties,
|
||||||
std::size_t* num_files) {
|
std::size_t* num_files) {
|
||||||
|
|
||||||
// Since we deref zero element in the vector it can not be empty
|
// Since we deref zero element in the vector it can not be empty
|
||||||
// otherwise we pass an address to some random memory
|
// otherwise we pass an address to some random memory
|
||||||
EXPECT_GT(ranges.size(), 0U);
|
EXPECT_GT(ranges.size(), 0U);
|
||||||
|
@ -524,10 +523,10 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
|
||||||
// effect
|
// effect
|
||||||
kWindowSize = 50;
|
kWindowSize = 50;
|
||||||
kNumDelsTrigger = 40;
|
kNumDelsTrigger = 40;
|
||||||
static_cast<CompactOnDeletionCollectorFactory*>
|
static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
|
||||||
(compact_on_del.get())->SetWindowSize(kWindowSize);
|
->SetWindowSize(kWindowSize);
|
||||||
static_cast<CompactOnDeletionCollectorFactory*>
|
static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
|
||||||
(compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger);
|
->SetDeletionTrigger(kNumDelsTrigger);
|
||||||
for (int i = 0; i < kNumKeys; ++i) {
|
for (int i = 0; i < kNumKeys; ++i) {
|
||||||
if (i >= kNumKeys - kWindowSize &&
|
if (i >= kNumKeys - kWindowSize &&
|
||||||
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
|
||||||
|
@ -543,10 +542,10 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
|
||||||
|
|
||||||
// Change the window size to disable delete triggered compaction
|
// Change the window size to disable delete triggered compaction
|
||||||
kWindowSize = 0;
|
kWindowSize = 0;
|
||||||
static_cast<CompactOnDeletionCollectorFactory*>
|
static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
|
||||||
(compact_on_del.get())->SetWindowSize(kWindowSize);
|
->SetWindowSize(kWindowSize);
|
||||||
static_cast<CompactOnDeletionCollectorFactory*>
|
static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
|
||||||
(compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger);
|
->SetDeletionTrigger(kNumDelsTrigger);
|
||||||
for (int i = 0; i < kNumKeys; ++i) {
|
for (int i = 0; i < kNumKeys; ++i) {
|
||||||
if (i >= kNumKeys - kWindowSize &&
|
if (i >= kNumKeys - kWindowSize &&
|
||||||
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
|
||||||
|
@ -611,13 +610,9 @@ TEST_P(DBTablePropertiesTest, RatioBasedDeletionTriggeredCompactionMarking) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
INSTANTIATE_TEST_CASE_P(
|
INSTANTIATE_TEST_CASE_P(DBTablePropertiesTest, DBTablePropertiesTest,
|
||||||
DBTablePropertiesTest,
|
::testing::Values("kCompactionStyleLevel",
|
||||||
DBTablePropertiesTest,
|
"kCompactionStyleUniversal"));
|
||||||
::testing::Values(
|
|
||||||
"kCompactionStyleLevel",
|
|
||||||
"kCompactionStyleUniversal"
|
|
||||||
));
|
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
|
|
@ -539,7 +539,6 @@ TEST_P(DBTestTailingIterator, SeekWithUpperBoundBug) {
|
||||||
const Slice upper_bound("cc", 3);
|
const Slice upper_bound("cc", 3);
|
||||||
read_options.iterate_upper_bound = &upper_bound;
|
read_options.iterate_upper_bound = &upper_bound;
|
||||||
|
|
||||||
|
|
||||||
// 1st L0 file
|
// 1st L0 file
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
|
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
|
@ -565,7 +564,6 @@ TEST_P(DBTestTailingIterator, SeekToFirstWithUpperBoundBug) {
|
||||||
const Slice upper_bound("cc", 3);
|
const Slice upper_bound("cc", 3);
|
||||||
read_options.iterate_upper_bound = &upper_bound;
|
read_options.iterate_upper_bound = &upper_bound;
|
||||||
|
|
||||||
|
|
||||||
// 1st L0 file
|
// 1st L0 file
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
|
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
|
|
|
@ -1135,7 +1135,7 @@ class DelayFilterFactory : public CompactionFilterFactory {
|
||||||
private:
|
private:
|
||||||
DBTestBase* db_test;
|
DBTestBase* db_test;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
|
|
||||||
|
@ -1490,7 +1490,7 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest, MinLevelToCompress1) {
|
TEST_F(DBTest, MinLevelToCompress1) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
|
@ -2843,7 +2843,7 @@ static void MTThreadBody(void* arg) {
|
||||||
fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
|
fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class MultiThreadedDBTest
|
class MultiThreadedDBTest
|
||||||
: public DBTest,
|
: public DBTest,
|
||||||
|
@ -2929,7 +2929,7 @@ static void GCThreadBody(void* arg) {
|
||||||
t->done = true;
|
t->done = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest, GroupCommitTest) {
|
TEST_F(DBTest, GroupCommitTest) {
|
||||||
do {
|
do {
|
||||||
|
@ -4645,7 +4645,7 @@ void VerifyOperationCount(Env* env, ThreadStatus::OperationType op_type,
|
||||||
}
|
}
|
||||||
ASSERT_EQ(op_count, expected_count);
|
ASSERT_EQ(op_count, expected_count);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest, GetThreadStatus) {
|
TEST_F(DBTest, GetThreadStatus) {
|
||||||
Options options;
|
Options options;
|
||||||
|
|
|
@ -691,7 +691,7 @@ namespace {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest2, WalFilterTest) {
|
TEST_F(DBTest2, WalFilterTest) {
|
||||||
class TestWalFilter : public WalFilter {
|
class TestWalFilter : public WalFilter {
|
||||||
|
@ -719,8 +719,7 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
|
|
||||||
if (current_record_index_ == apply_option_at_record_index_) {
|
if (current_record_index_ == apply_option_at_record_index_) {
|
||||||
option_to_return = wal_processing_option_;
|
option_to_return = wal_processing_option_;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
option_to_return = WalProcessingOption::kContinueProcessing;
|
option_to_return = WalProcessingOption::kContinueProcessing;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -784,8 +783,7 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
// databse
|
// databse
|
||||||
options.paranoid_checks = false;
|
options.paranoid_checks = false;
|
||||||
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
ASSERT_OK(status);
|
ASSERT_OK(status);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -815,8 +813,7 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||||
if (i == apply_option_for_record_index) {
|
if (i == apply_option_for_record_index) {
|
||||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -834,8 +831,7 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||||
if (i >= apply_option_for_record_index) {
|
if (i >= apply_option_for_record_index) {
|
||||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -977,8 +973,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||||
if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) {
|
if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) {
|
||||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1008,7 +1003,8 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
|
TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
|
||||||
class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
|
class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
|
||||||
public:
|
public:
|
||||||
WalProcessingOption LogRecord(const WriteBatch& batch, WriteBatch* new_batch,
|
WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||||
|
WriteBatch* new_batch,
|
||||||
bool* batch_changed) const override {
|
bool* batch_changed) const override {
|
||||||
*new_batch = batch;
|
*new_batch = batch;
|
||||||
Status s = new_batch->Put("key_extra", "value_extra");
|
Status s = new_batch->Put("key_extra", "value_extra");
|
||||||
|
@ -1085,6 +1081,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
// during recovery (i.e. aren't already flushed to SST file(s))
|
// during recovery (i.e. aren't already flushed to SST file(s))
|
||||||
// for verification against the keys we expect.
|
// for verification against the keys we expect.
|
||||||
std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
|
std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
void ColumnFamilyLogNumberMap(
|
void ColumnFamilyLogNumberMap(
|
||||||
const std::map<uint32_t, uint64_t>& cf_lognumber_map,
|
const std::map<uint32_t, uint64_t>& cf_lognumber_map,
|
||||||
|
@ -1103,11 +1100,13 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
const std::map<uint32_t, uint64_t>& cf_log_number_map_;
|
const std::map<uint32_t, uint64_t>& cf_log_number_map_;
|
||||||
std::map<uint32_t, std::vector<std::string>>& cf_wal_keys_;
|
std::map<uint32_t, std::vector<std::string>>& cf_wal_keys_;
|
||||||
unsigned long long log_number_;
|
unsigned long long log_number_;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
LogRecordBatchHandler(unsigned long long current_log_number,
|
LogRecordBatchHandler(
|
||||||
|
unsigned long long current_log_number,
|
||||||
const std::map<uint32_t, uint64_t>& cf_log_number_map,
|
const std::map<uint32_t, uint64_t>& cf_log_number_map,
|
||||||
std::map<uint32_t, std::vector<std::string>> & cf_wal_keys) :
|
std::map<uint32_t, std::vector<std::string>>& cf_wal_keys)
|
||||||
cf_log_number_map_(cf_log_number_map),
|
: cf_log_number_map_(cf_log_number_map),
|
||||||
cf_wal_keys_(cf_wal_keys),
|
cf_wal_keys_(cf_wal_keys),
|
||||||
log_number_(current_log_number) {}
|
log_number_(current_log_number) {}
|
||||||
|
|
||||||
|
@ -1120,8 +1119,8 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
// (i.e. isn't flushed to SST file(s) for column_family_id)
|
// (i.e. isn't flushed to SST file(s) for column_family_id)
|
||||||
// add it to the cf_wal_keys_ map for verification.
|
// add it to the cf_wal_keys_ map for verification.
|
||||||
if (log_number_ >= log_number_for_cf) {
|
if (log_number_ >= log_number_for_cf) {
|
||||||
cf_wal_keys_[column_family_id].push_back(std::string(key.data(),
|
cf_wal_keys_[column_family_id].push_back(
|
||||||
key.size()));
|
std::string(key.data(), key.size()));
|
||||||
}
|
}
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
|
@ -1208,8 +1207,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
// Reopen database with option to use WAL filter
|
// Reopen database with option to use WAL filter
|
||||||
options = OptionsForLogIterTest();
|
options = OptionsForLogIterTest();
|
||||||
options.wal_filter = &test_wal_filter_column_families;
|
options.wal_filter = &test_wal_filter_column_families;
|
||||||
Status status =
|
Status status = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||||
TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
|
||||||
ASSERT_TRUE(status.ok());
|
ASSERT_TRUE(status.ok());
|
||||||
|
|
||||||
// verify that handles_[0] only has post_flush keys
|
// verify that handles_[0] only has post_flush keys
|
||||||
|
@ -1960,7 +1958,8 @@ TEST_F(DBTest2, CompressionOptions) {
|
||||||
|
|
||||||
class CompactionStallTestListener : public EventListener {
|
class CompactionStallTestListener : public EventListener {
|
||||||
public:
|
public:
|
||||||
CompactionStallTestListener() : compacting_files_cnt_(0), compacted_files_cnt_(0) {}
|
CompactionStallTestListener()
|
||||||
|
: compacting_files_cnt_(0), compacted_files_cnt_(0) {}
|
||||||
|
|
||||||
void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override {
|
void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override {
|
||||||
ASSERT_EQ(ci.cf_name, "default");
|
ASSERT_EQ(ci.cf_name, "default");
|
||||||
|
@ -2039,7 +2038,8 @@ TEST_F(DBTest2, CompactionStall) {
|
||||||
options.level0_file_num_compaction_trigger);
|
options.level0_file_num_compaction_trigger);
|
||||||
ASSERT_GT(listener->compacted_files_cnt_.load(),
|
ASSERT_GT(listener->compacted_files_cnt_.load(),
|
||||||
10 - options.level0_file_num_compaction_trigger);
|
10 - options.level0_file_num_compaction_trigger);
|
||||||
ASSERT_EQ(listener->compacting_files_cnt_.load(), listener->compacted_files_cnt_.load());
|
ASSERT_EQ(listener->compacting_files_cnt_.load(),
|
||||||
|
listener->compacted_files_cnt_.load());
|
||||||
|
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
||||||
}
|
}
|
||||||
|
@ -2664,7 +2664,7 @@ namespace {
|
||||||
void CountSyncPoint() {
|
void CountSyncPoint() {
|
||||||
TEST_SYNC_POINT_CALLBACK("DBTest2::MarkedPoint", nullptr /* arg */);
|
TEST_SYNC_POINT_CALLBACK("DBTest2::MarkedPoint", nullptr /* arg */);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest2, SyncPointMarker) {
|
TEST_F(DBTest2, SyncPointMarker) {
|
||||||
std::atomic<int> sync_point_called(0);
|
std::atomic<int> sync_point_called(0);
|
||||||
|
@ -2899,7 +2899,6 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
|
||||||
size_t total_loaded_bytes_iter2 =
|
size_t total_loaded_bytes_iter2 =
|
||||||
options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES);
|
options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES);
|
||||||
|
|
||||||
|
|
||||||
// Read amp is on average 100% since we read all what we loaded in memory
|
// Read amp is on average 100% since we read all what we loaded in memory
|
||||||
if (k == 0) {
|
if (k == 0) {
|
||||||
ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2,
|
ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2,
|
||||||
|
@ -5503,16 +5502,20 @@ TEST_F(DBTest2, TestGetColumnFamilyHandleUnlocked) {
|
||||||
port::Thread user_thread1([&]() {
|
port::Thread user_thread1([&]() {
|
||||||
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[0]->GetID());
|
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[0]->GetID());
|
||||||
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
|
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
|
||||||
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked1");
|
TEST_SYNC_POINT(
|
||||||
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::ReadColumnFamilyHandle1");
|
"TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked1");
|
||||||
|
TEST_SYNC_POINT(
|
||||||
|
"TestGetColumnFamilyHandleUnlocked::ReadColumnFamilyHandle1");
|
||||||
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
|
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
|
||||||
});
|
});
|
||||||
|
|
||||||
port::Thread user_thread2([&]() {
|
port::Thread user_thread2([&]() {
|
||||||
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::PreGetColumnFamilyHandleUnlocked2");
|
TEST_SYNC_POINT(
|
||||||
|
"TestGetColumnFamilyHandleUnlocked::PreGetColumnFamilyHandleUnlocked2");
|
||||||
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[1]->GetID());
|
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[1]->GetID());
|
||||||
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
|
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
|
||||||
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked2");
|
TEST_SYNC_POINT(
|
||||||
|
"TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked2");
|
||||||
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
|
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -5666,7 +5669,7 @@ class DummyOldStats : public Statistics {
|
||||||
std::atomic<int> num_rt{0};
|
std::atomic<int> num_rt{0};
|
||||||
std::atomic<int> num_mt{0};
|
std::atomic<int> num_mt{0};
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest2, OldStatsInterface) {
|
TEST_F(DBTest2, OldStatsInterface) {
|
||||||
DummyOldStats* dos = new DummyOldStats();
|
DummyOldStats* dos = new DummyOldStats();
|
||||||
|
|
|
@ -29,7 +29,7 @@ int64_t MaybeCurrentTime(Env* env) {
|
||||||
env->GetCurrentTime(&time).PermitUncheckedError();
|
env->GetCurrentTime(&time).PermitUncheckedError();
|
||||||
return time;
|
return time;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// Special Env used to delay background operations
|
// Special Env used to delay background operations
|
||||||
|
|
||||||
|
@ -1308,12 +1308,14 @@ void DBTestBase::GetSstFiles(Env* env, std::string path,
|
||||||
std::vector<std::string>* files) {
|
std::vector<std::string>* files) {
|
||||||
EXPECT_OK(env->GetChildren(path, files));
|
EXPECT_OK(env->GetChildren(path, files));
|
||||||
|
|
||||||
files->erase(
|
files->erase(std::remove_if(files->begin(), files->end(),
|
||||||
std::remove_if(files->begin(), files->end(), [](std::string name) {
|
[](std::string name) {
|
||||||
uint64_t number;
|
uint64_t number;
|
||||||
FileType type;
|
FileType type;
|
||||||
return !(ParseFileName(name, &number, &type) && type == kTableFile);
|
return !(ParseFileName(name, &number, &type) &&
|
||||||
}), files->end());
|
type == kTableFile);
|
||||||
|
}),
|
||||||
|
files->end());
|
||||||
}
|
}
|
||||||
|
|
||||||
int DBTestBase::GetSstFileCount(std::string path) {
|
int DBTestBase::GetSstFileCount(std::string path) {
|
||||||
|
@ -1583,8 +1585,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
||||||
iter_cnt++;
|
iter_cnt++;
|
||||||
total_reads++;
|
total_reads++;
|
||||||
}
|
}
|
||||||
ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / "
|
ASSERT_EQ(data_iter, true_data.end())
|
||||||
<< true_data.size();
|
<< iter_cnt << " / " << true_data.size();
|
||||||
delete iter;
|
delete iter;
|
||||||
|
|
||||||
// Verify Iterator::Prev()
|
// Verify Iterator::Prev()
|
||||||
|
@ -1606,8 +1608,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
||||||
iter_cnt++;
|
iter_cnt++;
|
||||||
total_reads++;
|
total_reads++;
|
||||||
}
|
}
|
||||||
ASSERT_EQ(data_rev, true_data.rend()) << iter_cnt << " / "
|
ASSERT_EQ(data_rev, true_data.rend())
|
||||||
<< true_data.size();
|
<< iter_cnt << " / " << true_data.size();
|
||||||
|
|
||||||
// Verify Iterator::Seek()
|
// Verify Iterator::Seek()
|
||||||
for (auto kv : true_data) {
|
for (auto kv : true_data) {
|
||||||
|
@ -1637,8 +1639,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
||||||
iter_cnt++;
|
iter_cnt++;
|
||||||
total_reads++;
|
total_reads++;
|
||||||
}
|
}
|
||||||
ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / "
|
ASSERT_EQ(data_iter, true_data.end())
|
||||||
<< true_data.size();
|
<< iter_cnt << " / " << true_data.size();
|
||||||
|
|
||||||
// Verify ForwardIterator::Seek()
|
// Verify ForwardIterator::Seek()
|
||||||
for (auto kv : true_data) {
|
for (auto kv : true_data) {
|
||||||
|
|
|
@ -220,9 +220,7 @@ class SpecialEnv : public EnvWrapper {
|
||||||
Env::IOPriority GetIOPriority() override {
|
Env::IOPriority GetIOPriority() override {
|
||||||
return base_->GetIOPriority();
|
return base_->GetIOPriority();
|
||||||
}
|
}
|
||||||
bool use_direct_io() const override {
|
bool use_direct_io() const override { return base_->use_direct_io(); }
|
||||||
return base_->use_direct_io();
|
|
||||||
}
|
|
||||||
Status Allocate(uint64_t offset, uint64_t len) override {
|
Status Allocate(uint64_t offset, uint64_t len) override {
|
||||||
return base_->Allocate(offset, len);
|
return base_->Allocate(offset, len);
|
||||||
}
|
}
|
||||||
|
|
|
@ -39,8 +39,8 @@ class DBTestUniversalCompactionBase
|
||||||
|
|
||||||
class DBTestUniversalCompaction : public DBTestUniversalCompactionBase {
|
class DBTestUniversalCompaction : public DBTestUniversalCompactionBase {
|
||||||
public:
|
public:
|
||||||
DBTestUniversalCompaction() :
|
DBTestUniversalCompaction()
|
||||||
DBTestUniversalCompactionBase("/db_universal_compaction_test") {}
|
: DBTestUniversalCompactionBase("/db_universal_compaction_test") {}
|
||||||
};
|
};
|
||||||
|
|
||||||
class DBTestUniversalCompaction2 : public DBTestBase {
|
class DBTestUniversalCompaction2 : public DBTestBase {
|
||||||
|
@ -93,7 +93,7 @@ class KeepFilterFactory : public CompactionFilterFactory {
|
||||||
std::atomic_bool expect_full_compaction_;
|
std::atomic_bool expect_full_compaction_;
|
||||||
std::atomic_bool expect_manual_compaction_;
|
std::atomic_bool expect_manual_compaction_;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// Make sure we don't trigger a problem if the trigger condtion is given
|
// Make sure we don't trigger a problem if the trigger condtion is given
|
||||||
// to be 0, which is invalid.
|
// to be 0, which is invalid.
|
||||||
|
@ -563,8 +563,7 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (compaction_input_file_names.size() == 0) {
|
if (compaction_input_file_names.size() == 0) {
|
||||||
compaction_input_file_names.push_back(
|
compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
|
||||||
cf_meta.levels[0].files[0].name);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// expect fail since universal compaction only allow L0 output
|
// expect fail since universal compaction only allow L0 output
|
||||||
|
@ -574,27 +573,22 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
||||||
.ok());
|
.ok());
|
||||||
|
|
||||||
// expect ok and verify the compacted files no longer exist.
|
// expect ok and verify the compacted files no longer exist.
|
||||||
ASSERT_OK(dbfull()->CompactFiles(
|
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
|
||||||
CompactionOptions(), handles_[1],
|
|
||||||
compaction_input_file_names, 0));
|
compaction_input_file_names, 0));
|
||||||
|
|
||||||
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
||||||
VerifyCompactionResult(
|
VerifyCompactionResult(
|
||||||
cf_meta,
|
cf_meta, std::set<std::string>(compaction_input_file_names.begin(),
|
||||||
std::set<std::string>(compaction_input_file_names.begin(),
|
|
||||||
compaction_input_file_names.end()));
|
compaction_input_file_names.end()));
|
||||||
|
|
||||||
compaction_input_file_names.clear();
|
compaction_input_file_names.clear();
|
||||||
|
|
||||||
// Pick the first and the last file, expect everything is
|
// Pick the first and the last file, expect everything is
|
||||||
// compacted into one single file.
|
// compacted into one single file.
|
||||||
|
compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
|
||||||
compaction_input_file_names.push_back(
|
compaction_input_file_names.push_back(
|
||||||
cf_meta.levels[0].files[0].name);
|
cf_meta.levels[0].files[cf_meta.levels[0].files.size() - 1].name);
|
||||||
compaction_input_file_names.push_back(
|
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
|
||||||
cf_meta.levels[0].files[
|
|
||||||
cf_meta.levels[0].files.size() - 1].name);
|
|
||||||
ASSERT_OK(dbfull()->CompactFiles(
|
|
||||||
CompactionOptions(), handles_[1],
|
|
||||||
compaction_input_file_names, 0));
|
compaction_input_file_names, 0));
|
||||||
|
|
||||||
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
||||||
|
@ -640,8 +634,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
|
||||||
class DBTestUniversalCompactionMultiLevels
|
class DBTestUniversalCompactionMultiLevels
|
||||||
: public DBTestUniversalCompactionBase {
|
: public DBTestUniversalCompactionBase {
|
||||||
public:
|
public:
|
||||||
DBTestUniversalCompactionMultiLevels() :
|
DBTestUniversalCompactionMultiLevels()
|
||||||
DBTestUniversalCompactionBase(
|
: DBTestUniversalCompactionBase(
|
||||||
"/db_universal_compaction_multi_levels_test") {}
|
"/db_universal_compaction_multi_levels_test") {}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -725,12 +719,11 @@ INSTANTIATE_TEST_CASE_P(MultiLevels, DBTestUniversalCompactionMultiLevels,
|
||||||
::testing::Combine(::testing::Values(3, 20),
|
::testing::Combine(::testing::Values(3, 20),
|
||||||
::testing::Bool()));
|
::testing::Bool()));
|
||||||
|
|
||||||
class DBTestUniversalCompactionParallel :
|
class DBTestUniversalCompactionParallel : public DBTestUniversalCompactionBase {
|
||||||
public DBTestUniversalCompactionBase {
|
|
||||||
public:
|
public:
|
||||||
DBTestUniversalCompactionParallel() :
|
DBTestUniversalCompactionParallel()
|
||||||
DBTestUniversalCompactionBase(
|
: DBTestUniversalCompactionBase("/db_universal_compaction_prallel_test") {
|
||||||
"/db_universal_compaction_prallel_test") {}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
|
TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
|
||||||
|
@ -1567,7 +1560,6 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
|
||||||
verify_func(max_key3);
|
verify_func(max_key3);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
|
TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
|
||||||
if (!Snappy_Supported()) {
|
if (!Snappy_Supported()) {
|
||||||
return;
|
return;
|
||||||
|
@ -1829,8 +1821,8 @@ INSTANTIATE_TEST_CASE_P(NumLevels, DBTestUniversalCompaction,
|
||||||
class DBTestUniversalManualCompactionOutputPathId
|
class DBTestUniversalManualCompactionOutputPathId
|
||||||
: public DBTestUniversalCompactionBase {
|
: public DBTestUniversalCompactionBase {
|
||||||
public:
|
public:
|
||||||
DBTestUniversalManualCompactionOutputPathId() :
|
DBTestUniversalManualCompactionOutputPathId()
|
||||||
DBTestUniversalCompactionBase(
|
: DBTestUniversalCompactionBase(
|
||||||
"/db_universal_compaction_manual_pid_test") {}
|
"/db_universal_compaction_manual_pid_test") {}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -170,7 +170,8 @@ TEST_P(DBWriteTest, WriteStallRemoveNoSlowdownWrite) {
|
||||||
|
|
||||||
TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
||||||
Options options = GetOptions();
|
Options options = GetOptions();
|
||||||
options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger = 4;
|
options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger =
|
||||||
|
4;
|
||||||
std::vector<port::Thread> threads;
|
std::vector<port::Thread> threads;
|
||||||
std::atomic<int> thread_num(0);
|
std::atomic<int> thread_num(0);
|
||||||
port::Mutex mutex;
|
port::Mutex mutex;
|
||||||
|
@ -254,8 +255,9 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr));
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr));
|
||||||
// This would have triggered a write stall. Unblock the write group leader
|
// This would have triggered a write stall. Unblock the write group leader
|
||||||
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:2");
|
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:2");
|
||||||
// The leader is going to create missing newer links. When the leader finishes,
|
// The leader is going to create missing newer links. When the leader
|
||||||
// the next leader is going to delay writes and fail writers with no_slowdown
|
// finishes, the next leader is going to delay writes and fail writers with
|
||||||
|
// no_slowdown
|
||||||
|
|
||||||
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:3");
|
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:3");
|
||||||
for (auto& t : threads) {
|
for (auto& t : threads) {
|
||||||
|
@ -635,15 +637,15 @@ TEST_P(DBWriteTest, ConcurrentlyDisabledWAL) {
|
||||||
std::string wal_value = "0";
|
std::string wal_value = "0";
|
||||||
std::thread threads[10];
|
std::thread threads[10];
|
||||||
for (int t = 0; t < 10; t++) {
|
for (int t = 0; t < 10; t++) {
|
||||||
threads[t] = std::thread([t, wal_key_prefix, wal_value, no_wal_key_prefix, no_wal_value, this] {
|
threads[t] = std::thread([t, wal_key_prefix, wal_value, no_wal_key_prefix,
|
||||||
|
no_wal_value, this] {
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
ROCKSDB_NAMESPACE::WriteOptions write_option_disable;
|
ROCKSDB_NAMESPACE::WriteOptions write_option_disable;
|
||||||
write_option_disable.disableWAL = true;
|
write_option_disable.disableWAL = true;
|
||||||
ROCKSDB_NAMESPACE::WriteOptions write_option_default;
|
ROCKSDB_NAMESPACE::WriteOptions write_option_default;
|
||||||
std::string no_wal_key = no_wal_key_prefix + std::to_string(t) +
|
std::string no_wal_key =
|
||||||
"_" + std::to_string(i);
|
no_wal_key_prefix + std::to_string(t) + "_" + std::to_string(i);
|
||||||
ASSERT_OK(
|
ASSERT_OK(this->Put(no_wal_key, no_wal_value, write_option_disable));
|
||||||
this->Put(no_wal_key, no_wal_value, write_option_disable));
|
|
||||||
std::string wal_key =
|
std::string wal_key =
|
||||||
wal_key_prefix + std::to_string(i) + "_" + std::to_string(i);
|
wal_key_prefix + std::to_string(i) + "_" + std::to_string(i);
|
||||||
ASSERT_OK(this->Put(wal_key, wal_value, write_option_default));
|
ASSERT_OK(this->Put(wal_key, wal_value, write_option_default));
|
||||||
|
@ -657,7 +659,8 @@ TEST_P(DBWriteTest, ConcurrentlyDisabledWAL) {
|
||||||
}
|
}
|
||||||
uint64_t bytes_num = options.statistics->getTickerCount(
|
uint64_t bytes_num = options.statistics->getTickerCount(
|
||||||
ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES);
|
ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES);
|
||||||
// written WAL size should less than 100KB (even included HEADER & FOOTER overhead)
|
// written WAL size should less than 100KB (even included HEADER & FOOTER
|
||||||
|
// overhead)
|
||||||
ASSERT_LE(bytes_num, 1024 * 100);
|
ASSERT_LE(bytes_num, 1024 * 100);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -15,8 +15,7 @@
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
static std::string IKey(const std::string& user_key,
|
static std::string IKey(const std::string& user_key, uint64_t seq,
|
||||||
uint64_t seq,
|
|
||||||
ValueType vt) {
|
ValueType vt) {
|
||||||
std::string encoded;
|
std::string encoded;
|
||||||
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
|
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
|
||||||
|
@ -37,9 +36,7 @@ static std::string ShortSuccessor(const std::string& s) {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void TestKey(const std::string& key,
|
static void TestKey(const std::string& key, uint64_t seq, ValueType vt) {
|
||||||
uint64_t seq,
|
|
||||||
ValueType vt) {
|
|
||||||
std::string encoded = IKey(key, seq, vt);
|
std::string encoded = IKey(key, seq, vt);
|
||||||
|
|
||||||
Slice in(encoded);
|
Slice in(encoded);
|
||||||
|
@ -57,12 +54,18 @@ class FormatTest : public testing::Test {};
|
||||||
|
|
||||||
TEST_F(FormatTest, InternalKey_EncodeDecode) {
|
TEST_F(FormatTest, InternalKey_EncodeDecode) {
|
||||||
const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"};
|
const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"};
|
||||||
const uint64_t seq[] = {
|
const uint64_t seq[] = {1,
|
||||||
1, 2, 3,
|
2,
|
||||||
(1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1,
|
3,
|
||||||
(1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1,
|
(1ull << 8) - 1,
|
||||||
(1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1
|
1ull << 8,
|
||||||
};
|
(1ull << 8) + 1,
|
||||||
|
(1ull << 16) - 1,
|
||||||
|
1ull << 16,
|
||||||
|
(1ull << 16) + 1,
|
||||||
|
(1ull << 32) - 1,
|
||||||
|
1ull << 32,
|
||||||
|
(1ull << 32) + 1};
|
||||||
for (unsigned int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
|
for (unsigned int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
|
||||||
for (unsigned int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
|
for (unsigned int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
|
||||||
TestKey(keys[k], seq[s], kTypeValue);
|
TestKey(keys[k], seq[s], kTypeValue);
|
||||||
|
@ -74,27 +77,25 @@ TEST_F(FormatTest, InternalKey_EncodeDecode) {
|
||||||
TEST_F(FormatTest, InternalKeyShortSeparator) {
|
TEST_F(FormatTest, InternalKeyShortSeparator) {
|
||||||
// When user keys are same
|
// When user keys are same
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue)));
|
||||||
IKey("foo", 99, kTypeValue)));
|
ASSERT_EQ(
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue)));
|
||||||
IKey("foo", 101, kTypeValue)));
|
ASSERT_EQ(
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue)));
|
||||||
IKey("foo", 100, kTypeValue)));
|
ASSERT_EQ(
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion)));
|
||||||
IKey("foo", 100, kTypeDeletion)));
|
|
||||||
|
|
||||||
// When user keys are misordered
|
// When user keys are misordered
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue)));
|
||||||
IKey("bar", 99, kTypeValue)));
|
|
||||||
|
|
||||||
// When user keys are different, but correctly ordered
|
// When user keys are different, but correctly ordered
|
||||||
ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
|
ASSERT_EQ(
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
|
||||||
IKey("hello", 200, kTypeValue)));
|
Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue)));
|
||||||
|
|
||||||
ASSERT_EQ(IKey("ABC2", kMaxSequenceNumber, kValueTypeForSeek),
|
ASSERT_EQ(IKey("ABC2", kMaxSequenceNumber, kValueTypeForSeek),
|
||||||
Shorten(IKey("ABC1AAAAA", 100, kTypeValue),
|
Shorten(IKey("ABC1AAAAA", 100, kTypeValue),
|
||||||
|
@ -121,14 +122,14 @@ TEST_F(FormatTest, InternalKeyShortSeparator) {
|
||||||
Shorten(IKey("AAA1", 100, kTypeValue), IKey("AAA2", 200, kTypeValue)));
|
Shorten(IKey("AAA1", 100, kTypeValue), IKey("AAA2", 200, kTypeValue)));
|
||||||
|
|
||||||
// When start user key is prefix of limit user key
|
// When start user key is prefix of limit user key
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
ASSERT_EQ(
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
IKey("foo", 100, kTypeValue),
|
||||||
IKey("foobar", 200, kTypeValue)));
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue)));
|
||||||
|
|
||||||
// When limit user key is prefix of start user key
|
// When limit user key is prefix of start user key
|
||||||
ASSERT_EQ(IKey("foobar", 100, kTypeValue),
|
ASSERT_EQ(
|
||||||
Shorten(IKey("foobar", 100, kTypeValue),
|
IKey("foobar", 100, kTypeValue),
|
||||||
IKey("foo", 200, kTypeValue)));
|
Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue)));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(FormatTest, InternalKeyShortestSuccessor) {
|
TEST_F(FormatTest, InternalKeyShortestSuccessor) {
|
||||||
|
|
|
@ -10,9 +10,11 @@
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
|
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "db/db_impl/db_impl.h"
|
#include "db/db_impl/db_impl.h"
|
||||||
#include "db/db_test_util.h"
|
#include "db/db_test_util.h"
|
||||||
#include "db/version_set.h"
|
#include "db/version_set.h"
|
||||||
|
@ -63,10 +65,8 @@ class DeleteFileTest : public DBTestBase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
int numKeysInLevels(
|
int numKeysInLevels(std::vector<LiveFileMetaData>& metadata,
|
||||||
std::vector<LiveFileMetaData> &metadata,
|
|
||||||
std::vector<int>* keysperlevel = nullptr) {
|
std::vector<int>* keysperlevel = nullptr) {
|
||||||
|
|
||||||
if (keysperlevel != nullptr) {
|
if (keysperlevel != nullptr) {
|
||||||
keysperlevel->resize(numlevels_);
|
keysperlevel->resize(numlevels_);
|
||||||
}
|
}
|
||||||
|
@ -82,8 +82,7 @@ class DeleteFileTest : public DBTestBase {
|
||||||
}
|
}
|
||||||
fprintf(stderr, "level %d name %s smallest %s largest %s\n",
|
fprintf(stderr, "level %d name %s smallest %s largest %s\n",
|
||||||
metadata[i].level, metadata[i].name.c_str(),
|
metadata[i].level, metadata[i].name.c_str(),
|
||||||
metadata[i].smallestkey.c_str(),
|
metadata[i].smallestkey.c_str(), metadata[i].largestkey.c_str());
|
||||||
metadata[i].largestkey.c_str());
|
|
||||||
}
|
}
|
||||||
return numKeys;
|
return numKeys;
|
||||||
}
|
}
|
||||||
|
@ -481,8 +480,8 @@ TEST_F(DeleteFileTest, DeleteFileWithIterator) {
|
||||||
}
|
}
|
||||||
|
|
||||||
Status status = db_->DeleteFile(level2file);
|
Status status = db_->DeleteFile(level2file);
|
||||||
fprintf(stdout, "Deletion status %s: %s\n",
|
fprintf(stdout, "Deletion status %s: %s\n", level2file.c_str(),
|
||||||
level2file.c_str(), status.ToString().c_str());
|
status.ToString().c_str());
|
||||||
ASSERT_OK(status);
|
ASSERT_OK(status);
|
||||||
it->SeekToFirst();
|
it->SeekToFirst();
|
||||||
int numKeysIterated = 0;
|
int numKeysIterated = 0;
|
||||||
|
|
|
@ -234,8 +234,8 @@ void ErrorHandler::CancelErrorRecovery() {
|
||||||
// We'll release the lock before calling sfm, so make sure no new
|
// We'll release the lock before calling sfm, so make sure no new
|
||||||
// recovery gets scheduled at that point
|
// recovery gets scheduled at that point
|
||||||
auto_recovery_ = false;
|
auto_recovery_ = false;
|
||||||
SstFileManagerImpl* sfm = reinterpret_cast<SstFileManagerImpl*>(
|
SstFileManagerImpl* sfm =
|
||||||
db_options_.sst_file_manager.get());
|
reinterpret_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
|
||||||
if (sfm) {
|
if (sfm) {
|
||||||
// This may or may not cancel a pending recovery
|
// This may or may not cancel a pending recovery
|
||||||
db_mutex_->Unlock();
|
db_mutex_->Unlock();
|
||||||
|
@ -292,8 +292,8 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err,
|
||||||
bool found = false;
|
bool found = false;
|
||||||
|
|
||||||
{
|
{
|
||||||
auto entry = ErrorSeverityMap.find(std::make_tuple(reason, bg_err.code(),
|
auto entry = ErrorSeverityMap.find(
|
||||||
bg_err.subcode(), paranoid));
|
std::make_tuple(reason, bg_err.code(), bg_err.subcode(), paranoid));
|
||||||
if (entry != ErrorSeverityMap.end()) {
|
if (entry != ErrorSeverityMap.end()) {
|
||||||
sev = entry->second;
|
sev = entry->second;
|
||||||
found = true;
|
found = true;
|
||||||
|
@ -301,8 +301,8 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err,
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!found) {
|
if (!found) {
|
||||||
auto entry = DefaultErrorSeverityMap.find(std::make_tuple(reason,
|
auto entry = DefaultErrorSeverityMap.find(
|
||||||
bg_err.code(), paranoid));
|
std::make_tuple(reason, bg_err.code(), paranoid));
|
||||||
if (entry != DefaultErrorSeverityMap.end()) {
|
if (entry != DefaultErrorSeverityMap.end()) {
|
||||||
sev = entry->second;
|
sev = entry->second;
|
||||||
found = true;
|
found = true;
|
||||||
|
|
|
@ -49,8 +49,7 @@ class ErrorHandler {
|
||||||
void EnableAutoRecovery() { auto_recovery_ = true; }
|
void EnableAutoRecovery() { auto_recovery_ = true; }
|
||||||
|
|
||||||
Status::Severity GetErrorSeverity(BackgroundErrorReason reason,
|
Status::Severity GetErrorSeverity(BackgroundErrorReason reason,
|
||||||
Status::Code code,
|
Status::Code code, Status::SubCode subcode);
|
||||||
Status::SubCode subcode);
|
|
||||||
|
|
||||||
const Status& SetBGError(const Status& bg_err, BackgroundErrorReason reason);
|
const Status& SetBGError(const Status& bg_err, BackgroundErrorReason reason);
|
||||||
|
|
||||||
|
|
|
@ -23,7 +23,7 @@ template <class T>
|
||||||
inline T SafeDivide(T a, T b) {
|
inline T SafeDivide(T a, T b) {
|
||||||
return b == 0 ? 0 : a / b;
|
return b == 0 ? 0 : a / b;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) {
|
void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) {
|
||||||
*jwriter << "time_micros"
|
*jwriter << "time_micros"
|
||||||
|
|
|
@ -39,9 +39,9 @@ class EventHelpers {
|
||||||
const std::string& file_checksum,
|
const std::string& file_checksum,
|
||||||
const std::string& file_checksum_func_name);
|
const std::string& file_checksum_func_name);
|
||||||
static void LogAndNotifyTableFileDeletion(
|
static void LogAndNotifyTableFileDeletion(
|
||||||
EventLogger* event_logger, int job_id,
|
EventLogger* event_logger, int job_id, uint64_t file_number,
|
||||||
uint64_t file_number, const std::string& file_path,
|
const std::string& file_path, const Status& status,
|
||||||
const Status& status, const std::string& db_name,
|
const std::string& db_name,
|
||||||
const std::vector<std::shared_ptr<EventListener>>& listeners);
|
const std::vector<std::shared_ptr<EventListener>>& listeners);
|
||||||
static void NotifyOnErrorRecoveryEnd(
|
static void NotifyOnErrorRecoveryEnd(
|
||||||
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
||||||
|
|
|
@ -106,9 +106,8 @@ Status ExternalSstFileIngestionJob::Prepare(
|
||||||
for (IngestedFileInfo& f : files_to_ingest_) {
|
for (IngestedFileInfo& f : files_to_ingest_) {
|
||||||
f.copy_file = false;
|
f.copy_file = false;
|
||||||
const std::string path_outside_db = f.external_file_path;
|
const std::string path_outside_db = f.external_file_path;
|
||||||
const std::string path_inside_db =
|
const std::string path_inside_db = TableFileName(
|
||||||
TableFileName(cfd_->ioptions()->cf_paths, f.fd.GetNumber(),
|
cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
|
||||||
f.fd.GetPathId());
|
|
||||||
if (ingestion_options_.move_files) {
|
if (ingestion_options_.move_files) {
|
||||||
status =
|
status =
|
||||||
fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
|
fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
|
||||||
|
@ -491,7 +490,8 @@ void ExternalSstFileIngestionJob::UpdateStats() {
|
||||||
stream.StartArray();
|
stream.StartArray();
|
||||||
|
|
||||||
for (IngestedFileInfo& f : files_to_ingest_) {
|
for (IngestedFileInfo& f : files_to_ingest_) {
|
||||||
InternalStats::CompactionStats stats(CompactionReason::kExternalSstIngestion, 1);
|
InternalStats::CompactionStats stats(
|
||||||
|
CompactionReason::kExternalSstIngestion, 1);
|
||||||
stats.micros = total_time;
|
stats.micros = total_time;
|
||||||
// If actual copy occurred for this file, then we need to count the file
|
// If actual copy occurred for this file, then we need to count the file
|
||||||
// size as the actual bytes written. If the file was linked, then we ignore
|
// size as the actual bytes written. If the file was linked, then we ignore
|
||||||
|
@ -591,8 +591,8 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
|
||||||
std::unique_ptr<FSRandomAccessFile> sst_file;
|
std::unique_ptr<FSRandomAccessFile> sst_file;
|
||||||
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
||||||
|
|
||||||
status = fs_->NewRandomAccessFile(external_file, env_options_,
|
status =
|
||||||
&sst_file, nullptr);
|
fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
|
||||||
if (!status.ok()) {
|
if (!status.ok()) {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
@ -912,9 +912,8 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
|
||||||
// If the file system does not support random write, then we should not.
|
// If the file system does not support random write, then we should not.
|
||||||
// Otherwise we should.
|
// Otherwise we should.
|
||||||
std::unique_ptr<FSRandomRWFile> rwfile;
|
std::unique_ptr<FSRandomRWFile> rwfile;
|
||||||
Status status =
|
Status status = fs_->NewRandomRWFile(file_to_ingest->internal_file_path,
|
||||||
fs_->NewRandomRWFile(file_to_ingest->internal_file_path, env_options_,
|
env_options_, &rwfile, nullptr);
|
||||||
&rwfile, nullptr);
|
|
||||||
TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::NewRandomRWFile",
|
TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::NewRandomRWFile",
|
||||||
&status);
|
&status);
|
||||||
if (status.ok()) {
|
if (status.ok()) {
|
||||||
|
|
|
@ -301,7 +301,8 @@ TEST_F(ExternalSSTFileTest, Basic) {
|
||||||
|
|
||||||
SstFileWriter sst_file_writer(EnvOptions(), options);
|
SstFileWriter sst_file_writer(EnvOptions(), options);
|
||||||
|
|
||||||
// Current file size should be 0 after sst_file_writer init and before open a file.
|
// Current file size should be 0 after sst_file_writer init and before open
|
||||||
|
// a file.
|
||||||
ASSERT_EQ(sst_file_writer.FileSize(), 0);
|
ASSERT_EQ(sst_file_writer.FileSize(), 0);
|
||||||
|
|
||||||
// file1.sst (0 => 99)
|
// file1.sst (0 => 99)
|
||||||
|
@ -2318,7 +2319,6 @@ TEST_F(ExternalSSTFileTest, SkipBloomFilter) {
|
||||||
table_options.cache_index_and_filter_blocks = true;
|
table_options.cache_index_and_filter_blocks = true;
|
||||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||||
|
|
||||||
|
|
||||||
// Create external SST file and include bloom filters
|
// Create external SST file and include bloom filters
|
||||||
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
|
|
@ -338,8 +338,7 @@ class FaultInjectionTest
|
||||||
FaultInjectionTest::kValExpectNoError));
|
FaultInjectionTest::kValExpectNoError));
|
||||||
}
|
}
|
||||||
|
|
||||||
void NoWriteTestPreFault() {
|
void NoWriteTestPreFault() {}
|
||||||
}
|
|
||||||
|
|
||||||
void NoWriteTestReopenWithFault(ResetMethod reset_method) {
|
void NoWriteTestReopenWithFault(ResetMethod reset_method) {
|
||||||
CloseDB();
|
CloseDB();
|
||||||
|
|
|
@ -8,8 +8,10 @@
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
#include "db/file_indexer.h"
|
#include "db/file_indexer.h"
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <functional>
|
#include <functional>
|
||||||
|
|
||||||
#include "db/version_edit.h"
|
#include "db/version_edit.h"
|
||||||
#include "rocksdb/comparator.h"
|
#include "rocksdb/comparator.h"
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,7 @@
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "memory/arena.h"
|
#include "memory/arena.h"
|
||||||
#include "port/port.h"
|
#include "port/port.h"
|
||||||
#include "util/autovector.h"
|
#include "util/autovector.h"
|
||||||
|
|
|
@ -8,7 +8,9 @@
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
#include "db/file_indexer.h"
|
#include "db/file_indexer.h"
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "db/dbformat.h"
|
#include "db/dbformat.h"
|
||||||
#include "db/version_edit.h"
|
#include "db/version_edit.h"
|
||||||
#include "port/stack_trace.h"
|
#include "port/stack_trace.h"
|
||||||
|
@ -73,8 +75,8 @@ class FileIndexerTest : public testing::Test {
|
||||||
}
|
}
|
||||||
|
|
||||||
void GetNextLevelIndex(const uint32_t level, const uint32_t file_index,
|
void GetNextLevelIndex(const uint32_t level, const uint32_t file_index,
|
||||||
const int cmp_smallest, const int cmp_largest, int32_t* left_index,
|
const int cmp_smallest, const int cmp_largest,
|
||||||
int32_t* right_index) {
|
int32_t* left_index, int32_t* right_index) {
|
||||||
*left_index = 100;
|
*left_index = 100;
|
||||||
*right_index = 100;
|
*right_index = 100;
|
||||||
indexer->GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest,
|
indexer->GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest,
|
||||||
|
|
|
@ -69,8 +69,7 @@ TEST_F(FileNameTest, Parse) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Errors
|
// Errors
|
||||||
static const char* errors[] = {
|
static const char* errors[] = {"",
|
||||||
"",
|
|
||||||
"foo",
|
"foo",
|
||||||
"foo-dx-100.log",
|
"foo-dx-100.log",
|
||||||
".log",
|
".log",
|
||||||
|
@ -96,8 +95,7 @@ TEST_F(FileNameTest, Parse) {
|
||||||
"184467440737095516150.log",
|
"184467440737095516150.log",
|
||||||
"100",
|
"100",
|
||||||
"100.",
|
"100.",
|
||||||
"100.lop"
|
"100.lop"};
|
||||||
};
|
|
||||||
for (unsigned int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
|
for (unsigned int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
|
||||||
std::string f = errors[i];
|
std::string f = errors[i];
|
||||||
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
|
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
|
||||||
|
|
|
@ -136,16 +136,13 @@ FlushJob::FlushJob(
|
||||||
TEST_SYNC_POINT("FlushJob::FlushJob()");
|
TEST_SYNC_POINT("FlushJob::FlushJob()");
|
||||||
}
|
}
|
||||||
|
|
||||||
FlushJob::~FlushJob() {
|
FlushJob::~FlushJob() { ThreadStatusUtil::ResetThreadStatus(); }
|
||||||
ThreadStatusUtil::ResetThreadStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
void FlushJob::ReportStartedFlush() {
|
void FlushJob::ReportStartedFlush() {
|
||||||
ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
|
ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
|
||||||
db_options_.enable_thread_tracking);
|
db_options_.enable_thread_tracking);
|
||||||
ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
|
ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
|
||||||
ThreadStatusUtil::SetThreadOperationProperty(
|
ThreadStatusUtil::SetThreadOperationProperty(ThreadStatus::COMPACTION_JOB_ID,
|
||||||
ThreadStatus::COMPACTION_JOB_ID,
|
|
||||||
job_context_->job_id);
|
job_context_->job_id);
|
||||||
IOSTATS_RESET(bytes_written);
|
IOSTATS_RESET(bytes_written);
|
||||||
}
|
}
|
||||||
|
@ -156,8 +153,7 @@ void FlushJob::ReportFlushInputSize(const autovector<MemTable*>& mems) {
|
||||||
input_size += mem->ApproximateMemoryUsage();
|
input_size += mem->ApproximateMemoryUsage();
|
||||||
}
|
}
|
||||||
ThreadStatusUtil::IncreaseThreadOperationProperty(
|
ThreadStatusUtil::IncreaseThreadOperationProperty(
|
||||||
ThreadStatus::FLUSH_BYTES_MEMTABLES,
|
ThreadStatus::FLUSH_BYTES_MEMTABLES, input_size);
|
||||||
input_size);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void FlushJob::RecordFlushIOStats() {
|
void FlushJob::RecordFlushIOStats() {
|
||||||
|
@ -220,8 +216,7 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,
|
||||||
double mempurge_threshold =
|
double mempurge_threshold =
|
||||||
mutable_cf_options_.experimental_mempurge_threshold;
|
mutable_cf_options_.experimental_mempurge_threshold;
|
||||||
|
|
||||||
AutoThreadOperationStageUpdater stage_run(
|
AutoThreadOperationStageUpdater stage_run(ThreadStatus::STAGE_FLUSH_RUN);
|
||||||
ThreadStatus::STAGE_FLUSH_RUN);
|
|
||||||
if (mems_.empty()) {
|
if (mems_.empty()) {
|
||||||
ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush",
|
ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush",
|
||||||
cfd_->GetName().c_str());
|
cfd_->GetName().c_str());
|
||||||
|
@ -906,8 +901,7 @@ Status FlushJob::WriteLevel0Table() {
|
||||||
}
|
}
|
||||||
const uint64_t current_time = static_cast<uint64_t>(_current_time);
|
const uint64_t current_time = static_cast<uint64_t>(_current_time);
|
||||||
|
|
||||||
uint64_t oldest_key_time =
|
uint64_t oldest_key_time = mems_.front()->ApproximateOldestKeyTime();
|
||||||
mems_.front()->ApproximateOldestKeyTime();
|
|
||||||
|
|
||||||
// It's not clear whether oldest_key_time is always available. In case
|
// It's not clear whether oldest_key_time is always available. In case
|
||||||
// it is not available, use current_time.
|
// it is not available, use current_time.
|
||||||
|
|
|
@ -214,8 +214,8 @@ TEST_F(FlushJobTest, NonEmpty) {
|
||||||
// Note: the first two blob references will not be considered when resolving
|
// Note: the first two blob references will not be considered when resolving
|
||||||
// the oldest blob file referenced (the first one is inlined TTL, while the
|
// the oldest blob file referenced (the first one is inlined TTL, while the
|
||||||
// second one is TTL and thus points to a TTL blob file).
|
// second one is TTL and thus points to a TTL blob file).
|
||||||
constexpr std::array<uint64_t, 6> blob_file_numbers{{
|
constexpr std::array<uint64_t, 6> blob_file_numbers{
|
||||||
kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
|
{kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
|
||||||
for (size_t i = 0; i < blob_file_numbers.size(); ++i) {
|
for (size_t i = 0; i < blob_file_numbers.size(); ++i) {
|
||||||
std::string key(std::to_string(i + 10001));
|
std::string key(std::to_string(i + 10001));
|
||||||
std::string blob_index;
|
std::string blob_index;
|
||||||
|
|
|
@ -104,9 +104,7 @@ class ForwardLevelIterator : public InternalIterator {
|
||||||
status_ = Status::NotSupported("ForwardLevelIterator::Prev()");
|
status_ = Status::NotSupported("ForwardLevelIterator::Prev()");
|
||||||
valid_ = false;
|
valid_ = false;
|
||||||
}
|
}
|
||||||
bool Valid() const override {
|
bool Valid() const override { return valid_; }
|
||||||
return valid_;
|
|
||||||
}
|
|
||||||
void SeekToFirst() override {
|
void SeekToFirst() override {
|
||||||
assert(file_iter_ != nullptr);
|
assert(file_iter_ != nullptr);
|
||||||
if (!status_.ok()) {
|
if (!status_.ok()) {
|
||||||
|
@ -249,9 +247,7 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
|
||||||
immutable_status_.PermitUncheckedError();
|
immutable_status_.PermitUncheckedError();
|
||||||
}
|
}
|
||||||
|
|
||||||
ForwardIterator::~ForwardIterator() {
|
ForwardIterator::~ForwardIterator() { Cleanup(true); }
|
||||||
Cleanup(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv,
|
void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv,
|
||||||
bool background_purge_on_iterator_cleanup) {
|
bool background_purge_on_iterator_cleanup) {
|
||||||
|
@ -284,13 +280,13 @@ struct SVCleanupParams {
|
||||||
SuperVersion* sv;
|
SuperVersion* sv;
|
||||||
bool background_purge_on_iterator_cleanup;
|
bool background_purge_on_iterator_cleanup;
|
||||||
};
|
};
|
||||||
}
|
} // anonymous namespace
|
||||||
|
|
||||||
// Used in PinnedIteratorsManager to release pinned SuperVersion
|
// Used in PinnedIteratorsManager to release pinned SuperVersion
|
||||||
void ForwardIterator::DeferredSVCleanup(void* arg) {
|
void ForwardIterator::DeferredSVCleanup(void* arg) {
|
||||||
auto d = reinterpret_cast<SVCleanupParams*>(arg);
|
auto d = reinterpret_cast<SVCleanupParams*>(arg);
|
||||||
ForwardIterator::SVCleanup(
|
ForwardIterator::SVCleanup(d->db, d->sv,
|
||||||
d->db, d->sv, d->background_purge_on_iterator_cleanup);
|
d->background_purge_on_iterator_cleanup);
|
||||||
delete d;
|
delete d;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -547,8 +543,7 @@ void ForwardIterator::Next() {
|
||||||
assert(valid_);
|
assert(valid_);
|
||||||
bool update_prev_key = false;
|
bool update_prev_key = false;
|
||||||
|
|
||||||
if (sv_ == nullptr ||
|
if (sv_ == nullptr || sv_->version_number != cfd_->GetSuperVersionNumber()) {
|
||||||
sv_->version_number != cfd_->GetSuperVersionNumber()) {
|
|
||||||
std::string current_key = key().ToString();
|
std::string current_key = key().ToString();
|
||||||
Slice old_key(current_key.data(), current_key.size());
|
Slice old_key(current_key.data(), current_key.size());
|
||||||
|
|
||||||
|
@ -578,7 +573,6 @@ void ForwardIterator::Next() {
|
||||||
update_prev_key = true;
|
update_prev_key = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if (update_prev_key) {
|
if (update_prev_key) {
|
||||||
prev_key_.SetInternalKey(current_->key());
|
prev_key_.SetInternalKey(current_->key());
|
||||||
is_prev_set_ = true;
|
is_prev_set_ = true;
|
||||||
|
@ -1043,8 +1037,8 @@ uint32_t ForwardIterator::FindFileInRange(
|
||||||
f->largest.Encode(), k) < 0;
|
f->largest.Encode(), k) < 0;
|
||||||
};
|
};
|
||||||
const auto& b = files.begin();
|
const auto& b = files.begin();
|
||||||
return static_cast<uint32_t>(std::lower_bound(b + left,
|
return static_cast<uint32_t>(
|
||||||
b + right, internal_key, cmp) - b);
|
std::lower_bound(b + left, b + right, internal_key, cmp) - b);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) {
|
void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) {
|
||||||
|
|
|
@ -7,9 +7,9 @@
|
||||||
#include "rocksdb/comparator.h"
|
#include "rocksdb/comparator.h"
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
|
|
||||||
|
#include <queue>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <queue>
|
|
||||||
|
|
||||||
#include "memory/arena.h"
|
#include "memory/arena.h"
|
||||||
#include "rocksdb/db.h"
|
#include "rocksdb/db.h"
|
||||||
|
@ -35,6 +35,7 @@ class MinIterComparator {
|
||||||
bool operator()(InternalIterator* a, InternalIterator* b) {
|
bool operator()(InternalIterator* a, InternalIterator* b) {
|
||||||
return comparator_->Compare(a->key(), b->key()) > 0;
|
return comparator_->Compare(a->key(), b->key()) > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
const CompareInterface* comparator_;
|
const CompareInterface* comparator_;
|
||||||
};
|
};
|
||||||
|
@ -92,8 +93,8 @@ class ForwardIterator : public InternalIterator {
|
||||||
// either done immediately or deferred until this iterator is unpinned by
|
// either done immediately or deferred until this iterator is unpinned by
|
||||||
// PinnedIteratorsManager.
|
// PinnedIteratorsManager.
|
||||||
void SVCleanup();
|
void SVCleanup();
|
||||||
static void SVCleanup(
|
static void SVCleanup(DBImpl* db, SuperVersion* sv,
|
||||||
DBImpl* db, SuperVersion* sv, bool background_purge_on_iterator_cleanup);
|
bool background_purge_on_iterator_cleanup);
|
||||||
static void DeferredSVCleanup(void* arg);
|
static void DeferredSVCleanup(void* arg);
|
||||||
|
|
||||||
void RebuildIterators(bool refresh_sv);
|
void RebuildIterators(bool refresh_sv);
|
||||||
|
@ -107,9 +108,9 @@ class ForwardIterator : public InternalIterator {
|
||||||
void UpdateCurrent();
|
void UpdateCurrent();
|
||||||
bool NeedToSeekImmutable(const Slice& internal_key);
|
bool NeedToSeekImmutable(const Slice& internal_key);
|
||||||
void DeleteCurrentIter();
|
void DeleteCurrentIter();
|
||||||
uint32_t FindFileInRange(
|
uint32_t FindFileInRange(const std::vector<FileMetaData*>& files,
|
||||||
const std::vector<FileMetaData*>& files, const Slice& internal_key,
|
const Slice& internal_key, uint32_t left,
|
||||||
uint32_t left, uint32_t right);
|
uint32_t right);
|
||||||
|
|
||||||
bool IsOverUpperBound(const Slice& internal_key) const;
|
bool IsOverUpperBound(const Slice& internal_key) const;
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,7 @@ int main() {
|
||||||
int main() { return 0; }
|
int main() { return 0; }
|
||||||
#else
|
#else
|
||||||
#include <semaphore.h>
|
#include <semaphore.h>
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <bitset>
|
#include <bitset>
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
@ -281,8 +282,9 @@ struct StatsThread {
|
||||||
}
|
}
|
||||||
auto now = std::chrono::steady_clock::now();
|
auto now = std::chrono::steady_clock::now();
|
||||||
double elapsed =
|
double elapsed =
|
||||||
std::chrono::duration_cast<std::chrono::duration<double> >(
|
std::chrono::duration_cast<std::chrono::duration<double> >(now -
|
||||||
now - tlast).count();
|
tlast)
|
||||||
|
.count();
|
||||||
uint64_t w = ::stats.written.load();
|
uint64_t w = ::stats.written.load();
|
||||||
uint64_t r = ::stats.read.load();
|
uint64_t r = ::stats.read.load();
|
||||||
fprintf(stderr,
|
fprintf(stderr,
|
||||||
|
|
|
@ -228,8 +228,8 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo(
|
||||||
std::unique_ptr<FSRandomAccessFile> sst_file;
|
std::unique_ptr<FSRandomAccessFile> sst_file;
|
||||||
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
||||||
|
|
||||||
status = fs_->NewRandomAccessFile(external_file, env_options_,
|
status =
|
||||||
&sst_file, nullptr);
|
fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
|
||||||
if (!status.ok()) {
|
if (!status.ok()) {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
|
@ -980,13 +980,14 @@ class InternalStats {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool GetIntProperty(const DBPropertyInfo& /*property_info*/, uint64_t* /*value*/,
|
bool GetIntProperty(const DBPropertyInfo& /*property_info*/,
|
||||||
DBImpl* /*db*/) const {
|
uint64_t* /*value*/, DBImpl* /*db*/) const {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool GetIntPropertyOutOfMutex(const DBPropertyInfo& /*property_info*/,
|
bool GetIntPropertyOutOfMutex(const DBPropertyInfo& /*property_info*/,
|
||||||
Version* /*version*/, uint64_t* /*value*/) const {
|
Version* /*version*/,
|
||||||
|
uint64_t* /*value*/) const {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
@ -54,8 +54,7 @@ struct SuperVersionContext {
|
||||||
|
|
||||||
inline bool HaveSomethingToDelete() const {
|
inline bool HaveSomethingToDelete() const {
|
||||||
#ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION
|
#ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION
|
||||||
return !superversions_to_free.empty() ||
|
return !superversions_to_free.empty() || !write_stall_notifications.empty();
|
||||||
!write_stall_notifications.empty();
|
|
||||||
#else
|
#else
|
||||||
return !superversions_to_free.empty();
|
return !superversions_to_free.empty();
|
||||||
#endif
|
#endif
|
||||||
|
@ -77,7 +76,8 @@ struct SuperVersionContext {
|
||||||
(void)new_cond;
|
(void)new_cond;
|
||||||
(void)name;
|
(void)name;
|
||||||
(void)ioptions;
|
(void)ioptions;
|
||||||
#endif // !defined(ROCKSDB_LITE) && !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION)
|
#endif // !defined(ROCKSDB_LITE) &&
|
||||||
|
// !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION)
|
||||||
}
|
}
|
||||||
|
|
||||||
void Clean() {
|
void Clean() {
|
||||||
|
@ -139,8 +139,7 @@ struct JobContext {
|
||||||
CandidateFileInfo(std::string name, std::string path)
|
CandidateFileInfo(std::string name, std::string path)
|
||||||
: file_name(std::move(name)), file_path(std::move(path)) {}
|
: file_name(std::move(name)), file_path(std::move(path)) {}
|
||||||
bool operator==(const CandidateFileInfo& other) const {
|
bool operator==(const CandidateFileInfo& other) const {
|
||||||
return file_name == other.file_name &&
|
return file_name == other.file_name && file_path == other.file_path;
|
||||||
file_path == other.file_path;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -172,9 +172,9 @@ TEST_F(EventListenerTest, OnSingleDBCompactionTest) {
|
||||||
|
|
||||||
TestCompactionListener* listener = new TestCompactionListener(this);
|
TestCompactionListener* listener = new TestCompactionListener(this);
|
||||||
options.listeners.emplace_back(listener);
|
options.listeners.emplace_back(listener);
|
||||||
std::vector<std::string> cf_names = {
|
std::vector<std::string> cf_names = {"pikachu", "ilya", "muromec",
|
||||||
"pikachu", "ilya", "muromec", "dobrynia",
|
"dobrynia", "nikitich", "alyosha",
|
||||||
"nikitich", "alyosha", "popovich"};
|
"popovich"};
|
||||||
CreateAndReopenWithCF(cf_names, options);
|
CreateAndReopenWithCF(cf_names, options);
|
||||||
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
|
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
|
||||||
|
|
||||||
|
@ -214,8 +214,7 @@ class TestFlushListener : public EventListener {
|
||||||
virtual ~TestFlushListener() {
|
virtual ~TestFlushListener() {
|
||||||
prev_fc_info_.status.PermitUncheckedError(); // Ignore the status
|
prev_fc_info_.status.PermitUncheckedError(); // Ignore the status
|
||||||
}
|
}
|
||||||
void OnTableFileCreated(
|
void OnTableFileCreated(const TableFileCreationInfo& info) override {
|
||||||
const TableFileCreationInfo& info) override {
|
|
||||||
// remember the info for later checking the FlushJobInfo.
|
// remember the info for later checking the FlushJobInfo.
|
||||||
prev_fc_info_ = info;
|
prev_fc_info_ = info;
|
||||||
ASSERT_GT(info.db_name.size(), 0U);
|
ASSERT_GT(info.db_name.size(), 0U);
|
||||||
|
@ -250,8 +249,7 @@ class TestFlushListener : public EventListener {
|
||||||
#endif // ROCKSDB_USING_THREAD_STATUS
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
||||||
}
|
}
|
||||||
|
|
||||||
void OnFlushCompleted(
|
void OnFlushCompleted(DB* db, const FlushJobInfo& info) override {
|
||||||
DB* db, const FlushJobInfo& info) override {
|
|
||||||
flushed_dbs_.push_back(db);
|
flushed_dbs_.push_back(db);
|
||||||
flushed_column_family_names_.push_back(info.cf_name);
|
flushed_column_family_names_.push_back(info.cf_name);
|
||||||
if (info.triggered_writes_slowdown) {
|
if (info.triggered_writes_slowdown) {
|
||||||
|
@ -317,9 +315,9 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
|
||||||
#endif // ROCKSDB_USING_THREAD_STATUS
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
||||||
TestFlushListener* listener = new TestFlushListener(options.env, this);
|
TestFlushListener* listener = new TestFlushListener(options.env, this);
|
||||||
options.listeners.emplace_back(listener);
|
options.listeners.emplace_back(listener);
|
||||||
std::vector<std::string> cf_names = {
|
std::vector<std::string> cf_names = {"pikachu", "ilya", "muromec",
|
||||||
"pikachu", "ilya", "muromec", "dobrynia",
|
"dobrynia", "nikitich", "alyosha",
|
||||||
"nikitich", "alyosha", "popovich"};
|
"popovich"};
|
||||||
options.table_properties_collector_factories.push_back(
|
options.table_properties_collector_factories.push_back(
|
||||||
std::make_shared<TestPropertiesCollectorFactory>());
|
std::make_shared<TestPropertiesCollectorFactory>());
|
||||||
CreateAndReopenWithCF(cf_names, options);
|
CreateAndReopenWithCF(cf_names, options);
|
||||||
|
@ -421,9 +419,9 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
|
||||||
listeners.emplace_back(new TestFlushListener(options.env, this));
|
listeners.emplace_back(new TestFlushListener(options.env, this));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<std::string> cf_names = {
|
std::vector<std::string> cf_names = {"pikachu", "ilya", "muromec",
|
||||||
"pikachu", "ilya", "muromec", "dobrynia",
|
"dobrynia", "nikitich", "alyosha",
|
||||||
"nikitich", "alyosha", "popovich"};
|
"popovich"};
|
||||||
|
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
for (int i = 0; i < kNumListeners; ++i) {
|
for (int i = 0; i < kNumListeners; ++i) {
|
||||||
|
@ -452,8 +450,8 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
|
||||||
|
|
||||||
for (int d = 0; d < kNumDBs; ++d) {
|
for (int d = 0; d < kNumDBs; ++d) {
|
||||||
for (size_t c = 0; c < cf_names.size(); ++c) {
|
for (size_t c = 0; c < cf_names.size(); ++c) {
|
||||||
ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c],
|
ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c], cf_names[c],
|
||||||
cf_names[c], cf_names[c]));
|
cf_names[c]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -483,7 +481,6 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
for (auto handles : vec_handles) {
|
for (auto handles : vec_handles) {
|
||||||
for (auto h : handles) {
|
for (auto h : handles) {
|
||||||
delete h;
|
delete h;
|
||||||
|
@ -889,6 +886,7 @@ TEST_F(EventListenerTest, TableFileCreationListenersTest) {
|
||||||
class MemTableSealedListener : public EventListener {
|
class MemTableSealedListener : public EventListener {
|
||||||
private:
|
private:
|
||||||
SequenceNumber latest_seq_number_;
|
SequenceNumber latest_seq_number_;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
MemTableSealedListener() {}
|
MemTableSealedListener() {}
|
||||||
void OnMemTableSealed(const MemTableInfo& info) override {
|
void OnMemTableSealed(const MemTableInfo& info) override {
|
||||||
|
|
|
@ -21,8 +21,7 @@
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
namespace log {
|
namespace log {
|
||||||
|
|
||||||
Reader::Reporter::~Reporter() {
|
Reader::Reporter::~Reporter() {}
|
||||||
}
|
|
||||||
|
|
||||||
Reader::Reader(std::shared_ptr<Logger> info_log,
|
Reader::Reader(std::shared_ptr<Logger> info_log,
|
||||||
std::unique_ptr<SequentialFileReader>&& _file,
|
std::unique_ptr<SequentialFileReader>&& _file,
|
||||||
|
@ -241,8 +240,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
|
||||||
FALLTHROUGH_INTENDED;
|
FALLTHROUGH_INTENDED;
|
||||||
|
|
||||||
case kBadRecordChecksum:
|
case kBadRecordChecksum:
|
||||||
if (recycled_ &&
|
if (recycled_ && wal_recovery_mode ==
|
||||||
wal_recovery_mode ==
|
|
||||||
WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
||||||
scratch->clear();
|
scratch->clear();
|
||||||
return false;
|
return false;
|
||||||
|
@ -297,9 +295,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t Reader::LastRecordOffset() {
|
uint64_t Reader::LastRecordOffset() { return last_record_offset_; }
|
||||||
return last_record_offset_;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t Reader::LastRecordEnd() {
|
uint64_t Reader::LastRecordEnd() {
|
||||||
return end_of_buffer_offset_ - buffer_.size();
|
return end_of_buffer_offset_ - buffer_.size();
|
||||||
|
|
|
@ -85,9 +85,7 @@ class Reader {
|
||||||
uint64_t LastRecordEnd();
|
uint64_t LastRecordEnd();
|
||||||
|
|
||||||
// returns true if the reader has encountered an eof condition.
|
// returns true if the reader has encountered an eof condition.
|
||||||
bool IsEOF() {
|
bool IsEOF() { return eof_; }
|
||||||
return eof_;
|
|
||||||
}
|
|
||||||
|
|
||||||
// returns true if the reader has encountered read error.
|
// returns true if the reader has encountered read error.
|
||||||
bool hasReadError() const { return read_error_; }
|
bool hasReadError() const { return read_error_; }
|
||||||
|
|
|
@ -185,9 +185,7 @@ class LogTest
|
||||||
ASSERT_OK(writer_->AddRecord(Slice(msg)));
|
ASSERT_OK(writer_->AddRecord(Slice(msg)));
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t WrittenBytes() const {
|
size_t WrittenBytes() const { return dest_contents().size(); }
|
||||||
return dest_contents().size();
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string Read(const WALRecoveryMode wal_recovery_mode =
|
std::string Read(const WALRecoveryMode wal_recovery_mode =
|
||||||
WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
||||||
|
@ -235,13 +233,9 @@ class LogTest
|
||||||
source_->force_error_position_ = position;
|
source_->force_error_position_ = position;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t DroppedBytes() const {
|
size_t DroppedBytes() const { return report_.dropped_bytes_; }
|
||||||
return report_.dropped_bytes_;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string ReportMessage() const {
|
std::string ReportMessage() const { return report_.message_; }
|
||||||
return report_.message_;
|
|
||||||
}
|
|
||||||
|
|
||||||
void ForceEOF(size_t position = 0) {
|
void ForceEOF(size_t position = 0) {
|
||||||
source_->force_eof_ = true;
|
source_->force_eof_ = true;
|
||||||
|
|
|
@ -58,6 +58,5 @@ class LogsWithPrepTracker {
|
||||||
// both logs_with_prep_ and prepared_section_completed_.
|
// both logs_with_prep_ and prepared_section_completed_.
|
||||||
std::unordered_map<uint64_t, uint64_t> prepared_section_completed_;
|
std::unordered_map<uint64_t, uint64_t> prepared_section_completed_;
|
||||||
std::mutex prepared_section_completed_mutex_;
|
std::mutex prepared_section_completed_mutex_;
|
||||||
|
|
||||||
};
|
};
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
|
@ -10,6 +10,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
|
||||||
#include "rocksdb/slice.h"
|
#include "rocksdb/slice.h"
|
||||||
#include "rocksdb/types.h"
|
#include "rocksdb/types.h"
|
||||||
|
|
||||||
|
|
|
@ -10,9 +10,10 @@
|
||||||
#include "db/malloc_stats.h"
|
#include "db/malloc_stats.h"
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
#include <memory>
|
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
#include "port/jemalloc_helper.h"
|
#include "port/jemalloc_helper.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
|
@ -42,9 +42,7 @@ std::string Key1(int i) {
|
||||||
return buf;
|
return buf;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string Key2(int i) {
|
std::string Key2(int i) { return Key1(i) + "_xxx"; }
|
||||||
return Key1(i) + "_xxx";
|
|
||||||
}
|
|
||||||
|
|
||||||
class ManualCompactionTest : public testing::Test {
|
class ManualCompactionTest : public testing::Test {
|
||||||
public:
|
public:
|
||||||
|
|
|
@ -330,9 +330,8 @@ int MemTable::KeyComparator::operator()(const char* prefix_len_key1,
|
||||||
return comparator.CompareKeySeq(k1, k2);
|
return comparator.CompareKeySeq(k1, k2);
|
||||||
}
|
}
|
||||||
|
|
||||||
int MemTable::KeyComparator::operator()(const char* prefix_len_key,
|
int MemTable::KeyComparator::operator()(
|
||||||
const KeyComparator::DecodedType& key)
|
const char* prefix_len_key, const KeyComparator::DecodedType& key) const {
|
||||||
const {
|
|
||||||
// Internal keys are encoded as length-prefixed strings.
|
// Internal keys are encoded as length-prefixed strings.
|
||||||
Slice a = GetLengthPrefixedSlice(prefix_len_key);
|
Slice a = GetLengthPrefixedSlice(prefix_len_key);
|
||||||
return comparator.CompareKeySeq(a, key);
|
return comparator.CompareKeySeq(a, key);
|
||||||
|
@ -914,7 +913,7 @@ struct Saver {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
static bool SaveValue(void* arg, const char* entry) {
|
static bool SaveValue(void* arg, const char* entry) {
|
||||||
TEST_SYNC_POINT_CALLBACK("Memtable::SaveValue:Begin:entry", &entry);
|
TEST_SYNC_POINT_CALLBACK("Memtable::SaveValue:Begin:entry", &entry);
|
||||||
|
|
|
@ -448,9 +448,7 @@ class MemTable {
|
||||||
// persisted.
|
// persisted.
|
||||||
// REQUIRES: external synchronization to prevent simultaneous
|
// REQUIRES: external synchronization to prevent simultaneous
|
||||||
// operations on the same MemTable.
|
// operations on the same MemTable.
|
||||||
void MarkFlushed() {
|
void MarkFlushed() { table_->MarkFlushed(); }
|
||||||
table_->MarkFlushed();
|
|
||||||
}
|
|
||||||
|
|
||||||
// return true if the current MemTableRep supports merge operator.
|
// return true if the current MemTableRep supports merge operator.
|
||||||
bool IsMergeOperatorSupported() const {
|
bool IsMergeOperatorSupported() const {
|
||||||
|
|
|
@ -4,9 +4,11 @@
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
#include "db/memtable_list.h"
|
#include "db/memtable_list.h"
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "db/merge_context.h"
|
#include "db/merge_context.h"
|
||||||
#include "db/version_set.h"
|
#include "db/version_set.h"
|
||||||
#include "db/write_controller.h"
|
#include "db/write_controller.h"
|
||||||
|
|
|
@ -8,6 +8,7 @@
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "rocksdb/slice.h"
|
#include "rocksdb/slice.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
|
@ -74,8 +74,7 @@ bool AssociativeMergeOperator::FullMergeV2(
|
||||||
|
|
||||||
// Call the user defined simple merge on the operands;
|
// Call the user defined simple merge on the operands;
|
||||||
// NOTE: It is assumed that the client's merge-operator will handle any errors.
|
// NOTE: It is assumed that the client's merge-operator will handle any errors.
|
||||||
bool AssociativeMergeOperator::PartialMerge(
|
bool AssociativeMergeOperator::PartialMerge(const Slice& key,
|
||||||
const Slice& key,
|
|
||||||
const Slice& left_operand,
|
const Slice& left_operand,
|
||||||
const Slice& right_operand,
|
const Slice& right_operand,
|
||||||
std::string* new_value,
|
std::string* new_value,
|
||||||
|
|
|
@ -10,10 +10,12 @@
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
|
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "db/db_impl/db_impl.h"
|
#include "db/db_impl/db_impl.h"
|
||||||
#include "db/db_test_util.h"
|
#include "db/db_test_util.h"
|
||||||
#include "db/version_set.h"
|
#include "db/version_set.h"
|
||||||
|
@ -28,7 +30,6 @@
|
||||||
#include "test_util/testutil.h"
|
#include "test_util/testutil.h"
|
||||||
#include "util/string_util.h"
|
#include "util/string_util.h"
|
||||||
|
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
class ObsoleteFilesTest : public DBTestBase {
|
class ObsoleteFilesTest : public DBTestBase {
|
||||||
|
|
|
@ -59,7 +59,7 @@ void VerifyOptionsFileName(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(OptionsFileTest, NumberOfOptionsFiles) {
|
TEST_F(OptionsFileTest, NumberOfOptionsFiles) {
|
||||||
const int kReopenCount = 20;
|
const int kReopenCount = 20;
|
||||||
|
|
|
@ -103,8 +103,9 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (FLAGS_verbose) {
|
if (FLAGS_verbose) {
|
||||||
std::cout << "Get user key comparison: \n" << hist_get.ToString()
|
std::cout << "Get user key comparison: \n"
|
||||||
<< "Get time: \n" << hist_get_time.ToString();
|
<< hist_get.ToString() << "Get time: \n"
|
||||||
|
<< hist_get_time.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
|
@ -139,7 +140,8 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
|
||||||
hist_seek.Add(get_perf_context()->user_key_comparison_count);
|
hist_seek.Add(get_perf_context()->user_key_comparison_count);
|
||||||
if (FLAGS_verbose) {
|
if (FLAGS_verbose) {
|
||||||
std::cout << "seek cmp: " << get_perf_context()->user_key_comparison_count
|
std::cout << "seek cmp: " << get_perf_context()->user_key_comparison_count
|
||||||
<< " ikey skipped " << get_perf_context()->internal_key_skipped_count
|
<< " ikey skipped "
|
||||||
|
<< get_perf_context()->internal_key_skipped_count
|
||||||
<< " idelete skipped "
|
<< " idelete skipped "
|
||||||
<< get_perf_context()->internal_delete_skipped_count
|
<< get_perf_context()->internal_delete_skipped_count
|
||||||
<< " elapsed: " << elapsed_nanos << "ns\n";
|
<< " elapsed: " << elapsed_nanos << "ns\n";
|
||||||
|
@ -322,7 +324,8 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
|
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
|
||||||
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
|
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
|
||||||
hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
|
hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
|
||||||
hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
|
hist_mget_num_memtable_checked.Add(
|
||||||
|
get_perf_context()->get_from_memtable_count);
|
||||||
hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
|
hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
|
||||||
hist_mget.Add(get_perf_context()->user_key_comparison_count);
|
hist_mget.Add(get_perf_context()->user_key_comparison_count);
|
||||||
}
|
}
|
||||||
|
@ -337,12 +340,14 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
<< hist_write_wal_time.ToString() << "\n"
|
<< hist_write_wal_time.ToString() << "\n"
|
||||||
<< " Writing Mem Table time: \n"
|
<< " Writing Mem Table time: \n"
|
||||||
<< hist_write_memtable_time.ToString() << "\n"
|
<< hist_write_memtable_time.ToString() << "\n"
|
||||||
<< " Write Delay: \n" << hist_write_delay_time.ToString() << "\n"
|
<< " Write Delay: \n"
|
||||||
|
<< hist_write_delay_time.ToString() << "\n"
|
||||||
<< " Waiting for Batch time: \n"
|
<< " Waiting for Batch time: \n"
|
||||||
<< hist_write_thread_wait_nanos.ToString() << "\n"
|
<< hist_write_thread_wait_nanos.ToString() << "\n"
|
||||||
<< " Scheduling Flushes and Compactions Time: \n"
|
<< " Scheduling Flushes and Compactions Time: \n"
|
||||||
<< hist_write_scheduling_time.ToString() << "\n"
|
<< hist_write_scheduling_time.ToString() << "\n"
|
||||||
<< " Total DB mutex nanos: \n" << total_db_mutex_nanos << "\n";
|
<< " Total DB mutex nanos: \n"
|
||||||
|
<< total_db_mutex_nanos << "\n";
|
||||||
|
|
||||||
std::cout << "Get(): Time to get snapshot: \n"
|
std::cout << "Get(): Time to get snapshot: \n"
|
||||||
<< hist_get_snapshot.ToString()
|
<< hist_get_snapshot.ToString()
|
||||||
|
@ -352,8 +357,8 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
<< hist_get_files.ToString() << "\n"
|
<< hist_get_files.ToString() << "\n"
|
||||||
<< " Number of memtables checked: \n"
|
<< " Number of memtables checked: \n"
|
||||||
<< hist_num_memtable_checked.ToString() << "\n"
|
<< hist_num_memtable_checked.ToString() << "\n"
|
||||||
<< " Time to post process: \n" << hist_get_post_process.ToString()
|
<< " Time to post process: \n"
|
||||||
<< "\n";
|
<< hist_get_post_process.ToString() << "\n";
|
||||||
|
|
||||||
std::cout << "MultiGet(): Time to get snapshot: \n"
|
std::cout << "MultiGet(): Time to get snapshot: \n"
|
||||||
<< hist_mget_snapshot.ToString()
|
<< hist_mget_snapshot.ToString()
|
||||||
|
@ -440,7 +445,8 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
|
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
|
||||||
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
|
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
|
||||||
hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
|
hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
|
||||||
hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
|
hist_mget_num_memtable_checked.Add(
|
||||||
|
get_perf_context()->get_from_memtable_count);
|
||||||
hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
|
hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
|
||||||
hist_mget.Add(get_perf_context()->user_key_comparison_count);
|
hist_mget.Add(get_perf_context()->user_key_comparison_count);
|
||||||
}
|
}
|
||||||
|
@ -459,8 +465,8 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
<< hist_get_files.ToString() << "\n"
|
<< hist_get_files.ToString() << "\n"
|
||||||
<< " Number of memtables checked: \n"
|
<< " Number of memtables checked: \n"
|
||||||
<< hist_num_memtable_checked.ToString() << "\n"
|
<< hist_num_memtable_checked.ToString() << "\n"
|
||||||
<< " Time to post process: \n" << hist_get_post_process.ToString()
|
<< " Time to post process: \n"
|
||||||
<< "\n";
|
<< hist_get_post_process.ToString() << "\n";
|
||||||
|
|
||||||
std::cout << "ReadOnly MultiGet(): Time to get snapshot: \n"
|
std::cout << "ReadOnly MultiGet(): Time to get snapshot: \n"
|
||||||
<< hist_mget_snapshot.ToString()
|
<< hist_mget_snapshot.ToString()
|
||||||
|
@ -556,7 +562,8 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (FLAGS_verbose) {
|
if (FLAGS_verbose) {
|
||||||
std::cout << "Put time:\n" << hist_put_time.ToString() << "WAL time:\n"
|
std::cout << "Put time:\n"
|
||||||
|
<< hist_put_time.ToString() << "WAL time:\n"
|
||||||
<< hist_wal_time.ToString() << "time diff:\n"
|
<< hist_wal_time.ToString() << "time diff:\n"
|
||||||
<< hist_time_diff.ToString();
|
<< hist_time_diff.ToString();
|
||||||
}
|
}
|
||||||
|
@ -584,7 +591,8 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
|
||||||
}
|
}
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
if (FLAGS_verbose) {
|
if (FLAGS_verbose) {
|
||||||
std::cout << "Seek:\n" << hist_seek.ToString() << "Next:\n"
|
std::cout << "Seek:\n"
|
||||||
|
<< hist_seek.ToString() << "Next:\n"
|
||||||
<< hist_next.ToString();
|
<< hist_next.ToString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -806,14 +814,18 @@ TEST_F(PerfContextTest, PerfContextByLevelGetSet) {
|
||||||
.bloom_filter_full_positive);
|
.bloom_filter_full_positive);
|
||||||
ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[2]
|
ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[2]
|
||||||
.bloom_filter_full_true_positive);
|
.bloom_filter_full_true_positive);
|
||||||
ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[0]
|
ASSERT_EQ(
|
||||||
.block_cache_hit_count);
|
1,
|
||||||
ASSERT_EQ(5, (*(get_perf_context()->level_to_perf_context))[2]
|
(*(get_perf_context()->level_to_perf_context))[0].block_cache_hit_count);
|
||||||
.block_cache_hit_count);
|
ASSERT_EQ(
|
||||||
ASSERT_EQ(2, (*(get_perf_context()->level_to_perf_context))[3]
|
5,
|
||||||
.block_cache_miss_count);
|
(*(get_perf_context()->level_to_perf_context))[2].block_cache_hit_count);
|
||||||
ASSERT_EQ(4, (*(get_perf_context()->level_to_perf_context))[1]
|
ASSERT_EQ(
|
||||||
.block_cache_miss_count);
|
2,
|
||||||
|
(*(get_perf_context()->level_to_perf_context))[3].block_cache_miss_count);
|
||||||
|
ASSERT_EQ(
|
||||||
|
4,
|
||||||
|
(*(get_perf_context()->level_to_perf_context))[1].block_cache_miss_count);
|
||||||
std::string zero_excluded = get_perf_context()->ToString(true);
|
std::string zero_excluded = get_perf_context()->ToString(true);
|
||||||
ASSERT_NE(std::string::npos,
|
ASSERT_NE(std::string::npos,
|
||||||
zero_excluded.find("bloom_filter_useful = 1@level5, 2@level7"));
|
zero_excluded.find("bloom_filter_useful = 1@level5, 2@level7"));
|
||||||
|
|
|
@ -38,7 +38,6 @@
|
||||||
#include "util/string_util.h"
|
#include "util/string_util.h"
|
||||||
#include "utilities/merge_operators.h"
|
#include "utilities/merge_operators.h"
|
||||||
|
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
class PlainTableKeyDecoderTest : public testing::Test {};
|
class PlainTableKeyDecoderTest : public testing::Test {};
|
||||||
|
|
||||||
|
@ -148,9 +147,7 @@ class PlainTableDBTest : public testing::Test,
|
||||||
|
|
||||||
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
|
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
|
||||||
|
|
||||||
void Reopen(Options* options = nullptr) {
|
void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); }
|
||||||
ASSERT_OK(TryReopen(options));
|
|
||||||
}
|
|
||||||
|
|
||||||
void Close() {
|
void Close() {
|
||||||
delete db_;
|
delete db_;
|
||||||
|
@ -200,9 +197,7 @@ class PlainTableDBTest : public testing::Test,
|
||||||
return db_->Put(WriteOptions(), k, v);
|
return db_->Put(WriteOptions(), k, v);
|
||||||
}
|
}
|
||||||
|
|
||||||
Status Delete(const std::string& k) {
|
Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); }
|
||||||
return db_->Delete(WriteOptions(), k);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
|
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
|
||||||
ReadOptions options;
|
ReadOptions options;
|
||||||
|
@ -217,7 +212,6 @@ class PlainTableDBTest : public testing::Test,
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
int NumTableFilesAtLevel(int level) {
|
int NumTableFilesAtLevel(int level) {
|
||||||
std::string property;
|
std::string property;
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty(
|
||||||
|
@ -503,15 +497,16 @@ TEST_P(PlainTableDBTest, Flush) {
|
||||||
ASSERT_GT(int_num, 0U);
|
ASSERT_GT(int_num, 0U);
|
||||||
|
|
||||||
TablePropertiesCollection ptc;
|
TablePropertiesCollection ptc;
|
||||||
ASSERT_OK(
|
ASSERT_OK(reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(
|
||||||
reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
|
&ptc));
|
||||||
ASSERT_EQ(1U, ptc.size());
|
ASSERT_EQ(1U, ptc.size());
|
||||||
auto row = ptc.begin();
|
auto row = ptc.begin();
|
||||||
auto tp = row->second;
|
auto tp = row->second;
|
||||||
|
|
||||||
if (full_scan_mode) {
|
if (full_scan_mode) {
|
||||||
// Does not support Get/Seek
|
// Does not support Get/Seek
|
||||||
std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ReadOptions()));
|
std::unique_ptr<Iterator> iter(
|
||||||
|
dbfull()->NewIterator(ReadOptions()));
|
||||||
iter->SeekToFirst();
|
iter->SeekToFirst();
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ("0000000000000bar", iter->key().ToString());
|
ASSERT_EQ("0000000000000bar", iter->key().ToString());
|
||||||
|
@ -863,7 +858,7 @@ namespace {
|
||||||
std::string MakeLongKey(size_t length, char c) {
|
std::string MakeLongKey(size_t length, char c) {
|
||||||
return std::string(length, c);
|
return std::string(length, c);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(PlainTableDBTest, IteratorLargeKeys) {
|
TEST_P(PlainTableDBTest, IteratorLargeKeys) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
|
@ -878,15 +873,10 @@ TEST_P(PlainTableDBTest, IteratorLargeKeys) {
|
||||||
options.prefix_extractor.reset();
|
options.prefix_extractor.reset();
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
|
|
||||||
std::string key_list[] = {
|
std::string key_list[] = {MakeLongKey(30, '0'), MakeLongKey(16, '1'),
|
||||||
MakeLongKey(30, '0'),
|
MakeLongKey(32, '2'), MakeLongKey(60, '3'),
|
||||||
MakeLongKey(16, '1'),
|
MakeLongKey(90, '4'), MakeLongKey(50, '5'),
|
||||||
MakeLongKey(32, '2'),
|
MakeLongKey(26, '6')};
|
||||||
MakeLongKey(60, '3'),
|
|
||||||
MakeLongKey(90, '4'),
|
|
||||||
MakeLongKey(50, '5'),
|
|
||||||
MakeLongKey(26, '6')
|
|
||||||
};
|
|
||||||
|
|
||||||
for (size_t i = 0; i < 7; i++) {
|
for (size_t i = 0; i < 7; i++) {
|
||||||
ASSERT_OK(Put(key_list[i], std::to_string(i)));
|
ASSERT_OK(Put(key_list[i], std::to_string(i)));
|
||||||
|
@ -913,7 +903,7 @@ namespace {
|
||||||
std::string MakeLongKeyWithPrefix(size_t length, char c) {
|
std::string MakeLongKeyWithPrefix(size_t length, char c) {
|
||||||
return "00000000" + std::string(length - 8, c);
|
return "00000000" + std::string(length - 8, c);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
|
TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
|
@ -1315,8 +1305,7 @@ TEST_P(PlainTableDBTest, AdaptiveTable) {
|
||||||
options.create_if_missing = false;
|
options.create_if_missing = false;
|
||||||
std::shared_ptr<TableFactory> block_based_factory(
|
std::shared_ptr<TableFactory> block_based_factory(
|
||||||
NewBlockBasedTableFactory());
|
NewBlockBasedTableFactory());
|
||||||
std::shared_ptr<TableFactory> plain_table_factory(
|
std::shared_ptr<TableFactory> plain_table_factory(NewPlainTableFactory());
|
||||||
NewPlainTableFactory());
|
|
||||||
std::shared_ptr<TableFactory> dummy_factory;
|
std::shared_ptr<TableFactory> dummy_factory;
|
||||||
options.table_factory.reset(NewAdaptiveTableFactory(
|
options.table_factory.reset(NewAdaptiveTableFactory(
|
||||||
block_based_factory, block_based_factory, plain_table_factory));
|
block_based_factory, block_based_factory, plain_table_factory));
|
||||||
|
|
|
@ -77,13 +77,11 @@ inline Slice TestKeyToSlice(std::string &s, const TestKey& test_key) {
|
||||||
}
|
}
|
||||||
|
|
||||||
inline const TestKey SliceToTestKey(const Slice& slice) {
|
inline const TestKey SliceToTestKey(const Slice& slice) {
|
||||||
return TestKey(DecodeFixed64(slice.data()),
|
return TestKey(DecodeFixed64(slice.data()), DecodeFixed64(slice.data() + 8));
|
||||||
DecodeFixed64(slice.data() + 8));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
class TestKeyComparator : public Comparator {
|
class TestKeyComparator : public Comparator {
|
||||||
public:
|
public:
|
||||||
|
|
||||||
// Compare needs to be aware of the possibility of a and/or b is
|
// Compare needs to be aware of the possibility of a and/or b is
|
||||||
// prefix only
|
// prefix only
|
||||||
int Compare(const Slice& a, const Slice& b) const override {
|
int Compare(const Slice& a, const Slice& b) const override {
|
||||||
|
@ -215,7 +213,7 @@ class SamePrefixTransform : public SliceTransform {
|
||||||
bool FullLengthEnabled(size_t* /*len*/) const override { return false; }
|
bool FullLengthEnabled(size_t* /*len*/) const override { return false; }
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class PrefixTest : public testing::Test {
|
class PrefixTest : public testing::Test {
|
||||||
public:
|
public:
|
||||||
|
@ -244,9 +242,7 @@ class PrefixTest : public testing::Test {
|
||||||
return std::shared_ptr<DB>(db);
|
return std::shared_ptr<DB>(db);
|
||||||
}
|
}
|
||||||
|
|
||||||
void FirstOption() {
|
void FirstOption() { option_config_ = kBegin; }
|
||||||
option_config_ = kBegin;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool NextOptions(int bucket_count) {
|
bool NextOptions(int bucket_count) {
|
||||||
// skip some options
|
// skip some options
|
||||||
|
@ -350,8 +346,7 @@ TEST_F(PrefixTest, TestResult) {
|
||||||
FirstOption();
|
FirstOption();
|
||||||
while (NextOptions(num_buckets)) {
|
while (NextOptions(num_buckets)) {
|
||||||
std::cout << "*** Mem table: " << options.memtable_factory->Name()
|
std::cout << "*** Mem table: " << options.memtable_factory->Name()
|
||||||
<< " number of buckets: " << num_buckets
|
<< " number of buckets: " << num_buckets << std::endl;
|
||||||
<< std::endl;
|
|
||||||
ASSERT_OK(DestroyDB(kDbName, Options()));
|
ASSERT_OK(DestroyDB(kDbName, Options()));
|
||||||
auto db = OpenDb();
|
auto db = OpenDb();
|
||||||
WriteOptions write_options;
|
WriteOptions write_options;
|
||||||
|
@ -615,8 +610,9 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::cout << "Put key comparison: \n" << hist_put_comparison.ToString()
|
std::cout << "Put key comparison: \n"
|
||||||
<< "Put time: \n" << hist_put_time.ToString();
|
<< hist_put_comparison.ToString() << "Put time: \n"
|
||||||
|
<< hist_put_time.ToString();
|
||||||
|
|
||||||
// test seek existing keys
|
// test seek existing keys
|
||||||
HistogramImpl hist_seek_time;
|
HistogramImpl hist_seek_time;
|
||||||
|
@ -635,8 +631,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
auto key_prefix = options.prefix_extractor->Transform(key);
|
auto key_prefix = options.prefix_extractor->Transform(key);
|
||||||
uint64_t total_keys = 0;
|
uint64_t total_keys = 0;
|
||||||
for (iter->Seek(key);
|
for (iter->Seek(key);
|
||||||
iter->Valid() && iter->key().starts_with(key_prefix);
|
iter->Valid() && iter->key().starts_with(key_prefix); iter->Next()) {
|
||||||
iter->Next()) {
|
|
||||||
if (FLAGS_trigger_deadlock) {
|
if (FLAGS_trigger_deadlock) {
|
||||||
std::cout << "Behold the deadlock!\n";
|
std::cout << "Behold the deadlock!\n";
|
||||||
db->Delete(write_options, iter->key());
|
db->Delete(write_options, iter->key());
|
||||||
|
@ -645,12 +640,12 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
}
|
}
|
||||||
hist_seek_time.Add(timer.ElapsedNanos());
|
hist_seek_time.Add(timer.ElapsedNanos());
|
||||||
hist_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
|
hist_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
|
||||||
ASSERT_EQ(total_keys, FLAGS_items_per_prefix - FLAGS_items_per_prefix/2);
|
ASSERT_EQ(total_keys,
|
||||||
|
FLAGS_items_per_prefix - FLAGS_items_per_prefix / 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::cout << "Seek key comparison: \n"
|
std::cout << "Seek key comparison: \n"
|
||||||
<< hist_seek_comparison.ToString()
|
<< hist_seek_comparison.ToString() << "Seek time: \n"
|
||||||
<< "Seek time: \n"
|
|
||||||
<< hist_seek_time.ToString();
|
<< hist_seek_time.ToString();
|
||||||
|
|
||||||
// test non-existing keys
|
// test non-existing keys
|
||||||
|
@ -658,8 +653,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
HistogramImpl hist_no_seek_comparison;
|
HistogramImpl hist_no_seek_comparison;
|
||||||
|
|
||||||
for (auto prefix = FLAGS_total_prefixes;
|
for (auto prefix = FLAGS_total_prefixes;
|
||||||
prefix < FLAGS_total_prefixes + 10000;
|
prefix < FLAGS_total_prefixes + 10000; prefix++) {
|
||||||
prefix++) {
|
|
||||||
TestKey test_key(prefix, 0);
|
TestKey test_key(prefix, 0);
|
||||||
std::string s;
|
std::string s;
|
||||||
Slice key = TestKeyToSlice(s, test_key);
|
Slice key = TestKeyToSlice(s, test_key);
|
||||||
|
@ -668,7 +662,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
StopWatchNano timer(SystemClock::Default().get(), true);
|
StopWatchNano timer(SystemClock::Default().get(), true);
|
||||||
iter->Seek(key);
|
iter->Seek(key);
|
||||||
hist_no_seek_time.Add(timer.ElapsedNanos());
|
hist_no_seek_time.Add(timer.ElapsedNanos());
|
||||||
hist_no_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
|
hist_no_seek_comparison.Add(
|
||||||
|
get_perf_context()->user_key_comparison_count);
|
||||||
ASSERT_TRUE(!iter->Valid());
|
ASSERT_TRUE(!iter->Valid());
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
}
|
}
|
||||||
|
|
|
@ -502,7 +502,7 @@ class TruncatedRangeDelMergingIter : public InternalIterator {
|
||||||
size_t ts_sz_;
|
size_t ts_sz_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator>
|
std::unique_ptr<FragmentedRangeTombstoneIterator>
|
||||||
CompactionRangeDelAggregator::NewIterator(const Slice* lower_bound,
|
CompactionRangeDelAggregator::NewIterator(const Slice* lower_bound,
|
||||||
|
|
|
@ -192,7 +192,7 @@ void VerifyFragmentedRangeDels(
|
||||||
EXPECT_FALSE(iter->Valid());
|
EXPECT_FALSE(iter->Valid());
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorTest, EmptyTruncatedIter) {
|
TEST_F(RangeDelAggregatorTest, EmptyTruncatedIter) {
|
||||||
auto range_del_iter = MakeRangeDelIter({});
|
auto range_del_iter = MakeRangeDelIter({});
|
||||||
|
|
13
db/repair.cc
13
db/repair.cc
|
@ -332,7 +332,8 @@ class Repairer {
|
||||||
void ConvertLogFilesToTables() {
|
void ConvertLogFilesToTables() {
|
||||||
const auto& wal_dir = immutable_db_options_.GetWalDir();
|
const auto& wal_dir = immutable_db_options_.GetWalDir();
|
||||||
for (size_t i = 0; i < logs_.size(); i++) {
|
for (size_t i = 0; i < logs_.size(); i++) {
|
||||||
// we should use LogFileName(wal_dir, logs_[i]) here. user might uses wal_dir option.
|
// we should use LogFileName(wal_dir, logs_[i]) here. user might uses
|
||||||
|
// wal_dir option.
|
||||||
std::string logname = LogFileName(wal_dir, logs_[i]);
|
std::string logname = LogFileName(wal_dir, logs_[i]);
|
||||||
Status status = ConvertLogToTable(wal_dir, logs_[i]);
|
Status status = ConvertLogToTable(wal_dir, logs_[i]);
|
||||||
if (!status.ok()) {
|
if (!status.ok()) {
|
||||||
|
@ -393,8 +394,8 @@ class Repairer {
|
||||||
int counter = 0;
|
int counter = 0;
|
||||||
while (reader.ReadRecord(&record, &scratch)) {
|
while (reader.ReadRecord(&record, &scratch)) {
|
||||||
if (record.size() < WriteBatchInternal::kHeader) {
|
if (record.size() < WriteBatchInternal::kHeader) {
|
||||||
reporter.Corruption(
|
reporter.Corruption(record.size(),
|
||||||
record.size(), Status::Corruption("log record too small"));
|
Status::Corruption("log record too small"));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
Status record_status = WriteBatchInternal::SetContents(&batch, record);
|
Status record_status = WriteBatchInternal::SetContents(&batch, record);
|
||||||
|
@ -715,8 +716,7 @@ Status GetDefaultCFOptions(
|
||||||
} // anonymous namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
Status RepairDB(const std::string& dbname, const DBOptions& db_options,
|
Status RepairDB(const std::string& dbname, const DBOptions& db_options,
|
||||||
const std::vector<ColumnFamilyDescriptor>& column_families
|
const std::vector<ColumnFamilyDescriptor>& column_families) {
|
||||||
) {
|
|
||||||
ColumnFamilyOptions default_cf_opts;
|
ColumnFamilyOptions default_cf_opts;
|
||||||
Status status = GetDefaultCFOptions(column_families, &default_cf_opts);
|
Status status = GetDefaultCFOptions(column_families, &default_cf_opts);
|
||||||
if (!status.ok()) {
|
if (!status.ok()) {
|
||||||
|
@ -756,8 +756,7 @@ Status RepairDB(const std::string& dbname, const Options& options) {
|
||||||
DBOptions db_options(opts);
|
DBOptions db_options(opts);
|
||||||
ColumnFamilyOptions cf_options(opts);
|
ColumnFamilyOptions cf_options(opts);
|
||||||
|
|
||||||
Repairer repairer(dbname, db_options,
|
Repairer repairer(dbname, db_options, {}, cf_options /* default_cf_opts */,
|
||||||
{}, cf_options /* default_cf_opts */,
|
|
||||||
cf_options /* unknown_cf_opts */,
|
cf_options /* unknown_cf_opts */,
|
||||||
true /* create_unknown_cfs */);
|
true /* create_unknown_cfs */);
|
||||||
Status status = repairer.Run();
|
Status status = repairer.Run();
|
||||||
|
|
|
@ -387,8 +387,7 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
|
||||||
ASSERT_EQ(fname_to_props.size(), 2U);
|
ASSERT_EQ(fname_to_props.size(), 2U);
|
||||||
for (const auto& fname_and_props : fname_to_props) {
|
for (const auto& fname_and_props : fname_to_props) {
|
||||||
std::string comparator_name(rev_opts.comparator->Name());
|
std::string comparator_name(rev_opts.comparator->Name());
|
||||||
ASSERT_EQ(comparator_name,
|
ASSERT_EQ(comparator_name, fname_and_props.second->comparator_name);
|
||||||
fname_and_props.second->comparator_name);
|
|
||||||
}
|
}
|
||||||
Close();
|
Close();
|
||||||
|
|
||||||
|
|
|
@ -3,14 +3,13 @@
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
#include "rocksdb/snapshot.h"
|
|
||||||
|
|
||||||
#include "rocksdb/db.h"
|
#include "rocksdb/db.h"
|
||||||
|
#include "rocksdb/snapshot.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
ManagedSnapshot::ManagedSnapshot(DB* db) : db_(db),
|
ManagedSnapshot::ManagedSnapshot(DB* db)
|
||||||
snapshot_(db->GetSnapshot()) {}
|
: db_(db), snapshot_(db->GetSnapshot()) {}
|
||||||
|
|
||||||
ManagedSnapshot::ManagedSnapshot(DB* db, const Snapshot* _snapshot)
|
ManagedSnapshot::ManagedSnapshot(DB* db, const Snapshot* _snapshot)
|
||||||
: db_(db), snapshot_(_snapshot) {}
|
: db_(db), snapshot_(_snapshot) {}
|
||||||
|
|
|
@ -72,8 +72,14 @@ class SnapshotList {
|
||||||
assert(list_.next_ != &list_ || 0 == count_);
|
assert(list_.next_ != &list_ || 0 == count_);
|
||||||
return list_.next_ == &list_;
|
return list_.next_ == &list_;
|
||||||
}
|
}
|
||||||
SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; }
|
SnapshotImpl* oldest() const {
|
||||||
SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; }
|
assert(!empty());
|
||||||
|
return list_.next_;
|
||||||
|
}
|
||||||
|
SnapshotImpl* newest() const {
|
||||||
|
assert(!empty());
|
||||||
|
return list_.prev_;
|
||||||
|
}
|
||||||
|
|
||||||
SnapshotImpl* New(SnapshotImpl* s, SequenceNumber seq, uint64_t unix_time,
|
SnapshotImpl* New(SnapshotImpl* s, SequenceNumber seq, uint64_t unix_time,
|
||||||
bool is_write_conflict_boundary,
|
bool is_write_conflict_boundary,
|
||||||
|
|
|
@ -38,7 +38,7 @@ static void DeleteEntry(const Slice& /*key*/, void* value) {
|
||||||
T* typed_value = reinterpret_cast<T*>(value);
|
T* typed_value = reinterpret_cast<T*>(value);
|
||||||
delete typed_value;
|
delete typed_value;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
// Generate the regular and coroutine versions of some methods by
|
// Generate the regular and coroutine versions of some methods by
|
||||||
|
@ -79,7 +79,7 @@ void AppendVarint64(IterKey* key, uint64_t v) {
|
||||||
|
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
const int kLoadConcurency = 128;
|
const int kLoadConcurency = 128;
|
||||||
|
|
||||||
|
@ -103,8 +103,7 @@ TableCache::TableCache(const ImmutableOptions& ioptions,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
TableCache::~TableCache() {
|
TableCache::~TableCache() {}
|
||||||
}
|
|
||||||
|
|
||||||
TableReader* TableCache::GetTableReaderFromHandle(Cache::Handle* handle) {
|
TableReader* TableCache::GetTableReaderFromHandle(Cache::Handle* handle) {
|
||||||
return reinterpret_cast<TableReader*>(cache_->Value(handle));
|
return reinterpret_cast<TableReader*>(cache_->Value(handle));
|
||||||
|
|
|
@ -27,7 +27,7 @@ uint64_t GetUint64Property(const UserCollectedProperties& props,
|
||||||
return GetVarint64(&raw, &val) ? val : 0;
|
return GetVarint64(&raw, &val) ? val : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key,
|
Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key,
|
||||||
const Slice& value,
|
const Slice& value,
|
||||||
|
@ -54,13 +54,12 @@ Status UserKeyTablePropertiesCollector::Finish(
|
||||||
return collector_->Finish(properties);
|
return collector_->Finish(properties);
|
||||||
}
|
}
|
||||||
|
|
||||||
UserCollectedProperties
|
UserCollectedProperties UserKeyTablePropertiesCollector::GetReadableProperties()
|
||||||
UserKeyTablePropertiesCollector::GetReadableProperties() const {
|
const {
|
||||||
return collector_->GetReadableProperties();
|
return collector_->GetReadableProperties();
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t GetDeletedKeys(
|
uint64_t GetDeletedKeys(const UserCollectedProperties& props) {
|
||||||
const UserCollectedProperties& props) {
|
|
||||||
bool property_present_ignored;
|
bool property_present_ignored;
|
||||||
return GetUint64Property(props, TablePropertiesNames::kDeletedKeys,
|
return GetUint64Property(props, TablePropertiesNames::kDeletedKeys,
|
||||||
&property_present_ignored);
|
&property_present_ignored);
|
||||||
|
@ -68,8 +67,8 @@ uint64_t GetDeletedKeys(
|
||||||
|
|
||||||
uint64_t GetMergeOperands(const UserCollectedProperties& props,
|
uint64_t GetMergeOperands(const UserCollectedProperties& props,
|
||||||
bool* property_present) {
|
bool* property_present) {
|
||||||
return GetUint64Property(
|
return GetUint64Property(props, TablePropertiesNames::kMergeOperands,
|
||||||
props, TablePropertiesNames::kMergeOperands, property_present);
|
property_present);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
|
@ -62,8 +62,7 @@ Status TransactionLogIteratorImpl::OpenLogFile(
|
||||||
// If cannot open file in DB directory.
|
// If cannot open file in DB directory.
|
||||||
// Try the archive dir, as it could have moved in the meanwhile.
|
// Try the archive dir, as it could have moved in the meanwhile.
|
||||||
fname = ArchivedLogFileName(dir_, log_file->LogNumber());
|
fname = ArchivedLogFileName(dir_, log_file->LogNumber());
|
||||||
s = fs->NewSequentialFile(fname, optimized_env_options,
|
s = fs->NewSequentialFile(fname, optimized_env_options, &file, nullptr);
|
||||||
&file, nullptr);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
|
@ -124,8 +123,8 @@ void TransactionLogIteratorImpl::SeekToStartSequence(uint64_t start_file_index,
|
||||||
}
|
}
|
||||||
while (RestrictedRead(&record)) {
|
while (RestrictedRead(&record)) {
|
||||||
if (record.size() < WriteBatchInternal::kHeader) {
|
if (record.size() < WriteBatchInternal::kHeader) {
|
||||||
reporter_.Corruption(
|
reporter_.Corruption(record.size(),
|
||||||
record.size(), Status::Corruption("very small log record"));
|
Status::Corruption("very small log record"));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
UpdateCurrentWriteBatch(record);
|
UpdateCurrentWriteBatch(record);
|
||||||
|
@ -137,7 +136,8 @@ void TransactionLogIteratorImpl::SeekToStartSequence(uint64_t start_file_index,
|
||||||
reporter_.Info(current_status_.ToString().c_str());
|
reporter_.Info(current_status_.ToString().c_str());
|
||||||
return;
|
return;
|
||||||
} else if (strict) {
|
} else if (strict) {
|
||||||
reporter_.Info("Could seek required sequence number. Iterator will "
|
reporter_.Info(
|
||||||
|
"Could seek required sequence number. Iterator will "
|
||||||
"continue.");
|
"continue.");
|
||||||
}
|
}
|
||||||
is_valid_ = true;
|
is_valid_ = true;
|
||||||
|
@ -189,8 +189,8 @@ void TransactionLogIteratorImpl::NextImpl(bool internal) {
|
||||||
}
|
}
|
||||||
while (RestrictedRead(&record)) {
|
while (RestrictedRead(&record)) {
|
||||||
if (record.size() < WriteBatchInternal::kHeader) {
|
if (record.size() < WriteBatchInternal::kHeader) {
|
||||||
reporter_.Corruption(
|
reporter_.Corruption(record.size(),
|
||||||
record.size(), Status::Corruption("very small log record"));
|
Status::Corruption("very small log record"));
|
||||||
continue;
|
continue;
|
||||||
} else {
|
} else {
|
||||||
// started_ should be true if called by application
|
// started_ should be true if called by application
|
||||||
|
|
|
@ -23,12 +23,11 @@ namespace ROCKSDB_NAMESPACE {
|
||||||
class LogFileImpl : public LogFile {
|
class LogFileImpl : public LogFile {
|
||||||
public:
|
public:
|
||||||
LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
|
LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
|
||||||
uint64_t sizeBytes) :
|
uint64_t sizeBytes)
|
||||||
logNumber_(logNum),
|
: logNumber_(logNum),
|
||||||
type_(logType),
|
type_(logType),
|
||||||
startSequence_(startSeq),
|
startSequence_(startSeq),
|
||||||
sizeFileBytes_(sizeBytes) {
|
sizeFileBytes_(sizeBytes) {}
|
||||||
}
|
|
||||||
|
|
||||||
std::string PathName() const override {
|
std::string PathName() const override {
|
||||||
if (type_ == kArchivedLogFile) {
|
if (type_ == kArchivedLogFile) {
|
||||||
|
@ -54,7 +53,6 @@ class LogFileImpl : public LogFile {
|
||||||
WalFileType type_;
|
WalFileType type_;
|
||||||
SequenceNumber startSequence_;
|
SequenceNumber startSequence_;
|
||||||
uint64_t sizeFileBytes_;
|
uint64_t sizeFileBytes_;
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
class TransactionLogIteratorImpl : public TransactionLogIterator {
|
class TransactionLogIteratorImpl : public TransactionLogIterator {
|
||||||
|
|
|
@ -6,8 +6,10 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
|
|
||||||
#include "util/autovector.h"
|
#include "util/autovector.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
|
@ -20,9 +20,7 @@
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
namespace {
|
namespace {} // anonymous namespace
|
||||||
|
|
||||||
} // anonymous namespace
|
|
||||||
|
|
||||||
uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) {
|
uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) {
|
||||||
assert(number <= kFileNumberMask);
|
assert(number <= kFileNumberMask);
|
||||||
|
@ -501,8 +499,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case kCompactCursor:
|
case kCompactCursor:
|
||||||
if (GetLevel(&input, &level, &msg) &&
|
if (GetLevel(&input, &level, &msg) && GetInternalKey(&input, &key)) {
|
||||||
GetInternalKey(&input, &key)) {
|
|
||||||
// Here we re-use the output format of compact pointer in LevelDB
|
// Here we re-use the output format of compact pointer in LevelDB
|
||||||
// to persist compact_cursors_
|
// to persist compact_cursors_
|
||||||
compact_cursors_.push_back(std::make_pair(level, key));
|
compact_cursors_.push_back(std::make_pair(level, key));
|
||||||
|
|
|
@ -146,8 +146,8 @@ struct FileDescriptor {
|
||||||
return packed_number_and_path_id & kFileNumberMask;
|
return packed_number_and_path_id & kFileNumberMask;
|
||||||
}
|
}
|
||||||
uint32_t GetPathId() const {
|
uint32_t GetPathId() const {
|
||||||
return static_cast<uint32_t>(
|
return static_cast<uint32_t>(packed_number_and_path_id /
|
||||||
packed_number_and_path_id / (kFileNumberMask + 1));
|
(kFileNumberMask + 1));
|
||||||
}
|
}
|
||||||
uint64_t GetFileSize() const { return file_size; }
|
uint64_t GetFileSize() const { return file_size; }
|
||||||
};
|
};
|
||||||
|
@ -316,11 +316,7 @@ struct FdWithKeyRange {
|
||||||
Slice largest_key; // slice that contain largest key
|
Slice largest_key; // slice that contain largest key
|
||||||
|
|
||||||
FdWithKeyRange()
|
FdWithKeyRange()
|
||||||
: fd(),
|
: fd(), file_metadata(nullptr), smallest_key(), largest_key() {}
|
||||||
file_metadata(nullptr),
|
|
||||||
smallest_key(),
|
|
||||||
largest_key() {
|
|
||||||
}
|
|
||||||
|
|
||||||
FdWithKeyRange(FileDescriptor _fd, Slice _smallest_key, Slice _largest_key,
|
FdWithKeyRange(FileDescriptor _fd, Slice _smallest_key, Slice _largest_key,
|
||||||
FileMetaData* _file_metadata)
|
FileMetaData* _file_metadata)
|
||||||
|
|
|
@ -93,23 +93,19 @@ namespace {
|
||||||
// Find File in LevelFilesBrief data structure
|
// Find File in LevelFilesBrief data structure
|
||||||
// Within an index range defined by left and right
|
// Within an index range defined by left and right
|
||||||
int FindFileInRange(const InternalKeyComparator& icmp,
|
int FindFileInRange(const InternalKeyComparator& icmp,
|
||||||
const LevelFilesBrief& file_level,
|
const LevelFilesBrief& file_level, const Slice& key,
|
||||||
const Slice& key,
|
uint32_t left, uint32_t right) {
|
||||||
uint32_t left,
|
|
||||||
uint32_t right) {
|
|
||||||
auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool {
|
auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool {
|
||||||
return icmp.InternalKeyComparator::Compare(f.largest_key, k) < 0;
|
return icmp.InternalKeyComparator::Compare(f.largest_key, k) < 0;
|
||||||
};
|
};
|
||||||
const auto& b = file_level.files;
|
const auto& b = file_level.files;
|
||||||
return static_cast<int>(std::lower_bound(b + left,
|
return static_cast<int>(std::lower_bound(b + left, b + right, key, cmp) - b);
|
||||||
b + right, key, cmp) - b);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Status OverlapWithIterator(const Comparator* ucmp,
|
Status OverlapWithIterator(const Comparator* ucmp,
|
||||||
const Slice& smallest_user_key,
|
const Slice& smallest_user_key,
|
||||||
const Slice& largest_user_key,
|
const Slice& largest_user_key,
|
||||||
InternalIterator* iter,
|
InternalIterator* iter, bool* overlap) {
|
||||||
bool* overlap) {
|
|
||||||
InternalKey range_start(smallest_user_key, kMaxSequenceNumber,
|
InternalKey range_start(smallest_user_key, kMaxSequenceNumber,
|
||||||
kValueTypeForSeek);
|
kValueTypeForSeek);
|
||||||
iter->Seek(range_start.Encode());
|
iter->Seek(range_start.Encode());
|
||||||
|
@ -187,9 +183,9 @@ class FilePicker {
|
||||||
// Do key range filtering of files or/and fractional cascading if:
|
// Do key range filtering of files or/and fractional cascading if:
|
||||||
// (1) not all the files are in level 0, or
|
// (1) not all the files are in level 0, or
|
||||||
// (2) there are more than 3 current level files
|
// (2) there are more than 3 current level files
|
||||||
// If there are only 3 or less current level files in the system, we skip
|
// If there are only 3 or less current level files in the system, we
|
||||||
// the key range filtering. In this case, more likely, the system is
|
// skip the key range filtering. In this case, more likely, the system
|
||||||
// highly tuned to minimize number of tables queried by each query,
|
// is highly tuned to minimize number of tables queried by each query,
|
||||||
// so it is unlikely that key range filtering is more efficient than
|
// so it is unlikely that key range filtering is more efficient than
|
||||||
// querying the files.
|
// querying the files.
|
||||||
if (num_levels_ > 1 || curr_file_level_->num_files > 3) {
|
if (num_levels_ > 1 || curr_file_level_->num_files > 3) {
|
||||||
|
@ -211,11 +207,9 @@ class FilePicker {
|
||||||
// Setup file search bound for the next level based on the
|
// Setup file search bound for the next level based on the
|
||||||
// comparison results
|
// comparison results
|
||||||
if (curr_level_ > 0) {
|
if (curr_level_ > 0) {
|
||||||
file_indexer_->GetNextLevelIndex(curr_level_,
|
file_indexer_->GetNextLevelIndex(
|
||||||
curr_index_in_curr_level_,
|
curr_level_, curr_index_in_curr_level_, cmp_smallest,
|
||||||
cmp_smallest, cmp_largest,
|
cmp_largest, &search_left_bound_, &search_right_bound_);
|
||||||
&search_left_bound_,
|
|
||||||
&search_right_bound_);
|
|
||||||
}
|
}
|
||||||
// Key falls out of current file's range
|
// Key falls out of current file's range
|
||||||
if (cmp_smallest < 0 || cmp_largest > 0) {
|
if (cmp_smallest < 0 || cmp_largest > 0) {
|
||||||
|
@ -846,8 +840,7 @@ Version::~Version() {
|
||||||
}
|
}
|
||||||
|
|
||||||
int FindFile(const InternalKeyComparator& icmp,
|
int FindFile(const InternalKeyComparator& icmp,
|
||||||
const LevelFilesBrief& file_level,
|
const LevelFilesBrief& file_level, const Slice& key) {
|
||||||
const Slice& key) {
|
|
||||||
return FindFileInRange(icmp, file_level, key, 0,
|
return FindFileInRange(icmp, file_level, key, 0,
|
||||||
static_cast<uint32_t>(file_level.num_files));
|
static_cast<uint32_t>(file_level.num_files));
|
||||||
}
|
}
|
||||||
|
@ -882,24 +875,23 @@ void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool AfterFile(const Comparator* ucmp,
|
static bool AfterFile(const Comparator* ucmp, const Slice* user_key,
|
||||||
const Slice* user_key, const FdWithKeyRange* f) {
|
const FdWithKeyRange* f) {
|
||||||
// nullptr user_key occurs before all keys and is therefore never after *f
|
// nullptr user_key occurs before all keys and is therefore never after *f
|
||||||
return (user_key != nullptr &&
|
return (user_key != nullptr &&
|
||||||
ucmp->CompareWithoutTimestamp(*user_key,
|
ucmp->CompareWithoutTimestamp(*user_key,
|
||||||
ExtractUserKey(f->largest_key)) > 0);
|
ExtractUserKey(f->largest_key)) > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool BeforeFile(const Comparator* ucmp,
|
static bool BeforeFile(const Comparator* ucmp, const Slice* user_key,
|
||||||
const Slice* user_key, const FdWithKeyRange* f) {
|
const FdWithKeyRange* f) {
|
||||||
// nullptr user_key occurs after all keys and is therefore never before *f
|
// nullptr user_key occurs after all keys and is therefore never before *f
|
||||||
return (user_key != nullptr &&
|
return (user_key != nullptr &&
|
||||||
ucmp->CompareWithoutTimestamp(*user_key,
|
ucmp->CompareWithoutTimestamp(*user_key,
|
||||||
ExtractUserKey(f->smallest_key)) < 0);
|
ExtractUserKey(f->smallest_key)) < 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool SomeFileOverlapsRange(
|
bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
|
||||||
const InternalKeyComparator& icmp,
|
|
||||||
bool disjoint_sorted_files,
|
bool disjoint_sorted_files,
|
||||||
const LevelFilesBrief& file_level,
|
const LevelFilesBrief& file_level,
|
||||||
const Slice* smallest_user_key,
|
const Slice* smallest_user_key,
|
||||||
|
@ -1025,9 +1017,7 @@ class LevelIterator final : public InternalIterator {
|
||||||
return file_iter_.iter() ? file_iter_.status() : Status::OK();
|
return file_iter_.iter() ? file_iter_.status() : Status::OK();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool PrepareValue() override {
|
bool PrepareValue() override { return file_iter_.PrepareValue(); }
|
||||||
return file_iter_.PrepareValue();
|
|
||||||
}
|
|
||||||
|
|
||||||
inline bool MayBeOutOfLowerBound() override {
|
inline bool MayBeOutOfLowerBound() override {
|
||||||
assert(Valid());
|
assert(Valid());
|
||||||
|
@ -1561,8 +1551,7 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
|
||||||
if (fname != nullptr) {
|
if (fname != nullptr) {
|
||||||
file_name = *fname;
|
file_name = *fname;
|
||||||
} else {
|
} else {
|
||||||
file_name =
|
file_name = TableFileName(ioptions->cf_paths, file_meta->fd.GetNumber(),
|
||||||
TableFileName(ioptions->cf_paths, file_meta->fd.GetNumber(),
|
|
||||||
file_meta->fd.GetPathId());
|
file_meta->fd.GetPathId());
|
||||||
}
|
}
|
||||||
s = ioptions->fs->NewRandomAccessFile(file_name, file_options_, &file,
|
s = ioptions->fs->NewRandomAccessFile(file_name, file_options_, &file,
|
||||||
|
@ -1690,8 +1679,8 @@ Status Version::GetPropertiesOfTablesInRange(
|
||||||
false);
|
false);
|
||||||
for (const auto& file_meta : files) {
|
for (const auto& file_meta : files) {
|
||||||
auto fname =
|
auto fname =
|
||||||
TableFileName(cfd_->ioptions()->cf_paths,
|
TableFileName(cfd_->ioptions()->cf_paths, file_meta->fd.GetNumber(),
|
||||||
file_meta->fd.GetNumber(), file_meta->fd.GetPathId());
|
file_meta->fd.GetPathId());
|
||||||
if (props->count(fname) == 0) {
|
if (props->count(fname) == 0) {
|
||||||
// 1. If the table is already present in table cache, load table
|
// 1. If the table is already present in table cache, load table
|
||||||
// properties from there.
|
// properties from there.
|
||||||
|
@ -1788,8 +1777,7 @@ void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) {
|
||||||
files.back().num_deletions = file->num_deletions;
|
files.back().num_deletions = file->num_deletions;
|
||||||
level_size += file->fd.GetFileSize();
|
level_size += file->fd.GetFileSize();
|
||||||
}
|
}
|
||||||
cf_meta->levels.emplace_back(
|
cf_meta->levels.emplace_back(level, level_size, std::move(files));
|
||||||
level, level_size, std::move(files));
|
|
||||||
cf_meta->size += level_size;
|
cf_meta->size += level_size;
|
||||||
}
|
}
|
||||||
for (const auto& meta : vstorage->GetBlobFiles()) {
|
for (const auto& meta : vstorage->GetBlobFiles()) {
|
||||||
|
@ -1880,10 +1868,8 @@ uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const {
|
||||||
|
|
||||||
if (current_num_samples_ < file_count) {
|
if (current_num_samples_ < file_count) {
|
||||||
// casting to avoid overflowing
|
// casting to avoid overflowing
|
||||||
return
|
return static_cast<uint64_t>(
|
||||||
static_cast<uint64_t>(
|
(est * static_cast<double>(file_count) / current_num_samples_));
|
||||||
(est * static_cast<double>(file_count) / current_num_samples_)
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
return est;
|
return est;
|
||||||
}
|
}
|
||||||
|
@ -2020,8 +2006,8 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
|
||||||
/*smallest_compaction_key=*/nullptr,
|
/*smallest_compaction_key=*/nullptr,
|
||||||
/*largest_compaction_key=*/nullptr,
|
/*largest_compaction_key=*/nullptr,
|
||||||
/*allow_unprepared_value=*/false));
|
/*allow_unprepared_value=*/false));
|
||||||
status = OverlapWithIterator(
|
status = OverlapWithIterator(ucmp, smallest_user_key, largest_user_key,
|
||||||
ucmp, smallest_user_key, largest_user_key, iter.get(), overlap);
|
iter.get(), overlap);
|
||||||
if (!status.ok() || *overlap) {
|
if (!status.ok() || *overlap) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -2035,8 +2021,8 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
|
||||||
cfd_->internal_stats()->GetFileReadHist(level),
|
cfd_->internal_stats()->GetFileReadHist(level),
|
||||||
TableReaderCaller::kUserIterator, IsFilterSkipped(level), level,
|
TableReaderCaller::kUserIterator, IsFilterSkipped(level), level,
|
||||||
&range_del_agg));
|
&range_del_agg));
|
||||||
status = OverlapWithIterator(
|
status = OverlapWithIterator(ucmp, smallest_user_key, largest_user_key,
|
||||||
ucmp, smallest_user_key, largest_user_key, iter.get(), overlap);
|
iter.get(), overlap);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (status.ok() && *overlap == false &&
|
if (status.ok() && *overlap == false &&
|
||||||
|
@ -2905,8 +2891,8 @@ bool Version::IsFilterSkipped(int level, bool is_file_last_in_level) {
|
||||||
void VersionStorageInfo::GenerateLevelFilesBrief() {
|
void VersionStorageInfo::GenerateLevelFilesBrief() {
|
||||||
level_files_brief_.resize(num_non_empty_levels_);
|
level_files_brief_.resize(num_non_empty_levels_);
|
||||||
for (int level = 0; level < num_non_empty_levels_; level++) {
|
for (int level = 0; level < num_non_empty_levels_; level++) {
|
||||||
DoGenerateLevelFilesBrief(
|
DoGenerateLevelFilesBrief(&level_files_brief_[level], files_[level],
|
||||||
&level_files_brief_[level], files_[level], &arena_);
|
&arena_);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2940,8 +2926,7 @@ void Version::PrepareAppend(const MutableCFOptions& mutable_cf_options,
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) {
|
bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) {
|
||||||
if (file_meta->init_stats_from_file ||
|
if (file_meta->init_stats_from_file || file_meta->compensated_file_size > 0) {
|
||||||
file_meta->compensated_file_size > 0) {
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
std::shared_ptr<const TableProperties> tp;
|
std::shared_ptr<const TableProperties> tp;
|
||||||
|
@ -3806,7 +3791,7 @@ void SortFileByRoundRobin(const InternalKeyComparator& icmp,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
void VersionStorageInfo::UpdateFilesByCompactionPri(
|
void VersionStorageInfo::UpdateFilesByCompactionPri(
|
||||||
const ImmutableOptions& ioptions, const MutableCFOptions& options) {
|
const ImmutableOptions& ioptions, const MutableCFOptions& options) {
|
||||||
|
@ -3978,9 +3963,7 @@ void VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Version::Ref() {
|
void Version::Ref() { ++refs_; }
|
||||||
++refs_;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool Version::Unref() {
|
bool Version::Unref() {
|
||||||
assert(refs_ >= 1);
|
assert(refs_ >= 1);
|
||||||
|
@ -4112,9 +4095,8 @@ void VersionStorageInfo::GetCleanInputsWithinInterval(
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
GetOverlappingInputsRangeBinarySearch(level, begin, end, inputs,
|
GetOverlappingInputsRangeBinarySearch(level, begin, end, inputs, hint_index,
|
||||||
hint_index, file_index,
|
file_index, true /* within_interval */);
|
||||||
true /* within_interval */);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store in "*inputs" all files in "level" that overlap [begin,end]
|
// Store in "*inputs" all files in "level" that overlap [begin,end]
|
||||||
|
@ -4277,8 +4259,7 @@ const char* VersionStorageInfo::LevelFileSummary(FileSummaryStorage* scratch,
|
||||||
"#%" PRIu64 "(seq=%" PRIu64 ",sz=%s,%d) ",
|
"#%" PRIu64 "(seq=%" PRIu64 ",sz=%s,%d) ",
|
||||||
f->fd.GetNumber(), f->fd.smallest_seqno, sztxt,
|
f->fd.GetNumber(), f->fd.smallest_seqno, sztxt,
|
||||||
static_cast<int>(f->being_compacted));
|
static_cast<int>(f->being_compacted));
|
||||||
if (ret < 0 || ret >= sz)
|
if (ret < 0 || ret >= sz) break;
|
||||||
break;
|
|
||||||
len += ret;
|
len += ret;
|
||||||
}
|
}
|
||||||
// overwrite the last space (only if files_[level].size() is non-zero)
|
// overwrite the last space (only if files_[level].size() is non-zero)
|
||||||
|
@ -4456,8 +4437,8 @@ uint64_t VersionStorageInfo::EstimateLiveDataSize() const {
|
||||||
// no potential overlap, we can safely insert the rest of this level
|
// no potential overlap, we can safely insert the rest of this level
|
||||||
// (if the level is not 0) into the map without checking again because
|
// (if the level is not 0) into the map without checking again because
|
||||||
// the elements in the level are sorted and non-overlapping.
|
// the elements in the level are sorted and non-overlapping.
|
||||||
auto lb = (found_end && l != 0) ?
|
auto lb = (found_end && l != 0) ? ranges.end()
|
||||||
ranges.end() : ranges.lower_bound(&file->smallest);
|
: ranges.lower_bound(&file->smallest);
|
||||||
found_end = (lb == ranges.end());
|
found_end = (lb == ranges.end());
|
||||||
if (found_end || internal_comparator_->Compare(
|
if (found_end || internal_comparator_->Compare(
|
||||||
file->largest, (*lb).second->smallest) < 0) {
|
file->largest, (*lb).second->smallest) < 0) {
|
||||||
|
@ -5674,7 +5655,7 @@ std::string ManifestPicker::GetNextManifest(uint64_t* number,
|
||||||
}
|
}
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
Status VersionSet::TryRecover(
|
Status VersionSet::TryRecover(
|
||||||
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
|
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
|
||||||
|
@ -5885,9 +5866,9 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
|
||||||
VersionEdit ve;
|
VersionEdit ve;
|
||||||
InstrumentedMutex dummy_mutex;
|
InstrumentedMutex dummy_mutex;
|
||||||
InstrumentedMutexLock l(&dummy_mutex);
|
InstrumentedMutexLock l(&dummy_mutex);
|
||||||
return versions.LogAndApply(
|
return versions.LogAndApply(versions.GetColumnFamilySet()->GetDefault(),
|
||||||
versions.GetColumnFamilySet()->GetDefault(),
|
mutable_cf_options, &ve, &dummy_mutex, nullptr,
|
||||||
mutable_cf_options, &ve, &dummy_mutex, nullptr, true);
|
true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the checksum information including the checksum and checksum function
|
// Get the checksum information including the checksum and checksum function
|
||||||
|
@ -5973,9 +5954,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
|
||||||
std::unique_ptr<FSSequentialFile> file;
|
std::unique_ptr<FSSequentialFile> file;
|
||||||
const std::shared_ptr<FileSystem>& fs = options.env->GetFileSystem();
|
const std::shared_ptr<FileSystem>& fs = options.env->GetFileSystem();
|
||||||
s = fs->NewSequentialFile(
|
s = fs->NewSequentialFile(
|
||||||
dscname,
|
dscname, fs->OptimizeForManifestRead(file_options_), &file, nullptr);
|
||||||
fs->OptimizeForManifestRead(file_options_), &file,
|
|
||||||
nullptr);
|
|
||||||
if (!s.ok()) {
|
if (!s.ok()) {
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
@ -6078,8 +6057,8 @@ Status VersionSet::WriteCurrentStateToManifest(
|
||||||
cfd->internal_comparator().user_comparator()->Name());
|
cfd->internal_comparator().user_comparator()->Name());
|
||||||
std::string record;
|
std::string record;
|
||||||
if (!edit.EncodeTo(&record)) {
|
if (!edit.EncodeTo(&record)) {
|
||||||
return Status::Corruption(
|
return Status::Corruption("Unable to Encode VersionEdit:" +
|
||||||
"Unable to Encode VersionEdit:" + edit.DebugString(true));
|
edit.DebugString(true));
|
||||||
}
|
}
|
||||||
io_s = log->AddRecord(record);
|
io_s = log->AddRecord(record);
|
||||||
if (!io_s.ok()) {
|
if (!io_s.ok()) {
|
||||||
|
@ -6137,9 +6116,10 @@ Status VersionSet::WriteCurrentStateToManifest(
|
||||||
edit.SetLogNumber(log_number);
|
edit.SetLogNumber(log_number);
|
||||||
|
|
||||||
if (cfd->GetID() == 0) {
|
if (cfd->GetID() == 0) {
|
||||||
// min_log_number_to_keep is for the whole db, not for specific column family.
|
// min_log_number_to_keep is for the whole db, not for specific column
|
||||||
// So it does not need to be set for every column family, just need to be set once.
|
// family. So it does not need to be set for every column family, just
|
||||||
// Since default CF can never be dropped, we set the min_log to the default CF here.
|
// need to be set once. Since default CF can never be dropped, we set
|
||||||
|
// the min_log to the default CF here.
|
||||||
uint64_t min_log = min_log_number_to_keep();
|
uint64_t min_log = min_log_number_to_keep();
|
||||||
if (min_log != 0) {
|
if (min_log != 0) {
|
||||||
edit.SetMinLogNumberToKeep(min_log);
|
edit.SetMinLogNumberToKeep(min_log);
|
||||||
|
@ -6155,8 +6135,8 @@ Status VersionSet::WriteCurrentStateToManifest(
|
||||||
|
|
||||||
std::string record;
|
std::string record;
|
||||||
if (!edit.EncodeTo(&record)) {
|
if (!edit.EncodeTo(&record)) {
|
||||||
return Status::Corruption(
|
return Status::Corruption("Unable to Encode VersionEdit:" +
|
||||||
"Unable to Encode VersionEdit:" + edit.DebugString(true));
|
edit.DebugString(true));
|
||||||
}
|
}
|
||||||
io_s = log->AddRecord(record);
|
io_s = log->AddRecord(record);
|
||||||
if (!io_s.ok()) {
|
if (!io_s.ok()) {
|
||||||
|
@ -6588,8 +6568,8 @@ void VersionSet::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
|
||||||
filemetadata.largestkey = file->largest.user_key().ToString();
|
filemetadata.largestkey = file->largest.user_key().ToString();
|
||||||
filemetadata.smallest_seqno = file->fd.smallest_seqno;
|
filemetadata.smallest_seqno = file->fd.smallest_seqno;
|
||||||
filemetadata.largest_seqno = file->fd.largest_seqno;
|
filemetadata.largest_seqno = file->fd.largest_seqno;
|
||||||
filemetadata.num_reads_sampled = file->stats.num_reads_sampled.load(
|
filemetadata.num_reads_sampled =
|
||||||
std::memory_order_relaxed);
|
file->stats.num_reads_sampled.load(std::memory_order_relaxed);
|
||||||
filemetadata.being_compacted = file->being_compacted;
|
filemetadata.being_compacted = file->being_compacted;
|
||||||
filemetadata.num_entries = file->num_entries;
|
filemetadata.num_entries = file->num_entries;
|
||||||
filemetadata.num_deletions = file->num_deletions;
|
filemetadata.num_deletions = file->num_deletions;
|
||||||
|
|
|
@ -230,9 +230,7 @@ class VersionStorageInfo {
|
||||||
double blob_garbage_collection_age_cutoff,
|
double blob_garbage_collection_age_cutoff,
|
||||||
double blob_garbage_collection_force_threshold);
|
double blob_garbage_collection_force_threshold);
|
||||||
|
|
||||||
bool level0_non_overlapping() const {
|
bool level0_non_overlapping() const { return level0_non_overlapping_; }
|
||||||
return level0_non_overlapping_;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Updates the oldest snapshot and related internal state, like the bottommost
|
// Updates the oldest snapshot and related internal state, like the bottommost
|
||||||
// files marked for compaction.
|
// files marked for compaction.
|
||||||
|
@ -814,8 +812,8 @@ class Version {
|
||||||
|
|
||||||
Status OverlapWithLevelIterator(const ReadOptions&, const FileOptions&,
|
Status OverlapWithLevelIterator(const ReadOptions&, const FileOptions&,
|
||||||
const Slice& smallest_user_key,
|
const Slice& smallest_user_key,
|
||||||
const Slice& largest_user_key,
|
const Slice& largest_user_key, int level,
|
||||||
int level, bool* overlap);
|
bool* overlap);
|
||||||
|
|
||||||
// Lookup the value for key or get all merge operands for key.
|
// Lookup the value for key or get all merge operands for key.
|
||||||
// If do_merge = true (default) then lookup value for key.
|
// If do_merge = true (default) then lookup value for key.
|
||||||
|
|
|
@ -481,7 +481,8 @@ TEST_F(VersionStorageInfoTest, EstimateLiveDataSize2) {
|
||||||
|
|
||||||
TEST_F(VersionStorageInfoTest, GetOverlappingInputs) {
|
TEST_F(VersionStorageInfoTest, GetOverlappingInputs) {
|
||||||
// Two files that overlap at the range deletion tombstone sentinel.
|
// Two files that overlap at the range deletion tombstone sentinel.
|
||||||
Add(1, 1U, {"a", 0, kTypeValue}, {"b", kMaxSequenceNumber, kTypeRangeDeletion}, 1);
|
Add(1, 1U, {"a", 0, kTypeValue},
|
||||||
|
{"b", kMaxSequenceNumber, kTypeRangeDeletion}, 1);
|
||||||
Add(1, 2U, {"b", 0, kTypeValue}, {"c", 0, kTypeValue}, 1);
|
Add(1, 2U, {"b", 0, kTypeValue}, {"c", 0, kTypeValue}, 1);
|
||||||
// Two files that overlap at the same user key.
|
// Two files that overlap at the same user key.
|
||||||
Add(1, 3U, {"d", 0, kTypeValue}, {"e", kMaxSequenceNumber, kTypeValue}, 1);
|
Add(1, 3U, {"d", 0, kTypeValue}, {"e", kMaxSequenceNumber, kTypeValue}, 1);
|
||||||
|
@ -492,24 +493,26 @@ TEST_F(VersionStorageInfoTest, GetOverlappingInputs) {
|
||||||
|
|
||||||
UpdateVersionStorageInfo();
|
UpdateVersionStorageInfo();
|
||||||
|
|
||||||
ASSERT_EQ("1,2", GetOverlappingFiles(
|
ASSERT_EQ("1,2",
|
||||||
1, {"a", 0, kTypeValue}, {"b", 0, kTypeValue}));
|
GetOverlappingFiles(1, {"a", 0, kTypeValue}, {"b", 0, kTypeValue}));
|
||||||
ASSERT_EQ("1", GetOverlappingFiles(
|
ASSERT_EQ("1",
|
||||||
1, {"a", 0, kTypeValue}, {"b", kMaxSequenceNumber, kTypeRangeDeletion}));
|
GetOverlappingFiles(1, {"a", 0, kTypeValue},
|
||||||
ASSERT_EQ("2", GetOverlappingFiles(
|
{"b", kMaxSequenceNumber, kTypeRangeDeletion}));
|
||||||
1, {"b", kMaxSequenceNumber, kTypeValue}, {"c", 0, kTypeValue}));
|
ASSERT_EQ("2", GetOverlappingFiles(1, {"b", kMaxSequenceNumber, kTypeValue},
|
||||||
ASSERT_EQ("3,4", GetOverlappingFiles(
|
{"c", 0, kTypeValue}));
|
||||||
1, {"d", 0, kTypeValue}, {"e", 0, kTypeValue}));
|
ASSERT_EQ("3,4",
|
||||||
ASSERT_EQ("3", GetOverlappingFiles(
|
GetOverlappingFiles(1, {"d", 0, kTypeValue}, {"e", 0, kTypeValue}));
|
||||||
1, {"d", 0, kTypeValue}, {"e", kMaxSequenceNumber, kTypeRangeDeletion}));
|
ASSERT_EQ("3",
|
||||||
ASSERT_EQ("3,4", GetOverlappingFiles(
|
GetOverlappingFiles(1, {"d", 0, kTypeValue},
|
||||||
1, {"e", kMaxSequenceNumber, kTypeValue}, {"f", 0, kTypeValue}));
|
{"e", kMaxSequenceNumber, kTypeRangeDeletion}));
|
||||||
ASSERT_EQ("3,4", GetOverlappingFiles(
|
ASSERT_EQ("3,4", GetOverlappingFiles(1, {"e", kMaxSequenceNumber, kTypeValue},
|
||||||
1, {"e", 0, kTypeValue}, {"f", 0, kTypeValue}));
|
{"f", 0, kTypeValue}));
|
||||||
ASSERT_EQ("5", GetOverlappingFiles(
|
ASSERT_EQ("3,4",
|
||||||
1, {"g", 0, kTypeValue}, {"h", 0, kTypeValue}));
|
GetOverlappingFiles(1, {"e", 0, kTypeValue}, {"f", 0, kTypeValue}));
|
||||||
ASSERT_EQ("6", GetOverlappingFiles(
|
ASSERT_EQ("5",
|
||||||
1, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}));
|
GetOverlappingFiles(1, {"g", 0, kTypeValue}, {"h", 0, kTypeValue}));
|
||||||
|
ASSERT_EQ("6",
|
||||||
|
GetOverlappingFiles(1, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(VersionStorageInfoTest, FileLocationAndMetaDataByNumber) {
|
TEST_F(VersionStorageInfoTest, FileLocationAndMetaDataByNumber) {
|
||||||
|
@ -944,8 +947,8 @@ class FindLevelFileTest : public testing::Test {
|
||||||
Slice smallest_slice = smallest_key.Encode();
|
Slice smallest_slice = smallest_key.Encode();
|
||||||
Slice largest_slice = largest_key.Encode();
|
Slice largest_slice = largest_key.Encode();
|
||||||
|
|
||||||
char* mem = arena_.AllocateAligned(
|
char* mem =
|
||||||
smallest_slice.size() + largest_slice.size());
|
arena_.AllocateAligned(smallest_slice.size() + largest_slice.size());
|
||||||
memcpy(mem, smallest_slice.data(), smallest_slice.size());
|
memcpy(mem, smallest_slice.data(), smallest_slice.size());
|
||||||
memcpy(mem + smallest_slice.size(), largest_slice.data(),
|
memcpy(mem + smallest_slice.size(), largest_slice.data(),
|
||||||
largest_slice.size());
|
largest_slice.size());
|
||||||
|
@ -955,8 +958,7 @@ class FindLevelFileTest : public testing::Test {
|
||||||
auto& file = file_level_.files[num];
|
auto& file = file_level_.files[num];
|
||||||
file.fd = FileDescriptor(num + 1, 0, 0);
|
file.fd = FileDescriptor(num + 1, 0, 0);
|
||||||
file.smallest_key = Slice(mem, smallest_slice.size());
|
file.smallest_key = Slice(mem, smallest_slice.size());
|
||||||
file.largest_key = Slice(mem + smallest_slice.size(),
|
file.largest_key = Slice(mem + smallest_slice.size(), largest_slice.size());
|
||||||
largest_slice.size());
|
|
||||||
file_level_.num_files++;
|
file_level_.num_files++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -355,7 +355,8 @@ Status WalManager::RetainProbableWalFiles(VectorLogPtr& all_logs,
|
||||||
// Binary Search. avoid opening all files.
|
// Binary Search. avoid opening all files.
|
||||||
while (end >= start) {
|
while (end >= start) {
|
||||||
int64_t mid = start + (end - start) / 2; // Avoid overflow.
|
int64_t mid = start + (end - start) / 2; // Avoid overflow.
|
||||||
SequenceNumber current_seq_num = all_logs.at(static_cast<size_t>(mid))->StartSequence();
|
SequenceNumber current_seq_num =
|
||||||
|
all_logs.at(static_cast<size_t>(mid))->StartSequence();
|
||||||
if (current_seq_num == target) {
|
if (current_seq_num == target) {
|
||||||
end = mid;
|
end = mid;
|
||||||
break;
|
break;
|
||||||
|
@ -366,7 +367,8 @@ Status WalManager::RetainProbableWalFiles(VectorLogPtr& all_logs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// end could be -ve.
|
// end could be -ve.
|
||||||
size_t start_index = static_cast<size_t>(std::max(static_cast<int64_t>(0), end));
|
size_t start_index =
|
||||||
|
static_cast<size_t>(std::max(static_cast<int64_t>(0), end));
|
||||||
// The last wal file is always included
|
// The last wal file is always included
|
||||||
all_logs.erase(all_logs.begin(), all_logs.begin() + start_index);
|
all_logs.erase(all_logs.begin(), all_logs.begin() + start_index);
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
|
@ -468,9 +470,8 @@ Status WalManager::ReadFirstLine(const std::string& fname,
|
||||||
};
|
};
|
||||||
|
|
||||||
std::unique_ptr<FSSequentialFile> file;
|
std::unique_ptr<FSSequentialFile> file;
|
||||||
Status status = fs_->NewSequentialFile(fname,
|
Status status = fs_->NewSequentialFile(
|
||||||
fs_->OptimizeForLogRead(file_options_),
|
fname, fs_->OptimizeForLogRead(file_options_), &file, nullptr);
|
||||||
&file, nullptr);
|
|
||||||
std::unique_ptr<SequentialFileReader> file_reader(
|
std::unique_ptr<SequentialFileReader> file_reader(
|
||||||
new SequentialFileReader(std::move(file), fname, io_tracer_));
|
new SequentialFileReader(std::move(file), fname, io_tracer_));
|
||||||
|
|
||||||
|
|
|
@ -11,11 +11,11 @@
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <deque>
|
#include <deque>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
|
#include <memory>
|
||||||
#include <set>
|
#include <set>
|
||||||
|
#include <string>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <string>
|
|
||||||
#include <memory>
|
|
||||||
|
|
||||||
#include "db/version_set.h"
|
#include "db/version_set.h"
|
||||||
#include "file/file_util.h"
|
#include "file/file_util.h"
|
||||||
|
|
|
@ -88,7 +88,8 @@ class WalManagerTest : public testing::Test {
|
||||||
std::unique_ptr<WritableFileWriter> file_writer;
|
std::unique_ptr<WritableFileWriter> file_writer;
|
||||||
ASSERT_OK(WritableFileWriter::Create(fs, fname, env_options_, &file_writer,
|
ASSERT_OK(WritableFileWriter::Create(fs, fname, env_options_, &file_writer,
|
||||||
nullptr));
|
nullptr));
|
||||||
current_log_writer_.reset(new log::Writer(std::move(file_writer), 0, false));
|
current_log_writer_.reset(
|
||||||
|
new log::Writer(std::move(file_writer), 0, false));
|
||||||
}
|
}
|
||||||
|
|
||||||
void CreateArchiveLogs(int num_logs, int entries_per_log) {
|
void CreateArchiveLogs(int num_logs, int entries_per_log) {
|
||||||
|
@ -215,7 +216,7 @@ int CountRecords(TransactionLogIterator* iter) {
|
||||||
EXPECT_OK(iter->status());
|
EXPECT_OK(iter->status());
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(WalManagerTest, WALArchivalSizeLimit) {
|
TEST_F(WalManagerTest, WALArchivalSizeLimit) {
|
||||||
db_options_.WAL_ttl_seconds = 0;
|
db_options_.WAL_ttl_seconds = 0;
|
||||||
|
|
|
@ -158,7 +158,7 @@ struct BatchContentClassifier : public WriteBatch::Handler {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
} // anon namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
struct SavePoints {
|
struct SavePoints {
|
||||||
std::stack<SavePoint, autovector<SavePoint>> stack;
|
std::stack<SavePoint, autovector<SavePoint>> stack;
|
||||||
|
@ -240,9 +240,7 @@ void WriteBatch::Handler::LogData(const Slice& /*blob*/) {
|
||||||
// them.
|
// them.
|
||||||
}
|
}
|
||||||
|
|
||||||
bool WriteBatch::Handler::Continue() {
|
bool WriteBatch::Handler::Continue() { return true; }
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
void WriteBatch::Clear() {
|
void WriteBatch::Clear() {
|
||||||
rep_.clear();
|
rep_.clear();
|
||||||
|
@ -779,7 +777,7 @@ Status CheckColumnFamilyTimestampSize(ColumnFamilyHandle* column_family,
|
||||||
}
|
}
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
Status WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id,
|
Status WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id,
|
||||||
const Slice& key, const Slice& value) {
|
const Slice& key, const Slice& value) {
|
||||||
|
@ -1746,7 +1744,6 @@ Status WriteBatch::VerifyChecksum() const {
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
class MemTableInserter : public WriteBatch::Handler {
|
class MemTableInserter : public WriteBatch::Handler {
|
||||||
|
|
||||||
SequenceNumber sequence_;
|
SequenceNumber sequence_;
|
||||||
ColumnFamilyMemTables* const cf_mems_;
|
ColumnFamilyMemTables* const cf_mems_;
|
||||||
FlushScheduler* const flush_scheduler_;
|
FlushScheduler* const flush_scheduler_;
|
||||||
|
@ -1818,8 +1815,8 @@ class MemTableInserter : public WriteBatch::Handler {
|
||||||
new (&duplicate_detector_) DuplicateDetector(db_);
|
new (&duplicate_detector_) DuplicateDetector(db_);
|
||||||
dup_dectector_on_ = true;
|
dup_dectector_on_ = true;
|
||||||
}
|
}
|
||||||
return reinterpret_cast<DuplicateDetector*>
|
return reinterpret_cast<DuplicateDetector*>(&duplicate_detector_)
|
||||||
(&duplicate_detector_)->IsDuplicateKeySeq(column_family_id, key, sequence_);
|
->IsDuplicateKeySeq(column_family_id, key, sequence_);
|
||||||
}
|
}
|
||||||
|
|
||||||
const ProtectionInfoKVOC64* NextProtectionInfo() {
|
const ProtectionInfoKVOC64* NextProtectionInfo() {
|
||||||
|
@ -1895,12 +1892,11 @@ class MemTableInserter : public WriteBatch::Handler {
|
||||||
|
|
||||||
~MemTableInserter() override {
|
~MemTableInserter() override {
|
||||||
if (dup_dectector_on_) {
|
if (dup_dectector_on_) {
|
||||||
reinterpret_cast<DuplicateDetector*>
|
reinterpret_cast<DuplicateDetector*>(&duplicate_detector_)
|
||||||
(&duplicate_detector_)->~DuplicateDetector();
|
->~DuplicateDetector();
|
||||||
}
|
}
|
||||||
if (post_info_created_) {
|
if (post_info_created_) {
|
||||||
reinterpret_cast<MemPostInfoMap*>
|
reinterpret_cast<MemPostInfoMap*>(&mem_post_info_map_)->~MemPostInfoMap();
|
||||||
(&mem_post_info_map_)->~MemPostInfoMap();
|
|
||||||
}
|
}
|
||||||
if (hint_created_) {
|
if (hint_created_) {
|
||||||
for (auto iter : GetHintMap()) {
|
for (auto iter : GetHintMap()) {
|
||||||
|
@ -2864,7 +2860,7 @@ class MemTableInserter : public WriteBatch::Handler {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// This function can only be called in these conditions:
|
// This function can only be called in these conditions:
|
||||||
// 1) During Recovery()
|
// 1) During Recovery()
|
||||||
|
@ -3035,7 +3031,7 @@ class ProtectionInfoUpdater : public WriteBatch::Handler {
|
||||||
WriteBatch::ProtectionInfo* const prot_info_ = nullptr;
|
WriteBatch::ProtectionInfo* const prot_info_ = nullptr;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
Status WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
|
Status WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
|
||||||
assert(contents.size() >= WriteBatchInternal::kHeader);
|
assert(contents.size() >= WriteBatchInternal::kHeader);
|
||||||
|
|
|
@ -77,7 +77,6 @@ struct WriteBatch::ProtectionInfo {
|
||||||
// WriteBatch that we don't want in the public WriteBatch interface.
|
// WriteBatch that we don't want in the public WriteBatch interface.
|
||||||
class WriteBatchInternal {
|
class WriteBatchInternal {
|
||||||
public:
|
public:
|
||||||
|
|
||||||
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
|
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
|
||||||
static constexpr size_t kHeader = 12;
|
static constexpr size_t kHeader = 12;
|
||||||
|
|
||||||
|
@ -149,13 +148,9 @@ class WriteBatchInternal {
|
||||||
// This offset is only valid if the batch is not empty.
|
// This offset is only valid if the batch is not empty.
|
||||||
static size_t GetFirstOffset(WriteBatch* batch);
|
static size_t GetFirstOffset(WriteBatch* batch);
|
||||||
|
|
||||||
static Slice Contents(const WriteBatch* batch) {
|
static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); }
|
||||||
return Slice(batch->rep_);
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t ByteSize(const WriteBatch* batch) {
|
static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); }
|
||||||
return batch->rep_.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
static Status SetContents(WriteBatch* batch, const Slice& contents);
|
static Status SetContents(WriteBatch* batch, const Slice& contents);
|
||||||
|
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue