mirror of https://github.com/facebook/rocksdb.git
Ran clang-format on db/ directory (#10910)
Summary: Ran `find ./db/ -type f | xargs clang-format -i`. Excluded minor changes it tried to make on db/db_impl/. Everything else it changed was directly under db/ directory. Included minor manual touchups mentioned in PR commit history. Pull Request resolved: https://github.com/facebook/rocksdb/pull/10910 Reviewed By: riversand963 Differential Revision: D40880683 Pulled By: ajkr fbshipit-source-id: cfe26cda05b3fb9a72e3cb82c286e21d8c5c4174
This commit is contained in:
parent
ff9ad2c39b
commit
5cf6ab6f31
|
@ -8,6 +8,7 @@
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
#include "db/arena_wrapped_db_iter.h"
|
#include "db/arena_wrapped_db_iter.h"
|
||||||
|
|
||||||
#include "memory/arena.h"
|
#include "memory/arena.h"
|
||||||
#include "rocksdb/env.h"
|
#include "rocksdb/env.h"
|
||||||
#include "rocksdb/iterator.h"
|
#include "rocksdb/iterator.h"
|
||||||
|
|
|
@ -9,7 +9,9 @@
|
||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "db/db_impl/db_impl.h"
|
#include "db/db_impl/db_impl.h"
|
||||||
#include "db/db_iter.h"
|
#include "db/db_iter.h"
|
||||||
#include "db/range_del_aggregator.h"
|
#include "db/range_del_aggregator.h"
|
||||||
|
|
|
@ -281,7 +281,8 @@ Status BuildTable(
|
||||||
meta->fd.file_size = file_size;
|
meta->fd.file_size = file_size;
|
||||||
meta->marked_for_compaction = builder->NeedCompact();
|
meta->marked_for_compaction = builder->NeedCompact();
|
||||||
assert(meta->fd.GetFileSize() > 0);
|
assert(meta->fd.GetFileSize() > 0);
|
||||||
tp = builder->GetTableProperties(); // refresh now that builder is finished
|
tp = builder
|
||||||
|
->GetTableProperties(); // refresh now that builder is finished
|
||||||
if (memtable_payload_bytes != nullptr &&
|
if (memtable_payload_bytes != nullptr &&
|
||||||
memtable_garbage_bytes != nullptr) {
|
memtable_garbage_bytes != nullptr) {
|
||||||
const CompactionIterationStats& ci_stats = c_iter.iter_stats();
|
const CompactionIterationStats& ci_stats = c_iter.iter_stats();
|
||||||
|
|
343
db/c_test.c
343
db/c_test.c
|
@ -49,32 +49,32 @@ static void StartPhase(const char* name) {
|
||||||
}
|
}
|
||||||
#ifdef _MSC_VER
|
#ifdef _MSC_VER
|
||||||
#pragma warning(push)
|
#pragma warning(push)
|
||||||
#pragma warning (disable: 4996) // getenv security warning
|
#pragma warning(disable : 4996) // getenv security warning
|
||||||
#endif
|
#endif
|
||||||
static const char* GetTempDir(void) {
|
static const char* GetTempDir(void) {
|
||||||
const char* ret = getenv("TEST_TMPDIR");
|
const char* ret = getenv("TEST_TMPDIR");
|
||||||
if (ret == NULL || ret[0] == '\0')
|
if (ret == NULL || ret[0] == '\0')
|
||||||
#ifdef OS_WIN
|
#ifdef OS_WIN
|
||||||
ret = getenv("TEMP");
|
ret = getenv("TEMP");
|
||||||
#else
|
#else
|
||||||
ret = "/tmp";
|
ret = "/tmp";
|
||||||
#endif
|
#endif
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
#ifdef _MSC_VER
|
#ifdef _MSC_VER
|
||||||
#pragma warning(pop)
|
#pragma warning(pop)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#define CheckNoError(err) \
|
#define CheckNoError(err) \
|
||||||
if ((err) != NULL) { \
|
if ((err) != NULL) { \
|
||||||
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \
|
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \
|
||||||
abort(); \
|
abort(); \
|
||||||
}
|
}
|
||||||
|
|
||||||
#define CheckCondition(cond) \
|
#define CheckCondition(cond) \
|
||||||
if (!(cond)) { \
|
if (!(cond)) { \
|
||||||
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \
|
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \
|
||||||
abort(); \
|
abort(); \
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckEqual(const char* expected, const char* v, size_t n) {
|
static void CheckEqual(const char* expected, const char* v, size_t n) {
|
||||||
|
@ -98,21 +98,15 @@ static void Free(char** ptr) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckValue(
|
static void CheckValue(char* err, const char* expected, char** actual,
|
||||||
char* err,
|
size_t actual_length) {
|
||||||
const char* expected,
|
|
||||||
char** actual,
|
|
||||||
size_t actual_length) {
|
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
CheckEqual(expected, *actual, actual_length);
|
CheckEqual(expected, *actual, actual_length);
|
||||||
Free(actual);
|
Free(actual);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckGet(
|
static void CheckGet(rocksdb_t* db, const rocksdb_readoptions_t* options,
|
||||||
rocksdb_t* db,
|
const char* key, const char* expected) {
|
||||||
const rocksdb_readoptions_t* options,
|
|
||||||
const char* key,
|
|
||||||
const char* expected) {
|
|
||||||
char* err = NULL;
|
char* err = NULL;
|
||||||
size_t val_len;
|
size_t val_len;
|
||||||
char* val;
|
char* val;
|
||||||
|
@ -122,12 +116,9 @@ static void CheckGet(
|
||||||
Free(&val);
|
Free(&val);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckGetCF(
|
static void CheckGetCF(rocksdb_t* db, const rocksdb_readoptions_t* options,
|
||||||
rocksdb_t* db,
|
rocksdb_column_family_handle_t* handle, const char* key,
|
||||||
const rocksdb_readoptions_t* options,
|
const char* expected) {
|
||||||
rocksdb_column_family_handle_t* handle,
|
|
||||||
const char* key,
|
|
||||||
const char* expected) {
|
|
||||||
char* err = NULL;
|
char* err = NULL;
|
||||||
size_t val_len;
|
size_t val_len;
|
||||||
char* val;
|
char* val;
|
||||||
|
@ -174,8 +165,8 @@ static void CheckMultiGetValues(size_t num_keys, char** values,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckIter(rocksdb_iterator_t* iter,
|
static void CheckIter(rocksdb_iterator_t* iter, const char* key,
|
||||||
const char* key, const char* val) {
|
const char* val) {
|
||||||
size_t len;
|
size_t len;
|
||||||
const char* str;
|
const char* str;
|
||||||
str = rocksdb_iter_key(iter, &len);
|
str = rocksdb_iter_key(iter, &len);
|
||||||
|
@ -185,10 +176,9 @@ static void CheckIter(rocksdb_iterator_t* iter,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Callback from rocksdb_writebatch_iterate()
|
// Callback from rocksdb_writebatch_iterate()
|
||||||
static void CheckPut(void* ptr,
|
static void CheckPut(void* ptr, const char* k, size_t klen, const char* v,
|
||||||
const char* k, size_t klen,
|
size_t vlen) {
|
||||||
const char* v, size_t vlen) {
|
int* state = (int*)ptr;
|
||||||
int* state = (int*) ptr;
|
|
||||||
CheckCondition(*state < 2);
|
CheckCondition(*state < 2);
|
||||||
switch (*state) {
|
switch (*state) {
|
||||||
case 0:
|
case 0:
|
||||||
|
@ -205,7 +195,7 @@ static void CheckPut(void* ptr,
|
||||||
|
|
||||||
// Callback from rocksdb_writebatch_iterate()
|
// Callback from rocksdb_writebatch_iterate()
|
||||||
static void CheckDel(void* ptr, const char* k, size_t klen) {
|
static void CheckDel(void* ptr, const char* k, size_t klen) {
|
||||||
int* state = (int*) ptr;
|
int* state = (int*)ptr;
|
||||||
CheckCondition(*state == 2);
|
CheckCondition(*state == 2);
|
||||||
CheckEqual("bar", k, klen);
|
CheckEqual("bar", k, klen);
|
||||||
(*state)++;
|
(*state)++;
|
||||||
|
@ -213,14 +203,16 @@ static void CheckDel(void* ptr, const char* k, size_t klen) {
|
||||||
|
|
||||||
static void CmpDestroy(void* arg) { (void)arg; }
|
static void CmpDestroy(void* arg) { (void)arg; }
|
||||||
|
|
||||||
static int CmpCompare(void* arg, const char* a, size_t alen,
|
static int CmpCompare(void* arg, const char* a, size_t alen, const char* b,
|
||||||
const char* b, size_t blen) {
|
size_t blen) {
|
||||||
(void)arg;
|
(void)arg;
|
||||||
size_t n = (alen < blen) ? alen : blen;
|
size_t n = (alen < blen) ? alen : blen;
|
||||||
int r = memcmp(a, b, n);
|
int r = memcmp(a, b, n);
|
||||||
if (r == 0) {
|
if (r == 0) {
|
||||||
if (alen < blen) r = -1;
|
if (alen < blen)
|
||||||
else if (alen > blen) r = +1;
|
r = -1;
|
||||||
|
else if (alen > blen)
|
||||||
|
r = +1;
|
||||||
}
|
}
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
@ -405,11 +397,9 @@ static const char* MergeOperatorName(void* arg) {
|
||||||
return "TestMergeOperator";
|
return "TestMergeOperator";
|
||||||
}
|
}
|
||||||
static char* MergeOperatorFullMerge(
|
static char* MergeOperatorFullMerge(
|
||||||
void* arg,
|
void* arg, const char* key, size_t key_length, const char* existing_value,
|
||||||
const char* key, size_t key_length,
|
size_t existing_value_length, const char* const* operands_list,
|
||||||
const char* existing_value, size_t existing_value_length,
|
const size_t* operands_list_length, int num_operands,
|
||||||
const char* const* operands_list, const size_t* operands_list_length,
|
|
||||||
int num_operands,
|
|
||||||
unsigned char* success, size_t* new_value_length) {
|
unsigned char* success, size_t* new_value_length) {
|
||||||
(void)arg;
|
(void)arg;
|
||||||
(void)key;
|
(void)key;
|
||||||
|
@ -425,12 +415,12 @@ static char* MergeOperatorFullMerge(
|
||||||
memcpy(result, "fake", 4);
|
memcpy(result, "fake", 4);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
static char* MergeOperatorPartialMerge(
|
static char* MergeOperatorPartialMerge(void* arg, const char* key,
|
||||||
void* arg,
|
size_t key_length,
|
||||||
const char* key, size_t key_length,
|
const char* const* operands_list,
|
||||||
const char* const* operands_list, const size_t* operands_list_length,
|
const size_t* operands_list_length,
|
||||||
int num_operands,
|
int num_operands, unsigned char* success,
|
||||||
unsigned char* success, size_t* new_value_length) {
|
size_t* new_value_length) {
|
||||||
(void)arg;
|
(void)arg;
|
||||||
(void)key;
|
(void)key;
|
||||||
(void)key_length;
|
(void)key_length;
|
||||||
|
@ -444,18 +434,16 @@ static char* MergeOperatorPartialMerge(
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckTxnGet(
|
static void CheckTxnGet(rocksdb_transaction_t* txn,
|
||||||
rocksdb_transaction_t* txn,
|
const rocksdb_readoptions_t* options, const char* key,
|
||||||
const rocksdb_readoptions_t* options,
|
const char* expected) {
|
||||||
const char* key,
|
char* err = NULL;
|
||||||
const char* expected) {
|
size_t val_len;
|
||||||
char* err = NULL;
|
char* val;
|
||||||
size_t val_len;
|
val = rocksdb_transaction_get(txn, options, key, strlen(key), &val_len, &err);
|
||||||
char* val;
|
CheckNoError(err);
|
||||||
val = rocksdb_transaction_get(txn, options, key, strlen(key), &val_len, &err);
|
CheckEqual(expected, val, val_len);
|
||||||
CheckNoError(err);
|
Free(&val);
|
||||||
CheckEqual(expected, val, val_len);
|
|
||||||
Free(&val);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckTxnGetCF(rocksdb_transaction_t* txn,
|
static void CheckTxnGetCF(rocksdb_transaction_t* txn,
|
||||||
|
@ -502,11 +490,9 @@ static void CheckTxnPinGetCF(rocksdb_transaction_t* txn,
|
||||||
rocksdb_pinnableslice_destroy(p);
|
rocksdb_pinnableslice_destroy(p);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void CheckTxnDBGet(
|
static void CheckTxnDBGet(rocksdb_transactiondb_t* txn_db,
|
||||||
rocksdb_transactiondb_t* txn_db,
|
const rocksdb_readoptions_t* options, const char* key,
|
||||||
const rocksdb_readoptions_t* options,
|
const char* expected) {
|
||||||
const char* key,
|
|
||||||
const char* expected) {
|
|
||||||
char* err = NULL;
|
char* err = NULL;
|
||||||
size_t val_len;
|
size_t val_len;
|
||||||
char* val;
|
char* val;
|
||||||
|
@ -632,7 +618,7 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_t* db;
|
rocksdb_t* db;
|
||||||
rocksdb_comparator_t* cmp;
|
rocksdb_comparator_t* cmp;
|
||||||
rocksdb_cache_t* cache;
|
rocksdb_cache_t* cache;
|
||||||
rocksdb_dbpath_t *dbpath;
|
rocksdb_dbpath_t* dbpath;
|
||||||
rocksdb_env_t* env;
|
rocksdb_env_t* env;
|
||||||
rocksdb_options_t* options;
|
rocksdb_options_t* options;
|
||||||
rocksdb_compactoptions_t* coptions;
|
rocksdb_compactoptions_t* coptions;
|
||||||
|
@ -649,30 +635,20 @@ int main(int argc, char** argv) {
|
||||||
char* err = NULL;
|
char* err = NULL;
|
||||||
int run = -1;
|
int run = -1;
|
||||||
|
|
||||||
snprintf(dbname, sizeof(dbname),
|
snprintf(dbname, sizeof(dbname), "%s/rocksdb_c_test-%d", GetTempDir(),
|
||||||
"%s/rocksdb_c_test-%d",
|
|
||||||
GetTempDir(),
|
|
||||||
((int) geteuid()));
|
|
||||||
|
|
||||||
snprintf(dbbackupname, sizeof(dbbackupname),
|
|
||||||
"%s/rocksdb_c_test-%d-backup",
|
|
||||||
GetTempDir(),
|
|
||||||
((int) geteuid()));
|
|
||||||
|
|
||||||
snprintf(dbcheckpointname, sizeof(dbcheckpointname),
|
|
||||||
"%s/rocksdb_c_test-%d-checkpoint",
|
|
||||||
GetTempDir(),
|
|
||||||
((int) geteuid()));
|
|
||||||
|
|
||||||
snprintf(sstfilename, sizeof(sstfilename),
|
|
||||||
"%s/rocksdb_c_test-%d-sst",
|
|
||||||
GetTempDir(),
|
|
||||||
((int)geteuid()));
|
((int)geteuid()));
|
||||||
|
|
||||||
snprintf(dbpathname, sizeof(dbpathname),
|
snprintf(dbbackupname, sizeof(dbbackupname), "%s/rocksdb_c_test-%d-backup",
|
||||||
"%s/rocksdb_c_test-%d-dbpath",
|
GetTempDir(), ((int)geteuid()));
|
||||||
GetTempDir(),
|
|
||||||
((int) geteuid()));
|
snprintf(dbcheckpointname, sizeof(dbcheckpointname),
|
||||||
|
"%s/rocksdb_c_test-%d-checkpoint", GetTempDir(), ((int)geteuid()));
|
||||||
|
|
||||||
|
snprintf(sstfilename, sizeof(sstfilename), "%s/rocksdb_c_test-%d-sst",
|
||||||
|
GetTempDir(), ((int)geteuid()));
|
||||||
|
|
||||||
|
snprintf(dbpathname, sizeof(dbpathname), "%s/rocksdb_c_test-%d-dbpath",
|
||||||
|
GetTempDir(), ((int)geteuid()));
|
||||||
|
|
||||||
StartPhase("create_objects");
|
StartPhase("create_objects");
|
||||||
cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
|
cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
|
||||||
|
@ -746,7 +722,8 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_destroy_db(options, dbbackupname, &err);
|
rocksdb_destroy_db(options, dbbackupname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_backup_engine_t *be = rocksdb_backup_engine_open(options, dbbackupname, &err);
|
rocksdb_backup_engine_t* be =
|
||||||
|
rocksdb_backup_engine_open(options, dbbackupname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_backup_engine_create_new_backup(be, db, &err);
|
rocksdb_backup_engine_create_new_backup(be, db, &err);
|
||||||
|
@ -759,7 +736,8 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_backup_engine_create_new_backup(be, db, &err);
|
rocksdb_backup_engine_create_new_backup(be, db, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
const rocksdb_backup_engine_info_t* bei = rocksdb_backup_engine_get_backup_info(be);
|
const rocksdb_backup_engine_info_t* bei =
|
||||||
|
rocksdb_backup_engine_get_backup_info(be);
|
||||||
CheckCondition(rocksdb_backup_engine_info_count(bei) > 1);
|
CheckCondition(rocksdb_backup_engine_info_count(bei) > 1);
|
||||||
rocksdb_backup_engine_info_destroy(bei);
|
rocksdb_backup_engine_info_destroy(bei);
|
||||||
|
|
||||||
|
@ -778,9 +756,11 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_destroy_db(options, dbname, &err);
|
rocksdb_destroy_db(options, dbname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_restore_options_t *restore_options = rocksdb_restore_options_create();
|
rocksdb_restore_options_t* restore_options =
|
||||||
|
rocksdb_restore_options_create();
|
||||||
rocksdb_restore_options_set_keep_log_files(restore_options, 0);
|
rocksdb_restore_options_set_keep_log_files(restore_options, 0);
|
||||||
rocksdb_backup_engine_restore_db_from_latest_backup(be, dbname, dbname, restore_options, &err);
|
rocksdb_backup_engine_restore_db_from_latest_backup(be, dbname, dbname,
|
||||||
|
restore_options, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
rocksdb_restore_options_destroy(restore_options);
|
rocksdb_restore_options_destroy(restore_options);
|
||||||
|
|
||||||
|
@ -799,7 +779,8 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_destroy_db(options, dbcheckpointname, &err);
|
rocksdb_destroy_db(options, dbcheckpointname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_checkpoint_t* checkpoint = rocksdb_checkpoint_object_create(db, &err);
|
rocksdb_checkpoint_t* checkpoint =
|
||||||
|
rocksdb_checkpoint_object_create(db, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err);
|
rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err);
|
||||||
|
@ -976,10 +957,10 @@ int main(int argc, char** argv) {
|
||||||
StartPhase("writebatch_vectors");
|
StartPhase("writebatch_vectors");
|
||||||
{
|
{
|
||||||
rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
|
rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
|
||||||
const char* k_list[2] = { "z", "ap" };
|
const char* k_list[2] = {"z", "ap"};
|
||||||
const size_t k_sizes[2] = { 1, 2 };
|
const size_t k_sizes[2] = {1, 2};
|
||||||
const char* v_list[3] = { "x", "y", "z" };
|
const char* v_list[3] = {"x", "y", "z"};
|
||||||
const size_t v_sizes[3] = { 1, 1, 1 };
|
const size_t v_sizes[3] = {1, 1, 1};
|
||||||
rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
|
rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
|
||||||
rocksdb_write(db, woptions, wb, &err);
|
rocksdb_write(db, woptions, wb, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
@ -1041,13 +1022,17 @@ int main(int argc, char** argv) {
|
||||||
CheckCondition(count == 3);
|
CheckCondition(count == 3);
|
||||||
size_t size;
|
size_t size;
|
||||||
char* value;
|
char* value;
|
||||||
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "box", 3, &size, &err);
|
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "box", 3, &size,
|
||||||
|
&err);
|
||||||
CheckValue(err, "c", &value, size);
|
CheckValue(err, "c", &value, size);
|
||||||
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "bar", 3, &size, &err);
|
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "bar", 3, &size,
|
||||||
|
&err);
|
||||||
CheckValue(err, NULL, &value, size);
|
CheckValue(err, NULL, &value, size);
|
||||||
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "foo", 3, &size, &err);
|
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions,
|
||||||
|
"foo", 3, &size, &err);
|
||||||
CheckValue(err, "hello", &value, size);
|
CheckValue(err, "hello", &value, size);
|
||||||
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "box", 3, &size, &err);
|
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions,
|
||||||
|
"box", 3, &size, &err);
|
||||||
CheckValue(err, "c", &value, size);
|
CheckValue(err, "c", &value, size);
|
||||||
rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
|
rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
@ -1064,10 +1049,10 @@ int main(int argc, char** argv) {
|
||||||
StartPhase("writebatch_wi_vectors");
|
StartPhase("writebatch_wi_vectors");
|
||||||
{
|
{
|
||||||
rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1);
|
rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1);
|
||||||
const char* k_list[2] = { "z", "ap" };
|
const char* k_list[2] = {"z", "ap"};
|
||||||
const size_t k_sizes[2] = { 1, 2 };
|
const size_t k_sizes[2] = {1, 2};
|
||||||
const char* v_list[3] = { "x", "y", "z" };
|
const char* v_list[3] = {"x", "y", "z"};
|
||||||
const size_t v_sizes[3] = { 1, 1, 1 };
|
const size_t v_sizes[3] = {1, 1, 1};
|
||||||
rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
|
rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
|
||||||
rocksdb_write_writebatch_wi(db, woptions, wb, &err);
|
rocksdb_write_writebatch_wi(db, woptions, wb, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
@ -1156,13 +1141,14 @@ int main(int argc, char** argv) {
|
||||||
|
|
||||||
StartPhase("multiget");
|
StartPhase("multiget");
|
||||||
{
|
{
|
||||||
const char* keys[3] = { "box", "foo", "notfound" };
|
const char* keys[3] = {"box", "foo", "notfound"};
|
||||||
const size_t keys_sizes[3] = { 3, 3, 8 };
|
const size_t keys_sizes[3] = {3, 3, 8};
|
||||||
char* vals[3];
|
char* vals[3];
|
||||||
size_t vals_sizes[3];
|
size_t vals_sizes[3];
|
||||||
char* errs[3];
|
char* errs[3];
|
||||||
const char* expected[3] = {"c", "hello", NULL};
|
const char* expected[3] = {"c", "hello", NULL};
|
||||||
rocksdb_multi_get(db, roptions, 3, keys, keys_sizes, vals, vals_sizes, errs);
|
rocksdb_multi_get(db, roptions, 3, keys, keys_sizes, vals, vals_sizes,
|
||||||
|
errs);
|
||||||
CheckMultiGetValues(3, vals, vals_sizes, errs, expected);
|
CheckMultiGetValues(3, vals, vals_sizes, errs, expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1180,10 +1166,10 @@ int main(int argc, char** argv) {
|
||||||
char keybuf[100];
|
char keybuf[100];
|
||||||
char valbuf[100];
|
char valbuf[100];
|
||||||
uint64_t sizes[2];
|
uint64_t sizes[2];
|
||||||
const char* start[2] = { "a", "k00000000000000010000" };
|
const char* start[2] = {"a", "k00000000000000010000"};
|
||||||
size_t start_len[2] = { 1, 21 };
|
size_t start_len[2] = {1, 21};
|
||||||
const char* limit[2] = { "k00000000000000010000", "z" };
|
const char* limit[2] = {"k00000000000000010000", "z"};
|
||||||
size_t limit_len[2] = { 21, 1 };
|
size_t limit_len[2] = {21, 1};
|
||||||
rocksdb_writeoptions_set_sync(woptions, 0);
|
rocksdb_writeoptions_set_sync(woptions, 0);
|
||||||
for (i = 0; i < n; i++) {
|
for (i = 0; i < n; i++) {
|
||||||
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
|
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
|
||||||
|
@ -1393,8 +1379,8 @@ int main(int argc, char** argv) {
|
||||||
factory);
|
factory);
|
||||||
db = CheckCompaction(db, options_with_filter_factory, roptions, woptions);
|
db = CheckCompaction(db, options_with_filter_factory, roptions, woptions);
|
||||||
|
|
||||||
rocksdb_options_set_compaction_filter_factory(
|
rocksdb_options_set_compaction_filter_factory(options_with_filter_factory,
|
||||||
options_with_filter_factory, NULL);
|
NULL);
|
||||||
rocksdb_options_destroy(options_with_filter_factory);
|
rocksdb_options_destroy(options_with_filter_factory);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1449,7 +1435,8 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_close(db);
|
rocksdb_close(db);
|
||||||
|
|
||||||
size_t cflen;
|
size_t cflen;
|
||||||
char** column_fams = rocksdb_list_column_families(db_options, dbname, &cflen, &err);
|
char** column_fams =
|
||||||
|
rocksdb_list_column_families(db_options, dbname, &cflen, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
CheckEqual("default", column_fams[0], 7);
|
CheckEqual("default", column_fams[0], 7);
|
||||||
CheckEqual("cf1", column_fams[1], 3);
|
CheckEqual("cf1", column_fams[1], 3);
|
||||||
|
@ -1465,7 +1452,8 @@ int main(int argc, char** argv) {
|
||||||
LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names,
|
LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names,
|
||||||
NULL);
|
NULL);
|
||||||
|
|
||||||
db = rocksdb_open_column_families(db_options, dbname, 2, cf_names, cf_opts, handles, &err);
|
db = rocksdb_open_column_families(db_options, dbname, 2, cf_names, cf_opts,
|
||||||
|
handles, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_put_cf(db, woptions, handles[1], "foo", 3, "hello", 5, &err);
|
rocksdb_put_cf(db, woptions, handles[1], "foo", 3, "hello", 5, &err);
|
||||||
|
@ -1483,11 +1471,10 @@ int main(int argc, char** argv) {
|
||||||
&err);
|
&err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_flushoptions_t *flush_options = rocksdb_flushoptions_create();
|
rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create();
|
||||||
rocksdb_flushoptions_set_wait(flush_options, 1);
|
rocksdb_flushoptions_set_wait(flush_options, 1);
|
||||||
rocksdb_flush_cf(db, flush_options, handles[1], &err);
|
rocksdb_flush_cf(db, flush_options, handles[1], &err);
|
||||||
CheckNoError(err)
|
CheckNoError(err) rocksdb_flushoptions_destroy(flush_options);
|
||||||
rocksdb_flushoptions_destroy(flush_options);
|
|
||||||
|
|
||||||
CheckGetCF(db, roptions, handles[1], "foo", "hello");
|
CheckGetCF(db, roptions, handles[1], "foo", "hello");
|
||||||
CheckPinGetCF(db, roptions, handles[1], "foo", "hello");
|
CheckPinGetCF(db, roptions, handles[1], "foo", "hello");
|
||||||
|
@ -1524,27 +1511,29 @@ int main(int argc, char** argv) {
|
||||||
rocksdb_flush_wal(db, 1, &err);
|
rocksdb_flush_wal(db, 1, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
const char* keys[3] = { "box", "box", "barfooxx" };
|
const char* keys[3] = {"box", "box", "barfooxx"};
|
||||||
const rocksdb_column_family_handle_t* get_handles[3] = { handles[0], handles[1], handles[1] };
|
const rocksdb_column_family_handle_t* get_handles[3] = {
|
||||||
const size_t keys_sizes[3] = { 3, 3, 8 };
|
handles[0], handles[1], handles[1]};
|
||||||
|
const size_t keys_sizes[3] = {3, 3, 8};
|
||||||
char* vals[3];
|
char* vals[3];
|
||||||
size_t vals_sizes[3];
|
size_t vals_sizes[3];
|
||||||
char* errs[3];
|
char* errs[3];
|
||||||
rocksdb_multi_get_cf(db, roptions, get_handles, 3, keys, keys_sizes, vals, vals_sizes, errs);
|
rocksdb_multi_get_cf(db, roptions, get_handles, 3, keys, keys_sizes, vals,
|
||||||
|
vals_sizes, errs);
|
||||||
|
|
||||||
int i;
|
int i;
|
||||||
for (i = 0; i < 3; i++) {
|
for (i = 0; i < 3; i++) {
|
||||||
CheckEqual(NULL, errs[i], 0);
|
CheckEqual(NULL, errs[i], 0);
|
||||||
switch (i) {
|
switch (i) {
|
||||||
case 0:
|
case 0:
|
||||||
CheckEqual(NULL, vals[i], vals_sizes[i]); // wrong cf
|
CheckEqual(NULL, vals[i], vals_sizes[i]); // wrong cf
|
||||||
break;
|
break;
|
||||||
case 1:
|
case 1:
|
||||||
CheckEqual("c", vals[i], vals_sizes[i]); // bingo
|
CheckEqual("c", vals[i], vals_sizes[i]); // bingo
|
||||||
break;
|
break;
|
||||||
case 2:
|
case 2:
|
||||||
CheckEqual(NULL, vals[i], vals_sizes[i]); // normal not found
|
CheckEqual(NULL, vals[i], vals_sizes[i]); // normal not found
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Free(&vals[i]);
|
Free(&vals[i]);
|
||||||
}
|
}
|
||||||
|
@ -1592,7 +1581,8 @@ int main(int argc, char** argv) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rocksdb_iterator_t* iter = rocksdb_create_iterator_cf(db, roptions, handles[1]);
|
rocksdb_iterator_t* iter =
|
||||||
|
rocksdb_create_iterator_cf(db, roptions, handles[1]);
|
||||||
CheckCondition(!rocksdb_iter_valid(iter));
|
CheckCondition(!rocksdb_iter_valid(iter));
|
||||||
rocksdb_iter_seek_to_first(iter);
|
rocksdb_iter_seek_to_first(iter);
|
||||||
CheckCondition(rocksdb_iter_valid(iter));
|
CheckCondition(rocksdb_iter_valid(iter));
|
||||||
|
@ -1605,9 +1595,11 @@ int main(int argc, char** argv) {
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
rocksdb_iter_destroy(iter);
|
rocksdb_iter_destroy(iter);
|
||||||
|
|
||||||
rocksdb_column_family_handle_t* iters_cf_handles[2] = { handles[0], handles[1] };
|
rocksdb_column_family_handle_t* iters_cf_handles[2] = {handles[0],
|
||||||
|
handles[1]};
|
||||||
rocksdb_iterator_t* iters_handles[2];
|
rocksdb_iterator_t* iters_handles[2];
|
||||||
rocksdb_create_iterators(db, roptions, iters_cf_handles, iters_handles, 2, &err);
|
rocksdb_create_iterators(db, roptions, iters_cf_handles, iters_handles, 2,
|
||||||
|
&err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
iter = iters_handles[0];
|
iter = iters_handles[0];
|
||||||
|
@ -1652,7 +1644,8 @@ int main(int argc, char** argv) {
|
||||||
{
|
{
|
||||||
// Create new database
|
// Create new database
|
||||||
rocksdb_options_set_allow_mmap_reads(options, 1);
|
rocksdb_options_set_allow_mmap_reads(options, 1);
|
||||||
rocksdb_options_set_prefix_extractor(options, rocksdb_slicetransform_create_fixed_prefix(3));
|
rocksdb_options_set_prefix_extractor(
|
||||||
|
options, rocksdb_slicetransform_create_fixed_prefix(3));
|
||||||
rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
|
rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
|
||||||
rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16);
|
rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16);
|
||||||
rocksdb_options_set_allow_concurrent_memtable_write(options, 0);
|
rocksdb_options_set_allow_concurrent_memtable_write(options, 0);
|
||||||
|
@ -1747,8 +1740,9 @@ int main(int argc, char** argv) {
|
||||||
// amount of memory used within memtables should grow
|
// amount of memory used within memtables should grow
|
||||||
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >=
|
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >=
|
||||||
rocksdb_approximate_memory_usage_get_mem_table_total(mu1));
|
rocksdb_approximate_memory_usage_get_mem_table_total(mu1));
|
||||||
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >=
|
CheckCondition(
|
||||||
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1));
|
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >=
|
||||||
|
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1));
|
||||||
|
|
||||||
rocksdb_memory_consumers_destroy(consumers);
|
rocksdb_memory_consumers_destroy(consumers);
|
||||||
rocksdb_approximate_memory_usage_destroy(mu1);
|
rocksdb_approximate_memory_usage_destroy(mu1);
|
||||||
|
@ -2839,53 +2833,57 @@ int main(int argc, char** argv) {
|
||||||
db = rocksdb_open(options, dbname, &err);
|
db = rocksdb_open(options, dbname, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
rocksdb_put(db, woptions, "a", 1, "0", 1, &err); CheckNoError(err);
|
rocksdb_put(db, woptions, "a", 1, "0", 1, &err);
|
||||||
rocksdb_put(db, woptions, "foo", 3, "bar", 3, &err); CheckNoError(err);
|
CheckNoError(err);
|
||||||
rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err); CheckNoError(err);
|
rocksdb_put(db, woptions, "foo", 3, "bar", 3, &err);
|
||||||
rocksdb_put(db, woptions, "g1", 2, "0", 1, &err); CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err);
|
||||||
|
CheckNoError(err);
|
||||||
|
rocksdb_put(db, woptions, "g1", 2, "0", 1, &err);
|
||||||
|
CheckNoError(err);
|
||||||
|
|
||||||
// testing basic case with no iterate_upper_bound and no prefix_extractor
|
// testing basic case with no iterate_upper_bound and no prefix_extractor
|
||||||
{
|
{
|
||||||
rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0);
|
rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0);
|
||||||
rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
|
rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
|
||||||
|
|
||||||
rocksdb_iter_seek(iter, "foo", 3);
|
rocksdb_iter_seek(iter, "foo", 3);
|
||||||
CheckCondition(rocksdb_iter_valid(iter));
|
CheckCondition(rocksdb_iter_valid(iter));
|
||||||
CheckIter(iter, "foo", "bar");
|
CheckIter(iter, "foo", "bar");
|
||||||
|
|
||||||
rocksdb_iter_next(iter);
|
rocksdb_iter_next(iter);
|
||||||
CheckCondition(rocksdb_iter_valid(iter));
|
CheckCondition(rocksdb_iter_valid(iter));
|
||||||
CheckIter(iter, "foo1", "bar1");
|
CheckIter(iter, "foo1", "bar1");
|
||||||
|
|
||||||
rocksdb_iter_next(iter);
|
rocksdb_iter_next(iter);
|
||||||
CheckCondition(rocksdb_iter_valid(iter));
|
CheckCondition(rocksdb_iter_valid(iter));
|
||||||
CheckIter(iter, "g1", "0");
|
CheckIter(iter, "g1", "0");
|
||||||
|
|
||||||
rocksdb_iter_destroy(iter);
|
rocksdb_iter_destroy(iter);
|
||||||
}
|
}
|
||||||
|
|
||||||
// testing iterate_upper_bound and forward iterator
|
// testing iterate_upper_bound and forward iterator
|
||||||
// to make sure it stops at bound
|
// to make sure it stops at bound
|
||||||
{
|
{
|
||||||
// iterate_upper_bound points beyond the last expected entry
|
// iterate_upper_bound points beyond the last expected entry
|
||||||
rocksdb_readoptions_set_iterate_upper_bound(roptions, "foo2", 4);
|
rocksdb_readoptions_set_iterate_upper_bound(roptions, "foo2", 4);
|
||||||
|
|
||||||
rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
|
rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
|
||||||
|
|
||||||
rocksdb_iter_seek(iter, "foo", 3);
|
rocksdb_iter_seek(iter, "foo", 3);
|
||||||
CheckCondition(rocksdb_iter_valid(iter));
|
CheckCondition(rocksdb_iter_valid(iter));
|
||||||
CheckIter(iter, "foo", "bar");
|
CheckIter(iter, "foo", "bar");
|
||||||
|
|
||||||
rocksdb_iter_next(iter);
|
rocksdb_iter_next(iter);
|
||||||
CheckCondition(rocksdb_iter_valid(iter));
|
CheckCondition(rocksdb_iter_valid(iter));
|
||||||
CheckIter(iter, "foo1", "bar1");
|
CheckIter(iter, "foo1", "bar1");
|
||||||
|
|
||||||
rocksdb_iter_next(iter);
|
rocksdb_iter_next(iter);
|
||||||
// should stop here...
|
// should stop here...
|
||||||
CheckCondition(!rocksdb_iter_valid(iter));
|
CheckCondition(!rocksdb_iter_valid(iter));
|
||||||
|
|
||||||
rocksdb_iter_destroy(iter);
|
rocksdb_iter_destroy(iter);
|
||||||
rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0);
|
rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3009,7 +3007,7 @@ int main(int argc, char** argv) {
|
||||||
snapshot = rocksdb_transactiondb_create_snapshot(txn_db);
|
snapshot = rocksdb_transactiondb_create_snapshot(txn_db);
|
||||||
rocksdb_readoptions_set_snapshot(roptions, snapshot);
|
rocksdb_readoptions_set_snapshot(roptions, snapshot);
|
||||||
|
|
||||||
rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hey", 3, &err);
|
rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hey", 3, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
|
|
||||||
CheckTxnDBGet(txn_db, roptions, "foo", "hello");
|
CheckTxnDBGet(txn_db, roptions, "foo", "hello");
|
||||||
|
@ -3021,7 +3019,8 @@ int main(int argc, char** argv) {
|
||||||
|
|
||||||
// iterate
|
// iterate
|
||||||
rocksdb_transaction_put(txn, "bar", 3, "hi", 2, &err);
|
rocksdb_transaction_put(txn, "bar", 3, "hi", 2, &err);
|
||||||
rocksdb_iterator_t* iter = rocksdb_transaction_create_iterator(txn, roptions);
|
rocksdb_iterator_t* iter =
|
||||||
|
rocksdb_transaction_create_iterator(txn, roptions);
|
||||||
CheckCondition(!rocksdb_iter_valid(iter));
|
CheckCondition(!rocksdb_iter_valid(iter));
|
||||||
rocksdb_iter_seek_to_first(iter);
|
rocksdb_iter_seek_to_first(iter);
|
||||||
CheckCondition(rocksdb_iter_valid(iter));
|
CheckCondition(rocksdb_iter_valid(iter));
|
||||||
|
|
|
@ -192,8 +192,7 @@ Status CheckCFPathsSupported(const DBOptions& db_options,
|
||||||
return Status::NotSupported(
|
return Status::NotSupported(
|
||||||
"More than one CF paths are only supported in "
|
"More than one CF paths are only supported in "
|
||||||
"universal and level compaction styles. ");
|
"universal and level compaction styles. ");
|
||||||
} else if (cf_options.cf_paths.empty() &&
|
} else if (cf_options.cf_paths.empty() && db_options.db_paths.size() > 1) {
|
||||||
db_options.db_paths.size() > 1) {
|
|
||||||
return Status::NotSupported(
|
return Status::NotSupported(
|
||||||
"More than one DB paths are only supported in "
|
"More than one DB paths are only supported in "
|
||||||
"universal and level compaction styles. ");
|
"universal and level compaction styles. ");
|
||||||
|
@ -205,7 +204,7 @@ Status CheckCFPathsSupported(const DBOptions& db_options,
|
||||||
namespace {
|
namespace {
|
||||||
const uint64_t kDefaultTtl = 0xfffffffffffffffe;
|
const uint64_t kDefaultTtl = 0xfffffffffffffffe;
|
||||||
const uint64_t kDefaultPeriodicCompSecs = 0xfffffffffffffffe;
|
const uint64_t kDefaultPeriodicCompSecs = 0xfffffffffffffffe;
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
|
ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
|
||||||
const ColumnFamilyOptions& src) {
|
const ColumnFamilyOptions& src) {
|
||||||
|
@ -353,7 +352,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
|
||||||
// were not deleted yet, when we open the DB we will find these .trash files
|
// were not deleted yet, when we open the DB we will find these .trash files
|
||||||
// and schedule them to be deleted (or delete immediately if SstFileManager
|
// and schedule them to be deleted (or delete immediately if SstFileManager
|
||||||
// was not used)
|
// was not used)
|
||||||
auto sfm = static_cast<SstFileManagerImpl*>(db_options.sst_file_manager.get());
|
auto sfm =
|
||||||
|
static_cast<SstFileManagerImpl*>(db_options.sst_file_manager.get());
|
||||||
for (size_t i = 0; i < result.cf_paths.size(); i++) {
|
for (size_t i = 0; i < result.cf_paths.size(); i++) {
|
||||||
DeleteScheduler::CleanupDirectory(db_options.env, sfm,
|
DeleteScheduler::CleanupDirectory(db_options.env, sfm,
|
||||||
result.cf_paths[i].path)
|
result.cf_paths[i].path)
|
||||||
|
@ -610,8 +610,8 @@ ColumnFamilyData::ColumnFamilyData(
|
||||||
compaction_picker_.reset(
|
compaction_picker_.reset(
|
||||||
new FIFOCompactionPicker(ioptions_, &internal_comparator_));
|
new FIFOCompactionPicker(ioptions_, &internal_comparator_));
|
||||||
} else if (ioptions_.compaction_style == kCompactionStyleNone) {
|
} else if (ioptions_.compaction_style == kCompactionStyleNone) {
|
||||||
compaction_picker_.reset(new NullCompactionPicker(
|
compaction_picker_.reset(
|
||||||
ioptions_, &internal_comparator_));
|
new NullCompactionPicker(ioptions_, &internal_comparator_));
|
||||||
ROCKS_LOG_WARN(ioptions_.logger,
|
ROCKS_LOG_WARN(ioptions_.logger,
|
||||||
"Column family %s does not use any background compaction. "
|
"Column family %s does not use any background compaction. "
|
||||||
"Compactions can only be done via CompactFiles\n",
|
"Compactions can only be done via CompactFiles\n",
|
||||||
|
@ -878,7 +878,7 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
|
||||||
return static_cast<int>(res);
|
return static_cast<int>(res);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause>
|
std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause>
|
||||||
ColumnFamilyData::GetWriteStallConditionAndCause(
|
ColumnFamilyData::GetWriteStallConditionAndCause(
|
||||||
|
@ -919,7 +919,7 @@ ColumnFamilyData::GetWriteStallConditionAndCause(
|
||||||
}
|
}
|
||||||
|
|
||||||
WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||||
const MutableCFOptions& mutable_cf_options) {
|
const MutableCFOptions& mutable_cf_options) {
|
||||||
auto write_stall_condition = WriteStallCondition::kNormal;
|
auto write_stall_condition = WriteStallCondition::kNormal;
|
||||||
if (current_ != nullptr) {
|
if (current_ != nullptr) {
|
||||||
auto* vstorage = current_->storage_info();
|
auto* vstorage = current_->storage_info();
|
||||||
|
@ -1012,7 +1012,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||||
mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
|
mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
|
||||||
(compaction_needed_bytes -
|
(compaction_needed_bytes -
|
||||||
mutable_cf_options.soft_pending_compaction_bytes_limit) >
|
mutable_cf_options.soft_pending_compaction_bytes_limit) >
|
||||||
3 * (mutable_cf_options.hard_pending_compaction_bytes_limit -
|
3 *
|
||||||
|
(mutable_cf_options.hard_pending_compaction_bytes_limit -
|
||||||
mutable_cf_options.soft_pending_compaction_bytes_limit) /
|
mutable_cf_options.soft_pending_compaction_bytes_limit) /
|
||||||
4;
|
4;
|
||||||
|
|
||||||
|
@ -1305,8 +1306,8 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ColumnFamilyData::InstallSuperVersion(
|
void ColumnFamilyData::InstallSuperVersion(SuperVersionContext* sv_context,
|
||||||
SuperVersionContext* sv_context, InstrumentedMutex* db_mutex) {
|
InstrumentedMutex* db_mutex) {
|
||||||
db_mutex->AssertHeld();
|
db_mutex->AssertHeld();
|
||||||
return InstallSuperVersion(sv_context, mutable_cf_options_);
|
return InstallSuperVersion(sv_context, mutable_cf_options_);
|
||||||
}
|
}
|
||||||
|
@ -1483,8 +1484,8 @@ Env::WriteLifeTimeHint ColumnFamilyData::CalculateSSTWriteHint(int level) {
|
||||||
// than base_level.
|
// than base_level.
|
||||||
return Env::WLTH_MEDIUM;
|
return Env::WLTH_MEDIUM;
|
||||||
}
|
}
|
||||||
return static_cast<Env::WriteLifeTimeHint>(level - base_level +
|
return static_cast<Env::WriteLifeTimeHint>(
|
||||||
static_cast<int>(Env::WLTH_MEDIUM));
|
level - base_level + static_cast<int>(Env::WLTH_MEDIUM));
|
||||||
}
|
}
|
||||||
|
|
||||||
Status ColumnFamilyData::AddDirectories(
|
Status ColumnFamilyData::AddDirectories(
|
||||||
|
@ -1580,8 +1581,8 @@ ColumnFamilyData* ColumnFamilySet::GetColumnFamily(uint32_t id) const {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ColumnFamilyData* ColumnFamilySet::GetColumnFamily(const std::string& name)
|
ColumnFamilyData* ColumnFamilySet::GetColumnFamily(
|
||||||
const {
|
const std::string& name) const {
|
||||||
auto cfd_iter = column_families_.find(name);
|
auto cfd_iter = column_families_.find(name);
|
||||||
if (cfd_iter != column_families_.end()) {
|
if (cfd_iter != column_families_.end()) {
|
||||||
auto cfd = GetColumnFamily(cfd_iter->second);
|
auto cfd = GetColumnFamily(cfd_iter->second);
|
||||||
|
|
|
@ -163,8 +163,8 @@ extern const double kIncSlowdownRatio;
|
||||||
class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
|
class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
|
||||||
public:
|
public:
|
||||||
// create while holding the mutex
|
// create while holding the mutex
|
||||||
ColumnFamilyHandleImpl(
|
ColumnFamilyHandleImpl(ColumnFamilyData* cfd, DBImpl* db,
|
||||||
ColumnFamilyData* cfd, DBImpl* db, InstrumentedMutex* mutex);
|
InstrumentedMutex* mutex);
|
||||||
// destroy without mutex
|
// destroy without mutex
|
||||||
virtual ~ColumnFamilyHandleImpl();
|
virtual ~ColumnFamilyHandleImpl();
|
||||||
virtual ColumnFamilyData* cfd() const { return cfd_; }
|
virtual ColumnFamilyData* cfd() const { return cfd_; }
|
||||||
|
@ -189,7 +189,8 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
|
||||||
class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
|
class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
|
||||||
public:
|
public:
|
||||||
ColumnFamilyHandleInternal()
|
ColumnFamilyHandleInternal()
|
||||||
: ColumnFamilyHandleImpl(nullptr, nullptr, nullptr), internal_cfd_(nullptr) {}
|
: ColumnFamilyHandleImpl(nullptr, nullptr, nullptr),
|
||||||
|
internal_cfd_(nullptr) {}
|
||||||
|
|
||||||
void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
|
void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
|
||||||
virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
|
virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
|
||||||
|
@ -357,7 +358,7 @@ class ColumnFamilyData {
|
||||||
Version* current() { return current_; }
|
Version* current() { return current_; }
|
||||||
Version* dummy_versions() { return dummy_versions_; }
|
Version* dummy_versions() { return dummy_versions_; }
|
||||||
void SetCurrent(Version* _current);
|
void SetCurrent(Version* _current);
|
||||||
uint64_t GetNumLiveVersions() const; // REQUIRE: DB mutex held
|
uint64_t GetNumLiveVersions() const; // REQUIRE: DB mutex held
|
||||||
uint64_t GetTotalSstFilesSize() const; // REQUIRE: DB mutex held
|
uint64_t GetTotalSstFilesSize() const; // REQUIRE: DB mutex held
|
||||||
uint64_t GetLiveSstFilesSize() const; // REQUIRE: DB mutex held
|
uint64_t GetLiveSstFilesSize() const; // REQUIRE: DB mutex held
|
||||||
uint64_t GetTotalBlobFileSize() const; // REQUIRE: DB mutex held
|
uint64_t GetTotalBlobFileSize() const; // REQUIRE: DB mutex held
|
||||||
|
@ -552,7 +553,7 @@ class ColumnFamilyData {
|
||||||
Version* dummy_versions_; // Head of circular doubly-linked list of versions.
|
Version* dummy_versions_; // Head of circular doubly-linked list of versions.
|
||||||
Version* current_; // == dummy_versions->prev_
|
Version* current_; // == dummy_versions->prev_
|
||||||
|
|
||||||
std::atomic<int> refs_; // outstanding references to ColumnFamilyData
|
std::atomic<int> refs_; // outstanding references to ColumnFamilyData
|
||||||
std::atomic<bool> initialized_;
|
std::atomic<bool> initialized_;
|
||||||
std::atomic<bool> dropped_; // true if client dropped it
|
std::atomic<bool> dropped_; // true if client dropped it
|
||||||
|
|
||||||
|
@ -656,8 +657,7 @@ class ColumnFamilySet {
|
||||||
// ColumnFamilySet supports iteration
|
// ColumnFamilySet supports iteration
|
||||||
class iterator {
|
class iterator {
|
||||||
public:
|
public:
|
||||||
explicit iterator(ColumnFamilyData* cfd)
|
explicit iterator(ColumnFamilyData* cfd) : current_(cfd) {}
|
||||||
: current_(cfd) {}
|
|
||||||
// NOTE: minimum operators for for-loop iteration
|
// NOTE: minimum operators for for-loop iteration
|
||||||
iterator& operator++() {
|
iterator& operator++() {
|
||||||
current_ = current_->next_;
|
current_ = current_->next_;
|
||||||
|
|
|
@ -39,9 +39,7 @@ class EnvCounter : public SpecialEnv {
|
||||||
public:
|
public:
|
||||||
explicit EnvCounter(Env* base)
|
explicit EnvCounter(Env* base)
|
||||||
: SpecialEnv(base), num_new_writable_file_(0) {}
|
: SpecialEnv(base), num_new_writable_file_(0) {}
|
||||||
int GetNumberOfNewWritableFileCalls() {
|
int GetNumberOfNewWritableFileCalls() { return num_new_writable_file_; }
|
||||||
return num_new_writable_file_;
|
|
||||||
}
|
|
||||||
Status NewWritableFile(const std::string& f, std::unique_ptr<WritableFile>* r,
|
Status NewWritableFile(const std::string& f, std::unique_ptr<WritableFile>* r,
|
||||||
const EnvOptions& soptions) override {
|
const EnvOptions& soptions) override {
|
||||||
++num_new_writable_file_;
|
++num_new_writable_file_;
|
||||||
|
@ -187,7 +185,7 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
}
|
}
|
||||||
|
|
||||||
Status OpenReadOnly(std::vector<std::string> cf,
|
Status OpenReadOnly(std::vector<std::string> cf,
|
||||||
std::vector<ColumnFamilyOptions> options = {}) {
|
std::vector<ColumnFamilyOptions> options = {}) {
|
||||||
std::vector<ColumnFamilyDescriptor> column_families;
|
std::vector<ColumnFamilyDescriptor> column_families;
|
||||||
names_.clear();
|
names_.clear();
|
||||||
for (size_t i = 0; i < cf.size(); ++i) {
|
for (size_t i = 0; i < cf.size(); ++i) {
|
||||||
|
@ -201,20 +199,17 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE // ReadOnlyDB is not supported
|
#ifndef ROCKSDB_LITE // ReadOnlyDB is not supported
|
||||||
void AssertOpenReadOnly(std::vector<std::string> cf,
|
void AssertOpenReadOnly(std::vector<std::string> cf,
|
||||||
std::vector<ColumnFamilyOptions> options = {}) {
|
std::vector<ColumnFamilyOptions> options = {}) {
|
||||||
ASSERT_OK(OpenReadOnly(cf, options));
|
ASSERT_OK(OpenReadOnly(cf, options));
|
||||||
}
|
}
|
||||||
#endif // !ROCKSDB_LITE
|
#endif // !ROCKSDB_LITE
|
||||||
|
|
||||||
|
|
||||||
void Open(std::vector<std::string> cf,
|
void Open(std::vector<std::string> cf,
|
||||||
std::vector<ColumnFamilyOptions> options = {}) {
|
std::vector<ColumnFamilyOptions> options = {}) {
|
||||||
ASSERT_OK(TryOpen(cf, options));
|
ASSERT_OK(TryOpen(cf, options));
|
||||||
}
|
}
|
||||||
|
|
||||||
void Open() {
|
void Open() { Open({"default"}); }
|
||||||
Open({"default"});
|
|
||||||
}
|
|
||||||
|
|
||||||
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
|
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
|
||||||
|
|
||||||
|
@ -253,7 +248,7 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
}
|
}
|
||||||
|
|
||||||
void Destroy(const std::vector<ColumnFamilyDescriptor>& column_families =
|
void Destroy(const std::vector<ColumnFamilyDescriptor>& column_families =
|
||||||
std::vector<ColumnFamilyDescriptor>()) {
|
std::vector<ColumnFamilyDescriptor>()) {
|
||||||
Close();
|
Close();
|
||||||
ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_),
|
ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_),
|
||||||
column_families));
|
column_families));
|
||||||
|
@ -335,9 +330,7 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf]));
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf]));
|
||||||
}
|
}
|
||||||
|
|
||||||
void WaitForCompaction() {
|
void WaitForCompaction() { ASSERT_OK(dbfull()->TEST_WaitForCompact()); }
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t MaxTotalInMemoryState() {
|
uint64_t MaxTotalInMemoryState() {
|
||||||
return dbfull()->TEST_MaxTotalInMemoryState();
|
return dbfull()->TEST_MaxTotalInMemoryState();
|
||||||
|
@ -354,9 +347,7 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
Status Merge(int cf, const std::string& key, const std::string& value) {
|
Status Merge(int cf, const std::string& key, const std::string& value) {
|
||||||
return db_->Merge(WriteOptions(), handles_[cf], Slice(key), Slice(value));
|
return db_->Merge(WriteOptions(), handles_[cf], Slice(key), Slice(value));
|
||||||
}
|
}
|
||||||
Status Flush(int cf) {
|
Status Flush(int cf) { return db_->Flush(FlushOptions(), handles_[cf]); }
|
||||||
return db_->Flush(FlushOptions(), handles_[cf]);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string Get(int cf, const std::string& key) {
|
std::string Get(int cf, const std::string& key) {
|
||||||
ReadOptions options;
|
ReadOptions options;
|
||||||
|
@ -409,8 +400,8 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
ASSERT_EQ(value, FilesPerLevel(cf));
|
ASSERT_EQ(value, FilesPerLevel(cf));
|
||||||
#else
|
#else
|
||||||
(void) value;
|
(void)value;
|
||||||
(void) cf;
|
(void)cf;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -426,7 +417,7 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
ASSERT_EQ(expected_value, CountLiveFiles());
|
ASSERT_EQ(expected_value, CountLiveFiles());
|
||||||
#else
|
#else
|
||||||
(void) expected_value;
|
(void)expected_value;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -476,7 +467,7 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
#ifndef ROCKSDB_LITE // GetSortedWalFiles is not supported
|
#ifndef ROCKSDB_LITE // GetSortedWalFiles is not supported
|
||||||
ASSERT_EQ(value, CountLiveLogFiles());
|
ASSERT_EQ(value, CountLiveLogFiles());
|
||||||
#else
|
#else
|
||||||
(void) value;
|
(void)value;
|
||||||
#endif // !ROCKSDB_LITE
|
#endif // !ROCKSDB_LITE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -521,14 +512,14 @@ class ColumnFamilyTestBase : public testing::Test {
|
||||||
return static_cast<int>(files.size());
|
return static_cast<int>(files.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
void RecalculateWriteStallConditions(ColumnFamilyData* cfd,
|
void RecalculateWriteStallConditions(
|
||||||
const MutableCFOptions& mutable_cf_options) {
|
ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options) {
|
||||||
// add lock to avoid race condition between
|
// add lock to avoid race condition between
|
||||||
// `RecalculateWriteStallConditions` which writes to CFStats and
|
// `RecalculateWriteStallConditions` which writes to CFStats and
|
||||||
// background `DBImpl::DumpStats()` threads which read CFStats
|
// background `DBImpl::DumpStats()` threads which read CFStats
|
||||||
dbfull()->TEST_LockMutex();
|
dbfull()->TEST_LockMutex();
|
||||||
cfd->RecalculateWriteStallConditions(mutable_cf_options);
|
cfd->RecalculateWriteStallConditions(mutable_cf_options);
|
||||||
dbfull()-> TEST_UnlockMutex();
|
dbfull()->TEST_UnlockMutex();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<ColumnFamilyHandle*> handles_;
|
std::vector<ColumnFamilyHandle*> handles_;
|
||||||
|
@ -970,8 +961,7 @@ TEST_P(ColumnFamilyTest, FlushTest) {
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < 3; ++i) {
|
for (int i = 0; i < 3; ++i) {
|
||||||
uint64_t max_total_in_memory_state =
|
uint64_t max_total_in_memory_state = MaxTotalInMemoryState();
|
||||||
MaxTotalInMemoryState();
|
|
||||||
ASSERT_OK(Flush(i));
|
ASSERT_OK(Flush(i));
|
||||||
AssertMaxTotalInMemoryState(max_total_in_memory_state);
|
AssertMaxTotalInMemoryState(max_total_in_memory_state);
|
||||||
}
|
}
|
||||||
|
@ -1209,7 +1199,7 @@ TEST_P(ColumnFamilyTest, DifferentWriteBufferSizes) {
|
||||||
WaitForFlush(2);
|
WaitForFlush(2);
|
||||||
AssertNumberOfImmutableMemtables({0, 0, 0, 0});
|
AssertNumberOfImmutableMemtables({0, 0, 0, 0});
|
||||||
AssertCountLiveLogFiles(12);
|
AssertCountLiveLogFiles(12);
|
||||||
PutRandomData(1, 2*200, 1000);
|
PutRandomData(1, 2 * 200, 1000);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertNumberOfImmutableMemtables({0, 0, 0, 0});
|
AssertNumberOfImmutableMemtables({0, 0, 0, 0});
|
||||||
AssertCountLiveLogFiles(7);
|
AssertCountLiveLogFiles(7);
|
||||||
|
@ -2123,7 +2113,6 @@ TEST_P(ColumnFamilyTest, ReadOnlyDBTest) {
|
||||||
ASSERT_EQ("bla", Get(1, "foo"));
|
ASSERT_EQ("bla", Get(1, "foo"));
|
||||||
ASSERT_EQ("blablablabla", Get(2, "foo"));
|
ASSERT_EQ("blablablabla", Get(2, "foo"));
|
||||||
|
|
||||||
|
|
||||||
// test newiterators
|
// test newiterators
|
||||||
{
|
{
|
||||||
std::vector<Iterator*> iterators;
|
std::vector<Iterator*> iterators;
|
||||||
|
@ -2488,7 +2477,7 @@ void DropSingleColumnFamily(ColumnFamilyTest* cf_test, int cf_id,
|
||||||
}
|
}
|
||||||
test_stage = kChildThreadFinishDroppingColumnFamily;
|
test_stage = kChildThreadFinishDroppingColumnFamily;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(ColumnFamilyTest, CreateAndDropRace) {
|
TEST_P(ColumnFamilyTest, CreateAndDropRace) {
|
||||||
const int kCfCount = 5;
|
const int kCfCount = 5;
|
||||||
|
|
|
@ -348,9 +348,7 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SetDB(DB* db) {
|
void SetDB(DB* db) { db_ = db; }
|
||||||
db_ = db;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char* Name() const override { return "FilterWithGet"; }
|
const char* Name() const override { return "FilterWithGet"; }
|
||||||
|
|
||||||
|
@ -358,7 +356,6 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
|
||||||
DB* db_;
|
DB* db_;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
std::shared_ptr<FilterWithGet> cf(new FilterWithGet());
|
std::shared_ptr<FilterWithGet> cf(new FilterWithGet());
|
||||||
|
|
||||||
Options options;
|
Options options;
|
||||||
|
@ -385,7 +382,6 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
|
||||||
db->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), {fname}, 0));
|
db->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), {fname}, 0));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
delete db;
|
delete db;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -400,10 +396,9 @@ TEST_F(CompactFilesTest, SentinelCompressionType) {
|
||||||
}
|
}
|
||||||
// Check that passing `CompressionType::kDisableCompressionOption` to
|
// Check that passing `CompressionType::kDisableCompressionOption` to
|
||||||
// `CompactFiles` causes it to use the column family compression options.
|
// `CompactFiles` causes it to use the column family compression options.
|
||||||
for (auto compaction_style :
|
for (auto compaction_style : {CompactionStyle::kCompactionStyleLevel,
|
||||||
{CompactionStyle::kCompactionStyleLevel,
|
CompactionStyle::kCompactionStyleUniversal,
|
||||||
CompactionStyle::kCompactionStyleUniversal,
|
CompactionStyle::kCompactionStyleNone}) {
|
||||||
CompactionStyle::kCompactionStyleNone}) {
|
|
||||||
ASSERT_OK(DestroyDB(db_name_, Options()));
|
ASSERT_OK(DestroyDB(db_name_, Options()));
|
||||||
Options options;
|
Options options;
|
||||||
options.compaction_style = compaction_style;
|
options.compaction_style = compaction_style;
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
#include "util/string_util.h"
|
#include "util/string_util.h"
|
||||||
#include "utilities/merge_operators.h"
|
#include "utilities/merge_operators.h"
|
||||||
|
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
|
@ -249,7 +248,7 @@ class TwoStrComparator : public Comparator {
|
||||||
|
|
||||||
void FindShortSuccessor(std::string* /*key*/) const override {}
|
void FindShortSuccessor(std::string* /*key*/) const override {}
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class ComparatorDBTest
|
class ComparatorDBTest
|
||||||
: public testing::Test,
|
: public testing::Test,
|
||||||
|
@ -470,7 +469,7 @@ void VerifySuccessor(const Slice& s, const Slice& t) {
|
||||||
ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(t, s));
|
ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(t, s));
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(ComparatorDBTest, IsSameLengthImmediateSuccessor) {
|
TEST_P(ComparatorDBTest, IsSameLengthImmediateSuccessor) {
|
||||||
{
|
{
|
||||||
|
|
|
@ -26,8 +26,7 @@ Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
|
||||||
}
|
}
|
||||||
|
|
||||||
Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family,
|
Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family,
|
||||||
const RangePtr* ranges, size_t n,
|
const RangePtr* ranges, size_t n, bool include_end) {
|
||||||
bool include_end) {
|
|
||||||
return (static_cast_with_check<DBImpl>(db->GetRootDB()))
|
return (static_cast_with_check<DBImpl>(db->GetRootDB()))
|
||||||
->DeleteFilesInRanges(column_family, ranges, n, include_end);
|
->DeleteFilesInRanges(column_family, ranges, n, include_end);
|
||||||
}
|
}
|
||||||
|
@ -47,9 +46,8 @@ Status VerifySstFileChecksum(const Options& options,
|
||||||
InternalKeyComparator internal_comparator(options.comparator);
|
InternalKeyComparator internal_comparator(options.comparator);
|
||||||
ImmutableOptions ioptions(options);
|
ImmutableOptions ioptions(options);
|
||||||
|
|
||||||
Status s = ioptions.fs->NewRandomAccessFile(file_path,
|
Status s = ioptions.fs->NewRandomAccessFile(
|
||||||
FileOptions(env_options),
|
file_path, FileOptions(env_options), &file, nullptr);
|
||||||
&file, nullptr);
|
|
||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
s = ioptions.fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr);
|
s = ioptions.fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr);
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -65,7 +65,7 @@ class ErrorEnv : public EnvWrapper {
|
||||||
return target()->NewWritableFile(fname, result, soptions);
|
return target()->NewWritableFile(fname, result, soptions);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
class CorruptionTest : public testing::Test {
|
class CorruptionTest : public testing::Test {
|
||||||
public:
|
public:
|
||||||
std::shared_ptr<Env> env_guard_;
|
std::shared_ptr<Env> env_guard_;
|
||||||
|
@ -138,9 +138,7 @@ class CorruptionTest : public testing::Test {
|
||||||
return DB::Open(opt, dbname_, &db_);
|
return DB::Open(opt, dbname_, &db_);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Reopen(Options* options = nullptr) {
|
void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); }
|
||||||
ASSERT_OK(TryReopen(options));
|
|
||||||
}
|
|
||||||
|
|
||||||
void RepairDB() {
|
void RepairDB() {
|
||||||
delete db_;
|
delete db_;
|
||||||
|
@ -156,7 +154,7 @@ class CorruptionTest : public testing::Test {
|
||||||
DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
|
DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
|
||||||
ASSERT_OK(dbi->TEST_FlushMemTable());
|
ASSERT_OK(dbi->TEST_FlushMemTable());
|
||||||
}
|
}
|
||||||
//if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
|
// if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
|
||||||
Slice key = Key(i + start, &key_space);
|
Slice key = Key(i + start, &key_space);
|
||||||
batch.Clear();
|
batch.Clear();
|
||||||
ASSERT_OK(batch.Put(key, Value(i + start, &value_space)));
|
ASSERT_OK(batch.Put(key, Value(i + start, &value_space)));
|
||||||
|
@ -183,8 +181,7 @@ class CorruptionTest : public testing::Test {
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
uint64_t key;
|
uint64_t key;
|
||||||
Slice in(iter->key());
|
Slice in(iter->key());
|
||||||
if (!ConsumeDecimalNumber(&in, &key) ||
|
if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
|
||||||
!in.empty() ||
|
|
||||||
key < next_expected) {
|
key < next_expected) {
|
||||||
bad_keys++;
|
bad_keys++;
|
||||||
continue;
|
continue;
|
||||||
|
@ -200,10 +197,11 @@ class CorruptionTest : public testing::Test {
|
||||||
iter->status().PermitUncheckedError();
|
iter->status().PermitUncheckedError();
|
||||||
delete iter;
|
delete iter;
|
||||||
|
|
||||||
fprintf(stderr,
|
fprintf(
|
||||||
"expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%llu\n",
|
stderr,
|
||||||
min_expected, max_expected, correct, bad_keys, bad_values,
|
"expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%llu\n",
|
||||||
static_cast<unsigned long long>(missed));
|
min_expected, max_expected, correct, bad_keys, bad_values,
|
||||||
|
static_cast<unsigned long long>(missed));
|
||||||
ASSERT_LE(min_expected, correct);
|
ASSERT_LE(min_expected, correct);
|
||||||
ASSERT_GE(max_expected, correct);
|
ASSERT_GE(max_expected, correct);
|
||||||
}
|
}
|
||||||
|
@ -217,8 +215,7 @@ class CorruptionTest : public testing::Test {
|
||||||
std::string fname;
|
std::string fname;
|
||||||
int picked_number = -1;
|
int picked_number = -1;
|
||||||
for (size_t i = 0; i < filenames.size(); i++) {
|
for (size_t i = 0; i < filenames.size(); i++) {
|
||||||
if (ParseFileName(filenames[i], &number, &type) &&
|
if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
|
||||||
type == filetype &&
|
|
||||||
static_cast<int>(number) > picked_number) { // Pick latest file
|
static_cast<int>(number) > picked_number) { // Pick latest file
|
||||||
fname = dbname_ + "/" + filenames[i];
|
fname = dbname_ + "/" + filenames[i];
|
||||||
picked_number = static_cast<int>(number);
|
picked_number = static_cast<int>(number);
|
||||||
|
@ -244,7 +241,6 @@ class CorruptionTest : public testing::Test {
|
||||||
FAIL() << "no file found at level";
|
FAIL() << "no file found at level";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
int Property(const std::string& name) {
|
int Property(const std::string& name) {
|
||||||
std::string property;
|
std::string property;
|
||||||
int result;
|
int result;
|
||||||
|
|
|
@ -77,9 +77,7 @@ class CuckooTableDBTest : public testing::Test {
|
||||||
return db_->Put(WriteOptions(), k, v);
|
return db_->Put(WriteOptions(), k, v);
|
||||||
}
|
}
|
||||||
|
|
||||||
Status Delete(const std::string& k) {
|
Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); }
|
||||||
return db_->Delete(WriteOptions(), k);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string Get(const std::string& k) {
|
std::string Get(const std::string& k) {
|
||||||
ReadOptions options;
|
ReadOptions options;
|
||||||
|
@ -313,23 +311,21 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) {
|
||||||
// Write some keys using plain table.
|
// Write some keys using plain table.
|
||||||
std::shared_ptr<TableFactory> block_based_factory(
|
std::shared_ptr<TableFactory> block_based_factory(
|
||||||
NewBlockBasedTableFactory());
|
NewBlockBasedTableFactory());
|
||||||
std::shared_ptr<TableFactory> plain_table_factory(
|
std::shared_ptr<TableFactory> plain_table_factory(NewPlainTableFactory());
|
||||||
NewPlainTableFactory());
|
std::shared_ptr<TableFactory> cuckoo_table_factory(NewCuckooTableFactory());
|
||||||
std::shared_ptr<TableFactory> cuckoo_table_factory(
|
|
||||||
NewCuckooTableFactory());
|
|
||||||
options.create_if_missing = false;
|
options.create_if_missing = false;
|
||||||
options.table_factory.reset(NewAdaptiveTableFactory(
|
options.table_factory.reset(
|
||||||
plain_table_factory, block_based_factory, plain_table_factory,
|
NewAdaptiveTableFactory(plain_table_factory, block_based_factory,
|
||||||
cuckoo_table_factory));
|
plain_table_factory, cuckoo_table_factory));
|
||||||
Reopen(&options);
|
Reopen(&options);
|
||||||
ASSERT_OK(Put("key4", "v4"));
|
ASSERT_OK(Put("key4", "v4"));
|
||||||
ASSERT_OK(Put("key1", "v5"));
|
ASSERT_OK(Put("key1", "v5"));
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
|
|
||||||
// Write some keys using block based table.
|
// Write some keys using block based table.
|
||||||
options.table_factory.reset(NewAdaptiveTableFactory(
|
options.table_factory.reset(
|
||||||
block_based_factory, block_based_factory, plain_table_factory,
|
NewAdaptiveTableFactory(block_based_factory, block_based_factory,
|
||||||
cuckoo_table_factory));
|
plain_table_factory, cuckoo_table_factory));
|
||||||
Reopen(&options);
|
Reopen(&options);
|
||||||
ASSERT_OK(Put("key5", "v6"));
|
ASSERT_OK(Put("key5", "v6"));
|
||||||
ASSERT_OK(Put("key2", "v7"));
|
ASSERT_OK(Put("key2", "v7"));
|
||||||
|
|
|
@ -2805,7 +2805,7 @@ TEST_F(DBBasicTest, MultiGetIOBufferOverrun) {
|
||||||
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
|
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
|
||||||
table_options.block_size = 16 * 1024;
|
table_options.block_size = 16 * 1024;
|
||||||
ASSERT_TRUE(table_options.block_size >
|
ASSERT_TRUE(table_options.block_size >
|
||||||
BlockBasedTable::kMultiGetReadStackBufSize);
|
BlockBasedTable::kMultiGetReadStackBufSize);
|
||||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
|
|
||||||
|
@ -2914,7 +2914,7 @@ class TableFileListener : public EventListener {
|
||||||
InstrumentedMutex mutex_;
|
InstrumentedMutex mutex_;
|
||||||
std::unordered_map<std::string, std::vector<std::string>> cf_to_paths_;
|
std::unordered_map<std::string, std::vector<std::string>> cf_to_paths_;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBasicTest, LastSstFileNotInManifest) {
|
TEST_F(DBBasicTest, LastSstFileNotInManifest) {
|
||||||
// If the last sst file is not tracked in MANIFEST,
|
// If the last sst file is not tracked in MANIFEST,
|
||||||
|
|
|
@ -414,7 +414,7 @@ class ReadOnlyCacheWrapper : public CacheWrapper {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBlockCacheTest, TestWithSameCompressed) {
|
TEST_F(DBBlockCacheTest, TestWithSameCompressed) {
|
||||||
auto table_options = GetTableOptions();
|
auto table_options = GetTableOptions();
|
||||||
|
@ -1973,7 +1973,7 @@ struct CacheKeyDecoder {
|
||||||
DownwardInvolution(decoded_session_counter));
|
DownwardInvolution(decoded_session_counter));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(CacheKeyTest, Encodings) {
|
TEST_F(CacheKeyTest, Encodings) {
|
||||||
// This test primarily verifies this claim from cache_key.cc:
|
// This test primarily verifies this claim from cache_key.cc:
|
||||||
|
|
|
@ -43,7 +43,7 @@ const std::string kStandard128Ribbon =
|
||||||
test::Standard128RibbonFilterPolicy::kClassName();
|
test::Standard128RibbonFilterPolicy::kClassName();
|
||||||
const std::string kAutoBloom = BloomFilterPolicy::kClassName();
|
const std::string kAutoBloom = BloomFilterPolicy::kClassName();
|
||||||
const std::string kAutoRibbon = RibbonFilterPolicy::kClassName();
|
const std::string kAutoRibbon = RibbonFilterPolicy::kClassName();
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// DB tests related to bloom filter.
|
// DB tests related to bloom filter.
|
||||||
|
|
||||||
|
@ -622,7 +622,7 @@ class AlwaysTrueFilterPolicy : public ReadOnlyBuiltinFilterPolicy {
|
||||||
bool skip_;
|
bool skip_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
|
TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
|
||||||
constexpr int maxKey = 10;
|
constexpr int maxKey = 10;
|
||||||
|
@ -767,10 +767,10 @@ INSTANTIATE_TEST_CASE_P(
|
||||||
|
|
||||||
INSTANTIATE_TEST_CASE_P(
|
INSTANTIATE_TEST_CASE_P(
|
||||||
FormatLatest, DBBloomFilterTestWithParam,
|
FormatLatest, DBBloomFilterTestWithParam,
|
||||||
::testing::Values(
|
::testing::Values(std::make_tuple(kAutoBloom, true, kLatestFormatVersion),
|
||||||
std::make_tuple(kAutoBloom, true, kLatestFormatVersion),
|
std::make_tuple(kAutoBloom, false, kLatestFormatVersion),
|
||||||
std::make_tuple(kAutoBloom, false, kLatestFormatVersion),
|
std::make_tuple(kAutoRibbon, false,
|
||||||
std::make_tuple(kAutoRibbon, false, kLatestFormatVersion)));
|
kLatestFormatVersion)));
|
||||||
#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
||||||
|
|
||||||
TEST_F(DBBloomFilterTest, BloomFilterRate) {
|
TEST_F(DBBloomFilterTest, BloomFilterRate) {
|
||||||
|
@ -840,7 +840,7 @@ std::vector<CompatibilityConfig> kCompatibilityConfigs = {
|
||||||
BlockBasedTableOptions().format_version},
|
BlockBasedTableOptions().format_version},
|
||||||
{kCompatibilityRibbonPolicy, true, BlockBasedTableOptions().format_version},
|
{kCompatibilityRibbonPolicy, true, BlockBasedTableOptions().format_version},
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
|
TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
|
@ -1678,7 +1678,7 @@ class TestingContextCustomFilterPolicy
|
||||||
private:
|
private:
|
||||||
mutable std::string test_report_;
|
mutable std::string test_report_;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
|
TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
|
||||||
auto policy = std::make_shared<TestingContextCustomFilterPolicy>(15, 8, 5);
|
auto policy = std::make_shared<TestingContextCustomFilterPolicy>(15, 8, 5);
|
||||||
|
@ -2186,16 +2186,14 @@ INSTANTIATE_TEST_CASE_P(DBBloomFilterTestVaryPrefixAndFormatVer,
|
||||||
std::make_tuple(false, 2),
|
std::make_tuple(false, 2),
|
||||||
std::make_tuple(false, 3),
|
std::make_tuple(false, 3),
|
||||||
std::make_tuple(false, 4),
|
std::make_tuple(false, 4),
|
||||||
std::make_tuple(false, 5),
|
std::make_tuple(false, 5), std::make_tuple(true, 2),
|
||||||
std::make_tuple(true, 2),
|
std::make_tuple(true, 3), std::make_tuple(true, 4),
|
||||||
std::make_tuple(true, 3),
|
|
||||||
std::make_tuple(true, 4),
|
|
||||||
std::make_tuple(true, 5)));
|
std::make_tuple(true, 5)));
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
namespace {
|
namespace {
|
||||||
static const std::string kPlainTable = "test_PlainTableBloom";
|
static const std::string kPlainTable = "test_PlainTableBloom";
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class BloomStatsTestWithParam
|
class BloomStatsTestWithParam
|
||||||
: public DBBloomFilterTest,
|
: public DBBloomFilterTest,
|
||||||
|
@ -2408,7 +2406,7 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
|
||||||
dbtest->Flush();
|
dbtest->Flush();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBBloomFilterTest, PrefixScan) {
|
TEST_F(DBBloomFilterTest, PrefixScan) {
|
||||||
while (ChangeFilterOptions()) {
|
while (ChangeFilterOptions()) {
|
||||||
|
@ -3169,7 +3167,7 @@ std::pair<uint64_t, uint64_t> CheckedAndUseful(uint64_t checked,
|
||||||
uint64_t useful) {
|
uint64_t useful) {
|
||||||
return {checked, useful};
|
return {checked, useful};
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// This uses a prefix_extractor + comparator combination that violates
|
// This uses a prefix_extractor + comparator combination that violates
|
||||||
// one of the old obsolete, unnecessary axioms of prefix extraction:
|
// one of the old obsolete, unnecessary axioms of prefix extraction:
|
||||||
|
@ -3377,7 +3375,7 @@ class NonIdempotentFixed4Transform : public SliceTransform {
|
||||||
|
|
||||||
bool InDomain(const Slice& src) const override { return src.size() >= 5; }
|
bool InDomain(const Slice& src) const override { return src.size() >= 5; }
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// This uses a prefix_extractor + comparator combination that violates
|
// This uses a prefix_extractor + comparator combination that violates
|
||||||
// two of the old obsolete, unnecessary axioms of prefix extraction:
|
// two of the old obsolete, unnecessary axioms of prefix extraction:
|
||||||
|
|
|
@ -255,9 +255,8 @@ Options DeletionTriggerOptions(Options options) {
|
||||||
return options;
|
return options;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool HaveOverlappingKeyRanges(
|
bool HaveOverlappingKeyRanges(const Comparator* c, const SstFileMetaData& a,
|
||||||
const Comparator* c,
|
const SstFileMetaData& b) {
|
||||||
const SstFileMetaData& a, const SstFileMetaData& b) {
|
|
||||||
if (c->CompareWithoutTimestamp(a.smallestkey, b.smallestkey) >= 0) {
|
if (c->CompareWithoutTimestamp(a.smallestkey, b.smallestkey) >= 0) {
|
||||||
if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) {
|
if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) {
|
||||||
// b.smallestkey <= a.smallestkey <= b.largestkey
|
// b.smallestkey <= a.smallestkey <= b.largestkey
|
||||||
|
@ -282,18 +281,15 @@ bool HaveOverlappingKeyRanges(
|
||||||
// Identifies all files between level "min_level" and "max_level"
|
// Identifies all files between level "min_level" and "max_level"
|
||||||
// which has overlapping key range with "input_file_meta".
|
// which has overlapping key range with "input_file_meta".
|
||||||
void GetOverlappingFileNumbersForLevelCompaction(
|
void GetOverlappingFileNumbersForLevelCompaction(
|
||||||
const ColumnFamilyMetaData& cf_meta,
|
const ColumnFamilyMetaData& cf_meta, const Comparator* comparator,
|
||||||
const Comparator* comparator,
|
int min_level, int max_level, const SstFileMetaData* input_file_meta,
|
||||||
int min_level, int max_level,
|
|
||||||
const SstFileMetaData* input_file_meta,
|
|
||||||
std::set<std::string>* overlapping_file_names) {
|
std::set<std::string>* overlapping_file_names) {
|
||||||
std::set<const SstFileMetaData*> overlapping_files;
|
std::set<const SstFileMetaData*> overlapping_files;
|
||||||
overlapping_files.insert(input_file_meta);
|
overlapping_files.insert(input_file_meta);
|
||||||
for (int m = min_level; m <= max_level; ++m) {
|
for (int m = min_level; m <= max_level; ++m) {
|
||||||
for (auto& file : cf_meta.levels[m].files) {
|
for (auto& file : cf_meta.levels[m].files) {
|
||||||
for (auto* included_file : overlapping_files) {
|
for (auto* included_file : overlapping_files) {
|
||||||
if (HaveOverlappingKeyRanges(
|
if (HaveOverlappingKeyRanges(comparator, *included_file, file)) {
|
||||||
comparator, *included_file, file)) {
|
|
||||||
overlapping_files.insert(&file);
|
overlapping_files.insert(&file);
|
||||||
overlapping_file_names->insert(file.name);
|
overlapping_file_names->insert(file.name);
|
||||||
break;
|
break;
|
||||||
|
@ -316,12 +312,9 @@ void VerifyCompactionResult(
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
const SstFileMetaData* PickFileRandomly(
|
const SstFileMetaData* PickFileRandomly(const ColumnFamilyMetaData& cf_meta,
|
||||||
const ColumnFamilyMetaData& cf_meta,
|
Random* rand, int* level = nullptr) {
|
||||||
Random* rand,
|
auto file_id = rand->Uniform(static_cast<int>(cf_meta.file_count)) + 1;
|
||||||
int* level = nullptr) {
|
|
||||||
auto file_id = rand->Uniform(static_cast<int>(
|
|
||||||
cf_meta.file_count)) + 1;
|
|
||||||
for (auto& level_meta : cf_meta.levels) {
|
for (auto& level_meta : cf_meta.levels) {
|
||||||
if (file_id <= level_meta.files.size()) {
|
if (file_id <= level_meta.files.size()) {
|
||||||
if (level != nullptr) {
|
if (level != nullptr) {
|
||||||
|
@ -747,7 +740,6 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
|
TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
|
||||||
const int kNumKeysPerFile = 100;
|
const int kNumKeysPerFile = 100;
|
||||||
|
|
||||||
|
@ -890,7 +882,7 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
|
||||||
|
|
||||||
TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
|
TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.write_buffer_size = 100000000; // Large write buffer
|
options.write_buffer_size = 100000000; // Large write buffer
|
||||||
options.max_subcompactions = max_subcompactions_;
|
options.max_subcompactions = max_subcompactions_;
|
||||||
CreateAndReopenWithCF({"pikachu"}, options);
|
CreateAndReopenWithCF({"pikachu"}, options);
|
||||||
|
|
||||||
|
@ -1076,7 +1068,7 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
|
||||||
compact_opt.compression = kNoCompression;
|
compact_opt.compression = kNoCompression;
|
||||||
compact_opt.output_file_size_limit = 4096;
|
compact_opt.output_file_size_limit = 4096;
|
||||||
const size_t key_len =
|
const size_t key_len =
|
||||||
static_cast<size_t>(compact_opt.output_file_size_limit) / 5;
|
static_cast<size_t>(compact_opt.output_file_size_limit) / 5;
|
||||||
|
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
|
||||||
|
@ -1254,14 +1246,8 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveNonOverlappingFiles) {
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
// non overlapping ranges
|
// non overlapping ranges
|
||||||
std::vector<std::pair<int32_t, int32_t>> ranges = {
|
std::vector<std::pair<int32_t, int32_t>> ranges = {
|
||||||
{100, 199},
|
{100, 199}, {300, 399}, {0, 99}, {200, 299},
|
||||||
{300, 399},
|
{600, 699}, {400, 499}, {500, 550}, {551, 599},
|
||||||
{0, 99},
|
|
||||||
{200, 299},
|
|
||||||
{600, 699},
|
|
||||||
{400, 499},
|
|
||||||
{500, 550},
|
|
||||||
{551, 599},
|
|
||||||
};
|
};
|
||||||
int32_t value_size = 10 * 1024; // 10 KB
|
int32_t value_size = 10 * 1024; // 10 KB
|
||||||
|
|
||||||
|
@ -1304,14 +1290,15 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveNonOverlappingFiles) {
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
// Same ranges as above but overlapping
|
// Same ranges as above but overlapping
|
||||||
ranges = {
|
ranges = {
|
||||||
{100, 199},
|
{100, 199},
|
||||||
{300, 399},
|
{300, 399},
|
||||||
{0, 99},
|
{0, 99},
|
||||||
{200, 299},
|
{200, 299},
|
||||||
{600, 699},
|
{600, 699},
|
||||||
{400, 499},
|
{400, 499},
|
||||||
{500, 560}, // this range overlap with the next one
|
{500, 560}, // this range overlap with the next
|
||||||
{551, 599},
|
// one
|
||||||
|
{551, 599},
|
||||||
};
|
};
|
||||||
for (size_t i = 0; i < ranges.size(); i++) {
|
for (size_t i = 0; i < ranges.size(); i++) {
|
||||||
for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) {
|
for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) {
|
||||||
|
@ -1914,7 +1901,7 @@ TEST_F(DBCompactionTest, DeleteFilesInRanges) {
|
||||||
ASSERT_EQ("0,0,10", FilesPerLevel(0));
|
ASSERT_EQ("0,0,10", FilesPerLevel(0));
|
||||||
|
|
||||||
// file [0 => 100), [200 => 300), ... [800, 900)
|
// file [0 => 100), [200 => 300), ... [800, 900)
|
||||||
for (auto i = 0; i < 10; i+=2) {
|
for (auto i = 0; i < 10; i += 2) {
|
||||||
for (auto j = 0; j < 100; j++) {
|
for (auto j = 0; j < 100; j++) {
|
||||||
auto k = i * 100 + j;
|
auto k = i * 100 + j;
|
||||||
ASSERT_OK(Put(Key(k), values[k]));
|
ASSERT_OK(Put(Key(k), values[k]));
|
||||||
|
@ -2356,14 +2343,14 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionCFPathUse) {
|
||||||
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_2", 4 * 1024 * 1024);
|
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_2", 4 * 1024 * 1024);
|
||||||
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 1024 * 1024 * 1024);
|
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 1024 * 1024 * 1024);
|
||||||
option_vector.emplace_back(DBOptions(options), cf_opt1);
|
option_vector.emplace_back(DBOptions(options), cf_opt1);
|
||||||
CreateColumnFamilies({"one"},option_vector[1]);
|
CreateColumnFamilies({"one"}, option_vector[1]);
|
||||||
|
|
||||||
// Configure CF2 specific paths.
|
// Configure CF2 specific paths.
|
||||||
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 500 * 1024);
|
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 500 * 1024);
|
||||||
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_2", 4 * 1024 * 1024);
|
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_2", 4 * 1024 * 1024);
|
||||||
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 1024 * 1024 * 1024);
|
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 1024 * 1024 * 1024);
|
||||||
option_vector.emplace_back(DBOptions(options), cf_opt2);
|
option_vector.emplace_back(DBOptions(options), cf_opt2);
|
||||||
CreateColumnFamilies({"two"},option_vector[2]);
|
CreateColumnFamilies({"two"}, option_vector[2]);
|
||||||
|
|
||||||
ReopenWithColumnFamilies({"default", "one", "two"}, option_vector);
|
ReopenWithColumnFamilies({"default", "one", "two"}, option_vector);
|
||||||
|
|
||||||
|
@ -2736,7 +2723,6 @@ TEST_P(DBCompactionTestWithParam, ManualCompaction) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) {
|
TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760);
|
options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760);
|
||||||
|
@ -2873,14 +2859,13 @@ TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) {
|
||||||
auto file_meta = PickFileRandomly(cf_meta, &rnd, &level);
|
auto file_meta = PickFileRandomly(cf_meta, &rnd, &level);
|
||||||
compaction_input_file_names.push_back(file_meta->name);
|
compaction_input_file_names.push_back(file_meta->name);
|
||||||
GetOverlappingFileNumbersForLevelCompaction(
|
GetOverlappingFileNumbersForLevelCompaction(
|
||||||
cf_meta, options.comparator, level, output_level,
|
cf_meta, options.comparator, level, output_level, file_meta,
|
||||||
file_meta, &overlapping_file_names);
|
&overlapping_file_names);
|
||||||
}
|
}
|
||||||
|
|
||||||
ASSERT_OK(dbfull()->CompactFiles(
|
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
|
||||||
CompactionOptions(), handles_[1],
|
compaction_input_file_names,
|
||||||
compaction_input_file_names,
|
output_level));
|
||||||
output_level));
|
|
||||||
|
|
||||||
// Make sure all overlapping files do not exist after compaction
|
// Make sure all overlapping files do not exist after compaction
|
||||||
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
||||||
|
@ -2903,8 +2888,7 @@ TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) {
|
||||||
options.write_buffer_size = kKeysPerBuffer * kKvSize;
|
options.write_buffer_size = kKeysPerBuffer * kKvSize;
|
||||||
options.max_write_buffer_number = 2;
|
options.max_write_buffer_number = 2;
|
||||||
options.target_file_size_base =
|
options.target_file_size_base =
|
||||||
options.write_buffer_size *
|
options.write_buffer_size * (options.max_write_buffer_number - 1);
|
||||||
(options.max_write_buffer_number - 1);
|
|
||||||
options.level0_file_num_compaction_trigger = kNumL1Files;
|
options.level0_file_num_compaction_trigger = kNumL1Files;
|
||||||
options.max_bytes_for_level_base =
|
options.max_bytes_for_level_base =
|
||||||
options.level0_file_num_compaction_trigger *
|
options.level0_file_num_compaction_trigger *
|
||||||
|
@ -2924,10 +2908,9 @@ TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) {
|
||||||
|
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
|
||||||
const int kNumInsertedKeys =
|
const int kNumInsertedKeys = options.level0_file_num_compaction_trigger *
|
||||||
options.level0_file_num_compaction_trigger *
|
(options.max_write_buffer_number - 1) *
|
||||||
(options.max_write_buffer_number - 1) *
|
kKeysPerBuffer;
|
||||||
kKeysPerBuffer;
|
|
||||||
|
|
||||||
Random rnd(301);
|
Random rnd(301);
|
||||||
std::vector<std::string> keys;
|
std::vector<std::string> keys;
|
||||||
|
@ -3625,9 +3608,8 @@ TEST_F(DBCompactionTest, CompactFilesPendingL0Bug) {
|
||||||
ASSERT_EQ(kNumL0Files, cf_meta.levels[0].files.size());
|
ASSERT_EQ(kNumL0Files, cf_meta.levels[0].files.size());
|
||||||
std::vector<std::string> input_filenames;
|
std::vector<std::string> input_filenames;
|
||||||
input_filenames.push_back(cf_meta.levels[0].files.front().name);
|
input_filenames.push_back(cf_meta.levels[0].files.front().name);
|
||||||
ASSERT_OK(dbfull()
|
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), input_filenames,
|
||||||
->CompactFiles(CompactionOptions(), input_filenames,
|
0 /* output_level */));
|
||||||
0 /* output_level */));
|
|
||||||
TEST_SYNC_POINT("DBCompactionTest::CompactFilesPendingL0Bug:ManualCompacted");
|
TEST_SYNC_POINT("DBCompactionTest::CompactFilesPendingL0Bug:ManualCompacted");
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
||||||
}
|
}
|
||||||
|
@ -4510,9 +4492,9 @@ TEST_F(DBCompactionTest, LevelPeriodicAndTtlCompaction) {
|
||||||
const int kValueSize = 100;
|
const int kValueSize = 100;
|
||||||
|
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.ttl = 10 * 60 * 60; // 10 hours
|
options.ttl = 10 * 60 * 60; // 10 hours
|
||||||
options.periodic_compaction_seconds = 48 * 60 * 60; // 2 days
|
options.periodic_compaction_seconds = 48 * 60 * 60; // 2 days
|
||||||
options.max_open_files = -1; // needed for both periodic and ttl compactions
|
options.max_open_files = -1; // needed for both periodic and ttl compactions
|
||||||
env_->SetMockSleep();
|
env_->SetMockSleep();
|
||||||
options.env = env_;
|
options.env = env_;
|
||||||
|
|
||||||
|
@ -4934,7 +4916,7 @@ TEST_F(DBCompactionTest, CompactRangeSkipFlushAfterDelay) {
|
||||||
{"DBImpl::FlushMemTable:StallWaitDone", "CompactionJob::Run():End"}});
|
{"DBImpl::FlushMemTable:StallWaitDone", "CompactionJob::Run():End"}});
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
||||||
|
|
||||||
//used for the delayable flushes
|
// used for the delayable flushes
|
||||||
FlushOptions flush_opts;
|
FlushOptions flush_opts;
|
||||||
flush_opts.allow_write_stall = true;
|
flush_opts.allow_write_stall = true;
|
||||||
for (int i = 0; i < kNumL0FilesLimit - 1; ++i) {
|
for (int i = 0; i < kNumL0FilesLimit - 1; ++i) {
|
||||||
|
@ -4953,7 +4935,8 @@ TEST_F(DBCompactionTest, CompactRangeSkipFlushAfterDelay) {
|
||||||
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
||||||
ASSERT_OK(dbfull()->Flush(flush_opts));
|
ASSERT_OK(dbfull()->Flush(flush_opts));
|
||||||
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
||||||
TEST_SYNC_POINT("DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush");
|
TEST_SYNC_POINT(
|
||||||
|
"DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush");
|
||||||
manual_compaction_thread.join();
|
manual_compaction_thread.join();
|
||||||
|
|
||||||
// If CompactRange's flush was skipped, the final Put above will still be
|
// If CompactRange's flush was skipped, the final Put above will still be
|
||||||
|
@ -5246,10 +5229,10 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
|
||||||
}
|
}
|
||||||
|
|
||||||
std::shared_ptr<ConcurrentTaskLimiter> unique_limiter(
|
std::shared_ptr<ConcurrentTaskLimiter> unique_limiter(
|
||||||
NewConcurrentTaskLimiter("unique_limiter", -1));
|
NewConcurrentTaskLimiter("unique_limiter", -1));
|
||||||
|
|
||||||
const char* cf_names[] = {"default", "0", "1", "2", "3", "4", "5",
|
const char* cf_names[] = {"default", "0", "1", "2", "3", "4", "5", "6", "7",
|
||||||
"6", "7", "8", "9", "a", "b", "c", "d", "e", "f" };
|
"8", "9", "a", "b", "c", "d", "e", "f"};
|
||||||
const unsigned int cf_count = sizeof cf_names / sizeof cf_names[0];
|
const unsigned int cf_count = sizeof cf_names / sizeof cf_names[0];
|
||||||
|
|
||||||
std::unordered_map<std::string, CompactionLimiter*> cf_to_limiter;
|
std::unordered_map<std::string, CompactionLimiter*> cf_to_limiter;
|
||||||
|
@ -5261,10 +5244,10 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
|
||||||
options.level0_file_num_compaction_trigger = 4;
|
options.level0_file_num_compaction_trigger = 4;
|
||||||
options.level0_slowdown_writes_trigger = 64;
|
options.level0_slowdown_writes_trigger = 64;
|
||||||
options.level0_stop_writes_trigger = 64;
|
options.level0_stop_writes_trigger = 64;
|
||||||
options.max_background_jobs = kMaxBackgroundThreads; // Enough threads
|
options.max_background_jobs = kMaxBackgroundThreads; // Enough threads
|
||||||
options.memtable_factory.reset(
|
options.memtable_factory.reset(
|
||||||
test::NewSpecialSkipListFactory(kNumKeysPerFile));
|
test::NewSpecialSkipListFactory(kNumKeysPerFile));
|
||||||
options.max_write_buffer_number = 10; // Enough memtables
|
options.max_write_buffer_number = 10; // Enough memtables
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
|
||||||
std::vector<Options> option_vector;
|
std::vector<Options> option_vector;
|
||||||
|
@ -5292,9 +5275,8 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
|
||||||
CreateColumnFamilies({cf_names[cf]}, option_vector[cf]);
|
CreateColumnFamilies({cf_names[cf]}, option_vector[cf]);
|
||||||
}
|
}
|
||||||
|
|
||||||
ReopenWithColumnFamilies(std::vector<std::string>(cf_names,
|
ReopenWithColumnFamilies(
|
||||||
cf_names + cf_count),
|
std::vector<std::string>(cf_names, cf_names + cf_count), option_vector);
|
||||||
option_vector);
|
|
||||||
|
|
||||||
port::Mutex mutex;
|
port::Mutex mutex;
|
||||||
|
|
||||||
|
@ -5356,7 +5338,7 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
|
||||||
// Enough L0 files to trigger compaction
|
// Enough L0 files to trigger compaction
|
||||||
for (unsigned int cf = 0; cf < cf_count; cf++) {
|
for (unsigned int cf = 0; cf < cf_count; cf++) {
|
||||||
ASSERT_EQ(NumTableFilesAtLevel(0, cf),
|
ASSERT_EQ(NumTableFilesAtLevel(0, cf),
|
||||||
options.level0_file_num_compaction_trigger);
|
options.level0_file_num_compaction_trigger);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create more files for one column family, which triggers speed up
|
// Create more files for one column family, which triggers speed up
|
||||||
|
@ -5399,7 +5381,7 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
|
||||||
|
|
||||||
// flush one more file to cf 1
|
// flush one more file to cf 1
|
||||||
for (int i = 0; i < kNumKeysPerFile; i++) {
|
for (int i = 0; i < kNumKeysPerFile; i++) {
|
||||||
ASSERT_OK(Put(cf_test, Key(keyIndex++), ""));
|
ASSERT_OK(Put(cf_test, Key(keyIndex++), ""));
|
||||||
}
|
}
|
||||||
// put extra key to trigger flush
|
// put extra key to trigger flush
|
||||||
ASSERT_OK(Put(cf_test, "", ""));
|
ASSERT_OK(Put(cf_test, "", ""));
|
||||||
|
@ -5434,9 +5416,7 @@ TEST_P(DBCompactionDirectIOTest, DirectIO) {
|
||||||
});
|
});
|
||||||
if (options.use_direct_io_for_flush_and_compaction) {
|
if (options.use_direct_io_for_flush_and_compaction) {
|
||||||
SyncPoint::GetInstance()->SetCallBack(
|
SyncPoint::GetInstance()->SetCallBack(
|
||||||
"SanitizeOptions:direct_io", [&](void* /*arg*/) {
|
"SanitizeOptions:direct_io", [&](void* /*arg*/) { readahead = true; });
|
||||||
readahead = true;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
SyncPoint::GetInstance()->EnableProcessing();
|
SyncPoint::GetInstance()->EnableProcessing();
|
||||||
CreateAndReopenWithCF({"pikachu"}, options);
|
CreateAndReopenWithCF({"pikachu"}, options);
|
||||||
|
@ -8404,8 +8384,8 @@ int main(int argc, char** argv) {
|
||||||
::testing::InitGoogleTest(&argc, argv);
|
::testing::InitGoogleTest(&argc, argv);
|
||||||
return RUN_ALL_TESTS();
|
return RUN_ALL_TESTS();
|
||||||
#else
|
#else
|
||||||
(void) argc;
|
(void)argc;
|
||||||
(void) argv;
|
(void)argv;
|
||||||
return 0;
|
return 0;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
|
@ -500,8 +500,8 @@ int main(int argc, char** argv) {
|
||||||
::testing::InitGoogleTest(&argc, argv);
|
::testing::InitGoogleTest(&argc, argv);
|
||||||
return RUN_ALL_TESTS();
|
return RUN_ALL_TESTS();
|
||||||
#else
|
#else
|
||||||
(void) argc;
|
(void)argc;
|
||||||
(void) argv;
|
(void)argv;
|
||||||
return 0;
|
return 0;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
|
@ -43,7 +43,7 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
|
||||||
|
|
||||||
Env* target = GetTargetEnv();
|
Env* target = GetTargetEnv();
|
||||||
int hits = 0;
|
int hits = 0;
|
||||||
for (auto it = fileNames.begin() ; it != fileNames.end(); ++it) {
|
for (auto it = fileNames.begin(); it != fileNames.end(); ++it) {
|
||||||
if (*it == "LOCK") {
|
if (*it == "LOCK") {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
@ -64,24 +64,24 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
|
||||||
ASSERT_OK(status);
|
ASSERT_OK(status);
|
||||||
|
|
||||||
if (data.ToString().find("foo567") != std::string::npos) {
|
if (data.ToString().find("foo567") != std::string::npos) {
|
||||||
hits++;
|
hits++;
|
||||||
//std::cout << "Hit in " << filePath << "\n";
|
// std::cout << "Hit in " << filePath << "\n";
|
||||||
}
|
}
|
||||||
if (data.ToString().find("v1.fetdq") != std::string::npos) {
|
if (data.ToString().find("v1.fetdq") != std::string::npos) {
|
||||||
hits++;
|
hits++;
|
||||||
//std::cout << "Hit in " << filePath << "\n";
|
// std::cout << "Hit in " << filePath << "\n";
|
||||||
}
|
}
|
||||||
if (data.ToString().find("bar123") != std::string::npos) {
|
if (data.ToString().find("bar123") != std::string::npos) {
|
||||||
hits++;
|
hits++;
|
||||||
//std::cout << "Hit in " << filePath << "\n";
|
// std::cout << "Hit in " << filePath << "\n";
|
||||||
}
|
}
|
||||||
if (data.ToString().find("v2.dfgkjdfghsd") != std::string::npos) {
|
if (data.ToString().find("v2.dfgkjdfghsd") != std::string::npos) {
|
||||||
hits++;
|
hits++;
|
||||||
//std::cout << "Hit in " << filePath << "\n";
|
// std::cout << "Hit in " << filePath << "\n";
|
||||||
}
|
}
|
||||||
if (data.ToString().find("dfgk") != std::string::npos) {
|
if (data.ToString().find("dfgk") != std::string::npos) {
|
||||||
hits++;
|
hits++;
|
||||||
//std::cout << "Hit in " << filePath << "\n";
|
// std::cout << "Hit in " << filePath << "\n";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (encrypted_env_) {
|
if (encrypted_env_) {
|
||||||
|
@ -119,7 +119,7 @@ TEST_F(DBEncryptionTest, ReadEmptyFile) {
|
||||||
ASSERT_TRUE(data.empty());
|
ASSERT_TRUE(data.empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
|
|
@ -65,8 +65,7 @@ Status DBImpl::FlushForGetLiveFiles() {
|
||||||
}
|
}
|
||||||
|
|
||||||
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
||||||
uint64_t* manifest_file_size,
|
uint64_t* manifest_file_size, bool flush_memtable) {
|
||||||
bool flush_memtable) {
|
|
||||||
*manifest_file_size = 0;
|
*manifest_file_size = 0;
|
||||||
|
|
||||||
mutex_.Lock();
|
mutex_.Lock();
|
||||||
|
|
|
@ -57,7 +57,7 @@ TEST_F(DBFlushTest, FlushWhileWritingManifest) {
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
FlushOptions no_wait;
|
FlushOptions no_wait;
|
||||||
no_wait.wait = false;
|
no_wait.wait = false;
|
||||||
no_wait.allow_write_stall=true;
|
no_wait.allow_write_stall = true;
|
||||||
|
|
||||||
SyncPoint::GetInstance()->LoadDependency(
|
SyncPoint::GetInstance()->LoadDependency(
|
||||||
{{"VersionSet::LogAndApply:WriteManifest",
|
{{"VersionSet::LogAndApply:WriteManifest",
|
||||||
|
@ -1822,8 +1822,8 @@ TEST_F(DBFlushTest, ManualFlushFailsInReadOnlyMode) {
|
||||||
ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable());
|
ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable());
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
uint64_t num_bg_errors;
|
uint64_t num_bg_errors;
|
||||||
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBackgroundErrors,
|
ASSERT_TRUE(
|
||||||
&num_bg_errors));
|
db_->GetIntProperty(DB::Properties::kBackgroundErrors, &num_bg_errors));
|
||||||
ASSERT_GT(num_bg_errors, 0);
|
ASSERT_GT(num_bg_errors, 0);
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
#include "db/db_info_dumper.h"
|
#include "db/db_info_dumper.h"
|
||||||
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <cinttypes>
|
#include <cinttypes>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
|
@ -392,7 +392,7 @@ struct ReferenceIterator {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// Use an internal iterator that sometimes returns errors and sometimes
|
// Use an internal iterator that sometimes returns errors and sometimes
|
||||||
// adds/removes entries on the fly. Do random operations on a DBIter and
|
// adds/removes entries on the fly. Do random operations on a DBIter and
|
||||||
|
@ -482,12 +482,11 @@ TEST_F(DBIteratorStressTest, StressTest) {
|
||||||
std::cout << "entries:";
|
std::cout << "entries:";
|
||||||
for (size_t i = 0; i < data.entries.size(); ++i) {
|
for (size_t i = 0; i < data.entries.size(); ++i) {
|
||||||
Entry& e = data.entries[i];
|
Entry& e = data.entries[i];
|
||||||
std::cout
|
std::cout << "\n idx " << i << ": \"" << e.key << "\": \""
|
||||||
<< "\n idx " << i << ": \"" << e.key << "\": \""
|
<< e.value << "\" seq: " << e.sequence << " type: "
|
||||||
<< e.value << "\" seq: " << e.sequence << " type: "
|
<< (e.type == kTypeValue ? "val"
|
||||||
<< (e.type == kTypeValue
|
: e.type == kTypeDeletion ? "del"
|
||||||
? "val"
|
: "merge");
|
||||||
: e.type == kTypeDeletion ? "del" : "merge");
|
|
||||||
}
|
}
|
||||||
std::cout << std::endl;
|
std::cout << std::endl;
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,12 +3,13 @@
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
#include <string>
|
|
||||||
#include <vector>
|
|
||||||
#include <algorithm>
|
|
||||||
#include <utility>
|
|
||||||
|
|
||||||
#include "db/db_iter.h"
|
#include "db/db_iter.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
#include "db/dbformat.h"
|
#include "db/dbformat.h"
|
||||||
#include "rocksdb/comparator.h"
|
#include "rocksdb/comparator.h"
|
||||||
#include "rocksdb/options.h"
|
#include "rocksdb/options.h"
|
||||||
|
@ -82,8 +83,8 @@ class TestIterator : public InternalIterator {
|
||||||
std::sort(data_.begin(), data_.end(),
|
std::sort(data_.begin(), data_.end(),
|
||||||
[this](std::pair<std::string, std::string> a,
|
[this](std::pair<std::string, std::string> a,
|
||||||
std::pair<std::string, std::string> b) {
|
std::pair<std::string, std::string> b) {
|
||||||
return (cmp.Compare(a.first, b.first) < 0);
|
return (cmp.Compare(a.first, b.first) < 0);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Removes the key from the set of keys over which this iterator iterates.
|
// Removes the key from the set of keys over which this iterator iterates.
|
||||||
|
@ -429,7 +430,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
|
||||||
db_iter->SeekToLast();
|
db_iter->SeekToLast();
|
||||||
|
|
||||||
ASSERT_TRUE(db_iter->Valid());
|
ASSERT_TRUE(db_iter->Valid());
|
||||||
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_key_skipped_count), 1);
|
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_key_skipped_count),
|
||||||
|
1);
|
||||||
ASSERT_EQ(db_iter->key().ToString(), "b");
|
ASSERT_EQ(db_iter->key().ToString(), "b");
|
||||||
|
|
||||||
SetPerfLevel(kDisable);
|
SetPerfLevel(kDisable);
|
||||||
|
@ -557,7 +559,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
|
||||||
db_iter->SeekToLast();
|
db_iter->SeekToLast();
|
||||||
|
|
||||||
ASSERT_TRUE(db_iter->Valid());
|
ASSERT_TRUE(db_iter->Valid());
|
||||||
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
|
ASSERT_EQ(
|
||||||
|
static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
|
||||||
ASSERT_EQ(db_iter->key().ToString(), "b");
|
ASSERT_EQ(db_iter->key().ToString(), "b");
|
||||||
|
|
||||||
SetPerfLevel(kDisable);
|
SetPerfLevel(kDisable);
|
||||||
|
@ -3013,7 +3016,6 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace8) {
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_F(DBIteratorTest, SeekPrefixTombstones) {
|
TEST_F(DBIteratorTest, SeekPrefixTombstones) {
|
||||||
ReadOptions ro;
|
ReadOptions ro;
|
||||||
Options options;
|
Options options;
|
||||||
|
|
|
@ -236,7 +236,7 @@ namespace {
|
||||||
std::string MakeLongKey(size_t length, char c) {
|
std::string MakeLongKey(size_t length, char c) {
|
||||||
return std::string(length, c);
|
return std::string(length, c);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(DBIteratorTest, IterLongKeys) {
|
TEST_P(DBIteratorTest, IterLongKeys) {
|
||||||
ASSERT_OK(Put(MakeLongKey(20, 0), "0"));
|
ASSERT_OK(Put(MakeLongKey(20, 0), "0"));
|
||||||
|
@ -1037,7 +1037,8 @@ TEST_P(DBIteratorTest, DBIteratorBoundTest) {
|
||||||
iter->Next();
|
iter->Next();
|
||||||
|
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 2);
|
ASSERT_EQ(
|
||||||
|
static_cast<int>(get_perf_context()->internal_delete_skipped_count), 2);
|
||||||
|
|
||||||
// now testing with iterate_bound
|
// now testing with iterate_bound
|
||||||
Slice prefix("c");
|
Slice prefix("c");
|
||||||
|
@ -1060,7 +1061,8 @@ TEST_P(DBIteratorTest, DBIteratorBoundTest) {
|
||||||
// even though the key is deleted
|
// even though the key is deleted
|
||||||
// hence internal_delete_skipped_count should be 0
|
// hence internal_delete_skipped_count should be 0
|
||||||
ASSERT_TRUE(!iter->Valid());
|
ASSERT_TRUE(!iter->Valid());
|
||||||
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
|
ASSERT_EQ(
|
||||||
|
static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1536,7 +1538,7 @@ class DBIteratorTestForPinnedData : public DBIteratorTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
delete iter;
|
delete iter;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
||||||
|
@ -2180,8 +2182,8 @@ TEST_P(DBIteratorTest, IteratorWithLocalStatistics) {
|
||||||
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV), (uint64_t)total_prev);
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV), (uint64_t)total_prev);
|
||||||
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV_FOUND),
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV_FOUND),
|
||||||
(uint64_t)total_prev_found);
|
(uint64_t)total_prev_found);
|
||||||
ASSERT_EQ(TestGetTickerCount(options, ITER_BYTES_READ), (uint64_t)total_bytes);
|
ASSERT_EQ(TestGetTickerCount(options, ITER_BYTES_READ),
|
||||||
|
(uint64_t)total_bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_P(DBIteratorTest, ReadAhead) {
|
TEST_P(DBIteratorTest, ReadAhead) {
|
||||||
|
@ -2310,8 +2312,8 @@ TEST_P(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
|
||||||
EXPECT_EQ(get_perf_context()->internal_merge_count, 0);
|
EXPECT_EQ(get_perf_context()->internal_merge_count, 0);
|
||||||
EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2);
|
EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2);
|
||||||
EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2);
|
EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2);
|
||||||
EXPECT_EQ(1, options.statistics->getTickerCount(
|
EXPECT_EQ(1,
|
||||||
NUMBER_OF_RESEEKS_IN_ITERATION));
|
options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_P(DBIteratorTest, Refresh) {
|
TEST_P(DBIteratorTest, Refresh) {
|
||||||
|
@ -2592,7 +2594,7 @@ TEST_P(DBIteratorTest, SkipStatistics) {
|
||||||
}
|
}
|
||||||
ASSERT_EQ(count, 3);
|
ASSERT_EQ(count, 3);
|
||||||
delete iter;
|
delete iter;
|
||||||
skip_count += 8; // 3 deletes + 3 original keys + 2 lower in sequence
|
skip_count += 8; // 3 deletes + 3 original keys + 2 lower in sequence
|
||||||
ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
|
ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
|
||||||
|
|
||||||
iter = NewIterator(ReadOptions());
|
iter = NewIterator(ReadOptions());
|
||||||
|
@ -2603,7 +2605,7 @@ TEST_P(DBIteratorTest, SkipStatistics) {
|
||||||
}
|
}
|
||||||
ASSERT_EQ(count, 3);
|
ASSERT_EQ(count, 3);
|
||||||
delete iter;
|
delete iter;
|
||||||
skip_count += 8; // Same as above, but in reverse order
|
skip_count += 8; // Same as above, but in reverse order
|
||||||
ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
|
ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
|
||||||
|
|
||||||
ASSERT_OK(Put("aa", "1"));
|
ASSERT_OK(Put("aa", "1"));
|
||||||
|
@ -2621,18 +2623,18 @@ TEST_P(DBIteratorTest, SkipStatistics) {
|
||||||
|
|
||||||
iter = NewIterator(ro);
|
iter = NewIterator(ro);
|
||||||
count = 0;
|
count = 0;
|
||||||
for(iter->Seek("aa"); iter->Valid(); iter->Next()) {
|
for (iter->Seek("aa"); iter->Valid(); iter->Next()) {
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
count++;
|
count++;
|
||||||
}
|
}
|
||||||
ASSERT_EQ(count, 1);
|
ASSERT_EQ(count, 1);
|
||||||
delete iter;
|
delete iter;
|
||||||
skip_count += 6; // 3 deletes + 3 original keys
|
skip_count += 6; // 3 deletes + 3 original keys
|
||||||
ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
|
ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
|
||||||
|
|
||||||
iter = NewIterator(ro);
|
iter = NewIterator(ro);
|
||||||
count = 0;
|
count = 0;
|
||||||
for(iter->SeekToLast(); iter->Valid(); iter->Prev()) {
|
for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
count++;
|
count++;
|
||||||
}
|
}
|
||||||
|
|
|
@ -55,14 +55,13 @@ SequenceNumber ReadRecords(std::unique_ptr<TransactionLogIterator>& iter,
|
||||||
return res.sequence;
|
return res.sequence;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExpectRecords(
|
void ExpectRecords(const int expected_no_records,
|
||||||
const int expected_no_records,
|
std::unique_ptr<TransactionLogIterator>& iter) {
|
||||||
std::unique_ptr<TransactionLogIterator>& iter) {
|
|
||||||
int num_records;
|
int num_records;
|
||||||
ReadRecords(iter, num_records);
|
ReadRecords(iter, num_records);
|
||||||
ASSERT_EQ(num_records, expected_no_records);
|
ASSERT_EQ(num_records, expected_no_records);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTestXactLogIterator, TransactionLogIterator) {
|
TEST_F(DBTestXactLogIterator, TransactionLogIterator) {
|
||||||
do {
|
do {
|
||||||
|
@ -95,10 +94,9 @@ TEST_F(DBTestXactLogIterator, TransactionLogIterator) {
|
||||||
TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
|
TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
|
||||||
static const int LOG_ITERATOR_RACE_TEST_COUNT = 2;
|
static const int LOG_ITERATOR_RACE_TEST_COUNT = 2;
|
||||||
static const char* sync_points[LOG_ITERATOR_RACE_TEST_COUNT][4] = {
|
static const char* sync_points[LOG_ITERATOR_RACE_TEST_COUNT][4] = {
|
||||||
{"WalManager::GetSortedWalFiles:1", "WalManager::PurgeObsoleteFiles:1",
|
{"WalManager::GetSortedWalFiles:1", "WalManager::PurgeObsoleteFiles:1",
|
||||||
"WalManager::PurgeObsoleteFiles:2", "WalManager::GetSortedWalFiles:2"},
|
"WalManager::PurgeObsoleteFiles:2", "WalManager::GetSortedWalFiles:2"},
|
||||||
{"WalManager::GetSortedWalsOfType:1",
|
{"WalManager::GetSortedWalsOfType:1", "WalManager::PurgeObsoleteFiles:1",
|
||||||
"WalManager::PurgeObsoleteFiles:1",
|
|
||||||
"WalManager::PurgeObsoleteFiles:2",
|
"WalManager::PurgeObsoleteFiles:2",
|
||||||
"WalManager::GetSortedWalsOfType:2"}};
|
"WalManager::GetSortedWalsOfType:2"}};
|
||||||
for (int test = 0; test < LOG_ITERATOR_RACE_TEST_COUNT; ++test) {
|
for (int test = 0; test < LOG_ITERATOR_RACE_TEST_COUNT; ++test) {
|
||||||
|
@ -300,8 +298,8 @@ int main(int argc, char** argv) {
|
||||||
::testing::InitGoogleTest(&argc, argv);
|
::testing::InitGoogleTest(&argc, argv);
|
||||||
return RUN_ALL_TESTS();
|
return RUN_ALL_TESTS();
|
||||||
#else
|
#else
|
||||||
(void) argc;
|
(void)argc;
|
||||||
(void) argv;
|
(void)argv;
|
||||||
return 0;
|
return 0;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
|
@ -224,8 +224,8 @@ TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamilies) {
|
||||||
// Now cf_path_0_ in cache_ has been properly decreased and cf_path_0_'s entry
|
// Now cf_path_0_ in cache_ has been properly decreased and cf_path_0_'s entry
|
||||||
// is dropped from cache
|
// is dropped from cache
|
||||||
ASSERT_EQ(0, cache_->Size());
|
ASSERT_EQ(0, cache_->Size());
|
||||||
ASSERT_OK(DestroyDB(dbname_, options,
|
ASSERT_OK(
|
||||||
{{"cf1", cf_options}, {"cf2", cf_options}}));
|
DestroyDB(dbname_, options, {{"cf1", cf_options}, {"cf2", cf_options}}));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) {
|
TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) {
|
||||||
|
@ -313,8 +313,8 @@ TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) {
|
||||||
delete db;
|
delete db;
|
||||||
ASSERT_EQ(0, cache_->Size());
|
ASSERT_EQ(0, cache_->Size());
|
||||||
}
|
}
|
||||||
ASSERT_OK(DestroyDB(dbname_, options,
|
ASSERT_OK(
|
||||||
{{"cf1", cf_options}, {"cf2", cf_options}}));
|
DestroyDB(dbname_, options, {{"cf1", cf_options}, {"cf2", cf_options}}));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(DBLogicalBlockSizeCacheTest, DestroyColumnFamilyHandle) {
|
TEST_F(DBLogicalBlockSizeCacheTest, DestroyColumnFamilyHandle) {
|
||||||
|
|
|
@ -39,7 +39,7 @@ class LimitedStringAppendMergeOp : public StringAppendTESTOperator {
|
||||||
private:
|
private:
|
||||||
size_t limit_ = 0;
|
size_t limit_ = 0;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class DBMergeOperandTest : public DBTestBase {
|
class DBMergeOperandTest : public DBTestBase {
|
||||||
public:
|
public:
|
||||||
|
|
|
@ -84,8 +84,7 @@ TEST_F(DBMergeOperatorTest, LimitMergeOperands) {
|
||||||
Options options;
|
Options options;
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
// Use only the latest two merge operands.
|
// Use only the latest two merge operands.
|
||||||
options.merge_operator =
|
options.merge_operator = std::make_shared<LimitedStringAppendMergeOp>(2, ',');
|
||||||
std::make_shared<LimitedStringAppendMergeOp>(2, ',');
|
|
||||||
options.env = env_;
|
options.env = env_;
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
// All K1 values are in memtable.
|
// All K1 values are in memtable.
|
||||||
|
@ -203,7 +202,6 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) {
|
||||||
VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}});
|
VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class MergeOperatorPinningTest : public DBMergeOperatorTest,
|
class MergeOperatorPinningTest : public DBMergeOperatorTest,
|
||||||
public testing::WithParamInterface<bool> {
|
public testing::WithParamInterface<bool> {
|
||||||
public:
|
public:
|
||||||
|
@ -471,7 +469,7 @@ TEST_F(DBMergeOperatorTest, TailingIteratorMemtableUnrefedBySomeoneElse) {
|
||||||
"DBIter::MergeValuesNewToOld:SteppedToNextOperand", [&](void*) {
|
"DBIter::MergeValuesNewToOld:SteppedToNextOperand", [&](void*) {
|
||||||
EXPECT_FALSE(stepped_to_next_operand);
|
EXPECT_FALSE(stepped_to_next_operand);
|
||||||
stepped_to_next_operand = true;
|
stepped_to_next_operand = true;
|
||||||
someone_else.reset(); // Unpin SuperVersion A
|
someone_else.reset(); // Unpin SuperVersion A
|
||||||
});
|
});
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
||||||
|
|
||||||
|
|
|
@ -402,7 +402,7 @@ TEST_F(DBOptionsTest, SetWalBytesPerSync) {
|
||||||
// Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its
|
// Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its
|
||||||
// empty and will not get the new wal_bytes_per_sync value.
|
// empty and will not get the new wal_bytes_per_sync value.
|
||||||
low_bytes_per_sync = counter;
|
low_bytes_per_sync = counter;
|
||||||
//5242880 = 1024 * 1024 * 5
|
// 5242880 = 1024 * 1024 * 5
|
||||||
ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}}));
|
ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}}));
|
||||||
ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync);
|
ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync);
|
||||||
counter = 0;
|
counter = 0;
|
||||||
|
@ -604,7 +604,7 @@ TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) {
|
||||||
TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
|
TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
|
||||||
Options options;
|
Options options;
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
options.max_background_compactions = 1; // default value
|
options.max_background_compactions = 1; // default value
|
||||||
options.env = env_;
|
options.env = env_;
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
|
ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
|
||||||
|
@ -627,7 +627,6 @@ TEST_F(DBOptionsTest, SetBackgroundFlushThreads) {
|
||||||
ASSERT_EQ(3, dbfull()->TEST_BGFlushesAllowed());
|
ASSERT_EQ(3, dbfull()->TEST_BGFlushesAllowed());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_F(DBOptionsTest, SetBackgroundJobs) {
|
TEST_F(DBOptionsTest, SetBackgroundJobs) {
|
||||||
Options options;
|
Options options;
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
|
@ -691,7 +690,8 @@ TEST_F(DBOptionsTest, SetDelayedWriteRateOption) {
|
||||||
options.delayed_write_rate = 2 * 1024U * 1024U;
|
options.delayed_write_rate = 2 * 1024U * 1024U;
|
||||||
options.env = env_;
|
options.env = env_;
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
ASSERT_EQ(2 * 1024U * 1024U, dbfull()->TEST_write_controler().max_delayed_write_rate());
|
ASSERT_EQ(2 * 1024U * 1024U,
|
||||||
|
dbfull()->TEST_write_controler().max_delayed_write_rate());
|
||||||
|
|
||||||
ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}}));
|
ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}}));
|
||||||
ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate());
|
ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate());
|
||||||
|
|
|
@ -270,7 +270,8 @@ void GetExpectedTableProperties(
|
||||||
const int kDeletionCount = kTableCount * kDeletionsPerTable;
|
const int kDeletionCount = kTableCount * kDeletionsPerTable;
|
||||||
const int kMergeCount = kTableCount * kMergeOperandsPerTable;
|
const int kMergeCount = kTableCount * kMergeOperandsPerTable;
|
||||||
const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable;
|
const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable;
|
||||||
const int kKeyCount = kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
|
const int kKeyCount =
|
||||||
|
kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
|
||||||
const int kAvgSuccessorSize = kKeySize / 5;
|
const int kAvgSuccessorSize = kKeySize / 5;
|
||||||
const int kEncodingSavePerKey = kKeySize / 4;
|
const int kEncodingSavePerKey = kKeySize / 4;
|
||||||
expected_tp->raw_key_size = kKeyCount * (kKeySize + 8);
|
expected_tp->raw_key_size = kKeyCount * (kKeySize + 8);
|
||||||
|
@ -281,7 +282,8 @@ void GetExpectedTableProperties(
|
||||||
expected_tp->num_merge_operands = kMergeCount;
|
expected_tp->num_merge_operands = kMergeCount;
|
||||||
expected_tp->num_range_deletions = kRangeDeletionCount;
|
expected_tp->num_range_deletions = kRangeDeletionCount;
|
||||||
expected_tp->num_data_blocks =
|
expected_tp->num_data_blocks =
|
||||||
kTableCount * (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
|
kTableCount *
|
||||||
|
(kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
|
||||||
kBlockSize;
|
kBlockSize;
|
||||||
expected_tp->data_size =
|
expected_tp->data_size =
|
||||||
kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize));
|
kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize));
|
||||||
|
@ -1120,7 +1122,8 @@ class CountingUserTblPropCollector : public TablePropertiesCollector {
|
||||||
std::string encoded;
|
std::string encoded;
|
||||||
PutVarint32(&encoded, count_);
|
PutVarint32(&encoded, count_);
|
||||||
*properties = UserCollectedProperties{
|
*properties = UserCollectedProperties{
|
||||||
{"CountingUserTblPropCollector", message_}, {"Count", encoded},
|
{"CountingUserTblPropCollector", message_},
|
||||||
|
{"Count", encoded},
|
||||||
};
|
};
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
|
@ -2122,7 +2125,7 @@ std::string PopMetaIndexKey(InternalIterator* meta_iter) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBPropertiesTest, TableMetaIndexKeys) {
|
TEST_F(DBPropertiesTest, TableMetaIndexKeys) {
|
||||||
// This is to detect unexpected churn in metaindex block keys. This is more
|
// This is to detect unexpected churn in metaindex block keys. This is more
|
||||||
|
|
|
@ -238,7 +238,8 @@ TEST_F(DBRangeDelTest, SentinelsOmittedFromOutputFile) {
|
||||||
const Snapshot* snapshot = db_->GetSnapshot();
|
const Snapshot* snapshot = db_->GetSnapshot();
|
||||||
|
|
||||||
// gaps between ranges creates sentinels in our internal representation
|
// gaps between ranges creates sentinels in our internal representation
|
||||||
std::vector<std::pair<std::string, std::string>> range_dels = {{"a", "b"}, {"c", "d"}, {"e", "f"}};
|
std::vector<std::pair<std::string, std::string>> range_dels = {
|
||||||
|
{"a", "b"}, {"c", "d"}, {"e", "f"}};
|
||||||
for (const auto& range_del : range_dels) {
|
for (const auto& range_del : range_dels) {
|
||||||
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
|
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
|
||||||
range_del.first, range_del.second));
|
range_del.first, range_del.second));
|
||||||
|
@ -567,8 +568,8 @@ TEST_F(DBRangeDelTest, PutDeleteRangeMergeFlush) {
|
||||||
std::string val;
|
std::string val;
|
||||||
PutFixed64(&val, 1);
|
PutFixed64(&val, 1);
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), "key", val));
|
ASSERT_OK(db_->Put(WriteOptions(), "key", val));
|
||||||
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
|
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "key",
|
||||||
"key", "key_"));
|
"key_"));
|
||||||
ASSERT_OK(db_->Merge(WriteOptions(), "key", val));
|
ASSERT_OK(db_->Merge(WriteOptions(), "key", val));
|
||||||
ASSERT_OK(db_->Flush(FlushOptions()));
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
||||||
|
|
||||||
|
@ -1332,7 +1333,7 @@ TEST_F(DBRangeDelTest, UntruncatedTombstoneDoesNotDeleteNewerKey) {
|
||||||
const int kFileBytes = 1 << 20;
|
const int kFileBytes = 1 << 20;
|
||||||
const int kValueBytes = 1 << 10;
|
const int kValueBytes = 1 << 10;
|
||||||
const int kNumFiles = 4;
|
const int kNumFiles = 4;
|
||||||
const int kMaxKey = kNumFiles* kFileBytes / kValueBytes;
|
const int kMaxKey = kNumFiles * kFileBytes / kValueBytes;
|
||||||
const int kKeysOverwritten = 10;
|
const int kKeysOverwritten = 10;
|
||||||
|
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
|
@ -1649,7 +1650,8 @@ TEST_F(DBRangeDelTest, RangeTombstoneWrittenToMinimalSsts) {
|
||||||
const auto& table_props = name_and_table_props.second;
|
const auto& table_props = name_and_table_props.second;
|
||||||
// The range tombstone should only be output to the second L1 SST.
|
// The range tombstone should only be output to the second L1 SST.
|
||||||
if (name.size() >= l1_metadata[1].name.size() &&
|
if (name.size() >= l1_metadata[1].name.size() &&
|
||||||
name.substr(name.size() - l1_metadata[1].name.size()).compare(l1_metadata[1].name) == 0) {
|
name.substr(name.size() - l1_metadata[1].name.size())
|
||||||
|
.compare(l1_metadata[1].name) == 0) {
|
||||||
ASSERT_EQ(1, table_props->num_range_deletions);
|
ASSERT_EQ(1, table_props->num_range_deletions);
|
||||||
++num_range_deletions;
|
++num_range_deletions;
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -499,7 +499,7 @@ class TraceFileEnv : public EnvWrapper {
|
||||||
private:
|
private:
|
||||||
std::atomic<int> files_closed_{0};
|
std::atomic<int> files_closed_{0};
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBSecondaryTest, SecondaryCloseFiles) {
|
TEST_F(DBSecondaryTest, SecondaryCloseFiles) {
|
||||||
Options options;
|
Options options;
|
||||||
|
|
|
@ -70,9 +70,9 @@ TEST_F(DBStatisticsTest, CompressionStatsTest) {
|
||||||
options.compression = kNoCompression;
|
options.compression = kNoCompression;
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
uint64_t currentCompressions =
|
uint64_t currentCompressions =
|
||||||
options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
|
options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
|
||||||
uint64_t currentDecompressions =
|
uint64_t currentDecompressions =
|
||||||
options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED);
|
options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED);
|
||||||
|
|
||||||
// Check that compressions do not occur when turned off
|
// Check that compressions do not occur when turned off
|
||||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||||
|
@ -80,14 +80,16 @@ TEST_F(DBStatisticsTest, CompressionStatsTest) {
|
||||||
ASSERT_OK(Put(Key(i), rnd.RandomString(128) + std::string(128, 'a')));
|
ASSERT_OK(Put(Key(i), rnd.RandomString(128) + std::string(128, 'a')));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED)
|
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED) -
|
||||||
- currentCompressions, 0);
|
currentCompressions,
|
||||||
|
0);
|
||||||
|
|
||||||
for (int i = 0; i < kNumKeysWritten; ++i) {
|
for (int i = 0; i < kNumKeysWritten; ++i) {
|
||||||
auto r = Get(Key(i));
|
auto r = Get(Key(i));
|
||||||
}
|
}
|
||||||
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED)
|
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED) -
|
||||||
- currentDecompressions, 0);
|
currentDecompressions,
|
||||||
|
0);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) {
|
TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) {
|
||||||
|
|
|
@ -52,7 +52,7 @@ void VerifyTableProperties(DB* db, uint64_t expected_entries_size) {
|
||||||
|
|
||||||
VerifySstUniqueIds(props);
|
VerifySstUniqueIds(props);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class DBTablePropertiesTest : public DBTestBase,
|
class DBTablePropertiesTest : public DBTestBase,
|
||||||
public testing::WithParamInterface<std::string> {
|
public testing::WithParamInterface<std::string> {
|
||||||
|
@ -240,7 +240,6 @@ TablePropertiesCollection
|
||||||
DBTablePropertiesTest::TestGetPropertiesOfTablesInRange(
|
DBTablePropertiesTest::TestGetPropertiesOfTablesInRange(
|
||||||
std::vector<Range> ranges, std::size_t* num_properties,
|
std::vector<Range> ranges, std::size_t* num_properties,
|
||||||
std::size_t* num_files) {
|
std::size_t* num_files) {
|
||||||
|
|
||||||
// Since we deref zero element in the vector it can not be empty
|
// Since we deref zero element in the vector it can not be empty
|
||||||
// otherwise we pass an address to some random memory
|
// otherwise we pass an address to some random memory
|
||||||
EXPECT_GT(ranges.size(), 0U);
|
EXPECT_GT(ranges.size(), 0U);
|
||||||
|
@ -469,12 +468,12 @@ INSTANTIATE_TEST_CASE_P(
|
||||||
|
|
||||||
class DeletionTriggeredCompactionTestListener : public EventListener {
|
class DeletionTriggeredCompactionTestListener : public EventListener {
|
||||||
public:
|
public:
|
||||||
void OnCompactionBegin(DB* , const CompactionJobInfo& ci) override {
|
void OnCompactionBegin(DB*, const CompactionJobInfo& ci) override {
|
||||||
ASSERT_EQ(ci.compaction_reason,
|
ASSERT_EQ(ci.compaction_reason,
|
||||||
CompactionReason::kFilesMarkedForCompaction);
|
CompactionReason::kFilesMarkedForCompaction);
|
||||||
}
|
}
|
||||||
|
|
||||||
void OnCompactionCompleted(DB* , const CompactionJobInfo& ci) override {
|
void OnCompactionCompleted(DB*, const CompactionJobInfo& ci) override {
|
||||||
ASSERT_EQ(ci.compaction_reason,
|
ASSERT_EQ(ci.compaction_reason,
|
||||||
CompactionReason::kFilesMarkedForCompaction);
|
CompactionReason::kFilesMarkedForCompaction);
|
||||||
}
|
}
|
||||||
|
@ -485,13 +484,13 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
|
||||||
int kWindowSize = 100;
|
int kWindowSize = 100;
|
||||||
int kNumDelsTrigger = 90;
|
int kNumDelsTrigger = 90;
|
||||||
std::shared_ptr<TablePropertiesCollectorFactory> compact_on_del =
|
std::shared_ptr<TablePropertiesCollectorFactory> compact_on_del =
|
||||||
NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger);
|
NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger);
|
||||||
|
|
||||||
Options opts = CurrentOptions();
|
Options opts = CurrentOptions();
|
||||||
opts.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
opts.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
||||||
opts.table_properties_collector_factories.emplace_back(compact_on_del);
|
opts.table_properties_collector_factories.emplace_back(compact_on_del);
|
||||||
|
|
||||||
if(GetParam() == "kCompactionStyleUniversal") {
|
if (GetParam() == "kCompactionStyleUniversal") {
|
||||||
opts.compaction_style = kCompactionStyleUniversal;
|
opts.compaction_style = kCompactionStyleUniversal;
|
||||||
}
|
}
|
||||||
Reopen(opts);
|
Reopen(opts);
|
||||||
|
@ -502,8 +501,8 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
MoveFilesToLevel(1);
|
MoveFilesToLevel(1);
|
||||||
|
|
||||||
DeletionTriggeredCompactionTestListener *listener =
|
DeletionTriggeredCompactionTestListener* listener =
|
||||||
new DeletionTriggeredCompactionTestListener();
|
new DeletionTriggeredCompactionTestListener();
|
||||||
opts.listeners.emplace_back(listener);
|
opts.listeners.emplace_back(listener);
|
||||||
Reopen(opts);
|
Reopen(opts);
|
||||||
|
|
||||||
|
@ -524,10 +523,10 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
|
||||||
// effect
|
// effect
|
||||||
kWindowSize = 50;
|
kWindowSize = 50;
|
||||||
kNumDelsTrigger = 40;
|
kNumDelsTrigger = 40;
|
||||||
static_cast<CompactOnDeletionCollectorFactory*>
|
static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
|
||||||
(compact_on_del.get())->SetWindowSize(kWindowSize);
|
->SetWindowSize(kWindowSize);
|
||||||
static_cast<CompactOnDeletionCollectorFactory*>
|
static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
|
||||||
(compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger);
|
->SetDeletionTrigger(kNumDelsTrigger);
|
||||||
for (int i = 0; i < kNumKeys; ++i) {
|
for (int i = 0; i < kNumKeys; ++i) {
|
||||||
if (i >= kNumKeys - kWindowSize &&
|
if (i >= kNumKeys - kWindowSize &&
|
||||||
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
|
||||||
|
@ -543,10 +542,10 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
|
||||||
|
|
||||||
// Change the window size to disable delete triggered compaction
|
// Change the window size to disable delete triggered compaction
|
||||||
kWindowSize = 0;
|
kWindowSize = 0;
|
||||||
static_cast<CompactOnDeletionCollectorFactory*>
|
static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
|
||||||
(compact_on_del.get())->SetWindowSize(kWindowSize);
|
->SetWindowSize(kWindowSize);
|
||||||
static_cast<CompactOnDeletionCollectorFactory*>
|
static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
|
||||||
(compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger);
|
->SetDeletionTrigger(kNumDelsTrigger);
|
||||||
for (int i = 0; i < kNumKeys; ++i) {
|
for (int i = 0; i < kNumKeys; ++i) {
|
||||||
if (i >= kNumKeys - kWindowSize &&
|
if (i >= kNumKeys - kWindowSize &&
|
||||||
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
|
||||||
|
@ -611,13 +610,9 @@ TEST_P(DBTablePropertiesTest, RatioBasedDeletionTriggeredCompactionMarking) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
INSTANTIATE_TEST_CASE_P(
|
INSTANTIATE_TEST_CASE_P(DBTablePropertiesTest, DBTablePropertiesTest,
|
||||||
DBTablePropertiesTest,
|
::testing::Values("kCompactionStyleLevel",
|
||||||
DBTablePropertiesTest,
|
"kCompactionStyleUniversal"));
|
||||||
::testing::Values(
|
|
||||||
"kCompactionStyleLevel",
|
|
||||||
"kCompactionStyleUniversal"
|
|
||||||
));
|
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
|
|
@ -399,7 +399,7 @@ TEST_P(DBTestTailingIterator, TailingIteratorSeekToSame) {
|
||||||
// Write rows with keys 00000, 00002, 00004 etc.
|
// Write rows with keys 00000, 00002, 00004 etc.
|
||||||
for (int i = 0; i < NROWS; ++i) {
|
for (int i = 0; i < NROWS; ++i) {
|
||||||
char buf[100];
|
char buf[100];
|
||||||
snprintf(buf, sizeof(buf), "%05d", 2*i);
|
snprintf(buf, sizeof(buf), "%05d", 2 * i);
|
||||||
std::string key(buf);
|
std::string key(buf);
|
||||||
std::string value("value");
|
std::string value("value");
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), key, value));
|
ASSERT_OK(db_->Put(WriteOptions(), key, value));
|
||||||
|
@ -539,7 +539,6 @@ TEST_P(DBTestTailingIterator, SeekWithUpperBoundBug) {
|
||||||
const Slice upper_bound("cc", 3);
|
const Slice upper_bound("cc", 3);
|
||||||
read_options.iterate_upper_bound = &upper_bound;
|
read_options.iterate_upper_bound = &upper_bound;
|
||||||
|
|
||||||
|
|
||||||
// 1st L0 file
|
// 1st L0 file
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
|
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
|
@ -565,7 +564,6 @@ TEST_P(DBTestTailingIterator, SeekToFirstWithUpperBoundBug) {
|
||||||
const Slice upper_bound("cc", 3);
|
const Slice upper_bound("cc", 3);
|
||||||
read_options.iterate_upper_bound = &upper_bound;
|
read_options.iterate_upper_bound = &upper_bound;
|
||||||
|
|
||||||
|
|
||||||
// 1st L0 file
|
// 1st L0 file
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
|
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
|
@ -599,8 +597,8 @@ int main(int argc, char** argv) {
|
||||||
::testing::InitGoogleTest(&argc, argv);
|
::testing::InitGoogleTest(&argc, argv);
|
||||||
return RUN_ALL_TESTS();
|
return RUN_ALL_TESTS();
|
||||||
#else
|
#else
|
||||||
(void) argc;
|
(void)argc;
|
||||||
(void) argv;
|
(void)argv;
|
||||||
return 0;
|
return 0;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
|
@ -1135,7 +1135,7 @@ class DelayFilterFactory : public CompactionFilterFactory {
|
||||||
private:
|
private:
|
||||||
DBTestBase* db_test;
|
DBTestBase* db_test;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
|
|
||||||
|
@ -1490,7 +1490,7 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest, MinLevelToCompress1) {
|
TEST_F(DBTest, MinLevelToCompress1) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
|
@ -2843,7 +2843,7 @@ static void MTThreadBody(void* arg) {
|
||||||
fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
|
fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class MultiThreadedDBTest
|
class MultiThreadedDBTest
|
||||||
: public DBTest,
|
: public DBTest,
|
||||||
|
@ -2929,7 +2929,7 @@ static void GCThreadBody(void* arg) {
|
||||||
t->done = true;
|
t->done = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest, GroupCommitTest) {
|
TEST_F(DBTest, GroupCommitTest) {
|
||||||
do {
|
do {
|
||||||
|
@ -4645,7 +4645,7 @@ void VerifyOperationCount(Env* env, ThreadStatus::OperationType op_type,
|
||||||
}
|
}
|
||||||
ASSERT_EQ(op_count, expected_count);
|
ASSERT_EQ(op_count, expected_count);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest, GetThreadStatus) {
|
TEST_F(DBTest, GetThreadStatus) {
|
||||||
Options options;
|
Options options;
|
||||||
|
|
345
db/db_test2.cc
345
db/db_test2.cc
|
@ -669,33 +669,33 @@ TEST_F(DBTest2, TestWriteBufferNoLimitWithCache) {
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
|
void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
|
||||||
const std::vector<Slice>& keys_must_not_exist) {
|
const std::vector<Slice>& keys_must_not_exist) {
|
||||||
// Ensure that expected keys exist
|
// Ensure that expected keys exist
|
||||||
std::vector<std::string> values;
|
std::vector<std::string> values;
|
||||||
if (keys_must_exist.size() > 0) {
|
if (keys_must_exist.size() > 0) {
|
||||||
std::vector<Status> status_list =
|
std::vector<Status> status_list =
|
||||||
db->MultiGet(ReadOptions(), keys_must_exist, &values);
|
db->MultiGet(ReadOptions(), keys_must_exist, &values);
|
||||||
for (size_t i = 0; i < keys_must_exist.size(); i++) {
|
for (size_t i = 0; i < keys_must_exist.size(); i++) {
|
||||||
ASSERT_OK(status_list[i]);
|
ASSERT_OK(status_list[i]);
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure that given keys don't exist
|
|
||||||
if (keys_must_not_exist.size() > 0) {
|
|
||||||
std::vector<Status> status_list =
|
|
||||||
db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
|
|
||||||
for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
|
|
||||||
ASSERT_TRUE(status_list[i].IsNotFound());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
// Ensure that given keys don't exist
|
||||||
|
if (keys_must_not_exist.size() > 0) {
|
||||||
|
std::vector<Status> status_list =
|
||||||
|
db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
|
||||||
|
for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
|
||||||
|
ASSERT_TRUE(status_list[i].IsNotFound());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest2, WalFilterTest) {
|
TEST_F(DBTest2, WalFilterTest) {
|
||||||
class TestWalFilter : public WalFilter {
|
class TestWalFilter : public WalFilter {
|
||||||
private:
|
private:
|
||||||
// Processing option that is requested to be applied at the given index
|
// Processing option that is requested to be applied at the given index
|
||||||
WalFilter::WalProcessingOption wal_processing_option_;
|
WalFilter::WalProcessingOption wal_processing_option_;
|
||||||
// Index at which to apply wal_processing_option_
|
// Index at which to apply wal_processing_option_
|
||||||
|
@ -705,12 +705,12 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
// Current record index, incremented with each record encountered.
|
// Current record index, incremented with each record encountered.
|
||||||
size_t current_record_index_;
|
size_t current_record_index_;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
|
TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
|
||||||
size_t apply_option_for_record_index)
|
size_t apply_option_for_record_index)
|
||||||
: wal_processing_option_(wal_processing_option),
|
: wal_processing_option_(wal_processing_option),
|
||||||
apply_option_at_record_index_(apply_option_for_record_index),
|
apply_option_at_record_index_(apply_option_for_record_index),
|
||||||
current_record_index_(0) {}
|
current_record_index_(0) {}
|
||||||
|
|
||||||
WalProcessingOption LogRecord(const WriteBatch& /*batch*/,
|
WalProcessingOption LogRecord(const WriteBatch& /*batch*/,
|
||||||
WriteBatch* /*new_batch*/,
|
WriteBatch* /*new_batch*/,
|
||||||
|
@ -719,8 +719,7 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
|
|
||||||
if (current_record_index_ == apply_option_at_record_index_) {
|
if (current_record_index_ == apply_option_at_record_index_) {
|
||||||
option_to_return = wal_processing_option_;
|
option_to_return = wal_processing_option_;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
option_to_return = WalProcessingOption::kContinueProcessing;
|
option_to_return = WalProcessingOption::kContinueProcessing;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -747,12 +746,12 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
|
|
||||||
// Test with all WAL processing options
|
// Test with all WAL processing options
|
||||||
for (int option = 0;
|
for (int option = 0;
|
||||||
option < static_cast<int>(
|
option < static_cast<int>(
|
||||||
WalFilter::WalProcessingOption::kWalProcessingOptionMax);
|
WalFilter::WalProcessingOption::kWalProcessingOptionMax);
|
||||||
option++) {
|
option++) {
|
||||||
Options options = OptionsForLogIterTest();
|
Options options = OptionsForLogIterTest();
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
CreateAndReopenWithCF({ "pikachu" }, options);
|
CreateAndReopenWithCF({"pikachu"}, options);
|
||||||
|
|
||||||
// Write given keys in given batches
|
// Write given keys in given batches
|
||||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||||
|
@ -764,28 +763,27 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
}
|
}
|
||||||
|
|
||||||
WalFilter::WalProcessingOption wal_processing_option =
|
WalFilter::WalProcessingOption wal_processing_option =
|
||||||
static_cast<WalFilter::WalProcessingOption>(option);
|
static_cast<WalFilter::WalProcessingOption>(option);
|
||||||
|
|
||||||
// Create a test filter that would apply wal_processing_option at the first
|
// Create a test filter that would apply wal_processing_option at the first
|
||||||
// record
|
// record
|
||||||
size_t apply_option_for_record_index = 1;
|
size_t apply_option_for_record_index = 1;
|
||||||
TestWalFilter test_wal_filter(wal_processing_option,
|
TestWalFilter test_wal_filter(wal_processing_option,
|
||||||
apply_option_for_record_index);
|
apply_option_for_record_index);
|
||||||
|
|
||||||
// Reopen database with option to use WAL filter
|
// Reopen database with option to use WAL filter
|
||||||
options = OptionsForLogIterTest();
|
options = OptionsForLogIterTest();
|
||||||
options.wal_filter = &test_wal_filter;
|
options.wal_filter = &test_wal_filter;
|
||||||
Status status =
|
Status status =
|
||||||
TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
TryReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||||
if (wal_processing_option ==
|
if (wal_processing_option ==
|
||||||
WalFilter::WalProcessingOption::kCorruptedRecord) {
|
WalFilter::WalProcessingOption::kCorruptedRecord) {
|
||||||
ASSERT_NOK(status);
|
ASSERT_NOK(status);
|
||||||
// In case of corruption we can turn off paranoid_checks to reopen
|
// In case of corruption we can turn off paranoid_checks to reopen
|
||||||
// databse
|
// databse
|
||||||
options.paranoid_checks = false;
|
options.paranoid_checks = false;
|
||||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
ASSERT_OK(status);
|
ASSERT_OK(status);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -794,56 +792,54 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
std::vector<Slice> keys_must_exist;
|
std::vector<Slice> keys_must_exist;
|
||||||
std::vector<Slice> keys_must_not_exist;
|
std::vector<Slice> keys_must_not_exist;
|
||||||
switch (wal_processing_option) {
|
switch (wal_processing_option) {
|
||||||
case WalFilter::WalProcessingOption::kCorruptedRecord:
|
case WalFilter::WalProcessingOption::kCorruptedRecord:
|
||||||
case WalFilter::WalProcessingOption::kContinueProcessing: {
|
case WalFilter::WalProcessingOption::kContinueProcessing: {
|
||||||
fprintf(stderr, "Testing with complete WAL processing\n");
|
fprintf(stderr, "Testing with complete WAL processing\n");
|
||||||
// we expect all records to be processed
|
// we expect all records to be processed
|
||||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
|
|
||||||
fprintf(stderr,
|
|
||||||
"Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
|
|
||||||
apply_option_for_record_index);
|
|
||||||
// We expect the record with apply_option_for_record_index to be not
|
|
||||||
// found.
|
|
||||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
|
||||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
|
||||||
if (i == apply_option_for_record_index) {
|
|
||||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
break;
|
case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
|
||||||
}
|
fprintf(stderr,
|
||||||
case WalFilter::WalProcessingOption::kStopReplay: {
|
"Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
|
||||||
fprintf(stderr,
|
apply_option_for_record_index);
|
||||||
"Testing with stopping replay from record %" ROCKSDB_PRIszt
|
// We expect the record with apply_option_for_record_index to be not
|
||||||
"\n",
|
// found.
|
||||||
apply_option_for_record_index);
|
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||||
// We expect records beyond apply_option_for_record_index to be not
|
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||||
// found.
|
if (i == apply_option_for_record_index) {
|
||||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
} else {
|
||||||
if (i >= apply_option_for_record_index) {
|
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
}
|
||||||
}
|
|
||||||
else {
|
|
||||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
break;
|
case WalFilter::WalProcessingOption::kStopReplay: {
|
||||||
}
|
fprintf(stderr,
|
||||||
default:
|
"Testing with stopping replay from record %" ROCKSDB_PRIszt
|
||||||
FAIL(); // unhandled case
|
"\n",
|
||||||
|
apply_option_for_record_index);
|
||||||
|
// We expect records beyond apply_option_for_record_index to be not
|
||||||
|
// found.
|
||||||
|
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||||
|
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||||
|
if (i >= apply_option_for_record_index) {
|
||||||
|
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
|
} else {
|
||||||
|
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
FAIL(); // unhandled case
|
||||||
}
|
}
|
||||||
|
|
||||||
bool checked_after_reopen = false;
|
bool checked_after_reopen = false;
|
||||||
|
@ -861,7 +857,7 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
//(even if they were skipped)
|
//(even if they were skipped)
|
||||||
// reopn database with option to use WAL filter
|
// reopn database with option to use WAL filter
|
||||||
options = OptionsForLogIterTest();
|
options = OptionsForLogIterTest();
|
||||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||||
|
|
||||||
checked_after_reopen = true;
|
checked_after_reopen = true;
|
||||||
}
|
}
|
||||||
|
@ -870,7 +866,7 @@ TEST_F(DBTest2, WalFilterTest) {
|
||||||
|
|
||||||
TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
class ChangeBatchHandler : public WriteBatch::Handler {
|
class ChangeBatchHandler : public WriteBatch::Handler {
|
||||||
private:
|
private:
|
||||||
// Batch to insert keys in
|
// Batch to insert keys in
|
||||||
WriteBatch* new_write_batch_;
|
WriteBatch* new_write_batch_;
|
||||||
// Number of keys to add in the new batch
|
// Number of keys to add in the new batch
|
||||||
|
@ -878,12 +874,12 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
// Number of keys added to new batch
|
// Number of keys added to new batch
|
||||||
size_t num_keys_added_;
|
size_t num_keys_added_;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
ChangeBatchHandler(WriteBatch* new_write_batch,
|
ChangeBatchHandler(WriteBatch* new_write_batch,
|
||||||
size_t num_keys_to_add_in_new_batch)
|
size_t num_keys_to_add_in_new_batch)
|
||||||
: new_write_batch_(new_write_batch),
|
: new_write_batch_(new_write_batch),
|
||||||
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
|
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
|
||||||
num_keys_added_(0) {}
|
num_keys_added_(0) {}
|
||||||
void Put(const Slice& key, const Slice& value) override {
|
void Put(const Slice& key, const Slice& value) override {
|
||||||
if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
|
if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
|
||||||
ASSERT_OK(new_write_batch_->Put(key, value));
|
ASSERT_OK(new_write_batch_->Put(key, value));
|
||||||
|
@ -893,7 +889,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
};
|
};
|
||||||
|
|
||||||
class TestWalFilterWithChangeBatch : public WalFilter {
|
class TestWalFilterWithChangeBatch : public WalFilter {
|
||||||
private:
|
private:
|
||||||
// Index at which to start changing records
|
// Index at which to start changing records
|
||||||
size_t change_records_from_index_;
|
size_t change_records_from_index_;
|
||||||
// Number of keys to add in the new batch
|
// Number of keys to add in the new batch
|
||||||
|
@ -901,12 +897,12 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
// Current record index, incremented with each record encountered.
|
// Current record index, incremented with each record encountered.
|
||||||
size_t current_record_index_;
|
size_t current_record_index_;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
TestWalFilterWithChangeBatch(size_t change_records_from_index,
|
TestWalFilterWithChangeBatch(size_t change_records_from_index,
|
||||||
size_t num_keys_to_add_in_new_batch)
|
size_t num_keys_to_add_in_new_batch)
|
||||||
: change_records_from_index_(change_records_from_index),
|
: change_records_from_index_(change_records_from_index),
|
||||||
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
|
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
|
||||||
current_record_index_(0) {}
|
current_record_index_(0) {}
|
||||||
|
|
||||||
WalProcessingOption LogRecord(const WriteBatch& batch,
|
WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||||
WriteBatch* new_batch,
|
WriteBatch* new_batch,
|
||||||
|
@ -925,7 +921,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
// object, however we modify it for our own purpose here and hence
|
// object, however we modify it for our own purpose here and hence
|
||||||
// cast the constness away.
|
// cast the constness away.
|
||||||
(const_cast<TestWalFilterWithChangeBatch*>(this)
|
(const_cast<TestWalFilterWithChangeBatch*>(this)
|
||||||
->current_record_index_)++;
|
->current_record_index_)++;
|
||||||
|
|
||||||
return WalProcessingOption::kContinueProcessing;
|
return WalProcessingOption::kContinueProcessing;
|
||||||
}
|
}
|
||||||
|
@ -944,7 +940,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
|
|
||||||
Options options = OptionsForLogIterTest();
|
Options options = OptionsForLogIterTest();
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
CreateAndReopenWithCF({ "pikachu" }, options);
|
CreateAndReopenWithCF({"pikachu"}, options);
|
||||||
|
|
||||||
// Write given keys in given batches
|
// Write given keys in given batches
|
||||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||||
|
@ -960,12 +956,12 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
size_t change_records_from_index = 1;
|
size_t change_records_from_index = 1;
|
||||||
size_t num_keys_to_add_in_new_batch = 1;
|
size_t num_keys_to_add_in_new_batch = 1;
|
||||||
TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
|
TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
|
||||||
change_records_from_index, num_keys_to_add_in_new_batch);
|
change_records_from_index, num_keys_to_add_in_new_batch);
|
||||||
|
|
||||||
// Reopen database with option to use WAL filter
|
// Reopen database with option to use WAL filter
|
||||||
options = OptionsForLogIterTest();
|
options = OptionsForLogIterTest();
|
||||||
options.wal_filter = &test_wal_filter_with_change_batch;
|
options.wal_filter = &test_wal_filter_with_change_batch;
|
||||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||||
|
|
||||||
// Ensure that all keys exist before change_records_from_index_
|
// Ensure that all keys exist before change_records_from_index_
|
||||||
// And after that index only single key exists
|
// And after that index only single key exists
|
||||||
|
@ -977,8 +973,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||||
if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) {
|
if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) {
|
||||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -999,7 +994,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
//(even if they were skipped)
|
//(even if they were skipped)
|
||||||
// reopn database with option to use WAL filter
|
// reopn database with option to use WAL filter
|
||||||
options = OptionsForLogIterTest();
|
options = OptionsForLogIterTest();
|
||||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||||
|
|
||||||
checked_after_reopen = true;
|
checked_after_reopen = true;
|
||||||
}
|
}
|
||||||
|
@ -1007,22 +1002,23 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||||
|
|
||||||
TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
|
TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
|
||||||
class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
|
class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
|
||||||
public:
|
public:
|
||||||
WalProcessingOption LogRecord(const WriteBatch& batch, WriteBatch* new_batch,
|
WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||||
bool* batch_changed) const override {
|
WriteBatch* new_batch,
|
||||||
*new_batch = batch;
|
bool* batch_changed) const override {
|
||||||
Status s = new_batch->Put("key_extra", "value_extra");
|
*new_batch = batch;
|
||||||
if (s.ok()) {
|
Status s = new_batch->Put("key_extra", "value_extra");
|
||||||
*batch_changed = true;
|
if (s.ok()) {
|
||||||
} else {
|
*batch_changed = true;
|
||||||
assert(false);
|
} else {
|
||||||
}
|
assert(false);
|
||||||
return WalProcessingOption::kContinueProcessing;
|
}
|
||||||
}
|
return WalProcessingOption::kContinueProcessing;
|
||||||
|
}
|
||||||
|
|
||||||
const char* Name() const override {
|
const char* Name() const override {
|
||||||
return "WalFilterTestWithChangeBatchExtraKeys";
|
return "WalFilterTestWithChangeBatchExtraKeys";
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
std::vector<std::vector<std::string>> batch_keys(3);
|
std::vector<std::vector<std::string>> batch_keys(3);
|
||||||
|
@ -1036,7 +1032,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
|
||||||
|
|
||||||
Options options = OptionsForLogIterTest();
|
Options options = OptionsForLogIterTest();
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
CreateAndReopenWithCF({ "pikachu" }, options);
|
CreateAndReopenWithCF({"pikachu"}, options);
|
||||||
|
|
||||||
// Write given keys in given batches
|
// Write given keys in given batches
|
||||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||||
|
@ -1059,7 +1055,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
|
||||||
// Reopen without filter, now reopen should succeed - previous
|
// Reopen without filter, now reopen should succeed - previous
|
||||||
// attempt to open must not have altered the db.
|
// attempt to open must not have altered the db.
|
||||||
options = OptionsForLogIterTest();
|
options = OptionsForLogIterTest();
|
||||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||||
|
|
||||||
std::vector<Slice> keys_must_exist;
|
std::vector<Slice> keys_must_exist;
|
||||||
std::vector<Slice> keys_must_not_exist; // empty vector
|
std::vector<Slice> keys_must_not_exist; // empty vector
|
||||||
|
@ -1075,7 +1071,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
|
||||||
|
|
||||||
TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
class TestWalFilterWithColumnFamilies : public WalFilter {
|
class TestWalFilterWithColumnFamilies : public WalFilter {
|
||||||
private:
|
private:
|
||||||
// column_family_id -> log_number map (provided to WALFilter)
|
// column_family_id -> log_number map (provided to WALFilter)
|
||||||
std::map<uint32_t, uint64_t> cf_log_number_map_;
|
std::map<uint32_t, uint64_t> cf_log_number_map_;
|
||||||
// column_family_name -> column_family_id map (provided to WALFilter)
|
// column_family_name -> column_family_id map (provided to WALFilter)
|
||||||
|
@ -1085,31 +1081,34 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
// during recovery (i.e. aren't already flushed to SST file(s))
|
// during recovery (i.e. aren't already flushed to SST file(s))
|
||||||
// for verification against the keys we expect.
|
// for verification against the keys we expect.
|
||||||
std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
|
std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
|
||||||
public:
|
|
||||||
void ColumnFamilyLogNumberMap(
|
|
||||||
const std::map<uint32_t, uint64_t>& cf_lognumber_map,
|
|
||||||
const std::map<std::string, uint32_t>& cf_name_id_map) override {
|
|
||||||
cf_log_number_map_ = cf_lognumber_map;
|
|
||||||
cf_name_id_map_ = cf_name_id_map;
|
|
||||||
}
|
|
||||||
|
|
||||||
WalProcessingOption LogRecordFound(unsigned long long log_number,
|
public:
|
||||||
const std::string& /*log_file_name*/,
|
void ColumnFamilyLogNumberMap(
|
||||||
const WriteBatch& batch,
|
const std::map<uint32_t, uint64_t>& cf_lognumber_map,
|
||||||
WriteBatch* /*new_batch*/,
|
const std::map<std::string, uint32_t>& cf_name_id_map) override {
|
||||||
bool* /*batch_changed*/) override {
|
cf_log_number_map_ = cf_lognumber_map;
|
||||||
class LogRecordBatchHandler : public WriteBatch::Handler {
|
cf_name_id_map_ = cf_name_id_map;
|
||||||
private:
|
}
|
||||||
const std::map<uint32_t, uint64_t> & cf_log_number_map_;
|
|
||||||
std::map<uint32_t, std::vector<std::string>> & cf_wal_keys_;
|
WalProcessingOption LogRecordFound(unsigned long long log_number,
|
||||||
|
const std::string& /*log_file_name*/,
|
||||||
|
const WriteBatch& batch,
|
||||||
|
WriteBatch* /*new_batch*/,
|
||||||
|
bool* /*batch_changed*/) override {
|
||||||
|
class LogRecordBatchHandler : public WriteBatch::Handler {
|
||||||
|
private:
|
||||||
|
const std::map<uint32_t, uint64_t>& cf_log_number_map_;
|
||||||
|
std::map<uint32_t, std::vector<std::string>>& cf_wal_keys_;
|
||||||
unsigned long long log_number_;
|
unsigned long long log_number_;
|
||||||
public:
|
|
||||||
LogRecordBatchHandler(unsigned long long current_log_number,
|
public:
|
||||||
const std::map<uint32_t, uint64_t> & cf_log_number_map,
|
LogRecordBatchHandler(
|
||||||
std::map<uint32_t, std::vector<std::string>> & cf_wal_keys) :
|
unsigned long long current_log_number,
|
||||||
cf_log_number_map_(cf_log_number_map),
|
const std::map<uint32_t, uint64_t>& cf_log_number_map,
|
||||||
cf_wal_keys_(cf_wal_keys),
|
std::map<uint32_t, std::vector<std::string>>& cf_wal_keys)
|
||||||
log_number_(current_log_number){}
|
: cf_log_number_map_(cf_log_number_map),
|
||||||
|
cf_wal_keys_(cf_wal_keys),
|
||||||
|
log_number_(current_log_number) {}
|
||||||
|
|
||||||
Status PutCF(uint32_t column_family_id, const Slice& key,
|
Status PutCF(uint32_t column_family_id, const Slice& key,
|
||||||
const Slice& /*value*/) override {
|
const Slice& /*value*/) override {
|
||||||
|
@ -1120,8 +1119,8 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
// (i.e. isn't flushed to SST file(s) for column_family_id)
|
// (i.e. isn't flushed to SST file(s) for column_family_id)
|
||||||
// add it to the cf_wal_keys_ map for verification.
|
// add it to the cf_wal_keys_ map for verification.
|
||||||
if (log_number_ >= log_number_for_cf) {
|
if (log_number_ >= log_number_for_cf) {
|
||||||
cf_wal_keys_[column_family_id].push_back(std::string(key.data(),
|
cf_wal_keys_[column_family_id].push_back(
|
||||||
key.size()));
|
std::string(key.data(), key.size()));
|
||||||
}
|
}
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
|
@ -1134,17 +1133,17 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
}
|
}
|
||||||
|
|
||||||
return WalProcessingOption::kContinueProcessing;
|
return WalProcessingOption::kContinueProcessing;
|
||||||
}
|
}
|
||||||
|
|
||||||
const char* Name() const override {
|
const char* Name() const override {
|
||||||
return "WalFilterTestWithColumnFamilies";
|
return "WalFilterTestWithColumnFamilies";
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::map<uint32_t, std::vector<std::string>>& GetColumnFamilyKeys() {
|
const std::map<uint32_t, std::vector<std::string>>& GetColumnFamilyKeys() {
|
||||||
return cf_wal_keys_;
|
return cf_wal_keys_;
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::map<std::string, uint32_t> & GetColumnFamilyNameIdMap() {
|
const std::map<std::string, uint32_t>& GetColumnFamilyNameIdMap() {
|
||||||
return cf_name_id_map_;
|
return cf_name_id_map_;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -1160,7 +1159,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
|
|
||||||
Options options = OptionsForLogIterTest();
|
Options options = OptionsForLogIterTest();
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
CreateAndReopenWithCF({ "pikachu" }, options);
|
CreateAndReopenWithCF({"pikachu"}, options);
|
||||||
|
|
||||||
// Write given keys in given batches
|
// Write given keys in given batches
|
||||||
for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
|
for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
|
||||||
|
@ -1174,7 +1173,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
|
ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
|
||||||
}
|
}
|
||||||
|
|
||||||
//Flush default column-family
|
// Flush default column-family
|
||||||
ASSERT_OK(db_->Flush(FlushOptions(), handles_[0]));
|
ASSERT_OK(db_->Flush(FlushOptions(), handles_[0]));
|
||||||
|
|
||||||
// Do some more writes
|
// Do some more writes
|
||||||
|
@ -1208,8 +1207,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
// Reopen database with option to use WAL filter
|
// Reopen database with option to use WAL filter
|
||||||
options = OptionsForLogIterTest();
|
options = OptionsForLogIterTest();
|
||||||
options.wal_filter = &test_wal_filter_column_families;
|
options.wal_filter = &test_wal_filter_column_families;
|
||||||
Status status =
|
Status status = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||||
TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
|
||||||
ASSERT_TRUE(status.ok());
|
ASSERT_TRUE(status.ok());
|
||||||
|
|
||||||
// verify that handles_[0] only has post_flush keys
|
// verify that handles_[0] only has post_flush keys
|
||||||
|
@ -1218,7 +1216,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
auto name_id_map = test_wal_filter_column_families.GetColumnFamilyNameIdMap();
|
auto name_id_map = test_wal_filter_column_families.GetColumnFamilyNameIdMap();
|
||||||
size_t index = 0;
|
size_t index = 0;
|
||||||
auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]];
|
auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]];
|
||||||
//default column-family, only post_flush keys are expected
|
// default column-family, only post_flush keys are expected
|
||||||
for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
|
for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
|
||||||
for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
|
for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
|
||||||
Slice key_from_the_log(keys_cf[index++]);
|
Slice key_from_the_log(keys_cf[index++]);
|
||||||
|
@ -1230,7 +1228,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||||
|
|
||||||
index = 0;
|
index = 0;
|
||||||
keys_cf = cf_wal_keys[name_id_map["pikachu"]];
|
keys_cf = cf_wal_keys[name_id_map["pikachu"]];
|
||||||
//pikachu column-family, all keys are expected
|
// pikachu column-family, all keys are expected
|
||||||
for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
|
for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
|
||||||
for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
|
for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
|
||||||
Slice key_from_the_log(keys_cf[index++]);
|
Slice key_from_the_log(keys_cf[index++]);
|
||||||
|
@ -1280,7 +1278,7 @@ TEST_F(DBTest2, PresetCompressionDict) {
|
||||||
#if LZ4_VERSION_NUMBER >= 10400 // r124+
|
#if LZ4_VERSION_NUMBER >= 10400 // r124+
|
||||||
compression_types.push_back(kLZ4Compression);
|
compression_types.push_back(kLZ4Compression);
|
||||||
compression_types.push_back(kLZ4HCCompression);
|
compression_types.push_back(kLZ4HCCompression);
|
||||||
#endif // LZ4_VERSION_NUMBER >= 10400
|
#endif // LZ4_VERSION_NUMBER >= 10400
|
||||||
if (ZSTD_Supported()) {
|
if (ZSTD_Supported()) {
|
||||||
compression_types.push_back(kZSTD);
|
compression_types.push_back(kZSTD);
|
||||||
}
|
}
|
||||||
|
@ -1960,7 +1958,8 @@ TEST_F(DBTest2, CompressionOptions) {
|
||||||
|
|
||||||
class CompactionStallTestListener : public EventListener {
|
class CompactionStallTestListener : public EventListener {
|
||||||
public:
|
public:
|
||||||
CompactionStallTestListener() : compacting_files_cnt_(0), compacted_files_cnt_(0) {}
|
CompactionStallTestListener()
|
||||||
|
: compacting_files_cnt_(0), compacted_files_cnt_(0) {}
|
||||||
|
|
||||||
void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override {
|
void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override {
|
||||||
ASSERT_EQ(ci.cf_name, "default");
|
ASSERT_EQ(ci.cf_name, "default");
|
||||||
|
@ -2039,7 +2038,8 @@ TEST_F(DBTest2, CompactionStall) {
|
||||||
options.level0_file_num_compaction_trigger);
|
options.level0_file_num_compaction_trigger);
|
||||||
ASSERT_GT(listener->compacted_files_cnt_.load(),
|
ASSERT_GT(listener->compacted_files_cnt_.load(),
|
||||||
10 - options.level0_file_num_compaction_trigger);
|
10 - options.level0_file_num_compaction_trigger);
|
||||||
ASSERT_EQ(listener->compacting_files_cnt_.load(), listener->compacted_files_cnt_.load());
|
ASSERT_EQ(listener->compacting_files_cnt_.load(),
|
||||||
|
listener->compacted_files_cnt_.load());
|
||||||
|
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
||||||
}
|
}
|
||||||
|
@ -2664,7 +2664,7 @@ namespace {
|
||||||
void CountSyncPoint() {
|
void CountSyncPoint() {
|
||||||
TEST_SYNC_POINT_CALLBACK("DBTest2::MarkedPoint", nullptr /* arg */);
|
TEST_SYNC_POINT_CALLBACK("DBTest2::MarkedPoint", nullptr /* arg */);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest2, SyncPointMarker) {
|
TEST_F(DBTest2, SyncPointMarker) {
|
||||||
std::atomic<int> sync_point_called(0);
|
std::atomic<int> sync_point_called(0);
|
||||||
|
@ -2797,7 +2797,7 @@ TEST_F(DBTest2, ReadAmpBitmap) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented
|
#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented
|
||||||
TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
|
TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
|
||||||
{
|
{
|
||||||
const int kIdBufLen = 100;
|
const int kIdBufLen = 100;
|
||||||
|
@ -2899,7 +2899,6 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
|
||||||
size_t total_loaded_bytes_iter2 =
|
size_t total_loaded_bytes_iter2 =
|
||||||
options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES);
|
options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES);
|
||||||
|
|
||||||
|
|
||||||
// Read amp is on average 100% since we read all what we loaded in memory
|
// Read amp is on average 100% since we read all what we loaded in memory
|
||||||
if (k == 0) {
|
if (k == 0) {
|
||||||
ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2,
|
ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2,
|
||||||
|
@ -2911,7 +2910,7 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif // !OS_SOLARIS
|
#endif // !OS_SOLARIS
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
|
TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
|
||||||
|
@ -5192,7 +5191,7 @@ TEST_F(DBTest2, TraceWithFilter) {
|
||||||
ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
|
ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
|
||||||
handles.clear();
|
handles.clear();
|
||||||
|
|
||||||
DB* db3 = nullptr;
|
DB* db3 = nullptr;
|
||||||
ASSERT_OK(DB::Open(db_opts, dbname3, column_families, &handles, &db3));
|
ASSERT_OK(DB::Open(db_opts, dbname3, column_families, &handles, &db3));
|
||||||
|
|
||||||
env_->SleepForMicroseconds(100);
|
env_->SleepForMicroseconds(100);
|
||||||
|
@ -5200,12 +5199,12 @@ TEST_F(DBTest2, TraceWithFilter) {
|
||||||
ASSERT_TRUE(db3->Get(ro, handles[0], "a", &value).IsNotFound());
|
ASSERT_TRUE(db3->Get(ro, handles[0], "a", &value).IsNotFound());
|
||||||
ASSERT_TRUE(db3->Get(ro, handles[0], "g", &value).IsNotFound());
|
ASSERT_TRUE(db3->Get(ro, handles[0], "g", &value).IsNotFound());
|
||||||
|
|
||||||
//The tracer will not record the READ ops.
|
// The tracer will not record the READ ops.
|
||||||
trace_opts.filter = TraceFilterType::kTraceFilterGet;
|
trace_opts.filter = TraceFilterType::kTraceFilterGet;
|
||||||
std::string trace_filename3 = dbname_ + "/rocksdb.trace_3";
|
std::string trace_filename3 = dbname_ + "/rocksdb.trace_3";
|
||||||
std::unique_ptr<TraceWriter> trace_writer3;
|
std::unique_ptr<TraceWriter> trace_writer3;
|
||||||
ASSERT_OK(
|
ASSERT_OK(
|
||||||
NewFileTraceWriter(env_, env_opts, trace_filename3, &trace_writer3));
|
NewFileTraceWriter(env_, env_opts, trace_filename3, &trace_writer3));
|
||||||
ASSERT_OK(db3->StartTrace(trace_opts, std::move(trace_writer3)));
|
ASSERT_OK(db3->StartTrace(trace_opts, std::move(trace_writer3)));
|
||||||
|
|
||||||
ASSERT_OK(db3->Put(wo, handles[0], "a", "1"));
|
ASSERT_OK(db3->Put(wo, handles[0], "a", "1"));
|
||||||
|
@ -5227,7 +5226,7 @@ TEST_F(DBTest2, TraceWithFilter) {
|
||||||
|
|
||||||
std::unique_ptr<TraceReader> trace_reader3;
|
std::unique_ptr<TraceReader> trace_reader3;
|
||||||
ASSERT_OK(
|
ASSERT_OK(
|
||||||
NewFileTraceReader(env_, env_opts, trace_filename3, &trace_reader3));
|
NewFileTraceReader(env_, env_opts, trace_filename3, &trace_reader3));
|
||||||
|
|
||||||
// Count the number of records in the trace file;
|
// Count the number of records in the trace file;
|
||||||
int count = 0;
|
int count = 0;
|
||||||
|
@ -5503,16 +5502,20 @@ TEST_F(DBTest2, TestGetColumnFamilyHandleUnlocked) {
|
||||||
port::Thread user_thread1([&]() {
|
port::Thread user_thread1([&]() {
|
||||||
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[0]->GetID());
|
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[0]->GetID());
|
||||||
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
|
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
|
||||||
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked1");
|
TEST_SYNC_POINT(
|
||||||
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::ReadColumnFamilyHandle1");
|
"TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked1");
|
||||||
|
TEST_SYNC_POINT(
|
||||||
|
"TestGetColumnFamilyHandleUnlocked::ReadColumnFamilyHandle1");
|
||||||
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
|
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
|
||||||
});
|
});
|
||||||
|
|
||||||
port::Thread user_thread2([&]() {
|
port::Thread user_thread2([&]() {
|
||||||
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::PreGetColumnFamilyHandleUnlocked2");
|
TEST_SYNC_POINT(
|
||||||
|
"TestGetColumnFamilyHandleUnlocked::PreGetColumnFamilyHandleUnlocked2");
|
||||||
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[1]->GetID());
|
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[1]->GetID());
|
||||||
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
|
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
|
||||||
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked2");
|
TEST_SYNC_POINT(
|
||||||
|
"TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked2");
|
||||||
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
|
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -5666,7 +5669,7 @@ class DummyOldStats : public Statistics {
|
||||||
std::atomic<int> num_rt{0};
|
std::atomic<int> num_rt{0};
|
||||||
std::atomic<int> num_mt{0};
|
std::atomic<int> num_mt{0};
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(DBTest2, OldStatsInterface) {
|
TEST_F(DBTest2, OldStatsInterface) {
|
||||||
DummyOldStats* dos = new DummyOldStats();
|
DummyOldStats* dos = new DummyOldStats();
|
||||||
|
|
|
@ -29,7 +29,7 @@ int64_t MaybeCurrentTime(Env* env) {
|
||||||
env->GetCurrentTime(&time).PermitUncheckedError();
|
env->GetCurrentTime(&time).PermitUncheckedError();
|
||||||
return time;
|
return time;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// Special Env used to delay background operations
|
// Special Env used to delay background operations
|
||||||
|
|
||||||
|
@ -125,54 +125,54 @@ DBTestBase::~DBTestBase() {
|
||||||
|
|
||||||
bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
|
bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
|
||||||
#ifdef ROCKSDB_LITE
|
#ifdef ROCKSDB_LITE
|
||||||
// These options are not supported in ROCKSDB_LITE
|
// These options are not supported in ROCKSDB_LITE
|
||||||
if (option_config == kHashSkipList ||
|
if (option_config == kHashSkipList ||
|
||||||
option_config == kPlainTableFirstBytePrefix ||
|
option_config == kPlainTableFirstBytePrefix ||
|
||||||
option_config == kPlainTableCappedPrefix ||
|
option_config == kPlainTableCappedPrefix ||
|
||||||
option_config == kPlainTableCappedPrefixNonMmap ||
|
option_config == kPlainTableCappedPrefixNonMmap ||
|
||||||
option_config == kPlainTableAllBytesPrefix ||
|
option_config == kPlainTableAllBytesPrefix ||
|
||||||
option_config == kVectorRep || option_config == kHashLinkList ||
|
option_config == kVectorRep || option_config == kHashLinkList ||
|
||||||
option_config == kUniversalCompaction ||
|
option_config == kUniversalCompaction ||
|
||||||
option_config == kUniversalCompactionMultiLevel ||
|
option_config == kUniversalCompactionMultiLevel ||
|
||||||
option_config == kUniversalSubcompactions ||
|
option_config == kUniversalSubcompactions ||
|
||||||
option_config == kFIFOCompaction ||
|
option_config == kFIFOCompaction ||
|
||||||
option_config == kConcurrentSkipList) {
|
option_config == kConcurrentSkipList) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
if ((skip_mask & kSkipUniversalCompaction) &&
|
if ((skip_mask & kSkipUniversalCompaction) &&
|
||||||
(option_config == kUniversalCompaction ||
|
(option_config == kUniversalCompaction ||
|
||||||
option_config == kUniversalCompactionMultiLevel ||
|
option_config == kUniversalCompactionMultiLevel ||
|
||||||
option_config == kUniversalSubcompactions)) {
|
option_config == kUniversalSubcompactions)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if ((skip_mask & kSkipMergePut) && option_config == kMergePut) {
|
if ((skip_mask & kSkipMergePut) && option_config == kMergePut) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if ((skip_mask & kSkipNoSeekToLast) &&
|
if ((skip_mask & kSkipNoSeekToLast) &&
|
||||||
(option_config == kHashLinkList || option_config == kHashSkipList)) {
|
(option_config == kHashLinkList || option_config == kHashSkipList)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if ((skip_mask & kSkipPlainTable) &&
|
if ((skip_mask & kSkipPlainTable) &&
|
||||||
(option_config == kPlainTableAllBytesPrefix ||
|
(option_config == kPlainTableAllBytesPrefix ||
|
||||||
option_config == kPlainTableFirstBytePrefix ||
|
option_config == kPlainTableFirstBytePrefix ||
|
||||||
option_config == kPlainTableCappedPrefix ||
|
option_config == kPlainTableCappedPrefix ||
|
||||||
option_config == kPlainTableCappedPrefixNonMmap)) {
|
option_config == kPlainTableCappedPrefixNonMmap)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if ((skip_mask & kSkipHashIndex) &&
|
if ((skip_mask & kSkipHashIndex) &&
|
||||||
(option_config == kBlockBasedTableWithPrefixHashIndex ||
|
(option_config == kBlockBasedTableWithPrefixHashIndex ||
|
||||||
option_config == kBlockBasedTableWithWholeKeyHashIndex)) {
|
option_config == kBlockBasedTableWithWholeKeyHashIndex)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) {
|
if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) {
|
if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Switch to a fresh database with the next option configuration to
|
// Switch to a fresh database with the next option configuration to
|
||||||
|
@ -424,13 +424,13 @@ Options DBTestBase::GetOptions(
|
||||||
options.allow_concurrent_memtable_write = false;
|
options.allow_concurrent_memtable_write = false;
|
||||||
options.unordered_write = false;
|
options.unordered_write = false;
|
||||||
break;
|
break;
|
||||||
case kDirectIO: {
|
case kDirectIO: {
|
||||||
options.use_direct_reads = true;
|
options.use_direct_reads = true;
|
||||||
options.use_direct_io_for_flush_and_compaction = true;
|
options.use_direct_io_for_flush_and_compaction = true;
|
||||||
options.compaction_readahead_size = 2 * 1024 * 1024;
|
options.compaction_readahead_size = 2 * 1024 * 1024;
|
||||||
SetupSyncPointsToMockDirectIO();
|
SetupSyncPointsToMockDirectIO();
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
case kMergePut:
|
case kMergePut:
|
||||||
options.merge_operator = MergeOperators::CreatePutOperator();
|
options.merge_operator = MergeOperators::CreatePutOperator();
|
||||||
|
@ -1308,12 +1308,14 @@ void DBTestBase::GetSstFiles(Env* env, std::string path,
|
||||||
std::vector<std::string>* files) {
|
std::vector<std::string>* files) {
|
||||||
EXPECT_OK(env->GetChildren(path, files));
|
EXPECT_OK(env->GetChildren(path, files));
|
||||||
|
|
||||||
files->erase(
|
files->erase(std::remove_if(files->begin(), files->end(),
|
||||||
std::remove_if(files->begin(), files->end(), [](std::string name) {
|
[](std::string name) {
|
||||||
uint64_t number;
|
uint64_t number;
|
||||||
FileType type;
|
FileType type;
|
||||||
return !(ParseFileName(name, &number, &type) && type == kTableFile);
|
return !(ParseFileName(name, &number, &type) &&
|
||||||
}), files->end());
|
type == kTableFile);
|
||||||
|
}),
|
||||||
|
files->end());
|
||||||
}
|
}
|
||||||
|
|
||||||
int DBTestBase::GetSstFileCount(std::string path) {
|
int DBTestBase::GetSstFileCount(std::string path) {
|
||||||
|
@ -1583,8 +1585,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
||||||
iter_cnt++;
|
iter_cnt++;
|
||||||
total_reads++;
|
total_reads++;
|
||||||
}
|
}
|
||||||
ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / "
|
ASSERT_EQ(data_iter, true_data.end())
|
||||||
<< true_data.size();
|
<< iter_cnt << " / " << true_data.size();
|
||||||
delete iter;
|
delete iter;
|
||||||
|
|
||||||
// Verify Iterator::Prev()
|
// Verify Iterator::Prev()
|
||||||
|
@ -1606,8 +1608,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
||||||
iter_cnt++;
|
iter_cnt++;
|
||||||
total_reads++;
|
total_reads++;
|
||||||
}
|
}
|
||||||
ASSERT_EQ(data_rev, true_data.rend()) << iter_cnt << " / "
|
ASSERT_EQ(data_rev, true_data.rend())
|
||||||
<< true_data.size();
|
<< iter_cnt << " / " << true_data.size();
|
||||||
|
|
||||||
// Verify Iterator::Seek()
|
// Verify Iterator::Seek()
|
||||||
for (auto kv : true_data) {
|
for (auto kv : true_data) {
|
||||||
|
@ -1637,8 +1639,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
||||||
iter_cnt++;
|
iter_cnt++;
|
||||||
total_reads++;
|
total_reads++;
|
||||||
}
|
}
|
||||||
ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / "
|
ASSERT_EQ(data_iter, true_data.end())
|
||||||
<< true_data.size();
|
<< iter_cnt << " / " << true_data.size();
|
||||||
|
|
||||||
// Verify ForwardIterator::Seek()
|
// Verify ForwardIterator::Seek()
|
||||||
for (auto kv : true_data) {
|
for (auto kv : true_data) {
|
||||||
|
|
|
@ -220,9 +220,7 @@ class SpecialEnv : public EnvWrapper {
|
||||||
Env::IOPriority GetIOPriority() override {
|
Env::IOPriority GetIOPriority() override {
|
||||||
return base_->GetIOPriority();
|
return base_->GetIOPriority();
|
||||||
}
|
}
|
||||||
bool use_direct_io() const override {
|
bool use_direct_io() const override { return base_->use_direct_io(); }
|
||||||
return base_->use_direct_io();
|
|
||||||
}
|
|
||||||
Status Allocate(uint64_t offset, uint64_t len) override {
|
Status Allocate(uint64_t offset, uint64_t len) override {
|
||||||
return base_->Allocate(offset, len);
|
return base_->Allocate(offset, len);
|
||||||
}
|
}
|
||||||
|
|
|
@ -39,8 +39,8 @@ class DBTestUniversalCompactionBase
|
||||||
|
|
||||||
class DBTestUniversalCompaction : public DBTestUniversalCompactionBase {
|
class DBTestUniversalCompaction : public DBTestUniversalCompactionBase {
|
||||||
public:
|
public:
|
||||||
DBTestUniversalCompaction() :
|
DBTestUniversalCompaction()
|
||||||
DBTestUniversalCompactionBase("/db_universal_compaction_test") {}
|
: DBTestUniversalCompactionBase("/db_universal_compaction_test") {}
|
||||||
};
|
};
|
||||||
|
|
||||||
class DBTestUniversalCompaction2 : public DBTestBase {
|
class DBTestUniversalCompaction2 : public DBTestBase {
|
||||||
|
@ -93,7 +93,7 @@ class KeepFilterFactory : public CompactionFilterFactory {
|
||||||
std::atomic_bool expect_full_compaction_;
|
std::atomic_bool expect_full_compaction_;
|
||||||
std::atomic_bool expect_manual_compaction_;
|
std::atomic_bool expect_manual_compaction_;
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
// Make sure we don't trigger a problem if the trigger condtion is given
|
// Make sure we don't trigger a problem if the trigger condtion is given
|
||||||
// to be 0, which is invalid.
|
// to be 0, which is invalid.
|
||||||
|
@ -563,8 +563,7 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (compaction_input_file_names.size() == 0) {
|
if (compaction_input_file_names.size() == 0) {
|
||||||
compaction_input_file_names.push_back(
|
compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
|
||||||
cf_meta.levels[0].files[0].name);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// expect fail since universal compaction only allow L0 output
|
// expect fail since universal compaction only allow L0 output
|
||||||
|
@ -574,28 +573,23 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
||||||
.ok());
|
.ok());
|
||||||
|
|
||||||
// expect ok and verify the compacted files no longer exist.
|
// expect ok and verify the compacted files no longer exist.
|
||||||
ASSERT_OK(dbfull()->CompactFiles(
|
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
|
||||||
CompactionOptions(), handles_[1],
|
compaction_input_file_names, 0));
|
||||||
compaction_input_file_names, 0));
|
|
||||||
|
|
||||||
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
||||||
VerifyCompactionResult(
|
VerifyCompactionResult(
|
||||||
cf_meta,
|
cf_meta, std::set<std::string>(compaction_input_file_names.begin(),
|
||||||
std::set<std::string>(compaction_input_file_names.begin(),
|
compaction_input_file_names.end()));
|
||||||
compaction_input_file_names.end()));
|
|
||||||
|
|
||||||
compaction_input_file_names.clear();
|
compaction_input_file_names.clear();
|
||||||
|
|
||||||
// Pick the first and the last file, expect everything is
|
// Pick the first and the last file, expect everything is
|
||||||
// compacted into one single file.
|
// compacted into one single file.
|
||||||
|
compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
|
||||||
compaction_input_file_names.push_back(
|
compaction_input_file_names.push_back(
|
||||||
cf_meta.levels[0].files[0].name);
|
cf_meta.levels[0].files[cf_meta.levels[0].files.size() - 1].name);
|
||||||
compaction_input_file_names.push_back(
|
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
|
||||||
cf_meta.levels[0].files[
|
compaction_input_file_names, 0));
|
||||||
cf_meta.levels[0].files.size() - 1].name);
|
|
||||||
ASSERT_OK(dbfull()->CompactFiles(
|
|
||||||
CompactionOptions(), handles_[1],
|
|
||||||
compaction_input_file_names, 0));
|
|
||||||
|
|
||||||
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
||||||
ASSERT_EQ(cf_meta.levels[0].files.size(), 1U);
|
ASSERT_EQ(cf_meta.levels[0].files.size(), 1U);
|
||||||
|
@ -604,7 +598,7 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
||||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
|
TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.compaction_style = kCompactionStyleUniversal;
|
options.compaction_style = kCompactionStyleUniversal;
|
||||||
options.write_buffer_size = 100 << 10; // 100KB
|
options.write_buffer_size = 100 << 10; // 100KB
|
||||||
options.num_levels = 7;
|
options.num_levels = 7;
|
||||||
options.disable_auto_compactions = true;
|
options.disable_auto_compactions = true;
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
@ -640,9 +634,9 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
|
||||||
class DBTestUniversalCompactionMultiLevels
|
class DBTestUniversalCompactionMultiLevels
|
||||||
: public DBTestUniversalCompactionBase {
|
: public DBTestUniversalCompactionBase {
|
||||||
public:
|
public:
|
||||||
DBTestUniversalCompactionMultiLevels() :
|
DBTestUniversalCompactionMultiLevels()
|
||||||
DBTestUniversalCompactionBase(
|
: DBTestUniversalCompactionBase(
|
||||||
"/db_universal_compaction_multi_levels_test") {}
|
"/db_universal_compaction_multi_levels_test") {}
|
||||||
};
|
};
|
||||||
|
|
||||||
TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
|
TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
|
||||||
|
@ -725,12 +719,11 @@ INSTANTIATE_TEST_CASE_P(MultiLevels, DBTestUniversalCompactionMultiLevels,
|
||||||
::testing::Combine(::testing::Values(3, 20),
|
::testing::Combine(::testing::Values(3, 20),
|
||||||
::testing::Bool()));
|
::testing::Bool()));
|
||||||
|
|
||||||
class DBTestUniversalCompactionParallel :
|
class DBTestUniversalCompactionParallel : public DBTestUniversalCompactionBase {
|
||||||
public DBTestUniversalCompactionBase {
|
|
||||||
public:
|
public:
|
||||||
DBTestUniversalCompactionParallel() :
|
DBTestUniversalCompactionParallel()
|
||||||
DBTestUniversalCompactionBase(
|
: DBTestUniversalCompactionBase("/db_universal_compaction_prallel_test") {
|
||||||
"/db_universal_compaction_prallel_test") {}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
|
TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
|
||||||
|
@ -919,8 +912,8 @@ INSTANTIATE_TEST_CASE_P(Parallel, DBTestUniversalCompactionParallel,
|
||||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
|
TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.compaction_style = kCompactionStyleUniversal;
|
options.compaction_style = kCompactionStyleUniversal;
|
||||||
options.write_buffer_size = 105 << 10; // 105KB
|
options.write_buffer_size = 105 << 10; // 105KB
|
||||||
options.arena_block_size = 4 << 10; // 4KB
|
options.arena_block_size = 4 << 10; // 4KB
|
||||||
options.target_file_size_base = 32 << 10; // 32KB
|
options.target_file_size_base = 32 << 10; // 32KB
|
||||||
options.level0_file_num_compaction_trigger = 4;
|
options.level0_file_num_compaction_trigger = 4;
|
||||||
options.num_levels = num_levels_;
|
options.num_levels = num_levels_;
|
||||||
|
@ -951,8 +944,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
|
||||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
|
TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.compaction_style = kCompactionStyleUniversal;
|
options.compaction_style = kCompactionStyleUniversal;
|
||||||
options.write_buffer_size = 105 << 10; // 105KB
|
options.write_buffer_size = 105 << 10; // 105KB
|
||||||
options.arena_block_size = 4 << 10; // 4KB
|
options.arena_block_size = 4 << 10; // 4KB
|
||||||
options.target_file_size_base = 32 << 10; // 32KB
|
options.target_file_size_base = 32 << 10; // 32KB
|
||||||
// trigger compaction if there are >= 4 files
|
// trigger compaction if there are >= 4 files
|
||||||
options.level0_file_num_compaction_trigger = 4;
|
options.level0_file_num_compaction_trigger = 4;
|
||||||
|
@ -1353,7 +1346,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCFPathUse) {
|
||||||
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 500 * 1024);
|
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 500 * 1024);
|
||||||
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_4", 1024 * 1024 * 1024);
|
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_4", 1024 * 1024 * 1024);
|
||||||
option_vector.emplace_back(DBOptions(options), cf_opt1);
|
option_vector.emplace_back(DBOptions(options), cf_opt1);
|
||||||
CreateColumnFamilies({"one"},option_vector[1]);
|
CreateColumnFamilies({"one"}, option_vector[1]);
|
||||||
|
|
||||||
// Configura CF2 specific paths.
|
// Configura CF2 specific paths.
|
||||||
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 300 * 1024);
|
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 300 * 1024);
|
||||||
|
@ -1361,7 +1354,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCFPathUse) {
|
||||||
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 500 * 1024);
|
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 500 * 1024);
|
||||||
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_4", 1024 * 1024 * 1024);
|
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_4", 1024 * 1024 * 1024);
|
||||||
option_vector.emplace_back(DBOptions(options), cf_opt2);
|
option_vector.emplace_back(DBOptions(options), cf_opt2);
|
||||||
CreateColumnFamilies({"two"},option_vector[2]);
|
CreateColumnFamilies({"two"}, option_vector[2]);
|
||||||
|
|
||||||
ReopenWithColumnFamilies({"default", "one", "two"}, option_vector);
|
ReopenWithColumnFamilies({"default", "one", "two"}, option_vector);
|
||||||
|
|
||||||
|
@ -1567,7 +1560,6 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
|
||||||
verify_func(max_key3);
|
verify_func(max_key3);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
|
TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
|
||||||
if (!Snappy_Supported()) {
|
if (!Snappy_Supported()) {
|
||||||
return;
|
return;
|
||||||
|
@ -1829,9 +1821,9 @@ INSTANTIATE_TEST_CASE_P(NumLevels, DBTestUniversalCompaction,
|
||||||
class DBTestUniversalManualCompactionOutputPathId
|
class DBTestUniversalManualCompactionOutputPathId
|
||||||
: public DBTestUniversalCompactionBase {
|
: public DBTestUniversalCompactionBase {
|
||||||
public:
|
public:
|
||||||
DBTestUniversalManualCompactionOutputPathId() :
|
DBTestUniversalManualCompactionOutputPathId()
|
||||||
DBTestUniversalCompactionBase(
|
: DBTestUniversalCompactionBase(
|
||||||
"/db_universal_compaction_manual_pid_test") {}
|
"/db_universal_compaction_manual_pid_test") {}
|
||||||
};
|
};
|
||||||
|
|
||||||
TEST_P(DBTestUniversalManualCompactionOutputPathId,
|
TEST_P(DBTestUniversalManualCompactionOutputPathId,
|
||||||
|
@ -2236,8 +2228,8 @@ int main(int argc, char** argv) {
|
||||||
::testing::InitGoogleTest(&argc, argv);
|
::testing::InitGoogleTest(&argc, argv);
|
||||||
return RUN_ALL_TESTS();
|
return RUN_ALL_TESTS();
|
||||||
#else
|
#else
|
||||||
(void) argc;
|
(void)argc;
|
||||||
(void) argv;
|
(void)argv;
|
||||||
return 0;
|
return 0;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
|
@ -170,7 +170,8 @@ TEST_P(DBWriteTest, WriteStallRemoveNoSlowdownWrite) {
|
||||||
|
|
||||||
TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
||||||
Options options = GetOptions();
|
Options options = GetOptions();
|
||||||
options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger = 4;
|
options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger =
|
||||||
|
4;
|
||||||
std::vector<port::Thread> threads;
|
std::vector<port::Thread> threads;
|
||||||
std::atomic<int> thread_num(0);
|
std::atomic<int> thread_num(0);
|
||||||
port::Mutex mutex;
|
port::Mutex mutex;
|
||||||
|
@ -195,7 +196,7 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
||||||
Status s = dbfull()->Put(wo, key, "bar");
|
Status s = dbfull()->Put(wo, key, "bar");
|
||||||
ASSERT_TRUE(s.ok() || s.IsIncomplete());
|
ASSERT_TRUE(s.ok() || s.IsIncomplete());
|
||||||
};
|
};
|
||||||
std::function<void(void *)> unblock_main_thread_func = [&](void *) {
|
std::function<void(void*)> unblock_main_thread_func = [&](void*) {
|
||||||
mutex.Lock();
|
mutex.Lock();
|
||||||
++writers;
|
++writers;
|
||||||
cv.SignalAll();
|
cv.SignalAll();
|
||||||
|
@ -254,8 +255,9 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr));
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr));
|
||||||
// This would have triggered a write stall. Unblock the write group leader
|
// This would have triggered a write stall. Unblock the write group leader
|
||||||
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:2");
|
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:2");
|
||||||
// The leader is going to create missing newer links. When the leader finishes,
|
// The leader is going to create missing newer links. When the leader
|
||||||
// the next leader is going to delay writes and fail writers with no_slowdown
|
// finishes, the next leader is going to delay writes and fail writers with
|
||||||
|
// no_slowdown
|
||||||
|
|
||||||
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:3");
|
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:3");
|
||||||
for (auto& t : threads) {
|
for (auto& t : threads) {
|
||||||
|
@ -623,42 +625,43 @@ TEST_P(DBWriteTest, LockWalInEffect) {
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_P(DBWriteTest, ConcurrentlyDisabledWAL) {
|
TEST_P(DBWriteTest, ConcurrentlyDisabledWAL) {
|
||||||
Options options = GetOptions();
|
Options options = GetOptions();
|
||||||
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
||||||
options.statistics->set_stats_level(StatsLevel::kAll);
|
options.statistics->set_stats_level(StatsLevel::kAll);
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
std::string wal_key_prefix = "WAL_KEY_";
|
std::string wal_key_prefix = "WAL_KEY_";
|
||||||
std::string no_wal_key_prefix = "K_";
|
std::string no_wal_key_prefix = "K_";
|
||||||
// 100 KB value each for NO-WAL operation
|
// 100 KB value each for NO-WAL operation
|
||||||
std::string no_wal_value(1024 * 100, 'X');
|
std::string no_wal_value(1024 * 100, 'X');
|
||||||
// 1B value each for WAL operation
|
// 1B value each for WAL operation
|
||||||
std::string wal_value = "0";
|
std::string wal_value = "0";
|
||||||
std::thread threads[10];
|
std::thread threads[10];
|
||||||
for (int t = 0; t < 10; t++) {
|
for (int t = 0; t < 10; t++) {
|
||||||
threads[t] = std::thread([t, wal_key_prefix, wal_value, no_wal_key_prefix, no_wal_value, this] {
|
threads[t] = std::thread([t, wal_key_prefix, wal_value, no_wal_key_prefix,
|
||||||
for(int i = 0; i < 10; i++) {
|
no_wal_value, this] {
|
||||||
ROCKSDB_NAMESPACE::WriteOptions write_option_disable;
|
for (int i = 0; i < 10; i++) {
|
||||||
write_option_disable.disableWAL = true;
|
ROCKSDB_NAMESPACE::WriteOptions write_option_disable;
|
||||||
ROCKSDB_NAMESPACE::WriteOptions write_option_default;
|
write_option_disable.disableWAL = true;
|
||||||
std::string no_wal_key = no_wal_key_prefix + std::to_string(t) +
|
ROCKSDB_NAMESPACE::WriteOptions write_option_default;
|
||||||
"_" + std::to_string(i);
|
std::string no_wal_key =
|
||||||
ASSERT_OK(
|
no_wal_key_prefix + std::to_string(t) + "_" + std::to_string(i);
|
||||||
this->Put(no_wal_key, no_wal_value, write_option_disable));
|
ASSERT_OK(this->Put(no_wal_key, no_wal_value, write_option_disable));
|
||||||
std::string wal_key =
|
std::string wal_key =
|
||||||
wal_key_prefix + std::to_string(i) + "_" + std::to_string(i);
|
wal_key_prefix + std::to_string(i) + "_" + std::to_string(i);
|
||||||
ASSERT_OK(this->Put(wal_key, wal_value, write_option_default));
|
ASSERT_OK(this->Put(wal_key, wal_value, write_option_default));
|
||||||
ASSERT_OK(dbfull()->SyncWAL());
|
ASSERT_OK(dbfull()->SyncWAL());
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
for (auto& t: threads) {
|
for (auto& t : threads) {
|
||||||
t.join();
|
t.join();
|
||||||
}
|
}
|
||||||
uint64_t bytes_num = options.statistics->getTickerCount(
|
uint64_t bytes_num = options.statistics->getTickerCount(
|
||||||
ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES);
|
ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES);
|
||||||
// written WAL size should less than 100KB (even included HEADER & FOOTER overhead)
|
// written WAL size should less than 100KB (even included HEADER & FOOTER
|
||||||
ASSERT_LE(bytes_num, 1024 * 100);
|
// overhead)
|
||||||
|
ASSERT_LE(bytes_num, 1024 * 100);
|
||||||
}
|
}
|
||||||
|
|
||||||
INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest,
|
INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest,
|
||||||
|
|
|
@ -15,8 +15,7 @@
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
static std::string IKey(const std::string& user_key,
|
static std::string IKey(const std::string& user_key, uint64_t seq,
|
||||||
uint64_t seq,
|
|
||||||
ValueType vt) {
|
ValueType vt) {
|
||||||
std::string encoded;
|
std::string encoded;
|
||||||
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
|
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
|
||||||
|
@ -37,9 +36,7 @@ static std::string ShortSuccessor(const std::string& s) {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void TestKey(const std::string& key,
|
static void TestKey(const std::string& key, uint64_t seq, ValueType vt) {
|
||||||
uint64_t seq,
|
|
||||||
ValueType vt) {
|
|
||||||
std::string encoded = IKey(key, seq, vt);
|
std::string encoded = IKey(key, seq, vt);
|
||||||
|
|
||||||
Slice in(encoded);
|
Slice in(encoded);
|
||||||
|
@ -56,13 +53,19 @@ static void TestKey(const std::string& key,
|
||||||
class FormatTest : public testing::Test {};
|
class FormatTest : public testing::Test {};
|
||||||
|
|
||||||
TEST_F(FormatTest, InternalKey_EncodeDecode) {
|
TEST_F(FormatTest, InternalKey_EncodeDecode) {
|
||||||
const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" };
|
const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"};
|
||||||
const uint64_t seq[] = {
|
const uint64_t seq[] = {1,
|
||||||
1, 2, 3,
|
2,
|
||||||
(1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1,
|
3,
|
||||||
(1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1,
|
(1ull << 8) - 1,
|
||||||
(1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1
|
1ull << 8,
|
||||||
};
|
(1ull << 8) + 1,
|
||||||
|
(1ull << 16) - 1,
|
||||||
|
1ull << 16,
|
||||||
|
(1ull << 16) + 1,
|
||||||
|
(1ull << 32) - 1,
|
||||||
|
1ull << 32,
|
||||||
|
(1ull << 32) + 1};
|
||||||
for (unsigned int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
|
for (unsigned int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
|
||||||
for (unsigned int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
|
for (unsigned int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
|
||||||
TestKey(keys[k], seq[s], kTypeValue);
|
TestKey(keys[k], seq[s], kTypeValue);
|
||||||
|
@ -74,27 +77,25 @@ TEST_F(FormatTest, InternalKey_EncodeDecode) {
|
||||||
TEST_F(FormatTest, InternalKeyShortSeparator) {
|
TEST_F(FormatTest, InternalKeyShortSeparator) {
|
||||||
// When user keys are same
|
// When user keys are same
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue)));
|
||||||
IKey("foo", 99, kTypeValue)));
|
ASSERT_EQ(
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue)));
|
||||||
IKey("foo", 101, kTypeValue)));
|
ASSERT_EQ(
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue)));
|
||||||
IKey("foo", 100, kTypeValue)));
|
ASSERT_EQ(
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion)));
|
||||||
IKey("foo", 100, kTypeDeletion)));
|
|
||||||
|
|
||||||
// When user keys are misordered
|
// When user keys are misordered
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue)));
|
||||||
IKey("bar", 99, kTypeValue)));
|
|
||||||
|
|
||||||
// When user keys are different, but correctly ordered
|
// When user keys are different, but correctly ordered
|
||||||
ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
|
ASSERT_EQ(
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
|
||||||
IKey("hello", 200, kTypeValue)));
|
Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue)));
|
||||||
|
|
||||||
ASSERT_EQ(IKey("ABC2", kMaxSequenceNumber, kValueTypeForSeek),
|
ASSERT_EQ(IKey("ABC2", kMaxSequenceNumber, kValueTypeForSeek),
|
||||||
Shorten(IKey("ABC1AAAAA", 100, kTypeValue),
|
Shorten(IKey("ABC1AAAAA", 100, kTypeValue),
|
||||||
|
@ -121,14 +122,14 @@ TEST_F(FormatTest, InternalKeyShortSeparator) {
|
||||||
Shorten(IKey("AAA1", 100, kTypeValue), IKey("AAA2", 200, kTypeValue)));
|
Shorten(IKey("AAA1", 100, kTypeValue), IKey("AAA2", 200, kTypeValue)));
|
||||||
|
|
||||||
// When start user key is prefix of limit user key
|
// When start user key is prefix of limit user key
|
||||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
ASSERT_EQ(
|
||||||
Shorten(IKey("foo", 100, kTypeValue),
|
IKey("foo", 100, kTypeValue),
|
||||||
IKey("foobar", 200, kTypeValue)));
|
Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue)));
|
||||||
|
|
||||||
// When limit user key is prefix of start user key
|
// When limit user key is prefix of start user key
|
||||||
ASSERT_EQ(IKey("foobar", 100, kTypeValue),
|
ASSERT_EQ(
|
||||||
Shorten(IKey("foobar", 100, kTypeValue),
|
IKey("foobar", 100, kTypeValue),
|
||||||
IKey("foo", 200, kTypeValue)));
|
Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue)));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(FormatTest, InternalKeyShortestSuccessor) {
|
TEST_F(FormatTest, InternalKeyShortestSuccessor) {
|
||||||
|
|
|
@ -10,9 +10,11 @@
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
|
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "db/db_impl/db_impl.h"
|
#include "db/db_impl/db_impl.h"
|
||||||
#include "db/db_test_util.h"
|
#include "db/db_test_util.h"
|
||||||
#include "db/version_set.h"
|
#include "db/version_set.h"
|
||||||
|
@ -55,7 +57,7 @@ class DeleteFileTest : public DBTestBase {
|
||||||
WriteOptions options;
|
WriteOptions options;
|
||||||
options.sync = false;
|
options.sync = false;
|
||||||
ReadOptions roptions;
|
ReadOptions roptions;
|
||||||
for (int i = startkey; i < (numkeys + startkey) ; i++) {
|
for (int i = startkey; i < (numkeys + startkey); i++) {
|
||||||
std::string temp = std::to_string(i);
|
std::string temp = std::to_string(i);
|
||||||
Slice key(temp);
|
Slice key(temp);
|
||||||
Slice value(temp);
|
Slice value(temp);
|
||||||
|
@ -63,10 +65,8 @@ class DeleteFileTest : public DBTestBase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
int numKeysInLevels(
|
int numKeysInLevels(std::vector<LiveFileMetaData>& metadata,
|
||||||
std::vector<LiveFileMetaData> &metadata,
|
std::vector<int>* keysperlevel = nullptr) {
|
||||||
std::vector<int> *keysperlevel = nullptr) {
|
|
||||||
|
|
||||||
if (keysperlevel != nullptr) {
|
if (keysperlevel != nullptr) {
|
||||||
keysperlevel->resize(numlevels_);
|
keysperlevel->resize(numlevels_);
|
||||||
}
|
}
|
||||||
|
@ -82,8 +82,7 @@ class DeleteFileTest : public DBTestBase {
|
||||||
}
|
}
|
||||||
fprintf(stderr, "level %d name %s smallest %s largest %s\n",
|
fprintf(stderr, "level %d name %s smallest %s largest %s\n",
|
||||||
metadata[i].level, metadata[i].name.c_str(),
|
metadata[i].level, metadata[i].name.c_str(),
|
||||||
metadata[i].smallestkey.c_str(),
|
metadata[i].smallestkey.c_str(), metadata[i].largestkey.c_str());
|
||||||
metadata[i].largestkey.c_str());
|
|
||||||
}
|
}
|
||||||
return numKeys;
|
return numKeys;
|
||||||
}
|
}
|
||||||
|
@ -214,7 +213,7 @@ TEST_F(DeleteFileTest, PurgeObsoleteFilesTest) {
|
||||||
|
|
||||||
// this time, we keep an iterator alive
|
// this time, we keep an iterator alive
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
Iterator *itr = nullptr;
|
Iterator* itr = nullptr;
|
||||||
CreateTwoLevels();
|
CreateTwoLevels();
|
||||||
itr = db_->NewIterator(ReadOptions());
|
itr = db_->NewIterator(ReadOptions());
|
||||||
ASSERT_OK(itr->status());
|
ASSERT_OK(itr->status());
|
||||||
|
@ -481,12 +480,12 @@ TEST_F(DeleteFileTest, DeleteFileWithIterator) {
|
||||||
}
|
}
|
||||||
|
|
||||||
Status status = db_->DeleteFile(level2file);
|
Status status = db_->DeleteFile(level2file);
|
||||||
fprintf(stdout, "Deletion status %s: %s\n",
|
fprintf(stdout, "Deletion status %s: %s\n", level2file.c_str(),
|
||||||
level2file.c_str(), status.ToString().c_str());
|
status.ToString().c_str());
|
||||||
ASSERT_OK(status);
|
ASSERT_OK(status);
|
||||||
it->SeekToFirst();
|
it->SeekToFirst();
|
||||||
int numKeysIterated = 0;
|
int numKeysIterated = 0;
|
||||||
while(it->Valid()) {
|
while (it->Valid()) {
|
||||||
numKeysIterated++;
|
numKeysIterated++;
|
||||||
it->Next();
|
it->Next();
|
||||||
}
|
}
|
||||||
|
|
|
@ -234,8 +234,8 @@ void ErrorHandler::CancelErrorRecovery() {
|
||||||
// We'll release the lock before calling sfm, so make sure no new
|
// We'll release the lock before calling sfm, so make sure no new
|
||||||
// recovery gets scheduled at that point
|
// recovery gets scheduled at that point
|
||||||
auto_recovery_ = false;
|
auto_recovery_ = false;
|
||||||
SstFileManagerImpl* sfm = reinterpret_cast<SstFileManagerImpl*>(
|
SstFileManagerImpl* sfm =
|
||||||
db_options_.sst_file_manager.get());
|
reinterpret_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
|
||||||
if (sfm) {
|
if (sfm) {
|
||||||
// This may or may not cancel a pending recovery
|
// This may or may not cancel a pending recovery
|
||||||
db_mutex_->Unlock();
|
db_mutex_->Unlock();
|
||||||
|
@ -292,8 +292,8 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err,
|
||||||
bool found = false;
|
bool found = false;
|
||||||
|
|
||||||
{
|
{
|
||||||
auto entry = ErrorSeverityMap.find(std::make_tuple(reason, bg_err.code(),
|
auto entry = ErrorSeverityMap.find(
|
||||||
bg_err.subcode(), paranoid));
|
std::make_tuple(reason, bg_err.code(), bg_err.subcode(), paranoid));
|
||||||
if (entry != ErrorSeverityMap.end()) {
|
if (entry != ErrorSeverityMap.end()) {
|
||||||
sev = entry->second;
|
sev = entry->second;
|
||||||
found = true;
|
found = true;
|
||||||
|
@ -301,8 +301,8 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err,
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!found) {
|
if (!found) {
|
||||||
auto entry = DefaultErrorSeverityMap.find(std::make_tuple(reason,
|
auto entry = DefaultErrorSeverityMap.find(
|
||||||
bg_err.code(), paranoid));
|
std::make_tuple(reason, bg_err.code(), paranoid));
|
||||||
if (entry != DefaultErrorSeverityMap.end()) {
|
if (entry != DefaultErrorSeverityMap.end()) {
|
||||||
sev = entry->second;
|
sev = entry->second;
|
||||||
found = true;
|
found = true;
|
||||||
|
|
|
@ -26,100 +26,99 @@ struct DBRecoverContext {
|
||||||
};
|
};
|
||||||
|
|
||||||
class ErrorHandler {
|
class ErrorHandler {
|
||||||
public:
|
public:
|
||||||
ErrorHandler(DBImpl* db, const ImmutableDBOptions& db_options,
|
ErrorHandler(DBImpl* db, const ImmutableDBOptions& db_options,
|
||||||
InstrumentedMutex* db_mutex)
|
InstrumentedMutex* db_mutex)
|
||||||
: db_(db),
|
: db_(db),
|
||||||
db_options_(db_options),
|
db_options_(db_options),
|
||||||
cv_(db_mutex),
|
cv_(db_mutex),
|
||||||
end_recovery_(false),
|
end_recovery_(false),
|
||||||
recovery_thread_(nullptr),
|
recovery_thread_(nullptr),
|
||||||
db_mutex_(db_mutex),
|
db_mutex_(db_mutex),
|
||||||
auto_recovery_(false),
|
auto_recovery_(false),
|
||||||
recovery_in_prog_(false),
|
recovery_in_prog_(false),
|
||||||
soft_error_no_bg_work_(false),
|
soft_error_no_bg_work_(false),
|
||||||
is_db_stopped_(false),
|
is_db_stopped_(false),
|
||||||
bg_error_stats_(db_options.statistics) {
|
bg_error_stats_(db_options.statistics) {
|
||||||
// Clear the checked flag for uninitialized errors
|
// Clear the checked flag for uninitialized errors
|
||||||
bg_error_.PermitUncheckedError();
|
bg_error_.PermitUncheckedError();
|
||||||
recovery_error_.PermitUncheckedError();
|
recovery_error_.PermitUncheckedError();
|
||||||
recovery_io_error_.PermitUncheckedError();
|
recovery_io_error_.PermitUncheckedError();
|
||||||
}
|
}
|
||||||
|
|
||||||
void EnableAutoRecovery() { auto_recovery_ = true; }
|
void EnableAutoRecovery() { auto_recovery_ = true; }
|
||||||
|
|
||||||
Status::Severity GetErrorSeverity(BackgroundErrorReason reason,
|
Status::Severity GetErrorSeverity(BackgroundErrorReason reason,
|
||||||
Status::Code code,
|
Status::Code code, Status::SubCode subcode);
|
||||||
Status::SubCode subcode);
|
|
||||||
|
|
||||||
const Status& SetBGError(const Status& bg_err, BackgroundErrorReason reason);
|
const Status& SetBGError(const Status& bg_err, BackgroundErrorReason reason);
|
||||||
|
|
||||||
Status GetBGError() const { return bg_error_; }
|
Status GetBGError() const { return bg_error_; }
|
||||||
|
|
||||||
Status GetRecoveryError() const { return recovery_error_; }
|
Status GetRecoveryError() const { return recovery_error_; }
|
||||||
|
|
||||||
Status ClearBGError();
|
Status ClearBGError();
|
||||||
|
|
||||||
bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); }
|
bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); }
|
||||||
|
|
||||||
bool IsBGWorkStopped() {
|
bool IsBGWorkStopped() {
|
||||||
assert(db_mutex_);
|
assert(db_mutex_);
|
||||||
db_mutex_->AssertHeld();
|
db_mutex_->AssertHeld();
|
||||||
return !bg_error_.ok() &&
|
return !bg_error_.ok() &&
|
||||||
(bg_error_.severity() >= Status::Severity::kHardError ||
|
(bg_error_.severity() >= Status::Severity::kHardError ||
|
||||||
!auto_recovery_ || soft_error_no_bg_work_);
|
!auto_recovery_ || soft_error_no_bg_work_);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool IsSoftErrorNoBGWork() { return soft_error_no_bg_work_; }
|
bool IsSoftErrorNoBGWork() { return soft_error_no_bg_work_; }
|
||||||
|
|
||||||
bool IsRecoveryInProgress() { return recovery_in_prog_; }
|
bool IsRecoveryInProgress() { return recovery_in_prog_; }
|
||||||
|
|
||||||
Status RecoverFromBGError(bool is_manual = false);
|
Status RecoverFromBGError(bool is_manual = false);
|
||||||
void CancelErrorRecovery();
|
void CancelErrorRecovery();
|
||||||
|
|
||||||
void EndAutoRecovery();
|
void EndAutoRecovery();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
DBImpl* db_;
|
DBImpl* db_;
|
||||||
const ImmutableDBOptions& db_options_;
|
const ImmutableDBOptions& db_options_;
|
||||||
Status bg_error_;
|
Status bg_error_;
|
||||||
// A separate Status variable used to record any errors during the
|
// A separate Status variable used to record any errors during the
|
||||||
// recovery process from hard errors
|
// recovery process from hard errors
|
||||||
Status recovery_error_;
|
Status recovery_error_;
|
||||||
// A separate IO Status variable used to record any IO errors during
|
// A separate IO Status variable used to record any IO errors during
|
||||||
// the recovery process. At the same time, recovery_error_ is also set.
|
// the recovery process. At the same time, recovery_error_ is also set.
|
||||||
IOStatus recovery_io_error_;
|
IOStatus recovery_io_error_;
|
||||||
// The condition variable used with db_mutex during auto resume for time
|
// The condition variable used with db_mutex during auto resume for time
|
||||||
// wait.
|
// wait.
|
||||||
InstrumentedCondVar cv_;
|
InstrumentedCondVar cv_;
|
||||||
bool end_recovery_;
|
bool end_recovery_;
|
||||||
std::unique_ptr<port::Thread> recovery_thread_;
|
std::unique_ptr<port::Thread> recovery_thread_;
|
||||||
|
|
||||||
InstrumentedMutex* db_mutex_;
|
InstrumentedMutex* db_mutex_;
|
||||||
// A flag indicating whether automatic recovery from errors is enabled
|
// A flag indicating whether automatic recovery from errors is enabled
|
||||||
bool auto_recovery_;
|
bool auto_recovery_;
|
||||||
bool recovery_in_prog_;
|
bool recovery_in_prog_;
|
||||||
// A flag to indicate that for the soft error, we should not allow any
|
// A flag to indicate that for the soft error, we should not allow any
|
||||||
// background work except the work is from recovery.
|
// background work except the work is from recovery.
|
||||||
bool soft_error_no_bg_work_;
|
bool soft_error_no_bg_work_;
|
||||||
|
|
||||||
// Used to store the context for recover, such as flush reason.
|
// Used to store the context for recover, such as flush reason.
|
||||||
DBRecoverContext recover_context_;
|
DBRecoverContext recover_context_;
|
||||||
std::atomic<bool> is_db_stopped_;
|
std::atomic<bool> is_db_stopped_;
|
||||||
|
|
||||||
// The pointer of DB statistics.
|
// The pointer of DB statistics.
|
||||||
std::shared_ptr<Statistics> bg_error_stats_;
|
std::shared_ptr<Statistics> bg_error_stats_;
|
||||||
|
|
||||||
const Status& HandleKnownErrors(const Status& bg_err,
|
const Status& HandleKnownErrors(const Status& bg_err,
|
||||||
BackgroundErrorReason reason);
|
BackgroundErrorReason reason);
|
||||||
Status OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery);
|
Status OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery);
|
||||||
void RecoverFromNoSpace();
|
void RecoverFromNoSpace();
|
||||||
const Status& StartRecoverFromRetryableBGIOError(const IOStatus& io_error);
|
const Status& StartRecoverFromRetryableBGIOError(const IOStatus& io_error);
|
||||||
void RecoverFromRetryableBGIOError();
|
void RecoverFromRetryableBGIOError();
|
||||||
// First, if it is in recovery and the recovery_error is ok. Set the
|
// First, if it is in recovery and the recovery_error is ok. Set the
|
||||||
// recovery_error_ to bg_err. Second, if the severity is higher than the
|
// recovery_error_ to bg_err. Second, if the severity is higher than the
|
||||||
// current bg_error_, overwrite it.
|
// current bg_error_, overwrite it.
|
||||||
void CheckAndSetRecoveryAndBGError(const Status& bg_err);
|
void CheckAndSetRecoveryAndBGError(const Status& bg_err);
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
|
@ -23,7 +23,7 @@ template <class T>
|
||||||
inline T SafeDivide(T a, T b) {
|
inline T SafeDivide(T a, T b) {
|
||||||
return b == 0 ? 0 : a / b;
|
return b == 0 ? 0 : a / b;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) {
|
void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) {
|
||||||
*jwriter << "time_micros"
|
*jwriter << "time_micros"
|
||||||
|
|
|
@ -39,9 +39,9 @@ class EventHelpers {
|
||||||
const std::string& file_checksum,
|
const std::string& file_checksum,
|
||||||
const std::string& file_checksum_func_name);
|
const std::string& file_checksum_func_name);
|
||||||
static void LogAndNotifyTableFileDeletion(
|
static void LogAndNotifyTableFileDeletion(
|
||||||
EventLogger* event_logger, int job_id,
|
EventLogger* event_logger, int job_id, uint64_t file_number,
|
||||||
uint64_t file_number, const std::string& file_path,
|
const std::string& file_path, const Status& status,
|
||||||
const Status& status, const std::string& db_name,
|
const std::string& db_name,
|
||||||
const std::vector<std::shared_ptr<EventListener>>& listeners);
|
const std::vector<std::shared_ptr<EventListener>>& listeners);
|
||||||
static void NotifyOnErrorRecoveryEnd(
|
static void NotifyOnErrorRecoveryEnd(
|
||||||
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
const std::vector<std::shared_ptr<EventListener>>& listeners,
|
||||||
|
|
|
@ -106,9 +106,8 @@ Status ExternalSstFileIngestionJob::Prepare(
|
||||||
for (IngestedFileInfo& f : files_to_ingest_) {
|
for (IngestedFileInfo& f : files_to_ingest_) {
|
||||||
f.copy_file = false;
|
f.copy_file = false;
|
||||||
const std::string path_outside_db = f.external_file_path;
|
const std::string path_outside_db = f.external_file_path;
|
||||||
const std::string path_inside_db =
|
const std::string path_inside_db = TableFileName(
|
||||||
TableFileName(cfd_->ioptions()->cf_paths, f.fd.GetNumber(),
|
cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
|
||||||
f.fd.GetPathId());
|
|
||||||
if (ingestion_options_.move_files) {
|
if (ingestion_options_.move_files) {
|
||||||
status =
|
status =
|
||||||
fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
|
fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
|
||||||
|
@ -491,7 +490,8 @@ void ExternalSstFileIngestionJob::UpdateStats() {
|
||||||
stream.StartArray();
|
stream.StartArray();
|
||||||
|
|
||||||
for (IngestedFileInfo& f : files_to_ingest_) {
|
for (IngestedFileInfo& f : files_to_ingest_) {
|
||||||
InternalStats::CompactionStats stats(CompactionReason::kExternalSstIngestion, 1);
|
InternalStats::CompactionStats stats(
|
||||||
|
CompactionReason::kExternalSstIngestion, 1);
|
||||||
stats.micros = total_time;
|
stats.micros = total_time;
|
||||||
// If actual copy occurred for this file, then we need to count the file
|
// If actual copy occurred for this file, then we need to count the file
|
||||||
// size as the actual bytes written. If the file was linked, then we ignore
|
// size as the actual bytes written. If the file was linked, then we ignore
|
||||||
|
@ -591,8 +591,8 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
|
||||||
std::unique_ptr<FSRandomAccessFile> sst_file;
|
std::unique_ptr<FSRandomAccessFile> sst_file;
|
||||||
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
||||||
|
|
||||||
status = fs_->NewRandomAccessFile(external_file, env_options_,
|
status =
|
||||||
&sst_file, nullptr);
|
fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
|
||||||
if (!status.ok()) {
|
if (!status.ok()) {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
@ -658,9 +658,9 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
|
||||||
assert(seqno_iter == uprops.end());
|
assert(seqno_iter == uprops.end());
|
||||||
file_to_ingest->original_seqno = 0;
|
file_to_ingest->original_seqno = 0;
|
||||||
if (ingestion_options_.allow_blocking_flush ||
|
if (ingestion_options_.allow_blocking_flush ||
|
||||||
ingestion_options_.allow_global_seqno) {
|
ingestion_options_.allow_global_seqno) {
|
||||||
return Status::InvalidArgument(
|
return Status::InvalidArgument(
|
||||||
"External SST file V1 does not support global seqno");
|
"External SST file V1 does not support global seqno");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return Status::InvalidArgument("External file version is not supported");
|
return Status::InvalidArgument("External file version is not supported");
|
||||||
|
@ -857,7 +857,7 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_SYNC_POINT_CALLBACK(
|
TEST_SYNC_POINT_CALLBACK(
|
||||||
"ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile",
|
"ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile",
|
||||||
&overlap_with_db);
|
&overlap_with_db);
|
||||||
file_to_ingest->picked_level = target_level;
|
file_to_ingest->picked_level = target_level;
|
||||||
|
@ -872,10 +872,10 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
|
||||||
auto* vstorage = cfd_->current()->storage_info();
|
auto* vstorage = cfd_->current()->storage_info();
|
||||||
// first check if new files fit in the bottommost level
|
// first check if new files fit in the bottommost level
|
||||||
int bottom_lvl = cfd_->NumberLevels() - 1;
|
int bottom_lvl = cfd_->NumberLevels() - 1;
|
||||||
if(!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) {
|
if (!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) {
|
||||||
return Status::InvalidArgument(
|
return Status::InvalidArgument(
|
||||||
"Can't ingest_behind file as it doesn't fit "
|
"Can't ingest_behind file as it doesn't fit "
|
||||||
"at the bottommost level!");
|
"at the bottommost level!");
|
||||||
}
|
}
|
||||||
|
|
||||||
// second check if despite allow_ingest_behind=true we still have 0 seqnums
|
// second check if despite allow_ingest_behind=true we still have 0 seqnums
|
||||||
|
@ -884,8 +884,8 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
|
||||||
for (auto file : vstorage->LevelFiles(lvl)) {
|
for (auto file : vstorage->LevelFiles(lvl)) {
|
||||||
if (file->fd.smallest_seqno == 0) {
|
if (file->fd.smallest_seqno == 0) {
|
||||||
return Status::InvalidArgument(
|
return Status::InvalidArgument(
|
||||||
"Can't ingest_behind file as despite allow_ingest_behind=true "
|
"Can't ingest_behind file as despite allow_ingest_behind=true "
|
||||||
"there are files with 0 seqno in database at upper levels!");
|
"there are files with 0 seqno in database at upper levels!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -912,9 +912,8 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
|
||||||
// If the file system does not support random write, then we should not.
|
// If the file system does not support random write, then we should not.
|
||||||
// Otherwise we should.
|
// Otherwise we should.
|
||||||
std::unique_ptr<FSRandomRWFile> rwfile;
|
std::unique_ptr<FSRandomRWFile> rwfile;
|
||||||
Status status =
|
Status status = fs_->NewRandomRWFile(file_to_ingest->internal_file_path,
|
||||||
fs_->NewRandomRWFile(file_to_ingest->internal_file_path, env_options_,
|
env_options_, &rwfile, nullptr);
|
||||||
&rwfile, nullptr);
|
|
||||||
TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::NewRandomRWFile",
|
TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::NewRandomRWFile",
|
||||||
&status);
|
&status);
|
||||||
if (status.ok()) {
|
if (status.ok()) {
|
||||||
|
|
|
@ -301,7 +301,8 @@ TEST_F(ExternalSSTFileTest, Basic) {
|
||||||
|
|
||||||
SstFileWriter sst_file_writer(EnvOptions(), options);
|
SstFileWriter sst_file_writer(EnvOptions(), options);
|
||||||
|
|
||||||
// Current file size should be 0 after sst_file_writer init and before open a file.
|
// Current file size should be 0 after sst_file_writer init and before open
|
||||||
|
// a file.
|
||||||
ASSERT_EQ(sst_file_writer.FileSize(), 0);
|
ASSERT_EQ(sst_file_writer.FileSize(), 0);
|
||||||
|
|
||||||
// file1.sst (0 => 99)
|
// file1.sst (0 => 99)
|
||||||
|
@ -2318,7 +2319,6 @@ TEST_F(ExternalSSTFileTest, SkipBloomFilter) {
|
||||||
table_options.cache_index_and_filter_blocks = true;
|
table_options.cache_index_and_filter_blocks = true;
|
||||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||||
|
|
||||||
|
|
||||||
// Create external SST file and include bloom filters
|
// Create external SST file and include bloom filters
|
||||||
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
|
|
@ -338,8 +338,7 @@ class FaultInjectionTest
|
||||||
FaultInjectionTest::kValExpectNoError));
|
FaultInjectionTest::kValExpectNoError));
|
||||||
}
|
}
|
||||||
|
|
||||||
void NoWriteTestPreFault() {
|
void NoWriteTestPreFault() {}
|
||||||
}
|
|
||||||
|
|
||||||
void NoWriteTestReopenWithFault(ResetMethod reset_method) {
|
void NoWriteTestReopenWithFault(ResetMethod reset_method) {
|
||||||
CloseDB();
|
CloseDB();
|
||||||
|
|
|
@ -8,8 +8,10 @@
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
#include "db/file_indexer.h"
|
#include "db/file_indexer.h"
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <functional>
|
#include <functional>
|
||||||
|
|
||||||
#include "db/version_edit.h"
|
#include "db/version_edit.h"
|
||||||
#include "rocksdb/comparator.h"
|
#include "rocksdb/comparator.h"
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,7 @@
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "memory/arena.h"
|
#include "memory/arena.h"
|
||||||
#include "port/port.h"
|
#include "port/port.h"
|
||||||
#include "util/autovector.h"
|
#include "util/autovector.h"
|
||||||
|
@ -66,7 +67,7 @@ class FileIndexer {
|
||||||
|
|
||||||
struct IndexUnit {
|
struct IndexUnit {
|
||||||
IndexUnit()
|
IndexUnit()
|
||||||
: smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {}
|
: smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {}
|
||||||
// During file search, a key is compared against smallest and largest
|
// During file search, a key is compared against smallest and largest
|
||||||
// from a FileMetaData. It can have 3 possible outcomes:
|
// from a FileMetaData. It can have 3 possible outcomes:
|
||||||
// (1) key is smaller than smallest, implying it is also smaller than
|
// (1) key is smaller than smallest, implying it is also smaller than
|
||||||
|
|
|
@ -8,7 +8,9 @@
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
#include "db/file_indexer.h"
|
#include "db/file_indexer.h"
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "db/dbformat.h"
|
#include "db/dbformat.h"
|
||||||
#include "db/version_edit.h"
|
#include "db/version_edit.h"
|
||||||
#include "port/stack_trace.h"
|
#include "port/stack_trace.h"
|
||||||
|
@ -73,8 +75,8 @@ class FileIndexerTest : public testing::Test {
|
||||||
}
|
}
|
||||||
|
|
||||||
void GetNextLevelIndex(const uint32_t level, const uint32_t file_index,
|
void GetNextLevelIndex(const uint32_t level, const uint32_t file_index,
|
||||||
const int cmp_smallest, const int cmp_largest, int32_t* left_index,
|
const int cmp_smallest, const int cmp_largest,
|
||||||
int32_t* right_index) {
|
int32_t* left_index, int32_t* right_index) {
|
||||||
*left_index = 100;
|
*left_index = 100;
|
||||||
*right_index = 100;
|
*right_index = 100;
|
||||||
indexer->GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest,
|
indexer->GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest,
|
||||||
|
|
|
@ -69,35 +69,33 @@ TEST_F(FileNameTest, Parse) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Errors
|
// Errors
|
||||||
static const char* errors[] = {
|
static const char* errors[] = {"",
|
||||||
"",
|
"foo",
|
||||||
"foo",
|
"foo-dx-100.log",
|
||||||
"foo-dx-100.log",
|
".log",
|
||||||
".log",
|
"",
|
||||||
"",
|
"manifest",
|
||||||
"manifest",
|
"CURREN",
|
||||||
"CURREN",
|
"CURRENTX",
|
||||||
"CURRENTX",
|
"MANIFES",
|
||||||
"MANIFES",
|
"MANIFEST",
|
||||||
"MANIFEST",
|
"MANIFEST-",
|
||||||
"MANIFEST-",
|
"XMANIFEST-3",
|
||||||
"XMANIFEST-3",
|
"MANIFEST-3x",
|
||||||
"MANIFEST-3x",
|
"META",
|
||||||
"META",
|
"METADB",
|
||||||
"METADB",
|
"METADB-",
|
||||||
"METADB-",
|
"XMETADB-3",
|
||||||
"XMETADB-3",
|
"METADB-3x",
|
||||||
"METADB-3x",
|
"LOC",
|
||||||
"LOC",
|
"LOCKx",
|
||||||
"LOCKx",
|
"LO",
|
||||||
"LO",
|
"LOGx",
|
||||||
"LOGx",
|
"18446744073709551616.log",
|
||||||
"18446744073709551616.log",
|
"184467440737095516150.log",
|
||||||
"184467440737095516150.log",
|
"100",
|
||||||
"100",
|
"100.",
|
||||||
"100.",
|
"100.lop"};
|
||||||
"100.lop"
|
|
||||||
};
|
|
||||||
for (unsigned int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
|
for (unsigned int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
|
||||||
std::string f = errors[i];
|
std::string f = errors[i];
|
||||||
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
|
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
|
||||||
|
|
|
@ -48,7 +48,7 @@
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
const char* GetFlushReasonString (FlushReason flush_reason) {
|
const char* GetFlushReasonString(FlushReason flush_reason) {
|
||||||
switch (flush_reason) {
|
switch (flush_reason) {
|
||||||
case FlushReason::kOthers:
|
case FlushReason::kOthers:
|
||||||
return "Other Reasons";
|
return "Other Reasons";
|
||||||
|
@ -136,17 +136,14 @@ FlushJob::FlushJob(
|
||||||
TEST_SYNC_POINT("FlushJob::FlushJob()");
|
TEST_SYNC_POINT("FlushJob::FlushJob()");
|
||||||
}
|
}
|
||||||
|
|
||||||
FlushJob::~FlushJob() {
|
FlushJob::~FlushJob() { ThreadStatusUtil::ResetThreadStatus(); }
|
||||||
ThreadStatusUtil::ResetThreadStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
void FlushJob::ReportStartedFlush() {
|
void FlushJob::ReportStartedFlush() {
|
||||||
ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
|
ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
|
||||||
db_options_.enable_thread_tracking);
|
db_options_.enable_thread_tracking);
|
||||||
ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
|
ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
|
||||||
ThreadStatusUtil::SetThreadOperationProperty(
|
ThreadStatusUtil::SetThreadOperationProperty(ThreadStatus::COMPACTION_JOB_ID,
|
||||||
ThreadStatus::COMPACTION_JOB_ID,
|
job_context_->job_id);
|
||||||
job_context_->job_id);
|
|
||||||
IOSTATS_RESET(bytes_written);
|
IOSTATS_RESET(bytes_written);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -156,8 +153,7 @@ void FlushJob::ReportFlushInputSize(const autovector<MemTable*>& mems) {
|
||||||
input_size += mem->ApproximateMemoryUsage();
|
input_size += mem->ApproximateMemoryUsage();
|
||||||
}
|
}
|
||||||
ThreadStatusUtil::IncreaseThreadOperationProperty(
|
ThreadStatusUtil::IncreaseThreadOperationProperty(
|
||||||
ThreadStatus::FLUSH_BYTES_MEMTABLES,
|
ThreadStatus::FLUSH_BYTES_MEMTABLES, input_size);
|
||||||
input_size);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void FlushJob::RecordFlushIOStats() {
|
void FlushJob::RecordFlushIOStats() {
|
||||||
|
@ -220,8 +216,7 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,
|
||||||
double mempurge_threshold =
|
double mempurge_threshold =
|
||||||
mutable_cf_options_.experimental_mempurge_threshold;
|
mutable_cf_options_.experimental_mempurge_threshold;
|
||||||
|
|
||||||
AutoThreadOperationStageUpdater stage_run(
|
AutoThreadOperationStageUpdater stage_run(ThreadStatus::STAGE_FLUSH_RUN);
|
||||||
ThreadStatus::STAGE_FLUSH_RUN);
|
|
||||||
if (mems_.empty()) {
|
if (mems_.empty()) {
|
||||||
ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush",
|
ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush",
|
||||||
cfd_->GetName().c_str());
|
cfd_->GetName().c_str());
|
||||||
|
@ -906,8 +901,7 @@ Status FlushJob::WriteLevel0Table() {
|
||||||
}
|
}
|
||||||
const uint64_t current_time = static_cast<uint64_t>(_current_time);
|
const uint64_t current_time = static_cast<uint64_t>(_current_time);
|
||||||
|
|
||||||
uint64_t oldest_key_time =
|
uint64_t oldest_key_time = mems_.front()->ApproximateOldestKeyTime();
|
||||||
mems_.front()->ApproximateOldestKeyTime();
|
|
||||||
|
|
||||||
// It's not clear whether oldest_key_time is always available. In case
|
// It's not clear whether oldest_key_time is always available. In case
|
||||||
// it is not available, use current_time.
|
// it is not available, use current_time.
|
||||||
|
|
|
@ -214,8 +214,8 @@ TEST_F(FlushJobTest, NonEmpty) {
|
||||||
// Note: the first two blob references will not be considered when resolving
|
// Note: the first two blob references will not be considered when resolving
|
||||||
// the oldest blob file referenced (the first one is inlined TTL, while the
|
// the oldest blob file referenced (the first one is inlined TTL, while the
|
||||||
// second one is TTL and thus points to a TTL blob file).
|
// second one is TTL and thus points to a TTL blob file).
|
||||||
constexpr std::array<uint64_t, 6> blob_file_numbers{{
|
constexpr std::array<uint64_t, 6> blob_file_numbers{
|
||||||
kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
|
{kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
|
||||||
for (size_t i = 0; i < blob_file_numbers.size(); ++i) {
|
for (size_t i = 0; i < blob_file_numbers.size(); ++i) {
|
||||||
std::string key(std::to_string(i + 10001));
|
std::string key(std::to_string(i + 10001));
|
||||||
std::string blob_index;
|
std::string blob_index;
|
||||||
|
|
|
@ -104,9 +104,7 @@ class ForwardLevelIterator : public InternalIterator {
|
||||||
status_ = Status::NotSupported("ForwardLevelIterator::Prev()");
|
status_ = Status::NotSupported("ForwardLevelIterator::Prev()");
|
||||||
valid_ = false;
|
valid_ = false;
|
||||||
}
|
}
|
||||||
bool Valid() const override {
|
bool Valid() const override { return valid_; }
|
||||||
return valid_;
|
|
||||||
}
|
|
||||||
void SeekToFirst() override {
|
void SeekToFirst() override {
|
||||||
assert(file_iter_ != nullptr);
|
assert(file_iter_ != nullptr);
|
||||||
if (!status_.ok()) {
|
if (!status_.ok()) {
|
||||||
|
@ -249,9 +247,7 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
|
||||||
immutable_status_.PermitUncheckedError();
|
immutable_status_.PermitUncheckedError();
|
||||||
}
|
}
|
||||||
|
|
||||||
ForwardIterator::~ForwardIterator() {
|
ForwardIterator::~ForwardIterator() { Cleanup(true); }
|
||||||
Cleanup(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv,
|
void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv,
|
||||||
bool background_purge_on_iterator_cleanup) {
|
bool background_purge_on_iterator_cleanup) {
|
||||||
|
@ -284,13 +280,13 @@ struct SVCleanupParams {
|
||||||
SuperVersion* sv;
|
SuperVersion* sv;
|
||||||
bool background_purge_on_iterator_cleanup;
|
bool background_purge_on_iterator_cleanup;
|
||||||
};
|
};
|
||||||
}
|
} // anonymous namespace
|
||||||
|
|
||||||
// Used in PinnedIteratorsManager to release pinned SuperVersion
|
// Used in PinnedIteratorsManager to release pinned SuperVersion
|
||||||
void ForwardIterator::DeferredSVCleanup(void* arg) {
|
void ForwardIterator::DeferredSVCleanup(void* arg) {
|
||||||
auto d = reinterpret_cast<SVCleanupParams*>(arg);
|
auto d = reinterpret_cast<SVCleanupParams*>(arg);
|
||||||
ForwardIterator::SVCleanup(
|
ForwardIterator::SVCleanup(d->db, d->sv,
|
||||||
d->db, d->sv, d->background_purge_on_iterator_cleanup);
|
d->background_purge_on_iterator_cleanup);
|
||||||
delete d;
|
delete d;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -547,8 +543,7 @@ void ForwardIterator::Next() {
|
||||||
assert(valid_);
|
assert(valid_);
|
||||||
bool update_prev_key = false;
|
bool update_prev_key = false;
|
||||||
|
|
||||||
if (sv_ == nullptr ||
|
if (sv_ == nullptr || sv_->version_number != cfd_->GetSuperVersionNumber()) {
|
||||||
sv_->version_number != cfd_->GetSuperVersionNumber()) {
|
|
||||||
std::string current_key = key().ToString();
|
std::string current_key = key().ToString();
|
||||||
Slice old_key(current_key.data(), current_key.size());
|
Slice old_key(current_key.data(), current_key.size());
|
||||||
|
|
||||||
|
@ -578,7 +573,6 @@ void ForwardIterator::Next() {
|
||||||
update_prev_key = true;
|
update_prev_key = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if (update_prev_key) {
|
if (update_prev_key) {
|
||||||
prev_key_.SetInternalKey(current_->key());
|
prev_key_.SetInternalKey(current_->key());
|
||||||
is_prev_set_ = true;
|
is_prev_set_ = true;
|
||||||
|
@ -635,7 +629,7 @@ bool ForwardIterator::PrepareValue() {
|
||||||
|
|
||||||
assert(!current_->Valid());
|
assert(!current_->Valid());
|
||||||
assert(!current_->status().ok());
|
assert(!current_->status().ok());
|
||||||
assert(current_ != mutable_iter_); // memtable iterator can't fail
|
assert(current_ != mutable_iter_); // memtable iterator can't fail
|
||||||
assert(immutable_status_.ok());
|
assert(immutable_status_.ok());
|
||||||
|
|
||||||
valid_ = false;
|
valid_ = false;
|
||||||
|
@ -950,11 +944,11 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
|
||||||
}
|
}
|
||||||
Slice prev_key = prev_key_.GetInternalKey();
|
Slice prev_key = prev_key_.GetInternalKey();
|
||||||
if (prefix_extractor_ && prefix_extractor_->Transform(target).compare(
|
if (prefix_extractor_ && prefix_extractor_->Transform(target).compare(
|
||||||
prefix_extractor_->Transform(prev_key)) != 0) {
|
prefix_extractor_->Transform(prev_key)) != 0) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (cfd_->internal_comparator().InternalKeyComparator::Compare(
|
if (cfd_->internal_comparator().InternalKeyComparator::Compare(
|
||||||
prev_key, target) >= (is_prev_inclusive_ ? 1 : 0)) {
|
prev_key, target) >= (is_prev_inclusive_ ? 1 : 0)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -963,8 +957,8 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (cfd_->internal_comparator().InternalKeyComparator::Compare(
|
if (cfd_->internal_comparator().InternalKeyComparator::Compare(
|
||||||
target, current_ == mutable_iter_ ? immutable_min_heap_.top()->key()
|
target, current_ == mutable_iter_ ? immutable_min_heap_.top()->key()
|
||||||
: current_->key()) > 0) {
|
: current_->key()) > 0) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
|
@ -1040,11 +1034,11 @@ uint32_t ForwardIterator::FindFileInRange(
|
||||||
uint32_t left, uint32_t right) {
|
uint32_t left, uint32_t right) {
|
||||||
auto cmp = [&](const FileMetaData* f, const Slice& k) -> bool {
|
auto cmp = [&](const FileMetaData* f, const Slice& k) -> bool {
|
||||||
return cfd_->internal_comparator().InternalKeyComparator::Compare(
|
return cfd_->internal_comparator().InternalKeyComparator::Compare(
|
||||||
f->largest.Encode(), k) < 0;
|
f->largest.Encode(), k) < 0;
|
||||||
};
|
};
|
||||||
const auto &b = files.begin();
|
const auto& b = files.begin();
|
||||||
return static_cast<uint32_t>(std::lower_bound(b + left,
|
return static_cast<uint32_t>(
|
||||||
b + right, internal_key, cmp) - b);
|
std::lower_bound(b + left, b + right, internal_key, cmp) - b);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) {
|
void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) {
|
||||||
|
|
|
@ -7,9 +7,9 @@
|
||||||
#include "rocksdb/comparator.h"
|
#include "rocksdb/comparator.h"
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
|
|
||||||
|
#include <queue>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <queue>
|
|
||||||
|
|
||||||
#include "memory/arena.h"
|
#include "memory/arena.h"
|
||||||
#include "rocksdb/db.h"
|
#include "rocksdb/db.h"
|
||||||
|
@ -35,6 +35,7 @@ class MinIterComparator {
|
||||||
bool operator()(InternalIterator* a, InternalIterator* b) {
|
bool operator()(InternalIterator* a, InternalIterator* b) {
|
||||||
return comparator_->Compare(a->key(), b->key()) > 0;
|
return comparator_->Compare(a->key(), b->key()) > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
const CompareInterface* comparator_;
|
const CompareInterface* comparator_;
|
||||||
};
|
};
|
||||||
|
@ -92,8 +93,8 @@ class ForwardIterator : public InternalIterator {
|
||||||
// either done immediately or deferred until this iterator is unpinned by
|
// either done immediately or deferred until this iterator is unpinned by
|
||||||
// PinnedIteratorsManager.
|
// PinnedIteratorsManager.
|
||||||
void SVCleanup();
|
void SVCleanup();
|
||||||
static void SVCleanup(
|
static void SVCleanup(DBImpl* db, SuperVersion* sv,
|
||||||
DBImpl* db, SuperVersion* sv, bool background_purge_on_iterator_cleanup);
|
bool background_purge_on_iterator_cleanup);
|
||||||
static void DeferredSVCleanup(void* arg);
|
static void DeferredSVCleanup(void* arg);
|
||||||
|
|
||||||
void RebuildIterators(bool refresh_sv);
|
void RebuildIterators(bool refresh_sv);
|
||||||
|
@ -107,9 +108,9 @@ class ForwardIterator : public InternalIterator {
|
||||||
void UpdateCurrent();
|
void UpdateCurrent();
|
||||||
bool NeedToSeekImmutable(const Slice& internal_key);
|
bool NeedToSeekImmutable(const Slice& internal_key);
|
||||||
void DeleteCurrentIter();
|
void DeleteCurrentIter();
|
||||||
uint32_t FindFileInRange(
|
uint32_t FindFileInRange(const std::vector<FileMetaData*>& files,
|
||||||
const std::vector<FileMetaData*>& files, const Slice& internal_key,
|
const Slice& internal_key, uint32_t left,
|
||||||
uint32_t left, uint32_t right);
|
uint32_t right);
|
||||||
|
|
||||||
bool IsOverUpperBound(const Slice& internal_key) const;
|
bool IsOverUpperBound(const Slice& internal_key) const;
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,7 @@ int main() {
|
||||||
int main() { return 0; }
|
int main() { return 0; }
|
||||||
#else
|
#else
|
||||||
#include <semaphore.h>
|
#include <semaphore.h>
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <bitset>
|
#include <bitset>
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
@ -281,8 +282,9 @@ struct StatsThread {
|
||||||
}
|
}
|
||||||
auto now = std::chrono::steady_clock::now();
|
auto now = std::chrono::steady_clock::now();
|
||||||
double elapsed =
|
double elapsed =
|
||||||
std::chrono::duration_cast<std::chrono::duration<double> >(
|
std::chrono::duration_cast<std::chrono::duration<double> >(now -
|
||||||
now - tlast).count();
|
tlast)
|
||||||
|
.count();
|
||||||
uint64_t w = ::stats.written.load();
|
uint64_t w = ::stats.written.load();
|
||||||
uint64_t r = ::stats.read.load();
|
uint64_t r = ::stats.read.load();
|
||||||
fprintf(stderr,
|
fprintf(stderr,
|
||||||
|
|
|
@ -228,8 +228,8 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo(
|
||||||
std::unique_ptr<FSRandomAccessFile> sst_file;
|
std::unique_ptr<FSRandomAccessFile> sst_file;
|
||||||
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
||||||
|
|
||||||
status = fs_->NewRandomAccessFile(external_file, env_options_,
|
status =
|
||||||
&sst_file, nullptr);
|
fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
|
||||||
if (!status.ok()) {
|
if (!status.ok()) {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
|
@ -353,7 +353,7 @@ class InternalStats {
|
||||||
this->num_output_records += c.num_output_records;
|
this->num_output_records += c.num_output_records;
|
||||||
this->count += c.count;
|
this->count += c.count;
|
||||||
int num_of_reasons = static_cast<int>(CompactionReason::kNumOfReasons);
|
int num_of_reasons = static_cast<int>(CompactionReason::kNumOfReasons);
|
||||||
for (int i = 0; i< num_of_reasons; i++) {
|
for (int i = 0; i < num_of_reasons; i++) {
|
||||||
counts[i] += c.counts[i];
|
counts[i] += c.counts[i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -648,8 +648,8 @@ class InternalStats {
|
||||||
struct CFStatsSnapshot {
|
struct CFStatsSnapshot {
|
||||||
// ColumnFamily-level stats
|
// ColumnFamily-level stats
|
||||||
CompactionStats comp_stats;
|
CompactionStats comp_stats;
|
||||||
uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush)
|
uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush)
|
||||||
uint64_t stall_count; // Stall count
|
uint64_t stall_count; // Stall count
|
||||||
// Stats from compaction jobs - bytes written, bytes read, duration.
|
// Stats from compaction jobs - bytes written, bytes read, duration.
|
||||||
uint64_t compact_bytes_write;
|
uint64_t compact_bytes_write;
|
||||||
uint64_t compact_bytes_read;
|
uint64_t compact_bytes_read;
|
||||||
|
@ -691,10 +691,10 @@ class InternalStats {
|
||||||
|
|
||||||
struct DBStatsSnapshot {
|
struct DBStatsSnapshot {
|
||||||
// DB-level stats
|
// DB-level stats
|
||||||
uint64_t ingest_bytes; // Bytes written by user
|
uint64_t ingest_bytes; // Bytes written by user
|
||||||
uint64_t wal_bytes; // Bytes written to WAL
|
uint64_t wal_bytes; // Bytes written to WAL
|
||||||
uint64_t wal_synced; // Number of times WAL is synced
|
uint64_t wal_synced; // Number of times WAL is synced
|
||||||
uint64_t write_with_wal; // Number of writes that request WAL
|
uint64_t write_with_wal; // Number of writes that request WAL
|
||||||
// These count the number of writes processed by the calling thread or
|
// These count the number of writes processed by the calling thread or
|
||||||
// another thread.
|
// another thread.
|
||||||
uint64_t write_other;
|
uint64_t write_other;
|
||||||
|
@ -980,13 +980,14 @@ class InternalStats {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool GetIntProperty(const DBPropertyInfo& /*property_info*/, uint64_t* /*value*/,
|
bool GetIntProperty(const DBPropertyInfo& /*property_info*/,
|
||||||
DBImpl* /*db*/) const {
|
uint64_t* /*value*/, DBImpl* /*db*/) const {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool GetIntPropertyOutOfMutex(const DBPropertyInfo& /*property_info*/,
|
bool GetIntPropertyOutOfMutex(const DBPropertyInfo& /*property_info*/,
|
||||||
Version* /*version*/, uint64_t* /*value*/) const {
|
Version* /*version*/,
|
||||||
|
uint64_t* /*value*/) const {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
@ -35,7 +35,7 @@ struct SuperVersionContext {
|
||||||
new_superversion; // if nullptr no new superversion
|
new_superversion; // if nullptr no new superversion
|
||||||
|
|
||||||
explicit SuperVersionContext(bool create_superversion = false)
|
explicit SuperVersionContext(bool create_superversion = false)
|
||||||
: new_superversion(create_superversion ? new SuperVersion() : nullptr) {}
|
: new_superversion(create_superversion ? new SuperVersion() : nullptr) {}
|
||||||
|
|
||||||
explicit SuperVersionContext(SuperVersionContext&& other) noexcept
|
explicit SuperVersionContext(SuperVersionContext&& other) noexcept
|
||||||
: superversions_to_free(std::move(other.superversions_to_free)),
|
: superversions_to_free(std::move(other.superversions_to_free)),
|
||||||
|
@ -54,8 +54,7 @@ struct SuperVersionContext {
|
||||||
|
|
||||||
inline bool HaveSomethingToDelete() const {
|
inline bool HaveSomethingToDelete() const {
|
||||||
#ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION
|
#ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION
|
||||||
return !superversions_to_free.empty() ||
|
return !superversions_to_free.empty() || !write_stall_notifications.empty();
|
||||||
!write_stall_notifications.empty();
|
|
||||||
#else
|
#else
|
||||||
return !superversions_to_free.empty();
|
return !superversions_to_free.empty();
|
||||||
#endif
|
#endif
|
||||||
|
@ -77,7 +76,8 @@ struct SuperVersionContext {
|
||||||
(void)new_cond;
|
(void)new_cond;
|
||||||
(void)name;
|
(void)name;
|
||||||
(void)ioptions;
|
(void)ioptions;
|
||||||
#endif // !defined(ROCKSDB_LITE) && !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION)
|
#endif // !defined(ROCKSDB_LITE) &&
|
||||||
|
// !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION)
|
||||||
}
|
}
|
||||||
|
|
||||||
void Clean() {
|
void Clean() {
|
||||||
|
@ -139,8 +139,7 @@ struct JobContext {
|
||||||
CandidateFileInfo(std::string name, std::string path)
|
CandidateFileInfo(std::string name, std::string path)
|
||||||
: file_name(std::move(name)), file_path(std::move(path)) {}
|
: file_name(std::move(name)), file_path(std::move(path)) {}
|
||||||
bool operator==(const CandidateFileInfo& other) const {
|
bool operator==(const CandidateFileInfo& other) const {
|
||||||
return file_name == other.file_name &&
|
return file_name == other.file_name && file_path == other.file_path;
|
||||||
file_path == other.file_path;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -89,7 +89,7 @@ class TestCompactionListener : public EventListener {
|
||||||
public:
|
public:
|
||||||
explicit TestCompactionListener(EventListenerTest* test) : test_(test) {}
|
explicit TestCompactionListener(EventListenerTest* test) : test_(test) {}
|
||||||
|
|
||||||
void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) override {
|
void OnCompactionCompleted(DB* db, const CompactionJobInfo& ci) override {
|
||||||
std::lock_guard<std::mutex> lock(mutex_);
|
std::lock_guard<std::mutex> lock(mutex_);
|
||||||
compacted_dbs_.push_back(db);
|
compacted_dbs_.push_back(db);
|
||||||
ASSERT_GT(ci.input_files.size(), 0U);
|
ASSERT_GT(ci.input_files.size(), 0U);
|
||||||
|
@ -172,9 +172,9 @@ TEST_F(EventListenerTest, OnSingleDBCompactionTest) {
|
||||||
|
|
||||||
TestCompactionListener* listener = new TestCompactionListener(this);
|
TestCompactionListener* listener = new TestCompactionListener(this);
|
||||||
options.listeners.emplace_back(listener);
|
options.listeners.emplace_back(listener);
|
||||||
std::vector<std::string> cf_names = {
|
std::vector<std::string> cf_names = {"pikachu", "ilya", "muromec",
|
||||||
"pikachu", "ilya", "muromec", "dobrynia",
|
"dobrynia", "nikitich", "alyosha",
|
||||||
"nikitich", "alyosha", "popovich"};
|
"popovich"};
|
||||||
CreateAndReopenWithCF(cf_names, options);
|
CreateAndReopenWithCF(cf_names, options);
|
||||||
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
|
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
|
||||||
|
|
||||||
|
@ -214,8 +214,7 @@ class TestFlushListener : public EventListener {
|
||||||
virtual ~TestFlushListener() {
|
virtual ~TestFlushListener() {
|
||||||
prev_fc_info_.status.PermitUncheckedError(); // Ignore the status
|
prev_fc_info_.status.PermitUncheckedError(); // Ignore the status
|
||||||
}
|
}
|
||||||
void OnTableFileCreated(
|
void OnTableFileCreated(const TableFileCreationInfo& info) override {
|
||||||
const TableFileCreationInfo& info) override {
|
|
||||||
// remember the info for later checking the FlushJobInfo.
|
// remember the info for later checking the FlushJobInfo.
|
||||||
prev_fc_info_ = info;
|
prev_fc_info_ = info;
|
||||||
ASSERT_GT(info.db_name.size(), 0U);
|
ASSERT_GT(info.db_name.size(), 0U);
|
||||||
|
@ -250,8 +249,7 @@ class TestFlushListener : public EventListener {
|
||||||
#endif // ROCKSDB_USING_THREAD_STATUS
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
||||||
}
|
}
|
||||||
|
|
||||||
void OnFlushCompleted(
|
void OnFlushCompleted(DB* db, const FlushJobInfo& info) override {
|
||||||
DB* db, const FlushJobInfo& info) override {
|
|
||||||
flushed_dbs_.push_back(db);
|
flushed_dbs_.push_back(db);
|
||||||
flushed_column_family_names_.push_back(info.cf_name);
|
flushed_column_family_names_.push_back(info.cf_name);
|
||||||
if (info.triggered_writes_slowdown) {
|
if (info.triggered_writes_slowdown) {
|
||||||
|
@ -317,9 +315,9 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
|
||||||
#endif // ROCKSDB_USING_THREAD_STATUS
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
||||||
TestFlushListener* listener = new TestFlushListener(options.env, this);
|
TestFlushListener* listener = new TestFlushListener(options.env, this);
|
||||||
options.listeners.emplace_back(listener);
|
options.listeners.emplace_back(listener);
|
||||||
std::vector<std::string> cf_names = {
|
std::vector<std::string> cf_names = {"pikachu", "ilya", "muromec",
|
||||||
"pikachu", "ilya", "muromec", "dobrynia",
|
"dobrynia", "nikitich", "alyosha",
|
||||||
"nikitich", "alyosha", "popovich"};
|
"popovich"};
|
||||||
options.table_properties_collector_factories.push_back(
|
options.table_properties_collector_factories.push_back(
|
||||||
std::make_shared<TestPropertiesCollectorFactory>());
|
std::make_shared<TestPropertiesCollectorFactory>());
|
||||||
CreateAndReopenWithCF(cf_names, options);
|
CreateAndReopenWithCF(cf_names, options);
|
||||||
|
@ -421,9 +419,9 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
|
||||||
listeners.emplace_back(new TestFlushListener(options.env, this));
|
listeners.emplace_back(new TestFlushListener(options.env, this));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<std::string> cf_names = {
|
std::vector<std::string> cf_names = {"pikachu", "ilya", "muromec",
|
||||||
"pikachu", "ilya", "muromec", "dobrynia",
|
"dobrynia", "nikitich", "alyosha",
|
||||||
"nikitich", "alyosha", "popovich"};
|
"popovich"};
|
||||||
|
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
for (int i = 0; i < kNumListeners; ++i) {
|
for (int i = 0; i < kNumListeners; ++i) {
|
||||||
|
@ -433,7 +431,7 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
|
||||||
ColumnFamilyOptions cf_opts(options);
|
ColumnFamilyOptions cf_opts(options);
|
||||||
|
|
||||||
std::vector<DB*> dbs;
|
std::vector<DB*> dbs;
|
||||||
std::vector<std::vector<ColumnFamilyHandle *>> vec_handles;
|
std::vector<std::vector<ColumnFamilyHandle*>> vec_handles;
|
||||||
|
|
||||||
for (int d = 0; d < kNumDBs; ++d) {
|
for (int d = 0; d < kNumDBs; ++d) {
|
||||||
ASSERT_OK(DestroyDB(dbname_ + std::to_string(d), options));
|
ASSERT_OK(DestroyDB(dbname_ + std::to_string(d), options));
|
||||||
|
@ -452,8 +450,8 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
|
||||||
|
|
||||||
for (int d = 0; d < kNumDBs; ++d) {
|
for (int d = 0; d < kNumDBs; ++d) {
|
||||||
for (size_t c = 0; c < cf_names.size(); ++c) {
|
for (size_t c = 0; c < cf_names.size(); ++c) {
|
||||||
ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c],
|
ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c], cf_names[c],
|
||||||
cf_names[c], cf_names[c]));
|
cf_names[c]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -483,7 +481,6 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
for (auto handles : vec_handles) {
|
for (auto handles : vec_handles) {
|
||||||
for (auto h : handles) {
|
for (auto h : handles) {
|
||||||
delete h;
|
delete h;
|
||||||
|
@ -887,16 +884,17 @@ TEST_F(EventListenerTest, TableFileCreationListenersTest) {
|
||||||
}
|
}
|
||||||
|
|
||||||
class MemTableSealedListener : public EventListener {
|
class MemTableSealedListener : public EventListener {
|
||||||
private:
|
private:
|
||||||
SequenceNumber latest_seq_number_;
|
SequenceNumber latest_seq_number_;
|
||||||
public:
|
|
||||||
|
public:
|
||||||
MemTableSealedListener() {}
|
MemTableSealedListener() {}
|
||||||
void OnMemTableSealed(const MemTableInfo& info) override {
|
void OnMemTableSealed(const MemTableInfo& info) override {
|
||||||
latest_seq_number_ = info.first_seqno;
|
latest_seq_number_ = info.first_seqno;
|
||||||
}
|
}
|
||||||
|
|
||||||
void OnFlushCompleted(DB* /*db*/,
|
void OnFlushCompleted(DB* /*db*/,
|
||||||
const FlushJobInfo& flush_job_info) override {
|
const FlushJobInfo& flush_job_info) override {
|
||||||
ASSERT_LE(flush_job_info.smallest_seqno, latest_seq_number_);
|
ASSERT_LE(flush_job_info.smallest_seqno, latest_seq_number_);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -911,8 +909,8 @@ TEST_F(EventListenerTest, MemTableSealedListenerTest) {
|
||||||
|
|
||||||
for (unsigned int i = 0; i < 10; i++) {
|
for (unsigned int i = 0; i < 10; i++) {
|
||||||
std::string tag = std::to_string(i);
|
std::string tag = std::to_string(i);
|
||||||
ASSERT_OK(Put("foo"+tag, "aaa"));
|
ASSERT_OK(Put("foo" + tag, "aaa"));
|
||||||
ASSERT_OK(Put("bar"+tag, "bbb"));
|
ASSERT_OK(Put("bar" + tag, "bbb"));
|
||||||
|
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,8 +21,7 @@
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
namespace log {
|
namespace log {
|
||||||
|
|
||||||
Reader::Reporter::~Reporter() {
|
Reader::Reporter::~Reporter() {}
|
||||||
}
|
|
||||||
|
|
||||||
Reader::Reader(std::shared_ptr<Logger> info_log,
|
Reader::Reader(std::shared_ptr<Logger> info_log,
|
||||||
std::unique_ptr<SequentialFileReader>&& _file,
|
std::unique_ptr<SequentialFileReader>&& _file,
|
||||||
|
@ -241,9 +240,8 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
|
||||||
FALLTHROUGH_INTENDED;
|
FALLTHROUGH_INTENDED;
|
||||||
|
|
||||||
case kBadRecordChecksum:
|
case kBadRecordChecksum:
|
||||||
if (recycled_ &&
|
if (recycled_ && wal_recovery_mode ==
|
||||||
wal_recovery_mode ==
|
WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
||||||
WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
|
||||||
scratch->clear();
|
scratch->clear();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -297,9 +295,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t Reader::LastRecordOffset() {
|
uint64_t Reader::LastRecordOffset() { return last_record_offset_; }
|
||||||
return last_record_offset_;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t Reader::LastRecordEnd() {
|
uint64_t Reader::LastRecordEnd() {
|
||||||
return end_of_buffer_offset_ - buffer_.size();
|
return end_of_buffer_offset_ - buffer_.size();
|
||||||
|
@ -361,11 +357,11 @@ void Reader::UnmarkEOFInternal() {
|
||||||
if (read_buffer.data() != backing_store_ + eof_offset_) {
|
if (read_buffer.data() != backing_store_ + eof_offset_) {
|
||||||
// Read did not write to backing_store_
|
// Read did not write to backing_store_
|
||||||
memmove(backing_store_ + eof_offset_, read_buffer.data(),
|
memmove(backing_store_ + eof_offset_, read_buffer.data(),
|
||||||
read_buffer.size());
|
read_buffer.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
buffer_ = Slice(backing_store_ + consumed_bytes,
|
buffer_ = Slice(backing_store_ + consumed_bytes,
|
||||||
eof_offset_ + added - consumed_bytes);
|
eof_offset_ + added - consumed_bytes);
|
||||||
|
|
||||||
if (added < remaining) {
|
if (added < remaining) {
|
||||||
eof_ = true;
|
eof_ = true;
|
||||||
|
@ -385,7 +381,7 @@ void Reader::ReportDrop(size_t bytes, const Status& reason) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Reader::ReadMore(size_t* drop_size, int *error) {
|
bool Reader::ReadMore(size_t* drop_size, int* error) {
|
||||||
if (!eof_ && !read_error_) {
|
if (!eof_ && !read_error_) {
|
||||||
// Last read was a full read, so this is a trailer to skip
|
// Last read was a full read, so this is a trailer to skip
|
||||||
buffer_.clear();
|
buffer_.clear();
|
||||||
|
|
|
@ -85,9 +85,7 @@ class Reader {
|
||||||
uint64_t LastRecordEnd();
|
uint64_t LastRecordEnd();
|
||||||
|
|
||||||
// returns true if the reader has encountered an eof condition.
|
// returns true if the reader has encountered an eof condition.
|
||||||
bool IsEOF() {
|
bool IsEOF() { return eof_; }
|
||||||
return eof_;
|
|
||||||
}
|
|
||||||
|
|
||||||
// returns true if the reader has encountered read error.
|
// returns true if the reader has encountered read error.
|
||||||
bool hasReadError() const { return read_error_; }
|
bool hasReadError() const { return read_error_; }
|
||||||
|
@ -122,8 +120,8 @@ class Reader {
|
||||||
|
|
||||||
// Internal state variables used for reading records
|
// Internal state variables used for reading records
|
||||||
Slice buffer_;
|
Slice buffer_;
|
||||||
bool eof_; // Last Read() indicated EOF by returning < kBlockSize
|
bool eof_; // Last Read() indicated EOF by returning < kBlockSize
|
||||||
bool read_error_; // Error occurred while reading from file
|
bool read_error_; // Error occurred while reading from file
|
||||||
|
|
||||||
// Offset of the file position indicator within the last block when an
|
// Offset of the file position indicator within the last block when an
|
||||||
// EOF was detected.
|
// EOF was detected.
|
||||||
|
@ -182,7 +180,7 @@ class Reader {
|
||||||
uint64_t* fragment_checksum = nullptr);
|
uint64_t* fragment_checksum = nullptr);
|
||||||
|
|
||||||
// Read some more
|
// Read some more
|
||||||
bool ReadMore(size_t* drop_size, int *error);
|
bool ReadMore(size_t* drop_size, int* error);
|
||||||
|
|
||||||
void UnmarkEOFInternal();
|
void UnmarkEOFInternal();
|
||||||
|
|
||||||
|
|
|
@ -128,7 +128,7 @@ class LogTest
|
||||||
size_t dropped_bytes_;
|
size_t dropped_bytes_;
|
||||||
std::string message_;
|
std::string message_;
|
||||||
|
|
||||||
ReportCollector() : dropped_bytes_(0) { }
|
ReportCollector() : dropped_bytes_(0) {}
|
||||||
void Corruption(size_t bytes, const Status& status) override {
|
void Corruption(size_t bytes, const Status& status) override {
|
||||||
dropped_bytes_ += bytes;
|
dropped_bytes_ += bytes;
|
||||||
message_.append(status.ToString());
|
message_.append(status.ToString());
|
||||||
|
@ -185,9 +185,7 @@ class LogTest
|
||||||
ASSERT_OK(writer_->AddRecord(Slice(msg)));
|
ASSERT_OK(writer_->AddRecord(Slice(msg)));
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t WrittenBytes() const {
|
size_t WrittenBytes() const { return dest_contents().size(); }
|
||||||
return dest_contents().size();
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string Read(const WALRecoveryMode wal_recovery_mode =
|
std::string Read(const WALRecoveryMode wal_recovery_mode =
|
||||||
WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
||||||
|
@ -235,13 +233,9 @@ class LogTest
|
||||||
source_->force_error_position_ = position;
|
source_->force_error_position_ = position;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t DroppedBytes() const {
|
size_t DroppedBytes() const { return report_.dropped_bytes_; }
|
||||||
return report_.dropped_bytes_;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string ReportMessage() const {
|
std::string ReportMessage() const { return report_.message_; }
|
||||||
return report_.message_;
|
|
||||||
}
|
|
||||||
|
|
||||||
void ForceEOF(size_t position = 0) {
|
void ForceEOF(size_t position = 0) {
|
||||||
source_->force_eof_ = true;
|
source_->force_eof_ = true;
|
||||||
|
@ -389,7 +383,7 @@ TEST_P(LogTest, BadRecordType) {
|
||||||
|
|
||||||
TEST_P(LogTest, TruncatedTrailingRecordIsIgnored) {
|
TEST_P(LogTest, TruncatedTrailingRecordIsIgnored) {
|
||||||
Write("foo");
|
Write("foo");
|
||||||
ShrinkSize(4); // Drop all payload as well as a header byte
|
ShrinkSize(4); // Drop all payload as well as a header byte
|
||||||
ASSERT_EQ("EOF", Read());
|
ASSERT_EQ("EOF", Read());
|
||||||
// Truncated last record is ignored, not treated as an error
|
// Truncated last record is ignored, not treated as an error
|
||||||
ASSERT_EQ(0U, DroppedBytes());
|
ASSERT_EQ(0U, DroppedBytes());
|
||||||
|
@ -581,7 +575,7 @@ TEST_P(LogTest, ErrorJoinsRecords) {
|
||||||
Write("correct");
|
Write("correct");
|
||||||
|
|
||||||
// Wipe the middle block
|
// Wipe the middle block
|
||||||
for (unsigned int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
|
for (unsigned int offset = kBlockSize; offset < 2 * kBlockSize; offset++) {
|
||||||
SetByte(offset, 'x');
|
SetByte(offset, 'x');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -100,7 +100,7 @@ class Writer {
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::unique_ptr<WritableFileWriter> dest_;
|
std::unique_ptr<WritableFileWriter> dest_;
|
||||||
size_t block_offset_; // Current offset in block
|
size_t block_offset_; // Current offset in block
|
||||||
uint64_t log_number_;
|
uint64_t log_number_;
|
||||||
bool recycle_log_files_;
|
bool recycle_log_files_;
|
||||||
|
|
||||||
|
|
|
@ -58,6 +58,5 @@ class LogsWithPrepTracker {
|
||||||
// both logs_with_prep_ and prepared_section_completed_.
|
// both logs_with_prep_ and prepared_section_completed_.
|
||||||
std::unordered_map<uint64_t, uint64_t> prepared_section_completed_;
|
std::unordered_map<uint64_t, uint64_t> prepared_section_completed_;
|
||||||
std::mutex prepared_section_completed_mutex_;
|
std::mutex prepared_section_completed_mutex_;
|
||||||
|
|
||||||
};
|
};
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
|
@ -10,6 +10,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
|
||||||
#include "rocksdb/slice.h"
|
#include "rocksdb/slice.h"
|
||||||
#include "rocksdb/types.h"
|
#include "rocksdb/types.h"
|
||||||
|
|
||||||
|
|
|
@ -10,9 +10,10 @@
|
||||||
#include "db/malloc_stats.h"
|
#include "db/malloc_stats.h"
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
#include <memory>
|
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
#include "port/jemalloc_helper.h"
|
#include "port/jemalloc_helper.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
|
@ -42,9 +42,7 @@ std::string Key1(int i) {
|
||||||
return buf;
|
return buf;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string Key2(int i) {
|
std::string Key2(int i) { return Key1(i) + "_xxx"; }
|
||||||
return Key1(i) + "_xxx";
|
|
||||||
}
|
|
||||||
|
|
||||||
class ManualCompactionTest : public testing::Test {
|
class ManualCompactionTest : public testing::Test {
|
||||||
public:
|
public:
|
||||||
|
@ -102,10 +100,10 @@ TEST_F(ManualCompactionTest, CompactTouchesAllKeys) {
|
||||||
for (int iter = 0; iter < 2; ++iter) {
|
for (int iter = 0; iter < 2; ++iter) {
|
||||||
DB* db;
|
DB* db;
|
||||||
Options options;
|
Options options;
|
||||||
if (iter == 0) { // level compaction
|
if (iter == 0) { // level compaction
|
||||||
options.num_levels = 3;
|
options.num_levels = 3;
|
||||||
options.compaction_style = CompactionStyle::kCompactionStyleLevel;
|
options.compaction_style = CompactionStyle::kCompactionStyleLevel;
|
||||||
} else { // universal compaction
|
} else { // universal compaction
|
||||||
options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
|
options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
|
||||||
}
|
}
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
|
|
|
@ -330,9 +330,8 @@ int MemTable::KeyComparator::operator()(const char* prefix_len_key1,
|
||||||
return comparator.CompareKeySeq(k1, k2);
|
return comparator.CompareKeySeq(k1, k2);
|
||||||
}
|
}
|
||||||
|
|
||||||
int MemTable::KeyComparator::operator()(const char* prefix_len_key,
|
int MemTable::KeyComparator::operator()(
|
||||||
const KeyComparator::DecodedType& key)
|
const char* prefix_len_key, const KeyComparator::DecodedType& key) const {
|
||||||
const {
|
|
||||||
// Internal keys are encoded as length-prefixed strings.
|
// Internal keys are encoded as length-prefixed strings.
|
||||||
Slice a = GetLengthPrefixedSlice(prefix_len_key);
|
Slice a = GetLengthPrefixedSlice(prefix_len_key);
|
||||||
return comparator.CompareKeySeq(a, key);
|
return comparator.CompareKeySeq(a, key);
|
||||||
|
@ -914,7 +913,7 @@ struct Saver {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
static bool SaveValue(void* arg, const char* entry) {
|
static bool SaveValue(void* arg, const char* entry) {
|
||||||
TEST_SYNC_POINT_CALLBACK("Memtable::SaveValue:Begin:entry", &entry);
|
TEST_SYNC_POINT_CALLBACK("Memtable::SaveValue:Begin:entry", &entry);
|
||||||
|
|
|
@ -88,7 +88,7 @@ class MemTable {
|
||||||
public:
|
public:
|
||||||
struct KeyComparator : public MemTableRep::KeyComparator {
|
struct KeyComparator : public MemTableRep::KeyComparator {
|
||||||
const InternalKeyComparator comparator;
|
const InternalKeyComparator comparator;
|
||||||
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
|
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {}
|
||||||
virtual int operator()(const char* prefix_len_key1,
|
virtual int operator()(const char* prefix_len_key1,
|
||||||
const char* prefix_len_key2) const override;
|
const char* prefix_len_key2) const override;
|
||||||
virtual int operator()(const char* prefix_len_key,
|
virtual int operator()(const char* prefix_len_key,
|
||||||
|
@ -448,9 +448,7 @@ class MemTable {
|
||||||
// persisted.
|
// persisted.
|
||||||
// REQUIRES: external synchronization to prevent simultaneous
|
// REQUIRES: external synchronization to prevent simultaneous
|
||||||
// operations on the same MemTable.
|
// operations on the same MemTable.
|
||||||
void MarkFlushed() {
|
void MarkFlushed() { table_->MarkFlushed(); }
|
||||||
table_->MarkFlushed();
|
|
||||||
}
|
|
||||||
|
|
||||||
// return true if the current MemTableRep supports merge operator.
|
// return true if the current MemTableRep supports merge operator.
|
||||||
bool IsMergeOperatorSupported() const {
|
bool IsMergeOperatorSupported() const {
|
||||||
|
@ -562,8 +560,8 @@ class MemTable {
|
||||||
std::atomic<size_t> write_buffer_size_;
|
std::atomic<size_t> write_buffer_size_;
|
||||||
|
|
||||||
// These are used to manage memtable flushes to storage
|
// These are used to manage memtable flushes to storage
|
||||||
bool flush_in_progress_; // started the flush
|
bool flush_in_progress_; // started the flush
|
||||||
bool flush_completed_; // finished the flush
|
bool flush_completed_; // finished the flush
|
||||||
uint64_t file_number_; // filled up after flush is complete
|
uint64_t file_number_; // filled up after flush is complete
|
||||||
|
|
||||||
// The updates to be applied to the transaction log when this
|
// The updates to be applied to the transaction log when this
|
||||||
|
|
|
@ -4,9 +4,11 @@
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
#include "db/memtable_list.h"
|
#include "db/memtable_list.h"
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "db/merge_context.h"
|
#include "db/merge_context.h"
|
||||||
#include "db/version_set.h"
|
#include "db/version_set.h"
|
||||||
#include "db/write_controller.h"
|
#include "db/write_controller.h"
|
||||||
|
|
|
@ -8,6 +8,7 @@
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "rocksdb/slice.h"
|
#include "rocksdb/slice.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
|
@ -167,7 +167,7 @@ class MergeHelper {
|
||||||
const CompactionFilter* compaction_filter_;
|
const CompactionFilter* compaction_filter_;
|
||||||
const std::atomic<bool>* shutting_down_;
|
const std::atomic<bool>* shutting_down_;
|
||||||
Logger* logger_;
|
Logger* logger_;
|
||||||
bool assert_valid_internal_key_; // enforce no internal key corruption?
|
bool assert_valid_internal_key_; // enforce no internal key corruption?
|
||||||
bool allow_single_operand_;
|
bool allow_single_operand_;
|
||||||
SequenceNumber latest_snapshot_;
|
SequenceNumber latest_snapshot_;
|
||||||
const SnapshotChecker* const snapshot_checker_;
|
const SnapshotChecker* const snapshot_checker_;
|
||||||
|
|
|
@ -74,12 +74,11 @@ bool AssociativeMergeOperator::FullMergeV2(
|
||||||
|
|
||||||
// Call the user defined simple merge on the operands;
|
// Call the user defined simple merge on the operands;
|
||||||
// NOTE: It is assumed that the client's merge-operator will handle any errors.
|
// NOTE: It is assumed that the client's merge-operator will handle any errors.
|
||||||
bool AssociativeMergeOperator::PartialMerge(
|
bool AssociativeMergeOperator::PartialMerge(const Slice& key,
|
||||||
const Slice& key,
|
const Slice& left_operand,
|
||||||
const Slice& left_operand,
|
const Slice& right_operand,
|
||||||
const Slice& right_operand,
|
std::string* new_value,
|
||||||
std::string* new_value,
|
Logger* logger) const {
|
||||||
Logger* logger) const {
|
|
||||||
return Merge(key, &left_operand, right_operand, new_value, logger);
|
return Merge(key, &left_operand, right_operand, new_value, logger);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -10,10 +10,12 @@
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
|
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "db/db_impl/db_impl.h"
|
#include "db/db_impl/db_impl.h"
|
||||||
#include "db/db_test_util.h"
|
#include "db/db_test_util.h"
|
||||||
#include "db/version_set.h"
|
#include "db/version_set.h"
|
||||||
|
@ -28,7 +30,6 @@
|
||||||
#include "test_util/testutil.h"
|
#include "test_util/testutil.h"
|
||||||
#include "util/string_util.h"
|
#include "util/string_util.h"
|
||||||
|
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
class ObsoleteFilesTest : public DBTestBase {
|
class ObsoleteFilesTest : public DBTestBase {
|
||||||
|
@ -40,7 +41,7 @@ class ObsoleteFilesTest : public DBTestBase {
|
||||||
void AddKeys(int numkeys, int startkey) {
|
void AddKeys(int numkeys, int startkey) {
|
||||||
WriteOptions options;
|
WriteOptions options;
|
||||||
options.sync = false;
|
options.sync = false;
|
||||||
for (int i = startkey; i < (numkeys + startkey) ; i++) {
|
for (int i = startkey; i < (numkeys + startkey); i++) {
|
||||||
std::string temp = std::to_string(i);
|
std::string temp = std::to_string(i);
|
||||||
Slice key(temp);
|
Slice key(temp);
|
||||||
Slice value(temp);
|
Slice value(temp);
|
||||||
|
@ -117,7 +118,7 @@ TEST_F(ObsoleteFilesTest, RaceForObsoleteFileDeletion) {
|
||||||
"ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"},
|
"ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"},
|
||||||
{"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles",
|
{"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles",
|
||||||
"ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"},
|
"ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"},
|
||||||
});
|
});
|
||||||
SyncPoint::GetInstance()->SetCallBack(
|
SyncPoint::GetInstance()->SetCallBack(
|
||||||
"DBImpl::DeleteObsoleteFileImpl:AfterDeletion", [&](void* arg) {
|
"DBImpl::DeleteObsoleteFileImpl:AfterDeletion", [&](void* arg) {
|
||||||
Status* p_status = reinterpret_cast<Status*>(arg);
|
Status* p_status = reinterpret_cast<Status*>(arg);
|
||||||
|
|
|
@ -59,7 +59,7 @@ void VerifyOptionsFileName(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(OptionsFileTest, NumberOfOptionsFiles) {
|
TEST_F(OptionsFileTest, NumberOfOptionsFiles) {
|
||||||
const int kReopenCount = 20;
|
const int kReopenCount = 20;
|
||||||
|
|
|
@ -39,31 +39,31 @@ const std::string kDbName =
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
std::shared_ptr<DB> OpenDb(bool read_only = false) {
|
std::shared_ptr<DB> OpenDb(bool read_only = false) {
|
||||||
DB* db;
|
DB* db;
|
||||||
Options options;
|
Options options;
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
options.max_open_files = -1;
|
options.max_open_files = -1;
|
||||||
options.write_buffer_size = FLAGS_write_buffer_size;
|
options.write_buffer_size = FLAGS_write_buffer_size;
|
||||||
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
|
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
|
||||||
options.min_write_buffer_number_to_merge =
|
options.min_write_buffer_number_to_merge =
|
||||||
FLAGS_min_write_buffer_number_to_merge;
|
FLAGS_min_write_buffer_number_to_merge;
|
||||||
|
|
||||||
if (FLAGS_use_set_based_memetable) {
|
if (FLAGS_use_set_based_memetable) {
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
options.prefix_extractor.reset(
|
options.prefix_extractor.reset(
|
||||||
ROCKSDB_NAMESPACE::NewFixedPrefixTransform(0));
|
ROCKSDB_NAMESPACE::NewFixedPrefixTransform(0));
|
||||||
options.memtable_factory.reset(NewHashSkipListRepFactory());
|
options.memtable_factory.reset(NewHashSkipListRepFactory());
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
}
|
}
|
||||||
|
|
||||||
Status s;
|
Status s;
|
||||||
if (!read_only) {
|
if (!read_only) {
|
||||||
s = DB::Open(options, kDbName, &db);
|
s = DB::Open(options, kDbName, &db);
|
||||||
} else {
|
} else {
|
||||||
s = DB::OpenForReadOnly(options, kDbName, &db);
|
s = DB::OpenForReadOnly(options, kDbName, &db);
|
||||||
}
|
}
|
||||||
EXPECT_OK(s);
|
EXPECT_OK(s);
|
||||||
return std::shared_ptr<DB>(db);
|
return std::shared_ptr<DB>(db);
|
||||||
}
|
}
|
||||||
|
|
||||||
class PerfContextTest : public testing::Test {};
|
class PerfContextTest : public testing::Test {};
|
||||||
|
@ -81,7 +81,7 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
|
||||||
ASSERT_OK(db->Put(write_options, key, value));
|
ASSERT_OK(db->Put(write_options, key, value));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < FLAGS_total_keys -1 ; ++i) {
|
for (int i = 0; i < FLAGS_total_keys - 1; ++i) {
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + std::to_string(i);
|
||||||
ASSERT_OK(db->Delete(write_options, key));
|
ASSERT_OK(db->Delete(write_options, key));
|
||||||
}
|
}
|
||||||
|
@ -103,8 +103,9 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (FLAGS_verbose) {
|
if (FLAGS_verbose) {
|
||||||
std::cout << "Get user key comparison: \n" << hist_get.ToString()
|
std::cout << "Get user key comparison: \n"
|
||||||
<< "Get time: \n" << hist_get_time.ToString();
|
<< hist_get.ToString() << "Get time: \n"
|
||||||
|
<< hist_get_time.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
|
@ -139,7 +140,8 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
|
||||||
hist_seek.Add(get_perf_context()->user_key_comparison_count);
|
hist_seek.Add(get_perf_context()->user_key_comparison_count);
|
||||||
if (FLAGS_verbose) {
|
if (FLAGS_verbose) {
|
||||||
std::cout << "seek cmp: " << get_perf_context()->user_key_comparison_count
|
std::cout << "seek cmp: " << get_perf_context()->user_key_comparison_count
|
||||||
<< " ikey skipped " << get_perf_context()->internal_key_skipped_count
|
<< " ikey skipped "
|
||||||
|
<< get_perf_context()->internal_key_skipped_count
|
||||||
<< " idelete skipped "
|
<< " idelete skipped "
|
||||||
<< get_perf_context()->internal_delete_skipped_count
|
<< get_perf_context()->internal_delete_skipped_count
|
||||||
<< " elapsed: " << elapsed_nanos << "ns\n";
|
<< " elapsed: " << elapsed_nanos << "ns\n";
|
||||||
|
@ -322,7 +324,8 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
|
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
|
||||||
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
|
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
|
||||||
hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
|
hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
|
||||||
hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
|
hist_mget_num_memtable_checked.Add(
|
||||||
|
get_perf_context()->get_from_memtable_count);
|
||||||
hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
|
hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
|
||||||
hist_mget.Add(get_perf_context()->user_key_comparison_count);
|
hist_mget.Add(get_perf_context()->user_key_comparison_count);
|
||||||
}
|
}
|
||||||
|
@ -337,12 +340,14 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
<< hist_write_wal_time.ToString() << "\n"
|
<< hist_write_wal_time.ToString() << "\n"
|
||||||
<< " Writing Mem Table time: \n"
|
<< " Writing Mem Table time: \n"
|
||||||
<< hist_write_memtable_time.ToString() << "\n"
|
<< hist_write_memtable_time.ToString() << "\n"
|
||||||
<< " Write Delay: \n" << hist_write_delay_time.ToString() << "\n"
|
<< " Write Delay: \n"
|
||||||
|
<< hist_write_delay_time.ToString() << "\n"
|
||||||
<< " Waiting for Batch time: \n"
|
<< " Waiting for Batch time: \n"
|
||||||
<< hist_write_thread_wait_nanos.ToString() << "\n"
|
<< hist_write_thread_wait_nanos.ToString() << "\n"
|
||||||
<< " Scheduling Flushes and Compactions Time: \n"
|
<< " Scheduling Flushes and Compactions Time: \n"
|
||||||
<< hist_write_scheduling_time.ToString() << "\n"
|
<< hist_write_scheduling_time.ToString() << "\n"
|
||||||
<< " Total DB mutex nanos: \n" << total_db_mutex_nanos << "\n";
|
<< " Total DB mutex nanos: \n"
|
||||||
|
<< total_db_mutex_nanos << "\n";
|
||||||
|
|
||||||
std::cout << "Get(): Time to get snapshot: \n"
|
std::cout << "Get(): Time to get snapshot: \n"
|
||||||
<< hist_get_snapshot.ToString()
|
<< hist_get_snapshot.ToString()
|
||||||
|
@ -352,8 +357,8 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
<< hist_get_files.ToString() << "\n"
|
<< hist_get_files.ToString() << "\n"
|
||||||
<< " Number of memtables checked: \n"
|
<< " Number of memtables checked: \n"
|
||||||
<< hist_num_memtable_checked.ToString() << "\n"
|
<< hist_num_memtable_checked.ToString() << "\n"
|
||||||
<< " Time to post process: \n" << hist_get_post_process.ToString()
|
<< " Time to post process: \n"
|
||||||
<< "\n";
|
<< hist_get_post_process.ToString() << "\n";
|
||||||
|
|
||||||
std::cout << "MultiGet(): Time to get snapshot: \n"
|
std::cout << "MultiGet(): Time to get snapshot: \n"
|
||||||
<< hist_mget_snapshot.ToString()
|
<< hist_mget_snapshot.ToString()
|
||||||
|
@ -440,7 +445,8 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
|
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
|
||||||
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
|
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
|
||||||
hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
|
hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
|
||||||
hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
|
hist_mget_num_memtable_checked.Add(
|
||||||
|
get_perf_context()->get_from_memtable_count);
|
||||||
hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
|
hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
|
||||||
hist_mget.Add(get_perf_context()->user_key_comparison_count);
|
hist_mget.Add(get_perf_context()->user_key_comparison_count);
|
||||||
}
|
}
|
||||||
|
@ -459,8 +465,8 @@ void ProfileQueries(bool enabled_time = false) {
|
||||||
<< hist_get_files.ToString() << "\n"
|
<< hist_get_files.ToString() << "\n"
|
||||||
<< " Number of memtables checked: \n"
|
<< " Number of memtables checked: \n"
|
||||||
<< hist_num_memtable_checked.ToString() << "\n"
|
<< hist_num_memtable_checked.ToString() << "\n"
|
||||||
<< " Time to post process: \n" << hist_get_post_process.ToString()
|
<< " Time to post process: \n"
|
||||||
<< "\n";
|
<< hist_get_post_process.ToString() << "\n";
|
||||||
|
|
||||||
std::cout << "ReadOnly MultiGet(): Time to get snapshot: \n"
|
std::cout << "ReadOnly MultiGet(): Time to get snapshot: \n"
|
||||||
<< hist_mget_snapshot.ToString()
|
<< hist_mget_snapshot.ToString()
|
||||||
|
@ -556,7 +562,8 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (FLAGS_verbose) {
|
if (FLAGS_verbose) {
|
||||||
std::cout << "Put time:\n" << hist_put_time.ToString() << "WAL time:\n"
|
std::cout << "Put time:\n"
|
||||||
|
<< hist_put_time.ToString() << "WAL time:\n"
|
||||||
<< hist_wal_time.ToString() << "time diff:\n"
|
<< hist_wal_time.ToString() << "time diff:\n"
|
||||||
<< hist_time_diff.ToString();
|
<< hist_time_diff.ToString();
|
||||||
}
|
}
|
||||||
|
@ -584,7 +591,8 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
|
||||||
}
|
}
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
if (FLAGS_verbose) {
|
if (FLAGS_verbose) {
|
||||||
std::cout << "Seek:\n" << hist_seek.ToString() << "Next:\n"
|
std::cout << "Seek:\n"
|
||||||
|
<< hist_seek.ToString() << "Next:\n"
|
||||||
<< hist_next.ToString();
|
<< hist_next.ToString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -614,7 +622,7 @@ TEST_F(PerfContextTest, DBMutexLockCounter) {
|
||||||
SystemClock::Default()->SleepForMicroseconds(100);
|
SystemClock::Default()->SleepForMicroseconds(100);
|
||||||
mutex.Unlock();
|
mutex.Unlock();
|
||||||
child_thread.join();
|
child_thread.join();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -806,14 +814,18 @@ TEST_F(PerfContextTest, PerfContextByLevelGetSet) {
|
||||||
.bloom_filter_full_positive);
|
.bloom_filter_full_positive);
|
||||||
ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[2]
|
ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[2]
|
||||||
.bloom_filter_full_true_positive);
|
.bloom_filter_full_true_positive);
|
||||||
ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[0]
|
ASSERT_EQ(
|
||||||
.block_cache_hit_count);
|
1,
|
||||||
ASSERT_EQ(5, (*(get_perf_context()->level_to_perf_context))[2]
|
(*(get_perf_context()->level_to_perf_context))[0].block_cache_hit_count);
|
||||||
.block_cache_hit_count);
|
ASSERT_EQ(
|
||||||
ASSERT_EQ(2, (*(get_perf_context()->level_to_perf_context))[3]
|
5,
|
||||||
.block_cache_miss_count);
|
(*(get_perf_context()->level_to_perf_context))[2].block_cache_hit_count);
|
||||||
ASSERT_EQ(4, (*(get_perf_context()->level_to_perf_context))[1]
|
ASSERT_EQ(
|
||||||
.block_cache_miss_count);
|
2,
|
||||||
|
(*(get_perf_context()->level_to_perf_context))[3].block_cache_miss_count);
|
||||||
|
ASSERT_EQ(
|
||||||
|
4,
|
||||||
|
(*(get_perf_context()->level_to_perf_context))[1].block_cache_miss_count);
|
||||||
std::string zero_excluded = get_perf_context()->ToString(true);
|
std::string zero_excluded = get_perf_context()->ToString(true);
|
||||||
ASSERT_NE(std::string::npos,
|
ASSERT_NE(std::string::npos,
|
||||||
zero_excluded.find("bloom_filter_useful = 1@level5, 2@level7"));
|
zero_excluded.find("bloom_filter_useful = 1@level5, 2@level7"));
|
||||||
|
|
|
@ -38,7 +38,6 @@
|
||||||
#include "util/string_util.h"
|
#include "util/string_util.h"
|
||||||
#include "utilities/merge_operators.h"
|
#include "utilities/merge_operators.h"
|
||||||
|
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
class PlainTableKeyDecoderTest : public testing::Test {};
|
class PlainTableKeyDecoderTest : public testing::Test {};
|
||||||
|
|
||||||
|
@ -148,9 +147,7 @@ class PlainTableDBTest : public testing::Test,
|
||||||
|
|
||||||
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
|
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
|
||||||
|
|
||||||
void Reopen(Options* options = nullptr) {
|
void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); }
|
||||||
ASSERT_OK(TryReopen(options));
|
|
||||||
}
|
|
||||||
|
|
||||||
void Close() {
|
void Close() {
|
||||||
delete db_;
|
delete db_;
|
||||||
|
@ -160,7 +157,7 @@ class PlainTableDBTest : public testing::Test,
|
||||||
bool mmap_mode() const { return mmap_mode_; }
|
bool mmap_mode() const { return mmap_mode_; }
|
||||||
|
|
||||||
void DestroyAndReopen(Options* options = nullptr) {
|
void DestroyAndReopen(Options* options = nullptr) {
|
||||||
//Destroy using last options
|
// Destroy using last options
|
||||||
Destroy(&last_options_);
|
Destroy(&last_options_);
|
||||||
ASSERT_OK(TryReopen(options));
|
ASSERT_OK(TryReopen(options));
|
||||||
}
|
}
|
||||||
|
@ -200,9 +197,7 @@ class PlainTableDBTest : public testing::Test,
|
||||||
return db_->Put(WriteOptions(), k, v);
|
return db_->Put(WriteOptions(), k, v);
|
||||||
}
|
}
|
||||||
|
|
||||||
Status Delete(const std::string& k) {
|
Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); }
|
||||||
return db_->Delete(WriteOptions(), k);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
|
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
|
||||||
ReadOptions options;
|
ReadOptions options;
|
||||||
|
@ -217,7 +212,6 @@ class PlainTableDBTest : public testing::Test,
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
int NumTableFilesAtLevel(int level) {
|
int NumTableFilesAtLevel(int level) {
|
||||||
std::string property;
|
std::string property;
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty(
|
||||||
|
@ -448,99 +442,100 @@ TEST_P(PlainTableDBTest, Flush) {
|
||||||
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
|
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
|
||||||
huge_page_tlb_size += 2 * 1024 * 1024) {
|
huge_page_tlb_size += 2 * 1024 * 1024) {
|
||||||
for (EncodingType encoding_type : {kPlain, kPrefix}) {
|
for (EncodingType encoding_type : {kPlain, kPrefix}) {
|
||||||
for (int bloom = -1; bloom <= 117; bloom += 117) {
|
for (int bloom = -1; bloom <= 117; bloom += 117) {
|
||||||
const int bloom_bits = std::max(bloom, 0);
|
const int bloom_bits = std::max(bloom, 0);
|
||||||
const bool full_scan_mode = bloom < 0;
|
const bool full_scan_mode = bloom < 0;
|
||||||
for (int total_order = 0; total_order <= 1; total_order++) {
|
for (int total_order = 0; total_order <= 1; total_order++) {
|
||||||
for (int store_index_in_file = 0; store_index_in_file <= 1;
|
for (int store_index_in_file = 0; store_index_in_file <= 1;
|
||||||
++store_index_in_file) {
|
++store_index_in_file) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
// Set only one bucket to force bucket conflict.
|
// Set only one bucket to force bucket conflict.
|
||||||
// Test index interval for the same prefix to be 1, 2 and 4
|
// Test index interval for the same prefix to be 1, 2 and 4
|
||||||
if (total_order) {
|
if (total_order) {
|
||||||
options.prefix_extractor.reset();
|
options.prefix_extractor.reset();
|
||||||
|
|
||||||
PlainTableOptions plain_table_options;
|
PlainTableOptions plain_table_options;
|
||||||
plain_table_options.user_key_len = 0;
|
plain_table_options.user_key_len = 0;
|
||||||
plain_table_options.bloom_bits_per_key = bloom_bits;
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
plain_table_options.hash_table_ratio = 0;
|
plain_table_options.hash_table_ratio = 0;
|
||||||
plain_table_options.index_sparseness = 2;
|
plain_table_options.index_sparseness = 2;
|
||||||
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
plain_table_options.encoding_type = encoding_type;
|
plain_table_options.encoding_type = encoding_type;
|
||||||
plain_table_options.full_scan_mode = full_scan_mode;
|
plain_table_options.full_scan_mode = full_scan_mode;
|
||||||
plain_table_options.store_index_in_file = store_index_in_file;
|
plain_table_options.store_index_in_file = store_index_in_file;
|
||||||
|
|
||||||
options.table_factory.reset(
|
options.table_factory.reset(
|
||||||
NewPlainTableFactory(plain_table_options));
|
NewPlainTableFactory(plain_table_options));
|
||||||
} else {
|
|
||||||
PlainTableOptions plain_table_options;
|
|
||||||
plain_table_options.user_key_len = 0;
|
|
||||||
plain_table_options.bloom_bits_per_key = bloom_bits;
|
|
||||||
plain_table_options.hash_table_ratio = 0.75;
|
|
||||||
plain_table_options.index_sparseness = 16;
|
|
||||||
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
|
||||||
plain_table_options.encoding_type = encoding_type;
|
|
||||||
plain_table_options.full_scan_mode = full_scan_mode;
|
|
||||||
plain_table_options.store_index_in_file = store_index_in_file;
|
|
||||||
|
|
||||||
options.table_factory.reset(
|
|
||||||
NewPlainTableFactory(plain_table_options));
|
|
||||||
}
|
|
||||||
DestroyAndReopen(&options);
|
|
||||||
uint64_t int_num;
|
|
||||||
ASSERT_TRUE(dbfull()->GetIntProperty(
|
|
||||||
"rocksdb.estimate-table-readers-mem", &int_num));
|
|
||||||
ASSERT_EQ(int_num, 0U);
|
|
||||||
|
|
||||||
ASSERT_OK(Put("1000000000000foo", "v1"));
|
|
||||||
ASSERT_OK(Put("0000000000000bar", "v2"));
|
|
||||||
ASSERT_OK(Put("1000000000000foo", "v3"));
|
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
|
||||||
|
|
||||||
ASSERT_TRUE(dbfull()->GetIntProperty(
|
|
||||||
"rocksdb.estimate-table-readers-mem", &int_num));
|
|
||||||
ASSERT_GT(int_num, 0U);
|
|
||||||
|
|
||||||
TablePropertiesCollection ptc;
|
|
||||||
ASSERT_OK(
|
|
||||||
reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
|
|
||||||
ASSERT_EQ(1U, ptc.size());
|
|
||||||
auto row = ptc.begin();
|
|
||||||
auto tp = row->second;
|
|
||||||
|
|
||||||
if (full_scan_mode) {
|
|
||||||
// Does not support Get/Seek
|
|
||||||
std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ReadOptions()));
|
|
||||||
iter->SeekToFirst();
|
|
||||||
ASSERT_TRUE(iter->Valid());
|
|
||||||
ASSERT_EQ("0000000000000bar", iter->key().ToString());
|
|
||||||
ASSERT_EQ("v2", iter->value().ToString());
|
|
||||||
iter->Next();
|
|
||||||
ASSERT_TRUE(iter->Valid());
|
|
||||||
ASSERT_EQ("1000000000000foo", iter->key().ToString());
|
|
||||||
ASSERT_EQ("v3", iter->value().ToString());
|
|
||||||
iter->Next();
|
|
||||||
ASSERT_TRUE(!iter->Valid());
|
|
||||||
ASSERT_TRUE(iter->status().ok());
|
|
||||||
} else {
|
|
||||||
if (!store_index_in_file) {
|
|
||||||
ASSERT_EQ(total_order ? "4" : "12",
|
|
||||||
(tp->user_collected_properties)
|
|
||||||
.at("plain_table_hash_table_size"));
|
|
||||||
ASSERT_EQ("0", (tp->user_collected_properties)
|
|
||||||
.at("plain_table_sub_index_size"));
|
|
||||||
} else {
|
} else {
|
||||||
ASSERT_EQ("0", (tp->user_collected_properties)
|
PlainTableOptions plain_table_options;
|
||||||
.at("plain_table_hash_table_size"));
|
plain_table_options.user_key_len = 0;
|
||||||
ASSERT_EQ("0", (tp->user_collected_properties)
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
.at("plain_table_sub_index_size"));
|
plain_table_options.hash_table_ratio = 0.75;
|
||||||
|
plain_table_options.index_sparseness = 16;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
plain_table_options.full_scan_mode = full_scan_mode;
|
||||||
|
plain_table_options.store_index_in_file = store_index_in_file;
|
||||||
|
|
||||||
|
options.table_factory.reset(
|
||||||
|
NewPlainTableFactory(plain_table_options));
|
||||||
|
}
|
||||||
|
DestroyAndReopen(&options);
|
||||||
|
uint64_t int_num;
|
||||||
|
ASSERT_TRUE(dbfull()->GetIntProperty(
|
||||||
|
"rocksdb.estimate-table-readers-mem", &int_num));
|
||||||
|
ASSERT_EQ(int_num, 0U);
|
||||||
|
|
||||||
|
ASSERT_OK(Put("1000000000000foo", "v1"));
|
||||||
|
ASSERT_OK(Put("0000000000000bar", "v2"));
|
||||||
|
ASSERT_OK(Put("1000000000000foo", "v3"));
|
||||||
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
|
|
||||||
|
ASSERT_TRUE(dbfull()->GetIntProperty(
|
||||||
|
"rocksdb.estimate-table-readers-mem", &int_num));
|
||||||
|
ASSERT_GT(int_num, 0U);
|
||||||
|
|
||||||
|
TablePropertiesCollection ptc;
|
||||||
|
ASSERT_OK(reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(
|
||||||
|
&ptc));
|
||||||
|
ASSERT_EQ(1U, ptc.size());
|
||||||
|
auto row = ptc.begin();
|
||||||
|
auto tp = row->second;
|
||||||
|
|
||||||
|
if (full_scan_mode) {
|
||||||
|
// Does not support Get/Seek
|
||||||
|
std::unique_ptr<Iterator> iter(
|
||||||
|
dbfull()->NewIterator(ReadOptions()));
|
||||||
|
iter->SeekToFirst();
|
||||||
|
ASSERT_TRUE(iter->Valid());
|
||||||
|
ASSERT_EQ("0000000000000bar", iter->key().ToString());
|
||||||
|
ASSERT_EQ("v2", iter->value().ToString());
|
||||||
|
iter->Next();
|
||||||
|
ASSERT_TRUE(iter->Valid());
|
||||||
|
ASSERT_EQ("1000000000000foo", iter->key().ToString());
|
||||||
|
ASSERT_EQ("v3", iter->value().ToString());
|
||||||
|
iter->Next();
|
||||||
|
ASSERT_TRUE(!iter->Valid());
|
||||||
|
ASSERT_TRUE(iter->status().ok());
|
||||||
|
} else {
|
||||||
|
if (!store_index_in_file) {
|
||||||
|
ASSERT_EQ(total_order ? "4" : "12",
|
||||||
|
(tp->user_collected_properties)
|
||||||
|
.at("plain_table_hash_table_size"));
|
||||||
|
ASSERT_EQ("0", (tp->user_collected_properties)
|
||||||
|
.at("plain_table_sub_index_size"));
|
||||||
|
} else {
|
||||||
|
ASSERT_EQ("0", (tp->user_collected_properties)
|
||||||
|
.at("plain_table_hash_table_size"));
|
||||||
|
ASSERT_EQ("0", (tp->user_collected_properties)
|
||||||
|
.at("plain_table_sub_index_size"));
|
||||||
|
}
|
||||||
|
ASSERT_EQ("v3", Get("1000000000000foo"));
|
||||||
|
ASSERT_EQ("v2", Get("0000000000000bar"));
|
||||||
}
|
}
|
||||||
ASSERT_EQ("v3", Get("1000000000000foo"));
|
|
||||||
ASSERT_EQ("v2", Get("0000000000000bar"));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -550,79 +545,79 @@ TEST_P(PlainTableDBTest, Flush2) {
|
||||||
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
|
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
|
||||||
huge_page_tlb_size += 2 * 1024 * 1024) {
|
huge_page_tlb_size += 2 * 1024 * 1024) {
|
||||||
for (EncodingType encoding_type : {kPlain, kPrefix}) {
|
for (EncodingType encoding_type : {kPlain, kPrefix}) {
|
||||||
for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
|
for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
|
||||||
for (int total_order = 0; total_order <= 1; total_order++) {
|
for (int total_order = 0; total_order <= 1; total_order++) {
|
||||||
for (int store_index_in_file = 0; store_index_in_file <= 1;
|
for (int store_index_in_file = 0; store_index_in_file <= 1;
|
||||||
++store_index_in_file) {
|
++store_index_in_file) {
|
||||||
if (encoding_type == kPrefix && total_order) {
|
if (encoding_type == kPrefix && total_order) {
|
||||||
continue;
|
continue;
|
||||||
|
}
|
||||||
|
if (!bloom_bits && store_index_in_file) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (total_order && store_index_in_file) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
bool expect_bloom_not_match = false;
|
||||||
|
Options options = CurrentOptions();
|
||||||
|
options.create_if_missing = true;
|
||||||
|
// Set only one bucket to force bucket conflict.
|
||||||
|
// Test index interval for the same prefix to be 1, 2 and 4
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
if (total_order) {
|
||||||
|
options.prefix_extractor = nullptr;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
plain_table_options.index_sparseness = 2;
|
||||||
|
} else {
|
||||||
|
plain_table_options.hash_table_ratio = 0.75;
|
||||||
|
plain_table_options.index_sparseness = 16;
|
||||||
|
}
|
||||||
|
plain_table_options.user_key_len = kPlainTableVariableLength;
|
||||||
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
plain_table_options.store_index_in_file = store_index_in_file;
|
||||||
|
options.table_factory.reset(new TestPlainTableFactory(
|
||||||
|
&expect_bloom_not_match, plain_table_options,
|
||||||
|
0 /* column_family_id */, kDefaultColumnFamilyName));
|
||||||
|
|
||||||
|
DestroyAndReopen(&options);
|
||||||
|
ASSERT_OK(Put("0000000000000bar", "b"));
|
||||||
|
ASSERT_OK(Put("1000000000000foo", "v1"));
|
||||||
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
|
|
||||||
|
ASSERT_OK(Put("1000000000000foo", "v2"));
|
||||||
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
|
ASSERT_EQ("v2", Get("1000000000000foo"));
|
||||||
|
|
||||||
|
ASSERT_OK(Put("0000000000000eee", "v3"));
|
||||||
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
|
ASSERT_EQ("v3", Get("0000000000000eee"));
|
||||||
|
|
||||||
|
ASSERT_OK(Delete("0000000000000bar"));
|
||||||
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
|
ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
|
||||||
|
|
||||||
|
ASSERT_OK(Put("0000000000000eee", "v5"));
|
||||||
|
ASSERT_OK(Put("9000000000000eee", "v5"));
|
||||||
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
|
ASSERT_EQ("v5", Get("0000000000000eee"));
|
||||||
|
|
||||||
|
// Test Bloom Filter
|
||||||
|
if (bloom_bits > 0) {
|
||||||
|
// Neither key nor value should exist.
|
||||||
|
expect_bloom_not_match = true;
|
||||||
|
ASSERT_EQ("NOT_FOUND", Get("5_not00000000bar"));
|
||||||
|
// Key doesn't exist any more but prefix exists.
|
||||||
|
if (total_order) {
|
||||||
|
ASSERT_EQ("NOT_FOUND", Get("1000000000000not"));
|
||||||
|
ASSERT_EQ("NOT_FOUND", Get("0000000000000not"));
|
||||||
|
}
|
||||||
|
expect_bloom_not_match = false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (!bloom_bits && store_index_in_file) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (total_order && store_index_in_file) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
bool expect_bloom_not_match = false;
|
|
||||||
Options options = CurrentOptions();
|
|
||||||
options.create_if_missing = true;
|
|
||||||
// Set only one bucket to force bucket conflict.
|
|
||||||
// Test index interval for the same prefix to be 1, 2 and 4
|
|
||||||
PlainTableOptions plain_table_options;
|
|
||||||
if (total_order) {
|
|
||||||
options.prefix_extractor = nullptr;
|
|
||||||
plain_table_options.hash_table_ratio = 0;
|
|
||||||
plain_table_options.index_sparseness = 2;
|
|
||||||
} else {
|
|
||||||
plain_table_options.hash_table_ratio = 0.75;
|
|
||||||
plain_table_options.index_sparseness = 16;
|
|
||||||
}
|
|
||||||
plain_table_options.user_key_len = kPlainTableVariableLength;
|
|
||||||
plain_table_options.bloom_bits_per_key = bloom_bits;
|
|
||||||
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
|
||||||
plain_table_options.encoding_type = encoding_type;
|
|
||||||
plain_table_options.store_index_in_file = store_index_in_file;
|
|
||||||
options.table_factory.reset(new TestPlainTableFactory(
|
|
||||||
&expect_bloom_not_match, plain_table_options,
|
|
||||||
0 /* column_family_id */, kDefaultColumnFamilyName));
|
|
||||||
|
|
||||||
DestroyAndReopen(&options);
|
|
||||||
ASSERT_OK(Put("0000000000000bar", "b"));
|
|
||||||
ASSERT_OK(Put("1000000000000foo", "v1"));
|
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
|
||||||
|
|
||||||
ASSERT_OK(Put("1000000000000foo", "v2"));
|
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
|
||||||
ASSERT_EQ("v2", Get("1000000000000foo"));
|
|
||||||
|
|
||||||
ASSERT_OK(Put("0000000000000eee", "v3"));
|
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
|
||||||
ASSERT_EQ("v3", Get("0000000000000eee"));
|
|
||||||
|
|
||||||
ASSERT_OK(Delete("0000000000000bar"));
|
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
|
||||||
ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
|
|
||||||
|
|
||||||
ASSERT_OK(Put("0000000000000eee", "v5"));
|
|
||||||
ASSERT_OK(Put("9000000000000eee", "v5"));
|
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
|
||||||
ASSERT_EQ("v5", Get("0000000000000eee"));
|
|
||||||
|
|
||||||
// Test Bloom Filter
|
|
||||||
if (bloom_bits > 0) {
|
|
||||||
// Neither key nor value should exist.
|
|
||||||
expect_bloom_not_match = true;
|
|
||||||
ASSERT_EQ("NOT_FOUND", Get("5_not00000000bar"));
|
|
||||||
// Key doesn't exist any more but prefix exists.
|
|
||||||
if (total_order) {
|
|
||||||
ASSERT_EQ("NOT_FOUND", Get("1000000000000not"));
|
|
||||||
ASSERT_EQ("NOT_FOUND", Get("0000000000000not"));
|
|
||||||
}
|
|
||||||
expect_bloom_not_match = false;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -675,129 +670,129 @@ TEST_P(PlainTableDBTest, Iterator) {
|
||||||
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
|
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
|
||||||
huge_page_tlb_size += 2 * 1024 * 1024) {
|
huge_page_tlb_size += 2 * 1024 * 1024) {
|
||||||
for (EncodingType encoding_type : {kPlain, kPrefix}) {
|
for (EncodingType encoding_type : {kPlain, kPrefix}) {
|
||||||
for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
|
for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
|
||||||
for (int total_order = 0; total_order <= 1; total_order++) {
|
for (int total_order = 0; total_order <= 1; total_order++) {
|
||||||
if (encoding_type == kPrefix && total_order == 1) {
|
if (encoding_type == kPrefix && total_order == 1) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
bool expect_bloom_not_match = false;
|
bool expect_bloom_not_match = false;
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
// Set only one bucket to force bucket conflict.
|
// Set only one bucket to force bucket conflict.
|
||||||
// Test index interval for the same prefix to be 1, 2 and 4
|
// Test index interval for the same prefix to be 1, 2 and 4
|
||||||
if (total_order) {
|
if (total_order) {
|
||||||
options.prefix_extractor = nullptr;
|
options.prefix_extractor = nullptr;
|
||||||
|
|
||||||
PlainTableOptions plain_table_options;
|
PlainTableOptions plain_table_options;
|
||||||
plain_table_options.user_key_len = 16;
|
plain_table_options.user_key_len = 16;
|
||||||
plain_table_options.bloom_bits_per_key = bloom_bits;
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
plain_table_options.hash_table_ratio = 0;
|
plain_table_options.hash_table_ratio = 0;
|
||||||
plain_table_options.index_sparseness = 2;
|
plain_table_options.index_sparseness = 2;
|
||||||
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
plain_table_options.encoding_type = encoding_type;
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
|
||||||
options.table_factory.reset(new TestPlainTableFactory(
|
options.table_factory.reset(new TestPlainTableFactory(
|
||||||
&expect_bloom_not_match, plain_table_options,
|
&expect_bloom_not_match, plain_table_options,
|
||||||
0 /* column_family_id */, kDefaultColumnFamilyName));
|
0 /* column_family_id */, kDefaultColumnFamilyName));
|
||||||
} else {
|
} else {
|
||||||
PlainTableOptions plain_table_options;
|
PlainTableOptions plain_table_options;
|
||||||
plain_table_options.user_key_len = 16;
|
plain_table_options.user_key_len = 16;
|
||||||
plain_table_options.bloom_bits_per_key = bloom_bits;
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
plain_table_options.hash_table_ratio = 0.75;
|
plain_table_options.hash_table_ratio = 0.75;
|
||||||
plain_table_options.index_sparseness = 16;
|
plain_table_options.index_sparseness = 16;
|
||||||
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
plain_table_options.encoding_type = encoding_type;
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
|
||||||
options.table_factory.reset(new TestPlainTableFactory(
|
options.table_factory.reset(new TestPlainTableFactory(
|
||||||
&expect_bloom_not_match, plain_table_options,
|
&expect_bloom_not_match, plain_table_options,
|
||||||
0 /* column_family_id */, kDefaultColumnFamilyName));
|
0 /* column_family_id */, kDefaultColumnFamilyName));
|
||||||
}
|
}
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
|
|
||||||
ASSERT_OK(Put("1000000000foo002", "v_2"));
|
ASSERT_OK(Put("1000000000foo002", "v_2"));
|
||||||
ASSERT_OK(Put("0000000000000bar", "random"));
|
ASSERT_OK(Put("0000000000000bar", "random"));
|
||||||
ASSERT_OK(Put("1000000000foo001", "v1"));
|
ASSERT_OK(Put("1000000000foo001", "v1"));
|
||||||
ASSERT_OK(Put("3000000000000bar", "bar_v"));
|
ASSERT_OK(Put("3000000000000bar", "bar_v"));
|
||||||
ASSERT_OK(Put("1000000000foo003", "v__3"));
|
ASSERT_OK(Put("1000000000foo003", "v__3"));
|
||||||
ASSERT_OK(Put("1000000000foo004", "v__4"));
|
ASSERT_OK(Put("1000000000foo004", "v__4"));
|
||||||
ASSERT_OK(Put("1000000000foo005", "v__5"));
|
ASSERT_OK(Put("1000000000foo005", "v__5"));
|
||||||
ASSERT_OK(Put("1000000000foo007", "v__7"));
|
ASSERT_OK(Put("1000000000foo007", "v__7"));
|
||||||
ASSERT_OK(Put("1000000000foo008", "v__8"));
|
ASSERT_OK(Put("1000000000foo008", "v__8"));
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
ASSERT_EQ("v1", Get("1000000000foo001"));
|
ASSERT_EQ("v1", Get("1000000000foo001"));
|
||||||
ASSERT_EQ("v__3", Get("1000000000foo003"));
|
ASSERT_EQ("v__3", Get("1000000000foo003"));
|
||||||
Iterator* iter = dbfull()->NewIterator(ReadOptions());
|
Iterator* iter = dbfull()->NewIterator(ReadOptions());
|
||||||
iter->Seek("1000000000foo000");
|
iter->Seek("1000000000foo000");
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ("1000000000foo001", iter->key().ToString());
|
ASSERT_EQ("1000000000foo001", iter->key().ToString());
|
||||||
ASSERT_EQ("v1", iter->value().ToString());
|
ASSERT_EQ("v1", iter->value().ToString());
|
||||||
|
|
||||||
iter->Next();
|
iter->Next();
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ("1000000000foo002", iter->key().ToString());
|
ASSERT_EQ("1000000000foo002", iter->key().ToString());
|
||||||
ASSERT_EQ("v_2", iter->value().ToString());
|
ASSERT_EQ("v_2", iter->value().ToString());
|
||||||
|
|
||||||
iter->Next();
|
iter->Next();
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ("1000000000foo003", iter->key().ToString());
|
ASSERT_EQ("1000000000foo003", iter->key().ToString());
|
||||||
ASSERT_EQ("v__3", iter->value().ToString());
|
ASSERT_EQ("v__3", iter->value().ToString());
|
||||||
|
|
||||||
iter->Next();
|
iter->Next();
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ("1000000000foo004", iter->key().ToString());
|
ASSERT_EQ("1000000000foo004", iter->key().ToString());
|
||||||
ASSERT_EQ("v__4", iter->value().ToString());
|
ASSERT_EQ("v__4", iter->value().ToString());
|
||||||
|
|
||||||
iter->Seek("3000000000000bar");
|
iter->Seek("3000000000000bar");
|
||||||
ASSERT_TRUE(iter->Valid());
|
|
||||||
ASSERT_EQ("3000000000000bar", iter->key().ToString());
|
|
||||||
ASSERT_EQ("bar_v", iter->value().ToString());
|
|
||||||
|
|
||||||
iter->Seek("1000000000foo000");
|
|
||||||
ASSERT_TRUE(iter->Valid());
|
|
||||||
ASSERT_EQ("1000000000foo001", iter->key().ToString());
|
|
||||||
ASSERT_EQ("v1", iter->value().ToString());
|
|
||||||
|
|
||||||
iter->Seek("1000000000foo005");
|
|
||||||
ASSERT_TRUE(iter->Valid());
|
|
||||||
ASSERT_EQ("1000000000foo005", iter->key().ToString());
|
|
||||||
ASSERT_EQ("v__5", iter->value().ToString());
|
|
||||||
|
|
||||||
iter->Seek("1000000000foo006");
|
|
||||||
ASSERT_TRUE(iter->Valid());
|
|
||||||
ASSERT_EQ("1000000000foo007", iter->key().ToString());
|
|
||||||
ASSERT_EQ("v__7", iter->value().ToString());
|
|
||||||
|
|
||||||
iter->Seek("1000000000foo008");
|
|
||||||
ASSERT_TRUE(iter->Valid());
|
|
||||||
ASSERT_EQ("1000000000foo008", iter->key().ToString());
|
|
||||||
ASSERT_EQ("v__8", iter->value().ToString());
|
|
||||||
|
|
||||||
if (total_order == 0) {
|
|
||||||
iter->Seek("1000000000foo009");
|
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ("3000000000000bar", iter->key().ToString());
|
ASSERT_EQ("3000000000000bar", iter->key().ToString());
|
||||||
}
|
ASSERT_EQ("bar_v", iter->value().ToString());
|
||||||
|
|
||||||
// Test Bloom Filter
|
iter->Seek("1000000000foo000");
|
||||||
if (bloom_bits > 0) {
|
ASSERT_TRUE(iter->Valid());
|
||||||
if (!total_order) {
|
ASSERT_EQ("1000000000foo001", iter->key().ToString());
|
||||||
// Neither key nor value should exist.
|
ASSERT_EQ("v1", iter->value().ToString());
|
||||||
expect_bloom_not_match = true;
|
|
||||||
iter->Seek("2not000000000bar");
|
iter->Seek("1000000000foo005");
|
||||||
ASSERT_TRUE(!iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
|
ASSERT_EQ("1000000000foo005", iter->key().ToString());
|
||||||
expect_bloom_not_match = false;
|
ASSERT_EQ("v__5", iter->value().ToString());
|
||||||
} else {
|
|
||||||
expect_bloom_not_match = true;
|
iter->Seek("1000000000foo006");
|
||||||
ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
|
ASSERT_TRUE(iter->Valid());
|
||||||
expect_bloom_not_match = false;
|
ASSERT_EQ("1000000000foo007", iter->key().ToString());
|
||||||
|
ASSERT_EQ("v__7", iter->value().ToString());
|
||||||
|
|
||||||
|
iter->Seek("1000000000foo008");
|
||||||
|
ASSERT_TRUE(iter->Valid());
|
||||||
|
ASSERT_EQ("1000000000foo008", iter->key().ToString());
|
||||||
|
ASSERT_EQ("v__8", iter->value().ToString());
|
||||||
|
|
||||||
|
if (total_order == 0) {
|
||||||
|
iter->Seek("1000000000foo009");
|
||||||
|
ASSERT_TRUE(iter->Valid());
|
||||||
|
ASSERT_EQ("3000000000000bar", iter->key().ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Test Bloom Filter
|
||||||
|
if (bloom_bits > 0) {
|
||||||
|
if (!total_order) {
|
||||||
|
// Neither key nor value should exist.
|
||||||
|
expect_bloom_not_match = true;
|
||||||
|
iter->Seek("2not000000000bar");
|
||||||
|
ASSERT_TRUE(!iter->Valid());
|
||||||
|
ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
|
||||||
|
expect_bloom_not_match = false;
|
||||||
|
} else {
|
||||||
|
expect_bloom_not_match = true;
|
||||||
|
ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
|
||||||
|
expect_bloom_not_match = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ASSERT_OK(iter->status());
|
||||||
|
delete iter;
|
||||||
}
|
}
|
||||||
ASSERT_OK(iter->status());
|
|
||||||
delete iter;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -863,7 +858,7 @@ namespace {
|
||||||
std::string MakeLongKey(size_t length, char c) {
|
std::string MakeLongKey(size_t length, char c) {
|
||||||
return std::string(length, c);
|
return std::string(length, c);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(PlainTableDBTest, IteratorLargeKeys) {
|
TEST_P(PlainTableDBTest, IteratorLargeKeys) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
|
@ -878,15 +873,10 @@ TEST_P(PlainTableDBTest, IteratorLargeKeys) {
|
||||||
options.prefix_extractor.reset();
|
options.prefix_extractor.reset();
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
|
|
||||||
std::string key_list[] = {
|
std::string key_list[] = {MakeLongKey(30, '0'), MakeLongKey(16, '1'),
|
||||||
MakeLongKey(30, '0'),
|
MakeLongKey(32, '2'), MakeLongKey(60, '3'),
|
||||||
MakeLongKey(16, '1'),
|
MakeLongKey(90, '4'), MakeLongKey(50, '5'),
|
||||||
MakeLongKey(32, '2'),
|
MakeLongKey(26, '6')};
|
||||||
MakeLongKey(60, '3'),
|
|
||||||
MakeLongKey(90, '4'),
|
|
||||||
MakeLongKey(50, '5'),
|
|
||||||
MakeLongKey(26, '6')
|
|
||||||
};
|
|
||||||
|
|
||||||
for (size_t i = 0; i < 7; i++) {
|
for (size_t i = 0; i < 7; i++) {
|
||||||
ASSERT_OK(Put(key_list[i], std::to_string(i)));
|
ASSERT_OK(Put(key_list[i], std::to_string(i)));
|
||||||
|
@ -913,7 +903,7 @@ namespace {
|
||||||
std::string MakeLongKeyWithPrefix(size_t length, char c) {
|
std::string MakeLongKeyWithPrefix(size_t length, char c) {
|
||||||
return "00000000" + std::string(length - 8, c);
|
return "00000000" + std::string(length - 8, c);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
|
TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
|
@ -1275,7 +1265,7 @@ TEST_P(PlainTableDBTest, CompactionTrigger) {
|
||||||
Random rnd(301);
|
Random rnd(301);
|
||||||
|
|
||||||
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
|
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
|
||||||
num++) {
|
num++) {
|
||||||
std::vector<std::string> values;
|
std::vector<std::string> values;
|
||||||
// Write 120KB (10 values, each 12K)
|
// Write 120KB (10 values, each 12K)
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
|
@ -1287,7 +1277,7 @@ TEST_P(PlainTableDBTest, CompactionTrigger) {
|
||||||
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
|
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
//generate one more file in level-0, and should trigger level-0 compaction
|
// generate one more file in level-0, and should trigger level-0 compaction
|
||||||
std::vector<std::string> values;
|
std::vector<std::string> values;
|
||||||
for (int i = 0; i < 12; i++) {
|
for (int i = 0; i < 12; i++) {
|
||||||
values.push_back(rnd.RandomString(10000));
|
values.push_back(rnd.RandomString(10000));
|
||||||
|
@ -1315,8 +1305,7 @@ TEST_P(PlainTableDBTest, AdaptiveTable) {
|
||||||
options.create_if_missing = false;
|
options.create_if_missing = false;
|
||||||
std::shared_ptr<TableFactory> block_based_factory(
|
std::shared_ptr<TableFactory> block_based_factory(
|
||||||
NewBlockBasedTableFactory());
|
NewBlockBasedTableFactory());
|
||||||
std::shared_ptr<TableFactory> plain_table_factory(
|
std::shared_ptr<TableFactory> plain_table_factory(NewPlainTableFactory());
|
||||||
NewPlainTableFactory());
|
|
||||||
std::shared_ptr<TableFactory> dummy_factory;
|
std::shared_ptr<TableFactory> dummy_factory;
|
||||||
options.table_factory.reset(NewAdaptiveTableFactory(
|
options.table_factory.reset(NewAdaptiveTableFactory(
|
||||||
block_based_factory, block_based_factory, plain_table_factory));
|
block_based_factory, block_based_factory, plain_table_factory));
|
||||||
|
|
|
@ -69,7 +69,7 @@ struct TestKey {
|
||||||
};
|
};
|
||||||
|
|
||||||
// return a slice backed by test_key
|
// return a slice backed by test_key
|
||||||
inline Slice TestKeyToSlice(std::string &s, const TestKey& test_key) {
|
inline Slice TestKeyToSlice(std::string& s, const TestKey& test_key) {
|
||||||
s.clear();
|
s.clear();
|
||||||
PutFixed64(&s, test_key.prefix);
|
PutFixed64(&s, test_key.prefix);
|
||||||
PutFixed64(&s, test_key.sorted);
|
PutFixed64(&s, test_key.sorted);
|
||||||
|
@ -77,20 +77,18 @@ inline Slice TestKeyToSlice(std::string &s, const TestKey& test_key) {
|
||||||
}
|
}
|
||||||
|
|
||||||
inline const TestKey SliceToTestKey(const Slice& slice) {
|
inline const TestKey SliceToTestKey(const Slice& slice) {
|
||||||
return TestKey(DecodeFixed64(slice.data()),
|
return TestKey(DecodeFixed64(slice.data()), DecodeFixed64(slice.data() + 8));
|
||||||
DecodeFixed64(slice.data() + 8));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
class TestKeyComparator : public Comparator {
|
class TestKeyComparator : public Comparator {
|
||||||
public:
|
public:
|
||||||
|
|
||||||
// Compare needs to be aware of the possibility of a and/or b is
|
// Compare needs to be aware of the possibility of a and/or b is
|
||||||
// prefix only
|
// prefix only
|
||||||
int Compare(const Slice& a, const Slice& b) const override {
|
int Compare(const Slice& a, const Slice& b) const override {
|
||||||
const TestKey kkey_a = SliceToTestKey(a);
|
const TestKey kkey_a = SliceToTestKey(a);
|
||||||
const TestKey kkey_b = SliceToTestKey(b);
|
const TestKey kkey_b = SliceToTestKey(b);
|
||||||
const TestKey *key_a = &kkey_a;
|
const TestKey* key_a = &kkey_a;
|
||||||
const TestKey *key_b = &kkey_b;
|
const TestKey* key_b = &kkey_b;
|
||||||
if (key_a->prefix != key_b->prefix) {
|
if (key_a->prefix != key_b->prefix) {
|
||||||
if (key_a->prefix < key_b->prefix) return -1;
|
if (key_a->prefix < key_b->prefix) return -1;
|
||||||
if (key_a->prefix > key_b->prefix) return 1;
|
if (key_a->prefix > key_b->prefix) return 1;
|
||||||
|
@ -215,7 +213,7 @@ class SamePrefixTransform : public SliceTransform {
|
||||||
bool FullLengthEnabled(size_t* /*len*/) const override { return false; }
|
bool FullLengthEnabled(size_t* /*len*/) const override { return false; }
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
class PrefixTest : public testing::Test {
|
class PrefixTest : public testing::Test {
|
||||||
public:
|
public:
|
||||||
|
@ -226,7 +224,7 @@ class PrefixTest : public testing::Test {
|
||||||
options.write_buffer_size = FLAGS_write_buffer_size;
|
options.write_buffer_size = FLAGS_write_buffer_size;
|
||||||
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
|
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
|
||||||
options.min_write_buffer_number_to_merge =
|
options.min_write_buffer_number_to_merge =
|
||||||
FLAGS_min_write_buffer_number_to_merge;
|
FLAGS_min_write_buffer_number_to_merge;
|
||||||
|
|
||||||
options.memtable_prefix_bloom_size_ratio =
|
options.memtable_prefix_bloom_size_ratio =
|
||||||
FLAGS_memtable_prefix_bloom_size_ratio;
|
FLAGS_memtable_prefix_bloom_size_ratio;
|
||||||
|
@ -239,21 +237,19 @@ class PrefixTest : public testing::Test {
|
||||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||||
options.allow_concurrent_memtable_write = false;
|
options.allow_concurrent_memtable_write = false;
|
||||||
|
|
||||||
Status s = DB::Open(options, kDbName, &db);
|
Status s = DB::Open(options, kDbName, &db);
|
||||||
EXPECT_OK(s);
|
EXPECT_OK(s);
|
||||||
return std::shared_ptr<DB>(db);
|
return std::shared_ptr<DB>(db);
|
||||||
}
|
}
|
||||||
|
|
||||||
void FirstOption() {
|
void FirstOption() { option_config_ = kBegin; }
|
||||||
option_config_ = kBegin;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool NextOptions(int bucket_count) {
|
bool NextOptions(int bucket_count) {
|
||||||
// skip some options
|
// skip some options
|
||||||
option_config_++;
|
option_config_++;
|
||||||
if (option_config_ < kEnd) {
|
if (option_config_ < kEnd) {
|
||||||
options.prefix_extractor.reset(NewFixedPrefixTransform(8));
|
options.prefix_extractor.reset(NewFixedPrefixTransform(8));
|
||||||
switch(option_config_) {
|
switch (option_config_) {
|
||||||
case kHashSkipList:
|
case kHashSkipList:
|
||||||
options.memtable_factory.reset(
|
options.memtable_factory.reset(
|
||||||
NewHashSkipListRepFactory(bucket_count, FLAGS_skiplist_height));
|
NewHashSkipListRepFactory(bucket_count, FLAGS_skiplist_height));
|
||||||
|
@ -350,8 +346,7 @@ TEST_F(PrefixTest, TestResult) {
|
||||||
FirstOption();
|
FirstOption();
|
||||||
while (NextOptions(num_buckets)) {
|
while (NextOptions(num_buckets)) {
|
||||||
std::cout << "*** Mem table: " << options.memtable_factory->Name()
|
std::cout << "*** Mem table: " << options.memtable_factory->Name()
|
||||||
<< " number of buckets: " << num_buckets
|
<< " number of buckets: " << num_buckets << std::endl;
|
||||||
<< std::endl;
|
|
||||||
ASSERT_OK(DestroyDB(kDbName, Options()));
|
ASSERT_OK(DestroyDB(kDbName, Options()));
|
||||||
auto db = OpenDb();
|
auto db = OpenDb();
|
||||||
WriteOptions write_options;
|
WriteOptions write_options;
|
||||||
|
@ -581,7 +576,7 @@ TEST_F(PrefixTest, PrefixValid) {
|
||||||
TEST_F(PrefixTest, DynamicPrefixIterator) {
|
TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
while (NextOptions(FLAGS_bucket_count)) {
|
while (NextOptions(FLAGS_bucket_count)) {
|
||||||
std::cout << "*** Mem table: " << options.memtable_factory->Name()
|
std::cout << "*** Mem table: " << options.memtable_factory->Name()
|
||||||
<< std::endl;
|
<< std::endl;
|
||||||
ASSERT_OK(DestroyDB(kDbName, Options()));
|
ASSERT_OK(DestroyDB(kDbName, Options()));
|
||||||
auto db = OpenDb();
|
auto db = OpenDb();
|
||||||
WriteOptions write_options;
|
WriteOptions write_options;
|
||||||
|
@ -600,7 +595,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
HistogramImpl hist_put_comparison;
|
HistogramImpl hist_put_comparison;
|
||||||
// insert x random prefix, each with y continuous element.
|
// insert x random prefix, each with y continuous element.
|
||||||
for (auto prefix : prefixes) {
|
for (auto prefix : prefixes) {
|
||||||
for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
|
for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
|
||||||
TestKey test_key(prefix, sorted);
|
TestKey test_key(prefix, sorted);
|
||||||
|
|
||||||
std::string s;
|
std::string s;
|
||||||
|
@ -615,8 +610,9 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::cout << "Put key comparison: \n" << hist_put_comparison.ToString()
|
std::cout << "Put key comparison: \n"
|
||||||
<< "Put time: \n" << hist_put_time.ToString();
|
<< hist_put_comparison.ToString() << "Put time: \n"
|
||||||
|
<< hist_put_time.ToString();
|
||||||
|
|
||||||
// test seek existing keys
|
// test seek existing keys
|
||||||
HistogramImpl hist_seek_time;
|
HistogramImpl hist_seek_time;
|
||||||
|
@ -635,8 +631,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
auto key_prefix = options.prefix_extractor->Transform(key);
|
auto key_prefix = options.prefix_extractor->Transform(key);
|
||||||
uint64_t total_keys = 0;
|
uint64_t total_keys = 0;
|
||||||
for (iter->Seek(key);
|
for (iter->Seek(key);
|
||||||
iter->Valid() && iter->key().starts_with(key_prefix);
|
iter->Valid() && iter->key().starts_with(key_prefix); iter->Next()) {
|
||||||
iter->Next()) {
|
|
||||||
if (FLAGS_trigger_deadlock) {
|
if (FLAGS_trigger_deadlock) {
|
||||||
std::cout << "Behold the deadlock!\n";
|
std::cout << "Behold the deadlock!\n";
|
||||||
db->Delete(write_options, iter->key());
|
db->Delete(write_options, iter->key());
|
||||||
|
@ -645,12 +640,12 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
}
|
}
|
||||||
hist_seek_time.Add(timer.ElapsedNanos());
|
hist_seek_time.Add(timer.ElapsedNanos());
|
||||||
hist_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
|
hist_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
|
||||||
ASSERT_EQ(total_keys, FLAGS_items_per_prefix - FLAGS_items_per_prefix/2);
|
ASSERT_EQ(total_keys,
|
||||||
|
FLAGS_items_per_prefix - FLAGS_items_per_prefix / 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::cout << "Seek key comparison: \n"
|
std::cout << "Seek key comparison: \n"
|
||||||
<< hist_seek_comparison.ToString()
|
<< hist_seek_comparison.ToString() << "Seek time: \n"
|
||||||
<< "Seek time: \n"
|
|
||||||
<< hist_seek_time.ToString();
|
<< hist_seek_time.ToString();
|
||||||
|
|
||||||
// test non-existing keys
|
// test non-existing keys
|
||||||
|
@ -658,8 +653,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
HistogramImpl hist_no_seek_comparison;
|
HistogramImpl hist_no_seek_comparison;
|
||||||
|
|
||||||
for (auto prefix = FLAGS_total_prefixes;
|
for (auto prefix = FLAGS_total_prefixes;
|
||||||
prefix < FLAGS_total_prefixes + 10000;
|
prefix < FLAGS_total_prefixes + 10000; prefix++) {
|
||||||
prefix++) {
|
|
||||||
TestKey test_key(prefix, 0);
|
TestKey test_key(prefix, 0);
|
||||||
std::string s;
|
std::string s;
|
||||||
Slice key = TestKeyToSlice(s, test_key);
|
Slice key = TestKeyToSlice(s, test_key);
|
||||||
|
@ -668,7 +662,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
||||||
StopWatchNano timer(SystemClock::Default().get(), true);
|
StopWatchNano timer(SystemClock::Default().get(), true);
|
||||||
iter->Seek(key);
|
iter->Seek(key);
|
||||||
hist_no_seek_time.Add(timer.ElapsedNanos());
|
hist_no_seek_time.Add(timer.ElapsedNanos());
|
||||||
hist_no_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
|
hist_no_seek_comparison.Add(
|
||||||
|
get_perf_context()->user_key_comparison_count);
|
||||||
ASSERT_TRUE(!iter->Valid());
|
ASSERT_TRUE(!iter->Valid());
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
}
|
}
|
||||||
|
|
|
@ -502,7 +502,7 @@ class TruncatedRangeDelMergingIter : public InternalIterator {
|
||||||
size_t ts_sz_;
|
size_t ts_sz_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator>
|
std::unique_ptr<FragmentedRangeTombstoneIterator>
|
||||||
CompactionRangeDelAggregator::NewIterator(const Slice* lower_bound,
|
CompactionRangeDelAggregator::NewIterator(const Slice* lower_bound,
|
||||||
|
|
|
@ -192,7 +192,7 @@ void VerifyFragmentedRangeDels(
|
||||||
EXPECT_FALSE(iter->Valid());
|
EXPECT_FALSE(iter->Valid());
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorTest, EmptyTruncatedIter) {
|
TEST_F(RangeDelAggregatorTest, EmptyTruncatedIter) {
|
||||||
auto range_del_iter = MakeRangeDelIter({});
|
auto range_del_iter = MakeRangeDelIter({});
|
||||||
|
|
15
db/repair.cc
15
db/repair.cc
|
@ -281,7 +281,7 @@ class Repairer {
|
||||||
std::vector<std::string> to_search_paths;
|
std::vector<std::string> to_search_paths;
|
||||||
|
|
||||||
for (size_t path_id = 0; path_id < db_options_.db_paths.size(); path_id++) {
|
for (size_t path_id = 0; path_id < db_options_.db_paths.size(); path_id++) {
|
||||||
to_search_paths.push_back(db_options_.db_paths[path_id].path);
|
to_search_paths.push_back(db_options_.db_paths[path_id].path);
|
||||||
}
|
}
|
||||||
|
|
||||||
// search wal_dir if user uses a customize wal_dir
|
// search wal_dir if user uses a customize wal_dir
|
||||||
|
@ -332,7 +332,8 @@ class Repairer {
|
||||||
void ConvertLogFilesToTables() {
|
void ConvertLogFilesToTables() {
|
||||||
const auto& wal_dir = immutable_db_options_.GetWalDir();
|
const auto& wal_dir = immutable_db_options_.GetWalDir();
|
||||||
for (size_t i = 0; i < logs_.size(); i++) {
|
for (size_t i = 0; i < logs_.size(); i++) {
|
||||||
// we should use LogFileName(wal_dir, logs_[i]) here. user might uses wal_dir option.
|
// we should use LogFileName(wal_dir, logs_[i]) here. user might uses
|
||||||
|
// wal_dir option.
|
||||||
std::string logname = LogFileName(wal_dir, logs_[i]);
|
std::string logname = LogFileName(wal_dir, logs_[i]);
|
||||||
Status status = ConvertLogToTable(wal_dir, logs_[i]);
|
Status status = ConvertLogToTable(wal_dir, logs_[i]);
|
||||||
if (!status.ok()) {
|
if (!status.ok()) {
|
||||||
|
@ -393,8 +394,8 @@ class Repairer {
|
||||||
int counter = 0;
|
int counter = 0;
|
||||||
while (reader.ReadRecord(&record, &scratch)) {
|
while (reader.ReadRecord(&record, &scratch)) {
|
||||||
if (record.size() < WriteBatchInternal::kHeader) {
|
if (record.size() < WriteBatchInternal::kHeader) {
|
||||||
reporter.Corruption(
|
reporter.Corruption(record.size(),
|
||||||
record.size(), Status::Corruption("log record too small"));
|
Status::Corruption("log record too small"));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
Status record_status = WriteBatchInternal::SetContents(&batch, record);
|
Status record_status = WriteBatchInternal::SetContents(&batch, record);
|
||||||
|
@ -715,8 +716,7 @@ Status GetDefaultCFOptions(
|
||||||
} // anonymous namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
Status RepairDB(const std::string& dbname, const DBOptions& db_options,
|
Status RepairDB(const std::string& dbname, const DBOptions& db_options,
|
||||||
const std::vector<ColumnFamilyDescriptor>& column_families
|
const std::vector<ColumnFamilyDescriptor>& column_families) {
|
||||||
) {
|
|
||||||
ColumnFamilyOptions default_cf_opts;
|
ColumnFamilyOptions default_cf_opts;
|
||||||
Status status = GetDefaultCFOptions(column_families, &default_cf_opts);
|
Status status = GetDefaultCFOptions(column_families, &default_cf_opts);
|
||||||
if (!status.ok()) {
|
if (!status.ok()) {
|
||||||
|
@ -756,8 +756,7 @@ Status RepairDB(const std::string& dbname, const Options& options) {
|
||||||
DBOptions db_options(opts);
|
DBOptions db_options(opts);
|
||||||
ColumnFamilyOptions cf_options(opts);
|
ColumnFamilyOptions cf_options(opts);
|
||||||
|
|
||||||
Repairer repairer(dbname, db_options,
|
Repairer repairer(dbname, db_options, {}, cf_options /* default_cf_opts */,
|
||||||
{}, cf_options /* default_cf_opts */,
|
|
||||||
cf_options /* unknown_cf_opts */,
|
cf_options /* unknown_cf_opts */,
|
||||||
true /* create_unknown_cfs */);
|
true /* create_unknown_cfs */);
|
||||||
Status status = repairer.Run();
|
Status status = repairer.Run();
|
||||||
|
|
|
@ -279,7 +279,7 @@ TEST_F(RepairTest, SeparateWalDir) {
|
||||||
ASSERT_EQ(total_ssts_size, 0);
|
ASSERT_EQ(total_ssts_size, 0);
|
||||||
}
|
}
|
||||||
std::string manifest_path =
|
std::string manifest_path =
|
||||||
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
||||||
|
|
||||||
Close();
|
Close();
|
||||||
ASSERT_OK(env_->FileExists(manifest_path));
|
ASSERT_OK(env_->FileExists(manifest_path));
|
||||||
|
@ -301,7 +301,7 @@ TEST_F(RepairTest, SeparateWalDir) {
|
||||||
ASSERT_EQ(Get("key"), "val");
|
ASSERT_EQ(Get("key"), "val");
|
||||||
ASSERT_EQ(Get("foo"), "bar");
|
ASSERT_EQ(Get("foo"), "bar");
|
||||||
|
|
||||||
} while(ChangeWalOptions());
|
} while (ChangeWalOptions());
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(RepairTest, RepairMultipleColumnFamilies) {
|
TEST_F(RepairTest, RepairMultipleColumnFamilies) {
|
||||||
|
@ -387,8 +387,7 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
|
||||||
ASSERT_EQ(fname_to_props.size(), 2U);
|
ASSERT_EQ(fname_to_props.size(), 2U);
|
||||||
for (const auto& fname_and_props : fname_to_props) {
|
for (const auto& fname_and_props : fname_to_props) {
|
||||||
std::string comparator_name(rev_opts.comparator->Name());
|
std::string comparator_name(rev_opts.comparator->Name());
|
||||||
ASSERT_EQ(comparator_name,
|
ASSERT_EQ(comparator_name, fname_and_props.second->comparator_name);
|
||||||
fname_and_props.second->comparator_name);
|
|
||||||
}
|
}
|
||||||
Close();
|
Close();
|
||||||
|
|
||||||
|
|
|
@ -3,14 +3,13 @@
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
#include "rocksdb/snapshot.h"
|
|
||||||
|
|
||||||
#include "rocksdb/db.h"
|
#include "rocksdb/db.h"
|
||||||
|
#include "rocksdb/snapshot.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
ManagedSnapshot::ManagedSnapshot(DB* db) : db_(db),
|
ManagedSnapshot::ManagedSnapshot(DB* db)
|
||||||
snapshot_(db->GetSnapshot()) {}
|
: db_(db), snapshot_(db->GetSnapshot()) {}
|
||||||
|
|
||||||
ManagedSnapshot::ManagedSnapshot(DB* db, const Snapshot* _snapshot)
|
ManagedSnapshot::ManagedSnapshot(DB* db, const Snapshot* _snapshot)
|
||||||
: db_(db), snapshot_(_snapshot) {}
|
: db_(db), snapshot_(_snapshot) {}
|
||||||
|
@ -21,6 +20,6 @@ ManagedSnapshot::~ManagedSnapshot() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const Snapshot* ManagedSnapshot::snapshot() { return snapshot_;}
|
const Snapshot* ManagedSnapshot::snapshot() { return snapshot_; }
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
|
@ -41,7 +41,7 @@ class SnapshotImpl : public Snapshot {
|
||||||
SnapshotImpl* prev_;
|
SnapshotImpl* prev_;
|
||||||
SnapshotImpl* next_;
|
SnapshotImpl* next_;
|
||||||
|
|
||||||
SnapshotList* list_; // just for sanity checks
|
SnapshotList* list_; // just for sanity checks
|
||||||
|
|
||||||
int64_t unix_time_;
|
int64_t unix_time_;
|
||||||
|
|
||||||
|
@ -56,7 +56,7 @@ class SnapshotList {
|
||||||
SnapshotList() {
|
SnapshotList() {
|
||||||
list_.prev_ = &list_;
|
list_.prev_ = &list_;
|
||||||
list_.next_ = &list_;
|
list_.next_ = &list_;
|
||||||
list_.number_ = 0xFFFFFFFFL; // placeholder marker, for debugging
|
list_.number_ = 0xFFFFFFFFL; // placeholder marker, for debugging
|
||||||
// Set all the variables to make UBSAN happy.
|
// Set all the variables to make UBSAN happy.
|
||||||
list_.list_ = nullptr;
|
list_.list_ = nullptr;
|
||||||
list_.unix_time_ = 0;
|
list_.unix_time_ = 0;
|
||||||
|
@ -72,8 +72,14 @@ class SnapshotList {
|
||||||
assert(list_.next_ != &list_ || 0 == count_);
|
assert(list_.next_ != &list_ || 0 == count_);
|
||||||
return list_.next_ == &list_;
|
return list_.next_ == &list_;
|
||||||
}
|
}
|
||||||
SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; }
|
SnapshotImpl* oldest() const {
|
||||||
SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; }
|
assert(!empty());
|
||||||
|
return list_.next_;
|
||||||
|
}
|
||||||
|
SnapshotImpl* newest() const {
|
||||||
|
assert(!empty());
|
||||||
|
return list_.prev_;
|
||||||
|
}
|
||||||
|
|
||||||
SnapshotImpl* New(SnapshotImpl* s, SequenceNumber seq, uint64_t unix_time,
|
SnapshotImpl* New(SnapshotImpl* s, SequenceNumber seq, uint64_t unix_time,
|
||||||
bool is_write_conflict_boundary,
|
bool is_write_conflict_boundary,
|
||||||
|
|
|
@ -38,7 +38,7 @@ static void DeleteEntry(const Slice& /*key*/, void* value) {
|
||||||
T* typed_value = reinterpret_cast<T*>(value);
|
T* typed_value = reinterpret_cast<T*>(value);
|
||||||
delete typed_value;
|
delete typed_value;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
// Generate the regular and coroutine versions of some methods by
|
// Generate the regular and coroutine versions of some methods by
|
||||||
|
@ -79,7 +79,7 @@ void AppendVarint64(IterKey* key, uint64_t v) {
|
||||||
|
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
const int kLoadConcurency = 128;
|
const int kLoadConcurency = 128;
|
||||||
|
|
||||||
|
@ -103,8 +103,7 @@ TableCache::TableCache(const ImmutableOptions& ioptions,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
TableCache::~TableCache() {
|
TableCache::~TableCache() {}
|
||||||
}
|
|
||||||
|
|
||||||
TableReader* TableCache::GetTableReaderFromHandle(Cache::Handle* handle) {
|
TableReader* TableCache::GetTableReaderFromHandle(Cache::Handle* handle) {
|
||||||
return reinterpret_cast<TableReader*>(cache_->Value(handle));
|
return reinterpret_cast<TableReader*>(cache_->Value(handle));
|
||||||
|
|
|
@ -27,7 +27,7 @@ uint64_t GetUint64Property(const UserCollectedProperties& props,
|
||||||
return GetVarint64(&raw, &val) ? val : 0;
|
return GetVarint64(&raw, &val) ? val : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key,
|
Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key,
|
||||||
const Slice& value,
|
const Slice& value,
|
||||||
|
@ -54,13 +54,12 @@ Status UserKeyTablePropertiesCollector::Finish(
|
||||||
return collector_->Finish(properties);
|
return collector_->Finish(properties);
|
||||||
}
|
}
|
||||||
|
|
||||||
UserCollectedProperties
|
UserCollectedProperties UserKeyTablePropertiesCollector::GetReadableProperties()
|
||||||
UserKeyTablePropertiesCollector::GetReadableProperties() const {
|
const {
|
||||||
return collector_->GetReadableProperties();
|
return collector_->GetReadableProperties();
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t GetDeletedKeys(
|
uint64_t GetDeletedKeys(const UserCollectedProperties& props) {
|
||||||
const UserCollectedProperties& props) {
|
|
||||||
bool property_present_ignored;
|
bool property_present_ignored;
|
||||||
return GetUint64Property(props, TablePropertiesNames::kDeletedKeys,
|
return GetUint64Property(props, TablePropertiesNames::kDeletedKeys,
|
||||||
&property_present_ignored);
|
&property_present_ignored);
|
||||||
|
@ -68,8 +67,8 @@ uint64_t GetDeletedKeys(
|
||||||
|
|
||||||
uint64_t GetMergeOperands(const UserCollectedProperties& props,
|
uint64_t GetMergeOperands(const UserCollectedProperties& props,
|
||||||
bool* property_present) {
|
bool* property_present) {
|
||||||
return GetUint64Property(
|
return GetUint64Property(props, TablePropertiesNames::kMergeOperands,
|
||||||
props, TablePropertiesNames::kMergeOperands, property_present);
|
property_present);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
|
@ -61,30 +61,30 @@ void MakeBuilder(
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
// Collects keys that starts with "A" in a table.
|
// Collects keys that starts with "A" in a table.
|
||||||
class RegularKeysStartWithA: public TablePropertiesCollector {
|
class RegularKeysStartWithA : public TablePropertiesCollector {
|
||||||
public:
|
public:
|
||||||
const char* Name() const override { return "RegularKeysStartWithA"; }
|
const char* Name() const override { return "RegularKeysStartWithA"; }
|
||||||
|
|
||||||
Status Finish(UserCollectedProperties* properties) override {
|
Status Finish(UserCollectedProperties* properties) override {
|
||||||
std::string encoded;
|
std::string encoded;
|
||||||
std::string encoded_num_puts;
|
std::string encoded_num_puts;
|
||||||
std::string encoded_num_deletes;
|
std::string encoded_num_deletes;
|
||||||
std::string encoded_num_single_deletes;
|
std::string encoded_num_single_deletes;
|
||||||
std::string encoded_num_size_changes;
|
std::string encoded_num_size_changes;
|
||||||
PutVarint32(&encoded, count_);
|
PutVarint32(&encoded, count_);
|
||||||
PutVarint32(&encoded_num_puts, num_puts_);
|
PutVarint32(&encoded_num_puts, num_puts_);
|
||||||
PutVarint32(&encoded_num_deletes, num_deletes_);
|
PutVarint32(&encoded_num_deletes, num_deletes_);
|
||||||
PutVarint32(&encoded_num_single_deletes, num_single_deletes_);
|
PutVarint32(&encoded_num_single_deletes, num_single_deletes_);
|
||||||
PutVarint32(&encoded_num_size_changes, num_size_changes_);
|
PutVarint32(&encoded_num_size_changes, num_size_changes_);
|
||||||
*properties = UserCollectedProperties{
|
*properties = UserCollectedProperties{
|
||||||
{"TablePropertiesTest", message_},
|
{"TablePropertiesTest", message_},
|
||||||
{"Count", encoded},
|
{"Count", encoded},
|
||||||
{"NumPuts", encoded_num_puts},
|
{"NumPuts", encoded_num_puts},
|
||||||
{"NumDeletes", encoded_num_deletes},
|
{"NumDeletes", encoded_num_deletes},
|
||||||
{"NumSingleDeletes", encoded_num_single_deletes},
|
{"NumSingleDeletes", encoded_num_single_deletes},
|
||||||
{"NumSizeChanges", encoded_num_size_changes},
|
{"NumSizeChanges", encoded_num_size_changes},
|
||||||
};
|
};
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
|
|
||||||
Status AddUserKey(const Slice& user_key, const Slice& /*value*/,
|
Status AddUserKey(const Slice& user_key, const Slice& /*value*/,
|
||||||
|
@ -338,7 +338,7 @@ void TestCustomizedTablePropertiesCollector(
|
||||||
TEST_P(TablePropertiesTest, CustomizedTablePropertiesCollector) {
|
TEST_P(TablePropertiesTest, CustomizedTablePropertiesCollector) {
|
||||||
// Test properties collectors with internal keys or regular keys
|
// Test properties collectors with internal keys or regular keys
|
||||||
// for block based table
|
// for block based table
|
||||||
for (bool encode_as_internal : { true, false }) {
|
for (bool encode_as_internal : {true, false}) {
|
||||||
Options options;
|
Options options;
|
||||||
BlockBasedTableOptions table_options;
|
BlockBasedTableOptions table_options;
|
||||||
table_options.flush_block_policy_factory =
|
table_options.flush_block_policy_factory =
|
||||||
|
@ -404,7 +404,7 @@ void TestInternalKeyPropertiesCollector(
|
||||||
// HACK: Set options.info_log to avoid writing log in
|
// HACK: Set options.info_log to avoid writing log in
|
||||||
// SanitizeOptions().
|
// SanitizeOptions().
|
||||||
options.info_log = std::make_shared<test::NullLogger>();
|
options.info_log = std::make_shared<test::NullLogger>();
|
||||||
options = SanitizeOptions("db", // just a place holder
|
options = SanitizeOptions("db", // just a place holder
|
||||||
options);
|
options);
|
||||||
ImmutableOptions ioptions(options);
|
ImmutableOptions ioptions(options);
|
||||||
GetIntTblPropCollectorFactory(ioptions, &int_tbl_prop_collector_factories);
|
GetIntTblPropCollectorFactory(ioptions, &int_tbl_prop_collector_factories);
|
||||||
|
|
|
@ -41,7 +41,7 @@ TransactionLogIteratorImpl::TransactionLogIteratorImpl(
|
||||||
current_status_.PermitUncheckedError(); // Clear on start
|
current_status_.PermitUncheckedError(); // Clear on start
|
||||||
reporter_.env = options_->env;
|
reporter_.env = options_->env;
|
||||||
reporter_.info_log = options_->info_log.get();
|
reporter_.info_log = options_->info_log.get();
|
||||||
SeekToStartSequence(); // Seek till starting sequence
|
SeekToStartSequence(); // Seek till starting sequence
|
||||||
}
|
}
|
||||||
|
|
||||||
Status TransactionLogIteratorImpl::OpenLogFile(
|
Status TransactionLogIteratorImpl::OpenLogFile(
|
||||||
|
@ -62,8 +62,7 @@ Status TransactionLogIteratorImpl::OpenLogFile(
|
||||||
// If cannot open file in DB directory.
|
// If cannot open file in DB directory.
|
||||||
// Try the archive dir, as it could have moved in the meanwhile.
|
// Try the archive dir, as it could have moved in the meanwhile.
|
||||||
fname = ArchivedLogFileName(dir_, log_file->LogNumber());
|
fname = ArchivedLogFileName(dir_, log_file->LogNumber());
|
||||||
s = fs->NewSequentialFile(fname, optimized_env_options,
|
s = fs->NewSequentialFile(fname, optimized_env_options, &file, nullptr);
|
||||||
&file, nullptr);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
|
@ -74,7 +73,7 @@ Status TransactionLogIteratorImpl::OpenLogFile(
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
BatchResult TransactionLogIteratorImpl::GetBatch() {
|
BatchResult TransactionLogIteratorImpl::GetBatch() {
|
||||||
assert(is_valid_); // cannot call in a non valid state.
|
assert(is_valid_); // cannot call in a non valid state.
|
||||||
BatchResult result;
|
BatchResult result;
|
||||||
result.sequence = current_batch_seq_;
|
result.sequence = current_batch_seq_;
|
||||||
|
@ -124,8 +123,8 @@ void TransactionLogIteratorImpl::SeekToStartSequence(uint64_t start_file_index,
|
||||||
}
|
}
|
||||||
while (RestrictedRead(&record)) {
|
while (RestrictedRead(&record)) {
|
||||||
if (record.size() < WriteBatchInternal::kHeader) {
|
if (record.size() < WriteBatchInternal::kHeader) {
|
||||||
reporter_.Corruption(
|
reporter_.Corruption(record.size(),
|
||||||
record.size(), Status::Corruption("very small log record"));
|
Status::Corruption("very small log record"));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
UpdateCurrentWriteBatch(record);
|
UpdateCurrentWriteBatch(record);
|
||||||
|
@ -137,11 +136,12 @@ void TransactionLogIteratorImpl::SeekToStartSequence(uint64_t start_file_index,
|
||||||
reporter_.Info(current_status_.ToString().c_str());
|
reporter_.Info(current_status_.ToString().c_str());
|
||||||
return;
|
return;
|
||||||
} else if (strict) {
|
} else if (strict) {
|
||||||
reporter_.Info("Could seek required sequence number. Iterator will "
|
reporter_.Info(
|
||||||
"continue.");
|
"Could seek required sequence number. Iterator will "
|
||||||
|
"continue.");
|
||||||
}
|
}
|
||||||
is_valid_ = true;
|
is_valid_ = true;
|
||||||
started_ = true; // set started_ as we could seek till starting sequence
|
started_ = true; // set started_ as we could seek till starting sequence
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
is_valid_ = false;
|
is_valid_ = false;
|
||||||
|
@ -182,15 +182,15 @@ void TransactionLogIteratorImpl::NextImpl(bool internal) {
|
||||||
// Runs every time until we can seek to the start sequence
|
// Runs every time until we can seek to the start sequence
|
||||||
SeekToStartSequence();
|
SeekToStartSequence();
|
||||||
}
|
}
|
||||||
while(true) {
|
while (true) {
|
||||||
assert(current_log_reader_);
|
assert(current_log_reader_);
|
||||||
if (current_log_reader_->IsEOF()) {
|
if (current_log_reader_->IsEOF()) {
|
||||||
current_log_reader_->UnmarkEOF();
|
current_log_reader_->UnmarkEOF();
|
||||||
}
|
}
|
||||||
while (RestrictedRead(&record)) {
|
while (RestrictedRead(&record)) {
|
||||||
if (record.size() < WriteBatchInternal::kHeader) {
|
if (record.size() < WriteBatchInternal::kHeader) {
|
||||||
reporter_.Corruption(
|
reporter_.Corruption(record.size(),
|
||||||
record.size(), Status::Corruption("very small log record"));
|
Status::Corruption("very small log record"));
|
||||||
continue;
|
continue;
|
||||||
} else {
|
} else {
|
||||||
// started_ should be true if called by application
|
// started_ should be true if called by application
|
||||||
|
|
|
@ -23,12 +23,11 @@ namespace ROCKSDB_NAMESPACE {
|
||||||
class LogFileImpl : public LogFile {
|
class LogFileImpl : public LogFile {
|
||||||
public:
|
public:
|
||||||
LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
|
LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
|
||||||
uint64_t sizeBytes) :
|
uint64_t sizeBytes)
|
||||||
logNumber_(logNum),
|
: logNumber_(logNum),
|
||||||
type_(logType),
|
type_(logType),
|
||||||
startSequence_(startSeq),
|
startSequence_(startSeq),
|
||||||
sizeFileBytes_(sizeBytes) {
|
sizeFileBytes_(sizeBytes) {}
|
||||||
}
|
|
||||||
|
|
||||||
std::string PathName() const override {
|
std::string PathName() const override {
|
||||||
if (type_ == kArchivedLogFile) {
|
if (type_ == kArchivedLogFile) {
|
||||||
|
@ -45,7 +44,7 @@ class LogFileImpl : public LogFile {
|
||||||
|
|
||||||
uint64_t SizeFileBytes() const override { return sizeFileBytes_; }
|
uint64_t SizeFileBytes() const override { return sizeFileBytes_; }
|
||||||
|
|
||||||
bool operator < (const LogFile& that) const {
|
bool operator<(const LogFile& that) const {
|
||||||
return LogNumber() < that.LogNumber();
|
return LogNumber() < that.LogNumber();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -54,7 +53,6 @@ class LogFileImpl : public LogFile {
|
||||||
WalFileType type_;
|
WalFileType type_;
|
||||||
SequenceNumber startSequence_;
|
SequenceNumber startSequence_;
|
||||||
uint64_t sizeFileBytes_;
|
uint64_t sizeFileBytes_;
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
class TransactionLogIteratorImpl : public TransactionLogIterator {
|
class TransactionLogIteratorImpl : public TransactionLogIterator {
|
||||||
|
|
|
@ -6,8 +6,10 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
|
|
||||||
#include "util/autovector.h"
|
#include "util/autovector.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
|
@ -1656,10 +1656,10 @@ TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) {
|
||||||
UpdateVersionStorageInfo(&new_vstorage);
|
UpdateVersionStorageInfo(&new_vstorage);
|
||||||
|
|
||||||
VersionBuilder version_builder2(env_options, &ioptions_, table_cache,
|
VersionBuilder version_builder2(env_options, &ioptions_, table_cache,
|
||||||
&new_vstorage, version_set);
|
&new_vstorage, version_set);
|
||||||
VersionStorageInfo new_vstorage2(&icmp_, ucmp_, options_.num_levels,
|
VersionStorageInfo new_vstorage2(&icmp_, ucmp_, options_.num_levels,
|
||||||
kCompactionStyleLevel, nullptr,
|
kCompactionStyleLevel, nullptr,
|
||||||
true /* force_consistency_checks */);
|
true /* force_consistency_checks */);
|
||||||
ASSERT_NOK(version_builder2.Apply(&version_edit));
|
ASSERT_NOK(version_builder2.Apply(&version_edit));
|
||||||
|
|
||||||
UnrefFilesInVersion(&new_vstorage);
|
UnrefFilesInVersion(&new_vstorage);
|
||||||
|
|
|
@ -20,9 +20,7 @@
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
namespace {
|
namespace {} // anonymous namespace
|
||||||
|
|
||||||
} // anonymous namespace
|
|
||||||
|
|
||||||
uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) {
|
uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) {
|
||||||
assert(number <= kFileNumberMask);
|
assert(number <= kFileNumberMask);
|
||||||
|
@ -501,8 +499,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case kCompactCursor:
|
case kCompactCursor:
|
||||||
if (GetLevel(&input, &level, &msg) &&
|
if (GetLevel(&input, &level, &msg) && GetInternalKey(&input, &key)) {
|
||||||
GetInternalKey(&input, &key)) {
|
|
||||||
// Here we re-use the output format of compact pointer in LevelDB
|
// Here we re-use the output format of compact pointer in LevelDB
|
||||||
// to persist compact_cursors_
|
// to persist compact_cursors_
|
||||||
compact_cursors_.push_back(std::make_pair(level, key));
|
compact_cursors_.push_back(std::make_pair(level, key));
|
||||||
|
|
|
@ -114,7 +114,7 @@ struct FileDescriptor {
|
||||||
// Table reader in table_reader_handle
|
// Table reader in table_reader_handle
|
||||||
TableReader* table_reader;
|
TableReader* table_reader;
|
||||||
uint64_t packed_number_and_path_id;
|
uint64_t packed_number_and_path_id;
|
||||||
uint64_t file_size; // File size in bytes
|
uint64_t file_size; // File size in bytes
|
||||||
SequenceNumber smallest_seqno; // The smallest seqno in this file
|
SequenceNumber smallest_seqno; // The smallest seqno in this file
|
||||||
SequenceNumber largest_seqno; // The largest seqno in this file
|
SequenceNumber largest_seqno; // The largest seqno in this file
|
||||||
|
|
||||||
|
@ -146,8 +146,8 @@ struct FileDescriptor {
|
||||||
return packed_number_and_path_id & kFileNumberMask;
|
return packed_number_and_path_id & kFileNumberMask;
|
||||||
}
|
}
|
||||||
uint32_t GetPathId() const {
|
uint32_t GetPathId() const {
|
||||||
return static_cast<uint32_t>(
|
return static_cast<uint32_t>(packed_number_and_path_id /
|
||||||
packed_number_and_path_id / (kFileNumberMask + 1));
|
(kFileNumberMask + 1));
|
||||||
}
|
}
|
||||||
uint64_t GetFileSize() const { return file_size; }
|
uint64_t GetFileSize() const { return file_size; }
|
||||||
};
|
};
|
||||||
|
@ -166,8 +166,8 @@ struct FileSampledStats {
|
||||||
|
|
||||||
struct FileMetaData {
|
struct FileMetaData {
|
||||||
FileDescriptor fd;
|
FileDescriptor fd;
|
||||||
InternalKey smallest; // Smallest internal key served by table
|
InternalKey smallest; // Smallest internal key served by table
|
||||||
InternalKey largest; // Largest internal key served by table
|
InternalKey largest; // Largest internal key served by table
|
||||||
|
|
||||||
// Needs to be disposed when refs becomes 0.
|
// Needs to be disposed when refs becomes 0.
|
||||||
Cache::Handle* table_reader_handle = nullptr;
|
Cache::Handle* table_reader_handle = nullptr;
|
||||||
|
@ -312,15 +312,11 @@ struct FileMetaData {
|
||||||
struct FdWithKeyRange {
|
struct FdWithKeyRange {
|
||||||
FileDescriptor fd;
|
FileDescriptor fd;
|
||||||
FileMetaData* file_metadata; // Point to all metadata
|
FileMetaData* file_metadata; // Point to all metadata
|
||||||
Slice smallest_key; // slice that contain smallest key
|
Slice smallest_key; // slice that contain smallest key
|
||||||
Slice largest_key; // slice that contain largest key
|
Slice largest_key; // slice that contain largest key
|
||||||
|
|
||||||
FdWithKeyRange()
|
FdWithKeyRange()
|
||||||
: fd(),
|
: fd(), file_metadata(nullptr), smallest_key(), largest_key() {}
|
||||||
file_metadata(nullptr),
|
|
||||||
smallest_key(),
|
|
||||||
largest_key() {
|
|
||||||
}
|
|
||||||
|
|
||||||
FdWithKeyRange(FileDescriptor _fd, Slice _smallest_key, Slice _largest_key,
|
FdWithKeyRange(FileDescriptor _fd, Slice _smallest_key, Slice _largest_key,
|
||||||
FileMetaData* _file_metadata)
|
FileMetaData* _file_metadata)
|
||||||
|
|
|
@ -93,23 +93,19 @@ namespace {
|
||||||
// Find File in LevelFilesBrief data structure
|
// Find File in LevelFilesBrief data structure
|
||||||
// Within an index range defined by left and right
|
// Within an index range defined by left and right
|
||||||
int FindFileInRange(const InternalKeyComparator& icmp,
|
int FindFileInRange(const InternalKeyComparator& icmp,
|
||||||
const LevelFilesBrief& file_level,
|
const LevelFilesBrief& file_level, const Slice& key,
|
||||||
const Slice& key,
|
uint32_t left, uint32_t right) {
|
||||||
uint32_t left,
|
|
||||||
uint32_t right) {
|
|
||||||
auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool {
|
auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool {
|
||||||
return icmp.InternalKeyComparator::Compare(f.largest_key, k) < 0;
|
return icmp.InternalKeyComparator::Compare(f.largest_key, k) < 0;
|
||||||
};
|
};
|
||||||
const auto &b = file_level.files;
|
const auto& b = file_level.files;
|
||||||
return static_cast<int>(std::lower_bound(b + left,
|
return static_cast<int>(std::lower_bound(b + left, b + right, key, cmp) - b);
|
||||||
b + right, key, cmp) - b);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Status OverlapWithIterator(const Comparator* ucmp,
|
Status OverlapWithIterator(const Comparator* ucmp,
|
||||||
const Slice& smallest_user_key,
|
const Slice& smallest_user_key,
|
||||||
const Slice& largest_user_key,
|
const Slice& largest_user_key,
|
||||||
InternalIterator* iter,
|
InternalIterator* iter, bool* overlap) {
|
||||||
bool* overlap) {
|
|
||||||
InternalKey range_start(smallest_user_key, kMaxSequenceNumber,
|
InternalKey range_start(smallest_user_key, kMaxSequenceNumber,
|
||||||
kValueTypeForSeek);
|
kValueTypeForSeek);
|
||||||
iter->Seek(range_start.Encode());
|
iter->Seek(range_start.Encode());
|
||||||
|
@ -187,9 +183,9 @@ class FilePicker {
|
||||||
// Do key range filtering of files or/and fractional cascading if:
|
// Do key range filtering of files or/and fractional cascading if:
|
||||||
// (1) not all the files are in level 0, or
|
// (1) not all the files are in level 0, or
|
||||||
// (2) there are more than 3 current level files
|
// (2) there are more than 3 current level files
|
||||||
// If there are only 3 or less current level files in the system, we skip
|
// If there are only 3 or less current level files in the system, we
|
||||||
// the key range filtering. In this case, more likely, the system is
|
// skip the key range filtering. In this case, more likely, the system
|
||||||
// highly tuned to minimize number of tables queried by each query,
|
// is highly tuned to minimize number of tables queried by each query,
|
||||||
// so it is unlikely that key range filtering is more efficient than
|
// so it is unlikely that key range filtering is more efficient than
|
||||||
// querying the files.
|
// querying the files.
|
||||||
if (num_levels_ > 1 || curr_file_level_->num_files > 3) {
|
if (num_levels_ > 1 || curr_file_level_->num_files > 3) {
|
||||||
|
@ -211,11 +207,9 @@ class FilePicker {
|
||||||
// Setup file search bound for the next level based on the
|
// Setup file search bound for the next level based on the
|
||||||
// comparison results
|
// comparison results
|
||||||
if (curr_level_ > 0) {
|
if (curr_level_ > 0) {
|
||||||
file_indexer_->GetNextLevelIndex(curr_level_,
|
file_indexer_->GetNextLevelIndex(
|
||||||
curr_index_in_curr_level_,
|
curr_level_, curr_index_in_curr_level_, cmp_smallest,
|
||||||
cmp_smallest, cmp_largest,
|
cmp_largest, &search_left_bound_, &search_right_bound_);
|
||||||
&search_left_bound_,
|
|
||||||
&search_right_bound_);
|
|
||||||
}
|
}
|
||||||
// Key falls out of current file's range
|
// Key falls out of current file's range
|
||||||
if (cmp_smallest < 0 || cmp_largest > 0) {
|
if (cmp_smallest < 0 || cmp_largest > 0) {
|
||||||
|
@ -846,22 +840,21 @@ Version::~Version() {
|
||||||
}
|
}
|
||||||
|
|
||||||
int FindFile(const InternalKeyComparator& icmp,
|
int FindFile(const InternalKeyComparator& icmp,
|
||||||
const LevelFilesBrief& file_level,
|
const LevelFilesBrief& file_level, const Slice& key) {
|
||||||
const Slice& key) {
|
|
||||||
return FindFileInRange(icmp, file_level, key, 0,
|
return FindFileInRange(icmp, file_level, key, 0,
|
||||||
static_cast<uint32_t>(file_level.num_files));
|
static_cast<uint32_t>(file_level.num_files));
|
||||||
}
|
}
|
||||||
|
|
||||||
void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
|
void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
|
||||||
const std::vector<FileMetaData*>& files,
|
const std::vector<FileMetaData*>& files,
|
||||||
Arena* arena) {
|
Arena* arena) {
|
||||||
assert(file_level);
|
assert(file_level);
|
||||||
assert(arena);
|
assert(arena);
|
||||||
|
|
||||||
size_t num = files.size();
|
size_t num = files.size();
|
||||||
file_level->num_files = num;
|
file_level->num_files = num;
|
||||||
char* mem = arena->AllocateAligned(num * sizeof(FdWithKeyRange));
|
char* mem = arena->AllocateAligned(num * sizeof(FdWithKeyRange));
|
||||||
file_level->files = new (mem)FdWithKeyRange[num];
|
file_level->files = new (mem) FdWithKeyRange[num];
|
||||||
|
|
||||||
for (size_t i = 0; i < num; i++) {
|
for (size_t i = 0; i < num; i++) {
|
||||||
Slice smallest_key = files[i]->smallest.Encode();
|
Slice smallest_key = files[i]->smallest.Encode();
|
||||||
|
@ -882,28 +875,27 @@ void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool AfterFile(const Comparator* ucmp,
|
static bool AfterFile(const Comparator* ucmp, const Slice* user_key,
|
||||||
const Slice* user_key, const FdWithKeyRange* f) {
|
const FdWithKeyRange* f) {
|
||||||
// nullptr user_key occurs before all keys and is therefore never after *f
|
// nullptr user_key occurs before all keys and is therefore never after *f
|
||||||
return (user_key != nullptr &&
|
return (user_key != nullptr &&
|
||||||
ucmp->CompareWithoutTimestamp(*user_key,
|
ucmp->CompareWithoutTimestamp(*user_key,
|
||||||
ExtractUserKey(f->largest_key)) > 0);
|
ExtractUserKey(f->largest_key)) > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool BeforeFile(const Comparator* ucmp,
|
static bool BeforeFile(const Comparator* ucmp, const Slice* user_key,
|
||||||
const Slice* user_key, const FdWithKeyRange* f) {
|
const FdWithKeyRange* f) {
|
||||||
// nullptr user_key occurs after all keys and is therefore never before *f
|
// nullptr user_key occurs after all keys and is therefore never before *f
|
||||||
return (user_key != nullptr &&
|
return (user_key != nullptr &&
|
||||||
ucmp->CompareWithoutTimestamp(*user_key,
|
ucmp->CompareWithoutTimestamp(*user_key,
|
||||||
ExtractUserKey(f->smallest_key)) < 0);
|
ExtractUserKey(f->smallest_key)) < 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool SomeFileOverlapsRange(
|
bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
|
||||||
const InternalKeyComparator& icmp,
|
bool disjoint_sorted_files,
|
||||||
bool disjoint_sorted_files,
|
const LevelFilesBrief& file_level,
|
||||||
const LevelFilesBrief& file_level,
|
const Slice* smallest_user_key,
|
||||||
const Slice* smallest_user_key,
|
const Slice* largest_user_key) {
|
||||||
const Slice* largest_user_key) {
|
|
||||||
const Comparator* ucmp = icmp.user_comparator();
|
const Comparator* ucmp = icmp.user_comparator();
|
||||||
if (!disjoint_sorted_files) {
|
if (!disjoint_sorted_files) {
|
||||||
// Need to check against all files
|
// Need to check against all files
|
||||||
|
@ -1025,9 +1017,7 @@ class LevelIterator final : public InternalIterator {
|
||||||
return file_iter_.iter() ? file_iter_.status() : Status::OK();
|
return file_iter_.iter() ? file_iter_.status() : Status::OK();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool PrepareValue() override {
|
bool PrepareValue() override { return file_iter_.PrepareValue(); }
|
||||||
return file_iter_.PrepareValue();
|
|
||||||
}
|
|
||||||
|
|
||||||
inline bool MayBeOutOfLowerBound() override {
|
inline bool MayBeOutOfLowerBound() override {
|
||||||
assert(Valid());
|
assert(Valid());
|
||||||
|
@ -1561,9 +1551,8 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
|
||||||
if (fname != nullptr) {
|
if (fname != nullptr) {
|
||||||
file_name = *fname;
|
file_name = *fname;
|
||||||
} else {
|
} else {
|
||||||
file_name =
|
file_name = TableFileName(ioptions->cf_paths, file_meta->fd.GetNumber(),
|
||||||
TableFileName(ioptions->cf_paths, file_meta->fd.GetNumber(),
|
file_meta->fd.GetPathId());
|
||||||
file_meta->fd.GetPathId());
|
|
||||||
}
|
}
|
||||||
s = ioptions->fs->NewRandomAccessFile(file_name, file_options_, &file,
|
s = ioptions->fs->NewRandomAccessFile(file_name, file_options_, &file,
|
||||||
nullptr);
|
nullptr);
|
||||||
|
@ -1690,8 +1679,8 @@ Status Version::GetPropertiesOfTablesInRange(
|
||||||
false);
|
false);
|
||||||
for (const auto& file_meta : files) {
|
for (const auto& file_meta : files) {
|
||||||
auto fname =
|
auto fname =
|
||||||
TableFileName(cfd_->ioptions()->cf_paths,
|
TableFileName(cfd_->ioptions()->cf_paths, file_meta->fd.GetNumber(),
|
||||||
file_meta->fd.GetNumber(), file_meta->fd.GetPathId());
|
file_meta->fd.GetPathId());
|
||||||
if (props->count(fname) == 0) {
|
if (props->count(fname) == 0) {
|
||||||
// 1. If the table is already present in table cache, load table
|
// 1. If the table is already present in table cache, load table
|
||||||
// properties from there.
|
// properties from there.
|
||||||
|
@ -1788,8 +1777,7 @@ void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) {
|
||||||
files.back().num_deletions = file->num_deletions;
|
files.back().num_deletions = file->num_deletions;
|
||||||
level_size += file->fd.GetFileSize();
|
level_size += file->fd.GetFileSize();
|
||||||
}
|
}
|
||||||
cf_meta->levels.emplace_back(
|
cf_meta->levels.emplace_back(level, level_size, std::move(files));
|
||||||
level, level_size, std::move(files));
|
|
||||||
cf_meta->size += level_size;
|
cf_meta->size += level_size;
|
||||||
}
|
}
|
||||||
for (const auto& meta : vstorage->GetBlobFiles()) {
|
for (const auto& meta : vstorage->GetBlobFiles()) {
|
||||||
|
@ -1880,10 +1868,8 @@ uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const {
|
||||||
|
|
||||||
if (current_num_samples_ < file_count) {
|
if (current_num_samples_ < file_count) {
|
||||||
// casting to avoid overflowing
|
// casting to avoid overflowing
|
||||||
return
|
return static_cast<uint64_t>(
|
||||||
static_cast<uint64_t>(
|
(est * static_cast<double>(file_count) / current_num_samples_));
|
||||||
(est * static_cast<double>(file_count) / current_num_samples_)
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
return est;
|
return est;
|
||||||
}
|
}
|
||||||
|
@ -2020,8 +2006,8 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
|
||||||
/*smallest_compaction_key=*/nullptr,
|
/*smallest_compaction_key=*/nullptr,
|
||||||
/*largest_compaction_key=*/nullptr,
|
/*largest_compaction_key=*/nullptr,
|
||||||
/*allow_unprepared_value=*/false));
|
/*allow_unprepared_value=*/false));
|
||||||
status = OverlapWithIterator(
|
status = OverlapWithIterator(ucmp, smallest_user_key, largest_user_key,
|
||||||
ucmp, smallest_user_key, largest_user_key, iter.get(), overlap);
|
iter.get(), overlap);
|
||||||
if (!status.ok() || *overlap) {
|
if (!status.ok() || *overlap) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -2035,8 +2021,8 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
|
||||||
cfd_->internal_stats()->GetFileReadHist(level),
|
cfd_->internal_stats()->GetFileReadHist(level),
|
||||||
TableReaderCaller::kUserIterator, IsFilterSkipped(level), level,
|
TableReaderCaller::kUserIterator, IsFilterSkipped(level), level,
|
||||||
&range_del_agg));
|
&range_del_agg));
|
||||||
status = OverlapWithIterator(
|
status = OverlapWithIterator(ucmp, smallest_user_key, largest_user_key,
|
||||||
ucmp, smallest_user_key, largest_user_key, iter.get(), overlap);
|
iter.get(), overlap);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (status.ok() && *overlap == false &&
|
if (status.ok() && *overlap == false &&
|
||||||
|
@ -2396,7 +2382,7 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!merge_operator_) {
|
if (!merge_operator_) {
|
||||||
*status = Status::InvalidArgument(
|
*status = Status::InvalidArgument(
|
||||||
"merge_operator is not properly initialized.");
|
"merge_operator is not properly initialized.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -2418,7 +2404,7 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
|
||||||
if (key_exists != nullptr) {
|
if (key_exists != nullptr) {
|
||||||
*key_exists = false;
|
*key_exists = false;
|
||||||
}
|
}
|
||||||
*status = Status::NotFound(); // Use an empty error message for speed
|
*status = Status::NotFound(); // Use an empty error message for speed
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2905,8 +2891,8 @@ bool Version::IsFilterSkipped(int level, bool is_file_last_in_level) {
|
||||||
void VersionStorageInfo::GenerateLevelFilesBrief() {
|
void VersionStorageInfo::GenerateLevelFilesBrief() {
|
||||||
level_files_brief_.resize(num_non_empty_levels_);
|
level_files_brief_.resize(num_non_empty_levels_);
|
||||||
for (int level = 0; level < num_non_empty_levels_; level++) {
|
for (int level = 0; level < num_non_empty_levels_; level++) {
|
||||||
DoGenerateLevelFilesBrief(
|
DoGenerateLevelFilesBrief(&level_files_brief_[level], files_[level],
|
||||||
&level_files_brief_[level], files_[level], &arena_);
|
&arena_);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2940,8 +2926,7 @@ void Version::PrepareAppend(const MutableCFOptions& mutable_cf_options,
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) {
|
bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) {
|
||||||
if (file_meta->init_stats_from_file ||
|
if (file_meta->init_stats_from_file || file_meta->compensated_file_size > 0) {
|
||||||
file_meta->compensated_file_size > 0) {
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
std::shared_ptr<const TableProperties> tp;
|
std::shared_ptr<const TableProperties> tp;
|
||||||
|
@ -3608,9 +3593,9 @@ struct Fsize {
|
||||||
// In normal mode: descending size
|
// In normal mode: descending size
|
||||||
bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) {
|
bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) {
|
||||||
return (first.file->compensated_file_size >
|
return (first.file->compensated_file_size >
|
||||||
second.file->compensated_file_size);
|
second.file->compensated_file_size);
|
||||||
}
|
}
|
||||||
} // anonymous namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
void VersionStorageInfo::AddFile(int level, FileMetaData* f) {
|
void VersionStorageInfo::AddFile(int level, FileMetaData* f) {
|
||||||
auto& level_files = files_[level];
|
auto& level_files = files_[level];
|
||||||
|
@ -3806,7 +3791,7 @@ void SortFileByRoundRobin(const InternalKeyComparator& icmp,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
void VersionStorageInfo::UpdateFilesByCompactionPri(
|
void VersionStorageInfo::UpdateFilesByCompactionPri(
|
||||||
const ImmutableOptions& ioptions, const MutableCFOptions& options) {
|
const ImmutableOptions& ioptions, const MutableCFOptions& options) {
|
||||||
|
@ -3978,9 +3963,7 @@ void VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Version::Ref() {
|
void Version::Ref() { ++refs_; }
|
||||||
++refs_;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool Version::Unref() {
|
bool Version::Unref() {
|
||||||
assert(refs_ >= 1);
|
assert(refs_ >= 1);
|
||||||
|
@ -4112,9 +4095,8 @@ void VersionStorageInfo::GetCleanInputsWithinInterval(
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
GetOverlappingInputsRangeBinarySearch(level, begin, end, inputs,
|
GetOverlappingInputsRangeBinarySearch(level, begin, end, inputs, hint_index,
|
||||||
hint_index, file_index,
|
file_index, true /* within_interval */);
|
||||||
true /* within_interval */);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store in "*inputs" all files in "level" that overlap [begin,end]
|
// Store in "*inputs" all files in "level" that overlap [begin,end]
|
||||||
|
@ -4277,8 +4259,7 @@ const char* VersionStorageInfo::LevelFileSummary(FileSummaryStorage* scratch,
|
||||||
"#%" PRIu64 "(seq=%" PRIu64 ",sz=%s,%d) ",
|
"#%" PRIu64 "(seq=%" PRIu64 ",sz=%s,%d) ",
|
||||||
f->fd.GetNumber(), f->fd.smallest_seqno, sztxt,
|
f->fd.GetNumber(), f->fd.smallest_seqno, sztxt,
|
||||||
static_cast<int>(f->being_compacted));
|
static_cast<int>(f->being_compacted));
|
||||||
if (ret < 0 || ret >= sz)
|
if (ret < 0 || ret >= sz) break;
|
||||||
break;
|
|
||||||
len += ret;
|
len += ret;
|
||||||
}
|
}
|
||||||
// overwrite the last space (only if files_[level].size() is non-zero)
|
// overwrite the last space (only if files_[level].size() is non-zero)
|
||||||
|
@ -4456,13 +4437,13 @@ uint64_t VersionStorageInfo::EstimateLiveDataSize() const {
|
||||||
// no potential overlap, we can safely insert the rest of this level
|
// no potential overlap, we can safely insert the rest of this level
|
||||||
// (if the level is not 0) into the map without checking again because
|
// (if the level is not 0) into the map without checking again because
|
||||||
// the elements in the level are sorted and non-overlapping.
|
// the elements in the level are sorted and non-overlapping.
|
||||||
auto lb = (found_end && l != 0) ?
|
auto lb = (found_end && l != 0) ? ranges.end()
|
||||||
ranges.end() : ranges.lower_bound(&file->smallest);
|
: ranges.lower_bound(&file->smallest);
|
||||||
found_end = (lb == ranges.end());
|
found_end = (lb == ranges.end());
|
||||||
if (found_end || internal_comparator_->Compare(
|
if (found_end || internal_comparator_->Compare(
|
||||||
file->largest, (*lb).second->smallest) < 0) {
|
file->largest, (*lb).second->smallest) < 0) {
|
||||||
ranges.emplace_hint(lb, &file->largest, file);
|
ranges.emplace_hint(lb, &file->largest, file);
|
||||||
size += file->fd.file_size;
|
size += file->fd.file_size;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -5674,7 +5655,7 @@ std::string ManifestPicker::GetNextManifest(uint64_t* number,
|
||||||
}
|
}
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
} // namespace
|
} // anonymous namespace
|
||||||
|
|
||||||
Status VersionSet::TryRecover(
|
Status VersionSet::TryRecover(
|
||||||
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
|
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
|
||||||
|
@ -5876,7 +5857,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
delete[] vstorage -> files_;
|
delete[] vstorage->files_;
|
||||||
vstorage->files_ = new_files_list;
|
vstorage->files_ = new_files_list;
|
||||||
vstorage->num_levels_ = new_levels;
|
vstorage->num_levels_ = new_levels;
|
||||||
vstorage->ResizeCompactCursors(new_levels);
|
vstorage->ResizeCompactCursors(new_levels);
|
||||||
|
@ -5885,9 +5866,9 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
|
||||||
VersionEdit ve;
|
VersionEdit ve;
|
||||||
InstrumentedMutex dummy_mutex;
|
InstrumentedMutex dummy_mutex;
|
||||||
InstrumentedMutexLock l(&dummy_mutex);
|
InstrumentedMutexLock l(&dummy_mutex);
|
||||||
return versions.LogAndApply(
|
return versions.LogAndApply(versions.GetColumnFamilySet()->GetDefault(),
|
||||||
versions.GetColumnFamilySet()->GetDefault(),
|
mutable_cf_options, &ve, &dummy_mutex, nullptr,
|
||||||
mutable_cf_options, &ve, &dummy_mutex, nullptr, true);
|
true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the checksum information including the checksum and checksum function
|
// Get the checksum information including the checksum and checksum function
|
||||||
|
@ -5973,9 +5954,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
|
||||||
std::unique_ptr<FSSequentialFile> file;
|
std::unique_ptr<FSSequentialFile> file;
|
||||||
const std::shared_ptr<FileSystem>& fs = options.env->GetFileSystem();
|
const std::shared_ptr<FileSystem>& fs = options.env->GetFileSystem();
|
||||||
s = fs->NewSequentialFile(
|
s = fs->NewSequentialFile(
|
||||||
dscname,
|
dscname, fs->OptimizeForManifestRead(file_options_), &file, nullptr);
|
||||||
fs->OptimizeForManifestRead(file_options_), &file,
|
|
||||||
nullptr);
|
|
||||||
if (!s.ok()) {
|
if (!s.ok()) {
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
@ -6078,8 +6057,8 @@ Status VersionSet::WriteCurrentStateToManifest(
|
||||||
cfd->internal_comparator().user_comparator()->Name());
|
cfd->internal_comparator().user_comparator()->Name());
|
||||||
std::string record;
|
std::string record;
|
||||||
if (!edit.EncodeTo(&record)) {
|
if (!edit.EncodeTo(&record)) {
|
||||||
return Status::Corruption(
|
return Status::Corruption("Unable to Encode VersionEdit:" +
|
||||||
"Unable to Encode VersionEdit:" + edit.DebugString(true));
|
edit.DebugString(true));
|
||||||
}
|
}
|
||||||
io_s = log->AddRecord(record);
|
io_s = log->AddRecord(record);
|
||||||
if (!io_s.ok()) {
|
if (!io_s.ok()) {
|
||||||
|
@ -6137,9 +6116,10 @@ Status VersionSet::WriteCurrentStateToManifest(
|
||||||
edit.SetLogNumber(log_number);
|
edit.SetLogNumber(log_number);
|
||||||
|
|
||||||
if (cfd->GetID() == 0) {
|
if (cfd->GetID() == 0) {
|
||||||
// min_log_number_to_keep is for the whole db, not for specific column family.
|
// min_log_number_to_keep is for the whole db, not for specific column
|
||||||
// So it does not need to be set for every column family, just need to be set once.
|
// family. So it does not need to be set for every column family, just
|
||||||
// Since default CF can never be dropped, we set the min_log to the default CF here.
|
// need to be set once. Since default CF can never be dropped, we set
|
||||||
|
// the min_log to the default CF here.
|
||||||
uint64_t min_log = min_log_number_to_keep();
|
uint64_t min_log = min_log_number_to_keep();
|
||||||
if (min_log != 0) {
|
if (min_log != 0) {
|
||||||
edit.SetMinLogNumberToKeep(min_log);
|
edit.SetMinLogNumberToKeep(min_log);
|
||||||
|
@ -6155,8 +6135,8 @@ Status VersionSet::WriteCurrentStateToManifest(
|
||||||
|
|
||||||
std::string record;
|
std::string record;
|
||||||
if (!edit.EncodeTo(&record)) {
|
if (!edit.EncodeTo(&record)) {
|
||||||
return Status::Corruption(
|
return Status::Corruption("Unable to Encode VersionEdit:" +
|
||||||
"Unable to Encode VersionEdit:" + edit.DebugString(true));
|
edit.DebugString(true));
|
||||||
}
|
}
|
||||||
io_s = log->AddRecord(record);
|
io_s = log->AddRecord(record);
|
||||||
if (!io_s.ok()) {
|
if (!io_s.ok()) {
|
||||||
|
@ -6479,7 +6459,7 @@ InternalIterator* VersionSet::MakeInputIterator(
|
||||||
const size_t space = (c->level() == 0 ? c->input_levels(0)->num_files +
|
const size_t space = (c->level() == 0 ? c->input_levels(0)->num_files +
|
||||||
c->num_input_levels() - 1
|
c->num_input_levels() - 1
|
||||||
: c->num_input_levels());
|
: c->num_input_levels());
|
||||||
InternalIterator** list = new InternalIterator* [space];
|
InternalIterator** list = new InternalIterator*[space];
|
||||||
size_t num = 0;
|
size_t num = 0;
|
||||||
for (size_t which = 0; which < c->num_input_levels(); which++) {
|
for (size_t which = 0; which < c->num_input_levels(); which++) {
|
||||||
if (c->input_levels(which)->num_files != 0) {
|
if (c->input_levels(which)->num_files != 0) {
|
||||||
|
@ -6588,8 +6568,8 @@ void VersionSet::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
|
||||||
filemetadata.largestkey = file->largest.user_key().ToString();
|
filemetadata.largestkey = file->largest.user_key().ToString();
|
||||||
filemetadata.smallest_seqno = file->fd.smallest_seqno;
|
filemetadata.smallest_seqno = file->fd.smallest_seqno;
|
||||||
filemetadata.largest_seqno = file->fd.largest_seqno;
|
filemetadata.largest_seqno = file->fd.largest_seqno;
|
||||||
filemetadata.num_reads_sampled = file->stats.num_reads_sampled.load(
|
filemetadata.num_reads_sampled =
|
||||||
std::memory_order_relaxed);
|
file->stats.num_reads_sampled.load(std::memory_order_relaxed);
|
||||||
filemetadata.being_compacted = file->being_compacted;
|
filemetadata.being_compacted = file->being_compacted;
|
||||||
filemetadata.num_entries = file->num_entries;
|
filemetadata.num_entries = file->num_entries;
|
||||||
filemetadata.num_deletions = file->num_deletions;
|
filemetadata.num_deletions = file->num_deletions;
|
||||||
|
|
|
@ -230,9 +230,7 @@ class VersionStorageInfo {
|
||||||
double blob_garbage_collection_age_cutoff,
|
double blob_garbage_collection_age_cutoff,
|
||||||
double blob_garbage_collection_force_threshold);
|
double blob_garbage_collection_force_threshold);
|
||||||
|
|
||||||
bool level0_non_overlapping() const {
|
bool level0_non_overlapping() const { return level0_non_overlapping_; }
|
||||||
return level0_non_overlapping_;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Updates the oldest snapshot and related internal state, like the bottommost
|
// Updates the oldest snapshot and related internal state, like the bottommost
|
||||||
// files marked for compaction.
|
// files marked for compaction.
|
||||||
|
@ -814,8 +812,8 @@ class Version {
|
||||||
|
|
||||||
Status OverlapWithLevelIterator(const ReadOptions&, const FileOptions&,
|
Status OverlapWithLevelIterator(const ReadOptions&, const FileOptions&,
|
||||||
const Slice& smallest_user_key,
|
const Slice& smallest_user_key,
|
||||||
const Slice& largest_user_key,
|
const Slice& largest_user_key, int level,
|
||||||
int level, bool* overlap);
|
bool* overlap);
|
||||||
|
|
||||||
// Lookup the value for key or get all merge operands for key.
|
// Lookup the value for key or get all merge operands for key.
|
||||||
// If do_merge = true (default) then lookup value for key.
|
// If do_merge = true (default) then lookup value for key.
|
||||||
|
@ -1032,10 +1030,10 @@ class Version {
|
||||||
const MergeOperator* merge_operator_;
|
const MergeOperator* merge_operator_;
|
||||||
|
|
||||||
VersionStorageInfo storage_info_;
|
VersionStorageInfo storage_info_;
|
||||||
VersionSet* vset_; // VersionSet to which this Version belongs
|
VersionSet* vset_; // VersionSet to which this Version belongs
|
||||||
Version* next_; // Next version in linked list
|
Version* next_; // Next version in linked list
|
||||||
Version* prev_; // Previous version in linked list
|
Version* prev_; // Previous version in linked list
|
||||||
int refs_; // Number of live refs to this version
|
int refs_; // Number of live refs to this version
|
||||||
const FileOptions file_options_;
|
const FileOptions file_options_;
|
||||||
const MutableCFOptions mutable_cf_options_;
|
const MutableCFOptions mutable_cf_options_;
|
||||||
// Cached value to avoid recomputing it on every read.
|
// Cached value to avoid recomputing it on every read.
|
||||||
|
@ -1398,7 +1396,7 @@ class VersionSet {
|
||||||
FileMetaData** metadata, ColumnFamilyData** cfd);
|
FileMetaData** metadata, ColumnFamilyData** cfd);
|
||||||
|
|
||||||
// This function doesn't support leveldb SST filenames
|
// This function doesn't support leveldb SST filenames
|
||||||
void GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata);
|
void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
|
||||||
|
|
||||||
void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) {
|
void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) {
|
||||||
assert(table_cache_);
|
assert(table_cache_);
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue