dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update SstFileWriter fuzzer to iterate and check all key-value pairs (#7761) 

Summary:
as title

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7761

Test Plan: cd fuzz && make sst_file_writer_fuzzer && ./sst_file_writer_fuzzer

Reviewed By: pdillinger

Differential Revision: D25430802

Pulled By: cheng-chang

fbshipit-source-id: 01436307df6f4c434bb608f44e1c8e4a1119f94f
This commit is contained in:
Cheng Chang 2020-12-11 16:07:48 -08:00 committed by Facebook GitHub Bot
parent b1ee191405
commit 491779514e
6 changed files with 174 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
fuzz/Makefile
View File

@ -3,7 +3,9 @@
# COPYING file in the root directory) and Apache 2.0 License
# (found in the LICENSE.Apache file in the root directory).
CWD = $(shell pwd)
ROOT_DIR = $(abspath $(shell pwd)/../)
include $(ROOT_DIR)/make_config.mk
PROTOBUF_CFLAGS = `pkg-config --cflags protobuf`
PROTOBUF_LDFLAGS = `pkg-config --libs protobuf`
@ -11,17 +13,17 @@ PROTOBUF_LDFLAGS = `pkg-config --libs protobuf`
PROTOBUF_MUTATOR_CFLAGS = `pkg-config --cflags libprotobuf-mutator`
PROTOBUF_MUTATOR_LDFLAGS = `pkg-config --libs libprotobuf-mutator`
ROCKSDB_INCLUDE_DIR = $(CWD)/../include
ROCKSDB_LIB_DIR = $(CWD)/..
ROCKSDB_INCLUDE_DIR = $(ROOT_DIR)/include
ROCKSDB_LIB_DIR = $(ROOT_DIR)
PROTO_IN = $(CWD)/proto
PROTO_OUT = $(CWD)/proto/gen
PROTO_IN = $(ROOT_DIR)/fuzz/proto
PROTO_OUT = $(ROOT_DIR)/fuzz/proto/gen
ifneq ($(FUZZ_ENV), ossfuzz)
CC = clang++
CCFLAGS += -std=c++14 -Wall -fsanitize=address,fuzzer
CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR)
LDFLAGS += $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb -lz -lbz2
CCFLAGS += -Wall -fsanitize=address,fuzzer
CFLAGS += $(PLATFORM_CXXFLAGS) $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR)
LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
else
# OSS-Fuzz sets various environment flags that are used for compilation.
# These environment flags depend on which type of sanitizer build is being
@ -36,14 +38,11 @@ else
# LIB_FUZZING_ENGINE="-fsanitize=fuzzer"
CC = $(CXX)
CCFLAGS = $(CXXFLAGS)
CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR)
LDFLAGS += $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb -lz -lbz2
CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR)
LDFLAGS += $(PLATFORM_LDFLAGS) $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
endif
.PHONY: librocksdb gen_proto
librocksdb:
cd $(CWD)/.. && make static_lib
.PHONY: gen_proto
gen_proto:
mkdir -p $(PROTO_OUT)
@ -52,11 +51,11 @@ gen_proto:
--cpp_out=$(PROTO_OUT) \
$(PROTO_IN)/*.proto
db_fuzzer: librocksdb db_fuzzer.cc
db_fuzzer: db_fuzzer.cc
$(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS)
db_map_fuzzer: librocksdb gen_proto db_map_fuzzer.cc proto/gen/db_operation.pb.cc
db_map_fuzzer: gen_proto db_map_fuzzer.cc proto/gen/db_operation.pb.cc
$(CC) $(CCFLAGS) -o db_map_fuzzer db_map_fuzzer.cc proto/gen/db_operation.pb.cc $(CFLAGS) $(LDFLAGS)
sst_file_writer_fuzzer: librocksdb gen_proto sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc
sst_file_writer_fuzzer: gen_proto sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc
$(CC) $(CCFLAGS) -o sst_file_writer_fuzzer sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc $(CFLAGS) $(LDFLAGS)

2
fuzz/README.md
View File

@ -56,6 +56,8 @@ protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations>
### Compile and link the fuzzer
In the rocksdb root directory, compile rocksdb library by `make static_lib`.
Go to the `fuzz` directory,
run `make sst_file_writer_fuzzer` to generate the fuzzer,
it will compile rocksdb static library, generate protobuf, then compile and link `sst_file_writer_fuzzer`.

26
fuzz/db_map_fuzzer.cc
View File

@ -12,28 +12,7 @@
#include "rocksdb/db.h"
#include "rocksdb/file_system.h"
#include "src/libfuzzer/libfuzzer_macro.h"
#define CHECK_OK(expression) \
do { \
auto status = (expression); \
if (!status.ok()) { \
std::cerr << status.ToString() << std::endl; \
abort(); \
} \
} while (0)
#define CHECK_EQ(a, b) \
if (a != b) { \
std::cerr << "(" << #a << "=" << a << ") != (" << #b << "=" << b << ")" \
<< std::endl; \
abort(); \
}
#define CHECK_TRUE(cond) \
if (!(cond)) { \
std::cerr << "\"" << #cond << "\" is false" << std::endl; \
abort(); \
}
#include "util.h"
protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations> reg = {
[](DBOperations* input, unsigned int /* seed */) {
@ -81,6 +60,9 @@ DEFINE_PROTO_FUZZER(DBOperations& input) {
kv[op.key()] = op.value();
break;
}
case OpType::MERGE: {
break;
}
case OpType::DELETE: {
CHECK_OK(db->Delete(rocksdb::WriteOptions(), op.key()));
kv.erase(op.key());

1
fuzz/proto/db_operation.proto
View File

@ -10,6 +10,7 @@ syntax = "proto2";
enum OpType {
PUT = 0;
MERGE = 1;
DELETE = 2;
DELETE_RANGE = 3;
}

169
fuzz/sst_file_writer_fuzzer.cc
View File

@ -5,20 +5,39 @@
#include <algorithm>
#include <iostream>
#include <memory>
#include <string>
#include "proto/gen/db_operation.pb.h"
#include "rocksdb/file_system.h"
#include "rocksdb/sst_file_reader.h"
#include "rocksdb/sst_file_writer.h"
#include "src/libfuzzer/libfuzzer_macro.h"
#include "table/table_reader.h"
#include "util.h"
using namespace ROCKSDB_NAMESPACE;
// Keys in SST file writer operations must be unique and in ascending order.
// For each DBOperation generated by the fuzzer, this function is called on
// it to deduplicate and sort the keys in the DBOperations.
protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations> reg = {
[](DBOperations* input, unsigned int /* seed */) {
const rocksdb::Comparator* comparator = rocksdb::BytewiseComparator();
const Comparator* comparator = BytewiseComparator();
auto ops = input->mutable_operations();
// Make sure begin <= end for DELETE_RANGE.
for (DBOperation& op : *ops) {
if (op.type() == OpType::DELETE_RANGE) {
auto begin = op.key();
auto end = op.value();
if (comparator->Compare(begin, end) > 0) {
std::swap(begin, end);
op.set_key(begin);
op.set_value(end);
}
}
}
std::sort(ops->begin(), ops->end(),
[&comparator](const DBOperation& a, const DBOperation& b) {
return comparator->Compare(a.key(), b.key()) < 0;
@ -32,15 +51,60 @@ protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations> reg = {
ops->erase(last, ops->end());
}};
#define CHECK_OK(status) \
if (!status.ok()) { \
std::cerr << status.ToString() << std::endl; \
abort(); \
TableReader* NewTableReader(const std::string& sst_file_path,
const Options& options,
const EnvOptions& env_options,
const ImmutableCFOptions& cf_ioptions) {
// This code block is similar to SstFileReader::Open.
uint64_t file_size = 0;
std::unique_ptr<RandomAccessFile> file;
std::unique_ptr<RandomAccessFileReader> file_reader;
std::unique_ptr<TableReader> table_reader;
Status s = options.env->GetFileSize(sst_file_path, &file_size);
if (s.ok()) {
s = options.env->NewRandomAccessFile(sst_file_path, &file, env_options);
}
if (s.ok()) {
file_reader.reset(new RandomAccessFileReader(
NewLegacyRandomAccessFileWrapper(file), sst_file_path));
}
if (s.ok()) {
TableReaderOptions t_opt(cf_ioptions, /*prefix_extractor=*/nullptr,
env_options, cf_ioptions.internal_comparator);
t_opt.largest_seqno = kMaxSequenceNumber;
s = options.table_factory->NewTableReader(t_opt, std::move(file_reader),
file_size, &table_reader,
/*prefetch=*/false);
}
if (!s.ok()) {
std::cerr << "Failed to create TableReader for " << sst_file_path << ": "
<< s.ToString() << std::endl;
abort();
}
return table_reader.release();
}
ValueType ToValueType(OpType op_type) {
switch (op_type) {
case OpType::PUT:
return ValueType::kTypeValue;
case OpType::MERGE:
return ValueType::kTypeMerge;
case OpType::DELETE:
return ValueType::kTypeDeletion;
case OpType::DELETE_RANGE:
return ValueType::kTypeRangeDeletion;
default:
std::cerr << "Unknown operation type " << static_cast<int>(op_type)
<< std::endl;
abort();
}
}
// Fuzzes DB operations as input, let SstFileWriter generate a SST file
// according to the operations, then let SstFileReader read the generated SST
// file to check its checksum.
// according to the operations, then let TableReader read and check all the
// key-value pairs from the generated SST file.
DEFINE_PROTO_FUZZER(DBOperations& input) {
if (input.operations().empty()) {
return;
@ -48,53 +112,76 @@ DEFINE_PROTO_FUZZER(DBOperations& input) {
std::string sstfile;
{
auto fs = rocksdb::FileSystem::Default();
auto fs = FileSystem::Default();
std::string dir;
rocksdb::IOOptions opt;
rocksdb::IOStatus s = fs->GetTestDirectory(opt, &dir, nullptr);
CHECK_OK(s);
IOOptions opt;
CHECK_OK(fs->GetTestDirectory(opt, &dir, nullptr));
sstfile = dir + "/SstFileWriterFuzzer.sst";
}
Options options;
EnvOptions env_options(options);
ImmutableCFOptions cf_ioptions(options);
// Generate sst file.
rocksdb::Options options;
rocksdb::EnvOptions env_options;
rocksdb::SstFileWriter writer(env_options, options);
rocksdb::Status s = writer.Open(sstfile);
CHECK_OK(s);
SstFileWriter writer(env_options, options);
CHECK_OK(writer.Open(sstfile));
for (const DBOperation& op : input.operations()) {
switch (op.type()) {
case OpType::PUT:
s = writer.Put(op.key(), op.value());
CHECK_OK(s);
case OpType::PUT: {
CHECK_OK(writer.Put(op.key(), op.value()));
break;
case OpType::MERGE:
s = writer.Merge(op.key(), op.value());
CHECK_OK(s);
}
case OpType::MERGE: {
CHECK_OK(writer.Merge(op.key(), op.value()));
break;
case OpType::DELETE:
s = writer.Delete(op.key());
CHECK_OK(s);
}
case OpType::DELETE: {
CHECK_OK(writer.Delete(op.key()));
break;
case OpType::DELETE_RANGE:
s = writer.DeleteRange(op.key(), op.value());
CHECK_OK(s);
}
case OpType::DELETE_RANGE: {
CHECK_OK(writer.DeleteRange(op.key(), op.value()));
break;
default:
std::cerr << "Unsupported operation" << static_cast<int>(op.type());
return;
}
default: {
std::cerr << "Unsupported operation" << static_cast<int>(op.type())
<< std::endl;
abort();
}
}
rocksdb::ExternalSstFileInfo info;
s = writer.Finish(&info);
CHECK_OK(s);
}
ExternalSstFileInfo info;
CHECK_OK(writer.Finish(&info));
// Verify checksum.
rocksdb::SstFileReader reader(options);
s = reader.Open(sstfile);
CHECK_OK(s);
s = reader.VerifyChecksum();
CHECK_OK(s);
// Iterate and verify key-value pairs.
std::unique_ptr<TableReader> table_reader(
NewTableReader(sstfile, options, env_options, cf_ioptions));
ReadOptions roptions;
CHECK_OK(table_reader->VerifyChecksum(roptions,
TableReaderCaller::kUncategorized));
std::unique_ptr<InternalIterator> it(
table_reader->NewIterator(roptions, /*prefix_extractor=*/nullptr,
/*arena=*/nullptr, /*skip_filters=*/true,
TableReaderCaller::kUncategorized));
it->SeekToFirst();
for (const DBOperation& op : input.operations()) {
if (op.type() == OpType::DELETE_RANGE) {
// InternalIterator cannot iterate over DELETE_RANGE entries.
continue;
}
CHECK_TRUE(it->Valid());
ParsedInternalKey ikey;
CHECK_OK(ParseInternalKey(it->key(), &ikey, /*log_err_key=*/true));
CHECK_EQ(ikey.user_key.ToString(), op.key());
CHECK_EQ(ikey.sequence, 0);
CHECK_EQ(ikey.type, ToValueType(op.type()));
if (op.type() != OpType::DELETE) {
CHECK_EQ(op.value(), it->value().ToString());
}
it->Next();
}
CHECK_TRUE(!it->Valid());
// Delete sst file.
remove(sstfile.c_str());

23
fuzz/util.h Normal file
View File

@ -0,0 +1,23 @@
#pragma once
#define CHECK_OK(expression) \
do { \
auto status = (expression); \
if (!status.ok()) { \
std::cerr << status.ToString() << std::endl; \
abort(); \
} \
} while (0)
#define CHECK_EQ(a, b) \
if (a != b) { \
std::cerr << "(" << #a << "=" << a << ") != (" << #b << "=" << b << ")" \
<< std::endl; \
abort(); \
}
#define CHECK_TRUE(cond) \
if (!(cond)) { \
std::cerr << "\"" << #cond << "\" is false" << std::endl; \
abort(); \
}