Fix compilation errors and add fuzzers to CircleCI (#9420)

Summary:
This PR does the following:
- Fix compilation and linking errors when building fuzzer
- Add the above to CircleCI
- Update documentation

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9420

Test Plan: CI

Reviewed By: jay-zhuang

Differential Revision: D33849452

Pulled By: riversand963

fbshipit-source-id: 0794e5d04a3f53bfd2216fe2b3cd827ca2083ac3

.circleci/config.yml

@@ -136,6 +136,21 @@ commands:
           command: |
             sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
 
+  install-libprotobuf-mutator:
+    steps:
+      - run:
+          name: Install libprotobuf-mutator libs
+          command: |
+            git clone --single-branch --branch master --depth 1 git@github.com:google/libprotobuf-mutator.git ~/libprotobuf-mutator
+            cd ~/libprotobuf-mutator && mkdir build && cd build
+            cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON
+            ninja && sudo ninja install
+      - run:
+          name: Setup environment variables
+          command: |
+            echo "export PKG_CONFIG_PATH=/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/" >> $BASH_ENV
+            echo "export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc" >> $BASH_ENV
+
 executors:
   windows-2xlarge:
     machine:
@@ -732,6 +747,23 @@ jobs:
             tools/check_format_compatible.sh
       - post-steps
 
+  build-fuzzers:
+    machine:
+      image: ubuntu-2004:202010-01
+    resource_class: large
+    steps:
+      - pre-steps
+      - install-clang-13
+      - run: sudo apt-get update -y && sudo apt-get install -y cmake ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
+      - install-libprotobuf-mutator
+      - run:
+          name: "Build rocksdb lib"
+          command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
+      - run:
+          name: "Build fuzzers"
+          command: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
+      - post-steps
+
 workflows:
   version: 2
   build-linux:
@@ -838,6 +870,9 @@ workflows:
   build-microbench:
     jobs:
       - build-linux-microbench
+  build-fuzzers:
+    jobs:
+      - build-fuzzers
   nightly:
     triggers:
       - schedule:

fuzz/.gitignore

@@ -0,0 +1,5 @@
+db_fuzzer
+db_map_fuzzer
+sst_file_writer_fuzzer
+
+proto/gen/*

fuzz/Makefile

@@ -20,10 +20,10 @@ PROTO_IN = $(ROOT_DIR)/fuzz/proto
 PROTO_OUT = $(ROOT_DIR)/fuzz/proto/gen
 
 ifneq ($(FUZZ_ENV), ossfuzz)
-CC = clang++
+CC = $(CXX)
 CCFLAGS += -Wall -fsanitize=address,fuzzer
 CFLAGS += $(PLATFORM_CXXFLAGS) $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR)
-LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
+LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
 else
 # OSS-Fuzz sets various environment flags that are used for compilation.
 # These environment flags depend on which type of sanitizer build is being
@@ -42,15 +42,21 @@ CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKS
 LDFLAGS += $(PLATFORM_LDFLAGS) $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
 endif
 
-.PHONY: gen_proto
+.PHONY: gen_proto clean
+
+# Set PROTOC_BIN when invoking `make` if a custom protoc is required.
+PROTOC_BIN ?= protoc
 
 gen_proto:
 	mkdir -p $(PROTO_OUT)
-	protoc \
+	$(PROTOC_BIN) \
 		--proto_path=$(PROTO_IN) \
 		--cpp_out=$(PROTO_OUT) \
 		$(PROTO_IN)/*.proto
 
+clean:
+	rm -rf db_fuzzer db_map_fuzzer sst_file_writer_fuzzer $(PROTO_OUT)
+
 db_fuzzer: db_fuzzer.cc
 	$(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS)
 

fuzz/README.md

@@ -15,6 +15,11 @@ Some tests rely on [structure aware fuzzing](https://github.com/google/fuzzing/b
 We use [protobuf](https://developers.google.com/protocol-buffers) to define structured input to the fuzzer,
 and use [libprotobuf-mutator](https://github.com/google/libprotobuf-mutator) as the custom libFuzzer mutator.
 So make sure you have protobuf and libprotobuf-mutator installed, and make sure `pkg-config` can find them.
+On some systems, there are both protobuf2 and protobuf3 in the package management system,
+make sure protobuf3 is installed.
+
+If you do not want to install protobuf library yourself, you can rely on libprotobuf-mutator to download protobuf
+for you. For details about installation, please refer to [libprotobuf-mutator README](https://github.com/google/libprotobuf-mutator#readme)
 
 ## Example
 

fuzz/db_fuzzer.cc

@@ -153,6 +153,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
         delete iter;
         break;
       }
+      case OP_COUNT:
+        break;
     }
   }
 

fuzz/sst_file_writer_fuzzer.cc

@@ -12,9 +12,33 @@
 #include "rocksdb/file_system.h"
 #include "rocksdb/sst_file_writer.h"
 #include "src/libfuzzer/libfuzzer_macro.h"
+#include "table/table_builder.h"
 #include "table/table_reader.h"
 #include "util.h"
 
+using ROCKSDB_NAMESPACE::BytewiseComparator;
+using ROCKSDB_NAMESPACE::Comparator;
+using ROCKSDB_NAMESPACE::EnvOptions;
+using ROCKSDB_NAMESPACE::ExternalSstFileInfo;
+using ROCKSDB_NAMESPACE::FileOptions;
+using ROCKSDB_NAMESPACE::FileSystem;
+using ROCKSDB_NAMESPACE::ImmutableCFOptions;
+using ROCKSDB_NAMESPACE::ImmutableOptions;
+using ROCKSDB_NAMESPACE::InternalIterator;
+using ROCKSDB_NAMESPACE::IOOptions;
+using ROCKSDB_NAMESPACE::kMaxSequenceNumber;
+using ROCKSDB_NAMESPACE::Options;
+using ROCKSDB_NAMESPACE::ParsedInternalKey;
+using ROCKSDB_NAMESPACE::ParseInternalKey;
+using ROCKSDB_NAMESPACE::RandomAccessFileReader;
+using ROCKSDB_NAMESPACE::ReadOptions;
+using ROCKSDB_NAMESPACE::SstFileWriter;
+using ROCKSDB_NAMESPACE::Status;
+using ROCKSDB_NAMESPACE::TableReader;
+using ROCKSDB_NAMESPACE::TableReaderCaller;
+using ROCKSDB_NAMESPACE::TableReaderOptions;
+using ROCKSDB_NAMESPACE::ValueType;
+
 // Keys in SST file writer operations must be unique and in ascending order.
 // For each DBOperation generated by the fuzzer, this function is called on
 // it to deduplicate and sort the keys in the DBOperations.
@@ -60,15 +84,15 @@ TableReader* NewTableReader(const std::string& sst_file_path,
   std::unique_ptr<TableReader> table_reader;
   const auto& fs = options.env->GetFileSystem();
   FileOptions fopts(env_options);
-  Status s = options.env->GetFileSize(sst_file_path, fopts.io_options,
+  Status s = options.env->GetFileSize(sst_file_path, &file_size);
-                                      &file_size, nullptr);
   if (s.ok()) {
     s = RandomAccessFileReader::Create(fs, sst_file_path, fopts, &file_reader,
                                        nullptr);
   }
   if (s.ok()) {
-    TableReaderOptions t_opt(cf_ioptions, /*prefix_extractor=*/nullptr,
+    ImmutableOptions iopts(options, cf_ioptions);
-                             env_options, cf_ioptions.internal_comparator);
+    TableReaderOptions t_opt(iopts, /*prefix_extractor=*/nullptr, env_options,
+                             cf_ioptions.internal_comparator);
     t_opt.largest_seqno = kMaxSequenceNumber;
     s = options.table_factory->NewTableReader(t_opt, std::move(file_reader),
                                               file_size, &table_reader,
@@ -153,7 +177,7 @@ DEFINE_PROTO_FUZZER(DBOperations& input) {
 
   // Iterate and verify key-value pairs.
   std::unique_ptr<TableReader> table_reader(
-      NewTableReader(sstfile, options, env_options, cf_ioptions));
+      ::NewTableReader(sstfile, options, env_options, cf_ioptions));
   ReadOptions roptions;
   CHECK_OK(table_reader->VerifyChecksum(roptions,
                                         TableReaderCaller::kUncategorized));