From 41237dd306b86982c2794cc60a381a017f3dbf59 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Mon, 18 Apr 2022 12:47:16 -0700 Subject: [PATCH] Add "no compression" job to CircleCI (#9850) Summary: Since they operate at distinct abstraction layers, I thought it was prudent to combine with EncryptedEnv CI test for each PR, for efficiency in testing. Also added supported compressions to sst_dump --help output so that CI job can verify no compiled-in compression support. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9850 Test Plan: CI, some manual stuff Reviewed By: riversand963 Differential Revision: D35682346 Pulled By: pdillinger fbshipit-source-id: be9879c1533fed304ee32c89fd9ba4b07c2b90cc --- .circleci/config.yml | 10 ++-- build_tools/fbcode_config.sh | 66 ++++++++++++++---------- build_tools/fbcode_config_platform009.sh | 46 ++++++++++------- build_tools/fbcode_config_platform010.sh | 48 ++++++++++------- tools/sst_dump_tool.cc | 16 +++++- 5 files changed, 116 insertions(+), 70 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index be32203c2e..acadd8e12b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -224,14 +224,16 @@ jobs: - run: make V=1 J=32 -j32 check - post-steps - build-linux-encrypted-env: + build-linux-encrypted_env-no_compression: machine: image: ubuntu-2004:202111-02 resource_class: 2xlarge steps: - pre-steps - install-gflags - - run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check + - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check + - run: | + ./sst_dump --help | egrep -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression - post-steps build-linux-shared_lib-alt_namespace-status_checked: @@ -803,9 +805,9 @@ workflows: jobs: - build-linux-cmake-with-folly - build-linux-cmake-with-benchmark - build-linux-encrypted-env: + build-linux-encrypted_env-no_compression: jobs: - - build-linux-encrypted-env + - build-linux-encrypted_env-no_compression build-linux-shared_lib-alt_namespace-status_checked: jobs: - build-linux-shared_lib-alt_namespace-status_checked diff --git a/build_tools/fbcode_config.sh b/build_tools/fbcode_config.sh index 060c88ad8d..4884a36272 100644 --- a/build_tools/fbcode_config.sh +++ b/build_tools/fbcode_config.sh @@ -21,38 +21,48 @@ LIBGCC_LIBS=" -L $LIBGCC_BASE/lib" GLIBC_INCLUDE="$GLIBC_BASE/include" GLIBC_LIBS=" -L $GLIBC_BASE/lib" -# snappy -SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" -if test -z $PIC_BUILD; then - SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" -else - SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a" -fi -CFLAGS+=" -DSNAPPY" - -if test -z $PIC_BUILD; then - # location of zlib headers and libraries - ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" - ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" - CFLAGS+=" -DZLIB" - - # location of bzip headers and libraries - BZIP_INCLUDE=" -I $BZIP2_BASE/include/" - BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a" - CFLAGS+=" -DBZIP2" - - LZ4_INCLUDE=" -I $LZ4_BASE/include/" - LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" - CFLAGS+=" -DLZ4" +if ! test $ROCKSDB_DISABLE_SNAPPY; then + # snappy + SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" + if test -z $PIC_BUILD; then + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" + else + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a" + fi + CFLAGS+=" -DSNAPPY" fi -ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" if test -z $PIC_BUILD; then - ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" -else - ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a" + if ! test $ROCKSDB_DISABLE_ZLIB; then + # location of zlib headers and libraries + ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" + ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" + CFLAGS+=" -DZLIB" + fi + + if ! test $ROCKSDB_DISABLE_BZIP; then + # location of bzip headers and libraries + BZIP_INCLUDE=" -I $BZIP2_BASE/include/" + BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a" + CFLAGS+=" -DBZIP2" + fi + + if ! test $ROCKSDB_DISABLE_LZ4; then + LZ4_INCLUDE=" -I $LZ4_BASE/include/" + LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" + CFLAGS+=" -DLZ4" + fi +fi + +if ! test $ROCKSDB_DISABLE_ZSTD; then + ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" + if test -z $PIC_BUILD; then + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" + else + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a" + fi + CFLAGS+=" -DZSTD -DZSTD_STATIC_LINKING_ONLY" fi -CFLAGS+=" -DZSTD -DZSTD_STATIC_LINKING_ONLY" # location of gflags headers and libraries GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" diff --git a/build_tools/fbcode_config_platform009.sh b/build_tools/fbcode_config_platform009.sh index 89ae4f098f..36e9a0f25c 100644 --- a/build_tools/fbcode_config_platform009.sh +++ b/build_tools/fbcode_config_platform009.sh @@ -27,28 +27,38 @@ else MAYBE_PIC=_pic fi -# snappy -SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" -SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a" -CFLAGS+=" -DSNAPPY" +if ! test $ROCKSDB_DISABLE_SNAPPY; then + # snappy + SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a" + CFLAGS+=" -DSNAPPY" +fi -# location of zlib headers and libraries -ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" -ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a" -CFLAGS+=" -DZLIB" +if ! test $ROCKSDB_DISABLE_ZLIB; then + # location of zlib headers and libraries + ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" + ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a" + CFLAGS+=" -DZLIB" +fi -# location of bzip headers and libraries -BZIP_INCLUDE=" -I $BZIP2_BASE/include/" -BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a" -CFLAGS+=" -DBZIP2" +if ! test $ROCKSDB_DISABLE_BZIP; then + # location of bzip headers and libraries + BZIP_INCLUDE=" -I $BZIP2_BASE/include/" + BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a" + CFLAGS+=" -DBZIP2" +fi -LZ4_INCLUDE=" -I $LZ4_BASE/include/" -LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a" -CFLAGS+=" -DLZ4" +if ! test $ROCKSDB_DISABLE_LZ4; then + LZ4_INCLUDE=" -I $LZ4_BASE/include/" + LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a" + CFLAGS+=" -DLZ4" +fi -ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" -ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a" -CFLAGS+=" -DZSTD" +if ! test $ROCKSDB_DISABLE_ZSTD; then + ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a" + CFLAGS+=" -DZSTD" +fi # location of gflags headers and libraries GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" diff --git a/build_tools/fbcode_config_platform010.sh b/build_tools/fbcode_config_platform010.sh index 346544f566..588dd88c25 100644 --- a/build_tools/fbcode_config_platform010.sh +++ b/build_tools/fbcode_config_platform010.sh @@ -29,28 +29,38 @@ else MAYBE_PIC=_pic fi -# snappy -SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" -SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a" -CFLAGS+=" -DSNAPPY" +if ! test $ROCKSDB_DISABLE_SNAPPY; then + # snappy + SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a" + CFLAGS+=" -DSNAPPY" +fi -# location of zlib headers and libraries -ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" -ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a" -CFLAGS+=" -DZLIB" +if ! test $ROCKSDB_DISABLE_ZLIB; then + # location of zlib headers and libraries + ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" + ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a" + CFLAGS+=" -DZLIB" +fi -# location of bzip headers and libraries -BZIP_INCLUDE=" -I $BZIP2_BASE/include/" -BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a" -CFLAGS+=" -DBZIP2" +if ! test $ROCKSDB_DISABLE_BZIP; then + # location of bzip headers and libraries + BZIP_INCLUDE=" -I $BZIP2_BASE/include/" + BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a" + CFLAGS+=" -DBZIP2" +fi -LZ4_INCLUDE=" -I $LZ4_BASE/include/" -LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a" -CFLAGS+=" -DLZ4" +if ! test $ROCKSDB_DISABLE_LZ4; then + LZ4_INCLUDE=" -I $LZ4_BASE/include/" + LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a" + CFLAGS+=" -DLZ4" +fi -ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" -ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a" -CFLAGS+=" -DZSTD" +if ! test $ROCKSDB_DISABLE_ZSTD; then + ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a" + CFLAGS+=" -DZSTD" +fi # location of gflags headers and libraries GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" @@ -108,7 +118,7 @@ if [ -z "$USE_CLANG" ]; then CXX="$GCC_BASE/bin/g++" AR="$GCC_BASE/bin/gcc-ar" - + CFLAGS+=" -B$BINUTILS" CFLAGS+=" -isystem $LIBGCC_INCLUDE" CFLAGS+=" -isystem $GLIBC_INCLUDE" diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc index 29048c00ea..622344e88d 100644 --- a/tools/sst_dump_tool.cc +++ b/tools/sst_dump_tool.cc @@ -10,7 +10,9 @@ #include #include +#include "options/options_helper.h" #include "port/port.h" +#include "rocksdb/convenience.h" #include "rocksdb/utilities/ldb_cmd.h" #include "table/sst_file_dumper.h" @@ -30,6 +32,16 @@ static const std::vector> namespace { void print_help(bool to_stderr) { + std::string supported_compressions; + for (CompressionType ct : GetSupportedCompressions()) { + if (!supported_compressions.empty()) { + supported_compressions += ", "; + } + std::string str; + Status s = GetStringFromCompressionType(&str, ct); + assert(s.ok()); + supported_compressions += str; + } fprintf( to_stderr ? stderr : stdout, R"(sst_dump --file= [--command=check|scan|raw|recompress|identify] @@ -88,6 +100,7 @@ void print_help(bool to_stderr) { kSnappyCompression> Can be combined with --command=recompress to run recompression for this list of compression types + Supported compression types: %s --parse_internal_key=<0xKEY> Convenience option to parse an internal key on the command line. Dumps the @@ -109,7 +122,8 @@ void print_help(bool to_stderr) { --compression_max_dict_buffer_bytes= Limit on buffer size from which we collect samples for dictionary generation. -)"); +)", + supported_compressions.c_str()); } // arg_name would include all prefix, e.g. "--my_arg="