mirror of
https://github.com/facebook/rocksdb.git
synced 2024-11-26 16:30:56 +00:00
fec5c8deb8
Summary: benchmark-linux is failing on main branch after https://github.com/facebook/rocksdb/issues/11074 with the following error msg: ``` /usr/bin/time -f '%e %U %S' -o /tmp/benchmark-results/8.0.0/benchmark_overwriteandwait.t1.s0.log.time numactl --interleave=all timeout 1200 ./db_bench --benchmarks=overwrite,waitforcompaction,stats --use_existing_db=1 --sync=0 --level0_file_num_compaction_trigger=4 --level0_slowdown_writes_trigger=20 --level0_stop_writes_trigger=30 --max_background_jobs=4 --max_write_buffer_number=8 --undefok=use_blob_cache,use_shared_block_and_blob_cache,blob_cache_size,blob_cache_numshardbits,prepopulate_blob_cache,multiread_batched,cache_low_pri_pool_ratio,prepopulate_block_cache --db=/tmp/rocksdb-benchmark-datadir --wal_dir=/tmp/rocksdb-benchmark-datadir --num=20000000 --key_size=20 --value_size=400 --block_size=8192 --cache_size=10737418240 --cache_numshardbits=6 --compression_max_dict_bytes=0 --compression_ratio=0.5 --compression_type=none --bytes_per_sync=1048576 --cache_index_and_filter_blocks=1 --cache_high_pri_pool_ratio=0.5 --cache_low_pri_pool_ratio=0 --benchmark_write_rate_limit=0 --write_buffer_size=16777216 --target_file_size_base=16777216 --max_bytes_for_level_base=67108864 --verify_checksum=1 --delete_obsolete_files_period_micros=62914560 --max_bytes_for_level_multiplier=8 --statistics=0 --stats_per_interval=1 --stats_interval_seconds=20 --report_interval_seconds=1 --histogram=1 --memtablerep=skip_list --bloom_bits=10 --open_files=-1 --subcompactions=1 --compaction_style=0 --num_levels=8 --min_level_to_compress=3 --level_compaction_dynamic_level_bytes=true --pin_l0_filter_and_index_blocks_in_cache=1 --duration=600 --threads=1 --merge_operator="put" --seed=1675372532 --report_file=/tmp/benchmark-results/8.0.0/benchmark_overwriteandwait.t1.s0.log.r.csv 2>&1 | tee -a /tmp/benchmark-results/8.0.0/benchmark_overwriteandwait.t1.s0.log /usr/bin/time: cannot run numactl: No such file or directory ``` This PR removes the newly added NUMA setting. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11180 Test Plan: check next main branch run for benchmark-linux Reviewed By: ajkr Differential Revision: D42975930 Pulled By: cbi42 fbshipit-source-id: f084d39aeba9877c0752502e879c5e612b507653
880 lines
28 KiB
YAML
880 lines
28 KiB
YAML
version: 2.1
|
|
|
|
orbs:
|
|
win: circleci/windows@5.0.0
|
|
|
|
commands:
|
|
install-cmake-on-macos:
|
|
steps:
|
|
- run:
|
|
name: Install cmake on macos
|
|
command: |
|
|
HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
|
|
|
|
install-jdk8-on-macos:
|
|
steps:
|
|
- run:
|
|
name: Install JDK 8 on macos
|
|
command: |
|
|
brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
|
|
|
|
increase-max-open-files-on-macos:
|
|
steps:
|
|
- run:
|
|
name: Increase max open files
|
|
command: |
|
|
sudo sysctl -w kern.maxfiles=1048576
|
|
sudo sysctl -w kern.maxfilesperproc=1048576
|
|
sudo launchctl limit maxfiles 1048576
|
|
|
|
pre-steps:
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Setup Environment Variables
|
|
command: |
|
|
echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
|
|
echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
|
|
echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
|
|
echo "export GTEST_COLOR=1" >> $BASH_ENV
|
|
echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
|
|
echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV
|
|
echo "export ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $BASH_ENV
|
|
echo "export BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $BASH_ENV
|
|
echo "export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $BASH_ENV
|
|
echo "export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $BASH_ENV
|
|
echo "export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $BASH_ENV
|
|
|
|
windows-build-steps:
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: "Install thirdparty dependencies"
|
|
command: |
|
|
echo "Installing CMake..."
|
|
choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y
|
|
mkdir $Env:THIRDPARTY_HOME
|
|
cd $Env:THIRDPARTY_HOME
|
|
echo "Building Snappy dependency..."
|
|
curl https://github.com/google/snappy/archive/refs/tags/1.1.8.zip -O snappy-1.1.8.zip
|
|
unzip -q snappy-1.1.8.zip
|
|
cd snappy-1.1.8
|
|
mkdir build
|
|
cd build
|
|
& $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" ..
|
|
msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
|
|
- run:
|
|
name: "Build RocksDB"
|
|
command: |
|
|
mkdir build
|
|
cd build
|
|
& $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 ..
|
|
cd ..
|
|
echo "Building with VS version: $Env:CMAKE_GENERATOR"
|
|
msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
|
|
- run:
|
|
name: "Test RocksDB"
|
|
shell: powershell.exe
|
|
command: |
|
|
build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
|
|
pre-steps-macos:
|
|
steps:
|
|
- pre-steps
|
|
|
|
post-steps:
|
|
steps:
|
|
- store_test_results: # store test result if there's any
|
|
path: /tmp/test-results
|
|
- store_artifacts: # store LOG for debugging if there's any
|
|
path: LOG
|
|
- run: # on fail, compress Test Logs for diagnosing the issue
|
|
name: Compress Test Logs
|
|
command: tar -cvzf t.tar.gz t
|
|
when: on_fail
|
|
- store_artifacts: # on fail, store Test Logs for diagnosing the issue
|
|
path: t.tar.gz
|
|
destination: test_logs
|
|
when: on_fail
|
|
- run: # store core dumps if there's any
|
|
command: |
|
|
mkdir -p /tmp/core_dumps
|
|
cp core.* /tmp/core_dumps
|
|
when: on_fail
|
|
- store_artifacts:
|
|
path: /tmp/core_dumps
|
|
when: on_fail
|
|
|
|
upgrade-cmake:
|
|
steps:
|
|
- run:
|
|
name: Upgrade cmake
|
|
command: |
|
|
sudo apt remove --purge cmake
|
|
sudo snap install cmake --classic
|
|
|
|
install-gflags:
|
|
steps:
|
|
- run:
|
|
name: Install gflags
|
|
command: |
|
|
sudo apt-get update -y && sudo apt-get install -y libgflags-dev
|
|
|
|
install-gflags-on-macos:
|
|
steps:
|
|
- run:
|
|
name: Install gflags on macos
|
|
command: |
|
|
HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
|
|
|
|
setup-folly:
|
|
steps:
|
|
- run:
|
|
name: Checkout folly sources
|
|
command: |
|
|
make checkout_folly
|
|
|
|
build-folly:
|
|
steps:
|
|
- run:
|
|
name: Build folly and dependencies
|
|
command: |
|
|
make build_folly
|
|
|
|
build-for-benchmarks:
|
|
steps:
|
|
- pre-steps
|
|
- run:
|
|
name: "Linux build for benchmarks"
|
|
command: #sized for the resource-class rocksdb-benchmark-sys1
|
|
make V=1 J=8 -j8 release
|
|
|
|
perform-benchmarks:
|
|
steps:
|
|
- run:
|
|
name: "Test low-variance benchmarks"
|
|
command: ./tools/benchmark_ci.py --db_dir /tmp/rocksdb-benchmark-datadir --output_dir /tmp/benchmark-results --num_keys 20000000
|
|
environment:
|
|
LD_LIBRARY_PATH: /usr/local/lib
|
|
# How long to run parts of the test(s)
|
|
DURATION_RO: 400
|
|
DURATION_RW: 600
|
|
# Keep threads within physical capacity of server (much lower than default)
|
|
NUM_THREADS: 1
|
|
MAX_BACKGROUND_JOBS: 4
|
|
# Don't run a couple of "optional" initial tests
|
|
CI_TESTS_ONLY: "true"
|
|
# Reduce configured size of levels to ensure more levels in the leveled compaction LSM tree
|
|
WRITE_BUFFER_SIZE_MB: 16
|
|
TARGET_FILE_SIZE_BASE_MB: 16
|
|
MAX_BYTES_FOR_LEVEL_BASE_MB: 64
|
|
# The benchmark host has 32GB memory
|
|
# The following values are tailored to work with that
|
|
# Note, tests may not exercise the targeted issues if the memory is increased on new test hosts.
|
|
COMPRESSION_TYPE: "none"
|
|
CACHE_INDEX_AND_FILTER_BLOCKS: 1
|
|
MIN_LEVEL_TO_COMPRESS: 3
|
|
CACHE_SIZE_MB: 10240
|
|
MB_WRITE_PER_SEC: 2
|
|
|
|
post-benchmarks:
|
|
steps:
|
|
- store_artifacts: # store the benchmark output
|
|
path: /tmp/benchmark-results
|
|
destination: test_logs
|
|
- run:
|
|
name: Send benchmark report to visualisation
|
|
command: |
|
|
set +e
|
|
set +o pipefail
|
|
./build_tools/benchmark_log_tool.py --tsvfile /tmp/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc
|
|
true
|
|
|
|
executors:
|
|
linux-docker:
|
|
docker:
|
|
# The image configuration is build_tools/ubuntu20_image/Dockerfile
|
|
# To update and build the image:
|
|
# $ cd build_tools/ubuntu20_image
|
|
# $ docker build -t zjay437/rocksdb:0.5 .
|
|
# $ docker push zjay437/rocksdb:0.5
|
|
# `zjay437` is the account name for zjay@meta.com which readwrite token is shared internally. To login:
|
|
# $ docker login --username zjay437
|
|
# Or please feel free to change it to your docker hub account for hosting the image, meta employee should already have the account and able to login with SSO.
|
|
# To avoid impacting the existing CI runs, please bump the version every time creating a new image
|
|
# to run the CI image environment locally:
|
|
# $ docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it zjay437/rocksdb:0.5 bash
|
|
# option `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined` is used to enable gdb to attach an existing process
|
|
- image: zjay437/rocksdb:0.6
|
|
|
|
jobs:
|
|
build-macos:
|
|
macos:
|
|
xcode: 12.5.1
|
|
resource_class: large
|
|
environment:
|
|
ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now
|
|
steps:
|
|
- increase-max-open-files-on-macos
|
|
- install-gflags-on-macos
|
|
- pre-steps-macos
|
|
- run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=32 -j32 all
|
|
- post-steps
|
|
|
|
build-macos-cmake:
|
|
macos:
|
|
xcode: 12.5.1
|
|
resource_class: large
|
|
parameters:
|
|
run_even_tests:
|
|
description: run even or odd tests, used to split tests to 2 groups
|
|
type: boolean
|
|
default: true
|
|
steps:
|
|
- increase-max-open-files-on-macos
|
|
- install-cmake-on-macos
|
|
- install-gflags-on-macos
|
|
- pre-steps-macos
|
|
- run:
|
|
name: "cmake generate project file"
|
|
command: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
|
|
- run:
|
|
name: "Build tests"
|
|
command: cd build && make V=1 -j32
|
|
- when:
|
|
condition: << parameters.run_even_tests >>
|
|
steps:
|
|
- run:
|
|
name: "Run even tests"
|
|
command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 0,,2
|
|
- when:
|
|
condition:
|
|
not: << parameters.run_even_tests >>
|
|
steps:
|
|
- run:
|
|
name: "Run odd tests"
|
|
command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 1,,2
|
|
- post-steps
|
|
|
|
build-linux:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: make V=1 J=32 -j32 check
|
|
- post-steps
|
|
|
|
build-linux-encrypted_env-no_compression:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
|
|
- run: |
|
|
./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression
|
|
- post-steps
|
|
|
|
build-linux-shared_lib-alt_namespace-status_checked:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check
|
|
- post-steps
|
|
|
|
build-linux-release:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: make V=1 -j32 release
|
|
- run: ./db_stress --version # ensure with gflags
|
|
- run: make clean
|
|
- run: apt-get remove -y libgflags-dev
|
|
- run: make V=1 -j32 release
|
|
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
|
|
- post-steps
|
|
|
|
build-linux-release-rtti:
|
|
executor: linux-docker
|
|
resource_class: xlarge
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
|
|
- run: ./db_stress --version # ensure with gflags
|
|
- run: make clean
|
|
- run: apt-get remove -y libgflags-dev
|
|
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
|
|
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
|
|
|
|
build-linux-clang-no_test_run:
|
|
executor: linux-docker
|
|
resource_class: xlarge
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all
|
|
- post-steps
|
|
|
|
build-linux-clang10-asan:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out
|
|
- post-steps
|
|
|
|
build-linux-clang10-mini-tsan:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge+
|
|
steps:
|
|
- pre-steps
|
|
- run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
|
|
- post-steps
|
|
|
|
build-linux-clang10-ubsan:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check # aligned new doesn't work for reason we haven't figured out
|
|
- post-steps
|
|
|
|
build-linux-valgrind:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: PORTABLE=1 make V=1 -j32 valgrind_test
|
|
- post-steps
|
|
|
|
build-linux-clang10-clang-analyze:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
|
|
- post-steps
|
|
- run:
|
|
name: "compress test report"
|
|
command: tar -cvzf scan_build_report.tar.gz scan_build_report
|
|
when: on_fail
|
|
- store_artifacts:
|
|
path: scan_build_report.tar.gz
|
|
destination: scan_build_report
|
|
when: on_fail
|
|
|
|
build-linux-runner:
|
|
machine: true
|
|
resource_class: facebook/rocksdb-benchmark-sys1
|
|
steps:
|
|
- pre-steps
|
|
- run:
|
|
name: "Checked Linux build (Runner)"
|
|
command: make V=1 J=8 -j8 check
|
|
environment:
|
|
LD_LIBRARY_PATH: /usr/local/lib
|
|
- post-steps
|
|
|
|
build-linux-cmake-with-folly:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- setup-folly
|
|
- build-folly
|
|
- run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)
|
|
- post-steps
|
|
|
|
build-linux-cmake-with-folly-lite-no-test:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- setup-folly
|
|
- run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)
|
|
- post-steps
|
|
|
|
build-linux-cmake-with-benchmark:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20
|
|
- post-steps
|
|
|
|
build-linux-unity-and-headers:
|
|
docker: # executor type
|
|
- image: gcc:latest
|
|
environment:
|
|
EXTRA_CXXFLAGS: -mno-avx512f # Warnings-as-error in avx512fintrin.h, would be used on newer hardware
|
|
resource_class: large
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: apt-get update -y && apt-get install -y libgflags-dev
|
|
- run: make V=1 -j8 unity_test
|
|
- run: make V=1 -j8 -k check-headers # could be moved to a different build
|
|
- post-steps
|
|
|
|
build-linux-gcc-7-with-folly:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- setup-folly
|
|
- build-folly
|
|
- run: USE_FOLLY=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 check
|
|
- post-steps
|
|
|
|
build-linux-gcc-7-with-folly-lite-no-test:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- setup-folly
|
|
- run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all
|
|
- post-steps
|
|
|
|
build-linux-gcc-8-no_test_run:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all
|
|
- post-steps
|
|
|
|
build-linux-cmake-with-folly-coroutines:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
environment:
|
|
CC: gcc-10
|
|
CXX: g++-10
|
|
steps:
|
|
- pre-steps
|
|
- setup-folly
|
|
- build-folly
|
|
- run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)
|
|
- post-steps
|
|
|
|
build-linux-gcc-10-cxx20-no_test_run:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all
|
|
- post-steps
|
|
|
|
build-linux-gcc-11-no_test_run:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench
|
|
- post-steps
|
|
|
|
build-linux-clang-13-no_test_run:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench
|
|
- post-steps
|
|
|
|
# Ensure ASAN+UBSAN with folly, and full testsuite with clang 13
|
|
build-linux-clang-13-asan-ubsan-with-folly:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- setup-folly
|
|
- build-folly
|
|
- run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check
|
|
- post-steps
|
|
|
|
# This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
|
|
build-linux-run-microbench:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run: DEBUG_LEVEL=0 make -j32 run_microbench
|
|
- post-steps
|
|
|
|
build-linux-mini-crashtest:
|
|
executor: linux-docker
|
|
resource_class: large
|
|
steps:
|
|
- pre-steps
|
|
- run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000' blackbox_crash_test_with_atomic_flush
|
|
- post-steps
|
|
|
|
build-linux-crashtest-tiered-storage-bb:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run:
|
|
name: "run crashtest"
|
|
command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS=--duration=10800 blackbox_crash_test_with_tiered_storage
|
|
no_output_timeout: 100m
|
|
- post-steps
|
|
|
|
build-linux-crashtest-tiered-storage-wb:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run:
|
|
name: "run crashtest"
|
|
command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS=--duration=10800 whitebox_crash_test_with_tiered_storage
|
|
no_output_timeout: 100m
|
|
- post-steps
|
|
|
|
build-windows-vs2022:
|
|
executor:
|
|
name: win/server-2022
|
|
size: 2xlarge
|
|
environment:
|
|
THIRDPARTY_HOME: C:/Users/circleci/thirdparty
|
|
CMAKE_HOME: C:/Program Files/CMake
|
|
CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
|
|
SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
|
|
SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
|
|
SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
|
|
CMAKE_GENERATOR: Visual Studio 17 2022
|
|
steps:
|
|
- windows-build-steps
|
|
|
|
build-windows-vs2019:
|
|
executor:
|
|
name: win/server-2019
|
|
size: 2xlarge
|
|
environment:
|
|
THIRDPARTY_HOME: C:/Users/circleci/thirdparty
|
|
CMAKE_HOME: C:/Program Files/CMake
|
|
CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
|
|
SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
|
|
SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
|
|
SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
|
|
CMAKE_GENERATOR: Visual Studio 16 2019
|
|
steps:
|
|
- windows-build-steps
|
|
|
|
build-linux-java:
|
|
executor: linux-docker
|
|
resource_class: large
|
|
steps:
|
|
- pre-steps
|
|
- run:
|
|
name: "Set Java Environment"
|
|
command: |
|
|
echo "JAVA_HOME=${JAVA_HOME}"
|
|
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
|
|
which java && java -version
|
|
which javac && javac -version
|
|
- run:
|
|
name: "Test RocksDBJava"
|
|
command: make V=1 J=8 -j8 jtest
|
|
- post-steps
|
|
|
|
build-linux-java-static:
|
|
executor: linux-docker
|
|
resource_class: large
|
|
steps:
|
|
- pre-steps
|
|
- run:
|
|
name: "Set Java Environment"
|
|
command: |
|
|
echo "JAVA_HOME=${JAVA_HOME}"
|
|
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
|
|
which java && java -version
|
|
which javac && javac -version
|
|
- run:
|
|
name: "Build RocksDBJava Static Library"
|
|
command: make V=1 J=8 -j8 rocksdbjavastatic
|
|
- post-steps
|
|
|
|
build-macos-java:
|
|
macos:
|
|
xcode: 12.5.1
|
|
resource_class: large
|
|
environment:
|
|
JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
|
|
ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash
|
|
steps:
|
|
- increase-max-open-files-on-macos
|
|
- install-gflags-on-macos
|
|
- install-jdk8-on-macos
|
|
- pre-steps-macos
|
|
- run:
|
|
name: "Set Java Environment"
|
|
command: |
|
|
echo "JAVA_HOME=${JAVA_HOME}"
|
|
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
|
|
which java && java -version
|
|
which javac && javac -version
|
|
- run:
|
|
name: "Test RocksDBJava"
|
|
command: make V=1 J=16 -j16 jtest
|
|
no_output_timeout: 20m
|
|
- post-steps
|
|
|
|
build-macos-java-static:
|
|
macos:
|
|
xcode: 12.5.1
|
|
resource_class: large
|
|
environment:
|
|
JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
|
|
steps:
|
|
- increase-max-open-files-on-macos
|
|
- install-gflags-on-macos
|
|
- install-cmake-on-macos
|
|
- install-jdk8-on-macos
|
|
- pre-steps-macos
|
|
- run:
|
|
name: "Set Java Environment"
|
|
command: |
|
|
echo "JAVA_HOME=${JAVA_HOME}"
|
|
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
|
|
which java && java -version
|
|
which javac && javac -version
|
|
- run:
|
|
name: "Build RocksDBJava x86 and ARM Static Libraries"
|
|
command: make V=1 J=16 -j16 rocksdbjavastaticosx
|
|
no_output_timeout: 20m
|
|
- post-steps
|
|
|
|
build-macos-java-static-universal:
|
|
macos:
|
|
xcode: 12.5.1
|
|
resource_class: large
|
|
environment:
|
|
JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
|
|
steps:
|
|
- increase-max-open-files-on-macos
|
|
- install-gflags-on-macos
|
|
- install-cmake-on-macos
|
|
- install-jdk8-on-macos
|
|
- pre-steps-macos
|
|
- run:
|
|
name: "Set Java Environment"
|
|
command: |
|
|
echo "JAVA_HOME=${JAVA_HOME}"
|
|
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
|
|
which java && java -version
|
|
which javac && javac -version
|
|
- run:
|
|
name: "Build RocksDBJava Universal Binary Static Library"
|
|
command: make V=1 J=16 -j16 rocksdbjavastaticosx_ub
|
|
no_output_timeout: 20m
|
|
- post-steps
|
|
|
|
build-examples:
|
|
executor: linux-docker
|
|
resource_class: large
|
|
steps:
|
|
- pre-steps
|
|
- run:
|
|
name: "Build examples"
|
|
command: |
|
|
make V=1 -j4 static_lib && cd examples && make V=1 -j4
|
|
- post-steps
|
|
|
|
build-cmake-mingw:
|
|
executor: linux-docker
|
|
resource_class: large
|
|
steps:
|
|
- pre-steps
|
|
- run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
|
|
- run:
|
|
name: "Build cmake-mingw"
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
echo "JAVA_HOME=${JAVA_HOME}"
|
|
which java && java -version
|
|
which javac && javac -version
|
|
mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
|
|
- post-steps
|
|
|
|
build-linux-non-shm:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
environment:
|
|
TEST_TMPDIR: /tmp/rocksdb_test_tmp
|
|
steps:
|
|
- pre-steps
|
|
- run: make V=1 -j32 check
|
|
- post-steps
|
|
|
|
build-linux-arm-test-full:
|
|
machine:
|
|
image: ubuntu-2004:202111-02
|
|
resource_class: arm.large
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- run: make V=1 J=4 -j4 check
|
|
- post-steps
|
|
|
|
build-linux-arm:
|
|
machine:
|
|
image: ubuntu-2004:202111-02
|
|
resource_class: arm.large
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some
|
|
- post-steps
|
|
|
|
build-linux-arm-cmake-no_test_run:
|
|
machine:
|
|
image: ubuntu-2004:202111-02
|
|
resource_class: arm.large
|
|
environment:
|
|
JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- run:
|
|
name: "Set Java Environment"
|
|
command: |
|
|
echo "JAVA_HOME=${JAVA_HOME}"
|
|
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
|
|
which java && java -version
|
|
which javac && javac -version
|
|
- run:
|
|
name: "Build with cmake"
|
|
command: |
|
|
mkdir build
|
|
cd build
|
|
cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
|
|
make -j4
|
|
- run:
|
|
name: "Build Java with cmake"
|
|
command: |
|
|
rm -rf build
|
|
mkdir build
|
|
cd build
|
|
cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
|
|
make -j4 rocksdb rocksdbjni
|
|
- post-steps
|
|
|
|
build-format-compatible:
|
|
executor: linux-docker
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- run:
|
|
name: "test"
|
|
command: |
|
|
export TEST_TMPDIR=/dev/shm/rocksdb
|
|
rm -rf /dev/shm/rocksdb
|
|
mkdir /dev/shm/rocksdb
|
|
tools/check_format_compatible.sh
|
|
- post-steps
|
|
|
|
build-fuzzers:
|
|
executor: linux-docker
|
|
resource_class: large
|
|
steps:
|
|
- pre-steps
|
|
- run:
|
|
name: "Build rocksdb lib"
|
|
command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
|
|
- run:
|
|
name: "Build fuzzers"
|
|
command: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
|
|
- post-steps
|
|
|
|
benchmark-linux: #use a private Circle CI runner (resource_class) to run the job
|
|
machine: true
|
|
resource_class: facebook/rocksdb-benchmark-sys1
|
|
steps:
|
|
- build-for-benchmarks
|
|
- perform-benchmarks
|
|
- post-benchmarks
|
|
|
|
workflows:
|
|
version: 2
|
|
jobs-linux-run-tests:
|
|
jobs:
|
|
- build-linux
|
|
- build-linux-cmake-with-folly
|
|
- build-linux-cmake-with-folly-lite-no-test
|
|
- build-linux-gcc-7-with-folly
|
|
- build-linux-gcc-7-with-folly-lite-no-test
|
|
- build-linux-cmake-with-folly-coroutines
|
|
- build-linux-cmake-with-benchmark
|
|
- build-linux-encrypted_env-no_compression
|
|
jobs-linux-run-tests-san:
|
|
jobs:
|
|
- build-linux-clang10-asan
|
|
- build-linux-clang10-ubsan
|
|
- build-linux-clang10-mini-tsan
|
|
- build-linux-shared_lib-alt_namespace-status_checked
|
|
jobs-linux-no-test-run:
|
|
jobs:
|
|
- build-linux-release
|
|
- build-linux-release-rtti
|
|
- build-examples
|
|
- build-fuzzers
|
|
- build-linux-clang-no_test_run
|
|
- build-linux-clang-13-no_test_run
|
|
- build-linux-gcc-8-no_test_run
|
|
- build-linux-gcc-10-cxx20-no_test_run
|
|
- build-linux-gcc-11-no_test_run
|
|
- build-linux-arm-cmake-no_test_run
|
|
jobs-linux-other-checks:
|
|
jobs:
|
|
- build-linux-clang10-clang-analyze
|
|
- build-linux-unity-and-headers
|
|
- build-linux-mini-crashtest
|
|
jobs-windows:
|
|
jobs:
|
|
- build-windows-vs2022
|
|
- build-windows-vs2019
|
|
- build-cmake-mingw
|
|
jobs-java:
|
|
jobs:
|
|
- build-linux-java
|
|
- build-linux-java-static
|
|
- build-macos-java
|
|
- build-macos-java-static
|
|
- build-macos-java-static-universal
|
|
jobs-macos:
|
|
jobs:
|
|
- build-macos
|
|
- build-macos-cmake:
|
|
run_even_tests: true
|
|
- build-macos-cmake:
|
|
run_even_tests: false
|
|
jobs-linux-arm:
|
|
jobs:
|
|
- build-linux-arm
|
|
build-fuzzers:
|
|
jobs:
|
|
- build-fuzzers
|
|
benchmark-linux:
|
|
triggers:
|
|
- schedule:
|
|
cron: "0 * * * *"
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
jobs:
|
|
- benchmark-linux
|
|
nightly:
|
|
triggers:
|
|
- schedule:
|
|
cron: "0 9 * * *"
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
jobs:
|
|
- build-format-compatible
|
|
- build-linux-arm-test-full
|
|
- build-linux-run-microbench
|
|
- build-linux-non-shm
|
|
- build-linux-clang-13-asan-ubsan-with-folly
|
|
- build-linux-valgrind
|