name: facebook/rocksdb/pr-jobs on: [push, pull_request] jobs: # NOTE: multiple workflows would be recommended, but the current GHA UI in # PRs doesn't make it clear when there's an overall error with a workflow, # making it easy to overlook something broken. Grouping everything into one # workflow minimizes the problem because it will be suspicious if there are # no GHA results. # # The if: ${{ github.repository_owner == 'facebook' }} lines prevent the # jobs from attempting to run on repo forks, because of a few problems: # * runs-on labels are repository (owner) specific, so the job might wait # for days waiting for a runner that simply isn't available. # * Pushes to branches on forks for pull requests (the normal process) would # run the workflow jobs twice: once in the pull-from fork and once for the PR # destination repo. This is wasteful and dumb. # * It is not known how to avoid copy-pasting the line to each job, # increasing the risk of misconfiguration, especially on forks that might # want to run with this GHA setup. # # DEBUGGING WITH SSH: Temporarily add this as a job step, either before the # step of interest without the "if:" line or after the failing step with the # "if:" line. Then use ssh command printed in CI output. # - name: Setup tmate session # TEMPORARY! # if: ${{ failure() }} # uses: mxschmitt/action-tmate@v3 # with: # limit-access-to-actor: true # ======================== Fast Initial Checks ====================== # check-format-and-targets: if: ${{ github.repository_owner == 'facebook' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.1.0 with: fetch-depth: 0 # Need full checkout to determine merge base fetch-tags: true - uses: "./.github/actions/setup-upstream" - name: Setup Python uses: actions/setup-python@v5 - name: Install Dependencies run: python -m pip install --upgrade pip - name: Install argparse run: pip install argparse - name: Download clang-format-diff.py run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py - name: Check format run: VERBOSE_CHECK=1 make check-format - name: Compare buckify output run: make check-buck-targets - name: Simple source code checks run: make check-sources # ========================= Linux With Tests ======================== # build-linux: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: make V=1 J=32 -j32 check - uses: "./.github/actions/post-steps" build-linux-cmake-mingw: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix - name: Build cmake-mingw run: |- export PATH=$JAVA_HOME/bin:$PATH echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - uses: "./.github/actions/build-folly" - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-lite-no-test: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)" - uses: "./.github/actions/post-steps" build-linux-gcc-7-with-folly: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - uses: "./.github/actions/build-folly" - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check - uses: "./.github/actions/post-steps" build-linux-gcc-7-with-folly-lite-no-test: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-coroutines: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb env: CC: gcc-10 CXX: g++-10 steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - uses: "./.github/actions/build-folly" - run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" - uses: "./.github/actions/post-steps" build-linux-cmake-with-benchmark: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 - uses: "./.github/actions/post-steps" build-linux-encrypted_env-no_compression: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check - run: "./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression\n" - uses: "./.github/actions/post-steps" # ======================== Linux No Test Runs ======================= # build-linux-release: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - run: make V=1 -j32 LIB_MODE=shared release - run: ls librocksdb.so - run: "./db_stress --version" - run: make clean - run: make V=1 -j32 release - run: ls librocksdb.a - run: "./db_stress --version" - run: make clean - run: apt-get remove -y libgflags-dev - run: make V=1 -j32 LIB_MODE=shared release - run: ls librocksdb.so - run: if ./db_stress --version; then false; else true; fi - run: make clean - run: make V=1 -j32 release - run: ls librocksdb.a - run: if ./db_stress --version; then false; else true; fi - uses: "./.github/actions/post-steps" build-linux-release-rtti: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench - run: "./db_stress --version" - run: make clean - run: apt-get remove -y libgflags-dev - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench - run: if ./db_stress --version; then false; else true; fi build-examples: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - name: Build examples run: make V=1 -j4 static_lib && cd examples && make V=1 -j4 - uses: "./.github/actions/post-steps" build-fuzzers: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - name: Build rocksdb lib run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib - name: Build fuzzers run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer - uses: "./.github/actions/post-steps" build-linux-clang-no_test_run: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all - uses: "./.github/actions/post-steps" build-linux-clang-13-no_test_run: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench - uses: "./.github/actions/post-steps" build-linux-gcc-8-no_test_run: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all - uses: "./.github/actions/post-steps" build-linux-gcc-10-cxx20-no_test_run: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all - uses: "./.github/actions/post-steps" build-linux-gcc-11-no_test_run: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench - uses: "./.github/actions/post-steps" # ======================== Linux Other Checks ======================= # build-linux-clang10-clang-analyze: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze - uses: "./.github/actions/post-steps" - name: compress test report run: tar -cvzf scan_build_report.tar.gz scan_build_report if: failure() - uses: actions/upload-artifact@v4.0.0 with: name: scan-build-report path: scan_build_report.tar.gz build-linux-unity-and-headers: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: gcc:latest options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - run: apt-get update -y && apt-get install -y libgflags-dev - name: Unity build run: make V=1 -j8 unity_test - run: make V=1 -j8 -k check-headers - uses: "./.github/actions/post-steps" build-linux-mini-crashtest: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000 --use_io_uring=0' blackbox_crash_test_with_atomic_flush - uses: "./.github/actions/post-steps" # ======================= Linux with Sanitizers ===================== # build-linux-clang10-asan: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check - uses: "./.github/actions/post-steps" build-linux-clang10-ubsan: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check - uses: "./.github/actions/post-steps" build-linux-clang10-mini-tsan: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check - uses: "./.github/actions/post-steps" build-linux-static_lib-alt_namespace-status_checked: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check - uses: "./.github/actions/post-steps" # ========================= MacOS with Tests ======================== # build-macos: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-13 env: ROCKSDB_DISABLE_JEMALLOC: 1 steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 14.3.1 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: Run all tests run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j16 all - uses: "./.github/actions/post-steps" build-macos-cmake: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-13 strategy: matrix: run_even_tests: [true, false] steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 14.3.1 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: cmake generate project file run: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. - name: Build tests run: cd build && make V=1 -j16 - name: Run even tests run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2 if: ${{ matrix.run_even_tests }} - name: Run odd tests run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2 if: ${{ ! matrix.run_even_tests }} - uses: "./.github/actions/post-steps" # ======================== Windows with Tests ======================= # # NOTE: some windows jobs are in "nightly" to save resources build-windows-vs2019: if: ${{ github.repository_owner == 'facebook' }} runs-on: windows-2019 env: CMAKE_GENERATOR: Visual Studio 16 2019 CMAKE_PORTABLE: 1 steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/windows-build-steps" # ============================ Java Jobs ============================ # build-linux-java: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: evolvedbinary/rocksjava:centos6_x64-be options: --shm-size=16gb steps: # The docker image is intentionally based on an OS that has an older GLIBC version. # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step. - name: Checkout env: GH_TOKEN: ${{ github.token }} run: | chown `whoami` . || true git clone --no-checkout https://oath2:$GH_TOKEN@github.com/${{ github.repository }}.git . git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }} git checkout --progress --force ${{ github.ref }} git log -1 --format='%H' - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Test RocksDBJava run: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest' # NOTE: post-steps skipped because of compatibility issues with docker image build-linux-java-static: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: evolvedbinary/rocksjava:centos6_x64-be options: --shm-size=16gb steps: # The docker image is intentionally based on an OS that has an older GLIBC version. # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step. - name: Checkout env: GH_TOKEN: ${{ github.token }} run: | chown `whoami` . || true git clone --no-checkout https://oath2:$GH_TOKEN@github.com/${{ github.repository }}.git . git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }} git checkout --progress --force ${{ github.ref }} git log -1 --format='%H' - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Build RocksDBJava Static Library run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' # NOTE: post-steps skipped because of compatibility issues with docker image build-macos-java: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-13 env: JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" ROCKSDB_DISABLE_JEMALLOC: 1 steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 14.3.1 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/install-jdk8-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Test RocksDBJava run: make V=1 J=16 -j16 jtest - uses: "./.github/actions/post-steps" build-macos-java-static: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-13 env: JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 14.3.1 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/install-jdk8-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Build RocksDBJava x86 and ARM Static Libraries run: make V=1 J=16 -j16 rocksdbjavastaticosx - uses: "./.github/actions/post-steps" build-macos-java-static-universal: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-13 env: JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 14.3.1 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/install-jdk8-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Build RocksDBJava Universal Binary Static Library run: make V=1 J=16 -j16 rocksdbjavastaticosx_ub - uses: "./.github/actions/post-steps" build-linux-java-pmd: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: evolvedbinary/rocksjava:rockylinux8_x64-be options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/install-maven" - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: PMD RocksDBJava run: make V=1 J=8 -j8 jpmd - uses: actions/upload-artifact@v4.0.0 with: name: pmd-report path: "${{ github.workspace }}/java/target/pmd.xml" - uses: actions/upload-artifact@v4.0.0 with: name: maven-site path: "${{ github.workspace }}/java/target/site"