rocksdb/.circleci/config.yml

version: 2.1

orbs:
  win: circleci/windows@2.4.0
  slack: circleci/slack@3.4.2

aliases:
  - &notify-on-main-failure
    fail_only: true
    only_for_branches: main

commands:
  install-cmake-on-macos:
    steps:
      - run:
          name: Install cmake on macos
          command: |
            HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake            

  install-jdk8-on-macos:
    steps:
      - run:
          name: Install JDK 8 on macos
          command: |
            brew install --cask adoptopenjdk/openjdk/adoptopenjdk8            

  increase-max-open-files-on-macos:
    steps:
      - run:
          name: Increase max open files
          command: |
            sudo sysctl -w kern.maxfiles=1048576
            sudo sysctl -w kern.maxfilesperproc=1048576
            sudo launchctl limit maxfiles 1048576            

  pre-steps:
    steps:
      - checkout
      - run:
          name: Setup Environment Variables
          command: |
            echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
            echo "export GTEST_COLOR=1" >> $BASH_ENV
            echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
            echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV
            echo "export ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $BASH_ENV
            echo "export BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $BASH_ENV
            echo "export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $BASH_ENV
            echo "export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $BASH_ENV
            echo "export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $BASH_ENV            

  pre-steps-macos:
      steps:
        - pre-steps

  post-steps:
    steps:
      - slack/status: *notify-on-main-failure
      - store_test_results: # store test result if there's any
          path: /tmp/test-results
      - store_artifacts: # store LOG for debugging if there's any
          path: LOG
      - run: # on fail, compress Test Logs for diagnosing the issue
         name: Compress Test Logs
         command: tar -cvzf t.tar.gz t
         when: on_fail
      - store_artifacts: # on fail, store Test Logs for diagnosing the issue
          path: t.tar.gz
          destination: test_logs
          when: on_fail

  install-clang-10:
    steps:
      - run:
          name: Install Clang 10
          command: |
            echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
            echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
            sudo apt-get update -y && sudo apt-get install -y clang-10            

  install-clang-13:
    steps:
      - run:
          name: Install Clang 13
          command: |
            echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
            echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
            wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
            sudo apt-get update -y && sudo apt-get install -y clang-13            

  install-gflags:
    steps:
      - run:
          name: Install gflags
          command: |
            sudo apt-get update -y && sudo apt-get install -y libgflags-dev            

  install-benchmark:
    steps:
      - run: # currently doesn't support ubuntu-1604 which doesn't have libbenchmark package, user can still install by building it youself
          name: Install benchmark
          command: |
            sudo apt-get update -y && sudo apt-get install -y libbenchmark-dev            

  upgrade-cmake:
    steps:
      - run:
          name: Upgrade cmake
          command: |
            sudo apt remove --purge cmake
            sudo snap install cmake --classic            

  install-gflags-on-macos:
    steps:
      - run:
          name: Install gflags on macos
          command: |
            HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags            

  install-gtest-parallel:
    steps:
      - run:
          name: Install gtest-parallel
          command: |
            git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
            echo 'export PATH=$HOME/gtest-parallel:$PATH' >> $BASH_ENV            

  install-compression-libs:
    steps:
      - run:
          name: Install compression libs
          command: |
            sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev            

executors:
  windows-2xlarge:
    machine:
      image: 'windows-server-2019-vs2019:stable'
      resource_class: windows.2xlarge
      shell: bash.exe

jobs:
  build-macos:
    macos:
      xcode: 12.5.1
    resource_class: large
    environment:
      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
      - pre-steps-macos
      - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-macos-cmake:
    macos:
      xcode: 12.5.1
    resource_class: large
    steps:
      - increase-max-open-files-on-macos
      - install-cmake-on-macos
      - install-gflags-on-macos
      - pre-steps-macos
      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - run: make V=1 J=32 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-encrypted-env:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-shared_lib-alt_namespace-status_checked:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-release:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
      - run: make V=1 -j8 release 2>&1 | .circleci/cat_ignore_eagain
      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
      - install-gflags
      - run: make V=1 -j8 release 2>&1 | .circleci/cat_ignore_eagain
      - run: ./db_stress --version # ensure with gflags
      - post-steps

  build-linux-release-rtti:
    machine:
      image: ubuntu-1604:201903-01
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
      - run: make clean
      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench 2>&1 | .circleci/cat_ignore_eagain
      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
      - run: make clean
      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench 2>&1 | .circleci/cat_ignore_eagain
      - run: ./db_stress --version # ensure with gflags

  build-linux-lite:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - run: LITE=1 make V=1 J=32 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-lite-release:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
      - run: LITE=1 make V=1 -j8 release 2>&1 | .circleci/cat_ignore_eagain
      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
      - install-gflags
      - run: LITE=1 make V=1 -j8 release 2>&1 | .circleci/cat_ignore_eagain
      - run: ./db_stress --version # ensure with gflags
      - post-steps

  build-linux-clang-no_test_run:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: xlarge
    steps:
      - checkout # check out the code in the project directory
      - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev libtbb-dev
      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-clang10-asan:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - install-clang-10
      - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check 2>&1 | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
      - post-steps

  build-linux-clang10-mini-tsan:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - install-clang-10
      - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check 2>&1 | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out.
      - post-steps

  build-linux-clang10-ubsan:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - install-clang-10
      - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check 2>&1 | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
      - post-steps

  build-linux-clang10-clang-analyze:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - install-clang-10
      - run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10
      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze 2>&1 | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
      - post-steps

  build-linux-cmake:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - upgrade-cmake
      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-cmake-ubuntu-20:
    machine:
      image: ubuntu-2004:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - install-benchmark
      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 && make microbench) 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-unity-and-headers:
    docker: # executor type
      - image: gcc:latest
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
      - run: apt-get update -y && apt-get install -y libgflags-dev
      - run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test 2>&1 | .circleci/cat_ignore_eagain
      - run: make V=1 -j8 -k check-headers 2>&1 | .circleci/cat_ignore_eagain # could be moved to a different build
      - post-steps

  build-linux-gcc-4_8-no_test_run:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
      - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j8 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  build-linux-gcc-8-no_test_run:
    machine:
      image: ubuntu-2004:202010-01
    resource_class: large
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
      - run: CC=gcc-8 CXX=g++-8 V=1 SKIP_LINK=1 make -j8 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  build-linux-gcc-9-no_test_run:
    machine:
      image: ubuntu-2004:202010-01
    resource_class: large
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo apt-get install gcc-9 g++-9 libgflags-dev
      - run: CC=gcc-9 CXX=g++-9 V=1 SKIP_LINK=1 make -j8 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  build-linux-gcc-10-cxx20-no_test_run:
    machine:
      image: ubuntu-2004:202010-01
    resource_class: xlarge
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
      - run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  build-linux-gcc-11-no_test_run:
    machine:
      image: ubuntu-2004:202010-01
    resource_class: xlarge
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get install gcc-11 g++-11 libgflags-dev
      - run: CC=gcc-11 CXX=g++-11 V=1 SKIP_LINK=1 make -j16 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  build-linux-clang-13-no_test_run:
    machine:
      image: ubuntu-2004:202010-01
    resource_class: xlarge
    steps:
      - pre-steps
      - install-clang-13
      - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j16 all 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
  build-linux-microbench:
    machine:
      image: ubuntu-2004:202010-01
    resource_class: xlarge
    steps:
      - pre-steps
      - install-benchmark
      - run: DEBUG_LEVEL=0 make microbench 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-windows:
    executor: windows-2xlarge
    parameters:
      extra_cmake_opt:
        default: ""
        type: string
      vs_year:
        default: "2019"
        type: string
      cmake_generator:
        default: "Visual Studio 16 2019"
        type: string
    environment:
      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
      CMAKE_HOME: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
      CMAKE_BIN: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.7
      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
      VS_YEAR: <<parameters.vs_year>>
      CMAKE_GENERATOR: <<parameters.cmake_generator>>
    steps:
      - checkout
      - run:
          name: "Setup VS"
          command: |
            if [[ "${VS_YEAR}" == "2019" ]]; then
              echo "VS2019 already present."
            elif [[ "${VS_YEAR}" == "2017" ]]; then
              echo "Installing VS2017..."
              powershell .circleci/vs2017_install.ps1
            elif [[ "${VS_YEAR}" == "2015" ]]; then
              echo "Installing VS2015..."
              powershell .circleci/vs2015_install.ps1
            fi            
      - store_artifacts:
          path: \Users\circleci\AppData\Local\Temp\vslogs.zip
      - run:
          name: "Install thirdparty dependencies"
          command: |
            mkdir ${THIRDPARTY_HOME}
            cd ${THIRDPARTY_HOME}
            echo "Installing CMake..."
            curl --fail --silent --show-error --output cmake-3.16.4-win64-x64.zip --location https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-win64-x64.zip
            unzip -q cmake-3.16.4-win64-x64.zip
            echo "Building Snappy dependency..."
            curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
            unzip -q snappy-1.1.7.zip
            cd snappy-1.1.7
            mkdir build
            cd build
            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" ..
            msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64            
      - run:
          name: "Build RocksDB"
          command: |
            mkdir build
            cd build
            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
            cd ..
            echo "Building with VS version: ${CMAKE_GENERATOR}"
            msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64            
      - run:
          name: "Test RocksDB"
          shell: powershell.exe
          command: |
            build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16            

  build-linux-java:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: large
    environment:
      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
    steps:
      - pre-steps
      - install-gflags
      - run:
          name: "Set Java Environment"
          command: |
            echo "JAVA_HOME=${JAVA_HOME}"
            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
            which java && java -version
            which javac && javac -version            
      - run:
          name: "Build RocksDBJava Shared Library"
          command: make V=1 J=8 -j8 rocksdbjava 2>&1 | .circleci/cat_ignore_eagain
      - run:
          name: "Test RocksDBJava"
          command: make V=1 J=8 -j8 jtest 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-java-static:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: large
    environment:
      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
    steps:
      - pre-steps
      - install-gflags
      - run:
          name: "Set Java Environment"
          command: |
            echo "JAVA_HOME=${JAVA_HOME}"
            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
            which java && java -version
            which javac && javac -version            
      - run:
          name: "Build RocksDBJava Static Library"
          command: make V=1 J=8 -j8 rocksdbjavastatic 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-macos-java:
    macos:
      xcode: 12.5.1
    resource_class: medium
    environment:
      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
      - install-jdk8-on-macos
      - pre-steps-macos
      - run:
          name: "Set Java Environment"
          command: |
            echo "JAVA_HOME=${JAVA_HOME}"
            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
            which java && java -version
            which javac && javac -version            
      - run:
          name: "Build RocksDBJava Shared Library"
          command: make V=1 J=8 -j8 rocksdbjava 2>&1 | .circleci/cat_ignore_eagain
      - run:
          name: "Test RocksDBJava"
          command: make V=1 J=8 -j8 jtest 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-macos-java-static:
    macos:
      xcode: 12.5.1
    resource_class: medium
    environment:
      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
      - install-cmake-on-macos
      - install-jdk8-on-macos
      - pre-steps-macos
      - run:
          name: "Set Java Environment"
          command: |
            echo "JAVA_HOME=${JAVA_HOME}"
            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
            which java && java -version
            which javac && javac -version            
      - run:
          name: "Build RocksDBJava x86 and ARM Static Libraries"
          command: make V=1 J=8 -j8 rocksdbjavastaticosx 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-macos-java-static-universal:
    macos:
      xcode: 12.5.1
    resource_class: medium
    environment:
      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
      - install-cmake-on-macos
      - install-jdk8-on-macos
      - pre-steps-macos
      - run:
          name: "Set Java Environment"
          command: |
            echo "JAVA_HOME=${JAVA_HOME}"
            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
            which java && java -version
            which javac && javac -version            
      - run:
          name: "Build RocksDBJava Universal Binary Static Library"
          command: make V=1 J=8 -j8 rocksdbjavastaticosx_ub 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-examples:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - pre-steps
      - install-gflags
      - run:
          name: "Build examples"
          command: |
            OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4 | ../.circleci/cat_ignore_eagain            
      - post-steps

  build-cmake-mingw:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - run: sudo apt-get update -y && sudo apt-get install -y mingw-w64
      - run: sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
      - run:
          name: "Build cmake-mingw"
          command: |
            sudo apt-get install snapd && sudo snap install cmake --beta --classic
            export PATH=/snap/bin:$PATH
            sudo apt-get install -y openjdk-8-jdk
            export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
            export PATH=$JAVA_HOME/bin:$PATH
            echo "JAVA_HOME=${JAVA_HOME}"
            which java && java -version
            which javac && javac -version
            mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni            
      - post-steps

  build-linux-non-shm:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    parameters:
      start_test:
        default: ""
        type: string
      end_test:
        default: ""
        type: string
    steps:
      - pre-steps
      - install-gflags
      - install-gtest-parallel
      - run:
          name: "Build unit tests"
          command: |
            echo "env: $(env)"
            echo "** done env"
            ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests            
      - run:
          name: "Run unit tests in parallel"
          command: |
            sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
            cat /tmp/test_list
            export TEST_TMPDIR=/tmp/rocksdb_test_tmp
            gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat  # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.            
      - post-steps

  build-linux-arm-test-full:
    machine:
      image: ubuntu-2004:202101-01
    resource_class: arm.large
    steps:
      - pre-steps
      - install-gflags
      - run: make V=1 J=4 -j4 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-arm:
    machine:
      image: ubuntu-2004:202101-01
    resource_class: arm.large
    steps:
      - pre-steps
      - install-gflags
      - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-arm-cmake-no_test_run:
    machine:
      image: ubuntu-2004:202101-01
    resource_class: arm.large
    environment:
      JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64
    steps:
      - pre-steps
      - install-gflags
      - run:
          name: "Set Java Environment"
          command: |
            echo "JAVA_HOME=${JAVA_HOME}"
            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
            which java && java -version
            which javac && javac -version            
      - run:
          name: "Build with cmake"
          command: |
            mkdir build
            cd build
            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
            make -j4            
      - run:
          name: "Build Java with cmake"
          command: |
            rm -rf build
            mkdir build
            cd build
            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
            make -j4 rocksdb rocksdbjni            
      - post-steps

  build-format-compatible:
    machine:
      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
      - install-gflags
      - install-compression-libs
      - run:
          name: "test"
          command: |
            export TEST_TMPDIR=/dev/shm/rocksdb
            rm -rf /dev/shm/rocksdb
            mkdir /dev/shm/rocksdb
            tools/check_format_compatible.sh            
      - post-steps

workflows:
  version: 2
  build-linux:
    jobs:
      - build-linux
  build-linux-cmake:
    jobs:
      - build-linux-cmake
      - build-linux-cmake-ubuntu-20
  build-linux-encrypted-env:
    jobs:
      - build-linux-encrypted-env
  build-linux-shared_lib-alt_namespace-status_checked:
    jobs:
      - build-linux-shared_lib-alt_namespace-status_checked
  build-linux-lite:
    jobs:
      - build-linux-lite
  build-linux-release:
    jobs:
      - build-linux-release
  build-linux-release-rtti:
    jobs:
      - build-linux-release-rtti
  build-linux-lite-release:
    jobs:
      - build-linux-lite-release
  build-linux-clang10-asan:
    jobs:
      - build-linux-clang10-asan
  build-linux-clang10-mini-tsan:
    jobs:
      - build-linux-clang10-mini-tsan
  build-linux-clang10-ubsan:
    jobs:
      - build-linux-clang10-ubsan
  build-linux-clang10-clang-analyze:
    jobs:
      - build-linux-clang10-clang-analyze
  build-linux-unity-and-headers:
    jobs:
      - build-linux-unity-and-headers
  build-windows-vs2019:
    jobs:
      - build-windows:
          name: "build-windows-vs2019"
  build-windows-vs2019-cxx20:
    jobs:
      - build-windows:
          name: "build-windows-vs2019-cxx20"
          extra_cmake_opt: -DCMAKE_CXX_STANDARD=20
  build-windows-vs2017:
    jobs:
      - build-windows:
          name: "build-windows-vs2017"
          vs_year: "2017"
          cmake_generator: "Visual Studio 15 Win64"
  build-java:
    jobs:
      - build-linux-java
      - build-linux-java-static
      - build-macos-java
      - build-macos-java-static
      - build-macos-java-static-universal
  build-examples:
    jobs:
      - build-examples
  build-linux-non-shm:
    jobs:
      - build-linux-non-shm:
          start_test: ""
          end_test: "db_options_test" # make sure unique in src.mk
      - build-linux-non-shm:
          start_test: "db_options_test" # make sure unique in src.mk
          end_test: "filename_test" # make sure unique in src.mk
      - build-linux-non-shm:
          start_test: "filename_test" # make sure unique in src.mk
          end_test: "statistics_test" # make sure unique in src.mk
      - build-linux-non-shm:
          start_test: "statistics_test" # make sure unique in src.mk
          end_test: ""
  build-linux-compilers-no_test_run:
    jobs:
      - build-linux-clang-no_test_run
      - build-linux-clang-13-no_test_run
      - build-linux-gcc-4_8-no_test_run
      - build-linux-gcc-8-no_test_run
      - build-linux-gcc-9-no_test_run
      - build-linux-gcc-10-cxx20-no_test_run
      - build-linux-gcc-11-no_test_run
      - build-linux-arm-cmake-no_test_run
  build-macos:
    jobs:
      - build-macos
  build-macos-cmake:
    jobs:
      - build-macos-cmake
  build-cmake-mingw:
    jobs:
      - build-cmake-mingw
  build-linux-arm:
    jobs:
      - build-linux-arm
  build-microbench:
    jobs:
      - build-linux-microbench
  nightly:
    triggers:
      - schedule:
          cron: "0 0 * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - build-format-compatible
      - build-linux-arm-test-full