2020-03-14 04:51:26 +00:00
version : 2.1
orbs :
win : circleci/windows@2.4.0
2020-09-23 21:40:53 +00:00
aliases :
2021-09-01 18:56:37 +00:00
- ¬ify-on-main-failure
2020-09-23 21:40:53 +00:00
fail_only : true
2021-09-01 18:56:37 +00:00
only_for_branches : main
2020-09-23 21:40:53 +00:00
commands :
2020-12-05 07:22:18 +00:00
install-cmake-on-macos :
steps :
- run :
name : Install cmake on macos
command : |
HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
2022-01-13 01:19:41 +00:00
install-jdk8-on-macos :
steps :
- run :
name : Install JDK 8 on macos
command : |
brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
2020-10-12 17:45:21 +00:00
increase-max-open-files-on-macos :
steps :
- run :
name : Increase max open files
command : |
sudo sysctl -w kern.maxfiles=1048576
sudo sysctl -w kern.maxfilesperproc=1048576
sudo launchctl limit maxfiles 1048576
2020-09-23 21:40:53 +00:00
pre-steps :
steps :
- checkout
- run :
name : Setup Environment Variables
command : |
echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
2021-06-01 21:42:06 +00:00
echo "export GTEST_COLOR=1" >> $BASH_ENV
echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV
2022-01-21 17:32:03 +00:00
echo "export ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $BASH_ENV
echo "export BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $BASH_ENV
echo "export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $BASH_ENV
echo "export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $BASH_ENV
echo "export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $BASH_ENV
2020-09-23 21:40:53 +00:00
2020-12-16 23:58:56 +00:00
pre-steps-macos :
steps :
2022-01-21 17:32:03 +00:00
- pre-steps
2020-12-16 23:58:56 +00:00
2020-09-23 21:40:53 +00:00
post-steps :
steps :
- store_test_results : # store test result if there's any
path : /tmp/test-results
- store_artifacts : # store LOG for debugging if there's any
path : LOG
2021-01-08 01:11:50 +00:00
- run : # on fail, compress Test Logs for diagnosing the issue
name : Compress Test Logs
command : tar -cvzf t.tar.gz t
when : on_fail
- store_artifacts : # on fail, store Test Logs for diagnosing the issue
path : t.tar.gz
destination : test_logs
when : on_fail
2020-09-23 21:40:53 +00:00
install-clang-10 :
steps :
- run :
name : Install Clang 10
command : |
2022-02-05 01:12:03 +00:00
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
2020-09-23 21:40:53 +00:00
echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
sudo apt-get update -y && sudo apt-get install -y clang-10
2022-01-11 18:35:22 +00:00
install-clang-13 :
steps :
- run :
name : Install Clang 13
command : |
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
sudo apt-get update -y && sudo apt-get install -y clang-13
2020-09-23 21:40:53 +00:00
install-gflags :
steps :
- run :
name : Install gflags
command : |
sudo apt-get update -y && sudo apt-get install -y libgflags-dev
2020-12-05 07:22:18 +00:00
2021-07-09 00:50:55 +00:00
install-benchmark :
steps :
2022-03-30 17:09:49 +00:00
- run :
name : Install ninja build
command : sudo apt-get update -y && sudo apt-get install -y ninja-build
2022-02-07 01:35:50 +00:00
- run :
2021-07-09 00:50:55 +00:00
name : Install benchmark
command : |
2022-03-30 17:09:49 +00:00
git clone --depth 1 --branch v1.6.1 https://github.com/google/benchmark.git ~/benchmark
cd ~/benchmark && mkdir build && cd build
cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0
ninja && sudo ninja install
2021-07-09 00:50:55 +00:00
2022-04-19 03:26:37 +00:00
install-valgrind :
steps :
- run :
name : Install valgrind
command : sudo apt-get update -y && sudo apt-get install -y valgrind
2021-06-01 21:42:06 +00:00
upgrade-cmake :
steps :
- run :
name : Upgrade cmake
command : |
sudo apt remove --purge cmake
sudo snap install cmake --classic
2020-10-12 17:45:21 +00:00
install-gflags-on-macos :
steps :
- run :
name : Install gflags on macos
command : |
HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
2020-09-23 21:40:53 +00:00
install-gtest-parallel :
steps :
- run :
name : Install gtest-parallel
command : |
2020-09-30 23:06:44 +00:00
git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
2020-12-11 04:36:44 +00:00
echo 'export PATH=$HOME/gtest-parallel:$PATH' >> $BASH_ENV
2020-03-14 04:51:26 +00:00
2021-02-10 04:47:06 +00:00
install-compression-libs :
steps :
- run :
name : Install compression libs
command : |
sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
2022-02-01 18:29:56 +00:00
install-libprotobuf-mutator :
steps :
- run :
name : Install libprotobuf-mutator libs
command : |
git clone --single-branch --branch master --depth 1 git@github.com:google/libprotobuf-mutator.git ~/libprotobuf-mutator
cd ~/libprotobuf-mutator && mkdir build && cd build
cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON
ninja && sudo ninja install
- run :
name : Setup environment variables
command : |
echo "export PKG_CONFIG_PATH=/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/" >> $BASH_ENV
echo "export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc" >> $BASH_ENV
2020-03-14 04:51:26 +00:00
executors :
windows-2xlarge :
machine :
2021-06-18 19:31:08 +00:00
image : 'windows-server-2019-vs2019:stable'
2020-03-14 04:51:26 +00:00
resource_class : windows.2xlarge
shell : bash.exe
jobs :
2020-10-12 17:45:21 +00:00
build-macos :
macos :
2022-01-20 17:40:34 +00:00
xcode : 12.5 .1
2021-05-22 01:16:05 +00:00
resource_class : large
2022-01-20 17:40:34 +00:00
environment :
2022-01-21 17:32:03 +00:00
ROCKSDB_DISABLE_JEMALLOC : 1 # jemalloc cause env_test hang, disable it for now
2020-10-12 17:45:21 +00:00
steps :
- increase-max-open-files-on-macos
- install-gflags-on-macos
2020-12-16 23:58:56 +00:00
- pre-steps-macos
2022-05-10 16:51:03 +00:00
- run : ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=32 -j32 all
2020-10-12 17:45:21 +00:00
- post-steps
2020-12-05 07:22:18 +00:00
build-macos-cmake :
macos :
2022-01-20 17:40:34 +00:00
xcode : 12.5 .1
2021-05-22 01:16:05 +00:00
resource_class : large
2022-03-03 19:58:51 +00:00
parameters :
run_even_tests :
description : run even or odd tests, used to split tests to 2 groups
type : boolean
default : true
2020-12-05 07:22:18 +00:00
steps :
- increase-max-open-files-on-macos
- install-cmake-on-macos
- install-gflags-on-macos
2020-12-16 23:58:56 +00:00
- pre-steps-macos
2022-03-03 19:58:51 +00:00
- run :
name : "cmake generate project file"
2022-05-10 16:51:03 +00:00
command : ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
2022-03-03 19:58:51 +00:00
- run :
name : "Build tests"
command : cd build && make V=1 -j32
- when :
condition : << parameters.run_even_tests >>
steps :
- run :
name : "Run even tests"
2022-05-10 16:51:03 +00:00
command : ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 0,,2
2022-03-03 19:58:51 +00:00
- when :
condition :
not : << parameters.run_even_tests >>
steps :
- run :
name : "Run odd tests"
2022-05-10 16:51:03 +00:00
command : ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 1,,2
2020-12-05 07:22:18 +00:00
- post-steps
2020-06-09 02:32:18 +00:00
build-linux :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-06-09 02:32:18 +00:00
resource_class : 2xlarge
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
2022-02-05 01:12:03 +00:00
- run : make V=1 J=32 -j32 check
2020-09-23 21:40:53 +00:00
- post-steps
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 18:24:00 +00:00
2022-04-18 19:47:16 +00:00
build-linux-encrypted_env-no_compression :
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 17:31:34 +00:00
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 17:31:34 +00:00
resource_class : 2xlarge
steps :
- pre-steps
- install-gflags
2022-04-18 19:47:16 +00:00
- run : ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
- run : |
./sst_dump --help | egrep -q 'Supported compression types : kNoCompression$' # Verify no compiled in compression
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 17:31:34 +00:00
- post-steps
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 18:24:00 +00:00
build-linux-shared_lib-alt_namespace-status_checked :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 18:24:00 +00:00
resource_class : 2xlarge
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
2022-02-05 01:12:03 +00:00
- run : ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check
2020-09-23 21:40:53 +00:00
- post-steps
2020-06-09 02:32:18 +00:00
build-linux-release :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2022-03-03 19:58:51 +00:00
resource_class : 2xlarge
2020-06-09 02:32:18 +00:00
steps :
- checkout # check out the code in the project directory
2022-03-03 19:58:51 +00:00
- run : make V=1 -j32 release
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 18:24:00 +00:00
- run : if ./db_stress --version; then false; else true; fi # ensure without gflags
2020-09-23 21:40:53 +00:00
- install-gflags
2022-03-03 19:58:51 +00:00
- run : make V=1 -j32 release
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 18:24:00 +00:00
- run : ./db_stress --version # ensure with gflags
2020-09-23 21:40:53 +00:00
- post-steps
2020-06-09 02:32:18 +00:00
2020-09-22 17:26:34 +00:00
build-linux-release-rtti :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2022-03-03 19:58:51 +00:00
resource_class : xlarge
2020-09-22 17:26:34 +00:00
steps :
- checkout # check out the code in the project directory
- run : make clean
2022-03-03 19:58:51 +00:00
- run : USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
2020-09-22 17:26:34 +00:00
- run : if ./db_stress --version; then false; else true; fi # ensure without gflags
- run : sudo apt-get update -y && sudo apt-get install -y libgflags-dev
- run : make clean
2022-03-03 19:58:51 +00:00
- run : USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
2020-09-22 17:26:34 +00:00
- run : ./db_stress --version # ensure with gflags
2020-06-09 02:32:18 +00:00
build-linux-lite :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2022-03-03 19:58:51 +00:00
resource_class : large
2020-06-09 02:32:18 +00:00
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
2022-03-03 19:58:51 +00:00
- run : LITE=1 make V=1 J=8 -j8 check
2020-09-23 21:40:53 +00:00
- post-steps
2020-06-09 02:32:18 +00:00
build-linux-lite-release :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-06-09 02:32:18 +00:00
resource_class : large
steps :
- checkout # check out the code in the project directory
2022-02-05 01:12:03 +00:00
- run : LITE=1 make V=1 -j8 release
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 18:24:00 +00:00
- run : if ./db_stress --version; then false; else true; fi # ensure without gflags
2020-09-23 21:40:53 +00:00
- install-gflags
2022-02-05 01:12:03 +00:00
- run : LITE=1 make V=1 -j8 release
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 18:24:00 +00:00
- run : ./db_stress --version # ensure with gflags
2020-09-23 21:40:53 +00:00
- post-steps
2020-06-09 02:32:18 +00:00
2020-12-22 08:19:44 +00:00
build-linux-clang-no_test_run :
2020-06-09 02:32:18 +00:00
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-12-22 08:19:44 +00:00
resource_class : xlarge
2020-06-09 02:32:18 +00:00
steps :
- checkout # check out the code in the project directory
2021-05-12 17:44:28 +00:00
- run : sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev libtbb-dev
2022-02-05 01:12:03 +00:00
- run : CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all
2020-09-23 21:40:53 +00:00
- post-steps
2020-06-24 23:20:55 +00:00
2020-06-26 00:40:08 +00:00
build-linux-clang10-asan :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-06-26 00:40:08 +00:00
resource_class : 2xlarge
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
- install-clang-10
2022-04-19 03:26:37 +00:00
- run : COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out
2020-09-23 21:40:53 +00:00
- post-steps
2020-06-26 00:40:08 +00:00
2020-07-14 22:31:57 +00:00
build-linux-clang10-mini-tsan :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-07-14 22:31:57 +00:00
resource_class : 2xlarge
2022-03-03 19:58:51 +00:00
# find test list by `make list_all_tests`
parameters :
start_test :
default : ""
type : string
end_test :
default : ""
type : string
2020-07-14 22:31:57 +00:00
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
- install-clang-10
2022-03-03 19:58:51 +00:00
- install-gtest-parallel
- run :
name : "Build unit tests"
command : |
echo "env: $(env)"
ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 --output-sync=target build_subset_tests
- run :
name : "Run unit tests in parallel"
command : |
sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
cat /tmp/test_list
gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
2020-09-23 21:40:53 +00:00
- post-steps
2020-07-14 22:31:57 +00:00
2020-07-11 03:02:16 +00:00
build-linux-clang10-ubsan :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-07-11 03:02:16 +00:00
resource_class : 2xlarge
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
- install-clang-10
2022-02-05 01:12:03 +00:00
- run : COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check # aligned new doesn't work for reason we haven't figured out
2020-09-23 21:40:53 +00:00
- post-steps
2020-07-11 03:02:16 +00:00
2022-04-19 03:26:37 +00:00
build-linux-valgrind :
machine :
image : ubuntu-2004:202111-02
resource_class : 2xlarge
steps :
- pre-steps
- install-gflags
- install-valgrind
- run : PORTABLE=1 make V=1 -j32 valgrind_test
- post-steps
2020-07-13 19:31:41 +00:00
build-linux-clang10-clang-analyze :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-07-13 19:31:41 +00:00
resource_class : 2xlarge
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
- install-clang-10
- run : sudo apt-get update -y && sudo apt-get install -y clang-tools-10
2022-02-05 01:12:03 +00:00
- run : CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
2020-09-23 21:40:53 +00:00
- post-steps
2020-07-13 19:31:41 +00:00
Meta-internal folly integration with F14FastMap (#9546)
Summary:
Especially after updating to C++17, I don't see a compelling case for
*requiring* any folly components in RocksDB. I was able to purge the existing
hard dependencies, and it can be quite difficult to strip out non-trivial components
from folly for use in RocksDB. (The prospect of doing that on F14 has changed
my mind on the best approach here.)
But this change creates an optional integration where we can plug in
components from folly at compile time, starting here with F14FastMap to replace
std::unordered_map when possible (probably no public APIs for example). I have
replaced the biggest CPU users of std::unordered_map with compile-time
pluggable UnorderedMap which will use F14FastMap when USE_FOLLY is set.
USE_FOLLY is always set in the Meta-internal buck build, and a simulation of
that is in the Makefile for public CI testing. A full folly build is not needed, but
checking out the full folly repo is much simpler for getting the dependency,
and anything else we might want to optionally integrate in the future.
Some picky details:
* I don't think the distributed mutex stuff is actually used, so it was easy to remove.
* I implemented an alternative to `folly::constexpr_log2` (which is much easier
in C++17 than C++11) so that I could pull out the hard dependencies on
`ConstexprMath.h`
* I had to add noexcept move constructors/operators to some types to make
F14's complainUnlessNothrowMoveAndDestroy check happy, and I added a
macro to make that easier in some common cases.
* Updated Meta-internal buck build to use folly F14Map (always)
No updates to HISTORY.md nor INSTALL.md as this is not (yet?) considered a
production integration for open source users.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9546
Test Plan:
CircleCI tests updated so that a couple of them use folly.
Most internal unit & stress/crash tests updated to use Meta-internal latest folly.
(Note: they should probably use buck but they currently use Makefile.)
Example performance improvement: when filter partitions are pinned in cache,
they are tracked by PartitionedFilterBlockReader::filter_map_ and we can build
a test that exercises that heavily. Build DB with
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench -benchmarks=fillrandom -num=10000000 -disable_wal=1 -write_buffer_size=30000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters
```
and test with (simultaneous runs with & without folly, ~20 times each to see
convergence)
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench_folly -readonly -use_existing_db -benchmarks=readrandom -num=10000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters -duration=40 -pin_l0_filter_and_index_blocks_in_cache
```
Average ops/s no folly: 26229.2
Average ops/s with folly: 26853.3 (+2.4%)
Reviewed By: ajkr
Differential Revision: D34181736
Pulled By: pdillinger
fbshipit-source-id: ffa6ad5104c2880321d8a1aa7187e00ab0d02e94
2022-04-13 14:34:01 +00:00
build-linux-cmake-with-folly :
2020-06-09 02:32:18 +00:00
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2021-06-01 21:42:06 +00:00
resource_class : 2xlarge
steps :
2021-07-15 20:36:48 +00:00
- pre-steps
2021-06-01 21:42:06 +00:00
- install-gflags
- upgrade-cmake
Meta-internal folly integration with F14FastMap (#9546)
Summary:
Especially after updating to C++17, I don't see a compelling case for
*requiring* any folly components in RocksDB. I was able to purge the existing
hard dependencies, and it can be quite difficult to strip out non-trivial components
from folly for use in RocksDB. (The prospect of doing that on F14 has changed
my mind on the best approach here.)
But this change creates an optional integration where we can plug in
components from folly at compile time, starting here with F14FastMap to replace
std::unordered_map when possible (probably no public APIs for example). I have
replaced the biggest CPU users of std::unordered_map with compile-time
pluggable UnorderedMap which will use F14FastMap when USE_FOLLY is set.
USE_FOLLY is always set in the Meta-internal buck build, and a simulation of
that is in the Makefile for public CI testing. A full folly build is not needed, but
checking out the full folly repo is much simpler for getting the dependency,
and anything else we might want to optionally integrate in the future.
Some picky details:
* I don't think the distributed mutex stuff is actually used, so it was easy to remove.
* I implemented an alternative to `folly::constexpr_log2` (which is much easier
in C++17 than C++11) so that I could pull out the hard dependencies on
`ConstexprMath.h`
* I had to add noexcept move constructors/operators to some types to make
F14's complainUnlessNothrowMoveAndDestroy check happy, and I added a
macro to make that easier in some common cases.
* Updated Meta-internal buck build to use folly F14Map (always)
No updates to HISTORY.md nor INSTALL.md as this is not (yet?) considered a
production integration for open source users.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9546
Test Plan:
CircleCI tests updated so that a couple of them use folly.
Most internal unit & stress/crash tests updated to use Meta-internal latest folly.
(Note: they should probably use buck but they currently use Makefile.)
Example performance improvement: when filter partitions are pinned in cache,
they are tracked by PartitionedFilterBlockReader::filter_map_ and we can build
a test that exercises that heavily. Build DB with
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench -benchmarks=fillrandom -num=10000000 -disable_wal=1 -write_buffer_size=30000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters
```
and test with (simultaneous runs with & without folly, ~20 times each to see
convergence)
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench_folly -readonly -use_existing_db -benchmarks=readrandom -num=10000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters -duration=40 -pin_l0_filter_and_index_blocks_in_cache
```
Average ops/s no folly: 26229.2
Average ops/s with folly: 26853.3 (+2.4%)
Reviewed By: ajkr
Differential Revision: D34181736
Pulled By: pdillinger
fbshipit-source-id: ffa6ad5104c2880321d8a1aa7187e00ab0d02e94
2022-04-13 14:34:01 +00:00
- run : make checkout_folly
- run : (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20)
2021-06-01 21:42:06 +00:00
- post-steps
Meta-internal folly integration with F14FastMap (#9546)
Summary:
Especially after updating to C++17, I don't see a compelling case for
*requiring* any folly components in RocksDB. I was able to purge the existing
hard dependencies, and it can be quite difficult to strip out non-trivial components
from folly for use in RocksDB. (The prospect of doing that on F14 has changed
my mind on the best approach here.)
But this change creates an optional integration where we can plug in
components from folly at compile time, starting here with F14FastMap to replace
std::unordered_map when possible (probably no public APIs for example). I have
replaced the biggest CPU users of std::unordered_map with compile-time
pluggable UnorderedMap which will use F14FastMap when USE_FOLLY is set.
USE_FOLLY is always set in the Meta-internal buck build, and a simulation of
that is in the Makefile for public CI testing. A full folly build is not needed, but
checking out the full folly repo is much simpler for getting the dependency,
and anything else we might want to optionally integrate in the future.
Some picky details:
* I don't think the distributed mutex stuff is actually used, so it was easy to remove.
* I implemented an alternative to `folly::constexpr_log2` (which is much easier
in C++17 than C++11) so that I could pull out the hard dependencies on
`ConstexprMath.h`
* I had to add noexcept move constructors/operators to some types to make
F14's complainUnlessNothrowMoveAndDestroy check happy, and I added a
macro to make that easier in some common cases.
* Updated Meta-internal buck build to use folly F14Map (always)
No updates to HISTORY.md nor INSTALL.md as this is not (yet?) considered a
production integration for open source users.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9546
Test Plan:
CircleCI tests updated so that a couple of them use folly.
Most internal unit & stress/crash tests updated to use Meta-internal latest folly.
(Note: they should probably use buck but they currently use Makefile.)
Example performance improvement: when filter partitions are pinned in cache,
they are tracked by PartitionedFilterBlockReader::filter_map_ and we can build
a test that exercises that heavily. Build DB with
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench -benchmarks=fillrandom -num=10000000 -disable_wal=1 -write_buffer_size=30000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters
```
and test with (simultaneous runs with & without folly, ~20 times each to see
convergence)
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench_folly -readonly -use_existing_db -benchmarks=readrandom -num=10000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters -duration=40 -pin_l0_filter_and_index_blocks_in_cache
```
Average ops/s no folly: 26229.2
Average ops/s with folly: 26853.3 (+2.4%)
Reviewed By: ajkr
Differential Revision: D34181736
Pulled By: pdillinger
fbshipit-source-id: ffa6ad5104c2880321d8a1aa7187e00ab0d02e94
2022-04-13 14:34:01 +00:00
build-linux-cmake-with-benchmark :
2021-06-01 21:42:06 +00:00
machine :
2022-02-07 01:35:50 +00:00
image : ubuntu-2004:202111-02
2021-06-01 21:42:06 +00:00
resource_class : 2xlarge
2020-06-09 02:32:18 +00:00
steps :
2021-07-15 20:36:48 +00:00
- pre-steps
2021-06-01 21:42:06 +00:00
- install-gflags
2021-07-09 00:50:55 +00:00
- install-benchmark
2022-02-05 01:12:03 +00:00
- run : (mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20)
2020-09-23 21:40:53 +00:00
- post-steps
2020-06-09 02:32:18 +00:00
2021-09-10 16:59:05 +00:00
build-linux-unity-and-headers :
2020-06-26 18:12:06 +00:00
docker : # executor type
- image : gcc:latest
Avoid some warnings-as-error in CircleCI+unity+AVX512F (#9978)
Summary:
Example failure when compiling on sufficiently new hardware and built-in headers:
```
In file included from /usr/local/lib/gcc/x86_64-linux-gnu/12.1.0/include/immintrin.h:49,
from ./util/bloom_impl.h:21,
from table/block_based/filter_policy.cc:31,
from unity.cc:167:
In function '__m512i _mm512_shuffle_epi32(__m512i, _MM_PERM_ENUM)',
inlined from 'void XXH3_accumulate_512_avx512(void*, const void*, const void*)' at util/xxhash.h:3605:58,
inlined from 'void XXH3_accumulate(xxh_u64*, const xxh_u8*, const xxh_u8*, size_t, XXH3_f_accumulate_512)' at util/xxhash.h:4229:17,
inlined from 'void XXH3_hashLong_internal_loop(xxh_u64*, const xxh_u8*, size_t, const xxh_u8*, size_t, XXH3_f_accumulate_512, XXH3_f_scrambleAcc)' at util/xxhash.h:4251:24,
inlined from 'XXH128_hash_t XXH3_hashLong_128b_internal(const void*, size_t, const xxh_u8*, size_t, XXH3_f_accumulate_512, XXH3_f_scrambleAcc)' at util/xxhash.h:5065:32,
inlined from 'XXH128_hash_t XXH3_hashLong_128b_withSecret(const void*, size_t, XXH64_hash_t, const void*, size_t)' at util/xxhash.h:5104:39:
/usr/local/lib/gcc/x86_64-linux-gnu/12.1.0/include/avx512fintrin.h:4459:50: error: '__Y' may be used uninitialized [-Werror=maybe-uninitialized]
```
https://app.circleci.com/pipelines/github/facebook/rocksdb/13295/workflows/1695fb5c-40c1-423b-96b4-45107dc3012d/jobs/360416
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9978
Test Plan:
I was able to re-run in CircleCI with ssh, see the failure, ssh in and
verify that adding -fno-avx512f fixed the failure. Will watch build-linux-unity-and-headers
Reviewed By: riversand963
Differential Revision: D36296028
Pulled By: pdillinger
fbshipit-source-id: ba5955cf2ac730f57d1d18c2f517e92f34be77a3
2022-05-10 22:24:40 +00:00
environment :
EXTRA_CXXFLAGS : -mno-avx512f # Warnings-as-error in avx512fintrin.h, would be used on newer hardware
2020-12-22 08:19:44 +00:00
resource_class : large
2020-06-26 18:12:06 +00:00
steps :
- checkout # check out the code in the project directory
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 18:24:00 +00:00
- run : apt-get update -y && apt-get install -y libgflags-dev
2022-05-09 20:38:46 +00:00
- run : make V=1 -j8 unity_test
2022-02-05 01:12:03 +00:00
- run : make V=1 -j8 -k check-headers # could be moved to a different build
2020-09-23 21:40:53 +00:00
- post-steps
2020-06-26 18:12:06 +00:00
Meta-internal folly integration with F14FastMap (#9546)
Summary:
Especially after updating to C++17, I don't see a compelling case for
*requiring* any folly components in RocksDB. I was able to purge the existing
hard dependencies, and it can be quite difficult to strip out non-trivial components
from folly for use in RocksDB. (The prospect of doing that on F14 has changed
my mind on the best approach here.)
But this change creates an optional integration where we can plug in
components from folly at compile time, starting here with F14FastMap to replace
std::unordered_map when possible (probably no public APIs for example). I have
replaced the biggest CPU users of std::unordered_map with compile-time
pluggable UnorderedMap which will use F14FastMap when USE_FOLLY is set.
USE_FOLLY is always set in the Meta-internal buck build, and a simulation of
that is in the Makefile for public CI testing. A full folly build is not needed, but
checking out the full folly repo is much simpler for getting the dependency,
and anything else we might want to optionally integrate in the future.
Some picky details:
* I don't think the distributed mutex stuff is actually used, so it was easy to remove.
* I implemented an alternative to `folly::constexpr_log2` (which is much easier
in C++17 than C++11) so that I could pull out the hard dependencies on
`ConstexprMath.h`
* I had to add noexcept move constructors/operators to some types to make
F14's complainUnlessNothrowMoveAndDestroy check happy, and I added a
macro to make that easier in some common cases.
* Updated Meta-internal buck build to use folly F14Map (always)
No updates to HISTORY.md nor INSTALL.md as this is not (yet?) considered a
production integration for open source users.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9546
Test Plan:
CircleCI tests updated so that a couple of them use folly.
Most internal unit & stress/crash tests updated to use Meta-internal latest folly.
(Note: they should probably use buck but they currently use Makefile.)
Example performance improvement: when filter partitions are pinned in cache,
they are tracked by PartitionedFilterBlockReader::filter_map_ and we can build
a test that exercises that heavily. Build DB with
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench -benchmarks=fillrandom -num=10000000 -disable_wal=1 -write_buffer_size=30000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters
```
and test with (simultaneous runs with & without folly, ~20 times each to see
convergence)
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench_folly -readonly -use_existing_db -benchmarks=readrandom -num=10000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters -duration=40 -pin_l0_filter_and_index_blocks_in_cache
```
Average ops/s no folly: 26229.2
Average ops/s with folly: 26853.3 (+2.4%)
Reviewed By: ajkr
Differential Revision: D34181736
Pulled By: pdillinger
fbshipit-source-id: ffa6ad5104c2880321d8a1aa7187e00ab0d02e94
2022-04-13 14:34:01 +00:00
build-linux-gcc-7-with-folly :
2020-09-08 19:08:05 +00:00
machine :
2022-02-07 01:35:50 +00:00
image : ubuntu-2004:202111-02
2022-02-05 01:12:03 +00:00
resource_class : 2xlarge
2020-09-08 19:08:05 +00:00
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
2022-02-05 01:12:03 +00:00
- run : sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-7 g++-7 libgflags-dev
Meta-internal folly integration with F14FastMap (#9546)
Summary:
Especially after updating to C++17, I don't see a compelling case for
*requiring* any folly components in RocksDB. I was able to purge the existing
hard dependencies, and it can be quite difficult to strip out non-trivial components
from folly for use in RocksDB. (The prospect of doing that on F14 has changed
my mind on the best approach here.)
But this change creates an optional integration where we can plug in
components from folly at compile time, starting here with F14FastMap to replace
std::unordered_map when possible (probably no public APIs for example). I have
replaced the biggest CPU users of std::unordered_map with compile-time
pluggable UnorderedMap which will use F14FastMap when USE_FOLLY is set.
USE_FOLLY is always set in the Meta-internal buck build, and a simulation of
that is in the Makefile for public CI testing. A full folly build is not needed, but
checking out the full folly repo is much simpler for getting the dependency,
and anything else we might want to optionally integrate in the future.
Some picky details:
* I don't think the distributed mutex stuff is actually used, so it was easy to remove.
* I implemented an alternative to `folly::constexpr_log2` (which is much easier
in C++17 than C++11) so that I could pull out the hard dependencies on
`ConstexprMath.h`
* I had to add noexcept move constructors/operators to some types to make
F14's complainUnlessNothrowMoveAndDestroy check happy, and I added a
macro to make that easier in some common cases.
* Updated Meta-internal buck build to use folly F14Map (always)
No updates to HISTORY.md nor INSTALL.md as this is not (yet?) considered a
production integration for open source users.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9546
Test Plan:
CircleCI tests updated so that a couple of them use folly.
Most internal unit & stress/crash tests updated to use Meta-internal latest folly.
(Note: they should probably use buck but they currently use Makefile.)
Example performance improvement: when filter partitions are pinned in cache,
they are tracked by PartitionedFilterBlockReader::filter_map_ and we can build
a test that exercises that heavily. Build DB with
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench -benchmarks=fillrandom -num=10000000 -disable_wal=1 -write_buffer_size=30000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters
```
and test with (simultaneous runs with & without folly, ~20 times each to see
convergence)
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench_folly -readonly -use_existing_db -benchmarks=readrandom -num=10000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters -duration=40 -pin_l0_filter_and_index_blocks_in_cache
```
Average ops/s no folly: 26229.2
Average ops/s with folly: 26853.3 (+2.4%)
Reviewed By: ajkr
Differential Revision: D34181736
Pulled By: pdillinger
fbshipit-source-id: ffa6ad5104c2880321d8a1aa7187e00ab0d02e94
2022-04-13 14:34:01 +00:00
- run : make checkout_folly
- run : USE_FOLLY=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 check
2020-12-22 08:19:44 +00:00
- post-steps
build-linux-gcc-8-no_test_run :
machine :
2022-02-07 01:35:50 +00:00
image : ubuntu-2004:202111-02
2022-03-03 19:58:51 +00:00
resource_class : xlarge
2020-12-22 08:19:44 +00:00
steps :
- pre-steps
2022-02-05 01:12:03 +00:00
- run : sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
2022-03-03 19:58:51 +00:00
- run : CC=gcc-8 CXX=g++-8 V=1 make -j16 all
2020-12-22 08:19:44 +00:00
- post-steps
build-linux-gcc-10-cxx20-no_test_run :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-12-22 08:19:44 +00:00
resource_class : xlarge
steps :
- pre-steps
- run : sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
2022-03-07 23:23:25 +00:00
- run : CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all
2020-09-23 21:40:53 +00:00
- post-steps
2020-09-08 19:08:05 +00:00
2021-10-18 19:21:25 +00:00
build-linux-gcc-11-no_test_run :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2021-10-18 19:21:25 +00:00
resource_class : xlarge
steps :
- pre-steps
2022-02-05 01:12:03 +00:00
- run : sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-11 g++-11 libgflags-dev
2022-03-30 17:09:49 +00:00
- install-benchmark
2022-03-07 23:23:25 +00:00
- run : CC=gcc-11 CXX=g++-11 V=1 make -j16 all microbench
2021-10-18 19:21:25 +00:00
- post-steps
2022-01-11 18:35:22 +00:00
build-linux-clang-13-no_test_run :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2022-01-11 18:35:22 +00:00
resource_class : xlarge
steps :
- pre-steps
- install-clang-13
2022-03-03 19:58:51 +00:00
- install-benchmark
- run : CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j16 all microbench
2022-01-11 18:35:22 +00:00
- post-steps
2022-04-19 03:26:37 +00:00
# Ensure ASAN+UBSAN with folly, and full testsuite with clang 13
build-linux-clang-13-asan-ubsan-with-folly :
machine :
image : ubuntu-2004:202111-02
resource_class : 2xlarge
steps :
- pre-steps
- install-clang-13
- install-gflags
- run : make checkout_folly
- run : CC=clang-13 CXX=clang++-13 USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check
- post-steps
2021-07-09 00:50:55 +00:00
# This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
2022-03-03 19:58:51 +00:00
build-linux-run-microbench :
2021-07-09 00:50:55 +00:00
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2022-03-03 19:58:51 +00:00
resource_class : 2xlarge
2021-07-09 00:50:55 +00:00
steps :
- pre-steps
- install-benchmark
2022-03-03 19:58:51 +00:00
- run : DEBUG_LEVEL=0 make -j32 run_microbench
2021-07-09 00:50:55 +00:00
- post-steps
2022-03-16 22:58:06 +00:00
build-linux-mini-crashtest :
machine :
image : ubuntu-2004:202111-02
resource_class : large
steps :
- pre-steps
- install-gflags
- install-compression-libs
2022-05-10 16:51:03 +00:00
- run : ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS=--duration=960 blackbox_crash_test_with_atomic_flush
2022-03-16 22:58:06 +00:00
- post-steps
2020-06-09 02:32:18 +00:00
build-windows :
2020-03-14 04:51:26 +00:00
executor : windows-2xlarge
2020-06-29 21:29:15 +00:00
parameters :
extra_cmake_opt :
default : ""
type : string
2020-07-14 01:08:08 +00:00
vs_year :
default : "2019"
type : string
cmake_generator :
default : "Visual Studio 16 2019"
type : string
2020-03-14 04:51:26 +00:00
environment :
THIRDPARTY_HOME : C:/Users/circleci/thirdparty
CMAKE_HOME : C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
CMAKE_BIN : C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
SNAPPY_HOME : C:/Users/circleci/thirdparty/snappy-1.1.7
SNAPPY_INCLUDE : C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
SNAPPY_LIB_DEBUG : C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
2020-07-14 01:08:08 +00:00
VS_YEAR : <<parameters.vs_year>>
CMAKE_GENERATOR : <<parameters.cmake_generator>>
2020-03-14 04:51:26 +00:00
steps :
- checkout
2020-07-14 01:08:08 +00:00
- run :
name : "Setup VS"
command : |
2021-01-11 21:31:03 +00:00
if [[ "${VS_YEAR}" == "2019" ]]; then
echo "VS2019 already present."
elif [[ "${VS_YEAR}" == "2017" ]]; then
echo "Installing VS2017..."
2020-07-14 01:08:08 +00:00
powershell .circleci/vs2017_install.ps1
elif [[ "${VS_YEAR}" == "2015" ]]; then
2021-01-11 21:31:03 +00:00
echo "Installing VS2015..."
2020-07-14 01:08:08 +00:00
powershell .circleci/vs2015_install.ps1
fi
2021-04-28 18:27:37 +00:00
- store_artifacts :
path : \Users\circleci\AppData\Local\Temp\vslogs.zip
2020-03-14 04:51:26 +00:00
- run :
name : "Install thirdparty dependencies"
command : |
mkdir ${THIRDPARTY_HOME}
cd ${THIRDPARTY_HOME}
echo "Installing CMake..."
curl --fail --silent --show-error --output cmake-3.16.4-win64-x64.zip --location https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-win64-x64.zip
unzip -q cmake-3.16.4-win64-x64.zip
echo "Building Snappy dependency..."
curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
unzip -q snappy-1.1.7.zip
cd snappy-1.1.7
mkdir build
cd build
${CMAKE_BIN} -G "${CMAKE_GENERATOR}" ..
msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
- run :
name : "Build RocksDB"
command : |
mkdir build
cd build
2020-06-29 21:29:15 +00:00
${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
2020-03-14 04:51:26 +00:00
cd ..
2021-01-11 21:31:03 +00:00
echo "Building with VS version: ${CMAKE_GENERATOR}"
2022-03-12 19:45:10 +00:00
msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
2020-03-14 04:51:26 +00:00
- run :
name : "Test RocksDB"
shell : powershell.exe
command : |
Fix MSVC-related build issues (#7439)
Summary:
This PR addresses some build and functional issues on MSVC targets, as a step towards an eventual goal of having RocksDB build successfully for Windows on ARM64.
Addressed issues include:
- BitsSetToOne and CountTrailingZeroBits do not compile on non-x64 MSVC targets. A fallback implementation of BitsSetToOne when Intel intrinsics are not available is added, based on the C++20 `<bit>` popcount implementation in Microsoft's STL.
- The implementation of FloorLog2 for MSVC targets (including x64) gives incorrect results. The unit test easily detects this, but CircleCI is currently configured to only run a specific set of tests for Windows CMake builds, so this seems to have been unnoticed.
- AsmVolatilePause does not use YieldProcessor on Windows ARM64 targets, even though it is available.
- When CondVar::TimedWait calls Microsoft STL's condition_variable::wait_for, it can potentially trigger a bug (just recently fixed in the upcoming VS 16.8's STL) that deadlocks various tests that wait for a timer to execute, since `Timer::Run` doesn't get a chance to execute before being blocked by the test function acquiring the mutex.
- In c_test, `GetTempDir` assumes a POSIX-style temp path.
- `NormalizePath` did not eliminate consecutive POSIX-style path separators on Windows, resulting in test failures in e.g., wal_manager_test.
- Various other test failures.
In a followup PR I hope to modify CircleCI's config.yml to invoke all RocksDB unit tests in Windows CMake builds with CTest, instead of the current use of `run_ci_db_test.ps1` which requires individual tests to be specified and is missing many of the existing tests.
Notes from peterd: FloorLog2 is not yet used in production code (it's for something in progress). I also added a few more inexpensive platform-dependent tests to Windows CircleCI runs. And included facebook/folly#1461 as requested
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7439
Reviewed By: jay-zhuang
Differential Revision: D24021563
Pulled By: pdillinger
fbshipit-source-id: 0ec2027c0d6a494d8a0fe38d9667fc2f7e29f7e7
2020-10-01 16:21:30 +00:00
build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
2020-06-09 02:32:18 +00:00
2020-07-13 22:04:27 +00:00
build-linux-java :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-12-16 23:58:56 +00:00
resource_class : large
environment :
JAVA_HOME : /usr/lib/jvm/java-1.8.0-openjdk-amd64
2020-07-13 22:04:27 +00:00
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
2020-07-13 22:04:27 +00:00
- run :
2020-12-16 23:58:56 +00:00
name : "Set Java Environment"
command : |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run :
name : "Test RocksDBJava"
2022-02-05 01:12:03 +00:00
command : make V=1 J=8 -j8 jtest
2020-12-16 23:58:56 +00:00
- post-steps
build-linux-java-static :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-12-16 23:58:56 +00:00
resource_class : large
environment :
JAVA_HOME : /usr/lib/jvm/java-1.8.0-openjdk-amd64
steps :
- pre-steps
- install-gflags
- run :
name : "Set Java Environment"
command : |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run :
name : "Build RocksDBJava Static Library"
2022-02-05 01:12:03 +00:00
command : make V=1 J=8 -j8 rocksdbjavastatic
2020-12-16 23:58:56 +00:00
- post-steps
build-macos-java :
macos :
2022-01-20 17:40:34 +00:00
xcode : 12.5 .1
2022-03-03 19:58:51 +00:00
resource_class : large
2020-12-16 23:58:56 +00:00
environment :
2021-05-04 17:33:31 +00:00
JAVA_HOME : /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
2022-01-21 17:32:03 +00:00
ROCKSDB_DISABLE_JEMALLOC : 1 # jemalloc causes java 8 crash
2020-12-16 23:58:56 +00:00
steps :
- increase-max-open-files-on-macos
- install-gflags-on-macos
2022-01-20 17:40:34 +00:00
- install-jdk8-on-macos
2020-12-16 23:58:56 +00:00
- pre-steps-macos
- run :
name : "Set Java Environment"
command : |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run :
name : "Test RocksDBJava"
2022-03-03 19:58:51 +00:00
command : make V=1 J=16 -j16 jtest
2020-12-16 23:58:56 +00:00
- post-steps
build-macos-java-static :
macos :
2022-01-13 01:19:41 +00:00
xcode : 12.5 .1
2022-03-03 19:58:51 +00:00
resource_class : large
2020-12-16 23:58:56 +00:00
environment :
2021-05-04 17:33:31 +00:00
JAVA_HOME : /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
2020-12-16 23:58:56 +00:00
steps :
- increase-max-open-files-on-macos
- install-gflags-on-macos
- install-cmake-on-macos
2022-01-13 01:19:41 +00:00
- install-jdk8-on-macos
2020-12-16 23:58:56 +00:00
- pre-steps-macos
- run :
name : "Set Java Environment"
2020-07-13 22:04:27 +00:00
command : |
echo "JAVA_HOME=${JAVA_HOME}"
2020-12-16 23:58:56 +00:00
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
2020-07-13 22:04:27 +00:00
which java && java -version
which javac && javac -version
2020-12-16 23:58:56 +00:00
- run :
2022-01-13 01:19:41 +00:00
name : "Build RocksDBJava x86 and ARM Static Libraries"
2022-03-03 19:58:51 +00:00
command : make V=1 J=16 -j16 rocksdbjavastaticosx
2022-01-13 01:19:41 +00:00
- post-steps
build-macos-java-static-universal :
macos :
xcode : 12.5 .1
2022-03-03 19:58:51 +00:00
resource_class : large
2022-01-13 01:19:41 +00:00
environment :
JAVA_HOME : /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
steps :
- increase-max-open-files-on-macos
- install-gflags-on-macos
- install-cmake-on-macos
- install-jdk8-on-macos
- pre-steps-macos
- run :
name : "Set Java Environment"
command : |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run :
name : "Build RocksDBJava Universal Binary Static Library"
2022-03-03 19:58:51 +00:00
command : make V=1 J=16 -j16 rocksdbjavastaticosx_ub
2020-09-23 21:40:53 +00:00
- post-steps
2020-07-13 22:04:27 +00:00
2020-07-16 00:46:18 +00:00
build-examples :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-11-18 02:18:13 +00:00
resource_class : large
2020-07-16 00:46:18 +00:00
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
2020-07-16 00:46:18 +00:00
- run :
name : "Build examples"
command : |
2022-02-05 01:12:03 +00:00
OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4
2020-09-23 21:40:53 +00:00
- post-steps
2020-07-16 00:46:18 +00:00
2020-11-18 02:18:13 +00:00
build-cmake-mingw :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2022-03-03 19:58:51 +00:00
resource_class : large
2020-11-18 02:18:13 +00:00
steps :
- pre-steps
- install-gflags
- run : sudo apt-get update -y && sudo apt-get install -y mingw-w64
- run : sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
- run :
name : "Build cmake-mingw"
command : |
sudo apt-get install snapd && sudo snap install cmake --beta --classic
export PATH=/snap/bin:$PATH
sudo apt-get install -y openjdk-8-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export PATH=$JAVA_HOME/bin:$PATH
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
- post-steps
2020-08-12 01:29:15 +00:00
build-linux-non-shm :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2020-08-12 01:29:15 +00:00
resource_class : 2xlarge
2022-03-03 19:58:51 +00:00
environment :
TEST_TMPDIR : /tmp/rocksdb_test_tmp
2020-08-12 01:29:15 +00:00
steps :
2020-09-23 21:40:53 +00:00
- pre-steps
- install-gflags
2022-03-03 19:58:51 +00:00
- run : make V=1 -j32 check
2020-09-23 21:40:53 +00:00
- post-steps
2020-08-12 01:29:15 +00:00
2021-04-20 15:41:32 +00:00
build-linux-arm-test-full :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2021-04-20 15:41:32 +00:00
resource_class : arm.large
steps :
- pre-steps
- install-gflags
2022-02-05 01:12:03 +00:00
- run : make V=1 J=4 -j4 check
2021-04-20 15:41:32 +00:00
- post-steps
2021-04-20 03:05:56 +00:00
build-linux-arm :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2021-04-20 03:05:56 +00:00
resource_class : arm.large
steps :
- pre-steps
- install-gflags
2022-02-05 01:12:03 +00:00
- run : ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some
2021-04-20 03:05:56 +00:00
- post-steps
build-linux-arm-cmake-no_test_run :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2021-04-20 03:05:56 +00:00
resource_class : arm.large
environment :
JAVA_HOME : /usr/lib/jvm/java-8-openjdk-arm64
steps :
- pre-steps
2021-06-01 21:42:06 +00:00
- install-gflags
2021-04-20 03:05:56 +00:00
- run :
name : "Set Java Environment"
command : |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run :
name : "Build with cmake"
command : |
mkdir build
cd build
2021-06-01 21:42:06 +00:00
cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
2021-04-20 03:05:56 +00:00
make -j4
- run :
name : "Build Java with cmake"
command : |
rm -rf build
mkdir build
cd build
2021-06-01 21:42:06 +00:00
cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
2021-04-20 03:05:56 +00:00
make -j4 rocksdb rocksdbjni
2021-04-20 15:41:32 +00:00
- post-steps
2021-04-20 03:05:56 +00:00
2021-02-10 04:47:06 +00:00
build-format-compatible :
machine :
2022-02-05 01:12:03 +00:00
image : ubuntu-2004:202111-02
2021-02-10 04:47:06 +00:00
resource_class : 2xlarge
steps :
- pre-steps
- install-gflags
- install-compression-libs
- run :
name : "test"
command : |
export TEST_TMPDIR=/dev/shm/rocksdb
rm -rf /dev/shm/rocksdb
mkdir /dev/shm/rocksdb
tools/check_format_compatible.sh
- post-steps
2022-02-01 18:29:56 +00:00
build-fuzzers :
machine :
2022-02-07 01:35:50 +00:00
image : ubuntu-2004:202111-02
2022-02-01 18:29:56 +00:00
resource_class : large
steps :
- pre-steps
- install-clang-13
- run : sudo apt-get update -y && sudo apt-get install -y cmake ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
- install-libprotobuf-mutator
- run :
name : "Build rocksdb lib"
command : CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
- run :
name : "Build fuzzers"
command : cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
- post-steps
2020-06-09 02:32:18 +00:00
workflows :
2021-02-10 04:47:06 +00:00
version : 2
2022-05-11 18:16:09 +00:00
jobs-linux-run-tests :
2020-06-09 02:32:18 +00:00
jobs :
- build-linux
Meta-internal folly integration with F14FastMap (#9546)
Summary:
Especially after updating to C++17, I don't see a compelling case for
*requiring* any folly components in RocksDB. I was able to purge the existing
hard dependencies, and it can be quite difficult to strip out non-trivial components
from folly for use in RocksDB. (The prospect of doing that on F14 has changed
my mind on the best approach here.)
But this change creates an optional integration where we can plug in
components from folly at compile time, starting here with F14FastMap to replace
std::unordered_map when possible (probably no public APIs for example). I have
replaced the biggest CPU users of std::unordered_map with compile-time
pluggable UnorderedMap which will use F14FastMap when USE_FOLLY is set.
USE_FOLLY is always set in the Meta-internal buck build, and a simulation of
that is in the Makefile for public CI testing. A full folly build is not needed, but
checking out the full folly repo is much simpler for getting the dependency,
and anything else we might want to optionally integrate in the future.
Some picky details:
* I don't think the distributed mutex stuff is actually used, so it was easy to remove.
* I implemented an alternative to `folly::constexpr_log2` (which is much easier
in C++17 than C++11) so that I could pull out the hard dependencies on
`ConstexprMath.h`
* I had to add noexcept move constructors/operators to some types to make
F14's complainUnlessNothrowMoveAndDestroy check happy, and I added a
macro to make that easier in some common cases.
* Updated Meta-internal buck build to use folly F14Map (always)
No updates to HISTORY.md nor INSTALL.md as this is not (yet?) considered a
production integration for open source users.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9546
Test Plan:
CircleCI tests updated so that a couple of them use folly.
Most internal unit & stress/crash tests updated to use Meta-internal latest folly.
(Note: they should probably use buck but they currently use Makefile.)
Example performance improvement: when filter partitions are pinned in cache,
they are tracked by PartitionedFilterBlockReader::filter_map_ and we can build
a test that exercises that heavily. Build DB with
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench -benchmarks=fillrandom -num=10000000 -disable_wal=1 -write_buffer_size=30000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters
```
and test with (simultaneous runs with & without folly, ~20 times each to see
convergence)
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench_folly -readonly -use_existing_db -benchmarks=readrandom -num=10000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters -duration=40 -pin_l0_filter_and_index_blocks_in_cache
```
Average ops/s no folly: 26229.2
Average ops/s with folly: 26853.3 (+2.4%)
Reviewed By: ajkr
Differential Revision: D34181736
Pulled By: pdillinger
fbshipit-source-id: ffa6ad5104c2880321d8a1aa7187e00ab0d02e94
2022-04-13 14:34:01 +00:00
- build-linux-cmake-with-folly
2022-05-11 18:16:09 +00:00
- build-linux-gcc-7-with-folly
Meta-internal folly integration with F14FastMap (#9546)
Summary:
Especially after updating to C++17, I don't see a compelling case for
*requiring* any folly components in RocksDB. I was able to purge the existing
hard dependencies, and it can be quite difficult to strip out non-trivial components
from folly for use in RocksDB. (The prospect of doing that on F14 has changed
my mind on the best approach here.)
But this change creates an optional integration where we can plug in
components from folly at compile time, starting here with F14FastMap to replace
std::unordered_map when possible (probably no public APIs for example). I have
replaced the biggest CPU users of std::unordered_map with compile-time
pluggable UnorderedMap which will use F14FastMap when USE_FOLLY is set.
USE_FOLLY is always set in the Meta-internal buck build, and a simulation of
that is in the Makefile for public CI testing. A full folly build is not needed, but
checking out the full folly repo is much simpler for getting the dependency,
and anything else we might want to optionally integrate in the future.
Some picky details:
* I don't think the distributed mutex stuff is actually used, so it was easy to remove.
* I implemented an alternative to `folly::constexpr_log2` (which is much easier
in C++17 than C++11) so that I could pull out the hard dependencies on
`ConstexprMath.h`
* I had to add noexcept move constructors/operators to some types to make
F14's complainUnlessNothrowMoveAndDestroy check happy, and I added a
macro to make that easier in some common cases.
* Updated Meta-internal buck build to use folly F14Map (always)
No updates to HISTORY.md nor INSTALL.md as this is not (yet?) considered a
production integration for open source users.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9546
Test Plan:
CircleCI tests updated so that a couple of them use folly.
Most internal unit & stress/crash tests updated to use Meta-internal latest folly.
(Note: they should probably use buck but they currently use Makefile.)
Example performance improvement: when filter partitions are pinned in cache,
they are tracked by PartitionedFilterBlockReader::filter_map_ and we can build
a test that exercises that heavily. Build DB with
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench -benchmarks=fillrandom -num=10000000 -disable_wal=1 -write_buffer_size=30000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters
```
and test with (simultaneous runs with & without folly, ~20 times each to see
convergence)
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench_folly -readonly -use_existing_db -benchmarks=readrandom -num=10000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters -duration=40 -pin_l0_filter_and_index_blocks_in_cache
```
Average ops/s no folly: 26229.2
Average ops/s with folly: 26853.3 (+2.4%)
Reviewed By: ajkr
Differential Revision: D34181736
Pulled By: pdillinger
fbshipit-source-id: ffa6ad5104c2880321d8a1aa7187e00ab0d02e94
2022-04-13 14:34:01 +00:00
- build-linux-cmake-with-benchmark
2022-04-18 19:47:16 +00:00
- build-linux-encrypted_env-no_compression
2020-06-09 02:32:18 +00:00
- build-linux-lite
2022-05-11 18:16:09 +00:00
jobs-linux-run-tests-san :
2020-06-26 00:40:08 +00:00
jobs :
- build-linux-clang10-asan
2022-05-11 18:16:09 +00:00
- build-linux-clang10-ubsan
2022-03-03 19:58:51 +00:00
- build-linux-clang10-mini-tsan :
start_test : ""
end_test : "env_test"
- build-linux-clang10-mini-tsan :
start_test : "env_test"
end_test : ""
2022-05-11 18:16:09 +00:00
- build-linux-shared_lib-alt_namespace-status_checked
jobs-linux-no-test-run :
2020-07-11 03:02:16 +00:00
jobs :
2022-05-11 18:16:09 +00:00
- build-linux-release
- build-linux-release-rtti
- build-linux-lite-release
- build-examples
- build-fuzzers
- build-linux-clang-no_test_run
- build-linux-clang-13-no_test_run
- build-linux-gcc-8-no_test_run
- build-linux-gcc-10-cxx20-no_test_run
- build-linux-gcc-11-no_test_run
- build-linux-arm-cmake-no_test_run
jobs-linux-other-checks :
2020-07-13 19:31:41 +00:00
jobs :
- build-linux-clang10-clang-analyze
2021-09-10 16:59:05 +00:00
- build-linux-unity-and-headers
2022-03-16 22:58:06 +00:00
- build-linux-mini-crashtest
2022-05-11 18:16:09 +00:00
jobs-windows :
2021-01-11 21:31:03 +00:00
jobs :
- build-windows :
name : "build-windows-vs2019"
- build-windows :
name : "build-windows-vs2019-cxx20"
extra_cmake_opt : -DCMAKE_CXX_STANDARD=20
2020-07-14 01:08:08 +00:00
- build-windows :
2021-01-11 21:31:03 +00:00
name : "build-windows-vs2017"
2020-07-14 01:08:08 +00:00
vs_year : "2017"
cmake_generator : "Visual Studio 15 Win64"
2022-05-11 18:16:09 +00:00
- build-cmake-mingw
jobs-java :
2020-07-13 22:04:27 +00:00
jobs :
- build-linux-java
2020-12-16 23:58:56 +00:00
- build-linux-java-static
- build-macos-java
- build-macos-java-static
2022-01-13 01:19:41 +00:00
- build-macos-java-static-universal
2022-05-11 18:16:09 +00:00
jobs-macos :
2020-10-12 17:45:21 +00:00
jobs :
- build-macos
2022-03-03 19:58:51 +00:00
- build-macos-cmake :
run_even_tests : true
- build-macos-cmake :
run_even_tests : false
2022-05-11 18:16:09 +00:00
jobs-linux-arm :
2021-04-20 03:05:56 +00:00
jobs :
- build-linux-arm
2021-02-10 04:47:06 +00:00
nightly :
triggers :
- schedule :
2022-05-11 18:16:09 +00:00
cron : "0 9 * * *"
2021-02-10 04:47:06 +00:00
filters :
branches :
only :
2021-09-01 18:56:37 +00:00
- main
2021-02-10 04:47:06 +00:00
jobs :
- build-format-compatible
2021-04-20 15:41:32 +00:00
- build-linux-arm-test-full
2022-03-03 19:58:51 +00:00
- build-linux-run-microbench
- build-linux-non-shm
2022-04-19 03:26:37 +00:00
- build-linux-clang-13-asan-ubsan-with-folly
- build-linux-valgrind