From 40944cbbdbdcfac694fc3b291ba1838e943a789b Mon Sep 17 00:00:00 2001 From: Andrew Kryczka Date: Sat, 22 Jun 2024 15:15:02 -0700 Subject: [PATCH] Fix folly build (#12795) Summary: - Updated pinned folly version to the latest - gcc/g++ 10 is required since https://github.com/facebook/folly/commit/2c1c617e9e so we had to modify the tests using gcc/g++ 7 - libsodium 1.0.17 is no longer downloadable from GitHub so I found it elsewhere. I will submit a PR for that upstream to folly - USE_FOLLY_LITE changes - added boost header dependency instead of commenting out the `#include`s since that approach stopped working - added "folly/lang/Exception.cpp" to the compilation Pull Request resolved: https://github.com/facebook/rocksdb/pull/12795 Reviewed By: hx235 Differential Revision: D58916693 Pulled By: ajkr fbshipit-source-id: b5f9bca2d929825846ac898b785972b071db62b1 --- .github/workflows/pr-jobs.yml | 20 ++++++++++++++++---- CMakeLists.txt | 9 ++++++++- Makefile | 23 +++++++++++++++++------ build_tools/build_detect_platform | 8 +++++++- src.mk | 1 + 5 files changed, 49 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr-jobs.yml b/.github/workflows/pr-jobs.yml index eaab8e6d73..627b4dfc03 100644 --- a/.github/workflows/pr-jobs.yml +++ b/.github/workflows/pr-jobs.yml @@ -90,6 +90,9 @@ jobs: container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb + env: + CC: gcc-10 + CXX: g++-10 steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -104,38 +107,47 @@ jobs: container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb + env: + CC: gcc-10 + CXX: g++-10 steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)" - uses: "./.github/actions/post-steps" - build-linux-gcc-7-with-folly: + build-linux-make-with-folly: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb + env: + CC: gcc-10 + CXX: g++-10 steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - uses: "./.github/actions/build-folly" - - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check + - run: USE_FOLLY=1 LIB_MODE=static V=1 make -j32 check - uses: "./.github/actions/post-steps" - build-linux-gcc-7-with-folly-lite-no-test: + build-linux-make-with-folly-lite-no-test: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 options: --shm-size=16gb + env: + CC: gcc-10 + CXX: g++-10 steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all + - run: USE_FOLLY_LITE=1 V=1 make -j32 all - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-coroutines: if: ${{ github.repository_owner == 'facebook' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index b4f1747c0d..93b884dd9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -617,7 +617,7 @@ if(USE_FOLLY) FMT_INST_PATH) exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE GFLAGS_INST_PATH) - set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.78.0) + set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.83.0) if(EXISTS ${FMT_INST_PATH}/lib64) set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt) else() @@ -1052,6 +1052,7 @@ if(USE_FOLLY_LITE) list(APPEND SOURCES third-party/folly/folly/container/detail/F14Table.cpp third-party/folly/folly/detail/Futex.cpp + third-party/folly/folly/lang/Exception.cpp third-party/folly/folly/lang/SafeAssert.cpp third-party/folly/folly/lang/ToAscii.cpp third-party/folly/folly/ScopeGuard.cpp @@ -1059,6 +1060,12 @@ if(USE_FOLLY_LITE) third-party/folly/folly/synchronization/DistributedMutex.cpp third-party/folly/folly/synchronization/ParkingLot.cpp) include_directories(${PROJECT_SOURCE_DIR}/third-party/folly) + exec_program(python3 ${PROJECT_SOURCE_DIR}/third-party/folly ARGS + build/fbcode_builder/getdeps.py show-source-dir boost OUTPUT_VARIABLE + BOOST_SOURCE_PATH) + exec_program(ls ARGS -d ${BOOST_SOURCE_PATH}/boost* OUTPUT_VARIABLE + BOOST_INCLUDE_DIR) + include_directories(${BOOST_INCLUDE_DIR}) add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG) list(APPEND THIRDPARTY_LIBS glog) endif() diff --git a/Makefile b/Makefile index 1dbaedf811..cb2e4ceea7 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,7 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \ export LIB_MODE="$(LIB_MODE)"; \ export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \ export USE_FOLLY="$(USE_FOLLY)"; \ + export USE_FOLLY_LITE="$(USE_FOLLY_LITE)"; \ "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) # this file is generated by the previous line to set build flags and sources include make_config.mk @@ -500,6 +501,17 @@ endif ifeq ($(USE_FOLLY_LITE),1) # Path to the Folly source code and include files FOLLY_DIR = ./third-party/folly +ifneq ($(strip $(BOOST_SOURCE_PATH)),) + BOOST_INCLUDE = $(shell (ls -d $(BOOST_SOURCE_PATH)/boost*/)) + # AIX: pre-defined system headers are surrounded by an extern "C" block + ifeq ($(PLATFORM), OS_AIX) + PLATFORM_CCFLAGS += -I$(BOOST_INCLUDE) + PLATFORM_CXXFLAGS += -I$(BOOST_INCLUDE) + else + PLATFORM_CCFLAGS += -isystem $(BOOST_INCLUDE) + PLATFORM_CXXFLAGS += -isystem $(BOOST_INCLUDE) + endif +endif # BOOST_SOURCE_PATH # AIX: pre-defined system headers are surrounded by an extern "C" block ifeq ($(PLATFORM), OS_AIX) PLATFORM_CCFLAGS += -I$(FOLLY_DIR) @@ -2472,14 +2484,15 @@ checkout_folly: fi @# Pin to a particular version for public CI, so that PR authors don't @# need to worry about folly breaking our integration. Update periodically - cd third-party/folly && git reset --hard beacd86d63cd71c904632262e6c36f60874d78ba - @# A hack to remove boost dependency. - @# NOTE: this hack is only needed if building using USE_FOLLY_LITE - perl -pi -e 's/^(#include .)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h + @# NOTE: this hack is only needed if building target build_folly in the future + perl -pi -e 's,https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz,https://download.libsodium.org/libsodium/releases/old/unsupported/libsodium-1.0.17.tar.gz,' third-party/folly/build/fbcode_builder/manifests/libsodium + @# NOTE: boost source will be needed for any build including `USE_FOLLY_LITE` builds as those depend on boost headers + cd third-party/folly && $(PYTHON) build/fbcode_builder/getdeps.py fetch boost CXX_M_FLAGS = $(filter -m%, $(CXXFLAGS)) @@ -2491,8 +2504,6 @@ build_folly: echo "Please run checkout_folly first"; \ false; \ fi - # Restore the original version of Invoke.h with boost dependency - cd third-party/folly && ${GIT_COMMAND} checkout folly/functional/Invoke.h cd third-party/folly && \ CXXFLAGS=" $(CXX_M_FLAGS) -DHAVE_CXX11_ATOMIC " $(PYTHON) build/fbcode_builder/getdeps.py build --no-tests diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 2e6d7bc20a..1472405c99 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -585,7 +585,7 @@ EOF PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark" fi fi - if test $USE_FOLLY; then + if test $USE_FOLLY || test $USE_FOLLY_LITE; then # Test whether libfolly library is installed $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < @@ -751,6 +751,11 @@ if [ "$USE_FOLLY" ]; then FOLLY_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-inst-dir folly` fi fi +if [ "$USE_FOLLY_LITE" ]; then + if [ "$FOLLY_DIR" ]; then + BOOST_SOURCE_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-source-dir boost` + fi +fi PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" @@ -792,6 +797,7 @@ echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT" echo "FIND=$FIND" >> "$OUTPUT" echo "WATCH=$WATCH" >> "$OUTPUT" echo "FOLLY_PATH=$FOLLY_PATH" >> "$OUTPUT" +echo "BOOST_SOURCE_PATH=$BOOST_SOURCE_PATH" >> "$OUTPUT" # This will enable some related identifiers for the preprocessor if test -n "$JEMALLOC"; then diff --git a/src.mk b/src.mk index 1f9c40a66b..7cbc69d140 100644 --- a/src.mk +++ b/src.mk @@ -407,6 +407,7 @@ TEST_LIB_SOURCES = \ FOLLY_SOURCES = \ $(FOLLY_DIR)/folly/container/detail/F14Table.cpp \ $(FOLLY_DIR)/folly/detail/Futex.cpp \ + $(FOLLY_DIR)/folly/lang/Exception.cpp \ $(FOLLY_DIR)/folly/lang/SafeAssert.cpp \ $(FOLLY_DIR)/folly/lang/ToAscii.cpp \ $(FOLLY_DIR)/folly/ScopeGuard.cpp \