Fix folly build (#12795)

Summary:
- Updated pinned folly version to the latest
- gcc/g++ 10 is required since https://github.com/facebook/folly/commit/2c1c617e9e so we had to modify the tests using gcc/g++ 7
- libsodium 1.0.17 is no longer downloadable from GitHub so I found it elsewhere. I will submit a PR for that upstream to folly
- USE_FOLLY_LITE changes
  - added boost header dependency instead of commenting out the `#include`s since that approach stopped working
  - added "folly/lang/Exception.cpp" to the compilation

Pull Request resolved: https://github.com/facebook/rocksdb/pull/12795

Reviewed By: hx235

Differential Revision: D58916693

Pulled By: ajkr

fbshipit-source-id: b5f9bca2d929825846ac898b785972b071db62b1
This commit is contained in:
Andrew Kryczka 2024-06-22 15:15:02 -07:00 committed by Facebook GitHub Bot
parent b4a84efb4e
commit 40944cbbdb
5 changed files with 49 additions and 12 deletions

View File

@ -90,6 +90,9 @@ jobs:
container: container:
image: zjay437/rocksdb:0.6 image: zjay437/rocksdb:0.6
options: --shm-size=16gb options: --shm-size=16gb
env:
CC: gcc-10
CXX: g++-10
steps: steps:
- uses: actions/checkout@v4.1.0 - uses: actions/checkout@v4.1.0
- uses: "./.github/actions/pre-steps" - uses: "./.github/actions/pre-steps"
@ -104,38 +107,47 @@ jobs:
container: container:
image: zjay437/rocksdb:0.6 image: zjay437/rocksdb:0.6
options: --shm-size=16gb options: --shm-size=16gb
env:
CC: gcc-10
CXX: g++-10
steps: steps:
- uses: actions/checkout@v4.1.0 - uses: actions/checkout@v4.1.0
- uses: "./.github/actions/pre-steps" - uses: "./.github/actions/pre-steps"
- uses: "./.github/actions/setup-folly" - uses: "./.github/actions/setup-folly"
- run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)" - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)"
- uses: "./.github/actions/post-steps" - uses: "./.github/actions/post-steps"
build-linux-gcc-7-with-folly: build-linux-make-with-folly:
if: ${{ github.repository_owner == 'facebook' }} if: ${{ github.repository_owner == 'facebook' }}
runs-on: runs-on:
labels: 16-core-ubuntu labels: 16-core-ubuntu
container: container:
image: zjay437/rocksdb:0.6 image: zjay437/rocksdb:0.6
options: --shm-size=16gb options: --shm-size=16gb
env:
CC: gcc-10
CXX: g++-10
steps: steps:
- uses: actions/checkout@v4.1.0 - uses: actions/checkout@v4.1.0
- uses: "./.github/actions/pre-steps" - uses: "./.github/actions/pre-steps"
- uses: "./.github/actions/setup-folly" - uses: "./.github/actions/setup-folly"
- uses: "./.github/actions/build-folly" - uses: "./.github/actions/build-folly"
- run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check - run: USE_FOLLY=1 LIB_MODE=static V=1 make -j32 check
- uses: "./.github/actions/post-steps" - uses: "./.github/actions/post-steps"
build-linux-gcc-7-with-folly-lite-no-test: build-linux-make-with-folly-lite-no-test:
if: ${{ github.repository_owner == 'facebook' }} if: ${{ github.repository_owner == 'facebook' }}
runs-on: runs-on:
labels: 16-core-ubuntu labels: 16-core-ubuntu
container: container:
image: zjay437/rocksdb:0.6 image: zjay437/rocksdb:0.6
options: --shm-size=16gb options: --shm-size=16gb
env:
CC: gcc-10
CXX: g++-10
steps: steps:
- uses: actions/checkout@v4.1.0 - uses: actions/checkout@v4.1.0
- uses: "./.github/actions/pre-steps" - uses: "./.github/actions/pre-steps"
- uses: "./.github/actions/setup-folly" - uses: "./.github/actions/setup-folly"
- run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all - run: USE_FOLLY_LITE=1 V=1 make -j32 all
- uses: "./.github/actions/post-steps" - uses: "./.github/actions/post-steps"
build-linux-cmake-with-folly-coroutines: build-linux-cmake-with-folly-coroutines:
if: ${{ github.repository_owner == 'facebook' }} if: ${{ github.repository_owner == 'facebook' }}

View File

@ -617,7 +617,7 @@ if(USE_FOLLY)
FMT_INST_PATH) FMT_INST_PATH)
exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE
GFLAGS_INST_PATH) GFLAGS_INST_PATH)
set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.78.0) set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.83.0)
if(EXISTS ${FMT_INST_PATH}/lib64) if(EXISTS ${FMT_INST_PATH}/lib64)
set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt) set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt)
else() else()
@ -1052,6 +1052,7 @@ if(USE_FOLLY_LITE)
list(APPEND SOURCES list(APPEND SOURCES
third-party/folly/folly/container/detail/F14Table.cpp third-party/folly/folly/container/detail/F14Table.cpp
third-party/folly/folly/detail/Futex.cpp third-party/folly/folly/detail/Futex.cpp
third-party/folly/folly/lang/Exception.cpp
third-party/folly/folly/lang/SafeAssert.cpp third-party/folly/folly/lang/SafeAssert.cpp
third-party/folly/folly/lang/ToAscii.cpp third-party/folly/folly/lang/ToAscii.cpp
third-party/folly/folly/ScopeGuard.cpp third-party/folly/folly/ScopeGuard.cpp
@ -1059,6 +1060,12 @@ if(USE_FOLLY_LITE)
third-party/folly/folly/synchronization/DistributedMutex.cpp third-party/folly/folly/synchronization/DistributedMutex.cpp
third-party/folly/folly/synchronization/ParkingLot.cpp) third-party/folly/folly/synchronization/ParkingLot.cpp)
include_directories(${PROJECT_SOURCE_DIR}/third-party/folly) include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
exec_program(python3 ${PROJECT_SOURCE_DIR}/third-party/folly ARGS
build/fbcode_builder/getdeps.py show-source-dir boost OUTPUT_VARIABLE
BOOST_SOURCE_PATH)
exec_program(ls ARGS -d ${BOOST_SOURCE_PATH}/boost* OUTPUT_VARIABLE
BOOST_INCLUDE_DIR)
include_directories(${BOOST_INCLUDE_DIR})
add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG) add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG)
list(APPEND THIRDPARTY_LIBS glog) list(APPEND THIRDPARTY_LIBS glog)
endif() endif()

View File

@ -103,6 +103,7 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \
export LIB_MODE="$(LIB_MODE)"; \ export LIB_MODE="$(LIB_MODE)"; \
export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \ export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \
export USE_FOLLY="$(USE_FOLLY)"; \ export USE_FOLLY="$(USE_FOLLY)"; \
export USE_FOLLY_LITE="$(USE_FOLLY_LITE)"; \
"$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
# this file is generated by the previous line to set build flags and sources # this file is generated by the previous line to set build flags and sources
include make_config.mk include make_config.mk
@ -500,6 +501,17 @@ endif
ifeq ($(USE_FOLLY_LITE),1) ifeq ($(USE_FOLLY_LITE),1)
# Path to the Folly source code and include files # Path to the Folly source code and include files
FOLLY_DIR = ./third-party/folly FOLLY_DIR = ./third-party/folly
ifneq ($(strip $(BOOST_SOURCE_PATH)),)
BOOST_INCLUDE = $(shell (ls -d $(BOOST_SOURCE_PATH)/boost*/))
# AIX: pre-defined system headers are surrounded by an extern "C" block
ifeq ($(PLATFORM), OS_AIX)
PLATFORM_CCFLAGS += -I$(BOOST_INCLUDE)
PLATFORM_CXXFLAGS += -I$(BOOST_INCLUDE)
else
PLATFORM_CCFLAGS += -isystem $(BOOST_INCLUDE)
PLATFORM_CXXFLAGS += -isystem $(BOOST_INCLUDE)
endif
endif # BOOST_SOURCE_PATH
# AIX: pre-defined system headers are surrounded by an extern "C" block # AIX: pre-defined system headers are surrounded by an extern "C" block
ifeq ($(PLATFORM), OS_AIX) ifeq ($(PLATFORM), OS_AIX)
PLATFORM_CCFLAGS += -I$(FOLLY_DIR) PLATFORM_CCFLAGS += -I$(FOLLY_DIR)
@ -2472,14 +2484,15 @@ checkout_folly:
fi fi
@# Pin to a particular version for public CI, so that PR authors don't @# Pin to a particular version for public CI, so that PR authors don't
@# need to worry about folly breaking our integration. Update periodically @# need to worry about folly breaking our integration. Update periodically
cd third-party/folly && git reset --hard beacd86d63cd71c904632262e6c36f60874d78ba cd third-party/folly && git reset --hard 17054c85c2dee0e84cbee7ef28d7bae7d0b1630e
@# A hack to remove boost dependency.
@# NOTE: this hack is only needed if building using USE_FOLLY_LITE
perl -pi -e 's/^(#include <boost)/\/\/$$1/' third-party/folly/folly/functional/Invoke.h
@# NOTE: this hack is required for clang in some cases @# NOTE: this hack is required for clang in some cases
perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp
@# NOTE: this hack is required for gcc in some cases @# NOTE: this hack is required for gcc in some cases
perl -pi -e 's/(__has_include.<experimental.memory_resource>.)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h perl -pi -e 's/(__has_include.<experimental.memory_resource>.)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h
@# NOTE: this hack is only needed if building target build_folly in the future
perl -pi -e 's,https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz,https://download.libsodium.org/libsodium/releases/old/unsupported/libsodium-1.0.17.tar.gz,' third-party/folly/build/fbcode_builder/manifests/libsodium
@# NOTE: boost source will be needed for any build including `USE_FOLLY_LITE` builds as those depend on boost headers
cd third-party/folly && $(PYTHON) build/fbcode_builder/getdeps.py fetch boost
CXX_M_FLAGS = $(filter -m%, $(CXXFLAGS)) CXX_M_FLAGS = $(filter -m%, $(CXXFLAGS))
@ -2491,8 +2504,6 @@ build_folly:
echo "Please run checkout_folly first"; \ echo "Please run checkout_folly first"; \
false; \ false; \
fi fi
# Restore the original version of Invoke.h with boost dependency
cd third-party/folly && ${GIT_COMMAND} checkout folly/functional/Invoke.h
cd third-party/folly && \ cd third-party/folly && \
CXXFLAGS=" $(CXX_M_FLAGS) -DHAVE_CXX11_ATOMIC " $(PYTHON) build/fbcode_builder/getdeps.py build --no-tests CXXFLAGS=" $(CXX_M_FLAGS) -DHAVE_CXX11_ATOMIC " $(PYTHON) build/fbcode_builder/getdeps.py build --no-tests

View File

@ -585,7 +585,7 @@ EOF
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark" PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark"
fi fi
fi fi
if test $USE_FOLLY; then if test $USE_FOLLY || test $USE_FOLLY_LITE; then
# Test whether libfolly library is installed # Test whether libfolly library is installed
$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <folly/synchronization/DistributedMutex.h> #include <folly/synchronization/DistributedMutex.h>
@ -751,6 +751,11 @@ if [ "$USE_FOLLY" ]; then
FOLLY_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-inst-dir folly` FOLLY_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-inst-dir folly`
fi fi
fi fi
if [ "$USE_FOLLY_LITE" ]; then
if [ "$FOLLY_DIR" ]; then
BOOST_SOURCE_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-source-dir boost`
fi
fi
PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
@ -792,6 +797,7 @@ echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
echo "FIND=$FIND" >> "$OUTPUT" echo "FIND=$FIND" >> "$OUTPUT"
echo "WATCH=$WATCH" >> "$OUTPUT" echo "WATCH=$WATCH" >> "$OUTPUT"
echo "FOLLY_PATH=$FOLLY_PATH" >> "$OUTPUT" echo "FOLLY_PATH=$FOLLY_PATH" >> "$OUTPUT"
echo "BOOST_SOURCE_PATH=$BOOST_SOURCE_PATH" >> "$OUTPUT"
# This will enable some related identifiers for the preprocessor # This will enable some related identifiers for the preprocessor
if test -n "$JEMALLOC"; then if test -n "$JEMALLOC"; then

1
src.mk
View File

@ -407,6 +407,7 @@ TEST_LIB_SOURCES = \
FOLLY_SOURCES = \ FOLLY_SOURCES = \
$(FOLLY_DIR)/folly/container/detail/F14Table.cpp \ $(FOLLY_DIR)/folly/container/detail/F14Table.cpp \
$(FOLLY_DIR)/folly/detail/Futex.cpp \ $(FOLLY_DIR)/folly/detail/Futex.cpp \
$(FOLLY_DIR)/folly/lang/Exception.cpp \
$(FOLLY_DIR)/folly/lang/SafeAssert.cpp \ $(FOLLY_DIR)/folly/lang/SafeAssert.cpp \
$(FOLLY_DIR)/folly/lang/ToAscii.cpp \ $(FOLLY_DIR)/folly/lang/ToAscii.cpp \
$(FOLLY_DIR)/folly/ScopeGuard.cpp \ $(FOLLY_DIR)/folly/ScopeGuard.cpp \