From be09943fb58a2dd3f70e6e30781ebfa3fcbcb8fa Mon Sep 17 00:00:00 2001 From: anand76 Date: Sun, 11 Sep 2022 21:40:11 -0700 Subject: [PATCH] Build and link libfolly with RocksDB (#10103) Summary: The current integration with folly requires cherry-picking folly source files to include in RocksDB for external CI builds. Its not scaleable as we depend on more features in folly, such as coroutines. This PR adds a dependency from RocksDB to the folly library when ```USE_FOLLY``` or ```USE_COROUTINES``` are set. We build folly using the build scripts in ```third-party/folly```, relying on it to download and build its dependencies. A new ```Makefile``` target, ```build_folly```, is provided to make building folly easier. A new option, ```USE_FOLLY_LITE``` is added to retain the old model of compiling selected folly sources with RocksDB. This might be useful for short-term development. Pull Request resolved: https://github.com/facebook/rocksdb/pull/10103 Reviewed By: pdillinger Differential Revision: D38426787 Pulled By: anand1976 fbshipit-source-id: 33bc84abd9fdc7e2567749f02aa1b2494eb62b2f --- .circleci/config.yml | 56 +++++++++++++++- CMakeLists.txt | 83 +++++++++++++++++++----- Makefile | 82 +++++++++++++++++++++-- build_tools/build_detect_platform | 21 ++++++ build_tools/dependencies_platform009.sh | 6 +- build_tools/fbcode_config_platform009.sh | 13 +--- 6 files changed, 221 insertions(+), 40 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a64f28606e..c96197d4e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,6 +72,14 @@ commands: path: /tmp/core_dumps when: on_fail + upgrade-cmake: + steps: + - run: + name: Upgrade cmake + command: | + sudo apt remove --purge cmake + sudo snap install cmake --classic + install-gflags: steps: - run: @@ -93,6 +101,13 @@ commands: command: | make checkout_folly + build-folly: + steps: + - run: + name: Build folly and dependencies + command: | + make build_folly + build-for-benchmarks: steps: - pre-steps @@ -158,7 +173,7 @@ executors: # to run the CI image environment locally: # $ docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it zjay437/rocksdb:0.5 bash # option `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined` is used to enable gdb to attach an existing process - - image: zjay437/rocksdb:0.5 + - image: zjay437/rocksdb:0.6 jobs: build-macos: @@ -355,7 +370,17 @@ jobs: steps: - pre-steps - setup-folly - - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) + - build-folly + - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20) + - post-steps + + build-linux-cmake-with-folly-lite-no-test: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - setup-folly + - run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20) - post-steps build-linux-cmake-with-benchmark: @@ -385,9 +410,19 @@ jobs: steps: - pre-steps - setup-folly + - build-folly - run: USE_FOLLY=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 check - post-steps + build-linux-gcc-7-with-folly-lite-no-test: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - setup-folly + - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all + - post-steps + build-linux-gcc-8-no_test_run: executor: linux-docker resource_class: 2xlarge @@ -396,6 +431,19 @@ jobs: - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all - post-steps + build-linux-cmake-with-folly-coroutines: + executor: linux-docker + resource_class: 2xlarge + environment: + CC: gcc-10 + CXX: g++-10 + steps: + - pre-steps + - setup-folly + - build-folly + - run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20) + - post-steps + build-linux-gcc-10-cxx20-no_test_run: executor: linux-docker resource_class: 2xlarge @@ -427,6 +475,7 @@ jobs: steps: - pre-steps - setup-folly + - build-folly - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check - post-steps @@ -777,7 +826,10 @@ workflows: jobs: - build-linux - build-linux-cmake-with-folly + - build-linux-cmake-with-folly-lite-no-test - build-linux-gcc-7-with-folly + - build-linux-gcc-7-with-folly-lite-no-test + - build-linux-cmake-with-folly-coroutines - build-linux-cmake-with-benchmark - build-linux-encrypted_env-no_compression - build-linux-lite diff --git a/CMakeLists.txt b/CMakeLists.txt index 1584ea1a80..b00f31d23d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, if (WITH_WINDOWS_UTF8_FILENAMES) add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES) endif() +option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON) if ($ENV{CIRCLECI}) message(STATUS "Build for CircieCI env, a few tests may be disabled") @@ -584,10 +585,61 @@ endif() include_directories(${PROJECT_SOURCE_DIR}) include_directories(${PROJECT_SOURCE_DIR}/include) + +if(USE_COROUTINES) + if(USE_FOLLY OR USE_FOLLY_LITE) + message(FATAL_ERROR "Please specify exactly one of USE_COROUTINES," + " USE_FOLLY, and USE_FOLLY_LITE") + endif() + set(CMAKE_CXX_STANDARD 20) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines -Wno-maybe-uninitialized") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-redundant-move") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-invalid-memory-model") + add_compile_definitions(USE_COROUTINES) + set(USE_FOLLY 1) +endif() + if(USE_FOLLY) - include_directories(${PROJECT_SOURCE_DIR}/third-party/folly) - add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG) - list(APPEND THIRDPARTY_LIBS glog) + if(USE_FOLLY_LITE) + message(FATAL_ERROR "Please specify one of USE_FOLLY or USE_FOLLY_LITE") + endif() + if(ROCKSDB_BUILD_SHARED) + message(FATAL_ERROR "Cannot build RocksDB shared library with folly") + endif() + set(ROCKSDB_BUILD_SHARED OFF) + set(GFLAGS_SHARED FALSE) + find_package(folly) + # If cmake could not find the folly-config.cmake file, fall back + # to looking in third-party/folly for folly and its dependencies + if(NOT FOLLY_LIBRARIES) + exec_program(python3 ${PROJECT_SOURCE_DIR}/third-party/folly ARGS + build/fbcode_builder/getdeps.py show-inst-dir OUTPUT_VARIABLE + FOLLY_INST_PATH) + exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../boost* OUTPUT_VARIABLE + BOOST_INST_PATH) + exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../fmt* OUTPUT_VARIABLE + FMT_INST_PATH) + exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE + GFLAGS_INST_PATH) + set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.78.0) + if(EXISTS ${FMT_INST_PATH}/lib64) + set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt) + else() + set(fmt_DIR ${FMT_INST_PATH}/lib/cmake/fmt) + endif() + set(gflags_DIR ${GFLAGS_INST_PATH}/lib/cmake/gflags) + + exec_program(sed ARGS -i 's/gflags_shared//g' + ${FOLLY_INST_PATH}/lib/cmake/folly/folly-targets.cmake) + + include(${FOLLY_INST_PATH}/lib/cmake/folly/folly-config.cmake) + endif() + + add_compile_definitions(USE_FOLLY FOLLY_NO_CONFIG HAVE_CXX11_ATOMIC) + list(APPEND THIRDPARTY_LIBS Folly::folly) + set(FOLLY_LIBS Folly::folly) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--copy-dt-needed-entries") endif() find_package(Threads REQUIRED) @@ -983,7 +1035,7 @@ else() env/io_posix.cc) endif() -if(USE_FOLLY) +if(USE_FOLLY_LITE) list(APPEND SOURCES third-party/folly/folly/container/detail/F14Table.cpp third-party/folly/folly/detail/Futex.cpp @@ -993,13 +1045,14 @@ if(USE_FOLLY) third-party/folly/folly/synchronization/AtomicNotification.cpp third-party/folly/folly/synchronization/DistributedMutex.cpp third-party/folly/folly/synchronization/ParkingLot.cpp) + include_directories(${PROJECT_SOURCE_DIR}/third-party/folly) + add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG) + list(APPEND THIRDPARTY_LIBS glog) endif() set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX}) set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX}) -option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON) - if(WIN32) set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) @@ -1425,7 +1478,7 @@ if(WITH_TESTS) add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND}) set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX}) add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE}) - target_link_libraries(${TESTUTILLIB} ${ROCKSDB_LIB}) + target_link_libraries(${TESTUTILLIB} ${ROCKSDB_LIB} ${FOLLY_LIBS}) if(MSVC) set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb") endif() @@ -1483,32 +1536,32 @@ if(WITH_BENCHMARK_TOOLS) cache/cache_bench.cc cache/cache_bench_tool.cc) target_link_libraries(cache_bench${ARTIFACT_SUFFIX} - ${ROCKSDB_LIB} ${GFLAGS_LIB}) + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) add_executable(memtablerep_bench${ARTIFACT_SUFFIX} memtable/memtablerep_bench.cc) target_link_libraries(memtablerep_bench${ARTIFACT_SUFFIX} - ${ROCKSDB_LIB} ${GFLAGS_LIB}) + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) add_executable(range_del_aggregator_bench${ARTIFACT_SUFFIX} db/range_del_aggregator_bench.cc) target_link_libraries(range_del_aggregator_bench${ARTIFACT_SUFFIX} - ${ROCKSDB_LIB} ${GFLAGS_LIB}) + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) add_executable(table_reader_bench${ARTIFACT_SUFFIX} table/table_reader_bench.cc) target_link_libraries(table_reader_bench${ARTIFACT_SUFFIX} - ${ROCKSDB_LIB} testharness ${GFLAGS_LIB}) + ${ROCKSDB_LIB} testharness ${GFLAGS_LIB} ${FOLLY_LIBS}) add_executable(filter_bench${ARTIFACT_SUFFIX} util/filter_bench.cc) target_link_libraries(filter_bench${ARTIFACT_SUFFIX} - ${ROCKSDB_LIB} ${GFLAGS_LIB}) + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) add_executable(hash_table_bench${ARTIFACT_SUFFIX} utilities/persistent_cache/hash_table_bench.cc) target_link_libraries(hash_table_bench${ARTIFACT_SUFFIX} - ${ROCKSDB_LIB} ${GFLAGS_LIB}) + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) endif() option(WITH_TRACE_TOOLS "build with trace tools" ON) @@ -1516,12 +1569,12 @@ if(WITH_TRACE_TOOLS) add_executable(block_cache_trace_analyzer_tool${ARTIFACT_SUFFIX} tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc) target_link_libraries(block_cache_trace_analyzer_tool${ARTIFACT_SUFFIX} - ${ROCKSDB_LIB} ${GFLAGS_LIB}) + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) add_executable(trace_analyzer${ARTIFACT_SUFFIX} tools/trace_analyzer.cc) target_link_libraries(trace_analyzer${ARTIFACT_SUFFIX} - ${ROCKSDB_LIB} ${GFLAGS_LIB}) + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) endif() diff --git a/Makefile b/Makefile index c7662a6ceb..102224e1fa 100644 --- a/Makefile +++ b/Makefile @@ -139,9 +139,14 @@ endif GIT_COMMAND ?= git ifeq ($(USE_COROUTINES), 1) USE_FOLLY = 1 - OPT += -DUSE_COROUTINES + # glog/logging.h requires HAVE_CXX11_ATOMIC + OPT += -DUSE_COROUTINES -DHAVE_CXX11_ATOMIC ROCKSDB_CXX_STANDARD = c++2a USE_RTTI = 1 +ifneq ($(USE_CLANG), 1) + ROCKSDB_CXX_STANDARD = c++20 + PLATFORM_CXXFLAGS += -fcoroutines +endif endif # if we're compiling for release, compile without debug code (-DNDEBUG) @@ -235,6 +240,7 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \ export USE_CLANG="$(USE_CLANG)"; \ export LIB_MODE="$(LIB_MODE)"; \ export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \ + export USE_FOLLY="$(USE_FOLLY)"; \ "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) # this file is generated by the previous line to set build flags and sources include make_config.mk @@ -415,6 +421,10 @@ ifndef DISABLE_JEMALLOC PLATFORM_CXXFLAGS += -DUSE_JEMALLOC PLATFORM_CCFLAGS += -DUSE_JEMALLOC endif + ifeq ($(USE_FOLLY_LITE),1) + PLATFORM_CXXFLAGS += -DUSE_JEMALLOC + PLATFORM_CCFLAGS += -DUSE_JEMALLOC + endif endif ifdef WITH_JEMALLOC_FLAG PLATFORM_LDFLAGS += -ljemalloc @@ -448,10 +458,57 @@ endif # This provides a Makefile simulation of a Meta-internal folly integration. # It is not validated for general use. +# +# USE_FOLLY links the build targets with libfolly.a. The latter could be +# built using 'make build_folly', or built externally and specified in +# the CXXFLAGS and EXTRA_LDFLAGS env variables. The build_detect_platform +# script tries to detect if an external folly dependency has been specified. +# If not, it exports FOLLY_PATH to the path of the installed Folly and +# dependency libraries. +# +# USE_FOLLY_LITE cherry picks source files from Folly to include in the +# RocksDB library. Its faster and has fewer dependencies on 3rd party +# libraries, but with limited functionality. For example, coroutine +# functionality is not available. ifeq ($(USE_FOLLY),1) - ifeq (,$(FOLLY_DIR)) - FOLLY_DIR = ./third-party/folly +ifeq ($(USE_FOLLY_LITE),1) +$(error Please specify only one of USE_FOLLY and USE_FOLLY_LITE) +endif +ifneq ($(strip $(FOLLY_PATH)),) + BOOST_PATH = $(shell (ls -d $(FOLLY_PATH)/../boost*)) + DBL_CONV_PATH = $(shell (ls -d $(FOLLY_PATH)/../double-conversion*)) + GFLAGS_PATH = $(shell (ls -d $(FOLLY_PATH)/../gflags*)) + GLOG_PATH = $(shell (ls -d $(FOLLY_PATH)/../glog*)) + LIBEVENT_PATH = $(shell (ls -d $(FOLLY_PATH)/../libevent*)) + XZ_PATH = $(shell (ls -d $(FOLLY_PATH)/../xz*)) + LIBSODIUM_PATH = $(shell (ls -d $(FOLLY_PATH)/../libsodium*)) + FMT_PATH = $(shell (ls -d $(FOLLY_PATH)/../fmt*)) + + # For some reason, glog and fmt libraries are under either lib or lib64 + GLOG_LIB_PATH = $(shell (ls -d $(GLOG_PATH)/lib*)) + FMT_LIB_PATH = $(shell (ls -d $(FMT_PATH)/lib*)) + + # AIX: pre-defined system headers are surrounded by an extern "C" block + ifeq ($(PLATFORM), OS_AIX) + PLATFORM_CCFLAGS += -I$(BOOST_PATH)/include -I$(DBL_CONV_PATH)/include -I$(GLOG_PATH)/include -I$(LIBEVENT_PATH)/include -I$(XZ_PATH)/include -I$(LIBSODIUM_PATH)/include -I$(FOLLY_PATH)/include -I$(FMT_PATH)/include + PLATFORM_CXXFLAGS += -I$(BOOST_PATH)/include -I$(DBL_CONV_PATH)/include -I$(GLOG_PATH)/include -I$(LIBEVENT_PATH)/include -I$(XZ_PATH)/include -I$(LIBSODIUM_PATH)/include -I$(FOLLY_PATH)/include -I$(FMT_PATH)/include + else + PLATFORM_CCFLAGS += -isystem $(BOOST_PATH)/include -isystem $(DBL_CONV_PATH)/include -isystem $(GLOG_PATH)/include -isystem $(LIBEVENT_PATH)/include -isystem $(XZ_PATH)/include -isystem $(LIBSODIUM_PATH)/include -isystem $(FOLLY_PATH)/include -isystem $(FMT_PATH)/include + PLATFORM_CXXFLAGS += -isystem $(BOOST_PATH)/include -isystem $(DBL_CONV_PATH)/include -isystem $(GLOG_PATH)/include -isystem $(LIBEVENT_PATH)/include -isystem $(XZ_PATH)/include -isystem $(LIBSODIUM_PATH)/include -isystem $(FOLLY_PATH)/include -isystem $(FMT_PATH)/include endif + + # Add -ldl at the end as gcc resolves a symbol in a library by searching only in libraries specified later + # in the command line + PLATFORM_LDFLAGS += $(BOOST_PATH)/lib/libboost_context.a $(BOOST_PATH)/lib/libboost_filesystem.a $(BOOST_PATH)/lib/libboost_atomic.a $(BOOST_PATH)/lib/libboost_program_options.a $(BOOST_PATH)/lib/libboost_regex.a $(BOOST_PATH)/lib/libboost_system.a $(BOOST_PATH)/lib/libboost_thread.a $(DBL_CONV_PATH)/lib/libdouble-conversion.a $(FOLLY_PATH)/lib/libfolly.a $(FMT_LIB_PATH)/libfmt.a $(GLOG_LIB_PATH)/libglog.so $(GFLAGS_PATH)/lib/libgflags.so.2.2 -ldl + PLATFORM_LDFLAGS += -Wl,-rpath=$(GFLAGS_PATH)/lib -Wl,-rpath=$(GLOG_LIB_PATH) -Wl,-rpath=$(LIBEVENT_PATH)/lib -Wl,-rpath=$(LIBSODIUM_PATH)/lib -Wl,-rpath=$(LIBEVENT_PATH)/lib +endif + PLATFORM_CCFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG + PLATFORM_CXXFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG +endif + +ifeq ($(USE_FOLLY_LITE),1) + # Path to the Folly source code and include files + FOLLY_DIR = ./third-party/folly # AIX: pre-defined system headers are surrounded by an extern "C" block ifeq ($(PLATFORM), OS_AIX) PLATFORM_CCFLAGS += -I$(FOLLY_DIR) @@ -550,7 +607,7 @@ LIB_OBJECTS += $(patsubst %.c, $(OBJ_DIR)/%.o, $(LIB_SOURCES_C)) LIB_OBJECTS += $(patsubst %.S, $(OBJ_DIR)/%.o, $(LIB_SOURCES_ASM)) endif -ifeq ($(USE_FOLLY),1) +ifeq ($(USE_FOLLY_LITE),1) LIB_OBJECTS += $(patsubst %.cpp, $(OBJ_DIR)/%.o, $(FOLLY_SOURCES)) endif @@ -2371,13 +2428,26 @@ checkout_folly: @# need to worry about folly breaking our integration. Update periodically cd third-party/folly && git reset --hard beacd86d63cd71c904632262e6c36f60874d78ba @# A hack to remove boost dependency. - @# NOTE: this hack is not needed if using FBCODE compiler config + @# NOTE: this hack is only needed if building using USE_FOLLY_LITE perl -pi -e 's/^(#include .)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h +build_folly: + FOLLY_INST_PATH=`cd third-party/folly; $(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir`; \ + if [ "$$FOLLY_INST_PATH" ]; then \ + rm -rf $${FOLLY_INST_PATH}/../../*; \ + else \ + echo "Please run checkout_folly first"; \ + false; \ + fi + # Restore the original version of Invoke.h with boost dependency + cd third-party/folly && ${GIT_COMMAND} checkout folly/functional/Invoke.h + cd third-party/folly && MAYBE_AVX2=`echo $(CXXFLAGS) | grep -o -- -DHAVE_AVX2 | sed 's/-DHAVE_AVX2/-mavx2/g' || true` && \ + CXXFLAGS=" $$MAYBE_AVX2 -DHAVE_CXX11_ATOMIC " $(PYTHON) build/fbcode_builder/getdeps.py build --no-tests + # --------------------------------------------------------------------------- # Build size testing # --------------------------------------------------------------------------- @@ -2462,7 +2532,7 @@ endif ifneq ($(SKIP_DEPENDS), 1) DEPFILES = $(patsubst %.cc, $(OBJ_DIR)/%.cc.d, $(ALL_SOURCES)) DEPFILES+ = $(patsubst %.c, $(OBJ_DIR)/%.c.d, $(LIB_SOURCES_C) $(TEST_MAIN_SOURCES_C)) -ifeq ($(USE_FOLLY),1) +ifeq ($(USE_FOLLY_LITE),1) DEPFILES +=$(patsubst %.cpp, $(OBJ_DIR)/%.cpp.d, $(FOLLY_SOURCES)) endif endif diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index c7f9261d84..15129411ac 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -277,6 +277,7 @@ if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then if [ "$FBCODE_BUILD" = "true" ]; then # Enable backtrace on fbcode since the necessary libraries are present COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE" + FOLLY_DIR="third-party/folly" fi true else @@ -601,6 +602,17 @@ EOF PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark" fi fi + if test $USE_FOLLY; then + # Test whether libfolly library is installed + $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF + if [ "$?" != 0 ]; then + FOLLY_DIR="./third-party/folly" + fi + fi + fi # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning. @@ -827,6 +839,13 @@ fi rm -f test.o test_dl.o +# Get the path for the folly installation dir +if [ "$USE_FOLLY" ]; then + if [ "$FOLLY_DIR" ]; then + FOLLY_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-inst-dir folly` + fi +fi + PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" @@ -866,6 +885,8 @@ echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT" echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT" echo "FIND=$FIND" >> "$OUTPUT" echo "WATCH=$WATCH" >> "$OUTPUT" +echo "FOLLY_PATH=$FOLLY_PATH" >> "$OUTPUT" + # This will enable some related identifiers for the preprocessor if test -n "$JEMALLOC"; then echo "JEMALLOC=1" >> "$OUTPUT" diff --git a/build_tools/dependencies_platform009.sh b/build_tools/dependencies_platform009.sh index 72a901f5b5..ce8dd4e06a 100644 --- a/build_tools/dependencies_platform009.sh +++ b/build_tools/dependencies_platform009.sh @@ -19,8 +19,4 @@ BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/08634589372fa5f237bfd374e8c644a836 VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/6ae525939ad02e5e676855082fbbc7828dbafeac/3.15.0/platform009/7f3b187 LUA_BASE=/mnt/gvfs/third-party2/lua/162efd9561a3d21f6869f4814011e9cf1b3ff4dc/5.3.4/platform009/a6271c4 BENCHMARK_BASE=/mnt/gvfs/third-party2/benchmark/30bf49ad6414325e17f3425b0edcb64239427ae3/1.6.1/platform009/7f3b187 -BOOST_BASE=/mnt/gvfs/third-party2/boost/201b7d74941e54b436dfa364a063aa6d2cd7de4c/1.69.0/platform009/8a7ffdf -GLOG_BASE=/mnt/gvfs/third-party2/glog/32d751bd5673375b438158717ab6a57c1cc57e3d/0.3.2_fb/platform009/10a364d/ -FMT_BASE=/mnt/gvfs/third-party2/fmt/ce0c25f67165f4d2c22a29b8ef50f5600d7873ca/6.1.1/platform009/7f3b187/ -DBL_CONV_BASE=/mnt/gvfs/third-party2/double_conversion/109b3d9696d71f1048678cd7da1e22505470543d/20141126/platform009/7f3b187/ -LIBEVENT_BASE=/mnt/gvfs/third-party2/libevent/4a4d3a79a76c2439b6bd471bf3586b3481dde75e/1.4.14b_hphp/platform009/7f3b187/ +GLOG_BASE=/mnt/gvfs/third-party2/glog/32d751bd5673375b438158717ab6a57c1cc57e3d/0.3.2_fb/platform009/10a364d diff --git a/build_tools/fbcode_config_platform009.sh b/build_tools/fbcode_config_platform009.sh index b4e64b6692..8c8ba092c6 100644 --- a/build_tools/fbcode_config_platform009.sh +++ b/build_tools/fbcode_config_platform009.sh @@ -68,20 +68,9 @@ CFLAGS+=" -DGFLAGS=gflags" BENCHMARK_INCLUDE=" -I $BENCHMARK_BASE/include/" BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark${MAYBE_PIC}.a" -BOOST_INCLUDE=" -I $BOOST_BASE/include/" - GLOG_INCLUDE=" -I $GLOG_BASE/include/" GLOG_LIBS=" $GLOG_BASE/lib/libglog${MAYBE_PIC}.a" -FMT_INCLUDE=" -I $FMT_BASE/include/" -FMT_LIBS=" $FMT_BASE/lib/libfmt${MAYBE_PIC}.a" - -DBL_CONV_INCLUDE=" -I $DBL_CONV_BASE/include/" -DBL_CONV_LIBS=" $DBL_CONV_BASE/lib/libdouble-conversion${MAYBE_PIC}.a" - -LIBEVENT_INCLUDE=" -I $LIBEVENT_BASE/include/" -LIBEVENT_LIBS=" $LIBEVENT_BASE/lib/libevent${MAYBE_PIC}.a" - # location of jemalloc JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/" JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc${MAYBE_PIC}.a" @@ -113,7 +102,7 @@ BINUTILS="$BINUTILS_BASE/bin" AR="$BINUTILS/ar" AS="$BINUTILS/as" -DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE $BENCHMARK_INCLUDE $BOOST_INCLUDE $GLOG_INCLUDE $FMT_INCLUDE $DBL_CONV_INCLUDE $LIBEVENT_INCLUDE" +DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE $BENCHMARK_INCLUDE $GLOG_INCLUDE" STDLIBS="-L $GCC_BASE/lib64"