2017-01-20 21:16:22 +00:00
|
|
|
# Prerequisites for Windows:
|
|
|
|
|
# This cmake build is for Windows 64-bit only.
|
2015-07-01 23:13:49 +00:00
|
|
|
#
|
|
|
|
|
# Prerequisites:
|
2016-10-12 20:40:20 +00:00
|
|
|
# You must have at least Visual Studio 2015 Update 3. Start the Developer Command Prompt window that is a part of Visual Studio installation.
|
2015-07-01 23:13:49 +00:00
|
|
|
# Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set.
|
2015-07-09 21:42:41 +00:00
|
|
|
# You must have git.exe in your %PATH% environment variable.
|
2015-07-01 23:13:49 +00:00
|
|
|
#
|
|
|
|
|
# To build Rocksdb for Windows is as easy as 1-2-3-4-5:
|
2015-07-15 21:51:51 +00:00
|
|
|
#
|
2015-07-09 21:42:41 +00:00
|
|
|
# 1. Update paths to third-party libraries in thirdparty.inc file
|
2015-07-01 23:13:49 +00:00
|
|
|
# 2. Create a new directory for build artifacts
|
|
|
|
|
# mkdir build
|
|
|
|
|
# cd build
|
2015-07-09 21:42:41 +00:00
|
|
|
# 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
|
|
|
|
|
# See thirdparty.inc for more information.
|
2018-01-05 22:52:57 +00:00
|
|
|
# sample command: cmake -G "Visual Studio 15 Win64" -DWITH_GFLAGS=1 -DWITH_SNAPPY=1 -DWITH_JEMALLOC=1 -DWITH_JNI=1 ..
|
2015-10-20 20:35:08 +00:00
|
|
|
# 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
|
|
|
|
|
# or simply /m ot use all avail cores)
|
|
|
|
|
# msbuild rocksdb.sln
|
|
|
|
|
#
|
|
|
|
|
# rocksdb.sln build features exclusions of test only code in Release. If you build ALL_BUILD then everything
|
|
|
|
|
# will be attempted but test only code does not build in Release mode.
|
|
|
|
|
#
|
2015-09-30 18:20:23 +00:00
|
|
|
# 5. And release mode (/m[:<N>] is also supported)
|
2015-10-20 20:35:08 +00:00
|
|
|
# msbuild rocksdb.sln /p:Configuration=Release
|
2015-07-01 23:13:49 +00:00
|
|
|
#
|
2017-01-20 21:16:22 +00:00
|
|
|
# Linux:
|
|
|
|
|
#
|
|
|
|
|
# 1. Install a recent toolchain such as devtoolset-3 if you're on a older distro. C++11 required.
|
|
|
|
|
# 2. mkdir build; cd build
|
|
|
|
|
# 3. cmake ..
|
|
|
|
|
# 4. make -j
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2017-09-19 18:56:45 +00:00
|
|
|
cmake_minimum_required(VERSION 2.8.12)
|
2015-07-01 23:13:49 +00:00
|
|
|
project(rocksdb)
|
2018-01-24 00:44:09 +00:00
|
|
|
enable_language(CXX)
|
|
|
|
|
enable_language(C)
|
|
|
|
|
enable_language(ASM)
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
if(POLICY CMP0042)
|
|
|
|
|
cmake_policy(SET CMP0042 NEW)
|
|
|
|
|
endif()
|
|
|
|
|
|
2018-01-16 22:13:38 +00:00
|
|
|
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")
|
2016-09-28 18:53:15 +00:00
|
|
|
|
2017-10-27 20:14:07 +00:00
|
|
|
option(WITH_JEMALLOC "build with JeMalloc" OFF)
|
2018-01-05 22:52:57 +00:00
|
|
|
option(WITH_SNAPPY "build with SNAPPY" OFF)
|
|
|
|
|
option(WITH_LZ4 "build with lz4" OFF)
|
|
|
|
|
option(WITH_ZLIB "build with zlib" OFF)
|
2017-04-06 20:54:49 +00:00
|
|
|
if(MSVC)
|
2018-01-05 22:52:57 +00:00
|
|
|
# Defaults currently different for GFLAGS.
|
|
|
|
|
# We will address find_package work a little later
|
|
|
|
|
option(WITH_GFLAGS "build with GFlags" OFF)
|
|
|
|
|
option(WITH_XPRESS "build with windows built in compression" OFF)
|
2016-09-28 18:53:15 +00:00
|
|
|
include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
|
|
|
|
|
else()
|
2017-04-06 20:54:49 +00:00
|
|
|
if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
|
2018-03-08 18:18:34 +00:00
|
|
|
# FreeBSD has jemalloc as default malloc
|
2017-04-09 18:15:35 +00:00
|
|
|
# but it does not have all the jemalloc files in include/...
|
2017-04-06 20:54:49 +00:00
|
|
|
set(WITH_JEMALLOC ON)
|
2017-04-09 18:15:35 +00:00
|
|
|
else()
|
|
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
find_package(JeMalloc REQUIRED)
|
|
|
|
|
add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
|
|
|
|
|
include_directories(${JEMALLOC_INCLUDE_DIR})
|
|
|
|
|
endif()
|
2017-04-16 18:41:12 +00:00
|
|
|
endif()
|
2018-01-05 22:52:57 +00:00
|
|
|
|
|
|
|
|
# No config file for this
|
|
|
|
|
option(WITH_GFLAGS "build with GFlags" ON)
|
|
|
|
|
if(WITH_GFLAGS)
|
|
|
|
|
find_package(gflags)
|
|
|
|
|
if(gflags_FOUND)
|
|
|
|
|
add_definitions(-DGFLAGS=1)
|
|
|
|
|
include_directories(${gflags_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${gflags_LIBRARIES})
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
2017-08-14 04:29:53 +00:00
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
if(WITH_SNAPPY)
|
|
|
|
|
find_package(snappy REQUIRED)
|
|
|
|
|
add_definitions(-DSNAPPY)
|
|
|
|
|
include_directories(${SNAPPY_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
|
|
|
|
|
endif()
|
2017-08-14 04:29:53 +00:00
|
|
|
|
|
|
|
|
if(WITH_ZLIB)
|
|
|
|
|
find_package(zlib REQUIRED)
|
|
|
|
|
add_definitions(-DZLIB)
|
|
|
|
|
include_directories(${ZLIB_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARIES})
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_BZ2 "build with bzip2" OFF)
|
|
|
|
|
if(WITH_BZ2)
|
|
|
|
|
find_package(bzip2 REQUIRED)
|
|
|
|
|
add_definitions(-DBZIP2)
|
|
|
|
|
include_directories(${BZIP2_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES})
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(WITH_LZ4)
|
|
|
|
|
find_package(lz4 REQUIRED)
|
|
|
|
|
add_definitions(-DLZ4)
|
|
|
|
|
include_directories(${LZ4_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARIES})
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_ZSTD "build with zstd" OFF)
|
|
|
|
|
if(WITH_ZSTD)
|
|
|
|
|
find_package(zstd REQUIRED)
|
|
|
|
|
add_definitions(-DZSTD)
|
|
|
|
|
include_directories(${ZSTD_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARIES})
|
|
|
|
|
endif()
|
2016-09-28 18:53:15 +00:00
|
|
|
endif()
|
|
|
|
|
|
2017-09-13 00:12:07 +00:00
|
|
|
string(TIMESTAMP GIT_DATE_TIME "%Y/%m/%d %H:%M:%S" UTC)
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2015-10-20 18:27:48 +00:00
|
|
|
find_package(Git)
|
|
|
|
|
|
2017-04-03 16:58:35 +00:00
|
|
|
if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
|
2016-09-28 18:53:15 +00:00
|
|
|
if(WIN32)
|
2015-10-20 18:27:48 +00:00
|
|
|
execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
|
2016-09-28 18:53:15 +00:00
|
|
|
else()
|
|
|
|
|
execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
|
|
|
|
|
endif()
|
2015-10-20 18:27:48 +00:00
|
|
|
else()
|
2016-09-28 18:53:15 +00:00
|
|
|
set(GIT_SHA 0)
|
2015-10-20 18:27:48 +00:00
|
|
|
endif()
|
|
|
|
|
|
2016-07-27 03:29:02 +00:00
|
|
|
string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2018-01-18 22:45:48 +00:00
|
|
|
set(SH_CMD "sh")
|
|
|
|
|
execute_process(COMMAND
|
|
|
|
|
${SH_CMD} -c "build_tools/version.sh full"
|
|
|
|
|
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
|
|
|
|
|
OUTPUT_VARIABLE ROCKSDB_VERSION
|
|
|
|
|
)
|
|
|
|
|
string(STRIP "${ROCKSDB_VERSION}" ROCKSDB_VERSION)
|
|
|
|
|
execute_process(COMMAND
|
|
|
|
|
${SH_CMD} -c "build_tools/version.sh major"
|
|
|
|
|
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
|
|
|
|
|
OUTPUT_VARIABLE ROCKSDB_VERSION_MAJOR
|
|
|
|
|
)
|
|
|
|
|
string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
|
2017-01-24 18:48:55 +00:00
|
|
|
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
option(WITH_MD_LIBRARY "build with MD" ON)
|
|
|
|
|
if(WIN32 AND MSVC)
|
|
|
|
|
if(WITH_MD_LIBRARY)
|
|
|
|
|
set(RUNTIME_LIBRARY "MD")
|
|
|
|
|
else()
|
|
|
|
|
set(RUNTIME_LIBRARY "MT")
|
2017-04-03 16:58:35 +00:00
|
|
|
endif()
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
endif()
|
2017-04-22 03:41:37 +00:00
|
|
|
|
2017-08-14 04:46:05 +00:00
|
|
|
set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
|
|
|
|
|
configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
|
|
|
|
|
add_library(build_version OBJECT ${BUILD_VERSION_CC})
|
|
|
|
|
target_include_directories(build_version PRIVATE
|
|
|
|
|
${CMAKE_CURRENT_SOURCE_DIR}/util)
|
|
|
|
|
if(MSVC)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
|
2017-10-19 17:48:47 +00:00
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
|
2017-08-14 04:46:05 +00:00
|
|
|
else()
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
|
2017-09-19 17:15:13 +00:00
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
|
2017-08-14 04:46:05 +00:00
|
|
|
if(MINGW)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
|
|
|
|
|
endif()
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
|
|
|
|
|
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
|
|
|
|
|
include(CheckCXXCompilerFlag)
|
|
|
|
|
CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
|
|
|
|
|
if(HAVE_OMIT_LEAF_FRAME_POINTER)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
2018-01-24 00:44:09 +00:00
|
|
|
include(CheckCCompilerFlag)
|
|
|
|
|
if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
|
|
|
|
|
CHECK_C_COMPILER_FLAG("-maltivec" HAS_ALTIVEC)
|
|
|
|
|
if(HAS_ALTIVEC)
|
|
|
|
|
message(STATUS " HAS_ALTIVEC yes")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maltivec")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maltivec")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power8")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power8")
|
|
|
|
|
endif(HAS_ALTIVEC)
|
|
|
|
|
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
|
|
|
|
|
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
option(PORTABLE "build a portable binary" OFF)
|
|
|
|
|
option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
|
|
|
|
|
if(PORTABLE)
|
|
|
|
|
# MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
|
|
|
|
|
# is available, it is available by default.
|
|
|
|
|
if(FORCE_SSE42 AND NOT MSVC)
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
2017-12-20 02:20:50 +00:00
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -mpclmul")
|
2017-04-22 03:41:37 +00:00
|
|
|
endif()
|
2017-04-16 18:41:12 +00:00
|
|
|
else()
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
if(MSVC)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
|
|
|
|
|
else()
|
2018-01-24 00:44:09 +00:00
|
|
|
if(NOT HAVE_POWER8)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
|
|
|
|
|
endif()
|
2017-04-16 18:41:12 +00:00
|
|
|
endif()
|
2017-04-03 16:58:35 +00:00
|
|
|
endif()
|
|
|
|
|
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
include(CheckCXXSourceCompiles)
|
2017-10-11 19:14:53 +00:00
|
|
|
if(NOT MSVC)
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
2017-12-20 02:20:50 +00:00
|
|
|
set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
|
2017-10-11 19:14:53 +00:00
|
|
|
endif()
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
|
|
|
|
#include <cstdint>
|
|
|
|
|
#include <nmmintrin.h>
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
2017-12-20 02:20:50 +00:00
|
|
|
#include <wmmintrin.h>
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
int main() {
|
|
|
|
|
volatile uint32_t x = _mm_crc32_u32(0, 0);
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
2017-12-20 02:20:50 +00:00
|
|
|
const auto a = _mm_set_epi64x(0, 0);
|
|
|
|
|
const auto b = _mm_set_epi64x(0, 0);
|
|
|
|
|
const auto c = _mm_clmulepi64_si128(a, b, 0x00);
|
|
|
|
|
auto d = _mm_cvtsi128_si64(c);
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
}
|
|
|
|
|
" HAVE_SSE42)
|
2017-10-11 19:14:53 +00:00
|
|
|
unset(CMAKE_REQUIRED_FLAGS)
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
if(HAVE_SSE42)
|
|
|
|
|
add_definitions(-DHAVE_SSE42)
|
|
|
|
|
elseif(FORCE_SSE42)
|
|
|
|
|
message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
|
|
|
|
#if defined(_MSC_VER) && !defined(__thread)
|
|
|
|
|
#define __thread __declspec(thread)
|
|
|
|
|
#endif
|
|
|
|
|
int main() {
|
|
|
|
|
static __thread int tls;
|
|
|
|
|
}
|
|
|
|
|
" HAVE_THREAD_LOCAL)
|
|
|
|
|
if(HAVE_THREAD_LOCAL)
|
|
|
|
|
add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
|
|
|
|
|
endif()
|
|
|
|
|
|
2016-11-01 08:11:02 +00:00
|
|
|
option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
|
|
|
|
|
if(FAIL_ON_WARNINGS)
|
|
|
|
|
if(MSVC)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX")
|
|
|
|
|
else() # assume GCC
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
option(WITH_ASAN "build with ASAN" OFF)
|
|
|
|
|
if(WITH_ASAN)
|
|
|
|
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
|
|
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
message(FATAL "ASAN does not work well with JeMalloc")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
option(WITH_TSAN "build with TSAN" OFF)
|
|
|
|
|
if(WITH_TSAN)
|
|
|
|
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -pie")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC")
|
|
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
message(FATAL "TSAN does not work well with JeMalloc")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
2015-11-19 00:23:19 +00:00
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
option(WITH_UBSAN "build with UBSAN" OFF)
|
|
|
|
|
if(WITH_UBSAN)
|
2017-05-30 18:05:28 +00:00
|
|
|
add_definitions(-DROCKSDB_UBSAN_RUN)
|
2016-09-28 18:53:15 +00:00
|
|
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined")
|
|
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
message(FATAL "UBSAN does not work well with JeMalloc")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2017-12-01 06:37:19 +00:00
|
|
|
find_package(NUMA)
|
|
|
|
|
if(NUMA_FOUND)
|
|
|
|
|
add_definitions(-DNUMA)
|
|
|
|
|
include_directories(${NUMA_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${NUMA_LIBRARIES})
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
find_package(TBB)
|
|
|
|
|
if(TBB_FOUND)
|
|
|
|
|
add_definitions(-DTBB)
|
|
|
|
|
include_directories(${TBB_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${TBB_LIBRARIES})
|
|
|
|
|
endif()
|
|
|
|
|
|
2015-09-29 19:22:48 +00:00
|
|
|
# Used to run CI build and tests so we can run faster
|
|
|
|
|
set(OPTIMIZE_DEBUG_DEFAULT 0) # Debug build is unoptimized by default use -DOPTDBG=1 to optimize
|
|
|
|
|
|
|
|
|
|
if(DEFINED OPTDBG)
|
|
|
|
|
set(OPTIMIZE_DEBUG ${OPTDBG})
|
|
|
|
|
else()
|
|
|
|
|
set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
|
|
|
|
|
endif()
|
|
|
|
|
|
2017-03-30 23:47:19 +00:00
|
|
|
if(MSVC)
|
2016-09-28 18:53:15 +00:00
|
|
|
if((${OPTIMIZE_DEBUG} EQUAL 1))
|
|
|
|
|
message(STATUS "Debug optimization is enabled")
|
2017-04-22 03:41:37 +00:00
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /${RUNTIME_LIBRARY}d")
|
2016-09-28 18:53:15 +00:00
|
|
|
else()
|
2017-04-22 03:41:37 +00:00
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /${RUNTIME_LIBRARY}d")
|
2016-09-28 18:53:15 +00:00
|
|
|
endif()
|
2017-04-22 03:41:37 +00:00
|
|
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
|
2016-09-28 18:53:15 +00:00
|
|
|
|
|
|
|
|
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
|
|
|
|
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
|
2015-09-29 19:22:48 +00:00
|
|
|
endif()
|
|
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
if(CMAKE_COMPILER_IS_GNUCXX)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-memcmp")
|
|
|
|
|
endif()
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
option(ROCKSDB_LITE "Build RocksDBLite version" OFF)
|
|
|
|
|
if(ROCKSDB_LITE)
|
|
|
|
|
add_definitions(-DROCKSDB_LITE)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
|
|
|
|
|
endif()
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
if(CMAKE_SYSTEM_NAME MATCHES "Cygwin")
|
|
|
|
|
add_definitions(-fno-builtin-memcmp -DCYGWIN)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
|
|
|
|
add_definitions(-DOS_MACOSX)
|
|
|
|
|
if(CMAKE_SYSTEM_PROCESSOR MATCHES arm)
|
|
|
|
|
add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE)
|
|
|
|
|
# no debug info for IOS, that will make our library big
|
|
|
|
|
add_definitions(-DNDEBUG)
|
|
|
|
|
endif()
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
|
|
|
|
|
add_definitions(-DOS_LINUX)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
|
|
|
|
|
add_definitions(-DOS_SOLARIS)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
|
|
|
|
|
add_definitions(-DOS_FREEBSD)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
|
|
|
|
|
add_definitions(-DOS_NETBSD)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
|
|
|
|
|
add_definitions(-DOS_OPENBSD)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly")
|
|
|
|
|
add_definitions(-DOS_DRAGONFLYBSD)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "Android")
|
|
|
|
|
add_definitions(-DOS_ANDROID)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
|
|
|
|
|
add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
|
2017-03-30 23:47:19 +00:00
|
|
|
if(MINGW)
|
|
|
|
|
add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA)
|
|
|
|
|
endif()
|
2016-09-28 18:53:15 +00:00
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(NOT WIN32)
|
|
|
|
|
add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_FALLOCATE "build with fallocate" ON)
|
|
|
|
|
if(WITH_FALLOCATE)
|
2017-12-01 06:37:19 +00:00
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
2016-09-28 18:53:15 +00:00
|
|
|
#include <fcntl.h>
|
|
|
|
|
#include <linux/falloc.h>
|
|
|
|
|
int main() {
|
|
|
|
|
int fd = open(\"/dev/null\", 0);
|
|
|
|
|
fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
|
|
|
|
|
}
|
|
|
|
|
" HAVE_FALLOCATE)
|
|
|
|
|
if(HAVE_FALLOCATE)
|
|
|
|
|
add_definitions(-DROCKSDB_FALLOCATE_PRESENT)
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
2017-12-01 06:37:19 +00:00
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
|
|
|
|
#include <fcntl.h>
|
|
|
|
|
int main() {
|
|
|
|
|
int fd = open(\"/dev/null\", 0);
|
|
|
|
|
sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
|
|
|
|
|
}
|
|
|
|
|
" HAVE_SYNC_FILE_RANGE_WRITE)
|
|
|
|
|
if(HAVE_SYNC_FILE_RANGE_WRITE)
|
|
|
|
|
add_definitions(-DROCKSDB_RANGESYNC_PRESENT)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
|
|
|
|
#include <pthread.h>
|
|
|
|
|
int main() {
|
|
|
|
|
(void) PTHREAD_MUTEX_ADAPTIVE_NP;
|
|
|
|
|
}
|
|
|
|
|
" HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
|
|
|
|
|
if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
|
|
|
|
|
add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
include(CheckCXXSymbolExists)
|
|
|
|
|
check_cxx_symbol_exists(malloc_usable_size malloc.h HAVE_MALLOC_USABLE_SIZE)
|
2016-09-28 18:53:15 +00:00
|
|
|
if(HAVE_MALLOC_USABLE_SIZE)
|
|
|
|
|
add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
|
|
|
|
|
endif()
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2017-12-01 06:37:19 +00:00
|
|
|
check_cxx_symbol_exists(sched_getcpu sched.h HAVE_SCHED_GETCPU)
|
|
|
|
|
if(HAVE_SCHED_GETCPU)
|
|
|
|
|
add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT)
|
|
|
|
|
endif()
|
|
|
|
|
|
2015-07-01 23:13:49 +00:00
|
|
|
include_directories(${PROJECT_SOURCE_DIR})
|
|
|
|
|
include_directories(${PROJECT_SOURCE_DIR}/include)
|
2016-09-28 18:53:15 +00:00
|
|
|
include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
|
|
|
|
|
find_package(Threads REQUIRED)
|
2015-07-01 23:13:49 +00:00
|
|
|
|
|
|
|
|
add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)
|
|
|
|
|
|
2015-10-20 20:35:08 +00:00
|
|
|
# Main library source code
|
2016-09-28 18:53:15 +00:00
|
|
|
|
2015-07-01 23:13:49 +00:00
|
|
|
set(SOURCES
|
2017-04-06 02:02:00 +00:00
|
|
|
cache/clock_cache.cc
|
|
|
|
|
cache/lru_cache.cc
|
|
|
|
|
cache/sharded_cache.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/builder.cc
|
|
|
|
|
db/c.cc
|
|
|
|
|
db/column_family.cc
|
2015-07-16 19:10:16 +00:00
|
|
|
db/compacted_db_impl.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/compaction.cc
|
2015-09-10 21:35:25 +00:00
|
|
|
db/compaction_iterator.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/compaction_job.cc
|
|
|
|
|
db/compaction_picker.cc
|
2017-04-07 03:06:34 +00:00
|
|
|
db/compaction_picker_universal.cc
|
2015-07-15 21:51:51 +00:00
|
|
|
db/convenience.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/db_filesnapshot.cc
|
|
|
|
|
db/db_impl.cc
|
2017-04-06 00:14:05 +00:00
|
|
|
db/db_impl_write.cc
|
|
|
|
|
db/db_impl_compaction_flush.cc
|
|
|
|
|
db/db_impl_files.cc
|
|
|
|
|
db/db_impl_open.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/db_impl_debug.cc
|
|
|
|
|
db/db_impl_experimental.cc
|
|
|
|
|
db/db_impl_readonly.cc
|
2016-01-26 01:49:58 +00:00
|
|
|
db/db_info_dumper.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/db_iter.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/dbformat.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/event_helpers.cc
|
|
|
|
|
db/experimental.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/external_sst_file_ingestion_job.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/file_indexer.cc
|
|
|
|
|
db/flush_job.cc
|
|
|
|
|
db/flush_scheduler.cc
|
|
|
|
|
db/forward_iterator.cc
|
|
|
|
|
db/internal_stats.cc
|
|
|
|
|
db/log_reader.cc
|
|
|
|
|
db/log_writer.cc
|
2017-06-01 05:41:44 +00:00
|
|
|
db/malloc_stats.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/managed_iterator.cc
|
|
|
|
|
db/memtable.cc
|
|
|
|
|
db/memtable_list.cc
|
|
|
|
|
db/merge_helper.cc
|
|
|
|
|
db/merge_operator.cc
|
Compaction Support for Range Deletion
Summary:
This diff introduces RangeDelAggregator, which takes ownership of iterators
provided to it via AddTombstones(). The tombstones are organized in a two-level
map (snapshot stripe -> begin key -> tombstone). Tombstone creation avoids data
copy by holding Slices returned by the iterator, which remain valid thanks to pinning.
For compaction, we create a hierarchical range tombstone iterator with structure
matching the iterator over compaction input data. An aggregator based on that
iterator is used by CompactionIterator to determine which keys are covered by
range tombstones. In case of merge operand, the same aggregator is used by
MergeHelper. Upon finishing each file in the compaction, relevant range tombstones
are added to the output file's range tombstone metablock and file boundaries are
updated accordingly.
To check whether a key is covered by range tombstone, RangeDelAggregator::ShouldDelete()
considers tombstones in the key's snapshot stripe. When this function is used outside of
compaction, it also checks newer stripes, which can contain covering tombstones. Currently
the intra-stripe check involves a linear scan; however, in the future we plan to collapse ranges
within a stripe such that binary search can be used.
RangeDelAggregator::AddToBuilder() adds all range tombstones in the table's key-range
to a new table's range tombstone meta-block. Since range tombstones may fall in the gap
between files, we may need to extend some files' key-ranges. The strategy is (1) first file
extends as far left as possible and other files do not extend left, (2) all files extend right
until either the start of the next file or the end of the last range tombstone in the gap,
whichever comes first.
One other notable change is adding release/move semantics to ScopedArenaIterator
such that it can be used to transfer ownership of an arena-allocated iterator, similar to
how unique_ptr is used for malloc'd data.
Depends on D61473
Test Plan: compaction_iterator_test, mock_table, end-to-end tests in D63927
Reviewers: sdong, IslamAbdelRahman, wanning, yhchiang, lightmark
Reviewed By: lightmark
Subscribers: andrewkr, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D62205
2016-10-18 19:04:56 +00:00
|
|
|
db/range_del_aggregator.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/repair.cc
|
2015-08-11 20:44:47 +00:00
|
|
|
db/snapshot_impl.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/table_cache.cc
|
|
|
|
|
db/table_properties_collector.cc
|
|
|
|
|
db/transaction_log_impl.cc
|
|
|
|
|
db/version_builder.cc
|
|
|
|
|
db/version_edit.cc
|
|
|
|
|
db/version_set.cc
|
|
|
|
|
db/wal_manager.cc
|
|
|
|
|
db/write_batch.cc
|
|
|
|
|
db/write_batch_base.cc
|
|
|
|
|
db/write_controller.cc
|
|
|
|
|
db/write_thread.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
env/env.cc
|
|
|
|
|
env/env_chroot.cc
|
2017-06-26 23:52:06 +00:00
|
|
|
env/env_encryption.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
env/env_hdfs.cc
|
2017-06-01 22:30:27 +00:00
|
|
|
env/mock_env.cc
|
2017-06-02 21:13:59 +00:00
|
|
|
memtable/alloc_tracker.cc
|
2015-10-16 21:10:33 +00:00
|
|
|
memtable/hash_cuckoo_rep.cc
|
|
|
|
|
memtable/hash_linklist_rep.cc
|
|
|
|
|
memtable/hash_skiplist_rep.cc
|
2016-01-26 01:49:58 +00:00
|
|
|
memtable/skiplistrep.cc
|
|
|
|
|
memtable/vectorrep.cc
|
2017-06-02 21:13:59 +00:00
|
|
|
memtable/write_buffer_manager.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
monitoring/histogram.cc
|
|
|
|
|
monitoring/histogram_windowing.cc
|
|
|
|
|
monitoring/instrumented_mutex.cc
|
|
|
|
|
monitoring/iostats_context.cc
|
|
|
|
|
monitoring/perf_context.cc
|
|
|
|
|
monitoring/perf_level.cc
|
|
|
|
|
monitoring/statistics.cc
|
|
|
|
|
monitoring/thread_status_impl.cc
|
|
|
|
|
monitoring/thread_status_updater.cc
|
|
|
|
|
monitoring/thread_status_util.cc
|
|
|
|
|
monitoring/thread_status_util_debug.cc
|
|
|
|
|
options/cf_options.cc
|
|
|
|
|
options/db_options.cc
|
|
|
|
|
options/options.cc
|
|
|
|
|
options/options_helper.cc
|
|
|
|
|
options/options_parser.cc
|
|
|
|
|
options/options_sanity_check.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
port/stack_trace.cc
|
|
|
|
|
table/adaptive_table_factory.cc
|
|
|
|
|
table/block.cc
|
|
|
|
|
table/block_based_filter_block.cc
|
|
|
|
|
table/block_based_table_builder.cc
|
|
|
|
|
table/block_based_table_factory.cc
|
|
|
|
|
table/block_based_table_reader.cc
|
|
|
|
|
table/block_builder.cc
|
2017-12-11 23:16:37 +00:00
|
|
|
table/block_fetcher.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
table/block_prefix_index.cc
|
|
|
|
|
table/bloom_block.cc
|
|
|
|
|
table/cuckoo_table_builder.cc
|
|
|
|
|
table/cuckoo_table_factory.cc
|
|
|
|
|
table/cuckoo_table_reader.cc
|
|
|
|
|
table/flush_block_policy.cc
|
|
|
|
|
table/format.cc
|
|
|
|
|
table/full_filter_block.cc
|
|
|
|
|
table/get_context.cc
|
2017-03-04 02:09:43 +00:00
|
|
|
table/index_builder.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
table/iterator.cc
|
2017-02-03 00:38:40 +00:00
|
|
|
table/merging_iterator.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
table/meta_blocks.cc
|
2017-03-04 02:09:43 +00:00
|
|
|
table/partitioned_filter_block.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
table/persistent_cache_helper.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
table/plain_table_builder.cc
|
|
|
|
|
table/plain_table_factory.cc
|
|
|
|
|
table/plain_table_index.cc
|
|
|
|
|
table/plain_table_key_coding.cc
|
|
|
|
|
table/plain_table_reader.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
table/sst_file_writer.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
table/table_properties.cc
|
|
|
|
|
table/two_level_iterator.cc
|
2016-02-17 19:49:52 +00:00
|
|
|
tools/db_bench_tool.cc
|
2015-10-06 22:52:09 +00:00
|
|
|
tools/dump/db_dump_tool.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
tools/ldb_cmd.cc
|
|
|
|
|
tools/ldb_tool.cc
|
|
|
|
|
tools/sst_dump_tool.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/arena.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
util/auto_roll_logger.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/bloom.cc
|
|
|
|
|
util/coding.cc
|
|
|
|
|
util/compaction_job_stats_impl.cc
|
|
|
|
|
util/comparator.cc
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-14 23:59:07 +00:00
|
|
|
util/concurrent_arena.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/crc32c.cc
|
2016-01-29 02:35:01 +00:00
|
|
|
util/delete_scheduler.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/dynamic_bloom.cc
|
|
|
|
|
util/event_logger.cc
|
2015-07-22 00:20:57 +00:00
|
|
|
util/file_reader_writer.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
util/file_util.cc
|
2017-04-04 01:27:24 +00:00
|
|
|
util/filename.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/filter_policy.cc
|
|
|
|
|
util/hash.cc
|
|
|
|
|
util/log_buffer.cc
|
|
|
|
|
util/murmurhash.cc
|
2015-11-06 16:07:08 +00:00
|
|
|
util/random.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/rate_limiter.cc
|
|
|
|
|
util/slice.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
util/sst_file_manager_impl.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/status.cc
|
2015-08-14 23:45:05 +00:00
|
|
|
util/status_message.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/string_util.cc
|
|
|
|
|
util/sync_point.cc
|
|
|
|
|
util/testutil.cc
|
|
|
|
|
util/thread_local.cc
|
2016-08-26 17:41:35 +00:00
|
|
|
util/threadpool_imp.cc
|
2016-03-15 17:57:33 +00:00
|
|
|
util/transaction_test_util.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/xxhash.cc
|
|
|
|
|
utilities/backupable/backupable_db.cc
|
2018-03-06 19:46:20 +00:00
|
|
|
utilities/blob_db/blob_compaction_filter.cc
|
2016-08-09 17:16:32 +00:00
|
|
|
utilities/blob_db/blob_db.cc
|
2017-05-10 21:54:35 +00:00
|
|
|
utilities/blob_db/blob_db_impl.cc
|
2017-05-31 05:16:32 +00:00
|
|
|
utilities/blob_db/blob_dump_tool.cc
|
2017-05-11 16:43:59 +00:00
|
|
|
utilities/blob_db/blob_file.cc
|
2017-05-10 21:54:35 +00:00
|
|
|
utilities/blob_db/blob_log_reader.cc
|
|
|
|
|
utilities/blob_db/blob_log_writer.cc
|
|
|
|
|
utilities/blob_db/blob_log_format.cc
|
2017-07-28 06:16:18 +00:00
|
|
|
utilities/blob_db/ttl_extractor.cc
|
2017-07-21 21:42:32 +00:00
|
|
|
utilities/cassandra/cassandra_compaction_filter.cc
|
|
|
|
|
utilities/cassandra/format.cc
|
|
|
|
|
utilities/cassandra/merge_operator.cc
|
2017-04-24 21:57:27 +00:00
|
|
|
utilities/checkpoint/checkpoint_impl.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
utilities/col_buf_decoder.cc
|
|
|
|
|
utilities/col_buf_encoder.cc
|
|
|
|
|
utilities/column_aware_encoding_util.cc
|
2016-01-28 14:44:31 +00:00
|
|
|
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
|
2016-08-05 22:56:22 +00:00
|
|
|
utilities/date_tiered/date_tiered_db_impl.cc
|
2017-05-12 21:59:57 +00:00
|
|
|
utilities/debug.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/document/document_db.cc
|
|
|
|
|
utilities/document/json_document.cc
|
|
|
|
|
utilities/document/json_document_builder.cc
|
2015-11-19 19:47:12 +00:00
|
|
|
utilities/env_mirror.cc
|
2017-04-04 18:12:47 +00:00
|
|
|
utilities/env_timed.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/geodb/geodb_impl.cc
|
|
|
|
|
utilities/leveldb_options/leveldb_options.cc
|
2016-11-16 23:27:02 +00:00
|
|
|
utilities/lua/rocks_lua_compaction_filter.cc
|
2015-11-04 01:52:17 +00:00
|
|
|
utilities/memory/memory_util.cc
|
2018-03-06 18:20:52 +00:00
|
|
|
utilities/merge_operators/bytesxor.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
utilities/merge_operators/max.cc
|
|
|
|
|
utilities/merge_operators/put.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/merge_operators/string_append/stringappend.cc
|
|
|
|
|
utilities/merge_operators/string_append/stringappend2.cc
|
|
|
|
|
utilities/merge_operators/uint64add.cc
|
2016-07-14 04:08:15 +00:00
|
|
|
utilities/option_change_migration/option_change_migration.cc
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
2015-11-12 14:52:43 +00:00
|
|
|
utilities/options/options_util.cc
|
2016-08-03 00:15:18 +00:00
|
|
|
utilities/persistent_cache/block_cache_tier.cc
|
|
|
|
|
utilities/persistent_cache/block_cache_tier_file.cc
|
|
|
|
|
utilities/persistent_cache/block_cache_tier_metadata.cc
|
2016-04-21 19:20:05 +00:00
|
|
|
utilities/persistent_cache/persistent_cache_tier.cc
|
2016-06-07 23:07:30 +00:00
|
|
|
utilities/persistent_cache/volatile_tier_impl.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/redis/redis_lists.cc
|
2016-06-30 01:44:22 +00:00
|
|
|
utilities/simulator_cache/sim_cache.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/spatialdb/spatial_db.cc
|
2015-08-04 03:42:55 +00:00
|
|
|
utilities/table_properties_collectors/compact_on_deletion_collector.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/transactions/optimistic_transaction_db_impl.cc
|
2017-08-07 23:07:40 +00:00
|
|
|
utilities/transactions/optimistic_transaction.cc
|
2017-10-06 17:26:38 +00:00
|
|
|
utilities/transactions/pessimistic_transaction.cc
|
2017-08-06 00:17:48 +00:00
|
|
|
utilities/transactions/pessimistic_transaction_db.cc
|
2017-10-06 17:26:38 +00:00
|
|
|
utilities/transactions/snapshot_checker.cc
|
|
|
|
|
utilities/transactions/transaction_base.cc
|
2015-09-08 19:36:48 +00:00
|
|
|
utilities/transactions/transaction_db_mutex_impl.cc
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 00:37:33 +00:00
|
|
|
utilities/transactions/transaction_lock_mgr.cc
|
|
|
|
|
utilities/transactions/transaction_util.cc
|
2017-08-07 23:07:40 +00:00
|
|
|
utilities/transactions/write_prepared_txn.cc
|
2017-11-02 18:05:55 +00:00
|
|
|
utilities/transactions/write_prepared_txn_db.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/ttl/db_ttl_impl.cc
|
|
|
|
|
utilities/write_batch_with_index/write_batch_with_index.cc
|
|
|
|
|
utilities/write_batch_with_index/write_batch_with_index_internal.cc
|
2016-09-28 18:53:15 +00:00
|
|
|
$<TARGET_OBJECTS:build_version>)
|
|
|
|
|
|
2017-10-11 19:14:53 +00:00
|
|
|
if(HAVE_SSE42 AND NOT FORCE_SSE42)
|
2017-10-19 17:48:47 +00:00
|
|
|
if(NOT MSVC)
|
2017-10-11 19:14:53 +00:00
|
|
|
set_source_files_properties(
|
|
|
|
|
util/crc32c.cc
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
2017-12-20 02:20:50 +00:00
|
|
|
PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
|
2017-10-19 17:48:47 +00:00
|
|
|
endif()
|
2017-10-11 19:14:53 +00:00
|
|
|
endif()
|
|
|
|
|
|
2018-01-24 00:44:09 +00:00
|
|
|
if(HAVE_POWER8)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
util/crc32c_ppc.c
|
|
|
|
|
util/crc32c_ppc_asm.S)
|
|
|
|
|
endif(HAVE_POWER8)
|
|
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
if(WIN32)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
port/win/io_win.cc
|
|
|
|
|
port/win/env_win.cc
|
|
|
|
|
port/win/env_default.cc
|
|
|
|
|
port/win/port_win.cc
|
|
|
|
|
port/win/win_logger.cc
|
2017-02-06 22:43:55 +00:00
|
|
|
port/win/win_thread.cc
|
2016-09-28 18:53:15 +00:00
|
|
|
port/win/xpress_win.cc)
|
2017-11-02 18:05:55 +00:00
|
|
|
|
2017-10-27 20:14:07 +00:00
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
port/win/win_jemalloc.cc)
|
|
|
|
|
endif()
|
2017-11-02 18:05:55 +00:00
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
else()
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
port/port_posix.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
env/env_posix.cc
|
|
|
|
|
env/io_posix.cc)
|
2016-09-28 18:53:15 +00:00
|
|
|
endif()
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2017-05-05 23:09:24 +00:00
|
|
|
set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
|
|
|
|
|
set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
|
|
|
|
|
set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
|
2017-01-20 21:16:22 +00:00
|
|
|
if(WIN32)
|
|
|
|
|
set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
|
|
|
|
|
set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
else()
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
2017-05-15 21:42:32 +00:00
|
|
|
set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
|
2017-01-20 21:16:22 +00:00
|
|
|
set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
|
|
|
|
|
add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
|
|
|
|
|
target_link_libraries(${ROCKSDB_SHARED_LIB}
|
|
|
|
|
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
|
|
|
|
|
LINKER_LANGUAGE CXX
|
2017-01-24 18:48:55 +00:00
|
|
|
VERSION ${ROCKSDB_VERSION}
|
|
|
|
|
SOVERSION ${ROCKSDB_VERSION_MAJOR}
|
2017-01-20 21:16:22 +00:00
|
|
|
CXX_STANDARD 11
|
|
|
|
|
OUTPUT_NAME "rocksdb")
|
|
|
|
|
endif()
|
|
|
|
|
|
2016-09-28 18:53:15 +00:00
|
|
|
option(WITH_LIBRADOS "Build with librados" OFF)
|
|
|
|
|
if(WITH_LIBRADOS)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
utilities/env_librados.cc)
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS rados)
|
|
|
|
|
endif()
|
2015-10-20 20:35:08 +00:00
|
|
|
|
2017-01-20 21:16:22 +00:00
|
|
|
add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
|
|
|
|
|
target_link_libraries(${ROCKSDB_STATIC_LIB}
|
2016-09-28 18:53:15 +00:00
|
|
|
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
|
|
|
|
|
if(WIN32)
|
2017-01-20 21:16:22 +00:00
|
|
|
add_library(${ROCKSDB_IMPORT_LIB} SHARED ${SOURCES})
|
|
|
|
|
target_link_libraries(${ROCKSDB_IMPORT_LIB}
|
|
|
|
|
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
|
2017-03-30 23:47:19 +00:00
|
|
|
COMPILE_DEFINITIONS "ROCKSDB_DLL;ROCKSDB_LIBRARY_EXPORTS")
|
|
|
|
|
if(MSVC)
|
|
|
|
|
set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES
|
|
|
|
|
COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_STATIC_LIB}.pdb")
|
|
|
|
|
set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
|
|
|
|
|
COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_IMPORT_LIB}.pdb")
|
|
|
|
|
endif()
|
2016-09-28 18:53:15 +00:00
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_JNI "build with JNI" OFF)
|
|
|
|
|
if(WITH_JNI OR JNI)
|
|
|
|
|
message(STATUS "JNI library is enabled")
|
|
|
|
|
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
|
2016-01-28 14:44:31 +00:00
|
|
|
else()
|
|
|
|
|
message(STATUS "JNI library is disabled")
|
|
|
|
|
endif()
|
|
|
|
|
|
2017-08-21 21:41:11 +00:00
|
|
|
# Installation and packaging
|
|
|
|
|
if(WIN32)
|
|
|
|
|
option(ROCKSDB_INSTALL_ON_WINDOWS "Enable install target on Windows" OFF)
|
|
|
|
|
endif()
|
|
|
|
|
if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
|
|
|
|
|
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
|
|
|
|
|
if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
|
|
|
|
|
# Change default installation prefix on Linux to /usr
|
|
|
|
|
set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "Install path prefix, prepended onto install directories." FORCE)
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
include(GNUInstallDirs)
|
2017-08-28 23:56:49 +00:00
|
|
|
include(CMakePackageConfigHelpers)
|
|
|
|
|
|
|
|
|
|
set(package_config_destination ${CMAKE_INSTALL_LIBDIR}/cmake/rocksdb)
|
|
|
|
|
|
|
|
|
|
configure_package_config_file(
|
2018-01-16 22:13:38 +00:00
|
|
|
${CMAKE_CURRENT_LIST_DIR}/cmake/RocksDBConfig.cmake.in RocksDBConfig.cmake
|
2017-08-28 23:56:49 +00:00
|
|
|
INSTALL_DESTINATION ${package_config_destination}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
write_basic_package_version_file(
|
|
|
|
|
RocksDBConfigVersion.cmake
|
|
|
|
|
VERSION ${ROCKSDB_VERSION}
|
|
|
|
|
COMPATIBILITY SameMajorVersion
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
|
|
|
|
|
|
|
|
|
|
install(
|
|
|
|
|
TARGETS ${ROCKSDB_STATIC_LIB}
|
|
|
|
|
EXPORT RocksDBTargets
|
|
|
|
|
COMPONENT devel
|
|
|
|
|
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
|
|
|
|
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
install(
|
|
|
|
|
TARGETS ${ROCKSDB_SHARED_LIB}
|
|
|
|
|
EXPORT RocksDBTargets
|
|
|
|
|
COMPONENT runtime
|
|
|
|
|
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
|
|
|
|
|
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
|
|
|
|
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
install(
|
|
|
|
|
EXPORT RocksDBTargets
|
|
|
|
|
COMPONENT devel
|
|
|
|
|
DESTINATION ${package_config_destination}
|
|
|
|
|
NAMESPACE RocksDB::
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
install(
|
|
|
|
|
FILES
|
|
|
|
|
${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfig.cmake
|
|
|
|
|
${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfigVersion.cmake
|
|
|
|
|
COMPONENT devel
|
|
|
|
|
DESTINATION ${package_config_destination}
|
|
|
|
|
)
|
2017-08-21 21:41:11 +00:00
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_TESTS "build with tests" ON)
|
|
|
|
|
if(WITH_TESTS)
|
2017-08-29 01:36:35 +00:00
|
|
|
set(TESTS
|
2017-04-06 02:02:00 +00:00
|
|
|
cache/cache_test.cc
|
|
|
|
|
cache/lru_cache_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/column_family_test.cc
|
|
|
|
|
db/compact_files_test.cc
|
2015-12-09 15:46:10 +00:00
|
|
|
db/compaction_iterator_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/compaction_job_stats_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/compaction_job_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/compaction_picker_test.cc
|
|
|
|
|
db/comparator_db_test.cc
|
|
|
|
|
db/corruption_test.cc
|
|
|
|
|
db/cuckoo_table_db_test.cc
|
2017-02-27 20:19:23 +00:00
|
|
|
db/db_basic_test.cc
|
2017-10-03 16:08:07 +00:00
|
|
|
db/db_blob_index_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/db_block_cache_test.cc
|
|
|
|
|
db/db_bloom_filter_test.cc
|
2015-07-14 23:03:47 +00:00
|
|
|
db/db_compaction_filter_test.cc
|
2015-07-21 10:05:57 +00:00
|
|
|
db/db_compaction_test.cc
|
2015-07-14 23:03:47 +00:00
|
|
|
db/db_dynamic_level_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/db_flush_test.cc
|
2015-07-20 23:05:28 +00:00
|
|
|
db/db_inplace_update_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/db_io_failure_test.cc
|
2016-01-20 23:17:52 +00:00
|
|
|
db/db_iter_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/db_iterator_test.cc
|
2015-07-20 22:51:33 +00:00
|
|
|
db/db_log_iter_test.cc
|
2016-11-14 02:58:17 +00:00
|
|
|
db/db_memtable_test.cc
|
2016-12-16 19:00:03 +00:00
|
|
|
db/db_merge_operator_test.cc
|
2016-07-12 22:30:38 +00:00
|
|
|
db/db_options_test.cc
|
2016-01-20 23:17:52 +00:00
|
|
|
db/db_properties_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/db_range_del_test.cc
|
|
|
|
|
db/db_sst_test.cc
|
2017-04-26 21:18:58 +00:00
|
|
|
db/db_statistics_test.cc
|
2016-01-20 23:17:52 +00:00
|
|
|
db/db_table_properties_test.cc
|
|
|
|
|
db/db_tailing_iter_test.cc
|
|
|
|
|
db/db_test.cc
|
2016-03-01 02:38:03 +00:00
|
|
|
db/db_test2.cc
|
2015-07-15 01:24:45 +00:00
|
|
|
db/db_universal_compaction_test.cc
|
2015-08-05 18:56:19 +00:00
|
|
|
db/db_wal_test.cc
|
2017-05-31 17:45:47 +00:00
|
|
|
db/db_write_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/dbformat_test.cc
|
|
|
|
|
db/deletefile_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/external_sst_file_basic_test.cc
|
|
|
|
|
db/external_sst_file_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/fault_injection_test.cc
|
|
|
|
|
db/file_indexer_test.cc
|
|
|
|
|
db/filename_test.cc
|
|
|
|
|
db/flush_job_test.cc
|
|
|
|
|
db/listener_test.cc
|
|
|
|
|
db/log_test.cc
|
2015-10-14 18:06:27 +00:00
|
|
|
db/manual_compaction_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/memtable_list_test.cc
|
2015-07-22 01:04:28 +00:00
|
|
|
db/merge_helper_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
db/merge_test.cc
|
2015-11-11 06:58:01 +00:00
|
|
|
db/options_file_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/perf_context_test.cc
|
|
|
|
|
db/plain_table_db_test.cc
|
|
|
|
|
db/prefix_test.cc
|
2016-03-18 22:18:42 +00:00
|
|
|
db/repair_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
db/table_properties_collector_test.cc
|
|
|
|
|
db/version_builder_test.cc
|
|
|
|
|
db/version_edit_test.cc
|
|
|
|
|
db/version_set_test.cc
|
|
|
|
|
db/wal_manager_test.cc
|
|
|
|
|
db/write_batch_test.cc
|
|
|
|
|
db/write_callback_test.cc
|
|
|
|
|
db/write_controller_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
env/env_basic_test.cc
|
|
|
|
|
env/env_test.cc
|
|
|
|
|
env/mock_env_test.cc
|
2017-04-06 20:59:31 +00:00
|
|
|
memtable/inlineskiplist_test.cc
|
|
|
|
|
memtable/skiplist_test.cc
|
2017-06-02 21:13:59 +00:00
|
|
|
memtable/write_buffer_manager_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
monitoring/histogram_test.cc
|
|
|
|
|
monitoring/iostats_context_test.cc
|
|
|
|
|
monitoring/statistics_test.cc
|
|
|
|
|
options/options_settable_test.cc
|
|
|
|
|
options/options_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
table/block_based_filter_block_test.cc
|
|
|
|
|
table/block_test.cc
|
2017-10-13 01:19:10 +00:00
|
|
|
table/cleanable_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
table/cuckoo_table_builder_test.cc
|
|
|
|
|
table/cuckoo_table_reader_test.cc
|
|
|
|
|
table/full_filter_block_test.cc
|
|
|
|
|
table/merger_test.cc
|
|
|
|
|
table/table_test.cc
|
2015-10-15 00:08:28 +00:00
|
|
|
tools/ldb_cmd_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
tools/reduce_levels_test.cc
|
2015-10-15 00:08:28 +00:00
|
|
|
tools/sst_dump_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/arena_test.cc
|
2017-04-04 01:27:24 +00:00
|
|
|
util/auto_roll_logger_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/autovector_test.cc
|
|
|
|
|
util/bloom_test.cc
|
|
|
|
|
util/coding_test.cc
|
|
|
|
|
util/crc32c_test.cc
|
2015-12-09 15:46:10 +00:00
|
|
|
util/delete_scheduler_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/dynamic_bloom_test.cc
|
|
|
|
|
util/event_logger_test.cc
|
RangeSync not to sync last 1MB of the file
Summary:
From other ones' investigation:
"sync_file_range() behavior highly depends on kernel version and filesystem.
xfs does neighbor page flushing outside of the specified ranges. For example, sync_file_range(fd, 8192, 16384) does not only trigger flushing page #3 to #4, but also flushing many more dirty pages (i.e. up to page#16)... Ranges of the sync_file_range() should be far enough from write() offset (at least 1MB)."
Test Plan: make all check
Reviewers: igor, rven, kradhakrishnan, yhchiang, IslamAbdelRahman, anthony
Reviewed By: anthony
Subscribers: yoshinorim, MarkCallaghan, sumeet, domas, dhruba, leveldb, ljin
Differential Revision: https://reviews.facebook.net/D15807
2015-07-20 21:46:15 +00:00
|
|
|
util/file_reader_writer_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
util/filelock_test.cc
|
2017-07-10 19:22:26 +00:00
|
|
|
util/hash_test.cc
|
2015-07-22 01:04:28 +00:00
|
|
|
util/heap_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/rate_limiter_test.cc
|
|
|
|
|
util/slice_transform_test.cc
|
2017-05-10 21:54:35 +00:00
|
|
|
util/timer_queue_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
util/thread_list_test.cc
|
|
|
|
|
util/thread_local_test.cc
|
|
|
|
|
utilities/backupable/backupable_db_test.cc
|
2017-05-31 05:16:32 +00:00
|
|
|
utilities/blob_db/blob_db_test.cc
|
2017-07-21 21:42:32 +00:00
|
|
|
utilities/cassandra/cassandra_functional_test.cc
|
|
|
|
|
utilities/cassandra/cassandra_format_test.cc
|
|
|
|
|
utilities/cassandra/cassandra_row_merge_test.cc
|
|
|
|
|
utilities/cassandra/cassandra_serialize_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/checkpoint/checkpoint_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
utilities/column_aware_encoding_test.cc
|
2016-08-05 22:56:22 +00:00
|
|
|
utilities/date_tiered/date_tiered_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/document/document_db_test.cc
|
|
|
|
|
utilities/document/json_document_test.cc
|
|
|
|
|
utilities/geodb/geodb_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
utilities/lua/rocks_lua_test.cc
|
2015-11-04 01:52:17 +00:00
|
|
|
utilities/memory/memory_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/merge_operators/string_append/stringappend_test.cc
|
2017-04-06 02:02:00 +00:00
|
|
|
utilities/object_registry_test.cc
|
2016-07-14 04:08:15 +00:00
|
|
|
utilities/option_change_migration/option_change_migration_test.cc
|
2015-11-12 22:53:19 +00:00
|
|
|
utilities/options/options_util_test.cc
|
2016-06-02 20:42:41 +00:00
|
|
|
utilities/persistent_cache/hash_table_test.cc
|
2016-06-07 23:07:30 +00:00
|
|
|
utilities/persistent_cache/persistent_cache_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/redis/redis_lists_test.cc
|
|
|
|
|
utilities/spatialdb/spatial_db_test.cc
|
2017-07-28 19:18:09 +00:00
|
|
|
utilities/simulator_cache/sim_cache_test.cc
|
2015-08-04 03:42:55 +00:00
|
|
|
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/transactions/optimistic_transaction_test.cc
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 00:37:33 +00:00
|
|
|
utilities/transactions/transaction_test.cc
|
2017-08-31 16:27:14 +00:00
|
|
|
utilities/transactions/write_prepared_transaction_test.cc
|
2015-07-01 23:13:49 +00:00
|
|
|
utilities/ttl/ttl_test.cc
|
|
|
|
|
utilities/write_batch_with_index/write_batch_with_index_test.cc
|
2017-08-29 01:36:35 +00:00
|
|
|
)
|
|
|
|
|
if(WITH_LIBRADOS)
|
|
|
|
|
list(APPEND TESTS utilities/env_librados_test.cc)
|
|
|
|
|
endif()
|
2015-10-20 20:35:08 +00:00
|
|
|
|
2017-08-29 01:36:35 +00:00
|
|
|
set(BENCHMARKS
|
|
|
|
|
cache/cache_bench.cc
|
|
|
|
|
memtable/memtablerep_bench.cc
|
|
|
|
|
tools/db_bench.cc
|
|
|
|
|
table/table_reader_bench.cc
|
|
|
|
|
utilities/column_aware_encoding_exp.cc
|
|
|
|
|
utilities/persistent_cache/hash_table_bench.cc)
|
|
|
|
|
add_library(testharness OBJECT util/testharness.cc)
|
|
|
|
|
foreach(sourcefile ${BENCHMARKS})
|
2016-09-28 18:53:15 +00:00
|
|
|
get_filename_component(exename ${sourcefile} NAME_WE)
|
|
|
|
|
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
|
|
|
|
|
$<TARGET_OBJECTS:testharness>)
|
2017-08-29 01:36:35 +00:00
|
|
|
target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
|
|
|
|
|
endforeach(sourcefile ${BENCHMARKS})
|
|
|
|
|
|
|
|
|
|
# For test util library that is build only in DEBUG mode
|
|
|
|
|
# and linked to tests. Add test only code that is not #ifdefed for Release here.
|
|
|
|
|
set(TESTUTIL_SOURCE
|
|
|
|
|
db/db_test_util.cc
|
|
|
|
|
monitoring/thread_status_updater_debug.cc
|
|
|
|
|
table/mock_table.cc
|
|
|
|
|
util/fault_injection_test_env.cc
|
|
|
|
|
utilities/cassandra/test_utils.cc
|
|
|
|
|
)
|
|
|
|
|
# test utilities are only build in debug
|
|
|
|
|
enable_testing()
|
|
|
|
|
add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
|
|
|
|
|
set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
|
|
|
|
|
add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
|
|
|
|
|
if(MSVC)
|
|
|
|
|
set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
|
|
|
|
|
endif()
|
|
|
|
|
set_target_properties(${TESTUTILLIB}
|
|
|
|
|
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Tests are excluded from Release builds
|
|
|
|
|
set(TEST_EXES ${TESTS})
|
|
|
|
|
|
|
|
|
|
foreach(sourcefile ${TEST_EXES})
|
|
|
|
|
get_filename_component(exename ${sourcefile} NAME_WE)
|
2018-01-16 22:13:38 +00:00
|
|
|
add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile}
|
2017-08-29 01:36:35 +00:00
|
|
|
$<TARGET_OBJECTS:testharness>)
|
2018-01-16 22:13:38 +00:00
|
|
|
set_target_properties(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX}
|
2017-08-29 01:36:35 +00:00
|
|
|
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
|
2018-01-16 22:13:38 +00:00
|
|
|
OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
|
2017-08-29 01:36:35 +00:00
|
|
|
)
|
2018-01-16 22:13:38 +00:00
|
|
|
target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
|
2017-08-29 01:36:35 +00:00
|
|
|
if(NOT "${exename}" MATCHES "db_sanity_test")
|
|
|
|
|
add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
|
2018-01-16 22:13:38 +00:00
|
|
|
add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
|
2017-08-29 01:36:35 +00:00
|
|
|
endif()
|
|
|
|
|
endforeach(sourcefile ${TEST_EXES})
|
|
|
|
|
|
|
|
|
|
# C executables must link to a shared object
|
|
|
|
|
set(C_TESTS db/c_test.c)
|
|
|
|
|
set(C_TEST_EXES ${C_TESTS})
|
|
|
|
|
|
|
|
|
|
foreach(sourcefile ${C_TEST_EXES})
|
|
|
|
|
string(REPLACE ".c" "" exename ${sourcefile})
|
|
|
|
|
string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
|
|
|
|
|
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
|
|
|
|
|
set_target_properties(${exename}${ARTIFACT_SUFFIX}
|
|
|
|
|
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
|
|
|
|
|
)
|
|
|
|
|
target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
|
2016-09-28 18:53:15 +00:00
|
|
|
add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
|
|
|
|
|
add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
|
2017-08-29 01:36:35 +00:00
|
|
|
endforeach(sourcefile ${C_TEST_EXES})
|
2017-08-21 21:41:11 +00:00
|
|
|
endif()
|
2017-01-24 18:48:55 +00:00
|
|
|
|
2017-08-21 21:41:11 +00:00
|
|
|
option(WITH_TOOLS "build with tools" ON)
|
|
|
|
|
if(WITH_TOOLS)
|
|
|
|
|
add_subdirectory(tools)
|
2017-01-24 18:48:55 +00:00
|
|
|
endif()
|