dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for building on s390x platform (#8962) 

Summary:
This PR adds support for building on s390x including updating travis CI. It uses the previous work in https://github.com/facebook/rocksdb/pull/6168 and adds some more changes to get all current tests (make check and jni tests) to pass. The tests were run with snappy, lz4, bzip2 and zstd all compiled in.

There are a few pieces still needed to get the travis build working that I don't think I can do. adamretter is this something you could help with?

1. A prebuilt https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/cmake-3.14.5-Linux-s390x.deb package
2. A https://hub.docker.com/r/evolvedbinary/rocksjava s390x image

Not sure if there is more required for travis. Happy to help in any way I can.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8962

Reviewed By: mrambacher

Differential Revision: D31802198

Pulled By: pdillinger

fbshipit-source-id: 683511466fa6b505f85ba5a9964a268c6151f0c2
This commit is contained in:
Jonathan Albrecht 2021-10-22 10:12:09 -07:00 committed by Facebook GitHub Bot
parent f72fd58565
commit e970248602
14 changed files with 109 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
.travis.yml
View File

@ -5,6 +5,7 @@ os:
arch:
- arm64
- ppc64le
- s390x
compiler:
- clang
- gcc
@ -59,6 +60,12 @@ matrix:
- os: linux
arch: ppc64le
env: JOB_NAME=make-gcc4.8
- os: linux
arch: s390x
env: JOB_NAME=cmake-mingw
- os: linux
arch: s390x
env: JOB_NAME=make-gcc4.8
- os: linux
compiler: clang
- if: type = pull_request AND commit_message !~ /FULL_CI/
@ -73,6 +80,10 @@ matrix:
os: linux
arch: ppc64le
env: TEST_GROUP=1
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: TEST_GROUP=1
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -81,6 +92,10 @@ matrix:
os: linux
arch: ppc64le
env: TEST_GROUP=2
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: TEST_GROUP=2
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -89,6 +104,10 @@ matrix:
os: linux
arch: ppc64le
env: TEST_GROUP=3
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: TEST_GROUP=3
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -97,6 +116,10 @@ matrix:
os: linux
arch: ppc64le
env: TEST_GROUP=4
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: TEST_GROUP=4
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -109,6 +132,10 @@ matrix:
os: linux
arch: ppc64le
env: JOB_NAME=java_test
- if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
os: linux
arch: s390x
env: JOB_NAME=java_test
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -117,6 +144,10 @@ matrix:
os: linux
arch: ppc64le
env: JOB_NAME=lite_build
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=lite_build
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -125,6 +156,10 @@ matrix:
os: linux
arch: ppc64le
env: JOB_NAME=examples
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=examples
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -133,6 +168,10 @@ matrix:
os: linux
arch: ppc64le
env: JOB_NAME=cmake-gcc8
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=cmake-gcc8
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -141,6 +180,10 @@ matrix:
os: linux
arch: ppc64le
env: JOB_NAME=cmake-gcc9
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=cmake-gcc9
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -149,6 +192,10 @@ matrix:
os: linux
arch: ppc64le
env: JOB_NAME=cmake-gcc9-c++20
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=cmake-gcc9-c++20
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: arm64
@ -157,6 +204,10 @@ matrix:
os: linux
arch: ppc64le
env: JOB_NAME=status_checked
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=status_checked
install:
- if [ "${JOB_NAME}" == cmake-gcc8 ]; then

14
CMakeLists.txt
View File

@ -267,6 +267,13 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
endif(HAS_ARMV8_CRC)
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
CHECK_C_COMPILER_FLAG("-march=native" HAS_S390X_MARCH_NATIVE)
if(HAS_S390X_MARCH_NATIVE)
message(STATUS " HAS_S390X_MARCH_NATIVE yes")
endif(HAS_S390X_MARCH_NATIVE)
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
option(PORTABLE "build a portable binary" OFF)
option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
option(FORCE_AVX "force building with AVX, even when PORTABLE=ON" OFF)
@ -292,12 +299,17 @@ if(PORTABLE)
if(FORCE_AVX2)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2 -mbmi -mlzcnt")
endif()
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196")
endif()
endif()
else()
if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
else()
if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64" AND NOT HAS_ARMV8_CRC)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x" AND NOT HAS_S390X_MARCH_NATIVE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196")
elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64" AND NOT HAS_ARMV8_CRC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
endif()
endif()

4
Makefile
View File

@ -1997,7 +1997,7 @@ JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
ifeq ($(PLATFORM), OS_SOLARIS)
ARCH := $(shell isainfo -b)
else ifeq ($(PLATFORM), OS_OPENBSD)
ifneq (,$(filter amd64 ppc64 ppc64le arm64 aarch64 sparc64, $(MACHINE)))
ifneq (,$(filter amd64 ppc64 ppc64le s390x arm64 aarch64 sparc64, $(MACHINE)))
ARCH := 64
else
ARCH := 32
@ -2017,7 +2017,7 @@ ifneq ($(origin JNI_LIBC), undefined)
JNI_LIBC_POSTFIX = -$(JNI_LIBC)
endif
ifneq (,$(filter ppc% arm64 aarch64 sparc64, $(MACHINE)))
ifneq (,$(filter ppc% s390x arm64 aarch64 sparc64, $(MACHINE)))
ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so
else
ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so

14
build_tools/build_detect_platform
View File

@ -638,13 +638,19 @@ if test "0$PORTABLE" -eq 0; then
# Tune for this POWER processor, treating '+' models as base models
POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
elif test -n "`echo $TARGET_ARCHITECTURE | grep -e^arm -e^aarch64`"; then
# TODO: Handle this with approprite options.
COMMON_FLAGS="$COMMON_FLAGS"
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^aarch64`"; then
COMMON_FLAGS="$COMMON_FLAGS"
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ \
-fsyntax-only -march=native - -o /dev/null 2>/dev/null; then
COMMON_FLAGS="$COMMON_FLAGS -march=native "
else
COMMON_FLAGS="$COMMON_FLAGS -march=z196 "
fi
COMMON_FLAGS="$COMMON_FLAGS"
elif [ "$TARGET_OS" == "IOS" ]; then
COMMON_FLAGS="$COMMON_FLAGS"
elif [ "$TARGET_OS" == "AIX" ] || [ "$TARGET_OS" == "SunOS" ]; then
@ -661,6 +667,10 @@ else
TRY_SSE_ETC="1"
fi
if test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
COMMON_FLAGS="$COMMON_FLAGS -march=z196 "
fi
if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then
# For portability compile for macOS 10.12 (2016) or newer
COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.12"

5
db/db_properties_test.cc
View File

@ -233,7 +233,7 @@ void VerifySimilar(uint64_t a, uint64_t b, double bias) {
void VerifyTableProperties(
const TableProperties& base_tp, const TableProperties& new_tp,
double filter_size_bias = CACHE_LINE_SIZE >= 256 ? 0.15 : 0.1,
double filter_size_bias = CACHE_LINE_SIZE >= 256 ? 0.18 : 0.1,
double index_size_bias = 0.1, double data_size_bias = 0.1,
double num_data_blocks_bias = 0.05) {
VerifySimilar(base_tp.data_size, new_tp.data_size, data_size_bias);
@ -627,7 +627,8 @@ TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) {
value_is_delta_encoded);
// Gives larger bias here as index block size, filter block size,
// and data block size become much harder to estimate in this test.
VerifyTableProperties(expected_tp, tp, 0.5, 0.4, 0.4, 0.25);
VerifyTableProperties(expected_tp, tp, CACHE_LINE_SIZE >= 256 ? 0.6 : 0.5,
0.4, 0.4, 0.25);
}
}
}

4
db/version_set_test.cc
View File

@ -2083,7 +2083,7 @@ class VersionSetAtomicGroupTest : public VersionSetTestBase,
});
SyncPoint::GetInstance()->SetCallBack(
"VersionEditHandlerBase::Iterate:Finish", [&](void* arg) {
num_recovered_edits_ = *reinterpret_cast<int*>(arg);
num_recovered_edits_ = *reinterpret_cast<size_t*>(arg);
});
SyncPoint::GetInstance()->SetCallBack(
"AtomicGroupReadBuffer::AddEdit:AtomicGroup",
@ -2123,7 +2123,7 @@ class VersionSetAtomicGroupTest : public VersionSetTestBase,
bool first_in_atomic_group_ = false;
bool last_in_atomic_group_ = false;
int num_edits_in_atomic_group_ = 0;
int num_recovered_edits_ = 0;
size_t num_recovered_edits_ = 0;
VersionEdit corrupted_edit_;
VersionEdit edit_with_incorrect_group_size_;
std::unique_ptr<log::Writer> log_writer_;

2
java/src/main/java/org/rocksdb/util/Environment.java
View File

@ -106,7 +106,7 @@ public class Environment {
if (isPowerPC() || isAarch64()) {
return String.format("%sjni-linux-%s%s", name, ARCH, getLibcPostfix());
} else if (isS390x()) {
return String.format("%sjni-linux%s", name, ARCH);
return String.format("%sjni-linux-%s", name, ARCH);
} else {
return String.format("%sjni-linux%s%s", name, arch, getLibcPostfix());
}

2
monitoring/statistics.h
View File

@ -100,7 +100,9 @@ class StatisticsImpl : public Statistics {
void operator delete[](void *p) { port::cacheline_aligned_free(p); }
};
#ifndef TEST_CACHE_LINE_SIZE
static_assert(sizeof(StatisticsData) % CACHE_LINE_SIZE == 0, "Expected " TOSTRING(CACHE_LINE_SIZE) "-byte aligned");
#endif
CoreLocalArray<StatisticsData> per_core_stats_;

4
port/port_posix.h
View File

@ -193,7 +193,11 @@ extern void InitOnce(OnceType* once, void (*initializer)());
#define ALIGN_AS(n) /*empty*/
#else
#if defined(__s390__)
#if defined(__GNUC__) && __GNUC__ < 6
#define CACHE_LINE_SIZE 64U
#else
#define CACHE_LINE_SIZE 256U
#endif
#elif defined(__powerpc__) || defined(__aarch64__)
#define CACHE_LINE_SIZE 128U
#else

7
third-party/folly/folly/Portability.h vendored
View File

@ -31,6 +31,12 @@
#define FOLLY_PPC64 0
#endif
#if defined(__s390x__)
#define FOLLY_S390X 1
#else
#define FOLLY_S390X 0
#endif
#if defined(__has_builtin)
#define FOLLY_HAS_BUILTIN(...) __has_builtin(__VA_ARGS__)
#else
@ -57,6 +63,7 @@ constexpr bool kIsArchArm = FOLLY_ARM == 1;
constexpr bool kIsArchAmd64 = FOLLY_X64 == 1;
constexpr bool kIsArchAArch64 = FOLLY_AARCH64 == 1;
constexpr bool kIsArchPPC64 = FOLLY_PPC64 == 1;
constexpr bool kIsArchS390X = FOLLY_S390X == 1;
} // namespace folly
namespace folly {

2
third-party/folly/folly/lang/Align.h vendored
View File

@ -120,7 +120,7 @@ struct alignas(max_align_v) max_align_t {};
//
// mimic: std::hardware_destructive_interference_size, C++17
constexpr std::size_t hardware_destructive_interference_size =
kIsArchArm ? 64 : 128;
(kIsArchArm || kIsArchS390X) ? 64 : 128;
static_assert(hardware_destructive_interference_size >= max_align_v, "math?");
// Memory locations within the same cache line are subject to constructive

3
util/bloom_test.cc
View File

@ -552,7 +552,8 @@ TEST_P(FullBloomTest, OptimizeForMemory) {
}
if (FLAGS_bits_per_key == 10) {
EXPECT_LE(total_fp_rate / double{nfilters}, 0.011);
EXPECT_GE(total_fp_rate / double{nfilters}, 0.008);
EXPECT_GE(total_fp_rate / double{nfilters},
CACHE_LINE_SIZE >= 256 ? 0.007 : 0.008);
}
int64_t ex_min_total_size = int64_t{FLAGS_bits_per_key} * total_keys / 8;

6
util/compression.h
View File

@ -1136,7 +1136,11 @@ inline CacheAllocationPtr LZ4_Uncompress(const UncompressionInfo& info,
if (input_length < 8) {
return nullptr;
}
memcpy(&output_len, input_data, sizeof(output_len));
if (port::kLittleEndian) {
memcpy(&output_len, input_data, sizeof(output_len));
} else {
memcpy(&output_len, input_data + 4, sizeof(output_len));
}
input_length -= 8;
input_data += 8;
}

4
utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h
View File

@ -133,6 +133,10 @@ static inline tokutime_t toku_time_now(void) {
return result;
#elif defined(__powerpc__)
return __ppc_get_timebase();
#elif defined(__s390x__)
uint64_t result;
asm volatile("stckf %0" : "=Q"(result) : : "cc");
return result;
#else
#error No timer implementation for this platform
#endif