Add support for Apple Silicon to RocksJava (#9254)

Summary:
Fixes facebook/rocksdb#7720

Updated Makefile with flags to define target architecture when compiling/linking,
and added goal `rocksdbjavastaticosxub` to build a OS X Universal Binary native library.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9254

Reviewed By: mrambacher

Differential Revision: D33551160

Pulled By: pdillinger

fbshipit-source-id: 9ce9962e03aacf55014545a6cdf638b5b14b8fa9
This commit is contained in:
Eric Thérond 2022-01-12 17:19:41 -08:00 committed by Facebook GitHub Bot
parent d247230aec
commit 5602b1d3d9
4 changed files with 124 additions and 21 deletions

View File

@ -24,6 +24,13 @@ commands:
command: |
HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
install-jdk8-on-macos:
steps:
- run:
name: Install JDK 8 on macos
command: |
brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
increase-max-open-files-on-macos:
steps:
- run:
@ -55,7 +62,7 @@ commands:
pre-steps-macos:
steps:
- pre-steps:
python-version: "3.6.0"
python-version: "3.7.8"
post-steps:
steps:
@ -569,7 +576,7 @@ jobs:
build-macos-java-static:
macos:
xcode: 11.3.0
xcode: 12.5.1
resource_class: medium
environment:
JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
@ -578,6 +585,7 @@ jobs:
- install-pyenv-on-macos
- install-gflags-on-macos
- install-cmake-on-macos
- install-jdk8-on-macos
- pre-steps-macos
- run:
name: "Set Java Environment"
@ -587,8 +595,33 @@ jobs:
which java && java -version
which javac && javac -version
- run:
name: "Build RocksDBJava Static Library"
command: make V=1 J=8 -j8 rocksdbjavastatic 2>&1 | .circleci/cat_ignore_eagain
name: "Build RocksDBJava x86 and ARM Static Libraries"
command: make V=1 J=8 -j8 rocksdbjavastaticosx 2>&1 | .circleci/cat_ignore_eagain
- post-steps
build-macos-java-static-universal:
macos:
xcode: 12.5.1
resource_class: medium
environment:
JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
steps:
- increase-max-open-files-on-macos
- install-pyenv-on-macos
- install-gflags-on-macos
- install-cmake-on-macos
- install-jdk8-on-macos
- pre-steps-macos
- run:
name: "Set Java Environment"
command: |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run:
name: "Build RocksDBJava Universal Binary Static Library"
command: make V=1 J=8 -j8 rocksdbjavastaticosx_ub 2>&1 | .circleci/cat_ignore_eagain
- post-steps
build-examples:
@ -793,6 +826,7 @@ workflows:
- build-linux-java-static
- build-macos-java
- build-macos-java-static
- build-macos-java-static-universal
build-examples:
jobs:
- build-examples

View File

@ -134,12 +134,6 @@ CFLAGS += -DHAVE_POWER8
HAVE_POWER8=1
endif
ifeq (,$(shell $(CXX) -fsyntax-only -march=armv8-a+crc+crypto -xc /dev/null 2>&1))
CXXFLAGS += -march=armv8-a+crc+crypto
CFLAGS += -march=armv8-a+crc+crypto
ARMCRC_SOURCE=1
endif
# if we're compiling for shared libraries, add the shared flags
ifeq ($(LIB_MODE),shared)
CXXFLAGS += $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL
@ -264,6 +258,16 @@ $(error pkg-config failed)
endif
endif
CXXFLAGS += $(ARCHFLAG)
ifeq (,$(shell $(CXX) -fsyntax-only -march=armv8-a+crc+crypto -xc /dev/null 2>&1))
ifneq ($(PLATFORM),OS_MACOSX)
CXXFLAGS += -march=armv8-a+crc+crypto
CFLAGS += -march=armv8-a+crc+crypto
ARMCRC_SOURCE=1
endif
endif
export JAVAC_ARGS
CLEAN_FILES += make_config.mk rocksdb.pc
@ -313,6 +317,19 @@ ifeq ($(LIB_MODE),shared)
EXEC_LDFLAGS += -Wl,-rpath -Wl,'$$ORIGIN'
endif
ifeq ($(PLATFORM), OS_MACOSX)
ifeq ($(ARCHFLAG), -arch arm64)
ifneq ($(MACHINE), arm64)
# If we're building on a non-arm64 machine but targeting arm64 Mac, we need to disable
# linking with jemalloc (as it won't be arm64-compatible) and remove some other options
# set during platform detection
DISABLE_JEMALLOC=1
PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CFLAGS))
PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CXXFLAGS))
endif
endif
endif
# ASAN doesn't work well with jemalloc. If we're compiling with ASAN, we should use regular malloc.
ifdef COMPILE_WITH_ASAN
DISABLE_JEMALLOC=1
@ -2055,11 +2072,13 @@ ifneq ($(origin JNI_LIBC), undefined)
JNI_LIBC_POSTFIX = -$(JNI_LIBC)
endif
ifeq (,$(ROCKSDBJNILIB))
ifneq (,$(filter ppc% s390x arm64 aarch64 sparc64, $(MACHINE)))
ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so
else
ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so
endif
endif
ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)
ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux$(ARCH)$(JNI_LIBC_POSTFIX).jar
ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
@ -2085,7 +2104,13 @@ ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
CURL_SSL_OPTS ?= --tlsv1
ifeq ($(PLATFORM), OS_MACOSX)
ifeq (,$(findstring librocksdbjni-osx,$(ROCKSDBJNILIB)))
ifeq ($(MACHINE),arm64)
ROCKSDBJNILIB = librocksdbjni-osx-aarch64.jnilib
else
ROCKSDBJNILIB = librocksdbjni-osx.jnilib
endif
endif
ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-osx.jar
SHA256_CMD = openssl sha256 -r
ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","")
@ -2094,6 +2119,7 @@ else
JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/
endif
endif
ifeq ($(PLATFORM), OS_FREEBSD)
JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/freebsd
ROCKSDBJNILIB = librocksdbjni-freebsd$(ARCH).so
@ -2128,7 +2154,11 @@ zlib-$(ZLIB_VER).tar.gz:
libz.a: zlib-$(ZLIB_VER).tar.gz
-rm -rf zlib-$(ZLIB_VER)
tar xvzf zlib-$(ZLIB_VER).tar.gz
cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' ./configure --static && $(MAKE)
if [ -n"$(ARCHFLAG)" ]; then \
cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' ./configure --static --archs="$(ARCHFLAG)" && $(MAKE); \
else \
cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' ./configure --static && $(MAKE); \
fi
cp zlib-$(ZLIB_VER)/libz.a .
bzip2-$(BZIP2_VER).tar.gz:
@ -2142,7 +2172,7 @@ bzip2-$(BZIP2_VER).tar.gz:
libbz2.a: bzip2-$(BZIP2_VER).tar.gz
-rm -rf bzip2-$(BZIP2_VER)
tar xvzf bzip2-$(BZIP2_VER).tar.gz
cd bzip2-$(BZIP2_VER) && $(MAKE) CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
cd bzip2-$(BZIP2_VER) && $(MAKE) CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 $(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' AR='ar ${EXTRA_ARFLAGS}' libbz2.a
cp bzip2-$(BZIP2_VER)/libbz2.a .
snappy-$(SNAPPY_VER).tar.gz:
@ -2157,7 +2187,7 @@ libsnappy.a: snappy-$(SNAPPY_VER).tar.gz
-rm -rf snappy-$(SNAPPY_VER)
tar xvzf snappy-$(SNAPPY_VER).tar.gz
mkdir snappy-$(SNAPPY_VER)/build
cd snappy-$(SNAPPY_VER)/build && CFLAGS='${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' CXXFLAGS='${JAVA_STATIC_DEPS_CXXFLAGS} ${EXTRA_CXXFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON ${PLATFORM_CMAKE_FLAGS} .. && $(MAKE) ${SNAPPY_MAKE_TARGET}
cd snappy-$(SNAPPY_VER)/build && CFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' CXXFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CXXFLAGS} ${EXTRA_CXXFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON ${PLATFORM_CMAKE_FLAGS} .. && $(MAKE) ${SNAPPY_MAKE_TARGET}
cp snappy-$(SNAPPY_VER)/build/libsnappy.a .
lz4-$(LZ4_VER).tar.gz:
@ -2171,7 +2201,7 @@ lz4-$(LZ4_VER).tar.gz:
liblz4.a: lz4-$(LZ4_VER).tar.gz
-rm -rf lz4-$(LZ4_VER)
tar xvzf lz4-$(LZ4_VER).tar.gz
cd lz4-$(LZ4_VER)/lib && $(MAKE) CFLAGS='-fPIC -O2 ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' all
cd lz4-$(LZ4_VER)/lib && $(MAKE) CFLAGS='-fPIC -O2 $(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' all
cp lz4-$(LZ4_VER)/lib/liblz4.a .
zstd-$(ZSTD_VER).tar.gz:
@ -2185,7 +2215,7 @@ zstd-$(ZSTD_VER).tar.gz:
libzstd.a: zstd-$(ZSTD_VER).tar.gz
-rm -rf zstd-$(ZSTD_VER)
tar xvzf zstd-$(ZSTD_VER).tar.gz
cd zstd-$(ZSTD_VER)/lib && DESTDIR=. PREFIX= $(MAKE) CFLAGS='-fPIC -O2 ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' libzstd.a
cd zstd-$(ZSTD_VER)/lib && DESTDIR=. PREFIX= $(MAKE) CFLAGS='-fPIC -O2 $(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' libzstd.a
cp zstd-$(ZSTD_VER)/lib/libzstd.a .
# A version of each $(LIB_OBJECTS) compiled with -fPIC and a fixed set of static compression libraries
@ -2207,6 +2237,29 @@ endif
$(MAKE) rocksdbjavastatic_deps
$(MAKE) rocksdbjavastatic_libobjects
$(MAKE) rocksdbjavastatic_javalib
$(MAKE) rocksdbjavastatic_jar
rocksdbjavastaticosx: rocksdbjavastaticosx_archs
mv java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx.jnilib
mv java/target/librocksdbjni-osx-arm64.jnilib java/target/librocksdbjni-osx-aarch64.jnilib
rocksdbjavastaticosx_ub: rocksdbjavastaticosx_archs
lipo -create -output ./java/target/$(ROCKSDBJNILIB) java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx-arm64.jnilib
$(MAKE) rocksdbjavastatic_jar
rocksdbjavastaticosx_archs:
$(MAKE) rocksdbjavastaticosx_arch_x86_64
$(MAKE) rocksdbjavastaticosx_arch_arm64
rocksdbjavastaticosx_arch_%:
ifeq ($(JAVA_HOME),)
$(error JAVA_HOME is not set)
endif
$(MAKE) clean-ext-libraries-bin
$(MAKE) clean-rocks
ARCHFLAG="-arch $*" $(MAKE) rocksdbjavastatic_deps
ARCHFLAG="-arch $*" $(MAKE) rocksdbjavastatic_libobjects
ARCHFLAG="-arch $*" ROCKSDBJNILIB="librocksdbjni-osx-$*.jnilib" $(MAKE) rocksdbjavastatic_javalib
ifeq ($(JAR_CMD),)
ifneq ($(JAVA_HOME),)
@ -2225,7 +2278,9 @@ rocksdbjavastatic_javalib:
cd java/target;if [ "$(DEBUG_LEVEL)" == "0" ]; then \
strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
fi
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
rocksdbjavastatic_jar:
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) *
@ -2238,14 +2293,14 @@ rocksdbjavastatic_deps: $(JAVA_COMPRESSIONS)
rocksdbjavastatic_libobjects: $(LIB_OBJECTS)
rocksdbjavastaticrelease: rocksdbjavastatic
rocksdbjavastaticrelease: rocksdbjavastaticosx
cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1
rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class

View File

@ -110,7 +110,11 @@ public class Environment {
return String.format("%sjni-linux%s%s", name, arch, getLibcPostfix());
}
} else if (isMac()) {
return String.format("%sjni-osx", name);
if (isAarch64()) {
return String.format("%sjni-osx-%s", name, ARCH);
} else {
return String.format("%sjni-osx", name);
}
} else if (isFreeBSD()) {
return String.format("%sjni-freebsd%s", name, is64Bit() ? "64" : "32");
} else if (isAix() && is64Bit()) {

View File

@ -43,7 +43,7 @@ public class EnvironmentTest {
@Test
public void mac64() {
setEnvironmentClassFields("mac", "64");
setEnvironmentClassFields("mac", "x86-64");
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".jnilib");
@ -53,6 +53,16 @@ public class EnvironmentTest {
isEqualTo("librocksdbjni.dylib");
}
@Test
public void macAarch64() {
setEnvironmentClassFields("mac", "aarch64");
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).isEqualTo(".jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx-aarch64.jnilib");
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.dylib");
}
@Test
public void nix32() {
// Linux
@ -184,7 +194,7 @@ public class EnvironmentTest {
}
@Test
public void aarch64() {
public void linuxArch64() {
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false);
setEnvironmentClassFields("Linux", "aarch64");
assertThat(Environment.isUnix()).isTrue();