*: update jemalloc and simplify build (#1)

- Remove all cross compilation and reduce supporting platforms - Remove complicated checks and builds - Update jemalloc to 5.2.1 - Fix gnzlbg/jemallocator#147 Signed-off-by: Jay Lee <BusyJayLee@gmail.com>
2024-12-03 14:53:05 +00:00 · 2020-07-21 11:18:43 +08:00 · 2020-07-21 11:18:43 +08:00 · 21c54a2446
parent a09f18a739
commit 21c54a2446
8 changed files with 667 additions and 446 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -7,61 +7,17 @@ jobs:
    # Linux
    - name: "aarch64-unknown-linux-gnu"
      env: TARGET=aarch64-unknown-linux-gnu NO_JEMALLOC_TESTS=1
-    - name: "arm-unknown-linux-gnueabi"
-      env: TARGET=arm-unknown-linux-gnueabi NO_JEMALLOC_TESTS=1
-    - name: "armv7-unknown-linux-gnueabihf"
-      env: TARGET=armv7-unknown-linux-gnueabihf NO_JEMALLOC_TESTS=1
-    - name: "i586-unknown-linux-gnu"
-      env: TARGET=i586-unknown-linux-gnu
-      addons: &gcc_multilib
-        apt:
-          packages:
-            - gcc-multilib
-    - name: "i686-unknown-linux-gnu (nightly)"
-      env: TARGET=i686-unknown-linux-gnu
-      addons: *gcc_multilib
-    - name: "i686-unknown-linux-gnu (beta)"
-      env: TARGET=i686-unknown-linux-gnu
-      addons: *gcc_multilib
-      rust: beta
-    - name: "i686-unknown-linux-gnu (stable)"
-      env: TARGET=i686-unknown-linux-gnu
-      addons: *gcc_multilib
-      rust: stable
-    # FIXME: blocked onhttps://github.com/jemalloc/jemalloc/issues/1464
-    # - name: "i686-unknown-linux-musl"
-    #   env: TARGET=i686-unknown-linux-musl NOBGT=1 NO_JEMALLOC_TESTS=1
-    - name: "mips-unknown-linux-gnu"
-      env: TARGET=mips-unknown-linux-gnu NO_JEMALLOC_TESTS=1
-    - name: "mips64-unknown-linux-gnuabi64"
-      env: TARGET=mips64-unknown-linux-gnuabi64 NO_JEMALLOC_TESTS=1
-    - name: "mips64el-unknown-linux-gnuabi64"
-      env: TARGET=mips64el-unknown-linux-gnuabi64 NO_JEMALLOC_TESTS=1
-    - name: "mipsel-unknown-linux-gnu"
-      env: TARGET=mipsel-unknown-linux-gnu NO_JEMALLOC_TESTS=1
-    - name: "powerpc-unknown-linux-gnu"
-      env: TARGET=powerpc-unknown-linux-gnu NO_JEMALLOC_TESTS=1
-    - name: "powerpc64-unknown-linux-gnu"
-      env: TARGET=powerpc64-unknown-linux-gnu NO_JEMALLOC_TESTS=1
+      arch: arm64
    - name: "powerpc64le-unknown-linux-gnu"
      env: TARGET=powerpc64le-unknown-linux-gnu NO_JEMALLOC_TESTS=1
+      arch: ppc64le
    - name: "x86_64-unknown-linux-gnu (nightly)"
-      env: TARGET=x86_64-unknown-linux-gnu VALGRIND=1 JEMALLOC_SYS_VERIFY_CONFIGURE=1
+      env: TARGET=x86_64-unknown-linux-gnu VALGRIND=1
      install: rustup component add llvm-tools-preview
      addons: &valgrind
        apt:
          packages:
            - valgrind
-            - autoconf
-    - name: "x86_64-unknown-linux-gnu (nightly - jemalloc's dev branch)"
-      env: TARGET=x86_64-unknown-linux-gnu VALGRIND=1 JEMALLOC_SYS_GIT_DEV_BRANCH=1
-      install: rustup component add llvm-tools-preview
-      addons: *valgrind
-    - name: "x86_64-unknown-linux-gnu (beta)"
-      env: TARGET=x86_64-unknown-linux-gnu VALGRIND=1
-      rust: beta
-      install: rustup component add llvm-tools-preview
-      addons: *valgrind
    - name: "x86_64-unknown-linux-gnu (stable)"
      env: TARGET=x86_64-unknown-linux-gnu VALGRIND=1
      rust: stable
@ -76,12 +32,11 @@ jobs:
        # - cargo test --features=alloc_trait --bench roundtrip
    - name: "x86_64-unknown-linux-musl"
      env: TARGET=x86_64-unknown-linux-musl NOBGT=1 NO_JEMALLOC_TESTS=1
-
-    # Android
-    - name: "aarch64-linux-android"
-      env: TARGET=aarch64-linux-android NO_JEMALLOC_TESTS=1
-    - name: "x86_64-linux-android"
-      env: TARGET=x86_64-linux-android
+      install: rustup target add x86_64-unknown-linux-musl
+      addons:
+        apt:
+          packages:
+            musl-tools

    # OSX
    # FIXME: cannot jemalloc tests fail due to:
@ -90,20 +45,10 @@ jobs:
    # FIXME: valgrind fails on OSX
    # https://github.com/gnzlbg/jemallocator/issues/86
    - name: "x86_64-apple-darwin (nightly)"
-      env: TARGET=x86_64-apple-darwin NO_JEMALLOC_TESTS=1 # JEMALLOC_SYS_VERIFY_CONFIGURE=1
-      os: osx
-      osx_image: xcode10
-      install: rustup component add llvm-tools-preview
-    - name: "x86_64-apple-darwin (nightly - jemalloc's dev branch)"
-      env: TARGET=x86_64-apple-darwin NO_JEMALLOC_TESTS=1 JEMALLOC_SYS_GIT_DEV_BRANCH=1
-      os: osx
-      osx_image: xcode10
-      install: rustup component add llvm-tools-preview
-    - name: "x86_64-apple-darwin (beta)"
      env: TARGET=x86_64-apple-darwin NO_JEMALLOC_TESTS=1
      os: osx
      osx_image: xcode10
-      rust: beta
+      install: rustup component add llvm-tools-preview
      install: rustup component add llvm-tools-preview
    - name: "x86_64-apple-darwin (stable)"
      env: TARGET=x86_64-apple-darwin NO_JEMALLOC_TESTS=1
--- a/Cross.toml
+++ b/Cross.toml
@ -1,16 +0,0 @@
-[build.env]
-passthrough = [
-    "RUST_BACKTRACE",
-    "RUST_TEST_THREADS",
-    "RUST_TEST_NOCAPTURE",
-    "NO_JEMALLOC_TESTS",
-    "TERM",
-    "JEMALLOC_SYS_RUN_JEMALLOC_TESTS",
-    "JEMALLOC_SYS_VERIFY_CONFIGURE",
-    "JEMALLOC_SYS_GIT_DEV_BRANCH",
-    "JEMALLOC_SYS_WITH_MALLOC_CONF",
-    "JEMALLOC_SYS_WITH_LG_PAGE",
-    "JEMALLOC_SYS_WITH_LG_HUGEPAGE",
-    "JEMALLOC_SYS_WITH_LG_QUANTUM",
-    "JEMALLOC_SYS_WITH_LG_VADDR"
-]
--- a/ci/run.sh
+++ b/ci/run.sh
@ -8,7 +8,6 @@ echo "Running tests for target: ${TARGET}, Rust version=${TRAVIS_RUST_VERSION}"
 export RUST_BACKTRACE=1
 export RUST_TEST_THREADS=1
 export RUST_TEST_NOCAPTURE=1
-export CARGO_CMD=cross

 # FIXME: workaround cargo breaking Travis-CI again:
 # https://github.com/rust-lang/cargo/issues/5721
@ -25,18 +24,6 @@ else
    export JEMALLOC_SYS_RUN_JEMALLOC_TESTS=1
 fi

-# Use cargo on native CI platforms:
-case "${TARGET}" in
-    "x86_64-unknown-linux-gnu") export CARGO_CMD=cargo ;;
-    *"windows"*) export CARGO_CMD=cargo ;;
-    *"apple"*) export CARGO_CMD=cargo ;;
-esac
-
-if [ "${CARGO_CMD}" = "cross" ]
-then
-    cargo install cross || echo "cross is already installed"
-fi
-
 if [ "${VALGRIND}" = "1" ]
 then
    case "${TARGET}" in
@ -56,7 +43,7 @@ fi
 if [ "${TARGET}" = "x86_64-unknown-linux-gnu" ] || [ "${TARGET}" = "x86_64-apple-darwin" ]
 then
    # Not using tee to avoid too much logs that exceeds travis' limit.
-    if ! ${CARGO_CMD} build -vv --target "${TARGET}" > build_no_std.txt 2>&1; then
+    if ! cargo build -vv --target "${TARGET}" > build_no_std.txt 2>&1; then
        tail -n 1024 build_no_std.txt
        exit 1
    fi
@ -75,42 +62,28 @@ then
    done
 fi

-${CARGO_CMD} test --target "${TARGET}"
+cargo test --target "${TARGET}"
+cargo test --target "${TARGET}" --features profiling
+cargo test --target "${TARGET}" --features debug
+cargo test --target "${TARGET}" --features stats
+cargo test --target "${TARGET}" --features 'debug profiling'

-if [ "${JEMALLOC_SYS_GIT_DEV_BRANCH}" = "1" ]; then
-    # FIXME: profiling tests broken on dev-branch
-    # https://github.com/jemalloc/jemalloc/issues/1477
-    :
-else
-    ${CARGO_CMD} test --target "${TARGET}" --features profiling
-fi
-
-${CARGO_CMD} test --target "${TARGET}" --features debug
-${CARGO_CMD} test --target "${TARGET}" --features stats
-if [ "${JEMALLOC_SYS_GIT_DEV_BRANCH}" = "1" ]; then
-    # FIXME: profiling tests broken on dev-branch
-    # https://github.com/jemalloc/jemalloc/issues/1477
-    :
-else
-    ${CARGO_CMD} test --target "${TARGET}" --features 'debug profiling'
-fi
-
-${CARGO_CMD} test --target "${TARGET}" \
-             --features unprefixed_malloc_on_supported_platforms
-${CARGO_CMD} test --target "${TARGET}" --no-default-features
-${CARGO_CMD} test --target "${TARGET}" --no-default-features \
-             --features background_threads_runtime_support
+cargo test --target "${TARGET}" \
+    --features unprefixed_malloc_on_supported_platforms
+cargo test --target "${TARGET}" --no-default-features
+cargo test --target "${TARGET}" --no-default-features \
+    --features background_threads_runtime_support

 if [ "${NOBGT}" = "1" ]
 then
    echo "enabling background threads by default at run-time is not tested"
 else
-    ${CARGO_CMD} test --target "${TARGET}" --features background_threads
+    cargo test --target "${TARGET}" --features background_threads
 fi

-${CARGO_CMD} test --target "${TARGET}" --release
-${CARGO_CMD} test --target "${TARGET}" --manifest-path jemalloc-sys/Cargo.toml
-${CARGO_CMD} test --target "${TARGET}" \
+cargo test --target "${TARGET}" --release
+cargo test --target "${TARGET}" --manifest-path jemalloc-sys/Cargo.toml
+cargo test --target "${TARGET}" \
             --manifest-path jemalloc-sys/Cargo.toml \
             --features unprefixed_malloc_on_supported_platforms

@ -120,9 +93,9 @@ case "${TARGET}" in
    "x86_64-unknown-linux-musl") ;;
    *)

-        ${CARGO_CMD} test --target "${TARGET}" \
-                     --manifest-path jemalloc-ctl/Cargo.toml \
-                     --no-default-features
+        cargo test --target "${TARGET}" \
+                   --manifest-path jemalloc-ctl/Cargo.toml \
+                   --no-default-features
        # FIXME: cross fails to pass features to jemalloc-ctl
        # ${CARGO_CMD} test --target "${TARGET}" \
        #             --manifest-path jemalloc-ctl \
@ -130,10 +103,9 @@ case "${TARGET}" in
        ;;
 esac

-${CARGO_CMD} test --target "${TARGET}" -p systest
-${CARGO_CMD} test --target "${TARGET}" \
-             --manifest-path jemallocator-global/Cargo.toml
-${CARGO_CMD} test --target "${TARGET}" \
+cargo test --target "${TARGET}" -p systest
+cargo test --target "${TARGET}" --manifest-path jemallocator-global/Cargo.toml
+cargo test --target "${TARGET}" \
             --manifest-path jemallocator-global/Cargo.toml \
             --features force_global_jemalloc

--- a/jemalloc-sys/README.md
+++ b/jemalloc-sys/README.md
@ -144,10 +144,6 @@ hyphens `-` are replaced with underscores `_`(see
  virtual address size on those platforms where it knows how, and picks a
  default otherwise. This option may be useful when cross-compiling.

-* `JEMALLOC_SYS_GIT_DEV_BRANCH`: when this environment variable is defined, the
-  latest commit from `jemalloc`'s dev branch is fetched from
-  `https://github.com/jemalloc/jemalloc` and built.
-
 [jemalloc_install]: https://github.com/jemalloc/jemalloc/blob/dev/INSTALL.md#advanced-configuration

 ## License
--- a/jemalloc-sys/build.rs
+++ b/jemalloc-sys/build.rs
@ -116,8 +116,6 @@ fn main() {
        println!("cargo:rustc-link-lib={}={}", kind, &stem[3..]);
        return;
    }
-
-    fs::create_dir_all(&build_dir).unwrap();
    // Disable -Wextra warnings - jemalloc doesn't compile free of warnings with
    // it enabled: https://github.com/jemalloc/jemalloc/issues/1196
    let compiler = cc::Build::new().extra_warnings(false).get_compiler();
@ -131,78 +129,31 @@ fn main() {
    info!("CFLAGS={:?}", cflags);

    assert!(out_dir.exists(), "OUT_DIR does not exist");
-    let (jemalloc_repo_dir, run_autoconf) = if env::var("JEMALLOC_SYS_GIT_DEV_BRANCH").is_ok() {
-        let jemalloc_repo = out_dir.join("jemalloc_repo");
-        if jemalloc_repo.exists() {
-            fs::remove_dir_all(jemalloc_repo.clone()).unwrap();
-        }
-        let mut cmd = Command::new("git");
-        cmd.arg("clone")
-            .arg("--depth=1")
-            .arg("--branch=dev")
-            .arg("--")
-            .arg("https://github.com/jemalloc/jemalloc")
-            .arg(format!("{}", jemalloc_repo.display()));
-        run(&mut cmd);
-        (jemalloc_repo, true)
-    } else {
-        (PathBuf::from("jemalloc"), false)
-    };
+    let jemalloc_repo_dir = PathBuf::from("jemalloc");
    info!("JEMALLOC_REPO_DIR={:?}", jemalloc_repo_dir);

-    let jemalloc_src_dir = out_dir.join("jemalloc");
-    info!("JEMALLOC_SRC_DIR={:?}", jemalloc_src_dir);
-
-    if jemalloc_src_dir.exists() {
-        fs::remove_dir_all(jemalloc_src_dir.clone()).unwrap();
+    if build_dir.exists() {
+        fs::remove_dir_all(build_dir.clone()).unwrap();
    }
-
    // Copy jemalloc submodule to the OUT_DIR
    let mut copy_options = fs_extra::dir::CopyOptions::new();
    copy_options.overwrite = true;
    copy_options.copy_inside = true;
-    fs_extra::dir::copy(&jemalloc_repo_dir, &jemalloc_src_dir, &copy_options)
+    fs_extra::dir::copy(&jemalloc_repo_dir, &build_dir, &copy_options)
        .expect("failed to copy jemalloc source code to OUT_DIR");
-    assert!(jemalloc_src_dir.exists());
+    assert!(build_dir.exists());

    // Configuration files
-    let config_files = ["configure" /*"VERSION"*/];
+    let config_files = ["configure", "VERSION"];

-    // Verify that the configuration files are up-to-date
-    let verify_configure = env::var("JEMALLOC_SYS_VERIFY_CONFIGURE").is_ok();
-    if verify_configure || run_autoconf {
-        info!("Verifying that configuration files in `configure/` are up-to-date... ");
-
-        // The configuration file from the configure/directory should be used.
-        // The jemalloc git submodule shouldn't contain any configuration files.
-        assert!(
-            !jemalloc_src_dir.join("configure").exists(),
-            "the jemalloc submodule contains configuration files"
-        );
-
-        // Run autoconf:
-        let mut cmd = Command::new("autoconf");
-        cmd.current_dir(jemalloc_src_dir.clone());
-        run(&mut cmd);
-
-        for f in &config_files {
-            if verify_configure {
-                let mut cmd = Command::new("diff");
-                run(cmd
-                    .arg(&jemalloc_src_dir.join(f))
-                    .arg(&Path::new("configure").join(f)));
-            }
-        }
-    } else {
-        // Copy the configuration files to jemalloc's source directory
-        for f in &config_files {
-            fs::copy(Path::new("configure").join(f), jemalloc_src_dir.join(f))
-                .expect("failed to copy config file to OUT_DIR");
-        }
+    // Copy the configuration files to jemalloc's source directory
+    for f in &config_files {
+        fs::copy(Path::new("configure").join(f), build_dir.join(f))
+            .expect("failed to copy config file to OUT_DIR");
    }

    // Run configure:
-    let configure = jemalloc_src_dir.join("configure");
+    let configure = build_dir.join("configure");
    let mut cmd = Command::new("sh");
    cmd.arg(
        configure
@ -311,13 +262,12 @@ fn main() {
    cmd.arg(format!("--build={}", gnu_target(&host)));
    cmd.arg(format!("--prefix={}", out_dir.display()));

-    run(&mut cmd);
+    run_and_log(&mut cmd, &build_dir.join("config.log"));

    // Make:
    let make = make_cmd(&host);
    run(Command::new(make)
        .current_dir(&build_dir)
-        .arg("srcroot=../jemalloc/")
        .arg("-j")
        .arg(num_jobs.clone()));

@ -326,22 +276,17 @@ fn main() {
        // Make tests:
        run(Command::new(make)
            .current_dir(&build_dir)
-            .arg("srcroot=../jemalloc/")
            .arg("-j")
            .arg(num_jobs.clone())
            .arg("tests"));

        // Run tests:
-        run(Command::new(make)
-            .current_dir(&build_dir)
-            .arg("srcroot=../jemalloc/")
-            .arg("check"));
+        run(Command::new(make).current_dir(&build_dir).arg("check"));
    }

    // Make install:
    run(Command::new(make)
        .current_dir(&build_dir)
-        .arg("srcroot=../jemalloc/")
        .arg("install_lib_static")
        .arg("install_include")
        .arg("-j")
@ -370,13 +315,24 @@ fn main() {
    println!("cargo:rerun-if-changed=jemalloc");
 }

+fn run_and_log(cmd: &mut Command, log_file: &Path) {
+    execute(cmd, || {
+        run(Command::new("tail").arg("-n").arg("100").arg(log_file));
+    })
+}
+
 fn run(cmd: &mut Command) {
+    execute(cmd, || ());
+}
+
+fn execute(cmd: &mut Command, on_fail: impl FnOnce()) {
    println!("running: {:?}", cmd);
    let status = match cmd.status() {
        Ok(status) => status,
        Err(e) => panic!("failed to execute command: {}", e),
    };
    if !status.success() {
+        on_fail();
        panic!(
            "command did not execute successfully: {:?}\n\
             expected success, got: {}",
--- a/jemalloc-sys/configure/VERSION
+++ b/jemalloc-sys/configure/VERSION
@ -1 +1 @@
-5.1.0-0-g61efbda7098de6fe64c362d309824864308c36d4
+5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756
--- a/jemalloc-sys/configure/configure
+++ b/jemalloc-sys/configure/configure
--- a/jemalloc-sys/jemalloc
+++ b/jemalloc-sys/jemalloc
@ -1 +1 @@
-Subproject commit 61efbda7098de6fe64c362d309824864308c36d4
+Subproject commit ea6b3e973b477b8061e0076bb257dbd7f3faa756