Allow jemalloc-sys to pick jemalloc from git

This PR allows automatically fetching the latest jemalloc from
its dev branch by defining the environment variable
`JEMALLOC_SYS_GIT_DEV_BRANCH`. This is useful when debugging
whether certain bugs are already fixed upstream, as well as
for detecting whether upstream changes break our tests.
This PR adds two build jobs that test the dev branch on the
tier-1 x86_64 linux and osx targets.

.travis.yml

@@ -53,6 +53,14 @@ matrix:
           packages:
             - valgrind
             - autoconf
+    - name: "x86_64-unknown-linux-gnu (nightly - jemalloc's dev branch)"
+      env: TARGET=x86_64-unknown-linux-gnu VALGRIND=1 JEMALLOC_SYS_VERIFY_CONFIGURE=1 JEMALLOC_SYS_GIT_DEV_BRANCH=1
+      install: rustup component add llvm-tools-preview
+      addons: &valgrind
+        apt:
+          packages:
+            - valgrind
+            - autoconf
     - name: "x86_64-unknown-linux-gnu (beta)"
       env: TARGET=x86_64-unknown-linux-gnu VALGRIND=1
       rust: beta
@@ -103,6 +111,11 @@ matrix:
       os: osx
       osx_image: xcode10
       install: rustup component add llvm-tools-preview
+    - name: "x86_64-apple-darwin (nightly - jemalloc's dev branch)"
+      env: TARGET=x86_64-apple-darwin NO_JEMALLOC_TESTS=1 JEMALLOC_SYS_VERIFY_CONFIGURE=1 JEMALLOC_SYS_GIT_DEV_BRANCH=1
+      os: osx
+      osx_image: xcode10
+      install: rustup component add llvm-tools-preview
     - name: "x86_64-apple-darwin (beta)"
       env: TARGET=x86_64-apple-darwin NO_JEMALLOC_TESTS=1
       os: osx

Cross.toml

@@ -7,6 +7,7 @@ passthrough = [
     "TERM",
     "JEMALLOC_SYS_RUN_JEMALLOC_TESTS",
     "JEMALLOC_SYS_VERIFY_CONFIGURE",
+    "JEMALLOC_SYS_GIT_DEV_BRANCH",
     "JEMALLOC_SYS_WITH_MALLOC_CONF",
     "JEMALLOC_SYS_WITH_LG_PAGE",
     "JEMALLOC_SYS_WITH_LG_HUGEPAGE",

jemalloc-sys/README.md

@@ -143,6 +143,10 @@ hyphens `-` are replaced with underscores `_`(see
   virtual address size on those platforms where it knows how, and picks a
   default otherwise. This option may be useful when cross-compiling.
 
+* `JEMALLOC_SYS_GIT_DEV_BRANCH`: when this environment variable is defined, the
+  latest commit from `jemalloc`'s dev branch is fetched from
+  `https://github.com/jemalloc/jemalloc` and built.
+
 [jemalloc_install]: https://github.com/jemalloc/jemalloc/blob/dev/INSTALL.md#advanced-configuration
 
 ## License

jemalloc-sys/build.rs

@@ -113,6 +113,26 @@ fn main() {
     info!("CC={:?}", compiler.path());
     info!("CFLAGS={:?}", cflags);
 
+    assert!(out_dir.exists(), "OUT_DIR does not exist");
+    let (jemalloc_repo_dir, run_autoconf) = if env::var("JEMALLOC_SYS_GIT_DEV_BRANCH").is_ok() {
+        let jemalloc_repo = out_dir.join("jemalloc_repo");
+        if jemalloc_repo.exists() {
+            fs::remove_dir_all(jemalloc_repo.clone()).unwrap();
+        }
+        let mut cmd = Command::new("git");
+        cmd.arg("clone")
+            .arg("--depth=1")
+            .arg("--branch=dev")
+            .arg("--")
+            .arg("https://github.com/jemalloc/jemalloc")
+            .arg(format!("{}", jemalloc_repo.display()));
+        run(&mut cmd);
+        (jemalloc_repo, true)
+    } else {
+        (PathBuf::from("jemalloc"), false)
+    };
+    info!("JEMALLOC_REPO_DIR={:?}", jemalloc_repo_dir);
+
     let jemalloc_src_dir = out_dir.join("jemalloc");
     info!("JEMALLOC_SRC_DIR={:?}", jemalloc_src_dir);
 
@@ -121,11 +141,10 @@ fn main() {
     }
 
     // Copy jemalloc submodule to the OUT_DIR
-    assert!(out_dir.exists(), "OUT_DIR does not exist");
     let mut copy_options = fs_extra::dir::CopyOptions::new();
     copy_options.overwrite = true;
     copy_options.copy_inside = true;
-    fs_extra::dir::copy(Path::new("jemalloc"), &jemalloc_src_dir, &copy_options)
+    fs_extra::dir::copy(&jemalloc_repo_dir, &jemalloc_src_dir, &copy_options)
         .expect("failed to copy jemalloc source code to OUT_DIR");
     assert!(jemalloc_src_dir.exists());
 
@@ -133,7 +152,8 @@ fn main() {
     let config_files = ["configure" /*"VERSION"*/];
 
     // Verify that the configuration files are up-to-date
-    if env::var("JEMALLOC_SYS_VERIFY_CONFIGURE").is_ok() {
+    let verify_configure = env::var("JEMALLOC_SYS_VERIFY_CONFIGURE").is_ok();
+    if verify_configure || run_autoconf {
         info!("Verifying that configuration files in `configure/` are up-to-date... ");
 
         // The configuration file from the configure/directory should be used.
@@ -163,6 +183,7 @@ fn main() {
                 content
             }
 
+            if verify_configure {
                 let current = read_content(&jemalloc_src_dir.join(f));
                 let reference = read_content(&Path::new("configure").join(f));
                 assert_eq!(
@@ -171,6 +192,7 @@ fn main() {
                     f
                 );
             }
+        }
     } else {
         // Copy the configuration files to jemalloc's source directory
         for f in &config_files {