From 70bdf5e93c2901a6d767060aadbebdb1812174dd Mon Sep 17 00:00:00 2001
From: Martin Larralde <martin.larralde@ens-cachan.fr>
Date: Fri, 4 May 2018 18:20:28 +0200
Subject: [PATCH] Tidy up Travis-CI (#152)

* Use Travis-CI build matrix to setup environment
* Setup Travis cache
* Use `sccache` to cache compiled artifacts
* Use `setup.sh` script to setup Travis-CI
* Add more tests to `ci/travis` to lighten `.travis.yml`
* Use `script` deployment to run `kcov` in Travis-CI
* Export `RUSTC_WRAPPER` in the Travis configuration directly
* Fix wrong Python `3.7` version in Travis configuration
* Fix `ci/travis/cover.sh`
* Prevent Travis-CI from cleaning build artifacts
* Recompile `kcov` only when needed in Travis-CI
* Collect coverage in parallel
* Add github-pages deployment script
---
 .travis.yml        | 103 ++++++++++++++++++---------------------------
 ci/travis/cover.sh |  21 +++++++++
 ci/travis/guide.sh |  29 +++++++++++++
 ci/travis/setup.sh |  71 +++++++++++++++++++++++++++++++
 ci/travis/test.sh  |  11 +++++
 5 files changed, 173 insertions(+), 62 deletions(-)
 create mode 100755 ci/travis/cover.sh
 create mode 100755 ci/travis/guide.sh
 create mode 100755 ci/travis/setup.sh
 create mode 100755 ci/travis/test.sh

diff --git a/.travis.yml b/.travis.yml
index 97c703c7..04220ec0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,22 +1,34 @@
+sudo: required
+dist: trusty
 language: python
 
-python:
-  - "2.7"
-  - "3.5"
-  - "3.6"
-  - "3.7-dev"
+cache:
+  pip: true
+  directories:
+  - "$HOME/.cargo"
+  - "$HOME/kcov"
 
-allow_failures:
+matrix:
+  include:
+  - python: "2.7"
+    env: FEATURES=python2
+  - python: "3.5"
+    env: FEATURES=python3
+  - python: "3.6"
+    env: FEATURES=python3
   - python: "3.7-dev"
+    env: FEATURES=python3
+  allow_failures:
+  - python: "3.7-dev"
+    env: FEATURES=python3
 
 env:
   global:
-    - RUST_VERSION=nightly
+    - TRAVIS_RUST_VERSION=nightly
     - RUSTFLAGS="-C link-dead-code"
     - RUST_BACKTRACE=1
-
-sudo: required
-dist: trusty
+    - SCCACHE_DIR="$HOME/.cargo/sccache"
+    - RUSTC_WRAPPER=sccache
 
 addons:
   apt:
@@ -29,59 +41,26 @@ addons:
       - binutils-dev
       - libiberty-dev
 
-# Add clippy
-before_script:
-  - |
-    if [[ "$TRAVIS_RUST_VERSION" == "nightly" ]]; then
-        ( ( cargo install clippy && export CLIPPY=true ) || export CLIPPY=false );
-    fi
-  - export PATH=$PATH:~/.cargo/bin
+before_install:
+  - source ./ci/travis/setup.sh
 
 install:
-  - python -c "import sysconfig; print('\n'.join(map(repr,sorted(sysconfig.get_config_vars().items()))))"
-  - mkdir ~/rust-installer
-  - curl -sL https://static.rust-lang.org/rustup.sh -o ~/rust-installer/rustup.sh
-  - sh ~/rust-installer/rustup.sh --prefix=~/rust --spec=$RUST_VERSION -y
-  - export PATH="$HOME/rust/bin:$PATH"
-  - python -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))"
-  - export PYTHON_LIB=$(python -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))")
-  # - find $PYTHON_LIB
-  - export LIBRARY_PATH="$LIBRARY_PATH:$PYTHON_LIB"
-  # delete any possible empty components
-  # https://github.com/google/pulldown-cmark/issues/122#issuecomment-364948741
-  - LIBRARY_PATH=$(echo $LIBRARY_PATH | sed -E -e 's/^:*//' -e 's/:*$//' -e 's/:+/:/g')
-  - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PYTHON_LIB:$HOME/rust/lib"
-  - rustc -V
+  - pip install setuptools-rust pytest pytest-benchmark
+
 script:
-  - make test
-  # - make clippy
+  - source ./ci/travis/test.sh
 
-after_success:
-  - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PULL_REQUEST" = "false" && "$TRAVIS_BRANCH" == "master" && "$TRAVIS_PYTHON_VERSION" == "3.6" ]]; then
-    cargo doc --no-deps &&
-    echo "<meta http-equiv=refresh content=0;url=os_balloon/index.html>" > target/doc/index.html &&
-    curl -sL https://github.com/rust-lang-nursery/mdBook/releases/download/v0.1.5/mdbook-v0.1.5-x86_64-unknown-linux-gnu.tar.gz | tar xvz -C $HOME/rust/bin &&
-    cd guide && mdbook build -d ../target/doc/guide && cd .. &&
-    git clone https://github.com/davisp/ghp-import.git &&
-    ./ghp-import/ghp_import.py -n -p -f -m "Documentation upload" -r https://"$GH_TOKEN"@github.com/"$TRAVIS_REPO_SLUG.git" target/doc &&
-    echo "Uploaded documentation"
-    fi
-
-  - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_BRANCH" == "master" && "$TRAVIS_PYTHON_VERSION" == "3.6" ]]; then
-      wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz &&
-      tar xzf master.tar.gz &&
-      cd kcov-master &&
-      mkdir build &&
-      cd build &&
-      cmake .. &&
-      make &&
-      make install DESTDIR=../../kcov-build &&
-      cd ../.. &&
-      rm -rf kcov-master &&
-      for file in target/debug/pyo3-*[^\.d]; do mkdir -p "target/cov/$(basename $file)"; ./kcov-build/usr/local/bin/kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$(basename $file)" "$file"; done &&
-      for file in target/debug/test_*[^\.d]; do mkdir -p "target/cov/$(basename $file)"; ./kcov-build/usr/local/bin/kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$(basename $file)" "$file"; done &&
-      bash <(curl -s https://codecov.io/bash) &&
-      echo "Uploaded code coverage"
-    fi
+deploy:
+  - provider: script
+    skip_cleanup: true
+    script: ./ci/travis/cover.sh
+    on:
+      branch: master
+  - provider: script
+    skip_cleanup: true
+    script: ./ci/travis/guide.sh
+    on:
+      branch: master
+      repo: althonos/pyo3
+      python: 3.6
+    
diff --git a/ci/travis/cover.sh b/ci/travis/cover.sh
new file mode 100755
index 00000000..94f9b84a
--- /dev/null
+++ b/ci/travis/cover.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+### Run kcov in parallel #######################################################
+
+rm -f target/debug/pyo3-*.d
+rm -f target/debug/test_*.d
+rm -f target/debug/test_doc-*
+
+# echo $FILES
+FILES=$(find . -path ./target/debug/pyo3-\* -or -path ./target/debug/test_\*)
+echo $FILES | xargs -n1 -P $(nproc) sh -c '
+  dir="target/cov/$(basename $@)"
+  mkdir -p $dir
+  echo "Collecting coverage data of $(basename $@)"
+  kcov --exclude-pattern=/.cargo,/usr/lib --verify $dir "$@" 2>&1 >/dev/null
+' _
+
+### Upload coverage ############################################################
+
+echo "Uploading code coverage"
+curl -SsL https://codecov.io/bash | bash
diff --git a/ci/travis/guide.sh b/ci/travis/guide.sh
new file mode 100755
index 00000000..f15e103f
--- /dev/null
+++ b/ci/travis/guide.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -e
+
+### Setup latest mdbook version ################################################
+
+INSTALLED=$(echo $(mdbook --version 2>/dev/null || echo "mdbook none") | cut -d' ' -f1)
+LATEST=0.1.5
+
+if [ "$LATEST" != "$INSTALLED" ]; then
+    URL=https://github.com/rust-lang-nursery/mdBook/releases/download/v${LATEST}/mdbook-v${LATEST}-x86_64-unknown-linux-gnu.tar.gz
+    curl -SsL $URL | tar xvz -C $HOME/.cargo/bin
+fi
+
+### Build API reference ########################################################
+
+cargo doc --no-deps
+echo "<meta http-equiv=refresh content=0;url=os_balloon/index.html>" > target/doc/index.html
+
+
+### Build guide ################################################################
+
+cd guide
+mdbook build -d ../target/doc/guide
+cd ..
+
+git clone https://github.com/davisp/ghp-import.git
+./ghp-import/ghp_import.py -n -p -f -m "Documentation upload" -r https://"$GH_TOKEN"@github.com/"$TRAVIS_REPO_SLUG.git" target/doc
+echo "Uploaded documentation"
diff --git a/ci/travis/setup.sh b/ci/travis/setup.sh
new file mode 100755
index 00000000..f92537e8
--- /dev/null
+++ b/ci/travis/setup.sh
@@ -0,0 +1,71 @@
+#!/bin/sh
+
+# Find the installed version of a binary, if any
+_installed() {
+    VERSION=$($@ --version 2>/dev/null || echo "$@ none")
+    echo $VERSION | rev | cut -d' ' -f1 | rev
+}
+
+# Find the latest available version of a binary on `crates.io`
+_latest() {
+    VERSION=$(cargo search -q "$@" | grep "$@" | cut -f2 -d"\"")
+    echo $VERSION
+}
+
+### Setup Rust toolchain #######################################################
+
+curl -SsL "https://sh.rustup.rs/" | sh -s -- -y --default-toolchain=$TRAVIS_RUST_VERSION
+export PATH=$PATH:$HOME/.cargo/bin
+
+
+### Setup sccache ##############################################################
+
+echo -n "Fetching latest available 'sccache' version... "
+INSTALLED=$(_installed sccache)
+LATEST=$(_latest sccache)
+echo "${LATEST} (installed: ${INSTALLED})"
+
+if [ "$INSTALLED" = "$LATEST" ]; then
+  echo "Using cached 'sccache'"
+else
+  echo "Installing latest 'sccache' from mozilla/sccache"
+  URL="https://github.com/mozilla/sccache/releases/download/${LATEST}/sccache-${LATEST}-x86_64-unknown-linux-musl.tar.gz"
+  curl -SsL $URL | tar xzv -C /tmp
+  mv /tmp/sccache-${LATEST}-x86_64-unknown-linux-musl/sccache $HOME/.cargo/bin/sccache
+fi
+
+mkdir -p $SCCACHE_DIR
+
+
+### Setup kcov #################################################################
+
+if [ ! -d "$HOME/kcov/.git" ]; then
+    git clone --depth=1 https://github.com/SimonKagstrom/kcov "$HOME/kcov"
+fi
+
+cd $HOME/kcov
+git pull
+cmake .
+make
+install src/kcov $HOME/.cargo/bin/kcov
+cd $TRAVIS_BUILD_DIR
+
+
+### Setup python linker flags ##################################################
+
+python -c """
+import sysconfig
+cfg = sorted(sysconfig.get_config_vars().items())
+print('\n'.join(['{}={}'.format(*x) for x in cfg]))
+"""
+
+export PYTHON_LIB=$(python -c "import sysconfig as s; print(s.get_config_var('LIBDIR'))")
+
+# find $PYTHON_LIB
+export LIBRARY_PATH="$LIBRARY_PATH:$PYTHON_LIB"
+
+# delete any possible empty components
+# https://github.com/google/pulldown-cmark/issues/122#issuecomment-364948741
+LIBRARY_PATH=$(echo $LIBRARY_PATH | sed -E -e 's/^:*//' -e 's/:*$//' -e 's/:+/:/g')
+
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PYTHON_LIB:$HOME/rust/lib"
diff --git a/ci/travis/test.sh b/ci/travis/test.sh
new file mode 100755
index 00000000..f91b1e4a
--- /dev/null
+++ b/ci/travis/test.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+cargo build --features $FEATURES
+cargo test --features $FEATURES
+
+for example in examples/*; do
+  cd $example
+  python setup.py install
+  pytest -v tests
+  cd $TRAVIS_BUILD_DIR
+done