diff --git a/GNUmakefile b/GNUmakefile index 26b3b64df..bb26911f8 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -186,7 +186,7 @@ check: ## Lint the source code .PHONY: checkscripts checkscripts: ## Lint shell scripts @echo "==> Linting scripts..." - @shellcheck ./scripts/* + @find scripts -type f -name '*.sh' | xargs shellcheck .PHONY: generate-all generate-all: generate-structs proto diff --git a/Vagrantfile b/Vagrantfile index 7a182f8f6..7fab7e224 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -125,6 +125,14 @@ def configureLinuxProvisioners(vmCfg) privileged: true, path: './scripts/vagrant-linux-priv-config.sh' + vmCfg.vm.provision "shell", + privileged: true, + path: './scripts/vagrant-linux-priv-dev.sh' + + vmCfg.vm.provision "shell", + privileged: true, + path: './scripts/vagrant-linux-priv-docker.sh' + vmCfg.vm.provision "shell", privileged: true, path: './scripts/vagrant-linux-priv-consul.sh' diff --git a/scripts/release/Dockerfile b/scripts/release/Dockerfile new file mode 100644 index 000000000..2957dc00f --- /dev/null +++ b/scripts/release/Dockerfile @@ -0,0 +1,45 @@ +# Dockerfile for building nomad binaries +# that mimics Vagrant environment as far as required +# for building the scripts and running provision scripts + +FROM ubuntu:16.04 + +RUN apt-get update; apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + git \ + sudo \ + tree \ + unzip \ + wget + +RUN useradd --create-home vagrant \ + && echo 'vagrant ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers + +# install priv packages +COPY ./scripts/vagrant-linux-priv-config.sh /tmp/scripts/vagrant-linux-priv-config.sh +RUN /tmp/scripts/vagrant-linux-priv-config.sh + +COPY ./scripts/vagrant-linux-priv-go.sh /tmp/scripts/vagrant-linux-priv-go.sh +RUN /tmp/scripts/vagrant-linux-priv-go.sh + +COPY ./scripts/vagrant-linux-priv-protoc.sh /tmp/scripts/vagrant-linux-priv-protoc.sh +RUN /tmp/scripts/vagrant-linux-priv-protoc.sh + +USER vagrant + +COPY ./scripts/vagrant-linux-unpriv-ui.sh /tmp/scripts/vagrant-linux-unpriv-ui.sh +RUN /tmp/scripts/vagrant-linux-unpriv-ui.sh + +COPY ./scripts/release/docker-build-all /tmp/scripts/docker-build-all + +# Update PATH with GO bin, yarn, and node +ENV NODE_VERSION=v8.11.2 +ENV GOPATH="/opt/gopath" \ + PATH="/home/vagrant/.nvm/versions/node/${NODE_VERSION}/bin:/home/vagrant/bin:/opt/gopath/bin:/home/vagrant/.yarn/bin:/home/vagrant/.config/yarn/global/node_modules/.bin:$PATH" + +RUN mkdir -p /opt/gopath/src/github.com/hashicorp/nomad +RUN mkdir -p /home/vagrant/bin \ + && git config --global user.email "nomad@hashicorp.com" \ + && git config --global user.name "Nomad Release Bot" diff --git a/scripts/release/Makefile.linux b/scripts/release/Makefile.linux new file mode 100644 index 000000000..053232f59 --- /dev/null +++ b/scripts/release/Makefile.linux @@ -0,0 +1,34 @@ + +NOMAD_VERSION = 0.9.0-dev + +NOMAD_MAIN_VERSION := $(shell echo $(NOMAD_VERSION) | cut -d- -f1) +NOMAD_PRERELEASE_VERSION := $(shell echo $(NOMAD_VERSION) | cut -d- -f2-) + +update_version: + @echo "updating version to $(NOMAD_MAIN_VERSION)-$(NOMAD_PRERELEASE_VERSION)" + @sed -i.bak -e 's|\(Version * = *"\)[^"]*|\1$(NOMAD_MAIN_VERSION)|g' version/version.go + @sed -i.bak -e 's|\(VersionPrerelease * = *"\)[^"]*|\1$(NOMAD_PRERELEASE_VERSION)|g' version/version.go + @rm -rf version/version.go.bak + +PRERELEASE_TARGET = prerelease +RELEASE_TARGET = release + +build_releases: + @echo "======>> installing dependencies" + $(MAKE) bootstrap + + @echo "======>> pre-releasing" + $(MAKE) $(PRERELEASE_TARGET) + + @echo "======>> committing generated files" + git add -A . +# skip comitting files if there are no generated files +# if prerelease process was a no-op + if ! git diff-index --quiet HEAD --; \ + then \ + git commit --author 'Nomad Release bot ' \ + --message "Generate files for $(NOMAD_VERSION) release"; \ + fi + + @echo "======>> building release artifacts" + $(MAKE) $(RELEASE_TARGET) diff --git a/scripts/release/docker-build-all b/scripts/release/docker-build-all new file mode 100755 index 000000000..a219260d3 --- /dev/null +++ b/scripts/release/docker-build-all @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# A script for building Linux and Windows nomad release binaries inside Docker +# +# This is a helper script file that is expected to be invoked +# within a docker container with an image created with the Dockerfile present on this directory. +# +# A sample way of invoking the script is +# ``` +# docker run --rm \ +# -v $(pwd)/.git:/tmp/nomad-git:ro \ +# -v /tmp/generated-repo:/tmp/artifacts:rw \ +# -e "PRERELEASE_TARGET=${PRERELEASE_TARGET}" \ +# -e "RELEASE_TARGET=${RELEASE_TARGET}" \ +# -e "NOMAD_VERSION=${NOMAD_VERSION}" \ +# nomad-builder:latest \ +# /tmp/scripts/docker-build-all +# ``` +# Namely the script takes the following arguments: +# * `/tmp/nomad-git` path being a read-only .git directory with HEAD being the sha to be released +# * `NOMAD_VERSION` env-var being the release version to be cut (e.g. `0.9.1-rc1`) +# * `PRERELEASE_TARGET` env-var being the prerelease make target, typically `prerelease`. Use `help` to skip `prerelease` step +# * `RELEASE_TARGET` env-var being the release make target, typically `release`. +# +# +# The script would then run prerelease steps, commits any generated files, and build all binary files +# and stores them to `/tmp/artifacts/repo`. + +set -o errexit +set -o xtrace + +cp -r /tmp/nomad-git /opt/gopath/src/github.com/hashicorp/nomad/.git + +cd /opt/gopath/src/github.com/hashicorp/nomad + +# checkout directory from .git and ensures a prestine state +git checkout . + +make -f ./scripts/release/Makefile.linux \ + "NOMAD_VERSION=${NOMAD_VERSION}" \ + "PRERELEASE_TARGET=${PRERELEASE_TARGET}" \ + "RELEASE_TARGET=${RELEASE_TARGET}" \ + update_version build_releases + +cp -r /opt/gopath/src/github.com/hashicorp/nomad \ + /tmp/artifacts/repo diff --git a/scripts/release/mac-remote-build b/scripts/release/mac-remote-build new file mode 100755 index 000000000..0c0b2387c --- /dev/null +++ b/scripts/release/mac-remote-build @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# A script for building macOS binary on a remote macOS host +# +# The helper is expected to be invoked with nomad repo as a first argument, e.g. +# `mac-remote-build ~/go/src/github.com/hashicorp/nomad`. +# +# The repository is required to have a HEAD with all generated files and udpated version committed. +# +# The script runs a host on `sharedmac-bot` host (assumes a corresponding entry in ~/.ssh/config). +# `REMOTE_MACOS_HOST` envvar can be set to point to another macOS host +# +# The script operates by creating a temporary workspace in the remote host to +# contain a clean go installation and gopath with the repository content. +# It should install all dependencies worth pinning, and *not* use system binaries +# that may influence the integrity of the release. +# + +set -o errexit + +REPO="$1" +RELEASE_TARGET="${2:-release}" + +if [[ -z "${REPO}" ]] +then + echo "repo path is required" + echo "Usage: $0 " + exit 1 +fi + +TMP_WORKSPACE="/tmp/nomad-workspace/$(date +%Y-%m-%d-%s)" +REPO_REMOTE_PATH="${TMP_WORKSPACE}/gopath/src/github.com/hashicorp/nomad" + +readonly remote_macos_host=${REMOTE_MACOS_HOST:-sharedmac-bot} + +echo "Using temp workspace: ${TMP_WORKSPACE}" +echo + +echo '=======>>>> Transfering repository' +ssh ${remote_macos_host} mkdir -p "${REPO_REMOTE_PATH}" +rsync -az \ + "${REPO}/.git" \ + "${remote_macos_host}:${REPO_REMOTE_PATH}" + +echo '=======>>>> Compiling Mac Binaries' +cat <<'EOF' | ssh ${remote_macos_host} /bin/bash -s "${TMP_WORKSPACE}" + +set -o errexit +set -o xtrace + +TMP_WORKSPACE="$1" +REPO_PATH="${TMP_WORKSPACE}/gopath/src/github.com/hashicorp/nomad" + + +mkdir -p "${TMP_WORKSPACE}/tmp" + +install_go() { + local go_version="1.11.6" + local download= + + download="https://storage.googleapis.com/golang/go${go_version}.darwin-amd64.tar.gz" + curl -sSL --fail -o "${TMP_WORKSPACE}/tmp/go.tar.gz" ${download} + + tar -C "${TMP_WORKSPACE}" -xf "${TMP_WORKSPACE}/tmp/go.tar.gz" +} + +install_release_deps() { + go get -u github.com/a8m/tree/cmd/tree +} + +compile() { + cd "${REPO_PATH}" + git checkout . + git status + git log -1 + make release +} + +install_go + +export GOPATH="${TMP_WORKSPACE}/gopath" +export PATH="${TMP_WORKSPACE}/go/bin:${GOPATH}/bin:$PATH" + +install_release_deps +compile + +EOF + +echo '=======>>>> Retreiving mac compiled binaries' +rsync -avz --ignore-existing ${remote_macos_host}:"${REPO_REMOTE_PATH}/pkg/" "${REPO}/pkg" + +ssh ${remote_macos_host} rm -rf "${TMP_WORKSPACE}" diff --git a/scripts/vagrant-linux-priv-config.sh b/scripts/vagrant-linux-priv-config.sh index 7c8b38c52..b2400c5c0 100755 --- a/scripts/vagrant-linux-priv-config.sh +++ b/scripts/vagrant-linux-priv-config.sh @@ -9,15 +9,6 @@ apt-get install -y software-properties-common # Add i386 architecture (for libraries) dpkg --add-architecture i386 -# Add the Docker repository -apt-key adv \ - --keyserver hkp://p80.pool.sks-keyservers.net:80 \ - --recv-keys 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 -add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) \ - stable" - # Update with i386, Go and Docker apt-get update @@ -35,7 +26,6 @@ apt-get install -y \ apt-get install -y \ curl \ default-jre \ - docker-ce \ htop \ jq \ qemu \ @@ -60,12 +50,6 @@ apt-get install -y \ # Ensure everything is up to date apt-get upgrade -y -# Restart Docker in case it got upgraded -systemctl restart docker.service - -# Ensure Docker can be used by vagrant user -usermod -aG docker vagrant - # Set hostname -> IP to make advertisement work as expected ip=$(ip route get 1 | awk '{print $NF; exit}') hostname=$(hostname) diff --git a/scripts/vagrant-linux-priv-dev.sh b/scripts/vagrant-linux-priv-dev.sh new file mode 100755 index 000000000..2b22a36a6 --- /dev/null +++ b/scripts/vagrant-linux-priv-dev.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Install Development utilities +apt-get install -y \ + curl \ + default-jre \ + htop \ + jq \ + qemu \ + silversearcher-ag \ + tree \ + unzip \ + vim + + +# Set hostname -> IP to make advertisement work as expected +ip=$(ip route get 1 | awk '{print $NF; exit}') +hostname=$(hostname) +sed -i -e "s/.*nomad.*/${ip} ${hostname}/" /etc/hosts + +# Ensure we cd into the working directory on login +if ! grep "cd /opt/gopath/src/github.com/hashicorp/nomad" /home/vagrant/.profile ; then + echo 'cd /opt/gopath/src/github.com/hashicorp/nomad' >> /home/vagrant/.profile +fi diff --git a/scripts/vagrant-linux-priv-docker.sh b/scripts/vagrant-linux-priv-docker.sh new file mode 100755 index 000000000..22a9c5717 --- /dev/null +++ b/scripts/vagrant-linux-priv-docker.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Add the Docker repository +apt-key adv \ + --keyserver hkp://p80.pool.sks-keyservers.net:80 \ + --recv-keys 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 +add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + +# Update with i386, Go and Docker +apt-get update + +apt-get install -y docker-ce + +# Restart Docker in case it got upgraded +systemctl restart docker.service + +# Ensure Docker can be used by vagrant user +usermod -aG docker vagrant