Merge pull request #5435 from hashicorp/release-scripts-20190318
Scripts for automating binary building
This commit is contained in:
Mahmood Ali
GitHub
commit
d4ed7aecbf
|
|
@ -186,7 +186,7 @@ check: ## Lint the source code
|
|||
.PHONY: checkscripts
|
||||
checkscripts: ## Lint shell scripts
|
||||
@echo "==> Linting scripts..."
|
||||
@shellcheck ./scripts/*
|
||||
@find scripts -type f -name '*.sh' | xargs shellcheck
|
||||
|
||||
.PHONY: generate-all
|
||||
generate-all: generate-structs proto
|
||||
|
|
|
|||
|
|
@ -125,6 +125,14 @@ def configureLinuxProvisioners(vmCfg)
|
|||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-config.sh'
|
||||
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-dev.sh'
|
||||
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-docker.sh'
|
||||
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-consul.sh'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
# Dockerfile for building nomad binaries
|
||||
# that mimics Vagrant environment as far as required
|
||||
# for building the scripts and running provision scripts
|
||||
|
||||
FROM ubuntu:16.04
|
||||
|
||||
RUN apt-get update; apt-get install -y \
|
||||
apt-transport-https \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
sudo \
|
||||
tree \
|
||||
unzip \
|
||||
wget
|
||||
|
||||
RUN useradd --create-home vagrant \
|
||||
&& echo 'vagrant ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers
|
||||
|
||||
# install priv packages
|
||||
COPY ./scripts/vagrant-linux-priv-config.sh /tmp/scripts/vagrant-linux-priv-config.sh
|
||||
RUN /tmp/scripts/vagrant-linux-priv-config.sh
|
||||
|
||||
COPY ./scripts/vagrant-linux-priv-go.sh /tmp/scripts/vagrant-linux-priv-go.sh
|
||||
RUN /tmp/scripts/vagrant-linux-priv-go.sh
|
||||
|
||||
COPY ./scripts/vagrant-linux-priv-protoc.sh /tmp/scripts/vagrant-linux-priv-protoc.sh
|
||||
RUN /tmp/scripts/vagrant-linux-priv-protoc.sh
|
||||
|
||||
USER vagrant
|
||||
|
||||
COPY ./scripts/vagrant-linux-unpriv-ui.sh /tmp/scripts/vagrant-linux-unpriv-ui.sh
|
||||
RUN /tmp/scripts/vagrant-linux-unpriv-ui.sh
|
||||
|
||||
COPY ./scripts/release/docker-build-all /tmp/scripts/docker-build-all
|
||||
|
||||
# Update PATH with GO bin, yarn, and node
|
||||
ENV NODE_VERSION=v8.11.2
|
||||
ENV GOPATH="/opt/gopath" \
|
||||
PATH="/home/vagrant/.nvm/versions/node/${NODE_VERSION}/bin:/home/vagrant/bin:/opt/gopath/bin:/home/vagrant/.yarn/bin:/home/vagrant/.config/yarn/global/node_modules/.bin:$PATH"
|
||||
|
||||
RUN mkdir -p /opt/gopath/src/github.com/hashicorp/nomad
|
||||
RUN mkdir -p /home/vagrant/bin \
|
||||
&& git config --global user.email "nomad@hashicorp.com" \
|
||||
&& git config --global user.name "Nomad Release Bot"
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
|
||||
NOMAD_VERSION = 0.9.0-dev
|
||||
|
||||
NOMAD_MAIN_VERSION := $(shell echo $(NOMAD_VERSION) | cut -d- -f1)
|
||||
NOMAD_PRERELEASE_VERSION := $(shell echo $(NOMAD_VERSION) | cut -d- -f2-)
|
||||
|
||||
update_version:
|
||||
@echo "updating version to $(NOMAD_MAIN_VERSION)-$(NOMAD_PRERELEASE_VERSION)"
|
||||
@sed -i.bak -e 's|\(Version * = *"\)[^"]*|\1$(NOMAD_MAIN_VERSION)|g' version/version.go
|
||||
@sed -i.bak -e 's|\(VersionPrerelease * = *"\)[^"]*|\1$(NOMAD_PRERELEASE_VERSION)|g' version/version.go
|
||||
@rm -rf version/version.go.bak
|
||||
|
||||
PRERELEASE_TARGET = prerelease
|
||||
RELEASE_TARGET = release
|
||||
|
||||
build_releases:
|
||||
@echo "======>> installing dependencies"
|
||||
$(MAKE) bootstrap
|
||||
|
||||
@echo "======>> pre-releasing"
|
||||
$(MAKE) $(PRERELEASE_TARGET)
|
||||
|
||||
@echo "======>> committing generated files"
|
||||
git add -A .
|
||||
# skip comitting files if there are no generated files
|
||||
# if prerelease process was a no-op
|
||||
if ! git diff-index --quiet HEAD --; \
|
||||
then \
|
||||
git commit --author 'Nomad Release bot <nomad@hashicorp.com>' \
|
||||
--message "Generate files for $(NOMAD_VERSION) release"; \
|
||||
fi
|
||||
|
||||
@echo "======>> building release artifacts"
|
||||
$(MAKE) $(RELEASE_TARGET)
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# A script for building Linux and Windows nomad release binaries inside Docker
|
||||
#
|
||||
# This is a helper script file that is expected to be invoked
|
||||
# within a docker container with an image created with the Dockerfile present on this directory.
|
||||
#
|
||||
# A sample way of invoking the script is
|
||||
# ```
|
||||
# docker run --rm \
|
||||
# -v $(pwd)/.git:/tmp/nomad-git:ro \
|
||||
# -v /tmp/generated-repo:/tmp/artifacts:rw \
|
||||
# -e "PRERELEASE_TARGET=${PRERELEASE_TARGET}" \
|
||||
# -e "RELEASE_TARGET=${RELEASE_TARGET}" \
|
||||
# -e "NOMAD_VERSION=${NOMAD_VERSION}" \
|
||||
# nomad-builder:latest \
|
||||
# /tmp/scripts/docker-build-all
|
||||
# ```
|
||||
# Namely the script takes the following arguments:
|
||||
# * `/tmp/nomad-git` path being a read-only .git directory with HEAD being the sha to be released
|
||||
# * `NOMAD_VERSION` env-var being the release version to be cut (e.g. `0.9.1-rc1`)
|
||||
# * `PRERELEASE_TARGET` env-var being the prerelease make target, typically `prerelease`. Use `help` to skip `prerelease` step
|
||||
# * `RELEASE_TARGET` env-var being the release make target, typically `release`.
|
||||
#
|
||||
#
|
||||
# The script would then run prerelease steps, commits any generated files, and build all binary files
|
||||
# and stores them to `/tmp/artifacts/repo`.
|
||||
|
||||
set -o errexit
|
||||
set -o xtrace
|
||||
|
||||
cp -r /tmp/nomad-git /opt/gopath/src/github.com/hashicorp/nomad/.git
|
||||
|
||||
cd /opt/gopath/src/github.com/hashicorp/nomad
|
||||
|
||||
# checkout directory from .git and ensures a prestine state
|
||||
git checkout .
|
||||
|
||||
make -f ./scripts/release/Makefile.linux \
|
||||
"NOMAD_VERSION=${NOMAD_VERSION}" \
|
||||
"PRERELEASE_TARGET=${PRERELEASE_TARGET}" \
|
||||
"RELEASE_TARGET=${RELEASE_TARGET}" \
|
||||
update_version build_releases
|
||||
|
||||
cp -r /opt/gopath/src/github.com/hashicorp/nomad \
|
||||
/tmp/artifacts/repo
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# A script for building macOS binary on a remote macOS host
|
||||
#
|
||||
# The helper is expected to be invoked with nomad repo as a first argument, e.g.
|
||||
# `mac-remote-build ~/go/src/github.com/hashicorp/nomad`.
|
||||
#
|
||||
# The repository is required to have a HEAD with all generated files and udpated version committed.
|
||||
#
|
||||
# The script runs a host on `sharedmac-bot` host (assumes a corresponding entry in ~/.ssh/config).
|
||||
# `REMOTE_MACOS_HOST` envvar can be set to point to another macOS host
|
||||
#
|
||||
# The script operates by creating a temporary workspace in the remote host to
|
||||
# contain a clean go installation and gopath with the repository content.
|
||||
# It should install all dependencies worth pinning, and *not* use system binaries
|
||||
# that may influence the integrity of the release.
|
||||
#
|
||||
|
||||
set -o errexit
|
||||
|
||||
REPO="$1"
|
||||
RELEASE_TARGET="${2:-release}"
|
||||
|
||||
if [[ -z "${REPO}" ]]
|
||||
then
|
||||
echo "repo path is required"
|
||||
echo "Usage: $0 <repo_path>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TMP_WORKSPACE="/tmp/nomad-workspace/$(date +%Y-%m-%d-%s)"
|
||||
REPO_REMOTE_PATH="${TMP_WORKSPACE}/gopath/src/github.com/hashicorp/nomad"
|
||||
|
||||
readonly remote_macos_host=${REMOTE_MACOS_HOST:-sharedmac-bot}
|
||||
|
||||
echo "Using temp workspace: ${TMP_WORKSPACE}"
|
||||
echo
|
||||
|
||||
echo '=======>>>> Transfering repository'
|
||||
ssh ${remote_macos_host} mkdir -p "${REPO_REMOTE_PATH}"
|
||||
rsync -az \
|
||||
"${REPO}/.git" \
|
||||
"${remote_macos_host}:${REPO_REMOTE_PATH}"
|
||||
|
||||
echo '=======>>>> Compiling Mac Binaries'
|
||||
cat <<'EOF' | ssh ${remote_macos_host} /bin/bash -s "${TMP_WORKSPACE}"
|
||||
|
||||
set -o errexit
|
||||
set -o xtrace
|
||||
|
||||
TMP_WORKSPACE="$1"
|
||||
REPO_PATH="${TMP_WORKSPACE}/gopath/src/github.com/hashicorp/nomad"
|
||||
|
||||
|
||||
mkdir -p "${TMP_WORKSPACE}/tmp"
|
||||
|
||||
install_go() {
|
||||
local go_version="1.11.6"
|
||||
local download=
|
||||
|
||||
download="https://storage.googleapis.com/golang/go${go_version}.darwin-amd64.tar.gz"
|
||||
curl -sSL --fail -o "${TMP_WORKSPACE}/tmp/go.tar.gz" ${download}
|
||||
|
||||
tar -C "${TMP_WORKSPACE}" -xf "${TMP_WORKSPACE}/tmp/go.tar.gz"
|
||||
}
|
||||
|
||||
install_release_deps() {
|
||||
go get -u github.com/a8m/tree/cmd/tree
|
||||
}
|
||||
|
||||
compile() {
|
||||
cd "${REPO_PATH}"
|
||||
git checkout .
|
||||
git status
|
||||
git log -1
|
||||
make release
|
||||
}
|
||||
|
||||
install_go
|
||||
|
||||
export GOPATH="${TMP_WORKSPACE}/gopath"
|
||||
export PATH="${TMP_WORKSPACE}/go/bin:${GOPATH}/bin:$PATH"
|
||||
|
||||
install_release_deps
|
||||
compile
|
||||
|
||||
EOF
|
||||
|
||||
echo '=======>>>> Retreiving mac compiled binaries'
|
||||
rsync -avz --ignore-existing ${remote_macos_host}:"${REPO_REMOTE_PATH}/pkg/" "${REPO}/pkg"
|
||||
|
||||
ssh ${remote_macos_host} rm -rf "${TMP_WORKSPACE}"
|
||||
|
|
@ -9,15 +9,6 @@ apt-get install -y software-properties-common
|
|||
# Add i386 architecture (for libraries)
|
||||
dpkg --add-architecture i386
|
||||
|
||||
# Add the Docker repository
|
||||
apt-key adv \
|
||||
--keyserver hkp://p80.pool.sks-keyservers.net:80 \
|
||||
--recv-keys 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
|
||||
add-apt-repository \
|
||||
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
|
||||
$(lsb_release -cs) \
|
||||
stable"
|
||||
|
||||
# Update with i386, Go and Docker
|
||||
apt-get update
|
||||
|
||||
|
|
@ -35,7 +26,6 @@ apt-get install -y \
|
|||
apt-get install -y \
|
||||
curl \
|
||||
default-jre \
|
||||
docker-ce \
|
||||
htop \
|
||||
jq \
|
||||
qemu \
|
||||
|
|
@ -60,12 +50,6 @@ apt-get install -y \
|
|||
# Ensure everything is up to date
|
||||
apt-get upgrade -y
|
||||
|
||||
# Restart Docker in case it got upgraded
|
||||
systemctl restart docker.service
|
||||
|
||||
# Ensure Docker can be used by vagrant user
|
||||
usermod -aG docker vagrant
|
||||
|
||||
# Set hostname -> IP to make advertisement work as expected
|
||||
ip=$(ip route get 1 | awk '{print $NF; exit}')
|
||||
hostname=$(hostname)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Install Development utilities
|
||||
apt-get install -y \
|
||||
curl \
|
||||
default-jre \
|
||||
htop \
|
||||
jq \
|
||||
qemu \
|
||||
silversearcher-ag \
|
||||
tree \
|
||||
unzip \
|
||||
vim
|
||||
|
||||
|
||||
# Set hostname -> IP to make advertisement work as expected
|
||||
ip=$(ip route get 1 | awk '{print $NF; exit}')
|
||||
hostname=$(hostname)
|
||||
sed -i -e "s/.*nomad.*/${ip} ${hostname}/" /etc/hosts
|
||||
|
||||
# Ensure we cd into the working directory on login
|
||||
if ! grep "cd /opt/gopath/src/github.com/hashicorp/nomad" /home/vagrant/.profile ; then
|
||||
echo 'cd /opt/gopath/src/github.com/hashicorp/nomad' >> /home/vagrant/.profile
|
||||
fi
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Add the Docker repository
|
||||
apt-key adv \
|
||||
--keyserver hkp://p80.pool.sks-keyservers.net:80 \
|
||||
--recv-keys 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
|
||||
add-apt-repository \
|
||||
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
|
||||
$(lsb_release -cs) \
|
||||
stable"
|
||||
|
||||
# Update with i386, Go and Docker
|
||||
apt-get update
|
||||
|
||||
apt-get install -y docker-ce
|
||||
|
||||
# Restart Docker in case it got upgraded
|
||||
systemctl restart docker.service
|
||||
|
||||
# Ensure Docker can be used by vagrant user
|
||||
usermod -aG docker vagrant
|
||||
Loading…
Reference in New Issue