build: Rework Vagrant to support multiple OS boxes

This commit reworks the Vagrantfile for Nomad in order to support
straightforward testing on more than one operating system, whilst
retaining the ability to stand up a test cluster running Ubuntu.

The following changes are made:

- Scripts have been extracted from the Vagrantfile into their own shell
  script files, in order that editors lint them.

- All scripts have been edited to lint with no warnings or errors for
  their respective shells.

- Scripts are named according to the operating system and privilege
  level which they run. We prefer to run a whole shell script as root
  versus prefixing (essentially) every command with `sudo` or an
  equivalent.

- The Linux development box has been separated from the test cluster,
  removing some of the more gnarly (and less portable) logic. The Linux
  development box is still primary and autostarts.

- A FreeBSD target has been added. The base box works for both
  Virtualbox and VMWare Fusion.

- A target is added to the GNUmakefile to stand up a test cluster, using
  the default provider, or overriding the provider by setting the PROVIDER
  variable in make:
	- `make testcluster`
	- `make testcluster PROVIDER=vmware_fusion`

- Machines in the test cluster have Avahi configured for zeroconf
  discovery. Each machine can ping each other machine at `hostname.local`
  - for example `nomad-server02.local`, `nomad-client03.local`.
This commit is contained in:
James Nugent 2017-09-07 13:49:22 -05:00
parent 85002aa8c1
commit 094fa23df6
14 changed files with 389 additions and 210 deletions

View File

@ -214,6 +214,16 @@ clean: ## Remove build artifacts
travis: ## Run Nomad test suites with output to prevent timeouts under Travis CI
@sh -C "$(PROJECT_ROOT)/scripts/travis.sh"
.PHONY: testcluster
testcluster: ## Bring up a Linux test cluster using Vagrant. Set PROVIDER if necessary.
vagrant up nomad-server01 \
nomad-server02 \
nomad-server03 \
nomad-client01 \
nomad-client02 \
nomad-client03 \
$(if $(PROVIDER),--provider $(PROVIDER))
HELP_FORMAT=" \033[36m%-25s\033[0m %s\n"
.PHONY: help
help: ## Display this usage information

259
Vagrantfile vendored
View File

@ -1,155 +1,146 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
#
# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2"
LINUX_BASE_BOX = "bento/ubuntu-16.04"
FREEBSD_BASE_BOX = "jen20/FreeBSD-11.1-RELEASE"
DEFAULT_CPU_COUNT = 2
$script = <<SCRIPT
GO_VERSION="1.9"
Vagrant.configure(2) do |config|
# Compilation and development boxes
config.vm.define "linux", autostart: true, primary: true do |vmCfg|
vmCfg.vm.box = LINUX_BASE_BOX
vmCfg.vm.hostname = "linux"
vmCfg = configureProviders vmCfg,
cpus: suggestedCPUCores()
export DEBIAN_FRONTEND=noninteractive
vmCfg = configureLinuxProvisioners(vmCfg)
sudo dpkg --add-architecture i386
sudo apt-get update
vmCfg.vm.synced_folder '.',
'/opt/gopath/src/github.com/hashicorp/nomad'
# Install base dependencies
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential curl git-core mercurial bzr \
libpcre3-dev pkg-config zip default-jre qemu silversearcher-ag \
jq htop vim unzip tree \
liblxc1 lxc-dev lxc-templates \
gcc-5-aarch64-linux-gnu binutils-aarch64-linux-gnu \
libc6-dev-i386 linux-libc-dev:i386 \
gcc-5-arm-linux-gnueabihf gcc-5-multilib-arm-linux-gnueabihf binutils-arm-linux-gnueabihf \
gcc-mingw-w64 binutils-mingw-w64
vmCfg.vm.provision "shell",
privileged: true,
path: './scripts/vagrant-linux-unpriv-bootstrap.sh'
end
# Setup go, for development of Nomad
SRCROOT="/opt/go"
SRCPATH="/opt/gopath"
config.vm.define "freebsd", autostart: false, primary: false do |vmCfg|
vmCfg.vm.box = FREEBSD_BASE_BOX
vmCfg.vm.hostname = "freebsd"
vmCfg = configureProviders vmCfg,
cpus: suggestedCPUCores()
# Get the ARCH
ARCH=`uname -m | sed 's|i686|386|' | sed 's|x86_64|amd64|'`
vmCfg.vm.synced_folder '.',
'/opt/gopath/src/github.com/hashicorp/nomad',
type: "nfs",
bsd__nfs_options: ['noatime']
# Install Go
if [[ $(go version) == "go version go${GO_VERSION} linux/${ARCH}" ]]; then
echo "Go ${GO_VERSION} ${ARCH} already installed; Skipping"
else
cd /tmp
wget -q https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${ARCH}.tar.gz
tar -xf go${GO_VERSION}.linux-${ARCH}.tar.gz
sudo rm -rf $SRCROOT
sudo mv go $SRCROOT
sudo chmod 775 $SRCROOT
sudo chown vagrant:vagrant $SRCROOT
fi
vmCfg.vm.provision "shell",
privileged: true,
path: './scripts/vagrant-freebsd-priv-config.sh'
# Setup the GOPATH; even though the shared folder spec gives the working
# directory the right user/group, we need to set it properly on the
# parent path to allow subsequent "go get" commands to work.
sudo mkdir -p $SRCPATH
sudo chown -R vagrant:vagrant $SRCPATH 2>/dev/null || true
# ^^ silencing errors here because we expect this to fail for the shared folder
vmCfg.vm.provision "shell",
privileged: false,
path: './scripts/vagrant-freebsd-unpriv-bootstrap.sh'
end
cat <<EOF >/tmp/gopath.sh
export GOPATH="$SRCPATH"
export GOROOT="$SRCROOT"
export PATH="$SRCROOT/bin:$SRCPATH/bin:\$PATH"
EOF
sudo mv /tmp/gopath.sh /etc/profile.d/gopath.sh
sudo chmod 0755 /etc/profile.d/gopath.sh
source /etc/profile.d/gopath.sh
# Test Cluster (Linux)
1.upto(3) do |n|
serverName = "nomad-server%02d" % [n]
clientName = "nomad-client%02d" % [n]
serverIP = "10.199.0.%d" % [10 + n]
clientIP = "10.199.0.%d" % [20 + n]
# Install Docker
if [[ -f /etc/apt/sources.list.d/docker.list ]]; then
echo "Docker repository already installed; Skipping"
else
echo deb https://apt.dockerproject.org/repo ubuntu-`lsb_release -c | awk '{print $2}'` main | sudo tee /etc/apt/sources.list.d/docker.list
sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
sudo apt-get update
fi
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y docker-engine
config.vm.define serverName, autostart: false, primary: false do |vmCfg|
vmCfg.vm.box = LINUX_BASE_BOX
vmCfg.vm.hostname = serverName
vmCfg = configureProviders(vmCfg)
vmCfg = configureLinuxProvisioners(vmCfg)
# Restart docker to make sure we get the latest version of the daemon if there is an upgrade
sudo service docker restart
vmCfg.vm.provider "virtualbox" do |_|
vmCfg.vm.network :private_network, ip: serverIP
end
vmCfg.vm.synced_folder '.',
'/opt/gopath/src/github.com/hashicorp/nomad'
vmCfg.vm.provision "shell",
privileged: true,
path: './scripts/vagrant-linux-priv-zeroconf.sh'
end
config.vm.define clientName, autostart: false, primary: false do |vmCfg|
vmCfg.vm.box = LINUX_BASE_BOX
vmCfg.vm.hostname = clientName
vmCfg = configureProviders(vmCfg)
vmCfg = configureLinuxProvisioners(vmCfg)
vmCfg.vm.provider "virtualbox" do |_|
vmCfg.vm.network :private_network, ip: clientIP
end
vmCfg.vm.synced_folder '.',
'/opt/gopath/src/github.com/hashicorp/nomad'
# Make sure we can actually use docker as the vagrant user
sudo usermod -aG docker vagrant
# Setup Nomad for development
cd /opt/gopath/src/github.com/hashicorp/nomad && make bootstrap
# Install rkt, consul and vault
bash scripts/install_rkt.sh
bash scripts/install_rkt_vagrant.sh
bash scripts/install_consul.sh
bash scripts/install_vault.sh
# Set hostname's IP to made advertisement Just Work
sudo sed -i -e "s/.*nomad.*/$(ip route get 1 | awk '{print $NF;exit}') $(hostname)/" /etc/hosts
# CD into the nomad working directory when we login to the VM
grep "cd /opt/gopath/src/github.com/hashicorp/nomad" ~/.profile || echo "cd /opt/gopath/src/github.com/hashicorp/nomad" >> ~/.profile
SCRIPT
def configureVM(vmCfg, vmParams={
numCPUs: DEFAULT_CPU_COUNT,
}
)
# When updating make sure to use a box that supports VMWare and VirtualBox
vmCfg.vm.box = "bento/ubuntu-16.04" # 16.04 LTS
vmCfg.vm.provision "shell", inline: $script, privileged: false
vmCfg.vm.synced_folder '.', '/opt/gopath/src/github.com/hashicorp/nomad'
# We're going to compile go and run a concurrent system, so give ourselves
# some extra resources. Nomad will have trouble working correctly with <2
# CPUs so we should use at least that many.
cpus = vmParams.fetch(:numCPUs, DEFAULT_CPU_COUNT)
memory = 2048
vmCfg.vm.provider "parallels" do |p, o|
p.memory = memory
p.cpus = cpus
end
vmCfg.vm.provider "virtualbox" do |v|
v.memory = memory
v.cpus = cpus
end
["vmware_fusion", "vmware_workstation"].each do |p|
vmCfg.vm.provider p do |v|
v.enable_vmrun_ip_lookup = false
v.gui = false
v.memory = memory
v.cpus = cpus
end
end
return vmCfg
vmCfg.vm.provision "shell",
privileged: true,
path: './scripts/vagrant-linux-priv-zeroconf.sh'
end
end
end
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
1.upto(3) do |n|
vmName = "nomad-server%02d" % [n]
isFirstBox = (n == 1)
def configureLinuxProvisioners(vmCfg)
vmCfg.vm.provision "shell",
privileged: true,
inline: 'rm -f /home/vagrant/linux.iso'
numCPUs = DEFAULT_CPU_COUNT
if isFirstBox and Object::RUBY_PLATFORM =~ /darwin/i
# Override the max CPUs for the first VM
numCPUs = [numCPUs, (`/usr/sbin/sysctl -n hw.ncpu`.to_i - 1)].max
end
vmCfg.vm.provision "shell",
privileged: true,
path: './scripts/vagrant-linux-priv-config.sh'
config.vm.define vmName, autostart: isFirstBox, primary: isFirstBox do |vmCfg|
vmCfg.vm.hostname = vmName
vmCfg = configureVM(vmCfg, {:numCPUs => numCPUs})
end
end
vmCfg.vm.provision "shell",
privileged: true,
path: './scripts/vagrant-linux-priv-consul.sh'
1.upto(3) do |n|
vmName = "nomad-client%02d" % [n]
config.vm.define vmName, autostart: false, primary: false do |vmCfg|
vmCfg.vm.hostname = vmName
vmCfg = configureVM(vmCfg)
end
end
vmCfg.vm.provision "shell",
privileged: true,
path: './scripts/vagrant-linux-priv-vault.sh'
vmCfg.vm.provision "shell",
privileged: true,
path: './scripts/vagrant-linux-priv-rkt.sh'
return vmCfg
end
def configureProviders(vmCfg, cpus: "2", memory: "2048")
vmCfg.vm.provider "virtualbox" do |v|
v.memory = memory
v.cpus = cpus
end
["vmware_fusion", "vmware_workstation"].each do |p|
vmCfg.vm.provider p do |v|
v.enable_vmrun_ip_lookup = false
v.vmx["memsize"] = memory
v.vmx["numvcpus"] = cpus
end
end
vmCfg.vm.provider "virtualbox" do |v|
v.memory = memory
v.cpus = cpus
end
return vmCfg
end
def suggestedCPUCores()
case RbConfig::CONFIG['host_os']
when /darwin/
Integer(`sysctl -n hw.ncpu`) / 2
when /linux/
Integer(`cat /proc/cpuinfo | grep processor | wc -l`) / 2
else
2
end
end

View File

@ -1,20 +0,0 @@
#!/bin/bash
set -e
CONSUL_VERSION="0.9.2"
CURDIR=`pwd`
if [[ $(which consul >/dev/null && consul version | head -n 1 | cut -d ' ' -f 2) == "v$CONSUL_VERSION" ]]; then
echo "Consul v$CONSUL_VERSION already installed; Skipping"
exit
fi
echo Fetching Consul...
cd /tmp/
wget -q https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip -O consul.zip
echo Installing Consul...
unzip consul.zip
sudo chmod +x consul
sudo mv consul /usr/bin/consul
cd ${CURDIR}

View File

@ -1,25 +0,0 @@
#!/bin/bash
set -e
RKT_VERSION="v1.18.0"
CMD="cp"
if [ ! -v DEST_DIR ]; then
DEST_DIR="/usr/local/bin"
CMD="sudo cp"
fi
if [[ $(which rkt >/dev/null && rkt version | head -n 1) == "rkt Version: 1.18.0" ]]; then
echo "rkt installed; Skipping"
else
printf "Fetching rkt-%s.tar.gz\n" "${RKT_VERSION}"
cd /tmp
wget -q https://github.com/coreos/rkt/releases/download/$RKT_VERSION/rkt-$RKT_VERSION.tar.gz -O rkt.tar.gz
tar xzf rkt.tar.gz
$CMD rkt-$RKT_VERSION/rkt $DEST_DIR
$CMD rkt-$RKT_VERSION/*.aci $DEST_DIR
fi
rkt version

View File

@ -1,11 +0,0 @@
#!/bin/bash
set -e
# Configure rkt networking
sudo mkdir -p /etc/rkt/net.d
if [[ -f /etc/rkt/net.d/99-network.conf ]]; then
echo "rkt network already configured; Skipping"
exit
fi
echo '{"name": "default", "type": "ptp", "ipMasq": false, "ipam": { "type": "host-local", "subnet": "172.16.28.0/24", "routes": [ { "dst": "0.0.0.0/0" } ] } }' | jq . | sudo tee -a /etc/rkt/net.d/99-network.conf

View File

@ -1,20 +0,0 @@
#!/bin/bash
set -e
VAULT_VERSION="0.7.0"
CURDIR=`pwd`
if [[ $(which vault >/dev/null && vault version | cut -d ' ' -f 2) == "v$VAULT_VERSION" ]]; then
echo "Vault v$VAULT_VERSION already installed; Skipping"
exit
fi
echo Fetching Vault ${VAULT_VERSION}...
cd /tmp/
wget -q https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip -O vault.zip
echo Installing Vault...
unzip vault.zip
sudo chmod +x vault
sudo mv vault /usr/bin/vault
cd ${CURDIR}

View File

@ -0,0 +1,35 @@
#!/bin/sh
chown vagrant:wheel \
/opt/gopath \
/opt/gopath/src \
/opt/gopath/src/github.com \
/opt/gopath/src/github.com/hashicorp
mkdir -p /usr/local/etc/pkg/repos
cat <<EOT > /usr/local/etc/pkg/repos/FreeBSD.conf
FreeBSD: {
url: "pkg+http://pkg.FreeBSD.org/\${ABI}/latest"
}
EOT
pkg update
pkg install -y \
editors/vim-lite \
devel/git \
devel/gmake \
lang/go \
security/ca_root_nss \
shells/bash
chsh -s /usr/local/bin/bash vagrant
chsh -s /usr/local/bin/bash root
cat <<EOT >> /home/vagrant/.profile
export GOPATH=/opt/gopath
export PATH=\$GOPATH/bin:\$PATH
cd /opt/gopath/src/github.com/hashicorp/nomad
EOT

View File

@ -0,0 +1,8 @@
#!/bin/sh
export GOPATH=/opt/gopath
PATH=$GOPATH/bin:$PATH
export PATH
cd /opt/gopath/src/github.com/hashicorp/nomad && gmake bootstrap

View File

@ -0,0 +1,107 @@
#!/usr/bin/env bash
export DEBIAN_FRONTEND=noninteractive
# Update and ensure we have apt-add-repository
apt-get update
apt-get install -y software-properties-common
# Add i386 architecture (for libraries)
dpkg --add-architecture i386
# Add a Golang PPA
sudo add-apt-repository ppa:gophers/archive
# Add the Docker repository
apt-key adv \
--keyserver hkp://p80.pool.sks-keyservers.net:80 \
--recv-keys 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
# Update with i386, Go and Docker
apt-get update
# Install Core build utilities for Linux
apt-get install -y \
build-essential \
git \
golang-1.9 \
libc6-dev-i386 \
liblxc1 \
libpcre3-dev \
linux-libc-dev:i386 \
lxc-dev \
lxc-templates \
pkg-config \
zip
# Install Development utilities
apt-get install -y \
curl \
default-jre \
docker-ce \
htop \
jq \
qemu \
silversearcher-ag \
tree \
unzip \
vim
# Install ARM build utilities
apt-get install -y \
binutils-aarch64-linux-gnu \
binutils-arm-linux-gnueabihf \
gcc-5-aarch64-linux-gnu \
gcc-5-arm-linux-gnueabihf \
gcc-5-multilib-arm-linux-gnueabihf
# Install Windows build utilities
apt-get install -y \
binutils-mingw-w64 \
gcc-mingw-w64
# Ensure everything is up to date
apt-get upgrade -y
# Ensure Go is on PATH
if [ ! -e /usr/bin/go ] ; then
ln -s /usr/lib/go-1.9/bin/go /usr/bin/go
fi
if [ ! -e /usr/bin/gofmt ] ; then
ln -s /usr/lib/go-1.9/bin/gofmt /usr/bin/gofmt
fi
# Ensure that the GOPATH tree is owned by vagrant:vagrant
mkdir -p /opt/gopath
chown vagrant:vagrant \
/opt/gopath \
/opt/gopath/src \
/opt/gopath/src/github.com \
/opt/gopath/src/github.com/hashicorp
# Ensure new sessions know about GOPATH
cat <<EOF > /etc/profile.d/gopath.sh
export GOPATH="/opt/gopath"
export PATH="/opt/gopath/bin:\$PATH"
EOF
chmod 755 /etc/profile.d/gopath.sh
# Restart Docker in case it got upgraded
systemctl restart docker.service
# Ensure Docker can be used by vagrant user
usermod -aG docker vagrant
# Set hostname -> IP to make advertisement work as expected
ip=$(ip route get 1 | awk '{print $NF; exit}')
hostname=$(hostname)
sed -i -e "s/.*nomad.*/${ip} ${hostname}/" /etc/hosts
# Ensure we cd into the working directory on login
if ! grep "cd /opt/gopath/src/github.com/hashicorp/nomad" /home/vagrant/.profile ; then
echo 'cd /opt/gopath/src/github.com/hashicorp/nomad' >> /home/vagrant/.profile
fi

View File

@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -o errexit
VERSION=0.9.2
DOWNLOAD=https://releases.hashicorp.com/consul/${VERSION}/consul_${VERSION}_linux_amd64.zip
function install_consul() {
if [[ -e /usr/bin/consul ]] ; then
if [ "v${VERSION}" == "$(consul version | head -n1 | awk '{print $2}')" ] ; then
return
fi
fi
wget -q -O /tmp/consul.zip ${DOWNLOAD}
unzip -d /tmp /tmp/consul.zip
mv /tmp/consul /usr/bin/consul
chmod +x /usr/bin/consul
}
install_consul

View File

@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -o errexit
VERSION=1.18.0
DOWNLOAD=https://github.com/coreos/rkt/releases/download/v${VERSION}/rkt-v${VERSION}.tar.gz
function install_rkt() {
if [[ -e /usr/local/bin/rkt ]] ; then
if [ "rkt Version: ${VERSION}" == "$(rkt version | head -n1)" ] ; then
return
fi
fi
wget -q -O /tmp/rkt.tar.gz "${DOWNLOAD}"
tar -C /tmp -xvf /tmp/rkt.tar.gz
mv /tmp/rkt-v${VERSION}/rkt /usr/local/bin
mv /tmp/rkt-v${VERSION}/*.aci /usr/local/bin
}
function configure_rkt_networking() {
if [[ -e /etc/rkt/net.d/99-network.conf ]] ; then
return
fi
mkdir -p /etc/rkt/net.d
cat <<EOT > /etc/rkt/net.d/99-network.conf
{
"name": "default",
"type": "ptp",
"ipMasq": false,
"ipam": {
"type": "host-local",
"subnet": "172.16.28.0/24",
"routes": [
{
"dst": "0.0.0.0/0"
}
]
}
}
EOT
}
install_rkt
configure_rkt_networking

View File

@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -o errexit
VERSION=0.7.0
DOWNLOAD=https://releases.hashicorp.com/vault/${VERSION}/vault_${VERSION}_linux_amd64.zip
function install_vault() {
if [[ -e /usr/bin/vault ]] ; then
if [ "v${VERSION}" = "$(vault version | head -n1 | awk '{print $2}')" ] ; then
return
fi
fi
wget -q -O /tmp/vault.zip ${DOWNLOAD}
unzip -d /tmp /tmp/vault.zip
mv /tmp/vault /usr/bin/vault
chmod +x /usr/bin/vault
}
install_vault

View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -o errexit
apt-get install -y \
avahi-daemon \
avahi-discover \
avahi-utils \
libnss-mdns \
mdns-scan

View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
cd /opt/gopath/src/github.com/hashicorp/nomad && make bootstrap