#!/bin/bash set -e # Disable interactive apt prompts export DEBIAN_FRONTEND=noninteractive cd /ops CONFIGDIR=/ops/shared/config CONSULVERSION=1.6.0 CONSULDOWNLOAD=https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip CONSULCONFIGDIR=/etc/consul.d CONSULDIR=/opt/consul VAULTVERSION=1.0.3 VAULTDOWNLOAD=https://releases.hashicorp.com/vault/${VAULTVERSION}/vault_${VAULTVERSION}_linux_amd64.zip VAULTCONFIGDIR=/etc/vault.d VAULTDIR=/opt/vault NOMADVERSION=0.9.0 NOMADDOWNLOAD=https://releases.hashicorp.com/nomad/${NOMADVERSION}/nomad_${NOMADVERSION}_linux_amd64.zip NOMADCONFIGDIR=/etc/nomad.d NOMADDIR=/opt/nomad CONSULTEMPLATEVERSION=0.20.0 CONSULTEMPLATEDOWNLOAD=https://releases.hashicorp.com/consul-template/${CONSULTEMPLATEVERSION}/consul-template_${CONSULTEMPLATEVERSION}_linux_amd64.zip CONSULTEMPLATECONFIGDIR=/etc/consul-template.d CONSULTEMPLATEDIR=/opt/consul-template HADOOP_VERSION=2.7.7 # Dependencies sudo apt-get install -y software-properties-common sudo apt-get update sudo apt-get install -y unzip tree redis-tools jq curl tmux # Numpy (for Spark) sudo apt-get install -y python-setuptools sudo easy_install pip sudo pip install numpy # Disable the firewall sudo ufw disable || echo "ufw not installed" # Consul curl -L $CONSULDOWNLOAD > consul.zip ## Install sudo unzip consul.zip -d /usr/local/bin sudo chmod 0755 /usr/local/bin/consul sudo chown root:root /usr/local/bin/consul ## Configure sudo mkdir -p $CONSULCONFIGDIR sudo chmod 755 $CONSULCONFIGDIR sudo mkdir -p $CONSULDIR sudo chmod 755 $CONSULDIR # Vault curl -L $VAULTDOWNLOAD > vault.zip ## Install sudo unzip vault.zip -d /usr/local/bin sudo chmod 0755 /usr/local/bin/vault sudo chown root:root /usr/local/bin/vault ## Configure sudo mkdir -p $VAULTCONFIGDIR sudo chmod 755 $VAULTCONFIGDIR sudo mkdir -p $VAULTDIR sudo chmod 755 $VAULTDIR # Nomad curl -L $NOMADDOWNLOAD > nomad.zip ## Install sudo unzip nomad.zip -d /usr/local/bin sudo chmod 0755 /usr/local/bin/nomad sudo chown root:root /usr/local/bin/nomad ## Configure sudo mkdir -p $NOMADCONFIGDIR sudo chmod 755 $NOMADCONFIGDIR sudo mkdir -p $NOMADDIR sudo chmod 755 $NOMADDIR # Consul Template curl -L $CONSULTEMPLATEDOWNLOAD > consul-template.zip ## Install sudo unzip consul-template.zip -d /usr/local/bin sudo chmod 0755 /usr/local/bin/consul-template sudo chown root:root /usr/local/bin/consul-template ## Configure sudo mkdir -p $CONSULTEMPLATECONFIGDIR sudo chmod 755 $CONSULTEMPLATECONFIGDIR sudo mkdir -p $CONSULTEMPLATEDIR sudo chmod 755 $CONSULTEMPLATEDIR # Docker distro=$(lsb_release -si | tr '[:upper:]' '[:lower:]') sudo apt-get install -y apt-transport-https ca-certificates gnupg2 curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/${distro} $(lsb_release -cs) stable" sudo apt-get update sudo apt-get install -y docker-ce if [[ ! -z ${INSTALL_NVIDIA_DOCKER+x} ]]; then # Install official NVIDIA driver package sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub sudo sh -c 'echo "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/cuda.list' sudo apt-get update && sudo apt-get install -y --no-install-recommends linux-headers-generic dkms cuda-drivers # Install nvidia-docker and nvidia-docker-plugin # from: https://github.com/NVIDIA/nvidia-docker#ubuntu-140416041804-debian-jessiestretch wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - distribution=$(. /etc/os-release;echo $ID$VERSION_ID) curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \ sudo tee /etc/apt/sources.list.d/nvidia-docker.list sudo apt-get update sudo apt-get install -y nvidia-docker2 fi # rkt VERSION=1.29.0 DOWNLOAD=https://github.com/rkt/rkt/releases/download/v${VERSION}/rkt-v${VERSION}.tar.gz function install_rkt() { wget -q -O /tmp/rkt.tar.gz "${DOWNLOAD}" tar -C /tmp -xvf /tmp/rkt.tar.gz sudo mv /tmp/rkt-v${VERSION}/rkt /usr/local/bin sudo mv /tmp/rkt-v${VERSION}/*.aci /usr/local/bin } function configure_rkt_networking() { sudo mkdir -p /etc/rkt/net.d sudo bash -c 'cat << EOT > /etc/rkt/net.d/99-network.conf { "name": "default", "type": "ptp", "ipMasq": false, "ipam": { "type": "host-local", "subnet": "172.16.28.0/24", "routes": [ { "dst": "0.0.0.0/0" } ] } } EOT' } install_rkt configure_rkt_networking # Java sudo add-apt-repository -y ppa:openjdk-r/ppa sudo apt-get update sudo apt-get install -y openjdk-8-jdk JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") # Spark sudo wget -P /ops/examples/spark https://nomad-spark.s3.amazonaws.com/spark-2.2.0-bin-nomad-0.7.0.tgz sudo tar -xf /ops/examples/spark/spark-2.2.0-bin-nomad-0.7.0.tgz --directory /ops/examples/spark sudo mv /ops/examples/spark/spark-2.2.0-bin-nomad-0.7.0 /usr/local/bin/spark sudo chown -R root:root /usr/local/bin/spark # Hadoop (to enable the HDFS CLI) wget -O - http://apache.mirror.iphh.net/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz | sudo tar xz -C /usr/local/