Update quick start AWS Terraform

This change started out as a quick update to RHEL 7 support (aka systemd), in
the process I realized most of the other platforms could use an update. While
trying to cleanup there I discovered I was repeating of bunch of information
that might be better maintained in one place - as a result:
 * consolidated server.sh and install.sh
 * removed upstart-join.conf in a favor of join flag in the consul start
 * removed platform specific folders and increased complexity of install.sh to
   include handling the differences
 * updated and extracted consul version
 * added a consistent ip_table.sh file to open ports on firewalls
 * updating consul service management configurations to enable proper restarting behavior for each platform
 * the configuration naming convention is <distro_origin>_file_name
 * added platform to the security group name so you can easily launch multpile platforms at once
 * fixes #1304

terraform/README.md

@@ -2,4 +2,4 @@
 
 This folder contains modules for Terraform that can setup Consul for
 various systems. The infrastructure provider that is used is designated
-by the folder above. See the `variables.tf` file in each for more documentation.
+by the folder above. See the `variables.tf` file in each for more documentation. 

terraform/aws/consul.tf

@@ -16,14 +16,10 @@ resource "aws_instance" "server" {
     }
 
     provisioner "file" {
-        source = "${path.module}/scripts/${var.platform}/upstart.conf"
+        source = "${path.module}/scripts/${lookup(var.service_conf, var.platform)}"
-        destination = "/tmp/upstart.conf"
+        destination = "/tmp/${lookup(var.service_conf_dest, var.platform)}"
     }
 
-    provisioner "file" {
-        source = "${path.module}/scripts/${var.platform}/upstart-join.conf"
-        destination = "/tmp/upstart-join.conf"
-    }
 
     provisioner "remote-exec" {
         inline = [
@@ -34,15 +30,15 @@ resource "aws_instance" "server" {
 
     provisioner "remote-exec" {
         scripts = [
-            "${path.module}/scripts/${var.platform}/install.sh",
+            "${path.module}/scripts/install.sh",
-            "${path.module}/scripts/${var.platform}/server.sh",
+            "${path.module}/scripts/service.sh",
-            "${path.module}/scripts/${var.platform}/service.sh",
+            "${path.module}/scripts/ip_tables.sh",
         ]
     }
 }
 
 resource "aws_security_group" "consul" {
-    name = "consul"
+    name = "consul_${var.platform}"
     description = "Consul internal traffic + maintenance."
 
     // These are for internal traffic

terraform/aws/scripts/centos6/install.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-set -e
-
-# Read the address to join from the file we provisioned
-JOIN_ADDRS=$(cat /tmp/consul-server-addr | tr -d '\n')
-
-# consul version to install
-CONSUL_VERSION=0.5.2
-
-echo "Installing dependencies..."
-yum update -y
-yum install -y unzip wget
-
-echo "Fetching Consul..."
-cd /tmp
-wget "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" -O consul.zip
-
-echo "Installing Consul..."
-unzip consul.zip >/dev/null
-chmod +x consul
-mv consul /usr/local/bin/consul
-mkdir -p /etc/consul.d
-mkdir -p /mnt/consul
-mkdir -p /etc/service
-
-#Enable consul port in iptables
-echo "Allow port 8301 in iptables"
-iptables -I INPUT -s 0/0 -p tcp --dport 8301 -j ACCEPT
-
-# Setup the join address
-cat >/tmp/consul-join << EOF
-export CONSUL_JOIN="${JOIN_ADDRS}"
-EOF
-mv /tmp/consul-join /etc/service/consul-join
-chmod 0644 /etc/service/consul-join
-
-echo "Installing Upstart service..."
-mv /tmp/upstart.conf /etc/init/consul.conf
-mv /tmp/upstart-join.conf /etc/init/consul-join.conf

terraform/aws/scripts/centos6/server.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-
-# Read from the file we created
-SERVER_COUNT=$(cat /tmp/consul-server-count | tr -d '\n')
-
-# Write the flags to a temporary file
-cat >/tmp/consul_flags << EOF
-export CONSUL_FLAGS="-server -bootstrap-expect=${SERVER_COUNT} -data-dir=/mnt/consul"
-EOF
-
-# Write it to the full service file
-mv /tmp/consul_flags /etc/service/consul
-chown root:root /etc/service/consul
-chmod 0644 /etc/service/consul

terraform/aws/scripts/centos6/service.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-set -e
-
-echo "Starting Consul..."
-start consul

terraform/aws/scripts/centos6/upstart-join.conf

@@ -1,25 +0,0 @@
-description "Join the consul cluster"
-
-start on started consul
-stop on stopped consul
-
-task
-
-script
-  if [ -f "/etc/service/consul-join" ]; then
-    . /etc/service/consul-join
-  fi
-
-  # Keep trying to join until it succeeds
-  set +e
-  while :; do
-    logger -t "consul-join" "Attempting join: ${CONSUL_JOIN}"
-    /usr/local/bin/consul join \
-      ${CONSUL_JOIN} \
-      >>/var/log/consul-join.log 2>&1
-    [ $? -eq 0 ] && break
-    sleep 5
-  done
-
-  logger -t "consul-join" "Join success!"
-end script

terraform/aws/scripts/debian_upstart.conf

@@ -1,6 +1,6 @@
 description "Consul agent"
 
-start on runlevel [2345]
+start on started networking
 stop on runlevel [!2345]
 
 respawn
@@ -24,3 +24,4 @@ script
     ${CONSUL_FLAGS} \
     >>/var/log/consul.log 2>&1
 end script
+

terraform/aws/scripts/install.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -e
+
+echo "Installing dependencies..."
+if [ -x "$(command -v apt-get)" ]; then
+  sudo apt-get update -y
+  sudo apt-get install -y unzip
+else
+  sudo yum update -y
+  sudo yum install -y unzip wget
+fi
+
+
+echo "Fetching Consul..."
+CONSUL=0.6.3
+cd /tmp
+wget https://releases.hashicorp.com/consul/${CONSUL}/consul_${CONSUL}_linux_amd64.zip -O consul.zip
+
+echo "Installing Consul..."
+unzip consul.zip >/dev/null
+chmod +x consul
+sudo mv consul /usr/local/bin/consul
+sudo mkdir -p /opt/consul/data
+
+# Read from the file we created
+SERVER_COUNT=$(cat /tmp/consul-server-count | tr -d '\n')
+CONSUL_JOIN=$(cat /tmp/consul-server-addr | tr -d '\n')
+
+# Write the flags to a temporary file
+cat >/tmp/consul_flags << EOF
+CONSUL_FLAGS="-server -bootstrap-expect=${SERVER_COUNT} -join=${CONSUL_JOIN} -data-dir=/opt/consul/data"
+EOF
+
+if [ -f /tmp/upstart.conf ];
+then
+  echo "Installing Upstart service..."
+  sudo mkdir -p /etc/consul.d
+  sudo mkdir -p /etc/service
+  sudo chown root:root /tmp/upstart.conf 
+  sudo mv /tmp/upstart.conf /etc/init/consul.conf
+  sudo chmod 0644 /etc/init/consul.conf
+  sudo mv /tmp/consul_flags /etc/service/consul
+  sudo chmod 0644 /etc/service/consul
+else
+  echo "Installing Systemd service..."
+  sudo mkdir -p /etc/systemd/system/consul.d
+  sudo chown root:root /tmp/consul.service 
+  sudo mv /tmp/consul.service /etc/systemd/system/consul.service
+  sudo chmod 0644 /etc/systemd/system/consul.service
+  sudo mv /tmp/consul_flags /etc/sysconfig/consul
+  sudo chown root:root /etc/sysconfig/consul
+  sudo chmod 0644 /etc/sysconfig/consul
+fi

terraform/aws/scripts/ip_tables.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+sudo iptables -I INPUT -s 0/0 -p tcp --dport 8300 -j ACCEPT
+sudo iptables -I INPUT -s 0/0 -p tcp --dport 8301 -j ACCEPT
+sudo iptables -I INPUT -s 0/0 -p tcp --dport 8302 -j ACCEPT
+
+if [ -d /etc/sysconfig ]; then
+  sudo iptables-save | sudo tee /etc/sysconfig/iptables
+else
+  sudo iptables-save | sudo tee /etc/iptables.rules
+fi

terraform/aws/scripts/rhel6/install.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-set -e
-
-# Read the address to join from the file we provisioned
-JOIN_ADDRS=$(cat /tmp/consul-server-addr | tr -d '\n')
-
-# consul version to install
-CONSUL_VERSION=0.5.2
-
-echo "Installing dependencies..."
-sudo yum update -y
-sudo yum install -y unzip wget
-
-echo "Fetching Consul..."
-cd /tmp
-wget "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" -O consul.zip
-
-echo "Installing Consul..."
-unzip consul.zip >/dev/null
-sudo chmod +x consul
-sudo mv consul /usr/local/bin/consul
-sudo mkdir -p /etc/consul.d
-sudo mkdir -p /mnt/consul
-sudo mkdir -p /etc/service
-
-# Setup the join address
-cat >/tmp/consul-join << EOF
-export CONSUL_JOIN="${JOIN_ADDRS}"
-EOF
-sudo mv /tmp/consul-join /etc/service/consul-join
-chmod 0644 /etc/service/consul-join
-
-echo "Installing Upstart service..."
-sudo chown root:root /tmp/upstart.conf
-sudo chown root:root /tmp/upstart-join.conf
-sudo mv /tmp/upstart.conf /etc/init/consul.conf
-sudo mv /tmp/upstart-join.conf /etc/init/consul-join.conf

terraform/aws/scripts/rhel6/server.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-
-# Read from the file we created
-SERVER_COUNT=$(cat /tmp/consul-server-count | tr -d '\n')
-
-# Write the flags to a temporary file
-cat >/tmp/consul_flags << EOF
-export CONSUL_FLAGS="-server -bootstrap-expect=${SERVER_COUNT} -data-dir=/mnt/consul"
-EOF
-
-# Write it to the full service file
-sudo mv /tmp/consul_flags /etc/service/consul
-sudo chown root:root /etc/service/consul
-sudo chmod 0644 /etc/service/consul

terraform/aws/scripts/rhel6/service.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-set -e
-
-echo "Starting Consul..."
-sudo start consul

terraform/aws/scripts/rhel6/upstart-join.conf

@@ -1,25 +0,0 @@
-description "Join the consul cluster"
-
-start on started consul
-stop on stopped consul
-
-task
-
-script
-  if [ -f "/etc/service/consul-join" ]; then
-    . /etc/service/consul-join
-  fi
-
-  # Keep trying to join until it succeeds
-  set +e
-  while :; do
-    logger -t "consul-join" "Attempting join: ${CONSUL_JOIN}"
-    /usr/local/bin/consul join \
-      ${CONSUL_JOIN} \
-      >>/var/log/consul-join.log 2>&1
-    [ $? -eq 0 ] && break
-    sleep 5
-  done
-
-  logger -t "consul-join" "Join success!"
-end script

terraform/aws/scripts/rhel_consul.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=consul agent
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+EnvironmentFile=-/etc/sysconfig/consul
+Restart=on-failure
+ExecStart=/usr/local/bin/consul agent $CONSUL_FLAGS -config-dir=/etc/systemd/system/consul.d
+ExecReload=/bin/kill -HUP $MAINPID
+KillSignal=SIGINT
+
+[Install]
+WantedBy=multi-user.target

terraform/aws/scripts/rhel_upstart.conf

@@ -1,6 +1,6 @@
 description "Consul agent"
 
-start on runlevel [2345]
+start on started network
 stop on runlevel [!2345]
 
 respawn

terraform/aws/scripts/service.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+echo "Starting Consul..."
+if [ -x "$(command -v systemctl)" ]; then
+  echo "using systemctl"
+  sudo systemctl enable consul.service
+  sudo systemctl start consul
+else 
+  echo "using upstart"
+  sudo start consul
+fi

terraform/aws/scripts/ubuntu/install.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-set -e
-
-# Read the address to join from the file we provisioned
-JOIN_ADDRS=$(cat /tmp/consul-server-addr | tr -d '\n')
-
-# consul version to install
-CONSUL_VERSION=0.5.2
-
-echo "Installing dependencies..."
-sudo apt-get update -y
-sudo apt-get install -y unzip
-
-echo "Fetching Consul..."
-cd /tmp
-wget "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" -O consul.zip
-
-echo "Installing Consul..."
-unzip consul.zip >/dev/null
-sudo chmod +x consul
-sudo mv consul /usr/local/bin/consul
-sudo mkdir -p /etc/consul.d
-sudo mkdir -p /mnt/consul
-sudo mkdir -p /etc/service
-
-# Setup the join address
-cat >/tmp/consul-join << EOF
-export CONSUL_JOIN="${JOIN_ADDRS}"
-EOF
-sudo mv /tmp/consul-join /etc/service/consul-join
-chmod 0644 /etc/service/consul-join
-
-echo "Installing Upstart service..."
-sudo mv /tmp/upstart.conf /etc/init/consul.conf
-sudo mv /tmp/upstart-join.conf /etc/init/consul-join.conf

terraform/aws/scripts/ubuntu/server.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-set -e
-
-# Read from the file we created
-SERVER_COUNT=$(cat /tmp/consul-server-count | tr -d '\n')
-
-# Write the flags to a temporary file
-cat >/tmp/consul_flags << EOF
-export CONSUL_FLAGS="-server -bootstrap-expect=${SERVER_COUNT} -data-dir=/mnt/consul"
-EOF
-
-# Write it to the full service file
-sudo mv /tmp/consul_flags /etc/service/consul
-chmod 0644 /etc/service/consul

terraform/aws/scripts/ubuntu/service.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-set -e
-
-echo "Starting Consul..."
-sudo start consul

terraform/aws/scripts/ubuntu/upstart-join.conf

@@ -1,25 +0,0 @@
-description "Join the consul cluster"
-
-start on started consul
-stop on stopped consul
-
-task
-
-script
-  if [ -f "/etc/service/consul-join" ]; then
-    . /etc/service/consul-join
-  fi
-
-  # Keep trying to join until it succeeds
-  set +e
-  while :; do
-    logger -t "consul-join" "Attempting join: ${CONSUL_JOIN}"
-    /usr/local/bin/consul join \
-      ${CONSUL_JOIN} \
-      >>/var/log/consul-join.log 2>&1
-    [ $? -eq 0 ] && break
-    sleep 5
-  done
-
-  logger -t "consul-join" "Join success!"
-end script

terraform/aws/scripts/ubuntu/upstart.conf

@@ -1,28 +0,0 @@
-description "Consul agent"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-respawn
-# This is to avoid Upstart re-spawning the process upon `consul leave`
-normal exit 0 INT
-# stop consul will not mark node as failed but left
-kill signal INT
-
-script
-  if [ -f "/etc/service/consul" ]; then
-    . /etc/service/consul
-  fi
-
-  # Make sure to use all our CPUs, because Consul can block a scheduler thread
-  export GOMAXPROCS=`nproc`
-
-  # Get the public IP
-  BIND=`ifconfig eth0 | grep "inet addr" | awk '{ print substr($2,6) }'`
-
-  exec /usr/local/bin/consul agent \
-    -config-dir="/etc/consul.d" \
-    -bind=$BIND \
-    ${CONSUL_FLAGS} \
-    >>/var/log/consul.log 2>&1
-end script

terraform/aws/variables.tf

@@ -5,29 +5,49 @@ variable "platform" {
 
 variable "user" {
     default = {
-        ubuntu = "ubuntu"
+        ubuntu  = "ubuntu"
-        rhel6 = "ec2-user"
+        rhel6   = "ec2-user"
-        centos6 = "root"
+        centos6 = "centos"
+        rhel7   = "ec2-user"
     }
 }
 
 variable "ami" {
-    description = "AWS AMI Id, if you change, make sure it is compatible with instance type, not all AMIs allow all instance types "
+    description           = "AWS AMI Id, if you change, make sure it is compatible with instance type, not all AMIs allow all instance types "
     default = {
-        us-east-1-ubuntu = "ami-83c525e8"
+        us-east-1-ubuntu  = "ami-fce3c696"
-        us-west-2-ubuntu = "ami-57e8d767"
+        us-west-2-ubuntu  = "ami-9abea4fb"
         eu-west-1-ubuntu = "ami-47a23a30"
         eu-central-1-ubuntu = "ami-accff2b1"
         ap-northeast-1-ubuntu = "ami-90815290"
         ap-southeast-1-ubuntu = "ami-0accf458"
         ap-southeast-2-ubuntu = "ami-1dc8b127"
-        us-east-1-rhel6 = "ami-b0fed2d8"
+        us-east-1-rhel6   = "ami-0d28fe66"
-        us-west-2-rhel6 = "ami-2faa861f"
+        us-west-2-rhel6   = "ami-3d3c0a0d"
-        us-east-1-centos6 = "ami-c2a818aa"
+        us-east-1-centos6 = "ami-57cd8732"
-        us-west-2-centos6 = "ami-81d092b1"
+        us-west-2-centos6 = "ami-1255b321"
+        us-east-1-rhel7   = "ami-2051294a"
+        us-west-2-rhel7   = "ami-775e4f16"
     }
 }
 
+variable "service_conf" {
+  default = {
+    ubuntu  = "debian_upstart.conf"
+    rhel6   = "rhel_upstart.conf"
+    centos6 = "rhel_upstart.conf"
+    rhel7   = "rhel_consul.service"
+  }
+}
+variable "service_conf_dest" {
+  default = {
+    ubuntu  = "upstart.conf"
+    rhel6   = "upstart.conf"
+    centos6 = "upstart.conf"
+    rhel7   = "consul.service"
+  }
+}
+
 variable "key_name" {
     description = "SSH key name in your AWS account for AWS instances."
 }