Merge pull request #5523 from hashicorp/upgrade_e2e

scripts for upgrade testing

e2e/upgrades/README.md

@@ -0,0 +1,46 @@
+Upgrade Tests
+=============
+
+This directory has a set of scripts for Nomad upgrade testing. The cluster's configs
+are also included here. For ease of testing, the configs and node names use names like
+`"server1"` and `"client1"` and the data directories are in `"/tmp/server1"`, "/tmp/client1"`, etc
+
+Start a cluster
+===============
+Use `run_cluster.sh` to start a 3 server, 2 client cluster locally.
+The script takes the path to the Nomad binary. For example, to run a
+0.8.7 Nomad cluster do:
+
+```
+$ ./run_cluster.sh /path/to/Nomad0.8.7
+```
+Run cluster also assumes that `consul` exists in your path and runs a dev agent
+
+Stop nodes
+==========
+Use the `kill_node` script to stop nodes. Note that for quorum, you can only
+stop one server at a time. For example, to stop server1 do :
+```
+$ ./kill_node.sh server1
+```
+This does `kill -9` and is not meant to test graceful shutdowns.
+
+To stop client1 do :
+```
+$ ./kill_node.sh client1
+```
+
+Start nodes
+===========
+Use the `run_node` script to start nodes. In general, upgrade testing involves
+shutting down a node that's running an older version of nomad and starting it
+back up with a newer binary. For example, to run Nomad 0.9 as server1 do:
+```
+$ ./run_node.sh /path/to/Nomad0.9 server1
+```
+This will use the same data directory, and should rejoin the quorum
+
+Another example - to start client1 do :
+```
+$ ./run_node.sh /path/to/Nomad0.9 client1
+```

e2e/upgrades/client1.hcl

@@ -0,0 +1,23 @@
+# Increase log verbosity
+log_level = "DEBUG"
+
+# Setup data dir
+data_dir = "/tmp/client1"
+
+# Give the agent a unique name. Defaults to hostname
+name = "client1"
+
+# Enable the client
+client {
+  enabled = true
+  server_join {
+    retry_join = ["127.0.0.1:4647", "127.0.0.1:5647", "127.0.0.1:6647"]
+  }
+  options = {
+    "driver.raw_exec.enable" = "1"
+  }
+}
+
+ports {
+  http = 7646
+}

e2e/upgrades/client2.hcl

@@ -0,0 +1,21 @@
+# Increase log verbosity
+log_level = "DEBUG"
+
+# Setup data dir
+data_dir = "/tmp/client2"
+
+# Give the agent a unique name. Defaults to hostname
+name = "client2"
+
+# Enable the client
+client {
+  enabled = true
+
+  server_join {
+    retry_join = ["127.0.0.1:4647", "127.0.0.1:5647", "127.0.0.1:6647"]
+  }
+}
+
+ports {
+  http = 8646
+}

e2e/upgrades/client3.hcl

@@ -0,0 +1,21 @@
+# Increase log verbosity
+log_level = "DEBUG"
+
+# Setup data dir
+data_dir = "/tmp/client1"
+
+# Give the agent a unique name. Defaults to hostname
+name = "client1"
+
+# Enable the client
+client {
+  enabled = true
+
+  server_join {
+    retry_join = ["127.0.0.1:4647", "127.0.0.1:5647", "127.0.0.1:6647"]
+  }
+}
+
+ports {
+  http = 7646
+}

e2e/upgrades/kill_node.sh

@@ -0,0 +1,12 @@
+#/bin/bash
+# This script takes the config file of the node being killed
+# usage ./_kill_node.sh client1
+if [ "$#" -ne 1 ]; then
+    echo "expected usage - ./kill_node.sh <client|server><1|2>"
+    exit 255
+fi
+CONFIG=$1
+echo "Killing $CONFIG"
+pid=`ps wwwaux | grep nomad | grep "$CONFIG.hcl" | awk 'BEGIN { FS = " " } ; { print $2 }'`
+echo "killing pid $pid"
+kill -9 $pid

e2e/upgrades/run_cluster.sh

@@ -0,0 +1,22 @@
+#/bin/bash
+# This script takes path to a binary and runs a 3 server, two node cluster
+if [ "$#" -ne 1 ]; then
+    echo "expected usage ./run_cluster.sh /path/to/nomad/binary"
+    exit 255
+fi
+NOMAD_BINARY=$1
+# launch server 
+( ${NOMAD_BINARY} agent -config=server1.hcl 2>&1 | tee "/tmp/server1/log" ; echo "Exit code: $?" >> "/tmp/server1/log" ) &
+
+( ${NOMAD_BINARY} agent -config=server2.hcl 2>&1 | tee "/tmp/server2/log" ; echo "Exit code: $?" >> "/tmp/server2/log" ) &
+
+( ${NOMAD_BINARY}  agent -config=server3.hcl 2>&1 | tee "/tmp/server3/log" ; echo "Exit code: $?" >> "/tmp/server3/log" ) &
+
+# launch client 1
+( ${NOMAD_BINARY} agent -config=client1.hcl 2>&1 | tee "/tmp/client1/log" ; echo "Exit code: $?" >> "/tmp/client1/log" ) &
+
+# launch client 2
+( ${NOMAD_BINARY} agent -config=client2.hcl 2>&1 | tee "/tmp/client2/log" ; echo "Exit code: $?" >> "/tmp/client2/log" ) &
+
+# launch consul
+(consul agent -dev)&

e2e/upgrades/run_node.sh

@@ -0,0 +1,12 @@
+#/bin/bash
+# this script runs a nomad node (client/server)
+# first arg is the binary and second arg is of the format (<client|server><num>).
+# this is only meant to be used within the context of the cluster created in run_cluster.sh 
+# e.g usage ./run_node.sh nomad client1
+if [ "$#" -ne 2 ]; then
+    echo "Expected usage ./run_node.sh /path/to/binary <client|server><1|2>"
+    exit 255
+fi
+NOMAD_BINARY=$1
+NODE=$2
+( $NOMAD_BINARY agent -config=${NODE}.hcl 2>&1 | tee "/tmp/$NODE/log" ; echo "Exit code: $?" >> "/tmp/$NODE/log" ) &

e2e/upgrades/server1.hcl

@@ -0,0 +1,20 @@
+# Increase log verbosity
+log_level = "DEBUG"
+
+# Setup data dir
+data_dir = "/tmp/server1"
+
+# Give the agent a unique name. Defaults to hostname
+name = "server1"
+
+# Enable the server
+server {
+  enabled = true
+
+  server_join {
+    retry_join = ["127.0.0.1:5648", "127.0.0.1:6648"]
+  }
+
+  # Self-elect, should be 3 or 5 for production
+  bootstrap_expect = 3
+}

e2e/upgrades/server2.hcl

@@ -0,0 +1,26 @@
+# Increase log verbosity
+log_level = "DEBUG"
+
+# Setup data dir
+data_dir = "/tmp/server2"
+
+# Give the agent a unique name. Defaults to hostname
+name = "server2"
+
+# Enable the server
+server {
+  enabled = true
+
+  server_join {
+    retry_join = ["127.0.0.1:4648", "127.0.0.1:6648"]
+  }
+
+  # Self-elect, should be 3 or 5 for production
+  bootstrap_expect = 3
+}
+
+ports {
+  http = 5646
+  rpc = 5647
+  serf = 5648
+}

e2e/upgrades/server3.hcl

@@ -0,0 +1,25 @@
+# Increase log verbosity
+log_level = "DEBUG"
+
+# Setup data dir
+data_dir = "/tmp/server3"
+# Give the agent a unique name. Defaults to hostname
+name = "server3"
+
+# Enable the server
+server {
+  enabled = true
+
+  server_join {
+    retry_join = ["127.0.0.1:4648", "127.0.0.1:5648"]
+  }
+
+  # Self-elect, should be 3 or 5 for production
+  bootstrap_expect = 3
+}
+
+ports {
+  http = 6646
+  rpc = 6647
+  serf = 6648
+}