diff --git a/e2e/upgrades/README.md b/e2e/upgrades/README.md new file mode 100644 index 000000000..ba52a40ef --- /dev/null +++ b/e2e/upgrades/README.md @@ -0,0 +1,46 @@ +Upgrade Tests +============= + +This directory has a set of scripts for Nomad upgrade testing. The cluster's configs +are also included here. For ease of testing, the configs and node names use names like +`"server1"` and `"client1"` and the data directories are in `"/tmp/server1"`, "/tmp/client1"`, etc + +Start a cluster +=============== +Use `run_cluster.sh` to start a 3 server, 2 client cluster locally. +The script takes the path to the Nomad binary. For example, to run a +0.8.7 Nomad cluster do: + +``` +$ ./run_cluster.sh /path/to/Nomad0.8.7 +``` +Run cluster also assumes that `consul` exists in your path and runs a dev agent + +Stop nodes +========== +Use the `kill_node` script to stop nodes. Note that for quorum, you can only +stop one server at a time. For example, to stop server1 do : +``` +$ ./kill_node.sh server1 +``` +This does `kill -9` and is not meant to test graceful shutdowns. + +To stop client1 do : +``` +$ ./kill_node.sh client1 +``` + +Start nodes +=========== +Use the `run_node` script to start nodes. In general, upgrade testing involves +shutting down a node that's running an older version of nomad and starting it +back up with a newer binary. For example, to run Nomad 0.9 as server1 do: +``` +$ ./run_node.sh /path/to/Nomad0.9 server1 +``` +This will use the same data directory, and should rejoin the quorum + +Another example - to start client1 do : +``` +$ ./run_node.sh /path/to/Nomad0.9 client1 +``` \ No newline at end of file diff --git a/e2e/upgrades/client1.hcl b/e2e/upgrades/client1.hcl new file mode 100644 index 000000000..7fb7429a9 --- /dev/null +++ b/e2e/upgrades/client1.hcl @@ -0,0 +1,23 @@ +# Increase log verbosity +log_level = "DEBUG" + +# Setup data dir +data_dir = "/tmp/client1" + +# Give the agent a unique name. Defaults to hostname +name = "client1" + +# Enable the client +client { + enabled = true + server_join { + retry_join = ["127.0.0.1:4647", "127.0.0.1:5647", "127.0.0.1:6647"] + } + options = { + "driver.raw_exec.enable" = "1" + } +} + +ports { + http = 7646 +} diff --git a/e2e/upgrades/client2.hcl b/e2e/upgrades/client2.hcl new file mode 100644 index 000000000..f6249fbb9 --- /dev/null +++ b/e2e/upgrades/client2.hcl @@ -0,0 +1,21 @@ +# Increase log verbosity +log_level = "DEBUG" + +# Setup data dir +data_dir = "/tmp/client2" + +# Give the agent a unique name. Defaults to hostname +name = "client2" + +# Enable the client +client { + enabled = true + + server_join { + retry_join = ["127.0.0.1:4647", "127.0.0.1:5647", "127.0.0.1:6647"] + } +} + +ports { + http = 8646 +} diff --git a/e2e/upgrades/client3.hcl b/e2e/upgrades/client3.hcl new file mode 100644 index 000000000..93db926b7 --- /dev/null +++ b/e2e/upgrades/client3.hcl @@ -0,0 +1,21 @@ +# Increase log verbosity +log_level = "DEBUG" + +# Setup data dir +data_dir = "/tmp/client1" + +# Give the agent a unique name. Defaults to hostname +name = "client1" + +# Enable the client +client { + enabled = true + + server_join { + retry_join = ["127.0.0.1:4647", "127.0.0.1:5647", "127.0.0.1:6647"] + } +} + +ports { + http = 7646 +} diff --git a/e2e/upgrades/kill_node.sh b/e2e/upgrades/kill_node.sh new file mode 100755 index 000000000..b4ac36b0c --- /dev/null +++ b/e2e/upgrades/kill_node.sh @@ -0,0 +1,12 @@ +#/bin/bash +# This script takes the config file of the node being killed +# usage ./_kill_node.sh client1 +if [ "$#" -ne 1 ]; then + echo "expected usage - ./kill_node.sh <1|2>" + exit 255 +fi +CONFIG=$1 +echo "Killing $CONFIG" +pid=`ps wwwaux | grep nomad | grep "$CONFIG.hcl" | awk 'BEGIN { FS = " " } ; { print $2 }'` +echo "killing pid $pid" +kill -9 $pid diff --git a/e2e/upgrades/run_cluster.sh b/e2e/upgrades/run_cluster.sh new file mode 100755 index 000000000..11704a6be --- /dev/null +++ b/e2e/upgrades/run_cluster.sh @@ -0,0 +1,22 @@ +#/bin/bash +# This script takes path to a binary and runs a 3 server, two node cluster +if [ "$#" -ne 1 ]; then + echo "expected usage ./run_cluster.sh /path/to/nomad/binary" + exit 255 +fi +NOMAD_BINARY=$1 +# launch server +( ${NOMAD_BINARY} agent -config=server1.hcl 2>&1 | tee "/tmp/server1/log" ; echo "Exit code: $?" >> "/tmp/server1/log" ) & + +( ${NOMAD_BINARY} agent -config=server2.hcl 2>&1 | tee "/tmp/server2/log" ; echo "Exit code: $?" >> "/tmp/server2/log" ) & + +( ${NOMAD_BINARY} agent -config=server3.hcl 2>&1 | tee "/tmp/server3/log" ; echo "Exit code: $?" >> "/tmp/server3/log" ) & + +# launch client 1 +( ${NOMAD_BINARY} agent -config=client1.hcl 2>&1 | tee "/tmp/client1/log" ; echo "Exit code: $?" >> "/tmp/client1/log" ) & + +# launch client 2 +( ${NOMAD_BINARY} agent -config=client2.hcl 2>&1 | tee "/tmp/client2/log" ; echo "Exit code: $?" >> "/tmp/client2/log" ) & + +# launch consul +(consul agent -dev)& diff --git a/e2e/upgrades/run_node.sh b/e2e/upgrades/run_node.sh new file mode 100755 index 000000000..7d6d7f012 --- /dev/null +++ b/e2e/upgrades/run_node.sh @@ -0,0 +1,12 @@ +#/bin/bash +# this script runs a nomad node (client/server) +# first arg is the binary and second arg is of the format (). +# this is only meant to be used within the context of the cluster created in run_cluster.sh +# e.g usage ./run_node.sh nomad client1 +if [ "$#" -ne 2 ]; then + echo "Expected usage ./run_node.sh /path/to/binary <1|2>" + exit 255 +fi +NOMAD_BINARY=$1 +NODE=$2 +( $NOMAD_BINARY agent -config=${NODE}.hcl 2>&1 | tee "/tmp/$NODE/log" ; echo "Exit code: $?" >> "/tmp/$NODE/log" ) & diff --git a/e2e/upgrades/server1.hcl b/e2e/upgrades/server1.hcl new file mode 100644 index 000000000..62b2c792a --- /dev/null +++ b/e2e/upgrades/server1.hcl @@ -0,0 +1,20 @@ +# Increase log verbosity +log_level = "DEBUG" + +# Setup data dir +data_dir = "/tmp/server1" + +# Give the agent a unique name. Defaults to hostname +name = "server1" + +# Enable the server +server { + enabled = true + + server_join { + retry_join = ["127.0.0.1:5648", "127.0.0.1:6648"] + } + + # Self-elect, should be 3 or 5 for production + bootstrap_expect = 3 +} diff --git a/e2e/upgrades/server2.hcl b/e2e/upgrades/server2.hcl new file mode 100644 index 000000000..c50fbf8b6 --- /dev/null +++ b/e2e/upgrades/server2.hcl @@ -0,0 +1,26 @@ +# Increase log verbosity +log_level = "DEBUG" + +# Setup data dir +data_dir = "/tmp/server2" + +# Give the agent a unique name. Defaults to hostname +name = "server2" + +# Enable the server +server { + enabled = true + + server_join { + retry_join = ["127.0.0.1:4648", "127.0.0.1:6648"] + } + + # Self-elect, should be 3 or 5 for production + bootstrap_expect = 3 +} + +ports { + http = 5646 + rpc = 5647 + serf = 5648 +} diff --git a/e2e/upgrades/server3.hcl b/e2e/upgrades/server3.hcl new file mode 100644 index 000000000..2c276dde0 --- /dev/null +++ b/e2e/upgrades/server3.hcl @@ -0,0 +1,25 @@ +# Increase log verbosity +log_level = "DEBUG" + +# Setup data dir +data_dir = "/tmp/server3" +# Give the agent a unique name. Defaults to hostname +name = "server3" + +# Enable the server +server { + enabled = true + + server_join { + retry_join = ["127.0.0.1:4648", "127.0.0.1:5648"] + } + + # Self-elect, should be 3 or 5 for production + bootstrap_expect = 3 +} + +ports { + http = 6646 + rpc = 6647 + serf = 6648 +}