Merge pull request #6373 from hashicorp/b-raft-proto-upgrade

raft protocol defaults to version 2

command/agent/command.go

@@ -1203,7 +1203,7 @@ Server Options:
 
   -raft-protocol=<num>
     The Raft protocol version to use. Used for enabling certain Autopilot
-    features. Defaults to 3.
+    features. Defaults to 2.
 
   -retry-join=<address>
     Address of an agent to join at start time with retries enabled.

e2e/upgrades/run_cluster.sh

@@ -5,6 +5,10 @@ if [ "$#" -ne 1 ]; then
     exit 255
 fi
 NOMAD_BINARY=$1
+
+# make sure the directories exist so tee can create logs in them
+mkdir -p /tmp/server{1,2,3} /tmp/client{1,2}
+
 # launch server 
 ( ${NOMAD_BINARY} agent -config=server1.hcl 2>&1 | tee "/tmp/server1/log" ; echo "Exit code: $?" >> "/tmp/server1/log" ) &
 

e2e/upgrades/run_node.sh

@@ -9,4 +9,4 @@ if [ "$#" -ne 2 ]; then
 fi
 NOMAD_BINARY=$1
 NODE=$2
-( $NOMAD_BINARY agent -config=${NODE}.hcl 2>&1 | tee "/tmp/$NODE/log" ; echo "Exit code: $?" >> "/tmp/$NODE/log" ) &
+( $NOMAD_BINARY agent -config=${NODE}.hcl 2>&1 | tee -a "/tmp/$NODE/log" ; echo "Exit code: $?" >> "/tmp/$NODE/log" ) &

e2e/upgrades/run_upgrade_safe.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+if [ $# -lt 2 ]; then
+    echo "usage: $0 path-nomad-v1 path-nomad-v2" 1>&2
+    exit 1
+fi
+
+v1="$1"; shift
+v2="$1"; shift
+
+sh run_cluster.sh "$v1" &
+
+function peers () {
+    $v1 operator raft list-peers | tail -n+2 | awk '{print $1 " " $2}'
+}
+
+while true; do
+    n=`peers | grep -c '\bserver[1-3]\b'`
+    [ "$n" = 3 ] && break
+done
+
+function wait_serf () {
+    echo "wait $1 \c"; date
+    while true; do
+	  peers \
+	      | egrep "$1.global [0-9a-f-][0-9a-f-]{35}$" \
+	      && break
+	  sleep 1
+    done
+    echo "done $1 \c"; date
+}
+
+for i in {3,2,1}; do
+    sh kill_node.sh server$i
+    sh run_node.sh "$v2" server$i &
+    wait_serf server$i
+done

e2e/upgrades/run_upgrade_unsafe.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+if [ $# -lt 2 ]; then
+    echo "usage: $0 path-nomad-v1 path-nomad-v2" 1>&2
+    exit 1
+fi
+
+v1="$1"; shift
+v2="$1"; shift
+
+# sh run_cluster.sh "$v1" >/dev/null &
+sh run_cluster.sh "$v1" &
+
+while true; do
+    n=`"$v1" operator raft list-peers | grep -c '\bserver[1-3]\b'`
+    [ "$n" = 3 ] && break
+done
+
+for i in {1,2,3}; do
+    sh kill_node.sh server$i
+    sh run_node.sh "$v2" server$i &
+done

nomad/config.go

@@ -398,9 +398,8 @@ func DefaultConfig() *Config {
 	// Disable shutdown on removal
 	c.RaftConfig.ShutdownOnRemove = false
 
-	// Default to Raft v3 to enable new Raft and autopilot features.
-	// Compatible with v2 servers.
-	c.RaftConfig.ProtocolVersion = 3
+	// Default to Raft v2, update to v3 to enable new Raft and autopilot features.
+	c.RaftConfig.ProtocolVersion = 2
 
 	return c
 }

nomad/server.go

@@ -1106,8 +1106,8 @@ func (s *Server) setupRaft() error {
 	s.config.RaftConfig.Logger = logger
 	s.config.RaftConfig.LogOutput = nil
 
-	// Our version of Raft protocol requires the LocalID to match the network
-	// address of the transport.
+	// Our version of Raft protocol 2 requires the LocalID to match the network
+	// address of the transport. Raft protocol 3 uses permanent ids.
 	s.config.RaftConfig.LocalID = raft.ServerID(trans.LocalAddr())
 	if s.config.RaftConfig.ProtocolVersion >= 3 {
 		s.config.RaftConfig.LocalID = raft.ServerID(s.config.NodeID)

website/source/guides/operations/autopilot.html.md

@@ -14,7 +14,10 @@ servers, monitoring the state of the Raft cluster, and stable server introductio
 
 To enable Autopilot features (with the exception of dead server cleanup),
 the `raft_protocol` setting in the [server stanza](/docs/configuration/server.html)
-must be set to 3 on all servers. In Nomad 0.8 and 0.9 this setting defaults to 2; in Nomad 0.10 it will default to 3.
+must be set to 3 on all servers. This setting defaults to 2; a cluster configured with protocol 2 can be upgraded
+to protocol 3 with a rolling update, provided time for membership to stabilize following each server update.
+During an upgrade from raft protocol 2 to 3, use the `nomad operator raft list-peers`
+command between server updates to verify that each server identifier is replaced with a UUID.
 For more information, see the [Version Upgrade section](/guides/upgrade/upgrade-specific.html#raft-protocol-version-compatibility)
 on Raft Protocol versions.