node pools: prevent panic on upsert during upgrades (#17474)

Whenever we write a Raft log entry for node pools, we need to first make sure
that all servers can safely apply the log without panicking. Gate upsert and
delete RPCs on all servers being upgraded to the minimum version.

nomad/leader.go

@@ -75,6 +75,10 @@ var minACLBindingRuleVersion = version.Must(version.NewVersion("1.5.0"))
 // servers must meet before the feature can be used.
 var minNomadServiceRegistrationVersion = version.Must(version.NewVersion("1.3.0"))
 
+// Any writes to node pools requires that all servers are on version 1.6.0 to
+// prevent older versions of the server from crashing.
+var minNodePoolsVersion = version.Must(version.NewVersion("1.6.0"))
+
 // monitorLeadership is used to monitor if we acquire or lose our role
 // as the leader in the Raft cluster. There is some work the leader is
 // expected to do, so we must react to changes

nomad/node_pool_endpoint.go

@@ -11,6 +11,7 @@ import (
 
 	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/go-memdb"
+
 	"github.com/hashicorp/nomad/acl"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/state"
@@ -186,6 +187,11 @@ func (n *NodePool) UpsertNodePools(args *structs.NodePoolUpsertRequest, reply *s
 		}
 	}
 
+	if !ServersMeetMinimumVersion(
+		n.srv.serf.Members(), n.srv.Region(), minNodePoolsVersion, true) {
+		return fmt.Errorf("all servers must be running version %v or later to upsert node pools", minNodePoolsVersion)
+	}
+
 	// Validate request.
 	if len(args.NodePools) == 0 {
 		return structs.NewErrRPCCodedf(http.StatusBadRequest, "must specify at least one node pool")
@@ -234,6 +240,11 @@ func (n *NodePool) DeleteNodePools(args *structs.NodePoolDeleteRequest, reply *s
 		}
 	}
 
+	if !ServersMeetMinimumVersion(
+		n.srv.serf.Members(), n.srv.Region(), minNodePoolsVersion, true) {
+		return fmt.Errorf("all servers must be running version %v or later to delete node pools", minNodePoolsVersion)
+	}
+
 	// Validate request.
 	if len(args.Names) == 0 {
 		return structs.NewErrRPCCodedf(http.StatusBadRequest, "must specify at least one node pool to delete")

version/version.go

@@ -19,7 +19,7 @@ var (
 	GitDescribe string
 
 	// The main version number that is being run at the moment.
-	Version = "1.5.7"
+	Version = "1.6.0"
 
 	// A pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release