d5aa72190f
Add structs and fields to support the Nomad Pools Governance Enterprise feature of controlling node pool access via namespaces. Nomad Enterprise allows users to specify a default node pool to be used by jobs that don't specify one. In order to accomplish this, it's necessary to distinguish between a job that explicitly uses the `default` node pool and one that did not specify any. If the `default` node pool is set during job canonicalization it's impossible to do this, so this commit allows a job to have an empty node pool value during registration but sets to `default` at the admission controller mutator. In order to guarantee state consistency the state store validates that the job node pool is set and exists before inserting it.
425 lines
15 KiB
Go
425 lines
15 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
//go:build !ent
|
|
// +build !ent
|
|
|
|
package nomad
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/hashicorp/go-memdb"
|
|
msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
|
|
"github.com/hashicorp/nomad/ci"
|
|
"github.com/hashicorp/nomad/command/agent/consul"
|
|
"github.com/hashicorp/nomad/helper/pointer"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/hashicorp/nomad/testutil"
|
|
"github.com/shoenig/test/must"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// TestJobEndpoint_Register_Connect_AllowUnauthenticatedFalse asserts that a job
|
|
// submission fails allow_unauthenticated is false, and either an invalid or no
|
|
// operator Consul token is provided.
|
|
func TestJobEndpoint_Register_Connect_AllowUnauthenticatedFalse_oss(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
c.ConsulConfig.AllowUnauthenticated = pointer.Of(false)
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
newJob := func(namespace string) *structs.Job {
|
|
// Create the register request
|
|
job := mock.Job()
|
|
job.TaskGroups[0].Networks[0].Mode = "bridge"
|
|
job.TaskGroups[0].Services = []*structs.Service{
|
|
{
|
|
Name: "service1", // matches consul.ExamplePolicyID1
|
|
PortLabel: "8080",
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{},
|
|
},
|
|
},
|
|
}
|
|
// For this test we only care about authorizing the connect service
|
|
job.TaskGroups[0].Tasks[0].Services = nil
|
|
|
|
// If testing with a Consul namespace, set it on the group
|
|
if namespace != "" {
|
|
job.TaskGroups[0].Consul = &structs.Consul{
|
|
Namespace: namespace,
|
|
}
|
|
}
|
|
return job
|
|
}
|
|
|
|
newRequest := func(job *structs.Job) *structs.JobRegisterRequest {
|
|
return &structs.JobRegisterRequest{
|
|
Job: job,
|
|
WriteRequest: structs.WriteRequest{
|
|
Region: "global",
|
|
Namespace: job.Namespace,
|
|
},
|
|
}
|
|
}
|
|
|
|
noTokenOnJob := func(t *testing.T, job *structs.Job) {
|
|
fsmState := s1.State()
|
|
ws := memdb.NewWatchSet()
|
|
storedJob, err := fsmState.JobByID(ws, job.Namespace, job.ID)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, storedJob)
|
|
require.Empty(t, storedJob.ConsulToken)
|
|
}
|
|
|
|
// Non-sense Consul ACL tokens that should be rejected
|
|
missingToken := ""
|
|
fakeToken := uuid.Generate()
|
|
|
|
// Consul ACL tokens in no Consul namespace
|
|
ossTokenNoPolicyNoNS := consul.ExampleOperatorTokenID3
|
|
ossTokenNoNS := consul.ExampleOperatorTokenID1
|
|
|
|
// Consul ACL tokens in "default" Consul namespace
|
|
entTokenNoPolicyDefaultNS := consul.ExampleOperatorTokenID20
|
|
entTokenDefaultNS := consul.ExampleOperatorTokenID21
|
|
|
|
// Consul ACL tokens in "banana" Consul namespace
|
|
entTokenNoPolicyBananaNS := consul.ExampleOperatorTokenID10
|
|
entTokenBananaNS := consul.ExampleOperatorTokenID11
|
|
|
|
t.Run("group consul namespace unset", func(t *testing.T) {
|
|
// When the group namespace is unset (which is always the case with
|
|
// Nomad OSS), Consul tokens with no namespace or are in the "default"
|
|
// namespace should be accepted (assuming a sufficient service policy).
|
|
namespace := ""
|
|
|
|
t.Run("no token provided", func(t *testing.T) {
|
|
request := newRequest(newJob(namespace))
|
|
request.Job.ConsulToken = missingToken
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, "job-submitter consul token denied: missing consul token")
|
|
})
|
|
|
|
t.Run("unknown token provided", func(t *testing.T) {
|
|
request := newRequest(newJob(namespace))
|
|
request.Job.ConsulToken = fakeToken
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, "job-submitter consul token denied: unable to read consul token: no such token")
|
|
})
|
|
|
|
t.Run("unauthorized oss token provided", func(t *testing.T) {
|
|
request := newRequest(newJob(namespace))
|
|
request.Job.ConsulToken = ossTokenNoPolicyNoNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: insufficient Consul ACL permissions to write service "service1"`)
|
|
})
|
|
|
|
t.Run("authorized oss token provided", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = ossTokenNoNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.NoError(t, err)
|
|
noTokenOnJob(t, job)
|
|
})
|
|
|
|
t.Run("unauthorized token in default namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenNoPolicyDefaultNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: insufficient Consul ACL permissions to write service "service1"`)
|
|
})
|
|
|
|
t.Run("authorized token in default namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenDefaultNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.NoError(t, err)
|
|
noTokenOnJob(t, job)
|
|
})
|
|
|
|
t.Run("unauthorized token in banana namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenNoPolicyBananaNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: consul ACL token requires using namespace "banana"`)
|
|
})
|
|
|
|
t.Run("authorized token in banana namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenBananaNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: consul ACL token requires using namespace "banana"`)
|
|
})
|
|
})
|
|
|
|
t.Run("group consul namespace banana", func(t *testing.T) {
|
|
// Nomad OSS does not respect setting the consul namespace field on the group,
|
|
// and for backwards compatibility accepts tokens in the "default" namespace
|
|
// for groups with no namespace set. The net result is setting the group namespace
|
|
// to something like "banana" and using a token in "default" namespace will
|
|
// be accepted in Nomad OSS (assuming sufficient service write policy).
|
|
//
|
|
// Using a Consul token in the non-"default" namespace will always fail in
|
|
// Nomad OSS, again because the group namespace is ignored.
|
|
namespace := "banana"
|
|
|
|
t.Run("no token provided", func(t *testing.T) {
|
|
request := newRequest(newJob(namespace))
|
|
request.Job.ConsulToken = missingToken
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, "job-submitter consul token denied: missing consul token")
|
|
})
|
|
|
|
t.Run("unknown token provided", func(t *testing.T) {
|
|
request := newRequest(newJob(namespace))
|
|
request.Job.ConsulToken = fakeToken
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, "job-submitter consul token denied: unable to read consul token: no such token")
|
|
})
|
|
|
|
t.Run("unauthorized oss token provided", func(t *testing.T) {
|
|
request := newRequest(newJob(namespace))
|
|
request.Job.ConsulToken = ossTokenNoPolicyNoNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: insufficient Consul ACL permissions to write service "service1"`)
|
|
})
|
|
|
|
t.Run("authorized oss token provided", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = ossTokenNoNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.NoError(t, err)
|
|
noTokenOnJob(t, job)
|
|
})
|
|
|
|
t.Run("unauthorized token in default namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenNoPolicyDefaultNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: insufficient Consul ACL permissions to write service "service1"`)
|
|
})
|
|
|
|
t.Run("authorized token in default namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenDefaultNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.NoError(t, err)
|
|
noTokenOnJob(t, job)
|
|
})
|
|
|
|
// Consul token in custom namespace will always fail in nomad oss
|
|
|
|
t.Run("unauthorized token in banana namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenNoPolicyBananaNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: consul ACL token requires using namespace "banana"`)
|
|
})
|
|
|
|
t.Run("authorized token in banana namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenBananaNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: consul ACL token requires using namespace "banana"`)
|
|
})
|
|
})
|
|
|
|
t.Run("group consul namespace default", func(t *testing.T) {
|
|
// Nomad OSS ignores the group consul namespace, and setting it as default
|
|
// should effectively be the same as leaving it unset.
|
|
namespace := "default"
|
|
|
|
t.Run("no token provided", func(t *testing.T) {
|
|
request := newRequest(newJob(namespace))
|
|
request.Job.ConsulToken = missingToken
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, "job-submitter consul token denied: missing consul token")
|
|
})
|
|
|
|
t.Run("unknown token provided", func(t *testing.T) {
|
|
request := newRequest(newJob(namespace))
|
|
request.Job.ConsulToken = fakeToken
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, "job-submitter consul token denied: unable to read consul token: no such token")
|
|
})
|
|
|
|
t.Run("unauthorized oss token provided", func(t *testing.T) {
|
|
request := newRequest(newJob(namespace))
|
|
request.Job.ConsulToken = ossTokenNoPolicyNoNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: insufficient Consul ACL permissions to write service "service1"`)
|
|
})
|
|
|
|
t.Run("authorized oss token provided", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = ossTokenNoNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.NoError(t, err)
|
|
noTokenOnJob(t, job)
|
|
})
|
|
|
|
t.Run("unauthorized token in default namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenNoPolicyDefaultNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: insufficient Consul ACL permissions to write service "service1"`)
|
|
})
|
|
|
|
t.Run("authorized token in default namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenDefaultNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.NoError(t, err)
|
|
noTokenOnJob(t, job)
|
|
})
|
|
|
|
t.Run("unauthorized token in banana namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenNoPolicyBananaNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: consul ACL token requires using namespace "banana"`)
|
|
})
|
|
|
|
t.Run("authorized token in banana namespace", func(t *testing.T) {
|
|
job := newJob(namespace)
|
|
request := newRequest(job)
|
|
request.Job.ConsulToken = entTokenBananaNS
|
|
var response structs.JobRegisterResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response)
|
|
require.EqualError(t, err, `job-submitter consul token denied: consul ACL token requires using namespace "banana"`)
|
|
})
|
|
})
|
|
}
|
|
|
|
func TestJobEndpoint_Register_NodePool(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s, cleanupS := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0
|
|
})
|
|
defer cleanupS()
|
|
codec := rpcClient(t, s)
|
|
testutil.WaitForLeader(t, s.RPC)
|
|
|
|
// Create test namespace.
|
|
ns := mock.Namespace()
|
|
nsReq := &structs.NamespaceUpsertRequest{
|
|
Namespaces: []*structs.Namespace{ns},
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var nsResp structs.GenericResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Namespace.UpsertNamespaces", nsReq, &nsResp)
|
|
must.NoError(t, err)
|
|
|
|
// Create test node pool.
|
|
pool := mock.NodePool()
|
|
poolReq := &structs.NodePoolUpsertRequest{
|
|
NodePools: []*structs.NodePool{pool},
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var poolResp structs.GenericResponse
|
|
err = msgpackrpc.CallWithCodec(codec, "NodePool.UpsertNodePools", poolReq, &poolResp)
|
|
must.NoError(t, err)
|
|
|
|
testCases := []struct {
|
|
name string
|
|
namespace string
|
|
nodePool string
|
|
expectedPool string
|
|
expectedErr string
|
|
}{
|
|
{
|
|
name: "job in default namespace uses default node pool",
|
|
namespace: structs.DefaultNamespace,
|
|
nodePool: "",
|
|
expectedPool: structs.NodePoolDefault,
|
|
},
|
|
{
|
|
name: "job without node pool uses default node pool",
|
|
namespace: ns.Name,
|
|
nodePool: "",
|
|
expectedPool: structs.NodePoolDefault,
|
|
},
|
|
{
|
|
name: "job can set node pool",
|
|
namespace: ns.Name,
|
|
nodePool: pool.Name,
|
|
expectedPool: pool.Name,
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
job := mock.Job()
|
|
job.Namespace = tc.namespace
|
|
job.NodePool = tc.nodePool
|
|
|
|
req := &structs.JobRegisterRequest{
|
|
Job: job,
|
|
WriteRequest: structs.WriteRequest{
|
|
Region: "global",
|
|
Namespace: job.Namespace,
|
|
},
|
|
}
|
|
var resp structs.JobRegisterResponse
|
|
err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)
|
|
|
|
if tc.expectedErr != "" {
|
|
must.ErrorContains(t, err, tc.expectedErr)
|
|
} else {
|
|
must.NoError(t, err)
|
|
|
|
got, err := s.State().JobByID(nil, job.Namespace, job.ID)
|
|
must.NoError(t, err)
|
|
must.Eq(t, tc.expectedPool, got.NodePool)
|
|
}
|
|
})
|
|
}
|
|
}
|