package nomad import ( "encoding/json" "fmt" "math/rand" "strings" "testing" "time" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" "github.com/shoenig/test" "github.com/shoenig/test/must" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" ) func TestSecureVariablesEndpoint_auth(t *testing.T) { ci.Parallel(t) srv, _, shutdown := TestACLServer(t, func(c *Config) { c.NumSchedulers = 0 // Prevent automatic dequeue }) defer shutdown() testutil.WaitForLeader(t, srv.RPC) const ns = "nondefault-namespace" alloc1 := mock.Alloc() alloc1.ClientStatus = structs.AllocClientStatusFailed alloc1.Job.Namespace = ns alloc1.Namespace = ns jobID := alloc1.JobID // create an alloc that will have no access to secure variables we create alloc2 := mock.Alloc() alloc2.Job.TaskGroups[0].Name = "other-no-permissions" alloc2.TaskGroup = "other-no-permissions" alloc2.ClientStatus = structs.AllocClientStatusRunning alloc2.Job.Namespace = ns alloc2.Namespace = ns alloc3 := mock.Alloc() alloc3.ClientStatus = structs.AllocClientStatusRunning alloc3.Job.Namespace = ns alloc3.Namespace = ns alloc3.Job.ParentID = jobID store := srv.fsm.State() must.NoError(t, store.UpsertNamespaces(1000, []*structs.Namespace{{Name: ns}})) must.NoError(t, store.UpsertAllocs( structs.MsgTypeTestSetup, 1001, []*structs.Allocation{alloc1, alloc2, alloc3})) claims1 := alloc1.ToTaskIdentityClaims(nil, "web") idToken, err := srv.encrypter.SignClaims(claims1) must.NoError(t, err) claims2 := alloc2.ToTaskIdentityClaims(nil, "web") noPermissionsToken, err := srv.encrypter.SignClaims(claims2) must.NoError(t, err) claims3 := alloc3.ToTaskIdentityClaims(alloc3.Job, "web") idDispatchToken, err := srv.encrypter.SignClaims(claims3) must.NoError(t, err) // corrupt the signature of the token idTokenParts := strings.Split(idToken, ".") must.Len(t, 3, idTokenParts) sig := []string(strings.Split(idTokenParts[2], "")) rand.Shuffle(len(sig), func(i, j int) { sig[i], sig[j] = sig[j], sig[i] }) idTokenParts[2] = strings.Join(sig, "") invalidIDToken := strings.Join(idTokenParts, ".") policy := mock.ACLPolicy() policy.Rules = `namespace "nondefault-namespace" { secure_variables { path "nomad/jobs/*" { capabilities = ["list"] } path "other/path" { capabilities = ["read"] } }}` policy.JobACL = &structs.JobACL{ Namespace: ns, JobID: jobID, Group: alloc1.TaskGroup, } policy.SetHash() err = store.UpsertACLPolicies(structs.MsgTypeTestSetup, 1100, []*structs.ACLPolicy{policy}) must.NoError(t, err) aclToken := mock.ACLToken() aclToken.Policies = []string{policy.Name} err = store.UpsertACLTokens(structs.MsgTypeTestSetup, 1150, []*structs.ACLToken{aclToken}) must.NoError(t, err) t.Run("terminal alloc should be denied", func(t *testing.T) { _, err = srv.staticEndpoints.SecureVariables.handleMixedAuthEndpoint( structs.QueryOptions{AuthToken: idToken, Namespace: ns}, "n/a", fmt.Sprintf("nomad/jobs/%s/web/web", jobID)) must.EqError(t, err, structs.ErrPermissionDenied.Error()) }) // make alloc non-terminal alloc1.ClientStatus = structs.AllocClientStatusRunning must.NoError(t, store.UpsertAllocs( structs.MsgTypeTestSetup, 1200, []*structs.Allocation{alloc1})) t.Run("wrong namespace should be denied", func(t *testing.T) { _, err = srv.staticEndpoints.SecureVariables.handleMixedAuthEndpoint( structs.QueryOptions{AuthToken: idToken, Namespace: structs.DefaultNamespace}, "n/a", fmt.Sprintf("nomad/jobs/%s/web/web", jobID)) must.EqError(t, err, structs.ErrPermissionDenied.Error()) }) testCases := []struct { name string token string cap string path string expectedErr error }{ { name: "valid claim for path with task secret", token: idToken, cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s/web/web", jobID), expectedErr: nil, }, { name: "valid claim for path with group secret", token: idToken, cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s/web", jobID), expectedErr: nil, }, { name: "valid claim for path with job secret", token: idToken, cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s", jobID), expectedErr: nil, }, { name: "valid claim for path with dispatch job secret", token: idDispatchToken, cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s", jobID), expectedErr: nil, }, { name: "valid claim for path with namespace secret", token: idToken, cap: "n/a", path: "nomad/jobs", expectedErr: nil, }, { name: "valid claim for job-attached policy", token: idToken, cap: acl.PolicyRead, path: "other/path", expectedErr: nil, }, { name: "valid claim for job-attached policy path denied", token: idToken, cap: acl.PolicyRead, path: "other/not-allowed", expectedErr: structs.ErrPermissionDenied, }, { name: "valid claim for job-attached policy capability denied", token: idToken, cap: acl.PolicyWrite, path: "other/path", expectedErr: structs.ErrPermissionDenied, }, { name: "valid claim for job-attached policy capability with cross-job access", token: idToken, cap: acl.PolicyList, path: "nomad/jobs/some-other", expectedErr: nil, }, { name: "valid claim with no permissions denied by path", token: noPermissionsToken, cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s/w", jobID), expectedErr: structs.ErrPermissionDenied, }, { name: "valid claim with no permissions allowed by namespace", token: noPermissionsToken, cap: "n/a", path: "nomad/jobs", expectedErr: nil, }, { name: "valid claim with no permissions denied by capability", token: noPermissionsToken, cap: acl.PolicyRead, path: fmt.Sprintf("nomad/jobs/%s/w", jobID), expectedErr: structs.ErrPermissionDenied, }, { name: "extra trailing slash is denied", token: idToken, cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s/web/", jobID), expectedErr: structs.ErrPermissionDenied, }, { name: "invalid prefix is denied", token: idToken, cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s/w", jobID), expectedErr: structs.ErrPermissionDenied, }, { name: "missing auth token is denied", cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s/web/web", jobID), expectedErr: structs.ErrPermissionDenied, }, { name: "invalid signature is denied", token: invalidIDToken, cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s/web/web", jobID), expectedErr: structs.ErrPermissionDenied, }, { name: "invalid claim for dispatched ID", token: idDispatchToken, cap: "n/a", path: fmt.Sprintf("nomad/jobs/%s", alloc3.JobID), expectedErr: structs.ErrPermissionDenied, }, { name: "acl token read policy is allowed to list", token: aclToken.SecretID, cap: acl.PolicyList, path: fmt.Sprintf("nomad/jobs/%s/web/web", jobID), expectedErr: nil, }, { name: "acl token read policy is not allowed to write", token: aclToken.SecretID, cap: acl.PolicyWrite, path: fmt.Sprintf("nomad/jobs/%s/web/web", jobID), expectedErr: structs.ErrPermissionDenied, }, } for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { _, err := srv.staticEndpoints.SecureVariables.handleMixedAuthEndpoint( structs.QueryOptions{AuthToken: tc.token, Namespace: ns}, tc.cap, tc.path) if tc.expectedErr == nil { must.NoError(t, err) } else { must.EqError(t, err, tc.expectedErr.Error()) } }) } } func TestSecureVariablesEndpoint_Apply_ACL(t *testing.T) { ci.Parallel(t) srv, rootToken, shutdown := TestACLServer(t, func(c *Config) { c.NumSchedulers = 0 // Prevent automatic dequeue }) defer shutdown() testutil.WaitForLeader(t, srv.RPC) codec := rpcClient(t, srv) state := srv.fsm.State() pol := mock.NamespacePolicyWithSecureVariables( structs.DefaultNamespace, "", []string{"list-jobs"}, map[string][]string{ "dropbox/*": {"write"}, }) writeToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", pol) sv1 := mock.SecureVariable() sv1.ModifyIndex = 0 var svHold *structs.SecureVariableDecrypted opMap := map[string]structs.SVOp{ "set": structs.SVOpSet, "cas": structs.SVOpCAS, "delete": structs.SVOpDelete, "delete-cas": structs.SVOpDeleteCAS, } for name, op := range opMap { t.Run(name+"/no token", func(t *testing.T) { sv1 := sv1 applyReq := structs.SecureVariablesApplyRequest{ Op: op, Var: sv1, WriteRequest: structs.WriteRequest{Region: "global"}, } applyResp := new(structs.SecureVariablesApplyResponse) err := msgpackrpc.CallWithCodec(codec, structs.SecureVariablesApplyRPCMethod, &applyReq, applyResp) must.EqError(t, err, structs.ErrPermissionDenied.Error()) }) } t.Run("cas/management token/new", func(t *testing.T) { applyReq := structs.SecureVariablesApplyRequest{ Op: structs.SVOpCAS, Var: sv1, WriteRequest: structs.WriteRequest{ Region: "global", AuthToken: rootToken.SecretID, }, } applyResp := new(structs.SecureVariablesApplyResponse) err := msgpackrpc.CallWithCodec(codec, structs.SecureVariablesApplyRPCMethod, &applyReq, applyResp) must.NoError(t, err) must.Eq(t, structs.SVOpResultOk, applyResp.Result) must.Equals(t, sv1.Items, applyResp.Output.Items) svHold = applyResp.Output }) t.Run("cas with current", func(t *testing.T) { must.NotNil(t, svHold) sv := svHold sv.Items["new"] = "newVal" applyReq := structs.SecureVariablesApplyRequest{ Op: structs.SVOpCAS, Var: sv, WriteRequest: structs.WriteRequest{ Region: "global", AuthToken: rootToken.SecretID, }, } applyResp := new(structs.SecureVariablesApplyResponse) applyReq.AuthToken = rootToken.SecretID err := msgpackrpc.CallWithCodec(codec, structs.SecureVariablesApplyRPCMethod, &applyReq, &applyResp) must.NoError(t, err) must.Eq(t, structs.SVOpResultOk, applyResp.Result) must.Equals(t, sv.Items, applyResp.Output.Items) svHold = applyResp.Output }) t.Run("cas with stale", func(t *testing.T) { must.NotNil(t, sv1) // TODO: query these directly must.NotNil(t, svHold) sv1 := sv1 svHold := svHold applyReq := structs.SecureVariablesApplyRequest{ Op: structs.SVOpCAS, Var: sv1, WriteRequest: structs.WriteRequest{ Region: "global", AuthToken: rootToken.SecretID, }, } applyResp := new(structs.SecureVariablesApplyResponse) applyReq.AuthToken = rootToken.SecretID err := msgpackrpc.CallWithCodec(codec, structs.SecureVariablesApplyRPCMethod, &applyReq, &applyResp) must.NoError(t, err) must.Eq(t, structs.SVOpResultConflict, applyResp.Result) must.Equals(t, svHold.SecureVariableMetadata, applyResp.Conflict.SecureVariableMetadata) must.Equals(t, svHold.Items, applyResp.Conflict.Items) }) sv3 := mock.SecureVariable() sv3.Path = "dropbox/a" sv3.ModifyIndex = 0 t.Run("cas/write-only/read own new", func(t *testing.T) { sv3 := sv3 applyReq := structs.SecureVariablesApplyRequest{ Op: structs.SVOpCAS, Var: sv3, WriteRequest: structs.WriteRequest{ Region: "global", AuthToken: writeToken.SecretID, }, } applyResp := new(structs.SecureVariablesApplyResponse) err := msgpackrpc.CallWithCodec(codec, structs.SecureVariablesApplyRPCMethod, &applyReq, &applyResp) must.NoError(t, err) must.Eq(t, structs.SVOpResultOk, applyResp.Result) must.Equals(t, sv3.Items, applyResp.Output.Items) svHold = applyResp.Output }) t.Run("cas/write only/conflict redacted", func(t *testing.T) { must.NotNil(t, sv3) must.NotNil(t, svHold) sv3 := sv3 svHold := svHold applyReq := structs.SecureVariablesApplyRequest{ Op: structs.SVOpCAS, Var: sv3, WriteRequest: structs.WriteRequest{ Region: "global", AuthToken: writeToken.SecretID, }, } applyResp := new(structs.SecureVariablesApplyResponse) err := msgpackrpc.CallWithCodec(codec, structs.SecureVariablesApplyRPCMethod, &applyReq, &applyResp) must.NoError(t, err) must.Eq(t, structs.SVOpResultRedacted, applyResp.Result) must.Eq(t, svHold.SecureVariableMetadata, applyResp.Conflict.SecureVariableMetadata) must.Nil(t, applyResp.Conflict.Items) }) t.Run("cas/write only/read own upsert", func(t *testing.T) { must.NotNil(t, svHold) sv := svHold sv.Items["upsert"] = "read" applyReq := structs.SecureVariablesApplyRequest{ Op: structs.SVOpCAS, Var: sv, WriteRequest: structs.WriteRequest{ Region: "global", AuthToken: writeToken.SecretID, }, } applyResp := new(structs.SecureVariablesApplyResponse) err := msgpackrpc.CallWithCodec(codec, structs.SecureVariablesApplyRPCMethod, &applyReq, &applyResp) must.NoError(t, err) must.Eq(t, structs.SVOpResultOk, applyResp.Result) must.Equals(t, sv.Items, applyResp.Output.Items) }) } func TestSecureVariablesEndpoint_ComplexACLPolicies(t *testing.T) { ci.Parallel(t) srv, _, shutdown := TestACLServer(t, func(c *Config) { c.NumSchedulers = 0 // Prevent automatic dequeue }) defer shutdown() testutil.WaitForLeader(t, srv.RPC) codec := rpcClient(t, srv) idx := uint64(1000) policyRules := ` namespace "dev" { secure_variables { path "*" { capabilities = ["list", "read"] } path "system/*" { capabilities = ["deny"] } path "config/system/*" { capabilities = ["deny"] } } } namespace "prod" { secure_variables { path "*" { capabilities = ["list"] } } } namespace "*" {} ` store := srv.fsm.State() must.NoError(t, store.UpsertNamespaces(1000, []*structs.Namespace{ {Name: "dev"}, {Name: "prod"}, {Name: "other"}})) idx++ token := mock.CreatePolicyAndToken(t, store, idx, "developer", policyRules) writeVar := func(ns, path string) { idx++ sv := mock.SecureVariableEncrypted() sv.Namespace = ns sv.Path = path resp := store.SVESet(idx, &structs.SVApplyStateRequest{ Op: structs.SVOpSet, Var: sv, }) must.NoError(t, resp.Error) } writeVar("dev", "system/never-list") writeVar("dev", "config/system/never-list") writeVar("dev", "config/can-read") writeVar("dev", "project/can-read") writeVar("prod", "system/can-list") writeVar("prod", "config/system/can-list") writeVar("prod", "config/can-list") writeVar("prod", "project/can-list") writeVar("other", "system/never-list") writeVar("other", "config/system/never-list") writeVar("other", "config/never-list") writeVar("other", "project/never-list") testListPrefix := func(ns, prefix string, expectedCount int, expectErr error) { t.Run(fmt.Sprintf("ns=%s-prefix=%s", ns, prefix), func(t *testing.T) { req := &structs.SecureVariablesListRequest{ QueryOptions: structs.QueryOptions{ Namespace: ns, Prefix: prefix, AuthToken: token.SecretID, Region: "global", }, } var resp structs.SecureVariablesListResponse if expectErr != nil { must.EqError(t, msgpackrpc.CallWithCodec(codec, "SecureVariables.List", req, &resp), expectErr.Error()) return } must.NoError(t, msgpackrpc.CallWithCodec(codec, "SecureVariables.List", req, &resp)) found := "found:\n" for _, sv := range resp.Data { found += fmt.Sprintf(" ns=%s path=%s\n", sv.Namespace, sv.Path) } must.Len(t, expectedCount, resp.Data, test.Sprintf("%s", found)) }) } testListPrefix("dev", "system", 0, nil) testListPrefix("dev", "config/system", 0, nil) testListPrefix("dev", "config", 1, nil) testListPrefix("dev", "project", 1, nil) testListPrefix("dev", "", 2, nil) testListPrefix("prod", "system", 1, nil) testListPrefix("prod", "config/system", 1, nil) testListPrefix("prod", "config", 2, nil) testListPrefix("prod", "project", 1, nil) testListPrefix("prod", "", 4, nil) testListPrefix("other", "system", 0, structs.ErrPermissionDenied) testListPrefix("other", "config/system", 0, structs.ErrPermissionDenied) testListPrefix("other", "config", 0, structs.ErrPermissionDenied) testListPrefix("other", "project", 0, structs.ErrPermissionDenied) testListPrefix("other", "", 0, structs.ErrPermissionDenied) testListPrefix("*", "system", 1, nil) testListPrefix("*", "config/system", 1, nil) testListPrefix("*", "config", 3, nil) testListPrefix("*", "project", 2, nil) testListPrefix("*", "", 6, nil) } func TestSecureVariablesEndpoint_GetSecureVariable_Blocking(t *testing.T) { ci.Parallel(t) s1, cleanupS1 := TestServer(t, nil) defer cleanupS1() state := s1.fsm.State() codec := rpcClient(t, s1) testutil.WaitForLeader(t, s1.RPC) // First create an unrelated variable. delay := 100 * time.Millisecond time.AfterFunc(delay, func() { writeVar(t, s1, 100, "default", "aaa") }) // Upsert the variable we are watching later delay = 200 * time.Millisecond time.AfterFunc(delay, func() { writeVar(t, s1, 200, "default", "bbb") }) // Lookup the variable req := &structs.SecureVariablesReadRequest{ Path: "bbb", QueryOptions: structs.QueryOptions{ Region: "global", MinQueryIndex: 150, MaxQueryTime: 500 * time.Millisecond, }, } var resp structs.SecureVariablesReadResponse start := time.Now() if err := msgpackrpc.CallWithCodec(codec, "SecureVariables.Read", req, &resp); err != nil { t.Fatalf("err: %v", err) } elapsed := time.Since(start) if elapsed < delay { t.Fatalf("should block (returned in %s) %#v", elapsed, resp) } if elapsed > req.MaxQueryTime { t.Fatalf("blocking query timed out %#v", resp) } if resp.Index != 200 { t.Fatalf("Bad index: %d %d", resp.Index, 200) } if resp.Data == nil || resp.Data.Path != "bbb" { t.Fatalf("bad: %#v", resp.Data) } // Variable update triggers watches delay = 100 * time.Millisecond time.AfterFunc(delay, func() { writeVar(t, s1, 300, "default", "bbb") }) req.QueryOptions.MinQueryIndex = 250 var resp2 structs.SecureVariablesReadResponse start = time.Now() if err := msgpackrpc.CallWithCodec(codec, "SecureVariables.Read", req, &resp2); err != nil { t.Fatalf("err: %v", err) } elapsed = time.Since(start) if elapsed < delay { t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) } if elapsed > req.MaxQueryTime { t.Fatal("blocking query timed out") } if resp2.Index != 300 { t.Fatalf("Bad index: %d %d", resp2.Index, 300) } if resp2.Data == nil || resp2.Data.Path != "bbb" { t.Fatalf("bad: %#v", resp2.Data) } // Variable delete triggers watches delay = 100 * time.Millisecond time.AfterFunc(delay, func() { sv := mock.SecureVariableEncrypted() sv.Path = "bbb" if resp := state.SVEDelete(400, &structs.SVApplyStateRequest{Op: structs.SVOpDelete, Var: sv}); !resp.IsOk() { t.Fatalf("err: %v", resp.Error) } }) req.QueryOptions.MinQueryIndex = 350 var resp3 structs.SecureVariablesReadResponse start = time.Now() if err := msgpackrpc.CallWithCodec(codec, "SecureVariables.Read", req, &resp3); err != nil { t.Fatalf("err: %v", err) } elapsed = time.Since(start) if elapsed < delay { t.Fatalf("should block (returned in %s) %#v", elapsed, resp) } if elapsed > req.MaxQueryTime { t.Fatal("blocking query timed out") } if resp3.Index != 400 { t.Fatalf("Bad index: %d %d", resp3.Index, 400) } if resp3.Data != nil { t.Fatalf("bad: %#v", resp3.Data) } } func writeVar(t *testing.T, s *Server, idx uint64, ns, path string) { store := s.fsm.State() sv := mock.SecureVariable() sv.Namespace = ns sv.Path = path bPlain, err := json.Marshal(sv.Items) must.NoError(t, err) bEnc, kID, err := s.encrypter.Encrypt(bPlain) must.NoError(t, err) sve := &structs.SecureVariableEncrypted{ SecureVariableMetadata: sv.SecureVariableMetadata, SecureVariableData: structs.SecureVariableData{ Data: bEnc, KeyID: kID, }, } resp := store.SVESet(idx, &structs.SVApplyStateRequest{ Op: structs.SVOpSet, Var: sve, }) must.NoError(t, resp.Error) }