From 2e8c36e698ac0a85b989a1fcab5e834f847f0f3e Mon Sep 17 00:00:00 2001 From: hc-github-team-nomad-core <82989552+hc-github-team-nomad-core@users.noreply.github.com> Date: Mon, 11 Dec 2023 13:29:16 -0600 Subject: [PATCH] backport of commit c983a8f0ad9a6a75dd0f74cd4e8fd4f929052fe0 (#19428) Co-authored-by: Tom Davies --- .changelog/18516.txt | 3 ++ e2e/connect/acls.go | 81 +++++++++++++++++++++++++++++++++++++++--- nomad/consul_policy.go | 15 ++++---- 3 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 .changelog/18516.txt diff --git a/.changelog/18516.txt b/.changelog/18516.txt new file mode 100644 index 000000000..44daffa47 --- /dev/null +++ b/.changelog/18516.txt @@ -0,0 +1,3 @@ +```release-note:bug +consul: uses token namespace to fetch policies for verification +``` diff --git a/e2e/connect/acls.go b/e2e/connect/acls.go index b742a3161..4b8c56bab 100644 --- a/e2e/connect/acls.go +++ b/e2e/connect/acls.go @@ -30,6 +30,7 @@ type ConnectACLsE2ETest struct { jobIDs []string consulPolicyIDs []string consulTokenIDs []string + consulNamespace string } func (tc *ConnectACLsE2ETest) BeforeAll(f *framework.F) { @@ -72,14 +73,28 @@ func (tc *ConnectACLsE2ETest) AfterEach(f *framework.F) { // cleanup consul tokens for _, id := range tc.consulTokenIDs { t.Log("cleanup: delete consul token id:", id) - _, err := tc.Consul().ACL().TokenDelete(id, &consulapi.WriteOptions{Token: tc.consulManagementToken}) + _, err := tc.Consul().ACL().TokenDelete(id, &consulapi.WriteOptions{ + Token: tc.consulManagementToken, + Namespace: tc.consulNamespace, + }) f.NoError(err) } // cleanup consul policies for _, id := range tc.consulPolicyIDs { t.Log("cleanup: delete consul policy id:", id) - _, err := tc.Consul().ACL().PolicyDelete(id, &consulapi.WriteOptions{Token: tc.consulManagementToken}) + _, err := tc.Consul().ACL().PolicyDelete(id, &consulapi.WriteOptions{ + Token: tc.consulManagementToken, + Namespace: tc.consulNamespace, + }) + f.NoError(err) + } + + if tc.consulNamespace != "" { + t.Log("cleanup: delete consul namespace:", tc.consulNamespace) + _, err := tc.Consul().Namespaces().Delete(tc.consulNamespace, &consulapi.WriteOptions{ + Token: tc.consulManagementToken, + }) f.NoError(err) } @@ -98,12 +113,14 @@ func (tc *ConnectACLsE2ETest) AfterEach(f *framework.F) { tc.jobIDs = []string{} tc.consulTokenIDs = []string{} tc.consulPolicyIDs = []string{} + tc.consulNamespace = "" } // todo(shoenig): follow up refactor with e2eutil.ConsulPolicy type consulPolicy struct { - Name string // e.g. nomad-operator - Rules string // e.g. service "" { policy="write" } + Name string // e.g. nomad-operator + Rules string // e.g. service "" { policy="write" } + Namespace string // e.g. default } // todo(shoenig): follow up refactor with e2eutil.ConsulPolicy @@ -112,17 +129,31 @@ func (tc *ConnectACLsE2ETest) createConsulPolicy(p consulPolicy, f *framework.F) Name: p.Name, Description: "test policy " + p.Name, Rules: p.Rules, + Namespace: p.Namespace, }, &consulapi.WriteOptions{Token: tc.consulManagementToken}) f.NoError(err, "failed to create consul policy") tc.consulPolicyIDs = append(tc.consulPolicyIDs, result.ID) return result.ID } +func (tc *ConnectACLsE2ETest) createConsulNamespace(namespace string, f *framework.F) string { + result, _, err := tc.Consul().Namespaces().Create(&consulapi.Namespace{ + Name: namespace, + }, &consulapi.WriteOptions{Token: tc.consulManagementToken}) + f.NoError(err, "failed to create consul namespace") + return result.Name +} + // todo(shoenig): follow up refactor with e2eutil.ConsulPolicy func (tc *ConnectACLsE2ETest) createOperatorToken(policyID string, f *framework.F) string { + return tc.createOperatorTokenNamespaced(policyID, "default", f) +} + +func (tc *ConnectACLsE2ETest) createOperatorTokenNamespaced(policyID string, namespace string, f *framework.F) string { token, _, err := tc.Consul().ACL().TokenCreate(&consulapi.ACLToken{ Description: "operator token", Policies: []*consulapi.ACLTokenPolicyLink{{ID: policyID}}, + Namespace: namespace, }, &consulapi.WriteOptions{Token: tc.consulManagementToken}) f.NoError(err, "failed to create operator token") tc.consulTokenIDs = append(tc.consulTokenIDs, token.AccessorID) @@ -250,6 +281,48 @@ func (tc *ConnectACLsE2ETest) TestConnectACLsConnectDemo(f *framework.F) { t.Log("connect legacy job with ACLs enable finished") } +func (tc *ConnectACLsE2ETest) TestConnectACLsConnectDemoNamespaced(f *framework.F) { + t := f.T() + + t.Log("test register Connect job w/ ACLs enabled w/ operator token") + + // === Setup ACL policy within a namespace and mint Operator token === + + // create a namespace + namespace := tc.createConsulNamespace("ns-"+uuid.Short(), f) + tc.consulNamespace = namespace + t.Log("created namespace:", namespace) + + // create a policy allowing writes of services "count-api" and "count-dashboard" + policyID := tc.createConsulPolicy(consulPolicy{ + Name: "nomad-operator-policy-" + uuid.Short(), + Rules: `service "count-api" { policy = "write" } service "count-dashboard" { policy = "write" }`, + Namespace: namespace, + }, f) + t.Log("created operator policy:", policyID) + + // create a Consul "operator token" blessed with the above policy + operatorToken := tc.createOperatorTokenNamespaced(policyID, namespace, f) + t.Log("created operator token:", operatorToken) + + jobID := connectJobID() + tc.jobIDs = append(tc.jobIDs, jobID) + + allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectJob, jobID, operatorToken) + f.Equal(2, len(allocs), "expected 2 allocs for connect demo", allocs) + allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) + f.Equal(2, len(allocIDs), "expected 2 allocIDs for connect demo", allocIDs) + e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) + + // === Check Consul SI tokens were generated for sidecars === + foundSITokens := tc.countSITokens(t) + f.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens) + f.Equal(1, foundSITokens["connect-proxy-count-api"], "expected 1 SI token for connect-proxy-count-api: %v", foundSITokens) + f.Equal(1, foundSITokens["connect-proxy-count-dashboard"], "expected 1 SI token for connect-proxy-count-dashboard: %v", foundSITokens) + + t.Log("connect legacy job with ACLs enable finished") +} + func (tc *ConnectACLsE2ETest) TestConnectACLsConnectNativeDemo(f *framework.F) { t := f.T() diff --git a/nomad/consul_policy.go b/nomad/consul_policy.go index 9adc318e0..93ac99431 100644 --- a/nomad/consul_policy.go +++ b/nomad/consul_policy.go @@ -122,7 +122,7 @@ func (c *consulACLsAPI) canReadKeystore(namespace string, token *api.ACLToken) ( // check each policy directly attached to the token for _, policyRef := range token.Policies { - if allowable, err := c.policyAllowsKeystoreRead(matches, namespace, policyRef.ID); err != nil { + if allowable, err := c.policyAllowsKeystoreRead(matches, namespace, policyRef.ID, token.Namespace); err != nil { return false, err } else if allowable { return true, nil @@ -133,13 +133,14 @@ func (c *consulACLsAPI) canReadKeystore(namespace string, token *api.ACLToken) ( for _, roleLink := range token.Roles { role, _, err := c.aclClient.RoleRead(roleLink.ID, &api.QueryOptions{ AllowStale: false, + Namespace: token.Namespace, }) if err != nil { return false, err } for _, policyLink := range role.Policies { - allowable, err := c.policyAllowsKeystoreRead(matches, namespace, policyLink.ID) + allowable, err := c.policyAllowsKeystoreRead(matches, namespace, policyLink.ID, token.Namespace) if err != nil { return false, err } else if allowable { @@ -173,7 +174,7 @@ func (c *consulACLsAPI) canWriteService(namespace, service string, token *api.AC // check each policy directly attached to the token for _, policyRef := range token.Policies { - if allowable, err := c.policyAllowsServiceWrite(matches, namespace, service, policyRef.ID); err != nil { + if allowable, err := c.policyAllowsServiceWrite(matches, namespace, service, policyRef.ID, token.Namespace); err != nil { return false, err } else if allowable { return true, nil @@ -190,7 +191,7 @@ func (c *consulACLsAPI) canWriteService(namespace, service string, token *api.AC } for _, policyLink := range role.Policies { - allowable, wErr := c.policyAllowsServiceWrite(matches, namespace, service, policyLink.ID) + allowable, wErr := c.policyAllowsServiceWrite(matches, namespace, service, policyLink.ID, token.Namespace) if wErr != nil { return false, wErr } else if allowable { @@ -202,9 +203,10 @@ func (c *consulACLsAPI) canWriteService(namespace, service string, token *api.AC return false, nil } -func (c *consulACLsAPI) policyAllowsServiceWrite(matches bool, namespace, service string, policyID string) (bool, error) { +func (c *consulACLsAPI) policyAllowsServiceWrite(matches bool, namespace, service string, policyID string, tokenNamespace string) (bool, error) { policy, _, err := c.aclClient.PolicyRead(policyID, &api.QueryOptions{ AllowStale: false, + Namespace: tokenNamespace, }) if err != nil { return false, err @@ -287,9 +289,10 @@ func (cp *ConsulPolicy) allowsServiceWrite(matches bool, namespace, task string) return false } -func (c *consulACLsAPI) policyAllowsKeystoreRead(matches bool, namespace, policyID string) (bool, error) { +func (c *consulACLsAPI) policyAllowsKeystoreRead(matches bool, namespace, policyID string, tokenNamespace string) (bool, error) { policy, _, err := c.aclClient.PolicyRead(policyID, &api.QueryOptions{ AllowStale: false, + Namespace: tokenNamespace, }) if err != nil { return false, err