luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

allow ACL policies to be associated with workload identity (#14140) 

The original design for workload identities and ACLs allows for operators to
extend the automatic capabilities of a workload by using a specially-named
policy. This has shown to be potentially unsafe because of naming collisions, so
instead we'll allow operators to explicitly attach a policy to a workload
identity.

This changeset adds workload identity fields to ACL policy objects and threads
that all the way down to the command line. It also a new secondary index to the
ACL policy table on namespace and job so that claim resolution can efficiently
query for related policies.
This commit is contained in:
Tim Gross 2022-08-22 16:41:21 -04:00 committed by GitHub
parent 29e63a6cb2
commit bf57d76ec7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 439 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/14140.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
cli: `acl policy info` output format has changed to improve readability with large policy documents
```

10
api/acl.go
View File

@ -215,10 +215,20 @@ type ACLPolicy struct {
Name string
Description string
Rules string
JobACL *JobACL
CreateIndex uint64
ModifyIndex uint64
}
// JobACL represents an ACL policy's attachment to a job, group, or task.
type JobACL struct {
Namespace string
JobID string
Group string
Task string
}
// ACLToken represents a client token which is used to Authenticate
type ACLToken struct {
AccessorID string

25
command/acl_bootstrap.go
View File

@ -130,16 +130,33 @@ func (c *ACLBootstrapCommand) Run(args []string) int {
return 0
}
// formatKVPolicy returns a K/V formatted policy
func formatKVPolicy(policy *api.ACLPolicy) string {
// formatACLPolicy returns formatted policy
func formatACLPolicy(policy *api.ACLPolicy) string {
output := []string{
fmt.Sprintf("Name|%s", policy.Name),
fmt.Sprintf("Description|%s", policy.Description),
fmt.Sprintf("Rules|%s", policy.Rules),
fmt.Sprintf("CreateIndex|%v", policy.CreateIndex),
fmt.Sprintf("ModifyIndex|%v", policy.ModifyIndex),
}
return formatKV(output)
formattedOut := formatKV(output)
if policy.JobACL != nil {
output := []string{
fmt.Sprintf("Namespace|%v", policy.JobACL.Namespace),
fmt.Sprintf("JobID|%v", policy.JobACL.JobID),
fmt.Sprintf("Group|%v", policy.JobACL.Group),
fmt.Sprintf("Task|%v", policy.JobACL.Task),
}
formattedOut += "\n\n[bold]Associated Workload[reset]\n"
formattedOut += formatKV(output)
}
// these are potentially large blobs so leave till the end
formattedOut += "\n\n[bold]Rules[reset]\n\n"
formattedOut += policy.Rules
return formattedOut
}
// formatKVACLToken returns a K/V formatted ACL token

46
command/acl_policy_apply.go
View File

@ -32,6 +32,21 @@ Apply Options:
-description
Specifies a human readable description for the policy.
-job
Attaches the policy to the specified job. Requires that -namespace is
also set.
-namespace
Attaches the policy to the specified namespace. Requires that -job is
also set.
-group
Attaches the policy to the specified task group. Requires that -namespace
and -job are also set.
-task
Attaches the policy to the specified task. Requires that -namespace, -job
and -group are also set.
`
return strings.TrimSpace(helpText)
}
@ -53,9 +68,16 @@ func (c *ACLPolicyApplyCommand) Name() string { return "acl policy apply" }
func (c *ACLPolicyApplyCommand) Run(args []string) int {
var description string
var jobID, group, task string // namespace is included in default flagset
flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
flags.Usage = func() { c.Ui.Output(c.Help()) }
flags.StringVar(&description, "description", "", "")
flags.StringVar(&jobID, "job", "", "attach policy to job")
flags.StringVar(&group, "group", "", "attach policy to group")
flags.StringVar(&task, "task", "", "attach policy to task")
if err := flags.Parse(args); err != nil {
return 1
}
@ -89,12 +111,36 @@ func (c *ACLPolicyApplyCommand) Run(args []string) int {
}
}
f := flags.Lookup("namespace")
namespace := f.Value.String()
if jobID != "" && namespace == "" {
c.Ui.Error("-namespace is required if -job is set")
return 1
}
if group != "" && jobID == "" {
c.Ui.Error("-job is required if -group is set")
return 1
}
if task != "" && group == "" {
c.Ui.Error("-group is required if -task is set")
return 1
}
// Construct the policy
ap := &api.ACLPolicy{
Name: policyName,
Description: description,
Rules: string(rawPolicy),
}
if namespace != "" {
ap.JobACL = &api.JobACL{
Namespace: namespace,
JobID: jobID,
Group: group,
Task: task,
}
}
// Get the HTTP client
client, err := c.Meta.Client()

2
command/acl_policy_info.go
View File

@ -74,6 +74,6 @@ func (c *ACLPolicyInfoCommand) Run(args []string) int {
return 1
}
c.Ui.Output(formatKVPolicy(policy))
c.Ui.Output(c.Colorize().Color(formatACLPolicy(policy)))
return 0
}

38
nomad/acl.go
View File

@ -180,27 +180,33 @@ func (s *Server) resolvePoliciesForClaims(claims *structs.IdentityClaims) ([]*st
return nil, fmt.Errorf("allocation does not exist")
}
// Find any implicit policies associated with this task
policies := []*structs.ACLPolicy{}
implicitPolicyNames := []string{
fmt.Sprintf("_:%s/%s/%s/%s", alloc.Namespace, alloc.Job.ID, alloc.TaskGroup, claims.TaskName),
fmt.Sprintf("_:%s/%s/%s", alloc.Namespace, alloc.Job.ID, alloc.TaskGroup),
fmt.Sprintf("_:%s/%s", alloc.Namespace, alloc.Job.ID),
fmt.Sprintf("_:%s", alloc.Namespace),
// Find any policies attached to the job
iter, err := snap.ACLPolicyByJob(nil, alloc.Namespace, alloc.Job.ID)
if err != nil {
return nil, err
}
for _, policyName := range implicitPolicyNames {
policy, err := snap.ACLPolicyByName(nil, policyName)
if err != nil {
return nil, err
policies := []*structs.ACLPolicy{}
for {
raw := iter.Next()
if raw == nil {
break
}
if policy == nil {
// Ignore policies that don't exist, since they don't
// grant any more privilege
policy := raw.(*structs.ACLPolicy)
if policy.JobACL == nil {
continue
}
policies = append(policies, policy)
switch {
case policy.JobACL.Group == "":
policies = append(policies, policy)
case policy.JobACL.Group != alloc.TaskGroup:
continue // don't bother checking task
case policy.JobACL.Task == "":
policies = append(policies, policy)
case policy.JobACL.Task == claims.TaskName:
policies = append(policies, policy)
}
}
return policies, nil
}

119
nomad/acl_test.go
View File

@ -4,6 +4,9 @@ import (
"testing"
lru "github.com/hashicorp/golang-lru"
"github.com/shoenig/test/must"
"github.com/stretchr/testify/assert"
"github.com/hashicorp/nomad/acl"
"github.com/hashicorp/nomad/ci"
"github.com/hashicorp/nomad/helper/uuid"
@ -11,7 +14,6 @@ import (
"github.com/hashicorp/nomad/nomad/state"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
"github.com/stretchr/testify/assert"
)
func TestResolveACLToken(t *testing.T) {
@ -63,7 +65,7 @@ func TestResolveACLToken(t *testing.T) {
assert.Nil(t, err)
assert.NotNil(t, aclObj)
// Check that the ACL object is sane
// Check that the ACL object looks reasonable
assert.Equal(t, false, aclObj.IsManagement())
allowed := aclObj.AllowNamespaceOperation("default", acl.NamespaceCapabilityListJobs)
assert.Equal(t, true, allowed)
@ -132,3 +134,116 @@ func TestResolveSecretToken(t *testing.T) {
}
}
func TestResolveClaims(t *testing.T) {
ci.Parallel(t)
srv, _, cleanup := TestACLServer(t, nil)
defer cleanup()
store := srv.fsm.State()
index := uint64(100)
alloc := mock.Alloc()
claims := &structs.IdentityClaims{
Namespace: alloc.Namespace,
JobID: alloc.Job.ID,
AllocationID: alloc.ID,
TaskName: alloc.Job.TaskGroups[0].Tasks[0].Name,
}
// unrelated policy
policy0 := mock.ACLPolicy()
// policy for job
policy1 := mock.ACLPolicy()
policy1.JobACL = &structs.JobACL{
Namespace: claims.Namespace,
JobID: claims.JobID,
}
// policy for job and group
policy2 := mock.ACLPolicy()
policy2.JobACL = &structs.JobACL{
Namespace: claims.Namespace,
JobID: claims.JobID,
Group: alloc.Job.TaskGroups[0].Name,
}
// policy for job and group and task
policy3 := mock.ACLPolicy()
policy3.JobACL = &structs.JobACL{
Namespace: claims.Namespace,
JobID: claims.JobID,
Group: alloc.Job.TaskGroups[0].Name,
Task: claims.TaskName,
}
// policy for job and group but different task
policy4 := mock.ACLPolicy()
policy4.JobACL = &structs.JobACL{
Namespace: claims.Namespace,
JobID: claims.JobID,
Group: alloc.Job.TaskGroups[0].Name,
Task: "another",
}
// policy for job but different group
policy5 := mock.ACLPolicy()
policy5.JobACL = &structs.JobACL{
Namespace: claims.Namespace,
JobID: claims.JobID,
Group: "another",
}
// policy for same namespace but different job
policy6 := mock.ACLPolicy()
policy6.JobACL = &structs.JobACL{
Namespace: claims.Namespace,
JobID: "another",
}
// policy for same job in different namespace
policy7 := mock.ACLPolicy()
policy7.JobACL = &structs.JobACL{
Namespace: "another",
JobID: claims.JobID,
}
index++
err := store.UpsertACLPolicies(structs.MsgTypeTestSetup, index, []*structs.ACLPolicy{
policy0, policy1, policy2, policy3, policy4, policy5, policy6, policy7})
must.NoError(t, err)
aclObj, err := srv.ResolveClaims(claims)
must.Nil(t, aclObj)
must.EqError(t, err, "allocation does not exist")
// upsert the allocation
index++
err = store.UpsertAllocs(structs.MsgTypeTestSetup, index, []*structs.Allocation{alloc})
must.NoError(t, err)
aclObj, err = srv.ResolveClaims(claims)
must.NoError(t, err)
must.NotNil(t, aclObj)
// Check that the ACL object looks reasonable
must.False(t, aclObj.IsManagement())
must.True(t, aclObj.AllowNamespaceOperation("default", acl.NamespaceCapabilityListJobs))
must.False(t, aclObj.AllowNamespaceOperation("other", acl.NamespaceCapabilityListJobs))
// Resolve the same claim again, should get cache value
aclObj2, err := srv.ResolveClaims(claims)
must.NoError(t, err)
must.NotNil(t, aclObj)
must.Eq(t, aclObj, aclObj2, must.Sprintf("expected cached value"))
policies, err := srv.resolvePoliciesForClaims(claims)
must.NoError(t, err)
must.Len(t, 3, policies)
must.Contains(t, policies, policy1)
must.Contains(t, policies, policy2)
must.Contains(t, policies, policy3)
}

21
nomad/secure_variables_endpoint_test.go
View File

@ -78,12 +78,16 @@ func TestSecureVariablesEndpoint_auth(t *testing.T) {
invalidIDToken := strings.Join(idTokenParts, ".")
policy := mock.ACLPolicy()
policy.Name = fmt.Sprintf("_:%s/%s/%s", ns, jobID, alloc1.TaskGroup)
policy.Rules = `namespace "nondefault-namespace" {
secure_variables {
path "nomad/jobs/*" { capabilities = ["read"] }
path "nomad/jobs/*" { capabilities = ["list"] }
path "other/path" { capabilities = ["read"] }
}}`
policy.JobACL = &structs.JobACL{
Namespace: ns,
JobID: jobID,
Group: alloc1.TaskGroup,
}
policy.SetHash()
err = store.UpsertACLPolicies(structs.MsgTypeTestSetup, 1100, []*structs.ACLPolicy{policy})
must.NoError(t, err)
@ -155,26 +159,33 @@ func TestSecureVariablesEndpoint_auth(t *testing.T) {
expectedErr: nil,
},
{
name: "valid claim for implied policy",
name: "valid claim for job-attached policy",
token: idToken,
cap: acl.PolicyRead,
path: "other/path",
expectedErr: nil,
},
{
name: "valid claim for implied policy path denied",
name: "valid claim for job-attached policy path denied",
token: idToken,
cap: acl.PolicyRead,
path: "other/not-allowed",
expectedErr: structs.ErrPermissionDenied,
},
{
name: "valid claim for implied policy capability denied",
name: "valid claim for job-attached policy capability denied",
token: idToken,
cap: acl.PolicyWrite,
path: "other/path",
expectedErr: structs.ErrPermissionDenied,
},
{
name: "valid claim for job-attached policy capability with cross-job access",
token: idToken,
cap: acl.PolicyList,
path: "nomad/jobs/some-other",
expectedErr: nil,
},
{
name: "valid claim with no permissions denied by path",
token: noPermissionsToken,

70
nomad/state/schema.go
View File

@ -775,10 +775,80 @@ func aclPolicyTableSchema() *memdb.TableSchema {
Field: "Name",
},
},
"job": {
Name: "job",
AllowMissing: true,
Unique: false,
Indexer: &ACLPolicyJobACLFieldIndex{},
},
},
}
}
// ACLPolicyJobACLFieldIndex is used to extract the policy's JobACL field and
// build an index on it.
type ACLPolicyJobACLFieldIndex struct{}
// FromObject is used to extract an index value from an
// object or to indicate that the index value is missing.
func (a *ACLPolicyJobACLFieldIndex) FromObject(obj interface{}) (bool, []byte, error) {
policy, ok := obj.(*structs.ACLPolicy)
if !ok {
return false, nil, fmt.Errorf("object %#v is not an ACLPolicy", obj)
}
if policy.JobACL == nil {
return false, nil, nil
}
ns := policy.JobACL.Namespace
if ns == "" {
return false, nil, nil
}
jobID := policy.JobACL.JobID
if jobID == "" {
return false, nil, fmt.Errorf(
"object %#v is not a valid ACLPolicy: JobACL.JobID without Namespace", obj)
}
val := ns + "\x00" + jobID + "\x00"
return true, []byte(val), nil
}
// FromArgs is used to build an exact index lookup based on arguments
func (a *ACLPolicyJobACLFieldIndex) FromArgs(args ...interface{}) ([]byte, error) {
if len(args) != 2 {
return nil, fmt.Errorf("must provide two arguments")
}
arg0, ok := args[0].(string)
if !ok {
return nil, fmt.Errorf("argument must be a string: %#v", args[0])
}
arg1, ok := args[1].(string)
if !ok {
return nil, fmt.Errorf("argument must be a string: %#v", args[0])
}
// Add the null character as a terminator
arg0 += "\x00" + arg1 + "\x00"
return []byte(arg0), nil
}
// PrefixFromArgs returns a prefix that should be used for scanning based on the arguments
func (a *ACLPolicyJobACLFieldIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) {
val, err := a.FromArgs(args...)
if err != nil {
return nil, err
}
// Strip the null terminator, the rest is a prefix
n := len(val)
if n > 0 {
return val[:n-1], nil
}
return val, nil
}
// aclTokenTableSchema returns the MemDB schema for the tokens table.
// This table is used to store the bearer tokens which are used to authenticate
func aclTokenTableSchema() *memdb.TableSchema {

14
nomad/state/state_store.go
View File

@ -5570,6 +5570,20 @@ func (s *StateStore) ACLPolicyByNamePrefix(ws memdb.WatchSet, prefix string) (me
return iter, nil
}
// ACLPolicyByJob is used to lookup policies that have been attached to a
// specific job
func (s *StateStore) ACLPolicyByJob(ws memdb.WatchSet, ns, jobID string) (memdb.ResultIterator, error) {
txn := s.db.ReadTxn()
iter, err := txn.Get("acl_policy", "job_prefix", ns, jobID)
if err != nil {
return nil, fmt.Errorf("acl policy lookup failed: %v", err)
}
ws.Add(iter.WatchCh())
return iter, nil
}
// ACLPolicies returns an iterator over all the acl policies
func (s *StateStore) ACLPolicies(ws memdb.WatchSet) (memdb.ResultIterator, error) {
txn := s.db.ReadTxn()

32
nomad/structs/structs.go
View File

@ -11718,11 +11718,21 @@ type ACLPolicy struct {
Description string // Human readable
Rules string // HCL or JSON format
RulesJSON *acl.Policy // Generated from Rules on read
JobACL *JobACL
Hash []byte
CreateIndex uint64
ModifyIndex uint64
}
// JobACL represents an ACL policy's attachment to a job, group, or task.
type JobACL struct {
Namespace string // namespace of the job
JobID string // ID of the job
Group string // ID of the group
Task string // ID of the task
}
// SetHash is used to compute and set the hash of the ACL policy
func (a *ACLPolicy) SetHash() []byte {
// Initialize a 256bit Blake2 hash (32 bytes)
@ -11736,6 +11746,13 @@ func (a *ACLPolicy) SetHash() []byte {
_, _ = hash.Write([]byte(a.Description))
_, _ = hash.Write([]byte(a.Rules))
if a.JobACL != nil {
_, _ = hash.Write([]byte(a.JobACL.Namespace))
_, _ = hash.Write([]byte(a.JobACL.JobID))
_, _ = hash.Write([]byte(a.JobACL.Group))
_, _ = hash.Write([]byte(a.JobACL.Task))
}
// Finalize the hash
hashVal := hash.Sum(nil)
@ -11768,6 +11785,21 @@ func (a *ACLPolicy) Validate() error {
err := fmt.Errorf("description longer than %d", maxPolicyDescriptionLength)
mErr.Errors = append(mErr.Errors, err)
}
if a.JobACL != nil {
if a.JobACL.JobID != "" && a.JobACL.Namespace == "" {
err := fmt.Errorf("namespace must be set to set job ID")
mErr.Errors = append(mErr.Errors, err)
}
if a.JobACL.Group != "" && a.JobACL.JobID == "" {
err := fmt.Errorf("job ID must be set to set group")
mErr.Errors = append(mErr.Errors, err)
}
if a.JobACL.Task != "" && a.JobACL.Group == "" {
err := fmt.Errorf("group must be set to set task")
mErr.Errors = append(mErr.Errors, err)
}
}
return mErr.ErrorOrNil()
}

22
website/content/docs/commands/acl/policy-apply.mdx
View File

@ -28,6 +28,19 @@ This command requires a management ACL token.
- `-description`: Sets the human readable description for the ACL policy.
- `-job`: Attaches the policy to the specified job. Requires that `-namespace` is
also set.
- `-namespace`: Attaches the policy to the specified namespace. Requires that
`-job` is also set.
- `-group`: Attaches the policy to the specified task group. Requires that
`-namespace` and `-job` are also set.
- `-task`: Attaches the policy to the specified task. Requires that `-namespace`,
`-job` and `-group` are also set.
## Examples
Create a new ACL Policy:
@ -36,3 +49,12 @@ Create a new ACL Policy:
$ nomad acl policy apply my-policy my-policy.json
Successfully wrote 'my-policy' ACL policy!
```
Associate an ACL Policy with a specific task:
```shell-session
$ nomad acl policy apply \
-namespace default -job example -group cache -task redis \
my-policy my-policy.json
Successfully wrote 'my-policy' ACL policy!
```

29
website/content/docs/commands/acl/policy-info.mdx
View File

@ -34,11 +34,38 @@ Fetch information on an existing ACL Policy:
$ nomad acl policy info my-policy
Name = my-policy
Description = <none>
Rules = {
CreateIndex = 749
ModifyIndex = 758
Rules
{
"Name": "my-policy",
"Description": "This is a great policy",
"Rules": "list_jobs"
}
```
If the ACL Policy is associated with a [Workload Identity], additional information will be shown:
```shell-session
$ nomad acl policy info my-policy
Name = my-policy
Description = <none>
CreateIndex = 749
ModifyIndex = 758
Associated Workload
Namespace = default
JobID = example
Group = cache
Task = redis
Rules
{
"Name": "my-policy",
"Description": "This is a great policy",
"Rules": "list_jobs"
}
```

12
website/content/docs/concepts/secure-variables.mdx
View File

@ -131,8 +131,9 @@ namespace "default" {
```
You can provide access to additional secrets by creating policies associated
with the task's [workload identity]. For example, to give the task above access to
set of shared secrets, you can create the following policy file:
with the task's [workload identity]. For example, to give the task above access
to all secrets in the "shared" namespace, you can create the following policy
file:
```hcl
namespace "shared" {
@ -144,11 +145,12 @@ namespace "shared" {
}
```
Then create the policy with to give the task read access to
all paths in the "shared" namespace:
Then create the policy and associate it with the specific task:
```shell-session
nomad acl policy apply "_:/default/example/cache/redis" ./policy.hcl
nomad acl policy apply \
-namespace default -job example -group cache -task redis \
redis-policy ./policy.hcl
```
See [Implicit Access to ACL Policies] for more details.

58
website/content/docs/concepts/workload-identity.mdx
View File

@ -20,33 +20,16 @@ workload identity includes the following identity claims:
}
```
## Implicit Access to ACL Policies
# Workload Associated ACL Policies
Nomad automatically attaches a set of implicit ACL policies to every workload
identity. The names of these policies start with the Nomad-owned prefix `_:`,
followed by the namespace, job ID, task group name, and task name.
You can associate additional ACL policies with workload identities by passing
the `-job`, `-group`, and `-task` flags to `nomad acl policy apply`. When Nomad
resolves a workload identity claim, it will automatically include policies that
match. If no matching policies exist, the workload identity does not have any
additional capabilities.
```
_:/$namespace/$job_id/$task_group/$task
_:/$namespace/$job_id/$task_group
_:/$namespace/$job_id
_:/$namespace
```
For example, a task named "redis", in a group named "cache", in a job named
"example", will automatically have the following policies:
```
_:/default/example/cache/redis
_:/default/example/cache
_:/default/example
_:/default
```
If these policies do not exist, the workload identity does not have any
additional capabilities. But you can create a policy with one of these names and
the task will automatically have access to them. For example, to give the task
above access to set of shared secrets, you can create the following policy file:
For example, to allow a workload access to secrets from the namespace "shared",
you can create the following policy file:
```hcl
namespace "shared" {
@ -58,11 +41,30 @@ namespace "shared" {
}
```
Then create the policy to give the task read access to
all paths in the "shared" namespace:
You can then apply this policy to a specific task:
```shell-session
nomad acl policy apply "_:/default/example/cache/redis" ./policy.hcl
nomad acl policy apply \
-namespace default -job example -group cache -task redis \
redis-policy ./policy.hcl
```
You can also apply this policy to all tasks in the group by omitting the `-task`
flag:
```shell-session
nomad acl policy apply \
-namespace default -job example -group cache \
redis-policy ./policy.hcl
```
And you can apply this policy to all groups in the job by omitting both the
`-group` and `-task` flag:
```shell-session
nomad acl policy apply \
-namespace default -job example \
redis-policy ./policy.hcl
```
## Using Workload Identity