From 0292f48396c7212c45eb2427448c8f322e883770 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Tue, 12 Jul 2022 13:43:25 +0200 Subject: [PATCH 01/19] server: add ACL token expiration config parameters. (#13667) This commit adds configuration parameters to control ACL token expirations. This includes both limits on the min and max TTL expiration values, as well as a GC threshold for expired tokens. --- command/agent/agent.go | 13 +++++++++++ command/agent/config.go | 35 ++++++++++++++++++++++++++++-- command/agent/config_parse.go | 2 ++ command/agent/config_parse_test.go | 17 ++++++++++----- command/agent/config_test.go | 20 ++++++++++------- command/agent/testdata/basic.hcl | 11 ++++++---- command/agent/testdata/basic.json | 5 ++++- nomad/config.go | 20 +++++++++++++++++ 8 files changed, 102 insertions(+), 21 deletions(-) diff --git a/command/agent/agent.go b/command/agent/agent.go index 0c8217d35..c1213bff9 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -241,6 +241,12 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { if agentConfig.ACL.ReplicationToken != "" { conf.ReplicationToken = agentConfig.ACL.ReplicationToken } + if agentConfig.ACL.TokenMinExpirationTTL != 0 { + conf.ACLTokenMinExpirationTTL = agentConfig.ACL.TokenMinExpirationTTL + } + if agentConfig.ACL.TokenMaxExpirationTTL != 0 { + conf.ACLTokenMaxExpirationTTL = agentConfig.ACL.TokenMaxExpirationTTL + } if agentConfig.Sentinel != nil { conf.SentinelConfig = agentConfig.Sentinel } @@ -377,6 +383,13 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { } conf.CSIPluginGCThreshold = dur } + if gcThreshold := agentConfig.Server.ACLTokenGCThreshold; gcThreshold != "" { + dur, err := time.ParseDuration(gcThreshold) + if err != nil { + return nil, err + } + conf.ACLTokenExpirationGCThreshold = dur + } if heartbeatGrace := agentConfig.Server.HeartbeatGrace; heartbeatGrace != 0 { conf.HeartbeatGrace = heartbeatGrace diff --git a/command/agent/config.go b/command/agent/config.go index 50e547592..649ca8829 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -355,6 +355,18 @@ type ACLConfig struct { // within the authoritative region. ReplicationToken string `hcl:"replication_token"` + // TokenMinExpirationTTL is used to enforce the lowest acceptable value for + // ACL token expiration. This is used by the Nomad servers to validate ACL + // tokens with an expiration value set upon creation. + TokenMinExpirationTTL time.Duration + TokenMinExpirationTTLHCL string `hcl:"token_min_expiration_ttl" json:"-"` + + // TokenMaxExpirationTTL is used to enforce the highest acceptable value + // for ACL token expiration. This is used by the Nomad servers to validate + // ACL tokens with an expiration value set upon creation. + TokenMaxExpirationTTL time.Duration + TokenMaxExpirationTTLHCL string `hcl:"token_max_expiration_ttl" json:"-"` + // ExtraKeysHCL is used by hcl to surface unexpected keys ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } @@ -417,7 +429,7 @@ type ServerConfig struct { EvalGCThreshold string `hcl:"eval_gc_threshold"` // DeploymentGCThreshold controls how "old" a deployment must be to be - // collected by GC. Age is not the only requirement for a deployment to be + // collected by GC. Age is not the only requirement for a deployment to be // GCed but the threshold can be used to filter by age. DeploymentGCThreshold string `hcl:"deployment_gc_threshold"` @@ -431,6 +443,10 @@ type ServerConfig struct { // GCed but the threshold can be used to filter by age. CSIPluginGCThreshold string `hcl:"csi_plugin_gc_threshold"` + // ACLTokenGCThreshold controls how "old" an expired ACL token must be to + // be collected by GC. + ACLTokenGCThreshold string `hcl:"acl_token_gc_threshold"` + // HeartbeatGrace is the grace period beyond the TTL to account for network, // processing delays and clock skew before marking a node as "down". HeartbeatGrace time.Duration @@ -932,7 +948,7 @@ func DevConfig(mode *devModeConfig) *Config { return conf } -// DefaultConfig is a the baseline configuration for Nomad +// DefaultConfig is the baseline configuration for Nomad. func DefaultConfig() *Config { return &Config{ LogLevel: "INFO", @@ -1470,6 +1486,18 @@ func (a *ACLConfig) Merge(b *ACLConfig) *ACLConfig { if b.PolicyTTLHCL != "" { result.PolicyTTLHCL = b.PolicyTTLHCL } + if b.TokenMinExpirationTTL != 0 { + result.TokenMinExpirationTTL = b.TokenMinExpirationTTL + } + if b.TokenMinExpirationTTLHCL != "" { + result.TokenMinExpirationTTLHCL = b.TokenMinExpirationTTLHCL + } + if b.TokenMaxExpirationTTL != 0 { + result.TokenMaxExpirationTTL = b.TokenMaxExpirationTTL + } + if b.TokenMaxExpirationTTLHCL != "" { + result.TokenMaxExpirationTTLHCL = b.TokenMaxExpirationTTLHCL + } if b.ReplicationToken != "" { result.ReplicationToken = b.ReplicationToken } @@ -1526,6 +1554,9 @@ func (s *ServerConfig) Merge(b *ServerConfig) *ServerConfig { if b.CSIPluginGCThreshold != "" { result.CSIPluginGCThreshold = b.CSIPluginGCThreshold } + if b.ACLTokenGCThreshold != "" { + result.ACLTokenGCThreshold = b.ACLTokenGCThreshold + } if b.HeartbeatGrace != 0 { result.HeartbeatGrace = b.HeartbeatGrace } diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index 363db9e1e..ea99cb36d 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -62,6 +62,8 @@ func ParseConfigFile(path string) (*Config, error) { {"gc_interval", &c.Client.GCInterval, &c.Client.GCIntervalHCL, nil}, {"acl.token_ttl", &c.ACL.TokenTTL, &c.ACL.TokenTTLHCL, nil}, {"acl.policy_ttl", &c.ACL.PolicyTTL, &c.ACL.PolicyTTLHCL, nil}, + {"acl.token_min_expiration_ttl", &c.ACL.TokenMinExpirationTTL, &c.ACL.TokenMinExpirationTTLHCL, nil}, + {"acl.token_max_expiration_ttl", &c.ACL.TokenMaxExpirationTTL, &c.ACL.TokenMaxExpirationTTLHCL, nil}, {"client.server_join.retry_interval", &c.Client.ServerJoin.RetryInterval, &c.Client.ServerJoin.RetryIntervalHCL, nil}, {"server.heartbeat_grace", &c.Server.HeartbeatGrace, &c.Server.HeartbeatGraceHCL, nil}, {"server.min_heartbeat_ttl", &c.Server.MinHeartbeatTTL, &c.Server.MinHeartbeatTTLHCL, nil}, diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index d2567c7fc..a1cbc0208 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -107,6 +107,7 @@ var basicConfig = &Config{ DeploymentGCThreshold: "12h", CSIVolumeClaimGCThreshold: "12h", CSIPluginGCThreshold: "12h", + ACLTokenGCThreshold: "12h", HeartbeatGrace: 30 * time.Second, HeartbeatGraceHCL: "30s", MinHeartbeatTTL: 33 * time.Second, @@ -143,12 +144,16 @@ var basicConfig = &Config{ LicensePath: "/tmp/nomad.hclic", }, ACL: &ACLConfig{ - Enabled: true, - TokenTTL: 60 * time.Second, - TokenTTLHCL: "60s", - PolicyTTL: 60 * time.Second, - PolicyTTLHCL: "60s", - ReplicationToken: "foobar", + Enabled: true, + TokenTTL: 60 * time.Second, + TokenTTLHCL: "60s", + PolicyTTL: 60 * time.Second, + PolicyTTLHCL: "60s", + TokenMinExpirationTTLHCL: "1h", + TokenMinExpirationTTL: 1 * time.Hour, + TokenMaxExpirationTTLHCL: "100h", + TokenMaxExpirationTTL: 100 * time.Hour, + ReplicationToken: "foobar", }, Audit: &config.AuditConfig{ Enabled: helper.BoolToPtr(true), diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 36e263a31..be29ada38 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -150,10 +150,12 @@ func TestConfig_Merge(t *testing.T) { EventBufferSize: helper.IntToPtr(0), }, ACL: &ACLConfig{ - Enabled: true, - TokenTTL: 60 * time.Second, - PolicyTTL: 60 * time.Second, - ReplicationToken: "foo", + Enabled: true, + TokenTTL: 60 * time.Second, + PolicyTTL: 60 * time.Second, + TokenMinExpirationTTL: 60 * time.Second, + TokenMaxExpirationTTL: 60 * time.Second, + ReplicationToken: "foo", }, Ports: &Ports{ HTTP: 4646, @@ -345,10 +347,12 @@ func TestConfig_Merge(t *testing.T) { EventBufferSize: helper.IntToPtr(100), }, ACL: &ACLConfig{ - Enabled: true, - TokenTTL: 20 * time.Second, - PolicyTTL: 20 * time.Second, - ReplicationToken: "foobar", + Enabled: true, + TokenTTL: 20 * time.Second, + PolicyTTL: 20 * time.Second, + TokenMinExpirationTTL: 20 * time.Second, + TokenMaxExpirationTTL: 20 * time.Second, + ReplicationToken: "foobar", }, Ports: &Ports{ HTTP: 20000, diff --git a/command/agent/testdata/basic.hcl b/command/agent/testdata/basic.hcl index 27754b957..700959c48 100644 --- a/command/agent/testdata/basic.hcl +++ b/command/agent/testdata/basic.hcl @@ -116,6 +116,7 @@ server { deployment_gc_threshold = "12h" csi_volume_claim_gc_threshold = "12h" csi_plugin_gc_threshold = "12h" + acl_token_gc_threshold = "12h" heartbeat_grace = "30s" min_heartbeat_ttl = "33s" max_heartbeats_per_second = 11.0 @@ -153,10 +154,12 @@ server { } acl { - enabled = true - token_ttl = "60s" - policy_ttl = "60s" - replication_token = "foobar" + enabled = true + token_ttl = "60s" + policy_ttl = "60s" + token_min_expiration_ttl = "1h" + token_max_expiration_ttl = "100h" + replication_token = "foobar" } audit { diff --git a/command/agent/testdata/basic.json b/command/agent/testdata/basic.json index 406e314a9..c60fc40db 100644 --- a/command/agent/testdata/basic.json +++ b/command/agent/testdata/basic.json @@ -4,7 +4,9 @@ "enabled": true, "policy_ttl": "60s", "replication_token": "foobar", - "token_ttl": "60s" + "token_ttl": "60s", + "token_min_expiration_ttl": "1h", + "token_max_expiration_ttl": "100h" } ], "audit": { @@ -254,6 +256,7 @@ ], "server": [ { + "acl_token_gc_threshold": "12h", "authoritative_region": "foobar", "bootstrap_expect": 5, "csi_plugin_gc_threshold": "12h", diff --git a/nomad/config.go b/nomad/config.go index 6ea8cfd09..8ca99723d 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -196,6 +196,14 @@ type Config struct { // one-time tokens. OneTimeTokenGCInterval time.Duration + // ACLTokenExpirationGCInterval is how often we dispatch a job to GC + // expired ACL tokens. + ACLTokenExpirationGCInterval time.Duration + + // ACLTokenExpirationGCThreshold controls how "old" an expired ACL token + // must be to be collected by GC. + ACLTokenExpirationGCThreshold time.Duration + // EvalNackTimeout controls how long we allow a sub-scheduler to // work on an evaluation before we consider it failed and Nack it. // This allows that evaluation to be handed to another sub-scheduler @@ -278,6 +286,14 @@ type Config struct { // the Authoritative Region. ReplicationToken string + // TokenMinExpirationTTL is used to enforce the lowest acceptable value for + // ACL token expiration. + ACLTokenMinExpirationTTL time.Duration + + // TokenMaxExpirationTTL is used to enforce the highest acceptable value + // for ACL token expiration. + ACLTokenMaxExpirationTTL time.Duration + // SentinelGCInterval is the interval that we GC unused policies. SentinelGCInterval time.Duration @@ -385,6 +401,8 @@ func DefaultConfig() *Config { CSIVolumeClaimGCInterval: 5 * time.Minute, CSIVolumeClaimGCThreshold: 5 * time.Minute, OneTimeTokenGCInterval: 10 * time.Minute, + ACLTokenExpirationGCInterval: 5 * time.Minute, + ACLTokenExpirationGCThreshold: 1 * time.Hour, EvalNackTimeout: 60 * time.Second, EvalDeliveryLimit: 3, EvalNackInitialReenqueueDelay: 1 * time.Second, @@ -405,6 +423,8 @@ func DefaultConfig() *Config { LicenseConfig: &LicenseConfig{}, EnableEventBroker: true, EventBufferSize: 100, + ACLTokenMinExpirationTTL: 1 * time.Minute, + ACLTokenMaxExpirationTTL: 24 * time.Hour, AutopilotConfig: &structs.AutopilotConfig{ CleanupDeadServers: true, LastContactThreshold: 200 * time.Millisecond, From 0cde3182ebc1b4308632b60a748bb746dea0d4e2 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 13 Jul 2022 15:40:34 +0200 Subject: [PATCH 02/19] core: add ACL token expiry state, struct, and RPC handling. (#13718) The ACL token state schema has been updated to utilise two new indexes which track expiration of tokens that are configured with an expiration TTL or time. A new state function allows listing ACL expired tokens which will be used by internal garbage collection. The ACL endpoint has been modified so that all validation happens within a single function call. This is easier to understand and see at a glance. The ACL token validation now also includes logic for expiry TTL and times. The ACL endpoint upsert tests have been condensed into a single, table driven test. There is a new token canonicalize which provides a single place for token canonicalization, rather than logic spread in the RPC handler. --- helper/funcs.go | 2 + nomad/acl_endpoint.go | 56 +++--- nomad/acl_endpoint_test.go | 239 ++++++++++++++++------- nomad/state/indexer/indexer.go | 78 ++++++++ nomad/state/indexer/indexer_test.go | 15 ++ nomad/state/indexer/time.go | 25 +++ nomad/state/indexer/time_test.go | 45 +++++ nomad/state/schema.go | 64 ++++++- nomad/state/state_store_acl.go | 59 ++++++ nomad/state/state_store_acl_test.go | 140 ++++++++++++++ nomad/structs/acl.go | 147 ++++++++++++++ nomad/structs/acl_test.go | 287 ++++++++++++++++++++++++++++ nomad/structs/structs.go | 91 +++++---- nomad/structs/structs_test.go | 47 ----- 14 files changed, 1094 insertions(+), 201 deletions(-) create mode 100644 nomad/state/indexer/indexer.go create mode 100644 nomad/state/indexer/indexer_test.go create mode 100644 nomad/state/indexer/time.go create mode 100644 nomad/state/indexer/time_test.go create mode 100644 nomad/state/state_store_acl.go create mode 100644 nomad/state/state_store_acl_test.go create mode 100644 nomad/structs/acl.go create mode 100644 nomad/structs/acl_test.go diff --git a/helper/funcs.go b/helper/funcs.go index 665017222..76e9dd56d 100644 --- a/helper/funcs.go +++ b/helper/funcs.go @@ -126,6 +126,8 @@ func StringToPtr(str string) *string { } // TimeToPtr returns the pointer to a time.Duration. +// +// Deprecated; use pointer.Of instead. func TimeToPtr(t time.Duration) *time.Duration { return &t } diff --git a/nomad/acl_endpoint.go b/nomad/acl_endpoint.go index 23a9e9913..0ca6427e9 100644 --- a/nomad/acl_endpoint.go +++ b/nomad/acl_endpoint.go @@ -468,7 +468,7 @@ func (a *ACL) UpsertTokens(args *structs.ACLTokenUpsertRequest, reply *structs.A // Validate non-zero set of tokens if len(args.Tokens) == 0 { - return structs.NewErrRPCCoded(400, "must specify as least one token") + return structs.NewErrRPCCoded(http.StatusBadRequest, "must specify as least one token") } // Force the request to the authoritative region if we are creating global tokens @@ -486,14 +486,15 @@ func (a *ACL) UpsertTokens(args *structs.ACLTokenUpsertRequest, reply *structs.A // the entire request as a single batch. if hasGlobal { if !allGlobal { - return structs.NewErrRPCCoded(400, "cannot upsert mixed global and non-global tokens") + return structs.NewErrRPCCoded(http.StatusBadRequest, + "cannot upsert mixed global and non-global tokens") } // Force the request to the authoritative region if it has global args.Region = a.srv.config.AuthoritativeRegion } - if done, err := a.srv.forward("ACL.UpsertTokens", args, args, reply); done { + if done, err := a.srv.forward(structs.ACLUpsertTokensRPCMethod, args, args, reply); done { return err } defer metrics.MeasureSince([]string{"nomad", "acl", "upsert_tokens"}, time.Now()) @@ -505,38 +506,41 @@ func (a *ACL) UpsertTokens(args *structs.ACLTokenUpsertRequest, reply *structs.A return structs.ErrPermissionDenied } - // Snapshot the state - state, err := a.srv.State().Snapshot() + // Snapshot the state so we can perform lookups against the accessor ID if + // needed. Do it here, so we only need to do this once no matter how many + // tokens we are upserting. + stateSnapshot, err := a.srv.State().Snapshot() if err != nil { return err } // Validate each token for idx, token := range args.Tokens { - if err := token.Validate(); err != nil { - return structs.NewErrRPCCodedf(400, "token %d invalid: %v", idx, err) - } - // Generate an accessor and secret ID if new - if token.AccessorID == "" { - token.AccessorID = uuid.Generate() - token.SecretID = uuid.Generate() - token.CreateTime = time.Now().UTC() + // Store any existing token found, so we can perform the correct update + // validation. + var existingToken *structs.ACLToken - } else { - // Verify the token exists - out, err := state.ACLTokenByAccessorID(nil, token.AccessorID) + // If the token is being updated, perform a lookup so can can validate + // the new changes against the old. + if token.AccessorID != "" { + out, err := stateSnapshot.ACLTokenByAccessorID(nil, token.AccessorID) if err != nil { - return structs.NewErrRPCCodedf(400, "token lookup failed: %v", err) + return structs.NewErrRPCCodedf(http.StatusBadRequest, "token lookup failed: %v", err) } if out == nil { - return structs.NewErrRPCCodedf(404, "cannot find token %s", token.AccessorID) + return structs.NewErrRPCCodedf(http.StatusBadRequest, "cannot find token %s", token.AccessorID) } + existingToken = out + } - // Cannot toggle the "Global" mode - if token.Global != out.Global { - return structs.NewErrRPCCodedf(400, "cannot toggle global mode of %s", token.AccessorID) - } + // Canonicalize sets information needed by the validation function, so + // this order must be maintained. + token.Canonicalize() + + if err := token.Validate(a.srv.config.ACLTokenMinExpirationTTL, + a.srv.config.ACLTokenMaxExpirationTTL, existingToken); err != nil { + return structs.NewErrRPCCodedf(http.StatusBadRequest, "token %d invalid: %v", idx, err) } // Compute the token hash @@ -549,14 +553,14 @@ func (a *ACL) UpsertTokens(args *structs.ACLTokenUpsertRequest, reply *structs.A return err } - // Populate the response. We do a lookup against the state to - // pickup the proper create / modify times. - state, err = a.srv.State().Snapshot() + // Populate the response. We do a lookup against the state to pick up the + // proper create / modify times. + stateSnapshot, err = a.srv.State().Snapshot() if err != nil { return err } for _, token := range args.Tokens { - out, err := state.ACLTokenByAccessorID(nil, token.AccessorID) + out, err := stateSnapshot.ACLTokenByAccessorID(nil, token.AccessorID) if err != nil { return structs.NewErrRPCCodedf(400, "token lookup failed: %v", err) } diff --git a/nomad/acl_endpoint_test.go b/nomad/acl_endpoint_test.go index 5a9785d40..87884d9bf 100644 --- a/nomad/acl_endpoint_test.go +++ b/nomad/acl_endpoint_test.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test/must" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1453,85 +1454,175 @@ func TestACLEndpoint_Bootstrap_Reset(t *testing.T) { func TestACLEndpoint_UpsertTokens(t *testing.T) { ci.Parallel(t) - s1, root, cleanupS1 := TestACLServer(t, nil) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) + // Each sub-test uses the same server to avoid creating a new one for each + // test. This means some care has to be taken with resource naming, but + // does avoid lots of calls to systems such as freeport. + testServer, rootACLToken, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + codec := rpcClient(t, testServer) + testutil.WaitForLeader(t, testServer.RPC) - // Create the register request - p1 := mock.ACLToken() - p1.AccessorID = "" // Blank to create + testCases := []struct { + name string + testFn func(testServer *Server, aclToken *structs.ACLToken) + }{ + { + name: "valid client token", + testFn: func(testServer *Server, aclToken *structs.ACLToken) { - // Lookup the tokens - req := &structs.ACLTokenUpsertRequest{ - Tokens: []*structs.ACLToken{p1}, - WriteRequest: structs.WriteRequest{ - Region: "global", - AuthToken: root.SecretID, + // Create the register request with a mocked token. We must set + // an empty accessorID, otherwise Nomad treats this as an + // update request. + p1 := mock.ACLToken() + p1.AccessorID = "" + + req := &structs.ACLTokenUpsertRequest{ + Tokens: []*structs.ACLToken{p1}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + var resp structs.ACLTokenUpsertResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, structs.ACLUpsertTokensRPCMethod, req, &resp)) + must.Greater(t, resp.Index, 0) + + // Get the token out from the response. + created := resp.Tokens[0] + require.NotEqual(t, "", created.AccessorID) + require.NotEqual(t, "", created.SecretID) + require.NotEqual(t, time.Time{}, created.CreateTime) + require.Equal(t, p1.Type, created.Type) + require.Equal(t, p1.Policies, created.Policies) + require.Equal(t, p1.Name, created.Name) + + // Check we created the token. + out, err := testServer.fsm.State().ACLTokenByAccessorID(nil, created.AccessorID) + require.Nil(t, err) + require.Equal(t, created, out) + + // Update the token type and policy list so we can try updating + // it. + req.Tokens[0] = created + created.Type = "management" + created.Policies = nil + + // Track the first upsert index, so we can test the next + // response against this and perform the update. + originalIndex := resp.Index + + require.NoError(t, msgpackrpc.CallWithCodec(codec, structs.ACLUpsertTokensRPCMethod, req, &resp)) + require.Greater(t, resp.Index, originalIndex) + + // Read the token from state and perform an equality check to + // ensure everything matches as we expect. + out, err = testServer.fsm.State().ACLTokenByAccessorID(nil, created.AccessorID) + require.Nil(t, err) + require.Equal(t, created, out) + }, + }, + { + name: "valid management token with expiration", + testFn: func(testServer *Server, aclToken *structs.ACLToken) { + + // Create our RPC request object which includes a management + // token with a TTL. + req := &structs.ACLTokenUpsertRequest{ + Tokens: []*structs.ACLToken{ + { + Name: "my-management-token-" + uuid.Generate(), + Type: structs.ACLManagementToken, + ExpirationTTL: 10 * time.Minute, + }, + }, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + + // Send the RPC request and ensure the expiration time is as + // expected. + var resp structs.ACLTokenUpsertResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, structs.ACLUpsertTokensRPCMethod, req, &resp)) + require.Equal(t, 10*time.Minute, resp.Tokens[0].ExpirationTime.Sub(resp.Tokens[0].CreateTime)) + }, + }, + { + name: "valid client token with expiration", + testFn: func(testServer *Server, aclToken *structs.ACLToken) { + + // Create an ACL policy so this can be associated to our client + // token. + policyReq := &structs.ACLPolicyUpsertRequest{ + Policies: []*structs.ACLPolicy{mock.ACLPolicy()}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + + var policyResp structs.GenericResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, structs.ACLUpsertPoliciesRPCMethod, policyReq, &policyResp)) + + // Create our RPC request object which includes a client token + // with a TTL that is associated to policies above. + tokenReq := &structs.ACLTokenUpsertRequest{ + Tokens: []*structs.ACLToken{ + { + Name: "my-client-token-" + uuid.Generate(), + Type: structs.ACLClientToken, + Policies: []string{policyReq.Policies[0].Name}, + ExpirationTTL: 10 * time.Minute, + }, + }, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + + // Send the RPC request and ensure the expiration time is as + // expected. + var tokenResp structs.ACLTokenUpsertResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, structs.ACLUpsertTokensRPCMethod, tokenReq, &tokenResp)) + require.Equal(t, 10*time.Minute, tokenResp.Tokens[0].ExpirationTime.Sub(tokenResp.Tokens[0].CreateTime)) + }, + }, + { + name: "invalid token type", + testFn: func(testServer *Server, aclToken *structs.ACLToken) { + + // Create our RPC request object which includes a token with an + // unknown type. This allows us to ensure the RPC handler calls + // the validation func. + tokenReq := &structs.ACLTokenUpsertRequest{ + Tokens: []*structs.ACLToken{ + { + Name: "my-blah-token-" + uuid.Generate(), + Type: "blah", + }, + }, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + + // Send the RPC request and ensure the expiration time is as + // expected. + var tokenResp structs.ACLTokenUpsertResponse + err := msgpackrpc.CallWithCodec(codec, structs.ACLUpsertTokensRPCMethod, tokenReq, &tokenResp) + require.ErrorContains(t, err, "token type must be client or management") + require.Empty(t, tokenResp.Tokens) + }, }, } - var resp structs.ACLTokenUpsertResponse - if err := msgpackrpc.CallWithCodec(codec, "ACL.UpsertTokens", req, &resp); err != nil { - t.Fatalf("err: %v", err) - } - assert.NotEqual(t, uint64(0), resp.Index) - // Get the token out from the response - created := resp.Tokens[0] - assert.NotEqual(t, "", created.AccessorID) - assert.NotEqual(t, "", created.SecretID) - assert.NotEqual(t, time.Time{}, created.CreateTime) - assert.Equal(t, p1.Type, created.Type) - assert.Equal(t, p1.Policies, created.Policies) - assert.Equal(t, p1.Name, created.Name) - - // Check we created the token - out, err := s1.fsm.State().ACLTokenByAccessorID(nil, created.AccessorID) - assert.Nil(t, err) - assert.Equal(t, created, out) - - // Update the token type - req.Tokens[0] = created - created.Type = "management" - created.Policies = nil - - // Upsert again - if err := msgpackrpc.CallWithCodec(codec, "ACL.UpsertTokens", req, &resp); err != nil { - t.Fatalf("err: %v", err) - } - assert.NotEqual(t, uint64(0), resp.Index) - - // Check we modified the token - out, err = s1.fsm.State().ACLTokenByAccessorID(nil, created.AccessorID) - assert.Nil(t, err) - assert.Equal(t, created, out) -} - -func TestACLEndpoint_UpsertTokens_Invalid(t *testing.T) { - ci.Parallel(t) - - s1, root, cleanupS1 := TestACLServer(t, nil) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Create the register request - p1 := mock.ACLToken() - p1.Type = "blah blah" - - // Lookup the tokens - req := &structs.ACLTokenUpsertRequest{ - Tokens: []*structs.ACLToken{p1}, - WriteRequest: structs.WriteRequest{ - Region: "global", - AuthToken: root.SecretID, - }, - } - var resp structs.GenericResponse - err := msgpackrpc.CallWithCodec(codec, "ACL.UpsertTokens", req, &resp) - assert.NotNil(t, err) - if !strings.Contains(err.Error(), "client or management") { - t.Fatalf("bad: %s", err) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn(testServer, rootACLToken) + }) } } diff --git a/nomad/state/indexer/indexer.go b/nomad/state/indexer/indexer.go new file mode 100644 index 000000000..3cdc8e93f --- /dev/null +++ b/nomad/state/indexer/indexer.go @@ -0,0 +1,78 @@ +package indexer + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "time" + + "github.com/hashicorp/go-memdb" +) + +var ( + // Ensure the required memdb interfaces are met at compile time. + _ memdb.Indexer = SingleIndexer{} + _ memdb.SingleIndexer = SingleIndexer{} +) + +// SingleIndexer implements both memdb.Indexer and memdb.SingleIndexer. It may +// be used in a memdb.IndexSchema to specify functions that generate the index +// value for memdb.Txn operations. +type SingleIndexer struct { + + // readIndex is used by memdb for Txn.Get, Txn.First, and other operations + // that read data. + ReadIndex + + // writeIndex is used by memdb for Txn.Insert, Txn.Delete, and other + // operations that write data to the index. + WriteIndex +} + +// ReadIndex implements memdb.Indexer. It exists so that a function can be used +// to provide the interface. +// +// Unlike memdb.Indexer, a readIndex function accepts only a single argument. To +// generate an index from multiple values, use a struct type with multiple fields. +type ReadIndex func(arg any) ([]byte, error) + +func (f ReadIndex) FromArgs(args ...interface{}) ([]byte, error) { + if len(args) != 1 { + return nil, fmt.Errorf("index supports only a single arg") + } + return f(args[0]) +} + +var ErrMissingValueForIndex = fmt.Errorf("object is missing a value for this index") + +// WriteIndex implements memdb.SingleIndexer. It exists so that a function +// can be used to provide this interface. +// +// Instead of a bool return value, writeIndex expects errMissingValueForIndex to +// indicate that an index could not be build for the object. It will translate +// this error into a false value to satisfy the memdb.SingleIndexer interface. +type WriteIndex func(raw any) ([]byte, error) + +func (f WriteIndex) FromObject(raw any) (bool, []byte, error) { + v, err := f(raw) + if errors.Is(err, ErrMissingValueForIndex) { + return false, nil, nil + } + return err == nil, v, err +} + +// IndexBuilder is a buffer used to construct memdb index values. +type IndexBuilder bytes.Buffer + +// Bytes returns the stored IndexBuilder value as a byte array. +func (b *IndexBuilder) Bytes() []byte { return (*bytes.Buffer)(b).Bytes() } + +// Time is used to write the passed time into the IndexBuilder for use as a +// memdb index value. +func (b *IndexBuilder) Time(t time.Time) { + val := t.Unix() + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(val)) + (*bytes.Buffer)(b).Write(buf) +} diff --git a/nomad/state/indexer/indexer_test.go b/nomad/state/indexer/indexer_test.go new file mode 100644 index 000000000..5d01c2608 --- /dev/null +++ b/nomad/state/indexer/indexer_test.go @@ -0,0 +1,15 @@ +package indexer + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_IndexBuilder_Time(t *testing.T) { + builder := &IndexBuilder{} + testTime := time.Date(1987, time.April, 13, 8, 3, 0, 0, time.UTC) + builder.Time(testTime) + require.Equal(t, []byte{0, 0, 0, 0, 32, 128, 155, 180}, builder.Bytes()) +} diff --git a/nomad/state/indexer/time.go b/nomad/state/indexer/time.go new file mode 100644 index 000000000..5daa71d5a --- /dev/null +++ b/nomad/state/indexer/time.go @@ -0,0 +1,25 @@ +package indexer + +import ( + "fmt" + "time" +) + +type TimeQuery struct { + Value time.Time +} + +// IndexFromTimeQuery can be used as a memdb.Indexer query via ReadIndex and +// allows querying by time. +func IndexFromTimeQuery(arg any) ([]byte, error) { + p, ok := arg.(*TimeQuery) + if !ok { + return nil, fmt.Errorf("unexpected type %T for TimeQuery index", arg) + } + + // Construct the index value and return the byte array representation of + // the time value. + var b IndexBuilder + b.Time(p.Value) + return b.Bytes(), nil +} diff --git a/nomad/state/indexer/time_test.go b/nomad/state/indexer/time_test.go new file mode 100644 index 000000000..77288d100 --- /dev/null +++ b/nomad/state/indexer/time_test.go @@ -0,0 +1,45 @@ +package indexer + +import ( + "testing" + "time" + + "github.com/hashicorp/nomad/ci" + "github.com/stretchr/testify/require" +) + +func Test_IndexFromTimeQuery(t *testing.T) { + ci.Parallel(t) + + testCases := []struct { + inputArg interface{} + expectedOutputBytes []byte + expectedOutputError error + name string + }{ + { + inputArg: &TimeQuery{ + Value: time.Date(1987, time.April, 13, 8, 3, 0, 0, time.UTC), + }, + expectedOutputBytes: []byte{0x0, 0x0, 0x0, 0x0, 0x20, 0x80, 0x9b, 0xb4}, + expectedOutputError: nil, + name: "generic test 1", + }, + { + inputArg: &TimeQuery{ + Value: time.Date(2022, time.April, 27, 14, 12, 0, 0, time.UTC), + }, + expectedOutputBytes: []byte{0x0, 0x0, 0x0, 0x0, 0x62, 0x69, 0x4f, 0x30}, + expectedOutputError: nil, + name: "generic test 2", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput, actualError := IndexFromTimeQuery(tc.inputArg) + require.Equal(t, tc.expectedOutputError, actualError) + require.Equal(t, tc.expectedOutputBytes, actualOutput) + }) + } +} diff --git a/nomad/state/schema.go b/nomad/state/schema.go index 50c9fb20f..182237e1e 100644 --- a/nomad/state/schema.go +++ b/nomad/state/schema.go @@ -5,7 +5,7 @@ import ( "sync" memdb "github.com/hashicorp/go-memdb" - + "github.com/hashicorp/nomad/nomad/state/indexer" "github.com/hashicorp/nomad/nomad/structs" ) @@ -17,11 +17,13 @@ const ( ) const ( - indexID = "id" - indexJob = "job" - indexNodeID = "node_id" - indexAllocID = "alloc_id" - indexServiceName = "service_name" + indexID = "id" + indexJob = "job" + indexNodeID = "node_id" + indexAllocID = "alloc_id" + indexServiceName = "service_name" + indexExpiresGlobal = "expires-global" + indexExpiresLocal = "expires-local" ) var ( @@ -816,10 +818,60 @@ func aclTokenTableSchema() *memdb.TableSchema { Field: "Global", }, }, + indexExpiresGlobal: { + Name: indexExpiresGlobal, + AllowMissing: true, + Unique: false, + Indexer: indexer.SingleIndexer{ + ReadIndex: indexer.ReadIndex(indexer.IndexFromTimeQuery), + WriteIndex: indexer.WriteIndex(indexExpiresGlobalFromACLToken), + }, + }, + indexExpiresLocal: { + Name: indexExpiresLocal, + AllowMissing: true, + Unique: false, + Indexer: indexer.SingleIndexer{ + ReadIndex: indexer.ReadIndex(indexer.IndexFromTimeQuery), + WriteIndex: indexer.WriteIndex(indexExpiresLocalFromACLToken), + }, + }, }, } } +func indexExpiresLocalFromACLToken(raw interface{}) ([]byte, error) { + return indexExpiresFromACLToken(raw, false) +} + +func indexExpiresGlobalFromACLToken(raw interface{}) ([]byte, error) { + return indexExpiresFromACLToken(raw, true) +} + +// indexExpiresFromACLToken implements the indexer.WriteIndex interface and +// allows us to use an ACL tokens ExpirationTime as an index, if it is a +// non-default value. This allows for efficient lookups when trying to deal +// with removal of expired tokens from state. +func indexExpiresFromACLToken(raw interface{}, global bool) ([]byte, error) { + p, ok := raw.(*structs.ACLToken) + if !ok { + return nil, fmt.Errorf("unexpected type %T for structs.ACLToken index", raw) + } + if p.Global != global { + return nil, indexer.ErrMissingValueForIndex + } + if !p.HasExpirationTime() { + return nil, indexer.ErrMissingValueForIndex + } + if p.ExpirationTime.Unix() < 0 { + return nil, fmt.Errorf("token expiration time cannot be before the unix epoch: %s", p.ExpirationTime) + } + + var b indexer.IndexBuilder + b.Time(*p.ExpirationTime) + return b.Bytes(), nil +} + // oneTimeTokenTableSchema returns the MemDB schema for the tokens table. // This table is used to store one-time tokens for ACL tokens func oneTimeTokenTableSchema() *memdb.TableSchema { diff --git a/nomad/state/state_store_acl.go b/nomad/state/state_store_acl.go new file mode 100644 index 000000000..78aab4403 --- /dev/null +++ b/nomad/state/state_store_acl.go @@ -0,0 +1,59 @@ +package state + +import ( + "fmt" + "time" + + "github.com/hashicorp/nomad/nomad/structs" +) + +// ACLTokensByExpired returns an array accessor IDs of expired ACL tokens. +// Their expiration is determined against the passed time.Time value. +// +// The function handles global and local tokens independently as determined by +// the global boolean argument. The number of returned IDs can be limited by +// the max integer, which is useful to limit the number of tokens we attempt to +// delete in a single transaction. +func (s *StateStore) ACLTokensByExpired(global bool, now time.Time, max int) ([]string, error) { + tnx := s.db.ReadTxn() + + iter, err := tnx.Get("acl_token", expiresIndexName(global)) + if err != nil { + return nil, fmt.Errorf("failed acl token listing: %v", err) + } + + var ( + accessorIDs []string + num int + ) + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + token := raw.(*structs.ACLToken) + + // The indexes mean if we come across an unexpired token, we can exit + // as we have found all currently expired tokens. + if !token.IsExpired(now) { + return accessorIDs, nil + } + + accessorIDs = append(accessorIDs, token.AccessorID) + + // Increment the counter. If this is at or above our limit, we return + // what we have so far. + num++ + if num >= max { + return accessorIDs, nil + } + } + + return accessorIDs, nil +} + +// expiresIndexName is a helper function to identify the correct ACL token +// table expiry index to use. +func expiresIndexName(global bool) string { + if global { + return indexExpiresGlobal + } + return indexExpiresLocal +} diff --git a/nomad/state/state_store_acl_test.go b/nomad/state/state_store_acl_test.go new file mode 100644 index 000000000..8974489a1 --- /dev/null +++ b/nomad/state/state_store_acl_test.go @@ -0,0 +1,140 @@ +package state + +import ( + "testing" + "time" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/helper/pointer" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" +) + +func TestStateStore_ACLTokensByExpired(t *testing.T) { + ci.Parallel(t) + testState := testStateStore(t) + + // This time is the threshold for all expiry calls to be based on. All + // tokens with expiry can use this as their base and use Add(). + expiryTimeThreshold := time.Date(2022, time.April, 27, 14, 50, 0, 0, time.UTC) + + // Generate two tokens without an expiry time. These tokens should never + // show up in calls to ACLTokensByExpired. + neverExpireLocalToken := mock.ACLToken() + neverExpireGlobalToken := mock.ACLToken() + neverExpireLocalToken.Global = true + + // Upsert the tokens into state and perform a global and local read of + // the state. + err := testState.UpsertACLTokens(structs.MsgTypeTestSetup, 10, []*structs.ACLToken{ + neverExpireLocalToken, neverExpireGlobalToken}) + require.NoError(t, err) + + ids, err := testState.ACLTokensByExpired(true, expiryTimeThreshold, 10) + require.NoError(t, err) + require.Len(t, ids, 0) + + ids, err = testState.ACLTokensByExpired(false, expiryTimeThreshold, 10) + require.NoError(t, err) + require.Len(t, ids, 0) + + // Generate, upsert, and test an expired local token. This token expired + // long ago and therefore before all others coming in the tests. It should + // therefore always be the first out. + expiredLocalToken := mock.ACLToken() + expiredLocalToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(-48 * time.Hour)) + + err = testState.UpsertACLTokens(structs.MsgTypeTestSetup, 20, []*structs.ACLToken{expiredLocalToken}) + require.NoError(t, err) + + ids, err = testState.ACLTokensByExpired(false, expiryTimeThreshold, 10) + require.NoError(t, err) + require.Len(t, ids, 1) + require.Equal(t, expiredLocalToken.AccessorID, ids[0]) + + // Generate, upsert, and test an expired global token. This token expired + // long ago and therefore before all others coming in the tests. It should + // therefore always be the first out. + expiredGlobalToken := mock.ACLToken() + expiredGlobalToken.Global = true + expiredGlobalToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(-48 * time.Hour)) + + err = testState.UpsertACLTokens(structs.MsgTypeTestSetup, 30, []*structs.ACLToken{expiredGlobalToken}) + require.NoError(t, err) + + ids, err = testState.ACLTokensByExpired(true, expiryTimeThreshold, 10) + require.NoError(t, err) + require.Len(t, ids, 1) + require.Equal(t, expiredGlobalToken.AccessorID, ids[0]) + + // This test function allows us to run the same test for local and global + // tokens. + testFn := func(oldID string, global bool) { + + // Track all the expected expired accessor IDs including the long + // expired token. + var expiredLocalAccessorIDs []string + expiredLocalAccessorIDs = append(expiredLocalAccessorIDs, oldID) + + // Generate and upsert a number of mixed expired, non-expired local tokens. + mixedLocalTokens := make([]*structs.ACLToken, 20) + for i := 0; i < 20; i++ { + mockedToken := mock.ACLToken() + mockedToken.Global = global + if i%2 == 0 { + expiredLocalAccessorIDs = append(expiredLocalAccessorIDs, mockedToken.AccessorID) + mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(-24 * time.Hour)) + } else { + mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(24 * time.Hour)) + } + mixedLocalTokens[i] = mockedToken + } + + err = testState.UpsertACLTokens(structs.MsgTypeTestSetup, 40, mixedLocalTokens) + require.NoError(t, err) + + // Use a max value higher than the number we have to check the full listing + // works as expected. Ensure our oldest expired token is first in the list. + ids, err = testState.ACLTokensByExpired(global, expiryTimeThreshold, 100) + require.NoError(t, err) + require.ElementsMatch(t, ids, expiredLocalAccessorIDs) + require.Equal(t, ids[0], oldID) + + // Use a lower max value than the number of known expired tokens to ensure + // this is working. + ids, err = testState.ACLTokensByExpired(global, expiryTimeThreshold, 3) + require.NoError(t, err) + require.Len(t, ids, 3) + require.Equal(t, ids[0], oldID) + } + + testFn(expiredLocalToken.AccessorID, false) + testFn(expiredGlobalToken.AccessorID, true) +} + +func Test_expiresIndexName(t *testing.T) { + testCases := []struct { + globalInput bool + expectedOutput string + name string + }{ + { + globalInput: false, + expectedOutput: indexExpiresLocal, + name: "local", + }, + { + globalInput: true, + expectedOutput: indexExpiresGlobal, + name: "global", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := expiresIndexName(tc.globalInput) + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} diff --git a/nomad/structs/acl.go b/nomad/structs/acl.go new file mode 100644 index 000000000..6d4d4b387 --- /dev/null +++ b/nomad/structs/acl.go @@ -0,0 +1,147 @@ +package structs + +import ( + "errors" + "fmt" + "time" + + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/nomad/helper/pointer" + "github.com/hashicorp/nomad/helper/uuid" +) + +const ( + // ACLUpsertPoliciesRPCMethod is the RPC method for batch creating or + // modifying ACL policies. + // + // Args: ACLPolicyUpsertRequest + // Reply: GenericResponse + ACLUpsertPoliciesRPCMethod = "ACL.UpsertPolicies" + + // ACLUpsertTokensRPCMethod is the RPC method for batch creating or + // modifying ACL tokens. + // + // Args: ACLTokenUpsertRequest + // Reply: ACLTokenUpsertResponse + ACLUpsertTokensRPCMethod = "ACL.UpsertTokens" +) + +// Canonicalize performs basic canonicalization on the ACL token object. It is +// important for callers to understand certain fields such as AccessorID are +// set if it is empty, so copies should be taken if needed before calling this +// function. +func (a *ACLToken) Canonicalize() { + + // If the accessor ID is empty, it means this is creation of a new token, + // therefore we need to generate base information. + if a.AccessorID == "" { + + a.AccessorID = uuid.Generate() + a.SecretID = uuid.Generate() + a.CreateTime = time.Now().UTC() + + // If the user has not set the expiration time, but has provided a TTL, we + // calculate and populate the former filed. + if a.ExpirationTime == nil && a.ExpirationTTL != 0 { + a.ExpirationTime = pointer.Of(a.CreateTime.Add(a.ExpirationTTL)) + } + } +} + +// Validate is used to check a token for reasonableness +func (a *ACLToken) Validate(minTTL, maxTTL time.Duration, existing *ACLToken) error { + var mErr multierror.Error + + // The human friendly name of an ACL token cannot exceed 256 characters. + if len(a.Name) > maxTokenNameLength { + mErr.Errors = append(mErr.Errors, errors.New("token name too long")) + } + + // The type of an ACL token must be set. An ACL token of type client must + // have associated policies, whereas a management token cannot be + // associated with policies. + switch a.Type { + case ACLClientToken: + if len(a.Policies) == 0 { + mErr.Errors = append(mErr.Errors, errors.New("client token missing policies")) + } + case ACLManagementToken: + if len(a.Policies) != 0 { + mErr.Errors = append(mErr.Errors, errors.New("management token cannot be associated with policies")) + } + default: + mErr.Errors = append(mErr.Errors, errors.New("token type must be client or management")) + } + + // There are different validation rules depending on whether the ACL token + // is being created or updated. + switch existing { + case nil: + if a.ExpirationTTL < 0 { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("token expiration TTL '%s' should not be negative", a.ExpirationTTL)) + } + + if a.ExpirationTime != nil && !a.ExpirationTime.IsZero() { + + if a.CreateTime.After(*a.ExpirationTime) { + mErr.Errors = append(mErr.Errors, errors.New("expiration time cannot be before create time")) + } + + // Create a time duration which details the time-til-expiry, so we can + // check this against the regions max and min values. + expiresIn := a.ExpirationTime.Sub(a.CreateTime) + if expiresIn > maxTTL { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("expiration time cannot be more than %s in the future (was %s)", + maxTTL, expiresIn)) + + } else if expiresIn < minTTL { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("expiration time cannot be less than %s in the future (was %s)", + minTTL, expiresIn)) + } + } + default: + if existing.Global != a.Global { + mErr.Errors = append(mErr.Errors, errors.New("cannot toggle global mode")) + } + if existing.ExpirationTTL != a.ExpirationTTL { + mErr.Errors = append(mErr.Errors, errors.New("cannot update expiration TTL")) + } + if existing.ExpirationTime != a.ExpirationTime { + mErr.Errors = append(mErr.Errors, errors.New("cannot update expiration time")) + } + } + + return mErr.ErrorOrNil() +} + +// HasExpirationTime checks whether the ACL token has an expiration time value +// set. +func (a *ACLToken) HasExpirationTime() bool { + if a == nil || a.ExpirationTime == nil { + return false + } + return !a.ExpirationTime.IsZero() +} + +// IsExpired compares the ACLToken.ExpirationTime against the passed t to +// identify whether the token is considered expired. The function can be called +// without checking whether the ACL token has an expiry time. +func (a *ACLToken) IsExpired(t time.Time) bool { + + // Check the token has an expiration time before potentially modifying the + // supplied time. This allows us to avoid extra work, if it isn't needed. + if !a.HasExpirationTime() { + return false + } + + // Check and ensure the time location is set to UTC. This is vital for + // consistency with multi-region global tokens. + if t.Location() != time.UTC { + t = t.UTC() + } + + return a.ExpirationTime.Before(t) || t.IsZero() +} diff --git a/nomad/structs/acl_test.go b/nomad/structs/acl_test.go new file mode 100644 index 000000000..3e58f1df8 --- /dev/null +++ b/nomad/structs/acl_test.go @@ -0,0 +1,287 @@ +package structs + +import ( + "testing" + "time" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/helper/pointer" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/stretchr/testify/require" +) + +func TestACLToken_Canonicalize(t *testing.T) { + testCases := []struct { + name string + testFn func() + }{ + { + name: "token with accessor", + testFn: func() { + mockToken := &ACLToken{ + AccessorID: uuid.Generate(), + SecretID: uuid.Generate(), + Name: "my cool token " + uuid.Generate(), + Type: "client", + Policies: []string{"foo", "bar"}, + Global: false, + CreateTime: time.Now().UTC(), + CreateIndex: 10, + ModifyIndex: 20, + } + mockToken.SetHash() + copiedMockToken := mockToken.Copy() + + mockToken.Canonicalize() + require.Equal(t, copiedMockToken, mockToken) + }, + }, + { + name: "token without accessor", + testFn: func() { + mockToken := &ACLToken{ + Name: "my cool token " + uuid.Generate(), + Type: "client", + Policies: []string{"foo", "bar"}, + Global: false, + } + + mockToken.Canonicalize() + require.NotEmpty(t, mockToken.AccessorID) + require.NotEmpty(t, mockToken.SecretID) + require.NotEmpty(t, mockToken.CreateTime) + }, + }, + { + name: "token with ttl without accessor", + testFn: func() { + mockToken := &ACLToken{ + Name: "my cool token " + uuid.Generate(), + Type: "client", + Policies: []string{"foo", "bar"}, + Global: false, + ExpirationTTL: 10 * time.Hour, + } + + mockToken.Canonicalize() + require.NotEmpty(t, mockToken.AccessorID) + require.NotEmpty(t, mockToken.SecretID) + require.NotEmpty(t, mockToken.CreateTime) + require.NotEmpty(t, mockToken.ExpirationTime) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn() + }) + } +} + +func TestACLTokenValidate(t *testing.T) { + ci.Parallel(t) + + testCases := []struct { + name string + inputACLToken *ACLToken + inputExistingACLToken *ACLToken + expectedErrorContains string + }{ + { + name: "missing type", + inputACLToken: &ACLToken{}, + inputExistingACLToken: nil, + expectedErrorContains: "client or management", + }, + { + name: "missing policies", + inputACLToken: &ACLToken{ + Type: ACLClientToken, + }, + inputExistingACLToken: nil, + expectedErrorContains: "missing policies", + }, + { + name: "invalid policies", + inputACLToken: &ACLToken{ + Type: ACLManagementToken, + Policies: []string{"foo"}, + }, + inputExistingACLToken: nil, + expectedErrorContains: "associated with policies", + }, + { + name: "name too long", + inputACLToken: &ACLToken{ + Type: ACLManagementToken, + Name: uuid.Generate() + uuid.Generate() + uuid.Generate() + uuid.Generate() + + uuid.Generate() + uuid.Generate() + uuid.Generate() + uuid.Generate(), + }, + inputExistingACLToken: nil, + expectedErrorContains: "name too long", + }, + { + name: "negative TTL", + inputACLToken: &ACLToken{ + Type: ACLManagementToken, + Name: "foo", + ExpirationTTL: -1 * time.Hour, + }, + inputExistingACLToken: nil, + expectedErrorContains: "should not be negative", + }, + { + name: "TTL too small", + inputACLToken: &ACLToken{ + Type: ACLManagementToken, + Name: "foo", + CreateTime: time.Date(2022, time.July, 11, 16, 23, 0, 0, time.UTC), + ExpirationTime: pointer.Of(time.Date(2022, time.July, 11, 16, 23, 10, 0, time.UTC)), + }, + inputExistingACLToken: nil, + expectedErrorContains: "expiration time cannot be less than", + }, + { + name: "TTL too large", + inputACLToken: &ACLToken{ + Type: ACLManagementToken, + Name: "foo", + CreateTime: time.Date(2022, time.July, 11, 16, 23, 0, 0, time.UTC), + ExpirationTime: pointer.Of(time.Date(2042, time.July, 11, 16, 23, 0, 0, time.UTC)), + }, + inputExistingACLToken: nil, + expectedErrorContains: "expiration time cannot be more than", + }, + { + name: "valid management", + inputACLToken: &ACLToken{ + Type: ACLManagementToken, + Name: "foo", + }, + inputExistingACLToken: nil, + expectedErrorContains: "", + }, + { + name: "valid client", + inputACLToken: &ACLToken{ + Type: ACLClientToken, + Name: "foo", + Policies: []string{"foo"}, + }, + inputExistingACLToken: nil, + expectedErrorContains: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutputError := tc.inputACLToken.Validate(1*time.Minute, 24*time.Hour, tc.inputExistingACLToken) + if tc.expectedErrorContains != "" { + require.ErrorContains(t, actualOutputError, tc.expectedErrorContains) + } else { + require.NoError(t, actualOutputError) + } + }) + } +} + +func TestACLToken_HasExpirationTime(t *testing.T) { + testCases := []struct { + name string + inputACLToken *ACLToken + expectedOutput bool `` + }{ + { + name: "nil acl token", + inputACLToken: nil, + expectedOutput: false, + }, + { + name: "default empty value", + inputACLToken: &ACLToken{}, + expectedOutput: false, + }, + { + name: "expiration set to now", + inputACLToken: &ACLToken{ + ExpirationTime: pointer.Of(time.Now().UTC()), + }, + expectedOutput: true, + }, + { + name: "expiration set to past", + inputACLToken: &ACLToken{ + ExpirationTime: pointer.Of(time.Date(2022, time.February, 21, 19, 35, 0, 0, time.UTC)), + }, + expectedOutput: true, + }, + { + name: "expiration set to future", + inputACLToken: &ACLToken{ + ExpirationTime: pointer.Of(time.Date(2087, time.April, 25, 12, 0, 0, 0, time.UTC)), + }, + expectedOutput: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputACLToken.HasExpirationTime() + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} + +func TestACLToken_IsExpired(t *testing.T) { + testCases := []struct { + name string + inputACLToken *ACLToken + inputTime time.Time + expectedOutput bool + }{ + { + name: "token without expiry", + inputACLToken: &ACLToken{}, + inputTime: time.Now().UTC(), + expectedOutput: false, + }, + { + name: "empty input time", + inputACLToken: &ACLToken{}, + inputTime: time.Time{}, + expectedOutput: false, + }, + { + name: "token not expired", + inputACLToken: &ACLToken{ + ExpirationTime: pointer.Of(time.Date(2022, time.May, 9, 10, 27, 0, 0, time.UTC)), + }, + inputTime: time.Date(2022, time.May, 9, 10, 26, 0, 0, time.UTC), + expectedOutput: false, + }, + { + name: "token expired", + inputACLToken: &ACLToken{ + ExpirationTime: pointer.Of(time.Date(2022, time.May, 9, 10, 27, 0, 0, time.UTC)), + }, + inputTime: time.Date(2022, time.May, 9, 10, 28, 0, 0, time.UTC), + expectedOutput: true, + }, + { + name: "empty input time", + inputACLToken: &ACLToken{ + ExpirationTime: pointer.Of(time.Date(2022, time.May, 9, 10, 27, 0, 0, time.UTC)), + }, + inputTime: time.Time{}, + expectedOutput: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputACLToken.IsExpired(tc.inputTime) + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index e5f1e7741..f003c1098 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -11721,14 +11721,26 @@ type ACLPolicyUpsertRequest struct { // ACLToken represents a client token which is used to Authenticate type ACLToken struct { - AccessorID string // Public Accessor ID (UUID) - SecretID string // Secret ID, private (UUID) - Name string // Human friendly name - Type string // Client or Management - Policies []string // Policies this token ties to - Global bool // Global or Region local - Hash []byte - CreateTime time.Time // Time of creation + AccessorID string // Public Accessor ID (UUID) + SecretID string // Secret ID, private (UUID) + Name string // Human friendly name + Type string // Client or Management + Policies []string // Policies this token ties to + Global bool // Global or Region local + Hash []byte + CreateTime time.Time // Time of creation + + // ExpirationTime represents the point after which a token should be + // considered revoked and is eligible for destruction. This time should + // always use UTC to account for multi-region global tokens. It is a + // pointer, so we can store nil, rather than the zero value of time.Time. + ExpirationTime *time.Time + + // ExpirationTTL is a convenience field for helping set ExpirationTime to a + // value of CreateTime+ExpirationTTL. This can only be set during token + // creation. This is a string version of a time.Duration like "2m". + ExpirationTTL time.Duration + CreateIndex uint64 ModifyIndex uint64 } @@ -11775,18 +11787,21 @@ var ( ) type ACLTokenListStub struct { - AccessorID string - Name string - Type string - Policies []string - Global bool - Hash []byte - CreateTime time.Time - CreateIndex uint64 - ModifyIndex uint64 + AccessorID string + Name string + Type string + Policies []string + Global bool + Hash []byte + CreateTime time.Time + ExpirationTime *time.Time + CreateIndex uint64 + ModifyIndex uint64 } -// SetHash is used to compute and set the hash of the ACL token +// SetHash is used to compute and set the hash of the ACL token. It only hashes +// fields which can be updated, and as such, does not hash fields such as +// ExpirationTime. func (a *ACLToken) SetHash() []byte { // Initialize a 256bit Blake2 hash (32 bytes) hash, err := blake2b.New256(nil) @@ -11816,39 +11831,19 @@ func (a *ACLToken) SetHash() []byte { func (a *ACLToken) Stub() *ACLTokenListStub { return &ACLTokenListStub{ - AccessorID: a.AccessorID, - Name: a.Name, - Type: a.Type, - Policies: a.Policies, - Global: a.Global, - Hash: a.Hash, - CreateTime: a.CreateTime, - CreateIndex: a.CreateIndex, - ModifyIndex: a.ModifyIndex, + AccessorID: a.AccessorID, + Name: a.Name, + Type: a.Type, + Policies: a.Policies, + Global: a.Global, + Hash: a.Hash, + CreateTime: a.CreateTime, + ExpirationTime: a.ExpirationTime, + CreateIndex: a.CreateIndex, + ModifyIndex: a.ModifyIndex, } } -// Validate is used to check a token for reasonableness -func (a *ACLToken) Validate() error { - var mErr multierror.Error - if len(a.Name) > maxTokenNameLength { - mErr.Errors = append(mErr.Errors, fmt.Errorf("token name too long")) - } - switch a.Type { - case ACLClientToken: - if len(a.Policies) == 0 { - mErr.Errors = append(mErr.Errors, fmt.Errorf("client token missing policies")) - } - case ACLManagementToken: - if len(a.Policies) != 0 { - mErr.Errors = append(mErr.Errors, fmt.Errorf("management token cannot be associated with policies")) - } - default: - mErr.Errors = append(mErr.Errors, fmt.Errorf("token type must be client or management")) - } - return mErr.ErrorOrNil() -} - // PolicySubset checks if a given set of policies is a subset of the token func (a *ACLToken) PolicySubset(policies []string) bool { // Hot-path the management tokens, superset of all policies. diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 376501670..7cb79ef70 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -5994,53 +5994,6 @@ func TestIsRecoverable(t *testing.T) { } } -func TestACLTokenValidate(t *testing.T) { - ci.Parallel(t) - - tk := &ACLToken{} - - // Missing a type - err := tk.Validate() - assert.NotNil(t, err) - if !strings.Contains(err.Error(), "client or management") { - t.Fatalf("bad: %v", err) - } - - // Missing policies - tk.Type = ACLClientToken - err = tk.Validate() - assert.NotNil(t, err) - if !strings.Contains(err.Error(), "missing policies") { - t.Fatalf("bad: %v", err) - } - - // Invalid policies - tk.Type = ACLManagementToken - tk.Policies = []string{"foo"} - err = tk.Validate() - assert.NotNil(t, err) - if !strings.Contains(err.Error(), "associated with policies") { - t.Fatalf("bad: %v", err) - } - - // Name too long policies - tk.Name = "" - for i := 0; i < 8; i++ { - tk.Name += uuid.Generate() - } - tk.Policies = nil - err = tk.Validate() - assert.NotNil(t, err) - if !strings.Contains(err.Error(), "too long") { - t.Fatalf("bad: %v", err) - } - - // Make it valid - tk.Name = "foo" - err = tk.Validate() - assert.Nil(t, err) -} - func TestACLTokenPolicySubset(t *testing.T) { ci.Parallel(t) From a8a8b1f84ffe4b53b46aea1c84fcc8668203290b Mon Sep 17 00:00:00 2001 From: James Rasell Date: Fri, 15 Jul 2022 15:20:50 +0200 Subject: [PATCH 03/19] acl: add token expiry checking to ACL token resolution. (#13756) This commit adds basic expiry checking when performing ACL token resolution. This expiry checking is local to each server and does not at this time take into account potential time skew on server hosts. A new error message has been created so clients whose token has expired get a clear message, rather than a generic token not found. The ACL resolution tests have been refactored into table driven tests, so additions are easier in the future. --- nomad/acl.go | 6 + nomad/acl_test.go | 309 +++++++++++++++++++++++++++------------- nomad/structs/errors.go | 2 + 3 files changed, 219 insertions(+), 98 deletions(-) diff --git a/nomad/acl.go b/nomad/acl.go index 90ecb417b..70587dd19 100644 --- a/nomad/acl.go +++ b/nomad/acl.go @@ -54,6 +54,9 @@ func resolveTokenFromSnapshotCache(snap *state.StateSnapshot, cache *lru.TwoQueu if token == nil { return nil, structs.ErrTokenNotFound } + if token.IsExpired(time.Now().UTC()) { + return nil, structs.ErrTokenExpired + } } // Check if this is a management token @@ -114,6 +117,9 @@ func (s *Server) ResolveSecretToken(secretID string) (*structs.ACLToken, error) if token == nil { return nil, structs.ErrTokenNotFound } + if token.IsExpired(time.Now().UTC()) { + return nil, structs.ErrTokenExpired + } } return token, nil diff --git a/nomad/acl_test.go b/nomad/acl_test.go index 867150639..14104a1df 100644 --- a/nomad/acl_test.go +++ b/nomad/acl_test.go @@ -2,133 +2,246 @@ package nomad import ( "testing" + "time" - lru "github.com/hashicorp/golang-lru" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/helper/pointer" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" - "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestResolveACLToken(t *testing.T) { ci.Parallel(t) - // Create mock state store and cache - state := state.TestStateStore(t) - cache, err := lru.New2Q(16) - assert.Nil(t, err) + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) - // Create a policy / token - policy := mock.ACLPolicy() - policy2 := mock.ACLPolicy() - token := mock.ACLToken() - token.Policies = []string{policy.Name, policy2.Name} - token2 := mock.ACLToken() - token2.Type = structs.ACLManagementToken - token2.Policies = nil - err = state.UpsertACLPolicies(structs.MsgTypeTestSetup, 100, []*structs.ACLPolicy{policy, policy2}) - assert.Nil(t, err) - err = state.UpsertACLTokens(structs.MsgTypeTestSetup, 110, []*structs.ACLToken{token, token2}) - assert.Nil(t, err) + testCases := []struct { + name string + testFn func(testServer *Server) + }{ + { + name: "leader token", + testFn: func(testServer *Server) { - snap, err := state.Snapshot() - assert.Nil(t, err) + // Check the leader ACL token is correctly set. + leaderACL := testServer.getLeaderAcl() + require.NotEmpty(t, leaderACL) - // Attempt resolution of blank token. Should return anonymous policy - aclObj, err := resolveTokenFromSnapshotCache(snap, cache, "") - assert.Nil(t, err) - assert.NotNil(t, aclObj) + // Resolve the token and ensure it's a management token. + aclResp, err := testServer.ResolveToken(leaderACL) + require.NoError(t, err) + require.NotNil(t, aclResp) + require.True(t, aclResp.IsManagement()) + }, + }, + { + name: "anonymous token", + testFn: func(testServer *Server) { - // Attempt resolution of unknown token. Should fail. - randID := uuid.Generate() - aclObj, err = resolveTokenFromSnapshotCache(snap, cache, randID) - assert.Equal(t, structs.ErrTokenNotFound, err) - assert.Nil(t, aclObj) + // Call the function with an empty input secret ID which is + // classed as representing anonymous access in clusters with + // ACLs enabled. + aclResp, err := testServer.ResolveToken("") + require.NoError(t, err) + require.NotNil(t, aclResp) + require.False(t, aclResp.IsManagement()) + }, + }, + { + name: "token not found", + testFn: func(testServer *Server) { - // Attempt resolution of management token. Should get singleton. - aclObj, err = resolveTokenFromSnapshotCache(snap, cache, token2.SecretID) - assert.Nil(t, err) - assert.NotNil(t, aclObj) - assert.Equal(t, true, aclObj.IsManagement()) - if aclObj != acl.ManagementACL { - t.Fatalf("expected singleton") + // Call the function with randomly generated secret ID which + // does not exist within state. + aclResp, err := testServer.ResolveToken(uuid.Generate()) + require.Equal(t, structs.ErrTokenNotFound, err) + require.Nil(t, aclResp) + }, + }, + { + name: "token expired", + testFn: func(testServer *Server) { + + // Create a mock token with an expiration time long in the + // past, and upsert. + token := mock.ACLToken() + token.ExpirationTime = pointer.Of(time.Date( + 1970, time.January, 1, 0, 0, 0, 0, time.UTC)) + + err := testServer.State().UpsertACLTokens( + structs.MsgTypeTestSetup, 10, []*structs.ACLToken{token}) + require.NoError(t, err) + + // Perform the function call which should result in finding the + // token has expired. + aclResp, err := testServer.ResolveToken(uuid.Generate()) + require.Equal(t, structs.ErrTokenNotFound, err) + require.Nil(t, aclResp) + }, + }, + { + name: "management token", + testFn: func(testServer *Server) { + + // Generate a management token and upsert this. + managementToken := mock.ACLToken() + managementToken.Type = structs.ACLManagementToken + managementToken.Policies = nil + + err := testServer.State().UpsertACLTokens( + structs.MsgTypeTestSetup, 10, []*structs.ACLToken{managementToken}) + require.NoError(t, err) + + // Resolve the token and check that we received a management + // ACL. + aclResp, err := testServer.ResolveToken(managementToken.SecretID) + require.Nil(t, err) + require.NotNil(t, aclResp) + require.True(t, aclResp.IsManagement()) + require.Equal(t, acl.ManagementACL, aclResp) + }, + }, + { + name: "client token", + testFn: func(testServer *Server) { + + // Generate a client token with associated policies and upsert + // these. + policy1 := mock.ACLPolicy() + policy2 := mock.ACLPolicy() + err := testServer.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2}) + + clientToken := mock.ACLToken() + clientToken.Policies = []string{policy1.Name, policy2.Name} + err = testServer.State().UpsertACLTokens( + structs.MsgTypeTestSetup, 20, []*structs.ACLToken{clientToken}) + require.NoError(t, err) + + // Resolve the token and check that we received a client + // ACL with appropriate permissions. + aclResp, err := testServer.ResolveToken(clientToken.SecretID) + require.Nil(t, err) + require.NotNil(t, aclResp) + require.False(t, aclResp.IsManagement()) + + allowed := aclResp.AllowNamespaceOperation("default", acl.NamespaceCapabilityListJobs) + require.True(t, allowed) + allowed = aclResp.AllowNamespaceOperation("other", acl.NamespaceCapabilityListJobs) + require.False(t, allowed) + + // Resolve the same token again and ensure we get the same + // result. + aclResp2, err := testServer.ResolveToken(clientToken.SecretID) + require.Nil(t, err) + require.NotNil(t, aclResp2) + require.Equal(t, aclResp, aclResp2) + + // Bust the cache by upserting the policy + err = testServer.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 30, []*structs.ACLPolicy{policy1}) + require.Nil(t, err) + + // Resolve the same token again, should get different value + aclResp3, err := testServer.ResolveToken(clientToken.SecretID) + require.Nil(t, err) + require.NotNil(t, aclResp3) + require.NotEqual(t, aclResp2, aclResp3) + }, + }, } - // Attempt resolution of client token - aclObj, err = resolveTokenFromSnapshotCache(snap, cache, token.SecretID) - assert.Nil(t, err) - assert.NotNil(t, aclObj) - - // Check that the ACL object is sane - assert.Equal(t, false, aclObj.IsManagement()) - allowed := aclObj.AllowNamespaceOperation("default", acl.NamespaceCapabilityListJobs) - assert.Equal(t, true, allowed) - allowed = aclObj.AllowNamespaceOperation("other", acl.NamespaceCapabilityListJobs) - assert.Equal(t, false, allowed) - - // Resolve the same token again, should get cache value - aclObj2, err := resolveTokenFromSnapshotCache(snap, cache, token.SecretID) - assert.Nil(t, err) - assert.NotNil(t, aclObj2) - if aclObj != aclObj2 { - t.Fatalf("expected cached value") - } - - // Bust the cache by upserting the policy - err = state.UpsertACLPolicies(structs.MsgTypeTestSetup, 120, []*structs.ACLPolicy{policy}) - assert.Nil(t, err) - snap, err = state.Snapshot() - assert.Nil(t, err) - - // Resolve the same token again, should get different value - aclObj3, err := resolveTokenFromSnapshotCache(snap, cache, token.SecretID) - assert.Nil(t, err) - assert.NotNil(t, aclObj3) - if aclObj == aclObj3 { - t.Fatalf("unexpected cached value") - } -} - -func TestResolveACLToken_LeaderToken(t *testing.T) { - ci.Parallel(t) - assert := assert.New(t) - s1, _, cleanupS1 := TestACLServer(t, nil) - defer cleanupS1() - testutil.WaitForLeader(t, s1.RPC) - - leaderAcl := s1.getLeaderAcl() - assert.NotEmpty(leaderAcl) - token, err := s1.ResolveToken(leaderAcl) - assert.Nil(err) - if assert.NotNil(token) { - assert.True(token.IsManagement()) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn(testServer) + }) } } func TestResolveSecretToken(t *testing.T) { ci.Parallel(t) - s1, _, cleanupS1 := TestACLServer(t, nil) - defer cleanupS1() - testutil.WaitForLeader(t, s1.RPC) + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) - state := s1.State() - leaderToken := s1.getLeaderAcl() - assert.NotEmpty(t, leaderToken) + testCases := []struct { + name string + testFn func(testServer *Server) + }{ + { + name: "valid token", + testFn: func(testServer *Server) { - token := mock.ACLToken() + // Generate and upsert a token. + token := mock.ACLToken() + err := testServer.State().UpsertACLTokens( + structs.MsgTypeTestSetup, 10, []*structs.ACLToken{token}) + require.NoError(t, err) - err := state.UpsertACLTokens(structs.MsgTypeTestSetup, 110, []*structs.ACLToken{token}) - assert.Nil(t, err) + // Attempt to look up the token and perform checks. + tokenResp, err := testServer.ResolveSecretToken(token.SecretID) + require.NoError(t, err) + require.NotNil(t, tokenResp) + require.Equal(t, token, tokenResp) + }, + }, + { + name: "anonymous token", + testFn: func(testServer *Server) { - respToken, err := s1.ResolveSecretToken(token.SecretID) - assert.Nil(t, err) - if assert.NotNil(t, respToken) { - assert.NotEmpty(t, respToken.AccessorID) + // Call the function with an empty input secret ID which is + // classed as representing anonymous access in clusters with + // ACLs enabled. + tokenResp, err := testServer.ResolveSecretToken("") + require.NoError(t, err) + require.NotNil(t, tokenResp) + require.Equal(t, structs.AnonymousACLToken, tokenResp) + }, + }, + { + name: "token not found", + testFn: func(testServer *Server) { + + // Call the function with randomly generated secret ID which + // does not exist within state. + tokenResp, err := testServer.ResolveSecretToken(uuid.Generate()) + require.Equal(t, structs.ErrTokenNotFound, err) + require.Nil(t, tokenResp) + }, + }, + { + name: "token expired", + testFn: func(testServer *Server) { + + // Create a mock token with an expiration time long in the + // past, and upsert. + token := mock.ACLToken() + token.ExpirationTime = pointer.Of(time.Date( + 1970, time.January, 1, 0, 0, 0, 0, time.UTC)) + + err := testServer.State().UpsertACLTokens( + structs.MsgTypeTestSetup, 10, []*structs.ACLToken{token}) + require.NoError(t, err) + + // Perform the function call which should result in finding the + // token has expired. + tokenResp, err := testServer.ResolveSecretToken(uuid.Generate()) + require.Equal(t, structs.ErrTokenNotFound, err) + require.Nil(t, tokenResp) + }, + }, } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn(testServer) + }) + } } diff --git a/nomad/structs/errors.go b/nomad/structs/errors.go index b98d81476..c96a5e9d0 100644 --- a/nomad/structs/errors.go +++ b/nomad/structs/errors.go @@ -12,6 +12,7 @@ const ( errNotReadyForConsistentReads = "Not ready to serve consistent reads" errNoRegionPath = "No path to region" errTokenNotFound = "ACL token not found" + errTokenExpired = "ACL token expired" errPermissionDenied = "Permission denied" errJobRegistrationDisabled = "Job registration, dispatch, and scale are disabled by the scheduler configuration" errNoNodeConn = "No path to node" @@ -48,6 +49,7 @@ var ( ErrNotReadyForConsistentReads = errors.New(errNotReadyForConsistentReads) ErrNoRegionPath = errors.New(errNoRegionPath) ErrTokenNotFound = errors.New(errTokenNotFound) + ErrTokenExpired = errors.New(errTokenExpired) ErrPermissionDenied = errors.New(errPermissionDenied) ErrJobRegistrationDisabled = errors.New(errJobRegistrationDisabled) ErrNoNodeConn = errors.New(errNoNodeConn) From 9264f07cc1d5c01b492828ed6f9752c72f9bacf2 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Tue, 19 Jul 2022 15:37:46 +0200 Subject: [PATCH 04/19] core: add expired token garbage collection periodic jobs. (#13805) Two new periodic core jobs have been added which handle removing expired local and global tokens from state. The local core job is run on every leader; the global core job is only run on the leader within the authoritative region. --- nomad/core_sched.go | 105 ++++++++++++++++++ nomad/core_sched_test.go | 163 ++++++++++++++++++++++++++++ nomad/leader.go | 94 +++++++++++----- nomad/leader_test.go | 22 ++++ nomad/state/state_store_acl.go | 32 +----- nomad/state/state_store_acl_test.go | 76 +++++++------ nomad/structs/acl.go | 16 +++ nomad/structs/structs.go | 10 ++ 8 files changed, 431 insertions(+), 87 deletions(-) diff --git a/nomad/core_sched.go b/nomad/core_sched.go index ff8605e02..2dbc8cf29 100644 --- a/nomad/core_sched.go +++ b/nomad/core_sched.go @@ -51,6 +51,10 @@ func (c *CoreScheduler) Process(eval *structs.Evaluation) error { return c.csiPluginGC(eval) case structs.CoreJobOneTimeTokenGC: return c.expiredOneTimeTokenGC(eval) + case structs.CoreJobLocalTokenExpiredGC: + return c.expiredACLTokenGC(eval, false) + case structs.CoreJobGlobalTokenExpiredGC: + return c.expiredACLTokenGC(eval, true) case structs.CoreJobForceGC: return c.forceGC(eval) default: @@ -78,6 +82,13 @@ func (c *CoreScheduler) forceGC(eval *structs.Evaluation) error { if err := c.expiredOneTimeTokenGC(eval); err != nil { return err } + if err := c.expiredACLTokenGC(eval, false); err != nil { + return err + } + if err := c.expiredACLTokenGC(eval, true); err != nil { + return err + } + // Node GC must occur after the others to ensure the allocations are // cleared. return c.nodeGC(eval) @@ -773,6 +784,100 @@ func (c *CoreScheduler) expiredOneTimeTokenGC(eval *structs.Evaluation) error { return c.srv.RPC("ACL.ExpireOneTimeTokens", req, &structs.GenericResponse{}) } +// expiredACLTokenGC handles running the garbage collector for expired ACL +// tokens. It can be used for both local and global tokens and includes +// behaviour to account for periodic and user actioned garbage collection +// invocations. +func (c *CoreScheduler) expiredACLTokenGC(eval *structs.Evaluation, global bool) error { + + // If ACLs are not enabled, we do not need to continue and should exit + // early. This is not an error condition as callers can blindly call this + // function without checking the configuration. If the caller wants this to + // be an error, they should check this config value themselves. + if !c.srv.config.ACLEnabled { + return nil + } + + // If the function has been triggered for global tokens, but we are not the + // authoritative region, we should exit. This is not an error condition as + // callers can blindly call this function without checking the + // configuration. If the caller wants this to be an error, they should + // check this config value themselves. + if global && c.srv.config.AuthoritativeRegion != c.srv.Region() { + return nil + } + + expiryThresholdIdx := c.getThreshold(eval, "expired_acl_token", + "acl_token_expiration_gc_threshold", c.srv.config.ACLTokenExpirationGCThreshold) + + expiredIter, err := c.snap.ACLTokensByExpired(global) + if err != nil { + return err + } + + var ( + expiredAccessorIDs []string + num int + ) + + // The memdb iterator contains all tokens which include an expiration time, + // however, as the caller, we do not know at which point in the array the + // tokens are no longer expired. This time therefore forms the basis at + // which we draw the line in the iteration loop and find the final expired + // token that is eligible for deletion. + now := time.Now().UTC() + + for raw := expiredIter.Next(); raw != nil; raw = expiredIter.Next() { + token := raw.(*structs.ACLToken) + + // The iteration order of the indexes mean if we come across an + // unexpired token, we can exit as we have found all currently expired + // tokens. + if !token.IsExpired(now) { + break + } + + // Check if the token is recent enough to skip, otherwise we'll delete + // it. + if token.CreateIndex > expiryThresholdIdx { + continue + } + + // Add the token accessor ID to the tracking array, thus marking it + // ready for deletion. + expiredAccessorIDs = append(expiredAccessorIDs, token.AccessorID) + + // Increment the counter. If this is at or above our limit, we return + // what we have so far. + if num++; num >= structs.ACLMaxExpiredBatchSize { + break + } + } + + // There is no need to call the RPC endpoint if we do not have any tokens + // to delete. + if len(expiredAccessorIDs) < 1 { + return nil + } + + // Log a nice, friendly debug message which could be useful when debugging + // garbage collection in environments with a high rate of token creation + // and expiration. + c.logger.Debug("expired ACL token GC found eligible tokens", + "num", len(expiredAccessorIDs)) + + // Set up and make the RPC request which will return any error performing + // the deletion. + req := structs.ACLTokenDeleteRequest{ + AccessorIDs: expiredAccessorIDs, + WriteRequest: structs.WriteRequest{ + Region: c.srv.Region(), + AuthToken: eval.LeaderACL, + }, + } + return c.srv.RPC(structs.ACLDeleteTokensRPCMethod, req, &structs.GenericResponse{}) +} + // getThreshold returns the index threshold for determining whether an // object is old enough to GC func (c *CoreScheduler) getThreshold(eval *structs.Evaluation, objectName, configName string, configThreshold time.Duration) uint64 { diff --git a/nomad/core_sched_test.go b/nomad/core_sched_test.go index 76aa6eb29..5ffd6d801 100644 --- a/nomad/core_sched_test.go +++ b/nomad/core_sched_test.go @@ -8,6 +8,7 @@ import ( memdb "github.com/hashicorp/go-memdb" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/helper/pointer" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/state" @@ -2513,3 +2514,165 @@ func TestCoreScheduler_FailLoop(t *testing.T) { out.TriggeredBy) } } + +func TestCoreScheduler_ExpiredACLTokenGC(t *testing.T) { + ci.Parallel(t) + + testServer, rootACLToken, testServerShutdown := TestACLServer(t, func(c *Config) { + c.NumSchedulers = 0 + }) + defer testServerShutdown() + testutil.WaitForLeader(t, testServer.RPC) + + now := time.Now().UTC() + + // Craft some specific local and global tokens. For each type, one is + // expired, one is not. + expiredGlobal := mock.ACLToken() + expiredGlobal.Global = true + expiredGlobal.ExpirationTime = pointer.Of(now.Add(-2 * time.Hour)) + + unexpiredGlobal := mock.ACLToken() + unexpiredGlobal.Global = true + unexpiredGlobal.ExpirationTime = pointer.Of(now.Add(2 * time.Hour)) + + expiredLocal := mock.ACLToken() + expiredLocal.ExpirationTime = pointer.Of(now.Add(-2 * time.Hour)) + + unexpiredLocal := mock.ACLToken() + unexpiredLocal.ExpirationTime = pointer.Of(now.Add(2 * time.Hour)) + + // Upsert these into state. + err := testServer.State().UpsertACLTokens(structs.MsgTypeTestSetup, 10, []*structs.ACLToken{ + expiredGlobal, unexpiredGlobal, expiredLocal, unexpiredLocal, + }) + require.NoError(t, err) + + // Overwrite the timetable. The existing timetable has an entry due to the + // ACL bootstrapping which makes witnessing a new index at a timestamp in + // the past impossible. + tt := NewTimeTable(timeTableGranularity, timeTableLimit) + tt.Witness(20, time.Now().UTC().Add(-1*testServer.config.ACLTokenExpirationGCThreshold)) + testServer.fsm.timetable = tt + + // Generate the core scheduler. + snap, err := testServer.State().Snapshot() + require.NoError(t, err) + coreScheduler := NewCoreScheduler(testServer, snap) + + // Trigger global and local periodic garbage collection runs. + index, err := testServer.State().LatestIndex() + require.NoError(t, err) + index++ + + globalGCEval := testServer.coreJobEval(structs.CoreJobGlobalTokenExpiredGC, index) + require.NoError(t, coreScheduler.Process(globalGCEval)) + + localGCEval := testServer.coreJobEval(structs.CoreJobLocalTokenExpiredGC, index) + require.NoError(t, coreScheduler.Process(localGCEval)) + + // Ensure the ACL tokens stored within state are as expected. + iter, err := testServer.State().ACLTokens(nil, state.SortDefault) + require.NoError(t, err) + + var tokens []*structs.ACLToken + for raw := iter.Next(); raw != nil; raw = iter.Next() { + tokens = append(tokens, raw.(*structs.ACLToken)) + } + require.ElementsMatch(t, []*structs.ACLToken{rootACLToken, unexpiredGlobal, unexpiredLocal}, tokens) +} + +func TestCoreScheduler_ExpiredACLTokenGC_Force(t *testing.T) { + ci.Parallel(t) + + testServer, rootACLToken, testServerShutdown := TestACLServer(t, func(c *Config) { + c.NumSchedulers = 0 + }) + defer testServerShutdown() + testutil.WaitForLeader(t, testServer.RPC) + + // This time is the threshold for all expiry calls to be based on. All + // tokens with expiry can use this as their base and use Add(). + expiryTimeThreshold := time.Now().UTC() + + // Track expired and non-expired tokens for local and global tokens in + // separate arrays, so we have a clear way to test state. + var expiredGlobalTokens, nonExpiredGlobalTokens, expiredLocalTokens, nonExpiredLocalTokens []*structs.ACLToken + + // Add the root ACL token to the appropriate array. This will be returned + // from state so must be accounted for and tested. + nonExpiredGlobalTokens = append(nonExpiredGlobalTokens, rootACLToken) + + // Generate and upsert a number of mixed expired, non-expired global + // tokens. + for i := 0; i < 20; i++ { + mockedToken := mock.ACLToken() + mockedToken.Global = true + if i%2 == 0 { + expiredGlobalTokens = append(expiredGlobalTokens, mockedToken) + mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(-24 * time.Hour)) + } else { + nonExpiredGlobalTokens = append(nonExpiredGlobalTokens, mockedToken) + mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(24 * time.Hour)) + } + } + + // Generate and upsert a number of mixed expired, non-expired local + // tokens. + for i := 0; i < 20; i++ { + mockedToken := mock.ACLToken() + mockedToken.Global = false + if i%2 == 0 { + expiredLocalTokens = append(expiredLocalTokens, mockedToken) + mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(-24 * time.Hour)) + } else { + nonExpiredLocalTokens = append(nonExpiredLocalTokens, mockedToken) + mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(24 * time.Hour)) + } + } + + allTokens := append(expiredGlobalTokens, nonExpiredGlobalTokens...) + allTokens = append(allTokens, expiredLocalTokens...) + allTokens = append(allTokens, nonExpiredLocalTokens...) + + // Upsert them all. + err := testServer.State().UpsertACLTokens(structs.MsgTypeTestSetup, 10, allTokens) + require.NoError(t, err) + + // This function provides an easy way to get all tokens out of the + // iterator. + fromIteratorFunc := func(iter memdb.ResultIterator) []*structs.ACLToken { + var tokens []*structs.ACLToken + for raw := iter.Next(); raw != nil; raw = iter.Next() { + tokens = append(tokens, raw.(*structs.ACLToken)) + } + return tokens + } + + // Check all the tokens are correctly stored within state. + iter, err := testServer.State().ACLTokens(nil, state.SortDefault) + require.NoError(t, err) + + tokens := fromIteratorFunc(iter) + require.ElementsMatch(t, allTokens, tokens) + + // Generate the core scheduler and trigger a forced garbage collection + // which should delete all expired tokens. + snap, err := testServer.State().Snapshot() + require.NoError(t, err) + coreScheduler := NewCoreScheduler(testServer, snap) + + index, err := testServer.State().LatestIndex() + require.NoError(t, err) + index++ + + forceGCEval := testServer.coreJobEval(structs.CoreJobForceGC, index) + require.NoError(t, coreScheduler.Process(forceGCEval)) + + // List all the remaining ACL tokens to be sure they are as expected. + iter, err = testServer.State().ACLTokens(nil, state.SortDefault) + require.NoError(t, err) + + tokens = fromIteratorFunc(iter) + require.ElementsMatch(t, append(nonExpiredGlobalTokens, nonExpiredLocalTokens...), tokens) +} diff --git a/nomad/leader.go b/nomad/leader.go index 8ad91bc4c..9d50d41b9 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-memdb" "github.com/hashicorp/go-version" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" @@ -341,7 +342,8 @@ func (s *Server) establishLeadership(stopCh chan struct{}) error { return err } - // Scheduler periodic jobs + // Schedule periodic jobs which include expired local ACL token garbage + // collection. go s.schedulePeriodic(stopCh) // Reap any failed evaluations @@ -373,12 +375,22 @@ func (s *Server) establishLeadership(stopCh chan struct{}) error { return err } - // Start replication of ACLs and Policies if they are enabled, - // and we are not the authoritative region. - if s.config.ACLEnabled && s.config.Region != s.config.AuthoritativeRegion { - go s.replicateACLPolicies(stopCh) - go s.replicateACLTokens(stopCh) - go s.replicateNamespaces(stopCh) + // If ACLs are enabled, the leader needs to start a number of long-lived + // routines. Exactly which routines, depends on whether this leader is + // running within the authoritative region or not. + if s.config.ACLEnabled { + + // The authoritative region is responsible for garbage collecting + // expired global tokens. Otherwise, non-authoritative regions need to + // replicate policies, tokens, and namespaces. + switch s.config.AuthoritativeRegion { + case s.config.Region: + go s.schedulePeriodicAuthoritative(stopCh) + default: + go s.replicateACLPolicies(stopCh) + go s.replicateACLTokens(stopCh) + go s.replicateNamespaces(stopCh) + } } // Setup any enterprise systems required. @@ -762,43 +774,35 @@ func (s *Server) schedulePeriodic(stopCh chan struct{}) { oneTimeTokenGC := time.NewTicker(s.config.OneTimeTokenGCInterval) defer oneTimeTokenGC.Stop() - // getLatest grabs the latest index from the state store. It returns true if - // the index was retrieved successfully. - getLatest := func() (uint64, bool) { - snapshotIndex, err := s.fsm.State().LatestIndex() - if err != nil { - s.logger.Error("failed to determine state store's index", "error", err) - return 0, false - } - - return snapshotIndex, true - } + // Set up the expired ACL local token garbage collection timer. + localTokenExpiredGC, localTokenExpiredGCStop := helper.NewSafeTimer(s.config.ACLTokenExpirationGCInterval) + defer localTokenExpiredGCStop() for { select { case <-evalGC.C: - if index, ok := getLatest(); ok { + if index, ok := s.getLatestIndex(); ok { s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobEvalGC, index)) } case <-nodeGC.C: - if index, ok := getLatest(); ok { + if index, ok := s.getLatestIndex(); ok { s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobNodeGC, index)) } case <-jobGC.C: - if index, ok := getLatest(); ok { + if index, ok := s.getLatestIndex(); ok { s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobJobGC, index)) } case <-deploymentGC.C: - if index, ok := getLatest(); ok { + if index, ok := s.getLatestIndex(); ok { s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobDeploymentGC, index)) } case <-csiPluginGC.C: - if index, ok := getLatest(); ok { + if index, ok := s.getLatestIndex(); ok { s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobCSIPluginGC, index)) } case <-csiVolumeClaimGC.C: - if index, ok := getLatest(); ok { + if index, ok := s.getLatestIndex(); ok { s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobCSIVolumeClaimGC, index)) } case <-oneTimeTokenGC.C: @@ -806,15 +810,55 @@ func (s *Server) schedulePeriodic(stopCh chan struct{}) { continue } - if index, ok := getLatest(); ok { + if index, ok := s.getLatestIndex(); ok { s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobOneTimeTokenGC, index)) } + case <-localTokenExpiredGC.C: + if index, ok := s.getLatestIndex(); ok { + s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobLocalTokenExpiredGC, index)) + } + localTokenExpiredGC.Reset(s.config.ACLTokenExpirationGCInterval) case <-stopCh: return } } } +// schedulePeriodicAuthoritative is a long-lived routine intended for use on +// the leader within the authoritative region only. It periodically queues work +// onto the _core scheduler for ACL based activities such as removing expired +// global ACL tokens. +func (s *Server) schedulePeriodicAuthoritative(stopCh chan struct{}) { + + // Set up the expired ACL global token garbage collection timer. + globalTokenExpiredGC, globalTokenExpiredGCStop := helper.NewSafeTimer(s.config.ACLTokenExpirationGCInterval) + defer globalTokenExpiredGCStop() + + for { + select { + case <-globalTokenExpiredGC.C: + if index, ok := s.getLatestIndex(); ok { + s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobGlobalTokenExpiredGC, index)) + } + globalTokenExpiredGC.Reset(s.config.ACLTokenExpirationGCInterval) + case <-stopCh: + return + } + } +} + +// getLatestIndex is a helper function which returns the latest index from the +// state store. The boolean return indicates whether the call has been +// successful or not. +func (s *Server) getLatestIndex() (uint64, bool) { + snapshotIndex, err := s.fsm.State().LatestIndex() + if err != nil { + s.logger.Error("failed to determine state store's index", "error", err) + return 0, false + } + return snapshotIndex, true +} + // coreJobEval returns an evaluation for a core job func (s *Server) coreJobEval(job string, modifyIndex uint64) *structs.Evaluation { return &structs.Evaluation{ diff --git a/nomad/leader_test.go b/nomad/leader_test.go index 62864d115..2bd1c1070 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/nomad/testutil" "github.com/hashicorp/raft" "github.com/hashicorp/serf/serf" + "github.com/shoenig/test/must" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1665,6 +1666,27 @@ func waitForStableLeadership(t *testing.T, servers []*Server) *Server { return leader } +func TestServer_getLatestIndex(t *testing.T) { + ci.Parallel(t) + + testServer, testServerCleanup := TestServer(t, nil) + defer testServerCleanup() + + // Test a new state store value. + idx, success := testServer.getLatestIndex() + require.True(t, success) + must.Eq(t, 1, idx) + + // Upsert something with a high index, and check again. + err := testServer.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 1013, []*structs.ACLPolicy{mock.ACLPolicy()}) + require.NoError(t, err) + + idx, success = testServer.getLatestIndex() + require.True(t, success) + must.Eq(t, 1013, idx) +} + func TestServer_handleEvalBrokerStateChange(t *testing.T) { ci.Parallel(t) diff --git a/nomad/state/state_store_acl.go b/nomad/state/state_store_acl.go index 78aab4403..43980d54f 100644 --- a/nomad/state/state_store_acl.go +++ b/nomad/state/state_store_acl.go @@ -2,9 +2,8 @@ package state import ( "fmt" - "time" - "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/go-memdb" ) // ACLTokensByExpired returns an array accessor IDs of expired ACL tokens. @@ -14,39 +13,14 @@ import ( // the global boolean argument. The number of returned IDs can be limited by // the max integer, which is useful to limit the number of tokens we attempt to // delete in a single transaction. -func (s *StateStore) ACLTokensByExpired(global bool, now time.Time, max int) ([]string, error) { +func (s *StateStore) ACLTokensByExpired(global bool) (memdb.ResultIterator, error) { tnx := s.db.ReadTxn() iter, err := tnx.Get("acl_token", expiresIndexName(global)) if err != nil { return nil, fmt.Errorf("failed acl token listing: %v", err) } - - var ( - accessorIDs []string - num int - ) - - for raw := iter.Next(); raw != nil; raw = iter.Next() { - token := raw.(*structs.ACLToken) - - // The indexes mean if we come across an unexpired token, we can exit - // as we have found all currently expired tokens. - if !token.IsExpired(now) { - return accessorIDs, nil - } - - accessorIDs = append(accessorIDs, token.AccessorID) - - // Increment the counter. If this is at or above our limit, we return - // what we have so far. - num++ - if num >= max { - return accessorIDs, nil - } - } - - return accessorIDs, nil + return iter, nil } // expiresIndexName is a helper function to identify the correct ACL token diff --git a/nomad/state/state_store_acl_test.go b/nomad/state/state_store_acl_test.go index 8974489a1..6636cb0fc 100644 --- a/nomad/state/state_store_acl_test.go +++ b/nomad/state/state_store_acl_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/helper/pointer" "github.com/hashicorp/nomad/nomad/mock" @@ -15,6 +16,16 @@ func TestStateStore_ACLTokensByExpired(t *testing.T) { ci.Parallel(t) testState := testStateStore(t) + // This function provides an easy way to get all tokens out of the + // iterator. + fromIteratorFunc := func(iter memdb.ResultIterator) []*structs.ACLToken { + var tokens []*structs.ACLToken + for raw := iter.Next(); raw != nil; raw = iter.Next() { + tokens = append(tokens, raw.(*structs.ACLToken)) + } + return tokens + } + // This time is the threshold for all expiry calls to be based on. All // tokens with expiry can use this as their base and use Add(). expiryTimeThreshold := time.Date(2022, time.April, 27, 14, 50, 0, 0, time.UTC) @@ -31,13 +42,15 @@ func TestStateStore_ACLTokensByExpired(t *testing.T) { neverExpireLocalToken, neverExpireGlobalToken}) require.NoError(t, err) - ids, err := testState.ACLTokensByExpired(true, expiryTimeThreshold, 10) + iter, err := testState.ACLTokensByExpired(true) require.NoError(t, err) - require.Len(t, ids, 0) + tokens := fromIteratorFunc(iter) + require.Len(t, tokens, 0) - ids, err = testState.ACLTokensByExpired(false, expiryTimeThreshold, 10) + iter, err = testState.ACLTokensByExpired(false) require.NoError(t, err) - require.Len(t, ids, 0) + tokens = fromIteratorFunc(iter) + require.Len(t, tokens, 0) // Generate, upsert, and test an expired local token. This token expired // long ago and therefore before all others coming in the tests. It should @@ -48,10 +61,11 @@ func TestStateStore_ACLTokensByExpired(t *testing.T) { err = testState.UpsertACLTokens(structs.MsgTypeTestSetup, 20, []*structs.ACLToken{expiredLocalToken}) require.NoError(t, err) - ids, err = testState.ACLTokensByExpired(false, expiryTimeThreshold, 10) + iter, err = testState.ACLTokensByExpired(false) require.NoError(t, err) - require.Len(t, ids, 1) - require.Equal(t, expiredLocalToken.AccessorID, ids[0]) + tokens = fromIteratorFunc(iter) + require.Len(t, tokens, 1) + require.Equal(t, expiredLocalToken.AccessorID, tokens[0].AccessorID) // Generate, upsert, and test an expired global token. This token expired // long ago and therefore before all others coming in the tests. It should @@ -63,54 +77,50 @@ func TestStateStore_ACLTokensByExpired(t *testing.T) { err = testState.UpsertACLTokens(structs.MsgTypeTestSetup, 30, []*structs.ACLToken{expiredGlobalToken}) require.NoError(t, err) - ids, err = testState.ACLTokensByExpired(true, expiryTimeThreshold, 10) + iter, err = testState.ACLTokensByExpired(true) require.NoError(t, err) - require.Len(t, ids, 1) - require.Equal(t, expiredGlobalToken.AccessorID, ids[0]) + tokens = fromIteratorFunc(iter) + require.Len(t, tokens, 1) + require.Equal(t, expiredGlobalToken.AccessorID, tokens[0].AccessorID) // This test function allows us to run the same test for local and global // tokens. - testFn := func(oldID string, global bool) { + testFn := func(oldToken *structs.ACLToken, global bool) { - // Track all the expected expired accessor IDs including the long + // Track all the expected expired ACL tokens, including the long // expired token. - var expiredLocalAccessorIDs []string - expiredLocalAccessorIDs = append(expiredLocalAccessorIDs, oldID) + var expiredTokens []*structs.ACLToken + expiredTokens = append(expiredTokens, oldToken) - // Generate and upsert a number of mixed expired, non-expired local tokens. - mixedLocalTokens := make([]*structs.ACLToken, 20) + // Generate and upsert a number of mixed expired, non-expired tokens. + mixedTokens := make([]*structs.ACLToken, 20) for i := 0; i < 20; i++ { mockedToken := mock.ACLToken() mockedToken.Global = global if i%2 == 0 { - expiredLocalAccessorIDs = append(expiredLocalAccessorIDs, mockedToken.AccessorID) + expiredTokens = append(expiredTokens, mockedToken) mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(-24 * time.Hour)) } else { mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(24 * time.Hour)) } - mixedLocalTokens[i] = mockedToken + mixedTokens[i] = mockedToken } - err = testState.UpsertACLTokens(structs.MsgTypeTestSetup, 40, mixedLocalTokens) + err = testState.UpsertACLTokens(structs.MsgTypeTestSetup, 40, mixedTokens) require.NoError(t, err) - // Use a max value higher than the number we have to check the full listing - // works as expected. Ensure our oldest expired token is first in the list. - ids, err = testState.ACLTokensByExpired(global, expiryTimeThreshold, 100) + // Check the full listing works as expected as the first 11 elements + // should all be our expired tokens. Ensure our oldest expired token is + // first in the list. + iter, err = testState.ACLTokensByExpired(global) require.NoError(t, err) - require.ElementsMatch(t, ids, expiredLocalAccessorIDs) - require.Equal(t, ids[0], oldID) - - // Use a lower max value than the number of known expired tokens to ensure - // this is working. - ids, err = testState.ACLTokensByExpired(global, expiryTimeThreshold, 3) - require.NoError(t, err) - require.Len(t, ids, 3) - require.Equal(t, ids[0], oldID) + tokens = fromIteratorFunc(iter) + require.ElementsMatch(t, expiredTokens, tokens[:11]) + require.Equal(t, tokens[0], oldToken) } - testFn(expiredLocalToken.AccessorID, false) - testFn(expiredGlobalToken.AccessorID, true) + testFn(expiredLocalToken, false) + testFn(expiredGlobalToken, true) } func Test_expiresIndexName(t *testing.T) { diff --git a/nomad/structs/acl.go b/nomad/structs/acl.go index 6d4d4b387..11d48fd09 100644 --- a/nomad/structs/acl.go +++ b/nomad/structs/acl.go @@ -24,6 +24,22 @@ const ( // Args: ACLTokenUpsertRequest // Reply: ACLTokenUpsertResponse ACLUpsertTokensRPCMethod = "ACL.UpsertTokens" + + // ACLDeleteTokensRPCMethod is the RPC method for batch deleting ACL + // tokens. + // + // Args: ACLTokenDeleteRequest + // Reply: GenericResponse + ACLDeleteTokensRPCMethod = "ACL.DeleteTokens" +) + +const ( + // ACLMaxExpiredBatchSize is the maximum number of expired ACL tokens that + // will be garbage collected in a single trigger. This number helps limit + // the replication pressure due to expired token deletion. If there are a + // large number of expired tokens pending garbage collection, this value is + // a potential limiting factor. + ACLMaxExpiredBatchSize = 4096 ) // Canonicalize performs basic canonicalization on the ACL token object. It is diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index f003c1098..d1375e6c5 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -10741,6 +10741,16 @@ const ( // tokens. We periodically scan for expired tokens and delete them. CoreJobOneTimeTokenGC = "one-time-token-gc" + // CoreJobLocalTokenExpiredGC is used for the garbage collection of + // expired local ACL tokens. We periodically scan for expired tokens and + // delete them. + CoreJobLocalTokenExpiredGC = "local-token-expired-gc" + + // CoreJobGlobalTokenExpiredGC is used for the garbage collection of + // expired global ACL tokens. We periodically scan for expired tokens and + // delete them. + CoreJobGlobalTokenExpiredGC = "global-token-expired-gc" + // CoreJobForceGC is used to force garbage collection of all GCable objects. CoreJobForceGC = "force-gc" ) From f6d12a3c002fd2f2fad61ef32d930a2ca34d0f00 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 20 Jul 2022 10:06:23 +0200 Subject: [PATCH 05/19] acl: enable configuration and visualisation of token expiration for users (#13846) * api: add ACL token expiry params to HTTP API * cli: allow setting and displaying ACL token expiry --- api/acl.go | 44 ++++++++++---- api/acl_test.go | 101 ++++++++++++++++++++++--------- command/acl_bootstrap.go | 9 +++ command/acl_bootstrap_test.go | 2 + command/acl_token_create.go | 21 ++++++- command/acl_token_create_test.go | 29 ++++++--- command/acl_token_list.go | 13 +++- 7 files changed, 166 insertions(+), 53 deletions(-) diff --git a/api/acl.go b/api/acl.go index 4a289c666..07b51deb4 100644 --- a/api/acl.go +++ b/api/acl.go @@ -221,24 +221,42 @@ type ACLPolicy struct { // ACLToken represents a client token which is used to Authenticate type ACLToken struct { - AccessorID string - SecretID string - Name string - Type string - Policies []string - Global bool - CreateTime time.Time + AccessorID string + SecretID string + Name string + Type string + Policies []string + Global bool + CreateTime time.Time + + // ExpirationTime represents the point after which a token should be + // considered revoked and is eligible for destruction. The zero value of + // time.Time does not respect json omitempty directives, so we must use a + // pointer. + ExpirationTime *time.Time `json:",omitempty"` + + // ExpirationTTL is a convenience field for helping set ExpirationTime to a + // value of CreateTime+ExpirationTTL. This can only be set during token + // creation. This is a string version of a time.Duration like "2m". + ExpirationTTL time.Duration `json:",omitempty"` + CreateIndex uint64 ModifyIndex uint64 } type ACLTokenListStub struct { - AccessorID string - Name string - Type string - Policies []string - Global bool - CreateTime time.Time + AccessorID string + Name string + Type string + Policies []string + Global bool + CreateTime time.Time + + // ExpirationTime represents the point after which a token should be + // considered revoked and is eligible for destruction. A nil value + // indicates no expiration has been set on the token. + ExpirationTime *time.Time `json:"expiration_time,omitempty"` + CreateIndex uint64 ModifyIndex uint64 } diff --git a/api/acl_test.go b/api/acl_test.go index 0b7dbc025..7a0434b54 100644 --- a/api/acl_test.go +++ b/api/acl_test.go @@ -2,9 +2,11 @@ package api import ( "testing" + "time" "github.com/hashicorp/nomad/api/internal/testutil" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestACLPolicies_ListUpsert(t *testing.T) { @@ -118,15 +120,10 @@ func TestACLTokens_List(t *testing.T) { // Expect out bootstrap token result, qm, err := at.List(nil) - if err != nil { - t.Fatalf("err: %s", err) - } - if qm.LastIndex == 0 { - t.Fatalf("bad index: %d", qm.LastIndex) - } - if n := len(result); n != 1 { - t.Fatalf("expected 1 token, got: %d", n) - } + require.NoError(t, err) + require.NotEqual(t, 0, qm.LastIndex) + require.Len(t, result, 1) + require.Nil(t, result[0].ExpirationTime) } func TestACLTokens_CreateUpdate(t *testing.T) { @@ -156,31 +153,81 @@ func TestACLTokens_CreateUpdate(t *testing.T) { // Verify the change took hold assert.Equal(t, out.Name, out2.Name) + + // Try updating the token to include a TTL which is not allowed. + out2.ExpirationTTL = 10 * time.Minute + out3, _, err := at.Update(out2, nil) + require.Error(t, err) + require.Nil(t, out3) } func TestACLTokens_Info(t *testing.T) { testutil.Parallel(t) - c, s, _ := makeACLClient(t, nil, nil) - defer s.Stop() - at := c.ACLTokens() - token := &ACLToken{ - Name: "foo", - Type: "client", - Policies: []string{"foo1"}, + testClient, testServer, _ := makeACLClient(t, nil, nil) + defer testServer.Stop() + + testCases := []struct { + name string + testFn func(client *Client) + }{ + { + name: "token without expiry", + testFn: func(client *Client) { + + token := &ACLToken{ + Name: "foo", + Type: "client", + Policies: []string{"foo1"}, + } + + // Create the token + out, wm, err := client.ACLTokens().Create(token, nil) + require.Nil(t, err) + assertWriteMeta(t, wm) + require.NotNil(t, out) + + // Query the token + out2, qm, err := client.ACLTokens().Info(out.AccessorID, nil) + require.Nil(t, err) + assertQueryMeta(t, qm) + require.Equal(t, out, out2) + }, + }, + { + name: "token with expiry", + testFn: func(client *Client) { + + token := &ACLToken{ + Name: "token-with-expiry", + Type: "client", + Policies: []string{"foo1"}, + ExpirationTTL: 10 * time.Minute, + } + + // Create the token + out, wm, err := client.ACLTokens().Create(token, nil) + require.Nil(t, err) + assertWriteMeta(t, wm) + require.NotNil(t, out) + + // Query the token and ensure it matches what was returned + // during the creation as well as ensuring the expiration time + // is set. + out2, qm, err := client.ACLTokens().Info(out.AccessorID, nil) + require.Nil(t, err) + assertQueryMeta(t, qm) + require.Equal(t, out, out2) + require.NotNil(t, out2.ExpirationTime) + }, + }, } - // Create the token - out, wm, err := at.Create(token, nil) - assert.Nil(t, err) - assertWriteMeta(t, wm) - assert.NotNil(t, out) - - // Query the token - out2, qm, err := at.Info(out.AccessorID, nil) - assert.Nil(t, err) - assertQueryMeta(t, qm) - assert.Equal(t, out, out2) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn(testClient) + }) + } } func TestACLTokens_Self(t *testing.T) { diff --git a/command/acl_bootstrap.go b/command/acl_bootstrap.go index f8970f938..f367f9cb9 100644 --- a/command/acl_bootstrap.go +++ b/command/acl_bootstrap.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "strings" + "time" "github.com/hashicorp/nomad/api" "github.com/posener/complete" @@ -163,8 +164,16 @@ func formatKVACLToken(token *api.ACLToken) string { // Add the generic output output = append(output, fmt.Sprintf("Create Time|%v", token.CreateTime), + fmt.Sprintf("Expiry Time |%s", expiryTimeString(token.ExpirationTime)), fmt.Sprintf("Create Index|%d", token.CreateIndex), fmt.Sprintf("Modify Index|%d", token.ModifyIndex), ) return formatKV(output) } + +func expiryTimeString(t *time.Time) string { + if t == nil || t.IsZero() { + return "" + } + return t.String() +} diff --git a/command/acl_bootstrap_test.go b/command/acl_bootstrap_test.go index c91588b23..96f489e02 100644 --- a/command/acl_bootstrap_test.go +++ b/command/acl_bootstrap_test.go @@ -36,6 +36,7 @@ func TestACLBootstrapCommand(t *testing.T) { out := ui.OutputWriter.String() assert.Contains(out, "Secret ID") + require.Contains(t, out, "Expiry Time = ") } // If a bootstrap token has already been created, attempts to create more should @@ -116,6 +117,7 @@ func TestACLBootstrapCommand_WithOperatorFileBootstrapToken(t *testing.T) { out := ui.OutputWriter.String() assert.Contains(t, out, mockToken.SecretID) + require.Contains(t, out, "Expiry Time = ") } // Attempting to bootstrap the server with an invalid operator provided token in a file should diff --git a/command/acl_token_create.go b/command/acl_token_create.go index eadb66ee6..de1962849 100644 --- a/command/acl_token_create.go +++ b/command/acl_token_create.go @@ -3,6 +3,7 @@ package command import ( "fmt" "strings" + "time" "github.com/hashicorp/nomad/api" "github.com/posener/complete" @@ -36,6 +37,11 @@ Create Options: -policy="" Specifies a policy to associate with the token. Can be specified multiple times, but only with client type tokens. + + -ttl + Specifies the time-to-live of the created ACL token. This takes the form of + a time duration such as "5m" and "1h". By default, tokens will be created + without a TTL and therefore never expire. ` return strings.TrimSpace(helpText) } @@ -47,6 +53,7 @@ func (c *ACLTokenCreateCommand) AutocompleteFlags() complete.Flags { "type": complete.PredictAnything, "global": complete.PredictNothing, "policy": complete.PredictAnything, + "ttl": complete.PredictAnything, }) } @@ -61,7 +68,7 @@ func (c *ACLTokenCreateCommand) Synopsis() string { func (c *ACLTokenCreateCommand) Name() string { return "acl token create" } func (c *ACLTokenCreateCommand) Run(args []string) int { - var name, tokenType string + var name, tokenType, ttl string var global bool var policies []string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) @@ -69,6 +76,7 @@ func (c *ACLTokenCreateCommand) Run(args []string) int { flags.StringVar(&name, "name", "", "") flags.StringVar(&tokenType, "type", "client", "") flags.BoolVar(&global, "global", false, "") + flags.StringVar(&ttl, "ttl", "", "") flags.Var((funcVar)(func(s string) error { policies = append(policies, s) return nil @@ -93,6 +101,17 @@ func (c *ACLTokenCreateCommand) Run(args []string) int { Global: global, } + // If the user set a TTL flag value, convert this to a time duration and + // add it to our token request object. + if ttl != "" { + ttlDuration, err := time.ParseDuration(ttl) + if err != nil { + c.Ui.Error(fmt.Sprintf("Failed to parse TTL as time duration: %s", err)) + return 1 + } + tk.ExpirationTTL = ttlDuration + } + // Get the HTTP client client, err := c.Meta.Client() if err != nil { diff --git a/command/acl_token_create_test.go b/command/acl_token_create_test.go index e24e4c507..8eb782686 100644 --- a/command/acl_token_create_test.go +++ b/command/acl_token_create_test.go @@ -1,18 +1,17 @@ package command import ( - "strings" "testing" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/command/agent" "github.com/mitchellh/cli" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestACLTokenCreateCommand(t *testing.T) { ci.Parallel(t) - assert := assert.New(t) + config := func(c *agent.Config) { c.ACL.Enabled = true } @@ -22,22 +21,32 @@ func TestACLTokenCreateCommand(t *testing.T) { // Bootstrap an initial ACL token token := srv.RootToken - assert.NotNil(token, "failed to bootstrap ACL token") + require.NotNil(t, token, "failed to bootstrap ACL token") ui := cli.NewMockUi() cmd := &ACLTokenCreateCommand{Meta: Meta{Ui: ui, flagAddress: url}} // Request to create a new token without providing a valid management token code := cmd.Run([]string{"-address=" + url, "-token=foo", "-policy=foo", "-type=client"}) - assert.Equal(1, code) + require.Equal(t, 1, code) - // Request to create a new token with a valid management token + // Request to create a new token with a valid management token that does + // not have an expiry set. code = cmd.Run([]string{"-address=" + url, "-token=" + token.SecretID, "-policy=foo", "-type=client"}) - assert.Equal(0, code) + require.Equal(t, 0, code) // Check the output out := ui.OutputWriter.String() - if !strings.Contains(out, "[foo]") { - t.Fatalf("bad: %v", out) - } + require.Contains(t, out, "[foo]") + require.Contains(t, out, "Expiry Time = ") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Create a new token that has an expiry TTL set and check the response. + code = cmd.Run([]string{"-address=" + url, "-token=" + token.SecretID, "-type=management", "-ttl=10m"}) + require.Equal(t, 0, code) + + out = ui.OutputWriter.String() + require.NotContains(t, out, "Expiry Time = ") } diff --git a/command/acl_token_list.go b/command/acl_token_list.go index 9e3abe4c5..5946302fd 100644 --- a/command/acl_token_list.go +++ b/command/acl_token_list.go @@ -3,6 +3,7 @@ package command import ( "fmt" "strings" + "time" "github.com/hashicorp/nomad/api" "github.com/posener/complete" @@ -108,9 +109,17 @@ func formatTokens(tokens []*api.ACLTokenListStub) string { } output := make([]string, 0, len(tokens)+1) - output = append(output, "Name|Type|Global|Accessor ID") + output = append(output, "Name|Type|Global|Accessor ID|Expired") for _, p := range tokens { - output = append(output, fmt.Sprintf("%s|%s|%t|%s", p.Name, p.Type, p.Global, p.AccessorID)) + expired := false + if p.ExpirationTime != nil && !p.ExpirationTime.IsZero() { + if p.ExpirationTime.Before(time.Now().UTC()) { + expired = true + } + } + + output = append(output, fmt.Sprintf( + "%s|%s|%t|%s|%v", p.Name, p.Type, p.Global, p.AccessorID, expired)) } return formatList(output) From e660c9a908ed8a76e08461968df7dd41fb23f52c Mon Sep 17 00:00:00 2001 From: James Rasell Date: Tue, 9 Aug 2022 09:33:41 +0200 Subject: [PATCH 06/19] core: add ACL role state schema and functionality. (#13955) This commit includes the new state schema for ACL roles along with state interaction functions for CRUD actions. The change also includes snapshot persist and restore functionality and the addition of FSM messages for Raft updates which will come via RPC endpoints. --- helper/raftutil/msgtypes.go | 2 + nomad/fsm.go | 80 ++++++ nomad/fsm_test.go | 104 +++++++ nomad/mock/mock.go | 16 ++ nomad/state/schema.go | 27 ++ nomad/state/state_store_acl.go | 197 +++++++++++++ nomad/state/state_store_acl_test.go | 295 ++++++++++++++++++++ nomad/state/state_store_restore.go | 9 + nomad/state/state_store_restore_test.go | 23 ++ nomad/structs/acl.go | 161 +++++++++++ nomad/structs/acl_test.go | 353 ++++++++++++++++++++++++ nomad/structs/structs.go | 2 + 12 files changed, 1269 insertions(+) diff --git a/helper/raftutil/msgtypes.go b/helper/raftutil/msgtypes.go index efd85afc5..d0249625f 100644 --- a/helper/raftutil/msgtypes.go +++ b/helper/raftutil/msgtypes.go @@ -58,6 +58,8 @@ var msgTypeNames = map[structs.MessageType]string{ structs.SecureVariableDeleteRequestType: "SecureVariableDeleteRequestType", structs.RootKeyMetaUpsertRequestType: "RootKeyMetaUpsertRequestType", structs.RootKeyMetaDeleteRequestType: "RootKeyMetaDeleteRequestType", + structs.ACLRolesUpsertRequestType: "ACLRolesUpsertRequestType", + structs.ACLRolesDeleteByIDRequestType: "ACLRolesDeleteByIDRequestType", structs.NamespaceUpsertRequestType: "NamespaceUpsertRequestType", structs.NamespaceDeleteRequestType: "NamespaceDeleteRequestType", } diff --git a/nomad/fsm.go b/nomad/fsm.go index 4339dcf72..abfca1ce7 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -58,6 +58,7 @@ const ( SecureVariablesSnapshot SnapshotType = 22 SecureVariablesQuotaSnapshot SnapshotType = 23 RootKeyMetaSnapshot SnapshotType = 24 + ACLRoleSnapshot SnapshotType = 25 // Namespace appliers were moved from enterprise and therefore start at 64 NamespaceSnapshot SnapshotType = 64 @@ -325,6 +326,10 @@ func (n *nomadFSM) Apply(log *raft.Log) interface{} { return n.applyRootKeyMetaUpsert(msgType, buf[1:], log.Index) case structs.RootKeyMetaDeleteRequestType: return n.applyRootKeyMetaDelete(msgType, buf[1:], log.Index) + case structs.ACLRolesUpsertRequestType: + return n.applyACLRolesUpsert(msgType, buf[1:], log.Index) + case structs.ACLRolesDeleteByIDRequestType: + return n.applyACLRolesDeleteByID(msgType, buf[1:], log.Index) } // Check enterprise only message types. @@ -1750,6 +1755,20 @@ func (n *nomadFSM) restoreImpl(old io.ReadCloser, filter *FSMFilter) error { if err := restore.RootKeyMetaRestore(keyMeta); err != nil { return err } + case ACLRoleSnapshot: + + // Create a new ACLRole object, so we can decode the message into + // it. + aclRole := new(structs.ACLRole) + + if err := dec.Decode(aclRole); err != nil { + return err + } + + // Perform the restoration. + if err := restore.ACLRoleRestore(aclRole); err != nil { + return err + } default: // Check if this is an enterprise only object being restored @@ -2010,6 +2029,36 @@ func (n *nomadFSM) applyDeleteServiceRegistrationByNodeID(msgType structs.Messag return nil } +func (n *nomadFSM) applyACLRolesUpsert(msgType structs.MessageType, buf []byte, index uint64) interface{} { + defer metrics.MeasureSince([]string{"nomad", "fsm", "apply_acl_role_upsert"}, time.Now()) + var req structs.ACLRolesUpsertRequest + if err := structs.Decode(buf, &req); err != nil { + panic(fmt.Errorf("failed to decode request: %v", err)) + } + + if err := n.state.UpsertACLRoles(msgType, index, req.ACLRoles); err != nil { + n.logger.Error("UpsertACLRoles failed", "error", err) + return err + } + + return nil +} + +func (n *nomadFSM) applyACLRolesDeleteByID(msgType structs.MessageType, buf []byte, index uint64) interface{} { + defer metrics.MeasureSince([]string{"nomad", "fsm", "apply_acl_role_delete_by_id"}, time.Now()) + var req structs.ACLRolesDeleteByIDRequest + if err := structs.Decode(buf, &req); err != nil { + panic(fmt.Errorf("failed to decode request: %v", err)) + } + + if err := n.state.DeleteACLRolesByID(msgType, index, req.ACLRoleIDs); err != nil { + n.logger.Error("DeleteACLRolesByID failed", "error", err) + return err + } + + return nil +} + type FSMFilter struct { evaluator *bexpr.Evaluator } @@ -2218,6 +2267,10 @@ func (s *nomadSnapshot) Persist(sink raft.SnapshotSink) error { sink.Cancel() return err } + if err := s.persistACLRoles(sink, encoder); err != nil { + sink.Cancel() + return err + } return nil } @@ -2845,6 +2898,33 @@ func (s *nomadSnapshot) persistRootKeyMeta(sink raft.SnapshotSink, return nil } +func (s *nomadSnapshot) persistACLRoles(sink raft.SnapshotSink, + encoder *codec.Encoder) error { + + // Get all the ACL roles. + ws := memdb.NewWatchSet() + aclRolesIter, err := s.snap.GetACLRoles(ws) + if err != nil { + return err + } + + for { + // Get the next item. + for raw := aclRolesIter.Next(); raw != nil; raw = aclRolesIter.Next() { + + // Prepare the request struct. + role := raw.(*structs.ACLRole) + + // Write out an ACL role snapshot. + sink.Write([]byte{byte(ACLRoleSnapshot)}) + if err := encoder.Encode(role); err != nil { + return err + } + } + return nil + } +} + // Release is a no-op, as we just need to GC the pointer // to the state store snapshot. There is nothing to explicitly // cleanup. diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index fdfcac309..3f18cad57 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -2893,6 +2893,43 @@ func TestFSM_SnapshotRestore_ServiceRegistrations(t *testing.T) { require.ElementsMatch(t, restoredRegs, serviceRegs) } +func TestFSM_SnapshotRestore_ACLRoles(t *testing.T) { + ci.Parallel(t) + + // Create our initial FSM which will be snapshotted. + fsm := testFSM(t) + testState := fsm.State() + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate and upsert some ACL roles. + aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + + // Perform a snapshot restore. + restoredFSM := testSnapshotRestore(t, fsm) + restoredState := restoredFSM.State() + + // List the ACL roles from restored state and ensure everything is as + // expected. + iter, err := restoredState.GetACLRoles(memdb.NewWatchSet()) + require.NoError(t, err) + + var restoredACLRoles []*structs.ACLRole + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + restoredACLRoles = append(restoredACLRoles, raw.(*structs.ACLRole)) + } + require.ElementsMatch(t, restoredACLRoles, aclRoles) +} + func TestFSM_ReconcileSummaries(t *testing.T) { ci.Parallel(t) // Add some state @@ -3413,6 +3450,73 @@ func TestFSM_SnapshotRestore_SecureVariables(t *testing.T) { require.ElementsMatch(t, restoredSVs, svs) } +func TestFSM_ApplyACLRolesUpsert(t *testing.T) { + ci.Parallel(t) + fsm := testFSM(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate the upsert request and apply the change. + req := structs.ACLRolesUpsertRequest{ + ACLRoles: []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()}, + } + buf, err := structs.Encode(structs.ACLRolesUpsertRequestType, req) + require.NoError(t, err) + require.Nil(t, fsm.Apply(makeLog(buf))) + + // Read out both ACL roles and perform an equality check using the hash. + ws := memdb.NewWatchSet() + out, err := fsm.State().GetACLRoleByName(ws, req.ACLRoles[0].Name) + require.NoError(t, err) + require.Equal(t, req.ACLRoles[0].Hash, out.Hash) + + out, err = fsm.State().GetACLRoleByName(ws, req.ACLRoles[1].Name) + require.NoError(t, err) + require.Equal(t, req.ACLRoles[1].Hash, out.Hash) +} + +func TestFSM_ApplyACLRolesDeleteByID(t *testing.T) { + ci.Parallel(t) + fsm := testFSM(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate and upsert two ACL roles. + aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, fsm.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + + // Build and apply our message. + req := structs.ACLRolesDeleteByIDRequest{ACLRoleIDs: []string{aclRoles[0].ID, aclRoles[1].ID}} + buf, err := structs.Encode(structs.ACLRolesDeleteByIDRequestType, req) + require.NoError(t, err) + require.Nil(t, fsm.Apply(makeLog(buf))) + + // List all ACL roles within state to ensure both have been removed. + ws := memdb.NewWatchSet() + iter, err := fsm.State().GetACLRoles(ws) + require.NoError(t, err) + + var count int + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count++ + } + require.Equal(t, 0, count) +} + func TestFSM_ACLEvents(t *testing.T) { ci.Parallel(t) diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index 09c781296..b4afc5de2 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -2438,3 +2438,19 @@ func mockSecureVariableMetadata() structs.SecureVariableMetadata { } return out } + +func ACLRole() *structs.ACLRole { + role := structs.ACLRole{ + ID: uuid.Generate(), + Name: fmt.Sprintf("acl-role-%s", uuid.Short()), + Description: "mocked-test-acl-role", + Policies: []*structs.ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + } + role.SetHash() + return &role +} diff --git a/nomad/state/schema.go b/nomad/state/schema.go index ecd15b30a..df16d1011 100644 --- a/nomad/state/schema.go +++ b/nomad/state/schema.go @@ -17,6 +17,7 @@ const ( TableSecureVariables = "secure_variables" TableSecureVariablesQuotas = "secure_variables_quota" TableRootKeyMeta = "secure_variables_root_key_meta" + TableACLRoles = "acl_roles" ) const ( @@ -29,6 +30,7 @@ const ( indexExpiresLocal = "expires-local" indexKeyID = "key_id" indexPath = "path" + indexName = "name" ) var ( @@ -80,6 +82,7 @@ func init() { secureVariablesTableSchema, secureVariablesQuotasTableSchema, secureVariablesRootKeyMetaSchema, + aclRolesTableSchema, }...) } @@ -1390,3 +1393,27 @@ func secureVariablesRootKeyMetaSchema() *memdb.TableSchema { }, } } + +func aclRolesTableSchema() *memdb.TableSchema { + return &memdb.TableSchema{ + Name: TableACLRoles, + Indexes: map[string]*memdb.IndexSchema{ + indexID: { + Name: indexID, + AllowMissing: false, + Unique: true, + Indexer: &memdb.StringFieldIndex{ + Field: "ID", + }, + }, + indexName: { + Name: indexName, + AllowMissing: false, + Unique: true, + Indexer: &memdb.StringFieldIndex{ + Field: "Name", + }, + }, + }, + } +} diff --git a/nomad/state/state_store_acl.go b/nomad/state/state_store_acl.go index 43980d54f..9fd579cf6 100644 --- a/nomad/state/state_store_acl.go +++ b/nomad/state/state_store_acl.go @@ -1,9 +1,11 @@ package state import ( + "errors" "fmt" "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/nomad/structs" ) // ACLTokensByExpired returns an array accessor IDs of expired ACL tokens. @@ -31,3 +33,198 @@ func expiresIndexName(global bool) string { } return indexExpiresLocal } + +// UpsertACLRoles is used to insert a number of ACL roles into the state store. +// It uses a single write transaction for efficiency, however, any error means +// no entries will be committed. +func (s *StateStore) UpsertACLRoles( + msgType structs.MessageType, index uint64, roles []*structs.ACLRole) error { + + // Grab a write transaction. + txn := s.db.WriteTxnMsgT(msgType, index) + defer txn.Abort() + + // updated tracks whether any inserts have been made. This allows us to + // skip updating the index table if we do not need to. + var updated bool + + // Iterate the array of roles. In the event of a single error, all inserts + // fail via the txn.Abort() defer. + for _, role := range roles { + + roleUpdated, err := s.upsertACLRoleTxn(index, txn, role) + if err != nil { + return err + } + + // Ensure we track whether any inserts have been made. + updated = updated || roleUpdated + } + + // If we did not perform any inserts, exit early. + if !updated { + return nil + } + + // Perform the index table update to mark the new insert. + if err := txn.Insert(tableIndex, &IndexEntry{TableACLRoles, index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } + + return txn.Commit() +} + +// upsertACLRoleTxn inserts a single ACL role into the state store using the +// provided write transaction. It is the responsibility of the caller to update +// the index table. +func (s *StateStore) upsertACLRoleTxn( + index uint64, txn *txn, role *structs.ACLRole) (bool, error) { + + // Ensure the role hash is not zero to provide defense in depth. This + // should be done outside the state store, so we do not spend time here + // and thus Raft, when it, can be avoided. + if len(role.Hash) == 0 { + role.SetHash() + } + + // This validation also happens within the RPC handler, but Raft latency + // could mean that by the time the state call is invoked, another Raft + // update has deleted policies detailed in role. Therefore, check again + // while in our write txn. + if err := s.validateACLRolePolicyLinksTxn(txn, role); err != nil { + return false, err + } + + existing, err := txn.First(TableACLRoles, indexID, role.ID) + if err != nil { + return false, fmt.Errorf("ACL role lookup failed: %v", err) + } + + // Set up the indexes correctly to ensure existing indexes are maintained. + if existing != nil { + exist := existing.(*structs.ACLRole) + if exist.Equals(role) { + return false, nil + } + role.CreateIndex = exist.CreateIndex + role.ModifyIndex = index + } else { + role.CreateIndex = index + role.ModifyIndex = index + } + + // Insert the role into the table. + if err := txn.Insert(TableACLRoles, role); err != nil { + return false, fmt.Errorf("ACL role insert failed: %v", err) + } + return true, nil +} + +// ValidateACLRolePolicyLinks ensures all ACL policies linked to from the ACL +// role exist within state. +func (s *StateStore) ValidateACLRolePolicyLinks(role *structs.ACLRole) error { + txn := s.db.ReadTxn() + return s.validateACLRolePolicyLinksTxn(txn, role) +} + +// validateACLRolePolicyLinksTxn is the same as ValidateACLRolePolicyLinks but +// allows callers to pass their own transaction. +func (s *StateStore) validateACLRolePolicyLinksTxn(txn *txn, role *structs.ACLRole) error { + for _, policyLink := range role.Policies { + _, existing, err := txn.FirstWatch("acl_policy", indexID, policyLink.Name) + if err != nil { + return fmt.Errorf("ACL policy lookup failed: %v", err) + } + if existing == nil { + return errors.New("ACL policy not found") + } + } + return nil +} + +// DeleteACLRolesByID is responsible for batch deleting ACL roles based on +// their ID. It uses a single write transaction for efficiency, however, any +// error means no entries will be committed. An error is produced if a role is +// not found within state which has been passed within the array. +func (s *StateStore) DeleteACLRolesByID( + msgType structs.MessageType, index uint64, roleIDs []string) error { + + txn := s.db.WriteTxnMsgT(msgType, index) + defer txn.Abort() + + for _, roleID := range roleIDs { + + existing, err := txn.First(TableACLRoles, indexID, roleID) + if err != nil { + return fmt.Errorf("ACL role lookup failed: %v", err) + } + if existing == nil { + return errors.New("ACL role not found") + } + + // Delete the existing entry from the table. + if err := txn.Delete(TableACLRoles, existing); err != nil { + return fmt.Errorf("ACL role deletion failed: %v", err) + } + } + + // Update the index table to indicate an update has occurred. + if err := txn.Insert(tableIndex, &IndexEntry{TableACLRoles, index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } + + return txn.Commit() +} + +// GetACLRoles returns an iterator that contains all ACL roles stored within +// state. +func (s *StateStore) GetACLRoles(ws memdb.WatchSet) (memdb.ResultIterator, error) { + txn := s.db.ReadTxn() + + // Walk the entire table to get all ACL roles. + iter, err := txn.Get(TableACLRoles, indexID) + if err != nil { + return nil, fmt.Errorf("ACL role lookup failed: %v", err) + } + ws.Add(iter.WatchCh()) + + return iter, nil +} + +// GetACLRoleByID returns a single ACL role specified by the input ID. The role +// object will be nil, if no matching entry was found; it is the responsibility +// of the caller to check for this. +func (s *StateStore) GetACLRoleByID(ws memdb.WatchSet, roleID string) (*structs.ACLRole, error) { + txn := s.db.ReadTxn() + + // Perform the ACL role lookup using the "id" index. + watchCh, existing, err := txn.FirstWatch(TableACLRoles, indexID, roleID) + if err != nil { + return nil, fmt.Errorf("ACL role lookup failed: %v", err) + } + ws.Add(watchCh) + + if existing != nil { + return existing.(*structs.ACLRole), nil + } + return nil, nil +} + +// GetACLRoleByName returns a single ACL role specified by the input name. The +// role object will be nil, if no matching entry was found; it is the +// responsibility of the caller to check for this. +func (s *StateStore) GetACLRoleByName(ws memdb.WatchSet, roleName string) (*structs.ACLRole, error) { + txn := s.db.ReadTxn() + + // Perform the ACL role lookup using the "name" index. + watchCh, existing, err := txn.FirstWatch(TableACLRoles, indexName, roleName) + if err != nil { + return nil, fmt.Errorf("ACL role lookup failed: %v", err) + } + ws.Add(watchCh) + + if existing != nil { + return existing.(*structs.ACLRole), nil + } + return nil, nil +} diff --git a/nomad/state/state_store_acl_test.go b/nomad/state/state_store_acl_test.go index 6636cb0fc..007de2b04 100644 --- a/nomad/state/state_store_acl_test.go +++ b/nomad/state/state_store_acl_test.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/nomad/helper/pointer" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + "github.com/shoenig/test/must" "github.com/stretchr/testify/require" ) @@ -148,3 +149,297 @@ func Test_expiresIndexName(t *testing.T) { }) } } + +func TestStateStore_UpsertACLRoles(t *testing.T) { + ci.Parallel(t) + testState := testStateStore(t) + + // Generate a mocked ACL role for testing and attempt to upsert this + // straight into state. It should fail because the ACL policies do not + // exist. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole()} + err := testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles) + require.ErrorContains(t, err, "policy not found") + + // Create the policies our ACL roles wants to link to and then try the + // upsert again. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles)) + + // Check that the index for the table was modified as expected. + initialIndex, err := testState.Index(TableACLRoles) + require.NoError(t, err) + must.Eq(t, 20, initialIndex) + + // List all the ACL roles in the table, so we can perform a number of tests + // on the return array. + ws := memdb.NewWatchSet() + iter, err := testState.GetACLRoles(ws) + require.NoError(t, err) + + // Count how many table entries we have, to ensure it is the expected + // number. + var count int + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count++ + + // Ensure the create and modify indexes are populated correctly. + aclRole := raw.(*structs.ACLRole) + must.Eq(t, 20, aclRole.CreateIndex) + must.Eq(t, 20, aclRole.ModifyIndex) + } + require.Equal(t, 1, count, "incorrect number of ACL roles found") + + // Try writing the same ACL roles to state which should not result in an + // update to the table index. + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 30, mockedACLRoles)) + reInsertActualIndex, err := testState.Index(TableACLRoles) + require.NoError(t, err) + must.Eq(t, 20, reInsertActualIndex) + + // Make a change to one of the ACL roles and ensure this update is accepted + // and the table index is updated. + updatedMockedACLRole := mockedACLRoles[0].Copy() + updatedMockedACLRole.Policies = []*structs.ACLRolePolicyLink{{Name: "mocked-test-policy-1"}} + updatedMockedACLRole.SetHash() + require.NoError(t, testState.UpsertACLRoles( + structs.MsgTypeTestSetup, 30, []*structs.ACLRole{updatedMockedACLRole})) + + // Check that the index for the table was modified as expected. + updatedIndex, err := testState.Index(TableACLRoles) + require.NoError(t, err) + must.Eq(t, 30, updatedIndex) + + // List the ACL roles in state. + iter, err = testState.GetACLRoles(ws) + require.NoError(t, err) + + // Count how many table entries we have, to ensure it is the expected + // number. + count = 0 + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count++ + + // Ensure the create and modify indexes are populated correctly. + aclRole := raw.(*structs.ACLRole) + must.Eq(t, 20, aclRole.CreateIndex) + must.Eq(t, 30, aclRole.ModifyIndex) + } + require.Equal(t, 1, count, "incorrect number of ACL roles found") +} + +func TestStateStore_ValidateACLRolePolicyLinks(t *testing.T) { + ci.Parallel(t) + testState := testStateStore(t) + + // Create our mocked role which includes two ACL policy links. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole()} + + // This should error as no policies exist within state. + err := testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles) + require.ErrorContains(t, err, "ACL policy not found") + + // Upsert one ACL policy and retry the role which should still fail. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + + require.NoError(t, testState.UpsertACLPolicies(structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1})) + err = testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles) + require.ErrorContains(t, err, "ACL policy not found") + + // Upsert the second ACL policy. The ACL role should now upsert into state + // without error. + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies(structs.MsgTypeTestSetup, 20, []*structs.ACLPolicy{policy2})) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 30, mockedACLRoles)) +} + +func TestStateStore_DeleteACLRolesByID(t *testing.T) { + ci.Parallel(t) + testState := testStateStore(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate a some mocked ACL roles for testing and upsert these straight + // into state. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + + // Try and delete a role using a name that doesn't exist. This should + // return an error and not change the index for the table. + err := testState.DeleteACLRolesByID(structs.MsgTypeTestSetup, 20, []string{"not-a-role"}) + require.ErrorContains(t, err, "ACL role not found") + + tableIndex, err := testState.Index(TableACLRoles) + require.NoError(t, err) + must.Eq(t, 10, tableIndex) + + // Delete one of the previously upserted ACL roles. This should succeed + // and modify the table index. + err = testState.DeleteACLRolesByID(structs.MsgTypeTestSetup, 20, []string{mockedACLRoles[0].ID}) + require.NoError(t, err) + + tableIndex, err = testState.Index(TableACLRoles) + require.NoError(t, err) + must.Eq(t, 20, tableIndex) + + // List the ACL roles and ensure we now only have one present and that it + // is the one we expect. + ws := memdb.NewWatchSet() + iter, err := testState.GetACLRoles(ws) + require.NoError(t, err) + + var aclRoles []*structs.ACLRole + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + aclRoles = append(aclRoles, raw.(*structs.ACLRole)) + } + + require.Len(t, aclRoles, 1, "incorrect number of ACL roles found") + require.True(t, aclRoles[0].Equals(mockedACLRoles[1])) + + // Delete the final remaining ACL role. This should succeed and modify the + // table index. + err = testState.DeleteACLRolesByID(structs.MsgTypeTestSetup, 30, []string{mockedACLRoles[1].ID}) + require.NoError(t, err) + + tableIndex, err = testState.Index(TableACLRoles) + require.NoError(t, err) + must.Eq(t, 30, tableIndex) + + // List the ACL roles and ensure we have zero entries. + iter, err = testState.GetACLRoles(ws) + require.NoError(t, err) + + aclRoles = []*structs.ACLRole{} + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + aclRoles = append(aclRoles, raw.(*structs.ACLRole)) + } + require.Len(t, aclRoles, 0, "incorrect number of ACL roles found") +} + +func TestStateStore_GetACLRoles(t *testing.T) { + ci.Parallel(t) + testState := testStateStore(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate a some mocked ACL roles for testing and upsert these straight + // into state. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + + // List the ACL roles and ensure they are exactly as we expect. + ws := memdb.NewWatchSet() + iter, err := testState.GetACLRoles(ws) + require.NoError(t, err) + + var aclRoles []*structs.ACLRole + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + aclRoles = append(aclRoles, raw.(*structs.ACLRole)) + } + + expected := mockedACLRoles + for i := range expected { + expected[i].CreateIndex = 10 + expected[i].ModifyIndex = 10 + } + + require.ElementsMatch(t, aclRoles, expected) +} + +func TestStateStore_GetACLRoleByID(t *testing.T) { + ci.Parallel(t) + testState := testStateStore(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate a some mocked ACL roles for testing and upsert these straight + // into state. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + + ws := memdb.NewWatchSet() + + // Try reading an ACL role that does not exist. + aclRole, err := testState.GetACLRoleByID(ws, "not-a-role") + require.NoError(t, err) + require.Nil(t, aclRole) + + // Read the two ACL roles that we should find. + aclRole, err = testState.GetACLRoleByID(ws, mockedACLRoles[0].ID) + require.NoError(t, err) + require.Equal(t, mockedACLRoles[0], aclRole) + + aclRole, err = testState.GetACLRoleByID(ws, mockedACLRoles[1].ID) + require.NoError(t, err) + require.Equal(t, mockedACLRoles[1], aclRole) +} + +func TestStateStore_GetACLRoleByName(t *testing.T) { + ci.Parallel(t) + testState := testStateStore(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate a some mocked ACL roles for testing and upsert these straight + // into state. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + + ws := memdb.NewWatchSet() + + // Try reading an ACL role that does not exist. + aclRole, err := testState.GetACLRoleByName(ws, "not-a-role") + require.NoError(t, err) + require.Nil(t, aclRole) + + // Read the two ACL roles that we should find. + aclRole, err = testState.GetACLRoleByName(ws, mockedACLRoles[0].Name) + require.NoError(t, err) + require.Equal(t, mockedACLRoles[0], aclRole) + + aclRole, err = testState.GetACLRoleByName(ws, mockedACLRoles[1].Name) + require.NoError(t, err) + require.Equal(t, mockedACLRoles[1], aclRole) +} diff --git a/nomad/state/state_store_restore.go b/nomad/state/state_store_restore.go index a746d0aa5..17faa5d5b 100644 --- a/nomad/state/state_store_restore.go +++ b/nomad/state/state_store_restore.go @@ -224,3 +224,12 @@ func (r *StateRestore) RootKeyMetaRestore(quota *structs.RootKeyMeta) error { } return nil } + +// ACLRoleRestore is used to restore a single ACL role into the acl_roles +// table. +func (r *StateRestore) ACLRoleRestore(aclRole *structs.ACLRole) error { + if err := r.txn.Insert(TableACLRoles, aclRole); err != nil { + return fmt.Errorf("ACL role insert failed: %v", err) + } + return nil +} diff --git a/nomad/state/state_store_restore_test.go b/nomad/state/state_store_restore_test.go index 4574ee495..e90a29146 100644 --- a/nomad/state/state_store_restore_test.go +++ b/nomad/state/state_store_restore_test.go @@ -604,3 +604,26 @@ func TestStateStore_SecureVariablesRestore(t *testing.T) { require.Equal(t, svs[i], out) } } + +func TestStateStore_ACLRoleRestore(t *testing.T) { + ci.Parallel(t) + testState := testStateStore(t) + + // Set up our test registrations and index. + expectedIndex := uint64(13) + aclRole := mock.ACLRole() + aclRole.CreateIndex = expectedIndex + aclRole.ModifyIndex = expectedIndex + + restore, err := testState.Restore() + require.NoError(t, err) + require.NoError(t, restore.ACLRoleRestore(aclRole)) + require.NoError(t, restore.Commit()) + + // Check the state is now populated as we expect and that we can find the + // restored registrations. + ws := memdb.NewWatchSet() + out, err := testState.GetACLRoleByName(ws, aclRole.Name) + require.NoError(t, err) + require.Equal(t, aclRole, out) +} diff --git a/nomad/structs/acl.go b/nomad/structs/acl.go index 11d48fd09..6ce3c5b0c 100644 --- a/nomad/structs/acl.go +++ b/nomad/structs/acl.go @@ -1,13 +1,17 @@ package structs import ( + "bytes" "errors" "fmt" + "regexp" "time" "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper/pointer" "github.com/hashicorp/nomad/helper/uuid" + "golang.org/x/crypto/blake2b" + "golang.org/x/exp/slices" ) const ( @@ -40,6 +44,14 @@ const ( // large number of expired tokens pending garbage collection, this value is // a potential limiting factor. ACLMaxExpiredBatchSize = 4096 + + // maxACLRoleDescriptionLength limits an ACL roles description length. + maxACLRoleDescriptionLength = 256 +) + +var ( + // validACLRoleName is used to validate an ACL role name. + validACLRoleName = regexp.MustCompile("^[a-zA-Z0-9-]{1,128}$") ) // Canonicalize performs basic canonicalization on the ACL token object. It is @@ -161,3 +173,152 @@ func (a *ACLToken) IsExpired(t time.Time) bool { return a.ExpirationTime.Before(t) || t.IsZero() } + +// ACLRole is an abstraction for the ACL system which allows the grouping of +// ACL policies into a single object. ACL tokens can be created and linked to +// a role; the token then inherits all the permissions granted by the policies. +type ACLRole struct { + + // ID is an internally generated UUID for this role and is controlled by + // Nomad. + ID string + + // Name is unique across the entire set of federated clusters and is + // supplied by the operator on role creation. The name can be modified by + // updating the role and including the Nomad generated ID. This update will + // not affect tokens created and linked to this role. This is a required + // field. + Name string + + // Description is a human-readable, operator set description that can + // provide additional context about the role. This is an operational field. + Description string + + // Policies is an array of ACL policy links. Although currently policies + // can only be linked using their name, in the future we will want to add + // IDs also and thus allow operators to specify either a name, an ID, or + // both. + Policies []*ACLRolePolicyLink + + // Hash is the hashed value of the role and is generated using all fields + // above this point. + Hash []byte + + CreateIndex uint64 + ModifyIndex uint64 +} + +// ACLRolePolicyLink is used to link a policy to an ACL role. We use a struct +// rather than a list of strings as in the future we will want to add IDs to +// policies and then link via these. +type ACLRolePolicyLink struct { + + // Name is the ACLPolicy.Name value which will be linked to the ACL role. + Name string +} + +// SetHash is used to compute and set the hash of the ACL role. This should be +// called every and each time a user specified field on the role is changed +// before updating the Nomad state store. +func (a *ACLRole) SetHash() []byte { + + // Initialize a 256bit Blake2 hash (32 bytes). + hash, err := blake2b.New256(nil) + if err != nil { + panic(err) + } + + // Write all the user set fields. + _, _ = hash.Write([]byte(a.Name)) + _, _ = hash.Write([]byte(a.Description)) + + for _, policyLink := range a.Policies { + _, _ = hash.Write([]byte(policyLink.Name)) + } + + // Finalize the hash. + hashVal := hash.Sum(nil) + + // Set and return the hash. + a.Hash = hashVal + return hashVal +} + +// Validate ensure the ACL role contains valid information which meets Nomad's +// internal requirements. This does not include any state calls, such as +// ensuring the linked policies exist. +func (a *ACLRole) Validate() error { + + var mErr multierror.Error + + if !validACLRoleName.MatchString(a.Name) { + mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid name '%s'", a.Name)) + } + + if len(a.Description) > maxACLRoleDescriptionLength { + mErr.Errors = append(mErr.Errors, fmt.Errorf("description longer than %d", maxACLRoleDescriptionLength)) + } + + if len(a.Policies) < 1 { + mErr.Errors = append(mErr.Errors, errors.New("at least one policy should be specified")) + } + + return mErr.ErrorOrNil() +} + +// Equals performs an equality check on the two service registrations. It +// handles nil objects. +func (a *ACLRole) Equals(o *ACLRole) bool { + if a == nil || o == nil { + return a == o + } + if len(a.Hash) == 0 { + a.SetHash() + } + if len(o.Hash) == 0 { + o.SetHash() + } + return bytes.Equal(a.Hash, o.Hash) +} + +// Copy creates a deep copy of the ACL role. This copy can then be safely +// modified. It handles nil objects. +func (a *ACLRole) Copy() *ACLRole { + if a == nil { + return nil + } + + c := new(ACLRole) + *c = *a + + c.Policies = slices.Clone(a.Policies) + c.Hash = slices.Clone(a.Hash) + + return c +} + +// ACLRolesUpsertRequest is the request object used to upsert one or more ACL +// roles. +type ACLRolesUpsertRequest struct { + ACLRoles []*ACLRole + WriteRequest +} + +// ACLRolesUpsertResponse is the response object when one or more ACL roles +// have been successfully upserted into state. +type ACLRolesUpsertResponse struct { + WriteMeta +} + +// ACLRolesDeleteByIDRequest is the request object to delete one or more ACL +// roles using the role ID. +type ACLRolesDeleteByIDRequest struct { + ACLRoleIDs []string + WriteRequest +} + +// ACLRolesDeleteByIDResponse is the response object when performing a deletion +// of one or more ACL roles using the role ID. +type ACLRolesDeleteByIDResponse struct { + WriteMeta +} diff --git a/nomad/structs/acl_test.go b/nomad/structs/acl_test.go index 3e58f1df8..d2621d415 100644 --- a/nomad/structs/acl_test.go +++ b/nomad/structs/acl_test.go @@ -1,6 +1,7 @@ package structs import ( + "fmt" "testing" "time" @@ -285,3 +286,355 @@ func TestACLToken_IsExpired(t *testing.T) { }) } } + +func TestACLRole_SetHash(t *testing.T) { + testCases := []struct { + name string + inputACLRole *ACLRole + expectedOutput []byte + }{ + { + name: "no hash set", + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{}, + }, + expectedOutput: []byte{ + 122, 193, 189, 171, 197, 13, 37, 81, 141, 213, 188, 212, 179, 223, 148, 160, + 171, 141, 155, 136, 21, 128, 252, 100, 149, 195, 236, 148, 94, 70, 173, 102, + }, + }, + { + name: "hash set with change", + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{ + 137, 147, 2, 29, 53, 94, 78, 13, 45, 51, 127, 193, 21, 248, 230, 126, 34, + 106, 216, 73, 248, 219, 209, 146, 204, 107, 185, 2, 89, 255, 198, 5, + }, + }, + expectedOutput: []byte{ + 122, 193, 189, 171, 197, 13, 37, 81, 141, 213, 188, 212, 179, 223, 148, 160, + 171, 141, 155, 136, 21, 128, 252, 100, 149, 195, 236, 148, 94, 70, 173, 102, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputACLRole.SetHash() + require.Equal(t, tc.expectedOutput, actualOutput) + require.Equal(t, tc.inputACLRole.Hash, actualOutput) + }) + } +} + +func TestACLRole_Validate(t *testing.T) { + testCases := []struct { + name string + inputACLRole *ACLRole + expectedError bool + expectedErrorContains string + }{ + { + name: "role name too long", + inputACLRole: &ACLRole{ + Name: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + expectedError: true, + expectedErrorContains: "invalid name", + }, + { + name: "role name too short", + inputACLRole: &ACLRole{ + Name: "", + }, + expectedError: true, + expectedErrorContains: "invalid name", + }, + { + name: "role name with invalid characters", + inputACLRole: &ACLRole{ + Name: "--#$%$^%_%%_?>", + }, + expectedError: true, + expectedErrorContains: "invalid name", + }, + { + name: "description too long", + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + expectedError: true, + expectedErrorContains: "description longer than", + }, + { + name: "no policies", + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "", + }, + expectedError: true, + expectedErrorContains: "at least one policy should be specified", + }, + { + name: "valid", + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "", + Policies: []*ACLRolePolicyLink{ + {Name: "policy-1"}, + }, + }, + expectedError: false, + expectedErrorContains: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputACLRole.Validate() + if tc.expectedError { + require.ErrorContains(t, actualOutput, tc.expectedErrorContains) + } else { + require.NoError(t, actualOutput) + } + }) + } +} + +func TestACLRole_Equals(t *testing.T) { + testCases := []struct { + name string + composedACLRole *ACLRole + inputACLRole *ACLRole + expectedOutput bool + }{ + { + name: "equal with hash set", + composedACLRole: &ACLRole{ + Name: "acl-role-", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{ + 122, 193, 189, 171, 197, 13, 37, 81, 141, 213, 188, 212, 179, 223, 148, 160, + 171, 141, 155, 136, 21, 128, 252, 100, 149, 195, 236, 148, 94, 70, 173, 102, + }, + }, + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{ + 122, 193, 189, 171, 197, 13, 37, 81, 141, 213, 188, 212, 179, 223, 148, 160, + 171, 141, 155, 136, 21, 128, 252, 100, 149, 195, 236, 148, 94, 70, 173, 102, + }, + }, + expectedOutput: true, + }, + { + name: "equal without hash set", + composedACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{}, + }, + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{}, + }, + expectedOutput: true, + }, + { + name: "both nil", + composedACLRole: nil, + inputACLRole: nil, + expectedOutput: true, + }, + { + name: "not equal composed nil", + composedACLRole: nil, + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{ + 122, 193, 189, 171, 197, 13, 37, 81, 141, 213, 188, 212, 179, 223, 148, 160, + 171, 141, 155, 136, 21, 128, 252, 100, 149, 195, 236, 148, 94, 70, 173, 102, + }, + }, + expectedOutput: false, + }, + { + name: "not equal input nil", + composedACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{ + 122, 193, 189, 171, 197, 13, 37, 81, 141, 213, 188, 212, 179, 223, 148, 160, + 171, 141, 155, 136, 21, 128, 252, 100, 149, 195, 236, 148, 94, 70, 173, 102, + }, + }, + inputACLRole: nil, + expectedOutput: false, + }, + { + name: "not equal with hash set", + composedACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{ + 122, 193, 189, 171, 197, 13, 37, 81, 141, 213, 188, 212, 179, 223, 148, 160, + 171, 141, 155, 136, 21, 128, 252, 100, 149, 195, 236, 148, 94, 70, 173, 102, + }, + }, + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{ + 137, 147, 2, 29, 53, 94, 78, 13, 45, 51, 127, 193, 21, 248, 230, 126, 34, + 106, 216, 73, 248, 219, 209, 146, 204, 107, 185, 2, 89, 255, 198, 5, + }, + }, + expectedOutput: false, + }, + { + name: "not equal without hash set", + composedACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{}, + }, + inputACLRole: &ACLRole{ + Name: "acl-role", + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{}, + }, + expectedOutput: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.composedACLRole.Equals(tc.inputACLRole) + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} + +func TestACLRole_Copy(t *testing.T) { + testCases := []struct { + name string + inputACLRole *ACLRole + }{ + { + name: "nil input", + inputACLRole: nil, + }, + { + name: "general 1", + inputACLRole: &ACLRole{ + Name: fmt.Sprintf("acl-role"), + Description: "mocked-test-acl-role", + Policies: []*ACLRolePolicyLink{ + {Name: "mocked-test-policy-1"}, + {Name: "mocked-test-policy-2"}, + }, + CreateIndex: 10, + ModifyIndex: 10, + Hash: []byte{ + 122, 193, 189, 171, 197, 13, 37, 81, 141, 213, 188, 212, 179, 223, 148, 160, + 171, 141, 155, 136, 21, 128, 252, 100, 149, 195, 236, 148, 94, 70, 173, 102, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputACLRole.Copy() + require.Equal(t, tc.inputACLRole, actualOutput) + }) + } +} + +func Test_ACLRolesUpsertRequest(t *testing.T) { + req := ACLRolesUpsertRequest{} + require.False(t, req.IsRead()) +} + +func Test_ACLRolesDeleteByIDRequest(t *testing.T) { + req := ACLRolesDeleteByIDRequest{} + require.False(t, req.IsRead()) +} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 4cc4c5512..f163a25d9 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -113,6 +113,8 @@ const ( SecureVariableDeleteRequestType MessageType = 51 RootKeyMetaUpsertRequestType MessageType = 52 RootKeyMetaDeleteRequestType MessageType = 53 + ACLRolesUpsertRequestType MessageType = 54 + ACLRolesDeleteByIDRequestType MessageType = 55 // Namespace types were moved from enterprise and therefore start at 64 NamespaceUpsertRequestType MessageType = 64 From 581a5bb6adad2f54390ce44657e67897eedba16d Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 11 Aug 2022 08:43:50 +0100 Subject: [PATCH 07/19] rpc: add ACL Role RPC endpoint for CRUD actions. New ACL Role RPC endpoints have been created to allow the creation, update, read, and deletion of ACL roles. All endpoints require a management token; in the future readers will also be allowed to view roles associated to their ACL token. The create endpoint in particular is responsible for deduplicating ACL policy links and ensuring named policies are found within state. This is done within the RPC handler so we perform a single loop through the links for slight efficiency. --- nomad/acl_endpoint.go | 340 +++++++++++++++++++++++- nomad/acl_endpoint_test.go | 387 ++++++++++++++++++++++++++++ nomad/state/state_store_acl.go | 21 +- nomad/state/state_store_acl_test.go | 45 ++++ nomad/structs/acl.go | 84 ++++++ nomad/structs/acl_test.go | 43 ++++ 6 files changed, 912 insertions(+), 8 deletions(-) diff --git a/nomad/acl_endpoint.go b/nomad/acl_endpoint.go index f4958df71..70b386f49 100644 --- a/nomad/acl_endpoint.go +++ b/nomad/acl_endpoint.go @@ -526,7 +526,7 @@ func (a *ACL) UpsertTokens(args *structs.ACLTokenUpsertRequest, reply *structs.A if token.AccessorID != "" { out, err := stateSnapshot.ACLTokenByAccessorID(nil, token.AccessorID) if err != nil { - return structs.NewErrRPCCodedf(http.StatusBadRequest, "token lookup failed: %v", err) + return structs.NewErrRPCCodedf(http.StatusInternalServerError, "token lookup failed: %v", err) } if out == nil { return structs.NewErrRPCCodedf(http.StatusBadRequest, "cannot find token %s", token.AccessorID) @@ -1028,3 +1028,341 @@ func (a *ACL) ExpireOneTimeTokens(args *structs.OneTimeTokenExpireRequest, reply reply.Index = index return nil } + +// UpsertRoles creates or updates ACL roles held within Nomad. +func (a *ACL) UpsertRoles( + args *structs.ACLRolesUpsertRequest, + reply *structs.ACLRolesUpsertResponse) error { + + // Only allow operators to upsert ACL roles when ACLs are enabled. + if !a.srv.config.ACLEnabled { + return aclDisabled + } + + // This endpoint always forwards to the authoritative region as ACL roles + // are global. + args.Region = a.srv.config.AuthoritativeRegion + + if done, err := a.srv.forward(structs.ACLUpsertRolesRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "acl", "upsert_roles"}, time.Now()) + + // Only tokens with management level permissions can create ACL roles. + if acl, err := a.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if acl == nil || !acl.IsManagement() { + return structs.ErrPermissionDenied + } + + // Snapshot the state so we can perform lookups against the ID and policy + // links if needed. Do it here, so we only need to do this once no matter + // how many roles we are upserting. + stateSnapshot, err := a.srv.State().Snapshot() + if err != nil { + return err + } + + // Validate each role. + for idx, role := range args.ACLRoles { + + // Perform all the static validation of the ACL role object. Use the + // array index as we cannot be sure the error was caused by a missing + // name. + if err := role.Validate(); err != nil { + return structs.NewErrRPCCodedf(http.StatusBadRequest, "role %d invalid: %v", idx, err) + } + + policyNames := make(map[string]struct{}) + var policiesLinks []*structs.ACLRolePolicyLink + + // We need to deduplicate the ACL policy links within this role as well + // as ensure the policies exist within state. + for _, policyLink := range role.Policies { + + // Perform a state look up for the policy. An error or not being + // able to find the policy is terminal. We can include the name in + // the error message as it has previously been validated. + existing, err := stateSnapshot.ACLPolicyByName(nil, policyLink.Name) + if err != nil { + return structs.NewErrRPCCodedf(http.StatusInternalServerError, "policy lookup failed: %v", err) + } + if existing == nil { + return structs.NewErrRPCCodedf(http.StatusBadRequest, "cannot find policy %s", policyLink.Name) + } + + // If the policy name is not found within our map, this means we + // have not seen it previously. We need to add this to our + // deduplicated array and also mark the policy name as seen, so we + // skip any future policies of the same name. + if _, ok := policyNames[policyLink.Name]; !ok { + policiesLinks = append(policiesLinks, policyLink) + policyNames[policyLink.Name] = struct{}{} + } + } + + // Stored the potentially updated policy links within our role. + role.Policies = policiesLinks + + // If the caller has passed a role ID, this call is considered an + // update to an existing role. We should therefore ensure it is found + // within state. + if role.ID != "" { + out, err := stateSnapshot.GetACLRoleByID(nil, role.ID) + if err != nil { + return structs.NewErrRPCCodedf(http.StatusBadRequest, "role lookup failed: %v", err) + } + if out == nil { + return structs.NewErrRPCCodedf(http.StatusBadRequest, "cannot find role %s", role.ID) + } + } + + role.Canonicalize() + role.SetHash() + } + + // Update via Raft. + out, index, err := a.srv.raftApply(structs.ACLRolesUpsertRequestType, args) + if err != nil { + return err + } + + // Check if the FSM response, which is an interface, contains an error. + if err, ok := out.(error); ok && err != nil { + return err + } + + // Populate the response. We do a lookup against the state to pick up the + // proper create / modify times. + stateSnapshot, err = a.srv.State().Snapshot() + if err != nil { + return err + } + for _, role := range args.ACLRoles { + lookupACLRole, err := stateSnapshot.GetACLRoleByName(nil, role.Name) + if err != nil { + return structs.NewErrRPCCodedf(400, "ACL role lookup failed: %v", err) + } + reply.ACLRoles = append(reply.ACLRoles, lookupACLRole) + } + + // Update the index. There is no need to floor this as we are writing to + // state and therefore will get a non-zero index response. + reply.Index = index + return nil +} + +// DeleteRolesByID is used to batch delete ACL roles using the ID as the +// deletion key. +func (a *ACL) DeleteRolesByID( + args *structs.ACLRolesDeleteByIDRequest, + reply *structs.ACLRolesDeleteByIDResponse) error { + + // Only allow operators to delete ACL roles when ACLs are enabled. + if !a.srv.config.ACLEnabled { + return aclDisabled + } + + // This endpoint always forwards to the authoritative region as ACL roles + // are global. + args.Region = a.srv.config.AuthoritativeRegion + + if done, err := a.srv.forward(structs.ACLDeleteRolesByIDRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "acl", "delete_roles"}, time.Now()) + + // Only tokens with management level permissions can create ACL roles. + if acl, err := a.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if acl == nil || !acl.IsManagement() { + return structs.ErrPermissionDenied + } + + // Update via Raft. + out, index, err := a.srv.raftApply(structs.ACLRolesDeleteByIDRequestType, args) + if err != nil { + return err + } + + // Check if the FSM response, which is an interface, contains an error. + if err, ok := out.(error); ok && err != nil { + return err + } + + // Update the index. There is no need to floor this as we are writing to + // state and therefore will get a non-zero index response. + reply.Index = index + return nil +} + +// ListRoles is used to list ACL roles within state. If not prefix is supplied, +// all ACL roles are listed, otherwise a prefix search is performed on the ACL +// role name. +func (a *ACL) ListRoles( + args *structs.ACLRolesListRequest, + reply *structs.ACLRolesListResponse) error { + + // Only allow operators to list ACL roles when ACLs are enabled. + if !a.srv.config.ACLEnabled { + return aclDisabled + } + + if done, err := a.srv.forward(structs.ACLListRolesRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "acl", "list_roles"}, time.Now()) + + // TODO (jrasell) allow callers to list role associated to their token. + if acl, err := a.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if acl == nil || !acl.IsManagement() { + return structs.ErrPermissionDenied + } + + // Set up and return the blocking query. + return a.srv.blockingRPC(&blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + + var ( + err error + iter memdb.ResultIterator + ) + + // If the operator supplied a prefix, perform a prefix search. + // Otherwise, list all ACL roles in state. + switch args.QueryOptions.Prefix { + case "": + iter, err = stateStore.GetACLRoles(ws) + default: + iter, err = stateStore.GetACLRoleByIDPrefix(ws, args.QueryOptions.Prefix) + } + if err != nil { + return err + } + + // Iterate all the results and add these to our reply object. There + // is no stub object for an ACL role and the hash is needed by the + // replication process. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + reply.ACLRoles = append(reply.ACLRoles, raw.(*structs.ACLRole)) + } + + // Use the index table to populate the query meta as we have no way + // of tracking the max index on deletes. + return a.srv.setReplyQueryMeta(stateStore, state.TableACLRoles, &reply.QueryMeta) + }, + }) +} + +// GetRoleByID is used to look up an individual ACL role using its ID. +func (a *ACL) GetRoleByID( + args *structs.ACLRoleByIDRequest, + reply *structs.ACLRoleByIDResponse) error { + + // Only allow operators to read an ACL role when ACLs are enabled. + if !a.srv.config.ACLEnabled { + return aclDisabled + } + + if done, err := a.srv.forward(structs.ACLGetRoleByIDRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "acl", "get_role_id"}, time.Now()) + + // TODO (jrasell) allow callers to detail a role associated to their token. + if acl, err := a.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if acl == nil || !acl.IsManagement() { + return structs.ErrPermissionDenied + } + + // Set up and return the blocking query. + return a.srv.blockingRPC(&blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + + // Perform a lookup for the ACL role. + out, err := stateStore.GetACLRoleByID(ws, args.RoleID) + if err != nil { + return err + } + + // Set the index correctly depending on whether the ACL role was + // found. + switch out { + case nil: + index, err := stateStore.Index(state.TableACLRoles) + if err != nil { + return err + } + reply.Index = index + default: + reply.Index = out.ModifyIndex + } + + // We didn't encounter an error looking up the index; set the ACL + // role on the reply and exit successfully. + reply.ACLRole = out + return nil + }, + }) +} + +// GetRoleByName is used to look up an individual ACL role using its name. +func (a *ACL) GetRoleByName( + args *structs.ACLRoleByNameRequest, + reply *structs.ACLRoleByNameResponse) error { + + // Only allow operators to read an ACL role when ACLs are enabled. + if !a.srv.config.ACLEnabled { + return aclDisabled + } + + if done, err := a.srv.forward(structs.ACLGetRoleByNameRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "acl", "get_role_name"}, time.Now()) + + // TODO (jrasell) allow callers to detail a role associated to their token. + if acl, err := a.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if acl == nil || !acl.IsManagement() { + return structs.ErrPermissionDenied + } + + // Set up and return the blocking query. + return a.srv.blockingRPC(&blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + + // Perform a lookup for the ACL role. + out, err := stateStore.GetACLRoleByName(ws, args.RoleName) + if err != nil { + return err + } + + // Set the index correctly depending on whether the ACL role was + // found. + switch out { + case nil: + index, err := stateStore.Index(state.TableACLRoles) + if err != nil { + return err + } + reply.Index = index + default: + reply.Index = out.ModifyIndex + } + + // We didn't encounter an error looking up the index; set the ACL + // role on the reply and exit successfully. + reply.ACLRole = out + return nil + }, + }) +} diff --git a/nomad/acl_endpoint_test.go b/nomad/acl_endpoint_test.go index 87884d9bf..a3ff68460 100644 --- a/nomad/acl_endpoint_test.go +++ b/nomad/acl_endpoint_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/hashicorp/go-memdb" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/helper/uuid" @@ -1776,3 +1777,389 @@ func TestACLEndpoint_OneTimeToken(t *testing.T) { require.NoError(t, err) require.Nil(t, ott) } + +func TestACL_UpsertRoles(t *testing.T) { + ci.Parallel(t) + + testServer, aclRootToken, testServerCleanupFn := TestACLServer(t, nil) + defer testServerCleanupFn() + codec := rpcClient(t, testServer) + testutil.WaitForLeader(t, testServer.RPC) + + // Create a mock ACL role and remove the ID so this looks like a creation. + aclRole1 := mock.ACLRole() + aclRole1.ID = "" + + // Attempt to upsert this role without setting an ACL token. This should + // fail. + aclRoleReq1 := &structs.ACLRolesUpsertRequest{ + ACLRoles: []*structs.ACLRole{aclRole1}, + WriteRequest: structs.WriteRequest{ + Region: "global", + }, + } + var aclRoleResp1 structs.ACLRolesUpsertResponse + err := msgpackrpc.CallWithCodec(codec, structs.ACLUpsertRolesRPCMethod, aclRoleReq1, &aclRoleResp1) + require.ErrorContains(t, err, "Permission denied") + + // Attempt to upsert this role again, this time setting the ACL root token. + // This should fail because the linked policies do not exist within state. + aclRoleReq2 := &structs.ACLRolesUpsertRequest{ + ACLRoles: []*structs.ACLRole{aclRole1}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp2 structs.ACLRolesUpsertResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLUpsertRolesRPCMethod, aclRoleReq2, &aclRoleResp2) + require.ErrorContains(t, err, "cannot find policy") + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testServer.fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Try the upsert a third time, which should succeed. + aclRoleReq3 := &structs.ACLRolesUpsertRequest{ + ACLRoles: []*structs.ACLRole{aclRole1}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp3 structs.ACLRolesUpsertResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLUpsertRolesRPCMethod, aclRoleReq3, &aclRoleResp3) + require.NoError(t, err) + require.Len(t, aclRoleResp3.ACLRoles, 1) + require.True(t, aclRole1.Equals(aclRoleResp3.ACLRoles[0])) + + // Perform an update of the ACL role by removing a policy and changing the + // name. + aclRole1Copy := aclRole1.Copy() + aclRole1Copy.Name = "updated-role-name" + aclRole1Copy.Policies = append(aclRole1Copy.Policies[:1], aclRole1Copy.Policies[1+1:]...) + aclRole1Copy.SetHash() + + aclRoleReq4 := &structs.ACLRolesUpsertRequest{ + ACLRoles: []*structs.ACLRole{aclRole1Copy}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp4 structs.ACLRolesUpsertResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLUpsertRolesRPCMethod, aclRoleReq4, &aclRoleResp4) + require.NoError(t, err) + require.Len(t, aclRoleResp4.ACLRoles, 1) + require.True(t, aclRole1Copy.Equals(aclRoleResp4.ACLRoles[0])) + require.Greater(t, aclRoleResp4.ACLRoles[0].ModifyIndex, aclRoleResp3.ACLRoles[0].ModifyIndex) + + // Create another ACL role that will fail validation. Attempting to upsert + // this ensures the handler is triggering the validation function. + aclRole2 := mock.ACLRole() + aclRole2.Policies = nil + + aclRoleReq5 := &structs.ACLRolesUpsertRequest{ + ACLRoles: []*structs.ACLRole{aclRole2}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp5 structs.ACLRolesUpsertResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLUpsertRolesRPCMethod, aclRoleReq5, &aclRoleResp5) + require.Error(t, err) + require.NotContains(t, err, "Permission denied") +} + +func TestACL_DeleteRolesByID(t *testing.T) { + ci.Parallel(t) + + testServer, aclRootToken, testServerCleanupFn := TestACLServer(t, nil) + defer testServerCleanupFn() + codec := rpcClient(t, testServer) + testutil.WaitForLeader(t, testServer.RPC) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testServer.fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create two ACL roles and put these directly into state. + aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + + // Attempt to delete an ACL role without setting an auth token. This should + // fail. + aclRoleReq1 := &structs.ACLRolesDeleteByIDRequest{ + ACLRoleIDs: []string{aclRoles[0].ID}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + }, + } + var aclRoleResp1 structs.ACLRolesDeleteByIDResponse + err := msgpackrpc.CallWithCodec(codec, structs.ACLDeleteRolesByIDRPCMethod, aclRoleReq1, &aclRoleResp1) + require.ErrorContains(t, err, "Permission denied") + + // Attempt to delete an ACL role now using a valid management token which + // should succeed. + aclRoleReq2 := &structs.ACLRolesDeleteByIDRequest{ + ACLRoleIDs: []string{aclRoles[0].ID}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp2 structs.ACLRolesDeleteByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLDeleteRolesByIDRPCMethod, aclRoleReq2, &aclRoleResp2) + require.NoError(t, err) + + // Ensure the deleted role is not found within state and that the other is. + ws := memdb.NewWatchSet() + iter, err := testServer.State().GetACLRoles(ws) + require.NoError(t, err) + + var aclRolesLookup []*structs.ACLRole + for raw := iter.Next(); raw != nil; raw = iter.Next() { + aclRolesLookup = append(aclRolesLookup, raw.(*structs.ACLRole)) + } + + require.Len(t, aclRolesLookup, 1) + require.True(t, aclRolesLookup[0].Equals(aclRoles[1])) + + // Try to delete the previously deleted ACL role, this should fail. + aclRoleReq3 := &structs.ACLRolesDeleteByIDRequest{ + ACLRoleIDs: []string{aclRoles[0].ID}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp3 structs.ACLRolesDeleteByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLDeleteRolesByIDRPCMethod, aclRoleReq3, &aclRoleResp3) + require.ErrorContains(t, err, "ACL role not found") +} + +func TestACL_ListRoles(t *testing.T) { + ci.Parallel(t) + + testServer, aclRootToken, testServerCleanupFn := TestACLServer(t, nil) + defer testServerCleanupFn() + codec := rpcClient(t, testServer) + testutil.WaitForLeader(t, testServer.RPC) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testServer.fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create two ACL roles with a known prefix and put these directly into + // state. + aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + aclRoles[0].ID = "prefix-" + uuid.Generate() + aclRoles[1].ID = "prefix-" + uuid.Generate() + require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + + // Try listing roles without a valid ACL token. + aclRoleReq1 := &structs.ACLRolesListRequest{ + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + }, + } + var aclRoleResp1 structs.ACLRolesListResponse + err := msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq1, &aclRoleResp1) + require.ErrorContains(t, err, "Permission denied") + + // Try listing roles with a valid ACL token. + aclRoleReq2 := &structs.ACLRolesListRequest{ + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp2 structs.ACLRolesListResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq2, &aclRoleResp2) + require.NoError(t, err) + require.Len(t, aclRoleResp2.ACLRoles, 2) + + // Try listing roles with a valid ACL token using a prefix that doesn't + // match anything. + aclRoleReq3 := &structs.ACLRolesListRequest{ + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + Prefix: "please", + }, + } + var aclRoleResp3 structs.ACLRolesListResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq3, &aclRoleResp3) + require.NoError(t, err) + require.Len(t, aclRoleResp3.ACLRoles, 0) + + // Try listing roles with a valid ACL token using a prefix that matches two + // entries. + aclRoleReq4 := &structs.ACLRolesListRequest{ + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + Prefix: "prefix-", + }, + } + var aclRoleResp4 structs.ACLRolesListResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq4, &aclRoleResp4) + require.NoError(t, err) + require.Len(t, aclRoleResp4.ACLRoles, 2) +} + +func TestACL_GetRoleByID(t *testing.T) { + ci.Parallel(t) + + testServer, aclRootToken, testServerCleanupFn := TestACLServer(t, nil) + defer testServerCleanupFn() + codec := rpcClient(t, testServer) + testutil.WaitForLeader(t, testServer.RPC) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testServer.fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create two ACL roles and put these directly into state. + aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + + // Try reading a role without setting a correct auth token. + aclRoleReq1 := &structs.ACLRoleByIDRequest{ + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + }, + } + var aclRoleResp1 structs.ACLRoleByIDResponse + err := msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByIDRPCMethod, aclRoleReq1, &aclRoleResp1) + require.ErrorContains(t, err, "Permission denied") + + // Try reading a role that doesn't exist. + aclRoleReq2 := &structs.ACLRoleByIDRequest{ + RoleID: "nope", + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp2 structs.ACLRoleByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByIDRPCMethod, aclRoleReq2, &aclRoleResp2) + require.NoError(t, err) + require.Nil(t, aclRoleResp2.ACLRole) + + // Read both our available ACL roles using a valid auth token. + aclRoleReq3 := &structs.ACLRoleByIDRequest{ + RoleID: aclRoles[0].ID, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp3 structs.ACLRoleByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByIDRPCMethod, aclRoleReq3, &aclRoleResp3) + require.NoError(t, err) + require.True(t, aclRoleResp3.ACLRole.Equals(aclRoles[0])) + + aclRoleReq4 := &structs.ACLRoleByIDRequest{ + RoleID: aclRoles[1].ID, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp4 structs.ACLRoleByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByIDRPCMethod, aclRoleReq4, &aclRoleResp4) + require.NoError(t, err) + require.True(t, aclRoleResp4.ACLRole.Equals(aclRoles[1])) +} + +func TestACL_GetRoleByName(t *testing.T) { + ci.Parallel(t) + + testServer, aclRootToken, testServerCleanupFn := TestACLServer(t, nil) + defer testServerCleanupFn() + codec := rpcClient(t, testServer) + testutil.WaitForLeader(t, testServer.RPC) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testServer.fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create two ACL roles and put these directly into state. + aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + + // Try reading a role without setting a correct auth token. + aclRoleReq1 := &structs.ACLRoleByNameRequest{ + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + }, + } + var aclRoleResp1 structs.ACLRoleByNameResponse + err := msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByNameRPCMethod, aclRoleReq1, &aclRoleResp1) + require.ErrorContains(t, err, "Permission denied") + + // Try reading a role that doesn't exist. + aclRoleReq2 := &structs.ACLRoleByNameRequest{ + RoleName: "nope", + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp2 structs.ACLRoleByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByNameRPCMethod, aclRoleReq2, &aclRoleResp2) + require.NoError(t, err) + require.Nil(t, aclRoleResp2.ACLRole) + + // Read both our available ACL roles using a valid auth token. + aclRoleReq3 := &structs.ACLRoleByNameRequest{ + RoleName: aclRoles[0].Name, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp3 structs.ACLRoleByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByNameRPCMethod, aclRoleReq3, &aclRoleResp3) + require.NoError(t, err) + require.True(t, aclRoleResp3.ACLRole.Equals(aclRoles[0])) + + aclRoleReq4 := &structs.ACLRoleByNameRequest{ + RoleName: aclRoles[1].Name, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp4 structs.ACLRoleByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByNameRPCMethod, aclRoleReq4, &aclRoleResp4) + require.NoError(t, err) + require.True(t, aclRoleResp4.ACLRole.Equals(aclRoles[1])) +} diff --git a/nomad/state/state_store_acl.go b/nomad/state/state_store_acl.go index 9fd579cf6..60df4e506 100644 --- a/nomad/state/state_store_acl.go +++ b/nomad/state/state_store_acl.go @@ -120,13 +120,6 @@ func (s *StateStore) upsertACLRoleTxn( return true, nil } -// ValidateACLRolePolicyLinks ensures all ACL policies linked to from the ACL -// role exist within state. -func (s *StateStore) ValidateACLRolePolicyLinks(role *structs.ACLRole) error { - txn := s.db.ReadTxn() - return s.validateACLRolePolicyLinksTxn(txn, role) -} - // validateACLRolePolicyLinksTxn is the same as ValidateACLRolePolicyLinks but // allows callers to pass their own transaction. func (s *StateStore) validateACLRolePolicyLinksTxn(txn *txn, role *structs.ACLRole) error { @@ -228,3 +221,17 @@ func (s *StateStore) GetACLRoleByName(ws memdb.WatchSet, roleName string) (*stru } return nil, nil } + +// GetACLRoleByIDPrefix is used to lookup ACL policies using a prefix to match +// on the ID. +func (s *StateStore) GetACLRoleByIDPrefix(ws memdb.WatchSet, idPrefix string) (memdb.ResultIterator, error) { + txn := s.db.ReadTxn() + + iter, err := txn.Get(TableACLRoles, indexID+"_prefix", idPrefix) + if err != nil { + return nil, fmt.Errorf("ACL role lookup failed: %v", err) + } + ws.Add(iter.WatchCh()) + + return iter, nil +} diff --git a/nomad/state/state_store_acl_test.go b/nomad/state/state_store_acl_test.go index 007de2b04..6cc6a0ae4 100644 --- a/nomad/state/state_store_acl_test.go +++ b/nomad/state/state_store_acl_test.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/helper/pointer" + "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/shoenig/test/must" @@ -443,3 +444,47 @@ func TestStateStore_GetACLRoleByName(t *testing.T) { require.NoError(t, err) require.Equal(t, mockedACLRoles[1], aclRole) } + +func TestStateStore_GetACLRoleByIDPrefix(t *testing.T) { + ci.Parallel(t) + testState := testStateStore(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate a some mocked ACL roles for testing and upsert these straight + // into state. Set the ID to something with a prefix we know so it is easy + // to test. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + mockedACLRoles[0].ID = "test-prefix-" + uuid.Generate() + mockedACLRoles[1].ID = "test-prefix-" + uuid.Generate() + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + + ws := memdb.NewWatchSet() + + // Try using a prefix that doesn't match any entries. + iter, err := testState.GetACLRoleByIDPrefix(ws, "nope") + require.NoError(t, err) + + var aclRoles []*structs.ACLRole + for raw := iter.Next(); raw != nil; raw = iter.Next() { + aclRoles = append(aclRoles, raw.(*structs.ACLRole)) + } + require.Len(t, aclRoles, 0) + + // Use a prefix which should match two entries in state. + iter, err = testState.GetACLRoleByIDPrefix(ws, "test-prefix-") + require.NoError(t, err) + + aclRoles = []*structs.ACLRole{} + for raw := iter.Next(); raw != nil; raw = iter.Next() { + aclRoles = append(aclRoles, raw.(*structs.ACLRole)) + } + require.Len(t, aclRoles, 2) +} diff --git a/nomad/structs/acl.go b/nomad/structs/acl.go index 6ce3c5b0c..e40a89d42 100644 --- a/nomad/structs/acl.go +++ b/nomad/structs/acl.go @@ -35,6 +35,40 @@ const ( // Args: ACLTokenDeleteRequest // Reply: GenericResponse ACLDeleteTokensRPCMethod = "ACL.DeleteTokens" + + // ACLUpsertRolesRPCMethod is the RPC method for batch creating or + // modifying ACL roles. + // + // Args: ACLRolesUpsertRequest + // Reply: ACLRolesUpsertResponse + ACLUpsertRolesRPCMethod = "ACL.UpsertRoles" + + // ACLDeleteRolesByIDRPCMethod the RPC method for batch deleting ACL + // roles by their ID. + // + // Args: ACLRolesDeleteByIDRequest + // Reply: ACLRolesDeleteByIDResponse + ACLDeleteRolesByIDRPCMethod = "ACL.DeleteRolesByID" + + // ACLListRolesRPCMethod is the RPC method for listing ACL roles. + // + // Args: ACLRolesListRequest + // Reply: ACLRolesListResponse + ACLListRolesRPCMethod = "ACL.ListRoles" + + // ACLGetRoleByIDRPCMethod is the RPC method for detailing an individual + // ACL role using its ID. + // + // Args: ACLRoleByIDRequest + // Reply: ACLRoleByIDResponse + ACLGetRoleByIDRPCMethod = "ACL.GetRoleByID" + + // ACLGetRoleByNameRPCMethod is the RPC method for detailing an individual + // ACL role using its name. + // + // Args: ACLRoleByNameRequest + // Reply: ACLRoleByNameResponse + ACLGetRoleByNameRPCMethod = "ACL.GetRoleByName" ) const ( @@ -266,6 +300,15 @@ func (a *ACLRole) Validate() error { return mErr.ErrorOrNil() } +// Canonicalize performs basic canonicalization on the ACL role object. It is +// important for callers to understand certain fields such as ID are set if it +// is empty, so copies should be taken if needed before calling this function. +func (a *ACLRole) Canonicalize() { + if a.ID == "" { + a.ID = uuid.Generate() + } +} + // Equals performs an equality check on the two service registrations. It // handles nil objects. func (a *ACLRole) Equals(o *ACLRole) bool { @@ -307,6 +350,7 @@ type ACLRolesUpsertRequest struct { // ACLRolesUpsertResponse is the response object when one or more ACL roles // have been successfully upserted into state. type ACLRolesUpsertResponse struct { + ACLRoles []*ACLRole WriteMeta } @@ -322,3 +366,43 @@ type ACLRolesDeleteByIDRequest struct { type ACLRolesDeleteByIDResponse struct { WriteMeta } + +// ACLRolesListRequest is the request object when performing ACL role listings. +type ACLRolesListRequest struct { + QueryOptions +} + +// ACLRolesListResponse is the response object when performing ACL role +// listings. +type ACLRolesListResponse struct { + ACLRoles []*ACLRole + QueryMeta +} + +// ACLRoleByIDRequest is the request object to perform a lookup of an ACL +// role using a specific ID. +type ACLRoleByIDRequest struct { + RoleID string + QueryOptions +} + +// ACLRoleByIDResponse is the response object when performing a lookup of an +// ACL role matching a specific ID. +type ACLRoleByIDResponse struct { + ACLRole *ACLRole + QueryMeta +} + +// ACLRoleByNameRequest is the request object to perform a lookup of an ACL +// role using a specific name. +type ACLRoleByNameRequest struct { + RoleName string + QueryOptions +} + +// ACLRoleByNameResponse is the response object when performing a lookup of an +// ACL role matching a specific name. +type ACLRoleByNameResponse struct { + ACLRole *ACLRole + QueryMeta +} diff --git a/nomad/structs/acl_test.go b/nomad/structs/acl_test.go index d2621d415..22e263e5e 100644 --- a/nomad/structs/acl_test.go +++ b/nomad/structs/acl_test.go @@ -418,6 +418,34 @@ func TestACLRole_Validate(t *testing.T) { } } +func TestACLRole_Canonicalize(t *testing.T) { + testCases := []struct { + name string + inputACLRole *ACLRole + }{ + { + name: "no ID set", + inputACLRole: &ACLRole{}, + }, + { + name: "id set", + inputACLRole: &ACLRole{ID: "some-random-uuid"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + existing := tc.inputACLRole.Copy() + tc.inputACLRole.Canonicalize() + if existing.ID == "" { + require.NotEmpty(t, tc.inputACLRole.ID) + } else { + require.Equal(t, existing.ID, tc.inputACLRole.ID) + } + }) + } +} + func TestACLRole_Equals(t *testing.T) { testCases := []struct { name string @@ -638,3 +666,18 @@ func Test_ACLRolesDeleteByIDRequest(t *testing.T) { req := ACLRolesDeleteByIDRequest{} require.False(t, req.IsRead()) } + +func Test_ACLRolesListRequest(t *testing.T) { + req := ACLRolesListRequest{} + require.True(t, req.IsRead()) +} + +func Test_ACLRoleByIDRequest(t *testing.T) { + req := ACLRoleByIDRequest{} + require.True(t, req.IsRead()) +} + +func Test_ACLRoleByNameRequest(t *testing.T) { + req := ACLRoleByNameRequest{} + require.True(t, req.IsRead()) +} From 9cd0dd2ff711f812fa8db2be4b72646dae255ffd Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 11 Aug 2022 08:44:19 +0100 Subject: [PATCH 08/19] http: add ACL Role HTTP endpoints for CRUD actions. These new endpoints are exposed under the /v1/acl/roles and /v1/acl/role endpoints. --- command/agent/acl_endpoint.go | 203 ++++++++++++++ command/agent/acl_endpoint_test.go | 437 +++++++++++++++++++++++++++++ command/agent/http.go | 5 + 3 files changed, 645 insertions(+) diff --git a/command/agent/acl_endpoint.go b/command/agent/acl_endpoint.go index df3f56851..ceaecb0b8 100644 --- a/command/agent/acl_endpoint.go +++ b/command/agent/acl_endpoint.go @@ -322,3 +322,206 @@ func (s *HTTPServer) ExchangeOneTimeToken(resp http.ResponseWriter, req *http.Re setIndex(resp, out.Index) return out, nil } + +// ACLRoleListRequest performs a listing of ACL roles and is callable via the +// /v1/acl/roles HTTP API. +func (s *HTTPServer) ACLRoleListRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + + // The endpoint only supports GET requests. + if req.Method != http.MethodGet { + return nil, CodedError(http.StatusMethodNotAllowed, ErrInvalidMethod) + } + + // Set up the request args and parse this to ensure the query options are + // set. + args := structs.ACLRolesListRequest{} + + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + // Perform the RPC request. + var reply structs.ACLRolesListResponse + if err := s.agent.RPC(structs.ACLListRolesRPCMethod, &args, &reply); err != nil { + return nil, err + } + + setMeta(resp, &reply.QueryMeta) + + if reply.ACLRoles == nil { + reply.ACLRoles = make([]*structs.ACLRole, 0) + } + return reply.ACLRoles, nil +} + +// ACLRoleRequest creates a new ACL role and is callable via the +// /v1/acl/role HTTP API. +func (s *HTTPServer) ACLRoleRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + + // // The endpoint only supports PUT or POST requests. + if !(req.Method == http.MethodPut || req.Method == http.MethodPost) { + return nil, CodedError(http.StatusMethodNotAllowed, ErrInvalidMethod) + } + + // Use the generic upsert function without setting an ID as this will be + // handled by the Nomad leader. + return s.aclRoleUpsertRequest(resp, req, "") +} + +// ACLRoleSpecificRequest is callable via the /v1/acl/role/ HTTP API and +// handles read via both the role name and ID, updates, and deletions. +func (s *HTTPServer) ACLRoleSpecificRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + + // Grab the suffix of the request, so we can further understand it. + reqSuffix := strings.TrimPrefix(req.URL.Path, "/v1/acl/role/") + + // Split the request suffix in order to identify whether this is a lookup + // of a service, or whether this includes a service and service identifier. + suffixParts := strings.Split(reqSuffix, "/") + + switch len(suffixParts) { + case 1: + // Ensure the role ID is not an empty string which is possible if the + // caller requested "/v1/acl/role/" + if suffixParts[0] == "" { + return nil, CodedError(http.StatusBadRequest, "missing ACL role ID") + } + return s.aclRoleRequest(resp, req, suffixParts[0]) + case 2: + // This endpoint only supports GET. + if req.Method != http.MethodGet { + return nil, CodedError(http.StatusMethodNotAllowed, ErrInvalidMethod) + } + + // Ensure that the path is correct, otherwise the call could use + // "/v1/acl/role/foobar/role-name" and successfully pass through here. + if suffixParts[0] != "name" { + return nil, CodedError(http.StatusBadRequest, "invalid URI") + } + + // Ensure the role name is not an empty string which is possible if the + // caller requested "/v1/acl/role/name/" + if suffixParts[1] == "" { + return nil, CodedError(http.StatusBadRequest, "missing ACL role name") + } + + return s.aclRoleGetByNameRequest(resp, req, suffixParts[1]) + + default: + return nil, CodedError(http.StatusBadRequest, "invalid URI") + } +} + +func (s *HTTPServer) aclRoleRequest( + resp http.ResponseWriter, req *http.Request, roleID string) (interface{}, error) { + + // Identify the method which indicates which downstream function should be + // called. + switch req.Method { + case http.MethodGet: + return s.aclRoleGetByIDRequest(resp, req, roleID) + case http.MethodDelete: + return s.aclRoleDeleteRequest(resp, req, roleID) + case http.MethodPost, http.MethodPut: + return s.aclRoleUpsertRequest(resp, req, roleID) + default: + return nil, CodedError(http.StatusMethodNotAllowed, ErrInvalidMethod) + } +} + +func (s *HTTPServer) aclRoleGetByIDRequest( + resp http.ResponseWriter, req *http.Request, roleID string) (interface{}, error) { + + args := structs.ACLRoleByIDRequest{ + RoleID: roleID, + } + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + var reply structs.ACLRoleByIDResponse + if err := s.agent.RPC(structs.ACLGetRoleByIDRPCMethod, &args, &reply); err != nil { + return nil, err + } + setMeta(resp, &reply.QueryMeta) + + if reply.ACLRole == nil { + return nil, CodedError(http.StatusNotFound, "ACL role not found") + } + return reply.ACLRole, nil +} + +func (s *HTTPServer) aclRoleDeleteRequest( + resp http.ResponseWriter, req *http.Request, roleID string) (interface{}, error) { + + args := structs.ACLRolesDeleteByIDRequest{ + ACLRoleIDs: []string{roleID}, + } + s.parseWriteRequest(req, &args.WriteRequest) + + var reply structs.ACLRolesDeleteByIDResponse + if err := s.agent.RPC(structs.ACLDeleteRolesByIDRPCMethod, &args, &reply); err != nil { + return nil, err + } + setIndex(resp, reply.Index) + return nil, nil + +} + +// aclRoleUpsertRequest handles upserting an ACL to the Nomad servers. It can +// handle both new creations, and updates to existing roles. +func (s *HTTPServer) aclRoleUpsertRequest( + resp http.ResponseWriter, req *http.Request, roleID string) (interface{}, error) { + + // Decode the ACL role. + var aclRole structs.ACLRole + if err := decodeBody(req, &aclRole); err != nil { + return nil, CodedError(http.StatusInternalServerError, err.Error()) + } + + // Ensure the request path ID matches the ACL role ID that was decoded. + // Only perform this check on updates as a generic error on creation might + // be confusing to operators as there is no specific role request path. + if roleID != "" && roleID != aclRole.ID { + return nil, CodedError(http.StatusBadRequest, "ACL role ID does not match request path") + } + + args := structs.ACLRolesUpsertRequest{ + ACLRoles: []*structs.ACLRole{&aclRole}, + } + s.parseWriteRequest(req, &args.WriteRequest) + + var out structs.ACLRolesUpsertResponse + if err := s.agent.RPC(structs.ACLUpsertRolesRPCMethod, &args, &out); err != nil { + return nil, err + } + setIndex(resp, out.Index) + + if len(out.ACLRoles) > 0 { + return out.ACLRoles[0], nil + } + return nil, nil + +} + +func (s *HTTPServer) aclRoleGetByNameRequest( + resp http.ResponseWriter, req *http.Request, roleName string) (interface{}, error) { + + args := structs.ACLRoleByNameRequest{ + RoleName: roleName, + } + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + var reply structs.ACLRoleByNameResponse + if err := s.agent.RPC(structs.ACLGetRoleByNameRPCMethod, &args, &reply); err != nil { + return nil, err + } + setMeta(resp, &reply.QueryMeta) + + if reply.ACLRole == nil { + return nil, CodedError(http.StatusNotFound, "ACL role not found") + } + return reply.ACLRole, nil +} diff --git a/command/agent/acl_endpoint_test.go b/command/agent/acl_endpoint_test.go index 0e52a0899..25350fb0f 100644 --- a/command/agent/acl_endpoint_test.go +++ b/command/agent/acl_endpoint_test.go @@ -1,11 +1,13 @@ package agent import ( + "fmt" "net/http" "net/http/httptest" "testing" "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/stretchr/testify/assert" @@ -558,3 +560,438 @@ func TestHTTP_OneTimeToken(t *testing.T) { require.EqualError(t, err, structs.ErrPermissionDenied.Error()) }) } + +func TestHTTPServer_ACLRoleListRequest(t *testing.T) { + ci.Parallel(t) + + testCases := []struct { + name string + testFn func(srv *TestAgent) + }{ + { + name: "no auth token set", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/acl/roles", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleListRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "Permission denied") + require.Nil(t, obj) + }, + }, + { + name: "invalid method", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodConnect, "/v1/acl/roles", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleListRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "Invalid method") + require.Nil(t, obj) + }, + }, + { + name: "no roles in state", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/acl/roles", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleListRequest(respW, req) + require.NoError(t, err) + require.Empty(t, obj.([]*structs.ACLRole)) + }, + }, + { + name: "roles in state", + testFn: func(srv *TestAgent) { + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, srv.server.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create two ACL roles and put these directly into state. + aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, srv.server.State().UpsertACLRoles(structs.MsgTypeTestSetup, 20, aclRoles)) + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/acl/roles", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleListRequest(respW, req) + require.NoError(t, err) + require.Len(t, obj.([]*structs.ACLRole), 2) + }, + }, + { + name: "roles in state using prefix", + testFn: func(srv *TestAgent) { + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, srv.server.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create two ACL roles and put these directly into state, one + // using a custom prefix. + aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + aclRoles[1].ID = "badger-badger-badger-" + uuid.Generate() + require.NoError(t, srv.server.State().UpsertACLRoles(structs.MsgTypeTestSetup, 20, aclRoles)) + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/acl/roles?prefix=badger-badger-badger", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleListRequest(respW, req) + require.NoError(t, err) + require.Len(t, obj.([]*structs.ACLRole), 1) + require.Contains(t, obj.([]*structs.ACLRole)[0].ID, "badger-badger-badger") + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + httpACLTest(t, nil, tc.testFn) + }) + } +} + +func TestHTTPServer_ACLRoleRequest(t *testing.T) { + ci.Parallel(t) + + testCases := []struct { + name string + testFn func(srv *TestAgent) + }{ + { + name: "no auth token set", + testFn: func(srv *TestAgent) { + + // Create a mock role to use in the request body. + mockACLRole := mock.ACLRole() + mockACLRole.ID = "" + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodPut, "/v1/acl/role", encodeReq(mockACLRole)) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "Permission denied") + require.Nil(t, obj) + }, + }, + { + name: "invalid method", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodConnect, "/v1/acl/role", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "Invalid method") + require.Nil(t, obj) + }, + }, + { + name: "successful upsert", + testFn: func(srv *TestAgent) { + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, srv.server.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create a mock role to use in the request body. + mockACLRole := mock.ACLRole() + mockACLRole.ID = "" + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodPut, "/v1/acl/role", encodeReq(mockACLRole)) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleRequest(respW, req) + require.NoError(t, err) + require.NotNil(t, obj) + require.Equal(t, obj.(*structs.ACLRole).Hash, mockACLRole.Hash) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + httpACLTest(t, nil, tc.testFn) + }) + } +} + +func TestHTTPServer_ACLRoleSpecificRequest(t *testing.T) { + ci.Parallel(t) + + testCases := []struct { + name string + testFn func(srv *TestAgent) + }{ + { + name: "invalid URI", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/acl/role/name/this/is/will/not/work", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleSpecificRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "invalid URI") + require.Nil(t, obj) + }, + }, + { + name: "invalid role name lookalike URI", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/acl/role/foobar/rolename", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleSpecificRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "invalid URI") + require.Nil(t, obj) + }, + }, + { + name: "missing role name", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/acl/role/name/", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleSpecificRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "missing ACL role name") + require.Nil(t, obj) + }, + }, + { + name: "missing role ID", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/acl/role/", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleSpecificRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "missing ACL role ID") + require.Nil(t, obj) + }, + }, + { + name: "role name incorrect method", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodConnect, "/v1/acl/role/name/foobar", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleSpecificRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "Invalid method") + require.Nil(t, obj) + }, + }, + { + name: "role ID incorrect method", + testFn: func(srv *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodConnect, "/v1/acl/role/foobar", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleSpecificRequest(respW, req) + require.Error(t, err) + require.ErrorContains(t, err, "Invalid method") + require.Nil(t, obj) + }, + }, + { + name: "get role by name", + testFn: func(srv *TestAgent) { + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, srv.server.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create a mock role and put directly into state. + mockACLRole := mock.ACLRole() + require.NoError(t, srv.server.State().UpsertACLRoles( + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{mockACLRole})) + + url := fmt.Sprintf("/v1/acl/role/name/%s", mockACLRole.Name) + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleSpecificRequest(respW, req) + require.NoError(t, err) + require.Equal(t, obj.(*structs.ACLRole).Hash, mockACLRole.Hash) + }, + }, + { + name: "get, update, and delete role by ID", + testFn: func(srv *TestAgent) { + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, srv.server.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create a mock role and put directly into state. + mockACLRole := mock.ACLRole() + require.NoError(t, srv.server.State().UpsertACLRoles( + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{mockACLRole})) + + url := fmt.Sprintf("/v1/acl/role/%s", mockACLRole.ID) + + // Build the HTTP request to read the role using its ID. + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err := srv.Server.ACLRoleSpecificRequest(respW, req) + require.NoError(t, err) + require.Equal(t, obj.(*structs.ACLRole).Hash, mockACLRole.Hash) + + // Update the role policy list and make the request via the + // HTTP API. + mockACLRole.Policies = []*structs.ACLRolePolicyLink{{Name: "mocked-test-policy-1"}} + + req, err = http.NewRequest(http.MethodPost, url, encodeReq(mockACLRole)) + require.NoError(t, err) + respW = httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err = srv.Server.ACLRoleSpecificRequest(respW, req) + require.NoError(t, err) + require.Equal(t, obj.(*structs.ACLRole).Policies, mockACLRole.Policies) + + // Delete the ACL role using its ID. + req, err = http.NewRequest(http.MethodDelete, url, nil) + require.NoError(t, err) + respW = httptest.NewRecorder() + + // Ensure we have a token set. + setToken(req, srv.RootToken) + + // Send the HTTP request. + obj, err = srv.Server.ACLRoleSpecificRequest(respW, req) + require.NoError(t, err) + require.Nil(t, obj) + + // Ensure the ACL role is no longer stored within state. + aclRole, err := srv.server.State().GetACLRoleByID(nil, mockACLRole.ID) + require.NoError(t, err) + require.Nil(t, aclRole) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + httpACLTest(t, nil, tc.testFn) + }) + } +} diff --git a/command/agent/http.go b/command/agent/http.go index b0676ec82..8a73e786a 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -381,6 +381,11 @@ func (s HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/acl/token", s.wrap(s.ACLTokenSpecificRequest)) s.mux.HandleFunc("/v1/acl/token/", s.wrap(s.ACLTokenSpecificRequest)) + // Register our ACL role handlers. + s.mux.HandleFunc("/v1/acl/roles", s.wrap(s.ACLRoleListRequest)) + s.mux.HandleFunc("/v1/acl/role", s.wrap(s.ACLRoleRequest)) + s.mux.HandleFunc("/v1/acl/role/", s.wrap(s.ACLRoleSpecificRequest)) + s.mux.Handle("/v1/client/fs/", wrapCORS(s.wrap(s.FsRequest))) s.mux.HandleFunc("/v1/client/gc", s.wrap(s.ClientGCRequest)) s.mux.Handle("/v1/client/stats", wrapCORS(s.wrap(s.ClientStatsRequest))) From 87249746830ec12026e89a16e83337f63605d076 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 11 Aug 2022 08:44:37 +0100 Subject: [PATCH 09/19] api: add ACL Role API implementation for CRUD actions. --- api/acl.go | 130 ++++++++++++++++++++++++++++++++++++++++++++++++ api/acl_test.go | 74 +++++++++++++++++++++++++++ 2 files changed, 204 insertions(+) diff --git a/api/acl.go b/api/acl.go index 07b51deb4..fd7f20786 100644 --- a/api/acl.go +++ b/api/acl.go @@ -1,6 +1,7 @@ package api import ( + "errors" "fmt" "time" ) @@ -202,6 +203,96 @@ func (a *ACLTokens) ExchangeOneTimeToken(secret string, q *WriteOptions) (*ACLTo return resp.Token, wm, nil } +var ( + // errMissingACLRoleID is the generic errors to use when a call is missing + // the required ACL Role ID parameter. + errMissingACLRoleID = errors.New("missing ACL role ID") +) + +// ACLRoles is used to query the ACL Role endpoints. +type ACLRoles struct { + client *Client +} + +// ACLRoles returns a new handle on the ACL roles API client. +func (c *Client) ACLRoles() *ACLRoles { + return &ACLRoles{client: c} +} + +// List is used to detail all the ACL roles currently stored within state. +func (a *ACLRoles) List(q *QueryOptions) ([]*ACLRole, *QueryMeta, error) { + var resp []*ACLRole + qm, err := a.client.query("/v1/acl/roles", &resp, q) + if err != nil { + return nil, nil, err + } + return resp, qm, nil +} + +// Create is used to create an ACL role. +func (a *ACLRoles) Create(role *ACLRole, w *WriteOptions) (*ACLRole, *WriteMeta, error) { + if role.ID != "" { + return nil, nil, errors.New("cannot specify ACL role ID") + } + var resp ACLRole + wm, err := a.client.write("/v1/acl/role", role, &resp, w) + if err != nil { + return nil, nil, err + } + return &resp, wm, nil +} + +// Update is used to update an existing ACL role. +func (a *ACLRoles) Update(role *ACLRole, w *WriteOptions) (*ACLRole, *WriteMeta, error) { + if role.ID == "" { + return nil, nil, errMissingACLRoleID + } + var resp ACLRole + wm, err := a.client.write("/v1/acl/role/"+role.ID, role, &resp, w) + if err != nil { + return nil, nil, err + } + return &resp, wm, nil +} + +// Delete is used to delete an ACL role. +func (a *ACLRoles) Delete(roleID string, w *WriteOptions) (*WriteMeta, error) { + if roleID == "" { + return nil, errMissingACLRoleID + } + wm, err := a.client.delete("/v1/acl/role/"+roleID, nil, nil, w) + if err != nil { + return nil, err + } + return wm, nil +} + +// Get is used to look up an ACL role. +func (a *ACLRoles) Get(roleID string, q *QueryOptions) (*ACLRole, *QueryMeta, error) { + if roleID == "" { + return nil, nil, errMissingACLRoleID + } + var resp ACLRole + qm, err := a.client.query("/v1/acl/role/"+roleID, &resp, q) + if err != nil { + return nil, nil, err + } + return &resp, qm, nil +} + +// GetByName is used to look up an ACL role using its name. +func (a *ACLRoles) GetByName(roleName string, q *QueryOptions) (*ACLRole, *QueryMeta, error) { + if roleName == "" { + return nil, nil, errors.New("missing ACL role name") + } + var resp ACLRole + qm, err := a.client.query("/v1/acl/role/name/"+roleName, &resp, q) + if err != nil { + return nil, nil, err + } + return &resp, qm, nil +} + // ACLPolicyListStub is used to for listing ACL policies type ACLPolicyListStub struct { Name string @@ -285,3 +376,42 @@ type OneTimeTokenExchangeResponse struct { type BootstrapRequest struct { BootstrapSecret string } + +// ACLRole is an abstraction for the ACL system which allows the grouping of +// ACL policies into a single object. ACL tokens can be created and linked to +// a role; the token then inherits all the permissions granted by the policies. +type ACLRole struct { + + // ID is an internally generated UUID for this role and is controlled by + // Nomad. It can be used after role creation to update the existing role. + ID string + + // Name is unique across the entire set of federated clusters and is + // supplied by the operator on role creation. The name can be modified by + // updating the role and including the Nomad generated ID. This update will + // not affect tokens created and linked to this role. This is a required + // field. + Name string + + // Description is a human-readable, operator set description that can + // provide additional context about the role. This is an optional field. + Description string + + // Policies is an array of ACL policy links. Although currently policies + // can only be linked using their name, in the future we will want to add + // IDs also and thus allow operators to specify either a name, an ID, or + // both. At least one entry is required. + Policies []*ACLRolePolicyLink + + CreateIndex uint64 + ModifyIndex uint64 +} + +// ACLRolePolicyLink is used to link a policy to an ACL role. We use a struct +// rather than a list of strings as in the future we will want to add IDs to +// policies and then link via these. +type ACLRolePolicyLink struct { + + // Name is the ACLPolicy.Name value which will be linked to the ACL role. + Name string +} diff --git a/api/acl_test.go b/api/acl_test.go index 7a0434b54..4487a7778 100644 --- a/api/acl_test.go +++ b/api/acl_test.go @@ -346,3 +346,77 @@ func TestACLTokens_BootstrapValidToken(t *testing.T) { assertWriteMeta(t, wm) assert.Equal(t, bootkn, out.SecretID) } + +func TestACLRoles(t *testing.T) { + testutil.Parallel(t) + + testClient, testServer, _ := makeACLClient(t, nil, nil) + defer testServer.Stop() + + // An initial listing shouldn't return any results. + aclRoleListResp, queryMeta, err := testClient.ACLRoles().List(nil) + require.NoError(t, err) + require.Empty(t, aclRoleListResp) + assertQueryMeta(t, queryMeta) + + // Create an ACL policy that can be referenced within the ACL role. + aclPolicy := ACLPolicy{ + Name: "acl-role-api-test", + Rules: `namespace "default" { + policy = "read" + } + `, + } + writeMeta, err := testClient.ACLPolicies().Upsert(&aclPolicy, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + + // Create an ACL role referencing the previously created policy. + role := ACLRole{ + Name: "acl-role-api-test", + Policies: []*ACLRolePolicyLink{{Name: aclPolicy.Name}}, + } + aclRoleCreateResp, writeMeta, err := testClient.ACLRoles().Create(&role, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + require.NotEmpty(t, aclRoleCreateResp.ID) + require.Equal(t, role.Name, aclRoleCreateResp.Name) + + // Another listing should return one result. + aclRoleListResp, queryMeta, err = testClient.ACLRoles().List(nil) + require.NoError(t, err) + require.Len(t, aclRoleListResp, 1) + assertQueryMeta(t, queryMeta) + + // Read the role using its ID. + aclRoleReadResp, queryMeta, err := testClient.ACLRoles().Get(aclRoleCreateResp.ID, nil) + require.NoError(t, err) + assertQueryMeta(t, queryMeta) + require.Equal(t, aclRoleCreateResp, aclRoleReadResp) + + // Read the role using its name. + aclRoleReadResp, queryMeta, err = testClient.ACLRoles().GetByName(aclRoleCreateResp.Name, nil) + require.NoError(t, err) + assertQueryMeta(t, queryMeta) + require.Equal(t, aclRoleCreateResp, aclRoleReadResp) + + // Update the role name. + role.Name = "acl-role-api-test-badger-badger-badger" + role.ID = aclRoleCreateResp.ID + aclRoleUpdateResp, writeMeta, err := testClient.ACLRoles().Update(&role, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + require.Equal(t, role.Name, aclRoleUpdateResp.Name) + require.Equal(t, role.ID, aclRoleUpdateResp.ID) + + // Delete the role. + writeMeta, err = testClient.ACLRoles().Delete(aclRoleCreateResp.ID, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + + // Make sure there are no ACL roles now present. + aclRoleListResp, queryMeta, err = testClient.ACLRoles().List(nil) + require.NoError(t, err) + require.Empty(t, aclRoleListResp) + assertQueryMeta(t, queryMeta) +} From 9c97560ded0ee761cdb732690be862470d9e82a1 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Fri, 12 Aug 2022 09:52:32 +0200 Subject: [PATCH 10/19] cli: add new acl role subcommands for CRUD role actions. (#14087) --- command/acl_role.go | 102 +++++++++++++++ command/acl_role_create.go | 148 ++++++++++++++++++++++ command/acl_role_create_test.go | 80 ++++++++++++ command/acl_role_delete.go | 83 ++++++++++++ command/acl_role_delete_test.go | 77 ++++++++++++ command/acl_role_info.go | 121 ++++++++++++++++++ command/acl_role_info_test.go | 95 ++++++++++++++ command/acl_role_list.go | 123 ++++++++++++++++++ command/acl_role_list_test.go | 77 ++++++++++++ command/acl_role_test.go | 61 +++++++++ command/acl_role_update.go | 217 ++++++++++++++++++++++++++++++++ command/acl_role_update_test.go | 124 ++++++++++++++++++ command/commands.go | 30 +++++ 13 files changed, 1338 insertions(+) create mode 100644 command/acl_role.go create mode 100644 command/acl_role_create.go create mode 100644 command/acl_role_create_test.go create mode 100644 command/acl_role_delete.go create mode 100644 command/acl_role_delete_test.go create mode 100644 command/acl_role_info.go create mode 100644 command/acl_role_info_test.go create mode 100644 command/acl_role_list.go create mode 100644 command/acl_role_list_test.go create mode 100644 command/acl_role_test.go create mode 100644 command/acl_role_update.go create mode 100644 command/acl_role_update_test.go diff --git a/command/acl_role.go b/command/acl_role.go new file mode 100644 index 000000000..284c59b9c --- /dev/null +++ b/command/acl_role.go @@ -0,0 +1,102 @@ +package command + +import ( + "fmt" + "sort" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" +) + +// Ensure ACLRoleCommand satisfies the cli.Command interface. +var _ cli.Command = &ACLRoleCommand{} + +// ACLRoleCommand implements cli.Command. +type ACLRoleCommand struct { + Meta +} + +// Help satisfies the cli.Command Help function. +func (a *ACLRoleCommand) Help() string { + helpText := ` +Usage: nomad acl role [options] [args] + + This command groups subcommands for interacting with ACL roles. Nomad's ACL + system can be used to control access to data and APIs. ACL roles are + associated with one or more ACL policies which grant specific capabilities. + For a full guide see: https://www.nomadproject.io/guides/acl.html + + Create an ACL role: + + $ nomad acl role create -name="name" -policy-name="policy-name" + + List all ACL roles: + + $ nomad acl role list + + Lookup a specific ACL role: + + $ nomad acl role info + + Update an ACL role: + + $ nomad acl role update -name="updated-name" + + Delete an ACL role: + + $ nomad acl role delete + + Please see the individual subcommand help for detailed usage information. +` + return strings.TrimSpace(helpText) +} + +// Synopsis satisfies the cli.Command Synopsis function. +func (a *ACLRoleCommand) Synopsis() string { return "Interact with ACL roles" } + +// Name returns the name of this command. +func (a *ACLRoleCommand) Name() string { return "acl role" } + +// Run satisfies the cli.Command Run function. +func (a *ACLRoleCommand) Run(_ []string) int { return cli.RunResultHelp } + +// formatACLRole formats and converts the ACL role API object into a string KV +// representation suitable for console output. +func formatACLRole(aclRole *api.ACLRole) string { + return formatKV([]string{ + fmt.Sprintf("ID|%s", aclRole.ID), + fmt.Sprintf("Name|%s", aclRole.Name), + fmt.Sprintf("Description|%s", aclRole.Description), + fmt.Sprintf("Policies|%s", strings.Join(aclRolePolicyLinkToStringList(aclRole.Policies), ",")), + fmt.Sprintf("Create Index|%d", aclRole.CreateIndex), + fmt.Sprintf("Modify Index|%d", aclRole.ModifyIndex), + }) +} + +// aclRolePolicyLinkToStringList converts an array of ACL role policy links to +// an array of string policy names. The returned array will be sorted. +func aclRolePolicyLinkToStringList(policyLinks []*api.ACLRolePolicyLink) []string { + policies := make([]string, len(policyLinks)) + for i, policy := range policyLinks { + policies[i] = policy.Name + } + sort.Strings(policies) + return policies +} + +// aclRolePolicyNamesToPolicyLinks takes a list of policy names as a string +// array and converts this to an array of ACL role policy links. Any duplicate +// names are removed. +func aclRolePolicyNamesToPolicyLinks(policyNames []string) []*api.ACLRolePolicyLink { + var policyLinks []*api.ACLRolePolicyLink + keys := make(map[string]struct{}) + + for _, policyName := range policyNames { + if _, ok := keys[policyName]; !ok { + policyLinks = append(policyLinks, &api.ACLRolePolicyLink{Name: policyName}) + keys[policyName] = struct{}{} + } + } + return policyLinks +} diff --git a/command/acl_role_create.go b/command/acl_role_create.go new file mode 100644 index 000000000..f1aa8482b --- /dev/null +++ b/command/acl_role_create.go @@ -0,0 +1,148 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +// Ensure ACLRoleCreateCommand satisfies the cli.Command interface. +var _ cli.Command = &ACLRoleCreateCommand{} + +// ACLRoleCreateCommand implements cli.Command. +type ACLRoleCreateCommand struct { + Meta + + name string + description string + policyNames []string + json bool + tmpl string +} + +// Help satisfies the cli.Command Help function. +func (a *ACLRoleCreateCommand) Help() string { + helpText := ` +Usage: nomad acl token create [options] + + Create is used to create new ACL roles. Use requires a management token. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + ` + +ACL Create Options: + + -name + Sets the human readable name for the ACL role. The name must be between + 1-128 characters and is a required parameter. + + -description + A free form text description of the role that must not exceed 256 + characters. + + -policy-name + Specifies a policy to associate with the role identified by their name. This + flag can be specified multiple times and must be specified at least once. + + -json + Output the ACL role in a JSON format. + + -t + Format and display the ACL role using a Go template. +` + return strings.TrimSpace(helpText) +} + +func (a *ACLRoleCreateCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(a.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-name": complete.PredictAnything, + "-description": complete.PredictAnything, + "-policy-name": complete.PredictAnything, + "-json": complete.PredictNothing, + "-t": complete.PredictAnything, + }) +} + +func (a *ACLRoleCreateCommand) AutocompleteArgs() complete.Predictor { return complete.PredictNothing } + +// Synopsis satisfies the cli.Command Synopsis function. +func (a *ACLRoleCreateCommand) Synopsis() string { return "Create a new ACL role" } + +// Name returns the name of this command. +func (a *ACLRoleCreateCommand) Name() string { return "acl role create" } + +// Run satisfies the cli.Command Run function. +func (a *ACLRoleCreateCommand) Run(args []string) int { + + flags := a.Meta.FlagSet(a.Name(), FlagSetClient) + flags.Usage = func() { a.Ui.Output(a.Help()) } + flags.StringVar(&a.name, "name", "", "") + flags.StringVar(&a.description, "description", "", "") + flags.Var((funcVar)(func(s string) error { + a.policyNames = append(a.policyNames, s) + return nil + }), "policy-name", "") + flags.BoolVar(&a.json, "json", false, "") + flags.StringVar(&a.tmpl, "t", "", "") + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got no arguments. + if len(flags.Args()) != 0 { + a.Ui.Error("This command takes no arguments") + a.Ui.Error(commandErrorText(a)) + return 1 + } + + // Perform some basic validation on the submitted role information to avoid + // sending API and RPC requests which will fail basic validation. + if a.name == "" { + a.Ui.Error("ACL role name must be specified using the -name flag") + return 1 + } + if len(a.policyNames) < 1 { + a.Ui.Error("At least one policy name must be specified using the -policy-name flag") + return 1 + } + + // Set up the ACL with the passed parameters. + aclRole := api.ACLRole{ + Name: a.name, + Description: a.description, + Policies: aclRolePolicyNamesToPolicyLinks(a.policyNames), + } + + // Get the HTTP client. + client, err := a.Meta.Client() + if err != nil { + a.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + // Create the ACL role via the API. + role, _, err := client.ACLRoles().Create(&aclRole, nil) + if err != nil { + a.Ui.Error(fmt.Sprintf("Error creating ACL role: %s", err)) + return 1 + } + + if a.json || len(a.tmpl) > 0 { + out, err := Format(a.json, a.tmpl, role) + if err != nil { + a.Ui.Error(err.Error()) + return 1 + } + + a.Ui.Output(out) + return 0 + } + + a.Ui.Output(formatACLRole(role)) + return 0 +} diff --git a/command/acl_role_create_test.go b/command/acl_role_create_test.go new file mode 100644 index 000000000..6b107df3a --- /dev/null +++ b/command/acl_role_create_test.go @@ -0,0 +1,80 @@ +package command + +import ( + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/command/agent" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +func TestACLRoleCreateCommand_Run(t *testing.T) { + ci.Parallel(t) + + // Build a test server with ACLs enabled. + srv, _, url := testServer(t, false, func(c *agent.Config) { + c.ACL.Enabled = true + }) + defer srv.Shutdown() + + // Wait for the server to start fully and ensure we have a bootstrap token. + testutil.WaitForLeader(t, srv.Agent.RPC) + rootACLToken := srv.RootToken + require.NotNil(t, rootACLToken) + + ui := cli.NewMockUi() + cmd := &ACLRoleCreateCommand{ + Meta: Meta{ + Ui: ui, + flagAddress: url, + }, + } + + // Test the basic validation on the command. + require.Equal(t, 1, cmd.Run([]string{"-address=" + url, "this-command-does-not-take-args"})) + require.Contains(t, ui.ErrorWriter.String(), "This command takes no arguments") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + require.Equal(t, 1, cmd.Run([]string{"-address=" + url})) + require.Contains(t, ui.ErrorWriter.String(), "ACL role name must be specified using the -name flag") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + require.Equal(t, 1, cmd.Run([]string{"-address=" + url, `-name="foobar"`})) + require.Contains(t, ui.ErrorWriter.String(), "At least one policy name must be specified using the -policy-name flag") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Create an ACL policy that can be referenced within the ACL role. + aclPolicy := structs.ACLPolicy{ + Name: "acl-role-cli-test-policy", + Rules: `namespace "default" { + policy = "read" + } + `, + } + err := srv.Agent.Server().State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{&aclPolicy}) + require.NoError(t, err) + + // Create an ACL role. + args := []string{ + "-address=" + url, "-token=" + rootACLToken.SecretID, "-name=acl-role-cli-test", + "-policy-name=acl-role-cli-test-policy", "-description=acl-role-all-the-things", + } + require.Equal(t, 0, cmd.Run(args)) + s := ui.OutputWriter.String() + require.Contains(t, s, "Name = acl-role-cli-test") + require.Contains(t, s, "Description = acl-role-all-the-things") + require.Contains(t, s, "Policies = acl-role-cli-test-policy") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() +} diff --git a/command/acl_role_delete.go b/command/acl_role_delete.go new file mode 100644 index 000000000..d7b472297 --- /dev/null +++ b/command/acl_role_delete.go @@ -0,0 +1,83 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +// Ensure ACLRoleDeleteCommand satisfies the cli.Command interface. +var _ cli.Command = &ACLRoleDeleteCommand{} + +// ACLRoleDeleteCommand implements cli.Command. +type ACLRoleDeleteCommand struct { + Meta +} + +// Help satisfies the cli.Command Help function. +func (a *ACLRoleDeleteCommand) Help() string { + helpText := ` +Usage: nomad acl role delete + + Delete is used to delete an existing ACL role. Use requires a management + token. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + + return strings.TrimSpace(helpText) +} + +func (a *ACLRoleDeleteCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(a.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{}) +} + +func (a *ACLRoleDeleteCommand) AutocompleteArgs() complete.Predictor { return complete.PredictNothing } + +// Synopsis satisfies the cli.Command Synopsis function. +func (a *ACLRoleDeleteCommand) Synopsis() string { return "Delete an existing ACL role" } + +// Name returns the name of this command. +func (a *ACLRoleDeleteCommand) Name() string { return "acl token delete" } + +// Run satisfies the cli.Command Run function. +func (a *ACLRoleDeleteCommand) Run(args []string) int { + + flags := a.Meta.FlagSet(a.Name(), FlagSetClient) + flags.Usage = func() { a.Ui.Output(a.Help()) } + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that the last argument is the role ID to delete. + if len(flags.Args()) != 1 { + a.Ui.Error("This command takes one argument: ") + a.Ui.Error(commandErrorText(a)) + return 1 + } + + aclRoleID := flags.Args()[0] + + // Get the HTTP client. + client, err := a.Meta.Client() + if err != nil { + a.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + // Delete the specified ACL role. + _, err = client.ACLRoles().Delete(aclRoleID, nil) + if err != nil { + a.Ui.Error(fmt.Sprintf("Error deleting ACL role: %s", err)) + return 1 + } + + // Give some feedback to indicate the deletion was successful. + a.Ui.Output(fmt.Sprintf("ACL role %s successfully deleted", aclRoleID)) + return 0 +} diff --git a/command/acl_role_delete_test.go b/command/acl_role_delete_test.go new file mode 100644 index 000000000..0afe31494 --- /dev/null +++ b/command/acl_role_delete_test.go @@ -0,0 +1,77 @@ +package command + +import ( + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/command/agent" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +func TestACLRoleDeleteCommand_Run(t *testing.T) { + ci.Parallel(t) + + // Build a test server with ACLs enabled. + srv, _, url := testServer(t, false, func(c *agent.Config) { + c.ACL.Enabled = true + }) + defer srv.Shutdown() + + // Wait for the server to start fully and ensure we have a bootstrap token. + testutil.WaitForLeader(t, srv.Agent.RPC) + rootACLToken := srv.RootToken + require.NotNil(t, rootACLToken) + + ui := cli.NewMockUi() + cmd := &ACLRoleDeleteCommand{ + Meta: Meta{ + Ui: ui, + flagAddress: url, + }, + } + + // Try and delete more than one ACL role. + code := cmd.Run([]string{"-address=" + url, "acl-role-1", "acl-role-2"}) + require.Equal(t, 1, code) + require.Contains(t, ui.ErrorWriter.String(), "This command takes one argument") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Try deleting a role that does not exist. + require.Equal(t, 1, cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID, "acl-role-1"})) + require.Contains(t, ui.ErrorWriter.String(), "ACL role not found") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Create an ACL policy that can be referenced within the ACL role. + aclPolicy := structs.ACLPolicy{ + Name: "acl-role-cli-test", + Rules: `namespace "default" { + policy = "read" + } + `, + } + err := srv.Agent.Server().State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{&aclPolicy}) + require.NoError(t, err) + + // Create an ACL role referencing the previously created policy. + aclRole := structs.ACLRole{ + ID: uuid.Generate(), + Name: "acl-role-cli-test", + Policies: []*structs.ACLRolePolicyLink{{Name: aclPolicy.Name}}, + } + err = srv.Agent.Server().State().UpsertACLRoles( + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}) + require.NoError(t, err) + + // Delete the existing ACL role. + require.Equal(t, 0, cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID, aclRole.ID})) + require.Contains(t, ui.OutputWriter.String(), "successfully deleted") +} diff --git a/command/acl_role_info.go b/command/acl_role_info.go new file mode 100644 index 000000000..897c4635d --- /dev/null +++ b/command/acl_role_info.go @@ -0,0 +1,121 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +// Ensure ACLRoleInfoCommand satisfies the cli.Command interface. +var _ cli.Command = &ACLRoleInfoCommand{} + +// ACLRoleInfoCommand implements cli.Command. +type ACLRoleInfoCommand struct { + Meta + + byName bool + json bool + tmpl string +} + +// Help satisfies the cli.Command Help function. +func (a *ACLRoleInfoCommand) Help() string { + helpText := ` +Usage: nomad acl role info [options] + + Info is used to fetch information on an existing ACL roles. Requires a + management token. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + ` + +ACL Info Options: + + -by-name + Look up the ACL role using its name as the identifier. The command defaults + to expecting the ACL ID as the argument. + + -json + Output the ACL role in a JSON format. + + -t + Format and display the ACL role using a Go template. +` + + return strings.TrimSpace(helpText) +} + +func (a *ACLRoleInfoCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(a.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-by-name": complete.PredictNothing, + "-json": complete.PredictNothing, + "-t": complete.PredictAnything, + }) +} + +func (a *ACLRoleInfoCommand) AutocompleteArgs() complete.Predictor { return complete.PredictNothing } + +// Synopsis satisfies the cli.Command Synopsis function. +func (a *ACLRoleInfoCommand) Synopsis() string { return "Fetch information on an existing ACL role" } + +// Name returns the name of this command. +func (a *ACLRoleInfoCommand) Name() string { return "acl role info" } + +// Run satisfies the cli.Command Run function. +func (a *ACLRoleInfoCommand) Run(args []string) int { + + flags := a.Meta.FlagSet(a.Name(), FlagSetClient) + flags.Usage = func() { a.Ui.Output(a.Help()) } + flags.BoolVar(&a.byName, "by-name", false, "") + flags.BoolVar(&a.json, "json", false, "") + flags.StringVar(&a.tmpl, "t", "", "") + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we have exactly one argument. + if len(flags.Args()) != 1 { + a.Ui.Error("This command takes one argument: ") + a.Ui.Error(commandErrorText(a)) + return 1 + } + + // Get the HTTP client. + client, err := a.Meta.Client() + if err != nil { + a.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + var ( + aclRole *api.ACLRole + apiErr error + ) + + aclRoleID := flags.Args()[0] + + // Use the correct API call depending on whether the lookup is by the name + // or the ID. + switch a.byName { + case true: + aclRole, _, apiErr = client.ACLRoles().GetByName(aclRoleID, nil) + default: + aclRole, _, apiErr = client.ACLRoles().Get(aclRoleID, nil) + } + + // Handle any error from the API. + if apiErr != nil { + a.Ui.Error(fmt.Sprintf("Error reading ACL role: %s", apiErr)) + return 1 + } + + // Format the output. + a.Ui.Output(formatACLRole(aclRole)) + return 0 +} diff --git a/command/acl_role_info_test.go b/command/acl_role_info_test.go new file mode 100644 index 000000000..0dfac81e8 --- /dev/null +++ b/command/acl_role_info_test.go @@ -0,0 +1,95 @@ +package command + +import ( + "fmt" + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/command/agent" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +func TestACLRoleInfoCommand_Run(t *testing.T) { + ci.Parallel(t) + + // Build a test server with ACLs enabled. + srv, _, url := testServer(t, false, func(c *agent.Config) { + c.ACL.Enabled = true + }) + defer srv.Shutdown() + + // Wait for the server to start fully and ensure we have a bootstrap token. + testutil.WaitForLeader(t, srv.Agent.RPC) + rootACLToken := srv.RootToken + require.NotNil(t, rootACLToken) + + ui := cli.NewMockUi() + cmd := &ACLRoleInfoCommand{ + Meta: Meta{ + Ui: ui, + flagAddress: url, + }, + } + + // Perform a lookup without specifying an ID. + require.Equal(t, 1, cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID})) + require.Contains(t, ui.ErrorWriter.String(), "This command takes one argument: ") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Perform a lookup specifying a random ID. + require.Equal(t, 1, cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID, uuid.Generate()})) + require.Contains(t, ui.ErrorWriter.String(), "ACL role not found") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Create an ACL policy that can be referenced within the ACL role. + aclPolicy := structs.ACLPolicy{ + Name: "acl-role-policy-cli-test", + Rules: `namespace "default" { + policy = "read" + } + `, + } + err := srv.Agent.Server().State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{&aclPolicy}) + require.NoError(t, err) + + // Create an ACL role referencing the previously created policy. + aclRole := structs.ACLRole{ + ID: uuid.Generate(), + Name: "acl-role-cli-test", + Policies: []*structs.ACLRolePolicyLink{{Name: aclPolicy.Name}}, + } + err = srv.Agent.Server().State().UpsertACLRoles( + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}) + require.NoError(t, err) + + // Look up the ACL role using its ID. + require.Equal(t, 0, cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID, aclRole.ID})) + s := ui.OutputWriter.String() + require.Contains(t, s, fmt.Sprintf("ID = %s", aclRole.ID)) + require.Contains(t, s, fmt.Sprintf("Name = %s", aclRole.Name)) + require.Contains(t, s, "Description = ") + require.Contains(t, s, fmt.Sprintf("Policies = %s", aclPolicy.Name)) + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Look up the ACL role using its Name. + require.Equal(t, 0, cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID, "-by-name", aclRole.Name})) + s = ui.OutputWriter.String() + require.Contains(t, s, fmt.Sprintf("ID = %s", aclRole.ID)) + require.Contains(t, s, fmt.Sprintf("Name = %s", aclRole.Name)) + require.Contains(t, s, "Description = ") + require.Contains(t, s, fmt.Sprintf("Policies = %s", aclPolicy.Name)) + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() +} diff --git a/command/acl_role_list.go b/command/acl_role_list.go new file mode 100644 index 000000000..c86d819b5 --- /dev/null +++ b/command/acl_role_list.go @@ -0,0 +1,123 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +// Ensure ACLRoleListCommand satisfies the cli.Command interface. +var _ cli.Command = &ACLRoleListCommand{} + +// ACLRoleListCommand implements cli.Command. +type ACLRoleListCommand struct { + Meta +} + +// Help satisfies the cli.Command Help function. +func (a *ACLRoleListCommand) Help() string { + helpText := ` +Usage: nomad acl role list [options] + + List is used to list existing ACL roles. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + ` + +ACL List Options: + + -json + Output the ACL roles in a JSON format. + + -t + Format and display the ACL roles using a Go template. +` + + return strings.TrimSpace(helpText) +} + +func (a *ACLRoleListCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(a.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-json": complete.PredictNothing, + "-t": complete.PredictAnything, + }) +} + +func (a *ACLRoleListCommand) AutocompleteArgs() complete.Predictor { return complete.PredictNothing } + +// Synopsis satisfies the cli.Command Synopsis function. +func (a *ACLRoleListCommand) Synopsis() string { return "List ACL roles" } + +// Name returns the name of this command. +func (a *ACLRoleListCommand) Name() string { return "acl role list" } + +// Run satisfies the cli.Command Run function. +func (a *ACLRoleListCommand) Run(args []string) int { + var json bool + var tmpl string + + flags := a.Meta.FlagSet(a.Name(), FlagSetClient) + flags.Usage = func() { a.Ui.Output(a.Help()) } + flags.BoolVar(&json, "json", false, "") + flags.StringVar(&tmpl, "t", "", "") + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got no arguments + if len(flags.Args()) != 0 { + a.Ui.Error("This command takes no arguments") + a.Ui.Error(commandErrorText(a)) + return 1 + } + + // Get the HTTP client + client, err := a.Meta.Client() + if err != nil { + a.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + // Fetch info on the policy + roles, _, err := client.ACLRoles().List(nil) + if err != nil { + a.Ui.Error(fmt.Sprintf("Error listing ACL roles: %s", err)) + return 1 + } + + if json || len(tmpl) > 0 { + out, err := Format(json, tmpl, roles) + if err != nil { + a.Ui.Error(err.Error()) + return 1 + } + + a.Ui.Output(out) + return 0 + } + + a.Ui.Output(formatACLRoles(roles)) + return 0 +} + +func formatACLRoles(roles []*api.ACLRole) string { + if len(roles) == 0 { + return "No ACL roles found" + } + + output := make([]string, 0, len(roles)+1) + output = append(output, "ID|Name|Description|Policies") + for _, role := range roles { + output = append(output, fmt.Sprintf( + "%s|%s|%s|%s", + role.ID, role.Name, role.Description, strings.Join(aclRolePolicyLinkToStringList(role.Policies), ","))) + } + + return formatList(output) +} diff --git a/command/acl_role_list_test.go b/command/acl_role_list_test.go new file mode 100644 index 000000000..9c529f591 --- /dev/null +++ b/command/acl_role_list_test.go @@ -0,0 +1,77 @@ +package command + +import ( + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/command/agent" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +func TestACLRoleListCommand_Run(t *testing.T) { + ci.Parallel(t) + + // Build a test server with ACLs enabled. + srv, _, url := testServer(t, false, func(c *agent.Config) { + c.ACL.Enabled = true + }) + defer srv.Shutdown() + + // Wait for the server to start fully and ensure we have a bootstrap token. + testutil.WaitForLeader(t, srv.Agent.RPC) + rootACLToken := srv.RootToken + require.NotNil(t, rootACLToken) + + ui := cli.NewMockUi() + cmd := &ACLRoleListCommand{ + Meta: Meta{ + Ui: ui, + flagAddress: url, + }, + } + + // Perform a list straight away without any roles held in state. + require.Equal(t, 0, cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID})) + require.Contains(t, ui.OutputWriter.String(), "No ACL roles found") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Create an ACL policy that can be referenced within the ACL role. + aclPolicy := structs.ACLPolicy{ + Name: "acl-role-policy-cli-test", + Rules: `namespace "default" { + policy = "read" + } + `, + } + err := srv.Agent.Server().State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{&aclPolicy}) + require.NoError(t, err) + + // Create an ACL role referencing the previously created policy. + aclRole := structs.ACLRole{ + ID: uuid.Generate(), + Name: "acl-role-cli-test", + Policies: []*structs.ACLRolePolicyLink{{Name: aclPolicy.Name}}, + } + err = srv.Agent.Server().State().UpsertACLRoles( + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}) + require.NoError(t, err) + + // Perform a listing to get the created role. + require.Equal(t, 0, cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID})) + s := ui.OutputWriter.String() + require.Contains(t, s, "ID") + require.Contains(t, s, "Name") + require.Contains(t, s, "Policies") + require.Contains(t, s, "acl-role-cli-test") + require.Contains(t, s, "acl-role-policy-cli-test") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() +} diff --git a/command/acl_role_test.go b/command/acl_role_test.go new file mode 100644 index 000000000..7031cc621 --- /dev/null +++ b/command/acl_role_test.go @@ -0,0 +1,61 @@ +package command + +import ( + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/stretchr/testify/require" +) + +func Test_formatACLRole(t *testing.T) { + inputACLRole := api.ACLRole{ + ID: "this-is-usually-a-uuid", + Name: "this-is-my-friendly-name", + Description: "this-is-my-friendly-name", + Policies: []*api.ACLRolePolicyLink{ + {Name: "policy-link-1"}, + {Name: "policy-link-2"}, + {Name: "policy-link-3"}, + {Name: "policy-link-4"}, + }, + CreateIndex: 13, + ModifyIndex: 1313, + } + expectedOutput := "ID = this-is-usually-a-uuid\nName = this-is-my-friendly-name\nDescription = this-is-my-friendly-name\nPolicies = policy-link-1,policy-link-2,policy-link-3,policy-link-4\nCreate Index = 13\nModify Index = 1313" + actualOutput := formatACLRole(&inputACLRole) + require.Equal(t, expectedOutput, actualOutput) +} + +func Test_aclRolePolicyLinkToStringList(t *testing.T) { + inputPolicyLinks := []*api.ACLRolePolicyLink{ + {Name: "z-policy-link-1"}, + {Name: "a-policy-link-2"}, + {Name: "policy-link-3"}, + {Name: "b-policy-link-4"}, + } + expectedOutput := []string{ + "a-policy-link-2", + "b-policy-link-4", + "policy-link-3", + "z-policy-link-1", + } + actualOutput := aclRolePolicyLinkToStringList(inputPolicyLinks) + require.Equal(t, expectedOutput, actualOutput) +} + +func Test_aclRolePolicyNamesToPolicyLinks(t *testing.T) { + inputPolicyNames := []string{ + "policy-link-1", "policy-link-2", "policy-link-3", "policy-link-4", + "policy-link-1", "policy-link-2", "policy-link-3", "policy-link-4", + "policy-link-1", "policy-link-2", "policy-link-3", "policy-link-4", + "policy-link-1", "policy-link-2", "policy-link-3", "policy-link-4", + } + expectedOutput := []*api.ACLRolePolicyLink{ + {Name: "policy-link-1"}, + {Name: "policy-link-2"}, + {Name: "policy-link-3"}, + {Name: "policy-link-4"}, + } + actualOutput := aclRolePolicyNamesToPolicyLinks(inputPolicyNames) + require.ElementsMatch(t, expectedOutput, actualOutput) +} diff --git a/command/acl_role_update.go b/command/acl_role_update.go new file mode 100644 index 000000000..fcb71902f --- /dev/null +++ b/command/acl_role_update.go @@ -0,0 +1,217 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +// Ensure ACLRoleUpdateCommand satisfies the cli.Command interface. +var _ cli.Command = &ACLRoleUpdateCommand{} + +// ACLRoleUpdateCommand implements cli.Command. +type ACLRoleUpdateCommand struct { + Meta + + name string + description string + policyNames []string + noMerge bool + json bool + tmpl string +} + +// Help satisfies the cli.Command Help function. +func (a *ACLRoleUpdateCommand) Help() string { + helpText := ` +Usage: nomad acl role update [options] + + Update is used to update an existing ACL token. Requires a management token. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + ` + +Update Options: + + -name + Sets the human readable name for the ACL role. The name must be between + 1-128 characters. + + -description + A free form text description of the role that must not exceed 256 + characters. + + -policy-name + Specifies a policy to associate with the role identified by their name. This + flag can be specified multiple times. + + -no-merge + Do not merge the current role information with what is provided to the + command. Instead overwrite all fields with the exception of the role ID + which is immutable. + + -json + Output the ACL role in a JSON format. + + -t + Format and display the ACL role using a Go template. +` + + return strings.TrimSpace(helpText) +} + +func (a *ACLRoleUpdateCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(a.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-name": complete.PredictAnything, + "-description": complete.PredictAnything, + "-no-merge": complete.PredictNothing, + "-policy-name": complete.PredictAnything, + "-json": complete.PredictNothing, + "-t": complete.PredictAnything, + }) +} + +func (a *ACLRoleUpdateCommand) AutocompleteArgs() complete.Predictor { return complete.PredictNothing } + +// Synopsis satisfies the cli.Command Synopsis function. +func (a *ACLRoleUpdateCommand) Synopsis() string { return "Update an existing ACL role" } + +// Name returns the name of this command. +func (*ACLRoleUpdateCommand) Name() string { return "acl role update" } + +// Run satisfies the cli.Command Run function. +func (a *ACLRoleUpdateCommand) Run(args []string) int { + + flags := a.Meta.FlagSet(a.Name(), FlagSetClient) + flags.Usage = func() { a.Ui.Output(a.Help()) } + flags.StringVar(&a.name, "name", "", "") + flags.StringVar(&a.description, "description", "", "") + flags.Var((funcVar)(func(s string) error { + a.policyNames = append(a.policyNames, s) + return nil + }), "policy-name", "") + flags.BoolVar(&a.noMerge, "no-merge", false, "") + flags.BoolVar(&a.json, "json", false, "") + flags.StringVar(&a.tmpl, "t", "", "") + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got exactly one argument which is expected to be the ACL + // role ID. + if len(flags.Args()) != 1 { + a.Ui.Error("This command takes one argument: ") + a.Ui.Error(commandErrorText(a)) + return 1 + } + + // Get the HTTP client. + client, err := a.Meta.Client() + if err != nil { + a.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + aclRoleID := flags.Args()[0] + + // Read the current role in both cases, so we can fail better if not found. + currentRole, _, err := client.ACLRoles().Get(aclRoleID, nil) + if err != nil { + a.Ui.Error(fmt.Sprintf("Error when retrieving ACL role: %v", err)) + return 1 + } + + var updatedRole api.ACLRole + + // Depending on whether we are merging or not, we need to take a different + // approach. + switch a.noMerge { + case true: + + // Perform some basic validation on the submitted role information to + // avoid sending API and RPC requests which will fail basic validation. + if a.name == "" { + a.Ui.Error("ACL role name must be specified using the -name flag") + return 1 + } + if len(a.policyNames) < 1 { + a.Ui.Error("At least one policy name must be specified using the -policy-name flag") + return 1 + } + + updatedRole = api.ACLRole{ + ID: aclRoleID, + Name: a.name, + Description: a.description, + Policies: aclRolePolicyNamesToPolicyLinks(a.policyNames), + } + default: + // Check that the operator specified at least one flag to update the ACL + // role with. + if len(a.policyNames) == 0 && a.name == "" && a.description == "" { + a.Ui.Error("Please provide at least one flag to update the ACL role") + a.Ui.Error(commandErrorText(a)) + return 1 + } + + updatedRole = *currentRole + + // If the operator specified a name or description, overwrite the + // existing value as these are simple strings. + if a.name != "" { + updatedRole.Name = a.name + } + if a.description != "" { + updatedRole.Description = a.description + } + + // In order to merge the policy updates, we need to identify if the + // specified policy names already exist within the ACL role linking. + for _, policyName := range a.policyNames { + + // Track whether we found the policy name already in the ACL role + // linking. + var found bool + + for _, existingLinkedPolicy := range currentRole.Policies { + if policyName == existingLinkedPolicy.Name { + found = true + break + } + } + + // If the policy name was not found, append this new link to the + // updated role. + if !found { + updatedRole.Policies = append(updatedRole.Policies, &api.ACLRolePolicyLink{Name: policyName}) + } + } + } + + // Update the ACL role with the new information via the API. + updatedACLRoleRead, _, err := client.ACLRoles().Update(&updatedRole, nil) + if err != nil { + a.Ui.Error(fmt.Sprintf("Error updating ACL role: %s", err)) + return 1 + } + + if a.json || len(a.tmpl) > 0 { + out, err := Format(a.json, a.tmpl, updatedACLRoleRead) + if err != nil { + a.Ui.Error(err.Error()) + return 1 + } + + a.Ui.Output(out) + return 0 + } + + // Format the output + a.Ui.Output(formatACLRole(updatedACLRoleRead)) + return 0 +} diff --git a/command/acl_role_update_test.go b/command/acl_role_update_test.go new file mode 100644 index 000000000..9c02d3696 --- /dev/null +++ b/command/acl_role_update_test.go @@ -0,0 +1,124 @@ +package command + +import ( + "fmt" + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/command/agent" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +func TestACLRoleUpdateCommand_Run(t *testing.T) { + ci.Parallel(t) + + // Build a test server with ACLs enabled. + srv, _, url := testServer(t, false, func(c *agent.Config) { + c.ACL.Enabled = true + }) + defer srv.Shutdown() + + // Wait for the server to start fully and ensure we have a bootstrap token. + testutil.WaitForLeader(t, srv.Agent.RPC) + rootACLToken := srv.RootToken + require.NotNil(t, rootACLToken) + + ui := cli.NewMockUi() + cmd := &ACLRoleUpdateCommand{ + Meta: Meta{ + Ui: ui, + flagAddress: url, + }, + } + + // Try calling the command without setting an ACL Role ID arg. + require.Equal(t, 1, cmd.Run([]string{"-address=" + url})) + require.Contains(t, ui.ErrorWriter.String(), "This command takes one argument") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Try calling the command with an ACL role ID that does not exist. + code := cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID, "catch-me-if-you-can"}) + require.Equal(t, 1, code) + require.Contains(t, ui.ErrorWriter.String(), "ACL role not found") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Create an ACL policy that can be referenced within the ACL role. + aclPolicy := structs.ACLPolicy{ + Name: "acl-role-cli-test-policy", + Rules: `namespace "default" { + policy = "read" + } + `, + } + err := srv.Agent.Server().State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{&aclPolicy}) + require.NoError(t, err) + + // Create an ACL role that can be used for updating. + aclRole := structs.ACLRole{ + ID: uuid.Generate(), + Name: "acl-role-cli-test", + Description: "my-lovely-role", + Policies: []*structs.ACLRolePolicyLink{{Name: aclPolicy.Name}}, + } + + err = srv.Agent.Server().State().UpsertACLRoles( + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}) + require.NoError(t, err) + + // Try a merge update without setting any parameters to update. + code = cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID, aclRole.ID}) + require.Equal(t, 1, code) + require.Contains(t, ui.ErrorWriter.String(), "Please provide at least one flag to update the ACL role") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Update the description using the merge method. + code = cmd.Run([]string{ + "-address=" + url, "-token=" + rootACLToken.SecretID, "-description=badger-badger-badger", aclRole.ID}) + require.Equal(t, 0, code) + s := ui.OutputWriter.String() + require.Contains(t, s, fmt.Sprintf("ID = %s", aclRole.ID)) + require.Contains(t, s, "Name = acl-role-cli-test") + require.Contains(t, s, "Description = badger-badger-badger") + require.Contains(t, s, "Policies = acl-role-cli-test-policy") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + // Try updating the role using no-merge without setting the required flags. + code = cmd.Run([]string{"-address=" + url, "-token=" + rootACLToken.SecretID, "-no-merge", aclRole.ID}) + require.Equal(t, 1, code) + require.Contains(t, ui.ErrorWriter.String(), "ACL role name must be specified using the -name flag") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + code = cmd.Run([]string{ + "-address=" + url, "-token=" + rootACLToken.SecretID, "-no-merge", "-name=update-role-name", aclRole.ID}) + require.Equal(t, 1, code) + require.Contains(t, ui.ErrorWriter.String(), "At least one policy name must be specified using the -policy-name flag") + + // Update the role using no-merge with all required flags set. + code = cmd.Run([]string{ + "-address=" + url, "-token=" + rootACLToken.SecretID, "-no-merge", "-name=update-role-name", + "-description=updated-description", "-policy-name=acl-role-cli-test-policy", aclRole.ID}) + require.Equal(t, 0, code) + s = ui.OutputWriter.String() + require.Contains(t, s, fmt.Sprintf("ID = %s", aclRole.ID)) + require.Contains(t, s, "Name = update-role-name") + require.Contains(t, s, "Description = updated-description") + require.Contains(t, s, "Policies = acl-role-cli-test-policy") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() +} diff --git a/command/commands.go b/command/commands.go index 2418f1033..d8daa7425 100644 --- a/command/commands.go +++ b/command/commands.go @@ -107,6 +107,36 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "acl role": func() (cli.Command, error) { + return &ACLRoleCommand{ + Meta: meta, + }, nil + }, + "acl role create": func() (cli.Command, error) { + return &ACLRoleCreateCommand{ + Meta: meta, + }, nil + }, + "acl role delete": func() (cli.Command, error) { + return &ACLRoleDeleteCommand{ + Meta: meta, + }, nil + }, + "acl role info": func() (cli.Command, error) { + return &ACLRoleInfoCommand{ + Meta: meta, + }, nil + }, + "acl role list": func() (cli.Command, error) { + return &ACLRoleListCommand{ + Meta: meta, + }, nil + }, + "acl role update": func() (cli.Command, error) { + return &ACLRoleUpdateCommand{ + Meta: meta, + }, nil + }, "acl token": func() (cli.Command, error) { return &ACLTokenCommand{ Meta: meta, From 9e3f1581fbf8a5f73633f92acadedaeb4af67000 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 17 Aug 2022 14:45:01 +0100 Subject: [PATCH 11/19] core: add ACL role functionality to ACL tokens. ACL tokens can now utilize ACL roles in order to provide API authorization. Each ACL token can be created and linked to an array of policies as well as an array of ACL role links. The link can be provided via the role name or ID, but internally, is always resolved to the ID as this is immutable whereas the name can be changed by operators. When resolving an ACL token, the policies linked from an ACL role are unpacked and combined with the policy array to form the complete auth set for the token. The ACL token creation endpoint handles deduplicating ACL role links as well as ensuring they exist within state. When reading a token, Nomad will also ensure the ACL role link is current. This handles ACL roles being deleted from under a token from a UX standpoint. --- nomad/acl.go | 53 +++++++-- nomad/acl_endpoint.go | 48 +++++++++ nomad/acl_endpoint_test.go | 123 +++++++++++++++++++++ nomad/acl_test.go | 160 +++++++++++++++++++++++++--- nomad/state/state_store.go | 16 ++- nomad/state/state_store_acl.go | 67 ++++++++++++ nomad/state/state_store_acl_test.go | 132 +++++++++++++++++++++++ nomad/structs/acl.go | 26 ++++- nomad/structs/acl_test.go | 16 ++- nomad/structs/structs.go | 20 +++- 10 files changed, 629 insertions(+), 32 deletions(-) diff --git a/nomad/acl.go b/nomad/acl.go index 76fe29096..dcdc07991 100644 --- a/nomad/acl.go +++ b/nomad/acl.go @@ -82,9 +82,9 @@ func (s *Server) ResolveClaims(claims *structs.IdentityClaims) (*acl.ACL, error) return aclObj, nil } -// resolveTokenFromSnapshotCache is used to resolve an ACL object from a snapshot of state, -// using a cache to avoid parsing and ACL construction when possible. It is split from resolveToken -// to simplify testing. +// resolveTokenFromSnapshotCache is used to resolve an ACL object from a +// snapshot of state, using a cache to avoid parsing and ACL construction when +// possible. It is split from resolveToken to simplify testing. func resolveTokenFromSnapshotCache(snap *state.StateSnapshot, cache *lru.TwoQueueCache, secretID string) (*acl.ACL, error) { // Lookup the ACL Token var token *structs.ACLToken @@ -111,22 +111,61 @@ func resolveTokenFromSnapshotCache(snap *state.StateSnapshot, cache *lru.TwoQueu return acl.ManagementACL, nil } - // Get all associated policies - policies := make([]*structs.ACLPolicy, 0, len(token.Policies)) + // Store all policies detailed in the token request, this includes the + // named policies and those referenced within the role link. + policies := make([]*structs.ACLPolicy, 0, len(token.Policies)+len(token.Roles)) + + // Iterate all the token policies and add these to our policy tracking + // array. for _, policyName := range token.Policies { policy, err := snap.ACLPolicyByName(nil, policyName) if err != nil { return nil, err } if policy == nil { - // Ignore policies that don't exist, since they don't grant any more privilege + // Ignore policies that don't exist, since they don't grant any + // more privilege. continue } - // Save the policy and update the cache key + // Add the policy to the tracking array. policies = append(policies, policy) } + // Iterate all the token role links, so we can unpack these and identify + // the ACL policies. + for _, roleLink := range token.Roles { + + // Any error reading the role means we cannot move forward. We just + // ignore any roles that have been detailed but are not within our + // state. + role, err := snap.GetACLRoleByID(nil, roleLink.ID) + if err != nil { + return nil, err + } + if role == nil { + continue + } + + // Unpack the policies held within the ACL role to form a single list + // of ACL policies that this token has available. + for _, policyLink := range role.Policies { + policy, err := snap.ACLPolicyByName(nil, policyLink.Name) + if err != nil { + return nil, err + } + + // Ignore policies that don't exist, since they don't grant any + // more privilege. + if policy == nil { + continue + } + + // Add the policy to the tracking array. + policies = append(policies, policy) + } + } + // Compile and cache the ACL object aclObj, err := structs.CompileACLObject(cache, policies) if err != nil { diff --git a/nomad/acl_endpoint.go b/nomad/acl_endpoint.go index 70b386f49..d4161dd30 100644 --- a/nomad/acl_endpoint.go +++ b/nomad/acl_endpoint.go @@ -543,6 +543,54 @@ func (a *ACL) UpsertTokens(args *structs.ACLTokenUpsertRequest, reply *structs.A return structs.NewErrRPCCodedf(http.StatusBadRequest, "token %d invalid: %v", idx, err) } + var normalizedRoleLinks []*structs.ACLTokenRoleLink + uniqueRoleIDs := make(map[string]struct{}) + + // Iterate, check, and normalize the ACL role links that the token has. + for _, roleLink := range token.Roles { + + var ( + existing *structs.ACLRole + roleIdentifier string + lookupErr error + ) + + // In the event the caller specified the role name, we need to + // identify the immutable ID. In either case, we need to ensure the + // role exists. + switch roleLink.ID { + case "": + roleIdentifier = roleLink.Name + existing, lookupErr = stateSnapshot.GetACLRoleByName(nil, roleIdentifier) + default: + roleIdentifier = roleLink.ID + existing, lookupErr = stateSnapshot.GetACLRoleByID(nil, roleIdentifier) + } + + // Handle any state lookup error or inability to locate the role + // within state. + if lookupErr != nil { + return structs.NewErrRPCCodedf(http.StatusInternalServerError, "role lookup failed: %v", lookupErr) + } + if existing == nil { + return structs.NewErrRPCCodedf(http.StatusBadRequest, "cannot find role %s", roleIdentifier) + } + + // Ensure the role ID is written to the object and that the name is + // emptied as it is possible the role name is updated in the future. + roleLink.ID = existing.ID + roleLink.Name = "" + + // Deduplicate role links by their ID. + if _, ok := uniqueRoleIDs[roleLink.ID]; !ok { + normalizedRoleLinks = append(normalizedRoleLinks, roleLink) + uniqueRoleIDs[roleLink.ID] = struct{}{} + } + } + + // Write the normalized array of ACL role links back to the token. + token.Roles = normalizedRoleLinks + // Compute the token hash token.SetHash() } diff --git a/nomad/acl_endpoint_test.go b/nomad/acl_endpoint_test.go index a3ff68460..dfff2caff 100644 --- a/nomad/acl_endpoint_test.go +++ b/nomad/acl_endpoint_test.go @@ -1618,6 +1618,129 @@ func TestACLEndpoint_UpsertTokens(t *testing.T) { require.Empty(t, tokenResp.Tokens) }, }, + { + name: "token with role links", + testFn: func(testServer *Server, aclToken *structs.ACLToken) { + + // Attempt to create a token with a link to a role that does + // not exist in state. + tokenReq1 := &structs.ACLTokenUpsertRequest{ + Tokens: []*structs.ACLToken{ + { + Name: "my-lovely-token-" + uuid.Generate(), + Type: structs.ACLClientToken, + Roles: []*structs.ACLTokenRoleLink{{Name: "cant-find-me"}}, + }, + }, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + + // Send the RPC request and ensure the expiration time is as + // expected. + var tokenResp1 structs.ACLTokenUpsertResponse + err := msgpackrpc.CallWithCodec(codec, structs.ACLUpsertTokensRPCMethod, tokenReq1, &tokenResp1) + require.ErrorContains(t, err, "cannot find role cant-find-me") + require.Empty(t, tokenResp1.Tokens) + + // Create an ACL policy that will be linked from an ACL role + // and enter this into state. + policy1 := mock.ACLPolicy() + + require.NoError(t, testServer.fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1})) + + // Create an ACL role that links to the above policy. + aclRole1 := mock.ACLRole() + aclRole1.Policies = []*structs.ACLRolePolicyLink{{Name: policy1.Name}} + + require.NoError(t, testServer.fsm.State().UpsertACLRoles( + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{aclRole1})) + + // Create a token which references the created ACL role. This + // role reference is duplicated to ensure the handler + // de-duplicates this before putting it into state. + // not exist in state. + tokenReq2 := &structs.ACLTokenUpsertRequest{ + Tokens: []*structs.ACLToken{ + { + Name: "my-lovely-token-" + uuid.Generate(), + Type: structs.ACLClientToken, + Roles: []*structs.ACLTokenRoleLink{ + {ID: aclRole1.ID}, + {ID: aclRole1.ID}, + {ID: aclRole1.ID}, + }, + }, + }, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + + // Send the RPC request and ensure the returned token is as + // expected. + var tokenResp2 structs.ACLTokenUpsertResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLUpsertTokensRPCMethod, tokenReq2, &tokenResp2) + require.NoError(t, err) + require.Len(t, tokenResp2.Tokens, 1) + require.Len(t, tokenResp2.Tokens[0].Policies, 0) + require.Len(t, tokenResp2.Tokens[0].Roles, 1) + require.Equal(t, []*structs.ACLTokenRoleLink{{ + ID: aclRole1.ID, Name: aclRole1.Name}}, tokenResp2.Tokens[0].Roles) + }, + }, + { + name: "token with role and policy links", + testFn: func(testServer *Server, aclToken *structs.ACLToken) { + + // Create two ACL policies that will be used for ACL role and + // policy linking. + policy1 := mock.ACLPolicy() + policy2 := mock.ACLPolicy() + + require.NoError(t, testServer.fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create an ACL role that links to one of the above policies. + aclRole1 := mock.ACLRole() + aclRole1.Policies = []*structs.ACLRolePolicyLink{{Name: policy1.Name}} + + require.NoError(t, testServer.fsm.State().UpsertACLRoles( + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{aclRole1})) + + // Create an ACL token with both ACL role and policy links. + tokenReq1 := &structs.ACLTokenUpsertRequest{ + Tokens: []*structs.ACLToken{ + { + Name: "my-lovely-token-" + uuid.Generate(), + Type: structs.ACLClientToken, + Policies: []string{policy2.Name}, + Roles: []*structs.ACLTokenRoleLink{{ID: aclRole1.ID}}, + }, + }, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + + // Send the RPC request and ensure the returned token has + // policy and ACL role links as expected. + var tokenResp1 structs.ACLTokenUpsertResponse + err := msgpackrpc.CallWithCodec(codec, structs.ACLUpsertTokensRPCMethod, tokenReq1, &tokenResp1) + require.NoError(t, err) + require.Len(t, tokenResp1.Tokens, 1) + require.Len(t, tokenResp1.Tokens[0].Policies, 1) + require.Len(t, tokenResp1.Tokens[0].Roles, 1) + require.Equal(t, policy2.Name, tokenResp1.Tokens[0].Policies[0]) + require.Equal(t, []*structs.ACLTokenRoleLink{{ + ID: aclRole1.ID, Name: aclRole1.Name}}, tokenResp1.Tokens[0].Roles) + }, + }, } for _, tc := range testCases { diff --git a/nomad/acl_test.go b/nomad/acl_test.go index 14104a1df..acd5455db 100644 --- a/nomad/acl_test.go +++ b/nomad/acl_test.go @@ -17,17 +17,17 @@ import ( func TestResolveACLToken(t *testing.T) { ci.Parallel(t) - testServer, _, testServerCleanup := TestACLServer(t, nil) - defer testServerCleanup() - testutil.WaitForLeader(t, testServer.RPC) - testCases := []struct { name string - testFn func(testServer *Server) + testFn func() }{ { name: "leader token", - testFn: func(testServer *Server) { + testFn: func() { + + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) // Check the leader ACL token is correctly set. leaderACL := testServer.getLeaderAcl() @@ -42,7 +42,11 @@ func TestResolveACLToken(t *testing.T) { }, { name: "anonymous token", - testFn: func(testServer *Server) { + testFn: func() { + + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) // Call the function with an empty input secret ID which is // classed as representing anonymous access in clusters with @@ -55,7 +59,11 @@ func TestResolveACLToken(t *testing.T) { }, { name: "token not found", - testFn: func(testServer *Server) { + testFn: func() { + + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) // Call the function with randomly generated secret ID which // does not exist within state. @@ -66,7 +74,11 @@ func TestResolveACLToken(t *testing.T) { }, { name: "token expired", - testFn: func(testServer *Server) { + testFn: func() { + + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) // Create a mock token with an expiration time long in the // past, and upsert. @@ -87,7 +99,11 @@ func TestResolveACLToken(t *testing.T) { }, { name: "management token", - testFn: func(testServer *Server) { + testFn: func() { + + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) // Generate a management token and upsert this. managementToken := mock.ACLToken() @@ -108,8 +124,12 @@ func TestResolveACLToken(t *testing.T) { }, }, { - name: "client token", - testFn: func(testServer *Server) { + name: "client token with policies only", + testFn: func() { + + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) // Generate a client token with associated policies and upsert // these. @@ -155,11 +175,125 @@ func TestResolveACLToken(t *testing.T) { require.NotEqual(t, aclResp2, aclResp3) }, }, + { + name: "client token with roles only", + testFn: func() { + + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) + + // Create a client token that only has a link to a role. + policy1 := mock.ACLPolicy() + policy2 := mock.ACLPolicy() + err := testServer.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2}) + + aclRole := mock.ACLRole() + aclRole.Policies = []*structs.ACLRolePolicyLink{ + {Name: policy1.Name}, + {Name: policy2.Name}, + } + err = testServer.State().UpsertACLRoles( + structs.MsgTypeTestSetup, 30, []*structs.ACLRole{aclRole}) + require.NoError(t, err) + + clientToken := mock.ACLToken() + clientToken.Policies = []string{} + clientToken.Roles = []*structs.ACLTokenRoleLink{{ID: aclRole.ID}} + err = testServer.State().UpsertACLTokens( + structs.MsgTypeTestSetup, 30, []*structs.ACLToken{clientToken}) + require.NoError(t, err) + + // Resolve the token and check that we received a client + // ACL with appropriate permissions. + aclResp, err := testServer.ResolveToken(clientToken.SecretID) + require.Nil(t, err) + require.NotNil(t, aclResp) + require.False(t, aclResp.IsManagement()) + + allowed := aclResp.AllowNamespaceOperation("default", acl.NamespaceCapabilityListJobs) + require.True(t, allowed) + allowed = aclResp.AllowNamespaceOperation("other", acl.NamespaceCapabilityListJobs) + require.False(t, allowed) + + // Remove the policies from the ACL role and ensure the resolution + // permissions are updated. + aclRole.Policies = []*structs.ACLRolePolicyLink{} + err = testServer.State().UpsertACLRoles( + structs.MsgTypeTestSetup, 40, []*structs.ACLRole{aclRole}) + require.NoError(t, err) + + aclResp, err = testServer.ResolveToken(clientToken.SecretID) + require.Nil(t, err) + require.NotNil(t, aclResp) + require.False(t, aclResp.IsManagement()) + require.False(t, aclResp.AllowNamespaceOperation("default", acl.NamespaceCapabilityListJobs)) + }, + }, + { + name: "client with roles and policies", + testFn: func() { + + testServer, _, testServerCleanup := TestACLServer(t, nil) + defer testServerCleanup() + testutil.WaitForLeader(t, testServer.RPC) + + // Generate two policies, each with a different namespace + // permission set. + policy1 := &structs.ACLPolicy{ + Name: "policy-" + uuid.Generate(), + Rules: `namespace "platform" { policy = "write"}`, + CreateIndex: 10, + ModifyIndex: 10, + } + policy1.SetHash() + policy2 := &structs.ACLPolicy{ + Name: "policy-" + uuid.Generate(), + Rules: `namespace "web" { policy = "write"}`, + CreateIndex: 10, + ModifyIndex: 10, + } + policy2.SetHash() + + err := testServer.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2}) + require.NoError(t, err) + + // Create a role which references the policy that has access to + // the web namespace. + aclRole := mock.ACLRole() + aclRole.Policies = []*structs.ACLRolePolicyLink{{Name: policy2.Name}} + err = testServer.State().UpsertACLRoles( + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{aclRole}) + require.NoError(t, err) + + // Create a token which references the policy and role. + clientToken := mock.ACLToken() + clientToken.Policies = []string{policy1.Name} + clientToken.Roles = []*structs.ACLTokenRoleLink{{ID: aclRole.ID}} + err = testServer.State().UpsertACLTokens( + structs.MsgTypeTestSetup, 30, []*structs.ACLToken{clientToken}) + require.NoError(t, err) + + // Resolve the token and check that we received a client + // ACL with appropriate permissions. + aclResp, err := testServer.ResolveToken(clientToken.SecretID) + require.Nil(t, err) + require.NotNil(t, aclResp) + require.False(t, aclResp.IsManagement()) + + allowed := aclResp.AllowNamespaceOperation("platform", acl.NamespaceCapabilityListJobs) + require.True(t, allowed) + allowed = aclResp.AllowNamespaceOperation("web", acl.NamespaceCapabilityListJobs) + require.True(t, allowed) + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - tc.testFn(testServer) + tc.testFn() }) } } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 737dd6b22..84d59370b 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -5660,10 +5660,20 @@ func (s *StateStore) ACLTokenByAccessorID(ws memdb.WatchSet, id string) (*struct } ws.Add(watchCh) - if existing != nil { - return existing.(*structs.ACLToken), nil + // If the existing token is nil, this indicates it does not exist in state. + if existing == nil { + return nil, nil } - return nil, nil + + // Assert the token type which allows us to perform additional work on the + // token that is needed before returning the call. + token := existing.(*structs.ACLToken) + + // Handle potential staleness of ACL role links. + if token, err = s.fixTokenRoleLinks(txn, token); err != nil { + return nil, err + } + return token, nil } // ACLTokenBySecretID is used to lookup a token by secret ID diff --git a/nomad/state/state_store_acl.go b/nomad/state/state_store_acl.go index 60df4e506..1f9f273ca 100644 --- a/nomad/state/state_store_acl.go +++ b/nomad/state/state_store_acl.go @@ -6,6 +6,7 @@ import ( "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/nomad/structs" + "golang.org/x/exp/slices" ) // ACLTokensByExpired returns an array accessor IDs of expired ACL tokens. @@ -189,6 +190,14 @@ func (s *StateStore) GetACLRoles(ws memdb.WatchSet) (memdb.ResultIterator, error // of the caller to check for this. func (s *StateStore) GetACLRoleByID(ws memdb.WatchSet, roleID string) (*structs.ACLRole, error) { txn := s.db.ReadTxn() + return s.getACLRoleByIDTxn(txn, ws, roleID) +} + +// getACLRoleByIDTxn allows callers to pass a read transaction in order to read +// a single ACL role specified by the input ID. The role object will be nil, if +// no matching entry was found; it is the responsibility of the caller to check +// for this. +func (s *StateStore) getACLRoleByIDTxn(txn ReadTxn, ws memdb.WatchSet, roleID string) (*structs.ACLRole, error) { // Perform the ACL role lookup using the "id" index. watchCh, existing, err := txn.FirstWatch(TableACLRoles, indexID, roleID) @@ -235,3 +244,61 @@ func (s *StateStore) GetACLRoleByIDPrefix(ws memdb.WatchSet, idPrefix string) (m return iter, nil } + +// fixTokenRoleLinks is a state helper that ensures the returned ACL token has +// an accurate representation of ACL role links. The role links could have +// become stale when a linked role was deleted or renamed. This will correct +// them and generates a newly allocated token only when fixes are needed. If +// the role links are still accurate, we just return the original token. +func (s *StateStore) fixTokenRoleLinks(txn ReadTxn, original *structs.ACLToken) (*structs.ACLToken, error) { + + // Track whether we have made an initial copy to ensure we are not + // operating on the token directly from state. + copied := false + + token := original + + // copyTokenFn is a helper function which copies the ACL token along with + // a certain number of ACL role links. + copyTokenFn := func(t *structs.ACLToken, numLinks int) *structs.ACLToken { + clone := t.Copy() + clone.Roles = slices.Clone(t.Roles[:numLinks]) + return clone + } + + for linkIndex, link := range original.Roles { + + // This should never happen, but guard against it anyway, so we log an + // error rather than panic. + if link.ID == "" { + return nil, errors.New("detected corrupted token within the state store: missing role link ID") + } + + role, err := s.getACLRoleByIDTxn(txn, nil, link.ID) + if err != nil { + return nil, err + } + + if role == nil { + if !copied { + // clone the token as we cannot touch the original + token = copyTokenFn(original, linkIndex) + copied = true + } + // if already owned then we just don't append it. + } else if role.Name != link.Name { + if !copied { + token = copyTokenFn(original, linkIndex) + copied = true + } + + // append the corrected policy + token.Roles = append(token.Roles, &structs.ACLTokenRoleLink{ID: link.ID, Name: role.Name}) + + } else if copied { + token.Roles = append(token.Roles, link) + } + } + + return token, nil +} diff --git a/nomad/state/state_store_acl_test.go b/nomad/state/state_store_acl_test.go index 6cc6a0ae4..16e375579 100644 --- a/nomad/state/state_store_acl_test.go +++ b/nomad/state/state_store_acl_test.go @@ -488,3 +488,135 @@ func TestStateStore_GetACLRoleByIDPrefix(t *testing.T) { } require.Len(t, aclRoles, 2) } + +func TestStateStore_fixTokenRoleLinks(t *testing.T) { + ci.Parallel(t) + + testCases := []struct { + name string + testFn func() + }{ + { + name: "no fix needed", + testFn: func() { + testState := testStateStore(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate a some mocked ACL roles for testing and upsert these straight + // into state. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles)) + + // Create an ACL token linking to the ACL role. + token1 := mock.ACLToken() + token1.Roles = []*structs.ACLTokenRoleLink{{ID: mockedACLRoles[0].ID}} + require.NoError(t, testState.UpsertACLTokens( + structs.MsgTypeTestSetup, 20, []*structs.ACLToken{token1})) + + // Perform the fix and check the returned token contains the + // correct roles. + readTxn := testState.db.ReadTxn() + outputToken, err := testState.fixTokenRoleLinks(readTxn, token1) + require.NoError(t, err) + require.Equal(t, outputToken.Roles, []*structs.ACLTokenRoleLink{{ + Name: mockedACLRoles[0].Name, ID: mockedACLRoles[0].ID, + }}) + }, + }, + { + name: "acl role from link deleted", + testFn: func() { + testState := testStateStore(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate a some mocked ACL roles for testing and upsert these straight + // into state. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles)) + + // Create an ACL token linking to the ACL roles. + token1 := mock.ACLToken() + token1.Roles = []*structs.ACLTokenRoleLink{{ID: mockedACLRoles[0].ID}, {ID: mockedACLRoles[1].ID}} + require.NoError(t, testState.UpsertACLTokens( + structs.MsgTypeTestSetup, 30, []*structs.ACLToken{token1})) + + // Now delete one of the ACL roles from state. + require.NoError(t, testState.DeleteACLRolesByID( + structs.MsgTypeTestSetup, 40, []string{mockedACLRoles[0].ID})) + + // Perform the fix and check the returned token contains the + // correct roles. + readTxn := testState.db.ReadTxn() + outputToken, err := testState.fixTokenRoleLinks(readTxn, token1) + require.NoError(t, err) + require.Len(t, outputToken.Roles, 1) + require.Equal(t, outputToken.Roles, []*structs.ACLTokenRoleLink{{ + Name: mockedACLRoles[1].Name, ID: mockedACLRoles[1].ID, + }}) + }, + }, + { + name: "acl role from link name changed", + testFn: func() { + testState := testStateStore(t) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testState.UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Generate a some mocked ACL roles for testing and upsert these straight + // into state. + mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles)) + + // Create an ACL token linking to the ACL roles. + token1 := mock.ACLToken() + token1.Roles = []*structs.ACLTokenRoleLink{{ID: mockedACLRoles[0].ID}, {ID: mockedACLRoles[1].ID}} + require.NoError(t, testState.UpsertACLTokens( + structs.MsgTypeTestSetup, 30, []*structs.ACLToken{token1})) + + // Now change the name of one of the ACL roles. + mockedACLRoles[0].Name = "badger-badger-badger" + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 40, mockedACLRoles)) + + // Perform the fix and check the returned token contains the + // correct roles. + readTxn := testState.db.ReadTxn() + outputToken, err := testState.fixTokenRoleLinks(readTxn, token1) + require.NoError(t, err) + require.Len(t, outputToken.Roles, 2) + require.ElementsMatch(t, outputToken.Roles, []*structs.ACLTokenRoleLink{ + {Name: mockedACLRoles[0].Name, ID: mockedACLRoles[0].ID}, + {Name: mockedACLRoles[1].Name, ID: mockedACLRoles[1].ID}, + }) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn() + }) + } +} diff --git a/nomad/structs/acl.go b/nomad/structs/acl.go index e40a89d42..8ab6862b1 100644 --- a/nomad/structs/acl.go +++ b/nomad/structs/acl.go @@ -88,6 +88,22 @@ var ( validACLRoleName = regexp.MustCompile("^[a-zA-Z0-9-]{1,128}$") ) +// ACLTokenRoleLink is used to link an ACL token to an ACL role. The ACL token +// can therefore inherit all the ACL policy permissions that the ACL role +// contains. +type ACLTokenRoleLink struct { + + // ID is the ACLRole.ID UUID. This field is immutable and represents the + // absolute truth for the link. + ID string + + // Name is the human friendly identifier for the ACL role and is a + // convenience field for operators. This field is always resolved to the + // ID and discarded before the token is stored in state. This is because + // operators can change the name of an ACL role. + Name string +} + // Canonicalize performs basic canonicalization on the ACL token object. It is // important for callers to understand certain fields such as AccessorID are // set if it is empty, so copies should be taken if needed before calling this @@ -120,16 +136,16 @@ func (a *ACLToken) Validate(minTTL, maxTTL time.Duration, existing *ACLToken) er } // The type of an ACL token must be set. An ACL token of type client must - // have associated policies, whereas a management token cannot be + // have associated policies or roles, whereas a management token cannot be // associated with policies. switch a.Type { case ACLClientToken: - if len(a.Policies) == 0 { - mErr.Errors = append(mErr.Errors, errors.New("client token missing policies")) + if len(a.Policies) == 0 && len(a.Roles) == 0 { + mErr.Errors = append(mErr.Errors, errors.New("client token missing policies or roles")) } case ACLManagementToken: - if len(a.Policies) != 0 { - mErr.Errors = append(mErr.Errors, errors.New("management token cannot be associated with policies")) + if len(a.Policies) != 0 || len(a.Roles) != 0 { + mErr.Errors = append(mErr.Errors, errors.New("management token cannot be associated with policies or roles")) } default: mErr.Errors = append(mErr.Errors, errors.New("token type must be client or management")) diff --git a/nomad/structs/acl_test.go b/nomad/structs/acl_test.go index 22e263e5e..1a9b426c4 100644 --- a/nomad/structs/acl_test.go +++ b/nomad/structs/acl_test.go @@ -25,6 +25,7 @@ func TestACLToken_Canonicalize(t *testing.T) { Name: "my cool token " + uuid.Generate(), Type: "client", Policies: []string{"foo", "bar"}, + Roles: []*ACLTokenRoleLink{}, Global: false, CreateTime: time.Now().UTC(), CreateIndex: 10, @@ -96,12 +97,12 @@ func TestACLTokenValidate(t *testing.T) { expectedErrorContains: "client or management", }, { - name: "missing policies", + name: "missing policies or roles", inputACLToken: &ACLToken{ Type: ACLClientToken, }, inputExistingACLToken: nil, - expectedErrorContains: "missing policies", + expectedErrorContains: "missing policies or roles", }, { name: "invalid policies", @@ -110,7 +111,16 @@ func TestACLTokenValidate(t *testing.T) { Policies: []string{"foo"}, }, inputExistingACLToken: nil, - expectedErrorContains: "associated with policies", + expectedErrorContains: "associated with policies or roles", + }, + { + name: "invalid roles", + inputACLToken: &ACLToken{ + Type: ACLManagementToken, + Roles: []*ACLTokenRoleLink{{Name: "foo"}}, + }, + inputExistingACLToken: nil, + expectedErrorContains: "associated with policies or roles", }, { name: "name too long", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index f163a25d9..24f91ab7b 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -11908,7 +11908,12 @@ type ACLToken struct { Name string // Human friendly name Type string // Client or Management Policies []string // Policies this token ties to - Global bool // Global or Region local + + // Roles represents the ACL roles that this token is tied to. The token + // will inherit the permissions of all policies detailed within the role. + Roles []*ACLTokenRoleLink + + Global bool // Global or Region local Hash []byte CreateTime time.Time // Time of creation @@ -11950,9 +11955,13 @@ func (a *ACLToken) Copy() *ACLToken { c.Policies = make([]string, len(a.Policies)) copy(c.Policies, a.Policies) + c.Hash = make([]byte, len(a.Hash)) copy(c.Hash, a.Hash) + c.Roles = make([]*ACLTokenRoleLink, len(a.Roles)) + copy(c.Roles, a.Roles) + return c } @@ -11973,6 +11982,7 @@ type ACLTokenListStub struct { Name string Type string Policies []string + Roles []*ACLTokenRoleLink Global bool Hash []byte CreateTime time.Time @@ -12003,6 +12013,13 @@ func (a *ACLToken) SetHash() []byte { _, _ = hash.Write([]byte("local")) } + // Iterate the ACL role links and hash the ID. The ID is immutable and the + // canonical way to reference a role. The name can be modified by + // operators, but won't impact the ACL token resolution. + for _, roleLink := range a.Roles { + _, _ = hash.Write([]byte(roleLink.ID)) + } + // Finalize the hash hashVal := hash.Sum(nil) @@ -12017,6 +12034,7 @@ func (a *ACLToken) Stub() *ACLTokenListStub { Name: a.Name, Type: a.Type, Policies: a.Policies, + Roles: a.Roles, Global: a.Global, Hash: a.Hash, CreateTime: a.CreateTime, From f5d8cb2d9010d777adafa83b9ee678baa77f26e8 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 17 Aug 2022 14:45:32 +0100 Subject: [PATCH 12/19] api: add ACL role linking to ACL tokens. --- api/acl.go | 20 +++++++ api/acl_test.go | 143 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) diff --git a/api/acl.go b/api/acl.go index fd7f20786..223474ecf 100644 --- a/api/acl.go +++ b/api/acl.go @@ -317,6 +317,11 @@ type ACLToken struct { Name string Type string Policies []string + + // Roles represents the ACL roles that this token is tied to. The token + // will inherit the permissions of all policies detailed within the role. + Roles []*ACLTokenRoleLink + Global bool CreateTime time.Time @@ -335,11 +340,26 @@ type ACLToken struct { ModifyIndex uint64 } +// ACLTokenRoleLink is used to link an ACL token to an ACL role. The ACL token +// can therefore inherit all the ACL policy permissions that the ACL role +// contains. +type ACLTokenRoleLink struct { + + // ID is the ACLRole.ID UUID. This field is immutable and represents the + // absolute truth for the link. + ID string + + // Name is the human friendly identifier for the ACL role and is a + // convenience field for operators. + Name string +} + type ACLTokenListStub struct { AccessorID string Name string Type string Policies []string + Roles []*ACLTokenRoleLink Global bool CreateTime time.Time diff --git a/api/acl_test.go b/api/acl_test.go index 4487a7778..4c412013d 100644 --- a/api/acl_test.go +++ b/api/acl_test.go @@ -159,6 +159,39 @@ func TestACLTokens_CreateUpdate(t *testing.T) { out3, _, err := at.Update(out2, nil) require.Error(t, err) require.Nil(t, out3) + + // Try adding a role link to our token, which should be possible. For this + // we need to create a policy and link to this from a role. + aclPolicy := ACLPolicy{ + Name: "acl-role-api-test", + Rules: `namespace "default" { policy = "read" }`, + } + writeMeta, err := c.ACLPolicies().Upsert(&aclPolicy, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + + // Create an ACL role referencing the previously created + // policy. + role := ACLRole{ + Name: "acl-role-api-test", + Policies: []*ACLRolePolicyLink{{Name: aclPolicy.Name}}, + } + aclRoleCreateResp, writeMeta, err := c.ACLRoles().Create(&role, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + require.NotEmpty(t, aclRoleCreateResp.ID) + require.Equal(t, role.Name, aclRoleCreateResp.Name) + + out2.Roles = []*ACLTokenRoleLink{{Name: aclRoleCreateResp.Name}} + out2.ExpirationTTL = 0 + + out3, writeMeta, err = at.Update(out2, nil) + require.NoError(t, err) + require.NotNil(t, out3) + require.Len(t, out3.Policies, 1) + require.Equal(t, out3.Policies[0], "foo1") + require.Len(t, out3.Roles, 1) + require.Equal(t, out3.Roles[0].Name, role.Name) } func TestACLTokens_Info(t *testing.T) { @@ -221,6 +254,116 @@ func TestACLTokens_Info(t *testing.T) { require.NotNil(t, out2.ExpirationTime) }, }, + { + name: "token with role link", + testFn: func(client *Client) { + + // Create an ACL policy that can be referenced within the ACL + // role. + aclPolicy := ACLPolicy{ + Name: "acl-role-api-test", + Rules: `namespace "default" { policy = "read" }`, + } + writeMeta, err := testClient.ACLPolicies().Upsert(&aclPolicy, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + + // Create an ACL role referencing the previously created + // policy. + role := ACLRole{ + Name: "acl-role-api-test", + Policies: []*ACLRolePolicyLink{{Name: aclPolicy.Name}}, + } + aclRoleCreateResp, writeMeta, err := testClient.ACLRoles().Create(&role, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + require.NotEmpty(t, aclRoleCreateResp.ID) + require.Equal(t, role.Name, aclRoleCreateResp.Name) + + // Create a token with a role linking. + token := &ACLToken{ + Name: "token-with-role-link", + Type: "client", + Roles: []*ACLTokenRoleLink{{Name: role.Name}}, + } + + out, wm, err := client.ACLTokens().Create(token, nil) + require.Nil(t, err) + assertWriteMeta(t, wm) + require.NotNil(t, out) + + // Query the token and ensure it matches what was returned + // during the creation. + out2, qm, err := client.ACLTokens().Info(out.AccessorID, nil) + require.Nil(t, err) + assertQueryMeta(t, qm) + require.Equal(t, out, out2) + require.Len(t, out.Roles, 1) + require.Equal(t, out.Roles[0].Name, aclPolicy.Name) + }, + }, + + { + name: "token with role and policy link", + testFn: func(client *Client) { + + // Create an ACL policy that can be referenced within the ACL + // role. + aclPolicy1 := ACLPolicy{ + Name: "acl-role-api-test-1", + Rules: `namespace "default" { policy = "read" }`, + } + writeMeta, err := testClient.ACLPolicies().Upsert(&aclPolicy1, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + + // Create another that can be referenced within the ACL token + // directly. + aclPolicy2 := ACLPolicy{ + Name: "acl-role-api-test-2", + Rules: `namespace "fawlty" { policy = "read" }`, + } + writeMeta, err = testClient.ACLPolicies().Upsert(&aclPolicy2, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + + // Create an ACL role referencing the previously created + // policy. + role := ACLRole{ + Name: "acl-role-api-test", + Policies: []*ACLRolePolicyLink{{Name: aclPolicy1.Name}}, + } + aclRoleCreateResp, writeMeta, err := testClient.ACLRoles().Create(&role, nil) + require.NoError(t, err) + assertWriteMeta(t, writeMeta) + require.NotEmpty(t, aclRoleCreateResp.ID) + require.Equal(t, role.Name, aclRoleCreateResp.Name) + + // Create a token with a role linking. + token := &ACLToken{ + Name: "token-with-role-link", + Type: "client", + Policies: []string{aclPolicy2.Name}, + Roles: []*ACLTokenRoleLink{{Name: role.Name}}, + } + + out, wm, err := client.ACLTokens().Create(token, nil) + require.Nil(t, err) + assertWriteMeta(t, wm) + require.NotNil(t, out) + require.Len(t, out.Policies, 1) + require.Equal(t, out.Policies[0], aclPolicy2.Name) + require.Len(t, out.Roles, 1) + require.Equal(t, out.Roles[0].Name, role.Name) + + // Query the token and ensure it matches what was returned + // during the creation. + out2, qm, err := client.ACLTokens().Info(out.AccessorID, nil) + require.Nil(t, err) + assertQueryMeta(t, qm) + require.Equal(t, out, out2) + }, + }, } for _, tc := range testCases { From 51a7df50bbd91a3cb8079f4469589052c3f487c7 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 17 Aug 2022 14:49:52 +0100 Subject: [PATCH 13/19] cli: add ability to create and view tokens with ACL role links. --- command/acl_bootstrap.go | 61 ++++++++++++++++++++++---------- command/acl_token_create.go | 54 ++++++++++++++++++++++++---- command/acl_token_create_test.go | 26 ++++++++++++++ command/acl_token_info.go | 2 +- command/acl_token_self.go | 2 +- command/acl_token_update.go | 2 +- helper/funcs.go | 11 ++++++ helper/funcs_test.go | 24 +++++++++++++ 8 files changed, 154 insertions(+), 28 deletions(-) diff --git a/command/acl_bootstrap.go b/command/acl_bootstrap.go index f367f9cb9..a1844365c 100644 --- a/command/acl_bootstrap.go +++ b/command/acl_bootstrap.go @@ -8,6 +8,7 @@ import ( "time" "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" "github.com/posener/complete" ) @@ -127,7 +128,7 @@ func (c *ACLBootstrapCommand) Run(args []string) int { } // Format the output - c.Ui.Output(formatKVACLToken(token)) + outputACLToken(c.Ui, token) return 0 } @@ -143,32 +144,56 @@ func formatKVPolicy(policy *api.ACLPolicy) string { return formatKV(output) } -// formatKVACLToken returns a K/V formatted ACL token -func formatKVACLToken(token *api.ACLToken) string { - // Add the fixed preamble - output := []string{ +// outputACLToken formats and outputs the ACL token via the UI in the correct +// format. +func outputACLToken(ui cli.Ui, token *api.ACLToken) { + + // Build the initial KV output which is always the same not matter whether + // the token is a management or client type. + kvOutput := []string{ fmt.Sprintf("Accessor ID|%s", token.AccessorID), fmt.Sprintf("Secret ID|%s", token.SecretID), fmt.Sprintf("Name|%s", token.Name), fmt.Sprintf("Type|%s", token.Type), fmt.Sprintf("Global|%v", token.Global), - } - - // Special case the policy output - if token.Type == "management" { - output = append(output, "Policies|n/a") - } else { - output = append(output, fmt.Sprintf("Policies|%v", token.Policies)) - } - - // Add the generic output - output = append(output, fmt.Sprintf("Create Time|%v", token.CreateTime), fmt.Sprintf("Expiry Time |%s", expiryTimeString(token.ExpirationTime)), fmt.Sprintf("Create Index|%d", token.CreateIndex), fmt.Sprintf("Modify Index|%d", token.ModifyIndex), - ) - return formatKV(output) + } + + // If the token is a management type, make it obvious that it is not + // possible to have policies or roles assigned to it and just output the + // KV data. + if token.Type == "management" { + kvOutput = append(kvOutput, "Policies|n/a", "Roles|n/a") + ui.Output(formatKV(kvOutput)) + } else { + + // Policies are only currently referenced by name, so keep the previous + // format. When/if policies gain an ID alongside name like roles, this + // output should follow that of the roles. + kvOutput = append(kvOutput, fmt.Sprintf("Policies|%v", token.Policies)) + + var roleOutput []string + + // If we have linked roles, add the ID and name in a list format to the + // output. Otherwise, make it clear there are no linked roles. + if len(token.Roles) > 0 { + roleOutput = append(roleOutput, "ID|Name") + for _, roleLink := range token.Roles { + roleOutput = append(roleOutput, roleLink.ID+"|"+roleLink.Name) + } + } else { + roleOutput = append(roleOutput, "") + } + + // Output the mixed formats of data, ensuring there is a space between + // the KV and list data. + ui.Output(formatKV(kvOutput)) + ui.Output("") + ui.Output(fmt.Sprintf("Roles\n%s", formatList(roleOutput))) + } } func expiryTimeString(t *time.Time) string { diff --git a/command/acl_token_create.go b/command/acl_token_create.go index de1962849..e15d38be6 100644 --- a/command/acl_token_create.go +++ b/command/acl_token_create.go @@ -5,12 +5,17 @@ import ( "strings" "time" + "github.com/hashicorp/go-set" "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" "github.com/posener/complete" ) type ACLTokenCreateCommand struct { Meta + + roleNames []string + roleIDs []string } func (c *ACLTokenCreateCommand) Help() string { @@ -38,6 +43,12 @@ Create Options: Specifies a policy to associate with the token. Can be specified multiple times, but only with client type tokens. + -role-id + ID of a role to use for this token. May be specified multiple times. + + -role-name + Name of a role to use for this token. May be specified multiple times. + -ttl Specifies the time-to-live of the created ACL token. This takes the form of a time duration such as "5m" and "1h". By default, tokens will be created @@ -49,11 +60,13 @@ Create Options: func (c *ACLTokenCreateCommand) AutocompleteFlags() complete.Flags { return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), complete.Flags{ - "name": complete.PredictAnything, - "type": complete.PredictAnything, - "global": complete.PredictNothing, - "policy": complete.PredictAnything, - "ttl": complete.PredictAnything, + "name": complete.PredictAnything, + "type": complete.PredictAnything, + "global": complete.PredictNothing, + "policy": complete.PredictAnything, + "role-id": complete.PredictAnything, + "role-name": complete.PredictAnything, + "ttl": complete.PredictAnything, }) } @@ -81,6 +94,14 @@ func (c *ACLTokenCreateCommand) Run(args []string) int { policies = append(policies, s) return nil }), "policy", "") + flags.Var((funcVar)(func(s string) error { + c.roleNames = append(c.roleNames, s) + return nil + }), "role-name", "") + flags.Var((funcVar)(func(s string) error { + c.roleIDs = append(c.roleIDs, s) + return nil + }), "role-id", "") if err := flags.Parse(args); err != nil { return 1 } @@ -93,11 +114,12 @@ func (c *ACLTokenCreateCommand) Run(args []string) int { return 1 } - // Setup the token + // Set up the token. tk := &api.ACLToken{ Name: name, Type: tokenType, Policies: policies, + Roles: generateACLTokenRoleLinks(c.roleNames, c.roleIDs), Global: global, } @@ -127,6 +149,24 @@ func (c *ACLTokenCreateCommand) Run(args []string) int { } // Format the output - c.Ui.Output(formatKVACLToken(token)) + outputACLToken(c.Ui, token) return 0 } + +// generateACLTokenRoleLinks takes the command input role links by ID and name +// and coverts this to the relevant API object. It handles de-duplicating +// entries to the best effort, so this doesn't need to be done on the leader. +func generateACLTokenRoleLinks(roleNames, roleIDs []string) []*api.ACLTokenRoleLink { + var tokenLinks []*api.ACLTokenRoleLink + + roleNameSet := set.From[string](roleNames).List() + roleNameFn := func(name string) *api.ACLTokenRoleLink { return &api.ACLTokenRoleLink{Name: name} } + + roleIDsSet := set.From[string](roleIDs).List() + roleIDFn := func(id string) *api.ACLTokenRoleLink { return &api.ACLTokenRoleLink{ID: id} } + + tokenLinks = append(tokenLinks, helper.ConvertSlice(roleNameSet, roleNameFn)...) + tokenLinks = append(tokenLinks, helper.ConvertSlice(roleIDsSet, roleIDFn)...) + + return tokenLinks +} diff --git a/command/acl_token_create_test.go b/command/acl_token_create_test.go index 8eb782686..b2d7c08cb 100644 --- a/command/acl_token_create_test.go +++ b/command/acl_token_create_test.go @@ -3,6 +3,7 @@ package command import ( "testing" + "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/command/agent" "github.com/mitchellh/cli" @@ -50,3 +51,28 @@ func TestACLTokenCreateCommand(t *testing.T) { out = ui.OutputWriter.String() require.NotContains(t, out, "Expiry Time = ") } + +func Test_generateACLTokenRoleLinks(t *testing.T) { + ci.Parallel(t) + + inputRoleNames := []string{ + "duplicate", + "policy1", + "policy2", + "duplicate", + } + inputRoleIDs := []string{ + "77a780d8-2dee-7c7f-7822-6f5471c5cbb2", + "56850b06-a343-a772-1a5c-ad083fd8a50e", + "77a780d8-2dee-7c7f-7822-6f5471c5cbb2", + "77a780d8-2dee-7c7f-7822-6f5471c5cbb2", + } + expectedOutput := []*api.ACLTokenRoleLink{ + {Name: "duplicate"}, + {Name: "policy1"}, + {Name: "policy2"}, + {ID: "77a780d8-2dee-7c7f-7822-6f5471c5cbb2"}, + {ID: "56850b06-a343-a772-1a5c-ad083fd8a50e"}, + } + require.ElementsMatch(t, generateACLTokenRoleLinks(inputRoleNames, inputRoleIDs), expectedOutput) +} diff --git a/command/acl_token_info.go b/command/acl_token_info.go index 7df77ee1d..cb0c12651 100644 --- a/command/acl_token_info.go +++ b/command/acl_token_info.go @@ -71,6 +71,6 @@ func (c *ACLTokenInfoCommand) Run(args []string) int { } // Format the output - c.Ui.Output(formatKVACLToken(token)) + outputACLToken(c.Ui, token) return 0 } diff --git a/command/acl_token_self.go b/command/acl_token_self.go index eac79e481..9e3516346 100644 --- a/command/acl_token_self.go +++ b/command/acl_token_self.go @@ -68,6 +68,6 @@ func (c *ACLTokenSelfCommand) Run(args []string) int { } // Format the output - c.Ui.Output(formatKVACLToken(token)) + outputACLToken(c.Ui, token) return 0 } diff --git a/command/acl_token_update.go b/command/acl_token_update.go index 7a471fc15..1880c9c61 100644 --- a/command/acl_token_update.go +++ b/command/acl_token_update.go @@ -127,6 +127,6 @@ func (c *ACLTokenUpdateCommand) Run(args []string) int { } // Format the output - c.Ui.Output(formatKVACLToken(updatedToken)) + outputACLToken(c.Ui, updatedToken) return 0 } diff --git a/helper/funcs.go b/helper/funcs.go index ccb96613a..24aa0159e 100644 --- a/helper/funcs.go +++ b/helper/funcs.go @@ -718,3 +718,14 @@ func NewSafeTimer(duration time.Duration) (*time.Timer, StopFunc) { return t, cancel } + +// ConvertSlice takes the input slice and generates a new one using the +// supplied conversion function to covert the element. This is useful when +// converting a slice of strings to a slice of structs which wraps the string. +func ConvertSlice[A, B any](original []A, conversion func(a A) B) []B { + result := make([]B, len(original)) + for i, element := range original { + result[i] = conversion(element) + } + return result +} diff --git a/helper/funcs_test.go b/helper/funcs_test.go index 685430210..be77678d6 100644 --- a/helper/funcs_test.go +++ b/helper/funcs_test.go @@ -546,3 +546,27 @@ func Test_NewSafeTimer(t *testing.T) { <-timer.C }) } + +func Test_ConvertSlice(t *testing.T) { + t.Run("string wrapper", func(t *testing.T) { + + type wrapper struct{ id string } + input := []string{"foo", "bar", "bad", "had"} + cFn := func(id string) *wrapper { return &wrapper{id: id} } + + expectedOutput := []*wrapper{{id: "foo"}, {id: "bar"}, {id: "bad"}, {id: "had"}} + actualOutput := ConvertSlice(input, cFn) + require.ElementsMatch(t, expectedOutput, actualOutput) + }) + + t.Run("int wrapper", func(t *testing.T) { + + type wrapper struct{ id int } + input := []int{10, 13, 1987, 2020} + cFn := func(id int) *wrapper { return &wrapper{id: id} } + + expectedOutput := []*wrapper{{id: 10}, {id: 13}, {id: 1987}, {id: 2020}} + actualOutput := ConvertSlice(input, cFn) + require.ElementsMatch(t, expectedOutput, actualOutput) + }) +} From 802d005ef5c99f838a535b727ebe2edf9ab7e1a1 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 22 Aug 2022 08:54:07 +0200 Subject: [PATCH 14/19] acl: add replication to ACL Roles from authoritative region. (#14176) ACL Roles along with policies and global token will be replicated from the authoritative region to all federated regions. This involves a new replication loop running on the federated leader. Policies and roles may be replicated at different times, meaning the policies and role references may not be present within the local state upon replication upsert. In order to bypass the RPC and state check, a new RPC request parameter has been added. This is used by the replication process; all other callers will trigger the ACL role policy validation check. There is a new ACL RPC endpoint to allow the reading of a set of ACL Roles which is required by the replication process and matches ACL Policies and Tokens. A bug within the ACL Role listing RPC has also been fixed which returned incorrect data during blocking queries where a deletion had occurred. --- command/acl_role_delete_test.go | 2 +- command/acl_role_info_test.go | 2 +- command/acl_role_list_test.go | 2 +- command/acl_role_update_test.go | 2 +- command/agent/acl_endpoint_test.go | 8 +- nomad/acl_endpoint.go | 80 ++++++++-- nomad/acl_endpoint_test.go | 139 ++++++++++++++++- nomad/acl_test.go | 6 +- nomad/fsm.go | 2 +- nomad/fsm_test.go | 4 +- nomad/leader.go | 232 +++++++++++++++++++++++++++- nomad/leader_test.go | 107 +++++++++++++ nomad/state/state_store_acl.go | 12 +- nomad/state/state_store_acl_test.go | 43 ++++-- nomad/structs/acl.go | 28 ++++ nomad/structs/acl_test.go | 5 + 16 files changed, 620 insertions(+), 54 deletions(-) diff --git a/command/acl_role_delete_test.go b/command/acl_role_delete_test.go index 0afe31494..217c06654 100644 --- a/command/acl_role_delete_test.go +++ b/command/acl_role_delete_test.go @@ -68,7 +68,7 @@ func TestACLRoleDeleteCommand_Run(t *testing.T) { Policies: []*structs.ACLRolePolicyLink{{Name: aclPolicy.Name}}, } err = srv.Agent.Server().State().UpsertACLRoles( - structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}) + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}, false) require.NoError(t, err) // Delete the existing ACL role. diff --git a/command/acl_role_info_test.go b/command/acl_role_info_test.go index 0dfac81e8..4f4cc5db5 100644 --- a/command/acl_role_info_test.go +++ b/command/acl_role_info_test.go @@ -68,7 +68,7 @@ func TestACLRoleInfoCommand_Run(t *testing.T) { Policies: []*structs.ACLRolePolicyLink{{Name: aclPolicy.Name}}, } err = srv.Agent.Server().State().UpsertACLRoles( - structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}) + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}, false) require.NoError(t, err) // Look up the ACL role using its ID. diff --git a/command/acl_role_list_test.go b/command/acl_role_list_test.go index 9c529f591..7c7855edf 100644 --- a/command/acl_role_list_test.go +++ b/command/acl_role_list_test.go @@ -60,7 +60,7 @@ func TestACLRoleListCommand_Run(t *testing.T) { Policies: []*structs.ACLRolePolicyLink{{Name: aclPolicy.Name}}, } err = srv.Agent.Server().State().UpsertACLRoles( - structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}) + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}, false) require.NoError(t, err) // Perform a listing to get the created role. diff --git a/command/acl_role_update_test.go b/command/acl_role_update_test.go index 9c02d3696..3e0449bc8 100644 --- a/command/acl_role_update_test.go +++ b/command/acl_role_update_test.go @@ -71,7 +71,7 @@ func TestACLRoleUpdateCommand_Run(t *testing.T) { } err = srv.Agent.Server().State().UpsertACLRoles( - structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}) + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{&aclRole}, false) require.NoError(t, err) // Try a merge update without setting any parameters to update. diff --git a/command/agent/acl_endpoint_test.go b/command/agent/acl_endpoint_test.go index 25350fb0f..00f61c1ed 100644 --- a/command/agent/acl_endpoint_test.go +++ b/command/agent/acl_endpoint_test.go @@ -636,7 +636,7 @@ func TestHTTPServer_ACLRoleListRequest(t *testing.T) { // Create two ACL roles and put these directly into state. aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, srv.server.State().UpsertACLRoles(structs.MsgTypeTestSetup, 20, aclRoles)) + require.NoError(t, srv.server.State().UpsertACLRoles(structs.MsgTypeTestSetup, 20, aclRoles, false)) // Build the HTTP request. req, err := http.NewRequest(http.MethodGet, "/v1/acl/roles", nil) @@ -669,7 +669,7 @@ func TestHTTPServer_ACLRoleListRequest(t *testing.T) { // using a custom prefix. aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} aclRoles[1].ID = "badger-badger-badger-" + uuid.Generate() - require.NoError(t, srv.server.State().UpsertACLRoles(structs.MsgTypeTestSetup, 20, aclRoles)) + require.NoError(t, srv.server.State().UpsertACLRoles(structs.MsgTypeTestSetup, 20, aclRoles, false)) // Build the HTTP request. req, err := http.NewRequest(http.MethodGet, "/v1/acl/roles?prefix=badger-badger-badger", nil) @@ -901,7 +901,7 @@ func TestHTTPServer_ACLRoleSpecificRequest(t *testing.T) { // Create a mock role and put directly into state. mockACLRole := mock.ACLRole() require.NoError(t, srv.server.State().UpsertACLRoles( - structs.MsgTypeTestSetup, 20, []*structs.ACLRole{mockACLRole})) + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{mockACLRole}, false)) url := fmt.Sprintf("/v1/acl/role/name/%s", mockACLRole.Name) @@ -935,7 +935,7 @@ func TestHTTPServer_ACLRoleSpecificRequest(t *testing.T) { // Create a mock role and put directly into state. mockACLRole := mock.ACLRole() require.NoError(t, srv.server.State().UpsertACLRoles( - structs.MsgTypeTestSetup, 20, []*structs.ACLRole{mockACLRole})) + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{mockACLRole}, false)) url := fmt.Sprintf("/v1/acl/role/%s", mockACLRole.ID) diff --git a/nomad/acl_endpoint.go b/nomad/acl_endpoint.go index d4161dd30..3de416ee7 100644 --- a/nomad/acl_endpoint.go +++ b/nomad/acl_endpoint.go @@ -1128,15 +1128,18 @@ func (a *ACL) UpsertRoles( // as ensure the policies exist within state. for _, policyLink := range role.Policies { - // Perform a state look up for the policy. An error or not being - // able to find the policy is terminal. We can include the name in - // the error message as it has previously been validated. - existing, err := stateSnapshot.ACLPolicyByName(nil, policyLink.Name) - if err != nil { - return structs.NewErrRPCCodedf(http.StatusInternalServerError, "policy lookup failed: %v", err) - } - if existing == nil { - return structs.NewErrRPCCodedf(http.StatusBadRequest, "cannot find policy %s", policyLink.Name) + // If the RPC does not allow for missing policies, perform a state + // look up for the policy. An error or not being able to find the + // policy is terminal. We can include the name in the error message + // as it has previously been validated. + if !args.AllowMissingPolicies { + existing, err := stateSnapshot.ACLPolicyByName(nil, policyLink.Name) + if err != nil { + return structs.NewErrRPCCodedf(http.StatusInternalServerError, "policy lookup failed: %v", err) + } + if existing == nil { + return structs.NewErrRPCCodedf(http.StatusBadRequest, "cannot find policy %s", policyLink.Name) + } } // If the policy name is not found within our map, this means we @@ -1274,6 +1277,12 @@ func (a *ACL) ListRoles( queryMeta: &reply.QueryMeta, run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + // The iteration below appends directly to the reply object, so in + // order for blocking queries to work properly we must ensure the + // ACLRoles are reset. This allows the blocking query run function + // to work as expected. + reply.ACLRoles = nil + var ( err error iter memdb.ResultIterator @@ -1305,6 +1314,59 @@ func (a *ACL) ListRoles( }) } +// GetRolesByID is used to get a set of ACL Roles as defined by their ID. This +// endpoint is used by the replication process and uses a specific response in +// order to make that process easier. +func (a *ACL) GetRolesByID(args *structs.ACLRolesByIDRequest, reply *structs.ACLRolesByIDResponse) error { + + // This endpoint is only used by the replication process which is only + // running on ACL enabled clusters, so this check should never be + // triggered. + if !a.srv.config.ACLEnabled { + return aclDisabled + } + + if done, err := a.srv.forward(structs.ACLGetRolesByIDRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "acl", "get_roles_id"}, time.Now()) + + // Check that the caller has a management token and that ACLs are enabled + // properly. + if acl, err := a.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if acl == nil || !acl.IsManagement() { + return structs.ErrPermissionDenied + } + + // Set up and return the blocking query + return a.srv.blockingRPC(&blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + + // Instantiate the output map to the correct maximum length. + reply.ACLRoles = make(map[string]*structs.ACLRole, len(args.ACLRoleIDs)) + + // Look for the ACL role and add this to our mapping if we have + // found it. + for _, roleID := range args.ACLRoleIDs { + out, err := stateStore.GetACLRoleByID(ws, roleID) + if err != nil { + return err + } + if out != nil { + reply.ACLRoles[out.ID] = out + } + } + + // Use the index table to populate the query meta as we have no way + // of tracking the max index on deletes. + return a.srv.setReplyQueryMeta(stateStore, state.TableACLRoles, &reply.QueryMeta) + }, + }) +} + // GetRoleByID is used to look up an individual ACL role using its ID. func (a *ACL) GetRoleByID( args *structs.ACLRoleByIDRequest, diff --git a/nomad/acl_endpoint_test.go b/nomad/acl_endpoint_test.go index dfff2caff..1e01eb1f2 100644 --- a/nomad/acl_endpoint_test.go +++ b/nomad/acl_endpoint_test.go @@ -1657,7 +1657,7 @@ func TestACLEndpoint_UpsertTokens(t *testing.T) { aclRole1.Policies = []*structs.ACLRolePolicyLink{{Name: policy1.Name}} require.NoError(t, testServer.fsm.State().UpsertACLRoles( - structs.MsgTypeTestSetup, 20, []*structs.ACLRole{aclRole1})) + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{aclRole1}, false)) // Create a token which references the created ACL role. This // role reference is duplicated to ensure the handler @@ -1710,7 +1710,7 @@ func TestACLEndpoint_UpsertTokens(t *testing.T) { aclRole1.Policies = []*structs.ACLRolePolicyLink{{Name: policy1.Name}} require.NoError(t, testServer.fsm.State().UpsertACLRoles( - structs.MsgTypeTestSetup, 20, []*structs.ACLRole{aclRole1})) + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{aclRole1}, false)) // Create an ACL token with both ACL role and policy links. tokenReq1 := &structs.ACLTokenUpsertRequest{ @@ -2019,7 +2019,7 @@ func TestACL_DeleteRolesByID(t *testing.T) { // Create two ACL roles and put these directly into state. aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles, false)) // Attempt to delete an ACL role without setting an auth token. This should // fail. @@ -2094,7 +2094,7 @@ func TestACL_ListRoles(t *testing.T) { aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} aclRoles[0].ID = "prefix-" + uuid.Generate() aclRoles[1].ID = "prefix-" + uuid.Generate() - require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles, false)) // Try listing roles without a valid ACL token. aclRoleReq1 := &structs.ACLRolesListRequest{ @@ -2145,6 +2145,133 @@ func TestACL_ListRoles(t *testing.T) { err = msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq4, &aclRoleResp4) require.NoError(t, err) require.Len(t, aclRoleResp4.ACLRoles, 2) + + // Now test a blocking query, where we wait for an update to the list which + // is triggered by a deletion. + type res struct { + err error + reply *structs.ACLRolesListResponse + } + resultCh := make(chan *res) + + go func(resultCh chan *res) { + aclRoleReq5 := &structs.ACLRolesListRequest{ + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + MinQueryIndex: aclRoleResp4.Index, + MaxQueryTime: 10 * time.Second, + }, + } + var aclRoleResp5 structs.ACLRolesListResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq5, &aclRoleResp5) + resultCh <- &res{err: err, reply: &aclRoleResp5} + }(resultCh) + + // Delete an ACL role from state which should return the blocking query. + require.NoError(t, testServer.fsm.State().DeleteACLRolesByID( + structs.MsgTypeTestSetup, aclRoleResp4.Index+10, []string{aclRoles[0].ID})) + + // Wait until the test within the routine is complete. + result := <-resultCh + require.NoError(t, result.err) + require.Len(t, result.reply.ACLRoles, 1) + require.NotEqual(t, result.reply.ACLRoles[0].ID, aclRoles[0].ID) +} + +func TestACL_GetRolesByID(t *testing.T) { + ci.Parallel(t) + + testServer, aclRootToken, testServerCleanupFn := TestACLServer(t, nil) + defer testServerCleanupFn() + codec := rpcClient(t, testServer) + testutil.WaitForLeader(t, testServer.RPC) + + // Try reading a role without setting a correct auth token. + aclRoleReq1 := &structs.ACLRolesByIDRequest{ + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + }, + } + var aclRoleResp1 structs.ACLRolesByIDResponse + err := msgpackrpc.CallWithCodec(codec, structs.ACLGetRolesByIDRPCMethod, aclRoleReq1, &aclRoleResp1) + require.ErrorContains(t, err, "Permission denied") + require.Empty(t, aclRoleResp1.ACLRoles) + + // Try reading a role that doesn't exist. + aclRoleReq2 := &structs.ACLRolesByIDRequest{ + ACLRoleIDs: []string{"nope"}, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp2 structs.ACLRolesByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRolesByIDRPCMethod, aclRoleReq2, &aclRoleResp2) + require.NoError(t, err) + require.Empty(t, aclRoleResp2.ACLRoles) + + // Create the policies our ACL roles wants to link to. + policy1 := mock.ACLPolicy() + policy1.Name = "mocked-test-policy-1" + policy2 := mock.ACLPolicy() + policy2.Name = "mocked-test-policy-2" + + require.NoError(t, testServer.fsm.State().UpsertACLPolicies( + structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) + + // Create two ACL roles and put these directly into state. + aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} + require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 20, aclRoles, false)) + + // Try reading both roles that are within state. + aclRoleReq3 := &structs.ACLRolesByIDRequest{ + ACLRoleIDs: []string{aclRoles[0].ID, aclRoles[1].ID}, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp3 structs.ACLRolesByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRolesByIDRPCMethod, aclRoleReq3, &aclRoleResp3) + require.NoError(t, err) + require.Len(t, aclRoleResp3.ACLRoles, 2) + require.Contains(t, aclRoleResp3.ACLRoles, aclRoles[0].ID) + require.Contains(t, aclRoleResp3.ACLRoles, aclRoles[1].ID) + + // Now test a blocking query, where we wait for an update to the set which + // is triggered by a deletion. + type res struct { + err error + reply *structs.ACLRolesByIDResponse + } + resultCh := make(chan *res) + + go func(resultCh chan *res) { + aclRoleReq4 := &structs.ACLRolesByIDRequest{ + ACLRoleIDs: []string{aclRoles[0].ID, aclRoles[1].ID}, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + MinQueryIndex: aclRoleResp3.Index, + MaxQueryTime: 10 * time.Second, + }, + } + var aclRoleResp4 structs.ACLRolesByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRolesByIDRPCMethod, aclRoleReq4, &aclRoleResp4) + resultCh <- &res{err: err, reply: &aclRoleResp4} + }(resultCh) + + // Delete an ACL role from state which should return the blocking query. + require.NoError(t, testServer.fsm.State().DeleteACLRolesByID( + structs.MsgTypeTestSetup, aclRoleResp3.Index+10, []string{aclRoles[0].ID})) + + // Wait for the result and then test it. + result := <-resultCh + require.NoError(t, result.err) + require.Len(t, result.reply.ACLRoles, 1) + _, ok := result.reply.ACLRoles[aclRoles[1].ID] + require.True(t, ok) } func TestACL_GetRoleByID(t *testing.T) { @@ -2166,7 +2293,7 @@ func TestACL_GetRoleByID(t *testing.T) { // Create two ACL roles and put these directly into state. aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles, false)) // Try reading a role without setting a correct auth token. aclRoleReq1 := &structs.ACLRoleByIDRequest{ @@ -2236,7 +2363,7 @@ func TestACL_GetRoleByName(t *testing.T) { // Create two ACL roles and put these directly into state. aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + require.NoError(t, testServer.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles, false)) // Try reading a role without setting a correct auth token. aclRoleReq1 := &structs.ACLRoleByNameRequest{ diff --git a/nomad/acl_test.go b/nomad/acl_test.go index acd5455db..4be4e1f4b 100644 --- a/nomad/acl_test.go +++ b/nomad/acl_test.go @@ -195,7 +195,7 @@ func TestResolveACLToken(t *testing.T) { {Name: policy2.Name}, } err = testServer.State().UpsertACLRoles( - structs.MsgTypeTestSetup, 30, []*structs.ACLRole{aclRole}) + structs.MsgTypeTestSetup, 30, []*structs.ACLRole{aclRole}, false) require.NoError(t, err) clientToken := mock.ACLToken() @@ -221,7 +221,7 @@ func TestResolveACLToken(t *testing.T) { // permissions are updated. aclRole.Policies = []*structs.ACLRolePolicyLink{} err = testServer.State().UpsertACLRoles( - structs.MsgTypeTestSetup, 40, []*structs.ACLRole{aclRole}) + structs.MsgTypeTestSetup, 40, []*structs.ACLRole{aclRole}, false) require.NoError(t, err) aclResp, err = testServer.ResolveToken(clientToken.SecretID) @@ -265,7 +265,7 @@ func TestResolveACLToken(t *testing.T) { aclRole := mock.ACLRole() aclRole.Policies = []*structs.ACLRolePolicyLink{{Name: policy2.Name}} err = testServer.State().UpsertACLRoles( - structs.MsgTypeTestSetup, 20, []*structs.ACLRole{aclRole}) + structs.MsgTypeTestSetup, 20, []*structs.ACLRole{aclRole}, false) require.NoError(t, err) // Create a token which references the policy and role. diff --git a/nomad/fsm.go b/nomad/fsm.go index abfca1ce7..0b9232231 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -2036,7 +2036,7 @@ func (n *nomadFSM) applyACLRolesUpsert(msgType structs.MessageType, buf []byte, panic(fmt.Errorf("failed to decode request: %v", err)) } - if err := n.state.UpsertACLRoles(msgType, index, req.ACLRoles); err != nil { + if err := n.state.UpsertACLRoles(msgType, index, req.ACLRoles, req.AllowMissingPolicies); err != nil { n.logger.Error("UpsertACLRoles failed", "error", err) return err } diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 3f18cad57..ff858c2d5 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -2911,7 +2911,7 @@ func TestFSM_SnapshotRestore_ACLRoles(t *testing.T) { // Generate and upsert some ACL roles. aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles, false)) // Perform a snapshot restore. restoredFSM := testSnapshotRestore(t, fsm) @@ -3497,7 +3497,7 @@ func TestFSM_ApplyACLRolesDeleteByID(t *testing.T) { // Generate and upsert two ACL roles. aclRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, fsm.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles)) + require.NoError(t, fsm.State().UpsertACLRoles(structs.MsgTypeTestSetup, 10, aclRoles, false)) // Build and apply our message. req := structs.ACLRolesDeleteByIDRequest{ACLRoleIDs: []string{aclRoles[0].ID, aclRoles[1].ID}} diff --git a/nomad/leader.go b/nomad/leader.go index 756f1d025..503981d66 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -395,6 +395,7 @@ func (s *Server) establishLeadership(stopCh chan struct{}) error { default: go s.replicateACLPolicies(stopCh) go s.replicateACLTokens(stopCh) + go s.replicateACLRoles(stopCh) go s.replicateNamespaces(stopCh) } } @@ -1689,6 +1690,229 @@ func diffACLTokens(store *state.StateStore, minIndex uint64, remoteList []*struc return } +// replicateACLRoles is used to replicate ACL Roles from the authoritative +// region to this region. The loop should only be run on the leader within the +// federated region. +func (s *Server) replicateACLRoles(stopCh chan struct{}) { + + // Generate our request object. We only need to do this once and reuse it + // for every RPC request. The MinQueryIndex is updated after every + // successful replication loop, so the next query acts as a blocking query + // and only returns upon a change in the authoritative region. + req := structs.ACLRolesListRequest{ + QueryOptions: structs.QueryOptions{ + AllowStale: true, + Region: s.config.AuthoritativeRegion, + }, + } + + // Create our replication rate limiter for ACL roles and log a lovely + // message to indicate the process is starting. + limiter := rate.NewLimiter(replicationRateLimit, int(replicationRateLimit)) + s.logger.Debug("starting ACL Role replication from authoritative region", + "authoritative_region", req.Region) + + // Enter the main ACL Role replication loop that will only exit when the + // stopCh is closed. + // + // Any error encountered will use the replicationBackoffContinue function + // which handles replication backoff and shutdown coordination in the event + // of an error inside the loop. + for { + select { + case <-stopCh: + return + default: + + // Rate limit how often we attempt replication. It is OK to ignore + // the error as the context will never be cancelled and the limit + // parameters are controlled internally. + _ = limiter.Wait(context.Background()) + + // Set the replication token on each replication iteration so that + // it is always current and can handle agent SIGHUP reloads. + req.AuthToken = s.ReplicationToken() + + var resp structs.ACLRolesListResponse + + // Make the list RPC request to the authoritative region, so we + // capture the latest ACL role listing. + err := s.forwardRegion(s.config.AuthoritativeRegion, structs.ACLListRolesRPCMethod, &req, &resp) + if err != nil { + s.logger.Error("failed to fetch ACL Roles from authoritative region", "error", err) + if s.replicationBackoffContinue(stopCh) { + continue + } else { + return + } + } + + // Perform a two-way diff on the ACL roles. + toDelete, toUpdate := diffACLRoles(s.State(), req.MinQueryIndex, resp.ACLRoles) + + // A significant amount of time could pass between the last check + // on whether we should stop the replication process. Therefore, do + // a check here, before calling Raft. + select { + case <-stopCh: + return + default: + } + + // If we have ACL roles to delete, make this call directly to Raft. + if len(toDelete) > 0 { + args := structs.ACLRolesDeleteByIDRequest{ACLRoleIDs: toDelete} + _, _, err := s.raftApply(structs.ACLRolesDeleteByIDRequestType, &args) + + // If the error was because we lost leadership while calling + // Raft, avoid logging as this can be confusing to operators. + if err != nil { + if err != raft.ErrLeadershipLost { + s.logger.Error("failed to delete ACL roles", "error", err) + } + if s.replicationBackoffContinue(stopCh) { + continue + } else { + return + } + } + } + + // Fetch any outdated policies. + var fetched []*structs.ACLRole + if len(toUpdate) > 0 { + req := structs.ACLRolesByIDRequest{ + ACLRoleIDs: toUpdate, + QueryOptions: structs.QueryOptions{ + Region: s.config.AuthoritativeRegion, + AuthToken: s.ReplicationToken(), + AllowStale: true, + MinQueryIndex: resp.Index - 1, + }, + } + var reply structs.ACLRolesByIDResponse + if err := s.forwardRegion(s.config.AuthoritativeRegion, structs.ACLGetRolesByIDRPCMethod, &req, &reply); err != nil { + s.logger.Error("failed to fetch ACL Roles from authoritative region", "error", err) + if s.replicationBackoffContinue(stopCh) { + continue + } else { + return + } + } + for _, aclRole := range reply.ACLRoles { + fetched = append(fetched, aclRole) + } + } + + // Update local tokens + if len(fetched) > 0 { + + // The replication of ACL roles and policies are independent, + // therefore we cannot ensure the policies linked within the + // role are present. We must set allow missing to true. + args := structs.ACLRolesUpsertRequest{ + ACLRoles: fetched, + AllowMissingPolicies: true, + } + + // Perform the upsert directly via Raft. + _, _, err := s.raftApply(structs.ACLRolesUpsertRequestType, &args) + if err != nil { + s.logger.Error("failed to update ACL roles", "error", err) + if s.replicationBackoffContinue(stopCh) { + continue + } else { + return + } + } + } + + // Update the minimum query index, blocks until there is a change. + req.MinQueryIndex = resp.Index + } + } +} + +// replicationBackoffContinue should be used when a replication loop encounters +// an error and wants to wait until either the backoff time has been met, or +// the stopCh has been closed. The boolean indicates whether the replication +// process should continue. +// +// Typical use: +// +// if s.replicationBackoffContinue(stopCh) { +// continue +// } else { +// return +// } +func (s *Server) replicationBackoffContinue(stopCh chan struct{}) bool { + + timer, timerStopFn := helper.NewSafeTimer(s.config.ReplicationBackoff) + defer timerStopFn() + + select { + case <-timer.C: + return true + case <-stopCh: + return false + } +} + +// diffACLRoles is used to perform a two-way diff between the local ACL Roles +// and the remote Roles to determine which tokens need to be deleted or +// updated. The returned array's contain ACL Role IDs. +func diffACLRoles( + store *state.StateStore, minIndex uint64, remoteList []*structs.ACLRole) ( + delete []string, update []string) { + + // The local ACL role tracking is keyed by the role ID and the value is the + // hash of the role. + local := make(map[string][]byte) + + // The remote ACL role tracking is keyed by the role ID; the value is an + // empty struct as we already have the full object. + remote := make(map[string]struct{}) + + // Read all the ACL role currently held within our local state. This panic + // will only happen as a developer making a mistake with naming the index + // to use. + iter, err := store.GetACLRoles(nil) + if err != nil { + panic(fmt.Sprintf("failed to iterate local ACL roles: %v", err)) + } + + // Iterate the local ACL roles and add them to our tracking of local roles. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + aclRole := raw.(*structs.ACLRole) + local[aclRole.ID] = aclRole.Hash + } + + // Iterate over the remote ACL roles. + for _, remoteACLRole := range remoteList { + remote[remoteACLRole.ID] = struct{}{} + + // Identify whether the ACL role is within the local state. If it is + // not, add this to our update list. + if localHash, ok := local[remoteACLRole.ID]; !ok { + update = append(update, remoteACLRole.ID) + + // Check if ACL role is newer remotely and there is a hash + // mismatch. + } else if remoteACLRole.ModifyIndex > minIndex && !bytes.Equal(localHash, remoteACLRole.Hash) { + update = append(update, remoteACLRole.ID) + } + } + + // If we have ACL roles within state which are no longer present in the + // authoritative region we should delete them. + for localACLRole := range local { + if _, ok := remote[localACLRole]; !ok { + delete = append(delete, localACLRole) + } + } + return +} + // getOrCreateAutopilotConfig is used to get the autopilot config, initializing it if necessary func (s *Server) getOrCreateAutopilotConfig() *structs.AutopilotConfig { state := s.fsm.State() @@ -1807,10 +2031,10 @@ func (s *Server) generateClusterID() (string, error) { // // The function checks the server is the leader and uses a mutex to avoid any // potential timings problems. Consider the following timings: -// - operator updates the configuration via the API -// - the RPC handler applies the change via Raft -// - leadership transitions with write barrier -// - the RPC handler call this function to enact the change +// - operator updates the configuration via the API +// - the RPC handler applies the change via Raft +// - leadership transitions with write barrier +// - the RPC handler call this function to enact the change // // The mutex also protects against a situation where leadership is revoked // while this function is being called. Ensuring the correct series of actions diff --git a/nomad/leader_test.go b/nomad/leader_test.go index 2bd1c1070..6cf117ba4 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -1026,6 +1026,113 @@ func TestLeader_DiffACLTokens(t *testing.T) { assert.Equal(t, []string{p3.AccessorID, p4.AccessorID}, update) } +func TestServer_replicationBackoffContinue(t *testing.T) { + ci.Parallel(t) + + testCases := []struct { + name string + testFn func() + }{ + { + name: "leadership lost", + testFn: func() { + + // Create a test server with a long enough backoff that we will + // be able to close the channel before it fires, but not too + // long that the test having problems means CI will hang + // forever. + testServer, testServerCleanup := TestServer(t, func(c *Config) { + c.ReplicationBackoff = 5 * time.Second + }) + defer testServerCleanup() + + // Create our stop channel which is used by the server to + // indicate leadership loss. + stopCh := make(chan struct{}) + + // The resultCh is used to block and collect the output from + // the test routine. + resultCh := make(chan bool, 1) + + // Run a routine to collect the result and close the channel + // straight away. + go func() { + output := testServer.replicationBackoffContinue(stopCh) + resultCh <- output + }() + + close(stopCh) + + actualResult := <-resultCh + require.False(t, actualResult) + }, + }, + { + name: "backoff continue", + testFn: func() { + + // Create a test server with a short backoff. + testServer, testServerCleanup := TestServer(t, func(c *Config) { + c.ReplicationBackoff = 10 * time.Nanosecond + }) + defer testServerCleanup() + + // Create our stop channel which is used by the server to + // indicate leadership loss. + stopCh := make(chan struct{}) + + // The resultCh is used to block and collect the output from + // the test routine. + resultCh := make(chan bool, 1) + + // Run a routine to collect the result without closing stopCh. + go func() { + output := testServer.replicationBackoffContinue(stopCh) + resultCh <- output + }() + + actualResult := <-resultCh + require.True(t, actualResult) + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn() + }) + } +} + +func Test_diffACLRoles(t *testing.T) { + ci.Parallel(t) + + stateStore := state.TestStateStore(t) + + // Build an initial baseline of ACL Roles. + aclRole0 := mock.ACLRole() + aclRole1 := mock.ACLRole() + aclRole2 := mock.ACLRole() + aclRole3 := mock.ACLRole() + + // Upsert these into our local state. Use copies, so we can alter the roles + // directly and use within the diff func. + err := stateStore.UpsertACLRoles(structs.MsgTypeTestSetup, 50, + []*structs.ACLRole{aclRole0.Copy(), aclRole1.Copy(), aclRole2.Copy(), aclRole3.Copy()}, true) + require.NoError(t, err) + + // Modify the ACL roles to create a number of differences. These roles + // represent the state of the authoritative region. + aclRole2.ModifyIndex = 50 + aclRole3.ModifyIndex = 200 + aclRole3.Hash = []byte{0, 1, 2, 3} + aclRole4 := mock.ACLRole() + + // Run the diff function and test the output. + toDelete, toUpdate := diffACLRoles(stateStore, 50, []*structs.ACLRole{aclRole2, aclRole3, aclRole4}) + require.ElementsMatch(t, []string{aclRole0.ID, aclRole1.ID}, toDelete) + require.ElementsMatch(t, []string{aclRole3.ID, aclRole4.ID}, toUpdate) +} + func TestLeader_UpgradeRaftVersion(t *testing.T) { ci.Parallel(t) diff --git a/nomad/state/state_store_acl.go b/nomad/state/state_store_acl.go index 1f9f273ca..faaa85ff5 100644 --- a/nomad/state/state_store_acl.go +++ b/nomad/state/state_store_acl.go @@ -39,7 +39,7 @@ func expiresIndexName(global bool) string { // It uses a single write transaction for efficiency, however, any error means // no entries will be committed. func (s *StateStore) UpsertACLRoles( - msgType structs.MessageType, index uint64, roles []*structs.ACLRole) error { + msgType structs.MessageType, index uint64, roles []*structs.ACLRole, allowMissingPolicies bool) error { // Grab a write transaction. txn := s.db.WriteTxnMsgT(msgType, index) @@ -53,7 +53,7 @@ func (s *StateStore) UpsertACLRoles( // fail via the txn.Abort() defer. for _, role := range roles { - roleUpdated, err := s.upsertACLRoleTxn(index, txn, role) + roleUpdated, err := s.upsertACLRoleTxn(index, txn, role, allowMissingPolicies) if err != nil { return err } @@ -79,7 +79,7 @@ func (s *StateStore) UpsertACLRoles( // provided write transaction. It is the responsibility of the caller to update // the index table. func (s *StateStore) upsertACLRoleTxn( - index uint64, txn *txn, role *structs.ACLRole) (bool, error) { + index uint64, txn *txn, role *structs.ACLRole, allowMissingPolicies bool) (bool, error) { // Ensure the role hash is not zero to provide defense in depth. This // should be done outside the state store, so we do not spend time here @@ -92,8 +92,10 @@ func (s *StateStore) upsertACLRoleTxn( // could mean that by the time the state call is invoked, another Raft // update has deleted policies detailed in role. Therefore, check again // while in our write txn. - if err := s.validateACLRolePolicyLinksTxn(txn, role); err != nil { - return false, err + if !allowMissingPolicies { + if err := s.validateACLRolePolicyLinksTxn(txn, role); err != nil { + return false, err + } } existing, err := txn.First(TableACLRoles, indexID, role.ID) diff --git a/nomad/state/state_store_acl_test.go b/nomad/state/state_store_acl_test.go index 16e375579..8458e88e7 100644 --- a/nomad/state/state_store_acl_test.go +++ b/nomad/state/state_store_acl_test.go @@ -159,7 +159,7 @@ func TestStateStore_UpsertACLRoles(t *testing.T) { // straight into state. It should fail because the ACL policies do not // exist. mockedACLRoles := []*structs.ACLRole{mock.ACLRole()} - err := testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles) + err := testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles, false) require.ErrorContains(t, err, "policy not found") // Create the policies our ACL roles wants to link to and then try the @@ -171,7 +171,7 @@ func TestStateStore_UpsertACLRoles(t *testing.T) { require.NoError(t, testState.UpsertACLPolicies( structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1, policy2})) - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles, false)) // Check that the index for the table was modified as expected. initialIndex, err := testState.Index(TableACLRoles) @@ -200,7 +200,7 @@ func TestStateStore_UpsertACLRoles(t *testing.T) { // Try writing the same ACL roles to state which should not result in an // update to the table index. - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 30, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 30, mockedACLRoles, false)) reInsertActualIndex, err := testState.Index(TableACLRoles) require.NoError(t, err) must.Eq(t, 20, reInsertActualIndex) @@ -211,7 +211,7 @@ func TestStateStore_UpsertACLRoles(t *testing.T) { updatedMockedACLRole.Policies = []*structs.ACLRolePolicyLink{{Name: "mocked-test-policy-1"}} updatedMockedACLRole.SetHash() require.NoError(t, testState.UpsertACLRoles( - structs.MsgTypeTestSetup, 30, []*structs.ACLRole{updatedMockedACLRole})) + structs.MsgTypeTestSetup, 30, []*structs.ACLRole{updatedMockedACLRole}, false)) // Check that the index for the table was modified as expected. updatedIndex, err := testState.Index(TableACLRoles) @@ -235,6 +235,17 @@ func TestStateStore_UpsertACLRoles(t *testing.T) { must.Eq(t, 30, aclRole.ModifyIndex) } require.Equal(t, 1, count, "incorrect number of ACL roles found") + + // Now try inserting an ACL role using the missing policies' argument to + // simulate replication. + replicatedACLRole := mock.ACLRole() + replicatedACLRole.Policies = []*structs.ACLRolePolicyLink{{Name: "nope"}} + require.NoError(t, testState.UpsertACLRoles( + structs.MsgTypeTestSetup, 40, []*structs.ACLRole{replicatedACLRole}, true)) + + replicatedACLRoleResp, err := testState.GetACLRoleByName(ws, replicatedACLRole.Name) + require.NoError(t, err) + must.Eq(t, replicatedACLRole.Hash, replicatedACLRoleResp.Hash) } func TestStateStore_ValidateACLRolePolicyLinks(t *testing.T) { @@ -245,7 +256,7 @@ func TestStateStore_ValidateACLRolePolicyLinks(t *testing.T) { mockedACLRoles := []*structs.ACLRole{mock.ACLRole()} // This should error as no policies exist within state. - err := testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles) + err := testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles, false) require.ErrorContains(t, err, "ACL policy not found") // Upsert one ACL policy and retry the role which should still fail. @@ -253,7 +264,7 @@ func TestStateStore_ValidateACLRolePolicyLinks(t *testing.T) { policy1.Name = "mocked-test-policy-1" require.NoError(t, testState.UpsertACLPolicies(structs.MsgTypeTestSetup, 10, []*structs.ACLPolicy{policy1})) - err = testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles) + err = testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles, false) require.ErrorContains(t, err, "ACL policy not found") // Upsert the second ACL policy. The ACL role should now upsert into state @@ -262,7 +273,7 @@ func TestStateStore_ValidateACLRolePolicyLinks(t *testing.T) { policy2.Name = "mocked-test-policy-2" require.NoError(t, testState.UpsertACLPolicies(structs.MsgTypeTestSetup, 20, []*structs.ACLPolicy{policy2})) - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 30, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 30, mockedACLRoles, false)) } func TestStateStore_DeleteACLRolesByID(t *testing.T) { @@ -281,7 +292,7 @@ func TestStateStore_DeleteACLRolesByID(t *testing.T) { // Generate a some mocked ACL roles for testing and upsert these straight // into state. mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles, false)) // Try and delete a role using a name that doesn't exist. This should // return an error and not change the index for the table. @@ -353,7 +364,7 @@ func TestStateStore_GetACLRoles(t *testing.T) { // Generate a some mocked ACL roles for testing and upsert these straight // into state. mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles, false)) // List the ACL roles and ensure they are exactly as we expect. ws := memdb.NewWatchSet() @@ -391,7 +402,7 @@ func TestStateStore_GetACLRoleByID(t *testing.T) { // Generate a some mocked ACL roles for testing and upsert these straight // into state. mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles, false)) ws := memdb.NewWatchSet() @@ -426,7 +437,7 @@ func TestStateStore_GetACLRoleByName(t *testing.T) { // Generate a some mocked ACL roles for testing and upsert these straight // into state. mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles, false)) ws := memdb.NewWatchSet() @@ -464,7 +475,7 @@ func TestStateStore_GetACLRoleByIDPrefix(t *testing.T) { mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} mockedACLRoles[0].ID = "test-prefix-" + uuid.Generate() mockedACLRoles[1].ID = "test-prefix-" + uuid.Generate() - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 10, mockedACLRoles, false)) ws := memdb.NewWatchSet() @@ -513,7 +524,7 @@ func TestStateStore_fixTokenRoleLinks(t *testing.T) { // Generate a some mocked ACL roles for testing and upsert these straight // into state. mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles, false)) // Create an ACL token linking to the ACL role. token1 := mock.ACLToken() @@ -548,7 +559,7 @@ func TestStateStore_fixTokenRoleLinks(t *testing.T) { // Generate a some mocked ACL roles for testing and upsert these straight // into state. mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles, false)) // Create an ACL token linking to the ACL roles. token1 := mock.ACLToken() @@ -588,7 +599,7 @@ func TestStateStore_fixTokenRoleLinks(t *testing.T) { // Generate a some mocked ACL roles for testing and upsert these straight // into state. mockedACLRoles := []*structs.ACLRole{mock.ACLRole(), mock.ACLRole()} - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 20, mockedACLRoles, false)) // Create an ACL token linking to the ACL roles. token1 := mock.ACLToken() @@ -598,7 +609,7 @@ func TestStateStore_fixTokenRoleLinks(t *testing.T) { // Now change the name of one of the ACL roles. mockedACLRoles[0].Name = "badger-badger-badger" - require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 40, mockedACLRoles)) + require.NoError(t, testState.UpsertACLRoles(structs.MsgTypeTestSetup, 40, mockedACLRoles, false)) // Perform the fix and check the returned token contains the // correct roles. diff --git a/nomad/structs/acl.go b/nomad/structs/acl.go index 8ab6862b1..4c7cc5c3e 100644 --- a/nomad/structs/acl.go +++ b/nomad/structs/acl.go @@ -56,6 +56,14 @@ const ( // Reply: ACLRolesListResponse ACLListRolesRPCMethod = "ACL.ListRoles" + // ACLGetRolesByIDRPCMethod is the RPC method for detailing a number of ACL + // roles using their ID. This is an internal only RPC endpoint and used by + // the ACL Role replication process. + // + // Args: ACLRolesByIDRequest + // Reply: ACLRolesByIDResponse + ACLGetRolesByIDRPCMethod = "ACL.GetRolesByID" + // ACLGetRoleByIDRPCMethod is the RPC method for detailing an individual // ACL role using its ID. // @@ -360,6 +368,12 @@ func (a *ACLRole) Copy() *ACLRole { // roles. type ACLRolesUpsertRequest struct { ACLRoles []*ACLRole + + // AllowMissingPolicies skips the ACL Role policy link verification and is + // used by the replication process. The replication cannot ensure policies + // are present before ACL Roles are replicated. + AllowMissingPolicies bool + WriteRequest } @@ -395,6 +409,20 @@ type ACLRolesListResponse struct { QueryMeta } +// ACLRolesByIDRequest is the request object when performing a lookup of +// multiple roles by the ID. +type ACLRolesByIDRequest struct { + ACLRoleIDs []string + QueryOptions +} + +// ACLRolesByIDResponse is the response object when performing a lookup of +// multiple roles by their IDs. +type ACLRolesByIDResponse struct { + ACLRoles map[string]*ACLRole + QueryMeta +} + // ACLRoleByIDRequest is the request object to perform a lookup of an ACL // role using a specific ID. type ACLRoleByIDRequest struct { diff --git a/nomad/structs/acl_test.go b/nomad/structs/acl_test.go index 1a9b426c4..ca41ab6c5 100644 --- a/nomad/structs/acl_test.go +++ b/nomad/structs/acl_test.go @@ -682,6 +682,11 @@ func Test_ACLRolesListRequest(t *testing.T) { require.True(t, req.IsRead()) } +func Test_ACLRolesByIDRequest(t *testing.T) { + req := ACLRolesByIDRequest{} + require.True(t, req.IsRead()) +} + func Test_ACLRoleByIDRequest(t *testing.T) { req := ACLRoleByIDRequest{} require.True(t, req.IsRead()) From 2736cf0dfa707ef12607fee8f3b85cd6816e02d1 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 22 Aug 2022 17:20:23 +0200 Subject: [PATCH 15/19] acl: make listing RPC and HTTP API a stub return object. (#14211) Making the ACL Role listing return object a stub future-proofs the endpoint. In the event the role object grows, we are not bound by having to return all fields within the list endpoint or change the signature of the endpoint to reduce the list return size. --- api/acl.go | 35 +++++++++++++++- command/acl_role_list.go | 2 +- command/agent/acl_endpoint.go | 2 +- command/agent/acl_endpoint_test.go | 8 ++-- nomad/acl_endpoint.go | 6 +-- nomad/leader.go | 2 +- nomad/leader_test.go | 3 +- nomad/structs/acl.go | 50 +++++++++++++++++++++- nomad/structs/acl_test.go | 66 ++++++++++++++++++++++++++++++ 9 files changed, 159 insertions(+), 15 deletions(-) diff --git a/api/acl.go b/api/acl.go index 223474ecf..0562a87d9 100644 --- a/api/acl.go +++ b/api/acl.go @@ -220,8 +220,8 @@ func (c *Client) ACLRoles() *ACLRoles { } // List is used to detail all the ACL roles currently stored within state. -func (a *ACLRoles) List(q *QueryOptions) ([]*ACLRole, *QueryMeta, error) { - var resp []*ACLRole +func (a *ACLRoles) List(q *QueryOptions) ([]*ACLRoleListStub, *QueryMeta, error) { + var resp []*ACLRoleListStub qm, err := a.client.query("/v1/acl/roles", &resp, q) if err != nil { return nil, nil, err @@ -435,3 +435,34 @@ type ACLRolePolicyLink struct { // Name is the ACLPolicy.Name value which will be linked to the ACL role. Name string } + +// ACLRoleListStub is the stub object returned when performing a listing of ACL +// roles. While it might not currently be different to the full response +// object, it allows us to future-proof the RPC in the event the ACLRole object +// grows over time. +type ACLRoleListStub struct { + + // ID is an internally generated UUID for this role and is controlled by + // Nomad. + ID string + + // Name is unique across the entire set of federated clusters and is + // supplied by the operator on role creation. The name can be modified by + // updating the role and including the Nomad generated ID. This update will + // not affect tokens created and linked to this role. This is a required + // field. + Name string + + // Description is a human-readable, operator set description that can + // provide additional context about the role. This is an operational field. + Description string + + // Policies is an array of ACL policy links. Although currently policies + // can only be linked using their name, in the future we will want to add + // IDs also and thus allow operators to specify either a name, an ID, or + // both. + Policies []*ACLRolePolicyLink + + CreateIndex uint64 + ModifyIndex uint64 +} diff --git a/command/acl_role_list.go b/command/acl_role_list.go index c86d819b5..97360b78c 100644 --- a/command/acl_role_list.go +++ b/command/acl_role_list.go @@ -106,7 +106,7 @@ func (a *ACLRoleListCommand) Run(args []string) int { return 0 } -func formatACLRoles(roles []*api.ACLRole) string { +func formatACLRoles(roles []*api.ACLRoleListStub) string { if len(roles) == 0 { return "No ACL roles found" } diff --git a/command/agent/acl_endpoint.go b/command/agent/acl_endpoint.go index ceaecb0b8..67942e86b 100644 --- a/command/agent/acl_endpoint.go +++ b/command/agent/acl_endpoint.go @@ -349,7 +349,7 @@ func (s *HTTPServer) ACLRoleListRequest(resp http.ResponseWriter, req *http.Requ setMeta(resp, &reply.QueryMeta) if reply.ACLRoles == nil { - reply.ACLRoles = make([]*structs.ACLRole, 0) + reply.ACLRoles = make([]*structs.ACLRoleListStub, 0) } return reply.ACLRoles, nil } diff --git a/command/agent/acl_endpoint_test.go b/command/agent/acl_endpoint_test.go index 00f61c1ed..7effd8511 100644 --- a/command/agent/acl_endpoint_test.go +++ b/command/agent/acl_endpoint_test.go @@ -618,7 +618,7 @@ func TestHTTPServer_ACLRoleListRequest(t *testing.T) { // Send the HTTP request. obj, err := srv.Server.ACLRoleListRequest(respW, req) require.NoError(t, err) - require.Empty(t, obj.([]*structs.ACLRole)) + require.Empty(t, obj.([]*structs.ACLRoleListStub)) }, }, { @@ -649,7 +649,7 @@ func TestHTTPServer_ACLRoleListRequest(t *testing.T) { // Send the HTTP request. obj, err := srv.Server.ACLRoleListRequest(respW, req) require.NoError(t, err) - require.Len(t, obj.([]*structs.ACLRole), 2) + require.Len(t, obj.([]*structs.ACLRoleListStub), 2) }, }, { @@ -682,8 +682,8 @@ func TestHTTPServer_ACLRoleListRequest(t *testing.T) { // Send the HTTP request. obj, err := srv.Server.ACLRoleListRequest(respW, req) require.NoError(t, err) - require.Len(t, obj.([]*structs.ACLRole), 1) - require.Contains(t, obj.([]*structs.ACLRole)[0].ID, "badger-badger-badger") + require.Len(t, obj.([]*structs.ACLRoleListStub), 1) + require.Contains(t, obj.([]*structs.ACLRoleListStub)[0].ID, "badger-badger-badger") }, }, } diff --git a/nomad/acl_endpoint.go b/nomad/acl_endpoint.go index 3de416ee7..5c06abe27 100644 --- a/nomad/acl_endpoint.go +++ b/nomad/acl_endpoint.go @@ -1300,11 +1300,9 @@ func (a *ACL) ListRoles( return err } - // Iterate all the results and add these to our reply object. There - // is no stub object for an ACL role and the hash is needed by the - // replication process. + // Iterate all the results and add these to our reply object. for raw := iter.Next(); raw != nil; raw = iter.Next() { - reply.ACLRoles = append(reply.ACLRoles, raw.(*structs.ACLRole)) + reply.ACLRoles = append(reply.ACLRoles, raw.(*structs.ACLRole).Stub()) } // Use the index table to populate the query meta as we have no way diff --git a/nomad/leader.go b/nomad/leader.go index 503981d66..c6335111d 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -1862,7 +1862,7 @@ func (s *Server) replicationBackoffContinue(stopCh chan struct{}) bool { // and the remote Roles to determine which tokens need to be deleted or // updated. The returned array's contain ACL Role IDs. func diffACLRoles( - store *state.StateStore, minIndex uint64, remoteList []*structs.ACLRole) ( + store *state.StateStore, minIndex uint64, remoteList []*structs.ACLRoleListStub) ( delete []string, update []string) { // The local ACL role tracking is keyed by the role ID and the value is the diff --git a/nomad/leader_test.go b/nomad/leader_test.go index 6cf117ba4..5289d413c 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -1128,7 +1128,8 @@ func Test_diffACLRoles(t *testing.T) { aclRole4 := mock.ACLRole() // Run the diff function and test the output. - toDelete, toUpdate := diffACLRoles(stateStore, 50, []*structs.ACLRole{aclRole2, aclRole3, aclRole4}) + toDelete, toUpdate := diffACLRoles(stateStore, 50, []*structs.ACLRoleListStub{ + aclRole2.Stub(), aclRole3.Stub(), aclRole4.Stub()}) require.ElementsMatch(t, []string{aclRole0.ID, aclRole1.ID}, toDelete) require.ElementsMatch(t, []string{aclRole3.ID, aclRole4.ID}, toUpdate) } diff --git a/nomad/structs/acl.go b/nomad/structs/acl.go index 4c7cc5c3e..ec4978eb8 100644 --- a/nomad/structs/acl.go +++ b/nomad/structs/acl.go @@ -364,6 +364,54 @@ func (a *ACLRole) Copy() *ACLRole { return c } +// Stub converts the ACLRole object into a ACLRoleListStub object. +func (a *ACLRole) Stub() *ACLRoleListStub { + return &ACLRoleListStub{ + ID: a.ID, + Name: a.Name, + Description: a.Description, + Policies: a.Policies, + Hash: a.Hash, + CreateIndex: a.CreateIndex, + ModifyIndex: a.ModifyIndex, + } +} + +// ACLRoleListStub is the stub object returned when performing a listing of ACL +// roles. While it might not currently be different to the full response +// object, it allows us to future-proof the RPC in the event the ACLRole object +// grows over time. +type ACLRoleListStub struct { + + // ID is an internally generated UUID for this role and is controlled by + // Nomad. + ID string + + // Name is unique across the entire set of federated clusters and is + // supplied by the operator on role creation. The name can be modified by + // updating the role and including the Nomad generated ID. This update will + // not affect tokens created and linked to this role. This is a required + // field. + Name string + + // Description is a human-readable, operator set description that can + // provide additional context about the role. This is an operational field. + Description string + + // Policies is an array of ACL policy links. Although currently policies + // can only be linked using their name, in the future we will want to add + // IDs also and thus allow operators to specify either a name, an ID, or + // both. + Policies []*ACLRolePolicyLink + + // Hash is the hashed value of the role and is generated using all fields + // above this point. + Hash []byte + + CreateIndex uint64 + ModifyIndex uint64 +} + // ACLRolesUpsertRequest is the request object used to upsert one or more ACL // roles. type ACLRolesUpsertRequest struct { @@ -405,7 +453,7 @@ type ACLRolesListRequest struct { // ACLRolesListResponse is the response object when performing ACL role // listings. type ACLRolesListResponse struct { - ACLRoles []*ACLRole + ACLRoles []*ACLRoleListStub QueryMeta } diff --git a/nomad/structs/acl_test.go b/nomad/structs/acl_test.go index ca41ab6c5..428c0b6b6 100644 --- a/nomad/structs/acl_test.go +++ b/nomad/structs/acl_test.go @@ -667,6 +667,72 @@ func TestACLRole_Copy(t *testing.T) { } } +func TestACLRole_Stub(t *testing.T) { + testCases := []struct { + name string + inputACLRole *ACLRole + expectedOutput *ACLRoleListStub + }{ + { + name: "partially hydrated", + inputACLRole: &ACLRole{ + ID: "1d6332c8-02d7-325e-f675-a9bb4aff0c51", + Name: "my-lovely-role", + Description: "", + Policies: []*ACLRolePolicyLink{ + {Name: "my-lovely-policy"}, + }, + Hash: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + CreateIndex: 24, + ModifyIndex: 24, + }, + expectedOutput: &ACLRoleListStub{ + ID: "1d6332c8-02d7-325e-f675-a9bb4aff0c51", + Name: "my-lovely-role", + Description: "", + Policies: []*ACLRolePolicyLink{ + {Name: "my-lovely-policy"}, + }, + Hash: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + CreateIndex: 24, + ModifyIndex: 24, + }, + }, + { + name: "hully hydrated", + inputACLRole: &ACLRole{ + ID: "1d6332c8-02d7-325e-f675-a9bb4aff0c51", + Name: "my-lovely-role", + Description: "this-is-my-lovely-role", + Policies: []*ACLRolePolicyLink{ + {Name: "my-lovely-policy"}, + }, + Hash: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + CreateIndex: 24, + ModifyIndex: 24, + }, + expectedOutput: &ACLRoleListStub{ + ID: "1d6332c8-02d7-325e-f675-a9bb4aff0c51", + Name: "my-lovely-role", + Description: "this-is-my-lovely-role", + Policies: []*ACLRolePolicyLink{ + {Name: "my-lovely-policy"}, + }, + Hash: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + CreateIndex: 24, + ModifyIndex: 24, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputACLRole.Stub() + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} + func Test_ACLRolesUpsertRequest(t *testing.T) { req := ACLRolesUpsertRequest{} require.False(t, req.IsRead()) From 9782d6d7ffc8312224ef564c61cb0be46e7b98f1 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 24 Aug 2022 13:51:51 +0200 Subject: [PATCH 16/19] acl: allow tokens to lookup linked roles. (#14227) When listing or reading an ACL role, roles linked to the ACL token used for authentication can be returned to the caller. --- command/agent/acl_endpoint_test.go | 5 +- go.mod | 4 +- go.sum | 8 +- nomad/acl_endpoint.go | 112 +++++++++++++++++++++++++--- nomad/acl_endpoint_test.go | 114 ++++++++++++++++++++++++++--- nomad/state/state_store.go | 16 +++- 6 files changed, 226 insertions(+), 33 deletions(-) diff --git a/command/agent/acl_endpoint_test.go b/command/agent/acl_endpoint_test.go index 7effd8511..6cbc07667 100644 --- a/command/agent/acl_endpoint_test.go +++ b/command/agent/acl_endpoint_test.go @@ -579,9 +579,8 @@ func TestHTTPServer_ACLRoleListRequest(t *testing.T) { // Send the HTTP request. obj, err := srv.Server.ACLRoleListRequest(respW, req) - require.Error(t, err) - require.ErrorContains(t, err, "Permission denied") - require.Nil(t, obj) + require.NoError(t, err) + require.Empty(t, obj) }, }, { diff --git a/go.mod b/go.mod index c1ddc2fd8..49e8c0218 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/hashicorp/go-plugin v1.4.3 github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.4 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 - github.com/hashicorp/go-set v0.1.2 + github.com/hashicorp/go-set v0.1.3 github.com/hashicorp/go-sockaddr v1.0.2 github.com/hashicorp/go-syslog v1.0.0 github.com/hashicorp/go-uuid v1.0.2 @@ -110,7 +110,7 @@ require ( github.com/ryanuber/go-glob v1.0.0 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 github.com/shirou/gopsutil/v3 v3.21.12 - github.com/shoenig/test v0.3.0 + github.com/shoenig/test v0.3.1 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c github.com/stretchr/testify v1.8.0 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 diff --git a/go.sum b/go.sum index 0dfc011d5..99906ce56 100644 --- a/go.sum +++ b/go.sum @@ -755,8 +755,8 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.1 h1:Yc026VyMyIpq1UWRnakHRG01U8fJm+nEfEmjoAb00n8= github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.1/go.mod h1:l8slYwnJA26yBz+ErHpp2IRCLr0vuOMGBORIz4rRiAs= -github.com/hashicorp/go-set v0.1.2 h1:WqFkeT32zKiD/l7zwO1RLF4YwctJwp6IByML0LLa0os= -github.com/hashicorp/go-set v0.1.2/go.mod h1:0jTQeDo6GKX0WMFUV4IicFkxXo9DuoRnUODngpsoYCk= +github.com/hashicorp/go-set v0.1.3 h1:1fyYno7QjlfAaMp1rdkMtMorFgSC5Te2TV+V60OD/cI= +github.com/hashicorp/go-set v0.1.3/go.mod h1:XFMEKCP3rGoZUBvdYwC9k2YVDj8PsMU/B0ITuYkl8IA= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= @@ -1178,8 +1178,8 @@ github.com/shirou/gopsutil v0.0.0-20181107111621-48177ef5f880/go.mod h1:5b4v6he4 github.com/shirou/gopsutil/v3 v3.21.12 h1:VoGxEW2hpmz0Vt3wUvHIl9fquzYLNpVpgNNB7pGJimA= github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shoenig/test v0.3.0 h1:H6tfSvgLrPHRR5NH9S40+lOfoyeH2PbswBr4twgn9Po= -github.com/shoenig/test v0.3.0/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0= +github.com/shoenig/test v0.3.1 h1:dhGZztS6nQuvJ0o0RtUiQHaEO4hhArh/WmWwik3Ols0= +github.com/shoenig/test v0.3.1/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= diff --git a/nomad/acl_endpoint.go b/nomad/acl_endpoint.go index 5c06abe27..f735cb56c 100644 --- a/nomad/acl_endpoint.go +++ b/nomad/acl_endpoint.go @@ -12,6 +12,7 @@ import ( metrics "github.com/armon/go-metrics" log "github.com/hashicorp/go-hclog" memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-set" policy "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" @@ -1264,13 +1265,35 @@ func (a *ACL) ListRoles( } defer metrics.MeasureSince([]string{"nomad", "acl", "list_roles"}, time.Now()) - // TODO (jrasell) allow callers to list role associated to their token. - if acl, err := a.srv.ResolveToken(args.AuthToken); err != nil { + // Resolve the token and ensure it has some form of permissions. + acl, err := a.srv.ResolveToken(args.AuthToken) + if err != nil { return err - } else if acl == nil || !acl.IsManagement() { + } else if acl == nil { return structs.ErrPermissionDenied } + // If the token is a management token, they can list all tokens. If not, + // the role set tracks which role links the token has and therefore which + // ones the caller can list. + isManagement := acl.IsManagement() + roleSet := &set.Set[string]{} + + // If the token is not a management token, we determine which roles are + // linked to the token and therefore can be listed by the caller. + if !isManagement { + token, err := a.requestACLToken(args.AuthToken) + if err != nil { + return err + } + if token == nil { + return structs.ErrTokenNotFound + } + + // Generate a set of Role IDs from the token role links. + roleSet = set.FromFunc(token.Roles, func(roleLink *structs.ACLTokenRoleLink) string { return roleLink.ID }) + } + // Set up and return the blocking query. return a.srv.blockingRPC(&blockingOptions{ queryOpts: &args.QueryOptions, @@ -1300,9 +1323,16 @@ func (a *ACL) ListRoles( return err } - // Iterate all the results and add these to our reply object. + // Iterate all the results and add these to our reply object. Check + // before appending to the reply that the caller is allowed to view + // the role. for raw := iter.Next(); raw != nil; raw = iter.Next() { - reply.ACLRoles = append(reply.ACLRoles, raw.(*structs.ACLRole).Stub()) + + role := raw.(*structs.ACLRole) + + if roleSet.Contains(role.ID) || isManagement { + reply.ACLRoles = append(reply.ACLRoles, role.Stub()) + } } // Use the index table to populate the query meta as we have no way @@ -1380,13 +1410,43 @@ func (a *ACL) GetRoleByID( } defer metrics.MeasureSince([]string{"nomad", "acl", "get_role_id"}, time.Now()) - // TODO (jrasell) allow callers to detail a role associated to their token. - if acl, err := a.srv.ResolveToken(args.AuthToken); err != nil { + // Resolve the token and ensure it has some form of permissions. + acl, err := a.srv.ResolveToken(args.AuthToken) + if err != nil { return err - } else if acl == nil || !acl.IsManagement() { + } else if acl == nil { return structs.ErrPermissionDenied } + // If the token is a management token, they can detail any token they so + // desire. + isManagement := acl.IsManagement() + + // If the token is not a management token, we determine if the caller wants + // to detail a role linked to their token. + if !isManagement { + aclToken, err := a.requestACLToken(args.AuthToken) + if err != nil { + return err + } + if aclToken == nil { + return structs.ErrTokenNotFound + } + + found := false + + for _, roleLink := range aclToken.Roles { + if roleLink.ID == args.RoleID { + found = true + break + } + } + + if !found { + return structs.ErrPermissionDenied + } + } + // Set up and return the blocking query. return a.srv.blockingRPC(&blockingOptions{ queryOpts: &args.QueryOptions, @@ -1435,13 +1495,43 @@ func (a *ACL) GetRoleByName( } defer metrics.MeasureSince([]string{"nomad", "acl", "get_role_name"}, time.Now()) - // TODO (jrasell) allow callers to detail a role associated to their token. - if acl, err := a.srv.ResolveToken(args.AuthToken); err != nil { + // Resolve the token and ensure it has some form of permissions. + acl, err := a.srv.ResolveToken(args.AuthToken) + if err != nil { return err - } else if acl == nil || !acl.IsManagement() { + } else if acl == nil { return structs.ErrPermissionDenied } + // If the token is a management token, they can detail any token they so + // desire. + isManagement := acl.IsManagement() + + // If the token is not a management token, we determine if the caller wants + // to detail a role linked to their token. + if !isManagement { + aclToken, err := a.requestACLToken(args.AuthToken) + if err != nil { + return err + } + if aclToken == nil { + return structs.ErrTokenNotFound + } + + found := false + + for _, roleLink := range aclToken.Roles { + if roleLink.Name == args.RoleName { + found = true + break + } + } + + if !found { + return structs.ErrPermissionDenied + } + } + // Set up and return the blocking query. return a.srv.blockingRPC(&blockingOptions{ queryOpts: &args.QueryOptions, diff --git a/nomad/acl_endpoint_test.go b/nomad/acl_endpoint_test.go index 1e01eb1f2..b134e3902 100644 --- a/nomad/acl_endpoint_test.go +++ b/nomad/acl_endpoint_test.go @@ -2099,12 +2099,13 @@ func TestACL_ListRoles(t *testing.T) { // Try listing roles without a valid ACL token. aclRoleReq1 := &structs.ACLRolesListRequest{ QueryOptions: structs.QueryOptions{ - Region: DefaultRegion, + Region: DefaultRegion, + AuthToken: uuid.Generate(), }, } var aclRoleResp1 structs.ACLRolesListResponse err := msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq1, &aclRoleResp1) - require.ErrorContains(t, err, "Permission denied") + require.ErrorContains(t, err, "ACL token not found") // Try listing roles with a valid ACL token. aclRoleReq2 := &structs.ACLRolesListRequest{ @@ -2146,6 +2147,28 @@ func TestACL_ListRoles(t *testing.T) { require.NoError(t, err) require.Len(t, aclRoleResp4.ACLRoles, 2) + // Generate and upsert an ACL Token which links to only one of the two + // roles within state. + aclToken := mock.ACLToken() + aclToken.Policies = nil + aclToken.Roles = []*structs.ACLTokenRoleLink{{ID: aclRoles[1].ID}} + + err = testServer.fsm.State().UpsertACLTokens(structs.MsgTypeTestSetup, 20, []*structs.ACLToken{aclToken}) + require.NoError(t, err) + + aclRoleReq5 := &structs.ACLRolesListRequest{ + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + var aclRoleResp5 structs.ACLRolesListResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq5, &aclRoleResp5) + require.NoError(t, err) + require.Len(t, aclRoleResp5.ACLRoles, 1) + require.Equal(t, aclRoleResp5.ACLRoles[0].ID, aclRoles[1].ID) + require.Equal(t, aclRoleResp5.ACLRoles[0].Name, aclRoles[1].Name) + // Now test a blocking query, where we wait for an update to the list which // is triggered by a deletion. type res struct { @@ -2155,7 +2178,7 @@ func TestACL_ListRoles(t *testing.T) { resultCh := make(chan *res) go func(resultCh chan *res) { - aclRoleReq5 := &structs.ACLRolesListRequest{ + aclRoleReq6 := &structs.ACLRolesListRequest{ QueryOptions: structs.QueryOptions{ Region: DefaultRegion, AuthToken: aclRootToken.SecretID, @@ -2163,9 +2186,9 @@ func TestACL_ListRoles(t *testing.T) { MaxQueryTime: 10 * time.Second, }, } - var aclRoleResp5 structs.ACLRolesListResponse - err = msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq5, &aclRoleResp5) - resultCh <- &res{err: err, reply: &aclRoleResp5} + var aclRoleResp6 structs.ACLRolesListResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLListRolesRPCMethod, aclRoleReq6, &aclRoleResp6) + resultCh <- &res{err: err, reply: &aclRoleResp6} }(resultCh) // Delete an ACL role from state which should return the blocking query. @@ -2248,7 +2271,7 @@ func TestACL_GetRolesByID(t *testing.T) { resultCh := make(chan *res) go func(resultCh chan *res) { - aclRoleReq4 := &structs.ACLRolesByIDRequest{ + aclRoleReq5 := &structs.ACLRolesByIDRequest{ ACLRoleIDs: []string{aclRoles[0].ID, aclRoles[1].ID}, QueryOptions: structs.QueryOptions{ Region: DefaultRegion, @@ -2257,9 +2280,9 @@ func TestACL_GetRolesByID(t *testing.T) { MaxQueryTime: 10 * time.Second, }, } - var aclRoleResp4 structs.ACLRolesByIDResponse - err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRolesByIDRPCMethod, aclRoleReq4, &aclRoleResp4) - resultCh <- &res{err: err, reply: &aclRoleResp4} + var aclRoleResp5 structs.ACLRolesByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRolesByIDRPCMethod, aclRoleReq5, &aclRoleResp5) + resultCh <- &res{err: err, reply: &aclRoleResp5} }(resultCh) // Delete an ACL role from state which should return the blocking query. @@ -2342,6 +2365,41 @@ func TestACL_GetRoleByID(t *testing.T) { err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByIDRPCMethod, aclRoleReq4, &aclRoleResp4) require.NoError(t, err) require.True(t, aclRoleResp4.ACLRole.Equals(aclRoles[1])) + + // Generate and upsert an ACL Token which links to only one of the two + // roles within state. + aclToken := mock.ACLToken() + aclToken.Policies = nil + aclToken.Roles = []*structs.ACLTokenRoleLink{{ID: aclRoles[1].ID}} + + err = testServer.fsm.State().UpsertACLTokens(structs.MsgTypeTestSetup, 20, []*structs.ACLToken{aclToken}) + require.NoError(t, err) + + // Try detailing the role that is tried to our ACL token. + aclRoleReq5 := &structs.ACLRoleByIDRequest{ + RoleID: aclRoles[1].ID, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + var aclRoleResp5 structs.ACLRoleByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByIDRPCMethod, aclRoleReq5, &aclRoleResp5) + require.NoError(t, err) + require.NotNil(t, aclRoleResp5.ACLRole) + require.Equal(t, aclRoleResp5.ACLRole.ID, aclRoles[1].ID) + + // Try detailing the role that is NOT tried to our ACL token. + aclRoleReq6 := &structs.ACLRoleByIDRequest{ + RoleID: aclRoles[0].ID, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + var aclRoleResp6 structs.ACLRoleByIDResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByIDRPCMethod, aclRoleReq6, &aclRoleResp6) + require.ErrorContains(t, err, "Permission denied") } func TestACL_GetRoleByName(t *testing.T) { @@ -2412,4 +2470,40 @@ func TestACL_GetRoleByName(t *testing.T) { err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByNameRPCMethod, aclRoleReq4, &aclRoleResp4) require.NoError(t, err) require.True(t, aclRoleResp4.ACLRole.Equals(aclRoles[1])) + + // Generate and upsert an ACL Token which links to only one of the two + // roles within state. + aclToken := mock.ACLToken() + aclToken.Policies = nil + aclToken.Roles = []*structs.ACLTokenRoleLink{{ID: aclRoles[1].ID}} + + err = testServer.fsm.State().UpsertACLTokens(structs.MsgTypeTestSetup, 20, []*structs.ACLToken{aclToken}) + require.NoError(t, err) + + // Try detailing the role that is tried to our ACL token. + aclRoleReq5 := &structs.ACLRoleByNameRequest{ + RoleName: aclRoles[1].Name, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + var aclRoleResp5 structs.ACLRoleByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByNameRPCMethod, aclRoleReq5, &aclRoleResp5) + require.NoError(t, err) + require.NotNil(t, aclRoleResp5.ACLRole) + require.Equal(t, aclRoleResp5.ACLRole.ID, aclRoles[1].ID) + require.Equal(t, aclRoleResp5.ACLRole.Name, aclRoles[1].Name) + + // Try detailing the role that is NOT tried to our ACL token. + aclRoleReq6 := &structs.ACLRoleByNameRequest{ + RoleName: aclRoles[0].Name, + QueryOptions: structs.QueryOptions{ + Region: DefaultRegion, + AuthToken: aclToken.SecretID, + }, + } + var aclRoleResp6 structs.ACLRoleByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLGetRoleByNameRPCMethod, aclRoleReq6, &aclRoleResp6) + require.ErrorContains(t, err, "Permission denied") } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 84d59370b..5a3d6028c 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -5690,10 +5690,20 @@ func (s *StateStore) ACLTokenBySecretID(ws memdb.WatchSet, secretID string) (*st } ws.Add(watchCh) - if existing != nil { - return existing.(*structs.ACLToken), nil + // If the existing token is nil, this indicates it does not exist in state. + if existing == nil { + return nil, nil } - return nil, nil + + // Assert the token type which allows us to perform additional work on the + // token that is needed before returning the call. + token := existing.(*structs.ACLToken) + + // Handle potential staleness of ACL role links. + if token, err = s.fixTokenRoleLinks(txn, token); err != nil { + return nil, err + } + return token, nil } // ACLTokenByAccessorIDPrefix is used to lookup tokens by prefix From 7401677e4e015866a3c4f908888202e7d2b88fdf Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 24 Aug 2022 15:14:49 +0100 Subject: [PATCH 17/19] cli: output none when a token has no expiration. --- command/acl_bootstrap.go | 2 +- command/acl_bootstrap_test.go | 4 ++-- command/acl_token_create_test.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/command/acl_bootstrap.go b/command/acl_bootstrap.go index a1844365c..b1d07968b 100644 --- a/command/acl_bootstrap.go +++ b/command/acl_bootstrap.go @@ -198,7 +198,7 @@ func outputACLToken(ui cli.Ui, token *api.ACLToken) { func expiryTimeString(t *time.Time) string { if t == nil || t.IsZero() { - return "" + return "" } return t.String() } diff --git a/command/acl_bootstrap_test.go b/command/acl_bootstrap_test.go index 96f489e02..58c247956 100644 --- a/command/acl_bootstrap_test.go +++ b/command/acl_bootstrap_test.go @@ -36,7 +36,7 @@ func TestACLBootstrapCommand(t *testing.T) { out := ui.OutputWriter.String() assert.Contains(out, "Secret ID") - require.Contains(t, out, "Expiry Time = ") + require.Contains(t, out, "Expiry Time = ") } // If a bootstrap token has already been created, attempts to create more should @@ -117,7 +117,7 @@ func TestACLBootstrapCommand_WithOperatorFileBootstrapToken(t *testing.T) { out := ui.OutputWriter.String() assert.Contains(t, out, mockToken.SecretID) - require.Contains(t, out, "Expiry Time = ") + require.Contains(t, out, "Expiry Time = ") } // Attempting to bootstrap the server with an invalid operator provided token in a file should diff --git a/command/acl_token_create_test.go b/command/acl_token_create_test.go index b2d7c08cb..b6b4223da 100644 --- a/command/acl_token_create_test.go +++ b/command/acl_token_create_test.go @@ -39,7 +39,7 @@ func TestACLTokenCreateCommand(t *testing.T) { // Check the output out := ui.OutputWriter.String() require.Contains(t, out, "[foo]") - require.Contains(t, out, "Expiry Time = ") + require.Contains(t, out, "Expiry Time = ") ui.OutputWriter.Reset() ui.ErrorWriter.Reset() @@ -49,7 +49,7 @@ func TestACLTokenCreateCommand(t *testing.T) { require.Equal(t, 0, code) out = ui.OutputWriter.String() - require.NotContains(t, out, "Expiry Time = ") + require.NotContains(t, out, "Expiry Time = ") } func Test_generateACLTokenRoleLinks(t *testing.T) { From 2ccc48c167c22abcbc47a51178bd6c636a3caca7 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 24 Aug 2022 15:15:02 +0100 Subject: [PATCH 18/19] cli: use policy flag for role creation and update. --- command/acl_role_create.go | 8 ++++---- command/acl_role_create_test.go | 4 ++-- command/acl_role_update.go | 8 ++++---- command/acl_role_update_test.go | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/command/acl_role_create.go b/command/acl_role_create.go index f1aa8482b..ed1ac0c0b 100644 --- a/command/acl_role_create.go +++ b/command/acl_role_create.go @@ -44,7 +44,7 @@ ACL Create Options: A free form text description of the role that must not exceed 256 characters. - -policy-name + -policy Specifies a policy to associate with the role identified by their name. This flag can be specified multiple times and must be specified at least once. @@ -62,7 +62,7 @@ func (a *ACLRoleCreateCommand) AutocompleteFlags() complete.Flags { complete.Flags{ "-name": complete.PredictAnything, "-description": complete.PredictAnything, - "-policy-name": complete.PredictAnything, + "-policy": complete.PredictAnything, "-json": complete.PredictNothing, "-t": complete.PredictAnything, }) @@ -86,7 +86,7 @@ func (a *ACLRoleCreateCommand) Run(args []string) int { flags.Var((funcVar)(func(s string) error { a.policyNames = append(a.policyNames, s) return nil - }), "policy-name", "") + }), "policy", "") flags.BoolVar(&a.json, "json", false, "") flags.StringVar(&a.tmpl, "t", "", "") if err := flags.Parse(args); err != nil { @@ -107,7 +107,7 @@ func (a *ACLRoleCreateCommand) Run(args []string) int { return 1 } if len(a.policyNames) < 1 { - a.Ui.Error("At least one policy name must be specified using the -policy-name flag") + a.Ui.Error("At least one policy name must be specified using the -policy flag") return 1 } diff --git a/command/acl_role_create_test.go b/command/acl_role_create_test.go index 6b107df3a..000bda470 100644 --- a/command/acl_role_create_test.go +++ b/command/acl_role_create_test.go @@ -47,7 +47,7 @@ func TestACLRoleCreateCommand_Run(t *testing.T) { ui.ErrorWriter.Reset() require.Equal(t, 1, cmd.Run([]string{"-address=" + url, `-name="foobar"`})) - require.Contains(t, ui.ErrorWriter.String(), "At least one policy name must be specified using the -policy-name flag") + require.Contains(t, ui.ErrorWriter.String(), "At least one policy name must be specified using the -policy flag") ui.OutputWriter.Reset() ui.ErrorWriter.Reset() @@ -67,7 +67,7 @@ func TestACLRoleCreateCommand_Run(t *testing.T) { // Create an ACL role. args := []string{ "-address=" + url, "-token=" + rootACLToken.SecretID, "-name=acl-role-cli-test", - "-policy-name=acl-role-cli-test-policy", "-description=acl-role-all-the-things", + "-policy=acl-role-cli-test-policy", "-description=acl-role-all-the-things", } require.Equal(t, 0, cmd.Run(args)) s := ui.OutputWriter.String() diff --git a/command/acl_role_update.go b/command/acl_role_update.go index fcb71902f..872cc7f39 100644 --- a/command/acl_role_update.go +++ b/command/acl_role_update.go @@ -45,7 +45,7 @@ Update Options: A free form text description of the role that must not exceed 256 characters. - -policy-name + -policy Specifies a policy to associate with the role identified by their name. This flag can be specified multiple times. @@ -70,7 +70,7 @@ func (a *ACLRoleUpdateCommand) AutocompleteFlags() complete.Flags { "-name": complete.PredictAnything, "-description": complete.PredictAnything, "-no-merge": complete.PredictNothing, - "-policy-name": complete.PredictAnything, + "-policy": complete.PredictAnything, "-json": complete.PredictNothing, "-t": complete.PredictAnything, }) @@ -94,7 +94,7 @@ func (a *ACLRoleUpdateCommand) Run(args []string) int { flags.Var((funcVar)(func(s string) error { a.policyNames = append(a.policyNames, s) return nil - }), "policy-name", "") + }), "policy", "") flags.BoolVar(&a.noMerge, "no-merge", false, "") flags.BoolVar(&a.json, "json", false, "") flags.StringVar(&a.tmpl, "t", "", "") @@ -140,7 +140,7 @@ func (a *ACLRoleUpdateCommand) Run(args []string) int { return 1 } if len(a.policyNames) < 1 { - a.Ui.Error("At least one policy name must be specified using the -policy-name flag") + a.Ui.Error("At least one policy name must be specified using the -policy flag") return 1 } diff --git a/command/acl_role_update_test.go b/command/acl_role_update_test.go index 3e0449bc8..9ae7adc11 100644 --- a/command/acl_role_update_test.go +++ b/command/acl_role_update_test.go @@ -106,12 +106,12 @@ func TestACLRoleUpdateCommand_Run(t *testing.T) { code = cmd.Run([]string{ "-address=" + url, "-token=" + rootACLToken.SecretID, "-no-merge", "-name=update-role-name", aclRole.ID}) require.Equal(t, 1, code) - require.Contains(t, ui.ErrorWriter.String(), "At least one policy name must be specified using the -policy-name flag") + require.Contains(t, ui.ErrorWriter.String(), "At least one policy name must be specified using the -policy flag") // Update the role using no-merge with all required flags set. code = cmd.Run([]string{ "-address=" + url, "-token=" + rootACLToken.SecretID, "-no-merge", "-name=update-role-name", - "-description=updated-description", "-policy-name=acl-role-cli-test-policy", aclRole.ID}) + "-description=updated-description", "-policy=acl-role-cli-test-policy", aclRole.ID}) require.Equal(t, 0, code) s = ui.OutputWriter.String() require.Contains(t, s, fmt.Sprintf("ID = %s", aclRole.ID)) From 7a0798663dfd3cb137afe40f07925522edf29b28 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 25 Aug 2022 09:20:43 +0100 Subject: [PATCH 19/19] acl: fix a bug where roles could be duplicated by name. An ACL roles name must be unique, however, a bug meant multiple roles of the same same could be created. This fixes that problem with checks in the RPC handler and state store. --- api/acl_test.go | 4 +-- nomad/acl_endpoint.go | 35 ++++++++++++++--------- nomad/acl_endpoint_test.go | 16 +++++++++++ nomad/state/state_store_acl.go | 44 +++++++++++++++++++++++++---- nomad/state/state_store_acl_test.go | 10 +++++++ 5 files changed, 89 insertions(+), 20 deletions(-) diff --git a/api/acl_test.go b/api/acl_test.go index 4c412013d..4a0f6e26d 100644 --- a/api/acl_test.go +++ b/api/acl_test.go @@ -330,7 +330,7 @@ func TestACLTokens_Info(t *testing.T) { // Create an ACL role referencing the previously created // policy. role := ACLRole{ - Name: "acl-role-api-test", + Name: "acl-role-api-test-role-and-policy", Policies: []*ACLRolePolicyLink{{Name: aclPolicy1.Name}}, } aclRoleCreateResp, writeMeta, err := testClient.ACLRoles().Create(&role, nil) @@ -341,7 +341,7 @@ func TestACLTokens_Info(t *testing.T) { // Create a token with a role linking. token := &ACLToken{ - Name: "token-with-role-link", + Name: "token-with-role-and-policy-link", Type: "client", Policies: []string{aclPolicy2.Name}, Roles: []*ACLTokenRoleLink{{Name: role.Name}}, diff --git a/nomad/acl_endpoint.go b/nomad/acl_endpoint.go index f735cb56c..46aaf8d3e 100644 --- a/nomad/acl_endpoint.go +++ b/nomad/acl_endpoint.go @@ -1122,6 +1122,28 @@ func (a *ACL) UpsertRoles( return structs.NewErrRPCCodedf(http.StatusBadRequest, "role %d invalid: %v", idx, err) } + // If the caller has passed a role ID, this call is considered an + // update to an existing role. We should therefore ensure it is found + // within state. Otherwise, the call is considered a new creation, and + // we must ensure a role of the same name does not exist. + if role.ID == "" { + existingRole, err := stateSnapshot.GetACLRoleByName(nil, role.Name) + if err != nil { + return structs.NewErrRPCCodedf(http.StatusInternalServerError, "role lookup failed: %v", err) + } + if existingRole != nil { + return structs.NewErrRPCCodedf(http.StatusBadRequest, "role with name %s already exists", role.Name) + } + } else { + existingRole, err := stateSnapshot.GetACLRoleByID(nil, role.ID) + if err != nil { + return structs.NewErrRPCCodedf(http.StatusInternalServerError, "role lookup failed: %v", err) + } + if existingRole == nil { + return structs.NewErrRPCCodedf(http.StatusBadRequest, "cannot find role %s", role.ID) + } + } + policyNames := make(map[string]struct{}) var policiesLinks []*structs.ACLRolePolicyLink @@ -1156,19 +1178,6 @@ func (a *ACL) UpsertRoles( // Stored the potentially updated policy links within our role. role.Policies = policiesLinks - // If the caller has passed a role ID, this call is considered an - // update to an existing role. We should therefore ensure it is found - // within state. - if role.ID != "" { - out, err := stateSnapshot.GetACLRoleByID(nil, role.ID) - if err != nil { - return structs.NewErrRPCCodedf(http.StatusBadRequest, "role lookup failed: %v", err) - } - if out == nil { - return structs.NewErrRPCCodedf(http.StatusBadRequest, "cannot find role %s", role.ID) - } - } - role.Canonicalize() role.SetHash() } diff --git a/nomad/acl_endpoint_test.go b/nomad/acl_endpoint_test.go index b134e3902..7d116bf67 100644 --- a/nomad/acl_endpoint_test.go +++ b/nomad/acl_endpoint_test.go @@ -1998,6 +1998,22 @@ func TestACL_UpsertRoles(t *testing.T) { err = msgpackrpc.CallWithCodec(codec, structs.ACLUpsertRolesRPCMethod, aclRoleReq5, &aclRoleResp5) require.Error(t, err) require.NotContains(t, err, "Permission denied") + + // Try and create a role with a name that already exists within state. + aclRole3 := mock.ACLRole() + aclRole3.ID = "" + aclRole3.Name = aclRole1.Name + + aclRoleReq6 := &structs.ACLRolesUpsertRequest{ + ACLRoles: []*structs.ACLRole{aclRole3}, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + AuthToken: aclRootToken.SecretID, + }, + } + var aclRoleResp6 structs.ACLRolesUpsertResponse + err = msgpackrpc.CallWithCodec(codec, structs.ACLUpsertRolesRPCMethod, aclRoleReq6, &aclRoleResp6) + require.ErrorContains(t, err, fmt.Sprintf("role with name %s already exists", aclRole1.Name)) } func TestACL_DeleteRolesByID(t *testing.T) { diff --git a/nomad/state/state_store_acl.go b/nomad/state/state_store_acl.go index faaa85ff5..c72bf969b 100644 --- a/nomad/state/state_store_acl.go +++ b/nomad/state/state_store_acl.go @@ -98,18 +98,52 @@ func (s *StateStore) upsertACLRoleTxn( } } - existing, err := txn.First(TableACLRoles, indexID, role.ID) + // This validation also happens within the RPC handler, but Raft latency + // could mean that by the time the state call is invoked, another Raft + // update has already written a role with the same name. We therefore need + // to check we are not trying to create a role with an existing name. + existingRaw, err := txn.First(TableACLRoles, indexName, role.Name) if err != nil { return false, fmt.Errorf("ACL role lookup failed: %v", err) } - // Set up the indexes correctly to ensure existing indexes are maintained. + // Track our type asserted role, so we only need to do this once. + var existing *structs.ACLRole + + // If we did not find an ACL Role within state with the same name, we need + // to check using the ID index as the operator might be performing an + // update on the role name. + // + // If we found an entry using the name index, we need to check that the ID + // matches the object within the request. + if existingRaw == nil { + existingRaw, err = txn.First(TableACLRoles, indexID, role.ID) + if err != nil { + return false, fmt.Errorf("ACL role lookup failed: %v", err) + } + if existingRaw != nil { + existing = existingRaw.(*structs.ACLRole) + } + } else { + existing = existingRaw.(*structs.ACLRole) + if existing.ID != role.ID { + return false, fmt.Errorf("ACL role with name %s already exists", role.Name) + } + } + + // Depending on whether this is an initial create, or an update, we need to + // check and set certain parameters. The most important is to ensure any + // create index is carried over. if existing != nil { - exist := existing.(*structs.ACLRole) - if exist.Equals(role) { + + // If the role already exists, check whether the update contains any + // difference. If it doesn't, we can avoid a state update as wel as + // updates to any blocking queries. + if existing.Equals(role) { return false, nil } - role.CreateIndex = exist.CreateIndex + + role.CreateIndex = existing.CreateIndex role.ModifyIndex = index } else { role.CreateIndex = index diff --git a/nomad/state/state_store_acl_test.go b/nomad/state/state_store_acl_test.go index 8458e88e7..602883679 100644 --- a/nomad/state/state_store_acl_test.go +++ b/nomad/state/state_store_acl_test.go @@ -1,6 +1,7 @@ package state import ( + "fmt" "testing" "time" @@ -246,6 +247,15 @@ func TestStateStore_UpsertACLRoles(t *testing.T) { replicatedACLRoleResp, err := testState.GetACLRoleByName(ws, replicatedACLRole.Name) require.NoError(t, err) must.Eq(t, replicatedACLRole.Hash, replicatedACLRoleResp.Hash) + + // Try adding a new ACL role, which has a name clash with an existing + // entry. + dupRoleName := mock.ACLRole() + dupRoleName.Name = mockedACLRoles[0].Name + + err = testState.UpsertACLRoles(structs.MsgTypeTestSetup, 50, + []*structs.ACLRole{dupRoleName}, false) + require.ErrorContains(t, err, fmt.Sprintf("ACL role with name %s already exists", dupRoleName.Name)) } func TestStateStore_ValidateACLRolePolicyLinks(t *testing.T) {