Merge PR #10010: Rate Limit Quotas: Allow Exempt Paths to be Configurable

vault/core.go

@@ -2522,14 +2522,26 @@ func (c *Core) setupQuotas(ctx context.Context, isPerfStandby bool) error {
 	return c.quotaManager.Setup(ctx, c.systemBarrierView, isPerfStandby)
 }
 
-// ApplyRateLimitQuota checks the request against all the applicable quota rules
+// ApplyRateLimitQuota checks the request against all the applicable quota rules.
+// If the given request's path is exempt, no rate limiting will be applied.
 func (c *Core) ApplyRateLimitQuota(req *quotas.Request) (quotas.Response, error) {
 	req.Type = quotas.TypeRateLimit
+
+	resp := quotas.Response{
+		Allowed: true,
+		Headers: make(map[string]string),
+	}
+
 	if c.quotaManager != nil {
+		// skip rate limit checks for paths that are exempt from rate limiting
+		if c.quotaManager.RateLimitPathExempt(req.Path) {
+			return resp, nil
+		}
+
 		return c.quotaManager.ApplyQuota(req)
 	}
 
-	return quotas.Response{Allowed: true}, nil
+	return resp, nil
 }
 
 // RateLimitAuditLoggingEnabled returns if the quota configuration allows audit

vault/external_tests/quotas/quotas_test.go

@@ -161,6 +161,57 @@ func TestQuotas_RateLimitQuota_DupName(t *testing.T) {
 	require.Len(t, s.Data, 1, "incorrect number of quotas")
 }
 
+func TestQuotas_RateLimitQuota_ExemptPaths(t *testing.T) {
+	conf, opts := teststorage.ClusterSetup(coreConfig, nil, nil)
+
+	cluster := vault.NewTestCluster(t, conf, opts)
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	core := cluster.Cores[0].Core
+	client := cluster.Cores[0].Client
+	vault.TestWaitActive(t, core)
+
+	_, err := client.Logical().Write("sys/quotas/rate-limit/rlq", map[string]interface{}{
+		"rate": 7.7,
+	})
+	require.NoError(t, err)
+
+	// ensure exempt paths are not empty by default
+	resp, err := client.Logical().Read("sys/quotas/config")
+	require.NoError(t, err)
+	require.NotEmpty(t, resp.Data["rate_limit_exempt_paths"].([]interface{}), "expected no exempt paths by default")
+
+	reqFunc := func(numSuccess, numFail *atomic.Int32) {
+		_, err := client.Logical().Read("sys/quotas/rate-limit/rlq")
+
+		if err != nil {
+			numFail.Add(1)
+		} else {
+			numSuccess.Add(1)
+		}
+	}
+
+	numSuccess, numFail, elapsed := testRPS(reqFunc, 5*time.Second)
+	ideal := 8 + (7.7 * float64(elapsed) / float64(time.Second))
+	want := int32(ideal + 1)
+	require.NotZerof(t, numFail, "expected some requests to fail; numSuccess: %d, elapsed: %d", numSuccess, elapsed)
+	require.Lessf(t, numSuccess, want, "too many successful requests;numSuccess: %d, numFail: %d, elapsed: %d", want, numSuccess, numFail, elapsed)
+
+	// allow time (1s) for rate limit to refill before updating the quota config
+	time.Sleep(time.Second)
+
+	_, err = client.Logical().Write("sys/quotas/config", map[string]interface{}{
+		"rate_limit_exempt_paths": []string{"sys/quotas/rate-limit"},
+	})
+	require.NoError(t, err)
+
+	// all requests should success
+	numSuccess, numFail, _ = testRPS(reqFunc, 5*time.Second)
+	require.NotZero(t, numSuccess)
+	require.Zero(t, numFail)
+}
+
 func TestQuotas_RateLimitQuota_Mount(t *testing.T) {
 	conf, opts := teststorage.ClusterSetup(coreConfig, nil, nil)
 	cluster := vault.NewTestCluster(t, conf, opts)

vault/logical_system_quotas.go

@@ -17,6 +17,10 @@ func (b *SystemBackend) quotasPaths() []*framework.Path {
 		{
 			Pattern: "quotas/config$",
 			Fields: map[string]*framework.FieldSchema{
+				"rate_limit_exempt_paths": {
+					Type:        framework.TypeStringSlice,
+					Description: "Specifies the list of exempt paths from all rate limit quotas. If empty no paths will be exempt.",
+				},
 				"enable_rate_limit_audit_logging": {
 					Type:        framework.TypeBool,
 					Description: "If set, starts audit logging of requests that get rejected due to rate limit quota rule violations.",
@@ -105,6 +109,7 @@ func (b *SystemBackend) handleQuotasConfigUpdate() framework.OperationFunc {
 
 		config.EnableRateLimitAuditLogging = d.Get("enable_rate_limit_audit_logging").(bool)
 		config.EnableRateLimitResponseHeaders = d.Get("enable_rate_limit_response_headers").(bool)
+		config.RateLimitExemptPaths = d.Get("rate_limit_exempt_paths").([]string)
 
 		entry, err := logical.StorageEntryJSON(quotas.ConfigPath, config)
 		if err != nil {
@@ -114,8 +119,17 @@ func (b *SystemBackend) handleQuotasConfigUpdate() framework.OperationFunc {
 			return nil, err
 		}
 
+		entry, err = logical.StorageEntryJSON(quotas.DefaultRateLimitExemptPathsToggle, true)
+		if err != nil {
+			return nil, err
+		}
+		if err := req.Storage.Put(ctx, entry); err != nil {
+			return nil, err
+		}
+
 		b.Core.quotaManager.SetEnableRateLimitAuditLogging(config.EnableRateLimitAuditLogging)
 		b.Core.quotaManager.SetEnableRateLimitResponseHeaders(config.EnableRateLimitResponseHeaders)
+		b.Core.quotaManager.SetRateLimitExemptPaths(config.RateLimitExemptPaths)
 
 		return nil, nil
 	}
@@ -128,6 +142,7 @@ func (b *SystemBackend) handleQuotasConfigRead() framework.OperationFunc {
 			Data: map[string]interface{}{
 				"enable_rate_limit_audit_logging":    config.EnableRateLimitAuditLogging,
 				"enable_rate_limit_response_headers": config.EnableRateLimitResponseHeaders,
+				"rate_limit_exempt_paths":            config.RateLimitExemptPaths,
 			},
 		}, nil
 	}

vault/quotas/quotas.go

@@ -11,6 +11,7 @@ import (
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-memdb"
 	"github.com/hashicorp/vault/helper/metricsutil"
+	"github.com/hashicorp/vault/sdk/helper/pathmanager"
 	"github.com/hashicorp/vault/sdk/logical"
 )
 
@@ -93,6 +94,13 @@ const (
 	// ConfigPath is the physical location where the quota configuration is
 	// persisted.
 	ConfigPath = StoragePrefix + "config"
+
+	// DefaultRateLimitExemptPathsToggle is the path to a toggle that allows us to
+	// determine if a Vault operator explicitly modified the exempt paths set for
+	// rate limit resource quotas. Specifically, when this toggle is false, we can
+	// infer a Vault node is operating with an initial default set and on a subsequent
+	// update to that set, we should not overwrite it on Setup.
+	DefaultRateLimitExemptPathsToggle = StoragePrefix + "default_rate_limit_exempt_paths_toggle"
 )
 
 var (
@@ -105,6 +113,16 @@ var (
 	ErrRateLimitQuotaExceeded = errors.New("rate limit quota exceeded")
 )
 
+var defaultExemptPaths = []string{
+	"/v1/sys/generate-recovery-token/attempt",
+	"/v1/sys/generate-recovery-token/update",
+	"/v1/sys/generate-root/attempt",
+	"/v1/sys/generate-root/update",
+	"/v1/sys/health",
+	"/v1/sys/seal-status",
+	"/v1/sys/unseal",
+}
+
 // Access provides information to reach back to the quota checker.
 type Access interface {
 	// QuotaID is the identifier of the quota that issued this access.
@@ -137,6 +155,8 @@ type Manager struct {
 	// config containing operator preferences and quota behaviors
 	config *Config
 
+	rateLimitPathManager *pathmanager.PathManager
+
 	storage logical.Storage
 	ctx     context.Context
 
@@ -192,6 +212,11 @@ type Config struct {
 	// EnableRateLimitResponseHeaders dictates if rate limit quota HTTP headers
 	// should be added to responses.
 	EnableRateLimitResponseHeaders bool `json:"enable_rate_limit_response_headers"`
+
+	// RateLimitExemptPaths defines the set of exempt paths used for all rate limit
+	// quotas. Any request path that exists in this set is exempt from rate limiting.
+	// If the set is empty, no paths are exempt.
+	RateLimitExemptPaths []string `json:"rate_limit_exempt_paths"`
 }
 
 // Request contains information required by the quota manager to query and
@@ -223,11 +248,12 @@ func NewManager(logger log.Logger, walkFunc leaseWalkFunc, ms *metricsutil.Clust
 	}
 
 	manager := &Manager{
-		db:         db,
-		logger:     logger,
-		metricSink: ms,
-		config:     new(Config),
-		lock:       new(sync.RWMutex),
+		db:                   db,
+		logger:               logger,
+		metricSink:           ms,
+		rateLimitPathManager: pathmanager.New(),
+		config:               new(Config),
+		lock:                 new(sync.RWMutex),
 	}
 
 	manager.init(walkFunc)
@@ -534,27 +560,87 @@ func (m *Manager) ApplyQuota(req *Request) (Response, error) {
 // SetEnableRateLimitAuditLogging updates the operator preference regarding the
 // audit logging behavior.
 func (m *Manager) SetEnableRateLimitAuditLogging(val bool) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.setEnableRateLimitAuditLoggingLocked(val)
+}
+
+func (m *Manager) setEnableRateLimitAuditLoggingLocked(val bool) {
 	m.config.EnableRateLimitAuditLogging = val
 }
 
 // SetEnableRateLimitResponseHeaders updates the operator preference regarding
 // the rate limit quota HTTP header behavior.
 func (m *Manager) SetEnableRateLimitResponseHeaders(val bool) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.setEnableRateLimitResponseHeadersLocked(val)
+}
+
+func (m *Manager) setEnableRateLimitResponseHeadersLocked(val bool) {
 	m.config.EnableRateLimitResponseHeaders = val
 }
 
+// SetRateLimitExemptPaths updates the rate limit exempt paths in the Manager's
+// configuration in addition to updating the path manager. Every call to
+// SetRateLimitExemptPaths will wipe out the existing path manager and set the
+// paths based on the provided argument.
+func (m *Manager) SetRateLimitExemptPaths(vals []string) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.setRateLimitExemptPathsLocked(vals)
+}
+
+func (m *Manager) setRateLimitExemptPathsLocked(vals []string) {
+	if vals == nil {
+		vals = []string{}
+	}
+	m.config.RateLimitExemptPaths = vals
+	m.rateLimitPathManager = pathmanager.New()
+	m.rateLimitPathManager.AddPaths(vals)
+}
+
 // RateLimitAuditLoggingEnabled returns if the quota configuration allows audit
 // logging of request rejections due to rate limiting quota rule violations.
 func (m *Manager) RateLimitAuditLoggingEnabled() bool {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
 	return m.config.EnableRateLimitAuditLogging
 }
 
 // RateLimitResponseHeadersEnabled returns if the quota configuration allows for
 // rate limit quota HTTP headers to be added to responses.
 func (m *Manager) RateLimitResponseHeadersEnabled() bool {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
 	return m.config.EnableRateLimitResponseHeaders
 }
 
+// RateLimitExemptPaths returns the list of exempt paths from all rate limit
+// resource quotas from the Manager's configuration.
+func (m *Manager) RateLimitExemptPaths() []string {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	return m.config.RateLimitExemptPaths
+}
+
+// RateLimitPathExempt returns a boolean dictating if a given path is exempt from
+// any rate limit quota. If not rate limit path manager is defined, false is
+// returned.
+func (m *Manager) RateLimitPathExempt(path string) bool {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	if m.rateLimitPathManager == nil {
+		return false
+	}
+
+	return m.rateLimitPathManager.HasPath(path)
+}
+
 // Config returns the operator preferences in the quota manager
 func (m *Manager) Config() *Config {
 	return m.config
@@ -658,6 +744,7 @@ func (m *Manager) Invalidate(key string) {
 
 		m.SetEnableRateLimitAuditLogging(config.EnableRateLimitAuditLogging)
 		m.SetEnableRateLimitResponseHeaders(config.EnableRateLimitResponseHeaders)
+		m.SetRateLimitExemptPaths(config.RateLimitExemptPaths)
 
 	default:
 		splitKeys := strings.Split(key, "/")
@@ -755,7 +842,31 @@ func (m *Manager) Setup(ctx context.Context, storage logical.Storage, isPerfStan
 	if err != nil {
 		return err
 	}
-	m.SetEnableRateLimitAuditLogging(config.EnableRateLimitAuditLogging)
+
+	entry, err := storage.Get(ctx, DefaultRateLimitExemptPathsToggle)
+	if err != nil {
+		return err
+	}
+
+	// Determine if we need to set the default set of exempt paths for rate limit
+	// resource quotas. We use a default set introduced in 1.5 when the toggle
+	// entry does not exist in storage or is false. The toggle is flipped , i.e.
+	// set to true when SetRateLimitExemptPaths is called during a config update.
+	var toggle bool
+	if entry != nil {
+		if err := entry.DecodeJSON(&toggle); err != nil {
+			return err
+		}
+	}
+
+	exemptPaths := defaultExemptPaths
+	if toggle {
+		exemptPaths = config.RateLimitExemptPaths
+	}
+
+	m.setEnableRateLimitAuditLoggingLocked(config.EnableRateLimitAuditLogging)
+	m.setEnableRateLimitResponseHeadersLocked(config.EnableRateLimitResponseHeaders)
+	m.setRateLimitExemptPathsLocked(exemptPaths)
 
 	// Load the quota rules for all supported types from storage and load it in
 	// the quota manager.

vault/quotas/quotas_rate_limit.go

@@ -11,14 +11,11 @@ import (
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/helper/metricsutil"
-	"github.com/hashicorp/vault/sdk/helper/pathmanager"
 	"github.com/sethvargo/go-limiter"
 	"github.com/sethvargo/go-limiter/httplimit"
 	"github.com/sethvargo/go-limiter/memorystore"
 )
 
-var rateLimitExemptPaths = pathmanager.New()
-
 const (
 	// DefaultRateLimitPurgeInterval defines the default purge interval used by a
 	// RateLimitQuota to remove stale client rate limiters.
@@ -32,19 +29,6 @@ const (
 	EnvVaultEnableRateLimitAuditLogging = "VAULT_ENABLE_RATE_LIMIT_AUDIT_LOGGING"
 )
 
-func init() {
-	rateLimitExemptPaths.AddPaths([]string{
-		"sys/internal/ui/mounts",
-		"sys/generate-recovery-token/attempt",
-		"sys/generate-recovery-token/update",
-		"sys/generate-root/attempt",
-		"sys/generate-root/update",
-		"sys/health",
-		"sys/seal-status",
-		"sys/unseal",
-	})
-}
-
 // Ensure that RateLimitQuota implements the Quota interface
 var _ Quota = (*RateLimitQuota)(nil)
 
@@ -251,12 +235,6 @@ func (rlq *RateLimitQuota) allow(req *Request) (Response, error) {
 		Headers: make(map[string]string),
 	}
 
-	// Skip rate limit checks for paths that are exempt from rate limiting.
-	if rateLimitExemptPaths.HasPath(req.Path) {
-		resp.Allowed = true
-		return resp, nil
-	}
-
 	if req.ClientAddress == "" {
 		return resp, fmt.Errorf("missing request client address in quota request")
 	}

website/pages/api-docs/system/quotas-config.mdx

@@ -17,6 +17,8 @@ The `/sys/quotas/config` endpoint is used to configure rate limit quotas.
 
 ### Parameters
 
+- `rate_limit_exempt_paths` `([]string: [])` - Specifies the list of exempt paths
+  from all rate limit quotas. If empty no paths will be exempt.
 - `enable_rate_limit_audit_logging` `(bool: false)` - If set, starts audit logging
   of requests that get rejected due to rate limit quota rule violations.
 - `enable_rate_limit_response_headers` `(bool: false)` - If set, additional rate
@@ -26,6 +28,16 @@ The `/sys/quotas/config` endpoint is used to configure rate limit quotas.
 
 ```json
 {
+  "rate_limit_exempt_paths": [
+		"sys/internal/ui/mounts",
+		"sys/generate-recovery-token/attempt",
+		"sys/generate-recovery-token/update",
+		"sys/generate-root/attempt",
+		"sys/generate-root/update",
+		"sys/health",
+		"sys/seal-status",
+		"sys/unseal"
+  ],
   "enable_rate_limit_audit_logging": true,
   "enable_rate_limit_response_headers": true,
 }
@@ -66,7 +78,17 @@ $ curl \
   "renewable": false,
   "data": {
     "enable_rate_limit_audit_logging": false,
-    "enable_rate_limit_response_headers": false
+    "enable_rate_limit_response_headers": false,
+    "rate_limit_exempt_paths": [
+      "sys/internal/ui/mounts",
+      "sys/generate-recovery-token/attempt",
+      "sys/generate-recovery-token/update",
+      "sys/generate-root/attempt",
+      "sys/generate-root/update",
+      "sys/health",
+      "sys/seal-status",
+      "sys/unseal"
+    ],
   },
   "warnings": null
 }

website/pages/docs/concepts/resource-quotas.mdx

@@ -50,7 +50,9 @@ and through enabling optional audit logging.
 
 ## Exempt Routes
 
-The following routes are always exempt from rate limiting:
+By default, the following paths are exempt from rate limiting. However, Vault
+operators can override the set of paths that are exempt from all rate limit
+resource quotas by updating the `rate_limit_exempt_paths` configuration field.
 
 - `/v1/sys/generate-recovery-token/attempt`
 - `/v1/sys/generate-recovery-token/update`