luxolus
/
open-vault
2
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

[VAULT-2852] deprecate req counters in oss (#12197)

This commit is contained in:
Pratyoy Mukhopadhyay 2021-07-29 10:21:40 -07:00 committed by GitHub
parent 7fac2ae196
commit 113b6885c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 47 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sdk/logical/error.go
View File

@ -47,6 +47,10 @@ var (
// with the data in the local node's storage, based on the provided // with the data in the local node's storage, based on the provided
// X-Vault-Index request header. // X-Vault-Index request header.
ErrMissingRequiredState = errors.New("required index state not present") ErrMissingRequiredState = errors.New("required index state not present")
// Error indicating that the requested path used to serve a purpose in older
// versions, but the functionality has now been removed
ErrPathFunctionalityRemoved = errors.New("functionality on this path has been removed")
) )
type HTTPCodedError interface { type HTTPCodedError interface {

2
sdk/logical/response_util.go
View File

@ -118,6 +118,8 @@ func RespondErrorCommon(req *Request, resp *Response, err error) (int, error) {
statusCode = http.StatusTooManyRequests statusCode = http.StatusTooManyRequests
case errwrap.Contains(err, ErrMissingRequiredState.Error()): case errwrap.Contains(err, ErrMissingRequiredState.Error()):
statusCode = http.StatusPreconditionFailed statusCode = http.StatusPreconditionFailed
case errwrap.Contains(err, ErrPathFunctionalityRemoved.Error()):
statusCode = http.StatusNotFound
} }
} }

80
vault/core.go
View File

@ -520,9 +520,6 @@ type Core struct {
// Telemetry objects // Telemetry objects
metricsHelper *metricsutil.MetricsHelper metricsHelper *metricsutil.MetricsHelper
// Stores request counters
counters counters
// raftFollowerStates tracks information about all the raft follower nodes. // raftFollowerStates tracks information about all the raft follower nodes.
raftFollowerStates *raft.FollowerStates raftFollowerStates *raft.FollowerStates
// Stop channel for raft TLS rotations // Stop channel for raft TLS rotations
@ -667,8 +664,6 @@ type CoreConfig struct {
MetricsHelper *metricsutil.MetricsHelper MetricsHelper *metricsutil.MetricsHelper
MetricSink *metricsutil.ClusterMetricSink MetricSink *metricsutil.ClusterMetricSink
CounterSyncInterval time.Duration
RecoveryMode bool RecoveryMode bool
ClusterNetworkLayer cluster.NetworkLayer ClusterNetworkLayer cluster.NetworkLayer
@ -754,11 +749,6 @@ func CreateCore(conf *CoreConfig) (*Core, error) {
conf.RawConfig = new(server.Config) conf.RawConfig = new(server.Config)
} }
syncInterval := conf.CounterSyncInterval
if syncInterval.Nanoseconds() == 0 {
syncInterval = 30 * time.Second
}
// secureRandomReader cannot be nil // secureRandomReader cannot be nil
if conf.SecureRandomReader == nil { if conf.SecureRandomReader == nil {
conf.SecureRandomReader = rand.Reader conf.SecureRandomReader = rand.Reader
@ -793,39 +783,35 @@ func CreateCore(conf *CoreConfig) (*Core, error) {
baseLogger: conf.Logger, baseLogger: conf.Logger,
logger: conf.Logger.Named("core"), logger: conf.Logger.Named("core"),
defaultLeaseTTL: conf.DefaultLeaseTTL, defaultLeaseTTL: conf.DefaultLeaseTTL,
maxLeaseTTL: conf.MaxLeaseTTL, maxLeaseTTL: conf.MaxLeaseTTL,
sentinelTraceDisabled: conf.DisableSentinelTrace, sentinelTraceDisabled: conf.DisableSentinelTrace,
cachingDisabled: conf.DisableCache, cachingDisabled: conf.DisableCache,
clusterName: conf.ClusterName, clusterName: conf.ClusterName,
clusterNetworkLayer: conf.ClusterNetworkLayer, clusterNetworkLayer: conf.ClusterNetworkLayer,
clusterPeerClusterAddrsCache: cache.New(3*clusterHeartbeatInterval, time.Second), clusterPeerClusterAddrsCache: cache.New(3*clusterHeartbeatInterval, time.Second),
enableMlock: !conf.DisableMlock, enableMlock: !conf.DisableMlock,
rawEnabled: conf.EnableRaw, rawEnabled: conf.EnableRaw,
shutdownDoneCh: make(chan struct{}), shutdownDoneCh: make(chan struct{}),
replicationState: new(uint32), replicationState: new(uint32),
atomicPrimaryClusterAddrs: new(atomic.Value), atomicPrimaryClusterAddrs: new(atomic.Value),
atomicPrimaryFailoverAddrs: new(atomic.Value), atomicPrimaryFailoverAddrs: new(atomic.Value),
localClusterPrivateKey: new(atomic.Value), localClusterPrivateKey: new(atomic.Value),
localClusterCert: new(atomic.Value), localClusterCert: new(atomic.Value),
localClusterParsedCert: new(atomic.Value), localClusterParsedCert: new(atomic.Value),
activeNodeReplicationState: new(uint32), activeNodeReplicationState: new(uint32),
keepHALockOnStepDown: new(uint32), keepHALockOnStepDown: new(uint32),
replicationFailure: new(uint32), replicationFailure: new(uint32),
disablePerfStandby: true, disablePerfStandby: true,
activeContextCancelFunc: new(atomic.Value), activeContextCancelFunc: new(atomic.Value),
allLoggers: conf.AllLoggers, allLoggers: conf.AllLoggers,
builtinRegistry: conf.BuiltinRegistry, builtinRegistry: conf.BuiltinRegistry,
neverBecomeActive: new(uint32), neverBecomeActive: new(uint32),
clusterLeaderParams: new(atomic.Value), clusterLeaderParams: new(atomic.Value),
metricsHelper: conf.MetricsHelper, metricsHelper: conf.MetricsHelper,
metricSink: conf.MetricSink, metricSink: conf.MetricSink,
secureRandomReader: conf.SecureRandomReader, secureRandomReader: conf.SecureRandomReader,
rawConfig: new(atomic.Value), rawConfig: new(atomic.Value),
counters: counters{
requests: new(uint64),
syncInterval: syncInterval,
},
recoveryMode: conf.RecoveryMode, recoveryMode: conf.RecoveryMode,
postUnsealStarted: new(uint32), postUnsealStarted: new(uint32),
raftJoinDoneCh: make(chan struct{}), raftJoinDoneCh: make(chan struct{}),
@ -1984,9 +1970,6 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c
if err := c.loadCORSConfig(ctx); err != nil { if err := c.loadCORSConfig(ctx); err != nil {
return err return err
} }
if err := c.loadCurrentRequestCounters(ctx, time.Now()); err != nil {
return err
}
if err := c.loadCredentials(ctx); err != nil { if err := c.loadCredentials(ctx); err != nil {
return err return err
} }
@ -2184,11 +2167,6 @@ func (c *Core) preSeal() error {
result = multierror.Append(result, fmt.Errorf("error unloading mounts: %w", err)) result = multierror.Append(result, fmt.Errorf("error unloading mounts: %w", err))
} }
// Zeros out the requests counter to avoid sending all requests for the month on stepdown
// when this runs on the active node. Unseal does the complementary operation of loading
// the counter from storage, so this operation should be a safe one.
atomic.StoreUint64(c.counters.requests, 0)
if err := enterprisePreSeal(c); err != nil { if err := enterprisePreSeal(c); err != nil {
result = multierror.Append(result, err) result = multierror.Append(result, err)
} }

23
vault/core_metrics.go
View File

@ -24,7 +24,7 @@ func (c *Core) metricsLoop(stopCh chan struct{}) {
identityCountTimer = nil identityCountTimer = nil
} }
writeTimer := time.Tick(c.counters.syncInterval) writeTimer := time.Tick(time.Second * 30)
// Do not process the writeTimer on DR Secondary nodes // Do not process the writeTimer on DR Secondary nodes
if c.IsDRSecondary() { if c.IsDRSecondary() {
writeTimer = nil writeTimer = nil
@ -104,23 +104,12 @@ func (c *Core) metricsLoop(stopCh chan struct{}) {
// should trigger // should trigger
continue continue
} }
if c.perfStandby { // already have lock here, do not re-acquire // Ship barrier encryption counts if a perf standby or the active node
err := syncCounters(c) // on a performance secondary cluster
if c.perfStandby || c.ReplicationState().HasState(consts.ReplicationPerformanceSecondary) { // already have lock here, do not re-acquire
err := syncBarrierEncryptionCounter(c)
if err != nil { if err != nil {
c.logger.Error("writing syncing counters", "err", err) c.logger.Error("writing syncing encryption counters", "err", err)
}
} else {
// Perf standbys will have synced above, but active nodes on a secondary cluster still need to ship
// barrier encryption counts
if c.ReplicationState().HasState(consts.ReplicationPerformanceSecondary) {
err := syncBarrierEncryptionCounter(c)
if err != nil {
c.logger.Error("writing syncing encryption counts", "err", err)
}
}
err := c.saveCurrentRequestCounters(context.Background(), time.Now())
if err != nil {
c.logger.Error("writing request counters to barrier", "err", err)
} }
} }
c.stateLock.RUnlock() c.stateLock.RUnlock()

166
vault/counters.go
View File

@ -2,12 +2,6 @@ package vault
import ( import (
"context" "context"
"fmt"
"sort"
"sync/atomic"
"time"
"github.com/hashicorp/vault/sdk/logical"
) )
const ( const (
@ -15,168 +9,8 @@ const (
// This storage path stores both the request counters in this file, and the activity log. // This storage path stores both the request counters in this file, and the activity log.
countersSubPath = "counters/" countersSubPath = "counters/"
requestCountersPath = "sys/counters/requests/"
) )
type counters struct {
// requests counts requests seen by Vault this month; does not include requests
// excluded by design, e.g. health checks and UI asset requests.
requests *uint64
// activePath is set at startup to the path we primed the requests counter from,
// or empty string if there wasn't a relevant path - either because this is the first
// time Vault starts with the feature enabled, or because Vault hadn't written
// out the request counter this month yet.
// Whenever we write out the counters, we update activePath if it's no longer
// accurate. This coincides with a reset of the counters.
// There's no lock because the only reader/writer of activePath is the goroutine
// doing background syncs.
activePath string
// syncInterval determines how often the counters get written to storage (on primary)
// or synced to primary.
syncInterval time.Duration
}
// RequestCounter stores the state of request counters for a single unspecified period.
type RequestCounter struct {
// Total is the number of requests seen during a given period.
Total *uint64 `json:"total"`
}
// DatedRequestCounter holds request counters from a single period of time.
type DatedRequestCounter struct {
// StartTime is when the period starts.
StartTime time.Time `json:"start_time"`
// RequestCounter counts requests.
RequestCounter
}
// loadAllRequestCounters returns all request counters found in storage,
// ordered by time (oldest first.)
func (c *Core) loadAllRequestCounters(ctx context.Context, now time.Time) ([]DatedRequestCounter, error) {
view := NewBarrierView(c.barrier, requestCountersPath)
datepaths, err := view.List(ctx, "")
if err != nil {
return nil, fmt.Errorf("failed to read request counters: %w", err)
}
var all []DatedRequestCounter
sort.Strings(datepaths)
for _, datepath := range datepaths {
datesubpaths, err := view.List(ctx, datepath)
if err != nil {
return nil, fmt.Errorf("failed to read request counters: %w", err)
}
sort.Strings(datesubpaths)
for _, datesubpath := range datesubpaths {
fullpath := datepath + datesubpath
counter, err := c.loadRequestCounters(ctx, fullpath)
if err != nil {
return nil, err
}
t, err := time.Parse(requestCounterDatePathFormat, fullpath)
if err != nil {
return nil, err
}
all = append(all, DatedRequestCounter{StartTime: t, RequestCounter: *counter})
}
}
start, _ := time.Parse(requestCounterDatePathFormat, now.Format(requestCounterDatePathFormat))
idx := sort.Search(len(all), func(i int) bool {
return !all[i].StartTime.Before(start)
})
cur := atomic.LoadUint64(c.counters.requests)
if idx < len(all) {
all[idx].RequestCounter.Total = &cur
} else {
all = append(all, DatedRequestCounter{StartTime: start, RequestCounter: RequestCounter{Total: &cur}})
}
return all, nil
}
// loadCurrentRequestCounters reads the current RequestCounter out of storage.
// The in-memory current request counter is populated with the value read, if any.
// now should be the current time; it is a parameter to facilitate testing.
func (c *Core) loadCurrentRequestCounters(ctx context.Context, now time.Time) error {
datepath := now.Format(requestCounterDatePathFormat)
counter, err := c.loadRequestCounters(ctx, datepath)
if err != nil {
return err
}
if counter != nil {
c.counters.activePath = datepath
atomic.StoreUint64(c.counters.requests, *counter.Total)
}
return nil
}
// loadRequestCounters reads a RequestCounter out of storage at location datepath.
// If nothing is found at that path, that isn't an error: a reference to a zero
// RequestCounter is returned.
func (c *Core) loadRequestCounters(ctx context.Context, datepath string) (*RequestCounter, error) {
view := NewBarrierView(c.barrier, requestCountersPath)
out, err := view.Get(ctx, datepath)
if err != nil {
return nil, fmt.Errorf("failed to read request counters: %w", err)
}
if out == nil {
return nil, nil
}
newCounters := &RequestCounter{}
err = out.DecodeJSON(newCounters)
if err != nil {
return nil, err
}
return newCounters, nil
}
// saveCurrentRequestCounters writes the current RequestCounter to storage.
// The in-memory current request counter is reset to zero after writing if
// we've entered a new month.
// now should be the current time; it is a parameter to facilitate testing.
func (c *Core) saveCurrentRequestCounters(ctx context.Context, now time.Time) error {
view := NewBarrierView(c.barrier, requestCountersPath)
requests := atomic.LoadUint64(c.counters.requests)
curDatePath := now.Format(requestCounterDatePathFormat)
// If activePath is empty string, we were started with nothing in storage
// for the current month, so we should not reset the in-mem counter.
// But if activePath is nonempty and not curDatePath, we should reset.
shouldReset, writeDatePath := false, curDatePath
if c.counters.activePath != "" && c.counters.activePath != curDatePath {
shouldReset, writeDatePath = true, c.counters.activePath
}
localCounters := &RequestCounter{
Total: &requests,
}
entry, err := logical.StorageEntryJSON(writeDatePath, localCounters)
if err != nil {
return fmt.Errorf("failed to create request counters entry: %w", err)
}
if err := view.Put(ctx, entry); err != nil {
return fmt.Errorf("failed to save request counters: %w", err)
}
if shouldReset {
atomic.StoreUint64(c.counters.requests, 0)
}
if c.counters.activePath != curDatePath {
c.counters.activePath = curDatePath
}
return nil
}
// ActiveTokens contains the number of active tokens. // ActiveTokens contains the number of active tokens.
type ActiveTokens struct { type ActiveTokens struct {
// ServiceTokens contains information about the number of active service // ServiceTokens contains information about the number of active service

104
vault/counters_test.go
View File

@ -1,8 +1,6 @@
package vault package vault
import ( import (
"context"
"sync/atomic"
"testing" "testing"
"time" "time"
@ -21,108 +19,6 @@ func testParseTime(t *testing.T, format, timeval string) time.Time {
return tm return tm
} }
// TestRequestCounterLoadCurrent exercises the code that primes the in-mem
// request counters from persistent storage.
func TestRequestCounterLoadCurrent(t *testing.T) {
c, _, _ := TestCoreUnsealed(t)
december2018 := testParseTime(t, time.RFC3339, "2018-12-05T09:44:12-05:00")
decemberRequests := uint64(555)
// It's December, and we got some requests. Persist the counter.
atomic.StoreUint64(c.counters.requests, decemberRequests)
err := c.saveCurrentRequestCounters(context.Background(), december2018)
if err != nil {
t.Fatal(err)
}
// It's still December, simulate being restarted. At startup the counter is
// zero initially, until we read the counter from storage post-unseal via
// loadCurrentRequestCounters.
atomic.StoreUint64(c.counters.requests, 0)
err = c.loadCurrentRequestCounters(context.Background(), december2018)
if err != nil {
t.Fatal(err)
}
if got := atomic.LoadUint64(c.counters.requests); got != decemberRequests {
t.Fatalf("expected=%d, got=%d", decemberRequests, got)
}
// Now simulate being restarted in January. We never wrote anything out during
// January, so the in-mem counter should remain zero.
january2019 := testParseTime(t, time.RFC3339, "2019-01-02T08:21:11-05:00")
atomic.StoreUint64(c.counters.requests, 0)
err = c.loadCurrentRequestCounters(context.Background(), january2019)
if err != nil {
t.Fatal(err)
}
if got := atomic.LoadUint64(c.counters.requests); got != 0 {
t.Fatalf("expected=%d, got=%d", 0, got)
}
}
// TestRequestCounterSaveCurrent exercises the code that saves the in-mem
// request counters to persistent storage.
func TestRequestCounterSaveCurrent(t *testing.T) {
c, _, _ := TestCoreUnsealed(t)
// storeSaveLoad stores newValue in the in-mem counter, saves it to storage,
// then verifies in-mem counter has value expectedPostLoad.
storeSaveLoad := func(newValue, expectedPostLoad uint64, now time.Time) {
t.Helper()
atomic.StoreUint64(c.counters.requests, newValue)
err := c.saveCurrentRequestCounters(context.Background(), now)
if err != nil {
t.Fatal(err)
}
if got := atomic.LoadUint64(c.counters.requests); got != expectedPostLoad {
t.Fatalf("expected=%d, got=%d", expectedPostLoad, got)
}
}
// Start in December. The first write ever should persist the current in-mem value.
december2018 := testParseTime(t, time.RFC3339, "2018-12-05T09:44:12-05:00")
decemberRequests := uint64(555)
storeSaveLoad(decemberRequests, decemberRequests, december2018)
// Update request count.
decemberRequests++
storeSaveLoad(decemberRequests, decemberRequests, december2018)
decemberStartTime := testParseTime(t, requestCounterDatePathFormat, december2018.Format(requestCounterDatePathFormat))
expected2018 := []DatedRequestCounter{
{StartTime: decemberStartTime, RequestCounter: RequestCounter{Total: &decemberRequests}},
}
all, err := c.loadAllRequestCounters(context.Background(), december2018)
if err != nil {
t.Fatal(err)
}
if diff := deep.Equal(all, expected2018); len(diff) != 0 {
t.Errorf("Expected=%v, got=%v, diff=%v", expected2018, all, diff)
}
// Now it's January. Saving after transition to new month should reset in-mem
// counter to zero, and also write zero to storage for the new month.
january2019 := testParseTime(t, time.RFC3339, "2019-01-02T08:21:11-05:00")
decemberRequests += 5
storeSaveLoad(decemberRequests, 0, january2019)
januaryRequests := uint64(333)
storeSaveLoad(januaryRequests, januaryRequests, january2019)
all, err = c.loadAllRequestCounters(context.Background(), january2019)
if err != nil {
t.Fatal(err)
}
januaryStartTime := testParseTime(t, requestCounterDatePathFormat, january2019.Format(requestCounterDatePathFormat))
expected2019 := expected2018
expected2019 = append(expected2019,
DatedRequestCounter{januaryStartTime, RequestCounter{&januaryRequests}})
if diff := deep.Equal(all, expected2019); len(diff) != 0 {
t.Errorf("Expected=%v, got=%v, diff=%v", expected2019, all, diff)
}
}
func testCountActiveTokens(t *testing.T, c *Core, root string) int { func testCountActiveTokens(t *testing.T, c *Core, root string) int {
t.Helper() t.Helper()

17
vault/logical_system.go
View File

@ -3548,18 +3548,9 @@ func (b *SystemBackend) pathInternalUIMountRead(ctx context.Context, req *logica
} }
func (b *SystemBackend) pathInternalCountersRequests(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { func (b *SystemBackend) pathInternalCountersRequests(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
counters, err := b.Core.loadAllRequestCounters(ctx, time.Now()) resp := logical.ErrorResponse("The functionality has been removed on this path")
if err != nil {
return nil, err
}
resp := &logical.Response{ return resp, logical.ErrPathFunctionalityRemoved
Data: map[string]interface{}{
"counters": counters,
},
}
return resp, nil
} }
func (b *SystemBackend) pathInternalCountersTokens(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { func (b *SystemBackend) pathInternalCountersTokens(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
@ -4754,8 +4745,8 @@ This path responds to the following HTTP methods.
"", "",
}, },
"internal-counters-requests": { "internal-counters-requests": {
"Count of requests seen by this Vault cluster over time.", "Currently unsupported. Previously, count of requests seen by this Vault cluster over time.",
"Count of requests seen by this Vault cluster over time. Not included in count: health checks, UI asset requests, requests forwarded from another cluster.", "Currently unsupported. Previously, count of requests seen by this Vault cluster over time. Not included in count: health checks, UI asset requests, requests forwarded from another cluster.",
}, },
"internal-counters-tokens": { "internal-counters-tokens": {
"Count of active tokens in this Vault cluster.", "Count of active tokens in this Vault cluster.",

2
vault/request_handling.go
View File

@ -5,7 +5,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
"sync/atomic"
"time" "time"
metrics "github.com/armon/go-metrics" metrics "github.com/armon/go-metrics"
@ -730,7 +729,6 @@ func (c *Core) doRouting(ctx context.Context, req *logical.Request) (*logical.Re
if shouldForward(c, resp, err) { if shouldForward(c, resp, err) {
return forward(ctx, c, req) return forward(ctx, c, req)
} }
atomic.AddUint64(c.counters.requests, 1)
return resp, err return resp, err
} }

1
vault/testing.go
View File

@ -1520,7 +1520,6 @@ func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *Te
coreConfig.DisableCache = base.DisableCache coreConfig.DisableCache = base.DisableCache
coreConfig.DevToken = base.DevToken coreConfig.DevToken = base.DevToken
coreConfig.CounterSyncInterval = base.CounterSyncInterval
coreConfig.RecoveryMode = base.RecoveryMode coreConfig.RecoveryMode = base.RecoveryMode
coreConfig.ActivityLogConfig = base.ActivityLogConfig coreConfig.ActivityLogConfig = base.ActivityLogConfig

47
website/content/api-docs/system/internal-counters.mdx
View File

@ -7,52 +7,9 @@ description: >-
# `/sys/internal/counters` # `/sys/internal/counters`
The `/sys/internal/counters` endpoints are used to return data about the number of Http Requests, Tokens, and Entities in Vault. The `/sys/internal/counters` endpoints are used to return data about the number of Tokens and Entities in Vault.
~> **NOTE**: This endpoint is only available in Vault version 1.3+. Backwards compatibility is not guaranteed. These endpoints are subject to change or may disappear without notice. ~> **NOTE**: These endpoints are only available in Vault version 1.3+. Backwards compatibility is not guaranteed. These endpoints are subject to change or may disappear without notice.
## Http Requests
This endpoint lists the number of Http Requests made per month.
| Method | Path |
| :----- | :-------------------------------- |
| `GET` | `/sys/internal/counters/requests` |
### Sample Request
```shell-session
$ curl \
--header "X-Vault-Token: ..." \
--request GET \
http://127.0.0.1:8200/v1/sys/internal/counters/requests
```
### Sample Response
```json
{
"request_id": "75cbaa46-e741-3eba-2be2-325b1ba8f03f",
"lease_id": "",
"renewable": false,
"lease_duration": 0,
"data": {
"counters": [
{
"start_time": "2019-05-01T00:00:00Z",
"total": 50
},
{
"start_time": "2019-04-01T00:00:00Z",
"total": 45
}
]
},
"wrap_info": null,
"warnings": null,
"auth": null
}
```
## Entities ## Entities