Change UseToken mechanics.
Add locking around UseToken and Lookup. Have UseToken flag an entry that needs to be revoked so that it can be done at the appropriate time, but so that Lookup in the interm doesn't return a value. The locking is a map of 4096 locks keyed off of the first three characters of the token ID which should provide good distribution.
This commit is contained in:
parent
1ffd5653c6
commit
2ebe49d3a1
|
@ -18,7 +18,7 @@ func TestFlagSet(t *testing.T) {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
FlagSetServer,
|
FlagSetServer,
|
||||||
[]string{"address", "ca-cert", "ca-path", "client-cert", "client-key", "insecure", "tls-skip-verify"},
|
[]string{"address", "ca-cert", "ca-path", "client-cert", "client-key", "insecure", "tls-skip-verify", "wrap-ttl"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -411,7 +411,7 @@ func (c *Core) checkToken(req *logical.Request) (*logical.Auth, *TokenEntry, err
|
||||||
|
|
||||||
acl, te, err := c.fetchACLandTokenEntry(req)
|
acl, te, err := c.fetchACLandTokenEntry(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, te, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if this is a root protected path
|
// Check if this is a root protected path
|
||||||
|
@ -450,13 +450,14 @@ func (c *Core) checkToken(req *logical.Request) (*logical.Auth, *TokenEntry, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check the standard non-root ACLs
|
// Check the standard non-root ACLs. Return the token entry if it's not
|
||||||
|
// allowed so we can decrement the use count.
|
||||||
allowed, rootPrivs := acl.AllowOperation(req.Operation, req.Path)
|
allowed, rootPrivs := acl.AllowOperation(req.Operation, req.Path)
|
||||||
if !allowed {
|
if !allowed {
|
||||||
return nil, nil, logical.ErrPermissionDenied
|
return nil, te, logical.ErrPermissionDenied
|
||||||
}
|
}
|
||||||
if rootPath && !rootPrivs {
|
if rootPath && !rootPrivs {
|
||||||
return nil, nil, logical.ErrPermissionDenied
|
return nil, te, logical.ErrPermissionDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create the auth response
|
// Create the auth response
|
||||||
|
@ -687,12 +688,20 @@ func (c *Core) Seal(token string) (retErr error) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Attempt to use the token (decrement num_uses)
|
// Attempt to use the token (decrement num_uses)
|
||||||
// If we can't, we still continue attempting the seal, so long as the token
|
// On error bail out; if the token has been revoked, bail out too
|
||||||
// has appropriate permissions
|
|
||||||
if te != nil {
|
if te != nil {
|
||||||
if err := c.tokenStore.UseToken(te); err != nil {
|
te, err = c.tokenStore.UseToken(te)
|
||||||
|
if err != nil {
|
||||||
c.logger.Printf("[ERR] core: failed to use token: %v", err)
|
c.logger.Printf("[ERR] core: failed to use token: %v", err)
|
||||||
retErr = ErrInternalError
|
return ErrInternalError
|
||||||
|
}
|
||||||
|
if te == nil {
|
||||||
|
// Token is no longer valid
|
||||||
|
return logical.ErrPermissionDenied
|
||||||
|
}
|
||||||
|
if te.NumUses == -1 {
|
||||||
|
// Token needs to be revoked
|
||||||
|
return c.tokenStore.Revoke(te.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -744,10 +753,19 @@ func (c *Core) StepDown(token string) error {
|
||||||
}
|
}
|
||||||
// Attempt to use the token (decrement num_uses)
|
// Attempt to use the token (decrement num_uses)
|
||||||
if te != nil {
|
if te != nil {
|
||||||
if err := c.tokenStore.UseToken(te); err != nil {
|
te, err = c.tokenStore.UseToken(te)
|
||||||
|
if err != nil {
|
||||||
c.logger.Printf("[ERR] core: failed to use token: %v", err)
|
c.logger.Printf("[ERR] core: failed to use token: %v", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if te == nil {
|
||||||
|
// Token has been revoked
|
||||||
|
return logical.ErrPermissionDenied
|
||||||
|
}
|
||||||
|
if te.NumUses == -1 {
|
||||||
|
// Token needs to be revoked
|
||||||
|
return c.tokenStore.Revoke(te.ID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify that this operation is allowed
|
// Verify that this operation is allowed
|
||||||
|
|
|
@ -146,10 +146,7 @@ func (c *Core) HandleRequest(req *logical.Request) (resp *logical.Response, err
|
||||||
// wrapped information, since the original response has been audit logged
|
// wrapped information, since the original response has been audit logged
|
||||||
if wrapping {
|
if wrapping {
|
||||||
wrappingResp := &logical.Response{
|
wrappingResp := &logical.Response{
|
||||||
WrapInfo: logical.WrapInfo{
|
WrapInfo: resp.WrapInfo,
|
||||||
Token: resp.WrapInfo.Token,
|
|
||||||
TTL: resp.WrapInfo.TTL,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
wrappingResp.CloneWarnings(resp)
|
wrappingResp.CloneWarnings(resp)
|
||||||
resp = wrappingResp
|
resp = wrappingResp
|
||||||
|
@ -162,45 +159,58 @@ func (c *Core) handleRequest(req *logical.Request) (retResp *logical.Response, r
|
||||||
defer metrics.MeasureSince([]string{"core", "handle_request"}, time.Now())
|
defer metrics.MeasureSince([]string{"core", "handle_request"}, time.Now())
|
||||||
|
|
||||||
// Validate the token
|
// Validate the token
|
||||||
auth, te, err := c.checkToken(req)
|
auth, te, ctErr := c.checkToken(req)
|
||||||
|
// We run this logic first because we want to decrement the use count even in the case of an error
|
||||||
if te != nil {
|
if te != nil {
|
||||||
defer func() {
|
// Attempt to use the token (decrement NumUses)
|
||||||
// Attempt to use the token (decrement num_uses)
|
var err error
|
||||||
// If a secret was generated and num_uses is currently 1, it will be
|
te, err = c.tokenStore.UseToken(te)
|
||||||
// immediately revoked; in that case, don't return the leased
|
if err != nil {
|
||||||
// credentials as they are now invalid.
|
|
||||||
if retResp != nil &&
|
|
||||||
te != nil && te.NumUses == 1 &&
|
|
||||||
retResp.Secret != nil &&
|
|
||||||
// Some backends return a TTL even without a Lease ID
|
|
||||||
retResp.Secret.LeaseID != "" {
|
|
||||||
retResp = logical.ErrorResponse("Secret cannot be returned; token had one use left, so leased credentials were immediately revoked.")
|
|
||||||
}
|
|
||||||
if err := c.tokenStore.UseToken(te); err != nil {
|
|
||||||
c.logger.Printf("[ERR] core: failed to use token: %v", err)
|
c.logger.Printf("[ERR] core: failed to use token: %v", err)
|
||||||
|
return nil, nil, ErrInternalError
|
||||||
|
}
|
||||||
|
if te == nil {
|
||||||
|
// Token has been revoked by this point
|
||||||
|
return nil, nil, logical.ErrPermissionDenied
|
||||||
|
}
|
||||||
|
if te.NumUses == -1 {
|
||||||
|
// We defer a revocation until after logic has run, since this is a
|
||||||
|
// valid request (this is the token's final use). We pass the ID in
|
||||||
|
// directly just to be safe in case something else modifies te later.
|
||||||
|
defer func(id string) {
|
||||||
|
err = c.tokenStore.Revoke(id)
|
||||||
|
if err != nil {
|
||||||
|
c.logger.Printf("[ERR] core: failed to revoke token: %v", err)
|
||||||
retResp = nil
|
retResp = nil
|
||||||
retAuth = nil
|
retAuth = nil
|
||||||
retErr = ErrInternalError
|
retErr = ErrInternalError
|
||||||
}
|
}
|
||||||
}()
|
if retResp != nil && retResp.Secret != nil &&
|
||||||
|
// Some backends return a TTL even without a Lease ID
|
||||||
|
retResp.Secret.LeaseID != "" {
|
||||||
|
retResp = logical.ErrorResponse("Secret cannot be returned; token had one use left, so leased credentials were immediately revoked.")
|
||||||
|
return
|
||||||
}
|
}
|
||||||
if err != nil {
|
}(te.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ctErr != nil {
|
||||||
// If it is an internal error we return that, otherwise we
|
// If it is an internal error we return that, otherwise we
|
||||||
// return invalid request so that the status codes can be correct
|
// return invalid request so that the status codes can be correct
|
||||||
var errType error
|
var errType error
|
||||||
switch err {
|
switch ctErr {
|
||||||
case ErrInternalError, logical.ErrPermissionDenied:
|
case ErrInternalError, logical.ErrPermissionDenied:
|
||||||
errType = err
|
errType = ctErr
|
||||||
default:
|
default:
|
||||||
errType = logical.ErrInvalidRequest
|
errType = logical.ErrInvalidRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.auditBroker.LogRequest(auth, req, err); err != nil {
|
if err := c.auditBroker.LogRequest(auth, req, ctErr); err != nil {
|
||||||
c.logger.Printf("[ERR] core: failed to audit request with path (%s): %v",
|
c.logger.Printf("[ERR] core: failed to audit request with path (%s): %v",
|
||||||
req.Path, err)
|
req.Path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return logical.ErrorResponse(err.Error()), nil, errType
|
return logical.ErrorResponse(ctErr.Error()), nil, errType
|
||||||
}
|
}
|
||||||
|
|
||||||
// Attach the display name
|
// Attach the display name
|
||||||
|
|
|
@ -5,7 +5,9 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/armon/go-metrics"
|
"github.com/armon/go-metrics"
|
||||||
|
@ -61,6 +63,8 @@ type TokenStore struct {
|
||||||
cubbyholeBackend *CubbyholeBackend
|
cubbyholeBackend *CubbyholeBackend
|
||||||
|
|
||||||
policyLookupFunc func(string) (*Policy, error)
|
policyLookupFunc func(string) (*Policy, error)
|
||||||
|
|
||||||
|
tokenLocks map[string]*sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTokenStore is used to construct a token store that is
|
// NewTokenStore is used to construct a token store that is
|
||||||
|
@ -87,6 +91,19 @@ func NewTokenStore(c *Core, config *logical.BackendConfig) (*TokenStore, error)
|
||||||
}
|
}
|
||||||
t.salt = salt
|
t.salt = salt
|
||||||
|
|
||||||
|
t.tokenLocks = map[string]*sync.RWMutex{}
|
||||||
|
for i := int64(0); i < 16; i++ {
|
||||||
|
for j := int64(0); j < 16; j++ {
|
||||||
|
for k := int64(0); k < 16; k++ {
|
||||||
|
t.tokenLocks[fmt.Sprintf("%s%s%s",
|
||||||
|
strconv.FormatInt(i, 16),
|
||||||
|
strconv.FormatInt(j, 16),
|
||||||
|
strconv.FormatInt(k, 16))] = &sync.RWMutex{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.tokenLocks["global"] = &sync.RWMutex{}
|
||||||
|
|
||||||
// Setup the framework endpoints
|
// Setup the framework endpoints
|
||||||
t.Backend = &framework.Backend{
|
t.Backend = &framework.Backend{
|
||||||
AuthRenew: t.authRenew,
|
AuthRenew: t.authRenew,
|
||||||
|
@ -547,33 +564,74 @@ func (ts *TokenStore) storeCommon(entry *TokenEntry, writeSecondary bool) error
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// UseToken is used to manage restricted use tokens and decrement
|
func (ts *TokenStore) getTokenLock(id string) *sync.RWMutex {
|
||||||
// their available uses. Note: this is potentially racy, but the simple
|
// Find our multilevel lock, or fall back to global
|
||||||
// solution of a global lock would be severely detrimental to performance. Also
|
var lock *sync.RWMutex
|
||||||
// note the specific revoke case below.
|
var ok bool
|
||||||
func (ts *TokenStore) UseToken(te *TokenEntry) error {
|
if len(id) >= 3 {
|
||||||
|
lock, ok = ts.tokenLocks[id[0:3]]
|
||||||
|
}
|
||||||
|
if !ok || lock == nil {
|
||||||
|
lock = ts.tokenLocks["global"]
|
||||||
|
}
|
||||||
|
|
||||||
|
return lock
|
||||||
|
}
|
||||||
|
|
||||||
|
// UseToken is used to manage restricted use tokens and decrement their
|
||||||
|
// available uses. Returns two values: a potentially updated entry or, if the
|
||||||
|
// token has been revoked, nil; and whether an error was encountered. The
|
||||||
|
// locking here isn't perfect, as other parts of the code may update an entry,
|
||||||
|
// but usually none after the entry is already created...so this is pretty
|
||||||
|
// good.
|
||||||
|
func (ts *TokenStore) UseToken(te *TokenEntry) (*TokenEntry, error) {
|
||||||
|
if te == nil {
|
||||||
|
return nil, fmt.Errorf("invalid token entry provided for use count decrementing")
|
||||||
|
}
|
||||||
|
|
||||||
|
lock := ts.getTokenLock(te.ID)
|
||||||
|
|
||||||
|
lock.RLock()
|
||||||
// If the token is not restricted, there is nothing to do
|
// If the token is not restricted, there is nothing to do
|
||||||
if te.NumUses == 0 {
|
if te.NumUses == 0 {
|
||||||
return nil
|
lock.RUnlock()
|
||||||
|
return te, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
lock.RUnlock()
|
||||||
|
lock.Lock()
|
||||||
|
defer lock.Unlock()
|
||||||
|
|
||||||
|
// Call lookupSalted to avoid lock contention
|
||||||
|
te, err := ts.lookupSalted(ts.SaltID(te.ID))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to refresh entry: %v", err)
|
||||||
|
}
|
||||||
|
if te == nil {
|
||||||
|
return nil, fmt.Errorf("token entry nil after refreshing to decrement use count; token has likely been used already")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if it's already been revoked at this point, if so, don't return an
|
||||||
|
// error, but do indicate with a nil response that the token is no longer
|
||||||
|
// valid
|
||||||
|
if te.NumUses == -1 {
|
||||||
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Decrement the count
|
// Decrement the count
|
||||||
te.NumUses -= 1
|
te.NumUses -= 1
|
||||||
|
|
||||||
// Revoke the token if there are no remaining uses.
|
// We need to indicate that this is no longer valid, but revocation is
|
||||||
// XXX: There is a race condition here with parallel
|
// deferred to the end of the call, so this will make sure that any Lookup
|
||||||
// requests using the same token. This would require
|
// that happens doesn't return an entry
|
||||||
// some global coordination to avoid, as we must ensure
|
|
||||||
// no requests using the same restricted token are handled
|
|
||||||
// in parallel.
|
|
||||||
if te.NumUses == 0 {
|
if te.NumUses == 0 {
|
||||||
return ts.Revoke(te.ID)
|
te.NumUses = -1
|
||||||
}
|
}
|
||||||
|
|
||||||
// Marshal the entry
|
// Marshal the entry
|
||||||
enc, err := json.Marshal(te)
|
enc, err := json.Marshal(te)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to encode entry: %v", err)
|
return nil, fmt.Errorf("failed to encode entry: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write under the primary ID
|
// Write under the primary ID
|
||||||
|
@ -581,17 +639,23 @@ func (ts *TokenStore) UseToken(te *TokenEntry) error {
|
||||||
path := lookupPrefix + saltedId
|
path := lookupPrefix + saltedId
|
||||||
le := &logical.StorageEntry{Key: path, Value: enc}
|
le := &logical.StorageEntry{Key: path, Value: enc}
|
||||||
if err := ts.view.Put(le); err != nil {
|
if err := ts.view.Put(le); err != nil {
|
||||||
return fmt.Errorf("failed to persist entry: %v", err)
|
return nil, fmt.Errorf("failed to persist entry: %v", err)
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
|
return te, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Lookup is used to find a token given its ID
|
// Lookup is used to find a token given its ID. It acquires a read lock, then calls lookupSalted.
|
||||||
func (ts *TokenStore) Lookup(id string) (*TokenEntry, error) {
|
func (ts *TokenStore) Lookup(id string) (*TokenEntry, error) {
|
||||||
defer metrics.MeasureSince([]string{"token", "lookup"}, time.Now())
|
defer metrics.MeasureSince([]string{"token", "lookup"}, time.Now())
|
||||||
if id == "" {
|
if id == "" {
|
||||||
return nil, fmt.Errorf("cannot lookup blank token")
|
return nil, fmt.Errorf("cannot lookup blank token")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lock := ts.getTokenLock(id)
|
||||||
|
lock.RLock()
|
||||||
|
defer lock.RUnlock()
|
||||||
|
|
||||||
return ts.lookupSalted(ts.SaltID(id))
|
return ts.lookupSalted(ts.SaltID(id))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -614,6 +678,12 @@ func (ts *TokenStore) lookupSalted(saltedId string) (*TokenEntry, error) {
|
||||||
if err := json.Unmarshal(raw.Value, entry); err != nil {
|
if err := json.Unmarshal(raw.Value, entry); err != nil {
|
||||||
return nil, fmt.Errorf("failed to decode entry: %v", err)
|
return nil, fmt.Errorf("failed to decode entry: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// This is a token that is awaiting deferred revocation
|
||||||
|
if entry.NumUses == -1 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
return entry, nil
|
return entry, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -244,10 +244,13 @@ func TestTokenStore_UseToken(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Root is an unlimited use token, should be a no-op
|
// Root is an unlimited use token, should be a no-op
|
||||||
err = ts.UseToken(ent)
|
te, err := ts.UseToken(ent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("err: %v", err)
|
t.Fatalf("err: %v", err)
|
||||||
}
|
}
|
||||||
|
if te == nil {
|
||||||
|
t.Fatalf("token entry after use was nil")
|
||||||
|
}
|
||||||
|
|
||||||
// Lookup the root token again
|
// Lookup the root token again
|
||||||
ent2, err := ts.Lookup(root)
|
ent2, err := ts.Lookup(root)
|
||||||
|
@ -266,10 +269,13 @@ func TestTokenStore_UseToken(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use the token
|
// Use the token
|
||||||
err = ts.UseToken(ent)
|
te, err = ts.UseToken(ent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("err: %v", err)
|
t.Fatalf("err: %v", err)
|
||||||
}
|
}
|
||||||
|
if te == nil {
|
||||||
|
t.Fatalf("token entry for use #1 was nil")
|
||||||
|
}
|
||||||
|
|
||||||
// Lookup the token
|
// Lookup the token
|
||||||
ent2, err = ts.Lookup(ent.ID)
|
ent2, err = ts.Lookup(ent.ID)
|
||||||
|
@ -283,10 +289,17 @@ func TestTokenStore_UseToken(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use the token
|
// Use the token
|
||||||
err = ts.UseToken(ent)
|
te, err = ts.UseToken(ent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("err: %v", err)
|
t.Fatalf("err: %v", err)
|
||||||
}
|
}
|
||||||
|
if te == nil {
|
||||||
|
t.Fatalf("token entry for use #2 was nil")
|
||||||
|
}
|
||||||
|
if te.NumUses != -1 {
|
||||||
|
t.Fatalf("token entry after use #2 did not have revoke flag")
|
||||||
|
}
|
||||||
|
ts.Revoke(te.ID)
|
||||||
|
|
||||||
// Lookup the token
|
// Lookup the token
|
||||||
ent2, err = ts.Lookup(ent.ID)
|
ent2, err = ts.Lookup(ent.ID)
|
||||||
|
|
Loading…
Reference in New Issue