Offline token revocation fix

This commit is contained in:
Jeff Mitchell 2018-06-03 18:14:51 -04:00
parent 217d2d0739
commit 5207099042
14 changed files with 578 additions and 385 deletions

View File

@ -1,5 +1,18 @@
## 0.10.2 (Unreleased)
SECURITY:
* The Vault team identified a race condition that could occur if a token's
lease expired while Vault was not running. In this case, when Vault came
back online, sometimes it would properly revoke the lease but other times it
would not, leading to a Vault token that no longer had an expiration and had
essentially unlimited lifetime. This race was per-token, not all-or-nothing
for all tokens that may have expired during Vault's downtime. We have fixed
the behavior and put extra checks in place to help prevent any similar
future issues. In addition, the logic we have put in place ensures that such
lease-less tokens can no longer be used (unless they are root tokens that
never had an expiration to begin with).
DEPRECATIONS/CHANGES:
* PKI duration return types: The PKI backend now returns durations (e.g. when

View File

@ -74,6 +74,12 @@ type Auth struct {
// The set of CIDRs that this token can be used with
BoundCIDRs []*sockaddr.SockAddrMarshaler `json:"bound_cidrs"`
// CreationPath is a path that the backend can return to use in the lease.
// This is currently only supported for the token store where roles may
// change the perceived path of the lease, even though they don't change
// the request path itself.
CreationPath string `json:"creation_path"`
}
func (a *Auth) GoString() string {

View File

@ -3,15 +3,15 @@ package logical
import (
"context"
"reflect"
"testing"
"time"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/helper/logging"
"github.com/mitchellh/go-testing-interface"
)
// TestRequest is a helper to create a purely in-memory Request struct.
func TestRequest(t *testing.T, op Operation, path string) *Request {
func TestRequest(t testing.T, op Operation, path string) *Request {
return &Request{
Operation: op,
Path: path,
@ -22,7 +22,7 @@ func TestRequest(t *testing.T, op Operation, path string) *Request {
// TestStorage is a helper that can be used from unit tests to verify
// the behavior of a Storage impl.
func TestStorage(t *testing.T, s Storage) {
func TestStorage(t testing.T, s Storage) {
keys, err := s.List(context.Background(), "")
if err != nil {
t.Fatalf("list error: %s", err)

View File

@ -5,6 +5,7 @@ import (
"reflect"
"sort"
"testing"
"time"
"github.com/hashicorp/vault/logical"
)
@ -73,10 +74,9 @@ path "secret/sample" {
Path: "secret/sample",
Policies: []string{"policy2"},
EntityID: entityID,
TTL: time.Hour,
}
if err := c.tokenStore.create(context.Background(), ent); err != nil {
t.Fatalf("err: %v", err)
}
testMakeTokenDirectly(t, c.tokenStore, ent)
actual, err := c.Capabilities(context.Background(), "capabilitiestoken", "secret/sample")
if err != nil {
@ -139,10 +139,9 @@ func TestCapabilities(t *testing.T) {
ID: "capabilitiestoken",
Path: "testpath",
Policies: []string{"dev"},
TTL: time.Hour,
}
if err := c.tokenStore.create(context.Background(), ent); err != nil {
t.Fatalf("err: %v", err)
}
testMakeTokenDirectly(t, c.tokenStore, ent)
actual, err = c.Capabilities(context.Background(), "capabilitiestoken", "foo/bar")
if err != nil {

View File

@ -494,7 +494,7 @@ func TestCore_HandleRequest_NoSlash(t *testing.T) {
// Test a root path is denied if non-root
func TestCore_HandleRequest_RootPath(t *testing.T) {
c, _, root := TestCoreUnsealed(t)
testCoreMakeToken(t, c, root, "child", "", []string{"test"})
testMakeTokenViaCore(t, c, root, "child", "", []string{"test"})
req := &logical.Request{
Operation: logical.ReadOperation,
@ -529,7 +529,7 @@ func TestCore_HandleRequest_RootPath_WithSudo(t *testing.T) {
}
// Child token (non-root) but with 'test' policy should have access
testCoreMakeToken(t, c, root, "child", "", []string{"test"})
testMakeTokenViaCore(t, c, root, "child", "", []string{"test"})
req = &logical.Request{
Operation: logical.ReadOperation,
Path: "sys/policy", // root protected!
@ -547,7 +547,7 @@ func TestCore_HandleRequest_RootPath_WithSudo(t *testing.T) {
// Check that standard permissions work
func TestCore_HandleRequest_PermissionDenied(t *testing.T) {
c, _, root := TestCoreUnsealed(t)
testCoreMakeToken(t, c, root, "child", "", []string{"test"})
testMakeTokenViaCore(t, c, root, "child", "", []string{"test"})
req := &logical.Request{
Operation: logical.UpdateOperation,
@ -567,7 +567,7 @@ func TestCore_HandleRequest_PermissionDenied(t *testing.T) {
// Check that standard permissions work
func TestCore_HandleRequest_PermissionAllowed(t *testing.T) {
c, _, root := TestCoreUnsealed(t)
testCoreMakeToken(t, c, root, "child", "", []string{"test"})
testMakeTokenViaCore(t, c, root, "child", "", []string{"test"})
// Set the 'test' policy object to permit access to secret/
req := &logical.Request{

View File

@ -624,9 +624,8 @@ func (m *ExpirationManager) revokePrefixCommon(prefix string, force bool) error
return errwrap.Wrapf(fmt.Sprintf("failed to revoke %q: {{err}}", prefix), err)
}
return nil
} else {
prefix = prefix + "/"
}
prefix = prefix + "/"
}
// Accumulate existing leases
@ -719,45 +718,6 @@ func (m *ExpirationManager) Renew(leaseID string, increment time.Duration) (*log
return resp, nil
}
// RestoreSaltedTokenCheck verifies that the token is not expired while running
// in restore mode. If we are not in restore mode, the lease has already been
// restored or the lease still has time left, it returns true.
func (m *ExpirationManager) RestoreSaltedTokenCheck(source string, saltedID string) (bool, error) {
defer metrics.MeasureSince([]string{"expire", "restore-token-check"}, time.Now())
// Return immediately if we are not in restore mode, expiration manager is
// already loaded
if !m.inRestoreMode() {
return true, nil
}
m.restoreModeLock.RLock()
defer m.restoreModeLock.RUnlock()
// Check again after we obtain the lock
if !m.inRestoreMode() {
return true, nil
}
leaseID := path.Join(source, saltedID)
m.lockLease(leaseID)
defer m.unlockLease(leaseID)
le, err := m.loadEntryInternal(leaseID, true, true)
if err != nil {
return false, err
}
if le != nil && !le.ExpireTime.IsZero() {
expires := le.ExpireTime.Sub(time.Now())
if expires <= 0 {
return false, nil
}
}
return true, nil
}
// RenewToken is used to renew a token which does not need to
// invoke a logical backend.
func (m *ExpirationManager) RenewToken(req *logical.Request, source string, token string,
@ -947,6 +907,7 @@ func (m *ExpirationManager) RegisterAuth(source string, auth *logical.Auth) erro
// Setup revocation timer
m.updatePending(&le, auth.LeaseTotal())
return nil
}

View File

@ -25,13 +25,13 @@ var (
// mockExpiration returns a mock expiration manager
func mockExpiration(t testing.TB) *ExpirationManager {
_, ts, _, _ := TestCoreWithTokenStore(t)
return ts.expiration
c, _, _ := TestCoreUnsealed(t)
return c.expiration
}
func mockBackendExpiration(t testing.TB, backend physical.Backend) (*Core, *ExpirationManager) {
c, ts, _, _ := TestCoreWithBackendTokenStore(t, backend)
return c, ts.expiration
c, _, _ := TestCoreUnsealedBackend(t, backend)
return c, c.expiration
}
func TestExpiration_Tidy(t *testing.T) {
@ -293,7 +293,8 @@ func BenchmarkExpiration_Restore_InMem(b *testing.B) {
}
func benchmarkExpirationBackend(b *testing.B, physicalBackend physical.Backend, numLeases int) {
c, exp := mockBackendExpiration(b, physicalBackend)
c, _, _ := TestCoreUnsealedBackend(b, physicalBackend)
exp := c.expiration
noop := &NoopBackend{}
view := NewBarrierView(c.barrier, "logical/")
meUUID, err := uuid.GenerateUUID()
@ -1603,7 +1604,7 @@ func TestLeaseEntry(t *testing.T) {
}
func TestExpiration_RevokeForce(t *testing.T) {
core, _, _, root := TestCoreWithTokenStore(t)
core, _, root := TestCoreUnsealed(t)
core.logicalBackends["badrenew"] = badRenewFactory
me := &MountEntry{
@ -1651,7 +1652,7 @@ func TestExpiration_RevokeForce(t *testing.T) {
}
func TestExpiration_RevokeForceSingle(t *testing.T) {
core, _, _, root := TestCoreWithTokenStore(t)
core, _, root := TestCoreUnsealed(t)
core.logicalBackends["badrenew"] = badRenewFactory
me := &MountEntry{

View File

@ -225,11 +225,9 @@ func TestIdentityStore_WrapInfoInheritance(t *testing.T) {
Path: "test",
Policies: []string{"default", responseWrappingPolicyName},
EntityID: entityID,
TTL: time.Hour,
}
if err := ts.create(context.Background(), te); err != nil {
t.Fatal(err)
}
testMakeTokenDirectly(t, ts, te)
wrapReq := &logical.Request{
Path: "sys/wrapping/wrap",
@ -258,18 +256,17 @@ func TestIdentityStore_WrapInfoInheritance(t *testing.T) {
}
func TestIdentityStore_TokenEntityInheritance(t *testing.T) {
_, ts, _, _ := TestCoreWithTokenStore(t)
c, _, _ := TestCoreUnsealed(t)
ts := c.tokenStore
// Create a token which has EntityID set
te := &TokenEntry{
Path: "test",
Policies: []string{"dev", "prod"},
EntityID: "testentityid",
TTL: time.Hour,
}
if err := ts.create(context.Background(), te); err != nil {
t.Fatal(err)
}
testMakeTokenDirectly(t, ts, te)
// Create a child token; this should inherit the EntityID
tokenReq := &logical.Request{
@ -301,14 +298,12 @@ func TestIdentityStore_TokenEntityInheritance(t *testing.T) {
func testCoreWithIdentityTokenGithub(t *testing.T) (*Core, *IdentityStore, *TokenStore, string) {
is, ghAccessor, core := testIdentityStoreWithGithubAuth(t)
ts := testTokenStore(t, core)
return core, is, ts, ghAccessor
return core, is, core.tokenStore, ghAccessor
}
func testCoreWithIdentityTokenGithubRoot(t *testing.T) (*Core, *IdentityStore, *TokenStore, string, string) {
is, ghAccessor, core, root := testIdentityStoreWithGithubAuthRoot(t)
ts := testTokenStore(t, core)
return core, is, ts, ghAccessor, root
return core, is, core.tokenStore, ghAccessor, root
}
func testIdentityStoreWithGithubAuth(t *testing.T) (*IdentityStore, string, *Core) {

View File

@ -442,7 +442,7 @@ func TestSystemBackend_PathCapabilities(t *testing.T) {
rootCheckFunc(t, resp)
// Create a non-root token
testMakeToken(t, core.tokenStore, rootToken, "tokenid", "", []string{"test"})
testMakeTokenViaBackend(t, core.tokenStore, rootToken, "tokenid", "", []string{"test"})
nonRootCheckFunc := func(t *testing.T, resp *logical.Response) {
expected1 := []string{"create", "sudo", "update"}
@ -544,7 +544,7 @@ func testCapabilities(t *testing.T, endpoint string) {
t.Fatalf("err: %v", err)
}
testMakeToken(t, core.tokenStore, rootToken, "tokenid", "", []string{"test"})
testMakeTokenViaBackend(t, core.tokenStore, rootToken, "tokenid", "", []string{"test"})
req = logical.TestRequest(t, logical.UpdateOperation, endpoint)
if endpoint == "capabilities-self" {
req.ClientToken = "tokenid"
@ -600,7 +600,7 @@ func TestSystemBackend_CapabilitiesAccessor_BC(t *testing.T) {
t.Fatalf("err: %v", err)
}
testMakeToken(t, core.tokenStore, rootToken, "tokenid", "", []string{"test"})
testMakeTokenViaBackend(t, core.tokenStore, rootToken, "tokenid", "", []string{"test"})
te, err = core.tokenStore.Lookup(context.Background(), "tokenid")
if err != nil {
@ -1264,8 +1264,10 @@ func TestSystemBackend_revokePrefix_origUrl(t *testing.T) {
}
}
func TestSystemBackend_revokePrefixAuth(t *testing.T) {
core, ts, _, _ := TestCoreWithTokenStore(t)
func TestSystemBackend_revokePrefixAuth_newUrl(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
ts := core.tokenStore
bc := &logical.BackendConfig{
Logger: core.logger,
System: logical.StaticSystemView{
@ -1284,11 +1286,9 @@ func TestSystemBackend_revokePrefixAuth(t *testing.T) {
te := &TokenEntry{
ID: "foo",
Path: "auth/github/login/bar",
TTL: time.Hour,
}
err = ts.create(context.Background(), te)
if err != nil {
t.Fatal(err)
}
testMakeTokenDirectly(t, ts, te)
te, err = ts.Lookup(context.Background(), "foo")
if err != nil {
@ -1329,7 +1329,8 @@ func TestSystemBackend_revokePrefixAuth(t *testing.T) {
}
func TestSystemBackend_revokePrefixAuth_origUrl(t *testing.T) {
core, ts, _, _ := TestCoreWithTokenStore(t)
core, _, _ := TestCoreUnsealed(t)
ts := core.tokenStore
bc := &logical.BackendConfig{
Logger: core.logger,
System: logical.StaticSystemView{
@ -1348,11 +1349,9 @@ func TestSystemBackend_revokePrefixAuth_origUrl(t *testing.T) {
te := &TokenEntry{
ID: "foo",
Path: "auth/github/login/bar",
TTL: time.Hour,
}
err = ts.create(context.Background(), te)
if err != nil {
t.Fatal(err)
}
testMakeTokenDirectly(t, ts, te)
te, err = ts.Lookup(context.Background(), "foo")
if err != nil {
@ -2396,7 +2395,7 @@ func TestSystemBackend_InternalUIMount(t *testing.T) {
t.Fatalf("Bad Response: %#v", resp)
}
testMakeToken(t, core.tokenStore, rootToken, "tokenid", "", []string{"secret"})
testMakeTokenViaBackend(t, core.tokenStore, rootToken, "tokenid", "", []string{"secret"})
req = logical.TestRequest(t, logical.ReadOperation, "internal/ui/mounts/kv")
req.ClientToken = "tokenid"

View File

@ -610,16 +610,7 @@ func (c *Core) handleRequest(ctx context.Context, req *logical.Request) (retResp
return nil, auth, retErr
}
// Register with the expiration manager. We use the token's actual path
// here because roles allow suffixes.
te, err := c.tokenStore.Lookup(ctx, resp.Auth.ClientToken)
if err != nil {
c.logger.Error("failed to look up token", "error", err)
retErr = multierror.Append(retErr, ErrInternalError)
return nil, auth, retErr
}
if err := c.expiration.RegisterAuth(te.Path, resp.Auth); err != nil {
if err := c.expiration.RegisterAuth(resp.Auth.CreationPath, resp.Auth); err != nil {
c.tokenStore.revokeOrphan(ctx, te.ID)
c.logger.Error("failed to register token lease", "request_path", req.Path, "error", err)
retErr = multierror.Append(retErr, ErrInternalError)

View File

@ -33,7 +33,6 @@ import (
"golang.org/x/net/http2"
cleanhttp "github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/go-uuid"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/audit"
"github.com/hashicorp/vault/helper/logging"
@ -305,58 +304,6 @@ func TestCoreUnsealedBackend(t testing.T, backend physical.Backend) (*Core, [][]
return core, keys, token
}
func testTokenStore(t testing.T, c *Core) *TokenStore {
me := &MountEntry{
Table: credentialTableType,
Path: "token/",
Type: "token",
Description: "token based credentials",
}
meUUID, err := uuid.GenerateUUID()
if err != nil {
t.Fatal(err)
}
me.UUID = meUUID
view := NewBarrierView(c.barrier, credentialBarrierPrefix+me.UUID+"/")
sysView := c.mountEntrySysView(me)
tokenstore, _ := c.newCredentialBackend(context.Background(), me, sysView, view)
ts := tokenstore.(*TokenStore)
err = c.router.Unmount(context.Background(), "auth/token/")
if err != nil {
t.Fatal(err)
}
err = c.router.Mount(ts, "auth/token/", &MountEntry{Table: credentialTableType, UUID: "authtokenuuid", Path: "auth/token", Accessor: "authtokenaccessor"}, ts.view)
if err != nil {
t.Fatal(err)
}
ts.SetExpirationManager(c.expiration)
return ts
}
// TestCoreWithTokenStore returns an in-memory core that has a token store
// mounted, so that logical token functions can be used
func TestCoreWithTokenStore(t testing.T) (*Core, *TokenStore, [][]byte, string) {
c, keys, root := TestCoreUnsealed(t)
return c, c.tokenStore, keys, root
}
// TestCoreWithBackendTokenStore returns a core that has a token store
// mounted and used the provided physical backend, so that logical token
// functions can be used
func TestCoreWithBackendTokenStore(t testing.T, backend physical.Backend) (*Core, *TokenStore, [][]byte, string) {
c, keys, root := TestCoreUnsealedBackend(t, backend)
ts := testTokenStore(t, c)
return c, ts, keys, root
}
// TestKeyCopy is a silly little function to just copy the key so that
// it can be used with Unseal easily.
func TestKeyCopy(key []byte) []byte {
@ -1205,6 +1152,7 @@ func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *Te
ClusterAddr: fmt.Sprintf("https://127.0.0.1:%d", listeners[0][0].Address.Port+105),
DisableMlock: true,
EnableUI: true,
EnableRaw: true,
}
if base != nil {
@ -1216,6 +1164,7 @@ func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *Te
coreConfig.PluginDirectory = base.PluginDirectory
coreConfig.Seal = base.Seal
coreConfig.DevToken = base.DevToken
coreConfig.EnableRaw = base.EnableRaw
if !coreConfig.DisableMlock {
base.DisableMlock = false

View File

@ -1028,19 +1028,6 @@ func (ts *TokenStore) lookupSalted(ctx context.Context, saltedID string, tainted
return nil, nil
}
// If we are still restoring the expiration manager, we want to ensure the
// token is not expired
if ts.expiration == nil {
return nil, errors.New("expiration manager is nil on tokenstore")
}
check, err := ts.expiration.RestoreSaltedTokenCheck(entry.Path, saltedID)
if err != nil {
return nil, errwrap.Wrapf("failed to check token in restore mode: {{err}}", err)
}
if !check {
return nil, nil
}
persistNeeded := false
// Upgrade the deprecated fields
@ -1076,6 +1063,48 @@ func (ts *TokenStore) lookupSalted(ctx context.Context, saltedID string, tainted
persistNeeded = true
}
// Perform these checks on upgraded fields, but before persisting
// If we are still restoring the expiration manager, we want to ensure the
// token is not expired
if ts.expiration == nil {
return nil, errors.New("expiration manager is nil on tokenstore")
}
le, err := ts.expiration.FetchLeaseTimesByToken(entry.Path, entry.ID)
if err != nil {
return nil, errwrap.Wrapf("failed to fetch lease times: {{err}}", err)
}
var ret *TokenEntry
switch {
// It's a root token with unlimited creation TTL (so never had an
// expiration); this may or may not have a lease (based on when it was
// generated, for later revocation purposes) but it doesn't matter, it's
// allowed
case len(entry.Policies) == 1 && entry.Policies[0] == "root" && entry.TTL == 0:
ret = entry
// It's any kind of expiring token with no lease, immediately delete it
case le == nil:
leaseID, err := ts.expiration.CreateOrFetchRevocationLeaseByToken(entry)
if err != nil {
return nil, err
}
err = ts.expiration.Revoke(leaseID)
if err != nil {
return nil, err
}
// Only return if we're not past lease expiration (or if tainted is true),
// otherwise assume expmgr is working on revocation
default:
if !le.ExpireTime.Before(time.Now()) || tainted {
ret = entry
}
}
// If fields are getting upgraded, store the changes
if persistNeeded {
if err := ts.store(ctx, entry); err != nil {
@ -1083,7 +1112,7 @@ func (ts *TokenStore) lookupSalted(ctx context.Context, saltedID string, tainted
}
}
return entry, nil
return ret, nil
}
// Revoke is used to invalidate a given token, any child tokens
@ -2070,6 +2099,7 @@ func (ts *TokenStore) handleCreateCommon(ctx context.Context, req *logical.Reque
EntityID: te.EntityID,
Period: periodToUse,
ExplicitMaxTTL: explicitMaxTTLToUse,
CreationPath: te.Path,
}
if ts.policyLookupFunc != nil {

View File

@ -1,13 +1,16 @@
package vault_test
import (
"encoding/base64"
"reflect"
"sort"
"strings"
"testing"
"time"
"github.com/hashicorp/vault/api"
credLdap "github.com/hashicorp/vault/builtin/credential/ldap"
"github.com/hashicorp/vault/helper/jsonutil"
vaulthttp "github.com/hashicorp/vault/http"
"github.com/hashicorp/vault/logical"
"github.com/hashicorp/vault/vault"
@ -336,3 +339,154 @@ func TestTokenStore_CIDRBlocks(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTokenStore_RevocationOnStartup(t *testing.T) {
cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
HandlerFunc: vaulthttp.Handler,
NumCores: 1,
})
cluster.Start()
defer cluster.Cleanup()
core := cluster.Cores[0].Core
vault.TestWaitActive(t, core)
client := cluster.Cores[0].Client
rootToken := client.Token()
type leaseEntry struct {
LeaseID string `json:"lease_id"`
ClientToken string `json:"client_token"`
Path string `json:"path"`
Data map[string]interface{} `json:"data"`
Secret *logical.Secret `json:"secret"`
Auth *logical.Auth `json:"auth"`
IssueTime time.Time `json:"issue_time"`
ExpireTime time.Time `json:"expire_time"`
LastRenewalTime time.Time `json:"last_renewal_time"`
}
var secret *api.Secret
var err error
var tokens []string
// Create tokens
for i := 0; i < 500; i++ {
secret, err = client.Auth().Token().Create(&api.TokenCreateRequest{
Policies: []string{"default"},
})
if err != nil {
t.Fatal(err)
}
tokens = append(tokens, secret.Auth.ClientToken)
}
const tokenPath string = "sys/raw/sys/token/id/"
secret, err = client.Logical().List(tokenPath)
if err != nil {
t.Fatal(err)
}
totalTokens := len(secret.Data["keys"].([]interface{}))
// Get the list of leases
const leasePath string = "sys/raw/sys/expire/id/auth/token/create/"
secret, err = client.Logical().List(leasePath)
if err != nil {
t.Fatal(err)
}
leases := secret.Data["keys"].([]interface{})
if len(leases) != 500 {
t.Fatalf("unexpected number of leases: %d", len(leases))
}
// Holds non-root leases
var validLeases []string
// Fake times in the past
for _, lease := range leases {
secret, err = client.Logical().Read(leasePath + lease.(string))
var entry leaseEntry
if err := jsonutil.DecodeJSON([]byte(secret.Data["value"].(string)), &entry); err != nil {
t.Fatal(err)
}
if entry.ExpireTime.IsZero() {
continue
}
validLeases = append(validLeases, lease.(string))
entry.IssueTime = entry.IssueTime.Add(-1 * time.Hour * 24 * 365)
entry.ExpireTime = entry.ExpireTime.Add(-1 * time.Hour * 24 * 365)
jsonEntry, err := jsonutil.EncodeJSON(&entry)
if err != nil {
t.Fatal(err)
}
if _, err := client.Logical().Write(leasePath+lease.(string), map[string]interface{}{
"value": string(jsonEntry),
}); err != nil {
t.Fatal(err)
}
}
if err := client.Sys().Seal(); err != nil {
t.Fatal(err)
}
var status *api.SealStatusResponse
for i := 0; i < len(cluster.BarrierKeys); i++ {
status, err = client.Sys().Unseal(string(base64.StdEncoding.EncodeToString(cluster.BarrierKeys[i])))
if err != nil {
t.Fatal(err)
}
if !status.Sealed {
break
}
}
if status.Sealed {
t.Fatal("did not unseal properly")
}
// Give lease loading some time to process
time.Sleep(5 * time.Second)
for i, token := range tokens {
client.SetToken(token)
_, err := client.Logical().Write("cubbyhole/foo", map[string]interface{}{
"value": "bar",
})
if err == nil {
t.Errorf("expected error but did not get one, token num %d", i)
}
}
expectedLeases := len(leases) - len(validLeases)
client.SetToken(rootToken)
secret, err = client.Logical().List(leasePath)
if err != nil {
t.Fatal(err)
}
switch {
case secret == nil:
if expectedLeases != 0 {
t.Fatalf("nil secret back but expected %d leases", expectedLeases)
}
case secret.Data == nil:
if expectedLeases != 0 {
t.Fatalf("nil secret data back but expected %d leases, secret is %#v", expectedLeases, *secret)
}
default:
leasesLeft := len(secret.Data["keys"].([]interface{}))
if leasesLeft != expectedLeases {
t.Fatalf("found %d leases left, expected %d", leasesLeft, expectedLeases)
}
}
expectedTokens := totalTokens - len(validLeases)
secret, err = client.Logical().List(tokenPath)
if err != nil {
t.Fatal(err)
}
tokensLeft := len(secret.Data["keys"].([]interface{}))
if tokensLeft != expectedTokens {
t.Fatalf("found %d tokens left, expected %d", tokensLeft, expectedTokens)
}
}

File diff suppressed because it is too large Load Diff