diff --git a/nomad/vault.go b/nomad/vault.go index 10b5a6062..ae0ba953a 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/rand" + "strconv" "sync" "sync/atomic" "time" @@ -140,6 +141,12 @@ type VaultStats struct { // TrackedForRevoke is the count of tokens that are being tracked to be // revoked since they could not be immediately revoked. TrackedForRevoke int + + // TokenTTL is the time-to-live duration for the current token + TokenTTL time.Duration + + // TokenExpiry Time is the recoreded expiry time of the current token + TokenExpiry time.Time } // PurgeVaultAccessor is called to remove VaultAccessors from the system. If @@ -203,8 +210,8 @@ type vaultClient struct { // childTTL is the TTL for child tokens. childTTL string - // lastRenewed is the time the token was last renewed - lastRenewed time.Time + // currentExpiration is the time the current tokean lease expires + currentExpiration time.Time tomb *tomb.Tomb logger log.Logger @@ -469,13 +476,11 @@ func (v *vaultClient) renewalLoop() { case <-authRenewTimer.C: // Renew the token and determine the new expiration err := v.renew() - currentExpiration := v.lastRenewed.Add(time.Duration(v.tokenData.CreationTTL) * time.Second) + currentExpiration := v.currentExpiration // Successfully renewed if err == nil { - // If we take the expiration (lastRenewed + auth duration) and - // subtract the current time, we get a duration until expiry. - // Set the timer to poke us after half of that time is up. + // Attempt to renew the token at half the expiration time durationUntilRenew := currentExpiration.Sub(time.Now()) / 2 v.logger.Info("successfully renewed token", "next_renewal", durationUntilRenew) @@ -548,7 +553,7 @@ func nextBackoff(backoff float64, expiry time.Time) float64 { } // renew attempts to renew our Vault token. If the renewal fails, an error is -// returned. This method updates the lastRenewed time +// returned. This method updates the currentExpiration time func (v *vaultClient) renew() error { // Track how long the request takes defer metrics.MeasureSince([]string{"nomad", "vault", "renew"}, time.Now()) @@ -571,7 +576,8 @@ func (v *vaultClient) renew() error { return fmt.Errorf("renewal successful but no lease duration returned") } - v.lastRenewed = time.Now() + v.currentExpiration = time.Now().Add(time.Duration(auth.LeaseDuration) * time.Second) + v.logger.Debug("successfully renewed server token") return nil } @@ -618,7 +624,6 @@ func (v *vaultClient) parseSelfToken() error { if err := mapstructure.WeakDecode(self.Data, &data); err != nil { return fmt.Errorf("failed to parse Vault token's data block: %v", err) } - root := false for _, p := range data.Policies { if p == "root" { @@ -626,10 +631,9 @@ func (v *vaultClient) parseSelfToken() error { break } } - - // Store the token data data.Root = root v.tokenData = &data + v.currentExpiration = time.Now().Add(time.Duration(data.TTL) * time.Second) // The criteria that must be met for the token to be valid are as follows: // 1) If token is non-root or is but has a creation ttl @@ -648,7 +652,7 @@ func (v *vaultClient) parseSelfToken() error { var mErr multierror.Error role := v.getRole() - if !root { + if !data.Root { // All non-root tokens must be renewable if !data.Renewable { multierror.Append(&mErr, fmt.Errorf("Vault token is not renewable or root")) @@ -680,7 +684,7 @@ func (v *vaultClient) parseSelfToken() error { } // Check we have the correct capabilities - if err := v.validateCapabilities(role, root); err != nil { + if err := v.validateCapabilities(role, data.Root); err != nil { multierror.Append(&mErr, err) } @@ -1206,8 +1210,23 @@ func (v *vaultClient) setLimit(l rate.Limit) { v.limiter = rate.NewLimiter(l, int(l)) } -// Stats is used to query the state of the blocked eval tracker. -func (v *vaultClient) Stats() *VaultStats { +func (v *vaultClient) Stats() map[string]string { + stat := v.stats() + + expireTimeStr := "" + + if !stat.TokenExpiry.IsZero() { + expireTimeStr = stat.TokenExpiry.Format(time.RFC3339) + } + + return map[string]string{ + "tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke), + "token_ttl": stat.TokenTTL.String(), + "token_expire_time": expireTimeStr, + } +} + +func (v *vaultClient) stats() *VaultStats { // Allocate a new stats struct stats := new(VaultStats) @@ -1215,6 +1234,11 @@ func (v *vaultClient) Stats() *VaultStats { stats.TrackedForRevoke = len(v.revoking) v.revLock.Unlock() + stats.TokenExpiry = v.currentExpiration + if !stats.TokenExpiry.IsZero() { + stats.TokenTTL = time.Until(stats.TokenExpiry) + } + return stats } @@ -1225,6 +1249,8 @@ func (v *vaultClient) EmitStats(period time.Duration, stopCh chan struct{}) { case <-time.After(period): stats := v.stats() metrics.SetGauge([]string{"nomad", "vault", "distributed_tokens_revoking"}, float32(stats.TrackedForRevoke)) + metrics.SetGauge([]string{"nomad", "vault", "token_ttl"}, float32(stats.TokenTTL/time.Millisecond)) + case <-stopCh: return } diff --git a/nomad/vault_test.go b/nomad/vault_test.go index 3de2e079f..67dcc0b1d 100644 --- a/nomad/vault_test.go +++ b/nomad/vault_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "golang.org/x/time/rate" "github.com/hashicorp/nomad/helper" @@ -528,6 +530,49 @@ func TestVaultClient_RenewalLoop(t *testing.T) { if ttl == 0 { t.Fatalf("token renewal failed; ttl %v", ttl) } + + if client.currentExpiration.Before(time.Now()) { + t.Fatalf("found current expiration to be in past %s", time.Until(client.currentExpiration)) + } +} + +func TestVaultClientRenewUpdatesExpiration(t *testing.T) { + t.Parallel() + v := testutil.NewTestVault(t) + defer v.Stop() + + // Set the configs token in a new test role + v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) + + // Start the client + logger := testlog.HCLogger(t) + client, err := NewVaultClient(v.Config, logger, nil) + if err != nil { + t.Fatalf("failed to build vault client: %v", err) + } + defer client.Stop() + + // Get the current TTL + a := v.Client.Auth().Token() + s2, err := a.Lookup(v.Config.Token) + if err != nil { + t.Fatalf("failed to lookup token: %v", err) + } + exp0 := time.Now().Add(time.Duration(parseTTLFromLookup(s2, t)) * time.Second) + + time.Sleep(1 * time.Second) + + err = client.renew() + require.NoError(t, err) + exp1 := client.currentExpiration + require.True(t, exp0.Before(exp1)) + + time.Sleep(1 * time.Second) + + err = client.renew() + require.NoError(t, err) + exp2 := client.currentExpiration + require.True(t, exp1.Before(exp2)) } func parseTTLFromLookup(s *vapi.Secret, t *testing.T) int64 { @@ -1114,7 +1159,7 @@ func TestVaultClient_RevokeTokens_PreEstablishs(t *testing.T) { t.Fatalf("didn't add to revoke loop") } - if client.Stats().TrackedForRevoke != 2 { + if client.stats().TrackedForRevoke != 2 { t.Fatalf("didn't add to revoke loop") } }