From 0ad63e5a20b62d22b97f17dd5d4acd8587f27fb1 Mon Sep 17 00:00:00 2001 From: Brian Kassouf Date: Thu, 18 Feb 2021 20:20:01 -0800 Subject: [PATCH] core/expiration: Add backoff jitter to the expiration retries (#10937) --- vault/expiration.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vault/expiration.go b/vault/expiration.go index 78f9f9476..43e521eb5 100644 --- a/vault/expiration.go +++ b/vault/expiration.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math/rand" "os" "path" "strconv" @@ -218,8 +219,7 @@ func (r *revocationJob) OnFailure(err error) { return } - // TODO vault 1.8 we added an exponential backoff library, check to see if it would be useful here - pending.timer.Reset((1 << pending.revokesAttempted) * revokeRetryBase) + pending.timer.Reset(revokeExponentialBackoff(pending.revokesAttempted)) r.m.pending.Store(r.leaseID, pending) } @@ -248,6 +248,15 @@ func expireLeaseStrategyFairsharing(ctx context.Context, m *ExpirationManager, l m.jobManager.AddJob(job, mountAccessor) } +func revokeExponentialBackoff(attempt uint8) time.Duration { + exp := (1 << attempt) * revokeRetryBase + randomDelta := 0.5 * float64(exp) + + // Allow backoff time to be a random value between exp +/- (0.5*exp) + backoffTime := (float64(exp) - randomDelta) + (rand.Float64() * (2 * randomDelta)) + return time.Duration(backoffTime) +} + // revokeIDFunc is invoked when a given ID is expired func expireLeaseStrategyRevoke(ctx context.Context, m *ExpirationManager, leaseID string, ns *namespace.Namespace) { for attempt := uint(0); attempt < maxRevokeAttempts; attempt++ {