open-nomad/helper/exptime/backoff.go

// Package exptime provides a generalized exponential backoff retry implementation.
//
// This package was copied from oss.indeed.com/go/libtime/decay and modified.
package exptime

import (
	"errors"
	"fmt"
	"math/rand"
	"time"
)

var (
	// ErrMaximumTimeExceeded indicates the maximum wait time has been exceeded.
	ErrMaximumTimeExceeded = errors.New("maximum backoff time exceeded")
)

// A TryFunc is what gets executed between retry wait periods during execution
// of Backoff. The keepRetrying return value is used to control whether a retry
// attempt should be made. This feature is useful in manipulating control flow
// in cases where it is known a retry will not be successful.
type TryFunc func() (keepRetrying bool, err error)

// BackoffOptions allow for fine-tuning backoff behavior.
type BackoffOptions struct {
	// MaxSleepTime represents the maximum amount of time
	// the exponential backoff system will spend sleeping,
	// accumulating the amount of time spent asleep between
	// retries.
	//
	// The algorithm starts at an interval of InitialGapSize
	// and increases exponentially (x2 each iteration) from there.
	// With no jitter, a MaxSleepTime of 10 seconds and InitialGapSize
	// of 1 millisecond would suggest a total of 15 attempts
	// (since the very last retry truncates the sleep time to
	// align exactly with MaxSleepTime).
	MaxSleepTime time.Duration

	// InitialGapSize sets the initial amount of time the algorithm
	// will sleep before the first retry (after the first attempt).
	// The actual amount of sleep time will include a random amount
	// of jitter, if MaxJitterSize is non-zero.
	InitialGapSize time.Duration

	// MaxJitterSize limits how much randomness we may
	// introduce in the duration of each retry interval.
	// The purpose of introducing jitter is to mitigate the
	// effect of thundering herds
	MaxJitterSize time.Duration

	// RandomSeed is used for generating a randomly computed
	// jitter size for each retry.
	RandomSeed int64

	// Sleeper is used to cause the process to sleep for
	// a computed amount of time. If not set, a default
	// implementation based on time.Sleep will be used.
	Sleeper Sleeper
}

// A Sleeper is a useful way for calling time.Sleep
// in a mock-able way for tests.
type Sleeper func(time.Duration)

// Backoff will attempt to execute function using a configurable
// exponential backoff algorithm. function is a TryFunc which requires
// two return parameters - a boolean for optimizing control flow, and
// an error for reporting failure conditions. If the first parameter is
// false, the backoff algorithm will abandon further retry attempts and
// simply return an error. Otherwise, if the returned error is non-nil, the
// backoff algorithm will sleep for an increasing amount of time, and
// then retry again later, until the maximum amount of sleep time has
// been consumed. Once function has executed successfully with no error,
// the backoff algorithm returns a nil error.
func Backoff(function TryFunc, options BackoffOptions) error {
	if options.MaxSleepTime <= 0 {
		panic("max sleep time must be > 0")
	}

	if options.InitialGapSize <= 0 {
		panic("initial gap size must be > 0")
	}

	if options.MaxJitterSize < 0 {
		panic("max jitter size must be >= 0")
	}

	if options.MaxJitterSize > (options.MaxSleepTime / 2) {
		panic("max jitter size is way too large")
	}

	if options.Sleeper == nil {
		options.Sleeper = time.Sleep
	}

	consumed := time.Duration(0)
	gap := options.InitialGapSize
	random := rand.New(rand.NewSource(options.RandomSeed))

	for consumed < options.MaxSleepTime {
		keepRetrying, err := function()
		if err != nil && !keepRetrying {
			return fmt.Errorf("exponential backoff instructed to stop retrying: %w", err)
		}

		// we can ignore keepRetrying at this point, since we know
		// what to do based on err
		if err == nil {
			return nil // success
		}

		// there was an error, and function wants to keep retrying
		// we will sleep, and then let the loop continue
		//
		// (random.Float64 returns a value [0.0, 1.0), which is used to
		// randomly scale the jitter from 0 to MaxJitterSize.
		jitter := nextJitter(random.Float64(), options.MaxJitterSize)
		duration := gap + jitter

		if (duration + consumed) > options.MaxSleepTime {
			// this will be our last try, force the duration
			// to line up with the maximum sleep time
			duration = options.MaxSleepTime - consumed
		}

		// sleep for the configured duration
		options.Sleeper(duration)

		// account for how long we intended to sleep
		consumed += duration

		// exponentially increase the gap
		gap *= 2
	}

	return ErrMaximumTimeExceeded
}

func nextJitter(fraction float64, maxSize time.Duration) time.Duration {
	scaled := fraction * float64(maxSize)
	return time.Duration(scaled)
}