0a19fe3b60
We use capped exponential backoff in several places in the code when handling failures. The code we've copy-and-pasted all over has a check to see if the backoff is greater than the limit, but this check happens after the bitshift and we always increment the number of attempts. This causes an overflow with a fairly small number of failures (ex. at one place I tested it occurs after only 24 iterations), resulting in a negative backoff which then never recovers. The backoff becomes a tight loop consuming resources and/or DoS'ing a Nomad RPC handler or an external API such as Vault. Note this doesn't occur in places where we cap the number of iterations so the loop breaks (usually to return an error), so long as the number of iterations is reasonable. Introduce a helper with a check on the cap before the bitshift to avoid overflow in all places this can occur. Fixes: #18199 Co-authored-by: stswidwinski <stan.swidwinski@gmail.com>
73 lines
1.6 KiB
Go
73 lines
1.6 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: BUSL-1.1
|
|
|
|
package helper
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/shoenig/test/must"
|
|
)
|
|
|
|
func Test_Backoff(t *testing.T) {
|
|
const MaxUint = ^uint64(0)
|
|
const MaxInt = int64(MaxUint >> 1)
|
|
|
|
cases := []struct {
|
|
name string
|
|
backoffBase time.Duration
|
|
backoffLimit time.Duration
|
|
attempt uint64
|
|
expectedResult time.Duration
|
|
}{
|
|
{
|
|
name: "backoff limit clamps for high base",
|
|
backoffBase: time.Hour,
|
|
backoffLimit: time.Minute,
|
|
attempt: 1,
|
|
expectedResult: time.Minute,
|
|
},
|
|
{
|
|
name: "backoff limit clamps for boundary attempt",
|
|
backoffBase: time.Hour,
|
|
backoffLimit: time.Minute,
|
|
attempt: 63,
|
|
expectedResult: time.Minute,
|
|
},
|
|
{
|
|
name: "small retry value",
|
|
backoffBase: time.Minute,
|
|
backoffLimit: time.Hour,
|
|
attempt: 0,
|
|
expectedResult: time.Minute,
|
|
},
|
|
{
|
|
name: "first retry value",
|
|
backoffBase: time.Minute,
|
|
backoffLimit: time.Hour,
|
|
attempt: 1,
|
|
expectedResult: 2 * time.Minute,
|
|
},
|
|
{
|
|
name: "fifth retry value",
|
|
backoffBase: time.Minute,
|
|
backoffLimit: time.Hour,
|
|
attempt: 5,
|
|
expectedResult: 32 * time.Minute,
|
|
},
|
|
{
|
|
name: "sixth retry value",
|
|
backoffBase: time.Minute,
|
|
backoffLimit: time.Hour,
|
|
attempt: 6,
|
|
expectedResult: time.Hour,
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
result := Backoff(tc.backoffBase, tc.backoffLimit, tc.attempt)
|
|
must.Eq(t, tc.expectedResult, result)
|
|
}
|
|
}
|