165 lines
3.6 KiB
Go
165 lines
3.6 KiB
Go
package nomad
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hashicorp/go-hclog"
|
|
"github.com/hashicorp/nomad/ci"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TesCachedtBadNodeTracker(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
config := DefaultCachedBadNodeTrackerConfig()
|
|
config.CacheSize = 3
|
|
|
|
tracker, err := NewCachedBadNodeTracker(hclog.NewNullLogger(), config)
|
|
require.NoError(t, err)
|
|
|
|
for i := 0; i < 10; i++ {
|
|
tracker.Add(fmt.Sprintf("node-%d", i+1))
|
|
}
|
|
|
|
require.Equal(t, config.CacheSize, tracker.cache.Len())
|
|
|
|
// Only track the most recent values.
|
|
expected := []string{"node-8", "node-9", "node-10"}
|
|
require.ElementsMatch(t, expected, tracker.cache.Keys())
|
|
}
|
|
|
|
func TestCachedBadNodeTracker_isBad(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
config := DefaultCachedBadNodeTrackerConfig()
|
|
config.CacheSize = 3
|
|
config.Threshold = 4
|
|
|
|
tracker, err := NewCachedBadNodeTracker(hclog.NewNullLogger(), config)
|
|
require.NoError(t, err)
|
|
|
|
// Populate cache.
|
|
tracker.Add("node-1")
|
|
|
|
tracker.Add("node-2")
|
|
tracker.Add("node-2")
|
|
|
|
tracker.Add("node-3")
|
|
tracker.Add("node-3")
|
|
tracker.Add("node-3")
|
|
tracker.Add("node-3")
|
|
tracker.Add("node-3")
|
|
tracker.Add("node-3")
|
|
|
|
testCases := []struct {
|
|
name string
|
|
nodeID string
|
|
bad bool
|
|
}{
|
|
{
|
|
name: "node-1 is not bad",
|
|
nodeID: "node-1",
|
|
bad: false,
|
|
},
|
|
{
|
|
name: "node-3 is bad",
|
|
nodeID: "node-3",
|
|
bad: true,
|
|
},
|
|
}
|
|
|
|
now := time.Now()
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
// Read value from cached.
|
|
v, ok := tracker.cache.Get(tc.nodeID)
|
|
require.True(t, ok)
|
|
|
|
// Check if it's bad.
|
|
stats := v.(*badNodeStats)
|
|
got := tracker.isBad(now, stats)
|
|
require.Equal(t, tc.bad, got)
|
|
})
|
|
}
|
|
|
|
future := time.Now().Add(2 * config.Window)
|
|
nodes := []string{"node-1", "node-2", "node-3"}
|
|
for _, n := range nodes {
|
|
t.Run(fmt.Sprintf("%s cache expires", n), func(t *testing.T) {
|
|
v, ok := tracker.cache.Get(n)
|
|
require.True(t, ok)
|
|
|
|
stats := v.(*badNodeStats)
|
|
bad := tracker.isBad(future, stats)
|
|
require.False(t, bad)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TesCachedtBadNodeTracker_rateLimit(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
config := DefaultCachedBadNodeTrackerConfig()
|
|
config.Threshold = 3
|
|
config.RateLimit = float64(1) // Get a new token every second.
|
|
config.BurstSize = 3
|
|
|
|
tracker, err := NewCachedBadNodeTracker(hclog.NewNullLogger(), config)
|
|
require.NoError(t, err)
|
|
|
|
tracker.Add("node-1")
|
|
tracker.Add("node-1")
|
|
tracker.Add("node-1")
|
|
tracker.Add("node-1")
|
|
tracker.Add("node-1")
|
|
|
|
v, ok := tracker.cache.Get("node-1")
|
|
require.True(t, ok)
|
|
|
|
stats := v.(*badNodeStats)
|
|
|
|
// Burst allows for max 3 operations.
|
|
now := time.Now()
|
|
require.True(t, tracker.isBad(now, stats))
|
|
require.True(t, tracker.isBad(now, stats))
|
|
require.True(t, tracker.isBad(now, stats))
|
|
require.False(t, tracker.isBad(now, stats))
|
|
|
|
// Wait for a new token.
|
|
time.Sleep(time.Second)
|
|
now = time.Now()
|
|
require.True(t, tracker.isBad(now, stats))
|
|
require.False(t, tracker.isBad(now, stats))
|
|
}
|
|
|
|
func TestBadNodeStats_score(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
window := time.Minute
|
|
stats := newBadNodeStats("node-1", window)
|
|
|
|
now := time.Now()
|
|
require.Equal(t, 0, stats.score(now))
|
|
|
|
stats.record(now)
|
|
stats.record(now)
|
|
stats.record(now)
|
|
require.Equal(t, 3, stats.score(now))
|
|
require.Len(t, stats.history, 3)
|
|
|
|
halfWindow := now.Add(window / 2)
|
|
stats.record(halfWindow)
|
|
require.Equal(t, 4, stats.score(halfWindow))
|
|
require.Len(t, stats.history, 4)
|
|
|
|
fullWindow := now.Add(window).Add(time.Second)
|
|
require.Equal(t, 1, stats.score(fullWindow))
|
|
require.Len(t, stats.history, 1)
|
|
|
|
afterWindow := now.Add(2 * window)
|
|
require.Equal(t, 0, stats.score(afterWindow))
|
|
require.Len(t, stats.history, 0)
|
|
}
|