diff --git a/.changelog/18838.txt b/.changelog/18838.txt new file mode 100644 index 000000000..5d8f8bb82 --- /dev/null +++ b/.changelog/18838.txt @@ -0,0 +1,3 @@ +```release-note:bug +scheduler (Enterprise): auto-unblock evals with associated quotas when node resources are freed up +``` diff --git a/nomad/blocked_evals.go b/nomad/blocked_evals.go index 11b244e4c..1971f5d92 100644 --- a/nomad/blocked_evals.go +++ b/nomad/blocked_evals.go @@ -567,10 +567,13 @@ func (b *BlockedEvals) unblock(computedClass, quota string, index uint64) { // never saw a node with the given computed class and thus needs to be // unblocked for correctness. for id, wrapped := range b.captured { - if quota != "" && wrapped.eval.QuotaLimitReached != quota { + if quota != "" && + wrapped.eval.QuotaLimitReached != "" && + wrapped.eval.QuotaLimitReached != quota { // We are unblocking based on quota and this eval doesn't match continue - } else if elig, ok := wrapped.eval.ClassEligibility[computedClass]; ok && !elig { + } + if elig, ok := wrapped.eval.ClassEligibility[computedClass]; ok && !elig { // Can skip because the eval has explicitly marked the node class // as ineligible. continue diff --git a/nomad/blocked_evals_test.go b/nomad/blocked_evals_test.go index 09a906e71..f1fc1bd2f 100644 --- a/nomad/blocked_evals_test.go +++ b/nomad/blocked_evals_test.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test/must" "github.com/stretchr/testify/require" ) @@ -322,6 +323,33 @@ func TestBlockedEvals_UnblockEligible_Quota(t *testing.T) { requireBlockedEvalsEnqueued(t, blocked, broker, 1) } +// The quota here is incidental. The eval is blocked due to something else, +// e.g. cpu exhausted, but there happens to also be a quota on the namespace. +func TestBlockedEvals_UnblockEligible_IncidentalQuota(t *testing.T) { + ci.Parallel(t) + + blocked, broker := testBlockedEvals(t) + + e := mock.BlockedEval() + e.Status = structs.EvalStatusBlocked + e.QuotaLimitReached = "" // explicitly not blocked due to quota limit + blocked.Block(e) + + // Verify block caused the eval to be tracked. + blockedStats := blocked.Stats() + must.Eq(t, 1, blockedStats.TotalBlocked) + must.MapLen(t, 1, blockedStats.BlockedResources.ByJob) + // but not due to quota. + must.Eq(t, 0, blockedStats.TotalQuotaLimit) + + // When unblocking, the quota name from the alloc is passed in, + // regardless of the cause of the initial blockage. + // Since the initial block in this test was due to something else, + // it should be unblocked without regard to quota. + blocked.UnblockQuota("foo", 1000) + requireBlockedEvalsEnqueued(t, blocked, broker, 1) +} + func TestBlockedEvals_UnblockIneligible_Quota(t *testing.T) { ci.Parallel(t) require := require.New(t)