Add failures_before_warning to checks (#10969)

Signed-off-by: Jakub Sokołowski <jakub@status.im>

* agent: add failures_before_warning setting

The new setting allows users to specify the number of check failures
that have to happen before a service status us updated to be `warning`.
This allows for more visibility for detected issues without creating
alerts and pinging administrators. Unlike the previous behavior, which
caused the service status to not update until it reached the configured
`failures_before_critical` setting, now Consul updates the Web UI view
with the `warning` state and the output of the service check when
`failures_before_warning` is breached.

The default value of `FailuresBeforeWarning` is the same as the value of
`FailuresBeforeCritical`, which allows for retaining the previous default
behavior of not triggering a warning.

When `FailuresBeforeWarning` is set to a value higher than that of
`FailuresBeforeCritical it has no effect as `FailuresBeforeCritical`
takes precedence.

Resolves: https://github.com/hashicorp/consul/issues/10680

Signed-off-by: Jakub Sokołowski <jakub@status.im>

Co-authored-by: Jakub Sokołowski <jakub@status.im>

.changelog/10969.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+checks: add failures_before_warning setting for interval checks.
+```
+

agent/agent.go

@@ -2459,6 +2459,11 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType,
 			maxOutputSize = chkType.OutputMaxSize
 		}
 
+		// FailuresBeforeWarning has to default to same value as FailuresBeforeCritical
+		if chkType.FailuresBeforeWarning == 0 {
+			chkType.FailuresBeforeWarning = chkType.FailuresBeforeCritical
+		}
+
 		// Get the address of the proxy for this service if it exists
 		// Need its config to know whether we should reroute checks to it
 		var proxy *structs.NodeService
@@ -2473,7 +2478,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType,
 			}
 		}
 
-		statusHandler := checks.NewStatusHandler(a.State, a.logger, chkType.SuccessBeforePassing, chkType.FailuresBeforeCritical)
+		statusHandler := checks.NewStatusHandler(a.State, a.logger, chkType.SuccessBeforePassing, chkType.FailuresBeforeWarning, chkType.FailuresBeforeCritical)
 		sid := check.CompoundServiceID()
 
 		cid := check.CompoundCheckID()

agent/checks/check.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	http2 "golang.org/x/net/http2"
 	"io"
 	"io/ioutil"
 	"net"
@@ -16,6 +15,8 @@ import (
 	"syscall"
 	"time"
 
+	http2 "golang.org/x/net/http2"
+
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/go-hclog"
 
@@ -907,17 +908,19 @@ type StatusHandler struct {
 	logger                 hclog.Logger
 	successBeforePassing   int
 	successCounter         int
+	failuresBeforeWarning  int
 	failuresBeforeCritical int
 	failuresCounter        int
 }
 
 // NewStatusHandler set counters values to threshold in order to immediatly update status after first check.
-func NewStatusHandler(inner CheckNotifier, logger hclog.Logger, successBeforePassing, failuresBeforeCritical int) *StatusHandler {
+func NewStatusHandler(inner CheckNotifier, logger hclog.Logger, successBeforePassing, failuresBeforeWarning, failuresBeforeCritical int) *StatusHandler {
 	return &StatusHandler{
 		logger:                 logger,
 		inner:                  inner,
 		successBeforePassing:   successBeforePassing,
 		successCounter:         successBeforePassing,
+		failuresBeforeWarning:  failuresBeforeWarning,
 		failuresBeforeCritical: failuresBeforeCritical,
 		failuresCounter:        failuresBeforeCritical,
 	}
@@ -950,10 +953,17 @@ func (s *StatusHandler) updateCheck(checkID structs.CheckID, status, output stri
 			s.inner.UpdateCheck(checkID, status, output)
 			return
 		}
-		s.logger.Warn("Check failed but has not reached failure threshold",
+		// Defaults to same value as failuresBeforeCritical if not set.
+		if s.failuresCounter >= s.failuresBeforeWarning {
+			s.logger.Warn("Check is now warning", "check", checkID.String())
+			s.inner.UpdateCheck(checkID, api.HealthWarning, output)
+			return
+		}
+		s.logger.Warn("Check failed but has not reached warning/failure threshold",
 			"check", checkID.String(),
 			"status", status,
 			"failure_count", s.failuresCounter,
+			"warning_threshold", s.failuresBeforeWarning,
 			"failure_threshold", s.failuresBeforeCritical,
 		)
 	}

agent/checks/check_test.go

@@ -49,7 +49,7 @@ func TestCheckMonitor_Script(t *testing.T) {
 		t.Run(tt.status, func(t *testing.T) {
 			notif := mock.NewNotify()
 			logger := testutil.Logger(t)
-			statusHandler := NewStatusHandler(notif, logger, 0, 0)
+			statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 
 			cid := structs.NewCheckID("foo", nil)
 			check := &CheckMonitor{
@@ -94,7 +94,7 @@ func TestCheckMonitor_Args(t *testing.T) {
 		t.Run(tt.status, func(t *testing.T) {
 			notif := mock.NewNotify()
 			logger := testutil.Logger(t)
-			statusHandler := NewStatusHandler(notif, logger, 0, 0)
+			statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 			cid := structs.NewCheckID("foo", nil)
 
 			check := &CheckMonitor{
@@ -128,7 +128,7 @@ func TestCheckMonitor_Timeout(t *testing.T) {
 	// t.Parallel() // timing test. no parallel
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 
 	cid := structs.NewCheckID("foo", nil)
 	check := &CheckMonitor{
@@ -163,7 +163,7 @@ func TestCheckMonitor_RandomStagger(t *testing.T) {
 	// t.Parallel() // timing test. no parallel
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 
 	cid := structs.NewCheckID("foo", nil)
 
@@ -195,7 +195,7 @@ func TestCheckMonitor_LimitOutput(t *testing.T) {
 	t.Parallel()
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 	cid := structs.NewCheckID("foo", nil)
 
 	check := &CheckMonitor{
@@ -354,7 +354,7 @@ func TestCheckHTTP(t *testing.T) {
 
 			notif := mock.NewNotify()
 			logger := testutil.Logger(t)
-			statusHandler := NewStatusHandler(notif, logger, 0, 0)
+			statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 
 			cid := structs.NewCheckID("foo", nil)
 
@@ -397,7 +397,7 @@ func TestCheckHTTP_Proxied(t *testing.T) {
 	notif := mock.NewNotify()
 
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 	cid := structs.NewCheckID("foo", nil)
 
 	check := &CheckHTTP{
@@ -433,7 +433,7 @@ func TestCheckHTTP_NotProxied(t *testing.T) {
 
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 	cid := structs.NewCheckID("foo", nil)
 
 	check := &CheckHTTP{
@@ -558,7 +558,7 @@ func TestCheckMaxOutputSize(t *testing.T) {
 		Interval:      2 * time.Millisecond,
 		Logger:        logger,
 		OutputMaxSize: maxOutputSize,
-		StatusHandler: NewStatusHandler(notif, logger, 0, 0),
+		StatusHandler: NewStatusHandler(notif, logger, 0, 0, 0),
 	}
 
 	check.Start()
@@ -586,7 +586,7 @@ func TestCheckHTTPTimeout(t *testing.T) {
 
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 
 	cid := structs.NewCheckID("bar", nil)
 
@@ -659,7 +659,7 @@ func TestCheckHTTPBody(t *testing.T) {
 				Timeout:       timeout,
 				Interval:      2 * time.Millisecond,
 				Logger:        logger,
-				StatusHandler: NewStatusHandler(notif, logger, 0, 0),
+				StatusHandler: NewStatusHandler(notif, logger, 0, 0, 0),
 			}
 			check.Start()
 			defer check.Stop()
@@ -690,7 +690,7 @@ func TestCheckHTTP_disablesKeepAlives(t *testing.T) {
 		HTTP:          "http://foo.bar/baz",
 		Interval:      10 * time.Second,
 		Logger:        logger,
-		StatusHandler: NewStatusHandler(notif, logger, 0, 0),
+		StatusHandler: NewStatusHandler(notif, logger, 0, 0, 0),
 	}
 
 	check.Start()
@@ -725,7 +725,7 @@ func TestCheckHTTP_TLS_SkipVerify(t *testing.T) {
 
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 
 	cid := structs.NewCheckID("skipverify_true", nil)
 	check := &CheckHTTP{
@@ -767,7 +767,7 @@ func TestCheckHTTP_TLS_BadVerify(t *testing.T) {
 
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 	cid := structs.NewCheckID("skipverify_false", nil)
 
 	check := &CheckHTTP{
@@ -819,7 +819,7 @@ func mockTCPServer(network string) net.Listener {
 func expectTCPStatus(t *testing.T, tcp string, status string) {
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 	cid := structs.NewCheckID("foo", nil)
 
 	check := &CheckTCP{
@@ -846,13 +846,12 @@ func TestStatusHandlerUpdateStatusAfterConsecutiveChecksThresholdIsReached(t *te
 	cid := structs.NewCheckID("foo", nil)
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 2, 3)
+	statusHandler := NewStatusHandler(notif, logger, 2, 2, 3)
 
 	// Set the initial status to passing after a single success
 	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
 
-	// Status should become critical after 3 failed checks only
+	// Status should still be passing after 1 failed check only
-	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
 	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
 
 	retry.Run(t, func(r *retry.R) {
@@ -860,10 +859,19 @@ func TestStatusHandlerUpdateStatusAfterConsecutiveChecksThresholdIsReached(t *te
 		require.Equal(r, api.HealthPassing, notif.State(cid))
 	})
 
+	// Status should become warning after 2 failed checks only
 	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
 
 	retry.Run(t, func(r *retry.R) {
 		require.Equal(r, 2, notif.Updates(cid))
+		require.Equal(r, api.HealthWarning, notif.State(cid))
+	})
+
+	// Status should become critical after 4 failed checks only
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 3, notif.Updates(cid))
 		require.Equal(r, api.HealthCritical, notif.State(cid))
 	})
 
@@ -871,14 +879,14 @@ func TestStatusHandlerUpdateStatusAfterConsecutiveChecksThresholdIsReached(t *te
 	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
 
 	retry.Run(t, func(r *retry.R) {
-		require.Equal(r, 2, notif.Updates(cid))
+		require.Equal(r, 3, notif.Updates(cid))
 		require.Equal(r, api.HealthCritical, notif.State(cid))
 	})
 
 	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
 
 	retry.Run(t, func(r *retry.R) {
-		require.Equal(r, 3, notif.Updates(cid))
+		require.Equal(r, 4, notif.Updates(cid))
 		require.Equal(r, api.HealthPassing, notif.State(cid))
 	})
 }
@@ -888,17 +896,18 @@ func TestStatusHandlerResetCountersOnNonIdenticalsConsecutiveChecks(t *testing.T
 	cid := structs.NewCheckID("foo", nil)
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 2, 3)
+	statusHandler := NewStatusHandler(notif, logger, 2, 2, 3)
 
 	// Set the initial status to passing after a single success
 	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
 
-	// Status should remain passing after FAIL PASS FAIL FAIL sequence
+	// Status should remain passing after FAIL PASS FAIL PASS FAIL sequence
 	// Although we have 3 FAILS, they are not consecutive
 
 	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
 	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
 	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
 	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
 
 	retry.Run(t, func(r *retry.R) {
@@ -906,11 +915,19 @@ func TestStatusHandlerResetCountersOnNonIdenticalsConsecutiveChecks(t *testing.T
 		require.Equal(r, api.HealthPassing, notif.State(cid))
 	})
 
-	// Critical after a 3rd consecutive FAIL
+	// Warning after a 2rd consecutive FAIL
 	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
 
 	retry.Run(t, func(r *retry.R) {
 		require.Equal(r, 2, notif.Updates(cid))
+		require.Equal(r, api.HealthWarning, notif.State(cid))
+	})
+
+	// Critical after a 3rd consecutive FAIL
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 3, notif.Updates(cid))
 		require.Equal(r, api.HealthCritical, notif.State(cid))
 	})
 
@@ -920,19 +937,137 @@ func TestStatusHandlerResetCountersOnNonIdenticalsConsecutiveChecks(t *testing.T
 	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
 
 	retry.Run(t, func(r *retry.R) {
-		require.Equal(r, 2, notif.Updates(cid))
+		require.Equal(r, 3, notif.Updates(cid))
 		require.Equal(r, api.HealthCritical, notif.State(cid))
 	})
 
 	// Passing after a 2nd consecutive PASS
 	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
 
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 4, notif.Updates(cid))
+		require.Equal(r, api.HealthPassing, notif.State(cid))
+	})
+}
+
+func TestStatusHandlerWarningAndCriticalThresholdsTheSameSetsCritical(t *testing.T) {
+	t.Parallel()
+	cid := structs.NewCheckID("foo", nil)
+	notif := mock.NewNotify()
+	logger := testutil.Logger(t)
+	statusHandler := NewStatusHandler(notif, logger, 2, 3, 3)
+
+	// Set the initial status to passing after a single success
+	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
+
+	// Status should remain passing after FAIL FAIL sequence
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 1, notif.Updates(cid))
+		require.Equal(r, api.HealthPassing, notif.State(cid))
+	})
+
+	// Critical and not Warning after a 3rd consecutive FAIL
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 2, notif.Updates(cid))
+		require.Equal(r, api.HealthCritical, notif.State(cid))
+	})
+
+	// Passing after consecutive PASS PASS sequence
+	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
+	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
+
 	retry.Run(t, func(r *retry.R) {
 		require.Equal(r, 3, notif.Updates(cid))
 		require.Equal(r, api.HealthPassing, notif.State(cid))
 	})
 }
 
+func TestStatusHandlerMaintainWarningStatusWhenCheckIsFlapping(t *testing.T) {
+	t.Parallel()
+	cid := structs.NewCheckID("foo", nil)
+	notif := mock.NewNotify()
+	logger := testutil.Logger(t)
+	statusHandler := NewStatusHandler(notif, logger, 3, 3, 5)
+
+	// Set the initial status to passing after a single success.
+	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
+
+	// Status should remain passing after a FAIL FAIL sequence.
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 1, notif.Updates(cid))
+		require.Equal(r, api.HealthPassing, notif.State(cid))
+	})
+
+	// Warning after a 3rd consecutive FAIL.
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 2, notif.Updates(cid))
+		require.Equal(r, api.HealthWarning, notif.State(cid))
+	})
+
+	// Status should remain passing after PASS FAIL FAIL FAIL PASS FAIL FAIL FAIL PASS sequence.
+	// Although we have 6 FAILS, they are not consecutive.
+	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	// The status gets updated due to failuresCounter being reset
+	// but the status itself remains as Warning.
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 3, notif.Updates(cid))
+		require.Equal(r, api.HealthWarning, notif.State(cid))
+	})
+
+	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	// Status doesn'tn change, but the state update is triggered.
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 4, notif.Updates(cid))
+		require.Equal(r, api.HealthWarning, notif.State(cid))
+	})
+
+	// Status should change only after 5 consecutive FAIL updates.
+	statusHandler.updateCheck(cid, api.HealthPassing, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	// The status doesn't change, but a status update is triggered.
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 5, notif.Updates(cid))
+		require.Equal(r, api.HealthWarning, notif.State(cid))
+	})
+
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	// The status doesn't change, but a status update is triggered.
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 6, notif.Updates(cid))
+		require.Equal(r, api.HealthWarning, notif.State(cid))
+	})
+
+	statusHandler.updateCheck(cid, api.HealthCritical, "bar")
+
+	// The FailuresBeforeCritical threshold is finally breached.
+	retry.Run(t, func(r *retry.R) {
+		require.Equal(r, 7, notif.Updates(cid))
+		require.Equal(r, api.HealthCritical, notif.State(cid))
+	})
+}
+
 func TestCheckTCPCritical(t *testing.T) {
 	t.Parallel()
 	var (
@@ -992,7 +1127,7 @@ func TestCheckH2PING(t *testing.T) {
 
 			notif := mock.NewNotify()
 			logger := testutil.Logger(t)
-			statusHandler := NewStatusHandler(notif, logger, 0, 0)
+			statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 			cid := structs.NewCheckID("foo", nil)
 			tlsCfg := &api.TLSConfig{
 				InsecureSkipVerify: true,
@@ -1044,7 +1179,7 @@ func TestCheckH2PING_TLS_BadVerify(t *testing.T) {
 
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 	cid := structs.NewCheckID("foo", nil)
 	tlsCfg := &api.TLSConfig{}
 	tlsClientCfg, err := api.SetupTLSConfig(tlsCfg)
@@ -1085,7 +1220,7 @@ func TestCheckH2PINGInvalidListener(t *testing.T) {
 
 	notif := mock.NewNotify()
 	logger := testutil.Logger(t)
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 	cid := structs.NewCheckID("foo", nil)
 	tlsCfg := &api.TLSConfig{
 		InsecureSkipVerify: true,
@@ -1388,7 +1523,7 @@ func TestCheck_Docker(t *testing.T) {
 
 			notif, upd := mock.NewNotifyChan()
 			logger := testutil.Logger(t)
-			statusHandler := NewStatusHandler(notif, logger, 0, 0)
+			statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 			id := structs.NewCheckID("chk", nil)
 
 			check := &CheckDocker{

agent/checks/grpc_test.go

@@ -113,7 +113,7 @@ func TestGRPC_Proxied(t *testing.T) {
 		Output: ioutil.Discard,
 	})
 
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 	cid := structs.NewCheckID("foo", nil)
 
 	check := &CheckGRPC{
@@ -147,7 +147,7 @@ func TestGRPC_NotProxied(t *testing.T) {
 		Output: ioutil.Discard,
 	})
 
-	statusHandler := NewStatusHandler(notif, logger, 0, 0)
+	statusHandler := NewStatusHandler(notif, logger, 0, 0, 0)
 	cid := structs.NewCheckID("foo", nil)
 
 	check := &CheckGRPC{

agent/config/builder.go

@@ -1415,6 +1415,12 @@ func (b *builder) validate(rt RuntimeConfig) error {
 			return fmt.Errorf("service %q: %s", s.Name, err)
 		}
 	}
+	// Check for errors in the node check definitions
+	for _, c := range rt.Checks {
+		if err := c.CheckType().Validate(); err != nil {
+			return fmt.Errorf("check %q: %w", c.Name, err)
+		}
+	}
 
 	// Validate the given Connect CA provider config
 	validCAProviders := map[string]bool{
@@ -1584,6 +1590,7 @@ func (b *builder) checkVal(v *CheckDefinition) *structs.CheckDefinition {
 		TTL:                            b.durationVal(fmt.Sprintf("check[%s].ttl", id), v.TTL),
 		SuccessBeforePassing:           intVal(v.SuccessBeforePassing),
 		FailuresBeforeCritical:         intVal(v.FailuresBeforeCritical),
+		FailuresBeforeWarning:          intValWithDefault(v.FailuresBeforeWarning, intVal(v.FailuresBeforeCritical)),
 		H2PING:                         stringVal(v.H2PING),
 		DeregisterCriticalServiceAfter: b.durationVal(fmt.Sprintf("check[%s].deregister_critical_service_after", id), v.DeregisterCriticalServiceAfter),
 		OutputMaxSize:                  intValWithDefault(v.OutputMaxSize, checks.DefaultBufSize),

agent/config/config.go

@@ -424,6 +424,7 @@ type CheckDefinition struct {
 	TTL                            *string             `mapstructure:"ttl"`
 	H2PING                         *string             `mapstructure:"h2ping"`
 	SuccessBeforePassing           *int                `mapstructure:"success_before_passing"`
+	FailuresBeforeWarning          *int                `mapstructure:"failures_before_warning"`
 	FailuresBeforeCritical         *int                `mapstructure:"failures_before_critical"`
 	DeregisterCriticalServiceAfter *string             `mapstructure:"deregister_critical_service_after" alias:"deregistercriticalserviceafter"`
 

agent/config/runtime.go

@@ -434,6 +434,9 @@ type RuntimeConfig struct {
 	//     tls_skip_verify = (true|false)
 	//     timeout = "duration"
 	//     ttl = "duration"
+	//     success_before_passing = int
+	//     failures_before_warning = int
+	//     failures_before_critical = int
 	//     deregister_critical_service_after = "duration"
 	//   },
 	//   ...

agent/config/runtime_test.go

@@ -2330,17 +2330,17 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 			`-data-dir=` + dataDir,
 		},
 		json: []string{
-			`{ "check": { "name": "a", "args": ["/bin/true"] } }`,
+			`{ "check": { "name": "a", "args": ["/bin/true"], "interval": "1s" } }`,
-			`{ "check": { "name": "b", "args": ["/bin/false"] } }`,
+			`{ "check": { "name": "b", "args": ["/bin/false"], "interval": "1s" } }`,
 		},
 		hcl: []string{
-			`check = { name = "a" args = ["/bin/true"] }`,
+			`check = { name = "a" args = ["/bin/true"] interval = "1s"}`,
-			`check = { name = "b" args = ["/bin/false"] }`,
+			`check = { name = "b" args = ["/bin/false"] interval = "1s" }`,
 		},
 		expected: func(rt *RuntimeConfig) {
 			rt.Checks = []*structs.CheckDefinition{
-				{Name: "a", ScriptArgs: []string{"/bin/true"}, OutputMaxSize: checks.DefaultBufSize},
+				{Name: "a", ScriptArgs: []string{"/bin/true"}, OutputMaxSize: checks.DefaultBufSize, Interval: time.Second},
-				{Name: "b", ScriptArgs: []string{"/bin/false"}, OutputMaxSize: checks.DefaultBufSize},
+				{Name: "b", ScriptArgs: []string{"/bin/false"}, OutputMaxSize: checks.DefaultBufSize, Interval: time.Second},
 			}
 			rt.DataDir = dataDir
 		},
@@ -2351,14 +2351,14 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 			`-data-dir=` + dataDir,
 		},
 		json: []string{
-			`{ "check": { "name": "a", "grpc": "localhost:12345/foo", "grpc_use_tls": true } }`,
+			`{ "check": { "name": "a", "grpc": "localhost:12345/foo", "grpc_use_tls": true, "interval": "1s" } }`,
 		},
 		hcl: []string{
-			`check = { name = "a" grpc = "localhost:12345/foo", grpc_use_tls = true }`,
+			`check = { name = "a" grpc = "localhost:12345/foo", grpc_use_tls = true interval = "1s" }`,
 		},
 		expected: func(rt *RuntimeConfig) {
 			rt.Checks = []*structs.CheckDefinition{
-				{Name: "a", GRPC: "localhost:12345/foo", GRPCUseTLS: true, OutputMaxSize: checks.DefaultBufSize},
+				{Name: "a", GRPC: "localhost:12345/foo", GRPCUseTLS: true, OutputMaxSize: checks.DefaultBufSize, Interval: time.Second},
 			}
 			rt.DataDir = dataDir
 		},
@@ -2478,7 +2478,8 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 							"name": "y",
 							"DockerContainerID": "z",
 							"DeregisterCriticalServiceAfter": "10s",
-							"ScriptArgs": ["a", "b"]
+							"ScriptArgs": ["a", "b"],
+							"Interval": "2s"
 						}
 					}
 				}`,
@@ -2500,6 +2501,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 						DockerContainerID = "z"
 						DeregisterCriticalServiceAfter = "10s"
 						ScriptArgs = ["a", "b"]
+						Interval = "2s"
 					}
 				}`,
 		},
@@ -2517,12 +2519,13 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 					EnableTagOverride: true,
 					Checks: []*structs.CheckType{
 						{
-							CheckID:                        types.CheckID("x"),
+							CheckID:                        "x",
 							Name:                           "y",
 							DockerContainerID:              "z",
 							DeregisterCriticalServiceAfter: 10 * time.Second,
 							ScriptArgs:                     []string{"a", "b"},
 							OutputMaxSize:                  checks.DefaultBufSize,
+							Interval:                       2 * time.Second,
 						},
 					},
 					Weights: &structs.Weights{
@@ -5299,7 +5302,6 @@ func TestLoad_FullConfig(t *testing.T) {
 				TLSServerName:                  "bdeb5f6a",
 				TLSSkipVerify:                  true,
 				Timeout:                        1813 * time.Second,
-				TTL:                            21743 * time.Second,
 				DeregisterCriticalServiceAfter: 14232 * time.Second,
 			},
 			{
@@ -5326,7 +5328,6 @@ func TestLoad_FullConfig(t *testing.T) {
 				TLSServerName:                  "6adc3bfb",
 				TLSSkipVerify:                  true,
 				Timeout:                        18506 * time.Second,
-				TTL:                            31006 * time.Second,
 				DeregisterCriticalServiceAfter: 2366 * time.Second,
 			},
 			{
@@ -5353,7 +5354,6 @@ func TestLoad_FullConfig(t *testing.T) {
 				TLSServerName:                  "7BdnzBYk",
 				TLSSkipVerify:                  true,
 				Timeout:                        5954 * time.Second,
-				TTL:                            30044 * time.Second,
 				DeregisterCriticalServiceAfter: 13209 * time.Second,
 			},
 		},
@@ -5559,7 +5559,6 @@ func TestLoad_FullConfig(t *testing.T) {
 						TLSServerName:                  "4f191d4F",
 						TLSSkipVerify:                  true,
 						Timeout:                        38333 * time.Second,
-						TTL:                            57201 * time.Second,
 						DeregisterCriticalServiceAfter: 44214 * time.Second,
 					},
 				},
@@ -5611,30 +5610,14 @@ func TestLoad_FullConfig(t *testing.T) {
 						TLSServerName:                  "f43ouY7a",
 						TLSSkipVerify:                  true,
 						Timeout:                        34738 * time.Second,
-						TTL:                            22773 * time.Second,
 						DeregisterCriticalServiceAfter: 84282 * time.Second,
 					},
 					&structs.CheckType{
-						CheckID:    "UHsDeLxG",
+						CheckID:                        "UHsDeLxG",
-						Name:       "PQSaPWlT",
+						Name:                           "PQSaPWlT",
-						Notes:      "jKChDOdl",
+						Notes:                          "jKChDOdl",
-						Status:     "5qFz6OZn",
+						Status:                         "5qFz6OZn",
-						ScriptArgs: []string{"NMtYWlT9", "vj74JXsm"},
-						HTTP:       "1LBDJhw4",
-						Header: map[string][]string{
-							"cXPmnv1M": {"imDqfaBx", "NFxZ1bQe"},
-							"vr7wY7CS": {"EtCoNPPL", "9vAarJ5s"},
-						},
-						Method:                         "wzByP903",
-						Body:                           "4I8ucZgZ",
 						OutputMaxSize:                  checks.DefaultBufSize,
-						TCP:                            "2exjZIGE",
-						H2PING:                         "jTDuR1DC",
-						Interval:                       5656 * time.Second,
-						DockerContainerID:              "5tDBWpfA",
-						Shell:                          "rlTpLM8s",
-						TLSServerName:                  "sOv5WTtp",
-						TLSSkipVerify:                  true,
 						Timeout:                        4868 * time.Second,
 						TTL:                            11222 * time.Second,
 						DeregisterCriticalServiceAfter: 68482 * time.Second,
@@ -5770,7 +5753,6 @@ func TestLoad_FullConfig(t *testing.T) {
 						TLSServerName:                  "axw5QPL5",
 						TLSSkipVerify:                  true,
 						Timeout:                        18913 * time.Second,
-						TTL:                            44743 * time.Second,
 						DeregisterCriticalServiceAfter: 8482 * time.Second,
 					},
 					&structs.CheckType{
@@ -5795,7 +5777,6 @@ func TestLoad_FullConfig(t *testing.T) {
 						TLSServerName:                  "7uwWOnUS",
 						TLSSkipVerify:                  true,
 						Timeout:                        38282 * time.Second,
-						TTL:                            1181 * time.Second,
 						DeregisterCriticalServiceAfter: 4992 * time.Second,
 					},
 					&structs.CheckType{
@@ -5820,7 +5801,6 @@ func TestLoad_FullConfig(t *testing.T) {
 						TLSServerName:                  "ECSHk8WF",
 						TLSSkipVerify:                  true,
 						Timeout:                        38483 * time.Second,
-						TTL:                            10943 * time.Second,
 						DeregisterCriticalServiceAfter: 68787 * time.Second,
 					},
 				},

agent/config/testdata/TestRuntimeConfig_Sanitize.golden

@@ -94,6 +94,7 @@
             "DeregisterCriticalServiceAfter": "0s",
             "DockerContainerID": "",
             "EnterpriseMeta": {},
+            "FailuresBeforeWarning": 0,
             "FailuresBeforeCritical": 0,
             "GRPC": "",
             "GRPCUseTLS": false,
@@ -295,6 +296,7 @@
                 "CheckID": "",
                 "DeregisterCriticalServiceAfter": "0s",
                 "DockerContainerID": "",
+                "FailuresBeforeWarning": 0,
                 "FailuresBeforeCritical": 0,
                 "GRPC": "",
                 "GRPCUseTLS": false,
@@ -416,4 +418,4 @@
     "Watches": [],
     "XDSAddrs": [],
     "XDSPort": 0
-}
+}

agent/config/testdata/full-config.hcl

@@ -117,7 +117,6 @@ check = {
     tls_server_name = "7BdnzBYk"
     tls_skip_verify = true
     timeout = "5954s"
-    ttl = "30044s"
     deregister_critical_service_after = "13209s"
 },
 checks = [
@@ -145,7 +144,6 @@ checks = [
         tls_server_name = "bdeb5f6a"
         tls_skip_verify = true
         timeout = "1813s"
-        ttl = "21743s"
         deregister_critical_service_after = "14232s"
     },
     {
@@ -172,7 +170,6 @@ checks = [
         tls_server_name = "6adc3bfb"
         tls_skip_verify = true
         timeout = "18506s"
-        ttl = "31006s"
         deregister_critical_service_after = "2366s"
     }
 ]
@@ -389,7 +386,6 @@ service = {
         tls_server_name = "ECSHk8WF"
         tls_skip_verify = true
         timeout = "38483s"
-        ttl = "10943s"
         deregister_critical_service_after = "68787s"
     }
     checks = [
@@ -415,7 +411,6 @@ service = {
             tls_server_name = "axw5QPL5"
             tls_skip_verify = true
             timeout = "18913s"
-            ttl = "44743s"
             deregister_critical_service_after = "8482s"
         },
         {
@@ -440,7 +435,6 @@ service = {
             tls_server_name = "7uwWOnUS"
             tls_skip_verify = true
             timeout = "38282s"
-            ttl = "1181s"
             deregister_critical_service_after = "4992s"
         }
     ]
@@ -479,7 +473,6 @@ services = [
             tls_server_name = "4f191d4F"
             tls_skip_verify = true
             timeout = "38333s"
-            ttl = "57201s"
             deregister_critical_service_after = "44214s"
         }
         connect {
@@ -521,7 +514,6 @@ services = [
                 tls_server_name = "f43ouY7a"
                 tls_skip_verify = true
                 timeout = "34738s"
-                ttl = "22773s"
                 deregister_critical_service_after = "84282s"
             },
             {
@@ -529,22 +521,7 @@ services = [
                 name = "PQSaPWlT"
                 notes = "jKChDOdl"
                 status = "5qFz6OZn"
-                args = ["NMtYWlT9", "vj74JXsm"]
-                http = "1LBDJhw4"
-                header = {
-                    "cXPmnv1M" = [ "imDqfaBx", "NFxZ1bQe" ],
-                    "vr7wY7CS" = [ "EtCoNPPL", "9vAarJ5s" ]
-                }
-                method = "wzByP903"
-                body = "4I8ucZgZ"
-                tcp = "2exjZIGE"
-                h2ping = "jTDuR1DC"
-                interval = "5656s"
                 output_max_size = 4096
-                docker_container_id = "5tDBWpfA"
-                shell = "rlTpLM8s"
-                tls_server_name = "sOv5WTtp"
-                tls_skip_verify = true
                 timeout = "4868s"
                 ttl = "11222s"
                 deregister_critical_service_after = "68482s"

agent/config/testdata/full-config.json

@@ -118,7 +118,6 @@
     "tls_server_name": "7BdnzBYk",
     "tls_skip_verify": true,
     "timeout": "5954s",
-    "ttl": "30044s",
     "deregister_critical_service_after": "13209s"
   },
   "checks": [
@@ -146,7 +145,6 @@
       "tls_server_name": "bdeb5f6a",
       "tls_skip_verify": true,
       "timeout": "1813s",
-      "ttl": "21743s",
       "deregister_critical_service_after": "14232s"
     },
     {
@@ -173,7 +171,6 @@
       "tls_server_name": "6adc3bfb",
       "tls_skip_verify": true,
       "timeout": "18506s",
-      "ttl": "31006s",
       "deregister_critical_service_after": "2366s"
     }
   ],
@@ -386,7 +383,6 @@
       "tls_server_name": "ECSHk8WF",
       "tls_skip_verify": true,
       "timeout": "38483s",
-      "ttl": "10943s",
       "deregister_critical_service_after": "68787s"
     },
     "checks": [
@@ -412,7 +408,6 @@
         "tls_server_name": "axw5QPL5",
         "tls_skip_verify": true,
         "timeout": "18913s",
-        "ttl": "44743s",
         "deregister_critical_service_after": "8482s"
       },
       {
@@ -437,7 +432,6 @@
         "tls_server_name": "7uwWOnUS",
         "tls_skip_verify": true,
         "timeout": "38282s",
-        "ttl": "1181s",
         "deregister_critical_service_after": "4992s"
       }
     ],
@@ -476,7 +470,6 @@
         "tls_server_name": "4f191d4F",
         "tls_skip_verify": true,
         "timeout": "38333s",
-        "ttl": "57201s",
         "deregister_critical_service_after": "44214s"
       },
       "connect": {
@@ -518,7 +511,6 @@
           "tls_server_name": "f43ouY7a",
           "tls_skip_verify": true,
           "timeout": "34738s",
-          "ttl": "22773s",
           "deregister_critical_service_after": "84282s"
         },
         {
@@ -526,22 +518,7 @@
           "name": "PQSaPWlT",
           "notes": "jKChDOdl",
           "status": "5qFz6OZn",
-          "args": ["NMtYWlT9", "vj74JXsm"],
-          "http": "1LBDJhw4",
-          "header": {
-            "cXPmnv1M": [ "imDqfaBx", "NFxZ1bQe" ],
-            "vr7wY7CS": [ "EtCoNPPL", "9vAarJ5s" ]
-          },
-          "method": "wzByP903",
-          "body": "4I8ucZgZ",
-          "tcp": "2exjZIGE",
-          "h2ping": "jTDuR1DC",
-          "interval": "5656s",
           "output_max_size": 4096,
-          "docker_container_id": "5tDBWpfA",
-          "shell": "rlTpLM8s",
-          "tls_server_name": "sOv5WTtp",
-          "tls_skip_verify": true,
           "timeout": "4868s",
           "ttl": "11222s",
           "deregister_critical_service_after": "68482s"

agent/structs/check_definition.go

@@ -41,6 +41,7 @@ type CheckDefinition struct {
 	Timeout                        time.Duration
 	TTL                            time.Duration
 	SuccessBeforePassing           int
+	FailuresBeforeWarning          int
 	FailuresBeforeCritical         int
 	DeregisterCriticalServiceAfter time.Duration
 	OutputMaxSize                  int
@@ -196,6 +197,7 @@ func (c *CheckDefinition) CheckType() *CheckType {
 		Timeout:                        c.Timeout,
 		TTL:                            c.TTL,
 		SuccessBeforePassing:           c.SuccessBeforePassing,
+		FailuresBeforeWarning:          c.FailuresBeforeWarning,
 		FailuresBeforeCritical:         c.FailuresBeforeCritical,
 		DeregisterCriticalServiceAfter: c.DeregisterCriticalServiceAfter,
 	}

agent/structs/check_type.go

@@ -49,6 +49,7 @@ type CheckType struct {
 	Timeout                time.Duration
 	TTL                    time.Duration
 	SuccessBeforePassing   int
+	FailuresBeforeWarning  int
 	FailuresBeforeCritical int
 
 	// Definition fields used when exposing checks through a proxy
@@ -182,6 +183,10 @@ func (c *CheckType) Validate() error {
 	if c.OutputMaxSize < 0 {
 		return fmt.Errorf("MaxOutputMaxSize must be positive")
 	}
+	if c.FailuresBeforeWarning > c.FailuresBeforeCritical {
+		return fmt.Errorf("FailuresBeforeWarning can't be higher than FailuresBeforeCritical")
+	}
+
 	return nil
 }
 

agent/structs/service_definition.go

@@ -1,8 +1,11 @@
 package structs
 
 import (
-	"github.com/hashicorp/consul/lib"
+	"fmt"
+
 	"github.com/hashicorp/go-multierror"
+
+	"github.com/hashicorp/consul/lib"
 )
 
 // ServiceDefinition is used to JSON decode the Service definitions. For
@@ -123,7 +126,11 @@ func (s *ServiceDefinition) Validate() error {
 	if err := s.NodeService().Validate(); err != nil {
 		result = multierror.Append(result, err)
 	}
-
+	for _, c := range s.Checks {
+		if err := c.Validate(); err != nil {
+			return fmt.Errorf("check %q: %s", c.Name, err)
+		}
+	}
 	return result
 }
 

api/agent.go

@@ -330,6 +330,7 @@ type AgentServiceCheck struct {
 	AliasNode              string              `json:",omitempty"`
 	AliasService           string              `json:",omitempty"`
 	SuccessBeforePassing   int                 `json:",omitempty"`
+	FailuresBeforeWarning  int                 `json:",omitempty"`
 	FailuresBeforeCritical int                 `json:",omitempty"`
 
 	// In Consul 0.7 and later, checks that are associated with a service

build-support/scripts/proto-gen.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 SCRIPT_NAME="$(basename ${BASH_SOURCE[0]})"
 pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null
 SCRIPT_DIR=$(pwd)

proto/pbservice/healthcheck.gen.go

@@ -30,6 +30,7 @@ func CheckTypeToStructs(s CheckType) structs.CheckType {
 	t.TTL = s.TTL
 	t.SuccessBeforePassing = int(s.SuccessBeforePassing)
 	t.FailuresBeforeCritical = int(s.FailuresBeforeCritical)
+	t.FailuresBeforeWarning = int(s.FailuresBeforeWarning)
 	t.ProxyHTTP = s.ProxyHTTP
 	t.ProxyGRPC = s.ProxyGRPC
 	t.DeregisterCriticalServiceAfter = s.DeregisterCriticalServiceAfter
@@ -62,6 +63,7 @@ func NewCheckTypeFromStructs(t structs.CheckType) CheckType {
 	s.TTL = t.TTL
 	s.SuccessBeforePassing = int32(t.SuccessBeforePassing)
 	s.FailuresBeforeCritical = int32(t.FailuresBeforeCritical)
+	s.FailuresBeforeWarning = int32(t.FailuresBeforeWarning)
 	s.ProxyHTTP = t.ProxyHTTP
 	s.ProxyGRPC = t.ProxyGRPC
 	s.DeregisterCriticalServiceAfter = t.DeregisterCriticalServiceAfter

proto/pbservice/healthcheck.pb.go

@@ -232,6 +232,8 @@ type CheckType struct {
 	// mog: func-to=int func-from=int32
 	SuccessBeforePassing int32 `protobuf:"varint,21,opt,name=SuccessBeforePassing,proto3" json:"SuccessBeforePassing,omitempty"`
 	// mog: func-to=int func-from=int32
+	FailuresBeforeWarning int32 `protobuf:"varint,29,opt,name=FailuresBeforeWarning,proto3" json:"FailuresBeforeWarning,omitempty"`
+	// mog: func-to=int func-from=int32
 	FailuresBeforeCritical int32 `protobuf:"varint,22,opt,name=FailuresBeforeCritical,proto3" json:"FailuresBeforeCritical,omitempty"`
 	// Definition fields used when exposing checks through a proxy
 	ProxyHTTP string `protobuf:"bytes,23,opt,name=ProxyHTTP,proto3" json:"ProxyHTTP,omitempty"`
@@ -289,74 +291,75 @@ func init() {
 func init() { proto.RegisterFile("proto/pbservice/healthcheck.proto", fileDescriptor_8a6f7448747c9fbe) }
 
 var fileDescriptor_8a6f7448747c9fbe = []byte{
-	// 1062 bytes of a gzipped FileDescriptorProto
+	// 1076 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0x41, 0x4f, 0xe3, 0x46,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0x5d, 0x4f, 0xe3, 0x46,
-	0x14, 0x8e, 0x09, 0x24, 0xf1, 0x64, 0x61, 0x61, 0x96, 0xa5, 0xb3, 0xec, 0xca, 0xa4, 0x74, 0x0f,
+	0x14, 0x8d, 0x09, 0x24, 0xf1, 0x04, 0x58, 0x98, 0x05, 0x3a, 0xcb, 0x6e, 0x4d, 0x4a, 0xf7, 0x81,
-	0x54, 0xa5, 0x8e, 0x44, 0xd5, 0xaa, 0xad, 0xd4, 0x56, 0x84, 0xb0, 0x90, 0x0a, 0x68, 0xea, 0xb8,
+	0xaa, 0x34, 0x91, 0xe8, 0x87, 0xda, 0x4a, 0x6d, 0x45, 0x08, 0x0b, 0xa9, 0x80, 0xa6, 0x4e, 0xba,
-	0x7b, 0xe8, 0xcd, 0x38, 0x93, 0xc4, 0xc2, 0xf1, 0x58, 0xe3, 0x31, 0x22, 0xfd, 0x15, 0x7b, 0xdc,
+	0x95, 0xfa, 0x66, 0x9c, 0x49, 0x62, 0xe1, 0x78, 0xac, 0xf1, 0x18, 0x91, 0xfe, 0x8a, 0x7d, 0xdc,
-	0xff, 0xd0, 0x3f, 0xc2, 0x91, 0x63, 0xa5, 0x4a, 0xb4, 0x85, 0x7f, 0xd1, 0x53, 0x35, 0x6f, 0xec,
+	0x9f, 0xc4, 0x23, 0x8f, 0x95, 0x2a, 0xd1, 0x2e, 0xfc, 0x8b, 0x3e, 0x55, 0x73, 0xc7, 0x0e, 0xf6,
-	0xe0, 0x6c, 0xbc, 0x25, 0x5d, 0x6d, 0x4f, 0xcc, 0xfb, 0xde, 0x7b, 0x33, 0x9e, 0x79, 0xdf, 0xf7,
+	0xc6, 0x5b, 0xd2, 0xd5, 0xee, 0x13, 0x73, 0xef, 0xb9, 0x77, 0xc6, 0x33, 0xf7, 0x9c, 0x13, 0xd0,
-	0x05, 0xf4, 0x61, 0xc8, 0x99, 0x60, 0xf5, 0xf0, 0x34, 0xa2, 0xfc, 0xdc, 0x73, 0x69, 0x7d, 0x40,
+	0x47, 0x3e, 0x67, 0x82, 0xd5, 0xfc, 0xd3, 0x80, 0xf2, 0x73, 0xc7, 0xa6, 0xb5, 0x01, 0xb5, 0x5c,
-	0x1d, 0x5f, 0x0c, 0xdc, 0x01, 0x75, 0xcf, 0x4c, 0xc8, 0x61, 0x7d, 0x9c, 0x5c, 0x37, 0xfa, 0x8c,
+	0x31, 0xb0, 0x07, 0xd4, 0x3e, 0xab, 0x02, 0x86, 0xf5, 0x31, 0xb8, 0x6e, 0xf4, 0x19, 0xeb, 0xbb,
-	0xf5, 0x7d, 0x5a, 0x87, 0xc4, 0x69, 0xdc, 0xab, 0x77, 0x63, 0xee, 0x08, 0x8f, 0x05, 0xaa, 0x74,
+	0xb4, 0x06, 0xc0, 0x69, 0xd8, 0xab, 0x75, 0x43, 0x6e, 0x09, 0x87, 0x79, 0xaa, 0x74, 0xfd, 0x71,
-	0xfd, 0x69, 0xba, 0x9b, 0xcb, 0x86, 0x43, 0x16, 0xd4, 0xd5, 0x9f, 0x24, 0xb9, 0xda, 0x67, 0x7d,
+	0xbc, 0x9b, 0xcd, 0x86, 0x43, 0xe6, 0xd5, 0xd4, 0x9f, 0x08, 0x5c, 0xe9, 0xb3, 0x3e, 0x53, 0x05,
-	0xa6, 0x0a, 0xe4, 0x4a, 0xa1, 0x9b, 0xbf, 0xcf, 0xa3, 0xea, 0x21, 0x9c, 0xb9, 0x27, 0xcf, 0xc4,
+	0x72, 0xa5, 0xb2, 0x9b, 0x7f, 0xce, 0xa2, 0xf2, 0x21, 0x9c, 0xb9, 0x27, 0xcf, 0xc4, 0x18, 0xcd,
-	0x18, 0xcd, 0x9f, 0xb0, 0x2e, 0x25, 0x5a, 0x4d, 0xdb, 0xd2, 0x2d, 0x58, 0xe3, 0x03, 0x54, 0x86,
+	0x9e, 0xb0, 0x2e, 0x25, 0x5a, 0x45, 0xdb, 0xd2, 0x4d, 0x58, 0xe3, 0x03, 0x54, 0x04, 0xb0, 0xd9,
-	0x64, 0xab, 0x49, 0xe6, 0x24, 0xdc, 0xf8, 0xf4, 0xef, 0xeb, 0x8d, 0x8f, 0xfb, 0x9e, 0x18, 0xc4,
+	0x20, 0x33, 0x32, 0x5d, 0xff, 0xec, 0x9f, 0xeb, 0x8d, 0x4f, 0xfa, 0x8e, 0x18, 0x84, 0xa7, 0x55,
-	0xa7, 0xa6, 0xcb, 0x86, 0xf5, 0x81, 0x13, 0x0d, 0x3c, 0x97, 0xf1, 0xb0, 0xee, 0xb2, 0x20, 0x8a,
+	0x9b, 0x0d, 0x6b, 0x03, 0x2b, 0x18, 0x38, 0x36, 0xe3, 0x7e, 0xcd, 0x66, 0x5e, 0x10, 0xba, 0x35,
-	0xfd, 0xba, 0x18, 0x85, 0x34, 0x32, 0x93, 0x26, 0x2b, 0xed, 0x86, 0xcd, 0x9d, 0x21, 0x25, 0xc5,
+	0x31, 0xf2, 0x69, 0x50, 0x8d, 0x9a, 0xcc, 0xb8, 0x1b, 0x36, 0xb7, 0x86, 0x94, 0xe4, 0xa3, 0xcd,
-	0x64, 0x73, 0x67, 0x48, 0xf1, 0x1a, 0x2a, 0x75, 0x84, 0x23, 0xe2, 0x88, 0xcc, 0x03, 0x9a, 0x44,
+	0xad, 0x21, 0xc5, 0x6b, 0xa8, 0xd0, 0x16, 0x96, 0x08, 0x03, 0x32, 0x0b, 0xd9, 0x28, 0xc2, 0x2b,
-	0x78, 0x15, 0x2d, 0x9c, 0x30, 0x41, 0x23, 0xb2, 0x00, 0xb0, 0x0a, 0x64, 0xf5, 0x0f, 0xb1, 0x08,
+	0x68, 0xee, 0x84, 0x09, 0x1a, 0x90, 0x39, 0x48, 0xab, 0x40, 0x56, 0xff, 0x14, 0x0a, 0x3f, 0x14,
-	0x63, 0x41, 0x4a, 0xaa, 0x5a, 0x45, 0xf8, 0x19, 0xd2, 0x3b, 0xea, 0x91, 0x5a, 0x4d, 0x52, 0x86,
+	0xa4, 0xa0, 0xaa, 0x55, 0x84, 0x9f, 0x20, 0xbd, 0xad, 0x1e, 0xa9, 0xd9, 0x20, 0x45, 0x80, 0xee,
-	0xd4, 0x1d, 0x80, 0x6b, 0xa8, 0x9a, 0x04, 0x70, 0x7c, 0x05, 0xf2, 0x59, 0x28, 0x53, 0x61, 0x3b,
+	0x12, 0xb8, 0x82, 0xca, 0x51, 0x00, 0xc7, 0x97, 0x00, 0x4f, 0xa6, 0x12, 0x15, 0x1d, 0xab, 0x1f,
-	0xfd, 0x88, 0xe8, 0xb5, 0x62, 0xa6, 0x42, 0x42, 0xf2, 0xdb, 0xed, 0x51, 0x48, 0xc9, 0x03, 0xf5,
+	0x10, 0xbd, 0x92, 0x4f, 0x54, 0xc8, 0x94, 0xfc, 0xf6, 0xce, 0xc8, 0xa7, 0x64, 0x5e, 0x7d, 0xbb,
-	0xed, 0x72, 0x8d, 0x5f, 0x20, 0xd4, 0xa4, 0x3d, 0x2f, 0xf0, 0xe4, 0x0c, 0x08, 0xaa, 0x69, 0x5b,
+	0x5c, 0xe3, 0x67, 0x08, 0x35, 0x68, 0xcf, 0xf1, 0x1c, 0x39, 0x03, 0x82, 0x2a, 0xda, 0x56, 0x79,
-	0xd5, 0x9d, 0x9a, 0x39, 0x9e, 0x97, 0x99, 0x79, 0xd8, 0xbb, 0xba, 0xc6, 0xfc, 0xe5, 0xf5, 0x46,
+	0xa7, 0x52, 0x1d, 0xcf, 0xab, 0x9a, 0x78, 0xd8, 0xbb, 0xba, 0xfa, 0xec, 0xe5, 0xf5, 0x46, 0xce,
-	0xc1, 0xca, 0x74, 0xe2, 0xaf, 0x90, 0x6e, 0x39, 0x3d, 0xd1, 0x0a, 0xba, 0xf4, 0x82, 0x54, 0x61,
+	0x4c, 0x74, 0xe2, 0x6f, 0x90, 0x6e, 0x5a, 0x3d, 0xd1, 0xf4, 0xba, 0xf4, 0x82, 0x94, 0x61, 0x9b,
-	0x9b, 0x15, 0x33, 0x19, 0xde, 0x38, 0xd1, 0xa8, 0xc8, 0xbe, 0xab, 0xeb, 0x0d, 0xcd, 0xba, 0xab,
+	0xe5, 0x6a, 0x34, 0xbc, 0x31, 0x50, 0x2f, 0xc9, 0xbe, 0xab, 0xeb, 0x0d, 0xcd, 0xbc, 0xab, 0xc6,
-	0xc6, 0x4d, 0xb4, 0xb4, 0x1f, 0x08, 0xca, 0x43, 0xee, 0x45, 0xf4, 0x98, 0x0a, 0x87, 0x2c, 0x42,
+	0x0d, 0xb4, 0xb8, 0xef, 0x09, 0xca, 0x7d, 0xee, 0x04, 0xf4, 0x98, 0x0a, 0x8b, 0x2c, 0x40, 0xff,
-	0xff, 0x5a, 0xda, 0x3f, 0x99, 0x4d, 0x0e, 0x7f, 0xa3, 0x47, 0x5e, 0x7f, 0xff, 0x22, 0x64, 0x11,
+	0x5a, 0xdc, 0x9f, 0x46, 0xa3, 0xc3, 0x5f, 0xeb, 0x91, 0xd7, 0xdf, 0xbf, 0xf0, 0x59, 0x40, 0xbb,
-	0xed, 0xb6, 0x19, 0x17, 0x64, 0xa9, 0xa6, 0x6d, 0x2d, 0x58, 0x59, 0x08, 0xaf, 0xa3, 0x4a, 0x4b,
+	0x2d, 0xc6, 0x05, 0x59, 0xac, 0x68, 0x5b, 0x73, 0x66, 0x32, 0x85, 0xd7, 0x51, 0xa9, 0x29, 0x7b,
-	0xf6, 0x9c, 0x3b, 0x3e, 0x79, 0x08, 0x4f, 0x30, 0x8e, 0x31, 0x41, 0x65, 0xdb, 0x1b, 0x52, 0x16,
+	0xce, 0x2d, 0x97, 0x3c, 0x80, 0x27, 0x18, 0xc7, 0x98, 0xa0, 0x62, 0xc7, 0x19, 0x52, 0x16, 0x0a,
-	0x0b, 0xb2, 0x0c, 0xa9, 0x34, 0xdc, 0xfc, 0x08, 0xc8, 0xd5, 0xa5, 0xfc, 0xa5, 0xe3, 0xc7, 0x54,
+	0xb2, 0x04, 0x50, 0x1c, 0x6e, 0x7e, 0x0c, 0xe4, 0xea, 0x52, 0xfe, 0xdc, 0x72, 0x43, 0x2a, 0x67,
-	0xce, 0x14, 0x16, 0x44, 0x83, 0xf7, 0x55, 0xc1, 0xe6, 0xab, 0x32, 0x7a, 0x9c, 0xfb, 0x52, 0xf2,
+	0x0a, 0x0b, 0xa2, 0xc1, 0xfb, 0xaa, 0x60, 0xf3, 0x45, 0x11, 0xad, 0x66, 0xbe, 0x94, 0x7c, 0xf3,
-	0xcd, 0x0f, 0x6d, 0xbb, 0x9d, 0x92, 0x51, 0xae, 0xf1, 0x73, 0xb4, 0x68, 0x1f, 0x75, 0xe4, 0x64,
+	0xc3, 0x4e, 0xa7, 0x15, 0x93, 0x51, 0xae, 0xf1, 0x53, 0xb4, 0xd0, 0x39, 0x6a, 0xcb, 0xc9, 0x50,
-	0x28, 0x87, 0x69, 0x3e, 0x82, 0xe4, 0x24, 0x98, 0x56, 0x9d, 0x79, 0xe1, 0x4b, 0xca, 0xbd, 0xde,
+	0x0e, 0xd3, 0x7c, 0x08, 0x60, 0x3a, 0x19, 0x57, 0x9d, 0x39, 0xfe, 0x73, 0xca, 0x9d, 0xde, 0x08,
-	0x08, 0x88, 0x5b, 0xb1, 0x26, 0x41, 0xfc, 0x3d, 0x2a, 0xa9, 0xcf, 0x23, 0xc5, 0x5a, 0x71, 0xab,
+	0x88, 0x5b, 0x32, 0xd3, 0x49, 0xfc, 0x23, 0x2a, 0xa8, 0xcf, 0x23, 0xf9, 0x4a, 0x7e, 0xab, 0xbc,
-	0xba, 0xb3, 0x7d, 0xdf, 0xec, 0x4c, 0x55, 0xbe, 0x1f, 0x08, 0x3e, 0x4a, 0x9e, 0x32, 0xd9, 0x41,
+	0xb3, 0x7d, 0xdf, 0xec, 0xaa, 0xaa, 0x7c, 0xdf, 0x13, 0x7c, 0x14, 0x3d, 0x65, 0xb4, 0x83, 0x64,
-	0x32, 0xf3, 0x98, 0x8a, 0x01, 0xeb, 0xa6, 0x3c, 0x56, 0x91, 0xbc, 0x43, 0x83, 0x75, 0x47, 0x04,
+	0xe6, 0x31, 0x15, 0x03, 0xd6, 0x8d, 0x79, 0xac, 0x22, 0x79, 0x87, 0x3a, 0xeb, 0x8e, 0x08, 0x56,
-	0xab, 0x3b, 0xc8, 0x35, 0x5e, 0x46, 0x45, 0x7b, 0xaf, 0x9d, 0x30, 0x5b, 0x2e, 0xf1, 0x77, 0x99,
+	0x77, 0x90, 0x6b, 0xbc, 0x84, 0xf2, 0x9d, 0xbd, 0x56, 0xc4, 0x6c, 0xb9, 0xc4, 0x3f, 0x24, 0x9e,
-	0xe7, 0x2d, 0xc1, 0x00, 0x9f, 0x98, 0x4a, 0xec, 0x66, 0x2a, 0x76, 0xb3, 0x99, 0x88, 0x5d, 0x11,
+	0xb7, 0x00, 0x03, 0x7c, 0x54, 0x55, 0x62, 0xaf, 0xc6, 0x62, 0xaf, 0x36, 0x22, 0xb1, 0x2b, 0x22,
-	0xe1, 0xf5, 0x1f, 0x1b, 0x5a, 0x66, 0x06, 0xcf, 0xd1, 0xa2, 0x92, 0xc2, 0xb1, 0x73, 0xd1, 0xf1,
+	0xbc, 0xfc, 0x6b, 0x43, 0x4b, 0xcc, 0xe0, 0x29, 0x5a, 0x50, 0x52, 0x38, 0xb6, 0x2e, 0xda, 0xce,
-	0x7e, 0xa1, 0x44, 0xaf, 0x69, 0x5b, 0x8b, 0xd6, 0x24, 0x88, 0xbf, 0xb9, 0x9b, 0x54, 0x79, 0xf6,
+	0xef, 0x94, 0xe8, 0x15, 0x6d, 0x6b, 0xc1, 0x4c, 0x27, 0xf1, 0x77, 0x77, 0x93, 0x2a, 0x4e, 0x7f,
-	0x53, 0xd2, 0x1e, 0x7c, 0x86, 0x8c, 0x26, 0xe5, 0xb4, 0xef, 0x45, 0x82, 0xf2, 0x3d, 0xee, 0x09,
+	0x4a, 0xdc, 0x83, 0xcf, 0x90, 0xd1, 0xa0, 0x9c, 0xf6, 0x9d, 0x40, 0x50, 0xbe, 0xc7, 0x1d, 0xe1,
-	0xcf, 0x75, 0xfc, 0x44, 0x24, 0xbb, 0x3d, 0x41, 0x39, 0x48, 0x6b, 0xc6, 0x5d, 0xef, 0xd9, 0x0a,
+	0xd8, 0x96, 0x1b, 0x89, 0x64, 0xb7, 0x27, 0x28, 0x07, 0x69, 0x4d, 0xb9, 0xeb, 0x3d, 0x5b, 0x61,
-	0x1b, 0x08, 0x75, 0x5c, 0xee, 0x85, 0x62, 0x97, 0xf7, 0x23, 0x82, 0x80, 0x31, 0x19, 0x04, 0x6f,
+	0x03, 0xa1, 0xb6, 0xcd, 0x1d, 0x5f, 0xec, 0xf2, 0x7e, 0x40, 0x10, 0x30, 0x26, 0x91, 0xc1, 0xdb,
-	0xa3, 0x95, 0x26, 0x73, 0xcf, 0x28, 0xdf, 0x63, 0x81, 0x70, 0xbc, 0x80, 0xf2, 0x56, 0x13, 0xc4,
+	0x68, 0xb9, 0xc1, 0xec, 0x33, 0xca, 0xf7, 0x98, 0x27, 0x2c, 0xc7, 0xa3, 0xbc, 0xd9, 0x00, 0xf1,
-	0xa3, 0x5b, 0xd3, 0x09, 0x49, 0xbd, 0xce, 0x80, 0xfa, 0x7e, 0xa2, 0x5f, 0x15, 0xc8, 0xa1, 0x1d,
+	0xe8, 0xe6, 0x24, 0x20, 0xa9, 0xd7, 0x1e, 0x50, 0xd7, 0x8d, 0xf4, 0xab, 0x02, 0x39, 0xb4, 0xc3,
-	0xee, 0xb4, 0x5b, 0x27, 0x07, 0x64, 0x55, 0x0d, 0x4d, 0x45, 0x72, 0x68, 0x07, 0x56, 0x7b, 0x0f,
+	0x9d, 0x56, 0xf3, 0xe4, 0x80, 0xac, 0xa8, 0xa1, 0xa9, 0x48, 0x0e, 0xed, 0xc0, 0x6c, 0xed, 0x81,
-	0xb4, 0xa4, 0x5b, 0xb0, 0x96, 0xdf, 0x23, 0xff, 0xfe, 0x14, 0x51, 0xfb, 0xa8, 0x03, 0x12, 0xa9,
+	0x96, 0x74, 0x13, 0xd6, 0xf2, 0x7b, 0xe4, 0xdf, 0x5f, 0x02, 0xda, 0x39, 0x6a, 0x83, 0x44, 0x4a,
-	0x58, 0x19, 0x44, 0x5a, 0xd0, 0xae, 0xef, 0x39, 0x11, 0xd8, 0xa7, 0x92, 0xc8, 0x1d, 0x80, 0x37,
+	0x66, 0x22, 0x23, 0x2d, 0x68, 0xd7, 0x75, 0xac, 0x00, 0xec, 0x53, 0x49, 0xe4, 0x2e, 0x81, 0x37,
-	0xd1, 0x03, 0x08, 0x92, 0x2b, 0x26, 0x42, 0x99, 0xc0, 0xf0, 0xe7, 0xa8, 0x68, 0xdb, 0x47, 0x64,
+	0xd1, 0x3c, 0x04, 0xd1, 0x15, 0x23, 0xa1, 0xa4, 0x72, 0xf8, 0x4b, 0x94, 0xef, 0x74, 0x8e, 0xc8,
-	0x65, 0xf6, 0x37, 0x94, 0xf5, 0xeb, 0x3f, 0xa6, 0x22, 0x03, 0x5a, 0x4a, 0x72, 0x9d, 0xd1, 0x51,
+	0xf2, 0xf4, 0x6f, 0x28, 0xeb, 0xd7, 0x7f, 0x8e, 0x45, 0x06, 0xb4, 0x94, 0xe4, 0x3a, 0xa3, 0xa3,
-	0xa2, 0x19, 0xb9, 0xc4, 0xdb, 0x68, 0xe1, 0x1c, 0x64, 0x37, 0x97, 0x58, 0xc3, 0x04, 0xcb, 0x53,
+	0x48, 0x33, 0x72, 0x89, 0xb7, 0xd1, 0xdc, 0x39, 0xc8, 0x6e, 0x26, 0xb2, 0x86, 0x14, 0xcb, 0x63,
-	0x75, 0x5a, 0xaa, 0xe8, 0xeb, 0xb9, 0x2f, 0xb5, 0xcd, 0x5f, 0x75, 0xa4, 0x03, 0xf5, 0xc1, 0xe6,
+	0x75, 0x9a, 0xaa, 0xe8, 0xdb, 0x99, 0xaf, 0xb5, 0xcd, 0x57, 0x3a, 0xd2, 0x81, 0xfa, 0x60, 0x73,
-	0x32, 0xfe, 0xaf, 0xbd, 0x17, 0xff, 0x9f, 0xcb, 0xf5, 0xff, 0x62, 0xbe, 0xff, 0xcf, 0x67, 0xfd,
+	0x09, 0xff, 0xd7, 0xde, 0x89, 0xff, 0xcf, 0x64, 0xfa, 0x7f, 0x3e, 0xdb, 0xff, 0x67, 0x93, 0xfe,
-	0x7f, 0x92, 0x14, 0x0b, 0x53, 0xa4, 0x48, 0x1d, 0xa3, 0x94, 0x71, 0x8c, 0x6f, 0xc7, 0x2a, 0x5f,
+	0x9f, 0x26, 0xc5, 0xdc, 0x04, 0x29, 0x62, 0xc7, 0x28, 0x24, 0x1c, 0xe3, 0xfb, 0xb1, 0xca, 0x57,
-	0x05, 0x95, 0x67, 0x1d, 0x7a, 0x7c, 0xc9, 0x99, 0x94, 0x5d, 0xce, 0x55, 0xf6, 0xfa, 0xb4, 0xb2,
+	0x40, 0xe5, 0x49, 0x87, 0x1e, 0x5f, 0x72, 0x2a, 0x65, 0x17, 0x33, 0x95, 0xbd, 0x3e, 0xa9, 0xec,
-	0x2b, 0xf9, 0xca, 0xd6, 0xdf, 0x45, 0xd9, 0x13, 0xbc, 0x42, 0xf7, 0xf1, 0xaa, 0x9a, 0xc3, 0xab,
+	0x52, 0xb6, 0xb2, 0xf5, 0xb7, 0x51, 0x76, 0x8a, 0x57, 0xe8, 0x3e, 0x5e, 0x95, 0x33, 0x78, 0x95,
-	0x5c, 0xa5, 0x3c, 0xb8, 0x57, 0x29, 0x8b, 0xf9, 0x4a, 0x79, 0x96, 0xab, 0x94, 0xa5, 0xb7, 0x2a,
+	0xa9, 0x94, 0xf9, 0x7b, 0x95, 0xb2, 0x90, 0xad, 0x94, 0x27, 0x99, 0x4a, 0x59, 0x7c, 0xa3, 0x52,
-	0xe5, 0xe1, 0x94, 0x52, 0xa6, 0x2c, 0xfc, 0xe9, 0x4c, 0x16, 0xbe, 0x9c, 0x67, 0xe1, 0x19, 0x47,
+	0x1e, 0x4c, 0x28, 0x65, 0xc2, 0xc2, 0x1f, 0x4f, 0x65, 0xe1, 0x4b, 0x59, 0x16, 0x9e, 0x70, 0xb4,
-	0x5b, 0x79, 0x07, 0x47, 0x4b, 0x24, 0x87, 0xff, 0x9b, 0xe4, 0xf0, 0x0e, 0x5a, 0xed, 0xc4, 0xae,
+	0xe5, 0xb7, 0x70, 0xb4, 0x48, 0x72, 0xf8, 0xff, 0x49, 0x0e, 0xef, 0xa0, 0x95, 0x76, 0x68, 0xdb,
-	0x4b, 0xa3, 0xa8, 0x41, 0x7b, 0x8c, 0xd3, 0xb6, 0x13, 0x45, 0x5e, 0xd0, 0x27, 0x8f, 0xe1, 0x87,
+	0x34, 0x08, 0xea, 0xb4, 0xc7, 0x38, 0x6d, 0x59, 0x41, 0xe0, 0x78, 0x7d, 0xb2, 0x0a, 0x3f, 0x9c,
-	0x33, 0x37, 0x87, 0xbf, 0x40, 0x6b, 0x2f, 0x1c, 0xcf, 0x8f, 0x39, 0x4d, 0x12, 0xa9, 0xeb, 0x91,
+	0x99, 0x18, 0xfe, 0x02, 0xad, 0x3e, 0xb3, 0x1c, 0x37, 0xe4, 0x34, 0x02, 0x7e, 0xb5, 0xb8, 0x27,
-	0x35, 0xe8, 0x7a, 0x4b, 0x56, 0xce, 0xbf, 0xcd, 0xd9, 0xc5, 0x08, 0x78, 0xfd, 0x81, 0x9a, 0xff,
+	0x9b, 0x3e, 0x84, 0xa6, 0x6c, 0x10, 0x7f, 0x85, 0xd6, 0xd2, 0x40, 0xec, 0x95, 0x64, 0x0d, 0xda,
-	0x18, 0x18, 0x67, 0x61, 0x08, 0x24, 0x93, 0x85, 0x49, 0xdc, 0x6f, 0xd8, 0x8f, 0xde, 0x9f, 0x61,
+	0xde, 0x80, 0x4a, 0xd6, 0xb4, 0x38, 0xbb, 0x18, 0x81, 0x1a, 0x3e, 0x50, 0xac, 0x19, 0x27, 0xc6,
-	0x4f, 0xfd, 0x04, 0x3d, 0x81, 0x7b, 0x4d, 0x82, 0xff, 0x83, 0x5b, 0x35, 0x8e, 0x2f, 0xff, 0x32,
+	0x28, 0x8c, 0x8e, 0x24, 0x50, 0x98, 0xdf, 0xfd, 0x36, 0xff, 0xf0, 0xdd, 0xd9, 0xfc, 0xc4, 0x0f,
-	0x0a, 0x97, 0x37, 0x86, 0x76, 0x75, 0x63, 0x68, 0x7f, 0xde, 0x18, 0xda, 0xab, 0x5b, 0xa3, 0xf0,
+	0xd7, 0x23, 0xb8, 0x57, 0x3a, 0xf9, 0x1e, 0x3c, 0xae, 0x7e, 0x7c, 0xf9, 0xca, 0xc8, 0x5d, 0xde,
-	0xfa, 0xd6, 0x28, 0x5c, 0xdd, 0x1a, 0x85, 0xdf, 0x6e, 0x8d, 0xc2, 0xcf, 0x9f, 0xfc, 0x9b, 0x59,
+	0x18, 0xda, 0xd5, 0x8d, 0xa1, 0xfd, 0x7d, 0x63, 0x68, 0x2f, 0x6e, 0x8d, 0xdc, 0xcb, 0x5b, 0x23,
-	0xbd, 0xf1, 0x2f, 0xf8, 0x69, 0x09, 0x80, 0xcf, 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, 0xce, 0x2e,
+	0x77, 0x75, 0x6b, 0xe4, 0xfe, 0xb8, 0x35, 0x72, 0xbf, 0x7d, 0xfa, 0x5f, 0x16, 0xf7, 0xda, 0x3f,
-	0x2d, 0x41, 0x9c, 0x0b, 0x00, 0x00,
+	0xee, 0xa7, 0x05, 0x48, 0x7c, 0xfe, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8b, 0x36, 0xc6, 0x76,
+	0xd2, 0x0b, 0x00, 0x00,
 }
 
 func (m *HealthCheck) Marshal() (dAtA []byte, err error) {
@@ -757,6 +760,13 @@ func (m *CheckType) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
+	if m.FailuresBeforeWarning != 0 {
+		i = encodeVarintHealthcheck(dAtA, i, uint64(m.FailuresBeforeWarning))
+		i--
+		dAtA[i] = 0x1
+		i--
+		dAtA[i] = 0xe8
+	}
 	if len(m.H2PING) > 0 {
 		i -= len(m.H2PING)
 		copy(dAtA[i:], m.H2PING)
@@ -1296,6 +1306,9 @@ func (m *CheckType) Size() (n int) {
 	if l > 0 {
 		n += 2 + l + sovHealthcheck(uint64(l))
 	}
+	if m.FailuresBeforeWarning != 0 {
+		n += 2 + sovHealthcheck(uint64(m.FailuresBeforeWarning))
+	}
 	return n
 }
 
@@ -3656,6 +3669,25 @@ func (m *CheckType) Unmarshal(dAtA []byte) error {
 			}
 			m.H2PING = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 29:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field FailuresBeforeWarning", wireType)
+			}
+			m.FailuresBeforeWarning = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowHealthcheck
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.FailuresBeforeWarning |= int32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		default:
 			iNdEx = preIndex
 			skippy, err := skipHealthcheck(dAtA[iNdEx:])

proto/pbservice/healthcheck.proto

@@ -136,6 +136,8 @@ message CheckType {
     // mog: func-to=int func-from=int32
     int32 SuccessBeforePassing = 21;
     // mog: func-to=int func-from=int32
+    int32 FailuresBeforeWarning = 29;
+    // mog: func-to=int func-from=int32
     int32 FailuresBeforeCritical = 22;
 
     // Definition fields used when exposing checks through a proxy

website/content/api-docs/agent/check.mdx

@@ -240,11 +240,16 @@ The table below shows this endpoint's support for
 
 - `SuccessBeforePassing` `(int: 0)` - Specifies the number of consecutive successful
   results required before check status transitions to passing. Available for HTTP,
-  TCP, gRPC, Docker & Monitor checks.
+  TCP, gRPC, Docker & Monitor checks. Added in Consul 1.7.0.
+
+- `FailuresBeforeWarning` `(int: 0)` - Specifies the number of consecutive unsuccessful
+  results required before check status transitions to warning. Defaults to the same value
+  as `FailuresBeforeCritical`. Values higher than `FailuresBeforeCritical` are invalid.
+  Available for HTTP, TCP, gRPC, Docker & Monitor checks. Added in Consul 1.11.0.
 
 - `FailuresBeforeCritical` `(int: 0)` - Specifies the number of consecutive unsuccessful
   results required before check status transitions to critical. Available for HTTP,
-  TCP, gRPC, Docker & Monitor checks.
+  TCP, gRPC, Docker & Monitor checks. Added in Consul 1.7.0.
 
 ### Sample Payload
 

website/content/docs/discovery/checks.mdx

@@ -413,12 +413,23 @@ key in your configuration file.
 }
 ```
 
-## Success/Failures before passing/critical
+## Success/Failures before passing/warning/critical
 
-In Consul 1.7.0 and later, a check may be configured to become passing/critical
+To prevent flapping health checks, and limit the load they cause on the cluster,
-only after a specified number of consecutive checks return passing/critical.
+a health check may be configured to become passing/warning/critical only after a
+specified number of consecutive checks return passing/critical.
 The status will not transition states until the configured threshold is reached.
 
+* `success_before_passing` - Number of consecutive successful results required
+  before check status transitions to passing. Defaults to `0`.  Added in Consul 1.7.0.
+* `failures_before_warning` - Number of consecutive unsuccessful results required
+  before check status transitions to warning. Defaults to the same value as that of
+  `failures_before_critical` to maintain the expected behavior of not changing the
+  status of serivce checks to `warning` before `critical` unless configured to do so.
+  Values higher than `failures_before_critical` are invalid. Added in Consul 1.11.0.
+* `failures_before_critical` - Number of consecutive unsuccessful results required
+  before check status transitions to critical. Defaults to `0`. Added in Consul 1.7.0.
+
 This feature is available for HTTP, TCP, gRPC, Docker & Monitor checks.
 By default, both passing and critical thresholds will be set to 0 so the check
 status will always reflect the last check result.
@@ -432,6 +443,7 @@ status will always reflect the last check result.
       "interval": "10s",
       "timeout": "1s",
       "success_before_passing": 3,
+      "failures_before_warning": 1,
       "failures_before_critical": 3
     }
   ]