testing: fix flakey check status test

This PR fixes a flakey test where we did not wait on the check
status to actually become failing (go too fast and you just get
a pending check).

Instead add a helper for waiting on any check in the alloc to become
the state we are looking for.

command/alloc_status_test.go

@@ -467,8 +467,8 @@ func TestAllocStatusCommand_NSD_Checks(t *testing.T) {
 	// Get an alloc id
 	allocID := getAllocFromJob(t, client, jobID)
 
-	// do not wait for alloc running - it will stay pending because the
-	// health-check will never pass
+	// wait for the check to be marked failure
+	waitForCheckStatus(t, client, allocID, "failure")
 
 	// Run command
 	cmd := &AllocStatusCommand{Meta: Meta{Ui: ui, flagAddress: url}}

command/testing_test.go

@@ -164,6 +164,26 @@ func waitForAllocRunning(t *testing.T, client *api.Client, allocID string) {
 	})
 }
 
+func waitForCheckStatus(t *testing.T, client *api.Client, allocID, status string) {
+	testutil.WaitForResult(func() (bool, error) {
+		results, err := client.Allocations().Checks(allocID, nil)
+		if err != nil {
+			return false, err
+		}
+
+		// pick a check, any check will do
+		for _, check := range results {
+			if check.Status == status {
+				return true, nil
+			}
+		}
+
+		return false, fmt.Errorf("no check with status: %s", status)
+	}, func(err error) {
+		t.Fatalf("timed out waiting for alloc to be running: %v", err)
+	})
+}
+
 func getAllocFromJob(t *testing.T, client *api.Client, jobID string) string {
 	var allocID string
 	if allocations, _, err := client.Jobs().Allocations(jobID, false, nil); err == nil {