Merge pull request #2683 from hashicorp/f-unit-test-robustify

Robustifies unit tests.

testutil/server.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/consul/consul/structs"
@@ -292,11 +293,13 @@ func (s *TestServer) waitForAPI() {
 // waitForLeader waits for the Consul server's HTTP API to become
 // available, and then waits for a known leader and an index of
 // 1 or more to be observed to confirm leader election is done.
-// It then waits to ensure the anti-entropy checks have completed.
+// It then waits to ensure the anti-entropy sync has completed.
 func (s *TestServer) waitForLeader() {
+	var index int64
 	WaitForResult(func() (bool, error) {
-		// Query the API and check the status code
-		resp, err := s.HttpClient.Get(s.url("/v1/catalog/nodes"))
+		// Query the API and check the status code.
+		url := s.url(fmt.Sprintf("/v1/catalog/nodes?index=%d&wait=10s", index))
+		resp, err := s.HttpClient.Get(url)
 		if err != nil {
 			return false, err
 		}
@@ -305,25 +308,27 @@ func (s *TestServer) waitForLeader() {
 			return false, err
 		}
 
-		// Ensure we have a leader and a node registration
+		// Ensure we have a leader and a node registration.
 		if leader := resp.Header.Get("X-Consul-KnownLeader"); leader != "true" {
-			fmt.Println(leader)
 			return false, fmt.Errorf("Consul leader status: %#v", leader)
 		}
-		if resp.Header.Get("X-Consul-Index") == "0" {
+		index, err = strconv.ParseInt(resp.Header.Get("X-Consul-Index"), 10, 64)
+		if err != nil {
+			return false, fmt.Errorf("Consul index was bad: %v", err)
+		}
+		if index == 0 {
 			return false, fmt.Errorf("Consul index is 0")
 		}
 
+		// Watch for the anti-entropy sync to finish.
 		var parsed []map[string]interface{}
 		dec := json.NewDecoder(resp.Body)
 		if err := dec.Decode(&parsed); err != nil {
 			return false, err
 		}
-
 		if len(parsed) < 1 {
 			return false, fmt.Errorf("No nodes")
 		}
-
 		taggedAddresses, ok := parsed[0]["TaggedAddresses"].(map[string]interface{})
 		if !ok {
 			return false, fmt.Errorf("Missing tagged addresses")

testutil/wait.go

@@ -10,22 +10,28 @@ import (
 type testFn func() (bool, error)
 type errorFn func(error)
 
-func WaitForResult(test testFn, error errorFn) {
-	retries := 400
+const (
+	baseWait = 1 * time.Millisecond
+	maxWait  = 100 * time.Millisecond
+)
 
-	for retries > 0 {
-		time.Sleep(100 * time.Millisecond)
-		retries--
-
-		success, err := test()
+func WaitForResult(try testFn, fail errorFn) {
+	var err error
+	wait := baseWait
+	for retries := 100; retries > 0; retries-- {
+		var success bool
+		success, err = try()
 		if success {
 			return
 		}
 
-		if retries == 0 {
-			error(err)
+		time.Sleep(wait)
+		wait *= 2
+		if wait > maxWait {
+			wait = maxWait
 		}
 	}
+	fail(err)
 }
 
 type rpcFn func(string, interface{}, interface{}) error