Test fixes (#2434)

* Test fixes in health_endpoint_test.go, agent_endpoint_test.go and rtt_test.go
* Don't reuse the same agent config in TestAgent_ReconnectConfigSettings

command/agent/agent_endpoint_test.go

@@ -189,9 +189,15 @@ func TestHTTPAgentJoin(t *testing.T) {
 		t.Fatalf("Err: %v", obj)
 	}
 
-	if len(a2.LANMembers()) != 2 {
+	if len(srv.agent.LANMembers()) != 2 {
 		t.Fatalf("should have 2 members")
 	}
+
+	testutil.WaitForResult(func() (bool, error) {
+		return len(a2.LANMembers()) == 2, nil
+	}, func(err error) {
+		t.Fatalf("should have 2 members")
+	})
 }
 
 func TestHTTPAgentJoin_WAN(t *testing.T) {
@@ -218,6 +224,10 @@ func TestHTTPAgentJoin_WAN(t *testing.T) {
 		t.Fatalf("Err: %v", obj)
 	}
 
+	if len(srv.agent.WANMembers()) != 2 {
+		t.Fatalf("should have 2 members")
+	}
+
 	testutil.WaitForResult(func() (bool, error) {
 		return len(a2.WANMembers()) == 2, nil
 	}, func(err error) {

command/agent/agent_test.go

@@ -248,6 +248,7 @@ func TestAgent_ReconnectConfigSettings(t *testing.T) {
 		}
 	}()
 
+	c = nextConfig()
 	c.ReconnectTimeoutLan = 24 * time.Hour
 	c.ReconnectTimeoutWan = 36 * time.Hour
 	func() {

command/agent/health_endpoint_test.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"reflect"
 	"testing"
-	"time"
 
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/hashicorp/consul/testutil"
@@ -126,25 +125,29 @@ func TestHealthChecksInState_DistanceSort(t *testing.T) {
 	if err := srv.agent.RPC("Coordinate.Update", &arg, &out); err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	time.Sleep(300 * time.Millisecond)
 
-	// Query again and now foo should have moved to the front of the line.
+	// Retry until foo moves to the front of the line.
+	testutil.WaitForResult(func() (bool, error) {
 		resp = httptest.NewRecorder()
 		obj, err = srv.HealthChecksInState(resp, req)
 		if err != nil {
-		t.Fatalf("err: %v", err)
+			return false, fmt.Errorf("err: %v", err)
 		}
 		assertIndex(t, resp)
 		nodes = obj.(structs.HealthChecks)
 		if len(nodes) != 2 {
-		t.Fatalf("bad: %v", nodes)
+			return false, fmt.Errorf("bad: %v", nodes)
 		}
 		if nodes[0].Node != "foo" {
-		t.Fatalf("bad: %v", nodes)
+			return false, fmt.Errorf("bad: %v", nodes)
 		}
 		if nodes[1].Node != "bar" {
-		t.Fatalf("bad: %v", nodes)
+			return false, fmt.Errorf("bad: %v", nodes)
 		}
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("failed to get sorted service nodes: %v", err)
+	})
 }
 
 func TestHealthNodeChecks(t *testing.T) {
@@ -320,25 +323,29 @@ func TestHealthServiceChecks_DistanceSort(t *testing.T) {
 	if err := srv.agent.RPC("Coordinate.Update", &arg, &out); err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	time.Sleep(300 * time.Millisecond)
 
-	// Query again and now foo should have moved to the front of the line.
+	// Retry until foo has moved to the front of the line.
+	testutil.WaitForResult(func() (bool, error) {
 		resp = httptest.NewRecorder()
 		obj, err = srv.HealthServiceChecks(resp, req)
 		if err != nil {
-		t.Fatalf("err: %v", err)
+			return false, fmt.Errorf("err: %v", err)
 		}
 		assertIndex(t, resp)
 		nodes = obj.(structs.HealthChecks)
 		if len(nodes) != 2 {
-		t.Fatalf("bad: %v", obj)
+			return false, fmt.Errorf("bad: %v", obj)
 		}
 		if nodes[0].Node != "foo" {
-		t.Fatalf("bad: %v", nodes)
+			return false, fmt.Errorf("bad: %v", nodes)
 		}
 		if nodes[1].Node != "bar" {
-		t.Fatalf("bad: %v", nodes)
+			return false, fmt.Errorf("bad: %v", nodes)
 		}
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("failed to get sorted service checks: %v", err)
+	})
 }
 
 func TestHealthServiceNodes(t *testing.T) {
@@ -487,25 +494,29 @@ func TestHealthServiceNodes_DistanceSort(t *testing.T) {
 	if err := srv.agent.RPC("Coordinate.Update", &arg, &out); err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	time.Sleep(300 * time.Millisecond)
 
-	// Query again and now foo should have moved to the front of the line.
+	// Retry until foo has moved to the front of the line.
+	testutil.WaitForResult(func() (bool, error) {
 		resp = httptest.NewRecorder()
 		obj, err = srv.HealthServiceNodes(resp, req)
 		if err != nil {
-		t.Fatalf("err: %v", err)
+			return false, fmt.Errorf("err: %v", err)
 		}
 		assertIndex(t, resp)
 		nodes = obj.(structs.CheckServiceNodes)
 		if len(nodes) != 2 {
-		t.Fatalf("bad: %v", obj)
+			return false, fmt.Errorf("bad: %v", obj)
 		}
 		if nodes[0].Node.Node != "foo" {
-		t.Fatalf("bad: %v", nodes)
+			return false, fmt.Errorf("bad: %v", nodes)
 		}
 		if nodes[1].Node.Node != "bar" {
-		t.Fatalf("bad: %v", nodes)
+			return false, fmt.Errorf("bad: %v", nodes)
 		}
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("failed to get sorted service nodes: %v", err)
+	})
 }
 
 func TestHealthServiceNodes_PassingFilter(t *testing.T) {

command/rtt_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/hashicorp/consul/command/agent"
 	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/serf/coordinate"
 	"github.com/mitchellh/cli"
 )
@@ -88,11 +89,7 @@ func TestRTTCommand_Run_LAN(t *testing.T) {
 		}
 	}
 
-	// Wait for the updates to get flushed to the data store.
+	// Ask for the RTT of two known nodes
-	time.Sleep(2 * updatePeriod)
-
-	// Try two known nodes.
-	{
 	ui := new(cli.MockUi)
 	c := &RTTCommand{Ui: ui}
 	args := []string{
@@ -100,18 +97,25 @@ func TestRTTCommand_Run_LAN(t *testing.T) {
 		a.config.NodeName,
 		"dogs",
 	}
+
+	// Wait for the updates to get flushed to the data store.
+	testutil.WaitForResult(func() (bool, error) {
 		code := c.Run(args)
 		if code != 0 {
-			t.Fatalf("bad: %d: %#v", code, ui.ErrorWriter.String())
+			return false, fmt.Errorf("bad: %d: %#v", code, ui.ErrorWriter.String())
 		}
 
 		// Make sure the proper RTT was reported in the output.
 		expected := fmt.Sprintf("rtt: %s", dist_str)
 		if !strings.Contains(ui.OutputWriter.String(), expected) {
-			t.Fatalf("bad: %#v", ui.OutputWriter.String())
+			return false, fmt.Errorf("bad: %#v", ui.OutputWriter.String())
-		}
 		}
 
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("failed to get proper RTT output: %v", err)
+	})
+
 	// Default to the agent's node.
 	{
 		ui := new(cli.MockUi)