test: make joinLAN/WAN reliable

only return if the members can see each other

agent/consul/helper_test.go

@@ -1,12 +1,34 @@
 package consul
 
 import (
+	"errors"
 	"fmt"
 	"testing"
+
+	"github.com/hashicorp/consul/testutil/retry"
+	"github.com/hashicorp/serf/serf"
 )
 
+func waitForLeader(servers ...*Server) error {
+	if len(servers) == 0 {
+		return errors.New("no servers")
+	}
+	dc := servers[0].config.Datacenter
+	for _, s := range servers {
+		if s.config.Datacenter != dc {
+			return fmt.Errorf("servers are in different datacenters %s and %s", s.config.Datacenter, dc)
+		}
+	}
+	for _, s := range servers {
+		if s.IsLeader() {
+			return nil
+		}
+	}
+	return errors.New("no leader")
+}
+
 // wantPeers determines whether the server has the given
-// number of peers.
+// number of raft peers.
 func wantPeers(s *Server, peers int) error {
 	n, err := s.numPeers()
 	if err != nil {
@@ -40,6 +62,7 @@ func joinAddrWAN(s *Server) string {
 
 type clientOrServer interface {
 	JoinLAN(addrs []string) (int, error)
+	LANMembers() []serf.Member
 }
 
 // joinLAN is a convenience function for
@@ -49,10 +72,25 @@ func joinLAN(t *testing.T, member clientOrServer, leader *Server) {
 	if member == nil || leader == nil {
 		panic("no server")
 	}
-	addr := []string{joinAddrLAN(leader)}
-	if _, err := member.JoinLAN(addr); err != nil {
+	var memberAddr string
+	switch x := member.(type) {
+	case *Server:
+		memberAddr = joinAddrLAN(x)
+	case *Client:
+		memberAddr = fmt.Sprintf("127.0.0.1:%d", x.config.SerfLANConfig.MemberlistConfig.BindPort)
+	}
+	leaderAddr := joinAddrLAN(leader)
+	if _, err := member.JoinLAN([]string{leaderAddr}); err != nil {
 		t.Fatal(err)
 	}
+	retry.Run(t, func(r *retry.R) {
+		if !seeEachOther(leader.LANMembers(), member.LANMembers(), leaderAddr, memberAddr) {
+			r.Fatalf("leader and member cannot see each other on LAN")
+		}
+	})
+	if !seeEachOther(leader.LANMembers(), member.LANMembers(), leaderAddr, memberAddr) {
+		t.Fatalf("leader and member cannot see each other on LAN")
+	}
 }
 
 // joinWAN is a convenience function for
@@ -62,8 +100,30 @@ func joinWAN(t *testing.T, member, leader *Server) {
 	if member == nil || leader == nil {
 		panic("no server")
 	}
-	addr := []string{joinAddrWAN(leader)}
-	if _, err := member.JoinWAN(addr); err != nil {
+	leaderAddr, memberAddr := joinAddrWAN(leader), joinAddrWAN(member)
+	if _, err := member.JoinWAN([]string{leaderAddr}); err != nil {
 		t.Fatal(err)
 	}
+	retry.Run(t, func(r *retry.R) {
+		if !seeEachOther(leader.WANMembers(), member.WANMembers(), leaderAddr, memberAddr) {
+			r.Fatalf("leader and member cannot see each other on WAN")
+		}
+	})
+	if !seeEachOther(leader.WANMembers(), member.WANMembers(), leaderAddr, memberAddr) {
+		t.Fatalf("leader and member cannot see each other on WAN")
+	}
+}
+
+func seeEachOther(a, b []serf.Member, addra, addrb string) bool {
+	return serfMembersContains(a, addrb) && serfMembersContains(b, addra)
+}
+
+func serfMembersContains(members []serf.Member, addr string) bool {
+	for _, m := range members {
+		maddr := fmt.Sprintf("%s:%d", m.Addr.String(), m.Port)
+		if maddr == addr {
+			return true
+		}
+	}
+	return false
 }