package consul import ( "fmt" "net/rpc" "os" "strings" "testing" "time" "github.com/hashicorp/consul/consul/structs" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/testutil" "github.com/hashicorp/net-rpc-msgpackrpc" ) // verifyNodeSort makes sure the order of the nodes in the slice is the same as // the expected order, expressed as a comma-separated string. func verifyNodeSort(t *testing.T, nodes structs.Nodes, expected string) { vec := make([]string, len(nodes)) for i, node := range nodes { vec[i] = node.Node } actual := strings.Join(vec, ",") if actual != expected { t.Fatalf("bad sort: %s != %s", actual, expected) } } // verifyServiceNodeSort makes sure the order of the nodes in the slice is the // same as the expected order, expressed as a comma-separated string. func verifyServiceNodeSort(t *testing.T, nodes structs.ServiceNodes, expected string) { vec := make([]string, len(nodes)) for i, node := range nodes { vec[i] = node.Node } actual := strings.Join(vec, ",") if actual != expected { t.Fatalf("bad sort: %s != %s", actual, expected) } } // verifyHealthCheckSort makes sure the order of the nodes in the slice is the // same as the expected order, expressed as a comma-separated string. func verifyHealthCheckSort(t *testing.T, checks structs.HealthChecks, expected string) { vec := make([]string, len(checks)) for i, check := range checks { vec[i] = check.Node } actual := strings.Join(vec, ",") if actual != expected { t.Fatalf("bad sort: %s != %s", actual, expected) } } // verifyCheckServiceNodeSort makes sure the order of the nodes in the slice is // the same as the expected order, expressed as a comma-separated string. func verifyCheckServiceNodeSort(t *testing.T, nodes structs.CheckServiceNodes, expected string) { vec := make([]string, len(nodes)) for i, node := range nodes { vec[i] = node.Node.Node } actual := strings.Join(vec, ",") if actual != expected { t.Fatalf("bad sort: %s != %s", actual, expected) } } // seedCoordinates uses the client to set up a set of nodes with a specific // set of distances from the origin. We also include the server so that we // can wait for the coordinates to get committed to the Raft log. // // Here's the layout of the nodes: // // node3 node2 node5 node4 node1 // | | | | | | | | | | | // 0 1 2 3 4 5 6 7 8 9 10 (ms) // func seedCoordinates(t *testing.T, codec rpc.ClientCodec, server *Server) { // Register some nodes. for i := 0; i < 5; i++ { req := structs.RegisterRequest{ Datacenter: "dc1", Node: fmt.Sprintf("node%d", i+1), Address: "127.0.0.1", } var reply struct{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &req, &reply); err != nil { t.Fatalf("err: %v", err) } } // Seed the fixed setup of the nodes. updates := []structs.CoordinateUpdateRequest{ structs.CoordinateUpdateRequest{ Datacenter: "dc1", Node: "node1", Coord: lib.GenerateCoordinate(10 * time.Millisecond), }, structs.CoordinateUpdateRequest{ Datacenter: "dc1", Node: "node2", Coord: lib.GenerateCoordinate(2 * time.Millisecond), }, structs.CoordinateUpdateRequest{ Datacenter: "dc1", Node: "node3", Coord: lib.GenerateCoordinate(1 * time.Millisecond), }, structs.CoordinateUpdateRequest{ Datacenter: "dc1", Node: "node4", Coord: lib.GenerateCoordinate(8 * time.Millisecond), }, structs.CoordinateUpdateRequest{ Datacenter: "dc1", Node: "node5", Coord: lib.GenerateCoordinate(3 * time.Millisecond), }, } // Apply the updates and wait a while for the batch to get committed to // the Raft log. for _, update := range updates { var out struct{} if err := msgpackrpc.CallWithCodec(codec, "Coordinate.Update", &update, &out); err != nil { t.Fatalf("err: %v", err) } } time.Sleep(2 * server.config.CoordinateUpdatePeriod) } func TestRTT_sortNodesByDistanceFrom(t *testing.T) { dir, server := testServer(t) defer os.RemoveAll(dir) defer server.Shutdown() codec := rpcClient(t, server) defer codec.Close() testutil.WaitForLeader(t, server.RPC, "dc1") seedCoordinates(t, codec, server) nodes := structs.Nodes{ &structs.Node{Node: "apple"}, &structs.Node{Node: "node1"}, &structs.Node{Node: "node2"}, &structs.Node{Node: "node3"}, &structs.Node{Node: "node4"}, &structs.Node{Node: "node5"}, } // The zero value for the source should not trigger any sorting. var source structs.QuerySource if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyNodeSort(t, nodes, "apple,node1,node2,node3,node4,node5") // Same for a source in some other DC. source.Node = "node1" source.Datacenter = "dc2" if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyNodeSort(t, nodes, "apple,node1,node2,node3,node4,node5") // Same for a source node in our DC that we have no coordinate for. source.Node = "apple" source.Datacenter = "dc1" if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyNodeSort(t, nodes, "apple,node1,node2,node3,node4,node5") // Now sort relative to node1, note that apple doesn't have any seeded // coordinate info so it should end up at the end, despite its lexical // hegemony. source.Node = "node1" if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyNodeSort(t, nodes, "node1,node4,node5,node2,node3,apple") } func TestRTT_sortNodesByDistanceFrom_Nodes(t *testing.T) { dir, server := testServer(t) defer os.RemoveAll(dir) defer server.Shutdown() codec := rpcClient(t, server) defer codec.Close() testutil.WaitForLeader(t, server.RPC, "dc1") seedCoordinates(t, codec, server) nodes := structs.Nodes{ &structs.Node{Node: "apple"}, &structs.Node{Node: "node1"}, &structs.Node{Node: "node2"}, &structs.Node{Node: "node3"}, &structs.Node{Node: "node4"}, &structs.Node{Node: "node5"}, } // Now sort relative to node1, note that apple doesn't have any // seeded coordinate info so it should end up at the end, despite // its lexical hegemony. var source structs.QuerySource source.Node = "node1" source.Datacenter = "dc1" if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyNodeSort(t, nodes, "node1,node4,node5,node2,node3,apple") // Try another sort from node2. Note that node5 and node3 are the // same distance away so the stable sort should preserve the order // they were in from the previous sort. source.Node = "node2" source.Datacenter = "dc1" if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyNodeSort(t, nodes, "node2,node5,node3,node4,node1,apple") // Let's exercise the stable sort explicitly to make sure we didn't // just get lucky. nodes[1], nodes[2] = nodes[2], nodes[1] if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyNodeSort(t, nodes, "node2,node3,node5,node4,node1,apple") } func TestRTT_sortNodesByDistanceFrom_ServiceNodes(t *testing.T) { dir, server := testServer(t) defer os.RemoveAll(dir) defer server.Shutdown() codec := rpcClient(t, server) defer codec.Close() testutil.WaitForLeader(t, server.RPC, "dc1") seedCoordinates(t, codec, server) nodes := structs.ServiceNodes{ &structs.ServiceNode{Node: "apple"}, &structs.ServiceNode{Node: "node1"}, &structs.ServiceNode{Node: "node2"}, &structs.ServiceNode{Node: "node3"}, &structs.ServiceNode{Node: "node4"}, &structs.ServiceNode{Node: "node5"}, } // Now sort relative to node1, note that apple doesn't have any // seeded coordinate info so it should end up at the end, despite // its lexical hegemony. var source structs.QuerySource source.Node = "node1" source.Datacenter = "dc1" if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyServiceNodeSort(t, nodes, "node1,node4,node5,node2,node3,apple") // Try another sort from node2. Note that node5 and node3 are the // same distance away so the stable sort should preserve the order // they were in from the previous sort. source.Node = "node2" source.Datacenter = "dc1" if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyServiceNodeSort(t, nodes, "node2,node5,node3,node4,node1,apple") // Let's exercise the stable sort explicitly to make sure we didn't // just get lucky. nodes[1], nodes[2] = nodes[2], nodes[1] if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyServiceNodeSort(t, nodes, "node2,node3,node5,node4,node1,apple") } func TestRTT_sortNodesByDistanceFrom_HealthChecks(t *testing.T) { dir, server := testServer(t) defer os.RemoveAll(dir) defer server.Shutdown() codec := rpcClient(t, server) defer codec.Close() testutil.WaitForLeader(t, server.RPC, "dc1") seedCoordinates(t, codec, server) checks := structs.HealthChecks{ &structs.HealthCheck{Node: "apple"}, &structs.HealthCheck{Node: "node1"}, &structs.HealthCheck{Node: "node2"}, &structs.HealthCheck{Node: "node3"}, &structs.HealthCheck{Node: "node4"}, &structs.HealthCheck{Node: "node5"}, } // Now sort relative to node1, note that apple doesn't have any // seeded coordinate info so it should end up at the end, despite // its lexical hegemony. var source structs.QuerySource source.Node = "node1" source.Datacenter = "dc1" if err := server.sortNodesByDistanceFrom(source, checks); err != nil { t.Fatalf("err: %v", err) } verifyHealthCheckSort(t, checks, "node1,node4,node5,node2,node3,apple") // Try another sort from node2. Note that node5 and node3 are the // same distance away so the stable sort should preserve the order // they were in from the previous sort. source.Node = "node2" source.Datacenter = "dc1" if err := server.sortNodesByDistanceFrom(source, checks); err != nil { t.Fatalf("err: %v", err) } verifyHealthCheckSort(t, checks, "node2,node5,node3,node4,node1,apple") // Let's exercise the stable sort explicitly to make sure we didn't // just get lucky. checks[1], checks[2] = checks[2], checks[1] if err := server.sortNodesByDistanceFrom(source, checks); err != nil { t.Fatalf("err: %v", err) } verifyHealthCheckSort(t, checks, "node2,node3,node5,node4,node1,apple") } func TestRTT_sortNodesByDistanceFrom_CheckServiceNodes(t *testing.T) { dir, server := testServer(t) defer os.RemoveAll(dir) defer server.Shutdown() codec := rpcClient(t, server) defer codec.Close() testutil.WaitForLeader(t, server.RPC, "dc1") seedCoordinates(t, codec, server) nodes := structs.CheckServiceNodes{ structs.CheckServiceNode{Node: &structs.Node{Node: "apple"}}, structs.CheckServiceNode{Node: &structs.Node{Node: "node1"}}, structs.CheckServiceNode{Node: &structs.Node{Node: "node2"}}, structs.CheckServiceNode{Node: &structs.Node{Node: "node3"}}, structs.CheckServiceNode{Node: &structs.Node{Node: "node4"}}, structs.CheckServiceNode{Node: &structs.Node{Node: "node5"}}, } // Now sort relative to node1, note that apple doesn't have any // seeded coordinate info so it should end up at the end, despite // its lexical hegemony. var source structs.QuerySource source.Node = "node1" source.Datacenter = "dc1" if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyCheckServiceNodeSort(t, nodes, "node1,node4,node5,node2,node3,apple") // Try another sort from node2. Note that node5 and node3 are the // same distance away so the stable sort should preserve the order // they were in from the previous sort. source.Node = "node2" source.Datacenter = "dc1" if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyCheckServiceNodeSort(t, nodes, "node2,node5,node3,node4,node1,apple") // Let's exercise the stable sort explicitly to make sure we didn't // just get lucky. nodes[1], nodes[2] = nodes[2], nodes[1] if err := server.sortNodesByDistanceFrom(source, nodes); err != nil { t.Fatalf("err: %v", err) } verifyCheckServiceNodeSort(t, nodes, "node2,node3,node5,node4,node1,apple") } /* // mockNodeMap is keyed by node name and the values are the coordinates of the // node. type mockNodeMap map[string]*coordinate.Coordinate // mockServer is used to provide a serfer interface for unit tests. The key is // DC, which selects a map from node name to coordinate for that node. type mockServer map[string]mockNodeMap // newMockServer is used to generate a serfer interface that presents a known DC // topology for unit tests. The server is in dc0. // // Here's the layout of the nodes: // // /---- dc1 ----\ /- dc2 -\ /- dc0 -\ // node2 node1 node3 node1 node1 // | | | | | | | | | | | // 0 1 2 3 4 5 6 7 8 9 10 (ms) // // We also include a node4 in dc1 with no known coordinate, as well as a // mysterious dcX with no nodes with known coordinates. // func newMockServer() *mockServer { s := make(mockServer) s["dc0"] = mockNodeMap{ "dc0.node1": lib.GenerateCoordinate(10 * time.Millisecond), } s["dc1"] = mockNodeMap{ "dc1.node1": lib.GenerateCoordinate(3 * time.Millisecond), "dc1.node2": lib.GenerateCoordinate(2 * time.Millisecond), "dc1.node3": lib.GenerateCoordinate(5 * time.Millisecond), "dc1.node4": nil, // no known coordinate } s["dc2"] = mockNodeMap{ "dc2.node1": lib.GenerateCoordinate(8 * time.Millisecond), } s["dcX"] = mockNodeMap{ "dcX.node1": nil, // no known coordinate } return &s } // See serfer. func (s *mockServer) GetDatacenter() string { return "dc0" } // See serfer. func (s *mockServer) GetCoordinate() (*coordinate.Coordinate, error) { return (*s)["dc0"]["dc0.node1"], nil } // See serfer. func (s *mockServer) GetCachedCoordinate(node string) (*coordinate.Coordinate, bool) { for _, nodes := range *s { for n, coord := range nodes { if n == node && coord != nil { return coord, true } } } return nil, false } // See serfer. func (s *mockServer) GetNodesForDatacenter(dc string) []string { nodes := make([]string, 0) if n, ok := (*s)[dc]; ok { for name := range n { nodes = append(nodes, name) } } sort.Strings(nodes) return nodes } func TestRTT_getDatacenterDistance(t *testing.T) { s := newMockServer() // The serfer's own DC is always 0 ms away. if dist, err := getDatacenterDistance(s, "dc0"); err != nil || dist != 0.0 { t.Fatalf("bad: %v err: %v", dist, err) } // Check a DC with no coordinates, which should give positive infinity. if dist, err := getDatacenterDistance(s, "dcX"); err != nil || dist != math.Inf(1.0) { t.Fatalf("bad: %v err: %v", dist, err) } // Similar for a totally unknown DC. if dist, err := getDatacenterDistance(s, "acdc"); err != nil || dist != math.Inf(1.0) { t.Fatalf("bad: %v err: %v", dist, err) } // Check the trivial median case (just one node). if dist, err := getDatacenterDistance(s, "dc2"); err != nil || dist != 0.002 { t.Fatalf("bad: %v err: %v", dist, err) } // Check the more interesting median case, note that there's a mystery // node4 in there that should be excluded to make the distances sort // like this: // // [0] node3 (0.005), [1] node1 (0.007), [2] node2 (0.008) // // So the median should be at index 3 / 2 = 1 -> 0.007. if dist, err := getDatacenterDistance(s, "dc1"); err != nil || dist != 0.007 { t.Fatalf("bad: %v err: %v", dist, err) } } func TestRTT_sortDatacentersByDistance(t *testing.T) { s := newMockServer() dcs := []string{"acdc", "dc0", "dc1", "dc2", "dcX"} if err := sortDatacentersByDistance(s, dcs); err != nil { t.Fatalf("err: %v", err) } expected := "dc0,dc2,dc1,acdc,dcX" if actual := strings.Join(dcs, ","); actual != expected { t.Fatalf("bad sort: %s != %s", actual, expected) } // Make sure the sort is stable and we didn't just get lucky. dcs = []string{"dcX", "dc0", "dc1", "dc2", "acdc"} if err := sortDatacentersByDistance(s, dcs); err != nil { t.Fatalf("err: %v", err) } expected = "dc0,dc2,dc1,dcX,acdc" if actual := strings.Join(dcs, ","); actual != expected { t.Fatalf("bad sort: %s != %s", actual, expected) } } func TestRTT_getDatacenterMaps(t *testing.T) { s := newMockServer() dcs := []string{"dc0", "acdc", "dc1", "dc2", "dcX"} maps := getDatacenterMaps(s, dcs) if len(maps) != 5 { t.Fatalf("bad: %v", maps) } if maps[0].Datacenter != "dc0" || len(maps[0].Coordinates) != 1 || maps[0].Coordinates[0].Node != "dc0.node1" { t.Fatalf("bad: %v", maps[0]) } verifyCoordinatesEqual(t, maps[0].Coordinates[0].Coord, lib.GenerateCoordinate(10*time.Millisecond)) if maps[1].Datacenter != "acdc" || len(maps[1].Coordinates) != 0 { t.Fatalf("bad: %v", maps[1]) } if maps[2].Datacenter != "dc1" || len(maps[2].Coordinates) != 3 || maps[2].Coordinates[0].Node != "dc1.node1" || maps[2].Coordinates[1].Node != "dc1.node2" || maps[2].Coordinates[2].Node != "dc1.node3" { t.Fatalf("bad: %v", maps[2]) } verifyCoordinatesEqual(t, maps[2].Coordinates[0].Coord, lib.GenerateCoordinate(3*time.Millisecond)) verifyCoordinatesEqual(t, maps[2].Coordinates[1].Coord, lib.GenerateCoordinate(2*time.Millisecond)) verifyCoordinatesEqual(t, maps[2].Coordinates[2].Coord, lib.GenerateCoordinate(5*time.Millisecond)) if maps[3].Datacenter != "dc2" || len(maps[3].Coordinates) != 1 || maps[3].Coordinates[0].Node != "dc2.node1" { t.Fatalf("bad: %v", maps[3]) } verifyCoordinatesEqual(t, maps[3].Coordinates[0].Coord, lib.GenerateCoordinate(8*time.Millisecond)) if maps[4].Datacenter != "dcX" || len(maps[4].Coordinates) != 0 { t.Fatalf("bad: %v", maps[4]) } } func TestRTT_getDatacentersByDistance(t *testing.T) { dir1, s1 := testServerWithConfig(t, func(c *Config) { c.Datacenter = "xxx" }) defer os.RemoveAll(dir1) defer s1.Shutdown() codec1 := rpcClient(t, s1) defer codec1.Close() dir2, s2 := testServerWithConfig(t, func(c *Config) { c.Datacenter = "dc1" }) defer os.RemoveAll(dir2) defer s2.Shutdown() codec2 := rpcClient(t, s2) defer codec2.Close() dir3, s3 := testServerWithConfig(t, func(c *Config) { c.Datacenter = "dc2" }) defer os.RemoveAll(dir3) defer s3.Shutdown() codec3 := rpcClient(t, s3) defer codec3.Close() testutil.WaitForLeader(t, s1.RPC, "xxx") testutil.WaitForLeader(t, s2.RPC, "dc1") testutil.WaitForLeader(t, s3.RPC, "dc2") // Do the WAN joins. addr := fmt.Sprintf("127.0.0.1:%d", s1.config.SerfWANConfig.MemberlistConfig.BindPort) if _, err := s2.JoinWAN([]string{addr}); err != nil { t.Fatalf("err: %v", err) } if _, err := s3.JoinWAN([]string{addr}); err != nil { t.Fatalf("err: %v", err) } testutil.WaitForResult( func() (bool, error) { return len(s1.WANMembers()) > 2, nil }, func(err error) { t.Fatalf("Failed waiting for WAN join: %v", err) }) // Get the DCs by distance. We don't have coordinate updates yet, but // having xxx show up first proves we are calling the distance sort, // since it would normally do a string sort. dcs, err := s1.getDatacentersByDistance() if err != nil { t.Fatalf("err: %s", err) } if len(dcs) != 3 || dcs[0] != "xxx" { t.Fatalf("bad: %v", dcs) } // Let's disable coordinates just to be sure. s1.config.DisableCoordinates = true dcs, err = s1.getDatacentersByDistance() if err != nil { t.Fatalf("err: %s", err) } if len(dcs) != 3 || dcs[0] != "dc1" { t.Fatalf("bad: %v", dcs) } } */