open-consul/agent/consul/servers/router_test.go

package servers

import (
	"fmt"
	"log"
	"net"
	"os"
	"reflect"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/consul/agent/consul/structs"
	"github.com/hashicorp/consul/lib"
	"github.com/hashicorp/consul/types"
	"github.com/hashicorp/serf/coordinate"
	"github.com/hashicorp/serf/serf"
)

type mockCluster struct {
	self    string
	members []serf.Member
	coords  map[string]*coordinate.Coordinate
	addr    int
}

func newMockCluster(self string) *mockCluster {
	return &mockCluster{
		self:   self,
		coords: make(map[string]*coordinate.Coordinate),
		addr:   1,
	}
}

func (m *mockCluster) NumNodes() int {
	return len(m.members)
}

func (m *mockCluster) Members() []serf.Member {
	return m.members
}

func (m *mockCluster) GetCoordinate() (*coordinate.Coordinate, error) {
	return m.coords[m.self], nil
}

func (m *mockCluster) GetCachedCoordinate(name string) (*coordinate.Coordinate, bool) {
	coord, ok := m.coords[name]
	return coord, ok
}

func (m *mockCluster) AddMember(dc string, name string, coord *coordinate.Coordinate) {
	member := serf.Member{
		Name: fmt.Sprintf("%s.%s", name, dc),
		Addr: net.ParseIP(fmt.Sprintf("127.0.0.%d", m.addr)),
		Port: 8300,
		Tags: map[string]string{
			"dc":    dc,
			"role":  "consul",
			"port":  "8300",
			"build": "0.8.0",
			"vsn":   "3",
		},
	}
	m.members = append(m.members, member)
	if coord != nil {
		m.coords[member.Name] = coord
	}
	m.addr++
}

// testCluster is used to generate a single WAN-like area with a known set of
// member and RTT topology.
//
// Here's the layout of the nodes:
//
//            /----   dc1         ----\         /-  dc2  -\ /-  dc0  -\
//             node2 node1       node3             node1       node0
//   |     |     |     |     |     |     |     |     |     |     |
//   0     1     2     3     4     5     6     7     8     9     10  (ms)
//
// We also include a node4 in dc1 with no known coordinate, as well as a
// mysterious dcX with no nodes with known coordinates.
func testCluster(self string) *mockCluster {
	c := newMockCluster(self)
	c.AddMember("dc0", "node0", lib.GenerateCoordinate(10*time.Millisecond))
	c.AddMember("dc1", "node1", lib.GenerateCoordinate(3*time.Millisecond))
	c.AddMember("dc1", "node2", lib.GenerateCoordinate(2*time.Millisecond))
	c.AddMember("dc1", "node3", lib.GenerateCoordinate(5*time.Millisecond))
	c.AddMember("dc1", "node4", nil)
	c.AddMember("dc2", "node1", lib.GenerateCoordinate(8*time.Millisecond))
	c.AddMember("dcX", "node1", nil)
	return c
}

func testRouter(dc string) *Router {
	logger := log.New(os.Stderr, "", log.LstdFlags)
	return NewRouter(logger, dc)
}

func TestRouter_Shutdown(t *testing.T) {
	r := testRouter("dc0")

	// Create a WAN-looking area.
	self := "node0.dc0"
	wan := testCluster(self)
	if err := r.AddArea(types.AreaWAN, wan, &fauxConnPool{}, false); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Add another area.
	otherID := types.AreaID("other")
	other := newMockCluster(self)
	other.AddMember("dcY", "node1", nil)
	if err := r.AddArea(otherID, other, &fauxConnPool{}, false); err != nil {
		t.Fatalf("err: %v", err)
	}
	_, _, ok := r.FindRoute("dcY")
	if !ok {
		t.Fatalf("bad")
	}

	// Shutdown and make sure we can't see any routes from before.
	r.Shutdown()
	_, _, ok = r.FindRoute("dcY")
	if ok {
		t.Fatalf("bad")
	}

	// You can't add areas once the router is shut down.
	err := r.AddArea(otherID, other, &fauxConnPool{}, false)
	if err == nil || !strings.Contains(err.Error(), "router is shut down") {
		t.Fatalf("err: %v", err)
	}
}

func TestRouter_Routing(t *testing.T) {
	r := testRouter("dc0")

	// Create a WAN-looking area.
	self := "node0.dc0"
	wan := testCluster(self)
	if err := r.AddArea(types.AreaWAN, wan, &fauxConnPool{}, false); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Adding the area should enable all the routes right away.
	if _, _, ok := r.FindRoute("dc0"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc1"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc2"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dcX"); !ok {
		t.Fatalf("bad")
	}

	// This hasn't been added yet.
	if _, _, ok := r.FindRoute("dcY"); ok {
		t.Fatalf("bad")
	}

	// Add another area.
	otherID := types.AreaID("other")
	other := newMockCluster(self)
	other.AddMember("dc0", "node0", nil)
	other.AddMember("dc1", "node1", nil)
	other.AddMember("dcY", "node1", nil)
	if err := r.AddArea(otherID, other, &fauxConnPool{}, false); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Now we should have a route to every DC.
	if _, _, ok := r.FindRoute("dc0"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc1"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc2"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dcX"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dcY"); !ok {
		t.Fatalf("bad")
	}

	// Get the route for dcY and then fail the server. This will still
	// give the server back since we have no other choice.
	_, s, ok := r.FindRoute("dcY")
	if !ok {
		t.Fatalf("bad")
	}
	if err := r.FailServer(otherID, s); err != nil {
		t.Fatalf("err: %v", err)
	}
	if _, _, ok := r.FindRoute("dcY"); !ok {
		t.Fatalf("bad")
	}

	// But if we remove the server we won't get a route.
	if err := r.RemoveServer(otherID, s); err != nil {
		t.Fatalf("err: %v", err)
	}
	if _, _, ok := r.FindRoute("dcY"); ok {
		t.Fatalf("bad")
	}

	// Make sure the dcY manager also got removed from the area and from
	// the index we use for routing.
	func() {
		r.RLock()
		defer r.RUnlock()

		area, ok := r.areas[otherID]
		if !ok {
			t.Fatalf("bad")
		}

		if _, ok := area.managers["dcY"]; ok {
			t.Fatalf("bad")
		}

		if _, ok := r.managers["dcY"]; ok {
			t.Fatalf("bad")
		}
	}()

	// Do similar for dc0, which will take two removes because the dc0 is
	// reachable from two different areas.
	_, s, ok = r.FindRoute("dc0")
	if !ok {
		t.Fatalf("bad")
	}
	if err := r.RemoveServer(types.AreaWAN, s); err != nil {
		t.Fatalf("err: %v", err)
	}
	if _, _, ok = r.FindRoute("dc0"); !ok {
		t.Fatalf("bad")
	}
	if err := r.RemoveServer(otherID, s); err != nil {
		t.Fatalf("err: %v", err)
	}
	if _, _, ok = r.FindRoute("dc0"); ok {
		t.Fatalf("bad")
	}

	// Now delete some areas.
	if _, _, ok = r.FindRoute("dc1"); !ok {
		t.Fatalf("bad")
	}
	if err := r.RemoveArea(types.AreaWAN); err != nil {
		t.Fatalf("err: %v", err)
	}
	if _, _, ok = r.FindRoute("dc1"); !ok {
		t.Fatalf("bad")
	}
	if err := r.RemoveArea(otherID); err != nil {
		t.Fatalf("err: %v", err)
	}
	if _, _, ok = r.FindRoute("dc1"); ok {
		t.Fatalf("bad")
	}
}

func TestRouter_Routing_Offline(t *testing.T) {
	r := testRouter("dc0")

	// Create a WAN-looking area.
	self := "node0.dc0"
	wan := testCluster(self)
	if err := r.AddArea(types.AreaWAN, wan, &fauxConnPool{1.0}, false); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Adding the area should enable all the routes right away.
	if _, _, ok := r.FindRoute("dc0"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc1"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc2"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dcX"); !ok {
		t.Fatalf("bad")
	}

	// Do a rebalance for dc1, which should knock it offline.
	func() {
		r.Lock()
		defer r.Unlock()

		area, ok := r.areas[types.AreaWAN]
		if !ok {
			t.Fatalf("bad")
		}

		info, ok := area.managers["dc1"]
		if !ok {
			t.Fatalf("bad")
		}
		info.manager.RebalanceServers()
	}()

	// Recheck all the routes.
	if _, _, ok := r.FindRoute("dc0"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc1"); ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc2"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dcX"); !ok {
		t.Fatalf("bad")
	}

	// Add another area with a route to dc1.
	otherID := types.AreaID("other")
	other := newMockCluster(self)
	other.AddMember("dc0", "node0", nil)
	other.AddMember("dc1", "node1", nil)
	if err := r.AddArea(otherID, other, &fauxConnPool{}, false); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Recheck all the routes and make sure it finds the one that's
	// online.
	if _, _, ok := r.FindRoute("dc0"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc1"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dc2"); !ok {
		t.Fatalf("bad")
	}
	if _, _, ok := r.FindRoute("dcX"); !ok {
		t.Fatalf("bad")
	}
}

func TestRouter_GetDatacenters(t *testing.T) {
	r := testRouter("dc0")

	self := "node0.dc0"
	wan := testCluster(self)
	if err := r.AddArea(types.AreaWAN, wan, &fauxConnPool{}, false); err != nil {
		t.Fatalf("err: %v", err)
	}

	actual := r.GetDatacenters()
	expected := []string{"dc0", "dc1", "dc2", "dcX"}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("bad: %#v", actual)
	}
}

func TestRouter_distanceSorter(t *testing.T) {
	actual := &datacenterSorter{
		Names: []string{"foo", "bar", "baz", "zoo"},
		Vec:   []float64{3.0, 1.0, 1.0, 0.0},
	}
	sort.Stable(actual)
	expected := &datacenterSorter{
		Names: []string{"zoo", "bar", "baz", "foo"},
		Vec:   []float64{0.0, 1.0, 1.0, 3.0},
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("bad: %#v", *expected)
	}
}

func TestRouter_GetDatacentersByDistance(t *testing.T) {
	r := testRouter("dc0")

	// Start with just the WAN area described in the diagram above.
	self := "node0.dc0"
	wan := testCluster(self)
	if err := r.AddArea(types.AreaWAN, wan, &fauxConnPool{}, false); err != nil {
		t.Fatalf("err: %v", err)
	}

	actual, err := r.GetDatacentersByDistance()
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	expected := []string{"dc0", "dc2", "dc1", "dcX"}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("bad: %#v", actual)
	}

	// Now add another area with a closer route for dc1.
	otherID := types.AreaID("other")
	other := newMockCluster(self)
	other.AddMember("dc0", "node0", lib.GenerateCoordinate(20*time.Millisecond))
	other.AddMember("dc1", "node1", lib.GenerateCoordinate(21*time.Millisecond))
	if err := r.AddArea(otherID, other, &fauxConnPool{}, false); err != nil {
		t.Fatalf("err: %v", err)
	}

	actual, err = r.GetDatacentersByDistance()
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	expected = []string{"dc0", "dc1", "dc2", "dcX"}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("bad: %#v", actual)
	}
}

func TestRouter_GetDatacenterMaps(t *testing.T) {
	r := testRouter("dc0")

	self := "node0.dc0"
	wan := testCluster(self)
	if err := r.AddArea(types.AreaWAN, wan, &fauxConnPool{}, false); err != nil {
		t.Fatalf("err: %v", err)
	}

	actual, err := r.GetDatacenterMaps()
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if len(actual) != 3 {
		t.Fatalf("bad: %#v", actual)
	}
	for _, entry := range actual {
		switch entry.Datacenter {
		case "dc0":
			if !reflect.DeepEqual(entry, structs.DatacenterMap{
				Datacenter: "dc0",
				AreaID:     types.AreaWAN,
				Coordinates: structs.Coordinates{
					&structs.Coordinate{
						Node:  "node0.dc0",
						Coord: lib.GenerateCoordinate(10 * time.Millisecond),
					},
				},
			}) {
				t.Fatalf("bad: %#v", entry)
			}
		case "dc1":
			if !reflect.DeepEqual(entry, structs.DatacenterMap{
				Datacenter: "dc1",
				AreaID:     types.AreaWAN,
				Coordinates: structs.Coordinates{
					&structs.Coordinate{
						Node:  "node1.dc1",
						Coord: lib.GenerateCoordinate(3 * time.Millisecond),
					},
					&structs.Coordinate{
						Node:  "node2.dc1",
						Coord: lib.GenerateCoordinate(2 * time.Millisecond),
					},
					&structs.Coordinate{
						Node:  "node3.dc1",
						Coord: lib.GenerateCoordinate(5 * time.Millisecond),
					},
				},
			}) {
				t.Fatalf("bad: %#v", entry)
			}
		case "dc2":
			if !reflect.DeepEqual(entry, structs.DatacenterMap{
				Datacenter: "dc2",
				AreaID:     types.AreaWAN,
				Coordinates: structs.Coordinates{
					&structs.Coordinate{
						Node:  "node1.dc2",
						Coord: lib.GenerateCoordinate(8 * time.Millisecond),
					},
				},
			}) {
				t.Fatalf("bad: %#v", entry)
			}
		default:
			t.Fatalf("bad: %#v", entry)
		}
	}
}