From 369e501dd511e4380867e09ea6a9094c33522de2 Mon Sep 17 00:00:00 2001 From: Evan Gilman Date: Wed, 2 Dec 2015 15:34:01 -0800 Subject: [PATCH 01/19] Store WanAddress on Node --- consul/state/state_store.go | 7 ++++--- consul/structs/structs.go | 15 +++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/consul/state/state_store.go b/consul/state/state_store.go index 412f36c2d..45f7d43f5 100644 --- a/consul/state/state_store.go +++ b/consul/state/state_store.go @@ -474,7 +474,7 @@ func (s *StateStore) EnsureRegistration(idx uint64, req *structs.RegisterRequest func (s *StateStore) ensureRegistrationTxn(tx *memdb.Txn, idx uint64, watches *DumbWatchManager, req *structs.RegisterRequest) error { // Add the node. - node := &structs.Node{Node: req.Node, Address: req.Address} + node := &structs.Node{Node: req.Node, Address: req.Address, WanAddress: req.WanAddress} if err := s.ensureNodeTxn(tx, idx, watches, node); err != nil { return fmt.Errorf("failed inserting node: %s", err) } @@ -1373,8 +1373,9 @@ func (s *StateStore) parseNodes(tx *memdb.Txn, idx uint64, // Create the wrapped node dump := &structs.NodeInfo{ - Node: node.Node, - Address: node.Address, + Node: node.Node, + Address: node.Address, + WanAddress: node.WanAddress, } // Query the node services diff --git a/consul/structs/structs.go b/consul/structs/structs.go index fd1ef08a9..9417cecf8 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -162,6 +162,7 @@ type RegisterRequest struct { Datacenter string Node string Address string + WanAddress string Service *NodeService Check *HealthCheck Checks HealthChecks @@ -245,8 +246,9 @@ func (r *ChecksInStateRequest) RequestDatacenter() string { // Used to return information about a node type Node struct { - Node string - Address string + Node string + Address string + WanAddress string RaftIndex } @@ -438,10 +440,11 @@ OUTER: // a node. This is currently used for the UI only, as it is // rather expensive to generate. type NodeInfo struct { - Node string - Address string - Services []*NodeService - Checks []*HealthCheck + Node string + Address string + WanAddress string + Services []*NodeService + Checks []*HealthCheck } // NodeDump is used to dump all the nodes with all their From fc61143b465fbdc5b9b86e29d4155c27c40929cf Mon Sep 17 00:00:00 2001 From: Evan Gilman Date: Mon, 7 Dec 2015 15:07:56 -0800 Subject: [PATCH 02/19] Store WanAddress during node registration --- consul/client.go | 3 +++ consul/fsm.go | 5 +++-- consul/leader.go | 2 ++ consul/server.go | 3 +++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/consul/client.go b/consul/client.go index e7155a144..8d5915117 100644 --- a/consul/client.go +++ b/consul/client.go @@ -181,6 +181,9 @@ func (c *Client) setupSerf(conf *serf.Config, ch chan serf.Event, path string) ( conf.RejoinAfterLeave = c.config.RejoinAfterLeave conf.Merge = &lanMergeDelegate{dc: c.config.Datacenter} conf.DisableCoordinates = c.config.DisableCoordinates + if wanAddr := c.config.SerfWANConfig.MemberlistConfig.AdvertiseAddr; wanAddr != "" { + conf.Tags["WanAddr"] = wanAddr + } if err := ensurePath(conf.SnapshotPath, false); err != nil { return nil, err } diff --git a/consul/fsm.go b/consul/fsm.go index 1d1049e96..5ccf7afcd 100644 --- a/consul/fsm.go +++ b/consul/fsm.go @@ -472,8 +472,9 @@ func (s *consulSnapshot) persistNodes(sink raft.SnapshotSink, for node := nodes.Next(); node != nil; node = nodes.Next() { n := node.(*structs.Node) req := structs.RegisterRequest{ - Node: n.Node, - Address: n.Address, + Node: n.Node, + Address: n.Address, + WanAddress: n.WanAddress, } // Register the node itself diff --git a/consul/leader.go b/consul/leader.go index 55c487b4f..ff926c98b 100644 --- a/consul/leader.go +++ b/consul/leader.go @@ -418,6 +418,7 @@ AFTER_CHECK: Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), + WanAddress: member.Tags["WanAddr"], Service: service, Check: &structs.HealthCheck{ Node: member.Name, @@ -460,6 +461,7 @@ func (s *Server) handleFailedMember(member serf.Member) error { Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), + WanAddress: member.Tags["WanAddr"], Check: &structs.HealthCheck{ Node: member.Name, CheckID: SerfCheckID, diff --git a/consul/server.go b/consul/server.go index 7a59cb594..5f15975c7 100644 --- a/consul/server.go +++ b/consul/server.go @@ -299,6 +299,9 @@ func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string, w if s.config.BootstrapExpect != 0 { conf.Tags["expect"] = fmt.Sprintf("%d", s.config.BootstrapExpect) } + if wanAddr := s.config.SerfWANConfig.MemberlistConfig.AdvertiseAddr; wanAddr != "" { + conf.Tags["WanAddr"] = wanAddr + } conf.MemberlistConfig.LogOutput = s.config.LogOutput conf.LogOutput = s.config.LogOutput conf.EventCh = ch From 1a828e39273cdd0d11920da0cf0c66650984eaa2 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Tue, 8 Dec 2015 18:31:58 -0800 Subject: [PATCH 03/19] Store WanAddress during Service/Check sync --- command/agent/local.go | 2 ++ consul/leader.go | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/command/agent/local.go b/command/agent/local.go index 545d11722..cb8db2775 100644 --- a/command/agent/local.go +++ b/command/agent/local.go @@ -526,6 +526,7 @@ func (l *localState) syncService(id string) error { Datacenter: l.config.Datacenter, Node: l.config.NodeName, Address: l.config.AdvertiseAddr, + WanAddress: l.config.ConsulConfig.SerfWANConfig.MemberlistConfig.AdvertiseAddr, Service: l.services[id], WriteRequest: structs.WriteRequest{Token: l.serviceToken(id)}, } @@ -583,6 +584,7 @@ func (l *localState) syncCheck(id string) error { Datacenter: l.config.Datacenter, Node: l.config.NodeName, Address: l.config.AdvertiseAddr, + WanAddress: l.config.ConsulConfig.SerfWANConfig.MemberlistConfig.AdvertiseAddr, Service: service, Check: l.checks[id], WriteRequest: structs.WriteRequest{Token: l.checkToken(id)}, diff --git a/consul/leader.go b/consul/leader.go index ff926c98b..8987120eb 100644 --- a/consul/leader.go +++ b/consul/leader.go @@ -380,6 +380,11 @@ func (s *Server) handleAliveMember(member serf.Member) error { return err } if node != nil && node.Address == member.Addr.String() { + // Check if the WAN address was updated + if node.WanAddress != member.Tags["WanAddr"] { + goto AFTER_CHECK + } + // Check if the associated service is available if service != nil { match := false From e166d2a4c70833189204640f8718ba41cfdca0b8 Mon Sep 17 00:00:00 2001 From: Evan Gilman Date: Tue, 8 Dec 2015 18:49:24 -0800 Subject: [PATCH 04/19] Use idiomatic name for wan_addr serf tag --- consul/client.go | 2 +- consul/leader.go | 6 +++--- consul/server.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/consul/client.go b/consul/client.go index 8d5915117..89aaf438a 100644 --- a/consul/client.go +++ b/consul/client.go @@ -182,7 +182,7 @@ func (c *Client) setupSerf(conf *serf.Config, ch chan serf.Event, path string) ( conf.Merge = &lanMergeDelegate{dc: c.config.Datacenter} conf.DisableCoordinates = c.config.DisableCoordinates if wanAddr := c.config.SerfWANConfig.MemberlistConfig.AdvertiseAddr; wanAddr != "" { - conf.Tags["WanAddr"] = wanAddr + conf.Tags["wan_addr"] = wanAddr } if err := ensurePath(conf.SnapshotPath, false); err != nil { return nil, err diff --git a/consul/leader.go b/consul/leader.go index 8987120eb..8f811ddee 100644 --- a/consul/leader.go +++ b/consul/leader.go @@ -381,7 +381,7 @@ func (s *Server) handleAliveMember(member serf.Member) error { } if node != nil && node.Address == member.Addr.String() { // Check if the WAN address was updated - if node.WanAddress != member.Tags["WanAddr"] { + if node.WanAddress != member.Tags["wan_addr"] { goto AFTER_CHECK } @@ -423,7 +423,7 @@ AFTER_CHECK: Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), - WanAddress: member.Tags["WanAddr"], + WanAddress: member.Tags["wan_addr"], Service: service, Check: &structs.HealthCheck{ Node: member.Name, @@ -466,7 +466,7 @@ func (s *Server) handleFailedMember(member serf.Member) error { Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), - WanAddress: member.Tags["WanAddr"], + WanAddress: member.Tags["wan_addr"], Check: &structs.HealthCheck{ Node: member.Name, CheckID: SerfCheckID, diff --git a/consul/server.go b/consul/server.go index 5f15975c7..36371e60f 100644 --- a/consul/server.go +++ b/consul/server.go @@ -300,7 +300,7 @@ func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string, w conf.Tags["expect"] = fmt.Sprintf("%d", s.config.BootstrapExpect) } if wanAddr := s.config.SerfWANConfig.MemberlistConfig.AdvertiseAddr; wanAddr != "" { - conf.Tags["WanAddr"] = wanAddr + conf.Tags["wan_addr"] = wanAddr } conf.MemberlistConfig.LogOutput = s.config.LogOutput conf.LogOutput = s.config.LogOutput From d123b4dece3f51a53c045f4b79a34c7c2a8dbc66 Mon Sep 17 00:00:00 2001 From: Evan Gilman Date: Wed, 9 Dec 2015 21:58:23 -0800 Subject: [PATCH 05/19] Rectify value of `AdvertiseAddrWan` when set elsewhere `AdvertiseAddrs` has been introduced as a configuration option, which duplicates a few other options, namely `AdvertiseAddrWan`. We need to use this value elsewhere, so rather than doing a precedence check every time we need to access it, rectify the value of `AdvertiseAddrWan` to match --- command/agent/agent.go | 1 + 1 file changed, 1 insertion(+) diff --git a/command/agent/agent.go b/command/agent/agent.go index 2336dff64..e047fae0d 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -287,6 +287,7 @@ func (a *Agent) consulConfig() *consul.Config { if a.config.AdvertiseAddrs.SerfWan != nil { base.SerfWANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddrs.SerfWan.IP.String() base.SerfWANConfig.MemberlistConfig.AdvertisePort = a.config.AdvertiseAddrs.SerfWan.Port + a.config.AdvertiseAddrWan = a.config.AdvertiseAddrs.SerfWan.IP.String() } if a.config.AdvertiseAddrs.RPC != nil { base.RPCAdvertise = a.config.AdvertiseAddrs.RPC From 0b741350b9d24ca5d05a786a66ac534fcc669b73 Mon Sep 17 00:00:00 2001 From: Evan Gilman Date: Tue, 22 Dec 2015 03:31:40 -0800 Subject: [PATCH 06/19] Implement `translate_wan_addrs` config option This knob tells consul whether it should prefer the WAN address (if set) when making service lookups in remote datacenters. This enables reachability for remote services which are behind a NAT. --- command/agent/config.go | 7 +++++++ command/agent/dns.go | 26 +++++++++++++++++++++----- command/agent/local.go | 4 ++-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/command/agent/config.go b/command/agent/config.go index 515443954..2b625eb1f 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -189,6 +189,10 @@ type Config struct { // Serf WAN IP. If not specified, the general advertise address is used. AdvertiseAddrWan string `mapstructure:"advertise_addr_wan"` + // TranslateWanAddrs controls whether or not Consul should prefer + // the AdvertiseAddrWan address when doing lookups in remote datacenters. + TranslateWanAddrs bool `mapstructure:"translate_wan_addrs"` + // Port configurations Ports PortConfig @@ -968,6 +972,9 @@ func MergeConfig(a, b *Config) *Config { if b.AdvertiseAddrWan != "" { result.AdvertiseAddrWan = b.AdvertiseAddrWan } + if b.TranslateWanAddrs == true { + result.TranslateWanAddrs = true + } if b.AdvertiseAddrs.SerfLan != nil { result.AdvertiseAddrs.SerfLan = b.AdvertiseAddrs.SerfLan result.AdvertiseAddrs.SerfLanRaw = b.AdvertiseAddrs.SerfLanRaw diff --git a/command/agent/dns.go b/command/agent/dns.go index 6cbb89f59..71343afe8 100644 --- a/command/agent/dns.go +++ b/command/agent/dns.go @@ -524,9 +524,15 @@ RPC: // Perform a random shuffle out.Nodes.Shuffle() + // Determine whether we should use the WAN address or not + var useWan bool + if d.agent.config.TranslateWanAddrs && datacenter != d.agent.config.Datacenter { + useWan = true + } + // Add various responses depending on the request qType := req.Question[0].Qtype - d.serviceNodeRecords(out.Nodes, req, resp, ttl) + d.serviceNodeRecords(out.Nodes, req, resp, ttl, useWan) if qType == dns.TypeSRV { d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl) @@ -620,9 +626,15 @@ RPC: return } + // Determine whether we should use the WAN address or not + var useWan bool + if d.agent.config.TranslateWanAddrs && datacenter != d.agent.config.Datacenter { + useWan = true + } + // Add various responses depending on the request. qType := req.Question[0].Qtype - d.serviceNodeRecords(out.Nodes, req, resp, ttl) + d.serviceNodeRecords(out.Nodes, req, resp, ttl, useWan) if qType == dns.TypeSRV { d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl) } @@ -646,18 +658,22 @@ RPC: } // serviceNodeRecords is used to add the node records for a service lookup -func (d *DNSServer) serviceNodeRecords(nodes structs.CheckServiceNodes, req, resp *dns.Msg, ttl time.Duration) { +func (d *DNSServer) serviceNodeRecords(nodes structs.CheckServiceNodes, req, resp *dns.Msg, ttl time.Duration, useWan bool) { qName := req.Question[0].Name qType := req.Question[0].Qtype handled := make(map[string]struct{}) for _, node := range nodes { - // Avoid duplicate entries, possible if a node has - // the same service on multiple ports, etc. + // Prefer the Service Address or WAN Address over the + // Node Address when configured addr := node.Node.Address if node.Service.Address != "" { addr = node.Service.Address + } else if useWan == true && node.Node.WanAddress != "" { + addr = node.Node.WanAddress } + // Avoid duplicate entries, possible if a node has + // the same service on multiple ports, etc. if _, ok := handled[addr]; ok { continue } diff --git a/command/agent/local.go b/command/agent/local.go index cb8db2775..7320db8dc 100644 --- a/command/agent/local.go +++ b/command/agent/local.go @@ -526,7 +526,7 @@ func (l *localState) syncService(id string) error { Datacenter: l.config.Datacenter, Node: l.config.NodeName, Address: l.config.AdvertiseAddr, - WanAddress: l.config.ConsulConfig.SerfWANConfig.MemberlistConfig.AdvertiseAddr, + WanAddress: l.config.AdvertiseAddrWan, Service: l.services[id], WriteRequest: structs.WriteRequest{Token: l.serviceToken(id)}, } @@ -584,7 +584,7 @@ func (l *localState) syncCheck(id string) error { Datacenter: l.config.Datacenter, Node: l.config.NodeName, Address: l.config.AdvertiseAddr, - WanAddress: l.config.ConsulConfig.SerfWANConfig.MemberlistConfig.AdvertiseAddr, + WanAddress: l.config.AdvertiseAddrWan, Service: service, Check: l.checks[id], WriteRequest: structs.WriteRequest{Token: l.checkToken(id)}, From d18dc8174960b0c60b2d0e297898b7c96e71fec5 Mon Sep 17 00:00:00 2001 From: Evan Gilman Date: Wed, 23 Dec 2015 14:52:47 -0800 Subject: [PATCH 07/19] Consider WAN addresses during Node and SRV record lookup --- command/agent/dns.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/command/agent/dns.go b/command/agent/dns.go index 71343afe8..bc150f054 100644 --- a/command/agent/dns.go +++ b/command/agent/dns.go @@ -402,8 +402,14 @@ RPC: return } + // Determine whether we should use the WAN address or not + addr := out.NodeServices.Node.Address + if d.agent.config.TranslateWanAddrs && datacenter != d.agent.config.Datacenter { + addr = out.NodeServices.Node.WanAddress + } + // Add the node record - records := d.formatNodeRecord(out.NodeServices.Node, out.NodeServices.Node.Address, + records := d.formatNodeRecord(out.NodeServices.Node, addr, req.Question[0].Name, qType, d.config.NodeTTL) if records != nil { resp.Answer = append(resp.Answer, records...) @@ -535,7 +541,7 @@ RPC: d.serviceNodeRecords(out.Nodes, req, resp, ttl, useWan) if qType == dns.TypeSRV { - d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl) + d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl, useWan) } // If the network is not TCP, restrict the number of responses @@ -636,7 +642,7 @@ RPC: qType := req.Question[0].Qtype d.serviceNodeRecords(out.Nodes, req, resp, ttl, useWan) if qType == dns.TypeSRV { - d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl) + d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl, useWan) } // If the network is not TCP, restrict the number of responses. @@ -688,7 +694,7 @@ func (d *DNSServer) serviceNodeRecords(nodes structs.CheckServiceNodes, req, res } // serviceARecords is used to add the SRV records for a service lookup -func (d *DNSServer) serviceSRVRecords(dc string, nodes structs.CheckServiceNodes, req, resp *dns.Msg, ttl time.Duration) { +func (d *DNSServer) serviceSRVRecords(dc string, nodes structs.CheckServiceNodes, req, resp *dns.Msg, ttl time.Duration, useWan bool) { handled := make(map[string]struct{}) for _, node := range nodes { // Avoid duplicate entries, possible if a node has @@ -718,6 +724,8 @@ func (d *DNSServer) serviceSRVRecords(dc string, nodes structs.CheckServiceNodes addr := node.Node.Address if node.Service.Address != "" { addr = node.Service.Address + } else if useWan == true && node.Node.WanAddress != "" { + addr = node.Node.WanAddress } // Add the extra record From 28ce1b079f876a058b7d4eaf27af053525b992f7 Mon Sep 17 00:00:00 2001 From: Evan Gilman Date: Wed, 23 Dec 2015 15:47:15 -0800 Subject: [PATCH 08/19] Explicitly set advertise address in agent tests Otherwise, the tests will fail when run on a machine with multiple private addresses --- command/agent/command_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/command/agent/command_test.go b/command/agent/command_test.go index 9f9317e7b..bfa68ffad 100644 --- a/command/agent/command_test.go +++ b/command/agent/command_test.go @@ -80,6 +80,7 @@ func TestRetryJoin(t *testing.T) { "-server", "-data-dir", tmpDir, "-node", fmt.Sprintf(`"%s"`, conf2.NodeName), + "-advertise", agent.config.BindAddr, "-retry-join", serfAddr, "-retry-interval", "1s", "-retry-join-wan", serfWanAddr, From a8fd9542dc7294a4474c141c8b8ef0fff919b016 Mon Sep 17 00:00:00 2001 From: Evan Gilman Date: Wed, 23 Dec 2015 16:07:27 -0800 Subject: [PATCH 09/19] Add tests for consul WAN translation --- command/agent/agent_test.go | 4 ++ command/agent/config_test.go | 19 +++++ command/agent/dns_test.go | 131 +++++++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+) diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go index 24fad74af..fbc0fb75a 100644 --- a/command/agent/agent_test.go +++ b/command/agent/agent_test.go @@ -168,6 +168,10 @@ func TestAgent_CheckAdvertiseAddrsSettings(t *testing.T) { if rpc != c.AdvertiseAddrs.RPC { t.Fatalf("RPC is not properly set to %v: %s", c.AdvertiseAddrs.RPC, rpc) } + advertiseWanAddress := agent.config.AdvertiseAddrWan + if serfWanAddr != advertiseWanAddress { + t.Fatalf("AdvertiseAddrWan is not properly set to '%s': %s", serfWanAddr, advertiseWanAddress) + } } func TestAgent_AddService(t *testing.T) { diff --git a/command/agent/config_test.go b/command/agent/config_test.go index d06feb8cb..89239fe74 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -253,6 +253,25 @@ func TestDecodeConfig(t *testing.T) { t.Fatalf("bad: %#v", config) } + // WAN address translation disabled by default + config, err = DecodeConfig(bytes.NewReader([]byte(`{}`))) + if err != nil { + t.Fatalf("err: %s", err) + } + if config.TranslateWanAddrs != false { + t.Fatalf("bad: %#v", config) + } + + // WAN address translation + input = `{"translate_wan_addrs": true}` + config, err = DecodeConfig(bytes.NewReader([]byte(input))) + if err != nil { + t.Fatalf("err: %s", err) + } + if config.TranslateWanAddrs != true { + t.Fatalf("bad: %#v", config) + } + // leave_on_terminate input = `{"leave_on_terminate": true}` config, err = DecodeConfig(bytes.NewReader([]byte(input))) diff --git a/command/agent/dns_test.go b/command/agent/dns_test.go index 2c2c8fd0a..2fca3afbc 100644 --- a/command/agent/dns_test.go +++ b/command/agent/dns_test.go @@ -117,6 +117,7 @@ func TestDNS_NodeLookup(t *testing.T) { Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", + WanAddress: "127.0.0.2", } var out struct{} @@ -715,6 +716,136 @@ func TestDNS_ServiceLookup_ServiceAddress(t *testing.T) { } } +func TestDNS_ServiceLookup_WanAddress(t *testing.T) { + dir1, srv1 := makeDNSServerConfig(t, + func(c *Config) { + c.Datacenter = "dc1" + c.TranslateWanAddrs = true + }, nil) + defer os.RemoveAll(dir1) + defer srv1.Shutdown() + + dir2, srv2 := makeDNSServerConfig(t, func(c *Config) { + c.Datacenter = "dc2" + c.TranslateWanAddrs = true + }, nil) + defer os.RemoveAll(dir2) + defer srv2.Shutdown() + + testutil.WaitForLeader(t, srv1.agent.RPC, "dc1") + testutil.WaitForLeader(t, srv2.agent.RPC, "dc2") + + // Join WAN cluster + addr := fmt.Sprintf("127.0.0.1:%d", + srv1.agent.config.Ports.SerfWan) + if _, err := srv2.agent.JoinWAN([]string{addr}); err != nil { + t.Fatalf("err: %v", err) + } + + testutil.WaitForResult( + func() (bool, error) { + return len(srv1.agent.WANMembers()) > 1, nil + }, + func(err error) { + t.Fatalf("Failed waiting for WAN join: %v", err) + }) + + // Register a remote node with a service. + { + args := &structs.RegisterRequest{ + Datacenter: "dc2", + Node: "foo", + Address: "127.0.0.1", + WanAddress: "127.0.0.2", + Service: &structs.NodeService{ + Service: "db", + }, + } + + var out struct{} + if err := srv2.agent.RPC("Catalog.Register", args, &out); err != nil { + t.Fatalf("err: %v", err) + } + } + + // Register an equivalent prepared query. + var id string + { + args := &structs.PreparedQueryRequest{ + Datacenter: "dc2", + Op: structs.PreparedQueryCreate, + Query: &structs.PreparedQuery{ + Service: structs.ServiceQuery{ + Service: "db", + }, + }, + } + if err := srv2.agent.RPC("PreparedQuery.Apply", args, &id); err != nil { + t.Fatalf("err: %v", err) + } + } + + // Look up the SRV record via service and prepared query. + questions := []string{ + "db.service.dc2.consul.", + id + ".query.dc2.consul.", + } + for _, question := range questions { + m := new(dns.Msg) + m.SetQuestion(question, dns.TypeSRV) + + c := new(dns.Client) + addr, _ := srv1.agent.config.ClientListener("", srv1.agent.config.Ports.DNS) + in, _, err := c.Exchange(m, addr.String()) + if err != nil { + t.Fatalf("err: %v", err) + } + + if len(in.Answer) != 1 { + t.Fatalf("Bad: %#v", in) + } + + aRec, ok := in.Extra[0].(*dns.A) + if !ok { + t.Fatalf("Bad: %#v", in.Extra[0]) + } + if aRec.Hdr.Name != "foo.node.dc2.consul." { + t.Fatalf("Bad: %#v", in.Extra[0]) + } + if aRec.A.String() != "127.0.0.2" { + t.Fatalf("Bad: %#v", in.Extra[0]) + } + } + + // Also check the A record directly + for _, question := range questions { + m := new(dns.Msg) + m.SetQuestion(question, dns.TypeA) + + c := new(dns.Client) + addr, _ := srv1.agent.config.ClientListener("", srv1.agent.config.Ports.DNS) + in, _, err := c.Exchange(m, addr.String()) + if err != nil { + t.Fatalf("err: %v", err) + } + + if len(in.Answer) != 1 { + t.Fatalf("Bad: %#v", in) + } + + aRec, ok := in.Answer[0].(*dns.A) + if !ok { + t.Fatalf("Bad: %#v", in.Answer[0]) + } + if aRec.Hdr.Name != question { + t.Fatalf("Bad: %#v", in.Answer[0]) + } + if aRec.A.String() != "127.0.0.2" { + t.Fatalf("Bad: %#v", in.Answer[0]) + } + } +} + func TestDNS_CaseInsensitiveServiceLookup(t *testing.T) { dir, srv := makeDNSServer(t) defer os.RemoveAll(dir) From 71cf39b5f8f9e4e6823f39cb7524916da9122c12 Mon Sep 17 00:00:00 2001 From: Evan Gilman Date: Tue, 26 Jan 2016 10:30:28 -0800 Subject: [PATCH 10/19] Use a map for additional node addresses --- command/agent/dns.go | 10 +++++----- command/agent/dns_test.go | 8 ++++++-- command/agent/local.go | 8 ++++++-- consul/fsm.go | 4 +++- consul/leader.go | 10 +++++++--- consul/state/state_store.go | 4 ++-- consul/structs/structs.go | 6 +++--- 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/command/agent/dns.go b/command/agent/dns.go index bc150f054..98863d11d 100644 --- a/command/agent/dns.go +++ b/command/agent/dns.go @@ -405,7 +405,7 @@ RPC: // Determine whether we should use the WAN address or not addr := out.NodeServices.Node.Address if d.agent.config.TranslateWanAddrs && datacenter != d.agent.config.Datacenter { - addr = out.NodeServices.Node.WanAddress + addr = out.NodeServices.Node.Addresses["wan"] } // Add the node record @@ -674,8 +674,8 @@ func (d *DNSServer) serviceNodeRecords(nodes structs.CheckServiceNodes, req, res addr := node.Node.Address if node.Service.Address != "" { addr = node.Service.Address - } else if useWan == true && node.Node.WanAddress != "" { - addr = node.Node.WanAddress + } else if useWan == true && node.Node.Addresses["wan"] != "" { + addr = node.Node.Addresses["wan"] } // Avoid duplicate entries, possible if a node has @@ -724,8 +724,8 @@ func (d *DNSServer) serviceSRVRecords(dc string, nodes structs.CheckServiceNodes addr := node.Node.Address if node.Service.Address != "" { addr = node.Service.Address - } else if useWan == true && node.Node.WanAddress != "" { - addr = node.Node.WanAddress + } else if useWan == true && node.Node.Addresses["wan"] != "" { + addr = node.Node.Addresses["wan"] } // Add the extra record diff --git a/command/agent/dns_test.go b/command/agent/dns_test.go index 2fca3afbc..0ef519c60 100644 --- a/command/agent/dns_test.go +++ b/command/agent/dns_test.go @@ -117,7 +117,9 @@ func TestDNS_NodeLookup(t *testing.T) { Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", - WanAddress: "127.0.0.2", + Addresses: map[string]string { + "wan": "127.0.0.2", + }, } var out struct{} @@ -756,7 +758,9 @@ func TestDNS_ServiceLookup_WanAddress(t *testing.T) { Datacenter: "dc2", Node: "foo", Address: "127.0.0.1", - WanAddress: "127.0.0.2", + Addresses: map[string]string { + "wan": "127.0.0.2", + }, Service: &structs.NodeService{ Service: "db", }, diff --git a/command/agent/local.go b/command/agent/local.go index 7320db8dc..0381ef488 100644 --- a/command/agent/local.go +++ b/command/agent/local.go @@ -526,7 +526,9 @@ func (l *localState) syncService(id string) error { Datacenter: l.config.Datacenter, Node: l.config.NodeName, Address: l.config.AdvertiseAddr, - WanAddress: l.config.AdvertiseAddrWan, + Addresses: map[string]string { + "wan": l.config.AdvertiseAddrWan, + }, Service: l.services[id], WriteRequest: structs.WriteRequest{Token: l.serviceToken(id)}, } @@ -584,7 +586,9 @@ func (l *localState) syncCheck(id string) error { Datacenter: l.config.Datacenter, Node: l.config.NodeName, Address: l.config.AdvertiseAddr, - WanAddress: l.config.AdvertiseAddrWan, + Addresses: map[string]string { + "wan": l.config.AdvertiseAddrWan, + }, Service: service, Check: l.checks[id], WriteRequest: structs.WriteRequest{Token: l.checkToken(id)}, diff --git a/consul/fsm.go b/consul/fsm.go index 5ccf7afcd..59231e65f 100644 --- a/consul/fsm.go +++ b/consul/fsm.go @@ -474,7 +474,9 @@ func (s *consulSnapshot) persistNodes(sink raft.SnapshotSink, req := structs.RegisterRequest{ Node: n.Node, Address: n.Address, - WanAddress: n.WanAddress, + Addresses: map[string]string { + "wan": n.Addresses["wan"], + }, } // Register the node itself diff --git a/consul/leader.go b/consul/leader.go index 8f811ddee..92909090c 100644 --- a/consul/leader.go +++ b/consul/leader.go @@ -381,7 +381,7 @@ func (s *Server) handleAliveMember(member serf.Member) error { } if node != nil && node.Address == member.Addr.String() { // Check if the WAN address was updated - if node.WanAddress != member.Tags["wan_addr"] { + if node.Addresses["wan"] != member.Tags["wan_addr"] { goto AFTER_CHECK } @@ -423,7 +423,9 @@ AFTER_CHECK: Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), - WanAddress: member.Tags["wan_addr"], + Addresses: map[string]string { + "wan": member.Tags["wan_addr"], + }, Service: service, Check: &structs.HealthCheck{ Node: member.Name, @@ -466,7 +468,9 @@ func (s *Server) handleFailedMember(member serf.Member) error { Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), - WanAddress: member.Tags["wan_addr"], + Addresses: map[string]string { + "wan": member.Tags["wan_addr"], + }, Check: &structs.HealthCheck{ Node: member.Name, CheckID: SerfCheckID, diff --git a/consul/state/state_store.go b/consul/state/state_store.go index 45f7d43f5..4ca07c732 100644 --- a/consul/state/state_store.go +++ b/consul/state/state_store.go @@ -474,7 +474,7 @@ func (s *StateStore) EnsureRegistration(idx uint64, req *structs.RegisterRequest func (s *StateStore) ensureRegistrationTxn(tx *memdb.Txn, idx uint64, watches *DumbWatchManager, req *structs.RegisterRequest) error { // Add the node. - node := &structs.Node{Node: req.Node, Address: req.Address, WanAddress: req.WanAddress} + node := &structs.Node{Node: req.Node, Address: req.Address, Addresses: req.Addresses} if err := s.ensureNodeTxn(tx, idx, watches, node); err != nil { return fmt.Errorf("failed inserting node: %s", err) } @@ -1375,7 +1375,7 @@ func (s *StateStore) parseNodes(tx *memdb.Txn, idx uint64, dump := &structs.NodeInfo{ Node: node.Node, Address: node.Address, - WanAddress: node.WanAddress, + Addresses: node.Addresses, } // Query the node services diff --git a/consul/structs/structs.go b/consul/structs/structs.go index 9417cecf8..0d86d77ed 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -162,7 +162,7 @@ type RegisterRequest struct { Datacenter string Node string Address string - WanAddress string + Addresses map[string]string Service *NodeService Check *HealthCheck Checks HealthChecks @@ -248,7 +248,7 @@ func (r *ChecksInStateRequest) RequestDatacenter() string { type Node struct { Node string Address string - WanAddress string + Addresses map[string]string RaftIndex } @@ -442,7 +442,7 @@ OUTER: type NodeInfo struct { Node string Address string - WanAddress string + Addresses map[string]string Services []*NodeService Checks []*HealthCheck } From 79bd1fd4bb1b0c8ce7ab4a05f209ccfdcc086ece Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 7 Feb 2016 10:37:34 -0800 Subject: [PATCH 11/19] Sets up config for more address tags down the road, renames struct members. --- command/agent/agent.go | 6 +++++- command/agent/agent_test.go | 8 +++++--- command/agent/config.go | 11 ++++++++++- command/agent/dns.go | 10 +++++----- command/agent/dns_test.go | 4 ++-- command/agent/local.go | 30 +++++++++++++----------------- consul/fsm.go | 8 +++----- consul/leader.go | 8 ++++---- consul/state/state_store.go | 12 ++++++++---- consul/structs/structs.go | 30 +++++++++++++++--------------- 10 files changed, 70 insertions(+), 57 deletions(-) diff --git a/command/agent/agent.go b/command/agent/agent.go index e047fae0d..29d8dc042 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -161,6 +161,11 @@ func Create(config *Config, logOutput io.Writer) (*Agent, error) { config.AdvertiseAddrWan = config.AdvertiseAddr } + // Create the default set of tagged addresses. + config.TaggedAddresses = map[string]string{ + "wan": config.AdvertiseAddrWan, + } + agent := &Agent{ config: config, logger: log.New(logOutput, "", log.LstdFlags), @@ -287,7 +292,6 @@ func (a *Agent) consulConfig() *consul.Config { if a.config.AdvertiseAddrs.SerfWan != nil { base.SerfWANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddrs.SerfWan.IP.String() base.SerfWANConfig.MemberlistConfig.AdvertisePort = a.config.AdvertiseAddrs.SerfWan.Port - a.config.AdvertiseAddrWan = a.config.AdvertiseAddrs.SerfWan.IP.String() } if a.config.AdvertiseAddrs.RPC != nil { base.RPCAdvertise = a.config.AdvertiseAddrs.RPC diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go index fbc0fb75a..fb28fa3df 100644 --- a/command/agent/agent_test.go +++ b/command/agent/agent_test.go @@ -168,9 +168,11 @@ func TestAgent_CheckAdvertiseAddrsSettings(t *testing.T) { if rpc != c.AdvertiseAddrs.RPC { t.Fatalf("RPC is not properly set to %v: %s", c.AdvertiseAddrs.RPC, rpc) } - advertiseWanAddress := agent.config.AdvertiseAddrWan - if serfWanAddr != advertiseWanAddress { - t.Fatalf("AdvertiseAddrWan is not properly set to '%s': %s", serfWanAddr, advertiseWanAddress) + expected := map[string]string{ + "wan": agent.config.AdvertiseAddrWan, + } + if !reflect.DeepEqual(agent.config.TaggedAddresses, expected) { + t.Fatalf("Tagged addresses not set up properly: %v", agent.config.TaggedAddresses) } } diff --git a/command/agent/config.go b/command/agent/config.go index 2b625eb1f..f56e3e740 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -190,7 +190,8 @@ type Config struct { AdvertiseAddrWan string `mapstructure:"advertise_addr_wan"` // TranslateWanAddrs controls whether or not Consul should prefer - // the AdvertiseAddrWan address when doing lookups in remote datacenters. + // the "wan" tagged address when doing lookups in remote datacenters. + // See TaggedAddresses below for more details. TranslateWanAddrs bool `mapstructure:"translate_wan_addrs"` // Port configurations @@ -199,6 +200,14 @@ type Config struct { // Address configurations Addresses AddressConfig + // Tagged addresses. These are used to publish a set of addresses for + // for a node, which can be used by the remote agent. We currently + // populate only the "wan" tag based on the SerfWan advertise address, + // but this structure is here for possible future features with other + // user-defined tags. The "wan" tag will be used by remote agents if + // they are configured with TranslateWanAddrs set to true. + TaggedAddresses map[string]string + // LeaveOnTerm controls if Serf does a graceful leave when receiving // the TERM signal. Defaults false. This can be changed on reload. LeaveOnTerm bool `mapstructure:"leave_on_terminate"` diff --git a/command/agent/dns.go b/command/agent/dns.go index 98863d11d..9749b19f0 100644 --- a/command/agent/dns.go +++ b/command/agent/dns.go @@ -405,7 +405,7 @@ RPC: // Determine whether we should use the WAN address or not addr := out.NodeServices.Node.Address if d.agent.config.TranslateWanAddrs && datacenter != d.agent.config.Datacenter { - addr = out.NodeServices.Node.Addresses["wan"] + addr = out.NodeServices.Node.TaggedAddresses["wan"] } // Add the node record @@ -674,8 +674,8 @@ func (d *DNSServer) serviceNodeRecords(nodes structs.CheckServiceNodes, req, res addr := node.Node.Address if node.Service.Address != "" { addr = node.Service.Address - } else if useWan == true && node.Node.Addresses["wan"] != "" { - addr = node.Node.Addresses["wan"] + } else if useWan == true && node.Node.TaggedAddresses["wan"] != "" { + addr = node.Node.TaggedAddresses["wan"] } // Avoid duplicate entries, possible if a node has @@ -724,8 +724,8 @@ func (d *DNSServer) serviceSRVRecords(dc string, nodes structs.CheckServiceNodes addr := node.Node.Address if node.Service.Address != "" { addr = node.Service.Address - } else if useWan == true && node.Node.Addresses["wan"] != "" { - addr = node.Node.Addresses["wan"] + } else if useWan == true && node.Node.TaggedAddresses["wan"] != "" { + addr = node.Node.TaggedAddresses["wan"] } // Add the extra record diff --git a/command/agent/dns_test.go b/command/agent/dns_test.go index 0ef519c60..cb017c912 100644 --- a/command/agent/dns_test.go +++ b/command/agent/dns_test.go @@ -117,7 +117,7 @@ func TestDNS_NodeLookup(t *testing.T) { Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", - Addresses: map[string]string { + TaggedAddresses: map[string]string{ "wan": "127.0.0.2", }, } @@ -758,7 +758,7 @@ func TestDNS_ServiceLookup_WanAddress(t *testing.T) { Datacenter: "dc2", Node: "foo", Address: "127.0.0.1", - Addresses: map[string]string { + TaggedAddresses: map[string]string{ "wan": "127.0.0.2", }, Service: &structs.NodeService{ diff --git a/command/agent/local.go b/command/agent/local.go index 0381ef488..15752555a 100644 --- a/command/agent/local.go +++ b/command/agent/local.go @@ -523,14 +523,12 @@ func (l *localState) deleteCheck(id string) error { // syncService is used to sync a service to the server func (l *localState) syncService(id string) error { req := structs.RegisterRequest{ - Datacenter: l.config.Datacenter, - Node: l.config.NodeName, - Address: l.config.AdvertiseAddr, - Addresses: map[string]string { - "wan": l.config.AdvertiseAddrWan, - }, - Service: l.services[id], - WriteRequest: structs.WriteRequest{Token: l.serviceToken(id)}, + Datacenter: l.config.Datacenter, + Node: l.config.NodeName, + Address: l.config.AdvertiseAddr, + TaggedAddresses: l.config.TaggedAddresses, + Service: l.services[id], + WriteRequest: structs.WriteRequest{Token: l.serviceToken(id)}, } // If the service has associated checks that are out of sync, @@ -583,15 +581,13 @@ func (l *localState) syncCheck(id string) error { } req := structs.RegisterRequest{ - Datacenter: l.config.Datacenter, - Node: l.config.NodeName, - Address: l.config.AdvertiseAddr, - Addresses: map[string]string { - "wan": l.config.AdvertiseAddrWan, - }, - Service: service, - Check: l.checks[id], - WriteRequest: structs.WriteRequest{Token: l.checkToken(id)}, + Datacenter: l.config.Datacenter, + Node: l.config.NodeName, + Address: l.config.AdvertiseAddr, + TaggedAddresses: l.config.TaggedAddresses, + Service: service, + Check: l.checks[id], + WriteRequest: structs.WriteRequest{Token: l.checkToken(id)}, } var out struct{} err := l.iface.RPC("Catalog.Register", &req, &out) diff --git a/consul/fsm.go b/consul/fsm.go index 59231e65f..9f786024a 100644 --- a/consul/fsm.go +++ b/consul/fsm.go @@ -472,11 +472,9 @@ func (s *consulSnapshot) persistNodes(sink raft.SnapshotSink, for node := nodes.Next(); node != nil; node = nodes.Next() { n := node.(*structs.Node) req := structs.RegisterRequest{ - Node: n.Node, - Address: n.Address, - Addresses: map[string]string { - "wan": n.Addresses["wan"], - }, + Node: n.Node, + Address: n.Address, + TaggedAddresses: n.TaggedAddresses, } // Register the node itself diff --git a/consul/leader.go b/consul/leader.go index 92909090c..374e7a9a2 100644 --- a/consul/leader.go +++ b/consul/leader.go @@ -381,7 +381,7 @@ func (s *Server) handleAliveMember(member serf.Member) error { } if node != nil && node.Address == member.Addr.String() { // Check if the WAN address was updated - if node.Addresses["wan"] != member.Tags["wan_addr"] { + if node.TaggedAddresses["wan"] != member.Tags["wan_addr"] { goto AFTER_CHECK } @@ -423,10 +423,10 @@ AFTER_CHECK: Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), - Addresses: map[string]string { + TaggedAddresses: map[string]string{ "wan": member.Tags["wan_addr"], }, - Service: service, + Service: service, Check: &structs.HealthCheck{ Node: member.Name, CheckID: SerfCheckID, @@ -468,7 +468,7 @@ func (s *Server) handleFailedMember(member serf.Member) error { Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), - Addresses: map[string]string { + TaggedAddresses: map[string]string{ "wan": member.Tags["wan_addr"], }, Check: &structs.HealthCheck{ diff --git a/consul/state/state_store.go b/consul/state/state_store.go index 4ca07c732..473775cae 100644 --- a/consul/state/state_store.go +++ b/consul/state/state_store.go @@ -474,7 +474,11 @@ func (s *StateStore) EnsureRegistration(idx uint64, req *structs.RegisterRequest func (s *StateStore) ensureRegistrationTxn(tx *memdb.Txn, idx uint64, watches *DumbWatchManager, req *structs.RegisterRequest) error { // Add the node. - node := &structs.Node{Node: req.Node, Address: req.Address, Addresses: req.Addresses} + node := &structs.Node{ + Node: req.Node, + Address: req.Address, + TaggedAddresses: req.TaggedAddresses, + } if err := s.ensureNodeTxn(tx, idx, watches, node); err != nil { return fmt.Errorf("failed inserting node: %s", err) } @@ -1373,9 +1377,9 @@ func (s *StateStore) parseNodes(tx *memdb.Txn, idx uint64, // Create the wrapped node dump := &structs.NodeInfo{ - Node: node.Node, - Address: node.Address, - Addresses: node.Addresses, + Node: node.Node, + Address: node.Address, + TaggedAddresses: node.TaggedAddresses, } // Query the node services diff --git a/consul/structs/structs.go b/consul/structs/structs.go index 0d86d77ed..3e7ef5955 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -159,13 +159,13 @@ type QueryMeta struct { // to register a node as providing a service. If no service // is provided, the node is registered. type RegisterRequest struct { - Datacenter string - Node string - Address string - Addresses map[string]string - Service *NodeService - Check *HealthCheck - Checks HealthChecks + Datacenter string + Node string + Address string + TaggedAddresses map[string]string + Service *NodeService + Check *HealthCheck + Checks HealthChecks WriteRequest } @@ -246,9 +246,9 @@ func (r *ChecksInStateRequest) RequestDatacenter() string { // Used to return information about a node type Node struct { - Node string - Address string - Addresses map[string]string + Node string + Address string + TaggedAddresses map[string]string RaftIndex } @@ -440,11 +440,11 @@ OUTER: // a node. This is currently used for the UI only, as it is // rather expensive to generate. type NodeInfo struct { - Node string - Address string - Addresses map[string]string - Services []*NodeService - Checks []*HealthCheck + Node string + Address string + TaggedAddresses map[string]string + Services []*NodeService + Checks []*HealthCheck } // NodeDump is used to dump all the nodes with all their From 800910137d2c46d9205ca9d73f3a1be9a9f3c47b Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 7 Feb 2016 10:56:47 -0800 Subject: [PATCH 12/19] Adds documentation for WAN address translation. --- website/source/docs/agent/dns.html.markdown | 9 ++++++ .../docs/agent/http/catalog.html.markdown | 20 +++++++++++-- .../docs/agent/http/health.html.markdown | 5 +++- .../source/docs/agent/options.html.markdown | 29 +++++++++++++------ 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/website/source/docs/agent/dns.html.markdown b/website/source/docs/agent/dns.html.markdown index 4a06353e9..a104fc607 100644 --- a/website/source/docs/agent/dns.html.markdown +++ b/website/source/docs/agent/dns.html.markdown @@ -207,3 +207,12 @@ By default, all DNS results served by Consul set a 0 TTL value. This disables caching of DNS results. However, there are many situations in which caching is desirable for performance and scalability. This is discussed more in the guide for [DNS Caching](/docs/guides/dns-cache.html). + +## WAN Address Translation + +Be default, Consul DNS queries will return a node's local address, even when +being queried from a remote datacenter. If you need to use a different address +to reach a node from outside its datacenter, you can configure this behavior +using the [`advertise-wan`](/docs/agent/options.html#_advertise-wan) and +[`translate_wan_addrs`](/docs/agent/options.html#translate_wan_addrs) configuration +options. diff --git a/website/source/docs/agent/http/catalog.html.markdown b/website/source/docs/agent/http/catalog.html.markdown index 323166f38..6b9d4cfc0 100644 --- a/website/source/docs/agent/http/catalog.html.markdown +++ b/website/source/docs/agent/http/catalog.html.markdown @@ -48,6 +48,9 @@ body must look something like: "v1" ], "Address": "127.0.0.1", + "TaggedAddresses": { + "wan": "127.0.0.1" + }, "Port": 8000 }, "Check": { @@ -64,7 +67,9 @@ body must look something like: The behavior of the endpoint depends on what keys are provided. The endpoint requires `Node` and `Address` to be provided while `Datacenter` will be defaulted to match that of the agent. If only those are provided, the endpoint will register -the node with the catalog. +the node with the catalog. `TaggedAddresses` can be used in conjunction with the +[`translate_wan_addrs`](/docs/agent/options.html#translate_wan_addrs) configuration +option. Currently only the "wan" tag is supported. If the `Service` key is provided, the service will also be registered. If `ID` is not provided, it will be defaulted to the value of the `Service.Service` property. @@ -191,10 +196,16 @@ It returns a JSON body like this: { "Node": "baz", "Address": "10.1.10.11" + "TaggedAddresses": { + "wan": "10.1.10.11" + } }, { "Node": "foobar", - "Address": "10.1.10.12" + "Address": "10.1.10.12", + "TaggedAddresses": { + "wan": "10.1.10.12" + } } ] ``` @@ -271,7 +282,10 @@ It returns a JSON body like this: { "Node": { "Node": "foobar", - "Address": "10.1.10.12" + "Address": "10.1.10.12", + "TaggedAddresses": { + "wan": "10.1.10.12" + } }, "Services": { "consul": { diff --git a/website/source/docs/agent/http/health.html.markdown b/website/source/docs/agent/http/health.html.markdown index 7aee45933..6f0c4a99e 100644 --- a/website/source/docs/agent/http/health.html.markdown +++ b/website/source/docs/agent/http/health.html.markdown @@ -127,7 +127,10 @@ It returns a JSON body like this: { "Node": { "Node": "foobar", - "Address": "10.1.10.12" + "Address": "10.1.10.12", + "TaggedAddresses": { + "wan": "10.1.10.12" + } }, "Service": { "ID": "redis", diff --git a/website/source/docs/agent/options.html.markdown b/website/source/docs/agent/options.html.markdown index dd4cd5851..3a2702357 100644 --- a/website/source/docs/agent/options.html.markdown +++ b/website/source/docs/agent/options.html.markdown @@ -41,18 +41,22 @@ The options below are all specified on the command-line. If this address is not routable, the node will be in a constant flapping state as other nodes will treat the non-routability as a failure. -* `-advertise-wan` - The advertise wan - address is used to change the address that we advertise to server nodes joining - through the WAN. By default, the [`-advertise`](#_advertise) address is advertised. - However, in some cases all members of all datacenters cannot be on the same - physical or virtual network, especially on hybrid setups mixing cloud and private datacenters. - This flag enables server nodes gossiping through the public network for the WAN while using - private VLANs for gossiping to each other and their client agents. +* `-advertise-wan` - The + advertise WAN address is used to change the address that we advertise to server nodes + joining through the WAN. This can also be set on client agents when used in combination + with the `translate_wan_addrs` configuration + option. By default, the [`-advertise`](#_advertise) address is advertised. However, in some + cases all members of all datacenters cannot be on the same physical or virtual network, + especially on hybrid setups mixing cloud and private datacenters. This flag enables server + nodes gossiping through the public network for the WAN while using private VLANs for gossiping + to each other and their client agents, and it allows client agents to be reached at this + address when being accessed from a remote datacenter if the remote datacenter is configured + with `translate_wan_addrs`. * `-atlas` - This flag enables [Atlas](https://atlas.hashicorp.com) integration. - It is used to provide the Atlas infrastructure name and the SCADA connection. The format of - this is `username/environment`. This enables Atlas features such as the Monitoring UI + It is used to provide the Atlas infrastructure name and the SCADA connection. The format of + this is `username/environment`. This enables Atlas features such as the Monitoring UI and node auto joining. * `-atlas-join` - When set, enables auto-join @@ -623,6 +627,13 @@ definitions support being updated during a reload. [`enable_syslog`](#enable_syslog) is provided, this controls to which facility messages are sent. By default, `LOCAL0` will be used. +* `translate_wan_addrs` If + set to true, Consul will prefer a node's configured WAN address + when servicing DNS requests for a node in a remote datacenter. This allows the node to be + reached within its own datacenter using its local address, and reached from other datacenters + using its WAN address, which is useful in hybrid setups with mixed networks. This is disabled + by default. + * `ui` - Equivalent to the [`-ui`](#_ui) command-line flag. From 81e416e70b92f14db02eec2c891a29cd9f2e7655 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 7 Feb 2016 11:26:19 -0800 Subject: [PATCH 13/19] Adds a test to make sure the local sync doesn't clobber tagged addresses. --- command/agent/local_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/command/agent/local_test.go b/command/agent/local_test.go index 0d6f8f53e..58a2f2329 100644 --- a/command/agent/local_test.go +++ b/command/agent/local_test.go @@ -120,6 +120,12 @@ func TestAgentAntiEntropy_Services(t *testing.T) { t.Fatalf("err: %v", err) } + // Make sure we sent along our tagged addresses when we synced. + addrs := services.NodeServices.Node.TaggedAddresses + if len(addrs) == 0 || !reflect.DeepEqual(addrs, conf.TaggedAddresses) { + t.Fatalf("bad: %v", addrs) + } + // We should have 6 services (consul included) if len(services.NodeServices.Services) != 6 { t.Fatalf("bad: %v", services.NodeServices.Services) @@ -627,6 +633,23 @@ func TestAgentAntiEntropy_Checks(t *testing.T) { t.Fatalf("should be in sync: %v %v", name, status) } } + + // Make sure we sent along our tagged addresses when we synced. + { + req := structs.NodeSpecificRequest{ + Datacenter: "dc1", + Node: agent.config.NodeName, + } + var services structs.IndexedNodeServices + if err := agent.RPC("Catalog.NodeServices", &req, &services); err != nil { + t.Fatalf("err: %v", err) + } + + addrs := services.NodeServices.Node.TaggedAddresses + if len(addrs) == 0 || !reflect.DeepEqual(addrs, conf.TaggedAddresses) { + t.Fatalf("bad: %v", addrs) + } + } } func TestAgentAntiEntropy_Check_DeferSync(t *testing.T) { From 3f50d2ae7e970b79a71768a10434d4d1fa795c97 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 7 Feb 2016 11:36:39 -0800 Subject: [PATCH 14/19] Adds an FSM persist and restore test for tagged addresses. --- consul/fsm_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/consul/fsm_test.go b/consul/fsm_test.go index e9ba05e15..5f8b32a32 100644 --- a/consul/fsm_test.go +++ b/consul/fsm_test.go @@ -360,7 +360,7 @@ func TestFSM_SnapshotRestore(t *testing.T) { // Add some state fsm.state.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}) - fsm.state.EnsureNode(2, &structs.Node{Node: "baz", Address: "127.0.0.2"}) + fsm.state.EnsureNode(2, &structs.Node{Node: "baz", Address: "127.0.0.2", TaggedAddresses: map[string]string{"hello": "1.2.3.4"}}) fsm.state.EnsureService(3, "foo", &structs.NodeService{ID: "web", Service: "web", Tags: nil, Address: "127.0.0.1", Port: 80}) fsm.state.EnsureService(4, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.1", Port: 5000}) fsm.state.EnsureService(5, "baz", &structs.NodeService{ID: "web", Service: "web", Tags: nil, Address: "127.0.0.2", Port: 80}) @@ -453,7 +453,18 @@ func TestFSM_SnapshotRestore(t *testing.T) { t.Fatalf("err: %s", err) } if len(nodes) != 2 { - t.Fatalf("Bad: %v", nodes) + t.Fatalf("bad: %v", nodes) + } + if nodes[0].Node != "baz" || + nodes[0].Address != "127.0.0.2" || + len(nodes[0].TaggedAddresses) != 1 || + nodes[0].TaggedAddresses["hello"] != "1.2.3.4" { + t.Fatalf("bad: %v", nodes[0]) + } + if nodes[1].Node != "foo" || + nodes[1].Address != "127.0.0.1" || + len(nodes[1].TaggedAddresses) != 0 { + t.Fatalf("bad: %v", nodes[1]) } _, fooSrv, err := fsm2.state.NodeServices("foo") From f163522f0cb9b4c617efead0c6e30b550e93e960 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 7 Feb 2016 13:12:42 -0800 Subject: [PATCH 15/19] Moves tagged wan address to be managed by anti-entropy, not serf. --- command/agent/local.go | 52 ++++++++++++++++++++++++++++++++ command/agent/local_test.go | 60 +++++++++++++++++++++++++++++++++++++ consul/client.go | 3 -- consul/leader.go | 13 +------- consul/server.go | 3 -- 5 files changed, 113 insertions(+), 18 deletions(-) diff --git a/command/agent/local.go b/command/agent/local.go index 15752555a..581536722 100644 --- a/command/agent/local.go +++ b/command/agent/local.go @@ -3,6 +3,7 @@ package agent import ( "fmt" "log" + "reflect" "strings" "sync" "sync/atomic" @@ -45,6 +46,10 @@ type localState struct { // iface is the consul interface to use for keeping in sync iface consul.Interface + // nodeInfoInSync tracks whether the server has our correct top-level + // node information in sync (currently only used for tagged addresses) + nodeInfoInSync bool + // Services tracks the local services services map[string]*structs.NodeService serviceStatus map[string]syncStatus @@ -361,6 +366,13 @@ func (l *localState) setSyncState() error { l.Lock() defer l.Unlock() + // Check the node info (currently limited to tagged addresses since + // everything else is managed by the Serf layer) + if !reflect.DeepEqual(out1.NodeServices.Node.TaggedAddresses, l.config.TaggedAddresses) { + l.nodeInfoInSync = false + } + + // Check all our services services := make(map[string]*structs.NodeService) if out1.NodeServices != nil { services = out1.NodeServices.Services @@ -440,6 +452,10 @@ func (l *localState) syncChanges() error { l.Lock() defer l.Unlock() + // We will do node-level info syncing at the end, since it will get + // updated by a service or check sync anyway, given how the register + // API works. + // Sync the services for id, status := range l.serviceStatus { if status.remoteDelete { @@ -475,6 +491,15 @@ func (l *localState) syncChanges() error { l.logger.Printf("[DEBUG] agent: Check '%s' in sync", id) } } + + // Now sync the node level info if we need to, and didn't do any of + // the other sync operations. + if !l.nodeInfoInSync { + if err := l.syncNodeInfo(); err != nil { + return err + } + } + return nil } @@ -554,6 +579,9 @@ func (l *localState) syncService(id string) error { err := l.iface.RPC("Catalog.Register", &req, &out) if err == nil { l.serviceStatus[id] = syncStatus{inSync: true} + // Given how the register API works, this info is also updated + // every time we sync a service. + l.nodeInfoInSync = true l.logger.Printf("[INFO] agent: Synced service '%s'", id) for _, check := range checks { l.checkStatus[check.CheckID] = syncStatus{inSync: true} @@ -593,6 +621,9 @@ func (l *localState) syncCheck(id string) error { err := l.iface.RPC("Catalog.Register", &req, &out) if err == nil { l.checkStatus[id] = syncStatus{inSync: true} + // Given how the register API works, this info is also updated + // every time we sync a service. + l.nodeInfoInSync = true l.logger.Printf("[INFO] agent: Synced check '%s'", id) } else if strings.Contains(err.Error(), permissionDenied) { l.checkStatus[id] = syncStatus{inSync: true} @@ -601,3 +632,24 @@ func (l *localState) syncCheck(id string) error { } return err } + +func (l *localState) syncNodeInfo() error { + req := structs.RegisterRequest{ + Datacenter: l.config.Datacenter, + Node: l.config.NodeName, + Address: l.config.AdvertiseAddr, + TaggedAddresses: l.config.TaggedAddresses, + WriteRequest: structs.WriteRequest{Token: l.config.ACLToken}, + } + var out struct{} + err := l.iface.RPC("Catalog.Register", &req, &out) + if err == nil { + l.nodeInfoInSync = true + l.logger.Printf("[INFO] agent: Synced node info") + } else if strings.Contains(err.Error(), permissionDenied) { + l.nodeInfoInSync = true + l.logger.Printf("[WARN] agent: Node info update blocked by ACLs") + return nil + } + return err +} diff --git a/command/agent/local_test.go b/command/agent/local_test.go index 58a2f2329..f7e453b0f 100644 --- a/command/agent/local_test.go +++ b/command/agent/local_test.go @@ -731,6 +731,66 @@ func TestAgentAntiEntropy_Check_DeferSync(t *testing.T) { }) } +func TestAgentAntiEntropy_NodeInfo(t *testing.T) { + conf := nextConfig() + dir, agent := makeAgent(t, conf) + defer os.RemoveAll(dir) + defer agent.Shutdown() + + testutil.WaitForLeader(t, agent.RPC, "dc1") + + // Register info + args := &structs.RegisterRequest{ + Datacenter: "dc1", + Node: agent.config.NodeName, + Address: "127.0.0.1", + } + var out struct{} + if err := agent.RPC("Catalog.Register", args, &out); err != nil { + t.Fatalf("err: %v", err) + } + + // Trigger anti-entropy run and wait + agent.StartSync() + time.Sleep(200 * time.Millisecond) + + // Verify that we are in sync + req := structs.NodeSpecificRequest{ + Datacenter: "dc1", + Node: agent.config.NodeName, + } + var services structs.IndexedNodeServices + if err := agent.RPC("Catalog.NodeServices", &req, &services); err != nil { + t.Fatalf("err: %v", err) + } + + // Make sure we synced our node info - this should have ridden on the + // "consul" service sync + addrs := services.NodeServices.Node.TaggedAddresses + if len(addrs) == 0 || !reflect.DeepEqual(addrs, conf.TaggedAddresses) { + t.Fatalf("bad: %v", addrs) + } + + // Blow away the catalog version of the node info + if err := agent.RPC("Catalog.Register", args, &out); err != nil { + t.Fatalf("err: %v", err) + } + + // Trigger anti-entropy run and wait + agent.StartSync() + time.Sleep(200 * time.Millisecond) + + // Verify that we are in sync - this should have been a sync of just the + // node info + if err := agent.RPC("Catalog.NodeServices", &req, &services); err != nil { + t.Fatalf("err: %v", err) + } + addrs = services.NodeServices.Node.TaggedAddresses + if len(addrs) == 0 || !reflect.DeepEqual(addrs, conf.TaggedAddresses) { + t.Fatalf("bad: %v", addrs) + } +} + func TestAgentAntiEntropy_deleteService_fails(t *testing.T) { l := new(localState) if err := l.deleteService(""); err == nil { diff --git a/consul/client.go b/consul/client.go index 89aaf438a..e7155a144 100644 --- a/consul/client.go +++ b/consul/client.go @@ -181,9 +181,6 @@ func (c *Client) setupSerf(conf *serf.Config, ch chan serf.Event, path string) ( conf.RejoinAfterLeave = c.config.RejoinAfterLeave conf.Merge = &lanMergeDelegate{dc: c.config.Datacenter} conf.DisableCoordinates = c.config.DisableCoordinates - if wanAddr := c.config.SerfWANConfig.MemberlistConfig.AdvertiseAddr; wanAddr != "" { - conf.Tags["wan_addr"] = wanAddr - } if err := ensurePath(conf.SnapshotPath, false); err != nil { return nil, err } diff --git a/consul/leader.go b/consul/leader.go index 374e7a9a2..55c487b4f 100644 --- a/consul/leader.go +++ b/consul/leader.go @@ -380,11 +380,6 @@ func (s *Server) handleAliveMember(member serf.Member) error { return err } if node != nil && node.Address == member.Addr.String() { - // Check if the WAN address was updated - if node.TaggedAddresses["wan"] != member.Tags["wan_addr"] { - goto AFTER_CHECK - } - // Check if the associated service is available if service != nil { match := false @@ -423,10 +418,7 @@ AFTER_CHECK: Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), - TaggedAddresses: map[string]string{ - "wan": member.Tags["wan_addr"], - }, - Service: service, + Service: service, Check: &structs.HealthCheck{ Node: member.Name, CheckID: SerfCheckID, @@ -468,9 +460,6 @@ func (s *Server) handleFailedMember(member serf.Member) error { Datacenter: s.config.Datacenter, Node: member.Name, Address: member.Addr.String(), - TaggedAddresses: map[string]string{ - "wan": member.Tags["wan_addr"], - }, Check: &structs.HealthCheck{ Node: member.Name, CheckID: SerfCheckID, diff --git a/consul/server.go b/consul/server.go index 36371e60f..7a59cb594 100644 --- a/consul/server.go +++ b/consul/server.go @@ -299,9 +299,6 @@ func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string, w if s.config.BootstrapExpect != 0 { conf.Tags["expect"] = fmt.Sprintf("%d", s.config.BootstrapExpect) } - if wanAddr := s.config.SerfWANConfig.MemberlistConfig.AdvertiseAddr; wanAddr != "" { - conf.Tags["wan_addr"] = wanAddr - } conf.MemberlistConfig.LogOutput = s.config.LogOutput conf.LogOutput = s.config.LogOutput conf.EventCh = ch From f398e1880e290a7f41870b256f636d9747b23460 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 7 Feb 2016 13:15:22 -0800 Subject: [PATCH 16/19] Adds a test for node registration and tagged addresses. --- consul/state/state_store_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/consul/state/state_store_test.go b/consul/state/state_store_test.go index c60365168..114745ca9 100644 --- a/consul/state/state_store_test.go +++ b/consul/state/state_store_test.go @@ -397,6 +397,9 @@ func TestStateStore_EnsureRegistration(t *testing.T) { req := &structs.RegisterRequest{ Node: "node1", Address: "1.2.3.4", + TaggedAddresses: map[string]string{ + "hello": "world", + }, } if err := s.EnsureRegistration(1, req); err != nil { t.Fatalf("err: %s", err) @@ -409,6 +412,8 @@ func TestStateStore_EnsureRegistration(t *testing.T) { t.Fatalf("err: %s", err) } if out.Node != "node1" || out.Address != "1.2.3.4" || + len(out.TaggedAddresses) != 1 || + out.TaggedAddresses["hello"] != "world" || out.CreateIndex != created || out.ModifyIndex != modified { t.Fatalf("bad node returned: %#v", out) } From a61a7403a7d80db21e1337cee94b60eda90bfdf1 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 7 Feb 2016 13:39:37 -0800 Subject: [PATCH 17/19] Factors address translation into a single function. --- command/agent/dns.go | 59 +++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/command/agent/dns.go b/command/agent/dns.go index 9749b19f0..a50486173 100644 --- a/command/agent/dns.go +++ b/command/agent/dns.go @@ -363,6 +363,19 @@ INVALID: resp.SetRcode(req, dns.RcodeNameError) } +// translateAddr is used to provide the final, translated address for a node, +// depending on how this agent and the other node are configured. +func (d *DNSServer) translateAddr(dc string, node *structs.Node) string { + addr := node.Address + if d.agent.config.TranslateWanAddrs && (d.agent.config.Datacenter != dc) { + wanAddr := node.TaggedAddresses["wan"] + if wanAddr != "" { + addr = wanAddr + } + } + return addr +} + // nodeLookup is used to handle a node query func (d *DNSServer) nodeLookup(network, datacenter, node string, req, resp *dns.Msg) { // Only handle ANY, A and AAAA type requests @@ -402,13 +415,8 @@ RPC: return } - // Determine whether we should use the WAN address or not - addr := out.NodeServices.Node.Address - if d.agent.config.TranslateWanAddrs && datacenter != d.agent.config.Datacenter { - addr = out.NodeServices.Node.TaggedAddresses["wan"] - } - // Add the node record + addr := d.translateAddr(datacenter, out.NodeServices.Node) records := d.formatNodeRecord(out.NodeServices.Node, addr, req.Question[0].Name, qType, d.config.NodeTTL) if records != nil { @@ -530,18 +538,12 @@ RPC: // Perform a random shuffle out.Nodes.Shuffle() - // Determine whether we should use the WAN address or not - var useWan bool - if d.agent.config.TranslateWanAddrs && datacenter != d.agent.config.Datacenter { - useWan = true - } - // Add various responses depending on the request qType := req.Question[0].Qtype - d.serviceNodeRecords(out.Nodes, req, resp, ttl, useWan) + d.serviceNodeRecords(datacenter, out.Nodes, req, resp, ttl) if qType == dns.TypeSRV { - d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl, useWan) + d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl) } // If the network is not TCP, restrict the number of responses @@ -632,17 +634,11 @@ RPC: return } - // Determine whether we should use the WAN address or not - var useWan bool - if d.agent.config.TranslateWanAddrs && datacenter != d.agent.config.Datacenter { - useWan = true - } - // Add various responses depending on the request. qType := req.Question[0].Qtype - d.serviceNodeRecords(out.Nodes, req, resp, ttl, useWan) + d.serviceNodeRecords(datacenter, out.Nodes, req, resp, ttl) if qType == dns.TypeSRV { - d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl, useWan) + d.serviceSRVRecords(datacenter, out.Nodes, req, resp, ttl) } // If the network is not TCP, restrict the number of responses. @@ -664,18 +660,16 @@ RPC: } // serviceNodeRecords is used to add the node records for a service lookup -func (d *DNSServer) serviceNodeRecords(nodes structs.CheckServiceNodes, req, resp *dns.Msg, ttl time.Duration, useWan bool) { +func (d *DNSServer) serviceNodeRecords(dc string, nodes structs.CheckServiceNodes, req, resp *dns.Msg, ttl time.Duration) { qName := req.Question[0].Name qType := req.Question[0].Qtype handled := make(map[string]struct{}) for _, node := range nodes { - // Prefer the Service Address or WAN Address over the - // Node Address when configured - addr := node.Node.Address + // Start with the translated address but use the service address, + // if specified. + addr := d.translateAddr(dc, node.Node) if node.Service.Address != "" { addr = node.Service.Address - } else if useWan == true && node.Node.TaggedAddresses["wan"] != "" { - addr = node.Node.TaggedAddresses["wan"] } // Avoid duplicate entries, possible if a node has @@ -694,7 +688,7 @@ func (d *DNSServer) serviceNodeRecords(nodes structs.CheckServiceNodes, req, res } // serviceARecords is used to add the SRV records for a service lookup -func (d *DNSServer) serviceSRVRecords(dc string, nodes structs.CheckServiceNodes, req, resp *dns.Msg, ttl time.Duration, useWan bool) { +func (d *DNSServer) serviceSRVRecords(dc string, nodes structs.CheckServiceNodes, req, resp *dns.Msg, ttl time.Duration) { handled := make(map[string]struct{}) for _, node := range nodes { // Avoid duplicate entries, possible if a node has @@ -720,12 +714,11 @@ func (d *DNSServer) serviceSRVRecords(dc string, nodes structs.CheckServiceNodes } resp.Answer = append(resp.Answer, srvRec) - // Determine advertised address - addr := node.Node.Address + // Start with the translated address but use the service address, + // if specified. + addr := d.translateAddr(dc, node.Node) if node.Service.Address != "" { addr = node.Service.Address - } else if useWan == true && node.Node.TaggedAddresses["wan"] != "" { - addr = node.Node.TaggedAddresses["wan"] } // Add the extra record From 3be9f160307fa2c53595ed03ababb5b1c926d8e1 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 7 Feb 2016 13:56:45 -0800 Subject: [PATCH 18/19] Increases wait to account for random stagger. --- command/agent/local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/agent/local_test.go b/command/agent/local_test.go index f7e453b0f..62c418f81 100644 --- a/command/agent/local_test.go +++ b/command/agent/local_test.go @@ -899,7 +899,7 @@ func TestAgent_sendCoordinate(t *testing.T) { testutil.WaitForLeader(t, agent.RPC, "dc1") // Wait a little while for an update. - time.Sleep(2 * conf.ConsulConfig.CoordinateUpdatePeriod) + time.Sleep(3 * conf.ConsulConfig.CoordinateUpdatePeriod) // Make sure the coordinate is present. req := structs.DCSpecificRequest{ From 6797b3ee85a02116618ef42e5f72f9ae5004e22a Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 7 Feb 2016 14:16:15 -0800 Subject: [PATCH 19/19] Adds a test to make sure we get the local address within the DC. --- command/agent/dns_test.go | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/command/agent/dns_test.go b/command/agent/dns_test.go index cb017c912..32f4c1f14 100644 --- a/command/agent/dns_test.go +++ b/command/agent/dns_test.go @@ -848,6 +848,62 @@ func TestDNS_ServiceLookup_WanAddress(t *testing.T) { t.Fatalf("Bad: %#v", in.Answer[0]) } } + + // Now query from the same DC and make sure we get the local address + for _, question := range questions { + m := new(dns.Msg) + m.SetQuestion(question, dns.TypeSRV) + + c := new(dns.Client) + addr, _ := srv2.agent.config.ClientListener("", srv2.agent.config.Ports.DNS) + in, _, err := c.Exchange(m, addr.String()) + if err != nil { + t.Fatalf("err: %v", err) + } + + if len(in.Answer) != 1 { + t.Fatalf("Bad: %#v", in) + } + + aRec, ok := in.Extra[0].(*dns.A) + if !ok { + t.Fatalf("Bad: %#v", in.Extra[0]) + } + if aRec.Hdr.Name != "foo.node.dc2.consul." { + t.Fatalf("Bad: %#v", in.Extra[0]) + } + if aRec.A.String() != "127.0.0.1" { + t.Fatalf("Bad: %#v", in.Extra[0]) + } + } + + // Also check the A record directly from DC2 + for _, question := range questions { + m := new(dns.Msg) + m.SetQuestion(question, dns.TypeA) + + c := new(dns.Client) + addr, _ := srv2.agent.config.ClientListener("", srv2.agent.config.Ports.DNS) + in, _, err := c.Exchange(m, addr.String()) + if err != nil { + t.Fatalf("err: %v", err) + } + + if len(in.Answer) != 1 { + t.Fatalf("Bad: %#v", in) + } + + aRec, ok := in.Answer[0].(*dns.A) + if !ok { + t.Fatalf("Bad: %#v", in.Answer[0]) + } + if aRec.Hdr.Name != question { + t.Fatalf("Bad: %#v", in.Answer[0]) + } + if aRec.A.String() != "127.0.0.1" { + t.Fatalf("Bad: %#v", in.Answer[0]) + } + } } func TestDNS_CaseInsensitiveServiceLookup(t *testing.T) {