From e66301ab99d53d74cecd52ad0a35e07eadda61f7 Mon Sep 17 00:00:00 2001 From: pepov Date: Sat, 21 Mar 2015 10:14:03 +0100 Subject: [PATCH 1/5] Add configuration option to specify a separate address for advertising on the wan --- command/agent/agent.go | 15 ++++++++++++++- command/agent/config.go | 7 +++++++ command/agent/config_test.go | 17 +++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/command/agent/agent.go b/command/agent/agent.go index c741240ac..1df338f8e 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -127,6 +127,15 @@ func Create(config *Config, logOutput io.Writer) (*Agent, error) { config.AdvertiseAddr = ip.String() } + // Try to get an advertise address for the wan + if config.AdvertiseAddrWan != "" { + if ip := net.ParseIP(config.AdvertiseAddrWan); ip == nil { + return nil, fmt.Errorf("Failed to parse advertise address for wan: %v", config.AdvertiseAddrWan) + } + } else { + config.AdvertiseAddrWan = config.AdvertiseAddr + } + agent := &Agent{ config: config, logger: log.New(logOutput, "", log.LstdFlags), @@ -225,7 +234,11 @@ func (a *Agent) consulConfig() *consul.Config { } if a.config.AdvertiseAddr != "" { base.SerfLANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddr - base.SerfWANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddr + if a.config.AdvertiseAddrWan != "" { + base.SerfWANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddrWan + } else { + base.SerfWANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddr + } base.RPCAdvertise = &net.TCPAddr{ IP: net.ParseIP(a.config.AdvertiseAddr), Port: a.config.Ports.Server, diff --git a/command/agent/config.go b/command/agent/config.go index 48b347c33..078d17292 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -142,6 +142,10 @@ type Config struct { // and Consul RPC IP. If not specified, bind address is used. AdvertiseAddr string `mapstructure:"advertise_addr"` + // AdvertiseAddrWan is the address we use for advertising our + // Serf WAN IP. If not specified, the general advertise address is used. + AdvertiseAddrWan string `mapstructure:"advertise_addr_wan"` + // Port configurations Ports PortConfig @@ -799,6 +803,9 @@ func MergeConfig(a, b *Config) *Config { if b.AdvertiseAddr != "" { result.AdvertiseAddr = b.AdvertiseAddr } + if b.AdvertiseAddrWan != "" { + result.AdvertiseAddrWan = b.AdvertiseAddrWan + } if b.Server == true { result.Server = b.Server } diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 945e2152d..5ca94261c 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -190,10 +190,27 @@ func TestDecodeConfig(t *testing.T) { t.Fatalf("bad: %#v", config) } + if config.AdvertiseAddrWan != "" { + t.Fatalf("bad: %#v", config) + } + if config.Ports.Server != 8000 { t.Fatalf("bad: %#v", config) } + // Advertise address for wan + input = `{"advertise_addr_wan": "127.0.0.5"}` + config, err = DecodeConfig(bytes.NewReader([]byte(input))) + if err != nil { + t.Fatalf("err: %s", err) + } + if config.AdvertiseAddr != "" { + t.Fatalf("bad: %#v", config) + } + if config.AdvertiseAddrWan != "127.0.0.5" { + t.Fatalf("bad: %#v", config) + } + // leave_on_terminate input = `{"leave_on_terminate": true}` config, err = DecodeConfig(bytes.NewReader([]byte(input))) From 0a04003e160411054a973e9532cf94b938690002 Mon Sep 17 00:00:00 2001 From: pepov Date: Sat, 28 Mar 2015 10:08:32 +0100 Subject: [PATCH 2/5] add AdvertiseAddrWan to TestMergeConfig and use different values for Addr values in b than in a --- command/agent/config_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 5ca94261c..7497eec08 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -1070,12 +1070,13 @@ func TestMergeConfig(t *testing.T) { MaxStale: 30 * time.Second, EnableTruncate: true, }, - Domain: "other", - LogLevel: "info", - NodeName: "baz", - ClientAddr: "127.0.0.1", - BindAddr: "127.0.0.1", - AdvertiseAddr: "127.0.0.1", + Domain: "other", + LogLevel: "info", + NodeName: "baz", + ClientAddr: "127.0.0.2", + BindAddr: "127.0.0.2", + AdvertiseAddr: "127.0.0.2", + AdvertiseAddrWan: "127.0.0.2", Ports: PortConfig{ DNS: 1, HTTP: 2, From 8abcf3d5415cdfb36f715cea8d12b018c1f6c6d3 Mon Sep 17 00:00:00 2001 From: pepov Date: Sat, 28 Mar 2015 10:47:28 +0100 Subject: [PATCH 3/5] add test using separate advertise addresses for wan and for lan --- consul/server_test.go | 99 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/consul/server_test.go b/consul/server_test.go index 0932a1f32..2029e4234 100644 --- a/consul/server_test.go +++ b/consul/server_test.go @@ -216,6 +216,105 @@ func TestServer_JoinWAN(t *testing.T) { }) } +func TestServer_JoinSeparateLanAndWanAddresses(t *testing.T) { + dir1, s1 := testServer(t) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + + dir2, s2 := testServerWithConfig(t, func(c *Config) { + c.NodeName = "s2" + c.Datacenter = "dc2" + // This wan address will be expected to be seen on s1 + c.SerfWANConfig.MemberlistConfig.AdvertiseAddr = "127.0.0.2" + // This lan address will be expected to be seen on s3 + c.SerfLANConfig.MemberlistConfig.AdvertiseAddr = "127.0.0.3" + }) + + defer os.RemoveAll(dir2) + defer s2.Shutdown() + + dir3, s3 := testServerDC(t, "dc2") + defer os.RemoveAll(dir3) + defer s3.Shutdown() + + // Join s2 to s1 on wan + addrs1 := fmt.Sprintf("127.0.0.1:%d", + s1.config.SerfWANConfig.MemberlistConfig.BindPort) + if _, err := s2.JoinWAN([]string{addrs1}); err != nil { + t.Fatalf("err: %v", err) + } + + // Join s3 to s2 on lan + addrs2 := fmt.Sprintf("127.0.0.1:%d", + s2.config.SerfLANConfig.MemberlistConfig.BindPort) + if _, err := s3.JoinLAN([]string{addrs2}); err != nil { + t.Fatalf("err: %v", err) + } + + // Check the WAN members on s1 + testutil.WaitForResult(func() (bool, error) { + return len(s1.WANMembers()) == 2, nil + }, func(err error) { + t.Fatalf("bad len") + }) + + // Check the WAN members on s2 + testutil.WaitForResult(func() (bool, error) { + return len(s2.WANMembers()) == 2, nil + }, func(err error) { + t.Fatalf("bad len") + }) + + // Check the LAN members on s2 + testutil.WaitForResult(func() (bool, error) { + return len(s2.LANMembers()) == 2, nil + }, func(err error) { + t.Fatalf("bad len") + }) + + // Check the LAN members on s3 + testutil.WaitForResult(func() (bool, error) { + return len(s3.LANMembers()) == 2, nil + }, func(err error) { + t.Fatalf("bad len") + }) + + // Check the remoteConsuls has both + if len(s1.remoteConsuls) != 2 { + t.Fatalf("remote consul missing") + } + + if len(s2.remoteConsuls) != 2 { + t.Fatalf("remote consul missing") + } + + if len(s2.localConsuls) != 2 { + t.Fatalf("local consul fellow s3 for s2 missing") + } + + // Get and check the wan address of s2 from s1 + var s2WanAddr string + for _, member := range s1.WANMembers() { + if member.Name == "s2.dc2" { + s2WanAddr = member.Addr.String() + } + } + if s2WanAddr != "127.0.0.2" { + t.Fatalf("s1 sees s2 on a wrong address: %s, expecting: %s", s2WanAddr, "127.0.0.2") + } + + // Get and check the lan address of s2 from s3 + var s2LanAddr string + for _, lanmember := range s3.LANMembers() { + if lanmember.Name == "s2" { + s2LanAddr = lanmember.Addr.String() + } + } + if s2LanAddr != "127.0.0.3" { + t.Fatalf("s3 sees s2 on a wrong address: %s, expecting: %s", s2LanAddr, "127.0.0.3") + } +} + func TestServer_LeaveLeader(t *testing.T) { dir1, s1 := testServer(t) defer os.RemoveAll(dir1) From 8da6d4f0653ce5ce80cb99fbde108e668a4ed8da Mon Sep 17 00:00:00 2001 From: pepov Date: Sat, 28 Mar 2015 15:48:06 +0100 Subject: [PATCH 4/5] add and test -advertise-wan cli flag --- command/agent/command.go | 1 + command/agent/command_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/command/agent/command.go b/command/agent/command.go index 05caa7a3f..149fa8084 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -80,6 +80,7 @@ func (c *Command) readConfig() *Config { cmdFlags.StringVar(&cmdConfig.ClientAddr, "client", "", "address to bind client listeners to (DNS, HTTP, HTTPS, RPC)") cmdFlags.StringVar(&cmdConfig.BindAddr, "bind", "", "address to bind server listeners to") cmdFlags.StringVar(&cmdConfig.AdvertiseAddr, "advertise", "", "address to advertise instead of bind addr") + cmdFlags.StringVar(&cmdConfig.AdvertiseAddrWan, "advertise-wan", "", "address to advertise on wan instead of bind or advertise addr") cmdFlags.StringVar(&cmdConfig.AtlasInfrastructure, "atlas", "", "infrastructure name in Atlas") cmdFlags.StringVar(&cmdConfig.AtlasToken, "atlas-token", "", "authentication token for Atlas") diff --git a/command/agent/command_test.go b/command/agent/command_test.go index 58129bba9..b54bd45c5 100644 --- a/command/agent/command_test.go +++ b/command/agent/command_test.go @@ -106,6 +106,32 @@ func TestRetryJoin(t *testing.T) { }) } +func TestReadCliConfig(t *testing.T) { + + shutdownCh := make(chan struct{}) + defer close(shutdownCh) + + tmpDir, err := ioutil.TempDir("", "consul") + if err != nil { + t.Fatalf("err: %s", err) + } + + cmd := &Command{ + args: []string{ + "-data-dir", tmpDir, + "-node", `"a"`, + "-advertise-wan", "1.2.3.4", + }, + ShutdownCh: shutdownCh, + Ui: new(cli.MockUi), + } + + config := cmd.readConfig() + if config.AdvertiseAddrWan != "1.2.3.4" { + t.Fatalf("expected -advertise-addr-wan 1.2.3.4 got %s", config.AdvertiseAddrWan) + } +} + func TestRetryJoinFail(t *testing.T) { conf := nextConfig() tmpDir, err := ioutil.TempDir("", "consul") From 14c8c86fee501ca7fb2b1c83d5949a7f01a9c312 Mon Sep 17 00:00:00 2001 From: pepov Date: Sat, 28 Mar 2015 16:04:09 +0100 Subject: [PATCH 5/5] update the docs: add advertise-wan cli and advertise_addr_wan json config --- website/source/docs/agent/options.html.markdown | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/website/source/docs/agent/options.html.markdown b/website/source/docs/agent/options.html.markdown index e1ae91113..0e00c406d 100644 --- a/website/source/docs/agent/options.html.markdown +++ b/website/source/docs/agent/options.html.markdown @@ -41,6 +41,14 @@ The options below are all specified on the command-line. If this address is not routable, the node will be in a constant flapping state as other nodes will treat the non-routability as a failure. +* `-advertise-wan` - The advertise wan + address is used to change the address that we advertise to server nodes joining + through the WAN. By default, the [`-advertise`](#_advertise) address is advertised. + However, in some cases all members of all datacenters cannot be on the same + physical or virtual network, especially on hybrid setups mixing cloud and private datacenters. + This flag enables server nodes gossiping through the public network for the WAN while using + private VLANs for gossiping to each other and their client agents. + * `-atlas` - This flag enables [Atlas](https://atlas.hashicorp.com) integration. It is used to provide the Atlas infrastructure name and the SCADA connection. @@ -312,6 +320,9 @@ definitions support being updated during a reload. * `advertise_addr` Equivalent to the [`-advertise` command-line flag](#_advertise). +* `advertise_addr_wan` Equivalent to + the [`-advertise-wan` command-line flag](#_advertise-wan). + * `atlas_acl_token` When provided, any requests made by Atlas will use this ACL token unless explicitly overriden. When not provided the [`acl_token`](#acl_token) is used. This can be set to 'anonymous' to reduce permission below