XDS Server Config (#4730)

* Config for the coming XDS server

* Default gRPC to 8502 for -dev mode; Re-merge the command Info output that shows gRPC.

agent/config/builder.go

@@ -340,6 +340,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 	httpPort := b.portVal("ports.http", c.Ports.HTTP)
 	httpsPort := b.portVal("ports.https", c.Ports.HTTPS)
 	serverPort := b.portVal("ports.server", c.Ports.Server)
+	grpcPort := b.portVal("ports.grpc", c.Ports.GRPC)
 	serfPortLAN := b.portVal("ports.serf_lan", c.Ports.SerfLAN)
 	serfPortWAN := b.portVal("ports.serf_wan", c.Ports.SerfWAN)
 	proxyMinPort := b.portVal("ports.proxy_min_port", c.Ports.ProxyMinPort)
@@ -445,6 +446,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 	dnsAddrs := b.makeAddrs(b.expandAddrs("addresses.dns", c.Addresses.DNS), clientAddrs, dnsPort)
 	httpAddrs := b.makeAddrs(b.expandAddrs("addresses.http", c.Addresses.HTTP), clientAddrs, httpPort)
 	httpsAddrs := b.makeAddrs(b.expandAddrs("addresses.https", c.Addresses.HTTPS), clientAddrs, httpsPort)
+	grpcAddrs := b.makeAddrs(b.expandAddrs("addresses.grpc", c.Addresses.GRPC), clientAddrs, grpcPort)
 
 	for _, a := range dnsAddrs {
 		if x, ok := a.(*net.TCPAddr); ok {
@@ -721,6 +723,8 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		EncryptKey:                              b.stringVal(c.EncryptKey),
 		EncryptVerifyIncoming:                   b.boolVal(c.EncryptVerifyIncoming),
 		EncryptVerifyOutgoing:                   b.boolVal(c.EncryptVerifyOutgoing),
+		GRPCPort:                                grpcPort,
+		GRPCAddrs:                               grpcAddrs,
 		KeyFile:                                 b.stringVal(c.KeyFile),
 		LeaveDrainTime:                          b.durationVal("performance.leave_drain_time", c.Performance.LeaveDrainTime),
 		LeaveOnTerm:                             leaveOnTerm,

agent/config/config.go

@@ -315,6 +315,7 @@ type Addresses struct {
 	DNS   *string `json:"dns,omitempty" hcl:"dns" mapstructure:"dns"`
 	HTTP  *string `json:"http,omitempty" hcl:"http" mapstructure:"http"`
 	HTTPS *string `json:"https,omitempty" hcl:"https" mapstructure:"https"`
+	GRPC  *string `json:"grpc,omitempty" hcl:"grpc" mapstructure:"grpc"`
 }
 
 type AdvertiseAddrsConfig struct {
@@ -575,6 +576,7 @@ type Ports struct {
 	SerfLAN        *int `json:"serf_lan,omitempty" hcl:"serf_lan" mapstructure:"serf_lan"`
 	SerfWAN        *int `json:"serf_wan,omitempty" hcl:"serf_wan" mapstructure:"serf_wan"`
 	Server         *int `json:"server,omitempty" hcl:"server" mapstructure:"server"`
+	GRPC           *int `json:"grpc,omitempty" hcl:"grpc" mapstructure:"grpc"`
 	ProxyMinPort   *int `json:"proxy_min_port,omitempty" hcl:"proxy_min_port" mapstructure:"proxy_min_port"`
 	ProxyMaxPort   *int `json:"proxy_max_port,omitempty" hcl:"proxy_max_port" mapstructure:"proxy_max_port"`
 	SidecarMinPort *int `json:"sidecar_min_port,omitempty" hcl:"sidecar_min_port" mapstructure:"sidecar_min_port"`

agent/config/default.go

@@ -102,6 +102,7 @@ func DefaultSource() Source {
 			dns = 8600
 			http = 8500
 			https = -1
+			grpc = -1
 			serf_lan = ` + strconv.Itoa(consul.DefaultLANSerfPort) + `
 			serf_wan = ` + strconv.Itoa(consul.DefaultWANSerfPort) + `
 			server = ` + strconv.Itoa(consul.DefaultRPCPort) + `
@@ -152,6 +153,9 @@ func DevSource() Source {
 		performance = {
 			raft_multiplier = 1
 		}
+		ports = {
+			grpc = 8502
+		}
 	`,
 	}
 }

agent/config/flags.go

@@ -59,7 +59,7 @@ func AddFlags(fs *flag.FlagSet, f *Flags) {
 	add(&f.Config.Ports.Server, "server-port", "Sets the server port to listen on.")
 	add(&f.Config.Bootstrap, "bootstrap", "Sets server to bootstrap mode.")
 	add(&f.Config.BootstrapExpect, "bootstrap-expect", "Sets server to expect bootstrap mode.")
-	add(&f.Config.ClientAddr, "client", "Sets the address to bind for client access. This includes RPC, DNS, HTTP and HTTPS (if configured).")
+	add(&f.Config.ClientAddr, "client", "Sets the address to bind for client access. This includes RPC, DNS, HTTP, HTTPS and gRPC (if configured).")
 	add(&f.ConfigFiles, "config-dir", "Path to a directory to read configuration files from. This will read every file ending in '.json' as configuration in this directory in alphabetical order. Can be specified multiple times.")
 	add(&f.ConfigFiles, "config-file", "Path to a JSON file to read configuration from. Can be specified multiple times.")
 	add(&f.ConfigFormat, "config-format", "Config files are in this format irrespective of their extension. Must be 'hcl' or 'json'")
@@ -72,6 +72,7 @@ func AddFlags(fs *flag.FlagSet, f *Flags) {
 	add(&f.Config.DNSDomain, "domain", "Domain to use for DNS interface.")
 	add(&f.Config.EnableScriptChecks, "enable-script-checks", "Enables health check scripts.")
 	add(&f.Config.EncryptKey, "encrypt", "Provides the gossip encryption key.")
+	add(&f.Config.Ports.GRPC, "grpc-port", "Sets the gRPC API port to listen on (currently needed for Envoy xDS only).")
 	add(&f.Config.Ports.HTTP, "http-port", "Sets the HTTP API port to listen on.")
 	add(&f.Config.StartJoinAddrsLAN, "join", "Address of an agent to join at start time. Can be specified multiple times.")
 	add(&f.Config.StartJoinAddrsWAN, "join-wan", "Address of an agent to join -wan at start time. Can be specified multiple times.")

agent/config/flags_test.go

@@ -48,6 +48,10 @@ func TestParseFlags(t *testing.T) {
 			args:  []string{`-dns-port`, `1`},
 			flags: Flags{Config: Config{Ports: Ports{DNS: pInt(1)}}},
 		},
+		{
+			args:  []string{`-grpc-port`, `1`},
+			flags: Flags{Config: Config{Ports: Ports{GRPC: pInt(1)}}},
+		},
 		{
 			args:  []string{`-serf-lan-port`, `1`},
 			flags: Flags{Config: Config{Ports: Ports{SerfLAN: pInt(1)}}},

agent/config/runtime.go

@@ -664,6 +664,28 @@ type RuntimeConfig struct {
 	// hcl: encrypt_verify_outgoing = (true|false)
 	EncryptVerifyOutgoing bool
 
+	// GRPCPort is the port the gRPC server listens on. Currently this only
+	// exposes the xDS and ext_authx APIs for Envoy and it disabled by default.
+	//
+	// hcl: ports { grpc = int }
+	// flags: -grpc-port int
+	GRPCPort int
+
+	// GRPCAddrs contains the list of TCP addresses and UNIX sockets the gRPC
+	// server will bind to. If the gRPC endpoint is disabled (ports.grpc <= 0)
+	// the list is empty.
+	//
+	// The addresses are taken from 'addresses.grpc' which should contain a
+	// space separated list of ip addresses, UNIX socket paths and/or
+	// go-sockaddr templates. UNIX socket paths must be written as
+	// 'unix://<full path>', e.g. 'unix:///var/run/consul-grpc.sock'.
+	//
+	// If 'addresses.grpc' was not provided the 'client_addr' addresses are
+	// used.
+	//
+	// hcl: client_addr = string addresses { grpc = string } ports { grpc = int }
+	GRPCAddrs []net.Addr
+
 	// HTTPAddrs contains the list of TCP addresses and UNIX sockets the HTTP
 	// server will bind to. If the HTTP endpoint is disabled (ports.http <= 0)
 	// the list is empty.

agent/config/runtime_test.go

@@ -295,6 +295,8 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				rt.GossipWANProbeTimeout = 100 * time.Millisecond
 				rt.GossipWANSuspicionMult = 3
 				rt.ConsulServerHealthInterval = 10 * time.Millisecond
+				rt.GRPCPort = 8502
+				rt.GRPCAddrs = []net.Addr{tcpAddr("127.0.0.1:8502")}
 			},
 		},
 		{
@@ -839,11 +841,11 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 			args: []string{`-data-dir=` + dataDir},
 			json: []string{`{
 					"client_addr":"0.0.0.0",
-					"ports": { "dns":-1, "http":-2, "https":-3 }
+					"ports": { "dns":-1, "http":-2, "https":-3, "grpc":-4 }
 				}`},
 			hcl: []string{`
 					client_addr = "0.0.0.0"
-					ports { dns = -1 http = -2 https = -3 }
+					ports { dns = -1 http = -2 https = -3 grpc = -4 }
 				`},
 			patch: func(rt *RuntimeConfig) {
 				rt.ClientAddrs = []*net.IPAddr{ipAddr("0.0.0.0")}
@@ -851,6 +853,8 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				rt.DNSAddrs = nil
 				rt.HTTPPort = -1
 				rt.HTTPAddrs = nil
+				// HTTPS and gRPC default to disabled so shouldn't be different from
+				// default rt.
 				rt.DataDir = dataDir
 			},
 		},
@@ -859,11 +863,11 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 			args: []string{`-data-dir=` + dataDir},
 			json: []string{`{
 					"client_addr":"0.0.0.0",
-					"ports":{ "dns": 1, "http": 2, "https": 3 }
+					"ports":{ "dns": 1, "http": 2, "https": 3, "grpc": 4 }
 				}`},
 			hcl: []string{`
 					client_addr = "0.0.0.0"
-					ports { dns = 1 http = 2 https = 3 }
+					ports { dns = 1 http = 2 https = 3 grpc = 4 }
 				`},
 			patch: func(rt *RuntimeConfig) {
 				rt.ClientAddrs = []*net.IPAddr{ipAddr("0.0.0.0")}
@@ -873,6 +877,8 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				rt.HTTPAddrs = []net.Addr{tcpAddr("0.0.0.0:2")}
 				rt.HTTPSPort = 3
 				rt.HTTPSAddrs = []net.Addr{tcpAddr("0.0.0.0:3")}
+				rt.GRPCPort = 4
+				rt.GRPCAddrs = []net.Addr{tcpAddr("0.0.0.0:4")}
 				rt.DataDir = dataDir
 			},
 		},
@@ -882,18 +888,20 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 			args: []string{`-data-dir=` + dataDir},
 			json: []string{`{
 					"client_addr":"0.0.0.0",
-					"addresses": { "dns": "1.1.1.1", "http": "2.2.2.2", "https": "3.3.3.3" },
+					"addresses": { "dns": "1.1.1.1", "http": "2.2.2.2", "https": "3.3.3.3", "grpc": "4.4.4.4" },
 					"ports":{}
 				}`},
 			hcl: []string{`
 					client_addr = "0.0.0.0"
-					addresses = { dns = "1.1.1.1" http = "2.2.2.2" https = "3.3.3.3" }
+					addresses = { dns = "1.1.1.1" http = "2.2.2.2" https = "3.3.3.3" grpc = "4.4.4.4" }
 					ports {}
 				`},
 			patch: func(rt *RuntimeConfig) {
 				rt.ClientAddrs = []*net.IPAddr{ipAddr("0.0.0.0")}
 				rt.DNSAddrs = []net.Addr{tcpAddr("1.1.1.1:8600"), udpAddr("1.1.1.1:8600")}
 				rt.HTTPAddrs = []net.Addr{tcpAddr("2.2.2.2:8500")}
+				// HTTPS and gRPC default to disabled so shouldn't be different from
+				// default rt.
 				rt.DataDir = dataDir
 			},
 		},
@@ -902,13 +910,13 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 			args: []string{`-data-dir=` + dataDir},
 			json: []string{`{
 					"client_addr":"0.0.0.0",
-					"addresses": { "dns": "1.1.1.1", "http": "2.2.2.2", "https": "3.3.3.3" },
+					"addresses": { "dns": "1.1.1.1", "http": "2.2.2.2", "https": "3.3.3.3", "grpc": "4.4.4.4" },
-					"ports": { "dns":-1, "http":-2, "https":-3 }
+					"ports": { "dns":-1, "http":-2, "https":-3, "grpc":-4 }
 				}`},
 			hcl: []string{`
 					client_addr = "0.0.0.0"
-					addresses = { dns = "1.1.1.1" http = "2.2.2.2" https = "3.3.3.3" }
+					addresses = { dns = "1.1.1.1" http = "2.2.2.2" https = "3.3.3.3" grpc = "4.4.4.4" }
-					ports { dns = -1 http = -2 https = -3 }
+					ports { dns = -1 http = -2 https = -3 grpc = -4 }
 				`},
 			patch: func(rt *RuntimeConfig) {
 				rt.ClientAddrs = []*net.IPAddr{ipAddr("0.0.0.0")}
@@ -916,6 +924,8 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				rt.DNSAddrs = nil
 				rt.HTTPPort = -1
 				rt.HTTPAddrs = nil
+				// HTTPS and gRPC default to disabled so shouldn't be different from
+				// default rt.
 				rt.DataDir = dataDir
 			},
 		},
@@ -924,13 +934,13 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 			args: []string{`-data-dir=` + dataDir},
 			json: []string{`{
 					"client_addr": "0.0.0.0",
-					"addresses": { "dns": "1.1.1.1", "http": "2.2.2.2", "https": "3.3.3.3" },
+					"addresses": { "dns": "1.1.1.1", "http": "2.2.2.2", "https": "3.3.3.3", "grpc": "4.4.4.4" },
-					"ports":{ "dns":1, "http":2, "https":3 }
+					"ports":{ "dns":1, "http":2, "https":3, "grpc":4 }
 				}`},
 			hcl: []string{`
 					client_addr = "0.0.0.0"
-					addresses = { dns = "1.1.1.1" http = "2.2.2.2" https = "3.3.3.3" }
+					addresses = { dns = "1.1.1.1" http = "2.2.2.2" https = "3.3.3.3" grpc = "4.4.4.4" }
-					ports { dns = 1 http = 2 https = 3 }
+					ports { dns = 1 http = 2 https = 3 grpc = 4 }
 				`},
 			patch: func(rt *RuntimeConfig) {
 				rt.ClientAddrs = []*net.IPAddr{ipAddr("0.0.0.0")}
@@ -940,6 +950,8 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				rt.HTTPAddrs = []net.Addr{tcpAddr("2.2.2.2:2")}
 				rt.HTTPSPort = 3
 				rt.HTTPSAddrs = []net.Addr{tcpAddr("3.3.3.3:3")}
+				rt.GRPCPort = 4
+				rt.GRPCAddrs = []net.Addr{tcpAddr("4.4.4.4:4")}
 				rt.DataDir = dataDir
 			},
 		},
@@ -948,11 +960,11 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 			args: []string{`-data-dir=` + dataDir},
 			json: []string{`{
 					"client_addr": "{{ printf \"1.2.3.4 2001:db8::1\" }}",
-					"ports":{ "dns":1, "http":2, "https":3 }
+					"ports":{ "dns":1, "http":2, "https":3, "grpc":4 }
 				}`},
 			hcl: []string{`
 					client_addr = "{{ printf \"1.2.3.4 2001:db8::1\" }}"
-					ports { dns = 1 http = 2 https = 3 }
+					ports { dns = 1 http = 2 https = 3 grpc = 4 }
 				`},
 			patch: func(rt *RuntimeConfig) {
 				rt.ClientAddrs = []*net.IPAddr{ipAddr("1.2.3.4"), ipAddr("2001:db8::1")}
@@ -962,6 +974,8 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				rt.HTTPAddrs = []net.Addr{tcpAddr("1.2.3.4:2"), tcpAddr("[2001:db8::1]:2")}
 				rt.HTTPSPort = 3
 				rt.HTTPSAddrs = []net.Addr{tcpAddr("1.2.3.4:3"), tcpAddr("[2001:db8::1]:3")}
+				rt.GRPCPort = 4
+				rt.GRPCAddrs = []net.Addr{tcpAddr("1.2.3.4:4"), tcpAddr("[2001:db8::1]:4")}
 				rt.DataDir = dataDir
 			},
 		},
@@ -973,9 +987,10 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 					"addresses": {
 						"dns": "{{ printf \"1.1.1.1 2001:db8::10 \" }}",
 						"http": "{{ printf \"2.2.2.2 unix://http 2001:db8::20 \" }}",
-						"https": "{{ printf \"3.3.3.3 unix://https 2001:db8::30 \" }}"
+						"https": "{{ printf \"3.3.3.3 unix://https 2001:db8::30 \" }}",
+						"grpc": "{{ printf \"4.4.4.4 unix://grpc 2001:db8::40 \" }}"
 					},
-					"ports":{ "dns":1, "http":2, "https":3 }
+					"ports":{ "dns":1, "http":2, "https":3, "grpc":4 }
 				}`},
 			hcl: []string{`
 					client_addr = "{{ printf \"1.2.3.4 2001:db8::1\" }}"
@@ -983,8 +998,9 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 						dns = "{{ printf \"1.1.1.1 2001:db8::10 \" }}"
 						http = "{{ printf \"2.2.2.2 unix://http 2001:db8::20 \" }}"
 						https = "{{ printf \"3.3.3.3 unix://https 2001:db8::30 \" }}"
+						grpc = "{{ printf \"4.4.4.4 unix://grpc 2001:db8::40 \" }}"
 					}
-					ports { dns = 1 http = 2 https = 3 }
+					ports { dns = 1 http = 2 https = 3 grpc = 4 }
 				`},
 			patch: func(rt *RuntimeConfig) {
 				rt.ClientAddrs = []*net.IPAddr{ipAddr("1.2.3.4"), ipAddr("2001:db8::1")}
@@ -994,6 +1010,8 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				rt.HTTPAddrs = []net.Addr{tcpAddr("2.2.2.2:2"), unixAddr("unix://http"), tcpAddr("[2001:db8::20]:2")}
 				rt.HTTPSPort = 3
 				rt.HTTPSAddrs = []net.Addr{tcpAddr("3.3.3.3:3"), unixAddr("unix://https"), tcpAddr("[2001:db8::30]:3")}
+				rt.GRPCPort = 4
+				rt.GRPCAddrs = []net.Addr{tcpAddr("4.4.4.4:4"), unixAddr("unix://grpc"), tcpAddr("[2001:db8::40]:4")}
 				rt.DataDir = dataDir
 			},
 		},
@@ -2779,7 +2797,8 @@ func TestFullConfig(t *testing.T) {
 			"addresses": {
 				"dns": "93.95.95.81",
 				"http": "83.39.91.39",
-				"https": "95.17.17.19"
+				"https": "95.17.17.19",
+				"grpc": "32.31.61.91"
 			},
 			"advertise_addr": "17.99.29.16",
 			"advertise_addr_wan": "78.63.37.19",
@@ -2971,6 +2990,7 @@ func TestFullConfig(t *testing.T) {
 				"http": 7999,
 				"https": 15127,
 				"server": 3757,
+				"grpc": 4881,
 				"proxy_min_port": 2000,
 				"proxy_max_port": 3000,
 				"sidecar_min_port": 8888,
@@ -3306,6 +3326,7 @@ func TestFullConfig(t *testing.T) {
 				dns = "93.95.95.81"
 				http = "83.39.91.39"
 				https = "95.17.17.19"
+				grpc = "32.31.61.91"
 			}
 			advertise_addr = "17.99.29.16"
 			advertise_addr_wan = "78.63.37.19"
@@ -3499,6 +3520,7 @@ func TestFullConfig(t *testing.T) {
 				http = 7999,
 				https = 15127
 				server = 3757
+				grpc = 4881
 				proxy_min_port = 2000
 				proxy_max_port = 3000
 				sidecar_min_port = 8888
@@ -4091,6 +4113,8 @@ func TestFullConfig(t *testing.T) {
 		EncryptKey:                       "A4wELWqH",
 		EncryptVerifyIncoming:            true,
 		EncryptVerifyOutgoing:            true,
+		GRPCPort:                         4881,
+		GRPCAddrs:                        []net.Addr{tcpAddr("32.31.61.91:4881")},
 		HTTPAddrs:                        []net.Addr{tcpAddr("83.39.91.39:7999")},
 		HTTPBlockEndpoints:               []string{"RBvAFcGD", "fWOWFznh"},
 		HTTPPort:                         7999,
@@ -4863,6 +4887,8 @@ func TestSanitize(t *testing.T) {
 		"EncryptKey": "hidden",
 		"EncryptVerifyIncoming": false,
 		"EncryptVerifyOutgoing": false,
+		"GRPCAddrs": [],
+		"GRPCPort": 0,
 		"HTTPAddrs": [
 			"tcp://1.2.3.4:5678",
 			"unix:///var/run/foo"

command/agent/agent.go

@@ -256,8 +256,8 @@ func (c *cmd) run(args []string) int {
 	c.UI.Info(fmt.Sprintf("     Node name: '%s'", config.NodeName))
 	c.UI.Info(fmt.Sprintf("    Datacenter: '%s' (Segment: '%s')", config.Datacenter, segment))
 	c.UI.Info(fmt.Sprintf("        Server: %v (Bootstrap: %v)", config.ServerMode, config.Bootstrap))
-	c.UI.Info(fmt.Sprintf("   Client Addr: %v (HTTP: %d, HTTPS: %d, DNS: %d)", config.ClientAddrs,
+	c.UI.Info(fmt.Sprintf("   Client Addr: %v (HTTP: %d, HTTPS: %d, gRPC: %d, DNS: %d)", config.ClientAddrs,
-		config.HTTPPort, config.HTTPSPort, config.DNSPort))
+		config.HTTPPort, config.HTTPSPort, config.GRPCPort, config.DNSPort))
 	c.UI.Info(fmt.Sprintf("  Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddrLAN,
 		config.SerfPortLAN, config.SerfPortWAN))
 	c.UI.Info(fmt.Sprintf("       Encrypt: Gossip: %v, TLS-Outgoing: %v, TLS-Incoming: %v",