From cb55fa37423f1793954cffc9baf82fe86aa0f3d3 Mon Sep 17 00:00:00 2001 From: Freddy Date: Thu, 26 Mar 2020 10:20:56 -0600 Subject: [PATCH] Enable CLI to register terminating gateways (#7500) * Enable CLI to register terminating gateways * Centralize gateway proxy configuration --- agent/agent_endpoint.go | 4 +- agent/agent_endpoint_test.go | 34 +++++++- agent/config/builder.go | 2 + agent/consul/helper_test.go | 15 +++- agent/consul/internal_endpoint_test.go | 9 +- agent/service_manager.go | 4 +- agent/service_manager_test.go | 56 ++++++++++++ agent/structs/structs.go | 41 ++++----- agent/structs/structs_test.go | 52 +++++++++++ agent/structs/testing_catalog.go | 9 ++ agent/xds/clusters.go | 2 +- agent/xds/config.go | 24 ++++-- agent/xds/config_test.go | 91 ++++++++++++++++++++ agent/xds/listeners.go | 2 +- api/agent.go | 4 + api/agent_test.go | 31 +++++++ command/connect/envoy/envoy.go | 74 +++++++++++----- command/connect/envoy/envoy_test.go | 57 ++++++++++-- command/connect/envoy/flags.go | 8 +- command/connect/envoy/flags_test.go | 6 +- command/connect/proxy/proxy.go | 14 +-- website/source/docs/connect/proxies/envoy.md | 20 +++-- 22 files changed, 465 insertions(+), 94 deletions(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 9f7a18fec..b182e5291 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -210,9 +210,7 @@ func buildAgentService(s *structs.NodeService) api.AgentService { as.Meta = map[string]string{} } // Attach Proxy config if exists - if s.Kind == structs.ServiceKindConnectProxy || - s.Kind == structs.ServiceKindMeshGateway { - + if s.Kind == structs.ServiceKindConnectProxy || s.IsGateway() { as.Proxy = s.Proxy.ToAPI() } diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index 915656f58..f7a37ea32 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -201,7 +201,7 @@ func TestAgent_Services_Sidecar(t *testing.T) { assert.NotContains(string(output), "locally_registered_as_sidecar") } -// Thie tests that a mesh gateway service is returned as expected. +// This tests that a mesh gateway service is returned as expected. func TestAgent_Services_MeshGateway(t *testing.T) { t.Parallel() @@ -233,6 +233,38 @@ func TestAgent_Services_MeshGateway(t *testing.T) { require.Equal(t, srv1.Proxy.ToAPI(), actual.Proxy) } +// This tests that a terminating gateway service is returned as expected. +func TestAgent_Services_TerminatingGateway(t *testing.T) { + t.Parallel() + + a := NewTestAgent(t, t.Name(), "") + defer a.Shutdown() + + testrpc.WaitForLeader(t, a.RPC, "dc1") + srv1 := &structs.NodeService{ + Kind: structs.ServiceKindTerminatingGateway, + ID: "tg-dc1-01", + Service: "tg-dc1", + Port: 8443, + Proxy: structs.ConnectProxyConfig{ + Config: map[string]interface{}{ + "foo": "bar", + }, + }, + } + require.NoError(t, a.State.AddService(srv1, "")) + + req, _ := http.NewRequest("GET", "/v1/agent/services", nil) + obj, err := a.srv.AgentServices(nil, req) + require.NoError(t, err) + val := obj.(map[string]*api.AgentService) + require.Len(t, val, 1) + actual := val["tg-dc1-01"] + require.NotNil(t, actual) + require.Equal(t, api.ServiceKindTerminatingGateway, actual.Kind) + require.Equal(t, srv1.Proxy.ToAPI(), actual.Proxy) +} + func TestAgent_Services_ACLFilter(t *testing.T) { t.Parallel() a := NewTestAgent(t, t.Name(), TestACLConfig()) diff --git a/agent/config/builder.go b/agent/config/builder.go index 5256c8b9b..6c6e93d87 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1410,6 +1410,8 @@ func (b *Builder) serviceKindVal(v *string) structs.ServiceKind { return structs.ServiceKindConnectProxy case string(structs.ServiceKindMeshGateway): return structs.ServiceKindMeshGateway + case string(structs.ServiceKindTerminatingGateway): + return structs.ServiceKindTerminatingGateway default: return structs.ServiceKindTypical } diff --git a/agent/consul/helper_test.go b/agent/consul/helper_test.go index 08955c86b..719210298 100644 --- a/agent/consul/helper_test.go +++ b/agent/consul/helper_test.go @@ -497,10 +497,23 @@ func registerTestCatalogEntries(t *testing.T, codec rpc.ClientCodec) { registerTestCatalogEntriesMap(t, codec, registrations) } -func registerTestCatalogEntriesMeshGateway(t *testing.T, codec rpc.ClientCodec) { +func registerTestCatalogProxyEntries(t *testing.T, codec rpc.ClientCodec) { t.Helper() registrations := map[string]*structs.RegisterRequest{ + "Service tg-gw": &structs.RegisterRequest{ + Datacenter: "dc1", + Node: "terminating-gateway", + ID: types.NodeID("3a9d7530-20d4-443a-98d3-c10fe78f09f4"), + Address: "10.1.2.2", + Service: &structs.NodeService{ + Kind: structs.ServiceKindTerminatingGateway, + ID: "tg-gw-01", + Service: "tg-gw", + Port: 8443, + Address: "198.18.1.3", + }, + }, "Service mg-gw": &structs.RegisterRequest{ Datacenter: "dc1", Node: "gateway", diff --git a/agent/consul/internal_endpoint_test.go b/agent/consul/internal_endpoint_test.go index 89f286e9c..bb61e9108 100644 --- a/agent/consul/internal_endpoint_test.go +++ b/agent/consul/internal_endpoint_test.go @@ -611,7 +611,7 @@ func TestInternal_ServiceDump_Kind(t *testing.T) { // prep the cluster with some data we can use in our filters registerTestCatalogEntries(t, codec) - registerTestCatalogEntriesMeshGateway(t, codec) + registerTestCatalogProxyEntries(t, codec) doRequest := func(t *testing.T, kind structs.ServiceKind) structs.CheckServiceNodes { t.Helper() @@ -633,6 +633,13 @@ func TestInternal_ServiceDump_Kind(t *testing.T) { require.Len(t, nodes, 9) }) + t.Run("Terminating Gateway", func(t *testing.T) { + nodes := doRequest(t, structs.ServiceKindTerminatingGateway) + require.Len(t, nodes, 1) + require.Equal(t, "tg-gw", nodes[0].Service.Service) + require.Equal(t, "tg-gw-01", nodes[0].Service.ID) + }) + t.Run("Mesh Gateway", func(t *testing.T) { nodes := doRequest(t, structs.ServiceKindMeshGateway) require.Len(t, nodes, 1) diff --git a/agent/service_manager.go b/agent/service_manager.go index d998f3151..b693ec6f4 100644 --- a/agent/service_manager.go +++ b/agent/service_manager.go @@ -120,9 +120,9 @@ func (s *ServiceManager) AddService(req *addServiceRequest) error { req.service.EnterpriseMeta.Normalize() - // For now only sidecar proxies have anything that can be configured + // For now only proxies have anything that can be configured // centrally. So bypass the whole manager for regular services. - if !req.service.IsSidecarProxy() && !req.service.IsMeshGateway() { + if !req.service.IsSidecarProxy() && !req.service.IsGateway() { // previousDefaults are ignored here because they are only relevant for central config. req.persistService = nil req.persistDefaults = nil diff --git a/agent/service_manager_test.go b/agent/service_manager_test.go index 7b211884e..e1a86ba8e 100644 --- a/agent/service_manager_test.go +++ b/agent/service_manager_test.go @@ -205,6 +205,62 @@ func TestServiceManager_RegisterMeshGateway(t *testing.T) { }, gateway) } +func TestServiceManager_RegisterTerminatingGateway(t *testing.T) { + require := require.New(t) + + a := NewTestAgent(t, t.Name(), "enable_central_service_config = true") + defer a.Shutdown() + + testrpc.WaitForLeader(t, a.RPC, "dc1") + + // Register a global proxy and service config + testApplyConfigEntries(t, a, + &structs.ProxyConfigEntry{ + Config: map[string]interface{}{ + "foo": 1, + }, + }, + &structs.ServiceConfigEntry{ + Kind: structs.ServiceDefaults, + Name: "terminating-gateway", + Protocol: "http", + }, + ) + + // Now register a terminating-gateway. + svc := &structs.NodeService{ + Kind: structs.ServiceKindTerminatingGateway, + ID: "terminating-gateway", + Service: "terminating-gateway", + Port: 443, + EnterpriseMeta: *structs.DefaultEnterpriseMeta(), + } + + require.NoError(a.AddService(svc, nil, false, "", ConfigSourceLocal)) + + // Verify gateway got global config loaded + gateway := a.State.Service(structs.NewServiceID("terminating-gateway", nil)) + require.NotNil(gateway) + require.Equal(&structs.NodeService{ + Kind: structs.ServiceKindTerminatingGateway, + ID: "terminating-gateway", + Service: "terminating-gateway", + Port: 443, + TaggedAddresses: map[string]structs.ServiceAddress{}, + Proxy: structs.ConnectProxyConfig{ + Config: map[string]interface{}{ + "foo": int64(1), + "protocol": "http", + }, + }, + Weights: &structs.Weights{ + Passing: 1, + Warning: 1, + }, + EnterpriseMeta: *structs.DefaultEnterpriseMeta(), + }, gateway) +} + func TestServiceManager_PersistService_API(t *testing.T) { // This is the ServiceManager version of TestAgent_PersistService and // TestAgent_PurgeService. diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 92ea6fde6..18c6681f6 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -890,21 +890,11 @@ const ( // service will proxy connections based off the SNI header set by other // connect proxies ServiceKindMeshGateway ServiceKind = "mesh-gateway" -) -func ServiceKindFromString(kind string) (ServiceKind, error) { - switch kind { - case string(ServiceKindTypical): - return ServiceKindTypical, nil - case string(ServiceKindConnectProxy): - return ServiceKindConnectProxy, nil - case string(ServiceKindMeshGateway): - return ServiceKindMeshGateway, nil - default: - // have to return something and it may as well be typical - return ServiceKindTypical, fmt.Errorf("Invalid service kind: %s", kind) - } -} + // ServiceKindTerminatingGateway is a Terminating Gateway for the Connect + // feature. This service will proxy connections to services outside the mesh. + ServiceKindTerminatingGateway ServiceKind = "terminating-gateway" +) // Type to hold a address and port of a service type ServiceAddress struct { @@ -1048,9 +1038,8 @@ func (s *NodeService) IsSidecarProxy() bool { return s.Kind == ServiceKindConnectProxy && s.Proxy.DestinationServiceID != "" } -func (s *NodeService) IsMeshGateway() bool { - // TODO (mesh-gateway) any other things to check? - return s.Kind == ServiceKindMeshGateway +func (s *NodeService) IsGateway() bool { + return s.Kind == ServiceKindMeshGateway || s.Kind == ServiceKindTerminatingGateway } // Validate validates the node service configuration. @@ -1147,36 +1136,36 @@ func (s *NodeService) Validate() error { } } - // MeshGateway validation - if s.Kind == ServiceKindMeshGateway { + // Gateway validation + if s.IsGateway() { // Gateways must have a port if s.Port == 0 { - result = multierror.Append(result, fmt.Errorf("Port must be non-zero for a Mesh Gateway")) + result = multierror.Append(result, fmt.Errorf("Port must be non-zero for a %s", s.Kind)) } // Gateways cannot have sidecars if s.Connect.SidecarService != nil { - result = multierror.Append(result, fmt.Errorf("Mesh Gateways cannot have a sidecar service defined")) + result = multierror.Append(result, fmt.Errorf("A %s cannot have a sidecar service defined", s.Kind)) } if s.Proxy.DestinationServiceName != "" { - result = multierror.Append(result, fmt.Errorf("The Proxy.DestinationServiceName configuration is invalid for Mesh Gateways")) + result = multierror.Append(result, fmt.Errorf("The Proxy.DestinationServiceName configuration is invalid for a %s", s.Kind)) } if s.Proxy.DestinationServiceID != "" { - result = multierror.Append(result, fmt.Errorf("The Proxy.DestinationServiceID configuration is invalid for Mesh Gateways")) + result = multierror.Append(result, fmt.Errorf("The Proxy.DestinationServiceID configuration is invalid for a %s", s.Kind)) } if s.Proxy.LocalServiceAddress != "" { - result = multierror.Append(result, fmt.Errorf("The Proxy.LocalServiceAddress configuration is invalid for Mesh Gateways")) + result = multierror.Append(result, fmt.Errorf("The Proxy.LocalServiceAddress configuration is invalid for a %s", s.Kind)) } if s.Proxy.LocalServicePort != 0 { - result = multierror.Append(result, fmt.Errorf("The Proxy.LocalServicePort configuration is invalid for Mesh Gateways")) + result = multierror.Append(result, fmt.Errorf("The Proxy.LocalServicePort configuration is invalid for a %s", s.Kind)) } if len(s.Proxy.Upstreams) != 0 { - result = multierror.Append(result, fmt.Errorf("The Proxy.Upstreams configuration is invalid for Mesh Gateways")) + result = multierror.Append(result, fmt.Errorf("The Proxy.Upstreams configuration is invalid for a %s", s.Kind)) } } diff --git a/agent/structs/structs_test.go b/agent/structs/structs_test.go index aeb55e324..2c51ca6d0 100644 --- a/agent/structs/structs_test.go +++ b/agent/structs/structs_test.go @@ -415,6 +415,58 @@ func TestStructs_NodeService_ValidateMeshGateway(t *testing.T) { } } +func TestStructs_NodeService_ValidateTerminatingGateway(t *testing.T) { + type testCase struct { + Modify func(*NodeService) + Err string + } + cases := map[string]testCase{ + "valid": testCase{ + func(x *NodeService) {}, + "", + }, + "sidecar-service": testCase{ + func(x *NodeService) { x.Connect.SidecarService = &ServiceDefinition{} }, + "cannot have a sidecar service", + }, + "proxy-destination-name": testCase{ + func(x *NodeService) { x.Proxy.DestinationServiceName = "foo" }, + "Proxy.DestinationServiceName configuration is invalid", + }, + "proxy-destination-id": testCase{ + func(x *NodeService) { x.Proxy.DestinationServiceID = "foo" }, + "Proxy.DestinationServiceID configuration is invalid", + }, + "proxy-local-address": testCase{ + func(x *NodeService) { x.Proxy.LocalServiceAddress = "127.0.0.1" }, + "Proxy.LocalServiceAddress configuration is invalid", + }, + "proxy-local-port": testCase{ + func(x *NodeService) { x.Proxy.LocalServicePort = 36 }, + "Proxy.LocalServicePort configuration is invalid", + }, + "proxy-upstreams": testCase{ + func(x *NodeService) { x.Proxy.Upstreams = []Upstream{Upstream{}} }, + "Proxy.Upstreams configuration is invalid", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ns := TestNodeServiceTerminatingGateway(t, "10.0.0.5") + tc.Modify(ns) + + err := ns.Validate() + if tc.Err == "" { + require.NoError(t, err) + } else { + require.Error(t, err) + require.Contains(t, strings.ToLower(err.Error()), strings.ToLower(tc.Err)) + } + }) + } +} + func TestStructs_NodeService_ValidateExposeConfig(t *testing.T) { type testCase struct { Modify func(*NodeService) diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index 1eca4a2e2..dfc526ee4 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -85,6 +85,15 @@ func TestNodeServiceMeshGateway(t testing.T) *NodeService { ServiceAddress{Address: "198.18.4.5", Port: 443}) } +func TestNodeServiceTerminatingGateway(t testing.T, address string) *NodeService { + return &NodeService{ + Kind: ServiceKindTerminatingGateway, + Port: 8443, + Service: "terminating-gateway", + Address: address, + } +} + func TestNodeServiceMeshGatewayWithAddrs(t testing.T, address string, port int, lanAddr, wanAddr ServiceAddress) *NodeService { return &NodeService{ Kind: ServiceKindMeshGateway, diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 6274d2eaa..afa22a0e2 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -480,7 +480,7 @@ func (s *Server) makeMeshGatewayCluster(clusterName string, cfgSnap *proxycfg.Co // defaults to use the mesh gateway timeout. func (s *Server) makeMeshGatewayClusterWithConnectTimeout(clusterName string, cfgSnap *proxycfg.ConfigSnapshot, connectTimeout time.Duration) (*envoy.Cluster, error) { - cfg, err := ParseMeshGatewayConfig(cfgSnap.Proxy.Config) + cfg, err := ParseGatewayConfig(cfgSnap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. diff --git a/agent/xds/config.go b/agent/xds/config.go index 146e561b1..a97161e4d 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -1,6 +1,7 @@ package xds import ( + "github.com/hashicorp/consul/lib" "strings" "github.com/hashicorp/consul/agent/structs" @@ -67,7 +68,7 @@ func ParseProxyConfig(m map[string]interface{}) (ProxyConfig, error) { return cfg, err } -type MeshGatewayConfig struct { +type GatewayConfig struct { // BindTaggedAddresses when set will cause all of the services tagged // addresses to have listeners bound to them in addition to the main service // address listener. This is only suitable when the tagged addresses are IP @@ -75,27 +76,32 @@ type MeshGatewayConfig struct { // for those addresses or where an external entity maps that IP to the Envoy // (like AWS EC2 mapping a public IP to the private interface) then this // cannot be used. See the BindAddresses config instead - // - // TODO - wow this is a verbose setting name. Maybe shorten this - BindTaggedAddresses bool `mapstructure:"envoy_mesh_gateway_bind_tagged_addresses"` + BindTaggedAddresses bool `mapstructure:"envoy_gateway_bind_tagged_addresses"` // BindAddresses additional bind addresses to configure listeners for - BindAddresses map[string]structs.ServiceAddress `mapstructure:"envoy_mesh_gateway_bind_addresses"` + BindAddresses map[string]structs.ServiceAddress `mapstructure:"envoy_gateway_bind_addresses"` // NoDefaultBind indicates that we should not bind to the default address of the // gateway service - NoDefaultBind bool `mapstructure:"envoy_mesh_gateway_no_default_bind"` + NoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind"` // ConnectTimeoutMs is the number of milliseconds to timeout making a new // connection to this upstream. Defaults to 5000 (5 seconds) if not set. ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"` } -// ParseMeshGatewayConfig returns the MeshGatewayConfig parsed from an opaque map. If an +// ParseGatewayConfig returns the GatewayConfig parsed from an opaque map. If an // error occurs during parsing, it is returned along with the default config. This // allows the caller to choose whether and how to report the error -func ParseMeshGatewayConfig(m map[string]interface{}) (MeshGatewayConfig, error) { - var cfg MeshGatewayConfig +func ParseGatewayConfig(m map[string]interface{}) (GatewayConfig, error) { + // Fixup for deprecated mesh gateway names + lib.TranslateKeys(m, map[string]string{ + "envoy_mesh_gateway_bind_tagged_addresses": "envoy_gateway_bind_tagged_addresses", + "envoy_mesh_gateway_bind_addresses": "envoy_gateway_bind_addresses", + "envoy_mesh_gateway_no_default_bind": "envoy_gateway_no_default_bind", + }) + + var cfg GatewayConfig err := mapstructure.WeakDecode(m, &cfg) if cfg.ConnectTimeoutMs < 1 { diff --git a/agent/xds/config_test.go b/agent/xds/config_test.go index ddb3b08cb..0957e6d6e 100644 --- a/agent/xds/config_test.go +++ b/agent/xds/config_test.go @@ -1,6 +1,7 @@ package xds import ( + "github.com/hashicorp/consul/agent/structs" "testing" "github.com/stretchr/testify/require" @@ -253,6 +254,96 @@ func TestParseUpstreamConfig(t *testing.T) { } } +func TestParseGatewayConfig(t *testing.T) { + tests := []struct { + name string + input map[string]interface{} + want GatewayConfig + }{ + { + name: "defaults - nil", + input: nil, + want: GatewayConfig{ + ConnectTimeoutMs: 5000, + }, + }, + { + name: "defaults - empty", + input: map[string]interface{}{}, + want: GatewayConfig{ + ConnectTimeoutMs: 5000, + }, + }, + { + name: "defaults - other stuff", + input: map[string]interface{}{ + "foo": "bar", + "envoy_foo": "envoy_bar", + }, + want: GatewayConfig{ + ConnectTimeoutMs: 5000, + }, + }, + { + name: "kitchen sink", + input: map[string]interface{}{ + "envoy_gateway_bind_tagged_addresses": true, + "envoy_gateway_bind_addresses": map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}}, + "envoy_gateway_no_default_bind": true, + "connect_timeout_ms": 10, + }, + want: GatewayConfig{ + ConnectTimeoutMs: 10, + BindTaggedAddresses: true, + NoDefaultBind: true, + BindAddresses: map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}}, + }, + }, + { + name: "deprecated kitchen sink", + input: map[string]interface{}{ + "envoy_mesh_gateway_bind_tagged_addresses": true, + "envoy_mesh_gateway_bind_addresses": map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}}, + "envoy_mesh_gateway_no_default_bind": true, + "connect_timeout_ms": 10, + }, + want: GatewayConfig{ + ConnectTimeoutMs: 10, + BindTaggedAddresses: true, + NoDefaultBind: true, + BindAddresses: map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}}, + }, + }, + { + name: "new fields override deprecated ones", + input: map[string]interface{}{ + // Deprecated + "envoy_mesh_gateway_bind_tagged_addresses": true, + "envoy_mesh_gateway_bind_addresses": map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}}, + "envoy_mesh_gateway_no_default_bind": true, + + // New + "envoy_gateway_bind_tagged_addresses": false, + "envoy_gateway_bind_addresses": map[string]structs.ServiceAddress{"bar": {Address: "127.0.0.1", Port: 8080}}, + "envoy_gateway_no_default_bind": false, + }, + want: GatewayConfig{ + ConnectTimeoutMs: 5000, + BindTaggedAddresses: false, + NoDefaultBind: false, + BindAddresses: map[string]structs.ServiceAddress{"bar": {Address: "127.0.0.1", Port: 8080}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseGatewayConfig(tt.input) + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + func intPointer(i int) *int { return &i } diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index ddff95c28..dbca837d6 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -181,7 +181,7 @@ func parseCheckPath(check structs.CheckType) (structs.ExposePath, error) { // listenersFromSnapshotMeshGateway returns the "listener" for a mesh-gateway service func (s *Server) listenersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) { - cfg, err := ParseMeshGatewayConfig(cfgSnap.Proxy.Config) + cfg, err := ParseGatewayConfig(cfgSnap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. diff --git a/api/agent.go b/api/agent.go index 929d3ccd3..6102ba741 100644 --- a/api/agent.go +++ b/api/agent.go @@ -28,6 +28,10 @@ const ( // service will proxy connections based off the SNI header set by other // connect proxies ServiceKindMeshGateway ServiceKind = "mesh-gateway" + + // ServiceKindTerminatingGateway is a Terminating Gateway for the Connect + // feature. This service will proxy connections to services outside the mesh. + ServiceKindTerminatingGateway ServiceKind = "terminating-gateway" ) // UpstreamDestType is the type of upstream discovery mechanism. diff --git a/api/agent_test.go b/api/agent_test.go index 085e991fd..956c6175c 100644 --- a/api/agent_test.go +++ b/api/agent_test.go @@ -1705,6 +1705,37 @@ func TestAgentService_Register_MeshGateway(t *testing.T) { require.Equal(t, "bar", svc.Proxy.Config["foo"]) } +func TestAgentService_Register_TerminatingGateway(t *testing.T) { + t.Parallel() + c, s := makeClient(t) + defer s.Stop() + + agent := c.Agent() + + reg := AgentServiceRegistration{ + Kind: ServiceKindTerminatingGateway, + Name: "terminating-gateway", + Address: "10.1.2.3", + Port: 8443, + Proxy: &AgentServiceConnectProxyConfig{ + Config: map[string]interface{}{ + "foo": "bar", + }, + }, + } + + err := agent.ServiceRegister(®) + require.NoError(t, err) + + svc, _, err := agent.Service("terminating-gateway", nil) + require.NoError(t, err) + require.NotNil(t, svc) + require.Equal(t, ServiceKindTerminatingGateway, svc.Kind) + require.NotNil(t, svc.Proxy) + require.Contains(t, svc.Proxy.Config, "foo") + require.Equal(t, "bar", svc.Proxy.Config["foo"]) +} + func TestAgentService_ExposeChecks(t *testing.T) { t.Parallel() c, s := makeClient(t) diff --git a/command/connect/envoy/envoy.go b/command/connect/envoy/envoy.go index 0ca642977..66c498872 100644 --- a/command/connect/envoy/envoy.go +++ b/command/connect/envoy/envoy.go @@ -46,6 +46,7 @@ type cmd struct { // flags meshGateway bool + gateway string proxyID string sidecarFor string adminAccessLogPath string @@ -64,10 +65,19 @@ type cmd struct { bindAddresses ServiceAddressMapValue exposeServers bool - meshGatewaySvcName string + gatewaySvcName string + gatewayKind api.ServiceKind } -const defaultEnvoyVersion = "1.13.1" +const ( + defaultEnvoyVersion = "1.13.1" + meshGatewayVal = "mesh" +) + +var supportedGateways = map[string]api.ServiceKind{ + "mesh": api.ServiceKindMeshGateway, + "terminating": api.ServiceKindTerminatingGateway, +} func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) @@ -75,9 +85,13 @@ func (c *cmd) init() { c.flags.StringVar(&c.proxyID, "proxy-id", os.Getenv("CONNECT_PROXY_ID"), "The proxy's ID on the local agent.") + // Deprecated in favor of `gateway` c.flags.BoolVar(&c.meshGateway, "mesh-gateway", false, "Configure Envoy as a Mesh Gateway.") + c.flags.StringVar(&c.gateway, "gateway", "", + "The type of gateway to register. One of: terminating or mesh") + c.flags.StringVar(&c.sidecarFor, "sidecar-for", os.Getenv("CONNECT_SIDECAR_FOR"), "The ID of a service instance on the local agent that this proxy should "+ "become a sidecar for. It requires that the proxy service is registered "+ @@ -128,7 +142,7 @@ func (c *cmd) init() { "address to use instead of the default binding rules given as `=:` "+ "pairs. This flag may be specified multiple times to add multiple bind addresses.") - c.flags.StringVar(&c.meshGatewaySvcName, "service", "mesh-gateway", + c.flags.StringVar(&c.gatewaySvcName, "service", "", "Service name to use for the registration") c.flags.BoolVar(&c.exposeServers, "expose-servers", false, @@ -195,8 +209,17 @@ func (c *cmd) Run(args []string) int { } c.client = client + // Fixup for deprecated mesh-gateway flag + if c.meshGateway && c.gateway != "" { + c.UI.Error("The mesh-gateway flag is deprecated and cannot be used alongside the gateway flag") + return 1 + } + if c.meshGateway { + c.gateway = meshGatewayVal + } + if c.exposeServers { - if !c.meshGateway { + if c.gateway != meshGatewayVal { c.UI.Error("'-expose-servers' can only be used for mesh gateways") return 1 } @@ -207,11 +230,22 @@ func (c *cmd) Run(args []string) int { } if c.register { - if !c.meshGateway { - c.UI.Error("Auto-Registration can only be used for mesh gateways") + if c.gateway == "" { + c.UI.Error("Auto-Registration can only be used for gateways") return 1 } + kind, ok := supportedGateways[c.gateway] + if !ok { + c.UI.Error("Gateway must be one of: terminating or mesh") + return 1 + } + c.gatewayKind = kind + + if c.gatewaySvcName == "" { + c.gatewaySvcName = string(c.gatewayKind) + } + taggedAddrs := make(map[string]api.ServiceAddress) lanAddr := c.lanAddress.Value() if lanAddr.Address != "" { @@ -231,13 +265,12 @@ func (c *cmd) Run(args []string) int { } var proxyConf *api.AgentServiceConnectProxyConfig - if len(c.bindAddresses.value) > 0 { // override all default binding rules and just bind to the user-supplied addresses proxyConf = &api.AgentServiceConnectProxyConfig{ Config: map[string]interface{}{ - "envoy_mesh_gateway_no_default_bind": true, - "envoy_mesh_gateway_bind_addresses": c.bindAddresses.value, + "envoy_gateway_no_default_bind": true, + "envoy_gateway_bind_addresses": c.bindAddresses.value, }, } } else if canBind(lanAddr) && canBind(wanAddr) { @@ -245,8 +278,8 @@ func (c *cmd) Run(args []string) int { // for creating the envoy listeners proxyConf = &api.AgentServiceConnectProxyConfig{ Config: map[string]interface{}{ - "envoy_mesh_gateway_no_default_bind": true, - "envoy_mesh_gateway_bind_tagged_addresses": true, + "envoy_gateway_no_default_bind": true, + "envoy_gateway_bind_tagged_addresses": true, }, } } else if !canBind(lanAddr) && lanAddr.Address != "" { @@ -260,15 +293,15 @@ func (c *cmd) Run(args []string) int { } svc := api.AgentServiceRegistration{ - Kind: api.ServiceKindMeshGateway, - Name: c.meshGatewaySvcName, + Kind: c.gatewayKind, + Name: c.gatewaySvcName, Address: lanAddr.Address, Port: lanAddr.Port, Meta: meta, TaggedAddresses: taggedAddrs, Proxy: proxyConf, Check: &api.AgentServiceCheck{ - Name: "Mesh Gateway Listening", + Name: fmt.Sprintf("%s listening", c.gatewayKind), TCP: ipaddr.FormatAddressPort(tcpCheckAddr, lanAddr.Port), Interval: "10s", DeregisterCriticalServiceAfter: c.deregAfterCritical, @@ -291,18 +324,19 @@ func (c *cmd) Run(args []string) int { return 1 } c.proxyID = proxyID - } else if c.proxyID == "" && c.meshGateway { - gatewaySvc, err := proxyCmd.LookupGatewayProxy(c.client) + } else if c.proxyID == "" && c.gateway != "" { + gatewaySvc, err := proxyCmd.LookupGatewayProxy(c.client, c.gatewayKind) if err != nil { c.UI.Error(err.Error()) return 1 } c.proxyID = gatewaySvc.ID - c.meshGatewaySvcName = gatewaySvc.Service + c.gatewaySvcName = gatewaySvc.Service } if c.proxyID == "" { - c.UI.Error("No proxy ID specified. One of -proxy-id or -sidecar-for/-mesh-gateway is required") + c.UI.Error("No proxy ID specified. One of -proxy-id or -sidecar-for/-gateway is " + + "required") return 1 } @@ -443,8 +477,8 @@ func (c *cmd) templateArgs() (*BootstrapTplArgs, error) { cluster := c.proxyID if c.sidecarFor != "" { cluster = c.sidecarFor - } else if c.meshGateway && c.meshGatewaySvcName != "" { - cluster = c.meshGatewaySvcName + } else if c.gateway != "" && c.gatewaySvcName != "" { + cluster = c.gatewaySvcName } adminAccessLogPath := c.adminAccessLogPath diff --git a/command/connect/envoy/envoy_test.go b/command/connect/envoy/envoy_test.go index 8b2d1996a..d5f0e5d15 100644 --- a/command/connect/envoy/envoy_test.go +++ b/command/connect/envoy/envoy_test.go @@ -3,6 +3,12 @@ package envoy import ( "encoding/json" "flag" + "github.com/hashicorp/consul/agent" + "github.com/hashicorp/consul/agent/xds" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" "io/ioutil" "net" "net/http" @@ -11,13 +17,6 @@ import ( "path/filepath" "strings" "testing" - - "github.com/hashicorp/consul/agent" - "github.com/hashicorp/consul/agent/xds" - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/sdk/testutil" - "github.com/mitchellh/cli" - "github.com/stretchr/testify/require" ) var update = flag.Bool("update", false, "update golden files") @@ -29,6 +28,50 @@ func TestEnvoyCommand_noTabs(t *testing.T) { } } +func TestEnvoyGateway_Validation(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + args []string + output string + }{ + { + "-register for non-gateway", + []string{"-register"}, + "Auto-Registration can only be used for gateways", + }, + { + "-mesh-gateway and -gateway cannot be combined", + []string{"-register", "-mesh-gateway", "-gateway", "mesh"}, + "The mesh-gateway flag is deprecated and cannot be used alongside the gateway flag", + }, + { + "no proxy registration specified nor discovered", + []string{""}, + "No proxy ID specified", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ui := cli.NewMockUi() + c := New(ui) + c.init() + + code := c.Run(tc.args) + if code == 0 { + t.Errorf("%s: expected non-zero exit", tc.name) + } + + output := ui.ErrorWriter.String() + if !strings.Contains(output, tc.output) { + t.Errorf("expected %q to contain %q", output, tc.output) + } + }) + } +} + // testSetAndResetEnv sets the env vars passed as KEY=value strings in the // current ENV and returns a func() that will undo it's work at the end of the // test for use with defer. diff --git a/command/connect/envoy/flags.go b/command/connect/envoy/flags.go index 5f6e2f465..e8bd9534a 100644 --- a/command/connect/envoy/flags.go +++ b/command/connect/envoy/flags.go @@ -11,7 +11,7 @@ import ( "github.com/hashicorp/go-sockaddr/template" ) -const defaultMeshGatewayPort int = 443 +const defaultGatewayPort int = 443 // ServiceAddressValue implements a flag.Value that may be used to parse an // addr:port string into an api.ServiceAddress. @@ -21,14 +21,14 @@ type ServiceAddressValue struct { func (s *ServiceAddressValue) String() string { if s == nil { - return fmt.Sprintf(":%d", defaultMeshGatewayPort) + return fmt.Sprintf(":%d", defaultGatewayPort) } return fmt.Sprintf("%v:%d", s.value.Address, s.value.Port) } func (s *ServiceAddressValue) Value() api.ServiceAddress { if s == nil || s.value.Port == 0 && s.value.Address == "" { - return api.ServiceAddress{Port: defaultMeshGatewayPort} + return api.ServiceAddress{Port: defaultGatewayPort} } return s.value } @@ -51,7 +51,7 @@ func parseAddress(raw string) (api.ServiceAddress, error) { return result, fmt.Errorf("Error parsing address %q: %v", x, err) } - port := defaultMeshGatewayPort + port := defaultGatewayPort if portStr != "" { port, err = strconv.Atoi(portStr) if err != nil { diff --git a/command/connect/envoy/flags_test.go b/command/connect/envoy/flags_test.go index 77dcf1c6c..0d5e83d5d 100644 --- a/command/connect/envoy/flags_test.go +++ b/command/connect/envoy/flags_test.go @@ -10,12 +10,12 @@ import ( func TestServiceAddressValue_Value(t *testing.T) { t.Run("nil receiver", func(t *testing.T) { var addr *ServiceAddressValue - require.Equal(t, addr.Value(), api.ServiceAddress{Port: defaultMeshGatewayPort}) + require.Equal(t, addr.Value(), api.ServiceAddress{Port: defaultGatewayPort}) }) t.Run("default value", func(t *testing.T) { addr := &ServiceAddressValue{} - require.Equal(t, addr.Value(), api.ServiceAddress{Port: defaultMeshGatewayPort}) + require.Equal(t, addr.Value(), api.ServiceAddress{Port: defaultGatewayPort}) }) t.Run("set value", func(t *testing.T) { @@ -40,7 +40,7 @@ func TestServiceAddressValue_Set(t *testing.T) { input: "8.8.8.8:", expectedValue: api.ServiceAddress{ Address: "8.8.8.8", - Port: defaultMeshGatewayPort, + Port: defaultGatewayPort, }, }, { diff --git a/command/connect/proxy/proxy.go b/command/connect/proxy/proxy.go index e367b6571..013f9d38f 100644 --- a/command/connect/proxy/proxy.go +++ b/command/connect/proxy/proxy.go @@ -248,13 +248,13 @@ func LookupProxyIDForSidecar(client *api.Client, sidecarFor string) (string, err return proxyIDs[0], nil } -// LookupGatewayProxyID finds the mesh-gateway service registered with the local +// LookupGatewayProxyID finds the gateway service registered with the local // agent if any and returns its service ID. It will return an ID if and only if -// there is exactly one registered mesh-gateway registered to the agent. -func LookupGatewayProxy(client *api.Client) (*api.AgentService, error) { - svcs, err := client.Agent().ServicesWithFilter("Kind == `mesh-gateway`") +// there is exactly one gateway of this kind registered to the agent. +func LookupGatewayProxy(client *api.Client, kind api.ServiceKind) (*api.AgentService, error) { + svcs, err := client.Agent().ServicesWithFilter(fmt.Sprintf("Kind == `%s`", kind)) if err != nil { - return nil, fmt.Errorf("Failed looking up mesh-gateway instances: %v", err) + return nil, fmt.Errorf("Failed looking up %s instances: %v", kind, err) } var proxyIDs []string @@ -264,14 +264,14 @@ func LookupGatewayProxy(client *api.Client) (*api.AgentService, error) { switch len(svcs) { case 0: - return nil, fmt.Errorf("No mesh-gateway services registered with this agent") + return nil, fmt.Errorf("No %s services registered with this agent", kind) case 1: for _, svc := range svcs { return svc, nil } return nil, fmt.Errorf("This should be unreachable") default: - return nil, fmt.Errorf("Cannot lookup the mesh-gateway's proxy ID because multiple are registered with the agent") + return nil, fmt.Errorf("Cannot lookup the %s's proxy ID because multiple are registered with the agent", kind) } } diff --git a/website/source/docs/connect/proxies/envoy.md b/website/source/docs/connect/proxies/envoy.md index dce51f706..208495da8 100644 --- a/website/source/docs/connect/proxies/envoy.md +++ b/website/source/docs/connect/proxies/envoy.md @@ -279,7 +279,7 @@ definition](/docs/connect/registration/service-registration.html) or since HTTP/2 has many requests per connection. For this configuration to be respected, a L7 protocol must be defined in the `protocol` field. -### Mesh Gateway Options +### Gateway Options These fields may also be overridden explicitly in the [proxy service definition](/docs/connect/registration/service-registration.html), or defined in @@ -287,25 +287,29 @@ the [global `proxy-defaults` configuration entry](/docs/agent/config_entries.html#proxy-defaults-proxy-defaults) to act as defaults that are inherited by all services. +Prior to 1.8.0 these settings were specific to Mesh Gateways. The deprecated +names such as `envoy_mesh_gateway_bind_addresses` and `envoy_mesh_gateway_no_default_bind` +will continue to be supported. + - `connect_timeout_ms` - The number of milliseconds to allow when making upstream connections before timing out. Defaults to 5000 (5 seconds). If the upstream service has the configuration option [`connect_timeout_ms`](/docs/agent/config-entries/service-resolver.html#connecttimeout) set for the `service-resolver`, that timeout value will take precedence over - this mesh gateway option. + this gateway option. -- `envoy_mesh_gateway_bind_tagged_addresses` - Indicates that the mesh gateway +- `envoy_gateway_bind_tagged_addresses` - Indicates that the gateway services tagged addresses should be bound to listeners in addition to the - default listener address. + default listener address. -- `envoy_mesh_gateway_bind_addresses` - A map of additional addresses to be bound. +- `envoy_gateway_bind_addresses` - A map of additional addresses to be bound. This map's keys are the name of the listeners to be created and the values are a map with two keys, address and port, that combined make the address to bind the listener to. These are bound in addition to the default address. -- `envoy_mesh_gateway_no_default_bind` - Prevents binding to the default address - of the mesh gateway service. This should be used with one of the other options - to configure the gateways bind addresses. +- `envoy_gateway_no_default_bind` - Prevents binding to the default address + of the gateway service. This should be used with one of the other options + to configure the gateway's bind addresses. ## Advanced Configuration