From 9f0a6c50b8e7c5abb3c96865f1208edd3d5aa9a6 Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesh Date: Thu, 19 Jan 2023 16:54:11 -0500 Subject: [PATCH] Add support for envoy readiness flags (#16015) * Add support for envoy readiness flags - add flags 'envoy-ready-bind-port` and `envoy-ready-bind-addr` on consul connect envoy to create a ready listener on that address. --- .changelog/16015.txt | 3 + command/connect/envoy/envoy.go | 12 + command/connect/envoy/envoy_test.go | 22 ++ .../testdata/envoy-readiness-probe.golden | 299 ++++++++++++++++++ website/content/commands/connect/envoy.mdx | 14 +- 5 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 .changelog/16015.txt create mode 100644 command/connect/envoy/testdata/envoy-readiness-probe.golden diff --git a/.changelog/16015.txt b/.changelog/16015.txt new file mode 100644 index 000000000..6d9e05294 --- /dev/null +++ b/.changelog/16015.txt @@ -0,0 +1,3 @@ +```release-note:feature +connect: add flags `envoy-ready-bind-port` and `envoy-ready-bind-address` to the `consul connect envoy` command that allows configuration of readiness probe on proxy for any service kind. +``` \ No newline at end of file diff --git a/command/connect/envoy/envoy.go b/command/connect/envoy/envoy.go index d009cc6c4..ad0988db5 100644 --- a/command/connect/envoy/envoy.go +++ b/command/connect/envoy/envoy.go @@ -75,6 +75,9 @@ type cmd struct { exposeServers bool omitDeprecatedTags bool + envoyReadyBindAddress string + envoyReadyBindPort int + gatewaySvcName string gatewayKind api.ServiceKind @@ -161,6 +164,11 @@ func (c *cmd) init() { c.flags.Var(&c.lanAddress, "address", "LAN address to advertise in the gateway service registration") + c.flags.StringVar(&c.envoyReadyBindAddress, "envoy-ready-bind-address", "", + "The address on which Envoy's readiness probe is available.") + c.flags.IntVar(&c.envoyReadyBindPort, "envoy-ready-bind-port", 0, + "The port on which Envoy's readiness probe is available.") + c.flags.Var(&c.wanAddress, "wan-address", "WAN address to advertise in the gateway service registration. For ingress gateways, "+ "only an IP address (without a port) is required.") @@ -704,6 +712,10 @@ func (c *cmd) generateConfig() ([]byte, error) { bsCfg.ReadyBindAddr = lanAddr } + if c.envoyReadyBindAddress != "" && c.envoyReadyBindPort != 0 { + bsCfg.ReadyBindAddr = fmt.Sprintf("%s:%d", c.envoyReadyBindAddress, c.envoyReadyBindPort) + } + if !c.disableCentralConfig { // Parse the bootstrap config if err := mapstructure.WeakDecode(svcProxyConfig.Config, &bsCfg); err != nil { diff --git a/command/connect/envoy/envoy_test.go b/command/connect/envoy/envoy_test.go index 8451b2eb6..bc2ba0fbb 100644 --- a/command/connect/envoy/envoy_test.go +++ b/command/connect/envoy/envoy_test.go @@ -996,6 +996,28 @@ func TestGenerateConfig(t *testing.T) { PrometheusScrapePath: "/metrics", }, }, + { + Name: "envoy-readiness-probe", + Flags: []string{"-proxy-id", "test-proxy", + "-envoy-ready-bind-address", "127.0.0.1", "-envoy-ready-bind-port", "21000"}, + WantArgs: BootstrapTplArgs{ + ProxyCluster: "test-proxy", + ProxyID: "test-proxy", + // We don't know this til after the lookup so it will be empty in the + // initial args call we are testing here. + ProxySourceService: "", + GRPC: GRPC{ + AgentAddress: "127.0.0.1", + AgentPort: "8502", // Note this is the gRPC port + }, + AdminAccessLogPath: "/dev/null", + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusBackendPort: "", + PrometheusScrapePath: "/metrics", + }, + }, { Name: "ingress-gateway-address-specified", Flags: []string{"-proxy-id", "ingress-gateway", "-gateway", "ingress", "-address", "1.2.3.4:7777"}, diff --git a/command/connect/envoy/testdata/envoy-readiness-probe.golden b/command/connect/envoy/testdata/envoy-readiness-probe.golden new file mode 100644 index 000000000..444528e88 --- /dev/null +++ b/command/connect/envoy/testdata/envoy-readiness-probe.golden @@ -0,0 +1,299 @@ +{ + "admin": { + "access_log_path": "/dev/null", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 19000 + } + } + }, + "node": { + "cluster": "test", + "id": "test-proxy", + "metadata": { + "namespace": "default", + "partition": "default" + } + }, + "layered_runtime": { + "layers": [ + { + "name": "base", + "static_layer": { + "re2.max_program_size.error_level": 1048576 + } + } + ] + }, + "static_resources": { + "clusters": [ + { + "name": "local_agent", + "ignore_health_on_host_removal": false, + "connect_timeout": "1s", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "local_agent", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 8502 + } + } + } + } + ] + } + ] + } + }, + { + "name": "self_admin", + "ignore_health_on_host_removal": false, + "connect_timeout": "5s", + "type": "STATIC", + "http_protocol_options": {}, + "loadAssignment": { + "clusterName": "self_admin", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 19000 + } + } + } + } + ] + } + ] + } + } + ], + "listeners": [ + { + "name": "envoy_ready_listener", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 21000 + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy_ready", + "codec_type": "HTTP1", + "route_config": { + "name": "self_admin_route", + "virtual_hosts": [ + { + "name": "self_admin", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "path": "/ready" + }, + "route": { + "cluster": "self_admin", + "prefix_rewrite": "/ready" + } + }, + { + "match": { + "prefix": "/" + }, + "direct_response": { + "status": 404 + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ] + } + } + ] + } + ] + } + ] + }, + "stats_config": { + "stats_tags": [ + { + "regex": "^cluster\\.(?:passthrough~)?((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.custom_hash" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service_subset" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.namespace" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:([^.]+)\\.)?[^.]+\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.partition" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.datacenter" + }, + { + "regex": "^cluster\\.([^.]+\\.(?:[^.]+\\.)?([^.]+)\\.external\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.peer" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.routing_type" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)", + "tag_name": "consul.destination.trust_domain" + }, + { + "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.target" + }, + { + "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)", + "tag_name": "consul.destination.full_target" + }, + { + "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.(([^.]+)(?:\\.[^.]+)?(?:\\.[^.]+)?\\.[^.]+\\.)", + "tag_name": "consul.upstream.service" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.[^.]+)?\\.([^.]+)\\.)", + "tag_name": "consul.upstream.datacenter" + }, + { + "regex": "^(?:tcp|http)\\.upstream_peered\\.([^.]+(?:\\.[^.]+)?\\.([^.]+)\\.)", + "tag_name": "consul.upstream.peer" + }, + { + "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.([^.]+(?:\\.([^.]+))?(?:\\.[^.]+)?\\.[^.]+\\.)", + "tag_name": "consul.upstream.namespace" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.([^.]+))?\\.[^.]+\\.)", + "tag_name": "consul.upstream.partition" + }, + { + "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.custom_hash" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.service_subset" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.service" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.namespace" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.datacenter" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)", + "tag_name": "consul.routing_type" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)", + "tag_name": "consul.trust_domain" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.target" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)", + "tag_name": "consul.full_target" + }, + { + "tag_name": "local_cluster", + "fixed_value": "test" + }, + { + "tag_name": "consul.source.service", + "fixed_value": "test" + }, + { + "tag_name": "consul.source.namespace", + "fixed_value": "default" + }, + { + "tag_name": "consul.source.partition", + "fixed_value": "default" + }, + { + "tag_name": "consul.source.datacenter", + "fixed_value": "dc1" + } + ], + "use_all_default_tags": true + }, + "dynamic_resources": { + "lds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "cds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "ads_config": { + "api_type": "DELTA_GRPC", + "transport_api_version": "V3", + "grpc_services": { + "initial_metadata": [ + { + "key": "x-consul-token", + "value": "" + } + ], + "envoy_grpc": { + "cluster_name": "local_agent" + } + } + } + } +} + diff --git a/website/content/commands/connect/envoy.mdx b/website/content/commands/connect/envoy.mdx index a1beb4f43..66fcd8990 100644 --- a/website/content/commands/connect/envoy.mdx +++ b/website/content/commands/connect/envoy.mdx @@ -75,6 +75,18 @@ Usage: `consul connect envoy [options] [-- pass-through options]` In cases where either assumption is violated this flag will prevent the command attempting to resolve config from the local agent. +- `-envoy-ready-bind-address` - By default the proxy does not have a readiness probe + configured on it. This flag in conjunction with the `envoy-ready-bind-port` flag + configures where the envoy readiness probe is configured on the proxy. A `/ready` HTTP + endpoint will be instantiated at the specified address and port. When configured, Consul + uses `/ready` HTTP endpoints to check proxy health. + +- `-envoy-ready-bind-port` - By default the proxy does not have a readiness probe + configured on it. This flag in conjunction with the `envoy-ready-bind-address` flag + configures where the envoy readiness probe is configured on the proxy. A `/ready` HTTP + endpoint will be instantiated at the specified address and port. When configured, Consul + uses `/ready` HTTP endpoints to check proxy health. + - `-prometheus-backend-port` - Sets the backend port for the "prometheus_backend" cluster that `envoy_prometheus_bind_addr` will point to. Without this flag, `envoy_prometheus_bind_addr` would point to the "self_admin" cluster where Envoy metrics @@ -161,7 +173,7 @@ compatibility with Envoy and prevent potential issues. Default is `false`. different port than the port specified in `-address` so that they do not conflict with the health check endpoint. -- `-admin-access-log-path` - +- `-admin-access-log-path` - **Deprecated in Consul 1.15.0 in favor of [`proxy-defaults` access logs](/docs/connect/config-entries/proxy-defaults#accesslogs).** The path to write the access log for the administration server. If no access log is desired specify `/dev/null`. By default it will