Add support for envoy readiness flags (#16015)

* Add support for envoy readiness flags
- add flags 'envoy-ready-bind-port` and `envoy-ready-bind-addr` on consul connect envoy to create a ready listener on that address.
This commit is contained in:
Ashwin Venkatesh 2023-01-19 16:54:11 -05:00 committed by GitHub
parent ec3ebec505
commit 9f0a6c50b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 349 additions and 1 deletions

3
.changelog/16015.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:feature
connect: add flags `envoy-ready-bind-port` and `envoy-ready-bind-address` to the `consul connect envoy` command that allows configuration of readiness probe on proxy for any service kind.
```

View File

@ -75,6 +75,9 @@ type cmd struct {
exposeServers bool exposeServers bool
omitDeprecatedTags bool omitDeprecatedTags bool
envoyReadyBindAddress string
envoyReadyBindPort int
gatewaySvcName string gatewaySvcName string
gatewayKind api.ServiceKind gatewayKind api.ServiceKind
@ -161,6 +164,11 @@ func (c *cmd) init() {
c.flags.Var(&c.lanAddress, "address", c.flags.Var(&c.lanAddress, "address",
"LAN address to advertise in the gateway service registration") "LAN address to advertise in the gateway service registration")
c.flags.StringVar(&c.envoyReadyBindAddress, "envoy-ready-bind-address", "",
"The address on which Envoy's readiness probe is available.")
c.flags.IntVar(&c.envoyReadyBindPort, "envoy-ready-bind-port", 0,
"The port on which Envoy's readiness probe is available.")
c.flags.Var(&c.wanAddress, "wan-address", c.flags.Var(&c.wanAddress, "wan-address",
"WAN address to advertise in the gateway service registration. For ingress gateways, "+ "WAN address to advertise in the gateway service registration. For ingress gateways, "+
"only an IP address (without a port) is required.") "only an IP address (without a port) is required.")
@ -704,6 +712,10 @@ func (c *cmd) generateConfig() ([]byte, error) {
bsCfg.ReadyBindAddr = lanAddr bsCfg.ReadyBindAddr = lanAddr
} }
if c.envoyReadyBindAddress != "" && c.envoyReadyBindPort != 0 {
bsCfg.ReadyBindAddr = fmt.Sprintf("%s:%d", c.envoyReadyBindAddress, c.envoyReadyBindPort)
}
if !c.disableCentralConfig { if !c.disableCentralConfig {
// Parse the bootstrap config // Parse the bootstrap config
if err := mapstructure.WeakDecode(svcProxyConfig.Config, &bsCfg); err != nil { if err := mapstructure.WeakDecode(svcProxyConfig.Config, &bsCfg); err != nil {

View File

@ -996,6 +996,28 @@ func TestGenerateConfig(t *testing.T) {
PrometheusScrapePath: "/metrics", PrometheusScrapePath: "/metrics",
}, },
}, },
{
Name: "envoy-readiness-probe",
Flags: []string{"-proxy-id", "test-proxy",
"-envoy-ready-bind-address", "127.0.0.1", "-envoy-ready-bind-port", "21000"},
WantArgs: BootstrapTplArgs{
ProxyCluster: "test-proxy",
ProxyID: "test-proxy",
// We don't know this til after the lookup so it will be empty in the
// initial args call we are testing here.
ProxySourceService: "",
GRPC: GRPC{
AgentAddress: "127.0.0.1",
AgentPort: "8502", // Note this is the gRPC port
},
AdminAccessLogPath: "/dev/null",
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
LocalAgentClusterName: xds.LocalAgentClusterName,
PrometheusBackendPort: "",
PrometheusScrapePath: "/metrics",
},
},
{ {
Name: "ingress-gateway-address-specified", Name: "ingress-gateway-address-specified",
Flags: []string{"-proxy-id", "ingress-gateway", "-gateway", "ingress", "-address", "1.2.3.4:7777"}, Flags: []string{"-proxy-id", "ingress-gateway", "-gateway", "ingress", "-address", "1.2.3.4:7777"},

View File

@ -0,0 +1,299 @@
{
"admin": {
"access_log_path": "/dev/null",
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
}
},
"node": {
"cluster": "test",
"id": "test-proxy",
"metadata": {
"namespace": "default",
"partition": "default"
}
},
"layered_runtime": {
"layers": [
{
"name": "base",
"static_layer": {
"re2.max_program_size.error_level": 1048576
}
}
]
},
"static_resources": {
"clusters": [
{
"name": "local_agent",
"ignore_health_on_host_removal": false,
"connect_timeout": "1s",
"type": "STATIC",
"http2_protocol_options": {},
"loadAssignment": {
"clusterName": "local_agent",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 8502
}
}
}
}
]
}
]
}
},
{
"name": "self_admin",
"ignore_health_on_host_removal": false,
"connect_timeout": "5s",
"type": "STATIC",
"http_protocol_options": {},
"loadAssignment": {
"clusterName": "self_admin",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
}
}
}
]
}
]
}
}
],
"listeners": [
{
"name": "envoy_ready_listener",
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 21000
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "envoy_ready",
"codec_type": "HTTP1",
"route_config": {
"name": "self_admin_route",
"virtual_hosts": [
{
"name": "self_admin",
"domains": [
"*"
],
"routes": [
{
"match": {
"path": "/ready"
},
"route": {
"cluster": "self_admin",
"prefix_rewrite": "/ready"
}
},
{
"match": {
"prefix": "/"
},
"direct_response": {
"status": 404
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.filters.http.router",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
]
}
}
]
}
]
}
]
},
"stats_config": {
"stats_tags": [
{
"regex": "^cluster\\.(?:passthrough~)?((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.destination.custom_hash"
},
{
"regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.destination.service_subset"
},
{
"regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.destination.service"
},
{
"regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.destination.namespace"
},
{
"regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:([^.]+)\\.)?[^.]+\\.internal[^.]*\\.[^.]+\\.consul\\.)",
"tag_name": "consul.destination.partition"
},
{
"regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)",
"tag_name": "consul.destination.datacenter"
},
{
"regex": "^cluster\\.([^.]+\\.(?:[^.]+\\.)?([^.]+)\\.external\\.[^.]+\\.consul\\.)",
"tag_name": "consul.destination.peer"
},
{
"regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
"tag_name": "consul.destination.routing_type"
},
{
"regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
"tag_name": "consul.destination.trust_domain"
},
{
"regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.destination.target"
},
{
"regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
"tag_name": "consul.destination.full_target"
},
{
"regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.(([^.]+)(?:\\.[^.]+)?(?:\\.[^.]+)?\\.[^.]+\\.)",
"tag_name": "consul.upstream.service"
},
{
"regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.[^.]+)?\\.([^.]+)\\.)",
"tag_name": "consul.upstream.datacenter"
},
{
"regex": "^(?:tcp|http)\\.upstream_peered\\.([^.]+(?:\\.[^.]+)?\\.([^.]+)\\.)",
"tag_name": "consul.upstream.peer"
},
{
"regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.([^.]+(?:\\.([^.]+))?(?:\\.[^.]+)?\\.[^.]+\\.)",
"tag_name": "consul.upstream.namespace"
},
{
"regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.([^.]+))?\\.[^.]+\\.)",
"tag_name": "consul.upstream.partition"
},
{
"regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.custom_hash"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.service_subset"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.service"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.namespace"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)",
"tag_name": "consul.datacenter"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
"tag_name": "consul.routing_type"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
"tag_name": "consul.trust_domain"
},
{
"regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.target"
},
{
"regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
"tag_name": "consul.full_target"
},
{
"tag_name": "local_cluster",
"fixed_value": "test"
},
{
"tag_name": "consul.source.service",
"fixed_value": "test"
},
{
"tag_name": "consul.source.namespace",
"fixed_value": "default"
},
{
"tag_name": "consul.source.partition",
"fixed_value": "default"
},
{
"tag_name": "consul.source.datacenter",
"fixed_value": "dc1"
}
],
"use_all_default_tags": true
},
"dynamic_resources": {
"lds_config": {
"ads": {},
"resource_api_version": "V3"
},
"cds_config": {
"ads": {},
"resource_api_version": "V3"
},
"ads_config": {
"api_type": "DELTA_GRPC",
"transport_api_version": "V3",
"grpc_services": {
"initial_metadata": [
{
"key": "x-consul-token",
"value": ""
}
],
"envoy_grpc": {
"cluster_name": "local_agent"
}
}
}
}
}

View File

@ -75,6 +75,18 @@ Usage: `consul connect envoy [options] [-- pass-through options]`
In cases where either assumption is violated this flag will prevent the In cases where either assumption is violated this flag will prevent the
command attempting to resolve config from the local agent. command attempting to resolve config from the local agent.
- `-envoy-ready-bind-address` - By default the proxy does not have a readiness probe
configured on it. This flag in conjunction with the `envoy-ready-bind-port` flag
configures where the envoy readiness probe is configured on the proxy. A `/ready` HTTP
endpoint will be instantiated at the specified address and port. When configured, Consul
uses `/ready` HTTP endpoints to check proxy health.
- `-envoy-ready-bind-port` - By default the proxy does not have a readiness probe
configured on it. This flag in conjunction with the `envoy-ready-bind-address` flag
configures where the envoy readiness probe is configured on the proxy. A `/ready` HTTP
endpoint will be instantiated at the specified address and port. When configured, Consul
uses `/ready` HTTP endpoints to check proxy health.
- `-prometheus-backend-port` - Sets the backend port for the "prometheus_backend" - `-prometheus-backend-port` - Sets the backend port for the "prometheus_backend"
cluster that `envoy_prometheus_bind_addr` will point to. Without this flag, cluster that `envoy_prometheus_bind_addr` will point to. Without this flag,
`envoy_prometheus_bind_addr` would point to the "self_admin" cluster where Envoy metrics `envoy_prometheus_bind_addr` would point to the "self_admin" cluster where Envoy metrics
@ -161,7 +173,7 @@ compatibility with Envoy and prevent potential issues. Default is `false`.
different port than the port specified in `-address` so that they do not different port than the port specified in `-address` so that they do not
conflict with the health check endpoint. conflict with the health check endpoint.
- `-admin-access-log-path` - - `-admin-access-log-path` -
**Deprecated in Consul 1.15.0 in favor of [`proxy-defaults` access logs](/docs/connect/config-entries/proxy-defaults#accesslogs).** **Deprecated in Consul 1.15.0 in favor of [`proxy-defaults` access logs](/docs/connect/config-entries/proxy-defaults#accesslogs).**
The path to write the access log for the administration The path to write the access log for the administration
server. If no access log is desired specify `/dev/null`. By default it will server. If no access log is desired specify `/dev/null`. By default it will