From 9ff49034e73af619ddd7cc2279d523473b820356 Mon Sep 17 00:00:00 2001 From: Nitya Dhanushkodi Date: Thu, 4 Mar 2021 14:15:47 -0800 Subject: [PATCH] Add flags to consul connect envoy for metrics merging. (#9768) Allows setting -prometheus-backend-port to configure the cluster envoy_prometheus_bind_addr points to. Allows setting -prometheus-scrape-path to configure which path envoy_prometheus_bind_addr exposes metrics on. -prometheus-backend-port is used by the consul-k8s metrics merging feature, to configure envoy_prometheus_bind_addr to point to the merged metrics endpoint that combines Envoy and service metrics so that one set of annotations on a Pod can scrape metrics from the service and it's Envoy sidecar. -prometheus-scrape-path is used to allow configurability of the path where prometheus metrics are exposed on envoy_prometheus_bind_addr. --- .changelog/9768.txt | 3 + command/connect/envoy/bootstrap_config.go | 32 +- .../connect/envoy/bootstrap_config_test.go | 310 ++++++++++++++---- command/connect/envoy/bootstrap_tpl.go | 8 + command/connect/envoy/envoy.go | 39 ++- command/connect/envoy/envoy_test.go | 53 +++ .../envoy/testdata/prometheus-metrics.golden | 265 +++++++++++++++ website/content/commands/connect/envoy.mdx | 13 + 8 files changed, 636 insertions(+), 87 deletions(-) create mode 100644 .changelog/9768.txt create mode 100644 command/connect/envoy/testdata/prometheus-metrics.golden diff --git a/.changelog/9768.txt b/.changelog/9768.txt new file mode 100644 index 000000000..925f11f9a --- /dev/null +++ b/.changelog/9768.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: adds new flags `prometheus-backend-port` and `prometheus-scrape-port` to `consul connect envoy` to support envoy_prometheus_bind_addr pointing to the merged metrics port when using Consul Connect on K8s. +``` diff --git a/command/connect/envoy/bootstrap_config.go b/command/connect/envoy/bootstrap_config.go index 7a4df51a3..af383c01d 100644 --- a/command/connect/envoy/bootstrap_config.go +++ b/command/connect/envoy/bootstrap_config.go @@ -213,19 +213,19 @@ func (c *BootstrapConfig) ConfigureArgs(args *BootstrapTplArgs, omitDeprecatedTa } // Setup prometheus if needed. This MUST happen after the Static*JSON is set above if c.PrometheusBindAddr != "" { - if err := c.generateListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", "/metrics", "/stats/prometheus"); err != nil { + if err := c.generateListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", args.PrometheusScrapePath, "/stats/prometheus", args.PrometheusBackendPort); err != nil { return err } } // Setup /stats proxy listener if needed. This MUST happen after the Static*JSON is set above if c.StatsBindAddr != "" { - if err := c.generateListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats"); err != nil { + if err := c.generateListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats", ""); err != nil { return err } } // Setup /ready proxy listener if needed. This MUST happen after the Static*JSON is set above if c.ReadyBindAddr != "" { - if err := c.generateListenerConfig(args, c.ReadyBindAddr, "envoy_ready", "path", "/ready", "/ready"); err != nil { + if err := c.generateListenerConfig(args, c.ReadyBindAddr, "envoy_ready", "path", "/ready", "/ready", ""); err != nil { return err } } @@ -549,20 +549,36 @@ func generateStatsTags(args *BootstrapTplArgs, initialTags []string, omitDepreca return tagJSONs, nil } -func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite string) error { +func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite, prometheusBackendPort string) error { host, port, err := net.SplitHostPort(bindAddr) if err != nil { return fmt.Errorf("invalid %s bind address: %s", name, err) } + // If prometheusBackendPort is set (not empty string), create + // "prometheus_backend" cluster with the prometheusBackendPort that the + // listener will point to, rather than the "self_admin" cluster. This is for + // the merged metrics feature in consul-k8s, so the + // envoy_prometheus_bind_addr listener will point to the merged Envoy and + // service metrics endpoint rather than the Envoy admin endpoint for + // metrics. This cluster will only be created once since it's only created + // when prometheusBackendPort is set, and prometheusBackendPort is only set + // when calling this function if c.PrometheusBindAddr is set. + clusterPort := args.AdminBindPort + clusterName := selfAdminName + if prometheusBackendPort != "" { + clusterPort = prometheusBackendPort + clusterName = "prometheus_backend" + } + clusterJSON := `{ - "name": "` + selfAdminName + `", + "name": "` + clusterName + `", "ignore_health_on_host_removal": false, "connect_timeout": "5s", "type": "STATIC", "http_protocol_options": {}, "loadAssignment": { - "clusterName": "` + selfAdminName + `", + "clusterName": "` + clusterName + `", "endpoints": [ { "lbEndpoints": [ @@ -571,7 +587,7 @@ func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAdd "address": { "socket_address": { "address": "127.0.0.1", - "port_value": ` + args.AdminBindPort + ` + "port_value": ` + clusterPort + ` } } } @@ -612,7 +628,7 @@ func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAdd "` + matchType + `": "` + matchValue + `" }, "route": { - "cluster": "self_admin", + "cluster": "` + clusterName + `", "prefix_rewrite": "` + prefixRewrite + `" } }, diff --git a/command/connect/envoy/bootstrap_config_test.go b/command/connect/envoy/bootstrap_config_test.go index c296c402a..35a0505b6 100644 --- a/command/connect/envoy/bootstrap_config_test.go +++ b/command/connect/envoy/bootstrap_config_test.go @@ -38,64 +38,206 @@ const ( ] } }` - expectedPromListener = `{ - "name": "envoy_prometheus_metrics_listener", - "address": { - "socket_address": { - "address": "0.0.0.0", - "port_value": 9000 - } - }, - "filter_chains": [ - { - "filters": [ - { - "name": "envoy.filters.network.http_connection_manager", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "stat_prefix": "envoy_prometheus_metrics", - "codec_type": "HTTP1", - "route_config": { - "name": "self_admin_route", - "virtual_hosts": [ - { - "name": "self_admin", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "path": "/metrics" - }, - "route": { - "cluster": "self_admin", - "prefix_rewrite": "/stats/prometheus" - } - }, - { - "match": { - "prefix": "/" - }, - "direct_response": { - "status": 404 - } - } - ] + expectedPrometheusBackendCluster = `{ + "name": "prometheus_backend", + "ignore_health_on_host_removal": false, + "connect_timeout": "5s", + "type": "STATIC", + "http_protocol_options": {}, + "loadAssignment": { + "clusterName": "prometheus_backend", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 20100 } - ] - }, - "http_filters": [ - { - "name": "envoy.filters.http.router" } - ] + } } - } - ] - } - ] + ] + } + ] + } }` + expectedPromListener = `{ + "name": "envoy_prometheus_metrics_listener", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9000 + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy_prometheus_metrics", + "codec_type": "HTTP1", + "route_config": { + "name": "self_admin_route", + "virtual_hosts": [ + { + "name": "self_admin", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "path": "/metrics" + }, + "route": { + "cluster": "self_admin", + "prefix_rewrite": "/stats/prometheus" + } + }, + { + "match": { + "prefix": "/" + }, + "direct_response": { + "status": 404 + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router" + } + ] + } + } + ] + } + ] + }` + expectedPromListenerCustomScrapePath = `{ + "name": "envoy_prometheus_metrics_listener", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9000 + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy_prometheus_metrics", + "codec_type": "HTTP1", + "route_config": { + "name": "self_admin_route", + "virtual_hosts": [ + { + "name": "self_admin", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "path": "/scrape-path" + }, + "route": { + "cluster": "self_admin", + "prefix_rewrite": "/stats/prometheus" + } + }, + { + "match": { + "prefix": "/" + }, + "direct_response": { + "status": 404 + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router" + } + ] + } + } + ] + } + ] + }` + expectedPromListenerWithPrometheusBackendCluster = `{ + "name": "envoy_prometheus_metrics_listener", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9000 + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy_prometheus_metrics", + "codec_type": "HTTP1", + "route_config": { + "name": "self_admin_route", + "virtual_hosts": [ + { + "name": "self_admin", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "path": "/metrics" + }, + "route": { + "cluster": "prometheus_backend", + "prefix_rewrite": "/stats/prometheus" + } + }, + { + "match": { + "prefix": "/" + }, + "direct_response": { + "status": 404 + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router" + } + ] + } + } + ] + } + ] + }` expectedStatsListener = `{ "name": "envoy_metrics_listener", "address": { @@ -457,8 +599,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { PrometheusBindAddr: "0.0.0.0:9000", }, baseArgs: BootstrapTplArgs{ - AdminBindAddress: "127.0.0.1", - AdminBindPort: "19000", + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + PrometheusScrapePath: "/metrics", }, wantArgs: BootstrapTplArgs{ AdminBindAddress: "127.0.0.1", @@ -466,8 +609,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { // Should add a static cluster for the self-proxy to admin StaticClustersJSON: expectedSelfAdminCluster, // Should add a static http listener too - StaticListenersJSON: expectedPromListener, - StatsConfigJSON: defaultStatsConfigJSON, + StaticListenersJSON: expectedPromListener, + StatsConfigJSON: defaultStatsConfigJSON, + PrometheusScrapePath: "/metrics", }, wantErr: false, }, @@ -479,8 +623,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { StaticListenersJSON: `{"baz":"qux"}`, }, baseArgs: BootstrapTplArgs{ - AdminBindAddress: "127.0.0.1", - AdminBindPort: "19000", + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + PrometheusScrapePath: "/scrape-path", }, wantArgs: BootstrapTplArgs{ AdminBindAddress: "127.0.0.1", @@ -488,8 +633,33 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { // Should add a static cluster for the self-proxy to admin StaticClustersJSON: `{"foo":"bar"},` + expectedSelfAdminCluster, // Should add a static http listener too - StaticListenersJSON: `{"baz":"qux"},` + expectedPromListener, - StatsConfigJSON: defaultStatsConfigJSON, + StaticListenersJSON: `{"baz":"qux"},` + expectedPromListenerCustomScrapePath, + StatsConfigJSON: defaultStatsConfigJSON, + PrometheusScrapePath: "/scrape-path", + }, + wantErr: false, + }, + { + name: "prometheus-bind-addr-with-prometheus-backend", + input: BootstrapConfig{ + PrometheusBindAddr: "0.0.0.0:9000", + }, + baseArgs: BootstrapTplArgs{ + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + PrometheusBackendPort: "20100", + PrometheusScrapePath: "/metrics", + }, + wantArgs: BootstrapTplArgs{ + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + // Should use the "prometheus_backend" cluster instead, which + // uses the PrometheusBackendPort rather than Envoy admin port + StaticClustersJSON: expectedPrometheusBackendCluster, + StaticListenersJSON: expectedPromListenerWithPrometheusBackendCluster, + StatsConfigJSON: defaultStatsConfigJSON, + PrometheusBackendPort: "20100", + PrometheusScrapePath: "/metrics", }, wantErr: false, }, @@ -635,8 +805,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { StatsBindAddr: "0.0.0.0:9000", }, baseArgs: BootstrapTplArgs{ - AdminBindAddress: "127.0.0.1", - AdminBindPort: "19000", + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + PrometheusScrapePath: "/metrics", }, wantArgs: BootstrapTplArgs{ AdminBindAddress: "127.0.0.1", @@ -648,7 +819,8 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { []string{expectedPromListener, expectedStatsListener, expectedReadyListener}, ", ", ), - StatsConfigJSON: defaultStatsConfigJSON, + StatsConfigJSON: defaultStatsConfigJSON, + PrometheusScrapePath: "/metrics", }, wantErr: false, }, @@ -660,8 +832,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { StatsBindAddr: "0.0.0.0:9000", }, baseArgs: BootstrapTplArgs{ - AdminBindAddress: "127.0.0.1", - AdminBindPort: "19000", + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + PrometheusScrapePath: "/metrics", }, omitDeprecatedTags: true, wantArgs: BootstrapTplArgs{ @@ -673,7 +846,8 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { ", ", ), // Should not have default stats config JSON when deprecated tags are omitted - StatsConfigJSON: updatedStatsConfigJSON, + StatsConfigJSON: updatedStatsConfigJSON, + PrometheusScrapePath: "/metrics", }, wantErr: false, }, diff --git a/command/connect/envoy/bootstrap_tpl.go b/command/connect/envoy/bootstrap_tpl.go index ce675c854..8fa78efb0 100644 --- a/command/connect/envoy/bootstrap_tpl.go +++ b/command/connect/envoy/bootstrap_tpl.go @@ -95,6 +95,14 @@ type BootstrapTplArgs struct { // EnvoyVersion is the envoy version, which is necessary to generate the // correct configuration. EnvoyVersion string + + // PrometheusBackendPort will configure a "prometheus_backend" cluster which + // envoy_prometheus_bind_addr will point to. + PrometheusBackendPort string + + // PrometheusScrapePath will configure the path where metrics are exposed on + // the envoy_prometheus_bind_addr listener. + PrometheusScrapePath string } // GRPC settings used in the bootstrap template. diff --git a/command/connect/envoy/envoy.go b/command/connect/envoy/envoy.go index 5bd617f30..a53867e7f 100644 --- a/command/connect/envoy/envoy.go +++ b/command/connect/envoy/envoy.go @@ -52,17 +52,19 @@ type cmd struct { directOut io.Writer // flags - meshGateway bool - gateway string - proxyID string - sidecarFor string - adminAccessLogPath string - adminBind string - envoyBin string - bootstrap bool - disableCentralConfig bool - grpcAddr string - envoyVersion string + meshGateway bool + gateway string + proxyID string + sidecarFor string + adminAccessLogPath string + adminBind string + envoyBin string + bootstrap bool + disableCentralConfig bool + grpcAddr string + envoyVersion string + prometheusBackendPort string + prometheusScrapePath string // mesh gateway registration information register bool @@ -165,6 +167,19 @@ func (c *cmd) init() { "consul.destination.[service|dc|...]. The old tags were preserved for backward compatibility,"+ "but can be disabled with this flag.") + c.flags.StringVar(&c.prometheusBackendPort, "prometheus-backend-port", "", + "Sets the backend port for the 'prometheus_backend' cluster that envoy_prometheus_bind_addr will point to. "+ + "Without this flag, envoy_prometheus_bind_addr would point to the 'self_admin' cluster where Envoy metrics are exposed. "+ + "The metrics merging feature in consul-k8s uses this to point to the merged metrics endpoint combining Envoy and service metrics. "+ + "Only applicable when envoy_prometheus_bind_addr is set in proxy config.") + + c.flags.StringVar(&c.prometheusScrapePath, "prometheus-scrape-path", "/metrics", + "Sets the path where Envoy will expose metrics on envoy_prometheus_bind_addr listener. "+ + "For example, if envoy_prometheus_bind_addr is 0.0.0.0:20200, and this flag is "+ + "set to /scrape-metrics, prometheus metrics would be scrapeable at "+ + "0.0.0.0:20200/scrape-metrics. "+ + "Only applicable when envoy_prometheus_bind_addr is set in proxy config.") + c.http = &flags.HTTPFlags{} flags.Merge(c.flags, c.http.ClientFlags()) flags.Merge(c.flags, c.http.NamespaceFlags()) @@ -479,6 +494,8 @@ func (c *cmd) templateArgs() (*BootstrapTplArgs, error) { Namespace: httpCfg.Namespace, EnvoyVersion: c.envoyVersion, Datacenter: httpCfg.Datacenter, + PrometheusBackendPort: c.prometheusBackendPort, + PrometheusScrapePath: c.prometheusScrapePath, }, nil } diff --git a/command/connect/envoy/envoy_test.go b/command/connect/envoy/envoy_test.go index 9ca218255..471688c00 100644 --- a/command/connect/envoy/envoy_test.go +++ b/command/connect/envoy/envoy_test.go @@ -146,6 +146,37 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusBackendPort: "", + PrometheusScrapePath: "/metrics", + }, + }, + { + Name: "prometheus-metrics", + Flags: []string{"-proxy-id", "test-proxy", + "-prometheus-backend-port", "20100", "-prometheus-scrape-path", "/scrape-path"}, + ProxyConfig: map[string]interface{}{ + // When envoy_prometheus_bind_addr is set, if + // PrometheusBackendPort is set, there will be a + // "prometheus_backend" cluster in the Envoy configuration. + "envoy_prometheus_bind_addr": "0.0.0.0:9000", + }, + WantArgs: BootstrapTplArgs{ + EnvoyVersion: defaultEnvoyVersion, + ProxyCluster: "test-proxy", + ProxyID: "test-proxy", + // We don't know this til after the lookup so it will be empty in the + // initial args call we are testing here. + ProxySourceService: "", + GRPC: GRPC{ + AgentAddress: "127.0.0.1", + AgentPort: "8502", // Note this is the gRPC port + }, + AdminAccessLogPath: "/dev/null", + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusBackendPort: "20100", + PrometheusScrapePath: "/scrape-path", }, }, { @@ -168,6 +199,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, Token: "c9a52720-bf6c-4aa6-b8bc-66881a5ade95", + PrometheusScrapePath: "/metrics", }, }, { @@ -192,6 +224,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, Token: "c9a52720-bf6c-4aa6-b8bc-66881a5ade95", + PrometheusScrapePath: "/metrics", }, }, { @@ -218,6 +251,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, Token: "c9a52720-bf6c-4aa6-b8bc-66881a5ade95", + PrometheusScrapePath: "/metrics", }, }, { @@ -245,6 +279,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, Token: "c9a52720-bf6c-4aa6-b8bc-66881a5ade95", + PrometheusScrapePath: "/metrics", }, }, { @@ -269,6 +304,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -295,6 +331,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -315,6 +352,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -339,6 +377,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -362,6 +401,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -408,6 +448,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -454,6 +495,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -496,6 +538,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -533,6 +576,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -575,6 +619,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -604,6 +649,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -663,6 +709,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -688,6 +735,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -706,6 +754,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -724,6 +773,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -742,6 +792,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -760,6 +811,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, { @@ -778,6 +830,7 @@ func TestGenerateConfig(t *testing.T) { AdminBindAddress: "127.0.0.1", AdminBindPort: "19000", LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", }, }, } diff --git a/command/connect/envoy/testdata/prometheus-metrics.golden b/command/connect/envoy/testdata/prometheus-metrics.golden new file mode 100644 index 000000000..328671e29 --- /dev/null +++ b/command/connect/envoy/testdata/prometheus-metrics.golden @@ -0,0 +1,265 @@ +{ + "admin": { + "access_log_path": "/dev/null", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 19000 + } + } + }, + "node": { + "cluster": "test-proxy", + "id": "test-proxy", + "metadata": { + "namespace": "default", + "envoy_version": "1.17.0" + } + }, + "static_resources": { + "clusters": [ + { + "name": "local_agent", + "ignore_health_on_host_removal": false, + "connect_timeout": "1s", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "local_agent", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 8502 + } + } + } + } + ] + } + ] + } + }, + { + "name": "prometheus_backend", + "ignore_health_on_host_removal": false, + "connect_timeout": "5s", + "type": "STATIC", + "http_protocol_options": {}, + "loadAssignment": { + "clusterName": "prometheus_backend", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 20100 + } + } + } + } + ] + } + ] + } + } + ], + "listeners": [ + { + "name": "envoy_prometheus_metrics_listener", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9000 + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy_prometheus_metrics", + "codec_type": "HTTP1", + "route_config": { + "name": "self_admin_route", + "virtual_hosts": [ + { + "name": "self_admin", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "path": "/scrape-path" + }, + "route": { + "cluster": "prometheus_backend", + "prefix_rewrite": "/stats/prometheus" + } + }, + { + "match": { + "prefix": "/" + }, + "direct_response": { + "status": 404 + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router" + } + ] + } + } + ] + } + ] + } + ] + }, + "stats_config": { + "stats_tags": [ + { + "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.custom_hash" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service_subset" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.namespace" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.datacenter" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.routing_type" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)", + "tag_name": "consul.destination.trust_domain" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.target" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)", + "tag_name": "consul.destination.full_target" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.(([^.]+)(?:\\.[^.]+)?\\.[^.]+\\.)", + "tag_name": "consul.upstream.service" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?\\.([^.]+)\\.)", + "tag_name": "consul.upstream.datacenter" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.([^.]+))?\\.[^.]+\\.)", + "tag_name": "consul.upstream.namespace" + }, + { + "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.custom_hash" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.service_subset" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.service" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.namespace" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.datacenter" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)", + "tag_name": "consul.routing_type" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)", + "tag_name": "consul.trust_domain" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.target" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)", + "tag_name": "consul.full_target" + }, + { + "tag_name": "local_cluster", + "fixed_value": "test-proxy" + }, + { + "tag_name": "consul.source.service", + "fixed_value": "test-proxy" + }, + { + "tag_name": "consul.source.namespace", + "fixed_value": "default" + }, + { + "tag_name": "consul.source.datacenter", + "fixed_value": "dc1" + } + ], + "use_all_default_tags": true + }, + "dynamic_resources": { + "lds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "cds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "ads_config": { + "api_type": "GRPC", + "transport_api_version": "V3", + "grpc_services": { + "initial_metadata": [ + { + "key": "x-consul-token", + "value": "" + } + ], + "envoy_grpc": { + "cluster_name": "local_agent" + } + } + } + } +} diff --git a/website/content/commands/connect/envoy.mdx b/website/content/commands/connect/envoy.mdx index 7ec8aae6d..5d8b10a0c 100644 --- a/website/content/commands/connect/envoy.mdx +++ b/website/content/commands/connect/envoy.mdx @@ -82,6 +82,19 @@ proxy configuration needed. In cases where either assumption is violated this flag will prevent the command attempting to resolve config from the local agent. +- `-prometheus-backend-port` - Sets the backend port for the "prometheus_backend" + cluster that `envoy_prometheus_bind_addr` will point to. Without this flag, + `envoy_prometheus_bind_addr` would point to the "self_admin" cluster where Envoy metrics + are exposed. The metrics merging feature in consul-k8s uses this to point to the + merged metrics endpoint combining Envoy and service metrics. + Only applicable when `envoy_prometheus_bind_addr` is set in proxy config. + +- `-prometheus-scrape-path` - Sets the path where Envoy will expose metrics on the + `envoy_prometheus_bind_addr` listener. Default is `/metrics`. For example, if `envoy_prometheus_bind_addr` + is `0.0.0.0:20200`, and this flag is set to `/scrape-metrics`, prometheus metrics would + be scrapable at `0.0.0.0:20200/scrape-metrics`. + Only applicable when `envoy_prometheus_bind_addr` is set in proxy config. + - `-- [pass-through options]` - Any options given after a double dash are passed directly through to the `envoy` invocation. See [Envoy's documentation](https://www.envoyproxy.io/docs) for more details. The command