Add flags to consul connect envoy for metrics merging. (#9768)

Allows setting -prometheus-backend-port to configure the cluster envoy_prometheus_bind_addr points to. Allows setting -prometheus-scrape-path to configure which path envoy_prometheus_bind_addr exposes metrics on. -prometheus-backend-port is used by the consul-k8s metrics merging feature, to configure envoy_prometheus_bind_addr to point to the merged metrics endpoint that combines Envoy and service metrics so that one set of annotations on a Pod can scrape metrics from the service and it's Envoy sidecar. -prometheus-scrape-path is used to allow configurability of the path where prometheus metrics are exposed on envoy_prometheus_bind_addr.
2021-03-04 14:15:47 -08:00 · 2021-03-04 14:15:47 -08:00 · 9ff49034e7
parent a2277882c1
commit 9ff49034e7
8 changed files with 636 additions and 87 deletions
--- a/.changelog/9768.txt
+++ b/.changelog/9768.txt
@ -0,0 +1,3 @@
+```release-note:improvement
+connect: adds new flags `prometheus-backend-port` and `prometheus-scrape-port` to `consul connect envoy` to support envoy_prometheus_bind_addr pointing to the merged metrics port when using Consul Connect on K8s.
+```
--- a/command/connect/envoy/bootstrap_config.go
+++ b/command/connect/envoy/bootstrap_config.go
@ -213,19 +213,19 @@ func (c *BootstrapConfig) ConfigureArgs(args *BootstrapTplArgs, omitDeprecatedTa
 	}
 	// Setup prometheus if needed. This MUST happen after the Static*JSON is set above
 	if c.PrometheusBindAddr != "" {
-		if err := c.generateListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", "/metrics", "/stats/prometheus"); err != nil {
+		if err := c.generateListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", args.PrometheusScrapePath, "/stats/prometheus", args.PrometheusBackendPort); err != nil {
 			return err
 		}
 	}
 	// Setup /stats proxy listener if needed. This MUST happen after the Static*JSON is set above
 	if c.StatsBindAddr != "" {
-		if err := c.generateListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats"); err != nil {
+		if err := c.generateListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats", ""); err != nil {
 			return err
 		}
 	}
 	// Setup /ready proxy listener if needed. This MUST happen after the Static*JSON is set above
 	if c.ReadyBindAddr != "" {
-		if err := c.generateListenerConfig(args, c.ReadyBindAddr, "envoy_ready", "path", "/ready", "/ready"); err != nil {
+		if err := c.generateListenerConfig(args, c.ReadyBindAddr, "envoy_ready", "path", "/ready", "/ready", ""); err != nil {
 			return err
 		}
 	}
@ -549,20 +549,36 @@ func generateStatsTags(args *BootstrapTplArgs, initialTags []string, omitDepreca
 	return tagJSONs, nil
 }

-func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite string) error {
+func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite, prometheusBackendPort string) error {
 	host, port, err := net.SplitHostPort(bindAddr)
 	if err != nil {
 		return fmt.Errorf("invalid %s bind address: %s", name, err)
 	}

+	// If prometheusBackendPort is set (not empty string), create
+	// "prometheus_backend" cluster with the prometheusBackendPort that the
+	// listener will point to, rather than the "self_admin" cluster. This is for
+	// the merged metrics feature in consul-k8s, so the
+	// envoy_prometheus_bind_addr listener will point to the merged Envoy and
+	// service metrics endpoint rather than the Envoy admin endpoint for
+	// metrics. This cluster will only be created once since it's only created
+	// when prometheusBackendPort is set, and prometheusBackendPort is only set
+	// when calling this function if c.PrometheusBindAddr is set.
+	clusterPort := args.AdminBindPort
+	clusterName := selfAdminName
+	if prometheusBackendPort != "" {
+		clusterPort = prometheusBackendPort
+		clusterName = "prometheus_backend"
+	}
+
 	clusterJSON := `{
-		"name": "` + selfAdminName + `",
+		"name": "` + clusterName + `",
 		"ignore_health_on_host_removal": false,
 		"connect_timeout": "5s",
 		"type": "STATIC",
 		"http_protocol_options": {},
 		"loadAssignment": {
-			"clusterName": "` + selfAdminName + `",
+			"clusterName": "` + clusterName + `",
 			"endpoints": [
 				{
 					"lbEndpoints": [
@ -571,7 +587,7 @@ func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAdd
 								"address": {
 									"socket_address": {
 										"address": "127.0.0.1",
-										"port_value": ` + args.AdminBindPort + `
+										"port_value": ` + clusterPort + `
 									}
 								}
 							}
@ -612,7 +628,7 @@ func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAdd
 													"` + matchType + `": "` + matchValue + `"
 												},
 												"route": {
-													"cluster": "self_admin",
+													"cluster": "` + clusterName + `",
 													"prefix_rewrite": "` + prefixRewrite + `"
 												}
 											},
--- a/command/connect/envoy/bootstrap_config_test.go
+++ b/command/connect/envoy/bootstrap_config_test.go
@ -38,64 +38,206 @@ const (
    ]
  }
 }`
-	expectedPromListener = `{
-  "name": "envoy_prometheus_metrics_listener",
-  "address": {
-    "socket_address": {
-      "address": "0.0.0.0",
-      "port_value": 9000
-    }
-  },
-  "filter_chains": [
-    {
-      "filters": [
-        {
-          "name": "envoy.filters.network.http_connection_manager",
-          "typedConfig": {
-            "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
-            "stat_prefix": "envoy_prometheus_metrics",
-            "codec_type": "HTTP1",
-            "route_config": {
-              "name": "self_admin_route",
-              "virtual_hosts": [
-                {
-                  "name": "self_admin",
-                  "domains": [
-                    "*"
-                  ],
-                  "routes": [
-                    {
-                      "match": {
-                        "path": "/metrics"
-                      },
-                      "route": {
-                        "cluster": "self_admin",
-                        "prefix_rewrite": "/stats/prometheus"
-                      }
-                    },
-                    {
-                      "match": {
-                        "prefix": "/"
-                      },
-                      "direct_response": {
-                        "status": 404
-                      }
-                    }
-                  ]
+	expectedPrometheusBackendCluster = `{
+  "name": "prometheus_backend",
+  "ignore_health_on_host_removal": false,
+  "connect_timeout": "5s",
+  "type": "STATIC",
+  "http_protocol_options": {},
+  "loadAssignment": {
+    "clusterName": "prometheus_backend",
+    "endpoints": [
+      {
+        "lbEndpoints": [
+          {
+            "endpoint": {
+              "address": {
+                "socket_address": {
+                  "address": "127.0.0.1",
+                  "port_value": 20100
                }
-              ]
-            },
-            "http_filters": [
-              {
-                "name": "envoy.filters.http.router"
              }
-            ]
+            }
          }
-        }
-      ]
-    }
-  ]
+        ]
+      }
+    ]
+  }
 }`
+	expectedPromListener = `{
+	"name": "envoy_prometheus_metrics_listener",
+	"address": {
+	  "socket_address": {
+		"address": "0.0.0.0",
+		"port_value": 9000
+	  }
+	},
+	"filter_chains": [
+	  {
+		"filters": [
+		  {
+			"name": "envoy.filters.network.http_connection_manager",
+			"typedConfig": {
+			  "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
+			  "stat_prefix": "envoy_prometheus_metrics",
+			  "codec_type": "HTTP1",
+			  "route_config": {
+				"name": "self_admin_route",
+				"virtual_hosts": [
+				  {
+					"name": "self_admin",
+					"domains": [
+					  "*"
+					],
+					"routes": [
+					  {
+						"match": {
+						  "path": "/metrics"
+						},
+						"route": {
+						  "cluster": "self_admin",
+						  "prefix_rewrite": "/stats/prometheus"
+						}
+					  },
+					  {
+						"match": {
+						  "prefix": "/"
+						},
+						"direct_response": {
+						  "status": 404
+						}
+					  }
+					]
+				  }
+				]
+			  },
+			  "http_filters": [
+				{
+				  "name": "envoy.filters.http.router"
+				}
+			  ]
+			}
+		  }
+		]
+	  }
+	]
+  }`
+	expectedPromListenerCustomScrapePath = `{
+	"name": "envoy_prometheus_metrics_listener",
+	"address": {
+	  "socket_address": {
+		"address": "0.0.0.0",
+		"port_value": 9000
+	  }
+	},
+	"filter_chains": [
+	  {
+		"filters": [
+		  {
+			"name": "envoy.filters.network.http_connection_manager",
+			"typedConfig": {
+			  "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
+			  "stat_prefix": "envoy_prometheus_metrics",
+			  "codec_type": "HTTP1",
+			  "route_config": {
+				"name": "self_admin_route",
+				"virtual_hosts": [
+				  {
+					"name": "self_admin",
+					"domains": [
+					  "*"
+					],
+					"routes": [
+					  {
+						"match": {
+						  "path": "/scrape-path"
+						},
+						"route": {
+						  "cluster": "self_admin",
+						  "prefix_rewrite": "/stats/prometheus"
+						}
+					  },
+					  {
+						"match": {
+						  "prefix": "/"
+						},
+						"direct_response": {
+						  "status": 404
+						}
+					  }
+					]
+				  }
+				]
+			  },
+			  "http_filters": [
+				{
+				  "name": "envoy.filters.http.router"
+				}
+			  ]
+			}
+		  }
+		]
+	  }
+	]
+  }`
+	expectedPromListenerWithPrometheusBackendCluster = `{
+	"name": "envoy_prometheus_metrics_listener",
+	"address": {
+	  "socket_address": {
+		"address": "0.0.0.0",
+		"port_value": 9000
+	  }
+	},
+	"filter_chains": [
+	  {
+		"filters": [
+		  {
+			"name": "envoy.filters.network.http_connection_manager",
+			"typedConfig": {
+			  "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
+			  "stat_prefix": "envoy_prometheus_metrics",
+			  "codec_type": "HTTP1",
+			  "route_config": {
+				"name": "self_admin_route",
+				"virtual_hosts": [
+				  {
+					"name": "self_admin",
+					"domains": [
+					  "*"
+					],
+					"routes": [
+					  {
+						"match": {
+						  "path": "/metrics"
+						},
+						"route": {
+						  "cluster": "prometheus_backend",
+						  "prefix_rewrite": "/stats/prometheus"
+						}
+					  },
+					  {
+						"match": {
+						  "prefix": "/"
+						},
+						"direct_response": {
+						  "status": 404
+						}
+					  }
+					]
+				  }
+				]
+			  },
+			  "http_filters": [
+				{
+				  "name": "envoy.filters.http.router"
+				}
+			  ]
+			}
+		  }
+		]
+	  }
+	]
+  }`
 	expectedStatsListener = `{
  "name": "envoy_metrics_listener",
  "address": {
@ -457,8 +599,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
 				PrometheusBindAddr: "0.0.0.0:9000",
 			},
 			baseArgs: BootstrapTplArgs{
-				AdminBindAddress: "127.0.0.1",
-				AdminBindPort:    "19000",
+				AdminBindAddress:     "127.0.0.1",
+				AdminBindPort:        "19000",
+				PrometheusScrapePath: "/metrics",
 			},
 			wantArgs: BootstrapTplArgs{
 				AdminBindAddress: "127.0.0.1",
@ -466,8 +609,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
 				// Should add a static cluster for the self-proxy to admin
 				StaticClustersJSON: expectedSelfAdminCluster,
 				// Should add a static http listener too
-				StaticListenersJSON: expectedPromListener,
-				StatsConfigJSON:     defaultStatsConfigJSON,
+				StaticListenersJSON:  expectedPromListener,
+				StatsConfigJSON:      defaultStatsConfigJSON,
+				PrometheusScrapePath: "/metrics",
 			},
 			wantErr: false,
 		},
@ -479,8 +623,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
 				StaticListenersJSON: `{"baz":"qux"}`,
 			},
 			baseArgs: BootstrapTplArgs{
-				AdminBindAddress: "127.0.0.1",
-				AdminBindPort:    "19000",
+				AdminBindAddress:     "127.0.0.1",
+				AdminBindPort:        "19000",
+				PrometheusScrapePath: "/scrape-path",
 			},
 			wantArgs: BootstrapTplArgs{
 				AdminBindAddress: "127.0.0.1",
@ -488,8 +633,33 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
 				// Should add a static cluster for the self-proxy to admin
 				StaticClustersJSON: `{"foo":"bar"},` + expectedSelfAdminCluster,
 				// Should add a static http listener too
-				StaticListenersJSON: `{"baz":"qux"},` + expectedPromListener,
-				StatsConfigJSON:     defaultStatsConfigJSON,
+				StaticListenersJSON:  `{"baz":"qux"},` + expectedPromListenerCustomScrapePath,
+				StatsConfigJSON:      defaultStatsConfigJSON,
+				PrometheusScrapePath: "/scrape-path",
+			},
+			wantErr: false,
+		},
+		{
+			name: "prometheus-bind-addr-with-prometheus-backend",
+			input: BootstrapConfig{
+				PrometheusBindAddr: "0.0.0.0:9000",
+			},
+			baseArgs: BootstrapTplArgs{
+				AdminBindAddress:      "127.0.0.1",
+				AdminBindPort:         "19000",
+				PrometheusBackendPort: "20100",
+				PrometheusScrapePath:  "/metrics",
+			},
+			wantArgs: BootstrapTplArgs{
+				AdminBindAddress: "127.0.0.1",
+				AdminBindPort:    "19000",
+				// Should use the "prometheus_backend" cluster instead, which
+				// uses the PrometheusBackendPort rather than Envoy admin port
+				StaticClustersJSON:    expectedPrometheusBackendCluster,
+				StaticListenersJSON:   expectedPromListenerWithPrometheusBackendCluster,
+				StatsConfigJSON:       defaultStatsConfigJSON,
+				PrometheusBackendPort: "20100",
+				PrometheusScrapePath:  "/metrics",
 			},
 			wantErr: false,
 		},
@ -635,8 +805,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
 				StatsBindAddr:      "0.0.0.0:9000",
 			},
 			baseArgs: BootstrapTplArgs{
-				AdminBindAddress: "127.0.0.1",
-				AdminBindPort:    "19000",
+				AdminBindAddress:     "127.0.0.1",
+				AdminBindPort:        "19000",
+				PrometheusScrapePath: "/metrics",
 			},
 			wantArgs: BootstrapTplArgs{
 				AdminBindAddress: "127.0.0.1",
@ -648,7 +819,8 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
 					[]string{expectedPromListener, expectedStatsListener, expectedReadyListener},
 					", ",
 				),
-				StatsConfigJSON: defaultStatsConfigJSON,
+				StatsConfigJSON:      defaultStatsConfigJSON,
+				PrometheusScrapePath: "/metrics",
 			},
 			wantErr: false,
 		},
@ -660,8 +832,9 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
 				StatsBindAddr:      "0.0.0.0:9000",
 			},
 			baseArgs: BootstrapTplArgs{
-				AdminBindAddress: "127.0.0.1",
-				AdminBindPort:    "19000",
+				AdminBindAddress:     "127.0.0.1",
+				AdminBindPort:        "19000",
+				PrometheusScrapePath: "/metrics",
 			},
 			omitDeprecatedTags: true,
 			wantArgs: BootstrapTplArgs{
@ -673,7 +846,8 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
 					", ",
 				),
 				// Should not have default stats config JSON when deprecated tags are omitted
-				StatsConfigJSON: updatedStatsConfigJSON,
+				StatsConfigJSON:      updatedStatsConfigJSON,
+				PrometheusScrapePath: "/metrics",
 			},
 			wantErr: false,
 		},
--- a/command/connect/envoy/bootstrap_tpl.go
+++ b/command/connect/envoy/bootstrap_tpl.go
@ -95,6 +95,14 @@ type BootstrapTplArgs struct {
 	// EnvoyVersion is the envoy version, which is necessary to generate the
 	// correct configuration.
 	EnvoyVersion string
+
+	// PrometheusBackendPort will configure a "prometheus_backend" cluster which
+	// envoy_prometheus_bind_addr will point to.
+	PrometheusBackendPort string
+
+	// PrometheusScrapePath will configure the path where metrics are exposed on
+	// the envoy_prometheus_bind_addr listener.
+	PrometheusScrapePath string
 }

 // GRPC settings used in the bootstrap template.
--- a/command/connect/envoy/envoy.go
+++ b/command/connect/envoy/envoy.go
@ -52,17 +52,19 @@ type cmd struct {
 	directOut io.Writer

 	// flags
-	meshGateway          bool
-	gateway              string
-	proxyID              string
-	sidecarFor           string
-	adminAccessLogPath   string
-	adminBind            string
-	envoyBin             string
-	bootstrap            bool
-	disableCentralConfig bool
-	grpcAddr             string
-	envoyVersion         string
+	meshGateway           bool
+	gateway               string
+	proxyID               string
+	sidecarFor            string
+	adminAccessLogPath    string
+	adminBind             string
+	envoyBin              string
+	bootstrap             bool
+	disableCentralConfig  bool
+	grpcAddr              string
+	envoyVersion          string
+	prometheusBackendPort string
+	prometheusScrapePath  string

 	// mesh gateway registration information
 	register           bool
@ -165,6 +167,19 @@ func (c *cmd) init() {
 			"consul.destination.[service|dc|...]. The old tags were preserved for backward compatibility,"+
 			"but can be disabled with this flag.")

+	c.flags.StringVar(&c.prometheusBackendPort, "prometheus-backend-port", "",
+		"Sets the backend port for the 'prometheus_backend' cluster that envoy_prometheus_bind_addr will point to. "+
+			"Without this flag, envoy_prometheus_bind_addr would point to the 'self_admin' cluster where Envoy metrics are exposed. "+
+			"The metrics merging feature in consul-k8s uses this to point to the merged metrics endpoint combining Envoy and service metrics. "+
+			"Only applicable when envoy_prometheus_bind_addr is set in proxy config.")
+
+	c.flags.StringVar(&c.prometheusScrapePath, "prometheus-scrape-path", "/metrics",
+		"Sets the path where Envoy will expose metrics on envoy_prometheus_bind_addr listener. "+
+			"For example, if envoy_prometheus_bind_addr is 0.0.0.0:20200, and this flag is "+
+			"set to /scrape-metrics, prometheus metrics would be scrapeable at "+
+			"0.0.0.0:20200/scrape-metrics. "+
+			"Only applicable when envoy_prometheus_bind_addr is set in proxy config.")
+
 	c.http = &flags.HTTPFlags{}
 	flags.Merge(c.flags, c.http.ClientFlags())
 	flags.Merge(c.flags, c.http.NamespaceFlags())
@ -479,6 +494,8 @@ func (c *cmd) templateArgs() (*BootstrapTplArgs, error) {
 		Namespace:             httpCfg.Namespace,
 		EnvoyVersion:          c.envoyVersion,
 		Datacenter:            httpCfg.Datacenter,
+		PrometheusBackendPort: c.prometheusBackendPort,
+		PrometheusScrapePath:  c.prometheusScrapePath,
 	}, nil
 }

--- a/command/connect/envoy/envoy_test.go
+++ b/command/connect/envoy/envoy_test.go
@ -146,6 +146,37 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusBackendPort: "",
+				PrometheusScrapePath:  "/metrics",
+			},
+		},
+		{
+			Name: "prometheus-metrics",
+			Flags: []string{"-proxy-id", "test-proxy",
+				"-prometheus-backend-port", "20100", "-prometheus-scrape-path", "/scrape-path"},
+			ProxyConfig: map[string]interface{}{
+				// When envoy_prometheus_bind_addr is set, if
+				// PrometheusBackendPort is set, there will be a
+				// "prometheus_backend" cluster in the Envoy configuration.
+				"envoy_prometheus_bind_addr": "0.0.0.0:9000",
+			},
+			WantArgs: BootstrapTplArgs{
+				EnvoyVersion: defaultEnvoyVersion,
+				ProxyCluster: "test-proxy",
+				ProxyID:      "test-proxy",
+				// We don't know this til after the lookup so it will be empty in the
+				// initial args call we are testing here.
+				ProxySourceService: "",
+				GRPC: GRPC{
+					AgentAddress: "127.0.0.1",
+					AgentPort:    "8502", // Note this is the gRPC port
+				},
+				AdminAccessLogPath:    "/dev/null",
+				AdminBindAddress:      "127.0.0.1",
+				AdminBindPort:         "19000",
+				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusBackendPort: "20100",
+				PrometheusScrapePath:  "/scrape-path",
 			},
 		},
 		{
@ -168,6 +199,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
 				Token:                 "c9a52720-bf6c-4aa6-b8bc-66881a5ade95",
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -192,6 +224,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
 				Token:                 "c9a52720-bf6c-4aa6-b8bc-66881a5ade95",
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -218,6 +251,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
 				Token:                 "c9a52720-bf6c-4aa6-b8bc-66881a5ade95",
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -245,6 +279,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
 				Token:                 "c9a52720-bf6c-4aa6-b8bc-66881a5ade95",
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -269,6 +304,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -295,6 +331,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -315,6 +352,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -339,6 +377,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -362,6 +401,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -408,6 +448,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -454,6 +495,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -496,6 +538,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -533,6 +576,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -575,6 +619,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -604,6 +649,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -663,6 +709,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -688,6 +735,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -706,6 +754,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -724,6 +773,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -742,6 +792,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -760,6 +811,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 		{
@ -778,6 +830,7 @@ func TestGenerateConfig(t *testing.T) {
 				AdminBindAddress:      "127.0.0.1",
 				AdminBindPort:         "19000",
 				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
 			},
 		},
 	}
--- a/command/connect/envoy/testdata/prometheus-metrics.golden
+++ b/command/connect/envoy/testdata/prometheus-metrics.golden
@ -0,0 +1,265 @@
+{
+  "admin": {
+    "access_log_path": "/dev/null",
+    "address": {
+      "socket_address": {
+        "address": "127.0.0.1",
+        "port_value": 19000
+      }
+    }
+  },
+  "node": {
+    "cluster": "test-proxy",
+    "id": "test-proxy",
+    "metadata": {
+      "namespace": "default",
+      "envoy_version": "1.17.0"
+    }
+  },
+  "static_resources": {
+    "clusters": [
+      {
+        "name": "local_agent",
+        "ignore_health_on_host_removal": false,
+        "connect_timeout": "1s",
+        "type": "STATIC",
+        "http2_protocol_options": {},
+        "loadAssignment": {
+          "clusterName": "local_agent",
+          "endpoints": [
+            {
+              "lbEndpoints": [
+                {
+                  "endpoint": {
+                    "address": {
+                      "socket_address": {
+                        "address": "127.0.0.1",
+                        "port_value": 8502
+                      }
+                    }
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      },
+      {
+        "name": "prometheus_backend",
+        "ignore_health_on_host_removal": false,
+        "connect_timeout": "5s",
+        "type": "STATIC",
+        "http_protocol_options": {},
+        "loadAssignment": {
+          "clusterName": "prometheus_backend",
+          "endpoints": [
+            {
+              "lbEndpoints": [
+                {
+                  "endpoint": {
+                    "address": {
+                      "socket_address": {
+                        "address": "127.0.0.1",
+                        "port_value": 20100
+                      }
+                    }
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      }
+    ],
+    "listeners": [
+      {
+        "name": "envoy_prometheus_metrics_listener",
+        "address": {
+          "socket_address": {
+            "address": "0.0.0.0",
+            "port_value": 9000
+          }
+        },
+        "filter_chains": [
+          {
+            "filters": [
+              {
+                "name": "envoy.filters.network.http_connection_manager",
+                "typedConfig": {
+                  "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
+                  "stat_prefix": "envoy_prometheus_metrics",
+                  "codec_type": "HTTP1",
+                  "route_config": {
+                    "name": "self_admin_route",
+                    "virtual_hosts": [
+                      {
+                        "name": "self_admin",
+                        "domains": [
+                          "*"
+                        ],
+                        "routes": [
+                          {
+                            "match": {
+                              "path": "/scrape-path"
+                            },
+                            "route": {
+                              "cluster": "prometheus_backend",
+                              "prefix_rewrite": "/stats/prometheus"
+                            }
+                          },
+                          {
+                            "match": {
+                              "prefix": "/"
+                            },
+                            "direct_response": {
+                              "status": 404
+                            }
+                          }
+                        ]
+                      }
+                    ]
+                  },
+                  "http_filters": [
+                    {
+                      "name": "envoy.filters.http.router"
+                    }
+                  ]
+                }
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "stats_config": {
+    "stats_tags": [
+      {
+        "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.custom_hash"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.service_subset"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.service"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.namespace"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.datacenter"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.routing_type"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
+        "tag_name": "consul.destination.trust_domain"
+      },
+      {
+        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.target"
+      },
+      {
+        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
+        "tag_name": "consul.destination.full_target"
+      },
+      {
+        "regex": "^(?:tcp|http)\\.upstream\\.(([^.]+)(?:\\.[^.]+)?\\.[^.]+\\.)",
+        "tag_name": "consul.upstream.service"
+      },
+      {
+        "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?\\.([^.]+)\\.)",
+        "tag_name": "consul.upstream.datacenter"
+      },
+      {
+        "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.([^.]+))?\\.[^.]+\\.)",
+        "tag_name": "consul.upstream.namespace"
+      },
+      {
+        "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.custom_hash"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.service_subset"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.service"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.namespace"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.datacenter"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.routing_type"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
+        "tag_name": "consul.trust_domain"
+      },
+      {
+        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.target"
+      },
+      {
+        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
+        "tag_name": "consul.full_target"
+      },
+      {
+        "tag_name": "local_cluster",
+        "fixed_value": "test-proxy"
+      },
+      {
+        "tag_name": "consul.source.service",
+        "fixed_value": "test-proxy"
+      },
+      {
+        "tag_name": "consul.source.namespace",
+        "fixed_value": "default"
+      },
+      {
+        "tag_name": "consul.source.datacenter",
+        "fixed_value": "dc1"
+      }
+    ],
+    "use_all_default_tags": true
+  },
+  "dynamic_resources": {
+    "lds_config": {
+      "ads": {},
+      "resource_api_version": "V3"
+    },
+    "cds_config": {
+      "ads": {},
+      "resource_api_version": "V3"
+    },
+    "ads_config": {
+      "api_type": "GRPC",
+      "transport_api_version": "V3",
+      "grpc_services": {
+        "initial_metadata": [
+          {
+            "key": "x-consul-token",
+            "value": ""
+          }
+        ],
+        "envoy_grpc": {
+          "cluster_name": "local_agent"
+        }
+      }
+    }
+  }
+}
--- a/website/content/commands/connect/envoy.mdx
+++ b/website/content/commands/connect/envoy.mdx
@ -82,6 +82,19 @@ proxy configuration needed.
  In cases where either assumption is violated this flag will prevent the
  command attempting to resolve config from the local agent.

+- `-prometheus-backend-port` - Sets the backend port for the "prometheus_backend"
+  cluster that `envoy_prometheus_bind_addr` will point to. Without this flag,
+  `envoy_prometheus_bind_addr` would point to the "self_admin" cluster where Envoy metrics
+  are exposed. The metrics merging feature in consul-k8s uses this to point to the
+  merged metrics endpoint combining Envoy and service metrics.
+	Only applicable when `envoy_prometheus_bind_addr` is set in proxy config.
+
+- `-prometheus-scrape-path` - Sets the path where Envoy will expose metrics on the
+  `envoy_prometheus_bind_addr` listener. Default is `/metrics`. For example, if `envoy_prometheus_bind_addr`
+  is `0.0.0.0:20200`, and this flag is set to `/scrape-metrics`, prometheus metrics would 
+  be scrapable at `0.0.0.0:20200/scrape-metrics`.
+	Only applicable when `envoy_prometheus_bind_addr` is set in proxy config.
+
 - `-- [pass-through options]` - Any options given after a double dash are passed
  directly through to the `envoy` invocation. See [Envoy's
  documentation](https://www.envoyproxy.io/docs) for more details. The command