Basic proxy telemetry working; not sure if it's too ugly; need to instrument things we care about

agent/agent_endpoint.go

@@ -1064,6 +1064,9 @@ func (s *HTTPServer) AgentConnectProxyConfig(resp http.ResponseWriter, req *http
 			// Add telemetry config
 			telemetry := s.agent.config.TelemetryConfig(false)
 			if len(telemetry) > 0 {
+				// Rely on the fact that TelemetryConfig makes a new map each call to
+				// override the prefix here without affecting other callers.
+				telemetry["MetricsPrefix"] = "consul.proxy." + target.ID
 				config["telemetry"] = telemetry
 			}
 

agent/agent_endpoint_test.go

@@ -2816,7 +2816,7 @@ func TestAgentConnectCALeafCert_goodNotLocal(t *testing.T) {
 				r.Fatalf("leaf has not updated")
 			}
 
-			// Got a new leaf. Sanity check it's a whole new key as well as differnt
+			// Got a new leaf. Sanity check it's a whole new key as well as different
 			// cert.
 			if issued.PrivateKeyPEM == issued2.PrivateKeyPEM {
 				r.Fatalf("new leaf has same private key as before")
@@ -2903,14 +2903,14 @@ func TestAgentConnectProxyConfig_Blocking(t *testing.T) {
 			"local_service_address": "127.0.0.1:8000",
 			"bind_port":             float64(1234),
 			"connect_timeout_ms":    float64(500),
-			"telemetry":             telemetryDefaults,
+			"telemetry":             makeTelemetryDefaults("test"),
 		},
 	}
 
 	ur, err := copystructure.Copy(expectedResponse)
 	require.NoError(t, err)
 	updatedResponse := ur.(*api.ConnectProxyConfig)
-	updatedResponse.ContentHash = "8f68aa2a4ba81b06"
+	updatedResponse.ContentHash = "29d7e419862bc848"
 	upstreams := updatedResponse.Config["upstreams"].([]interface{})
 	upstreams = append(upstreams,
 		map[string]interface{}{
@@ -3248,9 +3248,11 @@ func TestAgentConnectProxyConfig_aclServiceReadDeny(t *testing.T) {
 	require.True(acl.IsErrPermissionDenied(err))
 }
 
-var telemetryDefaults = map[string]interface{}{
-	"FilterDefault": true,
-	"MetricsPrefix": "consul",
+func makeTelemetryDefaults(targetID string) map[string]interface{} {
+	return map[string]interface{}{
+		"FilterDefault": true,
+		"MetricsPrefix": "consul.proxy." + targetID,
+	}
 }
 
 func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) {
@@ -3304,7 +3306,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) {
 				"bind_address":          "0.0.0.0",
 				"bind_port":             10000,            // "randomly" chosen from our range of 1
 				"local_service_address": "127.0.0.1:8000", // port from service reg
-				"telemetry":             telemetryDefaults,
+				"telemetry":             makeTelemetryDefaults(reg.ID),
 			},
 		},
 		{
@@ -3333,7 +3335,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) {
 				"bind_address":          "0.0.0.0",
 				"bind_port":             10000,            // "randomly" chosen from our range of 1
 				"local_service_address": "127.0.0.1:8000", // port from service reg
-				"telemetry":             telemetryDefaults,
+				"telemetry":             makeTelemetryDefaults(reg.ID),
 			},
 		},
 		{
@@ -3362,7 +3364,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) {
 				"bind_address":          "0.0.0.0",
 				"bind_port":             10000,            // "randomly" chosen from our range of 1
 				"local_service_address": "127.0.0.1:8000", // port from service reg
-				"telemetry":             telemetryDefaults,
+				"telemetry":             makeTelemetryDefaults(reg.ID),
 			},
 		},
 		{
@@ -3398,7 +3400,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) {
 				"local_service_address": "127.0.0.1:8000", // port from service reg
 				"connect_timeout_ms":    1000,
 				"foo":                   "bar",
-				"telemetry":             telemetryDefaults,
+				"telemetry":             makeTelemetryDefaults(reg.ID),
 			},
 		},
 		{
@@ -3441,7 +3443,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) {
 				"bind_port":             float64(1024),
 				"local_service_address": "127.0.0.1:9191",
 				"connect_timeout_ms":    float64(2000),
-				"telemetry":             telemetryDefaults,
+				"telemetry":             makeTelemetryDefaults(reg.ID),
 			},
 		},
 	}

connect/proxy/config.go

@@ -265,6 +265,10 @@ func (w *AgentConfigWatcher) handler(blockVal watch.BlockingParamVal,
 		ProxiedServiceNamespace: "default",
 	}
 
+	if t, ok := resp.Config["telemetry"].(map[string]interface{}); ok {
+		cfg.Telemetry = t
+	}
+
 	// Unmarshal configs
 	err := mapstructure.Decode(resp.Config, &cfg.PublicListener)
 	if err != nil {

connect/proxy/proxy.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/connect"
+	"github.com/hashicorp/consul/lib"
 )
 
 // Proxy implements the built-in connect proxy.
@@ -44,6 +45,15 @@ func (p *Proxy) Serve() error {
 			if cfg == nil {
 				// Initial setup
 
+				// Setup telemetry if configured
+				if len(newCfg.Telemetry) > 0 {
+					p.logger.Printf("[DEBUG] got Telemetry confg: %v", newCfg.Telemetry)
+					_, err := lib.StartupTelemetry(newCfg.Telemetry)
+					if err != nil {
+						p.logger.Printf("[ERR] proxy telemetry config error: %s", err)
+					}
+				}
+
 				// Setup Service instance now we know target ID etc
 				service, err := newCfg.Service(p.client, p.logger)
 				if err != nil {