From ced9b2bee4bfc10932ef846ed7b280f1bee7b34f Mon Sep 17 00:00:00 2001 From: Paul Banks Date: Thu, 7 Jun 2018 14:11:06 +0100 Subject: [PATCH] Expose telemetry config from RuntimeConfig to proxy config endpoint --- agent/agent_endpoint.go | 6 +++ agent/agent_endpoint_test.go | 13 ++++++- agent/config/config.go | 3 ++ agent/config/runtime.go | 63 +++++++++++++++++++++++++++++++ agent/config/runtime_test.go | 73 ++++++++++++++++++++++++++++++++++++ connect/proxy/config.go | 5 +++ 6 files changed, 162 insertions(+), 1 deletion(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 5f13ed5eb..63812aa62 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -1061,6 +1061,12 @@ func (s *HTTPServer) AgentConnectProxyConfig(resp http.ResponseWriter, req *http target.Port) } + // Add telemetry config + telemetry := s.agent.config.TelemetryConfig(false) + if len(telemetry) > 0 { + config["telemetry"] = telemetry + } + reply := &api.ConnectProxyConfig{ ProxyServiceID: proxy.Proxy.ProxyService.ID, TargetServiceID: target.ID, diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index cd8389cae..b8b82c74c 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -2903,13 +2903,14 @@ func TestAgentConnectProxyConfig_Blocking(t *testing.T) { "local_service_address": "127.0.0.1:8000", "bind_port": float64(1234), "connect_timeout_ms": float64(500), + "telemetry": telemetryDefaults, }, } ur, err := copystructure.Copy(expectedResponse) require.NoError(t, err) updatedResponse := ur.(*api.ConnectProxyConfig) - updatedResponse.ContentHash = "23b5b6b3767601e1" + updatedResponse.ContentHash = "8f68aa2a4ba81b06" upstreams := updatedResponse.Config["upstreams"].([]interface{}) upstreams = append(upstreams, map[string]interface{}{ @@ -3247,6 +3248,11 @@ func TestAgentConnectProxyConfig_aclServiceReadDeny(t *testing.T) { require.True(acl.IsErrPermissionDenied(err)) } +var telemetryDefaults = map[string]interface{}{ + "FilterDefault": true, + "MetricsPrefix": "consul", +} + func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) { t.Parallel() @@ -3298,6 +3304,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) { "bind_address": "0.0.0.0", "bind_port": 10000, // "randomly" chosen from our range of 1 "local_service_address": "127.0.0.1:8000", // port from service reg + "telemetry": telemetryDefaults, }, }, { @@ -3326,6 +3333,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) { "bind_address": "0.0.0.0", "bind_port": 10000, // "randomly" chosen from our range of 1 "local_service_address": "127.0.0.1:8000", // port from service reg + "telemetry": telemetryDefaults, }, }, { @@ -3354,6 +3362,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) { "bind_address": "0.0.0.0", "bind_port": 10000, // "randomly" chosen from our range of 1 "local_service_address": "127.0.0.1:8000", // port from service reg + "telemetry": telemetryDefaults, }, }, { @@ -3389,6 +3398,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) { "local_service_address": "127.0.0.1:8000", // port from service reg "connect_timeout_ms": 1000, "foo": "bar", + "telemetry": telemetryDefaults, }, }, { @@ -3431,6 +3441,7 @@ func TestAgentConnectProxyConfig_ConfigHandling(t *testing.T) { "bind_port": float64(1024), "local_service_address": "127.0.0.1:9191", "connect_timeout_ms": float64(2000), + "telemetry": telemetryDefaults, }, }, } diff --git a/agent/config/config.go b/agent/config/config.go index ab3b34c17..8919ae3e4 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -428,6 +428,9 @@ type Performance struct { RPCHoldTimeout *string `json:"rpc_hold_timeout" hcl:"rpc_hold_timeout" mapstructure:"rpc_hold_timeout"` } +// Telemetry defines the configuration for logging telemetry. NOTE: that any +// additions here that are reflected in RuntimeConfig also need to be added to +// RuntimeConfig.TelemetryConfig. type Telemetry struct { CirconusAPIApp *string `json:"circonus_api_app,omitempty" hcl:"circonus_api_app" mapstructure:"circonus_api_app"` CirconusAPIToken *string `json:"circonus_api_token,omitempty" json:"-" hcl:"circonus_api_token" mapstructure:"circonus_api_token" json:"-"` diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 695bff954..f0ab0f3bf 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -1331,6 +1331,69 @@ type RuntimeConfig struct { Watches []map[string]interface{} } +// TelemetryConfig returns all the Telemetry fields in a map for passing on to +// other processes that need to share the same telemetry config. This method +// must be kept in sync with additions to Telemetry* configs above. includeEmpty +// is not really useful in practice but allows us to verify in tests using +// reflection that new fields were not missed. This method avoids runtime +// reflection however it's tests validate it's completeness using reflection. +func (c *RuntimeConfig) TelemetryConfig(includeEmpty bool) map[string]interface{} { + t := make(map[string]interface{}) + + addString := func(key string, val string) { + if !includeEmpty && val == "" { + return + } + t[key] = val + } + + addBool := func(key string, val bool) { + if !includeEmpty && !val { + return + } + t[key] = val + } + + addDuration := func(key string, val time.Duration) { + if !includeEmpty && val == 0 { + return + } + t[key] = val + } + + addStrSlice := func(key string, val []string) { + if !includeEmpty && len(val) == 0 { + return + } + t[key] = val + } + + addString("CirconusAPIApp", c.TelemetryCirconusAPIApp) + addString("CirconusAPIToken", c.TelemetryCirconusAPIToken) + addString("CirconusAPIURL", c.TelemetryCirconusAPIURL) + addString("CirconusBrokerID", c.TelemetryCirconusBrokerID) + addString("CirconusBrokerSelectTag", c.TelemetryCirconusBrokerSelectTag) + addString("CirconusCheckDisplayName", c.TelemetryCirconusCheckDisplayName) + addString("CirconusCheckForceMetricActivation", c.TelemetryCirconusCheckForceMetricActivation) + addString("CirconusCheckID", c.TelemetryCirconusCheckID) + addString("CirconusCheckInstanceID", c.TelemetryCirconusCheckInstanceID) + addString("CirconusCheckSearchTag", c.TelemetryCirconusCheckSearchTag) + addString("CirconusCheckTags", c.TelemetryCirconusCheckTags) + addString("CirconusSubmissionInterval", c.TelemetryCirconusSubmissionInterval) + addString("CirconusSubmissionURL", c.TelemetryCirconusSubmissionURL) + addBool("DisableHostname", c.TelemetryDisableHostname) + addString("DogstatsdAddr", c.TelemetryDogstatsdAddr) + addStrSlice("DogstatsdTags", c.TelemetryDogstatsdTags) + addDuration("PrometheusRetentionTime", c.TelemetryPrometheusRetentionTime) + addBool("FilterDefault", c.TelemetryFilterDefault) + addStrSlice("AllowedPrefixes", c.TelemetryAllowedPrefixes) + addStrSlice("BlockedPrefixes", c.TelemetryBlockedPrefixes) + addString("MetricsPrefix", c.TelemetryMetricsPrefix) + addString("StatsdAddr", c.TelemetryStatsdAddr) + addString("StatsiteAddr", c.TelemetryStatsiteAddr) + return t +} + // IncomingHTTPSConfig returns the TLS configuration for HTTPS // connections to consul. func (c *RuntimeConfig) IncomingHTTPSConfig() (*tls.Config, error) { diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 9852347c6..76a645f8f 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -23,6 +23,7 @@ import ( "github.com/hashicorp/consul/types" "github.com/pascaldekloe/goe/verify" "github.com/sergi/go-diff/diffmatchpatch" + "github.com/stretchr/testify/require" ) type configTest struct { @@ -4532,3 +4533,75 @@ func metaPairs(n int, format string) string { panic("invalid format: " + format) } } + +func TestTelemetryConfig(t *testing.T) { + + tests := []struct { + name string + cfg RuntimeConfig + includeEmpty bool + }{ + { + name: "sanity check specific fields", + cfg: RuntimeConfig{ + TelemetryAllowedPrefixes: []string{"foo", "bar"}, + TelemetryStatsiteAddr: "localhost:1234", + TelemetryDisableHostname: true, + }, + includeEmpty: false, + }, + { + name: "verify all telemetry fields present", + cfg: RuntimeConfig{}, + includeEmpty: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Build expected response with reflection + cfgValue := reflect.ValueOf(tt.cfg) + structType := cfgValue.Type() + nFields := cfgValue.NumField() + + expect := make(map[string]interface{}) + + for i := 0; i < nFields; i++ { + name := structType.Field(i).Name + if strings.HasPrefix(name, "Telemetry") { + val := cfgValue.Field(i) + if !tt.includeEmpty { + // No built in way to check for zero-values for all types so only + // implementing this for the types we actually have for now. The test + // failure will hopefully catch the case where we add new types later. + switch val.Kind() { + case reflect.Slice: + if val.IsNil() { + continue + } + case reflect.Int, reflect.Int64: // time.Duration == int64 + if val.Int() == 0 { + continue + } + case reflect.String: + if val.String() == "" { + continue + } + case reflect.Bool: + if val.Bool() == false { + continue + } + default: + t.Fatalf("New type added to Telemetry* fields in RuntimeConfig." + + "Update this test.") + } + } + // non-zero values should be exported. + expect[strings.TrimPrefix(name, "Telemetry")] = val.Interface() + } + } + + require.Equal(t, expect, tt.cfg.TelemetryConfig(tt.includeEmpty)) + }) + } +} diff --git a/connect/proxy/config.go b/connect/proxy/config.go index b0e020ab2..012e150cd 100644 --- a/connect/proxy/config.go +++ b/connect/proxy/config.go @@ -36,6 +36,11 @@ type Config struct { // Upstreams configures outgoing proxies for remote connect services. Upstreams []UpstreamConfig `json:"upstreams" hcl:"upstreams"` + + // Telemetry stores configuration to configure go-metrics. It is typically + // passed the Telemetry block from the agent's config verbatim so that the + // proxy will log metrics to the same location(s) as the agent. + Telemetry map[string]interface{} } // Service returns the *connect.Service structure represented by this config.