grpc: fix grpc metrics

defaultMetrics was being set at package import time, which meant that it received an instance of
the original default. But lib/telemetry.InitTelemetry sets a new global when it is called.

This resulted in the metrics being sent nowhere.

This commit changes defaultMetrics to be a function, so it will return the global instance when
called. Since it is called after InitTelemetry it will return the correct metrics instance.

agent/grpc/client.go

@@ -61,7 +61,7 @@ func (c *ClientConnPool) ClientConn(datacenter string) (*grpc.ClientConn, error)
 		grpc.WithInsecure(),
 		grpc.WithContextDialer(c.dialer),
 		grpc.WithDisableRetry(),
-		grpc.WithStatsHandler(newStatsHandler(defaultMetrics)),
+		grpc.WithStatsHandler(newStatsHandler(defaultMetrics())),
 		// nolint:staticcheck // there is no other supported alternative to WithBalancerName
 		grpc.WithBalancerName("pick_first"))
 	if err != nil {

agent/grpc/handler.go

@@ -14,11 +14,12 @@ import (
 // The register function will be called with the grpc.Server to register
 // gRPC services with the server.
 func NewHandler(addr net.Addr, register func(server *grpc.Server)) *Handler {
+	metrics := defaultMetrics()
 	// We don't need to pass tls.Config to the server since it's multiplexed
 	// behind the RPC listener, which already has TLS configured.
 	srv := grpc.NewServer(
-		grpc.StatsHandler(newStatsHandler(defaultMetrics)),
-		grpc.StreamInterceptor((&activeStreamCounter{metrics: defaultMetrics}).Intercept),
+		grpc.StatsHandler(newStatsHandler(metrics)),
+		grpc.StreamInterceptor((&activeStreamCounter{metrics: metrics}).Intercept),
 	)
 	register(srv)
 

agent/grpc/stats.go

@@ -9,7 +9,7 @@ import (
 	"google.golang.org/grpc/stats"
 )
 
-var defaultMetrics = metrics.Default()
+var defaultMetrics = metrics.Default
 
 // statsHandler is a grpc/stats.StatsHandler which emits connection and
 // request metrics to go-metrics.

agent/grpc/stats_test.go

@@ -113,11 +113,14 @@ func patchGlobalMetrics(t *testing.T) (*fakeMetricsSink, func()) {
 		FilterDefault:    true,
 	}
 	var err error
-	defaultMetrics, err = metrics.New(cfg, sink)
+	defaultMetrics = func() *metrics.Metrics {
+		m, _ := metrics.New(cfg, sink)
+		return m
+	}
 	require.NoError(t, err)
 	reset := func() {
 		t.Helper()
-		defaultMetrics, err = metrics.New(cfg, &metrics.BlackholeSink{})
+		defaultMetrics = metrics.Default
 		require.NoError(t, err, "failed to reset global metrics")
 	}
 	return sink, reset