From 616cc9b6f8f0feb2d7d59aedff5d4501fa06f7e5 Mon Sep 17 00:00:00 2001
From: Daniel Nephin <dnephin@hashicorp.com>
Date: Wed, 4 Aug 2021 16:34:01 -0400
Subject: [PATCH] telemetry: improve cert expiry metrics

Emit the metric immediately so that after restarting an agent, the new expiry time will be
emitted. This is particularly important when this metric is being monitored, because we want
the alert to resovle itself immediately.

Also fixed a bug that was exposed in one of these metrics. The CARoot can be nil, so we have
to handle that case.
---
 agent/consul/leader_metrics.go | 41 +++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/agent/consul/leader_metrics.go b/agent/consul/leader_metrics.go
index 77cde7183..02a6c6d19 100644
--- a/agent/consul/leader_metrics.go
+++ b/agent/consul/leader_metrics.go
@@ -97,8 +97,11 @@ func signingCAExpiryMonitor(s *Server) CertExpirationMonitor {
 func getActiveIntermediateExpiry(s *Server) (time.Duration, error) {
 	state := s.fsm.State()
 	_, root, err := state.CARootActive(nil)
-	if err != nil {
-		return 0, err
+	switch {
+	case err != nil:
+		return 0, fmt.Errorf("failed to retrieve root CA: %w", err)
+	case root == nil:
+		return 0, fmt.Errorf("no active root CA")
 	}
 
 	// the CA used in a secondary DC is the active intermediate,
@@ -130,24 +133,32 @@ func (m CertExpirationMonitor) Monitor(ctx context.Context) error {
 
 	logger := m.Logger.With("metric", strings.Join(m.Key, "."))
 
+	fn := func() {
+		d, err := m.Query()
+		if err != nil {
+			logger.Warn("failed to emit certificate expiry metric", "error", err)
+			return
+		}
+
+		if d < 24*time.Hour {
+			logger.Warn("certificate will expire soon",
+				"time_to_expiry", d, "expiration", time.Now().Add(d))
+		}
+
+		expiry := d / time.Second
+		metrics.SetGaugeWithLabels(m.Key, float32(expiry), m.Labels)
+	}
+
+	// emit the metric immediately so that if a cert was just updated the
+	// new metric will be updated to the new expiration time.
+	fn()
+
 	for {
 		select {
 		case <-ctx.Done():
 			return nil
 		case <-ticker.C:
-			d, err := m.Query()
-			if err != nil {
-				logger.Warn("failed to emit certificate expiry metric", "error", err)
-				continue
-			}
-
-			if d < 24*time.Hour {
-				logger.Warn("certificate will expire soon",
-					"time_to_expiry", d, "expiration", time.Now().Add(d))
-			}
-
-			expiry := d / time.Second
-			metrics.SetGaugeWithLabels(m.Key, float32(expiry), m.Labels)
+			fn()
 		}
 	}
 }