open-vault/helper/metricsutil/wrapped_metrics.go
Christopher Swenson c57d053d28
Add database plugin metrics around connections (#16048)
Add database plugin metrics around connections

This is a replacement for #15923 that takes into account recent lock
cleanup.

I went ahead and added back in the hanging plugin test, which I meant to
add in #15944 but forgot.

I tested this by spinning up a statsd sink in the tests and verifying I
got a stream of metrics:

```
$ nc -u -l 8125 | grep backend
test.swenson-Q9Q0L72D39.secrets.database.backend.connections.count.pgx.5.:1.000000|g
test.swenson-Q9Q0L72D39.secrets.database.backend.connections.count.pgx.5.:0.000000|g
test.swenson-Q9Q0L72D39.secrets.database.backend.connections.count.pgx.5.:1.000000|g
test.swenson-Q9Q0L72D39.secrets.database.backend.connections.count.pgx.5.:0.000000|g
```

We have to rework the shared gauge code to work without a full
`ClusterMetricSink`, since we don't have access to the core metrics from
within a plugin.

This only reports metrics every 10 minutes by default, but it solves
some problems we would have had with the gauge values becoming stale and
needing to be re-sent.

Co-authored-by: Tom Proctor <tomhjp@users.noreply.github.com>
2022-06-27 09:34:45 -07:00

157 lines
4.9 KiB
Go

package metricsutil
import (
"strings"
"sync/atomic"
"time"
"github.com/armon/go-metrics"
"github.com/hashicorp/vault/helper/namespace"
)
// ClusterMetricSink serves as a shim around go-metrics
// and inserts a "cluster" label.
//
// It also provides a mechanism to limit the cardinality of the labels on a gauge
// (at each reporting interval, which isn't sufficient if there is variability in which
// labels are the top N) and a backoff mechanism for gauge computation.
type ClusterMetricSink struct {
// ClusterName is either the cluster ID, or a name provided
// in the telemetry configuration stanza.
//
// Because it may be set after the Core is initialized, we need
// to protect against concurrent access.
ClusterName atomic.Value
MaxGaugeCardinality int
GaugeInterval time.Duration
// Sink is the go-metrics instance to send to.
Sink metrics.MetricSink
// Constants that are helpful for metrics within the metrics sink
TelemetryConsts TelemetryConstConfig
}
type TelemetryConstConfig struct {
LeaseMetricsEpsilon time.Duration
NumLeaseMetricsTimeBuckets int
LeaseMetricsNameSpaceLabels bool
}
type Metrics interface {
SetGaugeWithLabels(key []string, val float32, labels []Label)
IncrCounterWithLabels(key []string, val float32, labels []Label)
AddSampleWithLabels(key []string, val float32, labels []Label)
AddDurationWithLabels(key []string, d time.Duration, labels []Label)
MeasureSinceWithLabels(key []string, start time.Time, labels []Label)
}
var _ Metrics = &ClusterMetricSink{}
// SinkWrapper implements `metricsutil.Metrics` using an instance of
// armon/go-metrics `MetricSink` as the underlying implementation.
type SinkWrapper struct {
metrics.MetricSink
}
func (s SinkWrapper) AddDurationWithLabels(key []string, d time.Duration, labels []Label) {
val := float32(d) / float32(time.Millisecond)
s.MetricSink.AddSampleWithLabels(key, val, labels)
}
func (s SinkWrapper) MeasureSinceWithLabels(key []string, start time.Time, labels []Label) {
elapsed := time.Now().Sub(start)
val := float32(elapsed) / float32(time.Millisecond)
s.MetricSink.AddSampleWithLabels(key, val, labels)
}
var _ Metrics = SinkWrapper{}
// Convenience alias
type Label = metrics.Label
func (m *ClusterMetricSink) SetGauge(key []string, val float32) {
m.Sink.SetGaugeWithLabels(key, val, []Label{{"cluster", m.ClusterName.Load().(string)}})
}
func (m *ClusterMetricSink) SetGaugeWithLabels(key []string, val float32, labels []Label) {
m.Sink.SetGaugeWithLabels(key, val,
append(labels, Label{"cluster", m.ClusterName.Load().(string)}))
}
func (m *ClusterMetricSink) IncrCounterWithLabels(key []string, val float32, labels []Label) {
m.Sink.IncrCounterWithLabels(key, val,
append(labels, Label{"cluster", m.ClusterName.Load().(string)}))
}
func (m *ClusterMetricSink) AddSample(key []string, val float32) {
m.Sink.AddSampleWithLabels(key, val, []Label{{"cluster", m.ClusterName.Load().(string)}})
}
func (m *ClusterMetricSink) AddSampleWithLabels(key []string, val float32, labels []Label) {
m.Sink.AddSampleWithLabels(key, val,
append(labels, Label{"cluster", m.ClusterName.Load().(string)}))
}
func (m *ClusterMetricSink) AddDurationWithLabels(key []string, d time.Duration, labels []Label) {
val := float32(d) / float32(time.Millisecond)
m.AddSampleWithLabels(key, val, labels)
}
func (m *ClusterMetricSink) MeasureSinceWithLabels(key []string, start time.Time, labels []Label) {
elapsed := time.Now().Sub(start)
val := float32(elapsed) / float32(time.Millisecond)
m.AddSampleWithLabels(key, val, labels)
}
// BlackholeSink is a default suitable for use in unit tests.
func BlackholeSink() *ClusterMetricSink {
conf := metrics.DefaultConfig("")
conf.EnableRuntimeMetrics = false
sink, _ := metrics.New(conf, &metrics.BlackholeSink{})
cms := &ClusterMetricSink{
ClusterName: atomic.Value{},
Sink: sink,
}
cms.ClusterName.Store("")
return cms
}
func NewClusterMetricSink(clusterName string, sink metrics.MetricSink) *ClusterMetricSink {
cms := &ClusterMetricSink{
ClusterName: atomic.Value{},
Sink: sink,
TelemetryConsts: TelemetryConstConfig{},
}
cms.ClusterName.Store(clusterName)
return cms
}
// SetDefaultClusterName changes the cluster name from its default value,
// if it has not previously been configured.
func (m *ClusterMetricSink) SetDefaultClusterName(clusterName string) {
// This is not a true compare-and-swap, but it should be
// consistent enough for normal uses
if m.ClusterName.Load().(string) == "" {
m.ClusterName.Store(clusterName)
}
}
// NamespaceLabel creates a metrics label for the given
// Namespace: root is "root"; others are path with the
// final '/' removed.
func NamespaceLabel(ns *namespace.Namespace) metrics.Label {
switch {
case ns == nil:
return metrics.Label{"namespace", "root"}
case ns.ID == namespace.RootNamespaceID:
return metrics.Label{"namespace", "root"}
default:
return metrics.Label{
"namespace",
strings.Trim(ns.Path, "/"),
}
}
}