2020-07-23 18:37:33 +00:00
|
|
|
|
// +build go1.9
|
2019-03-26 21:50:42 +00:00
|
|
|
|
|
2018-04-05 16:21:32 +00:00
|
|
|
|
package prometheus
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"fmt"
|
2020-07-23 18:37:33 +00:00
|
|
|
|
"log"
|
2020-08-11 16:17:43 +00:00
|
|
|
|
"regexp"
|
2018-04-05 16:21:32 +00:00
|
|
|
|
"strings"
|
|
|
|
|
"sync"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"github.com/armon/go-metrics"
|
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2020-07-23 18:37:33 +00:00
|
|
|
|
"github.com/prometheus/client_golang/prometheus/push"
|
2018-04-05 16:21:32 +00:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
var (
|
|
|
|
|
// DefaultPrometheusOpts is the default set of options used when creating a
|
|
|
|
|
// PrometheusSink.
|
|
|
|
|
DefaultPrometheusOpts = PrometheusOpts{
|
|
|
|
|
Expiration: 60 * time.Second,
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// PrometheusOpts is used to configure the Prometheus Sink
|
|
|
|
|
type PrometheusOpts struct {
|
|
|
|
|
// Expiration is the duration a metric is valid for, after which it will be
|
|
|
|
|
// untracked. If the value is zero, a metric is never expired.
|
|
|
|
|
Expiration time.Duration
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type PrometheusSink struct {
|
2020-07-23 18:37:33 +00:00
|
|
|
|
// If these will ever be copied, they should be converted to *sync.Map values and initialized appropriately
|
|
|
|
|
gauges sync.Map
|
|
|
|
|
summaries sync.Map
|
|
|
|
|
counters sync.Map
|
2018-04-05 16:21:32 +00:00
|
|
|
|
expiration time.Duration
|
|
|
|
|
}
|
|
|
|
|
|
2020-08-11 16:17:43 +00:00
|
|
|
|
type PrometheusGauge struct {
|
|
|
|
|
prometheus.Gauge
|
|
|
|
|
updatedAt time.Time
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type PrometheusSummary struct {
|
|
|
|
|
prometheus.Summary
|
|
|
|
|
updatedAt time.Time
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type PrometheusCounter struct {
|
|
|
|
|
prometheus.Counter
|
|
|
|
|
updatedAt time.Time
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-05 16:21:32 +00:00
|
|
|
|
// NewPrometheusSink creates a new PrometheusSink using the default options.
|
|
|
|
|
func NewPrometheusSink() (*PrometheusSink, error) {
|
|
|
|
|
return NewPrometheusSinkFrom(DefaultPrometheusOpts)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// NewPrometheusSinkFrom creates a new PrometheusSink using the passed options.
|
|
|
|
|
func NewPrometheusSinkFrom(opts PrometheusOpts) (*PrometheusSink, error) {
|
|
|
|
|
sink := &PrometheusSink{
|
2020-07-23 18:37:33 +00:00
|
|
|
|
gauges: sync.Map{},
|
|
|
|
|
summaries: sync.Map{},
|
|
|
|
|
counters: sync.Map{},
|
2018-04-05 16:21:32 +00:00
|
|
|
|
expiration: opts.Expiration,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return sink, prometheus.Register(sink)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Describe is needed to meet the Collector interface.
|
|
|
|
|
func (p *PrometheusSink) Describe(c chan<- *prometheus.Desc) {
|
|
|
|
|
// We must emit some description otherwise an error is returned. This
|
|
|
|
|
// description isn't shown to the user!
|
|
|
|
|
prometheus.NewGauge(prometheus.GaugeOpts{Name: "Dummy", Help: "Dummy"}).Describe(c)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Collect meets the collection interface and allows us to enforce our expiration
|
|
|
|
|
// logic to clean up ephemeral metrics if their value haven't been set for a
|
|
|
|
|
// duration exceeding our allowed expiration time.
|
|
|
|
|
func (p *PrometheusSink) Collect(c chan<- prometheus.Metric) {
|
|
|
|
|
expire := p.expiration != 0
|
|
|
|
|
now := time.Now()
|
2020-07-23 18:37:33 +00:00
|
|
|
|
p.gauges.Range(func(k, v interface{}) bool {
|
2020-08-11 16:17:43 +00:00
|
|
|
|
if v != nil {
|
|
|
|
|
lastUpdate := v.(*PrometheusGauge).updatedAt
|
|
|
|
|
if expire && lastUpdate.Add(p.expiration).Before(now) {
|
|
|
|
|
p.gauges.Delete(k)
|
|
|
|
|
} else {
|
|
|
|
|
v.(*PrometheusGauge).Collect(c)
|
|
|
|
|
}
|
2018-04-05 16:21:32 +00:00
|
|
|
|
}
|
2020-07-23 18:37:33 +00:00
|
|
|
|
return true
|
|
|
|
|
})
|
|
|
|
|
p.summaries.Range(func(k, v interface{}) bool {
|
2020-08-11 16:17:43 +00:00
|
|
|
|
if v != nil {
|
|
|
|
|
lastUpdate := v.(*PrometheusSummary).updatedAt
|
|
|
|
|
if expire && lastUpdate.Add(p.expiration).Before(now) {
|
|
|
|
|
p.summaries.Delete(k)
|
|
|
|
|
} else {
|
|
|
|
|
v.(*PrometheusSummary).Collect(c)
|
|
|
|
|
}
|
2018-04-05 16:21:32 +00:00
|
|
|
|
}
|
2020-07-23 18:37:33 +00:00
|
|
|
|
return true
|
|
|
|
|
})
|
|
|
|
|
p.counters.Range(func(k, v interface{}) bool {
|
2020-08-11 16:17:43 +00:00
|
|
|
|
if v != nil {
|
|
|
|
|
lastUpdate := v.(*PrometheusCounter).updatedAt
|
|
|
|
|
if expire && lastUpdate.Add(p.expiration).Before(now) {
|
|
|
|
|
p.counters.Delete(k)
|
|
|
|
|
} else {
|
|
|
|
|
v.(*PrometheusCounter).Collect(c)
|
|
|
|
|
}
|
2018-04-05 16:21:32 +00:00
|
|
|
|
}
|
2020-07-23 18:37:33 +00:00
|
|
|
|
return true
|
|
|
|
|
})
|
2018-04-05 16:21:32 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-03-26 21:50:42 +00:00
|
|
|
|
var forbiddenChars = regexp.MustCompile("[ .=\\-/]")
|
2018-04-05 16:21:32 +00:00
|
|
|
|
|
|
|
|
|
func (p *PrometheusSink) flattenKey(parts []string, labels []metrics.Label) (string, string) {
|
|
|
|
|
key := strings.Join(parts, "_")
|
|
|
|
|
key = forbiddenChars.ReplaceAllString(key, "_")
|
|
|
|
|
|
|
|
|
|
hash := key
|
|
|
|
|
for _, label := range labels {
|
|
|
|
|
hash += fmt.Sprintf(";%s=%s", label.Name, label.Value)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return key, hash
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func prometheusLabels(labels []metrics.Label) prometheus.Labels {
|
|
|
|
|
l := make(prometheus.Labels)
|
|
|
|
|
for _, label := range labels {
|
|
|
|
|
l[label.Name] = label.Value
|
|
|
|
|
}
|
|
|
|
|
return l
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (p *PrometheusSink) SetGauge(parts []string, val float32) {
|
|
|
|
|
p.SetGaugeWithLabels(parts, val, nil)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (p *PrometheusSink) SetGaugeWithLabels(parts []string, val float32, labels []metrics.Label) {
|
|
|
|
|
key, hash := p.flattenKey(parts, labels)
|
2020-08-11 16:17:43 +00:00
|
|
|
|
pg, ok := p.gauges.Load(hash)
|
|
|
|
|
|
|
|
|
|
// The sync.Map underlying gauges stores pointers to our structs. If we need to make updates,
|
|
|
|
|
// rather than modifying the underlying value directly, which would be racy, we make a local
|
|
|
|
|
// copy by dereferencing the pointer we get back, making the appropriate changes, and then
|
|
|
|
|
// storing a pointer to our local copy. The underlying Prometheus types are threadsafe,
|
|
|
|
|
// so there's no issues there. It's possible for racy updates to occur to the updatedAt
|
|
|
|
|
// value, but since we're always setting it to time.Now(), it doesn't really matter.
|
|
|
|
|
if ok {
|
|
|
|
|
localGauge := *pg.(*PrometheusGauge)
|
|
|
|
|
localGauge.Set(float64(val))
|
|
|
|
|
localGauge.updatedAt = time.Now()
|
|
|
|
|
p.gauges.Store(hash, &localGauge)
|
|
|
|
|
} else {
|
|
|
|
|
g := prometheus.NewGauge(prometheus.GaugeOpts{
|
2018-04-05 16:21:32 +00:00
|
|
|
|
Name: key,
|
|
|
|
|
Help: key,
|
|
|
|
|
ConstLabels: prometheusLabels(labels),
|
|
|
|
|
})
|
2020-08-11 16:17:43 +00:00
|
|
|
|
g.Set(float64(val))
|
|
|
|
|
pg = &PrometheusGauge{
|
|
|
|
|
g, time.Now(),
|
|
|
|
|
}
|
|
|
|
|
p.gauges.Store(hash, pg)
|
2018-04-05 16:21:32 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (p *PrometheusSink) AddSample(parts []string, val float32) {
|
|
|
|
|
p.AddSampleWithLabels(parts, val, nil)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (p *PrometheusSink) AddSampleWithLabels(parts []string, val float32, labels []metrics.Label) {
|
|
|
|
|
key, hash := p.flattenKey(parts, labels)
|
2020-08-11 16:17:43 +00:00
|
|
|
|
ps, ok := p.summaries.Load(hash)
|
|
|
|
|
|
|
|
|
|
if ok {
|
|
|
|
|
localSummary := *ps.(*PrometheusSummary)
|
|
|
|
|
localSummary.Observe(float64(val))
|
|
|
|
|
localSummary.updatedAt = time.Now()
|
|
|
|
|
p.summaries.Store(hash, &localSummary)
|
|
|
|
|
} else {
|
|
|
|
|
s := prometheus.NewSummary(prometheus.SummaryOpts{
|
2018-04-05 16:21:32 +00:00
|
|
|
|
Name: key,
|
|
|
|
|
Help: key,
|
|
|
|
|
MaxAge: 10 * time.Second,
|
|
|
|
|
ConstLabels: prometheusLabels(labels),
|
2020-07-23 18:37:33 +00:00
|
|
|
|
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
|
2018-04-05 16:21:32 +00:00
|
|
|
|
})
|
2020-08-11 16:17:43 +00:00
|
|
|
|
s.Observe(float64(val))
|
|
|
|
|
ps = &PrometheusSummary{
|
|
|
|
|
s, time.Now(),
|
|
|
|
|
}
|
|
|
|
|
p.summaries.Store(hash, ps)
|
2018-04-05 16:21:32 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// EmitKey is not implemented. Prometheus doesn’t offer a type for which an
|
|
|
|
|
// arbitrary number of values is retained, as Prometheus works with a pull
|
|
|
|
|
// model, rather than a push model.
|
|
|
|
|
func (p *PrometheusSink) EmitKey(key []string, val float32) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (p *PrometheusSink) IncrCounter(parts []string, val float32) {
|
|
|
|
|
p.IncrCounterWithLabels(parts, val, nil)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (p *PrometheusSink) IncrCounterWithLabels(parts []string, val float32, labels []metrics.Label) {
|
|
|
|
|
key, hash := p.flattenKey(parts, labels)
|
2020-08-11 16:17:43 +00:00
|
|
|
|
pc, ok := p.counters.Load(hash)
|
|
|
|
|
|
|
|
|
|
if ok {
|
|
|
|
|
localCounter := *pc.(*PrometheusCounter)
|
|
|
|
|
localCounter.Add(float64(val))
|
|
|
|
|
localCounter.updatedAt = time.Now()
|
|
|
|
|
p.counters.Store(hash, &localCounter)
|
|
|
|
|
} else {
|
|
|
|
|
c := prometheus.NewCounter(prometheus.CounterOpts{
|
2018-04-05 16:21:32 +00:00
|
|
|
|
Name: key,
|
|
|
|
|
Help: key,
|
|
|
|
|
ConstLabels: prometheusLabels(labels),
|
|
|
|
|
})
|
2020-08-11 16:17:43 +00:00
|
|
|
|
c.Add(float64(val))
|
|
|
|
|
pc = &PrometheusCounter{
|
|
|
|
|
c, time.Now(),
|
|
|
|
|
}
|
|
|
|
|
p.counters.Store(hash, pc)
|
2020-07-23 18:37:33 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type PrometheusPushSink struct {
|
|
|
|
|
*PrometheusSink
|
|
|
|
|
pusher *push.Pusher
|
|
|
|
|
address string
|
|
|
|
|
pushInterval time.Duration
|
|
|
|
|
stopChan chan struct{}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func NewPrometheusPushSink(address string, pushIterval time.Duration, name string) (*PrometheusPushSink, error) {
|
|
|
|
|
promSink := &PrometheusSink{
|
|
|
|
|
gauges: sync.Map{},
|
|
|
|
|
summaries: sync.Map{},
|
|
|
|
|
counters: sync.Map{},
|
|
|
|
|
expiration: 60 * time.Second,
|
2018-04-05 16:21:32 +00:00
|
|
|
|
}
|
2020-07-23 18:37:33 +00:00
|
|
|
|
|
|
|
|
|
pusher := push.New(address, name).Collector(promSink)
|
|
|
|
|
|
|
|
|
|
sink := &PrometheusPushSink{
|
|
|
|
|
promSink,
|
|
|
|
|
pusher,
|
|
|
|
|
address,
|
|
|
|
|
pushIterval,
|
|
|
|
|
make(chan struct{}),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
sink.flushMetrics()
|
|
|
|
|
return sink, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (s *PrometheusPushSink) flushMetrics() {
|
|
|
|
|
ticker := time.NewTicker(s.pushInterval)
|
|
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
|
for {
|
|
|
|
|
select {
|
|
|
|
|
case <-ticker.C:
|
|
|
|
|
err := s.pusher.Push()
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf("[ERR] Error pushing to Prometheus! Err: %s", err)
|
|
|
|
|
}
|
|
|
|
|
case <-s.stopChan:
|
|
|
|
|
ticker.Stop()
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (s *PrometheusPushSink) Shutdown() {
|
|
|
|
|
close(s.stopChan)
|
2018-04-05 16:21:32 +00:00
|
|
|
|
}
|