hoststats: add package for collecting host statistics including cpu memory and disk usage (#17038)
This commit is contained in:
Nick Ethier
GitHub
parent
265c50b1dc
commit
e5cdb702e5
|
|
@ -0,0 +1,3 @@
|
|||
```release-note:improvement
|
||||
agent: add new metrics to track cpu disk and memory usage for server hosts (defaults to: enabled)
|
||||
```
|
||||
|
|
@ -1106,6 +1106,9 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
|
|||
LocalProxyConfigResyncInterval: 30 * time.Second,
|
||||
}
|
||||
|
||||
// host metrics are enabled by default if consul is configured with HashiCorp Cloud Platform integration
|
||||
rt.Telemetry.EnableHostMetrics = boolValWithDefault(c.Telemetry.EnableHostMetrics, rt.IsCloudEnabled())
|
||||
|
||||
rt.TLS, err = b.buildTLSConfig(rt, c.TLS)
|
||||
if err != nil {
|
||||
return RuntimeConfig{}, err
|
||||
|
|
|
|||
|
|
@ -556,3 +556,22 @@ func TestBuilder_parsePrefixFilter(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuidler_hostMetricsWithCloud(t *testing.T) {
|
||||
devMode := true
|
||||
builderOpts := LoadOpts{
|
||||
DevMode: &devMode,
|
||||
DefaultConfig: FileSource{
|
||||
Name: "test",
|
||||
Format: "hcl",
|
||||
Data: `cloud{ resource_id = "abc" client_id = "abc" client_secret = "abc"}`,
|
||||
},
|
||||
}
|
||||
|
||||
result, err := Load(builderOpts)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, result.Warnings)
|
||||
cfg := result.RuntimeConfig
|
||||
require.NotNil(t, cfg)
|
||||
require.True(t, cfg.Telemetry.EnableHostMetrics)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -691,6 +691,7 @@ type Telemetry struct {
|
|||
CirconusSubmissionInterval *string `mapstructure:"circonus_submission_interval" json:"circonus_submission_interval,omitempty"`
|
||||
CirconusSubmissionURL *string `mapstructure:"circonus_submission_url" json:"circonus_submission_url,omitempty"`
|
||||
DisableHostname *bool `mapstructure:"disable_hostname" json:"disable_hostname,omitempty"`
|
||||
EnableHostMetrics *bool `mapstructure:"enable_host_metrics" json:"enable_host_metrics,omitempty"`
|
||||
DogstatsdAddr *string `mapstructure:"dogstatsd_addr" json:"dogstatsd_addr,omitempty"`
|
||||
DogstatsdTags []string `mapstructure:"dogstatsd_tags" json:"dogstatsd_tags,omitempty"`
|
||||
RetryFailedConfiguration *bool `mapstructure:"retry_failed_connection" json:"retry_failed_connection,omitempty"`
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ type testCase struct {
|
|||
desc string
|
||||
args []string
|
||||
setup func() // TODO: accept a testing.T instead of panic
|
||||
cleanup func()
|
||||
expected func(rt *RuntimeConfig)
|
||||
expectedErr string
|
||||
expectedWarnings []string
|
||||
|
|
@ -2308,9 +2309,9 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
|
|||
},
|
||||
setup: func() {
|
||||
os.Setenv("HCP_RESOURCE_ID", "env-id")
|
||||
t.Cleanup(func() {
|
||||
},
|
||||
cleanup: func() {
|
||||
os.Unsetenv("HCP_RESOURCE_ID")
|
||||
})
|
||||
},
|
||||
expected: func(rt *RuntimeConfig) {
|
||||
rt.DataDir = dataDir
|
||||
|
|
@ -2321,6 +2322,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
|
|||
|
||||
// server things
|
||||
rt.ServerMode = true
|
||||
rt.Telemetry.EnableHostMetrics = true
|
||||
rt.TLS.ServerMode = true
|
||||
rt.LeaveOnTerm = false
|
||||
rt.SkipLeaveOnInt = true
|
||||
|
|
@ -2337,9 +2339,9 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
|
|||
},
|
||||
setup: func() {
|
||||
os.Setenv("HCP_RESOURCE_ID", "env-id")
|
||||
t.Cleanup(func() {
|
||||
},
|
||||
cleanup: func() {
|
||||
os.Unsetenv("HCP_RESOURCE_ID")
|
||||
})
|
||||
},
|
||||
json: []string{`{
|
||||
"cloud": {
|
||||
|
|
@ -2360,6 +2362,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
|
|||
|
||||
// server things
|
||||
rt.ServerMode = true
|
||||
rt.Telemetry.EnableHostMetrics = true
|
||||
rt.TLS.ServerMode = true
|
||||
rt.LeaveOnTerm = false
|
||||
rt.SkipLeaveOnInt = true
|
||||
|
|
@ -6032,6 +6035,9 @@ func (tc testCase) run(format string, dataDir string) func(t *testing.T) {
|
|||
expected.ACLResolverSettings.EnterpriseMeta = *structs.NodeEnterpriseMetaInPartition(expected.PartitionOrDefault())
|
||||
|
||||
prototest.AssertDeepEqual(t, expected, actual, cmpopts.EquateEmpty())
|
||||
if tc.cleanup != nil {
|
||||
tc.cleanup()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -6754,6 +6760,7 @@ func TestLoad_FullConfig(t *testing.T) {
|
|||
Expiration: 15 * time.Second,
|
||||
Name: "ftO6DySn", // notice this is the same as the metrics prefix
|
||||
},
|
||||
EnableHostMetrics: true,
|
||||
},
|
||||
TLS: tlsutil.Config{
|
||||
InternalRPC: tlsutil.ProtocolConfig{
|
||||
|
|
|
|||
|
|
@ -465,6 +465,7 @@
|
|||
"DisableHostname": false,
|
||||
"DogstatsdAddr": "",
|
||||
"DogstatsdTags": [],
|
||||
"EnableHostMetrics": false,
|
||||
"FilterDefault": false,
|
||||
"MetricsPrefix": "",
|
||||
"PrometheusOpts": {
|
||||
|
|
|
|||
|
|
@ -690,6 +690,7 @@ telemetry {
|
|||
circonus_check_tags = "prvO4uBl"
|
||||
circonus_submission_interval = "DolzaflP"
|
||||
circonus_submission_url = "gTcbS93G"
|
||||
enable_host_metrics = true
|
||||
disable_hostname = true
|
||||
dogstatsd_addr = "0wSndumK"
|
||||
dogstatsd_tags = [ "3N81zSUB","Xtj8AnXZ" ]
|
||||
|
|
|
|||
|
|
@ -808,6 +808,7 @@
|
|||
"circonus_check_tags": "prvO4uBl",
|
||||
"circonus_submission_interval": "DolzaflP",
|
||||
"circonus_submission_url": "gTcbS93G",
|
||||
"enable_host_metrics": true,
|
||||
"disable_hostname": true,
|
||||
"dogstatsd_addr": "0wSndumK",
|
||||
"dogstatsd_tags": [
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
|
|
@ -41,6 +42,7 @@ import (
|
|||
"github.com/hashicorp/consul/agent/xds"
|
||||
"github.com/hashicorp/consul/ipaddr"
|
||||
"github.com/hashicorp/consul/lib"
|
||||
"github.com/hashicorp/consul/lib/hoststats"
|
||||
"github.com/hashicorp/consul/logging"
|
||||
"github.com/hashicorp/consul/tlsutil"
|
||||
)
|
||||
|
|
@ -59,6 +61,7 @@ type BaseDeps struct {
|
|||
WatchedFiles []string
|
||||
|
||||
deregisterBalancer, deregisterResolver func()
|
||||
stopHostCollector context.CancelFunc
|
||||
}
|
||||
|
||||
type ConfigLoader func(source config.Source) (config.LoadResult, error)
|
||||
|
|
@ -117,6 +120,11 @@ func NewBaseDeps(configLoader ConfigLoader, logOut io.Writer, providedLogger hcl
|
|||
if err != nil {
|
||||
return d, fmt.Errorf("failed to initialize telemetry: %w", err)
|
||||
}
|
||||
if !cfg.Telemetry.Disable && cfg.Telemetry.EnableHostMetrics {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
hoststats.NewCollector(ctx, d.Logger, cfg.DataDir)
|
||||
d.stopHostCollector = cancel
|
||||
}
|
||||
|
||||
d.TLSConfigurator, err = tlsutil.NewConfigurator(cfg.TLS, d.Logger)
|
||||
if err != nil {
|
||||
|
|
@ -214,11 +222,10 @@ func (bd BaseDeps) Close() {
|
|||
bd.AutoConfig.Stop()
|
||||
bd.MetricsConfig.Cancel()
|
||||
|
||||
if fn := bd.deregisterBalancer; fn != nil {
|
||||
for _, fn := range []func(){bd.deregisterBalancer, bd.deregisterResolver, bd.stopHostCollector} {
|
||||
if fn != nil {
|
||||
fn()
|
||||
}
|
||||
if fn := bd.deregisterResolver; fn != nil {
|
||||
fn()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -297,6 +304,10 @@ func getPrometheusDefs(cfg *config.RuntimeConfig, isServer bool) ([]prometheus.G
|
|||
serverGauges,
|
||||
}
|
||||
|
||||
if cfg.Telemetry.EnableHostMetrics {
|
||||
gauges = append(gauges, hoststats.Gauges)
|
||||
}
|
||||
|
||||
// TODO(ffmmm): conditionally add only leader specific metrics to gauges, counters, summaries, etc
|
||||
if isServer {
|
||||
gauges = append(gauges,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,189 @@
|
|||
package hoststats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/armon/go-metrics"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/shirou/gopsutil/v3/disk"
|
||||
"github.com/shirou/gopsutil/v3/host"
|
||||
"github.com/shirou/gopsutil/v3/mem"
|
||||
)
|
||||
|
||||
// Collector collects host resource usage stats
|
||||
type Collector struct {
|
||||
numCores int
|
||||
cpuCalculator map[string]*cpuStatsCalculator
|
||||
hostStats *HostStats
|
||||
hostStatsLock sync.RWMutex
|
||||
dataDir string
|
||||
|
||||
metrics Metrics
|
||||
baseLabels []metrics.Label
|
||||
|
||||
logger hclog.Logger
|
||||
}
|
||||
|
||||
// NewCollector returns a Collector. The dataDir is passed in
|
||||
// so that we can present the disk related statistics for the mountpoint where the dataDir exists
|
||||
func NewCollector(ctx context.Context, logger hclog.Logger, dataDir string, opts ...CollectorOption) *Collector {
|
||||
logger = logger.Named("host_stats")
|
||||
collector := initCollector(logger, dataDir)
|
||||
go collector.loop(ctx)
|
||||
return collector
|
||||
}
|
||||
|
||||
// initCollector initializes the Collector but does not start the collection loop
|
||||
func initCollector(logger hclog.Logger, dataDir string, opts ...CollectorOption) *Collector {
|
||||
numCores := runtime.NumCPU()
|
||||
statsCalculator := make(map[string]*cpuStatsCalculator)
|
||||
collector := &Collector{
|
||||
cpuCalculator: statsCalculator,
|
||||
numCores: numCores,
|
||||
logger: logger,
|
||||
dataDir: dataDir,
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(collector)
|
||||
}
|
||||
|
||||
if collector.metrics == nil {
|
||||
collector.metrics = metrics.Default()
|
||||
}
|
||||
return collector
|
||||
}
|
||||
|
||||
func (c *Collector) loop(ctx context.Context) {
|
||||
// Start collecting host stats right away and then keep collecting every
|
||||
// collection interval
|
||||
next := time.NewTimer(0)
|
||||
defer next.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-next.C:
|
||||
c.collect()
|
||||
next.Reset(hostStatsCollectionInterval)
|
||||
c.Stats().Emit(c.metrics, c.baseLabels)
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// collect will collect stats related to resource usage of the host
|
||||
func (c *Collector) collect() {
|
||||
hs := &HostStats{Timestamp: time.Now().UTC().UnixNano()}
|
||||
|
||||
// Determine up-time
|
||||
uptime, err := host.Uptime()
|
||||
if err != nil {
|
||||
c.logger.Error("failed to collect uptime stats", "error", err)
|
||||
uptime = 0
|
||||
}
|
||||
hs.Uptime = uptime
|
||||
|
||||
// Collect memory stats
|
||||
mstats, err := c.collectMemoryStats()
|
||||
if err != nil {
|
||||
c.logger.Error("failed to collect memory stats", "error", err)
|
||||
mstats = &MemoryStats{}
|
||||
}
|
||||
hs.Memory = mstats
|
||||
|
||||
// Collect cpu stats
|
||||
cpus, err := c.collectCPUStats()
|
||||
if err != nil {
|
||||
c.logger.Error("failed to collect cpu stats", "error", err)
|
||||
cpus = []*CPUStats{}
|
||||
}
|
||||
hs.CPU = cpus
|
||||
|
||||
// Collect disk stats
|
||||
diskStats, err := c.collectDiskStats(c.dataDir)
|
||||
if err != nil {
|
||||
c.logger.Error("failed to collect dataDir disk stats", "error", err)
|
||||
}
|
||||
hs.DataDirStats = diskStats
|
||||
|
||||
// Update the collected status object.
|
||||
c.hostStatsLock.Lock()
|
||||
c.hostStats = hs
|
||||
c.hostStatsLock.Unlock()
|
||||
}
|
||||
|
||||
func (c *Collector) collectDiskStats(dir string) (*DiskStats, error) {
|
||||
usage, err := disk.Usage(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to collect disk usage stats: %w", err)
|
||||
}
|
||||
return c.toDiskStats(usage), nil
|
||||
}
|
||||
|
||||
func (c *Collector) collectMemoryStats() (*MemoryStats, error) {
|
||||
memStats, err := mem.VirtualMemory()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mem := &MemoryStats{
|
||||
Total: memStats.Total,
|
||||
Available: memStats.Available,
|
||||
Used: memStats.Used,
|
||||
UsedPercent: memStats.UsedPercent,
|
||||
Free: memStats.Free,
|
||||
}
|
||||
|
||||
return mem, nil
|
||||
}
|
||||
|
||||
// Stats returns the host stats that has been collected
|
||||
func (c *Collector) Stats() *HostStats {
|
||||
c.hostStatsLock.RLock()
|
||||
defer c.hostStatsLock.RUnlock()
|
||||
|
||||
if c.hostStats == nil {
|
||||
return &HostStats{}
|
||||
}
|
||||
|
||||
return c.hostStats.Clone()
|
||||
}
|
||||
|
||||
// toDiskStats merges UsageStat and PartitionStat to create a DiskStat
|
||||
func (c *Collector) toDiskStats(usage *disk.UsageStat) *DiskStats {
|
||||
ds := DiskStats{
|
||||
Size: usage.Total,
|
||||
Used: usage.Used,
|
||||
Available: usage.Free,
|
||||
UsedPercent: usage.UsedPercent,
|
||||
InodesUsedPercent: usage.InodesUsedPercent,
|
||||
Path: usage.Path,
|
||||
}
|
||||
if math.IsNaN(ds.UsedPercent) {
|
||||
ds.UsedPercent = 0.0
|
||||
}
|
||||
if math.IsNaN(ds.InodesUsedPercent) {
|
||||
ds.InodesUsedPercent = 0.0
|
||||
}
|
||||
|
||||
return &ds
|
||||
}
|
||||
|
||||
type CollectorOption func(c *Collector)
|
||||
|
||||
func WithMetrics(m *metrics.Metrics) CollectorOption {
|
||||
return func(c *Collector) {
|
||||
c.metrics = m
|
||||
}
|
||||
}
|
||||
|
||||
func WithBaseLabels(labels []metrics.Label) CollectorOption {
|
||||
return func(c *Collector) {
|
||||
c.baseLabels = labels
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
package hoststats
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"github.com/shirou/gopsutil/v3/cpu"
|
||||
)
|
||||
|
||||
// cpuStatsCalculator calculates cpu usage percentages
|
||||
type cpuStatsCalculator struct {
|
||||
prev cpu.TimesStat
|
||||
prevBusy float64
|
||||
prevTotal float64
|
||||
}
|
||||
|
||||
// calculate the current cpu usage percentages.
|
||||
// Since the cpu.TimesStat captures the total time a cpu spent in various states
|
||||
// this function tracks the last seen stat and derives each cpu state's utilization
|
||||
// as a percentage of the total change in cpu time between calls.
|
||||
// The first time calculate is called CPUStats will report %100 idle
|
||||
// usage since there is not a previous value to calculate against
|
||||
func (h *cpuStatsCalculator) calculate(times cpu.TimesStat) *CPUStats {
|
||||
|
||||
// sum all none idle counters to get the total busy cpu time
|
||||
currentBusy := times.User + times.System + times.Nice + times.Iowait + times.Irq +
|
||||
times.Softirq + times.Steal + times.Guest + times.GuestNice
|
||||
// sum of the total cpu time
|
||||
currentTotal := currentBusy + times.Idle
|
||||
|
||||
// calculate how much cpu time has passed since last calculation
|
||||
deltaTotal := currentTotal - h.prevTotal
|
||||
|
||||
stats := &CPUStats{
|
||||
CPU: times.CPU,
|
||||
|
||||
// calculate each percentage as the ratio of the change
|
||||
// in each state's time to the total change in cpu time
|
||||
Idle: ((times.Idle - h.prev.Idle) / deltaTotal) * 100,
|
||||
User: ((times.User - h.prev.User) / deltaTotal) * 100,
|
||||
System: ((times.System - h.prev.System) / deltaTotal) * 100,
|
||||
Iowait: ((times.Iowait - h.prev.Iowait) / deltaTotal) * 100,
|
||||
Total: ((currentBusy - h.prevBusy) / deltaTotal) * 100,
|
||||
}
|
||||
|
||||
// Protect against any invalid values
|
||||
if math.IsNaN(stats.Idle) || math.IsInf(stats.Idle, 0) {
|
||||
stats.Idle = 100.0
|
||||
}
|
||||
if math.IsNaN(stats.User) || math.IsInf(stats.User, 0) {
|
||||
stats.User = 0.0
|
||||
}
|
||||
if math.IsNaN(stats.System) || math.IsInf(stats.System, 0) {
|
||||
stats.System = 0.0
|
||||
}
|
||||
if math.IsNaN(stats.Iowait) || math.IsInf(stats.Iowait, 0) {
|
||||
stats.Iowait = 0.0
|
||||
}
|
||||
if math.IsNaN(stats.Total) || math.IsInf(stats.Total, 0) {
|
||||
stats.Total = 0.0
|
||||
}
|
||||
|
||||
h.prev = times
|
||||
h.prevTotal = currentTotal
|
||||
h.prevBusy = currentBusy
|
||||
return stats
|
||||
}
|
||||
|
||||
func (c *Collector) collectCPUStats() (cpus []*CPUStats, err error) {
|
||||
|
||||
cpuStats, err := cpu.Times(true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cs := make([]*CPUStats, len(cpuStats))
|
||||
for idx, cpuStat := range cpuStats {
|
||||
percentCalculator, ok := c.cpuCalculator[cpuStat.CPU]
|
||||
if !ok {
|
||||
percentCalculator = &cpuStatsCalculator{}
|
||||
c.cpuCalculator[cpuStat.CPU] = percentCalculator
|
||||
}
|
||||
cs[idx] = percentCalculator.calculate(cpuStat)
|
||||
}
|
||||
|
||||
return cs, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
package hoststats
|
||||
|
||||
import (
|
||||
"math"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/consul/sdk/testutil"
|
||||
"github.com/shirou/gopsutil/v3/cpu"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHostStats_CPU(t *testing.T) {
|
||||
logger := testutil.Logger(t)
|
||||
cwd, err := os.Getwd()
|
||||
assert.Nil(t, err)
|
||||
hs := initCollector(logger, cwd)
|
||||
|
||||
// Collect twice so we can calculate percents we need to generate some work
|
||||
// so that the cpu values change
|
||||
hs.collect()
|
||||
for begin := time.Now(); time.Now().Sub(begin) < 100*time.Millisecond; {
|
||||
}
|
||||
hs.collect()
|
||||
stats := hs.Stats()
|
||||
assert.NotZero(t, len(stats.CPU))
|
||||
|
||||
for _, cpu := range stats.CPU {
|
||||
assert.False(t, math.IsNaN(cpu.Idle))
|
||||
assert.False(t, math.IsNaN(cpu.Total))
|
||||
assert.False(t, math.IsNaN(cpu.System))
|
||||
assert.False(t, math.IsNaN(cpu.User))
|
||||
|
||||
assert.False(t, math.IsInf(cpu.Idle, 0))
|
||||
assert.False(t, math.IsInf(cpu.Total, 0))
|
||||
assert.False(t, math.IsInf(cpu.System, 0))
|
||||
assert.False(t, math.IsInf(cpu.User, 0))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCpuStatsCalculator_Nan(t *testing.T) {
|
||||
times := cpu.TimesStat{
|
||||
User: 0.0,
|
||||
Idle: 100.0,
|
||||
System: 0.0,
|
||||
}
|
||||
|
||||
calculator := &cpuStatsCalculator{}
|
||||
calculator.calculate(times)
|
||||
stats := calculator.calculate(times)
|
||||
require.Equal(t, 100.0, stats.Idle)
|
||||
require.Zero(t, stats.User)
|
||||
require.Zero(t, stats.System)
|
||||
require.Zero(t, stats.Iowait)
|
||||
require.Zero(t, stats.Total)
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
package hoststats
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/armon/go-metrics"
|
||||
)
|
||||
|
||||
var hostStatsCollectionInterval = 10 * time.Second
|
||||
|
||||
// HostStats represents resource usage hoststats of the host running a Consul agent
|
||||
type HostStats struct {
|
||||
Memory *MemoryStats
|
||||
CPU []*CPUStats
|
||||
DataDirStats *DiskStats
|
||||
Uptime uint64
|
||||
Timestamp int64
|
||||
}
|
||||
|
||||
func (hs *HostStats) Clone() *HostStats {
|
||||
clone := &HostStats{}
|
||||
*clone = *hs
|
||||
return clone
|
||||
}
|
||||
|
||||
func (hs *HostStats) Emit(sink Metrics, baseLabels []metrics.Label) {
|
||||
|
||||
if hs.Memory != nil {
|
||||
sink.SetGaugeWithLabels([]string{"host", "memory", "total"}, float32(hs.Memory.Total), baseLabels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "memory", "available"}, float32(hs.Memory.Available), baseLabels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "memory", "used"}, float32(hs.Memory.Used), baseLabels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "memory", "used_percent"}, float32(hs.Memory.UsedPercent), baseLabels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "memory", "free"}, float32(hs.Memory.Free), baseLabels)
|
||||
}
|
||||
|
||||
for _, cpu := range hs.CPU {
|
||||
labels := append(baseLabels, metrics.Label{
|
||||
Name: "cpu",
|
||||
Value: cpu.CPU,
|
||||
})
|
||||
|
||||
sink.SetGaugeWithLabels([]string{"host", "cpu", "total"}, float32(cpu.Total), labels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "cpu", "user"}, float32(cpu.User), labels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "cpu", "idle"}, float32(cpu.Idle), labels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "cpu", "iowait"}, float32(cpu.Iowait), labels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "cpu", "system"}, float32(cpu.System), labels)
|
||||
}
|
||||
|
||||
if hs.DataDirStats != nil {
|
||||
diskLabels := append(baseLabels, metrics.Label{
|
||||
Name: "path",
|
||||
Value: hs.DataDirStats.Path,
|
||||
})
|
||||
|
||||
sink.SetGaugeWithLabels([]string{"host", "disk", "size"}, float32(hs.DataDirStats.Size), diskLabels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "disk", "used"}, float32(hs.DataDirStats.Used), diskLabels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "disk", "available"}, float32(hs.DataDirStats.Available), diskLabels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "disk", "used_percent"}, float32(hs.DataDirStats.UsedPercent), diskLabels)
|
||||
sink.SetGaugeWithLabels([]string{"host", "disk", "inodes_percent"}, float32(hs.DataDirStats.InodesUsedPercent), diskLabels)
|
||||
}
|
||||
|
||||
sink.SetGaugeWithLabels([]string{"host", "uptime"}, float32(hs.Uptime), baseLabels)
|
||||
}
|
||||
|
||||
// CPUStats represents hoststats related to cpu usage
|
||||
type CPUStats struct {
|
||||
CPU string
|
||||
User float64
|
||||
System float64
|
||||
Idle float64
|
||||
Iowait float64
|
||||
Total float64
|
||||
}
|
||||
|
||||
// MemoryStats represents hoststats related to virtual memory usage
|
||||
type MemoryStats struct {
|
||||
Total uint64
|
||||
Available uint64
|
||||
Used uint64
|
||||
UsedPercent float64
|
||||
Free uint64
|
||||
}
|
||||
|
||||
// DiskStats represents hoststats related to disk usage
|
||||
type DiskStats struct {
|
||||
Path string
|
||||
Size uint64
|
||||
Used uint64
|
||||
Available uint64
|
||||
UsedPercent float64
|
||||
InodesUsedPercent float64
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
package hoststats
|
||||
|
||||
import (
|
||||
"github.com/armon/go-metrics"
|
||||
"github.com/armon/go-metrics/prometheus"
|
||||
)
|
||||
|
||||
// Metrics defines an interface for the methods used to emit data to the go-metrics library.
|
||||
// `metrics.Default()` should always satisfy this interface.
|
||||
type Metrics interface {
|
||||
SetGaugeWithLabels(key []string, val float32, labels []metrics.Label)
|
||||
}
|
||||
|
||||
var Gauges = []prometheus.GaugeDefinition{
|
||||
{
|
||||
Name: []string{"host", "memory", "total"},
|
||||
Help: "Total physical memory in bytes",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "memory", "available"},
|
||||
Help: "Available physical memory in bytes",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "memory", "free"},
|
||||
Help: "Free physical memory in bytes",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "memory", "used"},
|
||||
Help: "Used physical memory in bytes",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "memory", "used_percent"},
|
||||
Help: "Percentage of physical memory in use",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "cpu", "total"},
|
||||
Help: "Total cpu utilization",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "cpu", "user"},
|
||||
Help: "User cpu utilization",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "cpu", "idle"},
|
||||
Help: "Idle cpu utilization",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "cpu", "iowait"},
|
||||
Help: "Iowait cpu utilization",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "cpu", "system"},
|
||||
Help: "System cpu utilization",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "disk", "size"},
|
||||
Help: "Size of disk in bytes",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "disk", "used"},
|
||||
Help: "Disk usage in bytes",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "disk", "available"},
|
||||
Help: "Available bytes on disk",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "disk", "used_percent"},
|
||||
Help: "Percentage of disk space usage",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "disk", "inodes_percent"},
|
||||
Help: "Percentage of disk inodes usage",
|
||||
},
|
||||
{
|
||||
Name: []string{"host", "uptime"},
|
||||
Help: "System uptime",
|
||||
},
|
||||
}
|
||||
|
|
@ -204,6 +204,11 @@ type TelemetryConfig struct {
|
|||
// hcl: telemetry { statsite_address = string }
|
||||
StatsiteAddr string `json:"statsite_address,omitempty" mapstructure:"statsite_address"`
|
||||
|
||||
// EnableHostMetrics will enable metrics collected about the host system such as cpu memory and disk usage.
|
||||
//
|
||||
// hcl: telemetry { enable_host_metrics = (true|false) }
|
||||
EnableHostMetrics bool `json:"enable_host_metrics,omitempty" mapstructure:"enable_host_metrics"`
|
||||
|
||||
// PrometheusOpts provides configuration for the PrometheusSink. Currently the only configuration
|
||||
// we acquire from hcl is the retention time. We also use definition slices that are set in agent setup
|
||||
// before being passed to InitTelemmetry.
|
||||
|
|
|
|||
|
|
@ -1831,6 +1831,9 @@ subsystem that provides Consul's service mesh capabilities.
|
|||
of global tags that will be added to all telemetry packets sent to DogStatsD.
|
||||
It is a list of strings, where each string looks like "my_tag_name:my_tag_value".
|
||||
|
||||
- `enable_host_metrics` ((#telemetry-enable_host_metrics))
|
||||
This enables reporting of host metrics about system resources, defaults to false.
|
||||
|
||||
- `filter_default` ((#telemetry-filter_default))
|
||||
This controls whether to allow metrics that have not been specified by the filter.
|
||||
Defaults to `true`, which will allow all metrics when no filters are provided.
|
||||
|
|
|
|||
|
|
@ -755,3 +755,32 @@ Consul attaches the following labels to metric values.
|
|||
| `peer_id` | The ID of a peer connected to the reporting cluster or leader. | Any UUID |
|
||||
| `partition` | <EnterpriseAlert inline /> Name of the partition that the peering is created in. | Any defined partition name in the cluster |
|
||||
|
||||
## Server Host Metrics
|
||||
|
||||
Consul servers can report the following metrics about the host's system resources.
|
||||
This feature must be enabled in the [agent telemetry configuration](/consul/docs/agent/config/config-files#telemetry-enable_host_metrics).
|
||||
Note that if the Consul server is operating inside a container these metrics
|
||||
still report host resource usage and do not report any resource limits placed
|
||||
on the container.
|
||||
|
||||
**Requirements:**
|
||||
- Consul 1.15.3+
|
||||
|
||||
| Metric | Description | Unit | Type |
|
||||
| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | ------- |
|
||||
| `consul.host.memory.total` | The total physical memory in bytes | mixed | mixed |
|
||||
| `consul.host.memory.available` | The available physical memory in bytes | mixed | mixed |
|
||||
| `consul.host.memory.free` | The free physical memory in bytes | mixed | mixed |
|
||||
| `consul.host.memory.used` | The used physical memory in bytes | mixed | mixed |
|
||||
| `consul.host.memory.used_percent` | The used physical memory as a percentage of total physical memory | mixed | mixed |
|
||||
| `consul.host.cpu.total` | The host's total cpu utilization
|
||||
| `consul.host.cpu.user` | The cpu utilization in user space
|
||||
| `consul.host.cpu.idle` | The cpu utilization in idle state
|
||||
| `consul.host.cpu.iowait` | The cpu utilization in iowait state
|
||||
| `consul.host.cpu.system` | The cpu utilization in system space
|
||||
| `consul.host.disk.size` | The size in bytes of the data_dir disk
|
||||
| `consul.host.disk.used` | The number of bytes used on the data_dir disk
|
||||
| `consul.host.disk.available` | The number of bytes available on the data_dir disk
|
||||
| `consul.host.disk.used_percent` | The percentage of disk space used on the data_dir disk
|
||||
| `consul.host.disk.inodes_percent` | The percentage of inode usage on the data_dir disk
|
||||
| `consul.host.uptime` | The uptime of the host in seconds
|
||||
|
|
|
|||
Loading…
Reference in New Issue