From e5cdb702e5a664d9b739f28c2005ec19a7aa2e58 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Tue, 30 May 2023 14:43:29 -0400 Subject: [PATCH] hoststats: add package for collecting host statistics including cpu memory and disk usage (#17038) --- .changelog/17038.txt | 3 + agent/config/builder.go | 3 + agent/config/builder_test.go | 19 ++ agent/config/config.go | 1 + agent/config/runtime_test.go | 19 +- .../TestRuntimeConfig_Sanitize.golden | 1 + agent/config/testdata/full-config.hcl | 1 + agent/config/testdata/full-config.json | 1 + agent/setup.go | 21 +- lib/hoststats/collector.go | 189 ++++++++++++++++++ lib/hoststats/cpu.go | 85 ++++++++ lib/hoststats/cpu_test.go | 58 ++++++ lib/hoststats/host.go | 92 +++++++++ lib/hoststats/metrics.go | 79 ++++++++ lib/telemetry.go | 5 + .../docs/agent/config/config-files.mdx | 3 + website/content/docs/agent/telemetry.mdx | 29 +++ 17 files changed, 598 insertions(+), 11 deletions(-) create mode 100644 .changelog/17038.txt create mode 100644 lib/hoststats/collector.go create mode 100644 lib/hoststats/cpu.go create mode 100644 lib/hoststats/cpu_test.go create mode 100644 lib/hoststats/host.go create mode 100644 lib/hoststats/metrics.go diff --git a/.changelog/17038.txt b/.changelog/17038.txt new file mode 100644 index 000000000..b3a47f98a --- /dev/null +++ b/.changelog/17038.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent: add new metrics to track cpu disk and memory usage for server hosts (defaults to: enabled) +``` diff --git a/agent/config/builder.go b/agent/config/builder.go index 87ee22940..063771b0f 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1106,6 +1106,9 @@ func (b *builder) build() (rt RuntimeConfig, err error) { LocalProxyConfigResyncInterval: 30 * time.Second, } + // host metrics are enabled by default if consul is configured with HashiCorp Cloud Platform integration + rt.Telemetry.EnableHostMetrics = boolValWithDefault(c.Telemetry.EnableHostMetrics, rt.IsCloudEnabled()) + rt.TLS, err = b.buildTLSConfig(rt, c.TLS) if err != nil { return RuntimeConfig{}, err diff --git a/agent/config/builder_test.go b/agent/config/builder_test.go index 28d5b2972..3eb81fdee 100644 --- a/agent/config/builder_test.go +++ b/agent/config/builder_test.go @@ -556,3 +556,22 @@ func TestBuilder_parsePrefixFilter(t *testing.T) { } }) } + +func TestBuidler_hostMetricsWithCloud(t *testing.T) { + devMode := true + builderOpts := LoadOpts{ + DevMode: &devMode, + DefaultConfig: FileSource{ + Name: "test", + Format: "hcl", + Data: `cloud{ resource_id = "abc" client_id = "abc" client_secret = "abc"}`, + }, + } + + result, err := Load(builderOpts) + require.NoError(t, err) + require.Empty(t, result.Warnings) + cfg := result.RuntimeConfig + require.NotNil(t, cfg) + require.True(t, cfg.Telemetry.EnableHostMetrics) +} diff --git a/agent/config/config.go b/agent/config/config.go index a1f414529..e26d6edc4 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -691,6 +691,7 @@ type Telemetry struct { CirconusSubmissionInterval *string `mapstructure:"circonus_submission_interval" json:"circonus_submission_interval,omitempty"` CirconusSubmissionURL *string `mapstructure:"circonus_submission_url" json:"circonus_submission_url,omitempty"` DisableHostname *bool `mapstructure:"disable_hostname" json:"disable_hostname,omitempty"` + EnableHostMetrics *bool `mapstructure:"enable_host_metrics" json:"enable_host_metrics,omitempty"` DogstatsdAddr *string `mapstructure:"dogstatsd_addr" json:"dogstatsd_addr,omitempty"` DogstatsdTags []string `mapstructure:"dogstatsd_tags" json:"dogstatsd_tags,omitempty"` RetryFailedConfiguration *bool `mapstructure:"retry_failed_connection" json:"retry_failed_connection,omitempty"` diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 1064829cd..3b1a77b2c 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -46,6 +46,7 @@ type testCase struct { desc string args []string setup func() // TODO: accept a testing.T instead of panic + cleanup func() expected func(rt *RuntimeConfig) expectedErr string expectedWarnings []string @@ -2308,9 +2309,9 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { }, setup: func() { os.Setenv("HCP_RESOURCE_ID", "env-id") - t.Cleanup(func() { - os.Unsetenv("HCP_RESOURCE_ID") - }) + }, + cleanup: func() { + os.Unsetenv("HCP_RESOURCE_ID") }, expected: func(rt *RuntimeConfig) { rt.DataDir = dataDir @@ -2321,6 +2322,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { // server things rt.ServerMode = true + rt.Telemetry.EnableHostMetrics = true rt.TLS.ServerMode = true rt.LeaveOnTerm = false rt.SkipLeaveOnInt = true @@ -2337,9 +2339,9 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { }, setup: func() { os.Setenv("HCP_RESOURCE_ID", "env-id") - t.Cleanup(func() { - os.Unsetenv("HCP_RESOURCE_ID") - }) + }, + cleanup: func() { + os.Unsetenv("HCP_RESOURCE_ID") }, json: []string{`{ "cloud": { @@ -2360,6 +2362,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { // server things rt.ServerMode = true + rt.Telemetry.EnableHostMetrics = true rt.TLS.ServerMode = true rt.LeaveOnTerm = false rt.SkipLeaveOnInt = true @@ -6032,6 +6035,9 @@ func (tc testCase) run(format string, dataDir string) func(t *testing.T) { expected.ACLResolverSettings.EnterpriseMeta = *structs.NodeEnterpriseMetaInPartition(expected.PartitionOrDefault()) prototest.AssertDeepEqual(t, expected, actual, cmpopts.EquateEmpty()) + if tc.cleanup != nil { + tc.cleanup() + } } } @@ -6754,6 +6760,7 @@ func TestLoad_FullConfig(t *testing.T) { Expiration: 15 * time.Second, Name: "ftO6DySn", // notice this is the same as the metrics prefix }, + EnableHostMetrics: true, }, TLS: tlsutil.Config{ InternalRPC: tlsutil.ProtocolConfig{ diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden index c17636eef..334f5f8c8 100644 --- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden +++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden @@ -465,6 +465,7 @@ "DisableHostname": false, "DogstatsdAddr": "", "DogstatsdTags": [], + "EnableHostMetrics": false, "FilterDefault": false, "MetricsPrefix": "", "PrometheusOpts": { diff --git a/agent/config/testdata/full-config.hcl b/agent/config/testdata/full-config.hcl index c29c334b9..660e10360 100644 --- a/agent/config/testdata/full-config.hcl +++ b/agent/config/testdata/full-config.hcl @@ -690,6 +690,7 @@ telemetry { circonus_check_tags = "prvO4uBl" circonus_submission_interval = "DolzaflP" circonus_submission_url = "gTcbS93G" + enable_host_metrics = true disable_hostname = true dogstatsd_addr = "0wSndumK" dogstatsd_tags = [ "3N81zSUB","Xtj8AnXZ" ] diff --git a/agent/config/testdata/full-config.json b/agent/config/testdata/full-config.json index 7640394a4..52dab37bf 100644 --- a/agent/config/testdata/full-config.json +++ b/agent/config/testdata/full-config.json @@ -808,6 +808,7 @@ "circonus_check_tags": "prvO4uBl", "circonus_submission_interval": "DolzaflP", "circonus_submission_url": "gTcbS93G", + "enable_host_metrics": true, "disable_hostname": true, "dogstatsd_addr": "0wSndumK", "dogstatsd_tags": [ diff --git a/agent/setup.go b/agent/setup.go index 9ed993aaf..fba5c2b5d 100644 --- a/agent/setup.go +++ b/agent/setup.go @@ -4,6 +4,7 @@ package agent import ( + "context" "fmt" "io" "net" @@ -41,6 +42,7 @@ import ( "github.com/hashicorp/consul/agent/xds" "github.com/hashicorp/consul/ipaddr" "github.com/hashicorp/consul/lib" + "github.com/hashicorp/consul/lib/hoststats" "github.com/hashicorp/consul/logging" "github.com/hashicorp/consul/tlsutil" ) @@ -59,6 +61,7 @@ type BaseDeps struct { WatchedFiles []string deregisterBalancer, deregisterResolver func() + stopHostCollector context.CancelFunc } type ConfigLoader func(source config.Source) (config.LoadResult, error) @@ -117,6 +120,11 @@ func NewBaseDeps(configLoader ConfigLoader, logOut io.Writer, providedLogger hcl if err != nil { return d, fmt.Errorf("failed to initialize telemetry: %w", err) } + if !cfg.Telemetry.Disable && cfg.Telemetry.EnableHostMetrics { + ctx, cancel := context.WithCancel(context.Background()) + hoststats.NewCollector(ctx, d.Logger, cfg.DataDir) + d.stopHostCollector = cancel + } d.TLSConfigurator, err = tlsutil.NewConfigurator(cfg.TLS, d.Logger) if err != nil { @@ -214,11 +222,10 @@ func (bd BaseDeps) Close() { bd.AutoConfig.Stop() bd.MetricsConfig.Cancel() - if fn := bd.deregisterBalancer; fn != nil { - fn() - } - if fn := bd.deregisterResolver; fn != nil { - fn() + for _, fn := range []func(){bd.deregisterBalancer, bd.deregisterResolver, bd.stopHostCollector} { + if fn != nil { + fn() + } } } @@ -297,6 +304,10 @@ func getPrometheusDefs(cfg *config.RuntimeConfig, isServer bool) ([]prometheus.G serverGauges, } + if cfg.Telemetry.EnableHostMetrics { + gauges = append(gauges, hoststats.Gauges) + } + // TODO(ffmmm): conditionally add only leader specific metrics to gauges, counters, summaries, etc if isServer { gauges = append(gauges, diff --git a/lib/hoststats/collector.go b/lib/hoststats/collector.go new file mode 100644 index 000000000..c4c57b35c --- /dev/null +++ b/lib/hoststats/collector.go @@ -0,0 +1,189 @@ +package hoststats + +import ( + "context" + "fmt" + "math" + "runtime" + "sync" + "time" + + "github.com/armon/go-metrics" + "github.com/hashicorp/go-hclog" + "github.com/shirou/gopsutil/v3/disk" + "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v3/mem" +) + +// Collector collects host resource usage stats +type Collector struct { + numCores int + cpuCalculator map[string]*cpuStatsCalculator + hostStats *HostStats + hostStatsLock sync.RWMutex + dataDir string + + metrics Metrics + baseLabels []metrics.Label + + logger hclog.Logger +} + +// NewCollector returns a Collector. The dataDir is passed in +// so that we can present the disk related statistics for the mountpoint where the dataDir exists +func NewCollector(ctx context.Context, logger hclog.Logger, dataDir string, opts ...CollectorOption) *Collector { + logger = logger.Named("host_stats") + collector := initCollector(logger, dataDir) + go collector.loop(ctx) + return collector +} + +// initCollector initializes the Collector but does not start the collection loop +func initCollector(logger hclog.Logger, dataDir string, opts ...CollectorOption) *Collector { + numCores := runtime.NumCPU() + statsCalculator := make(map[string]*cpuStatsCalculator) + collector := &Collector{ + cpuCalculator: statsCalculator, + numCores: numCores, + logger: logger, + dataDir: dataDir, + } + + for _, opt := range opts { + opt(collector) + } + + if collector.metrics == nil { + collector.metrics = metrics.Default() + } + return collector +} + +func (c *Collector) loop(ctx context.Context) { + // Start collecting host stats right away and then keep collecting every + // collection interval + next := time.NewTimer(0) + defer next.Stop() + for { + select { + case <-next.C: + c.collect() + next.Reset(hostStatsCollectionInterval) + c.Stats().Emit(c.metrics, c.baseLabels) + + case <-ctx.Done(): + return + } + } +} + +// collect will collect stats related to resource usage of the host +func (c *Collector) collect() { + hs := &HostStats{Timestamp: time.Now().UTC().UnixNano()} + + // Determine up-time + uptime, err := host.Uptime() + if err != nil { + c.logger.Error("failed to collect uptime stats", "error", err) + uptime = 0 + } + hs.Uptime = uptime + + // Collect memory stats + mstats, err := c.collectMemoryStats() + if err != nil { + c.logger.Error("failed to collect memory stats", "error", err) + mstats = &MemoryStats{} + } + hs.Memory = mstats + + // Collect cpu stats + cpus, err := c.collectCPUStats() + if err != nil { + c.logger.Error("failed to collect cpu stats", "error", err) + cpus = []*CPUStats{} + } + hs.CPU = cpus + + // Collect disk stats + diskStats, err := c.collectDiskStats(c.dataDir) + if err != nil { + c.logger.Error("failed to collect dataDir disk stats", "error", err) + } + hs.DataDirStats = diskStats + + // Update the collected status object. + c.hostStatsLock.Lock() + c.hostStats = hs + c.hostStatsLock.Unlock() +} + +func (c *Collector) collectDiskStats(dir string) (*DiskStats, error) { + usage, err := disk.Usage(dir) + if err != nil { + return nil, fmt.Errorf("failed to collect disk usage stats: %w", err) + } + return c.toDiskStats(usage), nil +} + +func (c *Collector) collectMemoryStats() (*MemoryStats, error) { + memStats, err := mem.VirtualMemory() + if err != nil { + return nil, err + } + mem := &MemoryStats{ + Total: memStats.Total, + Available: memStats.Available, + Used: memStats.Used, + UsedPercent: memStats.UsedPercent, + Free: memStats.Free, + } + + return mem, nil +} + +// Stats returns the host stats that has been collected +func (c *Collector) Stats() *HostStats { + c.hostStatsLock.RLock() + defer c.hostStatsLock.RUnlock() + + if c.hostStats == nil { + return &HostStats{} + } + + return c.hostStats.Clone() +} + +// toDiskStats merges UsageStat and PartitionStat to create a DiskStat +func (c *Collector) toDiskStats(usage *disk.UsageStat) *DiskStats { + ds := DiskStats{ + Size: usage.Total, + Used: usage.Used, + Available: usage.Free, + UsedPercent: usage.UsedPercent, + InodesUsedPercent: usage.InodesUsedPercent, + Path: usage.Path, + } + if math.IsNaN(ds.UsedPercent) { + ds.UsedPercent = 0.0 + } + if math.IsNaN(ds.InodesUsedPercent) { + ds.InodesUsedPercent = 0.0 + } + + return &ds +} + +type CollectorOption func(c *Collector) + +func WithMetrics(m *metrics.Metrics) CollectorOption { + return func(c *Collector) { + c.metrics = m + } +} + +func WithBaseLabels(labels []metrics.Label) CollectorOption { + return func(c *Collector) { + c.baseLabels = labels + } +} diff --git a/lib/hoststats/cpu.go b/lib/hoststats/cpu.go new file mode 100644 index 000000000..45633b40d --- /dev/null +++ b/lib/hoststats/cpu.go @@ -0,0 +1,85 @@ +package hoststats + +import ( + "math" + + "github.com/shirou/gopsutil/v3/cpu" +) + +// cpuStatsCalculator calculates cpu usage percentages +type cpuStatsCalculator struct { + prev cpu.TimesStat + prevBusy float64 + prevTotal float64 +} + +// calculate the current cpu usage percentages. +// Since the cpu.TimesStat captures the total time a cpu spent in various states +// this function tracks the last seen stat and derives each cpu state's utilization +// as a percentage of the total change in cpu time between calls. +// The first time calculate is called CPUStats will report %100 idle +// usage since there is not a previous value to calculate against +func (h *cpuStatsCalculator) calculate(times cpu.TimesStat) *CPUStats { + + // sum all none idle counters to get the total busy cpu time + currentBusy := times.User + times.System + times.Nice + times.Iowait + times.Irq + + times.Softirq + times.Steal + times.Guest + times.GuestNice + // sum of the total cpu time + currentTotal := currentBusy + times.Idle + + // calculate how much cpu time has passed since last calculation + deltaTotal := currentTotal - h.prevTotal + + stats := &CPUStats{ + CPU: times.CPU, + + // calculate each percentage as the ratio of the change + // in each state's time to the total change in cpu time + Idle: ((times.Idle - h.prev.Idle) / deltaTotal) * 100, + User: ((times.User - h.prev.User) / deltaTotal) * 100, + System: ((times.System - h.prev.System) / deltaTotal) * 100, + Iowait: ((times.Iowait - h.prev.Iowait) / deltaTotal) * 100, + Total: ((currentBusy - h.prevBusy) / deltaTotal) * 100, + } + + // Protect against any invalid values + if math.IsNaN(stats.Idle) || math.IsInf(stats.Idle, 0) { + stats.Idle = 100.0 + } + if math.IsNaN(stats.User) || math.IsInf(stats.User, 0) { + stats.User = 0.0 + } + if math.IsNaN(stats.System) || math.IsInf(stats.System, 0) { + stats.System = 0.0 + } + if math.IsNaN(stats.Iowait) || math.IsInf(stats.Iowait, 0) { + stats.Iowait = 0.0 + } + if math.IsNaN(stats.Total) || math.IsInf(stats.Total, 0) { + stats.Total = 0.0 + } + + h.prev = times + h.prevTotal = currentTotal + h.prevBusy = currentBusy + return stats +} + +func (c *Collector) collectCPUStats() (cpus []*CPUStats, err error) { + + cpuStats, err := cpu.Times(true) + if err != nil { + return nil, err + } + cs := make([]*CPUStats, len(cpuStats)) + for idx, cpuStat := range cpuStats { + percentCalculator, ok := c.cpuCalculator[cpuStat.CPU] + if !ok { + percentCalculator = &cpuStatsCalculator{} + c.cpuCalculator[cpuStat.CPU] = percentCalculator + } + cs[idx] = percentCalculator.calculate(cpuStat) + } + + return cs, nil +} diff --git a/lib/hoststats/cpu_test.go b/lib/hoststats/cpu_test.go new file mode 100644 index 000000000..5d5efbe97 --- /dev/null +++ b/lib/hoststats/cpu_test.go @@ -0,0 +1,58 @@ +package hoststats + +import ( + "math" + "os" + "testing" + "time" + + "github.com/hashicorp/consul/sdk/testutil" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHostStats_CPU(t *testing.T) { + logger := testutil.Logger(t) + cwd, err := os.Getwd() + assert.Nil(t, err) + hs := initCollector(logger, cwd) + + // Collect twice so we can calculate percents we need to generate some work + // so that the cpu values change + hs.collect() + for begin := time.Now(); time.Now().Sub(begin) < 100*time.Millisecond; { + } + hs.collect() + stats := hs.Stats() + assert.NotZero(t, len(stats.CPU)) + + for _, cpu := range stats.CPU { + assert.False(t, math.IsNaN(cpu.Idle)) + assert.False(t, math.IsNaN(cpu.Total)) + assert.False(t, math.IsNaN(cpu.System)) + assert.False(t, math.IsNaN(cpu.User)) + + assert.False(t, math.IsInf(cpu.Idle, 0)) + assert.False(t, math.IsInf(cpu.Total, 0)) + assert.False(t, math.IsInf(cpu.System, 0)) + assert.False(t, math.IsInf(cpu.User, 0)) + } +} + +func TestCpuStatsCalculator_Nan(t *testing.T) { + times := cpu.TimesStat{ + User: 0.0, + Idle: 100.0, + System: 0.0, + } + + calculator := &cpuStatsCalculator{} + calculator.calculate(times) + stats := calculator.calculate(times) + require.Equal(t, 100.0, stats.Idle) + require.Zero(t, stats.User) + require.Zero(t, stats.System) + require.Zero(t, stats.Iowait) + require.Zero(t, stats.Total) +} diff --git a/lib/hoststats/host.go b/lib/hoststats/host.go new file mode 100644 index 000000000..426cf43ea --- /dev/null +++ b/lib/hoststats/host.go @@ -0,0 +1,92 @@ +package hoststats + +import ( + "time" + + "github.com/armon/go-metrics" +) + +var hostStatsCollectionInterval = 10 * time.Second + +// HostStats represents resource usage hoststats of the host running a Consul agent +type HostStats struct { + Memory *MemoryStats + CPU []*CPUStats + DataDirStats *DiskStats + Uptime uint64 + Timestamp int64 +} + +func (hs *HostStats) Clone() *HostStats { + clone := &HostStats{} + *clone = *hs + return clone +} + +func (hs *HostStats) Emit(sink Metrics, baseLabels []metrics.Label) { + + if hs.Memory != nil { + sink.SetGaugeWithLabels([]string{"host", "memory", "total"}, float32(hs.Memory.Total), baseLabels) + sink.SetGaugeWithLabels([]string{"host", "memory", "available"}, float32(hs.Memory.Available), baseLabels) + sink.SetGaugeWithLabels([]string{"host", "memory", "used"}, float32(hs.Memory.Used), baseLabels) + sink.SetGaugeWithLabels([]string{"host", "memory", "used_percent"}, float32(hs.Memory.UsedPercent), baseLabels) + sink.SetGaugeWithLabels([]string{"host", "memory", "free"}, float32(hs.Memory.Free), baseLabels) + } + + for _, cpu := range hs.CPU { + labels := append(baseLabels, metrics.Label{ + Name: "cpu", + Value: cpu.CPU, + }) + + sink.SetGaugeWithLabels([]string{"host", "cpu", "total"}, float32(cpu.Total), labels) + sink.SetGaugeWithLabels([]string{"host", "cpu", "user"}, float32(cpu.User), labels) + sink.SetGaugeWithLabels([]string{"host", "cpu", "idle"}, float32(cpu.Idle), labels) + sink.SetGaugeWithLabels([]string{"host", "cpu", "iowait"}, float32(cpu.Iowait), labels) + sink.SetGaugeWithLabels([]string{"host", "cpu", "system"}, float32(cpu.System), labels) + } + + if hs.DataDirStats != nil { + diskLabels := append(baseLabels, metrics.Label{ + Name: "path", + Value: hs.DataDirStats.Path, + }) + + sink.SetGaugeWithLabels([]string{"host", "disk", "size"}, float32(hs.DataDirStats.Size), diskLabels) + sink.SetGaugeWithLabels([]string{"host", "disk", "used"}, float32(hs.DataDirStats.Used), diskLabels) + sink.SetGaugeWithLabels([]string{"host", "disk", "available"}, float32(hs.DataDirStats.Available), diskLabels) + sink.SetGaugeWithLabels([]string{"host", "disk", "used_percent"}, float32(hs.DataDirStats.UsedPercent), diskLabels) + sink.SetGaugeWithLabels([]string{"host", "disk", "inodes_percent"}, float32(hs.DataDirStats.InodesUsedPercent), diskLabels) + } + + sink.SetGaugeWithLabels([]string{"host", "uptime"}, float32(hs.Uptime), baseLabels) +} + +// CPUStats represents hoststats related to cpu usage +type CPUStats struct { + CPU string + User float64 + System float64 + Idle float64 + Iowait float64 + Total float64 +} + +// MemoryStats represents hoststats related to virtual memory usage +type MemoryStats struct { + Total uint64 + Available uint64 + Used uint64 + UsedPercent float64 + Free uint64 +} + +// DiskStats represents hoststats related to disk usage +type DiskStats struct { + Path string + Size uint64 + Used uint64 + Available uint64 + UsedPercent float64 + InodesUsedPercent float64 +} diff --git a/lib/hoststats/metrics.go b/lib/hoststats/metrics.go new file mode 100644 index 000000000..c89d40b81 --- /dev/null +++ b/lib/hoststats/metrics.go @@ -0,0 +1,79 @@ +package hoststats + +import ( + "github.com/armon/go-metrics" + "github.com/armon/go-metrics/prometheus" +) + +// Metrics defines an interface for the methods used to emit data to the go-metrics library. +// `metrics.Default()` should always satisfy this interface. +type Metrics interface { + SetGaugeWithLabels(key []string, val float32, labels []metrics.Label) +} + +var Gauges = []prometheus.GaugeDefinition{ + { + Name: []string{"host", "memory", "total"}, + Help: "Total physical memory in bytes", + }, + { + Name: []string{"host", "memory", "available"}, + Help: "Available physical memory in bytes", + }, + { + Name: []string{"host", "memory", "free"}, + Help: "Free physical memory in bytes", + }, + { + Name: []string{"host", "memory", "used"}, + Help: "Used physical memory in bytes", + }, + { + Name: []string{"host", "memory", "used_percent"}, + Help: "Percentage of physical memory in use", + }, + { + Name: []string{"host", "cpu", "total"}, + Help: "Total cpu utilization", + }, + { + Name: []string{"host", "cpu", "user"}, + Help: "User cpu utilization", + }, + { + Name: []string{"host", "cpu", "idle"}, + Help: "Idle cpu utilization", + }, + { + Name: []string{"host", "cpu", "iowait"}, + Help: "Iowait cpu utilization", + }, + { + Name: []string{"host", "cpu", "system"}, + Help: "System cpu utilization", + }, + { + Name: []string{"host", "disk", "size"}, + Help: "Size of disk in bytes", + }, + { + Name: []string{"host", "disk", "used"}, + Help: "Disk usage in bytes", + }, + { + Name: []string{"host", "disk", "available"}, + Help: "Available bytes on disk", + }, + { + Name: []string{"host", "disk", "used_percent"}, + Help: "Percentage of disk space usage", + }, + { + Name: []string{"host", "disk", "inodes_percent"}, + Help: "Percentage of disk inodes usage", + }, + { + Name: []string{"host", "uptime"}, + Help: "System uptime", + }, +} diff --git a/lib/telemetry.go b/lib/telemetry.go index 2d87707c3..b66ec721b 100644 --- a/lib/telemetry.go +++ b/lib/telemetry.go @@ -204,6 +204,11 @@ type TelemetryConfig struct { // hcl: telemetry { statsite_address = string } StatsiteAddr string `json:"statsite_address,omitempty" mapstructure:"statsite_address"` + // EnableHostMetrics will enable metrics collected about the host system such as cpu memory and disk usage. + // + // hcl: telemetry { enable_host_metrics = (true|false) } + EnableHostMetrics bool `json:"enable_host_metrics,omitempty" mapstructure:"enable_host_metrics"` + // PrometheusOpts provides configuration for the PrometheusSink. Currently the only configuration // we acquire from hcl is the retention time. We also use definition slices that are set in agent setup // before being passed to InitTelemmetry. diff --git a/website/content/docs/agent/config/config-files.mdx b/website/content/docs/agent/config/config-files.mdx index 3d6efc68f..d7ac6923e 100644 --- a/website/content/docs/agent/config/config-files.mdx +++ b/website/content/docs/agent/config/config-files.mdx @@ -1831,6 +1831,9 @@ subsystem that provides Consul's service mesh capabilities. of global tags that will be added to all telemetry packets sent to DogStatsD. It is a list of strings, where each string looks like "my_tag_name:my_tag_value". + - `enable_host_metrics` ((#telemetry-enable_host_metrics)) + This enables reporting of host metrics about system resources, defaults to false. + - `filter_default` ((#telemetry-filter_default)) This controls whether to allow metrics that have not been specified by the filter. Defaults to `true`, which will allow all metrics when no filters are provided. diff --git a/website/content/docs/agent/telemetry.mdx b/website/content/docs/agent/telemetry.mdx index 27626d9be..5e1b6b8a8 100644 --- a/website/content/docs/agent/telemetry.mdx +++ b/website/content/docs/agent/telemetry.mdx @@ -755,3 +755,32 @@ Consul attaches the following labels to metric values. | `peer_id` | The ID of a peer connected to the reporting cluster or leader. | Any UUID | | `partition` | Name of the partition that the peering is created in. | Any defined partition name in the cluster | +## Server Host Metrics + +Consul servers can report the following metrics about the host's system resources. +This feature must be enabled in the [agent telemetry configuration](/consul/docs/agent/config/config-files#telemetry-enable_host_metrics). +Note that if the Consul server is operating inside a container these metrics +still report host resource usage and do not report any resource limits placed +on the container. + +**Requirements:** +- Consul 1.15.3+ + +| Metric | Description | Unit | Type | +| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | ------- | +| `consul.host.memory.total` | The total physical memory in bytes | mixed | mixed | +| `consul.host.memory.available` | The available physical memory in bytes | mixed | mixed | +| `consul.host.memory.free` | The free physical memory in bytes | mixed | mixed | +| `consul.host.memory.used` | The used physical memory in bytes | mixed | mixed | +| `consul.host.memory.used_percent` | The used physical memory as a percentage of total physical memory | mixed | mixed | +| `consul.host.cpu.total` | The host's total cpu utilization +| `consul.host.cpu.user` | The cpu utilization in user space +| `consul.host.cpu.idle` | The cpu utilization in idle state +| `consul.host.cpu.iowait` | The cpu utilization in iowait state +| `consul.host.cpu.system` | The cpu utilization in system space +| `consul.host.disk.size` | The size in bytes of the data_dir disk +| `consul.host.disk.used` | The number of bytes used on the data_dir disk +| `consul.host.disk.available` | The number of bytes available on the data_dir disk +| `consul.host.disk.used_percent` | The percentage of disk space used on the data_dir disk +| `consul.host.disk.inodes_percent` | The percentage of inode usage on the data_dir disk +| `consul.host.uptime` | The uptime of the host in seconds