hoststats: add package for collecting host statistics including cpu memory and disk usage (#17038)

.changelog/17038.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+agent: add new metrics to track cpu disk and memory usage for server hosts (defaults to: enabled)
+```

agent/config/builder.go

@@ -1106,6 +1106,9 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 		LocalProxyConfigResyncInterval:    30 * time.Second,
 	}
 
+	// host metrics are enabled by default if consul is configured with HashiCorp Cloud Platform integration
+	rt.Telemetry.EnableHostMetrics = boolValWithDefault(c.Telemetry.EnableHostMetrics, rt.IsCloudEnabled())
+
 	rt.TLS, err = b.buildTLSConfig(rt, c.TLS)
 	if err != nil {
 		return RuntimeConfig{}, err

agent/config/builder_test.go

@@ -556,3 +556,22 @@ func TestBuilder_parsePrefixFilter(t *testing.T) {
 		}
 	})
 }
+
+func TestBuidler_hostMetricsWithCloud(t *testing.T) {
+	devMode := true
+	builderOpts := LoadOpts{
+		DevMode: &devMode,
+		DefaultConfig: FileSource{
+			Name:   "test",
+			Format: "hcl",
+			Data:   `cloud{ resource_id = "abc" client_id = "abc" client_secret = "abc"}`,
+		},
+	}
+
+	result, err := Load(builderOpts)
+	require.NoError(t, err)
+	require.Empty(t, result.Warnings)
+	cfg := result.RuntimeConfig
+	require.NotNil(t, cfg)
+	require.True(t, cfg.Telemetry.EnableHostMetrics)
+}

agent/config/config.go

@@ -691,6 +691,7 @@ type Telemetry struct {
 	CirconusSubmissionInterval         *string  `mapstructure:"circonus_submission_interval" json:"circonus_submission_interval,omitempty"`
 	CirconusSubmissionURL              *string  `mapstructure:"circonus_submission_url" json:"circonus_submission_url,omitempty"`
 	DisableHostname                    *bool    `mapstructure:"disable_hostname" json:"disable_hostname,omitempty"`
+	EnableHostMetrics                  *bool    `mapstructure:"enable_host_metrics" json:"enable_host_metrics,omitempty"`
 	DogstatsdAddr                      *string  `mapstructure:"dogstatsd_addr" json:"dogstatsd_addr,omitempty"`
 	DogstatsdTags                      []string `mapstructure:"dogstatsd_tags" json:"dogstatsd_tags,omitempty"`
 	RetryFailedConfiguration           *bool    `mapstructure:"retry_failed_connection" json:"retry_failed_connection,omitempty"`

agent/config/runtime_test.go

@@ -46,6 +46,7 @@ type testCase struct {
 	desc             string
 	args             []string
 	setup            func() // TODO: accept a testing.T instead of panic
+	cleanup          func()
 	expected         func(rt *RuntimeConfig)
 	expectedErr      string
 	expectedWarnings []string
@@ -2308,9 +2309,9 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 		},
 		setup: func() {
 			os.Setenv("HCP_RESOURCE_ID", "env-id")
-			t.Cleanup(func() {
+		},
-				os.Unsetenv("HCP_RESOURCE_ID")
+		cleanup: func() {
-			})
+			os.Unsetenv("HCP_RESOURCE_ID")
 		},
 		expected: func(rt *RuntimeConfig) {
 			rt.DataDir = dataDir
@@ -2321,6 +2322,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 
 			// server things
 			rt.ServerMode = true
+			rt.Telemetry.EnableHostMetrics = true
 			rt.TLS.ServerMode = true
 			rt.LeaveOnTerm = false
 			rt.SkipLeaveOnInt = true
@@ -2337,9 +2339,9 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 		},
 		setup: func() {
 			os.Setenv("HCP_RESOURCE_ID", "env-id")
-			t.Cleanup(func() {
+		},
-				os.Unsetenv("HCP_RESOURCE_ID")
+		cleanup: func() {
-			})
+			os.Unsetenv("HCP_RESOURCE_ID")
 		},
 		json: []string{`{
 			  "cloud": {
@@ -2360,6 +2362,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 
 			// server things
 			rt.ServerMode = true
+			rt.Telemetry.EnableHostMetrics = true
 			rt.TLS.ServerMode = true
 			rt.LeaveOnTerm = false
 			rt.SkipLeaveOnInt = true
@@ -6032,6 +6035,9 @@ func (tc testCase) run(format string, dataDir string) func(t *testing.T) {
 		expected.ACLResolverSettings.EnterpriseMeta = *structs.NodeEnterpriseMetaInPartition(expected.PartitionOrDefault())
 
 		prototest.AssertDeepEqual(t, expected, actual, cmpopts.EquateEmpty())
+		if tc.cleanup != nil {
+			tc.cleanup()
+		}
 	}
 }
 
@@ -6754,6 +6760,7 @@ func TestLoad_FullConfig(t *testing.T) {
 				Expiration: 15 * time.Second,
 				Name:       "ftO6DySn", // notice this is the same as the metrics prefix
 			},
+			EnableHostMetrics: true,
 		},
 		TLS: tlsutil.Config{
 			InternalRPC: tlsutil.ProtocolConfig{

agent/config/testdata/TestRuntimeConfig_Sanitize.golden

@@ -465,6 +465,7 @@
         "DisableHostname": false,
         "DogstatsdAddr": "",
         "DogstatsdTags": [],
+        "EnableHostMetrics": false,
         "FilterDefault": false,
         "MetricsPrefix": "",
         "PrometheusOpts": {

agent/config/testdata/full-config.hcl

@@ -690,6 +690,7 @@ telemetry {
     circonus_check_tags = "prvO4uBl"
     circonus_submission_interval = "DolzaflP"
     circonus_submission_url = "gTcbS93G"
+    enable_host_metrics = true
     disable_hostname = true
     dogstatsd_addr = "0wSndumK"
     dogstatsd_tags = [ "3N81zSUB","Xtj8AnXZ" ]

agent/config/testdata/full-config.json

@@ -808,6 +808,7 @@
     "circonus_check_tags": "prvO4uBl",
     "circonus_submission_interval": "DolzaflP",
     "circonus_submission_url": "gTcbS93G",
+    "enable_host_metrics": true,
     "disable_hostname": true,
     "dogstatsd_addr": "0wSndumK",
     "dogstatsd_tags": [

agent/setup.go

@@ -4,6 +4,7 @@
 package agent
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net"
@@ -41,6 +42,7 @@ import (
 	"github.com/hashicorp/consul/agent/xds"
 	"github.com/hashicorp/consul/ipaddr"
 	"github.com/hashicorp/consul/lib"
+	"github.com/hashicorp/consul/lib/hoststats"
 	"github.com/hashicorp/consul/logging"
 	"github.com/hashicorp/consul/tlsutil"
 )
@@ -59,6 +61,7 @@ type BaseDeps struct {
 	WatchedFiles  []string
 
 	deregisterBalancer, deregisterResolver func()
+	stopHostCollector                      context.CancelFunc
 }
 
 type ConfigLoader func(source config.Source) (config.LoadResult, error)
@@ -117,6 +120,11 @@ func NewBaseDeps(configLoader ConfigLoader, logOut io.Writer, providedLogger hcl
 	if err != nil {
 		return d, fmt.Errorf("failed to initialize telemetry: %w", err)
 	}
+	if !cfg.Telemetry.Disable && cfg.Telemetry.EnableHostMetrics {
+		ctx, cancel := context.WithCancel(context.Background())
+		hoststats.NewCollector(ctx, d.Logger, cfg.DataDir)
+		d.stopHostCollector = cancel
+	}
 
 	d.TLSConfigurator, err = tlsutil.NewConfigurator(cfg.TLS, d.Logger)
 	if err != nil {
@@ -214,11 +222,10 @@ func (bd BaseDeps) Close() {
 	bd.AutoConfig.Stop()
 	bd.MetricsConfig.Cancel()
 
-	if fn := bd.deregisterBalancer; fn != nil {
+	for _, fn := range []func(){bd.deregisterBalancer, bd.deregisterResolver, bd.stopHostCollector} {
-		fn()
+		if fn != nil {
-	}
+			fn()
-	if fn := bd.deregisterResolver; fn != nil {
+		}
-		fn()
 	}
 }
 
@@ -297,6 +304,10 @@ func getPrometheusDefs(cfg *config.RuntimeConfig, isServer bool) ([]prometheus.G
 		serverGauges,
 	}
 
+	if cfg.Telemetry.EnableHostMetrics {
+		gauges = append(gauges, hoststats.Gauges)
+	}
+
 	// TODO(ffmmm): conditionally add only leader specific metrics to gauges, counters, summaries, etc
 	if isServer {
 		gauges = append(gauges,

lib/hoststats/collector.go

@@ -0,0 +1,189 @@
+package hoststats
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/armon/go-metrics"
+	"github.com/hashicorp/go-hclog"
+	"github.com/shirou/gopsutil/v3/disk"
+	"github.com/shirou/gopsutil/v3/host"
+	"github.com/shirou/gopsutil/v3/mem"
+)
+
+// Collector collects host resource usage stats
+type Collector struct {
+	numCores      int
+	cpuCalculator map[string]*cpuStatsCalculator
+	hostStats     *HostStats
+	hostStatsLock sync.RWMutex
+	dataDir       string
+
+	metrics    Metrics
+	baseLabels []metrics.Label
+
+	logger hclog.Logger
+}
+
+// NewCollector returns a Collector. The dataDir is passed in
+// so that we can present the disk related statistics for the mountpoint where the dataDir exists
+func NewCollector(ctx context.Context, logger hclog.Logger, dataDir string, opts ...CollectorOption) *Collector {
+	logger = logger.Named("host_stats")
+	collector := initCollector(logger, dataDir)
+	go collector.loop(ctx)
+	return collector
+}
+
+// initCollector initializes the Collector but does not start the collection loop
+func initCollector(logger hclog.Logger, dataDir string, opts ...CollectorOption) *Collector {
+	numCores := runtime.NumCPU()
+	statsCalculator := make(map[string]*cpuStatsCalculator)
+	collector := &Collector{
+		cpuCalculator: statsCalculator,
+		numCores:      numCores,
+		logger:        logger,
+		dataDir:       dataDir,
+	}
+
+	for _, opt := range opts {
+		opt(collector)
+	}
+
+	if collector.metrics == nil {
+		collector.metrics = metrics.Default()
+	}
+	return collector
+}
+
+func (c *Collector) loop(ctx context.Context) {
+	// Start collecting host stats right away and then keep collecting every
+	// collection interval
+	next := time.NewTimer(0)
+	defer next.Stop()
+	for {
+		select {
+		case <-next.C:
+			c.collect()
+			next.Reset(hostStatsCollectionInterval)
+			c.Stats().Emit(c.metrics, c.baseLabels)
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// collect will collect stats related to resource usage of the host
+func (c *Collector) collect() {
+	hs := &HostStats{Timestamp: time.Now().UTC().UnixNano()}
+
+	// Determine up-time
+	uptime, err := host.Uptime()
+	if err != nil {
+		c.logger.Error("failed to collect uptime stats", "error", err)
+		uptime = 0
+	}
+	hs.Uptime = uptime
+
+	// Collect memory stats
+	mstats, err := c.collectMemoryStats()
+	if err != nil {
+		c.logger.Error("failed to collect memory stats", "error", err)
+		mstats = &MemoryStats{}
+	}
+	hs.Memory = mstats
+
+	// Collect cpu stats
+	cpus, err := c.collectCPUStats()
+	if err != nil {
+		c.logger.Error("failed to collect cpu stats", "error", err)
+		cpus = []*CPUStats{}
+	}
+	hs.CPU = cpus
+
+	// Collect disk stats
+	diskStats, err := c.collectDiskStats(c.dataDir)
+	if err != nil {
+		c.logger.Error("failed to collect dataDir disk stats", "error", err)
+	}
+	hs.DataDirStats = diskStats
+
+	// Update the collected status object.
+	c.hostStatsLock.Lock()
+	c.hostStats = hs
+	c.hostStatsLock.Unlock()
+}
+
+func (c *Collector) collectDiskStats(dir string) (*DiskStats, error) {
+	usage, err := disk.Usage(dir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to collect disk usage stats: %w", err)
+	}
+	return c.toDiskStats(usage), nil
+}
+
+func (c *Collector) collectMemoryStats() (*MemoryStats, error) {
+	memStats, err := mem.VirtualMemory()
+	if err != nil {
+		return nil, err
+	}
+	mem := &MemoryStats{
+		Total:       memStats.Total,
+		Available:   memStats.Available,
+		Used:        memStats.Used,
+		UsedPercent: memStats.UsedPercent,
+		Free:        memStats.Free,
+	}
+
+	return mem, nil
+}
+
+// Stats returns the host stats that has been collected
+func (c *Collector) Stats() *HostStats {
+	c.hostStatsLock.RLock()
+	defer c.hostStatsLock.RUnlock()
+
+	if c.hostStats == nil {
+		return &HostStats{}
+	}
+
+	return c.hostStats.Clone()
+}
+
+// toDiskStats merges UsageStat and PartitionStat to create a DiskStat
+func (c *Collector) toDiskStats(usage *disk.UsageStat) *DiskStats {
+	ds := DiskStats{
+		Size:              usage.Total,
+		Used:              usage.Used,
+		Available:         usage.Free,
+		UsedPercent:       usage.UsedPercent,
+		InodesUsedPercent: usage.InodesUsedPercent,
+		Path:              usage.Path,
+	}
+	if math.IsNaN(ds.UsedPercent) {
+		ds.UsedPercent = 0.0
+	}
+	if math.IsNaN(ds.InodesUsedPercent) {
+		ds.InodesUsedPercent = 0.0
+	}
+
+	return &ds
+}
+
+type CollectorOption func(c *Collector)
+
+func WithMetrics(m *metrics.Metrics) CollectorOption {
+	return func(c *Collector) {
+		c.metrics = m
+	}
+}
+
+func WithBaseLabels(labels []metrics.Label) CollectorOption {
+	return func(c *Collector) {
+		c.baseLabels = labels
+	}
+}

lib/hoststats/cpu.go

@@ -0,0 +1,85 @@
+package hoststats
+
+import (
+	"math"
+
+	"github.com/shirou/gopsutil/v3/cpu"
+)
+
+// cpuStatsCalculator calculates cpu usage percentages
+type cpuStatsCalculator struct {
+	prev      cpu.TimesStat
+	prevBusy  float64
+	prevTotal float64
+}
+
+// calculate the current cpu usage percentages.
+// Since the cpu.TimesStat captures the total time a cpu spent in various states
+// this function tracks the last seen stat and derives each cpu state's utilization
+// as a percentage of the total change in cpu time between calls.
+// The first time calculate is called CPUStats will report %100 idle
+// usage since there is not a previous value to calculate against
+func (h *cpuStatsCalculator) calculate(times cpu.TimesStat) *CPUStats {
+
+	// sum all none idle counters to get the total busy cpu time
+	currentBusy := times.User + times.System + times.Nice + times.Iowait + times.Irq +
+		times.Softirq + times.Steal + times.Guest + times.GuestNice
+	// sum of the total cpu time
+	currentTotal := currentBusy + times.Idle
+
+	// calculate how much cpu time has passed since last calculation
+	deltaTotal := currentTotal - h.prevTotal
+
+	stats := &CPUStats{
+		CPU: times.CPU,
+
+		// calculate each percentage as the ratio of the change
+		// in each state's time to the total change in cpu time
+		Idle:   ((times.Idle - h.prev.Idle) / deltaTotal) * 100,
+		User:   ((times.User - h.prev.User) / deltaTotal) * 100,
+		System: ((times.System - h.prev.System) / deltaTotal) * 100,
+		Iowait: ((times.Iowait - h.prev.Iowait) / deltaTotal) * 100,
+		Total:  ((currentBusy - h.prevBusy) / deltaTotal) * 100,
+	}
+
+	// Protect against any invalid values
+	if math.IsNaN(stats.Idle) || math.IsInf(stats.Idle, 0) {
+		stats.Idle = 100.0
+	}
+	if math.IsNaN(stats.User) || math.IsInf(stats.User, 0) {
+		stats.User = 0.0
+	}
+	if math.IsNaN(stats.System) || math.IsInf(stats.System, 0) {
+		stats.System = 0.0
+	}
+	if math.IsNaN(stats.Iowait) || math.IsInf(stats.Iowait, 0) {
+		stats.Iowait = 0.0
+	}
+	if math.IsNaN(stats.Total) || math.IsInf(stats.Total, 0) {
+		stats.Total = 0.0
+	}
+
+	h.prev = times
+	h.prevTotal = currentTotal
+	h.prevBusy = currentBusy
+	return stats
+}
+
+func (c *Collector) collectCPUStats() (cpus []*CPUStats, err error) {
+
+	cpuStats, err := cpu.Times(true)
+	if err != nil {
+		return nil, err
+	}
+	cs := make([]*CPUStats, len(cpuStats))
+	for idx, cpuStat := range cpuStats {
+		percentCalculator, ok := c.cpuCalculator[cpuStat.CPU]
+		if !ok {
+			percentCalculator = &cpuStatsCalculator{}
+			c.cpuCalculator[cpuStat.CPU] = percentCalculator
+		}
+		cs[idx] = percentCalculator.calculate(cpuStat)
+	}
+
+	return cs, nil
+}

lib/hoststats/cpu_test.go

@@ -0,0 +1,58 @@
+package hoststats
+
+import (
+	"math"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/sdk/testutil"
+	"github.com/shirou/gopsutil/v3/cpu"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHostStats_CPU(t *testing.T) {
+	logger := testutil.Logger(t)
+	cwd, err := os.Getwd()
+	assert.Nil(t, err)
+	hs := initCollector(logger, cwd)
+
+	// Collect twice so we can calculate percents we need to generate some work
+	// so that the cpu values change
+	hs.collect()
+	for begin := time.Now(); time.Now().Sub(begin) < 100*time.Millisecond; {
+	}
+	hs.collect()
+	stats := hs.Stats()
+	assert.NotZero(t, len(stats.CPU))
+
+	for _, cpu := range stats.CPU {
+		assert.False(t, math.IsNaN(cpu.Idle))
+		assert.False(t, math.IsNaN(cpu.Total))
+		assert.False(t, math.IsNaN(cpu.System))
+		assert.False(t, math.IsNaN(cpu.User))
+
+		assert.False(t, math.IsInf(cpu.Idle, 0))
+		assert.False(t, math.IsInf(cpu.Total, 0))
+		assert.False(t, math.IsInf(cpu.System, 0))
+		assert.False(t, math.IsInf(cpu.User, 0))
+	}
+}
+
+func TestCpuStatsCalculator_Nan(t *testing.T) {
+	times := cpu.TimesStat{
+		User:   0.0,
+		Idle:   100.0,
+		System: 0.0,
+	}
+
+	calculator := &cpuStatsCalculator{}
+	calculator.calculate(times)
+	stats := calculator.calculate(times)
+	require.Equal(t, 100.0, stats.Idle)
+	require.Zero(t, stats.User)
+	require.Zero(t, stats.System)
+	require.Zero(t, stats.Iowait)
+	require.Zero(t, stats.Total)
+}

lib/hoststats/host.go

@@ -0,0 +1,92 @@
+package hoststats
+
+import (
+	"time"
+
+	"github.com/armon/go-metrics"
+)
+
+var hostStatsCollectionInterval = 10 * time.Second
+
+// HostStats represents resource usage hoststats of the host running a Consul agent
+type HostStats struct {
+	Memory       *MemoryStats
+	CPU          []*CPUStats
+	DataDirStats *DiskStats
+	Uptime       uint64
+	Timestamp    int64
+}
+
+func (hs *HostStats) Clone() *HostStats {
+	clone := &HostStats{}
+	*clone = *hs
+	return clone
+}
+
+func (hs *HostStats) Emit(sink Metrics, baseLabels []metrics.Label) {
+
+	if hs.Memory != nil {
+		sink.SetGaugeWithLabels([]string{"host", "memory", "total"}, float32(hs.Memory.Total), baseLabels)
+		sink.SetGaugeWithLabels([]string{"host", "memory", "available"}, float32(hs.Memory.Available), baseLabels)
+		sink.SetGaugeWithLabels([]string{"host", "memory", "used"}, float32(hs.Memory.Used), baseLabels)
+		sink.SetGaugeWithLabels([]string{"host", "memory", "used_percent"}, float32(hs.Memory.UsedPercent), baseLabels)
+		sink.SetGaugeWithLabels([]string{"host", "memory", "free"}, float32(hs.Memory.Free), baseLabels)
+	}
+
+	for _, cpu := range hs.CPU {
+		labels := append(baseLabels, metrics.Label{
+			Name:  "cpu",
+			Value: cpu.CPU,
+		})
+
+		sink.SetGaugeWithLabels([]string{"host", "cpu", "total"}, float32(cpu.Total), labels)
+		sink.SetGaugeWithLabels([]string{"host", "cpu", "user"}, float32(cpu.User), labels)
+		sink.SetGaugeWithLabels([]string{"host", "cpu", "idle"}, float32(cpu.Idle), labels)
+		sink.SetGaugeWithLabels([]string{"host", "cpu", "iowait"}, float32(cpu.Iowait), labels)
+		sink.SetGaugeWithLabels([]string{"host", "cpu", "system"}, float32(cpu.System), labels)
+	}
+
+	if hs.DataDirStats != nil {
+		diskLabels := append(baseLabels, metrics.Label{
+			Name:  "path",
+			Value: hs.DataDirStats.Path,
+		})
+
+		sink.SetGaugeWithLabels([]string{"host", "disk", "size"}, float32(hs.DataDirStats.Size), diskLabels)
+		sink.SetGaugeWithLabels([]string{"host", "disk", "used"}, float32(hs.DataDirStats.Used), diskLabels)
+		sink.SetGaugeWithLabels([]string{"host", "disk", "available"}, float32(hs.DataDirStats.Available), diskLabels)
+		sink.SetGaugeWithLabels([]string{"host", "disk", "used_percent"}, float32(hs.DataDirStats.UsedPercent), diskLabels)
+		sink.SetGaugeWithLabels([]string{"host", "disk", "inodes_percent"}, float32(hs.DataDirStats.InodesUsedPercent), diskLabels)
+	}
+
+	sink.SetGaugeWithLabels([]string{"host", "uptime"}, float32(hs.Uptime), baseLabels)
+}
+
+// CPUStats represents hoststats related to cpu usage
+type CPUStats struct {
+	CPU    string
+	User   float64
+	System float64
+	Idle   float64
+	Iowait float64
+	Total  float64
+}
+
+// MemoryStats represents hoststats related to virtual memory usage
+type MemoryStats struct {
+	Total       uint64
+	Available   uint64
+	Used        uint64
+	UsedPercent float64
+	Free        uint64
+}
+
+// DiskStats represents hoststats related to disk usage
+type DiskStats struct {
+	Path              string
+	Size              uint64
+	Used              uint64
+	Available         uint64
+	UsedPercent       float64
+	InodesUsedPercent float64
+}

lib/hoststats/metrics.go

@@ -0,0 +1,79 @@
+package hoststats
+
+import (
+	"github.com/armon/go-metrics"
+	"github.com/armon/go-metrics/prometheus"
+)
+
+// Metrics defines an interface for the methods used to emit data to the go-metrics library.
+// `metrics.Default()` should always satisfy this interface.
+type Metrics interface {
+	SetGaugeWithLabels(key []string, val float32, labels []metrics.Label)
+}
+
+var Gauges = []prometheus.GaugeDefinition{
+	{
+		Name: []string{"host", "memory", "total"},
+		Help: "Total physical memory in bytes",
+	},
+	{
+		Name: []string{"host", "memory", "available"},
+		Help: "Available physical memory in bytes",
+	},
+	{
+		Name: []string{"host", "memory", "free"},
+		Help: "Free physical memory in bytes",
+	},
+	{
+		Name: []string{"host", "memory", "used"},
+		Help: "Used physical memory in bytes",
+	},
+	{
+		Name: []string{"host", "memory", "used_percent"},
+		Help: "Percentage of physical memory in use",
+	},
+	{
+		Name: []string{"host", "cpu", "total"},
+		Help: "Total cpu utilization",
+	},
+	{
+		Name: []string{"host", "cpu", "user"},
+		Help: "User cpu utilization",
+	},
+	{
+		Name: []string{"host", "cpu", "idle"},
+		Help: "Idle cpu utilization",
+	},
+	{
+		Name: []string{"host", "cpu", "iowait"},
+		Help: "Iowait cpu utilization",
+	},
+	{
+		Name: []string{"host", "cpu", "system"},
+		Help: "System cpu utilization",
+	},
+	{
+		Name: []string{"host", "disk", "size"},
+		Help: "Size of disk in bytes",
+	},
+	{
+		Name: []string{"host", "disk", "used"},
+		Help: "Disk usage in bytes",
+	},
+	{
+		Name: []string{"host", "disk", "available"},
+		Help: "Available bytes on disk",
+	},
+	{
+		Name: []string{"host", "disk", "used_percent"},
+		Help: "Percentage of disk space usage",
+	},
+	{
+		Name: []string{"host", "disk", "inodes_percent"},
+		Help: "Percentage of disk inodes usage",
+	},
+	{
+		Name: []string{"host", "uptime"},
+		Help: "System uptime",
+	},
+}

lib/telemetry.go

@@ -204,6 +204,11 @@ type TelemetryConfig struct {
 	// hcl: telemetry { statsite_address = string }
 	StatsiteAddr string `json:"statsite_address,omitempty" mapstructure:"statsite_address"`
 
+	// EnableHostMetrics will enable metrics collected about the host system such as cpu memory and disk usage.
+	//
+	// hcl: telemetry { enable_host_metrics = (true|false) }
+	EnableHostMetrics bool `json:"enable_host_metrics,omitempty" mapstructure:"enable_host_metrics"`
+
 	// PrometheusOpts provides configuration for the PrometheusSink. Currently the only configuration
 	// we acquire from hcl is the retention time. We also use definition slices that are set in agent setup
 	// before being passed to InitTelemmetry.

website/content/docs/agent/config/config-files.mdx

@@ -1831,6 +1831,9 @@ subsystem that provides Consul's service mesh capabilities.
     of global tags that will be added to all telemetry packets sent to DogStatsD.
     It is a list of strings, where each string looks like "my_tag_name:my_tag_value".
 
+  - `enable_host_metrics` ((#telemetry-enable_host_metrics))
+    This enables reporting of host metrics about system resources, defaults to false.
+
   - `filter_default` ((#telemetry-filter_default))
     This controls whether to allow metrics that have not been specified by the filter.
     Defaults to `true`, which will allow all metrics when no filters are provided.

website/content/docs/agent/telemetry.mdx

@@ -755,3 +755,32 @@ Consul attaches the following labels to metric values.
 | `peer_id`                             | The ID of a peer connected to the reporting cluster or leader.                   | Any UUID                                  |
 | `partition`                           | <EnterpriseAlert inline /> Name of the partition that the peering is created in. | Any defined partition name in the cluster |
 
+## Server Host Metrics
+
+Consul servers can report the following metrics about the host's system resources.
+This feature must be enabled in the [agent telemetry configuration](/consul/docs/agent/config/config-files#telemetry-enable_host_metrics).
+Note that if the Consul server is operating inside a container these metrics
+still report host resource usage and do not report any resource limits placed
+on the container.  
+
+**Requirements:**
+- Consul 1.15.3+
+
+| Metric                              | Description                                                                                                                                                                                                                                                       | Unit        | Type    |
+| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | ------- |
+| `consul.host.memory.total`          | The total physical memory in bytes                                                                                                                                                                                                   | mixed       | mixed   |
+| `consul.host.memory.available`      | The available physical memory in bytes                                                                                                                                                                                                   | mixed       | mixed   |
+| `consul.host.memory.free`           | The free physical memory in bytes                                                                                                                                                                                                   | mixed       | mixed   |
+| `consul.host.memory.used`           | The used physical memory in bytes                                                                                                                                                                                                   | mixed       | mixed   |
+| `consul.host.memory.used_percent`   | The used physical memory as a percentage of total physical memory                                                                                                                                                                                                 | mixed       | mixed   |
+| `consul.host.cpu.total`             | The host's total cpu utilization
+| `consul.host.cpu.user`              | The cpu utilization in user space
+| `consul.host.cpu.idle`              | The cpu utilization in idle state
+| `consul.host.cpu.iowait`            | The cpu utilization in iowait state
+| `consul.host.cpu.system`            | The cpu utilization in system space
+| `consul.host.disk.size`             | The size in bytes of the data_dir disk
+| `consul.host.disk.used`             | The number of bytes used on the data_dir disk
+| `consul.host.disk.available`        | The number of bytes available on the data_dir disk
+| `consul.host.disk.used_percent`     | The percentage of disk space used on the data_dir disk
+| `consul.host.disk.inodes_percent`   | The percentage of inode usage on the data_dir disk
+| `consul.host.uptime`                | The uptime of the host in seconds