feat(telemetry): add labels to serf and memberlist metrics (#14161)
* feat(telemetry): add labels to serf and memberlist metrics * changelog * doc update Co-authored-by: R.B. Boyer <4903+rboyer@users.noreply.github.com>
This commit is contained in:
parent
76b39a5453
commit
e7b5baa3cc
|
@ -0,0 +1,3 @@
|
||||||
|
```release-note:improvement
|
||||||
|
metrics: add labels of segment, partition, network area, network (lan or wan) to serf and memberlist metrics
|
||||||
|
```
|
|
@ -62,6 +62,8 @@ func (c *Client) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
addSerfMetricsLabels(conf, false, "", "", "")
|
||||||
|
|
||||||
addEnterpriseSerfTags(conf.Tags, c.config.AgentEnterpriseMeta())
|
addEnterpriseSerfTags(conf.Tags, c.config.AgentEnterpriseMeta())
|
||||||
|
|
||||||
conf.ReconnectTimeoutOverride = libserf.NewReconnectOverride(c.logger)
|
conf.ReconnectTimeoutOverride = libserf.NewReconnectOverride(c.logger)
|
||||||
|
|
|
@ -584,6 +584,7 @@ func CloneSerfLANConfig(base *serf.Config) *serf.Config {
|
||||||
cfg.MemberlistConfig.ProbeTimeout = base.MemberlistConfig.ProbeTimeout
|
cfg.MemberlistConfig.ProbeTimeout = base.MemberlistConfig.ProbeTimeout
|
||||||
cfg.MemberlistConfig.SuspicionMult = base.MemberlistConfig.SuspicionMult
|
cfg.MemberlistConfig.SuspicionMult = base.MemberlistConfig.SuspicionMult
|
||||||
cfg.MemberlistConfig.RetransmitMult = base.MemberlistConfig.RetransmitMult
|
cfg.MemberlistConfig.RetransmitMult = base.MemberlistConfig.RetransmitMult
|
||||||
|
cfg.MemberlistConfig.MetricLabels = base.MemberlistConfig.MetricLabels
|
||||||
|
|
||||||
// agent/keyring.go
|
// agent/keyring.go
|
||||||
cfg.MemberlistConfig.Keyring = base.MemberlistConfig.Keyring
|
cfg.MemberlistConfig.Keyring = base.MemberlistConfig.Keyring
|
||||||
|
@ -593,6 +594,7 @@ func CloneSerfLANConfig(base *serf.Config) *serf.Config {
|
||||||
cfg.ReapInterval = base.ReapInterval
|
cfg.ReapInterval = base.ReapInterval
|
||||||
cfg.TombstoneTimeout = base.TombstoneTimeout
|
cfg.TombstoneTimeout = base.TombstoneTimeout
|
||||||
cfg.MemberlistConfig.SecretKey = base.MemberlistConfig.SecretKey
|
cfg.MemberlistConfig.SecretKey = base.MemberlistConfig.SecretKey
|
||||||
|
cfg.MetricLabels = base.MetricLabels
|
||||||
|
|
||||||
return cfg
|
return cfg
|
||||||
}
|
}
|
||||||
|
|
|
@ -159,3 +159,18 @@ func (s *Server) addEnterpriseStats(stats map[string]map[string]string) {
|
||||||
func getSerfMemberEnterpriseMeta(member serf.Member) *acl.EnterpriseMeta {
|
func getSerfMemberEnterpriseMeta(member serf.Member) *acl.EnterpriseMeta {
|
||||||
return structs.NodeEnterpriseMetaInDefaultPartition()
|
return structs.NodeEnterpriseMetaInDefaultPartition()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func addSerfMetricsLabels(conf *serf.Config, wan bool, segment string, partition string, areaID string) {
|
||||||
|
conf.MetricLabels = []metrics.Label{}
|
||||||
|
|
||||||
|
networkMetric := metrics.Label{
|
||||||
|
Name: "network",
|
||||||
|
}
|
||||||
|
if wan {
|
||||||
|
networkMetric.Value = "wan"
|
||||||
|
} else {
|
||||||
|
networkMetric.Value = "lan"
|
||||||
|
}
|
||||||
|
|
||||||
|
conf.MetricLabels = append(conf.MetricLabels, networkMetric)
|
||||||
|
}
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/armon/go-metrics"
|
||||||
"github.com/hashicorp/go-hclog"
|
"github.com/hashicorp/go-hclog"
|
||||||
"github.com/hashicorp/memberlist"
|
"github.com/hashicorp/memberlist"
|
||||||
"github.com/hashicorp/raft"
|
"github.com/hashicorp/raft"
|
||||||
|
@ -180,6 +181,7 @@ func (s *Server) setupSerfConfig(opts setupSerfOptions) (*serf.Config, error) {
|
||||||
BindAddrs: []string{conf.MemberlistConfig.BindAddr},
|
BindAddrs: []string{conf.MemberlistConfig.BindAddr},
|
||||||
BindPort: conf.MemberlistConfig.BindPort,
|
BindPort: conf.MemberlistConfig.BindPort,
|
||||||
Logger: conf.MemberlistConfig.Logger,
|
Logger: conf.MemberlistConfig.Logger,
|
||||||
|
MetricLabels: []metrics.Label{{Name: "network", Value: "wan"}},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -230,6 +232,8 @@ func (s *Server) setupSerfConfig(opts setupSerfOptions) (*serf.Config, error) {
|
||||||
|
|
||||||
conf.ReconnectTimeoutOverride = libserf.NewReconnectOverride(s.logger)
|
conf.ReconnectTimeoutOverride = libserf.NewReconnectOverride(s.logger)
|
||||||
|
|
||||||
|
addSerfMetricsLabels(conf, opts.WAN, "", "", "")
|
||||||
|
|
||||||
addEnterpriseSerfTags(conf.Tags, s.config.AgentEnterpriseMeta())
|
addEnterpriseSerfTags(conf.Tags, s.config.AgentEnterpriseMeta())
|
||||||
|
|
||||||
if s.config.OverrideInitialSerfTags != nil {
|
if s.config.OverrideInitialSerfTags != nil {
|
||||||
|
|
6
go.mod
6
go.mod
|
@ -45,11 +45,11 @@ require (
|
||||||
github.com/hashicorp/golang-lru v0.5.4
|
github.com/hashicorp/golang-lru v0.5.4
|
||||||
github.com/hashicorp/hcl v1.0.0
|
github.com/hashicorp/hcl v1.0.0
|
||||||
github.com/hashicorp/hil v0.0.0-20200423225030-a18a1cd20038
|
github.com/hashicorp/hil v0.0.0-20200423225030-a18a1cd20038
|
||||||
github.com/hashicorp/memberlist v0.3.1
|
github.com/hashicorp/memberlist v0.4.0
|
||||||
github.com/hashicorp/raft v1.3.9
|
github.com/hashicorp/raft v1.3.9
|
||||||
github.com/hashicorp/raft-autopilot v0.1.6
|
github.com/hashicorp/raft-autopilot v0.1.6
|
||||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2
|
github.com/hashicorp/raft-boltdb/v2 v2.2.2
|
||||||
github.com/hashicorp/serf v0.9.8
|
github.com/hashicorp/serf v0.10.0
|
||||||
github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086
|
github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086
|
||||||
github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267
|
github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267
|
||||||
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493
|
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493
|
||||||
|
@ -77,7 +77,7 @@ require (
|
||||||
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
|
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
|
||||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
|
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
|
||||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
|
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10
|
||||||
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
|
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
|
||||||
google.golang.org/genproto v0.0.0-20200623002339-fbb79eadd5eb
|
google.golang.org/genproto v0.0.0-20200623002339-fbb79eadd5eb
|
||||||
google.golang.org/grpc v1.37.1
|
google.golang.org/grpc v1.37.1
|
||||||
|
|
10
go.sum
10
go.sum
|
@ -364,8 +364,9 @@ github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg
|
||||||
github.com/hashicorp/mdns v1.0.4 h1:sY0CMhFmjIPDMlTB+HfymFHCaYLhgifZ0QhjaYKD/UQ=
|
github.com/hashicorp/mdns v1.0.4 h1:sY0CMhFmjIPDMlTB+HfymFHCaYLhgifZ0QhjaYKD/UQ=
|
||||||
github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
|
github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
|
||||||
github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
|
github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
|
||||||
github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM=
|
|
||||||
github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
|
github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
|
||||||
|
github.com/hashicorp/memberlist v0.4.0 h1:k3uda5gZcltmafuFF+UFqNEl5PrH+yPZ4zkjp1f/H/8=
|
||||||
|
github.com/hashicorp/memberlist v0.4.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
|
||||||
github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
|
github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
|
||||||
github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
|
github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
|
||||||
github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
|
github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
|
||||||
|
@ -380,8 +381,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20211202195631-7d34b9fb3f42/go.mod h1:wc
|
||||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2 h1:rlkPtOllgIcKLxVT4nutqlTH2NRFn+tO1wwZk/4Dxqw=
|
github.com/hashicorp/raft-boltdb/v2 v2.2.2 h1:rlkPtOllgIcKLxVT4nutqlTH2NRFn+tO1wwZk/4Dxqw=
|
||||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2/go.mod h1:N8YgaZgNJLpZC+h+by7vDu5rzsRgONThTEeUS3zWbfY=
|
github.com/hashicorp/raft-boltdb/v2 v2.2.2/go.mod h1:N8YgaZgNJLpZC+h+by7vDu5rzsRgONThTEeUS3zWbfY=
|
||||||
github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
|
github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
|
||||||
github.com/hashicorp/serf v0.9.8 h1:JGklO/2Drf1QGa312EieQN3zhxQ+aJg6pG+aC3MFaVo=
|
github.com/hashicorp/serf v0.10.0 h1:89qvvpfMQnz6c2y4pv7j2vUUmeT1+5TSZMexuTbtsPs=
|
||||||
github.com/hashicorp/serf v0.9.8/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
|
github.com/hashicorp/serf v0.10.0/go.mod h1:bXN03oZc5xlH46k/K1qTrpXb9ERKyY1/i/N5mxvgrZw=
|
||||||
github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086 h1:OKsyxKi2sNmqm1Gv93adf2AID2FOBFdCbbZn9fGtIdg=
|
github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086 h1:OKsyxKi2sNmqm1Gv93adf2AID2FOBFdCbbZn9fGtIdg=
|
||||||
github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086/go.mod h1:R3Umvhlxi2TN7Ex2hzOowyeNb+SfbVWI973N+ctaFMk=
|
github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086/go.mod h1:R3Umvhlxi2TN7Ex2hzOowyeNb+SfbVWI973N+ctaFMk=
|
||||||
github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267 h1:e1ok06zGrWJW91rzRroyl5nRNqraaBe4d5hiKcVZuHM=
|
github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267 h1:e1ok06zGrWJW91rzRroyl5nRNqraaBe4d5hiKcVZuHM=
|
||||||
|
@ -793,8 +794,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
|
|
||||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
|
||||||
|
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
|
|
@ -605,6 +605,10 @@ Any metric in this section can be turned off with the [`prefix_filter`](/docs/ag
|
||||||
## Cluster Health
|
## Cluster Health
|
||||||
|
|
||||||
These metrics give insight into the health of the cluster as a whole.
|
These metrics give insight into the health of the cluster as a whole.
|
||||||
|
Query for the `consul.memberlist.*` and `consul.serf.*` metrics can be appended
|
||||||
|
with certain labels to further distinguish data between different gossip pools.
|
||||||
|
The supported label for OSS is `network`, while `segment`, `partition`, `area`
|
||||||
|
are allowed for <EnterpriseAlert inline />.
|
||||||
|
|
||||||
| Metric | Description | Unit | Type |
|
| Metric | Description | Unit | Type |
|
||||||
|----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
|
|----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
|
||||||
|
|
Loading…
Reference in New Issue