luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Metricsify new Consul client

This commit is contained in:
Michael Schurter 2017-04-18 16:23:39 -07:00
parent 59c687c940
commit 4db840b54e
2 changed files with 17 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
command/agent/consul/client.go
View File

@ -10,6 +10,7 @@ import (
"sync" "sync"
"time" "time"
metrics "github.com/armon/go-metrics"
"github.com/hashicorp/consul/api" "github.com/hashicorp/consul/api"
"github.com/hashicorp/nomad/client/driver" "github.com/hashicorp/nomad/client/driver"
"github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs"
@ -216,6 +217,9 @@ func (c *ServiceClient) merge(ops *operations) {
} }
delete(c.checks, cid) delete(c.checks, cid)
} }
metrics.SetGauge([]string{"client", "consul", "services"}, float32(len(c.services)))
metrics.SetGauge([]string{"client", "consul", "checks"}, float32(len(c.checks)))
metrics.SetGauge([]string{"client", "consul", "script_checks"}, float32(len(c.runningScripts)))
} }
// sync enqueued operations. // sync enqueued operations.
@ -224,11 +228,13 @@ func (c *ServiceClient) sync() error {
consulServices, err := c.client.Services() consulServices, err := c.client.Services()
if err != nil { if err != nil {
metrics.IncrCounter([]string{"client", "consul", "sync_failure"}, 1)
return fmt.Errorf("error querying Consul services: %v", err) return fmt.Errorf("error querying Consul services: %v", err)
} }
consulChecks, err := c.client.Checks() consulChecks, err := c.client.Checks()
if err != nil { if err != nil {
metrics.IncrCounter([]string{"client", "consul", "sync_failure"}, 1)
return fmt.Errorf("error querying Consul checks: %v", err) return fmt.Errorf("error querying Consul checks: %v", err)
} }
@ -244,9 +250,11 @@ func (c *ServiceClient) sync() error {
} }
// Unknown Nomad managed service; kill // Unknown Nomad managed service; kill
if err := c.client.ServiceDeregister(id); err != nil { if err := c.client.ServiceDeregister(id); err != nil {
metrics.IncrCounter([]string{"client", "consul", "sync_failure"}, 1)
return err return err
} }
sdereg++ sdereg++
metrics.IncrCounter([]string{"client", "consul", "service_deregisrations"}, 1)
} }
// Track services whose ports have changed as their checks may also // Track services whose ports have changed as their checks may also
@ -264,9 +272,11 @@ func (c *ServiceClient) sync() error {
portsChanged[id] = struct{}{} portsChanged[id] = struct{}{}
} }
if err = c.client.ServiceRegister(locals); err != nil { if err = c.client.ServiceRegister(locals); err != nil {
metrics.IncrCounter([]string{"client", "consul", "sync_failure"}, 1)
return err return err
} }
sreg++ sreg++
metrics.IncrCounter([]string{"client", "consul", "service_regisrations"}, 1)
} }
// Remove Nomad checks in Consul but unknown locally // Remove Nomad checks in Consul but unknown locally
@ -281,9 +291,11 @@ func (c *ServiceClient) sync() error {
} }
// Unknown Nomad managed check; kill // Unknown Nomad managed check; kill
if err := c.client.CheckDeregister(id); err != nil { if err := c.client.CheckDeregister(id); err != nil {
metrics.IncrCounter([]string{"client", "consul", "sync_failure"}, 1)
return err return err
} }
cdereg++ cdereg++
metrics.IncrCounter([]string{"client", "consul", "check_deregisrations"}, 1)
} }
// Add Nomad checks missing from Consul // Add Nomad checks missing from Consul
@ -295,9 +307,11 @@ func (c *ServiceClient) sync() error {
} }
} }
if err := c.client.CheckRegister(check); err != nil { if err := c.client.CheckRegister(check); err != nil {
metrics.IncrCounter([]string{"client", "consul", "sync_failure"}, 1)
return err return err
} }
creg++ creg++
metrics.IncrCounter([]string{"client", "consul", "check_regisrations"}, 1)
// Handle starting scripts // Handle starting scripts
if script, ok := c.scripts[id]; ok { if script, ok := c.scripts[id]; ok {

3
command/agent/consul/script.go
View File

@ -5,6 +5,7 @@ import (
"log" "log"
"time" "time"
metrics "github.com/armon/go-metrics"
"github.com/hashicorp/consul/api" "github.com/hashicorp/consul/api"
"github.com/hashicorp/nomad/client/driver" "github.com/hashicorp/nomad/client/driver"
"github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs"
@ -87,6 +88,7 @@ func (s *scriptCheck) run() *scriptHandle {
case <-timer.C: case <-timer.C:
timer.Reset(s.check.Interval) timer.Reset(s.check.Interval)
} }
metrics.IncrCounter([]string{"client", "consul", "script_runs"}, 1)
// Execute check script with timeout // Execute check script with timeout
execctx, cancel := context.WithTimeout(ctx, s.check.Timeout) execctx, cancel := context.WithTimeout(ctx, s.check.Timeout)
@ -96,6 +98,7 @@ func (s *scriptCheck) run() *scriptHandle {
// check removed during execution; exit // check removed during execution; exit
return return
case context.DeadlineExceeded: case context.DeadlineExceeded:
metrics.IncrCounter([]string{"client", "consul", "script_timeouts"}, 1)
// If no error was returned, set one to make sure the task goes critical // If no error was returned, set one to make sure the task goes critical
if err == nil { if err == nil {
err = context.DeadlineExceeded err = context.DeadlineExceeded