luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Merge pull request #4111 from hashicorp/b-undetected-set-health-to-false 

Immediately set driver health status to false when driver moves to undetected
This commit is contained in:
Chelsea Komlo 2018-04-10 18:30:31 -04:00 committed by GitHub
parent 73621d7ae9 d58b3e473c
commit eb5aac16e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 53 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
client/fingerprint_manager.go
View File

@ -49,6 +49,14 @@ func NewFingerprintManager(getConfig func() *config.Config,
}
}
// setNode updates the current client node
func (fm *FingerprintManager) setNode(node *structs.Node) {
fm.nodeLock.Lock()
defer fm.nodeLock.Unlock()
fm.node = node
}
// Run starts the process of fingerprinting the node. It does an initial pass,
// identifying whitelisted and blacklisted fingerprints/drivers. Then, for
// those which require periotic checking, it starts a periodic process for
@ -167,7 +175,13 @@ func (fm *FingerprintManager) setupDrivers(drivers []string) error {
return err
}
detected, err := fm.fingerprintDriver(name, d)
// Pass true for whether the health check is periodic here, so that the
// fingerprinter will not set the initial health check status (this is set
// below, with an empty health status so that a node event is not
// triggered)
// Later, the periodic health checker will update this value for drivers
// where health checks are enabled.
detected, err := fm.fingerprintDriver(name, d, true)
if err != nil {
fm.logger.Printf("[DEBUG] client.fingerprint_manager: fingerprinting driver %v failed: %+v", name, err)
return err
@ -181,9 +195,7 @@ func (fm *FingerprintManager) setupDrivers(drivers []string) error {
UpdateTime: time.Now(),
}
if node := fm.updateNodeFromDriver(name, nil, healthInfo); node != nil {
fm.nodeLock.Lock()
fm.node = node
fm.nodeLock.Unlock()
fm.setNode(node)
}
// Start a periodic watcher to detect changes to a drivers health and
@ -240,9 +252,7 @@ func (fm *FingerprintManager) fingerprint(name string, f fingerprint.Fingerprint
}
if node := fm.updateNodeAttributes(&response); node != nil {
fm.nodeLock.Lock()
fm.node = node
fm.nodeLock.Unlock()
fm.setNode(node)
}
return response.Detected, nil
@ -269,18 +279,21 @@ func (fm *FingerprintManager) watchDriver(d driver.Driver, name string) {
defer ticker.Stop()
fm.logger.Printf("[DEBUG] client.fingerprint_manager: fingerprinting driver %s every %v", name, fingerprintPeriod)
}
var isHealthCheckPeriodic bool
if isHealthCheck {
// Determine the interval at which to health check
req := &cstructs.HealthCheckIntervalRequest{}
var resp cstructs.HealthCheckIntervalResponse
var healthCheckResp cstructs.HealthCheckIntervalResponse
if err := hc.GetHealthCheckInterval(req, &resp); err != nil {
if err := hc.GetHealthCheckInterval(req, &healthCheckResp); err != nil {
fm.logger.Printf("[ERR] client.fingerprint_manager: error getting health check interval for driver %s: %v", name, err)
} else if resp.Eligible {
ticker := time.NewTicker(resp.Period)
} else if healthCheckResp.Eligible {
isHealthCheckPeriodic = true
ticker := time.NewTicker(healthCheckResp.Period)
healthTicker = ticker.C
defer ticker.Stop()
fm.logger.Printf("[DEBUG] client.fingerprint_manager: health checking driver %s every %v", name, resp.Period)
fm.logger.Printf("[DEBUG] client.fingerprint_manager: health checking driver %s every %v", name, healthCheckResp.Period)
}
}
@ -290,7 +303,7 @@ func (fm *FingerprintManager) watchDriver(d driver.Driver, name string) {
case <-fm.shutdownCh:
return
case <-fingerprintTicker:
if _, err := fm.fingerprintDriver(name, d); err != nil {
if _, err := fm.fingerprintDriver(name, d, isHealthCheckPeriodic); err != nil {
fm.logger.Printf("[DEBUG] client.fingerprint_manager: periodic fingerprinting for driver %v failed: %+v", name, err)
}
@ -310,19 +323,6 @@ func (fm *FingerprintManager) watchDriver(d driver.Driver, name string) {
if err := fm.runDriverHealthCheck(name, hc); err != nil {
fm.logger.Printf("[DEBUG] client.fingerprint_manager: health checking for %v failed: %v", name, err)
}
} else {
// If the driver is undetected, change the health status to unhealthy
// only once.
healthInfo := &structs.DriverInfo{
Healthy: false,
HealthDescription: fmt.Sprintf("Driver %s is not detected", name),
UpdateTime: time.Now(),
}
if node := fm.updateNodeFromDriver(name, nil, healthInfo); node != nil {
fm.nodeLock.Lock()
fm.node = node
fm.nodeLock.Unlock()
}
}
}
}
@ -332,7 +332,7 @@ func (fm *FingerprintManager) watchDriver(d driver.Driver, name string) {
// away from annotating a node's attributes to demonstrate support for a
// particular driver. Takes the FingerprintResponse and converts it to the
// proper DriverInfo update and then sets the prefix attributes as well
func (fm *FingerprintManager) fingerprintDriver(name string, f fingerprint.Fingerprint) (bool, error) {
func (fm *FingerprintManager) fingerprintDriver(name string, f fingerprint.Fingerprint, hasPeriodicHealthCheck bool) (bool, error) {
var response cstructs.FingerprintResponse
fm.nodeLock.Lock()
@ -345,9 +345,7 @@ func (fm *FingerprintManager) fingerprintDriver(name string, f fingerprint.Finge
}
if node := fm.updateNodeAttributes(&response); node != nil {
fm.nodeLock.Lock()
fm.node = node
fm.nodeLock.Unlock()
fm.setNode(node)
}
di := &structs.DriverInfo{
@ -361,9 +359,30 @@ func (fm *FingerprintManager) fingerprintDriver(name string, f fingerprint.Finge
delete(di.Attributes, driverKey)
if node := fm.updateNodeFromDriver(name, di, nil); node != nil {
fm.nodeLock.Lock()
fm.node = node
fm.nodeLock.Unlock()
fm.setNode(node)
}
// Get a copy of the current node
var driverExists, driverIsHealthy bool
fm.nodeLock.Lock()
driverInfo, driverExists := fm.node.Drivers[name]
if driverExists {
driverIsHealthy = driverInfo.Healthy
}
fm.nodeLock.Unlock()
// If either 1) the driver is undetected or 2) if the driver does not have
// periodic health checks enabled, set the health status to the match that
// of the fingerprinter
if !hasPeriodicHealthCheck || !response.Detected && driverExists && driverIsHealthy {
healthInfo := &structs.DriverInfo{
Healthy: response.Detected,
HealthDescription: fmt.Sprintf("Driver %s is detected: %t", name, response.Detected),
UpdateTime: time.Now(),
}
if node := fm.updateNodeFromDriver(name, nil, healthInfo); node != nil {
fm.setNode(node)
}
}
return response.Detected, nil
@ -381,9 +400,7 @@ func (fm *FingerprintManager) runDriverHealthCheck(name string, hc fingerprint.H
// case of periodic health checks, an error will occur if a health check
// fails
if node := fm.updateNodeFromDriver(name, nil, response.Drivers[name]); node != nil {
fm.nodeLock.Lock()
fm.node = node
fm.nodeLock.Unlock()
fm.setNode(node)
}
return nil