luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

template: protect use of template manager with a lock (#15192) 

This PR protects access to `templateHook.templateManager` with its lock. So
far we have not been able to reproduce the panic - but it seems either Poststart
is running without a Prestart being run first (should be impossible), or the
Update hook is running concurrently with Poststart, nil-ing out the templateManager
in a race with Poststart.

Fixes #15189
This commit is contained in:
Seth Hoenig 2022-11-10 08:30:27 -06:00 committed by GitHub
parent 87a34102f5
commit 6e3309ebc6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/15192.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:bug
template: Fixed a bug where template could cause agent panic on startup
```

25
client/allocrunner/taskrunner/template_hook.go
View File

@ -116,11 +116,18 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
}
func (h *templateHook) Poststart(ctx context.Context, req *interfaces.TaskPoststartRequest, resp *interfaces.TaskPoststartResponse) error {
h.managerLock.Lock()
defer h.managerLock.Unlock()
if h.templateManager == nil {
return nil
}
if req.DriverExec != nil {
h.templateManager.SetDriverHandle(req.DriverExec)
} else {
for _, template := range h.config.templates {
if template.ChangeMode == structs.TemplateChangeModeScript {
for _, tmpl := range h.config.templates {
if tmpl.ChangeMode == structs.TemplateChangeModeScript {
return fmt.Errorf("template has change mode set to 'script' but the driver it uses does not provide exec capability")
}
}
@ -166,17 +173,17 @@ func (h *templateHook) Stop(ctx context.Context, req *interfaces.TaskStopRequest
return nil
}
// Handle new Vault token
// Update is used to handle updates to vault and/or nomad tokens.
func (h *templateHook) Update(ctx context.Context, req *interfaces.TaskUpdateRequest, resp *interfaces.TaskUpdateResponse) error {
h.managerLock.Lock()
defer h.managerLock.Unlock()
// Nothing to do
// no template manager to manage
if h.templateManager == nil {
return nil
}
// Check if either the Nomad or Vault tokens have changed
// neither vault or nomad token has been updated, nothing to do
if req.VaultToken == h.vaultToken && req.NomadToken == h.nomadToken {
return nil
} else {
@ -184,15 +191,15 @@ func (h *templateHook) Update(ctx context.Context, req *interfaces.TaskUpdateReq
h.nomadToken = req.NomadToken
}
// Shutdown the old template
// shutdown the old template
h.templateManager.Stop()
h.templateManager = nil
// Create the new template
// create the new template
if _, err := h.newManager(); err != nil {
err := fmt.Errorf("failed to build template manager: %v", err)
err = fmt.Errorf("failed to build template manager: %v", err)
h.logger.Error("failed to build template manager", "error", err)
h.config.lifecycle.Kill(context.Background(),
_ = h.config.lifecycle.Kill(context.Background(),
structs.NewTaskEvent(structs.TaskKilling).
SetFailsTask().
SetDisplayMessage(fmt.Sprintf("Template update %v", err)))