CSI: set plugin socket path on restore (#12149)

The Prestart hook for task runner hooks doesn't get called when we
restore a task, because the task is already running. The Postrun hook
for CSI plugin supervisors needs the socket path to have been
populated so that the client has a valid path.
This commit is contained in:
Tim Gross 2022-03-01 10:22:52 -05:00 committed by GitHub
parent f2a4ad0949
commit f4dfaec589
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 26 additions and 17 deletions

View File

@ -129,12 +129,13 @@ func (*csiPluginSupervisorHook) Name() string {
} }
// Prestart is called before the task is started including after every // Prestart is called before the task is started including after every
// restart. This requires that the mount paths for a plugin be idempotent, // restart (but not after restore). This requires that the mount paths
// despite us not knowing the name of the plugin ahead of time. // for a plugin be idempotent, despite us not knowing the name of the
// Because of this, we use the allocid_taskname as the unique identifier for a // plugin ahead of time. Because of this, we use the allocid_taskname
// plugin on the filesystem. // as the unique identifier for a plugin on the filesystem.
func (h *csiPluginSupervisorHook) Prestart(ctx context.Context, func (h *csiPluginSupervisorHook) Prestart(ctx context.Context,
req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error { req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
// Create the mount directory that the container will access if it doesn't // Create the mount directory that the container will access if it doesn't
// already exist. Default to only nomad user access. // already exist. Default to only nomad user access.
if err := os.MkdirAll(h.mountPoint, 0700); err != nil && !os.IsExist(err) { if err := os.MkdirAll(h.mountPoint, 0700); err != nil && !os.IsExist(err) {
@ -167,19 +168,7 @@ func (h *csiPluginSupervisorHook) Prestart(ctx context.Context,
Readonly: false, Readonly: false,
} }
// TODO(tgross): https://github.com/hashicorp/nomad/issues/11786 h.setSocketHook()
// If we're already registered, we should be able to update the
// definition in the update hook
// For backwards compatibility, ensure that we don't overwrite the
// socketPath on client restart with existing plugin allocations.
pluginInfo, _ := h.runner.dynamicRegistry.PluginForAlloc(
string(h.task.CSIPluginConfig.Type), h.task.CSIPluginConfig.ID, h.alloc.ID)
if pluginInfo != nil {
h.socketPath = pluginInfo.ConnectionInfo.SocketPath
} else {
h.socketPath = filepath.Join(h.socketMountPoint, structs.CSISocketName)
}
switch h.caps.FSIsolation { switch h.caps.FSIsolation {
case drivers.FSIsolationNone: case drivers.FSIsolationNone:
@ -206,11 +195,29 @@ func (h *csiPluginSupervisorHook) Prestart(ctx context.Context,
return nil return nil
} }
func (h *csiPluginSupervisorHook) setSocketHook() {
// TODO(tgross): https://github.com/hashicorp/nomad/issues/11786
// If we're already registered, we should be able to update the
// definition in the update hook
// For backwards compatibility, ensure that we don't overwrite the
// socketPath on client restart with existing plugin allocations.
pluginInfo, _ := h.runner.dynamicRegistry.PluginForAlloc(
string(h.task.CSIPluginConfig.Type), h.task.CSIPluginConfig.ID, h.alloc.ID)
if pluginInfo != nil && pluginInfo.ConnectionInfo.SocketPath != "" {
h.socketPath = pluginInfo.ConnectionInfo.SocketPath
return
}
h.socketPath = filepath.Join(h.socketMountPoint, structs.CSISocketName)
}
// Poststart is called after the task has started. Poststart is not // Poststart is called after the task has started. Poststart is not
// called if the allocation is terminal. // called if the allocation is terminal.
// //
// The context is cancelled if the task is killed. // The context is cancelled if the task is killed.
func (h *csiPluginSupervisorHook) Poststart(_ context.Context, _ *interfaces.TaskPoststartRequest, _ *interfaces.TaskPoststartResponse) error { func (h *csiPluginSupervisorHook) Poststart(_ context.Context, _ *interfaces.TaskPoststartRequest, _ *interfaces.TaskPoststartResponse) error {
// If we're already running the supervisor routine, then we don't need to try // If we're already running the supervisor routine, then we don't need to try
// and restart it here as it only terminates on `Stop` hooks. // and restart it here as it only terminates on `Stop` hooks.
h.runningLock.Lock() h.runningLock.Lock()
@ -220,6 +227,8 @@ func (h *csiPluginSupervisorHook) Poststart(_ context.Context, _ *interfaces.Tas
} }
h.runningLock.Unlock() h.runningLock.Unlock()
h.setSocketHook()
go h.ensureSupervisorLoop(h.shutdownCtx) go h.ensureSupervisorLoop(h.shutdownCtx)
return nil return nil
} }