csi: provide `CSI_ENDPOINT` env var to plugins (#12050)

The CSI specification says:
> The CO SHALL provide the listen-address for the Plugin by way of the
`CSI_ENDPOINT` environment variable.

Note that plugins without filesystem isolation won't have the plugin
dir bind-mounted to their alloc dir, but we can provide a path to the
socket anyways.

Refactor to use opts struct for plugin supervisor hook config.
The parameter list for configuring the plugin supervisor hook has
grown enough where is makes sense to use an options struct similiar to
many of the other task runner hooks (ex. template).
This commit is contained in:
Tim Gross 2022-02-11 08:46:21 -05:00 committed by GitHub
parent 926458c5b2
commit 8ffe7aa76f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 44 additions and 9 deletions

3
.changelog/12050.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:bug
csi: provide `CSI_ENDPOINT` environment variable to plugin tasks
```

View File

@ -34,6 +34,8 @@ type csiPluginSupervisorHook struct {
runner *TaskRunner runner *TaskRunner
mountPoint string mountPoint string
caps *drivers.Capabilities
// eventEmitter is used to emit events to the task // eventEmitter is used to emit events to the task
eventEmitter ti.EventEmitter eventEmitter ti.EventEmitter
@ -48,6 +50,14 @@ type csiPluginSupervisorHook struct {
previousHealthState bool previousHealthState bool
} }
type csiPluginSupervisorHookConfig struct {
clientStateDirPath string
events ti.EventEmitter
runner *TaskRunner
capabilities *drivers.Capabilities
logger hclog.Logger
}
// The plugin supervisor uses the PrestartHook mechanism to setup the requisite // The plugin supervisor uses the PrestartHook mechanism to setup the requisite
// mount points and configuration for the task that exposes a CSI plugin. // mount points and configuration for the task that exposes a CSI plugin.
var _ interfaces.TaskPrestartHook = &csiPluginSupervisorHook{} var _ interfaces.TaskPrestartHook = &csiPluginSupervisorHook{}
@ -61,8 +71,8 @@ var _ interfaces.TaskPoststartHook = &csiPluginSupervisorHook{}
// with the catalog and to ensure any mounts are cleaned up. // with the catalog and to ensure any mounts are cleaned up.
var _ interfaces.TaskStopHook = &csiPluginSupervisorHook{} var _ interfaces.TaskStopHook = &csiPluginSupervisorHook{}
func newCSIPluginSupervisorHook(csiRootDir string, eventEmitter ti.EventEmitter, runner *TaskRunner, logger hclog.Logger) *csiPluginSupervisorHook { func newCSIPluginSupervisorHook(config *csiPluginSupervisorHookConfig) *csiPluginSupervisorHook {
task := runner.Task() task := config.runner.Task()
// The Plugin directory will look something like this: // The Plugin directory will look something like this:
// . // .
@ -72,19 +82,21 @@ func newCSIPluginSupervisorHook(csiRootDir string, eventEmitter ti.EventEmitter,
// {volume-id}/{usage-mode-hash}/ - Intermediary mount point that will be used by plugins that support NODE_STAGE_UNSTAGE capabilities. // {volume-id}/{usage-mode-hash}/ - Intermediary mount point that will be used by plugins that support NODE_STAGE_UNSTAGE capabilities.
// per-alloc/ // per-alloc/
// {alloc-id}/{volume-id}/{usage-mode-hash}/ - Mount Point that will be bind-mounted into tasks that utilise the volume // {alloc-id}/{volume-id}/{usage-mode-hash}/ - Mount Point that will be bind-mounted into tasks that utilise the volume
pluginRoot := filepath.Join(csiRootDir, string(task.CSIPluginConfig.Type), task.CSIPluginConfig.ID) pluginRoot := filepath.Join(config.clientStateDirPath, "csi",
string(task.CSIPluginConfig.Type), task.CSIPluginConfig.ID)
shutdownCtx, cancelFn := context.WithCancel(context.Background()) shutdownCtx, cancelFn := context.WithCancel(context.Background())
hook := &csiPluginSupervisorHook{ hook := &csiPluginSupervisorHook{
alloc: runner.Alloc(), alloc: config.runner.Alloc(),
runner: runner, runner: config.runner,
logger: logger, logger: config.logger,
task: task, task: task,
mountPoint: pluginRoot, mountPoint: pluginRoot,
caps: config.capabilities,
shutdownCtx: shutdownCtx, shutdownCtx: shutdownCtx,
shutdownCancelFn: cancelFn, shutdownCancelFn: cancelFn,
eventEmitter: eventEmitter, eventEmitter: config.events,
} }
return hook return hook
@ -119,6 +131,20 @@ func (h *csiPluginSupervisorHook) Prestart(ctx context.Context,
Readonly: false, Readonly: false,
} }
switch h.caps.FSIsolation {
case drivers.FSIsolationNone:
// Plugin tasks with no filesystem isolation won't have the
// plugin dir bind-mounted to their alloc dir, but we can
// provide them the path to the socket. These Nomad-only
// plugins will need to be aware of the csi directory layout
// in the client data dir
resp.Env = map[string]string{
"CSI_ENDPOINT": filepath.Join(h.mountPoint, "csi.sock")}
default:
resp.Env = map[string]string{
"CSI_ENDPOINT": filepath.Join(h.task.CSIPluginConfig.MountDir, "csi.sock")}
}
mounts := ensureMountpointInserted(h.runner.hookResources.getMounts(), configMount) mounts := ensureMountpointInserted(h.runner.hookResources.getMounts(), configMount)
mounts = ensureMountpointInserted(mounts, devMount) mounts = ensureMountpointInserted(mounts, devMount)

View File

@ -3,7 +3,6 @@ package taskrunner
import ( import (
"context" "context"
"fmt" "fmt"
"path/filepath"
"sync" "sync"
"time" "time"
@ -72,7 +71,14 @@ func (tr *TaskRunner) initHooks() {
// If the task has a CSI stanza, add the hook. // If the task has a CSI stanza, add the hook.
if task.CSIPluginConfig != nil { if task.CSIPluginConfig != nil {
tr.runnerHooks = append(tr.runnerHooks, newCSIPluginSupervisorHook(filepath.Join(tr.clientConfig.StateDir, "csi"), tr, tr, hookLogger)) tr.runnerHooks = append(tr.runnerHooks, newCSIPluginSupervisorHook(
&csiPluginSupervisorHookConfig{
clientStateDirPath: tr.clientConfig.StateDir,
events: tr,
runner: tr,
capabilities: tr.driverCapabilities,
logger: hookLogger,
}))
} }
// If Vault is enabled, add the hook // If Vault is enabled, add the hook