diff --git a/.changelog/13919.txt b/.changelog/13919.txt new file mode 100644 index 000000000..559d948d8 --- /dev/null +++ b/.changelog/13919.txt @@ -0,0 +1,3 @@ +```release-note:improvement +csi: Add `stage_publish_base_dir` field to `csi_plugin` block to support plugins that require a specific staging/publishing directory for mounts +``` diff --git a/api/tasks.go b/api/tasks.go index 6cdb44da3..98957ecc3 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -1032,14 +1032,17 @@ type TaskCSIPluginConfig struct { // CSIPluginType instructs Nomad on how to handle processing a plugin Type CSIPluginType `mapstructure:"type" hcl:"type,optional"` - // MountDir is the destination that nomad should mount in its CSI - // directory for the plugin. It will then expect a file called CSISocketName - // to be created by the plugin, and will provide references into - // "MountDir/CSIIntermediaryDirname/VolumeName/AllocID for mounts. - // - // Default is /csi. + // MountDir is the directory (within its container) in which the plugin creates a + // socket (called CSISocketName) for communication with Nomad. Default is /csi. MountDir string `mapstructure:"mount_dir" hcl:"mount_dir,optional"` + // StagePublishBaseDir is the base directory (within its container) in which the plugin + // mounts volumes being staged and bind mounts volumes being published. + // e.g. staging_target_path = {StagePublishBaseDir}/staging/{volume-id}/{usage-mode} + // e.g. target_path = {StagePublishBaseDir}/per-alloc/{alloc-id}/{volume-id}/{usage-mode} + // Default is /local/csi. + StagePublishBaseDir string `mapstructure:"stage_publish_base_dir" hcl:"stage_publish_base_dir,optional"` + // HealthTimeout is the time after which the CSI plugin tasks will be killed // if the CSI Plugin is not healthy. HealthTimeout time.Duration `mapstructure:"health_timeout" hcl:"health_timeout,optional"` @@ -1050,6 +1053,10 @@ func (t *TaskCSIPluginConfig) Canonicalize() { t.MountDir = "/csi" } + if t.StagePublishBaseDir == "" { + t.StagePublishBaseDir = filepath.Join("/local", "csi") + } + if t.HealthTimeout == 0 { t.HealthTimeout = 30 * time.Second } diff --git a/client/allocrunner/taskrunner/plugin_supervisor_hook.go b/client/allocrunner/taskrunner/plugin_supervisor_hook.go index 4696bc53f..03e52e66b 100644 --- a/client/allocrunner/taskrunner/plugin_supervisor_hook.go +++ b/client/allocrunner/taskrunner/plugin_supervisor_hook.go @@ -81,7 +81,7 @@ var _ interfaces.TaskStopHook = &csiPluginSupervisorHook{} // Per-allocation directories of unix domain sockets used to communicate // with the CSI plugin. Nomad creates the directory and the plugin creates // the socket file. This directory is bind-mounted to the -// csi_plugin.mount_config dir in the plugin task. +// csi_plugin.mount_dir in the plugin task. // // {plugin-type}/{plugin-id}/ // staging/ @@ -103,6 +103,16 @@ func newCSIPluginSupervisorHook(config *csiPluginSupervisorHookConfig) *csiPlugi socketMountPoint := filepath.Join(config.clientStateDirPath, "csi", "plugins", config.runner.Alloc().ID) + // In v1.3.0, Nomad started instructing CSI plugins to stage and publish + // within /local/csi. Plugins deployed after the introduction of + // StagePublishBaseDir default to StagePublishBaseDir = /local/csi. However, + // plugins deployed between v1.3.0 and the introduction of + // StagePublishBaseDir have StagePublishBaseDir = "". Default to /local/csi here + // to avoid breaking plugins that aren't redeployed. + if task.CSIPluginConfig.StagePublishBaseDir == "" { + task.CSIPluginConfig.StagePublishBaseDir = filepath.Join("/local", "csi") + } + if task.CSIPluginConfig.HealthTimeout == 0 { task.CSIPluginConfig.HealthTimeout = 30 * time.Second } @@ -157,8 +167,7 @@ func (h *csiPluginSupervisorHook) Prestart(ctx context.Context, } // where the staging and per-alloc directories will be mounted volumeStagingMounts := &drivers.MountConfig{ - // TODO(tgross): add this TaskPath to the CSIPluginConfig as well - TaskPath: "/local/csi", + TaskPath: h.task.CSIPluginConfig.StagePublishBaseDir, HostPath: h.mountPoint, Readonly: false, PropagationMode: "bidirectional", @@ -360,7 +369,7 @@ func (h *csiPluginSupervisorHook) registerPlugin(client csi.CSIPlugin, socketPat Options: map[string]string{ "Provider": info.Name, // vendor name "MountPoint": h.mountPoint, - "ContainerMountPoint": "/local/csi", + "ContainerMountPoint": h.task.CSIPluginConfig.StagePublishBaseDir, }, } } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index f7a74c4f8..30513725a 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1263,6 +1263,7 @@ func ApiCSIPluginConfigToStructsCSIPluginConfig(apiConfig *api.TaskCSIPluginConf sc.ID = apiConfig.ID sc.Type = structs.CSIPluginType(apiConfig.Type) sc.MountDir = apiConfig.MountDir + sc.StagePublishBaseDir = apiConfig.StagePublishBaseDir sc.HealthTimeout = apiConfig.HealthTimeout return sc } diff --git a/nomad/structs/csi.go b/nomad/structs/csi.go index eea20b597..75601706b 100644 --- a/nomad/structs/csi.go +++ b/nomad/structs/csi.go @@ -62,12 +62,17 @@ type TaskCSIPluginConfig struct { // Type instructs Nomad on how to handle processing a plugin Type CSIPluginType - // MountDir is the destination that nomad should mount in its CSI - // directory for the plugin. It will then expect a file called CSISocketName - // to be created by the plugin, and will provide references into - // "MountDir/CSIIntermediaryDirname/{VolumeName}/{AllocID} for mounts. + // MountDir is the directory (within its container) in which the plugin creates a + // socket (called CSISocketName) for communication with Nomad. Default is /csi. MountDir string + // StagePublishBaseDir is the base directory (within its container) in which the plugin + // mounts volumes being staged and bind mount volumes being published. + // e.g. staging_target_path = {StagePublishBaseDir}/staging/{volume-id}/{usage-mode} + // e.g. target_path = {StagePublishBaseDir}/per-alloc/{alloc-id}/{volume-id}/{usage-mode} + // Default is /local/csi. + StagePublishBaseDir string + // HealthTimeout is the time after which the CSI plugin tasks will be killed // if the CSI Plugin is not healthy. HealthTimeout time.Duration `mapstructure:"health_timeout" hcl:"health_timeout,optional"` diff --git a/website/content/docs/concepts/plugins/csi.mdx b/website/content/docs/concepts/plugins/csi.mdx index fa2231959..f711600fe 100644 --- a/website/content/docs/concepts/plugins/csi.mdx +++ b/website/content/docs/concepts/plugins/csi.mdx @@ -38,9 +38,10 @@ A CSI plugin task requires the [`csi_plugin`][csi_plugin] block: ```hcl csi_plugin { - id = "csi-hostpath" - type = "monolith" - mount_dir = "/csi" + id = "csi-hostpath" + type = "monolith" + mount_dir = "/csi" + stage_publish_base_dir = "/local/csi" } ``` @@ -73,7 +74,11 @@ Nomad exposes a Unix domain socket named `csi.sock` inside each CSI plugin task, and communicates over the gRPC protocol expected by the CSI specification. The `mount_dir` field tells Nomad where the plugin expects to find the socket file. The path to this socket is exposed in -the container as the `CSI_ENDPOINT` environment variable. +the container as the `CSI_ENDPOINT` environment variable. + +Some plugins also require the `stage_publish_base_dir` field, which +tells Nomad where to instruct the plugin to mount volumes for staging +and/or publishing. ### Plugin Lifecycle and State diff --git a/website/content/docs/job-specification/csi_plugin.mdx b/website/content/docs/job-specification/csi_plugin.mdx index 4048943e3..04a314a63 100644 --- a/website/content/docs/job-specification/csi_plugin.mdx +++ b/website/content/docs/job-specification/csi_plugin.mdx @@ -17,10 +17,11 @@ to claim [volumes][csi_volumes]. ```hcl csi_plugin { - id = "csi-hostpath" - type = "monolith" - mount_dir = "/csi" - health_timeout = "30s" + id = "csi-hostpath" + type = "monolith" + mount_dir = "/csi" + stage_publish__base_dir = "/local/csi" + health_timeout = "30s" } ``` @@ -40,9 +41,15 @@ csi_plugin { `node` at the same time, and these are called `monolith` plugins. Refer to your CSI plugin's documentation. -- `mount_dir` `(string: )` - The directory path inside the +- `mount_dir` `(string: )` - The directory path inside the container where the plugin will expect a Unix domain socket for - bidirectional communication with Nomad. + bidirectional communication with Nomad. This field is typically not + required. Refer to your CSI plugin's documentation for details. + +- `stage_publish_base_dir` `(string: )` - The base directory + path inside the container where the plugin will be instructed to + stage and publish volumes. This field is typically not required. + Refer to your CSI plugin's documentation for details. - `health_timeout` `(duration: )` - The duration that the plugin supervisor will wait before restarting an unhealthy