luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

CSI: use AccessMode/AttachmentMode from CSIVolumeClaim 

Registration of Nomad volumes previously allowed for a single volume
capability (access mode + attachment mode pair). The recent `volume create`
command requires that we pass a list of requested capabilities, but the
existing workflow for claiming volumes and attaching them on the client
assumed that the volume's single capability was correct and unchanging.

Add `AccessMode` and `AttachmentMode` to `CSIVolumeClaim`, use these fields to
set the initial claim value, and add backwards compatibility logic to handle
the existing volumes that already have claims without these fields.
This commit is contained in:
Tim Gross 2021-04-02 14:36:13 -04:00 committed by Tim Gross
parent dbcc2694b0
commit 276633673d
14 changed files with 670 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
api/tasks.go
View File

@ -377,13 +377,15 @@ func (m *MigrateStrategy) Copy() *MigrateStrategy {
// VolumeRequest is a representation of a storage volume that a TaskGroup wishes to use.
type VolumeRequest struct {
Name string `hcl:"name,label"`
Type string `hcl:"type,optional"`
Source string `hcl:"source,optional"`
ReadOnly bool `hcl:"read_only,optional"`
MountOptions *CSIMountOptions `hcl:"mount_options,block"`
PerAlloc bool `hcl:"per_alloc,optional"`
ExtraKeysHCL []string `hcl1:",unusedKeys,optional" json:"-"`
Name string `hcl:"name,label"`
Type string `hcl:"type,optional"`
Source string `hcl:"source,optional"`
ReadOnly bool `hcl:"read_only,optional"`
AccessMode string `hcl:"access_mode,optional"`
AttachmentMode string `hcl:"attachment_mode,optional"`
MountOptions *CSIMountOptions `hcl:"mount_options,block"`
PerAlloc bool `hcl:"per_alloc,optional"`
ExtraKeysHCL []string `hcl1:",unusedKeys,optional" json:"-"`
}
const (

15
client/allocrunner/csi_hook.go
View File

@ -54,8 +54,8 @@ func (c *csiHook) Prerun() error {
usageOpts := &csimanager.UsageOptions{
ReadOnly: pair.request.ReadOnly,
AttachmentMode: string(pair.volume.AttachmentMode),
AccessMode: string(pair.volume.AccessMode),
AttachmentMode: pair.request.AttachmentMode,
AccessMode: pair.request.AccessMode,
MountOptions: pair.request.MountOptions,
}
@ -171,10 +171,12 @@ func (c *csiHook) claimVolumesFromAlloc() (map[string]*volumeAndRequest, error)
}
req := &structs.CSIVolumeClaimRequest{
VolumeID: source,
AllocationID: c.alloc.ID,
NodeID: c.alloc.NodeID,
Claim: claimType,
VolumeID: source,
AllocationID: c.alloc.ID,
NodeID: c.alloc.NodeID,
Claim: claimType,
AccessMode: pair.request.AccessMode,
AttachmentMode: pair.request.AttachmentMode,
WriteRequest: structs.WriteRequest{
Region: c.alloc.Job.Region,
Namespace: c.alloc.Job.Namespace,
@ -191,6 +193,7 @@ func (c *csiHook) claimVolumesFromAlloc() (map[string]*volumeAndRequest, error)
return nil, fmt.Errorf("Unexpected nil volume returned for ID: %v", pair.request.Source)
}
result[alias].request = pair.request
result[alias].volume = resp.Volume
result[alias].publishContext = resp.PublishContext
}

4
client/csi_endpoint.go
View File

@ -468,8 +468,8 @@ func (c *CSI) NodeDetachVolume(req *structs.ClientCSINodeDetachVolumeRequest, re
usageOpts := &csimanager.UsageOptions{
ReadOnly: req.ReadOnly,
AttachmentMode: string(req.AttachmentMode),
AccessMode: string(req.AccessMode),
AttachmentMode: req.AttachmentMode,
AccessMode: req.AccessMode,
}
err = mounter.UnmountVolume(ctx, req.VolumeID, req.ExternalID, req.AllocID, usageOpts)

8
client/pluginmanager/csimanager/interface.go
View File

@ -15,8 +15,8 @@ type MountInfo struct {
type UsageOptions struct {
ReadOnly bool
AttachmentMode string
AccessMode string
AttachmentMode structs.CSIVolumeAttachmentMode
AccessMode structs.CSIVolumeAccessMode
MountOptions *structs.CSIMountOptions
}
@ -33,9 +33,9 @@ func (u *UsageOptions) ToFS() string {
sb.WriteString("rw-")
}
sb.WriteString(u.AttachmentMode)
sb.WriteString(string(u.AttachmentMode))
sb.WriteString("-")
sb.WriteString(u.AccessMode)
sb.WriteString(string(u.AccessMode))
return sb.String()
}

2
client/pluginmanager/csimanager/volume.go
View File

@ -127,7 +127,7 @@ func (v *volumeManager) ensureAllocDir(vol *structs.CSIVolume, alloc *structs.Al
}
func volumeCapability(vol *structs.CSIVolume, usage *UsageOptions) (*csi.VolumeCapability, error) {
capability, err := csi.VolumeCapabilityFromStructs(vol.AttachmentMode, vol.AccessMode)
capability, err := csi.VolumeCapabilityFromStructs(usage.AttachmentMode, usage.AccessMode)
if err != nil {
return nil, err
}

66
client/pluginmanager/csimanager/volume_test.go
View File

@ -148,43 +148,45 @@ func TestVolumeManager_stageVolume(t *testing.T) {
{
Name: "Returns an error when an invalid AttachmentMode is provided",
Volume: &structs.CSIVolume{
ID: "foo",
AttachmentMode: "nonsense",
ID: "foo",
},
UsageOptions: &UsageOptions{},
UsageOptions: &UsageOptions{AttachmentMode: "nonsense"},
ExpectedErr: errors.New("unknown volume attachment mode: nonsense"),
},
{
Name: "Returns an error when an invalid AccessMode is provided",
Volume: &structs.CSIVolume{
ID: "foo",
ID: "foo",
},
UsageOptions: &UsageOptions{
AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice,
AccessMode: "nonsense",
},
UsageOptions: &UsageOptions{},
ExpectedErr: errors.New("unknown volume access mode: nonsense"),
ExpectedErr: errors.New("unknown volume access mode: nonsense"),
},
{
Name: "Returns an error when the plugin returns an error",
Volume: &structs.CSIVolume{
ID: "foo",
ID: "foo",
},
UsageOptions: &UsageOptions{
AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice,
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
},
UsageOptions: &UsageOptions{},
PluginErr: errors.New("Some Unknown Error"),
ExpectedErr: errors.New("Some Unknown Error"),
PluginErr: errors.New("Some Unknown Error"),
ExpectedErr: errors.New("Some Unknown Error"),
},
{
Name: "Happy Path",
Volume: &structs.CSIVolume{
ID: "foo",
ID: "foo",
},
UsageOptions: &UsageOptions{
AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice,
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
},
UsageOptions: &UsageOptions{},
PluginErr: nil,
ExpectedErr: nil,
PluginErr: nil,
ExpectedErr: nil,
},
}
@ -293,11 +295,12 @@ func TestVolumeManager_publishVolume(t *testing.T) {
Name: "Returns an error when the plugin returns an error",
Allocation: structs.MockAlloc(),
Volume: &structs.CSIVolume{
ID: "foo",
ID: "foo",
},
UsageOptions: &UsageOptions{
AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice,
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
},
UsageOptions: &UsageOptions{},
PluginErr: errors.New("Some Unknown Error"),
ExpectedErr: errors.New("Some Unknown Error"),
ExpectedCSICallCount: 1,
@ -306,11 +309,12 @@ func TestVolumeManager_publishVolume(t *testing.T) {
Name: "Happy Path",
Allocation: structs.MockAlloc(),
Volume: &structs.CSIVolume{
ID: "foo",
ID: "foo",
},
UsageOptions: &UsageOptions{
AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice,
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
},
UsageOptions: &UsageOptions{},
PluginErr: nil,
ExpectedErr: nil,
ExpectedCSICallCount: 1,
@ -319,14 +323,15 @@ func TestVolumeManager_publishVolume(t *testing.T) {
Name: "Mount options in the volume",
Allocation: structs.MockAlloc(),
Volume: &structs.CSIVolume{
ID: "foo",
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
ID: "foo",
MountOptions: &structs.CSIMountOptions{
MountFlags: []string{"ro"},
},
},
UsageOptions: &UsageOptions{},
UsageOptions: &UsageOptions{
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
},
PluginErr: nil,
ExpectedErr: nil,
ExpectedCSICallCount: 1,
@ -342,14 +347,14 @@ func TestVolumeManager_publishVolume(t *testing.T) {
Name: "Mount options override in the request",
Allocation: structs.MockAlloc(),
Volume: &structs.CSIVolume{
ID: "foo",
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
ID: "foo",
MountOptions: &structs.CSIMountOptions{
MountFlags: []string{"ro"},
},
},
UsageOptions: &UsageOptions{
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
MountOptions: &structs.CSIMountOptions{
MountFlags: []string{"rw"},
},
@ -481,12 +486,13 @@ func TestVolumeManager_MountVolumeEvents(t *testing.T) {
manager := newVolumeManager(testlog.HCLogger(t), eventer, csiFake, tmpPath, tmpPath, true)
ctx := context.Background()
vol := &structs.CSIVolume{
ID: "vol",
Namespace: "ns",
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
ID: "vol",
Namespace: "ns",
}
alloc := mock.Alloc()
usage := &UsageOptions{}
usage := &UsageOptions{
AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter,
}
pubCtx := map[string]string{}
_, err := manager.MountVolume(ctx, vol, alloc, usage, pubCtx)
@ -500,7 +506,7 @@ func TestVolumeManager_MountVolumeEvents(t *testing.T) {
require.Equal(t, "unknown volume attachment mode: ", e.Details["error"])
events = events[1:]
vol.AttachmentMode = structs.CSIVolumeAttachmentModeFilesystem
usage.AttachmentMode = structs.CSIVolumeAttachmentModeFilesystem
_, err = manager.MountVolume(ctx, vol, alloc, usage, pubCtx)
require.NoError(t, err)

12
command/agent/job_endpoint.go
View File

@ -940,11 +940,13 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta
}
vol := &structs.VolumeRequest{
Name: v.Name,
Type: v.Type,
ReadOnly: v.ReadOnly,
Source: v.Source,
PerAlloc: v.PerAlloc,
Name: v.Name,
Type: v.Type,
ReadOnly: v.ReadOnly,
Source: v.Source,
AttachmentMode: structs.CSIVolumeAttachmentMode(v.AttachmentMode),
AccessMode: structs.CSIVolumeAccessMode(v.AccessMode),
PerAlloc: v.PerAlloc,
}
if v.MountOptions != nil {

5
nomad/csi_endpoint.go
View File

@ -377,7 +377,6 @@ func (v *CSIVolume) Claim(args *structs.CSIVolumeClaimRequest, reply *structs.CS
isNewClaim := args.Claim != structs.CSIVolumeClaimGC &&
args.State == structs.CSIVolumeClaimStateTaken
// COMPAT(1.0): the NodeID field was added after 0.11.0 and so we
// need to ensure it's been populated during upgrades from 0.11.0
// to later patch versions. Remove this block in 1.0
@ -470,8 +469,8 @@ func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest,
cReq := &cstructs.ClientCSIControllerAttachVolumeRequest{
VolumeID: vol.RemoteID(),
ClientCSINodeID: externalNodeID,
AttachmentMode: vol.AttachmentMode,
AccessMode: vol.AccessMode,
AttachmentMode: req.AttachmentMode,
AccessMode: req.AccessMode,
ReadOnly: req.Claim == structs.CSIVolumeClaimRead,
Secrets: vol.Secrets,
VolumeContext: vol.Context,

17
nomad/state/state_store.go
View File

@ -2479,6 +2479,23 @@ func (s *StateStore) CSIVolumeDenormalizeTxn(txn Txn, ws memdb.WatchSet, vol *st
}
}
// COMPAT: the AccessMode and AttachmentMode fields were added to claims
// in 1.1.0, so claims made before that may be missing this value. In this
// case, the volume will already have AccessMode/AttachmentMode until it
// no longer has any claims, so set from those values
for _, claim := range vol.ReadClaims {
if claim.AccessMode == "" || claim.AttachmentMode == "" {
claim.AccessMode = vol.AccessMode
claim.AttachmentMode = vol.AttachmentMode
}
}
for _, claim := range vol.WriteClaims {
if claim.AccessMode == "" || claim.AttachmentMode == "" {
claim.AccessMode = vol.AccessMode
claim.AttachmentMode = vol.AttachmentMode
}
}
return vol, nil
}

101
nomad/structs/csi.go
View File

@ -224,6 +224,8 @@ type CSIVolumeClaim struct {
NodeID string
ExternalNodeID string
Mode CSIVolumeClaimMode
AccessMode CSIVolumeAccessMode
AttachmentMode CSIVolumeAttachmentMode
State CSIVolumeClaimState
}
@ -247,13 +249,14 @@ type CSIVolume struct {
ExternalID string
Namespace string
Topologies []*CSITopology
AccessMode CSIVolumeAccessMode
AttachmentMode CSIVolumeAttachmentMode
AccessMode CSIVolumeAccessMode // *current* access mode
AttachmentMode CSIVolumeAttachmentMode // *current* attachment mode
MountOptions *CSIMountOptions
Secrets CSISecrets
Parameters map[string]string
Context map[string]string
Capacity int64 // bytes
Secrets CSISecrets
Parameters map[string]string
Context map[string]string
Capacity int64 // bytes
// These values are used only on volume creation but we record them
// so that we can diff the volume later
@ -376,19 +379,32 @@ func (v *CSIVolume) ReadSchedulable() bool {
return v.ResourceExhausted == time.Time{}
}
// WriteSchedulable determines if the volume is schedulable for writes, considering only
// volume health
// WriteSchedulable determines if the volume is schedulable for writes,
// considering only volume capabilities and plugin health
func (v *CSIVolume) WriteSchedulable() bool {
if !v.Schedulable {
return false
}
switch v.AccessMode {
case CSIVolumeAccessModeSingleNodeWriter, CSIVolumeAccessModeMultiNodeSingleWriter, CSIVolumeAccessModeMultiNodeMultiWriter:
case CSIVolumeAccessModeSingleNodeWriter,
CSIVolumeAccessModeMultiNodeSingleWriter,
CSIVolumeAccessModeMultiNodeMultiWriter:
return v.ResourceExhausted == time.Time{}
default:
return false
case CSIVolumeAccessModeUnknown:
// this volume was created but not currently claimed, so we check what
// it's capable of, not what it's been previously assigned
for _, cap := range v.RequestedCapabilities {
switch cap.AccessMode {
case CSIVolumeAccessModeSingleNodeWriter,
CSIVolumeAccessModeMultiNodeSingleWriter,
CSIVolumeAccessModeMultiNodeMultiWriter:
return v.ResourceExhausted == time.Time{}
}
}
}
return false
}
// WriteFreeClaims determines if there are any free write claims available
@ -401,9 +417,25 @@ func (v *CSIVolume) WriteFreeClaims() bool {
// we track node resource exhaustion through v.ResourceExhausted
// which is checked in WriteSchedulable
return true
default:
return false
case CSIVolumeAccessModeUnknown:
// this volume was created but not yet claimed, so we check what it's
// capable of, not what it's been assigned
if len(v.RequestedCapabilities) == 0 {
// COMPAT: a volume that was registered before 1.1.0 and has not
// had a change in claims could have no requested caps. It will
// get corrected on the first claim.
return true
}
for _, cap := range v.RequestedCapabilities {
switch cap.AccessMode {
case CSIVolumeAccessModeSingleNodeWriter, CSIVolumeAccessModeMultiNodeSingleWriter:
return len(v.WriteClaims) == 0
case CSIVolumeAccessModeMultiNodeMultiWriter:
return true
}
}
}
return false
}
// InUse tests whether any allocations are actively using the volume
@ -459,6 +491,23 @@ func (v *CSIVolume) Copy() *CSIVolume {
// Claim updates the allocations and changes the volume state
func (v *CSIVolume) Claim(claim *CSIVolumeClaim, alloc *Allocation) error {
// COMPAT: volumes registered prior to 1.1.0 will be missing caps for the
// volume on any claim. Correct this when we make the first change to a
// claim by setting its currently claimed capability as the only requested
// capability
if len(v.RequestedCapabilities) == 0 && v.AccessMode != "" && v.AttachmentMode != "" {
v.RequestedCapabilities = []*CSIVolumeCapability{
{
AccessMode: v.AccessMode,
AttachmentMode: v.AttachmentMode,
},
}
}
if v.AttachmentMode != CSIVolumeAttachmentModeUnknown &&
claim.AttachmentMode != CSIVolumeAttachmentModeUnknown &&
v.AttachmentMode != claim.AttachmentMode {
return fmt.Errorf("cannot change attachment mode of claimed volume")
}
if claim.State == CSIVolumeClaimStateTaken {
switch claim.Mode {
@ -494,6 +543,7 @@ func (v *CSIVolume) claimRead(claim *CSIVolumeClaim, alloc *Allocation) error {
delete(v.WriteClaims, claim.AllocationID)
delete(v.PastClaims, claim.AllocationID)
v.setModesFromClaim(claim)
return nil
}
@ -528,9 +578,24 @@ func (v *CSIVolume) claimWrite(claim *CSIVolumeClaim, alloc *Allocation) error {
delete(v.ReadClaims, alloc.ID)
delete(v.PastClaims, alloc.ID)
v.setModesFromClaim(claim)
return nil
}
// setModesFromClaim sets the volume AttachmentMode and AccessMode based on
// the first claim we make. Originally the volume AccessMode and
// AttachmentMode were set during registration, but this is incorrect once we
// started creating volumes ourselves. But we still want these values for CLI
// and UI status.
func (v *CSIVolume) setModesFromClaim(claim *CSIVolumeClaim) {
if v.AttachmentMode == CSIVolumeAttachmentModeUnknown {
v.AttachmentMode = claim.AttachmentMode
}
if v.AccessMode == CSIVolumeAccessModeUnknown {
v.AccessMode = claim.AccessMode
}
}
// claimRelease is called when the allocation has terminated and
// already stopped using the volume
func (v *CSIVolume) claimRelease(claim *CSIVolumeClaim) error {
@ -540,6 +605,12 @@ func (v *CSIVolume) claimRelease(claim *CSIVolumeClaim) error {
delete(v.ReadClaims, claim.AllocationID)
delete(v.WriteClaims, claim.AllocationID)
delete(v.PastClaims, claim.AllocationID)
// remove AccessMode/AttachmentMode if this is the last claim
if len(v.ReadClaims) == 0 && len(v.WriteClaims) == 0 && len(v.PastClaims) == 0 {
v.AccessMode = CSIVolumeAccessModeUnknown
v.AttachmentMode = CSIVolumeAttachmentModeUnknown
}
} else {
v.PastClaims[claim.AllocationID] = claim
}
@ -674,6 +745,8 @@ type CSIVolumeClaimRequest struct {
NodeID string
ExternalNodeID string
Claim CSIVolumeClaimMode
AccessMode CSIVolumeAccessMode
AttachmentMode CSIVolumeAttachmentMode
State CSIVolumeClaimState
WriteRequest
}
@ -684,6 +757,8 @@ func (req *CSIVolumeClaimRequest) ToClaim() *CSIVolumeClaim {
NodeID: req.NodeID,
ExternalNodeID: req.ExternalNodeID,
Mode: req.Claim,
AccessMode: req.AccessMode,
AttachmentMode: req.AttachmentMode,
State: req.State,
}
}

481
nomad/structs/csi_test.go
View File

@ -8,41 +8,470 @@ import (
"github.com/stretchr/testify/require"
)
// TestCSIVolumeClaim ensures that a volume claim workflows work as expected.
func TestCSIVolumeClaim(t *testing.T) {
vol := NewCSIVolume("", 0)
vol.AccessMode = CSIVolumeAccessModeMultiNodeSingleWriter
require := require.New(t)
vol := NewCSIVolume("vol0", 0)
vol.Schedulable = true
alloc := &Allocation{ID: "a1", Namespace: "n", JobID: "j"}
claim := &CSIVolumeClaim{
AllocationID: alloc.ID,
NodeID: "foo",
Mode: CSIVolumeClaimRead,
vol.AccessMode = CSIVolumeAccessModeUnknown
vol.AttachmentMode = CSIVolumeAttachmentModeUnknown
vol.RequestedCapabilities = []*CSIVolumeCapability{
{
AccessMode: CSIVolumeAccessModeMultiNodeSingleWriter,
AttachmentMode: CSIVolumeAttachmentModeFilesystem,
},
{
AccessMode: CSIVolumeAccessModeMultiNodeReader,
AttachmentMode: CSIVolumeAttachmentModeFilesystem,
},
}
require.NoError(t, vol.claimRead(claim, alloc))
require.True(t, vol.ReadSchedulable())
require.True(t, vol.WriteSchedulable())
require.NoError(t, vol.claimRead(claim, alloc))
alloc1 := &Allocation{ID: "a1", Namespace: "n", JobID: "j"}
alloc2 := &Allocation{ID: "a2", Namespace: "n", JobID: "j"}
alloc3 := &Allocation{ID: "a3", Namespace: "n", JobID: "j3"}
claim := &CSIVolumeClaim{
AllocationID: alloc1.ID,
NodeID: "foo",
State: CSIVolumeClaimStateTaken,
}
// claim a read and ensure we are still schedulable
claim.Mode = CSIVolumeClaimRead
claim.AccessMode = CSIVolumeAccessModeMultiNodeReader
claim.AttachmentMode = CSIVolumeAttachmentModeFilesystem
require.NoError(vol.Claim(claim, alloc1))
require.True(vol.ReadSchedulable())
require.False(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 0)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeReader, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
require.Len(vol.RequestedCapabilities, 2)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAccessModeMultiNodeReader,
vol.RequestedCapabilities[1].AccessMode)
// claim a write and ensure we can't upgrade capabilities.
claim.AccessMode = CSIVolumeAccessModeMultiNodeSingleWriter
claim.Mode = CSIVolumeClaimWrite
require.NoError(t, vol.claimWrite(claim, alloc))
require.True(t, vol.ReadSchedulable())
require.False(t, vol.WriteFreeClaims())
vol.claimRelease(claim)
require.True(t, vol.ReadSchedulable())
require.False(t, vol.WriteFreeClaims())
claim.AllocationID = alloc2.ID
require.EqualError(vol.Claim(claim, alloc2), "unschedulable")
require.True(vol.ReadSchedulable())
require.False(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 0)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeReader, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// release our last claim, including unpublish workflow
claim.AllocationID = alloc1.ID
claim.Mode = CSIVolumeClaimRead
claim.State = CSIVolumeClaimStateReadyToFree
vol.claimRelease(claim)
require.True(t, vol.ReadSchedulable())
require.True(t, vol.WriteFreeClaims())
vol.Claim(claim, nil)
require.Len(vol.ReadClaims, 0)
require.Len(vol.WriteClaims, 0)
require.Equal(CSIVolumeAccessModeUnknown, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeUnknown, vol.AttachmentMode)
require.Len(vol.RequestedCapabilities, 2)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAccessModeMultiNodeReader,
vol.RequestedCapabilities[1].AccessMode)
vol.AccessMode = CSIVolumeAccessModeMultiNodeMultiWriter
require.NoError(t, vol.claimWrite(claim, alloc))
require.NoError(t, vol.claimWrite(claim, alloc))
require.True(t, vol.WriteFreeClaims())
// claim a write on the now-unclaimed volume and ensure we can upgrade
// capabilities so long as they're in our RequestedCapabilities.
claim.AccessMode = CSIVolumeAccessModeMultiNodeSingleWriter
claim.Mode = CSIVolumeClaimWrite
claim.State = CSIVolumeClaimStateTaken
claim.AllocationID = alloc2.ID
require.NoError(vol.Claim(claim, alloc2))
require.Len(vol.ReadClaims, 0)
require.Len(vol.WriteClaims, 1)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
require.Len(vol.RequestedCapabilities, 2)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAccessModeMultiNodeReader,
vol.RequestedCapabilities[1].AccessMode)
// make the claim again to ensure its idempotent, and that the volume's
// access mode is unchanged.
require.NoError(vol.Claim(claim, alloc2))
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 0)
require.Len(vol.WriteClaims, 1)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// claim a read. ensure we are still schedulable and that we haven't
// changed the access mode
claim.AllocationID = alloc1.ID
claim.Mode = CSIVolumeClaimRead
claim.AccessMode = CSIVolumeAccessModeMultiNodeReader
claim.AttachmentMode = CSIVolumeAttachmentModeFilesystem
require.NoError(vol.Claim(claim, alloc1))
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 1)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// ensure we can't change the attachment mode for a claimed volume
claim.AttachmentMode = CSIVolumeAttachmentModeBlockDevice
claim.AllocationID = alloc3.ID
require.EqualError(vol.Claim(claim, alloc3),
"cannot change attachment mode of claimed volume")
claim.AttachmentMode = CSIVolumeAttachmentModeFilesystem
// denormalize-on-read (simulating a volume we've gotten out of the state
// store) and then ensure we cannot claim another write
vol.WriteAllocs[alloc2.ID] = alloc2
claim.Mode = CSIVolumeClaimWrite
require.EqualError(vol.Claim(claim, alloc3), "volume max claim reached")
// release the write claim but ensure it doesn't free up write claims
// until after we've unpublished
claim.AllocationID = alloc2.ID
claim.State = CSIVolumeClaimStateUnpublishing
vol.Claim(claim, nil)
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 1) // claim still exists until we're done
require.Len(vol.PastClaims, 1)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// complete the unpublish workflow
claim.State = CSIVolumeClaimStateReadyToFree
vol.Claim(claim, nil)
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.True(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 0)
require.Len(vol.WriteAllocs, 0)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// release our last claim, including unpublish workflow
claim.AllocationID = alloc1.ID
claim.Mode = CSIVolumeClaimRead
vol.Claim(claim, nil)
require.Len(vol.ReadClaims, 0)
require.Len(vol.WriteClaims, 0)
require.Equal(CSIVolumeAccessModeUnknown, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeUnknown, vol.AttachmentMode)
require.Len(vol.RequestedCapabilities, 2)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAccessModeMultiNodeReader,
vol.RequestedCapabilities[1].AccessMode)
}
// TestCSIVolumeClaim_CompatOldClaims ensures that volume created before
// v1.1.0 with claims that exist before v1.1.0 still work.
//
// COMPAT(1.3.0): safe to remove this test, but not the code, for 1.3.0
func TestCSIVolumeClaim_CompatOldClaims(t *testing.T) {
require := require.New(t)
vol := NewCSIVolume("vol0", 0)
vol.Schedulable = true
vol.AccessMode = CSIVolumeAccessModeMultiNodeSingleWriter
vol.AttachmentMode = CSIVolumeAttachmentModeFilesystem
alloc1 := &Allocation{ID: "a1", Namespace: "n", JobID: "j"}
alloc2 := &Allocation{ID: "a2", Namespace: "n", JobID: "j"}
alloc3 := &Allocation{ID: "a3", Namespace: "n", JobID: "j3"}
claim := &CSIVolumeClaim{
AllocationID: alloc1.ID,
NodeID: "foo",
State: CSIVolumeClaimStateTaken,
}
// claim a read and ensure we are still schedulable
claim.Mode = CSIVolumeClaimRead
require.NoError(vol.Claim(claim, alloc1))
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.True(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 0)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
require.Len(vol.RequestedCapabilities, 1)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem,
vol.RequestedCapabilities[0].AttachmentMode)
// claim a write and ensure we no longer have free write claims
claim.Mode = CSIVolumeClaimWrite
claim.AllocationID = alloc2.ID
require.NoError(vol.Claim(claim, alloc2))
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 1)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// denormalize-on-read (simulating a volume we've gotten out of the state
// store) and then ensure we cannot claim another write
vol.WriteAllocs[alloc2.ID] = alloc2
claim.AllocationID = alloc3.ID
require.EqualError(vol.Claim(claim, alloc3), "volume max claim reached")
// release the write claim but ensure it doesn't free up write claims
// until after we've unpublished
claim.AllocationID = alloc2.ID
claim.State = CSIVolumeClaimStateUnpublishing
vol.Claim(claim, nil)
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 1) // claim still exists until we're done
require.Len(vol.PastClaims, 1)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// complete the unpublish workflow
claim.State = CSIVolumeClaimStateReadyToFree
vol.Claim(claim, nil)
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.True(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 0)
require.Len(vol.WriteAllocs, 0)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// release our last claim, including unpublish workflow
claim.AllocationID = alloc1.ID
claim.Mode = CSIVolumeClaimRead
vol.Claim(claim, nil)
require.Len(vol.ReadClaims, 0)
require.Len(vol.WriteClaims, 0)
require.Equal(CSIVolumeAccessModeUnknown, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeUnknown, vol.AttachmentMode)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem,
vol.RequestedCapabilities[0].AttachmentMode)
}
// TestCSIVolumeClaim_CompatNewClaimsOK ensures that a volume created
// before v1.1.0 is compatible with new claims.
//
// COMPAT(1.3.0): safe to remove this test, but not the code, for 1.3.0
func TestCSIVolumeClaim_CompatNewClaimsOK(t *testing.T) {
require := require.New(t)
vol := NewCSIVolume("vol0", 0)
vol.Schedulable = true
vol.AccessMode = CSIVolumeAccessModeMultiNodeSingleWriter
vol.AttachmentMode = CSIVolumeAttachmentModeFilesystem
alloc1 := &Allocation{ID: "a1", Namespace: "n", JobID: "j"}
alloc2 := &Allocation{ID: "a2", Namespace: "n", JobID: "j"}
alloc3 := &Allocation{ID: "a3", Namespace: "n", JobID: "j3"}
claim := &CSIVolumeClaim{
AllocationID: alloc1.ID,
NodeID: "foo",
State: CSIVolumeClaimStateTaken,
}
// claim a read and ensure we are still schedulable
claim.Mode = CSIVolumeClaimRead
claim.AccessMode = CSIVolumeAccessModeMultiNodeReader
claim.AttachmentMode = CSIVolumeAttachmentModeFilesystem
require.NoError(vol.Claim(claim, alloc1))
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.True(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 0)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
require.Len(vol.RequestedCapabilities, 1)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem,
vol.RequestedCapabilities[0].AttachmentMode)
// claim a write and ensure we no longer have free write claims
claim.Mode = CSIVolumeClaimWrite
claim.AllocationID = alloc2.ID
require.NoError(vol.Claim(claim, alloc2))
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 1)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// ensure we can't change the attachment mode for a claimed volume
claim.AttachmentMode = CSIVolumeAttachmentModeBlockDevice
require.EqualError(vol.Claim(claim, alloc2),
"cannot change attachment mode of claimed volume")
claim.AttachmentMode = CSIVolumeAttachmentModeFilesystem
// denormalize-on-read (simulating a volume we've gotten out of the state
// store) and then ensure we cannot claim another write
vol.WriteAllocs[alloc2.ID] = alloc2
claim.AllocationID = alloc3.ID
require.EqualError(vol.Claim(claim, alloc3), "volume max claim reached")
// release the write claim but ensure it doesn't free up write claims
// until after we've unpublished
claim.AllocationID = alloc2.ID
claim.State = CSIVolumeClaimStateUnpublishing
vol.Claim(claim, nil)
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 1) // claim still exists until we're done
require.Len(vol.PastClaims, 1)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// complete the unpublish workflow
claim.State = CSIVolumeClaimStateReadyToFree
vol.Claim(claim, nil)
require.True(vol.ReadSchedulable())
require.True(vol.WriteSchedulable())
require.True(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 0)
require.Len(vol.WriteAllocs, 0)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
// release our last claim, including unpublish workflow
claim.AllocationID = alloc1.ID
claim.Mode = CSIVolumeClaimRead
vol.Claim(claim, nil)
require.Len(vol.ReadClaims, 0)
require.Len(vol.WriteClaims, 0)
require.Equal(CSIVolumeAccessModeUnknown, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeUnknown, vol.AttachmentMode)
require.Equal(CSIVolumeAccessModeMultiNodeSingleWriter,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem,
vol.RequestedCapabilities[0].AttachmentMode)
}
// TestCSIVolumeClaim_CompatNewClaimsNoUpgrade ensures that a volume created
// before v1.1.0 is compatible with new claims, but prevents unexpected
// capability upgrades.
//
// COMPAT(1.3.0): safe to remove this test, but not the code, for 1.3.0
func TestCSIVolumeClaim_CompatNewClaimsNoUpgrade(t *testing.T) {
require := require.New(t)
vol := NewCSIVolume("vol0", 0)
vol.Schedulable = true
vol.AccessMode = CSIVolumeAccessModeMultiNodeReader
vol.AttachmentMode = CSIVolumeAttachmentModeFilesystem
alloc1 := &Allocation{ID: "a1", Namespace: "n", JobID: "j"}
alloc2 := &Allocation{ID: "a2", Namespace: "n", JobID: "j"}
claim := &CSIVolumeClaim{
AllocationID: alloc1.ID,
NodeID: "foo",
State: CSIVolumeClaimStateTaken,
}
// claim a read and ensure we are still schedulable
claim.Mode = CSIVolumeClaimRead
claim.AccessMode = CSIVolumeAccessModeMultiNodeReader
claim.AttachmentMode = CSIVolumeAttachmentModeFilesystem
require.NoError(vol.Claim(claim, alloc1))
require.True(vol.ReadSchedulable())
require.False(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 0)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeReader, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
require.Len(vol.RequestedCapabilities, 1)
require.Equal(CSIVolumeAccessModeMultiNodeReader,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem,
vol.RequestedCapabilities[0].AttachmentMode)
// claim a write and ensure we can't upgrade capabilities.
claim.AccessMode = CSIVolumeAccessModeMultiNodeSingleWriter
claim.Mode = CSIVolumeClaimWrite
claim.AllocationID = alloc2.ID
require.EqualError(vol.Claim(claim, alloc2), "unschedulable")
require.True(vol.ReadSchedulable())
require.False(vol.WriteSchedulable())
require.False(vol.WriteFreeClaims())
require.Len(vol.ReadClaims, 1)
require.Len(vol.WriteClaims, 0)
require.Len(vol.PastClaims, 0)
require.Equal(CSIVolumeAccessModeMultiNodeReader, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem, vol.AttachmentMode)
require.Len(vol.RequestedCapabilities, 1)
require.Equal(CSIVolumeAccessModeMultiNodeReader,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem,
vol.RequestedCapabilities[0].AttachmentMode)
// release our last claim, including unpublish workflow
claim.AllocationID = alloc1.ID
claim.Mode = CSIVolumeClaimRead
claim.State = CSIVolumeClaimStateReadyToFree
vol.Claim(claim, nil)
require.Len(vol.ReadClaims, 0)
require.Len(vol.WriteClaims, 0)
require.Equal(CSIVolumeAccessModeUnknown, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeUnknown, vol.AttachmentMode)
require.Equal(CSIVolumeAccessModeMultiNodeReader,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem,
vol.RequestedCapabilities[0].AttachmentMode)
// claim a write on the now-unclaimed volume and ensure we still can't
// upgrade capabilities.
claim.AccessMode = CSIVolumeAccessModeMultiNodeSingleWriter
claim.Mode = CSIVolumeClaimWrite
claim.State = CSIVolumeClaimStateTaken
claim.AllocationID = alloc2.ID
require.EqualError(vol.Claim(claim, alloc2), "unschedulable")
require.Len(vol.ReadClaims, 0)
require.Len(vol.WriteClaims, 0)
require.Equal(CSIVolumeAccessModeUnknown, vol.AccessMode)
require.Equal(CSIVolumeAttachmentModeUnknown, vol.AttachmentMode)
require.Equal(CSIVolumeAccessModeMultiNodeReader,
vol.RequestedCapabilities[0].AccessMode)
require.Equal(CSIVolumeAttachmentModeFilesystem,
vol.RequestedCapabilities[0].AttachmentMode)
}
func TestVolume_Copy(t *testing.T) {

2
nomad/structs/structs_test.go
View File

@ -1088,7 +1088,7 @@ func TestTaskGroup_Validate(t *testing.T) {
},
}
err = tg.Validate(&Job{})
require.Contains(t, err.Error(), `volume has unrecognised type nothost`)
require.Contains(t, err.Error(), `volume has unrecognized type nothost`)
tg = &TaskGroup{
Volumes: map[string]*VolumeRequest{

37
nomad/structs/volumes.go
View File

@ -92,21 +92,44 @@ func HostVolumeSliceMerge(a, b []*ClientHostVolumeConfig) []*ClientHostVolumeCon
// VolumeRequest is a representation of a storage volume that a TaskGroup wishes to use.
type VolumeRequest struct {
Name string
Type string
Source string
ReadOnly bool
MountOptions *CSIMountOptions
PerAlloc bool
Name string
Type string
Source string
ReadOnly bool
AccessMode CSIVolumeAccessMode
AttachmentMode CSIVolumeAttachmentMode
MountOptions *CSIMountOptions
PerAlloc bool
}
func (v *VolumeRequest) Validate(canaries int) error {
if !(v.Type == VolumeTypeHost ||
v.Type == VolumeTypeCSI) {
return fmt.Errorf("volume has unrecognised type %s", v.Type)
return fmt.Errorf("volume has unrecognized type %s", v.Type)
}
var mErr multierror.Error
if v.Type == VolumeTypeHost && v.AttachmentMode != CSIVolumeAttachmentModeUnknown {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("host volumes cannot have an attachment mode"))
}
if v.Type == VolumeTypeHost && v.AccessMode != CSIVolumeAccessModeUnknown {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("host volumes cannot have an access mode"))
}
if v.AccessMode == CSIVolumeAccessModeSingleNodeReader || v.AccessMode == CSIVolumeAccessModeMultiNodeReader {
if !v.ReadOnly {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("%s volumes must be read-only", v.AccessMode))
}
}
if v.AttachmentMode == CSIVolumeAttachmentModeBlockDevice && v.MountOptions != nil {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("block devices cannot have mount options"))
}
if v.PerAlloc && canaries > 0 {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("volume cannot be per_alloc when canaries are in use"))

16
vendor/github.com/hashicorp/nomad/api/tasks.go generated vendored
View File

@ -377,13 +377,15 @@ func (m *MigrateStrategy) Copy() *MigrateStrategy {
// VolumeRequest is a representation of a storage volume that a TaskGroup wishes to use.
type VolumeRequest struct {
Name string `hcl:"name,label"`
Type string `hcl:"type,optional"`
Source string `hcl:"source,optional"`
ReadOnly bool `hcl:"read_only,optional"`
MountOptions *CSIMountOptions `hcl:"mount_options,block"`
PerAlloc bool `hcl:"per_alloc,optional"`
ExtraKeysHCL []string `hcl1:",unusedKeys,optional" json:"-"`
Name string `hcl:"name,label"`
Type string `hcl:"type,optional"`
Source string `hcl:"source,optional"`
ReadOnly bool `hcl:"read_only,optional"`
AccessMode string `hcl:"access_mode,optional"`
AttachmentMode string `hcl:"attachment_mode,optional"`
MountOptions *CSIMountOptions `hcl:"mount_options,block"`
PerAlloc bool `hcl:"per_alloc,optional"`
ExtraKeysHCL []string `hcl1:",unusedKeys,optional" json:"-"`
}
const (