open-nomad/nomad/csi_endpoint_test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package nomad

import (
	"errors"
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"

	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
	"github.com/shoenig/test"
	"github.com/shoenig/test/must"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/hashicorp/nomad/acl"
	"github.com/hashicorp/nomad/ci"
	"github.com/hashicorp/nomad/client"
	cconfig "github.com/hashicorp/nomad/client/config"
	cstructs "github.com/hashicorp/nomad/client/structs"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/lib/lang"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/state"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/csi"
	"github.com/hashicorp/nomad/testutil"
)

func TestCSIVolumeEndpoint_Get(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	ns := structs.DefaultNamespace

	state := srv.fsm.State()

	codec := rpcClient(t, srv)

	id0 := uuid.Generate()

	// Create the volume
	vols := []*structs.CSIVolume{{
		ID:        id0,
		Namespace: ns,
		PluginID:  "minnie",
		Secrets:   structs.CSISecrets{"mysecret": "secretvalue"},
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	}}
	err := state.UpsertCSIVolume(999, vols)
	require.NoError(t, err)

	// Create the register request
	req := &structs.CSIVolumeGetRequest{
		ID: id0,
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			Namespace: ns,
		},
	}

	var resp structs.CSIVolumeGetResponse
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req, &resp)
	require.NoError(t, err)
	require.Equal(t, uint64(999), resp.Index)
	require.Equal(t, vols[0].ID, resp.Volume.ID)
}

func TestCSIVolumeEndpoint_Get_ACL(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	ns := structs.DefaultNamespace

	state := srv.fsm.State()
	state.BootstrapACLTokens(structs.MsgTypeTestSetup, 1, 0, mock.ACLManagementToken())
	srv.config.ACLEnabled = true
	policy := mock.NamespacePolicy(ns, "", []string{acl.NamespaceCapabilityCSIReadVolume})
	validToken := mock.CreatePolicyAndToken(t, state, 1001, "csi-access", policy)

	codec := rpcClient(t, srv)

	id0 := uuid.Generate()

	// Create the volume
	vols := []*structs.CSIVolume{{
		ID:        id0,
		Namespace: ns,
		PluginID:  "minnie",
		Secrets:   structs.CSISecrets{"mysecret": "secretvalue"},
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	}}
	err := state.UpsertCSIVolume(999, vols)
	require.NoError(t, err)

	// Create the register request
	req := &structs.CSIVolumeGetRequest{
		ID: id0,
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			Namespace: ns,
			AuthToken: validToken.SecretID,
		},
	}

	var resp structs.CSIVolumeGetResponse
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req, &resp)
	require.NoError(t, err)
	require.Equal(t, uint64(999), resp.Index)
	require.Equal(t, vols[0].ID, resp.Volume.ID)
}

func TestCSIVolume_pluginValidateVolume(t *testing.T) {
	// bare minimum server for this method
	store := state.TestStateStore(t)
	srv := &Server{
		fsm: &nomadFSM{state: store},
	}
	// has our method under test
	csiVolume := &CSIVolume{srv: srv}
	// volume for which we will request a valid plugin
	vol := &structs.CSIVolume{PluginID: "neat-plugin"}

	// plugin not found
	got, err := csiVolume.pluginValidateVolume(vol)
	must.Nil(t, got, must.Sprint("nonexistent plugin should be nil"))
	must.ErrorContains(t, err, "no CSI plugin named")

	// we'll upsert this plugin after optionally modifying it
	basePlug := &structs.CSIPlugin{
		ID: vol.PluginID,
		// these should be set on the volume after success
		Provider: "neat-provider",
		Version:  "v0",
		// explicit zero values, because these modify behavior we care about
		ControllerRequired: false,
		ControllersHealthy: 0,
	}

	cases := []struct {
		name         string
		updatePlugin func(*structs.CSIPlugin)
		expectErr    string
	}{
		{
			name: "controller not required",
		},
		{
			name: "controller unhealthy",
			updatePlugin: func(p *structs.CSIPlugin) {
				p.ControllerRequired = true
			},
			expectErr: "no healthy controllers",
		},
		{
			name: "controller healthy",
			updatePlugin: func(p *structs.CSIPlugin) {
				p.ControllerRequired = true
				p.ControllersHealthy = 1
			},
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			vol := vol.Copy()
			plug := basePlug.Copy()

			if tc.updatePlugin != nil {
				tc.updatePlugin(plug)
			}
			must.NoError(t, store.UpsertCSIPlugin(1000, plug))

			got, err := csiVolume.pluginValidateVolume(vol)

			if tc.expectErr == "" {
				must.NoError(t, err)
				must.NotNil(t, got, must.Sprint("plugin should not be nil"))
				must.Eq(t, vol.Provider, plug.Provider)
				must.Eq(t, vol.ProviderVersion, plug.Version)
			} else {
				must.Error(t, err, must.Sprint("expect error:", tc.expectErr))
				must.ErrorContains(t, err, tc.expectErr)
				must.Nil(t, got, must.Sprint("plugin should be nil"))
			}
		})
	}
}

func TestCSIVolumeEndpoint_Register(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	store := srv.fsm.State()
	codec := rpcClient(t, srv)

	id0 := uuid.Generate()

	// Create the register request
	ns := mock.Namespace()
	store.UpsertNamespaces(900, []*structs.Namespace{ns})

	// Create the node and plugin
	node := mock.Node()
	node.CSINodePlugins = map[string]*structs.CSIInfo{
		"minnie": {PluginID: "minnie",
			Healthy: true,
			// Registers as node plugin that does not require a controller to skip
			// the client RPC during registration.
			NodeInfo: &structs.CSINodeInfo{},
		},
	}
	require.NoError(t, store.UpsertNode(structs.MsgTypeTestSetup, 1000, node))

	// Create the volume
	vols := []*structs.CSIVolume{{
		ID:             id0,
		Namespace:      ns.Name,
		PluginID:       "minnie",
		AccessMode:     structs.CSIVolumeAccessModeSingleNodeReader, // legacy field ignored
		AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice,  // legacy field ignored
		MountOptions: &structs.CSIMountOptions{
			FSType: "ext4", MountFlags: []string{"sensitive"}},
		Secrets:    structs.CSISecrets{"mysecret": "secretvalue"},
		Parameters: map[string]string{"myparam": "paramvalue"},
		Context:    map[string]string{"mycontext": "contextvalue"},
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	}}

	// Create the register request
	req1 := &structs.CSIVolumeRegisterRequest{
		Volumes: vols,
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: "",
		},
	}
	resp1 := &structs.CSIVolumeRegisterResponse{}
	err := msgpackrpc.CallWithCodec(codec, "CSIVolume.Register", req1, resp1)
	require.NoError(t, err)
	require.NotEqual(t, uint64(0), resp1.Index)

	// Get the volume back out
	req2 := &structs.CSIVolumeGetRequest{
		ID: id0,
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			Namespace: ns.Name,
		},
	}
	resp2 := &structs.CSIVolumeGetResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req2, resp2)
	require.NoError(t, err)
	require.Equal(t, resp1.Index, resp2.Index)
	require.Equal(t, vols[0].ID, resp2.Volume.ID)
	require.Equal(t, "csi.CSISecrets(map[mysecret:[REDACTED]])",
		resp2.Volume.Secrets.String())
	require.Equal(t, "csi.CSIOptions(FSType: ext4, MountFlags: [REDACTED])",
		resp2.Volume.MountOptions.String())

	// Registration does not update
	req1.Volumes[0].PluginID = "adam"
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Register", req1, resp1)
	require.Error(t, err, "exists")

	// Deregistration works
	req3 := &structs.CSIVolumeDeregisterRequest{
		VolumeIDs: []string{id0},
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: ns.Name,
		},
	}
	resp3 := &structs.CSIVolumeDeregisterResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Deregister", req3, resp3)
	require.NoError(t, err)

	// Volume is missing
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req2, resp2)
	require.NoError(t, err)
	require.Nil(t, resp2.Volume)
}

// TestCSIVolumeEndpoint_Claim exercises the VolumeClaim RPC, verifying that claims
// are honored only if the volume exists, the mode is permitted, and the volume
// is schedulable according to its count of claims.
func TestCSIVolumeEndpoint_Claim(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	index := uint64(1000)

	state := srv.fsm.State()
	codec := rpcClient(t, srv)
	id0 := uuid.Generate()
	alloc := mock.BatchAlloc()

	// Create a client node and alloc
	node := mock.Node()
	alloc.NodeID = node.ID
	summary := mock.JobSummary(alloc.JobID)
	index++
	require.NoError(t, state.UpsertJobSummary(index, summary))
	index++
	require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, index, []*structs.Allocation{alloc}))

	// Create an initial volume claim request; we expect it to fail
	// because there's no such volume yet.
	claimReq := &structs.CSIVolumeClaimRequest{
		VolumeID:       id0,
		AllocationID:   alloc.ID,
		Claim:          structs.CSIVolumeClaimWrite,
		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: structs.DefaultNamespace,
		},
	}
	claimResp := &structs.CSIVolumeClaimResponse{}
	err := msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp)
	require.EqualError(t, err, fmt.Sprintf("volume not found: %s", id0),
		"expected 'volume not found' error because volume hasn't yet been created")

	// Create a plugin and volume

	node.CSINodePlugins = map[string]*structs.CSIInfo{
		"minnie": {
			PluginID: "minnie",
			Healthy:  true,
			NodeInfo: &structs.CSINodeInfo{},
		},
	}
	index++
	err = state.UpsertNode(structs.MsgTypeTestSetup, index, node)
	require.NoError(t, err)

	vols := []*structs.CSIVolume{{
		ID:        id0,
		Namespace: structs.DefaultNamespace,
		PluginID:  "minnie",
		RequestedTopologies: &structs.CSITopologyRequest{
			Required: []*structs.CSITopology{
				{Segments: map[string]string{"foo": "bar"}}},
		},
		Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	}}
	index++
	err = state.UpsertCSIVolume(index, vols)
	require.NoError(t, err)

	// Verify that the volume exists, and is healthy
	volGetReq := &structs.CSIVolumeGetRequest{
		ID: id0,
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			Namespace: structs.DefaultNamespace,
		},
	}
	volGetResp := &structs.CSIVolumeGetResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", volGetReq, volGetResp)
	require.NoError(t, err)
	require.Equal(t, id0, volGetResp.Volume.ID)
	require.True(t, volGetResp.Volume.Schedulable)
	require.Len(t, volGetResp.Volume.ReadAllocs, 0)
	require.Len(t, volGetResp.Volume.WriteAllocs, 0)

	// Now our claim should succeed
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp)
	require.NoError(t, err)

	// Verify the claim was set
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", volGetReq, volGetResp)
	require.NoError(t, err)
	require.Equal(t, id0, volGetResp.Volume.ID)
	require.Len(t, volGetResp.Volume.ReadAllocs, 0)
	require.Len(t, volGetResp.Volume.WriteAllocs, 1)

	// Make another writer claim for a different job
	alloc2 := mock.Alloc()
	alloc2.JobID = uuid.Generate()
	summary = mock.JobSummary(alloc2.JobID)
	index++
	require.NoError(t, state.UpsertJobSummary(index, summary))
	index++
	require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, index, []*structs.Allocation{alloc2}))
	claimReq.AllocationID = alloc2.ID
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp)
	require.EqualError(t, err, structs.ErrCSIVolumeMaxClaims.Error(),
		"expected 'volume max claims reached' because we only allow 1 writer")

	// Fix the mode and our claim will succeed
	claimReq.Claim = structs.CSIVolumeClaimRead
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp)
	require.NoError(t, err)

	// Verify the new claim was set
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", volGetReq, volGetResp)
	require.NoError(t, err)
	require.Equal(t, id0, volGetResp.Volume.ID)
	require.Len(t, volGetResp.Volume.ReadAllocs, 1)
	require.Len(t, volGetResp.Volume.WriteAllocs, 1)

	// Claim is idempotent
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp)
	require.NoError(t, err)
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", volGetReq, volGetResp)
	require.NoError(t, err)
	require.Equal(t, id0, volGetResp.Volume.ID)
	require.Len(t, volGetResp.Volume.ReadAllocs, 1)
	require.Len(t, volGetResp.Volume.WriteAllocs, 1)

	// Make a second reader claim
	alloc3 := mock.Alloc()
	alloc3.JobID = uuid.Generate()
	summary = mock.JobSummary(alloc3.JobID)
	index++
	require.NoError(t, state.UpsertJobSummary(index, summary))
	index++
	require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, index, []*structs.Allocation{alloc3}))
	claimReq.AllocationID = alloc3.ID
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp)
	require.NoError(t, err)

	// Verify the new claim was set
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", volGetReq, volGetResp)
	require.NoError(t, err)
	require.Equal(t, id0, volGetResp.Volume.ID)
	require.Len(t, volGetResp.Volume.ReadAllocs, 2)
	require.Len(t, volGetResp.Volume.WriteAllocs, 1)
}

// TestCSIVolumeEndpoint_ClaimWithController exercises the VolumeClaim RPC
// when a controller is required.
func TestCSIVolumeEndpoint_ClaimWithController(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) {
		c.ACLEnabled = true
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	ns := structs.DefaultNamespace
	state := srv.fsm.State()
	state.BootstrapACLTokens(structs.MsgTypeTestSetup, 1, 0, mock.ACLManagementToken())

	policy := mock.NamespacePolicy(ns, "", []string{acl.NamespaceCapabilityCSIMountVolume}) +
		mock.PluginPolicy("read")
	accessToken := mock.CreatePolicyAndToken(t, state, 1001, "claim", policy)

	codec := rpcClient(t, srv)
	id0 := uuid.Generate()

	// Create a client node, plugin, alloc, and volume
	node := mock.Node()
	node.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early version
	node.CSIControllerPlugins = map[string]*structs.CSIInfo{
		"minnie": {
			PluginID: "minnie",
			Healthy:  true,
			ControllerInfo: &structs.CSIControllerInfo{
				SupportsAttachDetach: true,
			},
			RequiresControllerPlugin: true,
		},
	}
	node.CSINodePlugins = map[string]*structs.CSIInfo{
		"minnie": {
			PluginID: "minnie",
			Healthy:  true,
			NodeInfo: &structs.CSINodeInfo{},
		},
	}
	err := state.UpsertNode(structs.MsgTypeTestSetup, 1002, node)
	require.NoError(t, err)
	vols := []*structs.CSIVolume{{
		ID:                 id0,
		Namespace:          ns,
		PluginID:           "minnie",
		ControllerRequired: true,
		Secrets:            structs.CSISecrets{"mysecret": "secretvalue"},
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	}}
	err = state.UpsertCSIVolume(1003, vols)
	require.NoError(t, err)

	alloc := mock.BatchAlloc()
	alloc.NodeID = node.ID
	summary := mock.JobSummary(alloc.JobID)
	require.NoError(t, state.UpsertJobSummary(1004, summary))
	require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1005, []*structs.Allocation{alloc}))

	// Make the volume claim
	claimReq := &structs.CSIVolumeClaimRequest{
		VolumeID:     id0,
		AllocationID: alloc.ID,
		Claim:        structs.CSIVolumeClaimWrite,
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: ns,
			AuthToken: accessToken.SecretID,
		},
	}
	claimResp := &structs.CSIVolumeClaimResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp)
	// Because the node is not registered
	require.EqualError(t, err, "controller publish: controller attach volume: No path to node")

	// The node SecretID is authorized for all policies
	claimReq.AuthToken = node.SecretID
	claimReq.Namespace = ""
	claimResp = &structs.CSIVolumeClaimResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp)
	require.EqualError(t, err, "controller publish: controller attach volume: No path to node")
}

func TestCSIVolumeEndpoint_Unpublish(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) { c.NumSchedulers = 0 })
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	var err error
	index := uint64(1000)
	ns := structs.DefaultNamespace
	state := srv.fsm.State()
	state.BootstrapACLTokens(structs.MsgTypeTestSetup, 1, 0, mock.ACLManagementToken())

	policy := mock.NamespacePolicy(ns, "", []string{acl.NamespaceCapabilityCSIMountVolume}) +
		mock.PluginPolicy("read")
	index++
	accessToken := mock.CreatePolicyAndToken(t, state, index, "claim", policy)

	codec := rpcClient(t, srv)

	// setup: create a client node with a controller and node plugin
	node := mock.Node()
	node.Attributes["nomad.version"] = "0.11.0"
	node.CSINodePlugins = map[string]*structs.CSIInfo{
		"minnie": {PluginID: "minnie",
			Healthy:  true,
			NodeInfo: &structs.CSINodeInfo{},
		},
	}
	node.CSIControllerPlugins = map[string]*structs.CSIInfo{
		"minnie": {PluginID: "minnie",
			Healthy:                  true,
			ControllerInfo:           &structs.CSIControllerInfo{SupportsAttachDetach: true},
			RequiresControllerPlugin: true,
		},
	}
	index++
	must.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node))

	type tc struct {
		name           string
		startingState  structs.CSIVolumeClaimState
		endState       structs.CSIVolumeClaimState
		nodeID         string
		otherNodeID    string
		expectedErrMsg string
	}
	testCases := []tc{
		{
			name:          "success",
			startingState: structs.CSIVolumeClaimStateControllerDetached,
			nodeID:        node.ID,
			otherNodeID:   uuid.Generate(),
		},
		{
			name:          "non-terminal allocation on same node",
			startingState: structs.CSIVolumeClaimStateNodeDetached,
			nodeID:        node.ID,
			otherNodeID:   node.ID,
		},
		{
			name:           "unpublish previously detached node",
			startingState:  structs.CSIVolumeClaimStateNodeDetached,
			endState:       structs.CSIVolumeClaimStateNodeDetached,
			expectedErrMsg: "could not detach from controller: controller detach volume: No path to node",
			nodeID:         node.ID,
			otherNodeID:    uuid.Generate(),
		},
		{
			name:           "unpublish claim on garbage collected node",
			startingState:  structs.CSIVolumeClaimStateTaken,
			endState:       structs.CSIVolumeClaimStateNodeDetached,
			expectedErrMsg: "could not detach from controller: controller detach volume: No path to node",
			nodeID:         uuid.Generate(),
			otherNodeID:    uuid.Generate(),
		},
		{
			name:           "first unpublish",
			startingState:  structs.CSIVolumeClaimStateTaken,
			endState:       structs.CSIVolumeClaimStateNodeDetached,
			expectedErrMsg: "could not detach from controller: controller detach volume: No path to node",
			nodeID:         node.ID,
			otherNodeID:    uuid.Generate(),
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			// setup: register a volume
			volID := uuid.Generate()
			vol := &structs.CSIVolume{
				ID:                 volID,
				Namespace:          ns,
				PluginID:           "minnie",
				Secrets:            structs.CSISecrets{"mysecret": "secretvalue"},
				ControllerRequired: true,
				RequestedCapabilities: []*structs.CSIVolumeCapability{{
					AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
					AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
				}},
			}

			index++
			err = state.UpsertCSIVolume(index, []*structs.CSIVolume{vol})
			must.NoError(t, err)

			// setup: create an alloc that will claim our volume
			alloc := mock.BatchAlloc()
			alloc.NodeID = tc.nodeID
			alloc.ClientStatus = structs.AllocClientStatusRunning

			otherAlloc := mock.BatchAlloc()
			otherAlloc.NodeID = tc.otherNodeID
			otherAlloc.ClientStatus = structs.AllocClientStatusRunning

			index++
			must.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, index,
				[]*structs.Allocation{alloc, otherAlloc}))

			// setup: claim the volume for our to-be-failed alloc
			claim := &structs.CSIVolumeClaim{
				AllocationID:   alloc.ID,
				NodeID:         tc.nodeID,
				ExternalNodeID: "i-example",
				Mode:           structs.CSIVolumeClaimRead,
			}

			index++
			claim.State = structs.CSIVolumeClaimStateTaken
			err = state.CSIVolumeClaim(index, ns, volID, claim)
			must.NoError(t, err)

			// setup: claim the volume for our other alloc
			otherClaim := &structs.CSIVolumeClaim{
				AllocationID:   otherAlloc.ID,
				NodeID:         tc.otherNodeID,
				ExternalNodeID: "i-example",
				Mode:           structs.CSIVolumeClaimRead,
			}

			index++
			otherClaim.State = structs.CSIVolumeClaimStateTaken
			err = state.CSIVolumeClaim(index, ns, volID, otherClaim)
			must.NoError(t, err)

			// test: unpublish and check the results
			claim.State = tc.startingState
			req := &structs.CSIVolumeUnpublishRequest{
				VolumeID: volID,
				Claim:    claim,
				WriteRequest: structs.WriteRequest{
					Region:    "global",
					Namespace: ns,
					AuthToken: accessToken.SecretID,
				},
			}

			alloc = alloc.Copy()
			alloc.ClientStatus = structs.AllocClientStatusFailed
			index++
			must.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, index,
				[]*structs.Allocation{alloc}))

			err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Unpublish", req,
				&structs.CSIVolumeUnpublishResponse{})

			snap, snapErr := state.Snapshot()
			must.NoError(t, snapErr)

			vol, volErr := snap.CSIVolumeByID(nil, ns, volID)
			must.NoError(t, volErr)
			must.NotNil(t, vol)

			if tc.expectedErrMsg == "" {
				must.NoError(t, err)
				assert.Len(t, vol.ReadAllocs, 1)
			} else {
				must.Error(t, err)
				assert.Len(t, vol.ReadAllocs, 2)
				test.True(t, strings.Contains(err.Error(), tc.expectedErrMsg),
					test.Sprintf("error %v did not contain %q", err, tc.expectedErrMsg))
				claim = vol.PastClaims[alloc.ID]
				must.NotNil(t, claim)
				test.Eq(t, tc.endState, claim.State)
			}

		})
	}

}

func TestCSIVolumeEndpoint_List(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	state := srv.fsm.State()
	state.BootstrapACLTokens(structs.MsgTypeTestSetup, 1, 0, mock.ACLManagementToken())
	srv.config.ACLEnabled = true
	codec := rpcClient(t, srv)

	nsPolicy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityCSIReadVolume}) +
		mock.PluginPolicy("read")
	nsTok := mock.CreatePolicyAndToken(t, state, 1000, "csi-token-name", nsPolicy)

	// Empty list results
	req := &structs.CSIVolumeListRequest{
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			AuthToken: nsTok.SecretID,
			Namespace: structs.DefaultNamespace,
		},
	}
	var resp structs.CSIVolumeListResponse
	err := msgpackrpc.CallWithCodec(codec, "CSIVolume.List", req, &resp)
	require.NoError(t, err)
	require.NotNil(t, resp.Volumes)
	require.Equal(t, 0, len(resp.Volumes))

	// Create the volume
	id0 := uuid.Generate()
	id1 := uuid.Generate()
	vols := []*structs.CSIVolume{{
		ID:        id0,
		Namespace: structs.DefaultNamespace,
		PluginID:  "minnie",
		Secrets:   structs.CSISecrets{"mysecret": "secretvalue"},
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	}, {
		ID:        id1,
		Namespace: structs.DefaultNamespace,
		PluginID:  "adam",
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	}}
	err = state.UpsertCSIVolume(1002, vols)
	require.NoError(t, err)

	// Query everything in the namespace
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.List", req, &resp)
	require.NoError(t, err)

	require.Equal(t, uint64(1002), resp.Index)
	require.Equal(t, 2, len(resp.Volumes))
	ids := map[string]bool{vols[0].ID: true, vols[1].ID: true}
	for _, v := range resp.Volumes {
		delete(ids, v.ID)
	}
	require.Equal(t, 0, len(ids))

	// Query by PluginID in ns
	req = &structs.CSIVolumeListRequest{
		PluginID: "adam",
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			Namespace: structs.DefaultNamespace,
			AuthToken: nsTok.SecretID,
		},
	}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.List", req, &resp)
	require.NoError(t, err)
	require.Equal(t, 1, len(resp.Volumes))
	require.Equal(t, vols[1].ID, resp.Volumes[0].ID)
}

func TestCSIVolumeEndpoint_ListAllNamespaces(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	state := srv.fsm.State()
	codec := rpcClient(t, srv)

	// Create namespaces.
	ns0 := structs.DefaultNamespace
	ns1 := "namespace-1"
	ns2 := "namespace-2"
	err := state.UpsertNamespaces(1000, []*structs.Namespace{{Name: ns1}, {Name: ns2}})
	require.NoError(t, err)

	// Create volumes in multiple namespaces.
	id0 := uuid.Generate()
	id1 := uuid.Generate()
	id2 := uuid.Generate()
	vols := []*structs.CSIVolume{{
		ID:        id0,
		Namespace: ns0,
		PluginID:  "minnie",
		Secrets:   structs.CSISecrets{"mysecret": "secretvalue"},
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	}, {
		ID:        id1,
		Namespace: ns1,
		PluginID:  "adam",
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	}, {
		ID:        id2,
		Namespace: ns2,
		PluginID:  "beth",
		RequestedCapabilities: []*structs.CSIVolumeCapability{{
			AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
			AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		}},
	},
	}
	err = state.UpsertCSIVolume(1001, vols)
	require.NoError(t, err)

	// Lookup volumes in all namespaces
	get := &structs.CSIVolumeListRequest{
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			Namespace: "*",
		},
	}
	var resp structs.CSIVolumeListResponse
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.List", get, &resp)
	require.NoError(t, err)
	require.Equal(t, uint64(1001), resp.Index)
	require.Len(t, resp.Volumes, len(vols))

	// Lookup volumes in all namespaces with prefix
	get = &structs.CSIVolumeListRequest{
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			Prefix:    id0[:4],
			Namespace: "*",
		},
	}
	var resp2 structs.CSIVolumeListResponse
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.List", get, &resp2)
	require.NoError(t, err)
	require.Equal(t, uint64(1001), resp.Index)
	require.Len(t, resp2.Volumes, 1)
	require.Equal(t, vols[0].ID, resp2.Volumes[0].ID)
	require.Equal(t, structs.DefaultNamespace, resp2.Volumes[0].Namespace)
}

func TestCSIVolumeEndpoint_List_PaginationFiltering(t *testing.T) {
	ci.Parallel(t)
	s1, cleanupS1 := TestServer(t, nil)
	defer cleanupS1()
	codec := rpcClient(t, s1)
	testutil.WaitForLeader(t, s1.RPC)

	nonDefaultNS := "non-default"

	// create a set of volumes. these are in the order that the state store
	// will return them from the iterator (sorted by create index), for ease of
	// writing tests
	mocks := []struct {
		id        string
		namespace string
	}{
		{id: "vol-01"},                          // 0
		{id: "vol-02"},                          // 1
		{id: "vol-03", namespace: nonDefaultNS}, // 2
		{id: "vol-04"},                          // 3
		{id: "vol-05"},                          // 4
		{id: "vol-06"},                          // 5
		{id: "vol-07"},                          // 6
		{id: "vol-08"},                          // 7
		{},                                      // 9, missing volume
		{id: "vol-10"},                          // 10
	}

	state := s1.fsm.State()
	plugin := mock.CSIPlugin()

	// Create namespaces.
	err := state.UpsertNamespaces(999, []*structs.Namespace{{Name: nonDefaultNS}})
	require.NoError(t, err)

	for i, m := range mocks {
		if m.id == "" {
			continue
		}

		volume := mock.CSIVolume(plugin)
		volume.ID = m.id
		if m.namespace != "" { // defaults to "default"
			volume.Namespace = m.namespace
		}
		index := 1000 + uint64(i)
		require.NoError(t, state.UpsertCSIVolume(index, []*structs.CSIVolume{volume}))
	}

	cases := []struct {
		name              string
		namespace         string
		prefix            string
		filter            string
		nextToken         string
		pageSize          int32
		expectedNextToken string
		expectedIDs       []string
		expectedError     string
	}{
		{
			name:              "test01 size-2 page-1 default NS",
			pageSize:          2,
			expectedNextToken: "default.vol-04",
			expectedIDs: []string{
				"vol-01",
				"vol-02",
			},
		},
		{
			name:              "test02 size-2 page-1 default NS with prefix",
			prefix:            "vol",
			pageSize:          2,
			expectedNextToken: "default.vol-04",
			expectedIDs: []string{
				"vol-01",
				"vol-02",
			},
		},
		{
			name:              "test03 size-2 page-2 default NS",
			pageSize:          2,
			nextToken:         "default.vol-04",
			expectedNextToken: "default.vol-06",
			expectedIDs: []string{
				"vol-04",
				"vol-05",
			},
		},
		{
			name:              "test04 size-2 page-2 default NS with prefix",
			prefix:            "vol",
			pageSize:          2,
			nextToken:         "default.vol-04",
			expectedNextToken: "default.vol-06",
			expectedIDs: []string{
				"vol-04",
				"vol-05",
			},
		},
		{
			name:        "test05 no valid results with filters and prefix",
			prefix:      "cccc",
			pageSize:    2,
			nextToken:   "",
			expectedIDs: []string{},
		},
		{
			name:      "test06 go-bexpr filter",
			namespace: "*",
			filter:    `ID matches "^vol-0[123]"`,
			expectedIDs: []string{
				"vol-01",
				"vol-02",
				"vol-03",
			},
		},
		{
			name:              "test07 go-bexpr filter with pagination",
			namespace:         "*",
			filter:            `ID matches "^vol-0[123]"`,
			pageSize:          2,
			expectedNextToken: "non-default.vol-03",
			expectedIDs: []string{
				"vol-01",
				"vol-02",
			},
		},
		{
			name:      "test08 go-bexpr filter in namespace",
			namespace: "non-default",
			filter:    `Provider == "com.hashicorp:mock"`,
			expectedIDs: []string{
				"vol-03",
			},
		},
		{
			name:        "test09 go-bexpr wrong namespace",
			namespace:   "default",
			filter:      `Namespace == "non-default"`,
			expectedIDs: []string{},
		},
		{
			name:          "test10 go-bexpr invalid expression",
			filter:        `NotValid`,
			expectedError: "failed to read filter expression",
		},
		{
			name:          "test11 go-bexpr invalid field",
			filter:        `InvalidField == "value"`,
			expectedError: "error finding value in datum",
		},
		{
			name:      "test14 missing volume",
			pageSize:  1,
			nextToken: "default.vol-09",
			expectedIDs: []string{
				"vol-10",
			},
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			req := &structs.CSIVolumeListRequest{
				QueryOptions: structs.QueryOptions{
					Region:    "global",
					Namespace: tc.namespace,
					Prefix:    tc.prefix,
					Filter:    tc.filter,
					PerPage:   tc.pageSize,
					NextToken: tc.nextToken,
				},
			}
			var resp structs.CSIVolumeListResponse
			err := msgpackrpc.CallWithCodec(codec, "CSIVolume.List", req, &resp)
			if tc.expectedError == "" {
				require.NoError(t, err)
			} else {
				require.Error(t, err)
				require.Contains(t, err.Error(), tc.expectedError)
				return
			}

			gotIDs := []string{}
			for _, deployment := range resp.Volumes {
				gotIDs = append(gotIDs, deployment.ID)
			}
			require.Equal(t, tc.expectedIDs, gotIDs, "unexpected page of volumes")
			require.Equal(t, tc.expectedNextToken, resp.QueryMeta.NextToken, "unexpected NextToken")
		})
	}
}

func TestCSIVolumeEndpoint_Create(t *testing.T) {
	ci.Parallel(t)
	var err error
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()

	testutil.WaitForLeader(t, srv.RPC)

	fake := newMockClientCSI()
	fake.NextValidateError = nil
	fake.NextCreateError = nil
	fake.NextCreateResponse = &cstructs.ClientCSIControllerCreateVolumeResponse{
		ExternalVolumeID: "vol-12345",
		CapacityBytes:    42,
		VolumeContext:    map[string]string{"plugincontext": "bar"},
		Topologies: []*structs.CSITopology{
			{Segments: map[string]string{"rack": "R1"}},
		},
	}

	client, cleanup := client.TestClientWithRPCs(t,
		func(c *cconfig.Config) {
			c.Servers = []string{srv.config.RPCAddr.String()}
		},
		map[string]interface{}{"CSI": fake},
	)
	defer cleanup()

	node := client.UpdateConfig(func(c *cconfig.Config) {
		// client RPCs not supported on early versions
		c.Node.Attributes["nomad.version"] = "0.11.0"
	}).Node

	req0 := &structs.NodeRegisterRequest{
		Node:         node,
		WriteRequest: structs.WriteRequest{Region: "global"},
	}
	var resp0 structs.NodeUpdateResponse
	err = client.RPC("Node.Register", req0, &resp0)
	require.NoError(t, err)

	testutil.WaitForResult(func() (bool, error) {
		nodes := srv.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		t.Fatalf("should have a client")
	})

	ns := structs.DefaultNamespace

	state := srv.fsm.State()
	codec := rpcClient(t, srv)
	index := uint64(1000)

	node = client.UpdateConfig(func(c *cconfig.Config) {
		c.Node.CSIControllerPlugins = map[string]*structs.CSIInfo{
			"minnie": {
				PluginID: "minnie",
				Healthy:  true,
				ControllerInfo: &structs.CSIControllerInfo{
					SupportsAttachDetach: true,
					SupportsCreateDelete: true,
				},
				RequiresControllerPlugin: true,
			},
		}
		c.Node.CSINodePlugins = map[string]*structs.CSIInfo{
			"minnie": {
				PluginID: "minnie",
				Healthy:  true,
				NodeInfo: &structs.CSINodeInfo{},
			},
		}
	}).Node
	index++
	require.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node))

	// Create the volume
	volID := uuid.Generate()
	vols := []*structs.CSIVolume{{
		ID:             volID,
		Name:           "vol",
		Namespace:      "", // overriden by WriteRequest
		PluginID:       "minnie",
		AccessMode:     structs.CSIVolumeAccessModeSingleNodeReader, // legacy field ignored
		AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice,  // legacy field ignored
		MountOptions: &structs.CSIMountOptions{
			FSType: "ext4", MountFlags: []string{"sensitive"}}, // ignored in create
		Secrets:    structs.CSISecrets{"mysecret": "secretvalue"},
		Parameters: map[string]string{"myparam": "paramvalue"},
		Context:    map[string]string{"mycontext": "contextvalue"}, // dropped by create
		RequestedCapabilities: []*structs.CSIVolumeCapability{
			{
				AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
				AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
			},
		},
		Topologies: []*structs.CSITopology{
			{Segments: map[string]string{"rack": "R1"}},
			{Segments: map[string]string{"zone": "Z2"}},
		},
	}}

	// Create the create request
	req1 := &structs.CSIVolumeCreateRequest{
		Volumes: vols,
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: ns,
		},
	}
	resp1 := &structs.CSIVolumeCreateResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Create", req1, resp1)
	require.NoError(t, err)

	// Get the volume back out
	req2 := &structs.CSIVolumeGetRequest{
		ID: volID,
		QueryOptions: structs.QueryOptions{
			Region: "global",
		},
	}
	resp2 := &structs.CSIVolumeGetResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req2, resp2)
	require.NoError(t, err)
	require.Equal(t, resp1.Index, resp2.Index)

	vol := resp2.Volume
	require.NotNil(t, vol)
	require.Equal(t, volID, vol.ID)

	// these fields are set from the args
	require.Equal(t, "csi.CSISecrets(map[mysecret:[REDACTED]])",
		vol.Secrets.String())
	require.Equal(t, "csi.CSIOptions(FSType: ext4, MountFlags: [REDACTED])",
		vol.MountOptions.String())
	require.Equal(t, ns, vol.Namespace)
	require.Len(t, vol.RequestedCapabilities, 1)

	// these fields are set from the plugin and should have been written to raft
	require.Equal(t, "vol-12345", vol.ExternalID)
	require.Equal(t, int64(42), vol.Capacity)
	require.Equal(t, "bar", vol.Context["plugincontext"])
	require.Equal(t, "", vol.Context["mycontext"])
	require.Equal(t, map[string]string{"rack": "R1"}, vol.Topologies[0].Segments)
}

func TestCSIVolumeEndpoint_Delete(t *testing.T) {
	ci.Parallel(t)
	var err error
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()

	testutil.WaitForLeader(t, srv.RPC)

	fake := newMockClientCSI()
	fake.NextDeleteError = fmt.Errorf("should not see this")

	client, cleanup := client.TestClientWithRPCs(t,
		func(c *cconfig.Config) {
			c.Servers = []string{srv.config.RPCAddr.String()}
		},
		map[string]interface{}{"CSI": fake},
	)
	defer cleanup()

	node := client.UpdateConfig(func(c *cconfig.Config) {
		// client RPCs not supported on early versions
		c.Node.Attributes["nomad.version"] = "0.11.0"
	}).Node

	req0 := &structs.NodeRegisterRequest{
		Node:         node,
		WriteRequest: structs.WriteRequest{Region: "global"},
	}
	var resp0 structs.NodeUpdateResponse
	err = client.RPC("Node.Register", req0, &resp0)
	must.NoError(t, err)

	testutil.WaitForResult(func() (bool, error) {
		nodes := srv.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		t.Fatalf("should have a client")
	})

	ns := structs.DefaultNamespace

	state := srv.fsm.State()
	codec := rpcClient(t, srv)
	index := uint64(1000)

	node = client.UpdateConfig(func(c *cconfig.Config) {
		c.Node.CSIControllerPlugins = map[string]*structs.CSIInfo{
			"minnie": {
				PluginID: "minnie",
				Healthy:  true,
				ControllerInfo: &structs.CSIControllerInfo{
					SupportsAttachDetach: true,
				},
				RequiresControllerPlugin: true,
			},
		}
		c.Node.CSINodePlugins = map[string]*structs.CSIInfo{
			"minnie": {
				PluginID: "minnie",
				Healthy:  true,
				NodeInfo: &structs.CSINodeInfo{},
			},
		}
	}).Node
	index++
	must.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node))

	volID := uuid.Generate()
	noPluginVolID := uuid.Generate()
	vols := []*structs.CSIVolume{
		{
			ID:        volID,
			Namespace: structs.DefaultNamespace,
			PluginID:  "minnie",
			Secrets:   structs.CSISecrets{"mysecret": "secretvalue"},
		},
		{
			ID:        noPluginVolID,
			Namespace: structs.DefaultNamespace,
			PluginID:  "doesnt-exist",
			Secrets:   structs.CSISecrets{"mysecret": "secretvalue"},
		},
	}
	index++
	err = state.UpsertCSIVolume(index, vols)
	must.NoError(t, err)

	// Delete volumes

	// Create an invalid delete request, ensure it doesn't hit the plugin
	req1 := &structs.CSIVolumeDeleteRequest{
		VolumeIDs: []string{"bad", volID},
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: ns,
		},
		Secrets: structs.CSISecrets{
			"secret-key-1": "secret-val-1",
		},
	}
	resp1 := &structs.CSIVolumeCreateResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Delete", req1, resp1)
	must.EqError(t, err, "volume not found: bad")

	// Make sure the valid volume wasn't deleted
	req2 := &structs.CSIVolumeGetRequest{
		ID: volID,
		QueryOptions: structs.QueryOptions{
			Region: "global",
		},
	}
	resp2 := &structs.CSIVolumeGetResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req2, resp2)
	must.NoError(t, err)
	must.NotNil(t, resp2.Volume)

	// Fix the delete request
	fake.NextDeleteError = nil
	req1.VolumeIDs = []string{volID}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Delete", req1, resp1)
	must.NoError(t, err)

	// Make sure it was deregistered
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req2, resp2)
	must.NoError(t, err)
	must.Nil(t, resp2.Volume)

	// Create a delete request for a volume without plugin.
	req3 := &structs.CSIVolumeDeleteRequest{
		VolumeIDs: []string{noPluginVolID},
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: ns,
		},
		Secrets: structs.CSISecrets{
			"secret-key-1": "secret-val-1",
		},
	}
	resp3 := &structs.CSIVolumeCreateResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Delete", req3, resp3)
	must.EqError(t, err, fmt.Sprintf(`plugin "doesnt-exist" for volume "%s" not found`, noPluginVolID))
}

func TestCSIVolumeEndpoint_ListExternal(t *testing.T) {
	ci.Parallel(t)
	var err error
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()

	testutil.WaitForLeader(t, srv.RPC)

	fake := newMockClientCSI()
	fake.NextDeleteError = fmt.Errorf("should not see this")
	fake.NextListExternalResponse = &cstructs.ClientCSIControllerListVolumesResponse{
		Entries: []*structs.CSIVolumeExternalStub{
			{
				ExternalID:               "vol-12345",
				CapacityBytes:            70000,
				PublishedExternalNodeIDs: []string{"i-12345"},
			},
			{
				ExternalID:    "vol-abcde",
				CapacityBytes: 50000,
				IsAbnormal:    true,
				Status:        "something went wrong",
			},
			{
				ExternalID: "vol-00000",
				Status:     "you should not see me",
			},
		},
		NextToken: "page2",
	}

	client, cleanup := client.TestClientWithRPCs(t,
		func(c *cconfig.Config) {
			c.Servers = []string{srv.config.RPCAddr.String()}
		},
		map[string]interface{}{"CSI": fake},
	)
	defer cleanup()

	node := client.UpdateConfig(func(c *cconfig.Config) {
		// client RPCs not supported on early versions
		c.Node.Attributes["nomad.version"] = "0.11.0"
	}).Node

	req0 := &structs.NodeRegisterRequest{
		Node:         node,
		WriteRequest: structs.WriteRequest{Region: "global"},
	}
	var resp0 structs.NodeUpdateResponse
	err = client.RPC("Node.Register", req0, &resp0)
	require.NoError(t, err)

	testutil.WaitForResult(func() (bool, error) {
		nodes := srv.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		t.Fatalf("should have a client")
	})

	state := srv.fsm.State()
	codec := rpcClient(t, srv)
	index := uint64(1000)

	node = client.UpdateConfig(func(c *cconfig.Config) {
		c.Node.CSIControllerPlugins = map[string]*structs.CSIInfo{
			"minnie": {
				PluginID: "minnie",
				Healthy:  true,
				ControllerInfo: &structs.CSIControllerInfo{
					SupportsAttachDetach: true,
					SupportsListVolumes:  true,
				},
				RequiresControllerPlugin: true,
			},
		}
		c.Node.CSINodePlugins = map[string]*structs.CSIInfo{
			"minnie": {
				PluginID: "minnie",
				Healthy:  true,
				NodeInfo: &structs.CSINodeInfo{},
			},
		}
	}).Node
	index++
	require.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node))

	// List external volumes; note that none of these exist in the state store

	req := &structs.CSIVolumeExternalListRequest{
		PluginID: "minnie",
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			Namespace: structs.DefaultNamespace,
			PerPage:   2,
			NextToken: "page1",
		},
	}
	resp := &structs.CSIVolumeExternalListResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.ListExternal", req, resp)
	require.NoError(t, err)
	require.Len(t, resp.Volumes, 2)
	require.Equal(t, "vol-12345", resp.Volumes[0].ExternalID)
	require.Equal(t, "vol-abcde", resp.Volumes[1].ExternalID)
	require.True(t, resp.Volumes[1].IsAbnormal)
	require.Equal(t, "page2", resp.NextToken)
}

func TestCSIVolumeEndpoint_CreateSnapshot(t *testing.T) {
	ci.Parallel(t)
	var err error
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()

	testutil.WaitForLeader(t, srv.RPC)

	now := time.Now().Unix()
	fake := newMockClientCSI()
	fake.NextCreateSnapshotError = nil
	fake.NextCreateSnapshotResponse = &cstructs.ClientCSIControllerCreateSnapshotResponse{
		ID:                     "snap-12345",
		ExternalSourceVolumeID: "vol-12345",
		SizeBytes:              42,
		CreateTime:             now,
		IsReady:                true,
	}

	client, cleanup := client.TestClientWithRPCs(t,
		func(c *cconfig.Config) {
			c.Servers = []string{srv.config.RPCAddr.String()}
		},
		map[string]interface{}{"CSI": fake},
	)
	defer cleanup()

	req0 := &structs.NodeRegisterRequest{
		Node:         client.Node(),
		WriteRequest: structs.WriteRequest{Region: "global"},
	}
	var resp0 structs.NodeUpdateResponse
	err = client.RPC("Node.Register", req0, &resp0)
	require.NoError(t, err)

	testutil.WaitForResult(func() (bool, error) {
		nodes := srv.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		t.Fatalf("should have a client")
	})

	ns := structs.DefaultNamespace

	state := srv.fsm.State()
	codec := rpcClient(t, srv)
	index := uint64(1000)

	node := client.UpdateConfig(func(c *cconfig.Config) {
		c.Node.CSIControllerPlugins = map[string]*structs.CSIInfo{
			"minnie": {
				PluginID: "minnie",
				Healthy:  true,
				ControllerInfo: &structs.CSIControllerInfo{
					SupportsCreateDeleteSnapshot: true,
				},
				RequiresControllerPlugin: true,
			},
		}
	}).Node
	index++
	require.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node))

	// Create the volume
	vols := []*structs.CSIVolume{{
		ID:             "test-volume0",
		Namespace:      ns,
		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		PluginID:       "minnie",
		ExternalID:     "vol-12345",
	}}
	index++
	require.NoError(t, state.UpsertCSIVolume(index, vols))

	// Create the snapshot request
	req1 := &structs.CSISnapshotCreateRequest{
		Snapshots: []*structs.CSISnapshot{{
			Name:           "snap",
			SourceVolumeID: "test-volume0",
			Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
			Parameters:     map[string]string{"myparam": "paramvalue"},
		}},
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: ns,
		},
	}
	resp1 := &structs.CSISnapshotCreateResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.CreateSnapshot", req1, resp1)
	require.NoError(t, err)

	snap := resp1.Snapshots[0]
	require.Equal(t, "vol-12345", snap.ExternalSourceVolumeID)       // set by the args
	require.Equal(t, "snap-12345", snap.ID)                          // set by the plugin
	require.Equal(t, "csi.CSISecrets(map[])", snap.Secrets.String()) // should not be set
	require.Len(t, snap.Parameters, 0)                               // should not be set
}

func TestCSIVolumeEndpoint_DeleteSnapshot(t *testing.T) {
	ci.Parallel(t)
	var err error
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()

	testutil.WaitForLeader(t, srv.RPC)

	fake := newMockClientCSI()
	fake.NextDeleteSnapshotError = nil

	client, cleanup := client.TestClientWithRPCs(t,
		func(c *cconfig.Config) {
			c.Servers = []string{srv.config.RPCAddr.String()}
		},
		map[string]interface{}{"CSI": fake},
	)
	defer cleanup()

	req0 := &structs.NodeRegisterRequest{
		Node:         client.Node(),
		WriteRequest: structs.WriteRequest{Region: "global"},
	}
	var resp0 structs.NodeUpdateResponse
	err = client.RPC("Node.Register", req0, &resp0)
	require.NoError(t, err)

	testutil.WaitForResult(func() (bool, error) {
		nodes := srv.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		t.Fatalf("should have a client")
	})

	ns := structs.DefaultNamespace

	state := srv.fsm.State()
	codec := rpcClient(t, srv)
	index := uint64(1000)

	node := client.UpdateConfig(func(c *cconfig.Config) {
		c.Node.CSIControllerPlugins = map[string]*structs.CSIInfo{
			"minnie": {
				PluginID: "minnie",
				Healthy:  true,
				ControllerInfo: &structs.CSIControllerInfo{
					SupportsCreateDeleteSnapshot: true,
				},
				RequiresControllerPlugin: true,
			},
		}
	}).Node
	index++
	require.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node))

	// Delete the snapshot request
	req1 := &structs.CSISnapshotDeleteRequest{
		Snapshots: []*structs.CSISnapshot{
			{
				ID:       "snap-12345",
				PluginID: "minnie",
			},
			{
				ID:       "snap-34567",
				PluginID: "minnie",
			},
		},
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: ns,
		},
	}

	resp1 := &structs.CSISnapshotDeleteResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.DeleteSnapshot", req1, resp1)
	require.NoError(t, err)
}

func TestCSIVolumeEndpoint_ListSnapshots(t *testing.T) {
	ci.Parallel(t)
	var err error
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()

	testutil.WaitForLeader(t, srv.RPC)

	fake := newMockClientCSI()
	fake.NextListExternalSnapshotsResponse = &cstructs.ClientCSIControllerListSnapshotsResponse{
		Entries: []*structs.CSISnapshot{
			{
				ID:                     "snap-12345",
				ExternalSourceVolumeID: "vol-12345",
				SizeBytes:              70000,
				IsReady:                true,
			},
			{
				ID:                     "snap-abcde",
				ExternalSourceVolumeID: "vol-abcde",
				SizeBytes:              70000,
				IsReady:                false,
			},
			{
				ExternalSourceVolumeID: "you should not see me",
			},
		},
		NextToken: "page2",
	}

	client, cleanup := client.TestClientWithRPCs(t,
		func(c *cconfig.Config) {
			c.Servers = []string{srv.config.RPCAddr.String()}
		},
		map[string]interface{}{"CSI": fake},
	)
	defer cleanup()

	req0 := &structs.NodeRegisterRequest{
		Node:         client.Node(),
		WriteRequest: structs.WriteRequest{Region: "global"},
	}
	var resp0 structs.NodeUpdateResponse
	err = client.RPC("Node.Register", req0, &resp0)
	require.NoError(t, err)

	testutil.WaitForResult(func() (bool, error) {
		nodes := srv.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		t.Fatalf("should have a client")
	})

	state := srv.fsm.State()
	codec := rpcClient(t, srv)
	index := uint64(1000)

	node := client.UpdateConfig(func(c *cconfig.Config) {
		c.Node.CSIControllerPlugins = map[string]*structs.CSIInfo{
			"minnie": {
				PluginID: "minnie",
				Healthy:  true,
				ControllerInfo: &structs.CSIControllerInfo{
					SupportsListSnapshots: true,
				},
				RequiresControllerPlugin: true,
			},
		}
	}).Node
	index++
	require.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node))

	// List snapshots
	req := &structs.CSISnapshotListRequest{
		PluginID: "minnie",
		Secrets: structs.CSISecrets{
			"secret-key-1": "secret-val-1",
		},
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			Namespace: structs.DefaultNamespace,
			PerPage:   2,
			NextToken: "page1",
		},
	}
	resp := &structs.CSISnapshotListResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIVolume.ListSnapshots", req, resp)
	require.NoError(t, err)
	require.Len(t, resp.Snapshots, 2)
	require.Equal(t, "vol-12345", resp.Snapshots[0].ExternalSourceVolumeID)
	require.Equal(t, "vol-abcde", resp.Snapshots[1].ExternalSourceVolumeID)
	require.True(t, resp.Snapshots[0].IsReady)
	require.Equal(t, "page2", resp.NextToken)
}

func TestCSIVolume_expandVolume(t *testing.T) {
	ci.Parallel(t)

	srv, cleanupSrv := TestServer(t, nil)
	t.Cleanup(cleanupSrv)
	testutil.WaitForLeader(t, srv.RPC)
	t.Log("server started 👍")

	_, fake, _, fakeVolID := testClientWithCSI(t, srv)

	endpoint := NewCSIVolumeEndpoint(srv, nil)
	plug, vol, err := endpoint.volAndPluginLookup(structs.DefaultNamespace, fakeVolID)
	must.NoError(t, err)

	// ensure nil checks
	expectErr := "unexpected nil value"
	err = endpoint.expandVolume(nil, plug, &csi.CapacityRange{})
	must.EqError(t, err, expectErr)
	err = endpoint.expandVolume(vol, nil, &csi.CapacityRange{})
	must.EqError(t, err, expectErr)
	err = endpoint.expandVolume(vol, plug, nil)
	must.EqError(t, err, expectErr)

	// these tests must be run in order, as they mutate vol along the way
	cases := []struct {
		Name string

		NewMin int64
		NewMax int64

		ExpectMin      int64
		ExpectMax      int64
		ControllerResp int64 // new capacity for the mock controller response
		ExpectCapacity int64 // expected resulting capacity on the volume
		ExpectErr      string
	}{
		{
			// successful expansion from initial vol with no capacity values.
			Name:   "success",
			NewMin: 1000,
			NewMax: 2000,

			ExpectMin:      1000,
			ExpectMax:      2000,
			ControllerResp: 1000,
			ExpectCapacity: 1000,
		},
		{
			// with min/max both zero, no action should be taken,
			// so expect no change to desired or actual capacity on the volume.
			Name:   "zero",
			NewMin: 0,
			NewMax: 0,

			ExpectMin:      1000,
			ExpectMax:      2000,
			ControllerResp: 999999, // this should not come into play
			ExpectCapacity: 1000,
		},
		{
			// increasing min is what actually triggers an expand to occur.
			Name:   "increase min",
			NewMin: 1500,
			NewMax: 2000,

			ExpectMin:      1500,
			ExpectMax:      2000,
			ControllerResp: 1500,
			ExpectCapacity: 1500,
		},
		{
			// min going down is okay, but no expand should occur.
			Name:   "reduce min",
			NewMin: 500,
			NewMax: 2000,

			ExpectMin:      500,
			ExpectMax:      2000,
			ControllerResp: 999999,
			ExpectCapacity: 1500,
		},
		{
			// max going up is okay, but no expand should occur.
			Name:   "increase max",
			NewMin: 500,
			NewMax: 5000,

			ExpectMin:      500,
			ExpectMax:      5000,
			ControllerResp: 999999,
			ExpectCapacity: 1500,
		},
		{
			// max lower than min is logically impossible.
			Name:      "max below min",
			NewMin:    3,
			NewMax:    2,
			ExpectErr: "max requested capacity (2 B) less than or equal to min (3 B)",
		},
		{
			// volume size cannot be reduced.
			Name:      "max below current",
			NewMax:    2,
			ExpectErr: "max requested capacity (2 B) less than or equal to current (1.5 kB)",
		},
	}

	for _, tc := range cases {
		t.Run(tc.Name, func(t *testing.T) {
			fake.NextControllerExpandVolumeResponse = &cstructs.ClientCSIControllerExpandVolumeResponse{
				CapacityBytes: tc.ControllerResp,
				// this also exercises some node expand code, incidentally
				NodeExpansionRequired: true,
			}

			err = endpoint.expandVolume(vol, plug, &csi.CapacityRange{
				RequiredBytes: tc.NewMin,
				LimitBytes:    tc.NewMax,
			})

			if tc.ExpectErr != "" {
				must.EqError(t, err, tc.ExpectErr)
				return
			}

			must.NoError(t, err)

			test.Eq(t, tc.ExpectCapacity, vol.Capacity,
				test.Sprint("unexpected capacity"))
			test.Eq(t, tc.ExpectMin, vol.RequestedCapacityMin,
				test.Sprint("unexpected min"))
			test.Eq(t, tc.ExpectMax, vol.RequestedCapacityMax,
				test.Sprint("unexpected max"))
		})
	}

	// a nodeExpandVolume error should fail expandVolume too
	t.Run("node error", func(t *testing.T) {
		expect := "sad node expand"
		fake.NextNodeExpandError = errors.New(expect)
		fake.NextControllerExpandVolumeResponse = &cstructs.ClientCSIControllerExpandVolumeResponse{
			CapacityBytes:         2000,
			NodeExpansionRequired: true,
		}
		err = endpoint.expandVolume(vol, plug, &csi.CapacityRange{
			RequiredBytes: 2000,
		})
		test.ErrorContains(t, err, expect)
	})

}

func TestCSIVolume_nodeExpandVolume(t *testing.T) {
	ci.Parallel(t)

	srv, cleanupSrv := TestServer(t, nil)
	t.Cleanup(cleanupSrv)
	testutil.WaitForLeader(t, srv.RPC)
	t.Log("server started 👍")

	c, fake, _, fakeVolID := testClientWithCSI(t, srv)
	fakeClaim := fakeCSIClaim(c.NodeID())

	endpoint := NewCSIVolumeEndpoint(srv, nil)
	plug, vol, err := endpoint.volAndPluginLookup(structs.DefaultNamespace, fakeVolID)
	must.NoError(t, err)

	// there's not a lot of logic here -- validation has been done prior,
	// in (controller) expandVolume and what preceeds it.
	cases := []struct {
		Name  string
		Error error
	}{
		{
			Name: "ok",
		},
		{
			Name:  "not ok",
			Error: errors.New("test node expand fail"),
		},
	}

	for _, tc := range cases {
		t.Run(tc.Name, func(t *testing.T) {

			fake.NextNodeExpandError = tc.Error
			capacity := &csi.CapacityRange{
				RequiredBytes: 10,
				LimitBytes:    10,
			}

			err = endpoint.nodeExpandVolume(vol, plug, capacity)

			if tc.Error == nil {
				test.NoError(t, err)
			} else {
				must.Error(t, err)
				must.ErrorContains(t, err,
					fmt.Sprintf("CSI.NodeExpandVolume error: %s", tc.Error))
			}

			req := fake.LastNodeExpandRequest
			must.NotNil(t, req, must.Sprint("request should have happened"))
			test.Eq(t, fakeVolID, req.VolumeID)
			test.Eq(t, capacity, req.Capacity)
			test.Eq(t, "fake-csi-plugin", req.PluginID)
			test.Eq(t, "fake-csi-external-id", req.ExternalID)
			test.Eq(t, fakeClaim, req.Claim)

		})
	}
}

func TestCSIPluginEndpoint_RegisterViaFingerprint(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	deleteNodes := state.CreateTestCSIPlugin(srv.fsm.State(), "foo")
	defer deleteNodes()

	state := srv.fsm.State()
	state.BootstrapACLTokens(structs.MsgTypeTestSetup, 1, 0, mock.ACLManagementToken())
	srv.config.ACLEnabled = true
	codec := rpcClient(t, srv)

	// Get the plugin back out
	listJob := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})
	policy := mock.PluginPolicy("read") + listJob
	getToken := mock.CreatePolicyAndToken(t, state, 1001, "plugin-read", policy)

	req2 := &structs.CSIPluginGetRequest{
		ID: "foo",
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			AuthToken: getToken.SecretID,
		},
	}
	resp2 := &structs.CSIPluginGetResponse{}
	err := msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", req2, resp2)
	require.NoError(t, err)

	// Get requires plugin-read, not plugin-list
	lPolicy := mock.PluginPolicy("list")
	lTok := mock.CreatePolicyAndToken(t, state, 1003, "plugin-list", lPolicy)
	req2.AuthToken = lTok.SecretID
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", req2, resp2)
	require.Error(t, err, "Permission denied")

	// List plugins
	req3 := &structs.CSIPluginListRequest{
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			AuthToken: getToken.SecretID,
		},
	}
	resp3 := &structs.CSIPluginListResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.List", req3, resp3)
	require.NoError(t, err)
	require.Equal(t, 1, len(resp3.Plugins))

	// ensure that plugin->alloc denormalization does COW correctly
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.List", req3, resp3)
	require.NoError(t, err)
	require.Equal(t, 1, len(resp3.Plugins))

	// List allows plugin-list
	req3.AuthToken = lTok.SecretID
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.List", req3, resp3)
	require.NoError(t, err)
	require.Equal(t, 1, len(resp3.Plugins))

	// Deregistration works
	deleteNodes()

	// Plugin is missing
	req2.AuthToken = getToken.SecretID
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", req2, resp2)
	require.NoError(t, err)
	require.Nil(t, resp2.Plugin)
}

func TestCSIPluginEndpoint_RegisterViaJob(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, nil)
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	codec := rpcClient(t, srv)

	// Register a job that creates the plugin
	job := mock.Job()
	job.TaskGroups[0].Tasks[0].CSIPluginConfig = &structs.TaskCSIPluginConfig{
		ID:   "foo",
		Type: structs.CSIPluginTypeNode,
	}

	req1 := &structs.JobRegisterRequest{
		Job:          job,
		WriteRequest: structs.WriteRequest{Region: "global"},
	}
	resp1 := &structs.JobRegisterResponse{}
	err := msgpackrpc.CallWithCodec(codec, "Job.Register", req1, resp1)
	require.NoError(t, err)

	// Verify that the plugin exists and is unhealthy
	req2 := &structs.CSIPluginGetRequest{
		ID:           "foo",
		QueryOptions: structs.QueryOptions{Region: "global"},
	}
	resp2 := &structs.CSIPluginGetResponse{}
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", req2, resp2)
	require.NoError(t, err)
	require.NotNil(t, resp2.Plugin)
	require.Zero(t, resp2.Plugin.ControllersHealthy)
	require.Zero(t, resp2.Plugin.NodesHealthy)
	require.Equal(t, job.ID, resp2.Plugin.NodeJobs[structs.DefaultNamespace][job.ID].ID)

	// Health depends on node fingerprints
	deleteNodes := state.CreateTestCSIPlugin(srv.fsm.State(), "foo")
	defer deleteNodes()

	resp2.Plugin = nil
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", req2, resp2)
	require.NoError(t, err)
	require.NotNil(t, resp2.Plugin)
	require.NotZero(t, resp2.Plugin.ControllersHealthy)
	require.NotZero(t, resp2.Plugin.NodesHealthy)
	require.Equal(t, job.ID, resp2.Plugin.NodeJobs[structs.DefaultNamespace][job.ID].ID)

	// All fingerprints failing makes the plugin unhealthy, but does not delete it
	deleteNodes()
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", req2, resp2)
	require.NoError(t, err)
	require.NotNil(t, resp2.Plugin)
	require.Zero(t, resp2.Plugin.ControllersHealthy)
	require.Zero(t, resp2.Plugin.NodesHealthy)
	require.Equal(t, job.ID, resp2.Plugin.NodeJobs[structs.DefaultNamespace][job.ID].ID)

	// Job deregistration is necessary to gc the plugin
	req3 := &structs.JobDeregisterRequest{
		JobID: job.ID,
		Purge: true,
		WriteRequest: structs.WriteRequest{
			Region:    "global",
			Namespace: structs.DefaultNamespace,
		},
	}
	resp3 := &structs.JobDeregisterResponse{}
	err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req3, resp3)
	require.NoError(t, err)

	// Plugin has been gc'ed
	resp2.Plugin = nil
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", req2, resp2)
	require.NoError(t, err)
	require.Nil(t, resp2.Plugin)
}

func TestCSIPluginEndpoint_DeleteViaGC(t *testing.T) {
	ci.Parallel(t)
	srv, shutdown := TestServer(t, func(c *Config) {
		c.NumSchedulers = 0 // Prevent automatic dequeue
	})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	deleteNodes := state.CreateTestCSIPlugin(srv.fsm.State(), "foo")
	defer deleteNodes()

	state := srv.fsm.State()
	state.BootstrapACLTokens(structs.MsgTypeTestSetup, 1, 0, mock.ACLManagementToken())
	srv.config.ACLEnabled = true
	codec := rpcClient(t, srv)

	// Get the plugin back out
	listJob := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})
	policy := mock.PluginPolicy("read") + listJob
	getToken := mock.CreatePolicyAndToken(t, state, 1001, "plugin-read", policy)

	reqGet := &structs.CSIPluginGetRequest{
		ID: "foo",
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			AuthToken: getToken.SecretID,
		},
	}
	respGet := &structs.CSIPluginGetResponse{}
	err := msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", reqGet, respGet)
	require.NoError(t, err)
	require.NotNil(t, respGet.Plugin)

	// Delete plugin
	reqDel := &structs.CSIPluginDeleteRequest{
		ID: "foo",
		QueryOptions: structs.QueryOptions{
			Region:    "global",
			AuthToken: getToken.SecretID,
		},
	}
	respDel := &structs.CSIPluginDeleteResponse{}

	// Improper permissions
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Delete", reqDel, respDel)
	require.EqualError(t, err, structs.ErrPermissionDenied.Error())

	// Retry with management permissions
	reqDel.AuthToken = srv.getLeaderAcl()
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Delete", reqDel, respDel)
	require.EqualError(t, err, "plugin in use")

	// Plugin was not deleted
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", reqGet, respGet)
	require.NoError(t, err)
	require.NotNil(t, respGet.Plugin)

	// Empty the plugin
	plugin := respGet.Plugin.Copy()
	plugin.Controllers = map[string]*structs.CSIInfo{}
	plugin.Nodes = map[string]*structs.CSIInfo{}

	index, _ := state.LatestIndex()
	index++
	err = state.UpsertCSIPlugin(index, plugin)
	require.NoError(t, err)

	// Retry now that it's empty
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Delete", reqDel, respDel)
	require.NoError(t, err)

	// Plugin is deleted
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Get", reqGet, respGet)
	require.NoError(t, err)
	require.Nil(t, respGet.Plugin)

	// Safe to call on already-deleted plugnis
	err = msgpackrpc.CallWithCodec(codec, "CSIPlugin.Delete", reqDel, respDel)
	require.NoError(t, err)
}

func TestCSI_RPCVolumeAndPluginLookup(t *testing.T) {
	ci.Parallel(t)

	srv, shutdown := TestServer(t, func(c *Config) {})
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	state := srv.fsm.State()
	id0 := uuid.Generate()
	id1 := uuid.Generate()
	id2 := uuid.Generate()

	// Create a client node with a plugin
	node := mock.Node()
	node.CSIControllerPlugins = map[string]*structs.CSIInfo{
		"minnie": {PluginID: "minnie", Healthy: true, RequiresControllerPlugin: true,
			ControllerInfo: &structs.CSIControllerInfo{SupportsAttachDetach: true},
		},
	}
	node.CSINodePlugins = map[string]*structs.CSIInfo{
		"adam": {PluginID: "adam", Healthy: true},
	}
	err := state.UpsertNode(structs.MsgTypeTestSetup, 3, node)
	require.NoError(t, err)

	// Create 2 volumes
	vols := []*structs.CSIVolume{
		{
			ID:                 id0,
			Namespace:          structs.DefaultNamespace,
			PluginID:           "minnie",
			AccessMode:         structs.CSIVolumeAccessModeMultiNodeSingleWriter,
			AttachmentMode:     structs.CSIVolumeAttachmentModeFilesystem,
			ControllerRequired: true,
		},
		{
			ID:                 id1,
			Namespace:          structs.DefaultNamespace,
			PluginID:           "adam",
			AccessMode:         structs.CSIVolumeAccessModeMultiNodeSingleWriter,
			AttachmentMode:     structs.CSIVolumeAttachmentModeFilesystem,
			ControllerRequired: false,
		},
	}
	err = state.UpsertCSIVolume(1002, vols)
	require.NoError(t, err)

	// has controller
	c := NewCSIVolumeEndpoint(srv, nil)
	plugin, vol, err := c.volAndPluginLookup(structs.DefaultNamespace, id0)
	require.NotNil(t, plugin)
	require.NotNil(t, vol)
	require.NoError(t, err)

	// no controller
	plugin, vol, err = c.volAndPluginLookup(structs.DefaultNamespace, id1)
	require.Nil(t, plugin)
	require.NotNil(t, vol)
	require.NoError(t, err)

	// doesn't exist
	plugin, vol, err = c.volAndPluginLookup(structs.DefaultNamespace, id2)
	require.Nil(t, plugin)
	require.Nil(t, vol)
	require.EqualError(t, err, fmt.Sprintf("volume not found: %s", id2))
}

func TestCSI_SerializedControllerRPC(t *testing.T) {
	ci.Parallel(t)

	srv, shutdown := TestServer(t, func(c *Config) { c.NumSchedulers = 0 })
	defer shutdown()
	testutil.WaitForLeader(t, srv.RPC)

	var wg sync.WaitGroup
	wg.Add(3)

	timeCh := make(chan lang.Pair[string, time.Duration])

	testFn := func(pluginID string, dur time.Duration) {
		defer wg.Done()
		c := NewCSIVolumeEndpoint(srv, nil)
		now := time.Now()
		err := c.serializedControllerRPC(pluginID, func() error {
			time.Sleep(dur)
			return nil
		})
		elapsed := time.Since(now)
		timeCh <- lang.Pair[string, time.Duration]{pluginID, elapsed}
		must.NoError(t, err)
	}

	go testFn("plugin1", 50*time.Millisecond)
	go testFn("plugin2", 50*time.Millisecond)
	go testFn("plugin1", 50*time.Millisecond)

	totals := map[string]time.Duration{}
	for i := 0; i < 3; i++ {
		pair := <-timeCh
		totals[pair.First] += pair.Second
	}

	wg.Wait()

	// plugin1 RPCs should block each other
	must.GreaterEq(t, 150*time.Millisecond, totals["plugin1"])
	must.Less(t, 200*time.Millisecond, totals["plugin1"])

	// plugin1 RPCs should not block plugin2 RPCs
	must.GreaterEq(t, 50*time.Millisecond, totals["plugin2"])
	must.Less(t, 100*time.Millisecond, totals["plugin2"])
}

// testClientWithCSI sets up a client with a fake CSI plugin.
// Much of the plugin/volume configuration is only to pass validation;
// callers should modify MockClientCSI's Next* fields.
func testClientWithCSI(t *testing.T, srv *Server) (c *client.Client, m *MockClientCSI, plugID, volID string) {
	t.Helper()

	m = newMockClientCSI()
	plugID = "fake-csi-plugin"
	volID = "fake-csi-volume"

	c, cleanup := client.TestClientWithRPCs(t,
		func(c *cconfig.Config) {
			c.Servers = []string{srv.config.RPCAddr.String()}
			c.Node.CSIControllerPlugins = map[string]*structs.CSIInfo{
				plugID: {
					PluginID: plugID,
					Healthy:  true,
					ControllerInfo: &structs.CSIControllerInfo{
						// Supports.* everything, but Next* values must be set on the mock.
						SupportsAttachDetach:             true,
						SupportsClone:                    true,
						SupportsCondition:                true,
						SupportsCreateDelete:             true,
						SupportsCreateDeleteSnapshot:     true,
						SupportsExpand:                   true,
						SupportsGet:                      true,
						SupportsGetCapacity:              true,
						SupportsListSnapshots:            true,
						SupportsListVolumes:              true,
						SupportsListVolumesAttachedNodes: true,
						SupportsReadOnlyAttach:           true,
					},
					RequiresControllerPlugin: true,
				},
			}
			c.Node.CSINodePlugins = map[string]*structs.CSIInfo{
				plugID: {
					PluginID: plugID,
					Healthy:  true,
					NodeInfo: &structs.CSINodeInfo{
						ID:                c.Node.GetID(),
						SupportsCondition: true,
						SupportsExpand:    true,
						SupportsStats:     true,
					},
				},
			}
		},
		map[string]interface{}{"CSI": m}, // MockClientCSI
	)
	t.Cleanup(func() { test.NoError(t, cleanup()) })
	testutil.WaitForClient(t, srv.RPC, c.NodeID(), c.Region())
	t.Log("client started with fake CSI plugin 👍")

	// Register a minimum-viable fake volume
	req := &structs.CSIVolumeRegisterRequest{
		Volumes: []*structs.CSIVolume{{
			PluginID:   plugID,
			ID:         volID,
			ExternalID: "fake-csi-external-id",
			Namespace:  structs.DefaultNamespace,
			RequestedCapabilities: []*structs.CSIVolumeCapability{
				{
					AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
					AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
				},
			},
			WriteClaims: map[string]*structs.CSIVolumeClaim{
				"fake-csi-claim": fakeCSIClaim(c.NodeID()),
			},
		}},
		WriteRequest: structs.WriteRequest{Region: srv.Region()},
	}
	must.NoError(t, srv.RPC("CSIVolume.Register", req, &structs.CSIVolumeRegisterResponse{}))
	t.Logf("CSI volume %s registered 👍", volID)

	return c, m, plugID, volID
}

func fakeCSIClaim(nodeID string) *structs.CSIVolumeClaim {
	return &structs.CSIVolumeClaim{
		NodeID:         nodeID,
		AllocationID:   "fake-csi-alloc",
		ExternalNodeID: "fake-csi-external-node",
		Mode:           structs.CSIVolumeClaimWrite,
		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		State:          structs.CSIVolumeClaimStateTaken,
	}
}