luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

csi: support Secrets parameter in CSI RPCs (#7923) 

CSI plugins can require credentials for some publishing and
unpublishing workflow RPCs. Secrets are configured at the time of
volume registration, stored in the volume struct, and then passed
around as an opaque map by Nomad to the plugins.
This commit is contained in:
Tim Gross 2020-05-11 17:12:51 -04:00 committed by GitHub
parent 938e626750
commit 4374c1a837
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 105 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/csi.go
View File

@ -87,6 +87,8 @@ type CSIMountOptions struct {
ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` // report unexpected keys
}
type CSISecrets map[string]string
// CSIVolume is used for serialization, see also nomad/structs/csi.go
type CSIVolume struct {
ID string
@ -97,6 +99,7 @@ type CSIVolume struct {
AccessMode CSIVolumeAccessMode `hcl:"access_mode"`
AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"`
MountOptions *CSIMountOptions `hcl:"mount_options"`
Secrets CSISecrets `hcl:"secrets"`
// Allocations, tracking claim status
ReadAllocs map[string]*Allocation
@ -162,7 +165,6 @@ type CSIVolumeListStub struct {
Topologies []*CSITopology
AccessMode CSIVolumeAccessMode
AttachmentMode CSIVolumeAttachmentMode
MountOptions *CSIMountOptions
Schedulable bool
PluginID string
Provider string

1
client/csi_endpoint.go
View File

@ -61,6 +61,7 @@ func (c *CSI) ControllerValidateVolume(req *structs.ClientCSIControllerValidateV
// CSI ValidateVolumeCapabilities errors for timeout, codes.Unavailable and
// codes.ResourceExhausted are retried; all other errors are fatal.
return plugin.ControllerValidateCapabilities(ctx, req.VolumeID, caps,
req.Secrets,
grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout),
grpc_retry.WithMax(3),
grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)))

2
client/pluginmanager/csimanager/volume.go
View File

@ -170,6 +170,7 @@ func (v *volumeManager) stageVolume(ctx context.Context, vol *structs.CSIVolume,
publishContext,
pluginStagingPath,
capability,
vol.Secrets,
grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
grpc_retry.WithMax(3),
grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
@ -208,6 +209,7 @@ func (v *volumeManager) publishVolume(ctx context.Context, vol *structs.CSIVolum
TargetPath: pluginTargetPath,
VolumeCapability: capabilities,
Readonly: usage.ReadOnly,
Secrets: vol.Secrets,
},
grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
grpc_retry.WithMax(3),

19
client/structs/csi.go
View File

@ -35,6 +35,8 @@ type ClientCSIControllerValidateVolumeRequest struct {
AttachmentMode structs.CSIVolumeAttachmentMode
AccessMode structs.CSIVolumeAccessMode
Secrets structs.CSISecrets
// Parameters map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670
CSIControllerQuery
}
@ -66,6 +68,15 @@ type ClientCSIControllerAttachVolumeRequest struct {
// only works when the Controller has the PublishReadonly capability.
ReadOnly bool
// Secrets required by plugin to complete the controller publish
// volume request. This field is OPTIONAL.
Secrets structs.CSISecrets
// TODO https://github.com/hashicorp/nomad/issues/7771
// Volume context as returned by storage provider in CreateVolumeResponse.
// This field is optional.
// VolumeContext map[string]string
CSIControllerQuery
}
@ -82,8 +93,10 @@ func (c *ClientCSIControllerAttachVolumeRequest) ToCSIRequest() (*csi.Controller
return &csi.ControllerPublishVolumeRequest{
VolumeID: c.VolumeID,
NodeID: c.ClientCSINodeID,
ReadOnly: c.ReadOnly,
VolumeCapability: caps,
ReadOnly: c.ReadOnly,
Secrets: c.Secrets,
// VolumeContext: c.VolumeContext, TODO: https://github.com/hashicorp/nomad/issues/7771
}, nil
}
@ -117,6 +130,10 @@ type ClientCSIControllerDetachVolumeRequest struct {
// by the target node for this plugin name.
ClientCSINodeID string
// Secrets required by plugin to complete the controller unpublish
// volume request. This field is OPTIONAL.
Secrets structs.CSISecrets
CSIControllerQuery
}

4
command/agent/csi_endpoint.go
View File

@ -85,6 +85,10 @@ func (s *HTTPServer) csiVolumeGet(id string, resp http.ResponseWriter, req *http
return nil, CodedError(404, "volume not found")
}
// remove sensitive fields, as our redaction mechanism doesn't
// help serializing here
out.Volume.Secrets = nil
out.Volume.MountOptions = nil
return out.Volume, nil
}

6
command/node_status.go
View File

@ -569,19 +569,17 @@ func (c *NodeStatusCommand) outputNodeCSIVolumeInfo(client *api.Client, node *ap
// Output the volumes in name order
output := make([]string, 0, len(names)+1)
output = append(output, "ID|Name|Plugin ID|Schedulable|Provider|Access Mode|Mount Options")
output = append(output, "ID|Name|Plugin ID|Schedulable|Provider|Access Mode")
for _, name := range names {
v := volumes[name]
r := requests[v.ID]
output = append(output, fmt.Sprintf(
"%s|%s|%s|%t|%s|%s|%s",
"%s|%s|%s|%t|%s|%s",
v.ID,
name,
v.PluginID,
v.Schedulable,
v.Provider,
v.AccessMode,
csiVolMountOption(v.MountOptions, r.MountOptions),
))
}

4
command/volume_register_test.go
View File

@ -57,6 +57,9 @@ namespace = "n"
access_mode = "single-node-writer"
attachment_mode = "file-system"
plugin_id = "p"
secrets {
mysecret = "secretvalue"
}
`,
q: &api.CSIVolume{
ID: "foo",
@ -64,6 +67,7 @@ plugin_id = "p"
AccessMode: "single-node-writer",
AttachmentMode: "file-system",
PluginID: "p",
Secrets: api.CSISecrets{"mysecret": "secretvalue"},
},
err: "",
}, {

4
nomad/csi_endpoint.go
View File

@ -237,6 +237,8 @@ func (v *CSIVolume) controllerValidateVolume(req *structs.CSIVolumeRegisterReque
VolumeID: vol.RemoteID(),
AttachmentMode: vol.AttachmentMode,
AccessMode: vol.AccessMode,
Secrets: vol.Secrets,
// Parameters: TODO: https://github.com/hashicorp/nomad/issues/7670
}
cReq.PluginID = plugin.ID
cResp := &cstructs.ClientCSIControllerValidateVolumeResponse{}
@ -440,6 +442,8 @@ func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest,
AttachmentMode: vol.AttachmentMode,
AccessMode: vol.AccessMode,
ReadOnly: req.Claim == structs.CSIVolumeClaimRead,
Secrets: vol.Secrets,
// VolumeContext: TODO https://github.com/hashicorp/nomad/issues/7771
}
cReq.PluginID = plug.ID
cResp := &cstructs.ClientCSIControllerAttachVolumeResponse{}

6
nomad/csi_endpoint_test.go
View File

@ -37,6 +37,7 @@ func TestCSIVolumeEndpoint_Get(t *testing.T) {
AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
PluginID: "minnie",
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}
err := state.CSIVolumeRegister(999, vols)
require.NoError(t, err)
@ -84,6 +85,7 @@ func TestCSIVolumeEndpoint_Get_ACL(t *testing.T) {
AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
PluginID: "minnie",
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}
err := state.CSIVolumeRegister(999, vols)
require.NoError(t, err)
@ -139,6 +141,7 @@ func TestCSIVolumeEndpoint_Register(t *testing.T) {
PluginID: "minnie",
AccessMode: structs.CSIVolumeAccessModeMultiNodeReader,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}
// Create the register request
@ -255,6 +258,7 @@ func TestCSIVolumeEndpoint_Claim(t *testing.T) {
Topologies: []*structs.CSITopology{{
Segments: map[string]string{"foo": "bar"},
}},
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}
index++
err = state.CSIVolumeRegister(index, vols)
@ -373,6 +377,7 @@ func TestCSIVolumeEndpoint_ClaimWithController(t *testing.T) {
ControllerRequired: true,
AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}
err = state.CSIVolumeRegister(1003, vols)
@ -439,6 +444,7 @@ func TestCSIVolumeEndpoint_List(t *testing.T) {
AccessMode: structs.CSIVolumeAccessModeMultiNodeReader,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
PluginID: "minnie",
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}, {
ID: id1,
Namespace: structs.DefaultNamespace,

1
nomad/mock/mock.go
View File

@ -1311,6 +1311,7 @@ func CSIVolume(plugin *structs.CSIPlugin) *structs.CSIVolume {
AccessMode: structs.CSIVolumeAccessModeSingleNodeWriter,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
MountOptions: &structs.CSIMountOptions{},
Secrets: structs.CSISecrets{},
ReadAllocs: map[string]*structs.Allocation{},
WriteAllocs: map[string]*structs.Allocation{},
ReadClaims: map[string]*structs.CSIVolumeClaim{},

30
nomad/structs/csi.go
View File

@ -185,6 +185,27 @@ func (v *CSIMountOptions) GoString() string {
return v.String()
}
// CSISecrets contain optional additional configuration that can be used
// when specifying that a Volume should be used with VolumeAccessTypeMount.
type CSISecrets map[string]string
// CSISecrets implements the Stringer and GoStringer interfaces to prevent
// accidental leakage of secrets via logs.
var _ fmt.Stringer = &CSISecrets{}
var _ fmt.GoStringer = &CSISecrets{}
func (s *CSISecrets) String() string {
redacted := map[string]string{}
for k := range *s {
redacted[k] = "[REDACTED]"
}
return fmt.Sprintf("csi.CSISecrets(%v)", redacted)
}
func (s *CSISecrets) GoString() string {
return s.String()
}
type CSIVolumeClaim struct {
AllocationID string
NodeID string
@ -214,6 +235,7 @@ type CSIVolume struct {
AccessMode CSIVolumeAccessMode
AttachmentMode CSIVolumeAttachmentMode
MountOptions *CSIMountOptions
Secrets CSISecrets
// Allocations, tracking claim status
ReadAllocs map[string]*Allocation // AllocID -> Allocation
@ -249,7 +271,6 @@ type CSIVolListStub struct {
Topologies []*CSITopology
AccessMode CSIVolumeAccessMode
AttachmentMode CSIVolumeAttachmentMode
MountOptions *CSIMountOptions
CurrentReaders int
CurrentWriters int
Schedulable bool
@ -279,6 +300,9 @@ func (v *CSIVolume) newStructs() {
if v.Topologies == nil {
v.Topologies = []*CSITopology{}
}
if v.Secrets == nil {
v.Secrets = CSISecrets{}
}
v.ReadAllocs = map[string]*Allocation{}
v.WriteAllocs = map[string]*Allocation{}
@ -304,7 +328,6 @@ func (v *CSIVolume) Stub() *CSIVolListStub {
Topologies: v.Topologies,
AccessMode: v.AccessMode,
AttachmentMode: v.AttachmentMode,
MountOptions: v.MountOptions,
CurrentReaders: len(v.ReadAllocs),
CurrentWriters: len(v.WriteAllocs),
Schedulable: v.Schedulable,
@ -365,6 +388,9 @@ func (v *CSIVolume) Copy() *CSIVolume {
copy := *v
out := &copy
out.newStructs()
for k, v := range v.Secrets {
out.Secrets[k] = v
}
for k, v := range v.ReadAllocs {
out.ReadAllocs[k] = v

1
nomad/volumewatcher/volume_watcher.go
View File

@ -350,6 +350,7 @@ func (vw *volumeWatcher) controllerDetach(vol *structs.CSIVolume, claim *structs
cReq := &cstructs.ClientCSIControllerDetachVolumeRequest{
VolumeID: vol.RemoteID(),
ClientCSINodeID: targetCSIInfo.NodeInfo.ID,
Secrets: vol.Secrets,
}
cReq.PluginID = plug.ID
err = vw.rpc.ControllerDetachVolume(cReq,

9
plugins/csi/client.go
View File

@ -12,6 +12,7 @@ import (
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/grpc-middleware/logging"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/base"
"github.com/hashicorp/nomad/plugins/shared/hclspec"
"google.golang.org/grpc"
@ -293,7 +294,7 @@ func (c *client) ControllerUnpublishVolume(ctx context.Context, req *ControllerU
return &ControllerUnpublishVolumeResponse{}, nil
}
func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
if c == nil {
return fmt.Errorf("Client not initialized")
}
@ -314,6 +315,9 @@ func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID st
VolumeCapabilities: []*csipbv1.VolumeCapability{
capabilities.ToCSIRepresentation(),
},
// VolumeContext: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7771
// Parameters: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670
Secrets: secrets,
}
resp, err := c.controllerClient.ValidateVolumeCapabilities(ctx, req, opts...)
@ -461,7 +465,7 @@ func (c *client) NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error)
return result, nil
}
func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
if c == nil {
return fmt.Errorf("Client not initialized")
}
@ -483,6 +487,7 @@ func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishCo
PublishContext: publishContext,
StagingTargetPath: stagingTargetPath,
VolumeCapability: capabilities.ToCSIRepresentation(),
Secrets: secrets,
}
// NodeStageVolume's response contains no extra data. If err == nil, we were

5
plugins/csi/client_test.go
View File

@ -578,7 +578,7 @@ func TestClient_RPC_ControllerValidateVolume(t *testing.T) {
cc.NextErr = c.ResponseErr
err := client.ControllerValidateCapabilities(
context.TODO(), "volumeID", requestedCaps)
context.TODO(), "volumeID", requestedCaps, structs.CSISecrets{})
if c.ExpectedErr != nil {
require.Error(t, c.ExpectedErr, err, c.Name)
} else {
@ -616,7 +616,8 @@ func TestClient_RPC_NodeStageVolume(t *testing.T) {
nc.NextErr = c.ResponseErr
nc.NextStageVolumeResponse = c.Response
err := client.NodeStageVolume(context.TODO(), "foo", nil, "/foo", &VolumeCapability{})
err := client.NodeStageVolume(context.TODO(), "foo", nil, "/foo",
&VolumeCapability{}, structs.CSISecrets{})
if c.ExpectedErr != nil {
require.Error(t, c.ExpectedErr, err)
} else {

5
plugins/csi/fake/client.go
View File

@ -8,6 +8,7 @@ import (
"fmt"
"sync"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/base"
"github.com/hashicorp/nomad/plugins/csi"
"github.com/hashicorp/nomad/plugins/shared/hclspec"
@ -159,7 +160,7 @@ func (c *Client) ControllerUnpublishVolume(ctx context.Context, req *csi.Control
return c.NextControllerUnpublishVolumeResponse, c.NextControllerUnpublishVolumeErr
}
func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
c.Mu.Lock()
defer c.Mu.Unlock()
@ -191,7 +192,7 @@ func (c *Client) NodeGetInfo(ctx context.Context) (*csi.NodeGetInfoResponse, err
// NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
// to prepare a volume for usage on a host. If err == nil, the response should
// be assumed to be successful.
func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
c.Mu.Lock()
defer c.Mu.Unlock()

15
plugins/csi/plugin.go
View File

@ -43,7 +43,7 @@ type CSIPlugin interface {
// ControllerValidateCapabilities is used to validate that a volume exists and
// supports the requested capability.
ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, opts ...grpc.CallOption) error
ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error
// NodeGetCapabilities is used to return the available capabilities from the
// Node Service.
@ -56,7 +56,7 @@ type CSIPlugin interface {
// NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
// to prepare a volume for usage on a host. If err == nil, the response should
// be assumed to be successful.
NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, opts ...grpc.CallOption) error
NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error
// NodeUnstageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
// to undo the work performed by NodeStageVolume. If a volume has been staged,
@ -111,8 +111,9 @@ type NodePublishVolumeRequest struct {
Readonly bool
// Reserved for future use.
Secrets map[string]string
// Secrets required by plugins to complete the node publish volume
// request. This field is OPTIONAL.
Secrets structs.CSISecrets
}
func (r *NodePublishVolumeRequest) ToCSIRepresentation() *csipbv1.NodePublishVolumeRequest {
@ -233,6 +234,8 @@ type ControllerPublishVolumeRequest struct {
NodeID string
ReadOnly bool
VolumeCapability *VolumeCapability
Secrets structs.CSISecrets
// VolumeContext map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7771
}
func (r *ControllerPublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerPublishVolumeRequest {
@ -245,6 +248,8 @@ func (r *ControllerPublishVolumeRequest) ToCSIRepresentation() *csipbv1.Controll
NodeId: r.NodeID,
Readonly: r.ReadOnly,
VolumeCapability: r.VolumeCapability.ToCSIRepresentation(),
Secrets: r.Secrets,
// VolumeContext: r.VolumeContext, https://github.com/hashicorp/nomad/issues/7771
}
}
@ -265,6 +270,7 @@ type ControllerPublishVolumeResponse struct {
type ControllerUnpublishVolumeRequest struct {
VolumeID string
NodeID string
Secrets structs.CSISecrets
}
func (r *ControllerUnpublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerUnpublishVolumeRequest {
@ -275,6 +281,7 @@ func (r *ControllerUnpublishVolumeRequest) ToCSIRepresentation() *csipbv1.Contro
return &csipbv1.ControllerUnpublishVolumeRequest{
VolumeId: r.VolumeID,
NodeId: r.NodeID,
Secrets: r.Secrets,
}
}

7
website/pages/docs/commands/volume/register.mdx
View File

@ -46,6 +46,9 @@ mount_options {
fs_type = "ext4"
mount_flags = ["ro"]
}
secrets {
example_secret = "xyzzy"
}
```
## Volume Specification Parameters
@ -84,6 +87,10 @@ mount_options {
- `fs_type`: file system type (ex. `"ext4"`)
- `mount_flags`: the flags passed to `mount` (ex. `"ro,noatime"`)
- `secrets` <code>(map:nil)</code> - An optional key-value map of
strings used as credentials for publishing and unpublishing volumes.
[volume_specification]: #volume-specification
[csi]: https://github.com/container-storage-interface/spec
[csi_plugin]: /docs/job-specification/csi_plugin