From f1ab38e8451cd3e1fc6a265549b2bab4a90c7093 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Tue, 28 Jan 2020 13:19:56 +0100 Subject: [PATCH] volume_manager: Introduce helpers for staging This commit adds helpers that create and validate the staging directory for a given volume. It is currently missing usage options as the interfaces are not yet in place for those. The staging directory is only required when a volume has the STAGE_UNSTAGE Volume capability and has to live within the plugin root as the plugin needs to be able to create mounts inside it from within the container. --- client/pluginmanager/csimanager/volume.go | 31 +++++ .../pluginmanager/csimanager/volume_test.go | 108 ++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 client/pluginmanager/csimanager/volume_test.go diff --git a/client/pluginmanager/csimanager/volume.go b/client/pluginmanager/csimanager/volume.go index 158efd7b5..966caaff3 100644 --- a/client/pluginmanager/csimanager/volume.go +++ b/client/pluginmanager/csimanager/volume.go @@ -3,9 +3,12 @@ package csimanager import ( "context" "fmt" + "os" + "path/filepath" "time" "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/helper/mount" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/csi" ) @@ -51,6 +54,34 @@ func newVolumeManager(logger hclog.Logger, plugin csi.CSIPlugin, rootDir string, } } +func (v *volumeManager) stagingDirForVolume(vol *structs.CSIVolume) string { + return filepath.Join(v.mountRoot, StagingDirName, vol.ID, "todo-provide-usage-options") +} + +// ensureStagingDir attempts to create a directory for use when staging a volume +// and then validates that the path is not already a mount point for e.g an +// existing volume stage. +// +// Returns whether the directory is a pre-existing mountpoint, the staging path, +// and any errors that occured. +func (v *volumeManager) ensureStagingDir(vol *structs.CSIVolume) (bool, string, error) { + stagingPath := v.stagingDirForVolume(vol) + + // Make the staging path, owned by the Nomad User + if err := os.MkdirAll(stagingPath, 0700); err != nil && !os.IsExist(err) { + return false, "", fmt.Errorf("failed to create staging directory for volume (%s): %v", vol.ID, err) + } + + // Validate that it is not already a mount point + m := mount.New() + isNotMount, err := m.IsNotAMountPoint(stagingPath) + if err != nil { + return false, "", fmt.Errorf("mount point detection failed for volume (%s): %v", vol.ID, err) + } + + return !isNotMount, stagingPath, nil +} + // MountVolume performs the steps required for using a given volume // configuration for the provided allocation. // diff --git a/client/pluginmanager/csimanager/volume_test.go b/client/pluginmanager/csimanager/volume_test.go new file mode 100644 index 000000000..f69fc5d1c --- /dev/null +++ b/client/pluginmanager/csimanager/volume_test.go @@ -0,0 +1,108 @@ +package csimanager + +import ( + "io/ioutil" + "os" + "runtime" + "testing" + + "github.com/hashicorp/nomad/helper/testlog" + "github.com/hashicorp/nomad/nomad/structs" + csifake "github.com/hashicorp/nomad/plugins/csi/fake" + "github.com/stretchr/testify/require" +) + +func tmpDir(t testing.TB) string { + t.Helper() + dir, err := ioutil.TempDir("", "nomad") + require.NoError(t, err) + return dir +} + +func TestVolumeManager_ensureStagingDir(t *testing.T) { + t.Parallel() + + cases := []struct { + Name string + Volume *structs.CSIVolume + CreateDirAheadOfTime bool + MountDirAheadOfTime bool + + ExpectedErr error + ExpectedMountState bool + }{ + { + Name: "Creates a directory when one does not exist", + Volume: &structs.CSIVolume{ID: "foo"}, + }, + { + Name: "Does not fail because of a pre-existing directory", + Volume: &structs.CSIVolume{ID: "foo"}, + CreateDirAheadOfTime: true, + }, + { + Name: "Returns negative mount info", + Volume: &structs.CSIVolume{ID: "foo"}, + }, + { + Name: "Returns positive mount info", + Volume: &structs.CSIVolume{ID: "foo"}, + CreateDirAheadOfTime: true, + MountDirAheadOfTime: true, + ExpectedMountState: true, + }, + } + + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + // Step 1: Validate that the test case makes sense + if !tc.CreateDirAheadOfTime && tc.MountDirAheadOfTime { + require.Fail(t, "Cannot Mount without creating a dir") + } + + if tc.MountDirAheadOfTime { + // We can enable these tests by either mounting a fake device on linux + // e.g shipping a small ext4 image file and using that as a loopback + // device, but there's no convenient way to implement this. + t.Skip("TODO: Skipped because we don't detect bind mounts") + } + + // Step 2: Test Setup + tmpPath := tmpDir(t) + defer os.RemoveAll(tmpPath) + + csiFake := &csifake.Client{} + manager := newVolumeManager(testlog.HCLogger(t), csiFake, tmpPath, true) + expectedStagingPath := manager.stagingDirForVolume(tc.Volume) + + if tc.CreateDirAheadOfTime { + err := os.MkdirAll(expectedStagingPath, 0700) + require.NoError(t, err) + } + + // Step 3: Now we can do some testing + + detectedMount, path, testErr := manager.ensureStagingDir(tc.Volume) + if tc.ExpectedErr != nil { + require.EqualError(t, testErr, tc.ExpectedErr.Error()) + return // We don't perform extra validation if an error was detected. + } + + require.NoError(t, testErr) + require.Equal(t, tc.ExpectedMountState, detectedMount) + + // If the ensureStagingDir call had to create a directory itself, then here + // we validate that the directory exists and its permissions + if !tc.CreateDirAheadOfTime { + file, err := os.Lstat(path) + require.NoError(t, err) + require.True(t, file.IsDir()) + + // TODO: Figure out a windows equivalent of this test + if runtime.GOOS != "windows" { + require.Equal(t, os.FileMode(0700), file.Mode().Perm()) + } + } + }) + } +}