Add walkSecretsTree helper function (#20464)

changelog/20464.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Add walkSecretsTree helper function, which recursively walks secrets rooted at the given path
+```

command/kv_helpers.go

@@ -4,10 +4,12 @@
 package command
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
-	"path"
+	paths "path"
+	"sort"
 	"strings"
 
 	"github.com/hashicorp/go-secure-stdlib/strutil"
@@ -128,7 +130,7 @@ func isKVv2(path string, client *api.Client) (string, bool, error) {
 
 func addPrefixToKVPath(p, mountPath, apiPrefix string) string {
 	if p == mountPath || p == strings.TrimSuffix(mountPath, "/") {
-		return path.Join(mountPath, apiPrefix)
+		return paths.Join(mountPath, apiPrefix)
 	}
 
 	tp := strings.TrimPrefix(p, mountPath)
@@ -148,7 +150,7 @@ func addPrefixToKVPath(p, mountPath, apiPrefix string) string {
 		tp = strings.TrimPrefix(tp, mountPath)
 	}
 
-	return path.Join(mountPath, apiPrefix, tp)
+	return paths.Join(mountPath, apiPrefix, tp)
 }
 
 func getHeaderForMap(header string, data map[string]interface{}) string {
@@ -197,3 +199,65 @@ func padEqualSigns(header string, totalLen int) string {
 
 	return fmt.Sprintf("%s %s %s", strings.Repeat("=", equalSigns/2), header, strings.Repeat("=", equalSigns/2))
 }
+
+// walkSecretsTree dfs-traverses the secrets tree rooted at the given path
+// and calls the `visit` functor for each of the directory and leaf paths.
+// Note: for kv-v2, a "metadata" path is expected and "metadata" paths will be
+// returned in the visit functor.
+func walkSecretsTree(ctx context.Context, client *api.Client, path string, visit func(path string, directory bool) error) error {
+	resp, err := client.Logical().ListWithContext(ctx, path)
+	if err != nil {
+		return fmt.Errorf("could not list %q path: %w", path, err)
+	}
+
+	if resp == nil || resp.Data == nil {
+		return fmt.Errorf("no value found at %q: %w", path, err)
+	}
+
+	keysRaw, ok := resp.Data["keys"]
+	if !ok {
+		return fmt.Errorf("unexpected list response at %q", path)
+	}
+
+	keysRawSlice, ok := keysRaw.([]interface{})
+	if !ok {
+		return fmt.Errorf("unexpected list response type %T at %q", keysRaw, path)
+	}
+
+	keys := make([]string, 0, len(keysRawSlice))
+
+	for _, keyRaw := range keysRawSlice {
+		key, ok := keyRaw.(string)
+		if !ok {
+			return fmt.Errorf("unexpected key type %T at %q", keyRaw, path)
+		}
+		keys = append(keys, key)
+	}
+
+	// sort the keys for a deterministic output
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		// the keys are relative to the current path: combine them
+		child := paths.Join(path, key)
+
+		if strings.HasSuffix(key, "/") {
+			// visit the directory
+			if err := visit(child, true); err != nil {
+				return err
+			}
+
+			// this is not a leaf node: we need to go deeper...
+			if err := walkSecretsTree(ctx, client, child, visit); err != nil {
+				return err
+			}
+		} else {
+			// this is a leaf node: add it to the list
+			if err := visit(child, false); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

command/kv_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -1523,6 +1524,193 @@ func TestPadEqualSigns(t *testing.T) {
 	}
 }
 
+// TestWalkSecretsTree  the walkSecretsTree helper function
+func TestWalkSecretsTree(t *testing.T) {
+	// test setup
+	client, closer := testVaultServer(t)
+	defer closer()
+
+	// enable kv-v1 backend
+	if err := client.Sys().Mount("kv-v1/", &api.MountInput{
+		Type: "kv-v1",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	// enable kv-v2 backend
+	if err := client.Sys().Mount("kv-v2/", &api.MountInput{
+		Type: "kv-v2",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	ctx, cancelContextFunc := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancelContextFunc()
+
+	// populate secrets
+	for _, path := range []string{
+		"foo",
+		"app-1/foo",
+		"app-1/bar",
+		"app-1/nested/x/y/z",
+		"app-1/nested/x/y",
+		"app-1/nested/bar",
+	} {
+		if err := client.KVv1("kv-v1").Put(ctx, path, map[string]interface{}{
+			"password": "Hashi123",
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		if _, err := client.KVv2("kv-v2").Put(ctx, path, map[string]interface{}{
+			"password": "Hashi123",
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	type treePath struct {
+		path      string
+		directory bool
+	}
+
+	cases := []struct {
+		name          string
+		path          string
+		expected      []treePath
+		expectedError bool
+	}{
+		{
+			name: "kv-v1-simple",
+			path: "kv-v1/app-1/nested/x/y",
+			expected: []treePath{
+				{path: "kv-v1/app-1/nested/x/y/z", directory: false},
+			},
+			expectedError: false,
+		},
+		{
+			name: "kv-v2-simple",
+			path: "kv-v2/metadata/app-1/nested/x/y",
+			expected: []treePath{
+				{path: "kv-v2/metadata/app-1/nested/x/y/z", directory: false},
+			},
+			expectedError: false,
+		},
+		{
+			name: "kv-v1-nested",
+			path: "kv-v1/app-1/nested/",
+			expected: []treePath{
+				{path: "kv-v1/app-1/nested/bar", directory: false},
+				{path: "kv-v1/app-1/nested/x", directory: true},
+				{path: "kv-v1/app-1/nested/x/y", directory: false},
+				{path: "kv-v1/app-1/nested/x/y", directory: true},
+				{path: "kv-v1/app-1/nested/x/y/z", directory: false},
+			},
+			expectedError: false,
+		},
+		{
+			name: "kv-v2-nested",
+			path: "kv-v2/metadata/app-1/nested/",
+			expected: []treePath{
+				{path: "kv-v2/metadata/app-1/nested/bar", directory: false},
+				{path: "kv-v2/metadata/app-1/nested/x", directory: true},
+				{path: "kv-v2/metadata/app-1/nested/x/y", directory: false},
+				{path: "kv-v2/metadata/app-1/nested/x/y", directory: true},
+				{path: "kv-v2/metadata/app-1/nested/x/y/z", directory: false},
+			},
+			expectedError: false,
+		},
+		{
+			name: "kv-v1-all",
+			path: "kv-v1",
+			expected: []treePath{
+				{path: "kv-v1/app-1", directory: true},
+				{path: "kv-v1/app-1/bar", directory: false},
+				{path: "kv-v1/app-1/foo", directory: false},
+				{path: "kv-v1/app-1/nested", directory: true},
+				{path: "kv-v1/app-1/nested/bar", directory: false},
+				{path: "kv-v1/app-1/nested/x", directory: true},
+				{path: "kv-v1/app-1/nested/x/y", directory: false},
+				{path: "kv-v1/app-1/nested/x/y", directory: true},
+				{path: "kv-v1/app-1/nested/x/y/z", directory: false},
+				{path: "kv-v1/foo", directory: false},
+			},
+			expectedError: false,
+		},
+		{
+			name: "kv-v2-all",
+			path: "kv-v2/metadata",
+			expected: []treePath{
+				{path: "kv-v2/metadata/app-1", directory: true},
+				{path: "kv-v2/metadata/app-1/bar", directory: false},
+				{path: "kv-v2/metadata/app-1/foo", directory: false},
+				{path: "kv-v2/metadata/app-1/nested", directory: true},
+				{path: "kv-v2/metadata/app-1/nested/bar", directory: false},
+				{path: "kv-v2/metadata/app-1/nested/x", directory: true},
+				{path: "kv-v2/metadata/app-1/nested/x/y", directory: false},
+				{path: "kv-v2/metadata/app-1/nested/x/y", directory: true},
+				{path: "kv-v2/metadata/app-1/nested/x/y/z", directory: false},
+				{path: "kv-v2/metadata/foo", directory: false},
+			},
+			expectedError: false,
+		},
+		{
+			name:          "kv-v1-not-found",
+			path:          "kv-v1/does/not/exist",
+			expected:      nil,
+			expectedError: true,
+		},
+		{
+			name:          "kv-v2-not-found",
+			path:          "kv-v2/metadata/does/not/exist",
+			expected:      nil,
+			expectedError: true,
+		},
+		{
+			name:          "kv-v1-not-listable-leaf-node",
+			path:          "kv-v1/foo",
+			expected:      nil,
+			expectedError: true,
+		},
+		{
+			name:          "kv-v2-not-listable-leaf-node",
+			path:          "kv-v2/metadata/foo",
+			expected:      nil,
+			expectedError: true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			var descendants []treePath
+
+			err := walkSecretsTree(ctx, client, tc.path, func(path string, directory bool) error {
+				descendants = append(descendants, treePath{
+					path:      path,
+					directory: directory,
+				})
+				return nil
+			})
+
+			if tc.expectedError {
+				if err == nil {
+					t.Fatal("an error was expected but the test succeeded")
+				}
+			} else {
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if !reflect.DeepEqual(tc.expected, descendants) {
+					t.Fatalf("unexpected list output; want: %v, got: %v", tc.expected, descendants)
+				}
+			}
+		})
+	}
+}
+
 func createTokenForPolicy(t *testing.T, client *api.Client, policy string) (*api.SecretAuth, error) {
 	t.Helper()