Extend kv metadata to get, put, and patch (#12907)

* go get vault-plugin-secrets-kv@extend-kv-metadata-to-get-and-put

* test for custom_metadata in kv get, put, patch command output

* remove flagFormat-specific check from TestKVMetadataGetCommand

* rewrite custom metadata changelog entry

* go get vault-plugin-secrets-kv@master

* go mod tidy

changelog/12218.txt

@@ -1,3 +0,0 @@
-```release-note:feature
-secrets/kv: Add ability to specify version-agnostic custom key metadata
-```

changelog/12907.txt

@@ -0,0 +1,5 @@
+```release-note:feature
+**KV Custom Metadata**: Add ability in kv-v2 to specify version-agnostic custom key metadata via the
+metadata endpoint. The data will be present in responses made to the data endpoint independent of the
+calling token's `read` access to the metadata endpoint.
+```

command/kv_test.go

@@ -84,52 +84,54 @@ func kvPatchWithRetry(t *testing.T, client *api.Client, args []string, stdin *io
 func TestKVPutCommand(t *testing.T) {
 	t.Parallel()
 
+	v2ExpectedFields := []string{"created_time", "custom_metadata", "deletion_time", "deletion_time", "version"}
+
 	cases := []struct {
 		name       string
 		args       []string
-		out  string
+		outStrings []string
 		code       int
 	}{
 		{
 			"not_enough_args",
 			[]string{},
-			"Not enough arguments",
+			[]string{"Not enough arguments"},
 			1,
 		},
 		{
 			"empty_kvs",
 			[]string{"secret/write/foo"},
-			"Must supply data",
+			[]string{"Must supply data"},
 			1,
 		},
 		{
 			"kvs_no_value",
 			[]string{"secret/write/foo", "foo"},
-			"Failed to parse K=V data",
+			[]string{"Failed to parse K=V data"},
 			1,
 		},
 		{
 			"single_value",
 			[]string{"secret/write/foo", "foo=bar"},
-			"Success!",
+			[]string{"Success!"},
 			0,
 		},
 		{
 			"multi_value",
 			[]string{"secret/write/foo", "foo=bar", "zip=zap"},
-			"Success!",
+			[]string{"Success!"},
 			0,
 		},
 		{
 			"v2_single_value",
 			[]string{"kv/write/foo", "foo=bar"},
-			"created_time",
+			v2ExpectedFields,
 			0,
 		},
 		{
 			"v2_multi_value",
 			[]string{"kv/write/foo", "foo=bar", "zip=zap"},
-			"created_time",
+			v2ExpectedFields,
 			0,
 		},
 	}
@@ -153,8 +155,11 @@ func TestKVPutCommand(t *testing.T) {
 			if code != tc.code {
 				t.Errorf("expected %d to be %d", code, tc.code)
 			}
-			if !strings.Contains(combined, tc.out) {
+
-				t.Errorf("expected %q to contain %q", combined, tc.out)
+			for _, str := range tc.outStrings {
+				if !strings.Contains(combined, str) {
+					t.Errorf("expected %q to contain %q", combined, str)
+				}
 			}
 		})
 	}
@@ -178,8 +183,11 @@ func TestKVPutCommand(t *testing.T) {
 		if code != 0 {
 			t.Fatalf("expected 0 to be %d", code)
 		}
-		if !strings.Contains(combined, "created_time") {
+
-			t.Errorf("expected %q to contain %q", combined, "created_time")
+		for _, str := range v2ExpectedFields {
+			if !strings.Contains(combined, str) {
+				t.Errorf("expected %q to contain %q", combined, str)
+			}
 		}
 
 		ui, cmd := testKVPutCommand(t)
@@ -191,8 +199,11 @@ func TestKVPutCommand(t *testing.T) {
 			t.Fatalf("expected 0 to be %d", code)
 		}
 		combined = ui.OutputWriter.String() + ui.ErrorWriter.String()
-		if !strings.Contains(combined, "created_time") {
+
-			t.Errorf("expected %q to contain %q", combined, "created_time")
+		for _, str := range v2ExpectedFields {
+			if !strings.Contains(combined, str) {
+				t.Errorf("expected %q to contain %q", combined, str)
+			}
 		}
 
 		ui, cmd = testKVPutCommand(t)
@@ -366,72 +377,68 @@ func testKVGetCommand(tb testing.TB) (*cli.MockUi, *KVGetCommand) {
 func TestKVGetCommand(t *testing.T) {
 	t.Parallel()
 
+	baseV2ExpectedFields := []string{"created_time", "custom_metadata", "deletion_time", "deletion_time", "version"}
+
 	cases := []struct {
 		name       string
 		args       []string
-		out  string
+		outStrings []string
 		code       int
 	}{
 		{
 			"not_enough_args",
 			[]string{},
-			"Not enough arguments",
+			[]string{"Not enough arguments"},
 			1,
 		},
 		{
 			"too_many_args",
 			[]string{"foo", "bar"},
-			"Too many arguments",
+			[]string{"Too many arguments"},
 			1,
 		},
 		{
 			"not_found",
 			[]string{"secret/nope/not/once/never"},
-			"",
+			[]string{"No value found at secret/nope/not/once/never"},
 			2,
 		},
 		{
 			"default",
 			[]string{"secret/read/foo"},
-			"foo",
+			[]string{"foo"},
 			0,
 		},
 		{
 			"v1_field",
 			[]string{"-field", "foo", "secret/read/foo"},
-			"bar",
+			[]string{"bar"},
 			0,
 		},
 		{
 			"v2_field",
 			[]string{"-field", "foo", "kv/read/foo"},
-			"bar",
+			[]string{"bar"},
 			0,
 		},
 
 		{
 			"v2_not_found",
 			[]string{"kv/nope/not/once/never"},
-			"",
+			[]string{"No value found at kv/data/nope/not/once/never"},
 			2,
 		},
 
 		{
 			"v2_read",
 			[]string{"kv/read/foo"},
-			"foo",
+			append(baseV2ExpectedFields, "foo"),
-			0,
-		},
-		{
-			"v2_read",
-			[]string{"kv/read/foo"},
-			"version",
 			0,
 		},
 		{
 			"v2_read_version",
 			[]string{"--version", "1", "kv/read/foo"},
-			"foo",
+			append(baseV2ExpectedFields, "foo"),
 			0,
 		},
 	}
@@ -479,8 +486,11 @@ func TestKVGetCommand(t *testing.T) {
 				}
 
 				combined := ui.OutputWriter.String() + ui.ErrorWriter.String()
-				if !strings.Contains(combined, tc.out) {
+
-					t.Errorf("expected %q to contain %q", combined, tc.out)
+				for _, str := range tc.outStrings {
+					if !strings.Contains(combined, str) {
+						t.Errorf("expected %q to contain %q", combined, str)
+					}
 				}
 			})
 		}
@@ -508,28 +518,46 @@ func testKVMetadataGetCommand(tb testing.TB) (*cli.MockUi, *KVMetadataGetCommand
 func TestKVMetadataGetCommand(t *testing.T) {
 	t.Parallel()
 
+	expectedTopLevelFields := []string{
+		"cas_required",
+		"created_time",
+		"current_version",
+		"custom_metadata",
+		"delete_version_after",
+		"max_versions",
+		"oldest_version",
+		"updated_time",
+	}
+
+	expectedVersionFields := []string{
+		"created_time", // field is redundant
+		"deletion_time",
+		"destroyed",
+	}
+
 	cases := []struct {
 		name       string
 		args       []string
-		out  string
+		outStrings []string
 		code       int
 	}{
 		{
 			"v1",
 			[]string{"secret/foo"},
-			"Metadata not supported on KV Version 1",
+			[]string{"Metadata not supported on KV Version 1"},
 			1,
 		},
 		{
 			"metadata_exists",
 			[]string{"kv/foo"},
-			"current_version",
+			expectedTopLevelFields,
 			0,
 		},
+		// ensure that all top-level and version-level fields are output along with version num
 		{
 			"versions_exist",
 			[]string{"kv/foo"},
-			"deletion_time",
+			append(expectedTopLevelFields,  expectedVersionFields[:]...),
 			0,
 		},
 	}
@@ -571,8 +599,10 @@ func TestKVMetadataGetCommand(t *testing.T) {
 				}
 
 				combined := ui.OutputWriter.String() + ui.ErrorWriter.String()
-				if !strings.Contains(combined, tc.out) {
+				for _, str := range tc.outStrings {
-					t.Errorf("expected %q to contain %q", combined, tc.out)
+					if !strings.Contains(combined, str) {
+						t.Errorf("expected %q to contain %q", combined, str)
+					}
 				}
 			})
 		}
@@ -652,6 +682,19 @@ func TestKVPatchCommand_ArgValidation(t *testing.T) {
 	}
 }
 
+// expectedPatchFields produces a deterministic slice of
+// expected fields for patch command output since const
+// slices are not supported
+func expectedPatchFields() []string {
+	return []string{
+		"created_time",
+		"custom_metadata",
+		"deletion_time",
+		"destroyed",
+		"version",
+	}
+}
+
 func TestKvPatchCommand_StdinFull(t *testing.T) {
 	client, closer := testVaultServer(t)
 	defer closer()
@@ -677,7 +720,14 @@ func TestKvPatchCommand_StdinFull(t *testing.T) {
 	}()
 
 	args := []string{"kv/patch/foo", "-"}
-	code, _ := kvPatchWithRetry(t, client, args, stdinR)
+	code, combined := kvPatchWithRetry(t, client, args, stdinR)
+
+	for _, str := range expectedPatchFields() {
+		if !strings.Contains(combined, str) {
+			t.Errorf("expected %q to contain %q", combined, str)
+		}
+	}
+
 	if code != 0 {
 		t.Fatalf("expected code to be 0 but was %d for patch cmd with args %#v\n", code, args)
 	}
@@ -733,11 +783,17 @@ func TestKvPatchCommand_StdinValue(t *testing.T) {
 	}()
 
 	args := []string{"kv/patch/foo", "foo=-"}
-	code, _ := kvPatchWithRetry(t, client, args, stdinR)
+	code, combined := kvPatchWithRetry(t, client, args, stdinR)
 	if code != 0 {
 		t.Fatalf("expected code to be 0 but was %d for patch cmd with args %#v\n", code, args)
 	}
 
+	for _, str := range expectedPatchFields() {
+		if !strings.Contains(combined, str) {
+			t.Errorf("expected %q to contain %q", combined, str)
+		}
+	}
+
 	secret, err := client.Logical().Read("kv/data/patch/foo")
 	if err != nil {
 		t.Fatalf("read failed, err: %#v\n", err)
@@ -810,9 +866,10 @@ func TestKVPatchCommand_RWMethodSucceeds(t *testing.T) {
 		t.Fatalf("expected code to be 0 but was %d for patch cmd with args %#v\n", code, args)
 	}
 
-	expectedOutputSubstr := "created_time"
+	for _, str := range expectedPatchFields() {
-	if !strings.Contains(combined, expectedOutputSubstr) {
+		if !strings.Contains(combined, str) {
-		t.Fatalf("expected output %q to contain %q for patch cmd with args %#v\n", combined, expectedOutputSubstr, args)
+			t.Errorf("expected %q to contain %q", combined, str)
+		}
 	}
 
 	// Test multi value
@@ -823,8 +880,10 @@ func TestKVPatchCommand_RWMethodSucceeds(t *testing.T) {
 		t.Fatalf("expected code to be 0 but was %d for patch cmd with args %#v\n", code, args)
 	}
 
-	if !strings.Contains(combined, expectedOutputSubstr) {
+	for _, str := range expectedPatchFields() {
-		t.Fatalf("expected output %q to contain %q for patch cmd with args %#v\n", combined, expectedOutputSubstr, args)
+		if !strings.Contains(combined, str) {
+			t.Errorf("expected %q to contain %q", combined, str)
+		}
 	}
 }
 
@@ -833,21 +892,21 @@ func TestKVPatchCommand_CAS(t *testing.T) {
 		name       string
 		args       []string
 		expected   string
-		out      string
+		outStrings []string
 		code       int
 	}{
 		{
 			"right version",
 			[]string{"-cas", "1", "kv/foo", "bar=quux"},
 			"quux",
-			"",
+			expectedPatchFields(),
 			0,
 		},
 		{
 			"wrong version",
 			[]string{"-cas", "2", "kv/foo", "bar=wibble"},
 			"baz",
-			"check-and-set parameter did not match the current version",
+			[]string{"check-and-set parameter did not match the current version"},
 			2,
 		},
 	}
@@ -892,9 +951,9 @@ func TestKVPatchCommand_CAS(t *testing.T) {
 				t.Fatalf("expected code to be %d but was %d", tc.code, code)
 			}
 
-			if tc.out != "" {
+			for _, str := range tc.outStrings {
-				if !strings.Contains(combined, tc.out) {
+				if !strings.Contains(combined, str) {
-					t.Errorf("expected %q to contain %q", combined, tc.out)
+					t.Errorf("expected %q to contain %q", combined, str)
 				}
 			}
 

go.mod

@@ -104,13 +104,13 @@ require (
 	github.com/hashicorp/vault-plugin-secrets-azure v0.6.3-0.20210924190759-58a034528e35
 	github.com/hashicorp/vault-plugin-secrets-gcp v0.10.2
 	github.com/hashicorp/vault-plugin-secrets-gcpkms v0.9.0
-	github.com/hashicorp/vault-plugin-secrets-kv v0.5.7-0.20211013154503-eec8a1c892fb
+	github.com/hashicorp/vault-plugin-secrets-kv v0.5.7-0.20211026132900-bc1c42ddb53c
 	github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.4.0
 	github.com/hashicorp/vault-plugin-secrets-openldap v0.4.1-0.20210921171411-e86105e4986d
 	github.com/hashicorp/vault-plugin-secrets-terraform v0.3.0
 	github.com/hashicorp/vault-testing-stepwise v0.1.1
 	github.com/hashicorp/vault/api v1.2.0
-	github.com/hashicorp/vault/sdk v0.2.2-0.20211004171540-a8c7e135dd6a
+	github.com/hashicorp/vault/sdk v0.2.2-0.20211014165207-28bd5c3a0311
 	github.com/influxdata/influxdb v0.0.0-20190411212539-d24b7ba8c4c4
 	github.com/jcmturner/gokrb5/v8 v8.0.0
 	github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f

go.sum

@@ -401,6 +401,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-ldap/ldap v3.0.2+incompatible h1:kD5HQcAzlQ7yrhfn+h+MSABeAy/jAJhvIJ/QDllP44g=
+github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
 github.com/go-ldap/ldap/v3 v3.1.3/go.mod h1:3rbOH3jRS2u6jg2rJnKAMLE/xQyCKIveG2Sa/Cohzb8=
 github.com/go-ldap/ldap/v3 v3.1.7/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
 github.com/go-ldap/ldap/v3 v3.1.10/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
@@ -776,8 +778,8 @@ github.com/hashicorp/vault-plugin-secrets-gcp v0.10.2 h1:+DtlYJTsrFRInQpAo09KkYN
 github.com/hashicorp/vault-plugin-secrets-gcp v0.10.2/go.mod h1:psRQ/dm5XatoUKLDUeWrpP9icMJNtu/jmscUr37YGK4=
 github.com/hashicorp/vault-plugin-secrets-gcpkms v0.9.0 h1:7a0iWuFA/YNinQ1xXogyZHStolxMVtLV+sy1LpEHaZs=
 github.com/hashicorp/vault-plugin-secrets-gcpkms v0.9.0/go.mod h1:hhwps56f2ATeC4Smgghrc5JH9dXR31b4ehSf1HblP5Q=
-github.com/hashicorp/vault-plugin-secrets-kv v0.5.7-0.20211013154503-eec8a1c892fb h1:nZ2a4a1G0ALLAzKOWQbLzD5oljKo+pjMarbq3BwU0pM=
+github.com/hashicorp/vault-plugin-secrets-kv v0.5.7-0.20211026132900-bc1c42ddb53c h1:m6aJO2SrAf8bCLjyAtQJNiSuV0nM4TBKqrJpImrDtSY=
-github.com/hashicorp/vault-plugin-secrets-kv v0.5.7-0.20211013154503-eec8a1c892fb/go.mod h1:D/FQJ7zU5pD6FNJVUwaVtxr75ZsxIIqaG/Nh6RHt/xo=
+github.com/hashicorp/vault-plugin-secrets-kv v0.5.7-0.20211026132900-bc1c42ddb53c/go.mod h1:Luu1GqDOMnuJ2iqn6mFf38Dz8DQ8mgtyQRXrS7Bp8Xc=
 github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.4.0 h1:6ve+7hZmGn7OpML81iZUxYj2AaJptwys323S5XsvVas=
 github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.4.0/go.mod h1:4mdgPqlkO+vfFX1cFAWcxkeqz6JAtZgKxL/67q/58Oo=
 github.com/hashicorp/vault-plugin-secrets-openldap v0.4.1-0.20210921171411-e86105e4986d h1:o5Z9B1FztTYSnTQNzFr+iZJHPM8ZD23uV5A8gMxm2g0=
@@ -1578,6 +1580,7 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=