From 64448b62a4613b9467f003fa6fb167cfa9239360 Mon Sep 17 00:00:00 2001 From: VAL Date: Wed, 25 May 2022 11:17:13 -0700 Subject: [PATCH] KV helper methods for api package (#15305) * Add Read methods for KVClient * KV write helper * Add changelog * Add Delete method * Use extractVersionMetadata inside extractDataAndVersionMetadata * Return nil, nil for v1 writes * Add test for extracting version metadata * Split kv client into v1 and v2-specific clients * Add ability to set options on Put * Add test for KV helpers * Add custom metadata to top level and allow for getting versions as sorted slice * Update tests * Separate KV v1 and v2 into different files * Add test for GetVersionsAsList, rename Metadata key to VersionMetadata for clarity * Move structs and godoc comments to more appropriate files * Add more tests for extract methods * Rework custom metadata helper to be more consistent with other helpers * Remove KVSecret from custom metadata test now that we don't append to it as part of helper method * Return early for readability and make test value name less confusing --- api/kv.go | 50 +++ api/kv_test.go | 334 ++++++++++++++++ api/kv_v1.go | 57 +++ api/kv_v2.go | 401 ++++++++++++++++++++ changelog/15305.txt | 3 + vault/external_tests/api/kv_helpers_test.go | 169 +++++++++ 6 files changed, 1014 insertions(+) create mode 100644 api/kv.go create mode 100644 api/kv_test.go create mode 100644 api/kv_v1.go create mode 100644 api/kv_v2.go create mode 100644 changelog/15305.txt create mode 100644 vault/external_tests/api/kv_helpers_test.go diff --git a/api/kv.go b/api/kv.go new file mode 100644 index 000000000..a334c8e21 --- /dev/null +++ b/api/kv.go @@ -0,0 +1,50 @@ +package api + +// A KVSecret is a key-value secret returned by Vault's KV secrets engine, +// and is the most basic type of secret stored in Vault. +// +// Data contains the key-value pairs of the secret itself, +// while Metadata contains a subset of metadata describing +// this particular version of the secret. +// The Metadata field for a KV v1 secret will always be nil, as +// metadata is only supported starting in KV v2. +// +// The Raw field can be inspected for information about the lease, +// and passed to a LifetimeWatcher object for periodic renewal. +type KVSecret struct { + Data map[string]interface{} + VersionMetadata *KVVersionMetadata + CustomMetadata map[string]interface{} + Raw *Secret +} + +// KVv1 is used to return a client for reads and writes against +// a KV v1 secrets engine in Vault. +// +// The mount path is the location where the target KV secrets engine resides +// in Vault. +// +// While v1 is not necessarily deprecated, Vault development servers tend to +// use v2 as the version of the KV secrets engine, as this is what's mounted +// by default when a server is started in -dev mode. See the kvv2 struct. +// +// Learn more about the KV secrets engine here: +// https://www.vaultproject.io/docs/secrets/kv +func (c *Client) KVv1(mountPath string) *kvv1 { + return &kvv1{c: c, mountPath: mountPath} +} + +// KVv2 is used to return a client for reads and writes against +// a KV v2 secrets engine in Vault. +// +// The mount path is the location where the target KV secrets engine resides +// in Vault. +// +// Vault development servers tend to have "secret" as the mount path, +// as these are the default settings when a server is started in -dev mode. +// +// Learn more about the KV secrets engine here: +// https://www.vaultproject.io/docs/secrets/kv +func (c *Client) KVv2(mountPath string) *kvv2 { + return &kvv2{c: c, mountPath: mountPath} +} diff --git a/api/kv_test.go b/api/kv_test.go new file mode 100644 index 000000000..2c16b2d43 --- /dev/null +++ b/api/kv_test.go @@ -0,0 +1,334 @@ +package api + +import ( + "reflect" + "testing" + "time" +) + +func TestExtractVersionMetadata(t *testing.T) { + t.Parallel() + + inputCreatedTimeStr := "2022-05-06T23:02:04.865025Z" + inputDeletionTimeStr := "2022-06-17T01:15:03.279013Z" + expectedCreatedTimeParsed, err := time.Parse(time.RFC3339, inputCreatedTimeStr) + if err != nil { + t.Fatalf("unable to parse expected created time: %v", err) + } + expectedDeletionTimeParsed, err := time.Parse(time.RFC3339, inputDeletionTimeStr) + if err != nil { + t.Fatalf("unable to parse expected created time: %v", err) + } + + testCases := []struct { + name string + input *Secret + expected *KVVersionMetadata + }{ + { + name: "a secret", + input: &Secret{ + Data: map[string]interface{}{ + "data": map[string]interface{}{ + "password": "Hashi123", + }, + "metadata": map[string]interface{}{ + "version": 10, + "created_time": inputCreatedTimeStr, + "deletion_time": "", + "destroyed": false, + "custom_metadata": nil, + }, + }, + }, + expected: &KVVersionMetadata{ + Version: 10, + CreatedTime: expectedCreatedTimeParsed, + DeletionTime: time.Time{}, + Destroyed: false, + }, + }, + { + name: "a secret that has been deleted", + input: &Secret{ + Data: map[string]interface{}{ + "data": map[string]interface{}{ + "password": "Hashi123", + }, + "metadata": map[string]interface{}{ + "version": 10, + "created_time": inputCreatedTimeStr, + "deletion_time": inputDeletionTimeStr, + "destroyed": false, + "custom_metadata": nil, + }, + }, + }, + expected: &KVVersionMetadata{ + Version: 10, + CreatedTime: expectedCreatedTimeParsed, + DeletionTime: expectedDeletionTimeParsed, + Destroyed: false, + }, + }, + { + name: "a response from a Write operation", + input: &Secret{ + Data: map[string]interface{}{ + "version": 10, + "created_time": inputCreatedTimeStr, + "deletion_time": "", + "destroyed": false, + "custom_metadata": nil, + }, + }, + expected: &KVVersionMetadata{ + Version: 10, + CreatedTime: expectedCreatedTimeParsed, + DeletionTime: time.Time{}, + Destroyed: false, + }, + }, + } + + for _, tc := range testCases { + versionMetadata, err := extractVersionMetadata(tc.input) + if err != nil { + t.Fatalf("err: %s", err) + } + + if !reflect.DeepEqual(versionMetadata, tc.expected) { + t.Fatalf("%s: got\n%#v\nexpected\n%#v\n", tc.name, versionMetadata, tc.expected) + } + } +} + +func TestExtractDataAndVersionMetadata(t *testing.T) { + t.Parallel() + + inputCreatedTimeStr := "2022-05-06T23:02:04.865025Z" + inputDeletionTimeStr := "2022-06-17T01:15:03.279013Z" + expectedCreatedTimeParsed, err := time.Parse(time.RFC3339, inputCreatedTimeStr) + if err != nil { + t.Fatalf("unable to parse expected created time: %v", err) + } + expectedDeletionTimeParsed, err := time.Parse(time.RFC3339, inputDeletionTimeStr) + if err != nil { + t.Fatalf("unable to parse expected created time: %v", err) + } + + readResp := &Secret{ + Data: map[string]interface{}{ + "data": map[string]interface{}{ + "password": "Hashi123", + }, + "metadata": map[string]interface{}{ + "version": 10, + "created_time": inputCreatedTimeStr, + "deletion_time": "", + "destroyed": false, + "custom_metadata": nil, + }, + }, + } + + readRespDeleted := &Secret{ + Data: map[string]interface{}{ + "data": nil, + "metadata": map[string]interface{}{ + "version": 10, + "created_time": inputCreatedTimeStr, + "deletion_time": inputDeletionTimeStr, + "destroyed": false, + "custom_metadata": nil, + }, + }, + } + + testCases := []struct { + name string + input *Secret + expected *KVSecret + }{ + { + name: "a response from a Read operation", + input: readResp, + expected: &KVSecret{ + Data: map[string]interface{}{ + "password": "Hashi123", + }, + VersionMetadata: &KVVersionMetadata{ + Version: 10, + CreatedTime: expectedCreatedTimeParsed, + DeletionTime: time.Time{}, + Destroyed: false, + }, + // it's tempting to test some Secrets with custom_metadata but + // we can't in this test because it isn't until we call the + // extractCustomMetadata function that the custom metadata + // gets added onto the struct. See TestExtractCustomMetadata. + CustomMetadata: nil, + Raw: readResp, + }, + }, + { + name: "a secret that has been deleted and thus has nil data", + input: readRespDeleted, + expected: &KVSecret{ + Data: nil, + VersionMetadata: &KVVersionMetadata{ + Version: 10, + CreatedTime: expectedCreatedTimeParsed, + DeletionTime: expectedDeletionTimeParsed, + Destroyed: false, + }, + CustomMetadata: nil, + Raw: readRespDeleted, + }, + }, + } + + for _, tc := range testCases { + dvm, err := extractDataAndVersionMetadata(tc.input) + if err != nil { + t.Fatalf("err: %s", err) + } + + if !reflect.DeepEqual(dvm, tc.expected) { + t.Fatalf("%s: got\n%#v\nexpected\n%#v\n", tc.name, dvm, tc.expected) + } + } +} + +func TestExtractFullMetadata(t *testing.T) { + inputCreatedTimeStr := "2022-05-20T00:51:49.419794Z" + expectedCreatedTimeParsed, err := time.Parse(time.RFC3339, inputCreatedTimeStr) + if err != nil { + t.Fatalf("unable to parse expected created time: %v", err) + } + + inputUpdatedTimeStr := "2022-05-20T20:23:43.284488Z" + expectedUpdatedTimeParsed, err := time.Parse(time.RFC3339, inputUpdatedTimeStr) + if err != nil { + t.Fatalf("unable to parse expected updated time: %v", err) + } + + inputDeletedTimeStr := "2022-05-21T00:05:49.521697Z" + expectedDeletedTimeParsed, err := time.Parse(time.RFC3339, inputDeletedTimeStr) + if err != nil { + t.Fatalf("unable to parse expected deletion time: %v", err) + } + + metadataResp := &Secret{ + Data: map[string]interface{}{ + "cas_required": true, + "created_time": inputCreatedTimeStr, + "current_version": 2, + "custom_metadata": map[string]interface{}{ + "org": "eng", + }, + "delete_version_after": "200s", + "max_versions": 3, + "oldest_version": 1, + "updated_time": inputUpdatedTimeStr, + "versions": map[string]interface{}{ + "2": map[string]interface{}{ + "created_time": inputUpdatedTimeStr, + "deletion_time": "", + "destroyed": false, + }, + "1": map[string]interface{}{ + "created_time": inputCreatedTimeStr, + "deletion_time": inputDeletedTimeStr, + "destroyed": false, + }, + }, + }, + } + + testCases := []struct { + name string + input *Secret + expected *KVMetadata + }{ + { + name: "a metadata response", + input: metadataResp, + expected: &KVMetadata{ + CASRequired: true, + CreatedTime: expectedCreatedTimeParsed, + CurrentVersion: 2, + CustomMetadata: map[string]interface{}{ + "org": "eng", + }, + DeleteVersionAfter: time.Duration(200 * time.Second), + MaxVersions: 3, + OldestVersion: 1, + UpdatedTime: expectedUpdatedTimeParsed, + Versions: map[string]KVVersionMetadata{ + "2": { + Version: 2, + CreatedTime: expectedUpdatedTimeParsed, + DeletionTime: time.Time{}, + }, + "1": { + Version: 1, + CreatedTime: expectedCreatedTimeParsed, + DeletionTime: expectedDeletedTimeParsed, + }, + }, + }, + }, + } + + for _, tc := range testCases { + md, err := extractFullMetadata(tc.input) + if err != nil { + t.Fatalf("err: %s", err) + } + + if !reflect.DeepEqual(md, tc.expected) { + t.Fatalf("%s: got\n%#v\nexpected\n%#v\n", tc.name, md, tc.expected) + } + } +} + +func TestExtractCustomMetadata(t *testing.T) { + testCases := []struct { + name string + inputAPIResp *Secret + expected map[string]interface{} + }{ + { + name: "a read response with some custom metadata", + inputAPIResp: &Secret{ + Data: map[string]interface{}{ + "metadata": map[string]interface{}{ + "custom_metadata": map[string]interface{}{"org": "eng"}, + }, + }, + }, + expected: map[string]interface{}{"org": "eng"}, + }, + { + name: "a write response with some (pre-existing) custom metadata", + inputAPIResp: &Secret{ + Data: map[string]interface{}{ + "custom_metadata": map[string]interface{}{"org": "eng"}, + }, + }, + expected: map[string]interface{}{"org": "eng"}, + }, + } + + for _, tc := range testCases { + cm, err := extractCustomMetadata(tc.inputAPIResp) + if err != nil { + t.Fatalf("err: %s", err) + } + + if !reflect.DeepEqual(cm, tc.expected) { + t.Fatalf("%s: got\n%#v\nexpected\n%#v\n", tc.name, cm, tc.expected) + } + } +} diff --git a/api/kv_v1.go b/api/kv_v1.go new file mode 100644 index 000000000..1b0428dee --- /dev/null +++ b/api/kv_v1.go @@ -0,0 +1,57 @@ +package api + +import ( + "context" + "fmt" +) + +type kvv1 struct { + c *Client + mountPath string +} + +// Get returns a secret from the KV v1 secrets engine. +func (kv *kvv1) Get(ctx context.Context, secretPath string) (*KVSecret, error) { + pathToRead := fmt.Sprintf("%s/%s", kv.mountPath, secretPath) + + secret, err := kv.c.Logical().ReadWithContext(ctx, pathToRead) + if err != nil { + return nil, fmt.Errorf("error encountered while reading secret at %s: %w", pathToRead, err) + } + if secret == nil { + return nil, fmt.Errorf("no secret found at %s", pathToRead) + } + + return &KVSecret{ + Data: secret.Data, + VersionMetadata: nil, + Raw: secret, + }, nil +} + +// Put inserts a key-value secret (e.g. {"password": "Hashi123"}) into the +// KV v1 secrets engine. +// +// If the secret already exists, it will be overwritten. +func (kv *kvv1) Put(ctx context.Context, secretPath string, data map[string]interface{}) error { + pathToWriteTo := fmt.Sprintf("%s/%s", kv.mountPath, secretPath) + + _, err := kv.c.Logical().WriteWithContext(ctx, pathToWriteTo, data) + if err != nil { + return fmt.Errorf("error writing secret to %s: %w", pathToWriteTo, err) + } + + return nil +} + +// Delete deletes a secret from the KV v1 secrets engine. +func (kv *kvv1) Delete(ctx context.Context, secretPath string) error { + pathToDelete := fmt.Sprintf("%s/%s", kv.mountPath, secretPath) + + _, err := kv.c.Logical().DeleteWithContext(ctx, pathToDelete) + if err != nil { + return fmt.Errorf("error deleting secret at %s: %w", pathToDelete, err) + } + + return nil +} diff --git a/api/kv_v2.go b/api/kv_v2.go new file mode 100644 index 000000000..2b6149b16 --- /dev/null +++ b/api/kv_v2.go @@ -0,0 +1,401 @@ +package api + +import ( + "context" + "fmt" + "sort" + "strconv" + "time" + + "github.com/mitchellh/mapstructure" +) + +type kvv2 struct { + c *Client + mountPath string +} + +// KVMetadata is the full metadata for a given KV v2 secret. +type KVMetadata struct { + CASRequired bool `mapstructure:"cas_required"` + CreatedTime time.Time `mapstructure:"created_time"` + CurrentVersion int `mapstructure:"current_version"` + CustomMetadata map[string]interface{} `mapstructure:"custom_metadata"` + DeleteVersionAfter time.Duration `mapstructure:"delete_version_after"` + MaxVersions int `mapstructure:"max_versions"` + OldestVersion int `mapstructure:"oldest_version"` + UpdatedTime time.Time `mapstructure:"updated_time"` + // Keys are stringified ints, e.g. "3". To get a sorted slice of version metadata, use GetVersionsAsList. + Versions map[string]KVVersionMetadata `mapstructure:"versions"` + Raw *Secret +} + +// KVVersionMetadata is a subset of metadata for a given version of a KV v2 secret. +type KVVersionMetadata struct { + Version int `mapstructure:"version"` + CreatedTime time.Time `mapstructure:"created_time"` + DeletionTime time.Time `mapstructure:"deletion_time"` + Destroyed bool `mapstructure:"destroyed"` +} + +// Currently supported options: WithCheckAndSet +type KVOption func() (key string, value interface{}) + +// WithCheckAndSet can optionally be passed to perform a check-and-set +// operation. If not set, the write will be allowed. If cas is set to 0, a +// write will only be allowed if the key doesn't exist. If set to non-zero, +// the write will only be allowed if the key’s current version matches the +// version specified in the cas parameter. +func WithCheckAndSet(cas int) KVOption { + return func() (string, interface{}) { + return "cas", cas + } +} + +// Get returns the latest version of a secret from the KV v2 secrets engine. +// +// If the latest version has been deleted, an error will not be thrown, but +// the Data field on the returned secret will be nil, and the Metadata field +// will contain the deletion time. +func (kv *kvv2) Get(ctx context.Context, secretPath string) (*KVSecret, error) { + pathToRead := fmt.Sprintf("%s/data/%s", kv.mountPath, secretPath) + + secret, err := kv.c.Logical().ReadWithContext(ctx, pathToRead) + if err != nil { + return nil, fmt.Errorf("error encountered while reading secret at %s: %w", pathToRead, err) + } + if secret == nil { + return nil, fmt.Errorf("no secret found at %s", pathToRead) + } + + kvSecret, err := extractDataAndVersionMetadata(secret) + if err != nil { + return nil, fmt.Errorf("error parsing secret at %s: %w", pathToRead, err) + } + + cm, err := extractCustomMetadata(secret) + if err != nil { + return nil, fmt.Errorf("error reading custom metadata for secret at %s: %w", pathToRead, err) + } + kvSecret.CustomMetadata = cm + + return kvSecret, nil +} + +// GetVersion returns the data and metadata for a specific version of the +// given secret. +// +// If that version has been deleted, the Data field on the +// returned secret will be nil, and the Metadata field will contain the deletion time. +// +// GetVersionsAsList can provide a list of available versions sorted by +// version number, while the response from GetMetadata contains them as a map. +func (kv *kvv2) GetVersion(ctx context.Context, secretPath string, version int) (*KVSecret, error) { + pathToRead := fmt.Sprintf("%s/data/%s", kv.mountPath, secretPath) + + queryParams := map[string][]string{"version": {strconv.Itoa(version)}} + secret, err := kv.c.Logical().ReadWithDataWithContext(ctx, pathToRead, queryParams) + if err != nil { + return nil, err + } + if secret == nil { + return nil, fmt.Errorf("no secret with version %d found at %s", version, pathToRead) + } + + kvSecret, err := extractDataAndVersionMetadata(secret) + if err != nil { + return nil, fmt.Errorf("error parsing secret at %s: %w", pathToRead, err) + } + + cm, err := extractCustomMetadata(secret) + if err != nil { + return nil, fmt.Errorf("error reading custom metadata for secret at %s: %w", pathToRead, err) + } + kvSecret.CustomMetadata = cm + + return kvSecret, nil +} + +// GetVersionsAsList returns a subset of the metadata for each version of the secret, sorted by version number. +func (kv *kvv2) GetVersionsAsList(ctx context.Context, secretPath string) ([]KVVersionMetadata, error) { + pathToRead := fmt.Sprintf("%s/metadata/%s", kv.mountPath, secretPath) + + secret, err := kv.c.Logical().ReadWithContext(ctx, pathToRead) + if err != nil { + return nil, err + } + if secret == nil || secret.Data == nil { + return nil, fmt.Errorf("no secret metadata found at %s", pathToRead) + } + + md, err := extractFullMetadata(secret) + if err != nil { + return nil, fmt.Errorf("unable to extract metadata from secret to determine versions: %w", err) + } + + versionsList := make([]KVVersionMetadata, 0, len(md.Versions)) + for _, versionMetadata := range md.Versions { + versionsList = append(versionsList, versionMetadata) + } + + sort.Slice(versionsList, func(i, j int) bool { return versionsList[i].Version < versionsList[j].Version }) + return versionsList, nil +} + +// GetMetadata returns the full metadata for a given secret, including a map of +// its existing versions and their respective creation/deletion times, etc. +func (kv *kvv2) GetMetadata(ctx context.Context, secretPath string) (*KVMetadata, error) { + pathToRead := fmt.Sprintf("%s/metadata/%s", kv.mountPath, secretPath) + + secret, err := kv.c.Logical().ReadWithContext(ctx, pathToRead) + if err != nil { + return nil, err + } + if secret == nil || secret.Data == nil { + return nil, fmt.Errorf("no secret metadata found at %s", pathToRead) + } + + md, err := extractFullMetadata(secret) + if err != nil { + return nil, fmt.Errorf("unable to extract metadata from secret: %w", err) + } + + return md, nil +} + +// Put inserts a key-value secret (e.g. {"password": "Hashi123"}) +// into the KV v2 secrets engine. +// +// If the secret already exists, a new version will be created +// and the previous version can be accessed with the GetVersion method. +// GetMetadata can provide a list of available versions. +func (kv *kvv2) Put(ctx context.Context, secretPath string, data map[string]interface{}, opts ...KVOption) (*KVSecret, error) { + pathToWriteTo := fmt.Sprintf("%s/data/%s", kv.mountPath, secretPath) + + wrappedData := map[string]interface{}{ + "data": data, + } + + // Add options such as check-and-set, etc. + // We leave this as an optional arg so that most users + // can just pass plain key-value secret data without + // having to remember to put the extra layer "data" in there. + options := make(map[string]interface{}) + for _, opt := range opts { + k, v := opt() + options[k] = v + } + if len(opts) > 0 { + wrappedData["options"] = options + } + + secret, err := kv.c.Logical().WriteWithContext(ctx, pathToWriteTo, wrappedData) + if err != nil { + return nil, fmt.Errorf("error writing secret to %s: %w", pathToWriteTo, err) + } + if secret == nil { + return nil, fmt.Errorf("no secret was written to %s", pathToWriteTo) + } + + metadata, err := extractVersionMetadata(secret) + if err != nil { + return nil, fmt.Errorf("secret was written successfully, but unable to view version metadata from response: %w", err) + } + + kvSecret := &KVSecret{ + Data: nil, // secret.Data in this case is the metadata + VersionMetadata: metadata, + Raw: secret, + } + + cm, err := extractCustomMetadata(secret) + if err != nil { + return nil, fmt.Errorf("error reading custom metadata for secret at %s: %w", pathToWriteTo, err) + } + kvSecret.CustomMetadata = cm + + return kvSecret, nil +} + +// Delete deletes the most recent version of a secret from the KV v2 +// secrets engine. To delete an older version, use DeleteVersions. +func (kv *kvv2) Delete(ctx context.Context, secretPath string) error { + pathToDelete := fmt.Sprintf("%s/data/%s", kv.mountPath, secretPath) + + _, err := kv.c.Logical().DeleteWithContext(ctx, pathToDelete) + if err != nil { + return fmt.Errorf("error deleting secret at %s: %w", pathToDelete, err) + } + + return nil +} + +// DeleteVersions deletes the specified versions of a secret from the KV v2 +// secrets engine. To delete the latest version of a secret, just use Delete. +func (kv *kvv2) DeleteVersions(ctx context.Context, secretPath string, versions []int) error { + // verb and path are different when trying to delete past versions + pathToDelete := fmt.Sprintf("%s/delete/%s", kv.mountPath, secretPath) + + if len(versions) == 0 { + return nil + } + + var versionsToDelete []string + for _, version := range versions { + versionsToDelete = append(versionsToDelete, strconv.Itoa(version)) + } + versionsMap := map[string]interface{}{ + "versions": versionsToDelete, + } + _, err := kv.c.Logical().WriteWithContext(ctx, pathToDelete, versionsMap) + if err != nil { + return fmt.Errorf("error deleting secret at %s: %w", pathToDelete, err) + } + + return nil +} + +func extractCustomMetadata(secret *Secret) (map[string]interface{}, error) { + // Logical Writes return the metadata directly, Reads return it nested inside the "metadata" key + cmI, ok := secret.Data["custom_metadata"] + if !ok { + mI, ok := secret.Data["metadata"] + if !ok { // if that's not found, bail since it should have had one or the other + return nil, fmt.Errorf("secret is missing expected fields") + } + mM, ok := mI.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("unexpected type for 'metadata' element: %T (%#v)", mI, mI) + } + cmI, ok = mM["custom_metadata"] + if !ok { + return nil, fmt.Errorf("metadata missing expected field \"custom_metadata\":%v", mM) + } + } + + cm, ok := cmI.(map[string]interface{}) + if !ok && cmI != nil { + return nil, fmt.Errorf("unexpected type for 'metadata' element: %T (%#v)", cmI, cmI) + } + + return cm, nil +} + +func extractDataAndVersionMetadata(secret *Secret) (*KVSecret, error) { + // A nil map is a valid value for data: secret.Data will be nil when this + // version of the secret has been deleted, but the metadata is still + // available. + var data map[string]interface{} + if secret.Data != nil { + dataInterface, ok := secret.Data["data"] + if !ok { + return nil, fmt.Errorf("missing expected 'data' element") + } + + if dataInterface != nil { + data, ok = dataInterface.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("unexpected type for 'data' element: %T (%#v)", data, data) + } + } + } + + metadata, err := extractVersionMetadata(secret) + if err != nil { + return nil, fmt.Errorf("unable to get version metadata: %w", err) + } + + return &KVSecret{ + Data: data, + VersionMetadata: metadata, + Raw: secret, + }, nil +} + +func extractVersionMetadata(secret *Secret) (*KVVersionMetadata, error) { + var metadata *KVVersionMetadata + + if secret.Data == nil { + return nil, nil + } + + // Logical Writes return the metadata directly, Reads return it nested inside the "metadata" key + var metadataMap map[string]interface{} + metadataInterface, ok := secret.Data["metadata"] + if ok { + metadataMap, ok = metadataInterface.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("unexpected type for 'metadata' element: %T (%#v)", metadataInterface, metadataInterface) + } + } else { + metadataMap = secret.Data + } + + // deletion_time usually comes in as an empty string which can't be + // processed as time.RFC3339, so we reset it to a convertible value + if metadataMap["deletion_time"] == "" { + metadataMap["deletion_time"] = time.Time{} + } + + d, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), + Result: &metadata, + }) + if err != nil { + return nil, fmt.Errorf("error setting up decoder for API response: %w", err) + } + + err = d.Decode(metadataMap) + if err != nil { + return nil, fmt.Errorf("error decoding metadata from API response into VersionMetadata: %w", err) + } + + return metadata, nil +} + +func extractFullMetadata(secret *Secret) (*KVMetadata, error) { + var metadata *KVMetadata + + if secret.Data == nil { + return nil, nil + } + + if versions, ok := secret.Data["versions"]; ok { + versionsMap := versions.(map[string]interface{}) + if len(versionsMap) > 0 { + for version, metadata := range versionsMap { + metadataMap := metadata.(map[string]interface{}) + // deletion_time usually comes in as an empty string which can't be + // processed as time.RFC3339, so we reset it to a convertible value + if metadataMap["deletion_time"] == "" { + metadataMap["deletion_time"] = time.Time{} + } + versionInt, err := strconv.Atoi(version) + if err != nil { + return nil, fmt.Errorf("error converting version %s to integer: %w", version, err) + } + metadataMap["version"] = versionInt + versionsMap[version] = metadataMap // save the updated copy of the metadata map + } + } + secret.Data["versions"] = versionsMap // save the updated copy of the versions map + } + + d, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeHookFunc(time.RFC3339), + mapstructure.StringToTimeDurationHookFunc(), + ), + Result: &metadata, + }) + if err != nil { + return nil, fmt.Errorf("error setting up decoder for API response: %w", err) + } + + err = d.Decode(secret.Data) + if err != nil { + return nil, fmt.Errorf("error decoding metadata from API response into KVMetadata: %w", err) + } + + return metadata, nil +} diff --git a/changelog/15305.txt b/changelog/15305.txt new file mode 100644 index 000000000..be3607cb6 --- /dev/null +++ b/changelog/15305.txt @@ -0,0 +1,3 @@ +```release-note:improvement +api: KV helper methods to simplify the common use case of reading and writing KV secrets +``` \ No newline at end of file diff --git a/vault/external_tests/api/kv_helpers_test.go b/vault/external_tests/api/kv_helpers_test.go new file mode 100644 index 000000000..381318cb7 --- /dev/null +++ b/vault/external_tests/api/kv_helpers_test.go @@ -0,0 +1,169 @@ +package api + +import ( + "context" + "reflect" + "testing" + + logicalKv "github.com/hashicorp/vault-plugin-secrets-kv" + "github.com/hashicorp/vault/api" + vaulthttp "github.com/hashicorp/vault/http" + "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/vault" +) + +func TestKVHelpers(t *testing.T) { + t.Parallel() + + // initialize test cluster + coreConfig := &vault.CoreConfig{ + LogicalBackends: map[string]logical.Factory{ + "kv": logicalKv.Factory, + "kv-v2": logicalKv.VersionedKVFactory, + }, + } + + cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + + cluster.Start() + defer cluster.Cleanup() + + cores := cluster.Cores + core := cores[0].Core + client := cluster.Cores[0].Client + vault.TestWaitActive(t, core) + + // mount the KVv2 backend + // (the test cluster has already mounted the KVv1 backend at "secret") + err := client.Sys().MountWithContext(context.Background(), "secret-v2", &api.MountInput{ + Type: "kv-v2", + }) + if err != nil { + t.Fatal(err) + } + + secretData := map[string]interface{}{ + "foo": "bar", + } + + //// v1 //// + t.Run("kv v1 helpers", func(t *testing.T) { + if err := client.KVv1("secret").Put(context.Background(), "my-secret", secretData); err != nil { + t.Fatal(err) + } + + secret, err := client.KVv1("secret").Get(context.Background(), "my-secret") + if err != nil { + t.Fatal(err) + } + + if secret.Data["foo"] != "bar" { + t.Fatalf("kv v1 secret did not contain expected value") + } + + if err := client.KVv1("secret").Delete(context.Background(), "my-secret"); err != nil { + t.Fatal(err) + } + }) + + //// v2 //// + t.Run("kv v2 helpers", func(t *testing.T) { + // create a secret + writtenSecret, err := client.KVv2("secret-v2").Put(context.Background(), "my-secret", secretData) + if err != nil { + t.Fatal(err) + } + if writtenSecret == nil || writtenSecret.VersionMetadata == nil { + t.Fatal("kv v2 secret did not have expected contents") + } + + secret, err := client.KVv2("secret-v2").Get(context.Background(), "my-secret") + if err != nil { + t.Fatal(err) + } + if secret.Data["foo"] != "bar" { + t.Fatal("kv v2 secret did not contain expected value") + } + if secret.VersionMetadata.CreatedTime != writtenSecret.VersionMetadata.CreatedTime { + t.Fatal("the created_time on the secret did not match the response from when it was created") + } + + // get its full metadata + fullMetadata, err := client.KVv2("secret-v2").GetMetadata(context.Background(), "my-secret") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(secret.CustomMetadata, fullMetadata.CustomMetadata) { + t.Fatalf("custom metadata on the secret does not match the custom metadata in the full metadata") + } + + // create a second version + _, err = client.KVv2("secret-v2").Put(context.Background(), "my-secret", map[string]interface{}{ + "foo": "baz", + }) + if err != nil { + t.Fatal(err) + } + + s2, err := client.KVv2("secret-v2").Get(context.Background(), "my-secret") + if err != nil { + t.Fatal(err) + } + if s2.Data["foo"] != "baz" { + t.Fatalf("second version of secret did not have expected contents") + } + if s2.VersionMetadata.Version != 2 { + t.Fatalf("wrong version of kv v2 secret was read, expected 2 but got %d", s2.VersionMetadata.Version) + } + + // get a specific past version + s1, err := client.KVv2("secret-v2").GetVersion(context.Background(), "my-secret", 1) + if err != nil { + t.Fatal(err) + } + if s1.VersionMetadata.Version != 1 { + t.Fatalf("wrong version of kv v2 secret was read, expected 1 but got %d", s1.VersionMetadata.Version) + } + + // delete that version + if err = client.KVv2("secret-v2").DeleteVersions(context.Background(), "my-secret", []int{1}); err != nil { + t.Fatal(err) + } + + s1AfterDelete, err := client.KVv2("secret-v2").GetVersion(context.Background(), "my-secret", 1) + if err != nil { + t.Fatal(err) + } + + if s1AfterDelete.VersionMetadata.DeletionTime.IsZero() { + t.Fatalf("the deletion_time in the first version of the secret was not updated") + } + + if s1AfterDelete.Data != nil { + t.Fatalf("data still exists on the first version of the secret despite this version being deleted") + } + + // check that KVOption works + _, err = client.KVv2("secret-v2").Put(context.Background(), "my-secret", map[string]interface{}{ + "meow": "woof", + }, api.WithCheckAndSet(99)) + if err == nil { + t.Fatalf("expected error from trying to update different version from check-and-set value") + } + + versions, err := client.KVv2("secret-v2").GetVersionsAsList(context.Background(), "my-secret") + if err != nil { + t.Fatal(err) + } + + if len(versions) != 2 { + t.Fatalf("expected there to be 2 versions of the secret but got %d", len(versions)) + } + + if versions[0].Version != 1 { + t.Fatalf("incorrect value for version; expected 1 but got %d", versions[0].Version) + } + }) +}