Add the batch reference field, as in Transform, to Transit operations (#18243)

* Add the batch reference field, as in Transform, to Transit operations

* changelog

* docs

* More mapstructure tags

builtin/logical/transit/path_decrypt.go

@@ -19,6 +19,10 @@ type DecryptBatchResponseItem struct {
 	// Error, if set represents a failure encountered while encrypting a
 	// corresponding batch request item
 	Error string `json:"error,omitempty" structs:"error" mapstructure:"error"`
+
+	// Reference is an arbitrary caller supplied string value that will be placed on the
+	// batch response to ease correlation between inputs and outputs
+	Reference string `json:"reference" structs:"reference" mapstructure:"reference"`
 }
 
 func (b *backend) pathDecrypt() *framework.Path {
@@ -195,6 +199,10 @@ func (b *backend) pathDecryptWrite(ctx context.Context, req *logical.Request, d
 
 	resp := &logical.Response{}
 	if batchInputRaw != nil {
+		// Copy the references
+		for i := range batchInputItems {
+			batchResponseItems[i].Reference = batchInputItems[i].Reference
+		}
 		resp.Data = map[string]interface{}{
 			"batch_results": batchResponseItems,
 		}

builtin/logical/transit/path_decrypt_test.go

@@ -19,9 +19,9 @@ func TestTransit_BatchDecryption(t *testing.T) {
 	b, s := createBackendWithStorage(t)
 
 	batchEncryptionInput := []interface{}{
-		map[string]interface{}{"plaintext": ""},     // empty string
-		map[string]interface{}{"plaintext": "Cg=="}, // newline
-		map[string]interface{}{"plaintext": "dGhlIHF1aWNrIGJyb3duIGZveA=="},
+		map[string]interface{}{"plaintext": "", "reference": "foo"},     // empty string
+		map[string]interface{}{"plaintext": "Cg==", "reference": "bar"}, // newline
+		map[string]interface{}{"plaintext": "dGhlIHF1aWNrIGJyb3duIGZveA==", "reference": "baz"},
 	}
 	batchEncryptionData := map[string]interface{}{
 		"batch_input": batchEncryptionInput,
@@ -41,7 +41,7 @@ func TestTransit_BatchDecryption(t *testing.T) {
 	batchResponseItems := resp.Data["batch_results"].([]EncryptBatchResponseItem)
 	batchDecryptionInput := make([]interface{}, len(batchResponseItems))
 	for i, item := range batchResponseItems {
-		batchDecryptionInput[i] = map[string]interface{}{"ciphertext": item.Ciphertext}
+		batchDecryptionInput[i] = map[string]interface{}{"ciphertext": item.Ciphertext, "reference": item.Reference}
 	}
 	batchDecryptionData := map[string]interface{}{
 		"batch_input": batchDecryptionInput,
@@ -59,7 +59,8 @@ func TestTransit_BatchDecryption(t *testing.T) {
 	}
 
 	batchDecryptionResponseItems := resp.Data["batch_results"].([]DecryptBatchResponseItem)
-	expectedResult := "[{\"plaintext\":\"\"},{\"plaintext\":\"Cg==\"},{\"plaintext\":\"dGhlIHF1aWNrIGJyb3duIGZveA==\"}]"
+	// This seems fragile
+	expectedResult := "[{\"plaintext\":\"\",\"reference\":\"foo\"},{\"plaintext\":\"Cg==\",\"reference\":\"bar\"},{\"plaintext\":\"dGhlIHF1aWNrIGJyb3duIGZveA==\",\"reference\":\"baz\"}]"
 
 	jsonResponse, err := json.Marshal(batchDecryptionResponseItems)
 	if err != nil || err == nil && string(jsonResponse) != expectedResult {

builtin/logical/transit/path_encrypt.go

@@ -42,6 +42,10 @@ type BatchRequestItem struct {
 
 	// Associated Data for AEAD ciphers
 	AssociatedData string `json:"associated_data" struct:"associated_data" mapstructure:"associated_data"`
+
+	// Reference is an arbitrary caller supplied string value that will be placed on the
+	// batch response to ease correlation between inputs and outputs
+	Reference string `json:"reference" structs:"reference" mapstructure:"reference"`
 }
 
 // EncryptBatchResponseItem represents a response item for batch processing
@@ -56,6 +60,10 @@ type EncryptBatchResponseItem struct {
 	// Error, if set represents a failure encountered while encrypting a
 	// corresponding batch request item
 	Error string `json:"error,omitempty" structs:"error" mapstructure:"error"`
+
+	// Reference is an arbitrary caller supplied string value that will be placed on the
+	// batch response to ease correlation between inputs and outputs
+	Reference string `json:"reference"`
 }
 
 type AssocDataFactory struct {
@@ -261,6 +269,14 @@ func decodeBatchRequestItems(src interface{}, requirePlaintext bool, requireCiph
 				errs.Errors = append(errs.Errors, fmt.Sprintf("'[%d].associated_data' expected type 'string', got unconvertible type '%T'", i, item["associated_data"]))
 			}
 		}
+		if v, has := item["reference"]; has {
+			if !reflect.ValueOf(v).IsValid() {
+			} else if casted, ok := v.(string); ok {
+				(*dst)[i].Reference = casted
+			} else {
+				errs.Errors = append(errs.Errors, fmt.Sprintf("'[%d].reference' expected type 'string', got unconvertible type '%T'", i, item["reference"]))
+			}
+		}
 	}
 
 	if len(errs.Errors) > 0 {
@@ -471,6 +487,10 @@ func (b *backend) pathEncryptWrite(ctx context.Context, req *logical.Request, d
 
 	resp := &logical.Response{}
 	if batchInputRaw != nil {
+		// Copy the references
+		for i := range batchInputItems {
+			batchResponseItems[i].Reference = batchInputItems[i].Reference
+		}
 		resp.Data = map[string]interface{}{
 			"batch_results": batchResponseItems,
 		}

builtin/logical/transit/path_encrypt_test.go

@@ -225,7 +225,7 @@ func TestTransit_BatchEncryptionCase3(t *testing.T) {
 	}
 }
 
-// Case4: Test batch encryption with an existing key
+// Case4: Test batch encryption with an existing key (and test references)
 func TestTransit_BatchEncryptionCase4(t *testing.T) {
 	var resp *logical.Response
 	var err error
@@ -243,8 +243,8 @@ func TestTransit_BatchEncryptionCase4(t *testing.T) {
 	}
 
 	batchInput := []interface{}{
-		map[string]interface{}{"plaintext": "dGhlIHF1aWNrIGJyb3duIGZveA=="},
-		map[string]interface{}{"plaintext": "dGhlIHF1aWNrIGJyb3duIGZveA=="},
+		map[string]interface{}{"plaintext": "dGhlIHF1aWNrIGJyb3duIGZveA==", "reference": "b"},
+		map[string]interface{}{"plaintext": "dGhlIHF1aWNrIGJyb3duIGZveA==", "reference": "a"},
 	}
 
 	batchData := map[string]interface{}{
@@ -271,7 +271,7 @@ func TestTransit_BatchEncryptionCase4(t *testing.T) {
 
 	plaintext := "dGhlIHF1aWNrIGJyb3duIGZveA=="
 
-	for _, item := range batchResponseItems {
+	for i, item := range batchResponseItems {
 		if item.KeyVersion != 1 {
 			t.Fatalf("unexpected key version; got: %d, expected: %d", item.KeyVersion, 1)
 		}
@@ -287,6 +287,10 @@ func TestTransit_BatchEncryptionCase4(t *testing.T) {
 		if resp.Data["plaintext"] != plaintext {
 			t.Fatalf("bad: plaintext. Expected: %q, Actual: %q", plaintext, resp.Data["plaintext"])
 		}
+		inputItem := batchInput[i].(map[string]interface{})
+		if item.Reference != inputItem["reference"] {
+			t.Fatalf("reference mismatch.  Expected %s, Actual: %s", inputItem["reference"], item.Reference)
+		}
 	}
 }
 

builtin/logical/transit/path_hmac.go

@@ -35,6 +35,10 @@ type batchResponseHMACItem struct {
 	// For batch processing to successfully mimic previous handling for simple 'input',
 	// both output values are needed - though 'err' should never be serialized.
 	err error
+
+	// Reference is an arbitrary caller supplied string value that will be placed on the
+	// batch response to ease correlation between inputs and outputs
+	Reference string `json:"reference" mapstructure:"reference"`
 }
 
 func (b *backend) pathHMAC() *framework.Path {
@@ -201,6 +205,10 @@ func (b *backend) pathHMACWrite(ctx context.Context, req *logical.Request, d *fr
 	// Generate the response
 	resp := &logical.Response{}
 	if batchInputRaw != nil {
+		// Copy the references
+		for i := range batchInputItems {
+			response[i].Reference = batchInputItems[i]["reference"]
+		}
 		resp.Data = map[string]interface{}{
 			"batch_results": response,
 		}
@@ -362,6 +370,10 @@ func (b *backend) pathHMACVerify(ctx context.Context, req *logical.Request, d *f
 	// Generate the response
 	resp := &logical.Response{}
 	if batchInputRaw != nil {
+		// Copy the references
+		for i := range batchInputItems {
+			response[i].Reference = batchInputItems[i]["reference"]
+		}
 		resp.Data = map[string]interface{}{
 			"batch_results": response,
 		}

builtin/logical/transit/path_hmac_test.go

@@ -248,18 +248,18 @@ func TestTransit_batchHMAC(t *testing.T) {
 
 	req.Path = "hmac/foo"
 	batchInput := []batchRequestHMACItem{
-		{"input": "dGhlIHF1aWNrIGJyb3duIGZveA=="},
-		{"input": "dGhlIHF1aWNrIGJyb3duIGZveA=="},
-		{"input": ""},
-		{"input": ":;.?"},
+		{"input": "dGhlIHF1aWNrIGJyb3duIGZveA==", "reference": "one"},
+		{"input": "dGhlIHF1aWNrIGJyb3duIGZveA==", "reference": "two"},
+		{"input": "", "reference": "three"},
+		{"input": ":;.?", "reference": "four"},
 		{},
 	}
 
 	expected := []batchResponseHMACItem{
-		{HMAC: "vault:v1:UcBvm5VskkukzZHlPgm3p5P/Yr/PV6xpuOGZISya3A4="},
-		{HMAC: "vault:v1:UcBvm5VskkukzZHlPgm3p5P/Yr/PV6xpuOGZISya3A4="},
-		{HMAC: "vault:v1:BCfVv6rlnRsIKpjCZCxWvh5iYwSSabRXpX9XJniuNgc="},
-		{Error: "unable to decode input as base64: illegal base64 data at input byte 0"},
+		{HMAC: "vault:v1:UcBvm5VskkukzZHlPgm3p5P/Yr/PV6xpuOGZISya3A4=", Reference: "one"},
+		{HMAC: "vault:v1:UcBvm5VskkukzZHlPgm3p5P/Yr/PV6xpuOGZISya3A4=", Reference: "two"},
+		{HMAC: "vault:v1:BCfVv6rlnRsIKpjCZCxWvh5iYwSSabRXpX9XJniuNgc=", Reference: "three"},
+		{Error: "unable to decode input as base64: illegal base64 data at input byte 0", Reference: "four"},
 		{Error: "missing input for HMAC"},
 	}
 
@@ -286,6 +286,9 @@ func TestTransit_batchHMAC(t *testing.T) {
 		if expected[i].Error != "" && expected[i].Error != m.Error {
 			t.Fatalf("Expected Error %q got %q in result %d", expected[i].Error, m.Error, i)
 		}
+		if expected[i].Reference != m.Reference {
+			t.Fatalf("Expected references to match, Got %s, Expected %s", m.Reference, expected[i].Reference)
+		}
 	}
 
 	// Verify a previous version

builtin/logical/transit/path_rewrap.go

@@ -182,6 +182,10 @@ func (b *backend) pathRewrapWrite(ctx context.Context, req *logical.Request, d *
 
 	resp := &logical.Response{}
 	if batchInputRaw != nil {
+		// Copy the references
+		for i := range batchInputItems {
+			batchResponseItems[i].Reference = batchInputItems[i].Reference
+		}
 		resp.Data = map[string]interface{}{
 			"batch_results": batchResponseItems,
 		}

builtin/logical/transit/path_rewrap_test.go

@@ -224,8 +224,8 @@ func TestTransit_BatchRewrapCase3(t *testing.T) {
 	b, s := createBackendWithStorage(t)
 
 	batchEncryptionInput := []interface{}{
-		map[string]interface{}{"plaintext": "dmlzaGFsCg=="},
-		map[string]interface{}{"plaintext": "dGhlIHF1aWNrIGJyb3duIGZveA=="},
+		map[string]interface{}{"plaintext": "dmlzaGFsCg==", "reference": "ek"},
+		map[string]interface{}{"plaintext": "dGhlIHF1aWNrIGJyb3duIGZveA==", "reference": "do"},
 	}
 	batchEncryptionData := map[string]interface{}{
 		"batch_input": batchEncryptionInput,
@@ -245,7 +245,7 @@ func TestTransit_BatchRewrapCase3(t *testing.T) {
 
 	batchRewrapInput := make([]interface{}, len(batchEncryptionResponseItems))
 	for i, item := range batchEncryptionResponseItems {
-		batchRewrapInput[i] = map[string]interface{}{"ciphertext": item.Ciphertext}
+		batchRewrapInput[i] = map[string]interface{}{"ciphertext": item.Ciphertext, "reference": item.Reference}
 	}
 
 	batchRewrapData := map[string]interface{}{
@@ -289,6 +289,11 @@ func TestTransit_BatchRewrapCase3(t *testing.T) {
 	for i, eItem := range batchEncryptionResponseItems {
 		rItem := batchRewrapResponseItems[i]
 
+		inputRef := batchEncryptionInput[i].(map[string]interface{})["reference"]
+		if eItem.Reference != inputRef {
+			t.Fatalf("bad: reference mismatch. Expected %s, Actual: %s", inputRef, eItem.Reference)
+		}
+
 		if eItem.Ciphertext == rItem.Ciphertext {
 			t.Fatalf("bad: rewrap input and output are the same")
 		}
@@ -315,5 +320,6 @@ func TestTransit_BatchRewrapCase3(t *testing.T) {
 		if resp.Data["plaintext"] != plaintext1 && resp.Data["plaintext"] != plaintext2 {
 			t.Fatalf("bad: plaintext. Expected: %q or %q, Actual: %q", plaintext1, plaintext2, resp.Data["plaintext"])
 		}
+
 	}
 }

builtin/logical/transit/path_sign_verify.go

@@ -39,6 +39,10 @@ type batchResponseSignItem struct {
 	// For batch processing to successfully mimic previous handling for simple 'input',
 	// both output values are needed - though 'err' should never be serialized.
 	err error
+
+	// Reference is an arbitrary caller supplied string value that will be placed on the
+	// batch response to ease correlation between inputs and outputs
+	Reference string `json:"reference" mapstructure:"reference"`
 }
 
 // BatchRequestVerifyItem represents a request item for batch processing.
@@ -59,6 +63,10 @@ type batchResponseVerifyItem struct {
 	// For batch processing to successfully mimic previous handling for simple 'input',
 	// both output values are needed - though 'err' should never be serialized.
 	err error
+
+	// Reference is an arbitrary caller supplied string value that will be placed on the
+	// batch response to ease correlation between inputs and outputs
+	Reference string `json:"reference" mapstructure:"reference"`
 }
 
 const defaultHashAlgorithm = "sha2-256"
@@ -420,6 +428,10 @@ func (b *backend) pathSignWrite(ctx context.Context, req *logical.Request, d *fr
 	// Generate the response
 	resp := &logical.Response{}
 	if batchInputRaw != nil {
+		// Copy the references
+		for i := range batchInputItems {
+			response[i].Reference = batchInputItems[i]["reference"]
+		}
 		resp.Data = map[string]interface{}{
 			"batch_results": response,
 		}
@@ -636,6 +648,10 @@ func (b *backend) pathVerifyWrite(ctx context.Context, req *logical.Request, d *
 	// Generate the response
 	resp := &logical.Response{}
 	if batchInputRaw != nil {
+		// Copy the references
+		for i := range batchInputItems {
+			response[i].Reference = batchInputItems[i]["reference"]
+		}
 		resp.Data = map[string]interface{}{
 			"batch_results": response,
 		}

builtin/logical/transit/path_sign_verify_test.go

@@ -25,6 +25,7 @@ type signOutcome struct {
 	requestOk bool
 	valid     bool
 	keyValid  bool
+	reference string
 }
 
 func TestTransit_SignVerify_ECDSA(t *testing.T) {
@@ -483,6 +484,7 @@ func TestTransit_SignVerify_ED25519(t *testing.T) {
 			}
 			for i, v := range sig {
 				batchRequestItems[i]["signature"] = v
+				batchRequestItems[i]["reference"] = outcome[i].reference
 			}
 		} else if attachSig {
 			req.Data["signature"] = sig[0]
@@ -535,6 +537,9 @@ func TestTransit_SignVerify_ED25519(t *testing.T) {
 				if pubKeyRaw, ok := req.Data["public_key"]; ok {
 					validatePublicKey(t, batchRequestItems[i]["input"], sig[i], pubKeyRaw.([]byte), outcome[i].keyValid, postpath, b)
 				}
+				if v.Reference != outcome[i].reference {
+					t.Fatalf("verification failed, mismatched references %s vs %s", v.Reference, outcome[i].reference)
+				}
 			}
 			return
 		}
@@ -634,15 +639,18 @@ func TestTransit_SignVerify_ED25519(t *testing.T) {
 
 	// Test Batch Signing
 	batchInput := []batchRequestSignItem{
-		{"context": "abcd", "input": "dGhlIHF1aWNrIGJyb3duIGZveA=="},
-		{"context": "efgh", "input": "dGhlIHF1aWNrIGJyb3duIGZveA=="},
+		{"context": "abcd", "input": "dGhlIHF1aWNrIGJyb3duIGZveA==", "reference": "uno"},
+		{"context": "efgh", "input": "dGhlIHF1aWNrIGJyb3duIGZveA==", "reference": "dos"},
 	}
 
 	req.Data = map[string]interface{}{
 		"batch_input": batchInput,
 	}
 
-	outcome = []signOutcome{{requestOk: true, valid: true, keyValid: true}, {requestOk: true, valid: true, keyValid: true}}
+	outcome = []signOutcome{
+		{requestOk: true, valid: true, keyValid: true, reference: "uno"},
+		{requestOk: true, valid: true, keyValid: true, reference: "dos"},
+	}
 
 	sig = signRequest(req, false, "foo")
 	verifyRequest(req, false, outcome, "foo", sig, true)

changelog/18243.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+secrets/transit: Add an optional reference field to batch operation items
+which is repeated on batch responses to help more easily correlate inputs with outputs.
+```

website/content/api-docs/secret/transit.mdx

@@ -627,6 +627,12 @@ will be returned.
   for any given context (and thus, any given encryption key) this nonce value is
   **never reused**.
 
+- `reference` `(string: "")` -
+  A user-supplied string that will be present in the `reference` field on the
+  corresponding `batch_results` item in the response, to assist in understanding
+  which result corresponds to a particular input. Only valid on batch requests
+  when using ‘batch_input’ below.
+
 - `batch_input` `(array<object>: nil)` – Specifies a list of items to be
   encrypted in a single batch. When this parameter is set, if the parameters
   'plaintext', 'context' and 'nonce' are also set, they will be ignored. 
@@ -740,6 +746,12 @@ This endpoint decrypts the provided ciphertext using the named key.
   and the key was generated with Vault 0.6.1. Not required for keys created in
   0.6.2+.
 
+- `reference` `(string: "")` -
+  A user-supplied string that will be present in the `reference` field on the
+  corresponding `batch_results` item in the response, to assist in understanding
+  which result corresponds to a particular input. Only valid on batch requests
+  when using ‘batch_input’ below.
+
 - `batch_input` `(array<object>: nil)` – Specifies a list of items to be
   decrypted in a single batch. When this parameter is set, if the parameters
   'ciphertext', 'context' and 'nonce' are also set, they will be ignored. 
@@ -824,6 +836,12 @@ functionality to untrusted users or scripts.
   and the key was generated with Vault 0.6.1. Not required for keys created in
   0.6.2+.
 
+- `reference` `(string: "")` -
+  A user-supplied string that will be present in the `reference` field on the
+  corresponding `batch_results` item in the response, to assist in understanding
+  which result corresponds to a particular input. Only valid on batch requests
+  when using ‘batch_input’ below.
+
 - `batch_input` `(array<object>: nil)` – Specifies a list of items to be
   decrypted in a single batch. When this parameter is set, if the parameters
   'ciphertext', 'context' and 'nonce' are also set, they will be ignored. 
@@ -1085,6 +1103,12 @@ be used.
 - `input` `(string: "")` – Specifies the **base64 encoded** input data. One of
   `input` or `batch_input` must be supplied.
 
+- `reference` `(string: "")` -
+  A user-supplied string that will be present in the `reference` field on the
+  corresponding `batch_results` item in the response, to assist in understanding
+  which result corresponds to a particular input. Only valid on batch requests
+  when using ‘batch_input’ below.
+
 - `batch_input` `(array<object>: nil)` – Specifies a list of items for processing.
   When this parameter is set, if the parameter 'input' is also set, it will be
   ignored. Responses are returned in the 'batch_results' array component of the
@@ -1233,6 +1257,12 @@ supports signing.
 - `input` `(string: "")` – Specifies the **base64 encoded** input data. One of
   `input` or `batch_input` must be supplied.
 
+- `reference` `(string: "")` -
+  A user-supplied string that will be present in the `reference` field on the
+  corresponding `batch_results` item in the response, to assist in understanding
+  which result corresponds to a particular input. Only valid on batch requests
+  when using ‘batch_input’ below.
+
 - `batch_input` `(array<object>: nil)` – Specifies a list of items for processing.
   When this parameter is set, any supplied 'input' or 'context' parameters will be
   ignored. Responses are returned in the 'batch_results' array component of the
@@ -1417,6 +1447,12 @@ data.
   `/transit/hmac` function. Either this must be supplied or `signature` must be
   supplied.
 
+- `reference` `(string: "")` -
+  A user-supplied string that will be present in the `reference` field on the
+  corresponding `batch_results` item in the response, to assist in understanding
+  which result corresponds to a particular input. Only valid on batch requests
+  when using ‘batch_input’ below.
+
 - `batch_input` `(array<object>: nil)` – Specifies a list of items for processing.
   When this parameter is set, any supplied 'input', 'hmac' or 'signature' parameters
   will be ignored. 'batch_input' items should contain an 'input' parameter and