Enhance sys/raw to read and write values that cannot be encoded in json (#13537)

changelog/13537.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+sys/raw: Enhance sys/raw to read and write values that cannot be encoded in json.
+```

sdk/helper/compressutil/compress.go

@@ -141,10 +141,21 @@ func Compress(data []byte, config *CompressionConfig) ([]byte, error) {
 // If the first byte isn't a canary byte, then the utility returns a boolean
 // value indicating that the input was not compressed.
 func Decompress(data []byte) ([]byte, bool, error) {
+	bytes, _, notCompressed, err := DecompressWithCanary(data)
+	return bytes, notCompressed, err
+}
+
+// DecompressWithCanary checks if the first byte in the input matches the canary byte.
+// If the first byte is a canary byte, then the input past the canary byte
+// will be decompressed using the method specified in the given configuration. The type of compression used is also
+// returned. If the first byte isn't a canary byte, then the utility returns a boolean
+// value indicating that the input was not compressed.
+func DecompressWithCanary(data []byte) ([]byte, string, bool, error) {
 	var err error
 	var reader io.ReadCloser
+	var compressionType string
 	if data == nil || len(data) == 0 {
-		return nil, false, fmt.Errorf("'data' being decompressed is empty")
+		return nil, "", false, fmt.Errorf("'data' being decompressed is empty")
 	}
 
 	canary := data[0]
@@ -155,43 +166,47 @@ func Decompress(data []byte) ([]byte, bool, error) {
 	// byte and try to decompress the data that is after the canary.
 	case CompressionCanaryGzip:
 		if len(data) < 2 {
-			return nil, false, fmt.Errorf("invalid 'data' after the canary")
+			return nil, "", false, fmt.Errorf("invalid 'data' after the canary")
 		}
 		reader, err = gzip.NewReader(bytes.NewReader(cData))
+		compressionType = CompressionTypeGzip
 
 	case CompressionCanaryLZW:
 		if len(data) < 2 {
-			return nil, false, fmt.Errorf("invalid 'data' after the canary")
+			return nil, "", false, fmt.Errorf("invalid 'data' after the canary")
 		}
 		reader = lzw.NewReader(bytes.NewReader(cData), lzw.LSB, 8)
+		compressionType = CompressionTypeLZW
 
 	case CompressionCanarySnappy:
 		if len(data) < 2 {
-			return nil, false, fmt.Errorf("invalid 'data' after the canary")
+			return nil, "", false, fmt.Errorf("invalid 'data' after the canary")
 		}
 		reader = &CompressUtilReadCloser{
 			Reader: snappy.NewReader(bytes.NewReader(cData)),
 		}
+		compressionType = CompressionTypeSnappy
 
 	case CompressionCanaryLZ4:
 		if len(data) < 2 {
-			return nil, false, fmt.Errorf("invalid 'data' after the canary")
+			return nil, "", false, fmt.Errorf("invalid 'data' after the canary")
 		}
 		reader = &CompressUtilReadCloser{
 			Reader: lz4.NewReader(bytes.NewReader(cData)),
 		}
+		compressionType = CompressionTypeLZ4
 
 	default:
 		// If the first byte doesn't match the canary byte, it means
 		// that the content was not compressed at all. Indicate the
 		// caller that the input was not compressed.
-		return nil, true, nil
+		return nil, "", true, nil
 	}
 	if err != nil {
-		return nil, false, errwrap.Wrapf("failed to create a compression reader: {{err}}", err)
+		return nil, "", false, errwrap.Wrapf("failed to create a compression reader: {{err}}", err)
 	}
 	if reader == nil {
-		return nil, false, fmt.Errorf("failed to create a compression reader")
+		return nil, "", false, fmt.Errorf("failed to create a compression reader")
 	}
 
 	// Close the io.ReadCloser
@@ -200,8 +215,8 @@ func Decompress(data []byte) ([]byte, bool, error) {
 	// Read all the compressed data into a buffer
 	var buf bytes.Buffer
 	if _, err = io.Copy(&buf, reader); err != nil {
-		return nil, false, err
+		return nil, "", false, err
 	}
 
-	return buf.Bytes(), false, nil
+	return buf.Bytes(), compressionType, false, nil
 }

sdk/helper/compressutil/compress_test.go

@@ -86,6 +86,15 @@ func TestCompressUtil_CompressDecompress(t *testing.T) {
 		if !bytes.Equal(inputJSONBytes, decompressedJSONBytes) {
 			t.Fatalf("bad (%s): decompressed value;\nexpected: %q\nactual: %q", test.compressionType, string(inputJSONBytes), string(decompressedJSONBytes))
 		}
+
+		decompressedJSONBytes, compressionType, wasNotCompressed, err := DecompressWithCanary(compressedJSONBytes)
+		if err != nil {
+			t.Fatalf("decompress error (%s): %s", test.compressionType, err)
+		}
+
+		if compressionType != test.compressionConfig.Type {
+			t.Fatalf("bad compressionType value;\nexpected: %q\naction: %q", test.compressionConfig.Type, compressionType)
+		}
 	}
 }
 

vault/logical_raw.go

@@ -1,7 +1,9 @@
 package vault
 
 import (
+	"compress/gzip"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"strings"
 
@@ -46,6 +48,17 @@ func NewRawBackend(core *Core) *RawBackend {
 func (b *RawBackend) handleRawRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
 	path := data.Get("path").(string)
 
+	// Preserve pre-existing behavior to decompress if `compressed` is missing
+	compressed := true
+	if d, ok := data.GetOk("compressed"); ok {
+		compressed = d.(bool)
+	}
+
+	encoding := data.Get("encoding").(string)
+	if encoding != "" && encoding != "base64" {
+		return logical.ErrorResponse("invalid encoding '%s'", encoding), logical.ErrInvalidRequest
+	}
+
 	if b.recoveryMode {
 		b.logger.Info("reading", "path", path)
 	}
@@ -72,23 +85,32 @@ func (b *RawBackend) handleRawRead(ctx context.Context, req *logical.Request, da
 		return nil, nil
 	}
 
+	valueBytes := entry.Value
+	if compressed {
 		// Run this through the decompression helper to see if it's been compressed.
-	// If the input contained the compression canary, `outputBytes` will hold
-	// the decompressed data. If the input was not compressed, then `outputBytes`
+		// If the input contained the compression canary, `valueBytes` will hold
+		// the decompressed data. If the input was not compressed, then `valueBytes`
 		// will be nil.
-	outputBytes, _, err := compressutil.Decompress(entry.Value)
+		valueBytes, _, err = compressutil.Decompress(entry.Value)
 		if err != nil {
 			return handleErrorNoReadOnlyForward(err)
 		}
 
-	// `outputBytes` is nil if the input is uncompressed. In that case set it to the original input.
-	if outputBytes == nil {
-		outputBytes = entry.Value
+		// `valueBytes` is nil if the input is uncompressed. In that case set it to the original input.
+		if valueBytes == nil {
+			valueBytes = entry.Value
+		}
+	}
+
+	var value interface{} = string(valueBytes)
+	// Golang docs (https://pkg.go.dev/encoding/json#Marshal), []byte encodes as a base64-encoded string
+	if encoding == "base64" {
+		value = valueBytes
 	}
 
 	resp := &logical.Response{
 		Data: map[string]interface{}{
-			"value": string(outputBytes),
+			"value": value,
 		},
 	}
 	return resp, nil
@@ -97,6 +119,16 @@ func (b *RawBackend) handleRawRead(ctx context.Context, req *logical.Request, da
 // handleRawWrite is used to write directly to the barrier
 func (b *RawBackend) handleRawWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
 	path := data.Get("path").(string)
+	compressionType := ""
+	c, compressionTypeOk := data.GetOk("compression_type")
+	if compressionTypeOk {
+		compressionType = c.(string)
+	}
+
+	encoding := data.Get("encoding").(string)
+	if encoding != "" && encoding != "base64" {
+		return logical.ErrorResponse("invalid encoding '%s'", encoding), logical.ErrInvalidRequest
+	}
 
 	if b.recoveryMode {
 		b.logger.Info("writing", "path", path)
@@ -110,11 +142,83 @@ func (b *RawBackend) handleRawWrite(ctx context.Context, req *logical.Request, d
 		}
 	}
 
-	value := data.Get("value").(string)
+	v := data.Get("value").(string)
+	value := []byte(v)
+	if encoding == "base64" {
+		var err error
+		value, err = base64.StdEncoding.DecodeString(v)
+		if err != nil {
+			return logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest
+		}
+	}
+
+	if req.Operation == logical.UpdateOperation {
+		// Check if this is an existing value with compression applied, if so, use the same compression (or no compression)
+		entry, err := b.barrier.Get(ctx, path)
+		if err != nil {
+			return handleErrorNoReadOnlyForward(err)
+		}
+		if entry == nil {
+			err := fmt.Sprintf("cannot figure out compression type because entry does not exist")
+			return logical.ErrorResponse(err), logical.ErrInvalidRequest
+		}
+
+		// For cases where DecompressWithCanary errored, treat entry as non-compressed data.
+		_, existingCompressionType, _, _ := compressutil.DecompressWithCanary(entry.Value)
+
+		// Ensure compression_type matches existing entries' compression
+		// except allow writing non-compressed data over compressed data
+		if existingCompressionType != compressionType && compressionType != "" {
+			err := fmt.Sprintf("the entry uses a different compression scheme then compression_type")
+			return logical.ErrorResponse(err), logical.ErrInvalidRequest
+		}
+
+		if !compressionTypeOk {
+			compressionType = existingCompressionType
+		}
+	}
+
+	if compressionType != "" {
+		var config *compressutil.CompressionConfig
+		switch compressionType {
+		case compressutil.CompressionTypeLZ4:
+			config = &compressutil.CompressionConfig{
+				Type: compressutil.CompressionTypeLZ4,
+			}
+			break
+		case compressutil.CompressionTypeLZW:
+			config = &compressutil.CompressionConfig{
+				Type: compressutil.CompressionTypeLZW,
+			}
+			break
+		case compressutil.CompressionTypeGzip:
+			config = &compressutil.CompressionConfig{
+				Type:                 compressutil.CompressionTypeGzip,
+				GzipCompressionLevel: gzip.BestCompression,
+			}
+			break
+		case compressutil.CompressionTypeSnappy:
+			config = &compressutil.CompressionConfig{
+				Type: compressutil.CompressionTypeSnappy,
+			}
+			break
+		default:
+			err := fmt.Sprintf("invalid compression type '%s'", compressionType)
+			return logical.ErrorResponse(err), logical.ErrInvalidRequest
+		}
+
+		var err error
+		value, err = compressutil.Compress(value, config)
+		if err != nil {
+			return logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest
+		}
+	}
+
 	entry := &logical.StorageEntry{
 		Key:   path,
-		Value: []byte(value),
+		Value: value,
 	}
+
 	if err := b.barrier.Put(ctx, entry); err != nil {
 		return logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest
 	}
@@ -175,6 +279,16 @@ func (b *RawBackend) handleRawList(ctx context.Context, req *logical.Request, da
 	return logical.ListResponse(keys), nil
 }
 
+// existenceCheck checks if entry exists, used in handleRawWrite for update or create operations
+func (b *RawBackend) existenceCheck(ctx context.Context, request *logical.Request, data *framework.FieldData) (bool, error) {
+	path := data.Get("path").(string)
+	entry, err := b.barrier.Get(ctx, path)
+	if err != nil {
+		return false, err
+	}
+	return entry != nil, nil
+}
+
 func rawPaths(prefix string, r *RawBackend) []*framework.Path {
 	return []*framework.Path{
 		{
@@ -187,6 +301,15 @@ func rawPaths(prefix string, r *RawBackend) []*framework.Path {
 				"value": {
 					Type: framework.TypeString,
 				},
+				"compressed": {
+					Type: framework.TypeBool,
+				},
+				"encoding": {
+					Type: framework.TypeString,
+				},
+				"compression_type": {
+					Type: framework.TypeString,
+				},
 			},
 
 			Operations: map[logical.Operation]framework.OperationHandler{
@@ -198,6 +321,10 @@ func rawPaths(prefix string, r *RawBackend) []*framework.Path {
 					Callback: r.handleRawWrite,
 					Summary:  "Update the value of the key at the given path.",
 				},
+				logical.CreateOperation: &framework.PathOperation{
+					Callback: r.handleRawWrite,
+					Summary:  "Create a key with value at the given path.",
+				},
 				logical.DeleteOperation: &framework.PathOperation{
 					Callback: r.handleRawDelete,
 					Summary:  "Delete the key with given path.",
@@ -208,6 +335,7 @@ func rawPaths(prefix string, r *RawBackend) []*framework.Path {
 				},
 			},
 
+			ExistenceCheck:  r.existenceCheck,
 			HelpSynopsis:    strings.TrimSpace(sysHelp["raw"][0]),
 			HelpDescription: strings.TrimSpace(sysHelp["raw"][1]),
 		},

vault/logical_system_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/random"
 	"github.com/hashicorp/vault/sdk/framework"
+	"github.com/hashicorp/vault/sdk/helper/compressutil"
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
 	"github.com/hashicorp/vault/sdk/helper/salt"
@@ -1940,6 +1941,7 @@ func TestSystemBackend_disableAudit(t *testing.T) {
 }
 
 func TestSystemBackend_rawRead_Compressed(t *testing.T) {
+	t.Run("basic", func(t *testing.T) {
 		b := testSystemBackendRaw(t)
 
 		req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
@@ -1947,9 +1949,132 @@ func TestSystemBackend_rawRead_Compressed(t *testing.T) {
 		if err != nil {
 			t.Fatalf("err: %v", err)
 		}
-	if !strings.HasPrefix(resp.Data["value"].(string), "{\"type\":\"mounts\"") {
+		if !strings.HasPrefix(resp.Data["value"].(string), `{"type":"mounts"`) {
 			t.Fatalf("bad: %v", resp)
 		}
+	})
+
+	t.Run("base64", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"encoding": "base64",
+		}
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if _, ok := resp.Data["value"].([]byte); !ok {
+			t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"])
+		}
+
+		if !strings.HasPrefix(string(resp.Data["value"].([]byte)), `{"type":"mounts"`) {
+			t.Fatalf("bad: %v", resp)
+		}
+	})
+
+	t.Run("invalid_encoding", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"encoding": "invalid_encoding",
+		}
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != logical.ErrInvalidRequest {
+			t.Fatalf("err: %v", err)
+		}
+
+		if !resp.IsError() {
+			t.Fatalf("bad: %v", resp)
+		}
+	})
+
+	t.Run("compressed_false", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"compressed": false,
+		}
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if _, ok := resp.Data["value"].(string); !ok {
+			t.Fatalf("value is a not a string, it is %T", resp.Data["value"])
+		}
+
+		if !strings.HasPrefix(string(resp.Data["value"].(string)), string(compressutil.CompressionCanaryGzip)) {
+			t.Fatalf("bad: %v", resp)
+		}
+	})
+
+	t.Run("compressed_false_base64", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"compressed": false,
+			"encoding":   "base64",
+		}
+
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if _, ok := resp.Data["value"].([]byte); !ok {
+			t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"])
+		}
+
+		if !strings.HasPrefix(string(resp.Data["value"].([]byte)), string(compressutil.CompressionCanaryGzip)) {
+			t.Fatalf("bad: %v", resp)
+		}
+	})
+
+	t.Run("uncompressed_entry_with_prefix_byte", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+		req := logical.TestRequest(t, logical.CreateOperation, "raw/test_raw")
+		req.Data = map[string]interface{}{
+			"value": "414c1e7f-0a9a-49e0-9fc4-61af329d0724",
+		}
+
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		if resp != nil {
+			t.Fatalf("bad: %v", resp)
+		}
+
+		req = logical.TestRequest(t, logical.ReadOperation, "raw/test_raw")
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err == nil {
+			t.Fatalf("expected error if trying to read uncompressed entry with prefix byte")
+		}
+		if !resp.IsError() {
+			t.Fatalf("bad: %v", resp)
+		}
+
+		req = logical.TestRequest(t, logical.ReadOperation, "raw/test_raw")
+		req.Data = map[string]interface{}{
+			"compressed": false,
+		}
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		if resp.IsError() {
+			t.Fatalf("bad: %v", resp)
+		}
+		if resp.Data["value"].(string) != "414c1e7f-0a9a-49e0-9fc4-61af329d0724" {
+			t.Fatalf("bad: %v", resp)
+		}
+	})
 }
 
 func TestSystemBackend_rawRead_Protected(t *testing.T) {
@@ -1975,7 +2100,7 @@ func TestSystemBackend_rawWrite_Protected(t *testing.T) {
 func TestSystemBackend_rawReadWrite(t *testing.T) {
 	_, b, _ := testCoreSystemBackendRaw(t)
 
-	req := logical.TestRequest(t, logical.UpdateOperation, "raw/sys/policy/test")
+	req := logical.TestRequest(t, logical.CreateOperation, "raw/sys/policy/test")
 	req.Data["value"] = `path "secret/" { policy = "read" }`
 	resp, err := b.HandleRequest(namespace.RootContext(nil), req)
 	if err != nil {
@@ -1999,6 +2124,328 @@ func TestSystemBackend_rawReadWrite(t *testing.T) {
 	// simply parse this out directly via GetPolicy, so the test now ends here.
 }
 
+func TestSystemBackend_rawWrite_ExistanceCheck(t *testing.T) {
+	b := testSystemBackendRaw(t)
+	req := logical.TestRequest(t, logical.CreateOperation, "raw/core/mounts")
+	_, exist, err := b.HandleExistenceCheck(namespace.RootContext(nil), req)
+	if err != nil {
+		t.Fatalf("err: #{err}")
+	}
+	if !exist {
+		t.Fatalf("raw existence check failed for actual key")
+	}
+
+	req = logical.TestRequest(t, logical.CreateOperation, "raw/non_existent")
+	_, exist, err = b.HandleExistenceCheck(namespace.RootContext(nil), req)
+	if err != nil {
+		t.Fatalf("err: #{err}")
+	}
+	if exist {
+		t.Fatalf("raw existence check failed for non-existent key")
+	}
+}
+
+func TestSystemBackend_rawReadWrite_base64(t *testing.T) {
+	t.Run("basic", func(t *testing.T) {
+		_, b, _ := testCoreSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.CreateOperation, "raw/sys/policy/test")
+		req.Data = map[string]interface{}{
+			"value":    base64.StdEncoding.EncodeToString([]byte(`path "secret/" { policy = "read"[ }`)),
+			"encoding": "base64",
+		}
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		if resp != nil {
+			t.Fatalf("bad: %v", resp)
+		}
+
+		// Read via raw API
+		req = logical.TestRequest(t, logical.ReadOperation, "raw/sys/policy/test")
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		if !strings.HasPrefix(resp.Data["value"].(string), "path") {
+			t.Fatalf("bad: %v", resp)
+		}
+	})
+
+	t.Run("invalid_value", func(t *testing.T) {
+		_, b, _ := testCoreSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.CreateOperation, "raw/sys/policy/test")
+		req.Data = map[string]interface{}{
+			"value":    "invalid base64",
+			"encoding": "base64",
+		}
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err == nil {
+			t.Fatalf("no error")
+		}
+
+		if err != logical.ErrInvalidRequest {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		if !resp.IsError() {
+			t.Fatalf("response is not error: %v", resp)
+		}
+	})
+
+	t.Run("invalid_encoding", func(t *testing.T) {
+		_, b, _ := testCoreSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.CreateOperation, "raw/sys/policy/test")
+		req.Data = map[string]interface{}{
+			"value":    "text",
+			"encoding": "invalid_encoding",
+		}
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err == nil {
+			t.Fatalf("no error")
+		}
+
+		if err != logical.ErrInvalidRequest {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		if !resp.IsError() {
+			t.Fatalf("response is not error: %v", resp)
+		}
+	})
+}
+
+func TestSystemBackend_rawReadWrite_Compressed(t *testing.T) {
+	t.Run("use_existing_compression", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		mounts := resp.Data["value"].(string)
+		req = logical.TestRequest(t, logical.UpdateOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"value":            mounts,
+			"compression_type": compressutil.CompressionTypeGzip,
+		}
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		// Read back and check gzip was applied by looking for prefix byte
+		req = logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"compressed": false,
+			"encoding":   "base64",
+		}
+
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if _, ok := resp.Data["value"].([]byte); !ok {
+			t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"])
+		}
+
+		if !strings.HasPrefix(string(resp.Data["value"].([]byte)), string(compressutil.CompressionCanaryGzip)) {
+			t.Fatalf("bad: %v", resp)
+		}
+	})
+
+	t.Run("compression_type_matches_existing_compression", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		mounts := resp.Data["value"].(string)
+		req = logical.TestRequest(t, logical.UpdateOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"value": mounts,
+		}
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		// Read back and check gzip was applied by looking for prefix byte
+		req = logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"compressed": false,
+			"encoding":   "base64",
+		}
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if _, ok := resp.Data["value"].([]byte); !ok {
+			t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"])
+		}
+
+		if !strings.HasPrefix(string(resp.Data["value"].([]byte)), string(compressutil.CompressionCanaryGzip)) {
+			t.Fatalf("bad: %v", resp)
+		}
+	})
+
+	t.Run("write_uncompressed_over_existing_compressed", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		mounts := resp.Data["value"].(string)
+		req = logical.TestRequest(t, logical.UpdateOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"value":            mounts,
+			"compression_type": "",
+		}
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		// Read back and check gzip was not applied by looking for prefix byte
+		req = logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"compressed": false,
+			"encoding":   "base64",
+		}
+
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if _, ok := resp.Data["value"].([]byte); !ok {
+			t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"])
+		}
+
+		if !strings.HasPrefix(string(resp.Data["value"].([]byte)), `{"type":"mounts"`) {
+			t.Fatalf("bad: %v", resp)
+		}
+	})
+
+	t.Run("invalid_compression_type", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts")
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		mounts := resp.Data["value"].(string)
+		req = logical.TestRequest(t, logical.UpdateOperation, "raw/core/mounts")
+		req.Data = map[string]interface{}{
+			"value":            mounts,
+			"compression_type": "invalid_type",
+		}
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != logical.ErrInvalidRequest {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		if !resp.IsError() {
+			t.Fatalf("response is not error: %v", resp)
+		}
+	})
+
+	t.Run("update_non_existent_entry", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.UpdateOperation, "raw/non_existent")
+		req.Data = map[string]interface{}{
+			"value": "{}",
+		}
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != logical.ErrInvalidRequest {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		if !resp.IsError() {
+			t.Fatalf("response is not error: %v", resp)
+		}
+	})
+
+	t.Run("invalid_compression_over_existing_uncompressed_data", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.CreateOperation, "raw/test")
+		req.Data = map[string]interface{}{
+			"value": "{}",
+		}
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		if resp.IsError() {
+			t.Fatalf("response is an error: %v", resp)
+		}
+
+		req = logical.TestRequest(t, logical.UpdateOperation, "raw/test")
+		req.Data = map[string]interface{}{
+			"value":            "{}",
+			"compression_type": compressutil.CompressionTypeGzip,
+		}
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != logical.ErrInvalidRequest {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		if !resp.IsError() {
+			t.Fatalf("response is not error: %v", resp)
+		}
+	})
+
+	t.Run("wrong_compression_type_over_existing_compressed_data", func(t *testing.T) {
+		b := testSystemBackendRaw(t)
+
+		req := logical.TestRequest(t, logical.CreateOperation, "raw/test")
+		req.Data = map[string]interface{}{
+			"value":            "{}",
+			"compression_type": compressutil.CompressionTypeGzip,
+		}
+		resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		if resp.IsError() {
+			t.Fatalf("response is an error: %v", resp)
+		}
+
+		req = logical.TestRequest(t, logical.UpdateOperation, "raw/test")
+		req.Data = map[string]interface{}{
+			"value":            "{}",
+			"compression_type": compressutil.CompressionTypeSnappy,
+		}
+		resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+		if err != logical.ErrInvalidRequest {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		if !resp.IsError() {
+			t.Fatalf("response is not error: %v", resp)
+		}
+	})
+}
+
 func TestSystemBackend_rawDelete_Protected(t *testing.T) {
 	b := testSystemBackendRaw(t)
 

website/content/api-docs/system/raw.mdx

@@ -27,6 +27,11 @@ system.
 - `path` `(string: <required>)` – Specifies the raw path in the storage backend.
   This is specified as part of the URL.
 
+- `compressed` `(bool: true)` - Attempt to decompress the value.
+
+- `encoding` `(string: "")` - Specifies the encoding of the returned data. Defaults to no encoding.
+  "base64" returns the value encoded in base64.
+
 ### Sample Request
 
 ```shell-session
@@ -60,6 +65,13 @@ mount system.
 
 - `value` `(string: <required>)` – Specifies the value of the key.
 
+- `compression_type` `(string: "")` - Create/update using the compressed form of `value`. Supported `compression_type`
+  values are `gzip`, `lzw`, `lz4`, `snappy`, or `""`. `""` means no compression is used. If omitted and key already exists,
+  update uses the same compression (or no compression) as the existing value.
+
+- `encoding` `(string: "")` - Specifies the encoding of `value`. Defaults to no encoding.
+  Use "base64" if `value` is encoded in base64.
+
 ### Sample Payload
 
 ```json