From 194c9e32d30c9c76ee525c29bb08a14386104247 Mon Sep 17 00:00:00 2001 From: Sung Hon Wu Date: Thu, 20 Jan 2022 04:52:53 -0800 Subject: [PATCH] Enhance sys/raw to read and write values that cannot be encoded in json (#13537) --- changelog/13537.txt | 3 + sdk/helper/compressutil/compress.go | 37 +- sdk/helper/compressutil/compress_test.go | 9 + vault/logical_raw.go | 154 +++++++- vault/logical_system_test.go | 467 ++++++++++++++++++++++- website/content/api-docs/system/raw.mdx | 12 + 6 files changed, 648 insertions(+), 34 deletions(-) create mode 100644 changelog/13537.txt diff --git a/changelog/13537.txt b/changelog/13537.txt new file mode 100644 index 000000000..d1c31af93 --- /dev/null +++ b/changelog/13537.txt @@ -0,0 +1,3 @@ +```release-note:improvement +sys/raw: Enhance sys/raw to read and write values that cannot be encoded in json. +``` \ No newline at end of file diff --git a/sdk/helper/compressutil/compress.go b/sdk/helper/compressutil/compress.go index 356d4548f..4af0ac5dd 100644 --- a/sdk/helper/compressutil/compress.go +++ b/sdk/helper/compressutil/compress.go @@ -141,10 +141,21 @@ func Compress(data []byte, config *CompressionConfig) ([]byte, error) { // If the first byte isn't a canary byte, then the utility returns a boolean // value indicating that the input was not compressed. func Decompress(data []byte) ([]byte, bool, error) { + bytes, _, notCompressed, err := DecompressWithCanary(data) + return bytes, notCompressed, err +} + +// DecompressWithCanary checks if the first byte in the input matches the canary byte. +// If the first byte is a canary byte, then the input past the canary byte +// will be decompressed using the method specified in the given configuration. The type of compression used is also +// returned. If the first byte isn't a canary byte, then the utility returns a boolean +// value indicating that the input was not compressed. +func DecompressWithCanary(data []byte) ([]byte, string, bool, error) { var err error var reader io.ReadCloser + var compressionType string if data == nil || len(data) == 0 { - return nil, false, fmt.Errorf("'data' being decompressed is empty") + return nil, "", false, fmt.Errorf("'data' being decompressed is empty") } canary := data[0] @@ -155,43 +166,47 @@ func Decompress(data []byte) ([]byte, bool, error) { // byte and try to decompress the data that is after the canary. case CompressionCanaryGzip: if len(data) < 2 { - return nil, false, fmt.Errorf("invalid 'data' after the canary") + return nil, "", false, fmt.Errorf("invalid 'data' after the canary") } reader, err = gzip.NewReader(bytes.NewReader(cData)) + compressionType = CompressionTypeGzip case CompressionCanaryLZW: if len(data) < 2 { - return nil, false, fmt.Errorf("invalid 'data' after the canary") + return nil, "", false, fmt.Errorf("invalid 'data' after the canary") } reader = lzw.NewReader(bytes.NewReader(cData), lzw.LSB, 8) + compressionType = CompressionTypeLZW case CompressionCanarySnappy: if len(data) < 2 { - return nil, false, fmt.Errorf("invalid 'data' after the canary") + return nil, "", false, fmt.Errorf("invalid 'data' after the canary") } reader = &CompressUtilReadCloser{ Reader: snappy.NewReader(bytes.NewReader(cData)), } + compressionType = CompressionTypeSnappy case CompressionCanaryLZ4: if len(data) < 2 { - return nil, false, fmt.Errorf("invalid 'data' after the canary") + return nil, "", false, fmt.Errorf("invalid 'data' after the canary") } reader = &CompressUtilReadCloser{ Reader: lz4.NewReader(bytes.NewReader(cData)), } + compressionType = CompressionTypeLZ4 default: // If the first byte doesn't match the canary byte, it means // that the content was not compressed at all. Indicate the // caller that the input was not compressed. - return nil, true, nil + return nil, "", true, nil } if err != nil { - return nil, false, errwrap.Wrapf("failed to create a compression reader: {{err}}", err) + return nil, "", false, errwrap.Wrapf("failed to create a compression reader: {{err}}", err) } if reader == nil { - return nil, false, fmt.Errorf("failed to create a compression reader") + return nil, "", false, fmt.Errorf("failed to create a compression reader") } // Close the io.ReadCloser @@ -200,8 +215,8 @@ func Decompress(data []byte) ([]byte, bool, error) { // Read all the compressed data into a buffer var buf bytes.Buffer if _, err = io.Copy(&buf, reader); err != nil { - return nil, false, err + return nil, "", false, err } - return buf.Bytes(), false, nil -} + return buf.Bytes(), compressionType, false, nil +} \ No newline at end of file diff --git a/sdk/helper/compressutil/compress_test.go b/sdk/helper/compressutil/compress_test.go index 050c7ad8a..f85f3c935 100644 --- a/sdk/helper/compressutil/compress_test.go +++ b/sdk/helper/compressutil/compress_test.go @@ -86,6 +86,15 @@ func TestCompressUtil_CompressDecompress(t *testing.T) { if !bytes.Equal(inputJSONBytes, decompressedJSONBytes) { t.Fatalf("bad (%s): decompressed value;\nexpected: %q\nactual: %q", test.compressionType, string(inputJSONBytes), string(decompressedJSONBytes)) } + + decompressedJSONBytes, compressionType, wasNotCompressed, err := DecompressWithCanary(compressedJSONBytes) + if err != nil { + t.Fatalf("decompress error (%s): %s", test.compressionType, err) + } + + if compressionType != test.compressionConfig.Type { + t.Fatalf("bad compressionType value;\nexpected: %q\naction: %q", test.compressionConfig.Type, compressionType) + } } } diff --git a/vault/logical_raw.go b/vault/logical_raw.go index f845c9c48..3aa68bd1f 100644 --- a/vault/logical_raw.go +++ b/vault/logical_raw.go @@ -1,7 +1,9 @@ package vault import ( + "compress/gzip" "context" + "encoding/base64" "fmt" "strings" @@ -46,6 +48,17 @@ func NewRawBackend(core *Core) *RawBackend { func (b *RawBackend) handleRawRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { path := data.Get("path").(string) + // Preserve pre-existing behavior to decompress if `compressed` is missing + compressed := true + if d, ok := data.GetOk("compressed"); ok { + compressed = d.(bool) + } + + encoding := data.Get("encoding").(string) + if encoding != "" && encoding != "base64" { + return logical.ErrorResponse("invalid encoding '%s'", encoding), logical.ErrInvalidRequest + } + if b.recoveryMode { b.logger.Info("reading", "path", path) } @@ -72,23 +85,32 @@ func (b *RawBackend) handleRawRead(ctx context.Context, req *logical.Request, da return nil, nil } - // Run this through the decompression helper to see if it's been compressed. - // If the input contained the compression canary, `outputBytes` will hold - // the decompressed data. If the input was not compressed, then `outputBytes` - // will be nil. - outputBytes, _, err := compressutil.Decompress(entry.Value) - if err != nil { - return handleErrorNoReadOnlyForward(err) + valueBytes := entry.Value + if compressed { + // Run this through the decompression helper to see if it's been compressed. + // If the input contained the compression canary, `valueBytes` will hold + // the decompressed data. If the input was not compressed, then `valueBytes` + // will be nil. + valueBytes, _, err = compressutil.Decompress(entry.Value) + if err != nil { + return handleErrorNoReadOnlyForward(err) + } + + // `valueBytes` is nil if the input is uncompressed. In that case set it to the original input. + if valueBytes == nil { + valueBytes = entry.Value + } } - // `outputBytes` is nil if the input is uncompressed. In that case set it to the original input. - if outputBytes == nil { - outputBytes = entry.Value + var value interface{} = string(valueBytes) + // Golang docs (https://pkg.go.dev/encoding/json#Marshal), []byte encodes as a base64-encoded string + if encoding == "base64" { + value = valueBytes } resp := &logical.Response{ Data: map[string]interface{}{ - "value": string(outputBytes), + "value": value, }, } return resp, nil @@ -97,6 +119,16 @@ func (b *RawBackend) handleRawRead(ctx context.Context, req *logical.Request, da // handleRawWrite is used to write directly to the barrier func (b *RawBackend) handleRawWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { path := data.Get("path").(string) + compressionType := "" + c, compressionTypeOk := data.GetOk("compression_type") + if compressionTypeOk { + compressionType = c.(string) + } + + encoding := data.Get("encoding").(string) + if encoding != "" && encoding != "base64" { + return logical.ErrorResponse("invalid encoding '%s'", encoding), logical.ErrInvalidRequest + } if b.recoveryMode { b.logger.Info("writing", "path", path) @@ -110,11 +142,83 @@ func (b *RawBackend) handleRawWrite(ctx context.Context, req *logical.Request, d } } - value := data.Get("value").(string) + v := data.Get("value").(string) + value := []byte(v) + if encoding == "base64" { + var err error + value, err = base64.StdEncoding.DecodeString(v) + if err != nil { + return logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest + } + } + + if req.Operation == logical.UpdateOperation { + // Check if this is an existing value with compression applied, if so, use the same compression (or no compression) + entry, err := b.barrier.Get(ctx, path) + if err != nil { + return handleErrorNoReadOnlyForward(err) + } + if entry == nil { + err := fmt.Sprintf("cannot figure out compression type because entry does not exist") + return logical.ErrorResponse(err), logical.ErrInvalidRequest + } + + // For cases where DecompressWithCanary errored, treat entry as non-compressed data. + _, existingCompressionType, _, _ := compressutil.DecompressWithCanary(entry.Value) + + // Ensure compression_type matches existing entries' compression + // except allow writing non-compressed data over compressed data + if existingCompressionType != compressionType && compressionType != "" { + err := fmt.Sprintf("the entry uses a different compression scheme then compression_type") + return logical.ErrorResponse(err), logical.ErrInvalidRequest + } + + if !compressionTypeOk { + compressionType = existingCompressionType + } + } + + if compressionType != "" { + var config *compressutil.CompressionConfig + switch compressionType { + case compressutil.CompressionTypeLZ4: + config = &compressutil.CompressionConfig{ + Type: compressutil.CompressionTypeLZ4, + } + break + case compressutil.CompressionTypeLZW: + config = &compressutil.CompressionConfig{ + Type: compressutil.CompressionTypeLZW, + } + break + case compressutil.CompressionTypeGzip: + config = &compressutil.CompressionConfig{ + Type: compressutil.CompressionTypeGzip, + GzipCompressionLevel: gzip.BestCompression, + } + break + case compressutil.CompressionTypeSnappy: + config = &compressutil.CompressionConfig{ + Type: compressutil.CompressionTypeSnappy, + } + break + default: + err := fmt.Sprintf("invalid compression type '%s'", compressionType) + return logical.ErrorResponse(err), logical.ErrInvalidRequest + } + + var err error + value, err = compressutil.Compress(value, config) + if err != nil { + return logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest + } + } + entry := &logical.StorageEntry{ Key: path, - Value: []byte(value), + Value: value, } + if err := b.barrier.Put(ctx, entry); err != nil { return logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest } @@ -175,6 +279,16 @@ func (b *RawBackend) handleRawList(ctx context.Context, req *logical.Request, da return logical.ListResponse(keys), nil } +// existenceCheck checks if entry exists, used in handleRawWrite for update or create operations +func (b *RawBackend) existenceCheck(ctx context.Context, request *logical.Request, data *framework.FieldData) (bool, error) { + path := data.Get("path").(string) + entry, err := b.barrier.Get(ctx, path) + if err != nil { + return false, err + } + return entry != nil, nil +} + func rawPaths(prefix string, r *RawBackend) []*framework.Path { return []*framework.Path{ { @@ -187,6 +301,15 @@ func rawPaths(prefix string, r *RawBackend) []*framework.Path { "value": { Type: framework.TypeString, }, + "compressed": { + Type: framework.TypeBool, + }, + "encoding": { + Type: framework.TypeString, + }, + "compression_type": { + Type: framework.TypeString, + }, }, Operations: map[logical.Operation]framework.OperationHandler{ @@ -198,6 +321,10 @@ func rawPaths(prefix string, r *RawBackend) []*framework.Path { Callback: r.handleRawWrite, Summary: "Update the value of the key at the given path.", }, + logical.CreateOperation: &framework.PathOperation{ + Callback: r.handleRawWrite, + Summary: "Create a key with value at the given path.", + }, logical.DeleteOperation: &framework.PathOperation{ Callback: r.handleRawDelete, Summary: "Delete the key with given path.", @@ -208,6 +335,7 @@ func rawPaths(prefix string, r *RawBackend) []*framework.Path { }, }, + ExistenceCheck: r.existenceCheck, HelpSynopsis: strings.TrimSpace(sysHelp["raw"][0]), HelpDescription: strings.TrimSpace(sysHelp["raw"][1]), }, diff --git a/vault/logical_system_test.go b/vault/logical_system_test.go index 5de28dd36..f46f39510 100644 --- a/vault/logical_system_test.go +++ b/vault/logical_system_test.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/helper/random" "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/helper/compressutil" "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/helper/jsonutil" "github.com/hashicorp/vault/sdk/helper/salt" @@ -1940,16 +1941,140 @@ func TestSystemBackend_disableAudit(t *testing.T) { } func TestSystemBackend_rawRead_Compressed(t *testing.T) { - b := testSystemBackendRaw(t) + t.Run("basic", func(t *testing.T) { + b := testSystemBackendRaw(t) - req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") - resp, err := b.HandleRequest(namespace.RootContext(nil), req) - if err != nil { - t.Fatalf("err: %v", err) - } - if !strings.HasPrefix(resp.Data["value"].(string), "{\"type\":\"mounts\"") { - t.Fatalf("bad: %v", resp) - } + req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + if !strings.HasPrefix(resp.Data["value"].(string), `{"type":"mounts"`) { + t.Fatalf("bad: %v", resp) + } + }) + + t.Run("base64", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "encoding": "base64", + } + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + if _, ok := resp.Data["value"].([]byte); !ok { + t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"]) + } + + if !strings.HasPrefix(string(resp.Data["value"].([]byte)), `{"type":"mounts"`) { + t.Fatalf("bad: %v", resp) + } + }) + + t.Run("invalid_encoding", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "encoding": "invalid_encoding", + } + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != logical.ErrInvalidRequest { + t.Fatalf("err: %v", err) + } + + if !resp.IsError() { + t.Fatalf("bad: %v", resp) + } + }) + + t.Run("compressed_false", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "compressed": false, + } + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + if _, ok := resp.Data["value"].(string); !ok { + t.Fatalf("value is a not a string, it is %T", resp.Data["value"]) + } + + if !strings.HasPrefix(string(resp.Data["value"].(string)), string(compressutil.CompressionCanaryGzip)) { + t.Fatalf("bad: %v", resp) + } + }) + + t.Run("compressed_false_base64", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "compressed": false, + "encoding": "base64", + } + + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + if _, ok := resp.Data["value"].([]byte); !ok { + t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"]) + } + + if !strings.HasPrefix(string(resp.Data["value"].([]byte)), string(compressutil.CompressionCanaryGzip)) { + t.Fatalf("bad: %v", resp) + } + }) + + t.Run("uncompressed_entry_with_prefix_byte", func(t *testing.T) { + b := testSystemBackendRaw(t) + req := logical.TestRequest(t, logical.CreateOperation, "raw/test_raw") + req.Data = map[string]interface{}{ + "value": "414c1e7f-0a9a-49e0-9fc4-61af329d0724", + } + + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + if resp != nil { + t.Fatalf("bad: %v", resp) + } + + req = logical.TestRequest(t, logical.ReadOperation, "raw/test_raw") + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err == nil { + t.Fatalf("expected error if trying to read uncompressed entry with prefix byte") + } + if !resp.IsError() { + t.Fatalf("bad: %v", resp) + } + + req = logical.TestRequest(t, logical.ReadOperation, "raw/test_raw") + req.Data = map[string]interface{}{ + "compressed": false, + } + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + if resp.IsError() { + t.Fatalf("bad: %v", resp) + } + if resp.Data["value"].(string) != "414c1e7f-0a9a-49e0-9fc4-61af329d0724" { + t.Fatalf("bad: %v", resp) + } + }) } func TestSystemBackend_rawRead_Protected(t *testing.T) { @@ -1975,7 +2100,7 @@ func TestSystemBackend_rawWrite_Protected(t *testing.T) { func TestSystemBackend_rawReadWrite(t *testing.T) { _, b, _ := testCoreSystemBackendRaw(t) - req := logical.TestRequest(t, logical.UpdateOperation, "raw/sys/policy/test") + req := logical.TestRequest(t, logical.CreateOperation, "raw/sys/policy/test") req.Data["value"] = `path "secret/" { policy = "read" }` resp, err := b.HandleRequest(namespace.RootContext(nil), req) if err != nil { @@ -1999,6 +2124,328 @@ func TestSystemBackend_rawReadWrite(t *testing.T) { // simply parse this out directly via GetPolicy, so the test now ends here. } +func TestSystemBackend_rawWrite_ExistanceCheck(t *testing.T) { + b := testSystemBackendRaw(t) + req := logical.TestRequest(t, logical.CreateOperation, "raw/core/mounts") + _, exist, err := b.HandleExistenceCheck(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: #{err}") + } + if !exist { + t.Fatalf("raw existence check failed for actual key") + } + + req = logical.TestRequest(t, logical.CreateOperation, "raw/non_existent") + _, exist, err = b.HandleExistenceCheck(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: #{err}") + } + if exist { + t.Fatalf("raw existence check failed for non-existent key") + } +} + +func TestSystemBackend_rawReadWrite_base64(t *testing.T) { + t.Run("basic", func(t *testing.T) { + _, b, _ := testCoreSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.CreateOperation, "raw/sys/policy/test") + req.Data = map[string]interface{}{ + "value": base64.StdEncoding.EncodeToString([]byte(`path "secret/" { policy = "read"[ }`)), + "encoding": "base64", + } + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + if resp != nil { + t.Fatalf("bad: %v", resp) + } + + // Read via raw API + req = logical.TestRequest(t, logical.ReadOperation, "raw/sys/policy/test") + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + if !strings.HasPrefix(resp.Data["value"].(string), "path") { + t.Fatalf("bad: %v", resp) + } + }) + + t.Run("invalid_value", func(t *testing.T) { + _, b, _ := testCoreSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.CreateOperation, "raw/sys/policy/test") + req.Data = map[string]interface{}{ + "value": "invalid base64", + "encoding": "base64", + } + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err == nil { + t.Fatalf("no error") + } + + if err != logical.ErrInvalidRequest { + t.Fatalf("unexpected error: %v", err) + } + + if !resp.IsError() { + t.Fatalf("response is not error: %v", resp) + } + }) + + t.Run("invalid_encoding", func(t *testing.T) { + _, b, _ := testCoreSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.CreateOperation, "raw/sys/policy/test") + req.Data = map[string]interface{}{ + "value": "text", + "encoding": "invalid_encoding", + } + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err == nil { + t.Fatalf("no error") + } + + if err != logical.ErrInvalidRequest { + t.Fatalf("unexpected error: %v", err) + } + + if !resp.IsError() { + t.Fatalf("response is not error: %v", resp) + } + }) +} + +func TestSystemBackend_rawReadWrite_Compressed(t *testing.T) { + t.Run("use_existing_compression", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + mounts := resp.Data["value"].(string) + req = logical.TestRequest(t, logical.UpdateOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "value": mounts, + "compression_type": compressutil.CompressionTypeGzip, + } + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Read back and check gzip was applied by looking for prefix byte + req = logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "compressed": false, + "encoding": "base64", + } + + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + if _, ok := resp.Data["value"].([]byte); !ok { + t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"]) + } + + if !strings.HasPrefix(string(resp.Data["value"].([]byte)), string(compressutil.CompressionCanaryGzip)) { + t.Fatalf("bad: %v", resp) + } + }) + + t.Run("compression_type_matches_existing_compression", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + mounts := resp.Data["value"].(string) + req = logical.TestRequest(t, logical.UpdateOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "value": mounts, + } + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Read back and check gzip was applied by looking for prefix byte + req = logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "compressed": false, + "encoding": "base64", + } + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + if _, ok := resp.Data["value"].([]byte); !ok { + t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"]) + } + + if !strings.HasPrefix(string(resp.Data["value"].([]byte)), string(compressutil.CompressionCanaryGzip)) { + t.Fatalf("bad: %v", resp) + } + }) + + t.Run("write_uncompressed_over_existing_compressed", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + mounts := resp.Data["value"].(string) + req = logical.TestRequest(t, logical.UpdateOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "value": mounts, + "compression_type": "", + } + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Read back and check gzip was not applied by looking for prefix byte + req = logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "compressed": false, + "encoding": "base64", + } + + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + if _, ok := resp.Data["value"].([]byte); !ok { + t.Fatalf("value is a not an array of bytes, it is %T", resp.Data["value"]) + } + + if !strings.HasPrefix(string(resp.Data["value"].([]byte)), `{"type":"mounts"`) { + t.Fatalf("bad: %v", resp) + } + }) + + t.Run("invalid_compression_type", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.ReadOperation, "raw/core/mounts") + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + mounts := resp.Data["value"].(string) + req = logical.TestRequest(t, logical.UpdateOperation, "raw/core/mounts") + req.Data = map[string]interface{}{ + "value": mounts, + "compression_type": "invalid_type", + } + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != logical.ErrInvalidRequest { + t.Fatalf("unexpected error: %v", err) + } + + if !resp.IsError() { + t.Fatalf("response is not error: %v", resp) + } + }) + + t.Run("update_non_existent_entry", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.UpdateOperation, "raw/non_existent") + req.Data = map[string]interface{}{ + "value": "{}", + } + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != logical.ErrInvalidRequest { + t.Fatalf("unexpected error: %v", err) + } + + if !resp.IsError() { + t.Fatalf("response is not error: %v", resp) + } + }) + + t.Run("invalid_compression_over_existing_uncompressed_data", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.CreateOperation, "raw/test") + req.Data = map[string]interface{}{ + "value": "{}", + } + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if resp.IsError() { + t.Fatalf("response is an error: %v", resp) + } + + req = logical.TestRequest(t, logical.UpdateOperation, "raw/test") + req.Data = map[string]interface{}{ + "value": "{}", + "compression_type": compressutil.CompressionTypeGzip, + } + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != logical.ErrInvalidRequest { + t.Fatalf("unexpected error: %v", err) + } + + if !resp.IsError() { + t.Fatalf("response is not error: %v", resp) + } + }) + + t.Run("wrong_compression_type_over_existing_compressed_data", func(t *testing.T) { + b := testSystemBackendRaw(t) + + req := logical.TestRequest(t, logical.CreateOperation, "raw/test") + req.Data = map[string]interface{}{ + "value": "{}", + "compression_type": compressutil.CompressionTypeGzip, + } + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if resp.IsError() { + t.Fatalf("response is an error: %v", resp) + } + + req = logical.TestRequest(t, logical.UpdateOperation, "raw/test") + req.Data = map[string]interface{}{ + "value": "{}", + "compression_type": compressutil.CompressionTypeSnappy, + } + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != logical.ErrInvalidRequest { + t.Fatalf("unexpected error: %v", err) + } + + if !resp.IsError() { + t.Fatalf("response is not error: %v", resp) + } + }) +} + func TestSystemBackend_rawDelete_Protected(t *testing.T) { b := testSystemBackendRaw(t) diff --git a/website/content/api-docs/system/raw.mdx b/website/content/api-docs/system/raw.mdx index 418e5589d..44005cb61 100644 --- a/website/content/api-docs/system/raw.mdx +++ b/website/content/api-docs/system/raw.mdx @@ -27,6 +27,11 @@ system. - `path` `(string: )` – Specifies the raw path in the storage backend. This is specified as part of the URL. +- `compressed` `(bool: true)` - Attempt to decompress the value. + +- `encoding` `(string: "")` - Specifies the encoding of the returned data. Defaults to no encoding. + "base64" returns the value encoded in base64. + ### Sample Request ```shell-session @@ -60,6 +65,13 @@ mount system. - `value` `(string: )` – Specifies the value of the key. +- `compression_type` `(string: "")` - Create/update using the compressed form of `value`. Supported `compression_type` + values are `gzip`, `lzw`, `lz4`, `snappy`, or `""`. `""` means no compression is used. If omitted and key already exists, + update uses the same compression (or no compression) as the existing value. + +- `encoding` `(string: "")` - Specifies the encoding of `value`. Defaults to no encoding. + Use "base64" if `value` is encoded in base64. + ### Sample Payload ```json