Vault Raw Read Support (CLI & Client) (#14945)

* Expose raw request from client.Logical()

Not all Vault API endpoints return well-formatted JSON objects.
Sometimes, in the case of the PKI secrets engine, they're not even
printable (/pki/ca returns a binary (DER-encoded) certificate). While
this endpoint isn't authenticated, in general the API caller would
either need to use Client.RawRequestWithContext(...) directly (which
the docs advise against), or setup their own net/http client and
re-create much of Client and/or Client.Logical.

Instead, exposing the raw Request (via the new ReadRawWithData(...))
allows callers to directly consume these non-JSON endpoints like they
would nearly any other endpoint.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add raw formatter for direct []byte data

As mentioned in the previous commit, some API endpoints return non-JSON
data. We get as far as fetching this data (via client.Logical().Read),
but parsing it as an api.Secret fails (as in this case, it is non-JSON).
Given that we intend to update `vault read` to support such endpoints,
we'll need a "raw" formatter that accepts []byte-encoded data and simply
writes it to the UI.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add support for reading raw API endpoints

Some endpoints, such as `pki/ca` and `pki/ca/pem` return non-JSON
objects. When calling `vault read` on these endpoints, an error
is returned because they cannot be parsed as api.Secret instances:

> Error reading pki/ca/pem: invalid character '-' in numeric literal

Indeed, we go to all the trouble of (successfully) fetching this value,
only to be unable to Unmarshal into a Secrets value. Instead, add
support for a new -format=raw option, allowing these endpoints to be
consumed by callers of `vault read` directly.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add changelog entry

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Remove panic

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

api/logical.go

@@ -65,23 +65,7 @@ func (c *Logical) ReadWithDataWithContext(ctx context.Context, path string, data
 	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
 	defer cancelFunc()
 
-	r := c.c.NewRequest(http.MethodGet, "/v1/"+path)
+	resp, err := c.readRawWithDataWithContext(ctx, path, data)
-
-	var values url.Values
-	for k, v := range data {
-		if values == nil {
-			values = make(url.Values)
-		}
-		for _, val := range v {
-			values.Add(k, val)
-		}
-	}
-
-	if values != nil {
-		r.Params = values
-	}
-
-	resp, err := c.c.rawRequestWithContext(ctx, r)
 	if resp != nil {
 		defer resp.Body.Close()
 	}
@@ -106,6 +90,41 @@ func (c *Logical) ReadWithDataWithContext(ctx context.Context, path string, data
 	return ParseSecret(resp.Body)
 }
 
+func (c *Logical) ReadRaw(path string) (*Response, error) {
+	return c.ReadRawWithData(path, nil)
+}
+
+func (c *Logical) ReadRawWithData(path string, data map[string][]string) (*Response, error) {
+	return c.ReadRawWithDataWithContext(context.Background(), path, data)
+}
+
+func (c *Logical) ReadRawWithDataWithContext(ctx context.Context, path string, data map[string][]string) (*Response, error) {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	return c.readRawWithDataWithContext(ctx, path, data)
+}
+
+func (c *Logical) readRawWithDataWithContext(ctx context.Context, path string, data map[string][]string) (*Response, error) {
+	r := c.c.NewRequest(http.MethodGet, "/v1/"+path)
+
+	var values url.Values
+	for k, v := range data {
+		if values == nil {
+			values = make(url.Values)
+		}
+		for _, val := range v {
+			values.Add(k, val)
+		}
+	}
+
+	if values != nil {
+		r.Params = values
+	}
+
+	return c.c.RawRequestWithContext(ctx, r)
+}
+
 func (c *Logical) List(path string) (*Secret, error) {
 	return c.ListWithContext(context.Background(), path)
 }

changelog/14945.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Support the -format=raw option, to read non-JSON Vault endpoints and original response bodies.
+```

command/format.go

@@ -70,6 +70,7 @@ var Formatters = map[string]Formatter{
 	"yaml":   YamlFormatter{},
 	"yml":    YamlFormatter{},
 	"pretty": PrettyFormatter{},
+	"raw":    RawFormatter{},
 }
 
 func Format(ui cli.Ui) string {
@@ -125,6 +126,27 @@ func (j JsonFormatter) Output(ui cli.Ui, secret *api.Secret, data interface{}) e
 	return nil
 }
 
+// An output formatter for raw output of the original request object
+type RawFormatter struct{}
+
+func (r RawFormatter) Format(data interface{}) ([]byte, error) {
+	byte_data, ok := data.([]byte)
+	if !ok {
+		return nil, fmt.Errorf("unable to type assert to []byte: %T; please share the command run", data)
+	}
+
+	return byte_data, nil
+}
+
+func (r RawFormatter) Output(ui cli.Ui, secret *api.Secret, data interface{}) error {
+	b, err := r.Format(data)
+	if err != nil {
+		return err
+	}
+	ui.Output(string(b))
+	return nil
+}
+
 // An output formatter for yaml output format of an object
 type YamlFormatter struct{}
 

command/read.go

@@ -91,19 +91,40 @@ func (c *ReadCommand) Run(args []string) int {
 		return 1
 	}
 
-	secret, err := client.Logical().ReadWithData(path, data)
+	if Format(c.UI) != "raw" {
+		secret, err := client.Logical().ReadWithData(path, data)
+		if err != nil {
+			c.UI.Error(fmt.Sprintf("Error reading %s: %s", path, err))
+			return 2
+		}
+		if secret == nil {
+			c.UI.Error(fmt.Sprintf("No value found at %s", path))
+			return 2
+		}
+
+		if c.flagField != "" {
+			return PrintRawField(c.UI, secret, c.flagField)
+		}
+
+		return OutputSecret(c.UI, secret)
+	}
+
+	resp, err := client.Logical().ReadRawWithData(path, data)
 	if err != nil {
-		c.UI.Error(fmt.Sprintf("Error reading %s: %s", path, err))
+		c.UI.Error(fmt.Sprintf("Error reading: %s: %s", path, err))
 		return 2
 	}
-	if secret == nil {
+	if resp == nil || resp.Body == nil {
 		c.UI.Error(fmt.Sprintf("No value found at %s", path))
 		return 2
 	}
+	defer resp.Body.Close()
 
-	if c.flagField != "" {
+	contents, err := io.ReadAll(resp.Body)
-		return PrintRawField(c.UI, secret, c.flagField)
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error reading: %s: %s", path, err))
+		return 2
 	}
 
-	return OutputSecret(c.UI, secret)
+	return OutputData(c.UI, contents)
 }