Merge pull request #1650 from hashicorp/request-uuid

Added unique identifier to each request. Closes hashicorp/vault#1617

api/secret.go

@@ -9,6 +9,9 @@ import (
 
 // Secret is the structure returned for every secret within Vault.
 type Secret struct {
+	// The request ID that generated this response
+	RequestID string `json:"request_id"`
+
 	LeaseID       string `json:"lease_id"`
 	LeaseDuration int    `json:"lease_duration"`
 	Renewable     bool   `json:"renewable"`

audit/format_json.go

@@ -42,6 +42,7 @@ func (f *FormatJSON) FormatRequest(
 
 		Request: JSONRequest{
 			ClientToken: req.ClientToken,
+			ID:          req.ID,
 			Operation:   req.Operation,
 			Path:        req.Path,
 			Data:        req.Data,
@@ -112,6 +113,7 @@ func (f *FormatJSON) FormatResponse(
 
 		Request: JSONRequest{
 			ClientToken: req.ClientToken,
+			ID:          req.ID,
 			Operation:   req.Operation,
 			Path:        req.Path,
 			Data:        req.Data,
@@ -149,6 +151,7 @@ type JSONResponseEntry struct {
 }
 
 type JSONRequest struct {
+	ID          string                 `json:"id"`
 	Operation   logical.Operation      `json:"operation"`
 	ClientToken string                 `json:"client_token"`
 	Path        string                 `json:"path"`

http/logical.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/vault"
 )
@@ -65,7 +66,13 @@ func buildLogicalRequest(w http.ResponseWriter, r *http.Request) (*logical.Reque
 	}
 
 	var err error
+	request_id, err := uuid.GenerateUUID()
+	if err != nil {
+		return nil, http.StatusBadRequest, errwrap.Wrapf("failed to generate identifier for the request: {{err}}", err)
+	}
+
 	req := requestAuth(r, &logical.Request{
+		ID:         request_id,
 		Operation:  op,
 		Path:       path,
 		Data:       data,
@@ -135,11 +142,11 @@ func handleLogical(core *vault.Core, dataOnly bool, prepareRequestCallback Prepa
 		}
 
 		// Build the proper response
-		respondLogical(w, r, req.Path, dataOnly, resp)
+		respondLogical(w, r, req, dataOnly, resp)
 	})
 }
 
-func respondLogical(w http.ResponseWriter, r *http.Request, path string, dataOnly bool, resp *logical.Response) {
+func respondLogical(w http.ResponseWriter, r *http.Request, req *logical.Request, dataOnly bool, resp *logical.Response) {
 	var httpResp interface{}
 	if resp != nil {
 		if resp.Redirect != "" {
@@ -156,7 +163,7 @@ func respondLogical(w http.ResponseWriter, r *http.Request, path string, dataOnl
 
 		// Check if this is a raw response
 		if _, ok := resp.Data[logical.HTTPContentType]; ok {
-			respondRaw(w, r, path, resp)
+			respondRaw(w, r, req.Path, resp)
 			return
 		}
 
@@ -170,7 +177,9 @@ func respondLogical(w http.ResponseWriter, r *http.Request, path string, dataOnl
 				},
 			}
 		} else {
-			httpResp = logical.SanitizeResponse(resp)
+			sanitizedHttp := logical.SanitizeResponse(resp)
+			sanitizedHttp.RequestID = req.ID
+			httpResp = sanitizedHttp
 		}
 	}
 

http/logical_test.go

@@ -52,6 +52,7 @@ func TestLogical(t *testing.T) {
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)
 	delete(actual, "lease_id")
+	expected["request_id"] = actual["request_id"]
 	if !reflect.DeepEqual(actual, expected) {
 		t.Fatalf("bad:\nactual:\n%#v\nexpected:\n%#v", actual, expected)
 	}
@@ -157,6 +158,7 @@ func TestLogical_StandbyRedirect(t *testing.T) {
 	delete(actualDataMap, "creation_time")
 	delete(actualDataMap, "accessor")
 	actual["data"] = actualDataMap
+	expected["request_id"] = actual["request_id"]
 	delete(actual, "lease_id")
 	if !reflect.DeepEqual(actual, expected) {
 		t.Fatalf("bad: got %#v; expected %#v", actual, expected)
@@ -198,6 +200,7 @@ func TestLogical_CreateToken(t *testing.T) {
 	testResponseBody(t, resp, &actual)
 	delete(actual["auth"].(map[string]interface{}), "client_token")
 	delete(actual["auth"].(map[string]interface{}), "accessor")
+	expected["request_id"] = actual["request_id"]
 	if !reflect.DeepEqual(actual, expected) {
 		t.Fatalf("bad:\nexpected:\n%#v\nactual:\n%#v", expected, actual)
 	}

logical/request.go

@@ -10,53 +10,56 @@ import (
 // of a request being made to Vault. It is used to abstract
 // the details of the higher level request protocol from the handlers.
 type Request struct {
+	// Id is the uuid associated with each request
+	ID string `json:"id" structs:"id" mapstructure:"id"`
+
 	// Operation is the requested operation type
-	Operation Operation
+	Operation Operation `json:"operation" structs:"operation" mapstructure:"operation"`
 
 	// Path is the part of the request path not consumed by the
 	// routing. As an example, if the original request path is "prod/aws/foo"
 	// and the AWS logical backend is mounted at "prod/aws/", then the
 	// final path is "foo" since the mount prefix is trimmed.
-	Path string
+	Path string `json:"path" structs:"path" mapstructure:"path"`
 
 	// Request data is an opaque map that must have string keys.
-	Data map[string]interface{}
+	Data map[string]interface{} `json:"map" structs:"data" mapstructure:"data"`
 
 	// Storage can be used to durably store and retrieve state.
-	Storage Storage
+	Storage Storage `json:"storage" structs:"storage" mapstructure:"storage"`
 
 	// Secret will be non-nil only for Revoke and Renew operations
 	// to represent the secret that was returned prior.
-	Secret *Secret
+	Secret *Secret `json:"secret" structs:"secret" mapstructure:"secret"`
 
 	// Auth will be non-nil only for Renew operations
 	// to represent the auth that was returned prior.
-	Auth *Auth
+	Auth *Auth `json:"auth" structs:"auth" mapstructure:"auth"`
 
 	// Connection will be non-nil only for credential providers to
 	// inspect the connection information and potentially use it for
 	// authentication/protection.
-	Connection *Connection
+	Connection *Connection `json:"connection" structs:"connection" mapstructure:"connection"`
 
 	// ClientToken is provided to the core so that the identity
 	// can be verified and ACLs applied. This value is passed
 	// through to the logical backends but after being salted and
 	// hashed.
-	ClientToken string
+	ClientToken string `json:"client_token" structs:"client_token" mapstructure:"client_token"`
 
 	// DisplayName is provided to the logical backend to help associate
 	// dynamic secrets with the source entity. This is not a sensitive
 	// name, but is useful for operators.
-	DisplayName string
+	DisplayName string `json:"display_name" structs:"display_name" mapstructure:"display_name"`
 
 	// MountPoint is provided so that a logical backend can generate
 	// paths relative to itself. The `Path` is effectively the client
 	// request path with the MountPoint trimmed off.
-	MountPoint string
+	MountPoint string `json:"mount_point" structs:"mount_point" mapstructure:"mount_point"`
 
 	// WrapTTL contains the requested TTL of the token used to wrap the
 	// response in a cubbyhole.
-	WrapTTL time.Duration
+	WrapTTL time.Duration `json:"wrap_ttl" struct:"wrap_ttl" mapstructure:"wrap_ttl"`
 }
 
 // Get returns a data field and guards for nil Data

logical/response.go

@@ -31,51 +31,51 @@ const (
 type WrapInfo struct {
 	// Setting to non-zero specifies that the response should be wrapped.
 	// Specifies the desired TTL of the wrapping token.
-	TTL time.Duration
+	TTL time.Duration `json:"ttl" structs:"ttl" mapstructure:"ttl"`
 
 	// The token containing the wrapped response
-	Token string
+	Token string `json:"token" structs:"token" mapstructure:"token"`
 
 	// The creation time. This can be used with the TTL to figure out an
 	// expected expiration.
-	CreationTime time.Time
+	CreationTime time.Time `json:"creation_time" structs:"creation_time" mapstructure:"cration_time"`
 
 	// If the contained response is the output of a token creation call, the
 	// created token's accessor will be accessible here
-	WrappedAccessor string
+	WrappedAccessor string `json:"wrapped_accessor" structs:"wrapped_accessor" mapstructure:"wrapped_accessor"`
 }
 
 // Response is a struct that stores the response of a request.
 // It is used to abstract the details of the higher level request protocol.
 type Response struct {
 	// Secret, if not nil, denotes that this response represents a secret.
-	Secret *Secret
+	Secret *Secret `json:"secret" structs:"secret" mapstructure:"secret"`
 
 	// Auth, if not nil, contains the authentication information for
 	// this response. This is only checked and means something for
 	// credential backends.
-	Auth *Auth
+	Auth *Auth `json:"auth" structs:"auth" mapstructure:"auth"`
 
 	// Response data is an opaque map that must have string keys. For
 	// secrets, this data is sent down to the user as-is. To store internal
 	// data that you don't want the user to see, store it in
 	// Secret.InternalData.
-	Data map[string]interface{}
+	Data map[string]interface{} `json:"data" structs:"data" mapstructure:"data"`
 
 	// Redirect is an HTTP URL to redirect to for further authentication.
 	// This is only valid for credential backends. This will be blanked
 	// for any logical backend and ignored.
-	Redirect string
+	Redirect string `json:"redirect" structs:"redirect" mapstructure:"redirect"`
 
 	// Warnings allow operations or backends to return warnings in response
 	// to user actions without failing the action outright.
 	// Making it private helps ensure that it is easy for various parts of
 	// Vault (backend, core, etc.) to add warnings without accidentally
 	// replacing what exists.
-	warnings []string
+	warnings []string `json:"warnings" structs:"warnings" mapstructure:"warnings"`
 
 	// Information for wrapping the response in a cubbyhole
-	WrapInfo *WrapInfo
+	WrapInfo *WrapInfo `json:"wrap_info" structs:"wrap_info" mapstructure:"wrap_info"`
 }
 
 func init() {

logical/sanitize.go

@@ -35,6 +35,7 @@ func SanitizeResponse(input *Response) *HTTPResponse {
 }
 
 type HTTPResponse struct {
+	RequestID     string                 `json:"request_id"`
 	LeaseID       string                 `json:"lease_id"`
 	Renewable     bool                   `json:"renewable"`
 	LeaseDuration int                    `json:"lease_duration"`

vault/audit.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/armon/go-metrics"
+	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/audit"
 	"github.com/hashicorp/vault/helper/jsonutil"
@@ -351,17 +352,24 @@ func (a *AuditBroker) GetHash(name string, input string) (string, error) {
 
 // LogRequest is used to ensure all the audit backends have an opportunity to
 // log the given request and that *at least one* succeeds.
-func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) (reterr error) {
+func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) (retErr error) {
 	defer metrics.MeasureSince([]string{"audit", "log_request"}, time.Now())
 	a.l.RLock()
 	defer a.l.RUnlock()
 	defer func() {
 		if r := recover(); r != nil {
 			a.logger.Printf("[ERR] audit: panic logging: req path: %s", req.Path)
-			reterr = fmt.Errorf("panic generating audit log")
+			retErr = multierror.Append(retErr, fmt.Errorf("panic generating audit log"))
 		}
 	}()
 
+	// All logged requests must have an identifier
+	//if req.ID == "" {
+	//	a.logger.Printf("[ERR] audit: missing identifier in request object: %s", req.Path)
+	//	retErr = multierror.Append(retErr, fmt.Errorf("missing identifier in request object: %s", req.Path))
+	//	return
+	//}
+
 	// Ensure at least one backend logs
 	anyLogged := false
 	for name, be := range a.backends {
@@ -375,7 +383,8 @@ func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outer
 		}
 	}
 	if !anyLogged && len(a.backends) > 0 {
-		return fmt.Errorf("no audit backend succeeded in logging the request")
+		retErr = multierror.Append(retErr, fmt.Errorf("no audit backend succeeded in logging the request"))
+		return
 	}
 	return nil
 }

vault/audit_test.go

@@ -10,6 +10,8 @@ import (
 
 	"errors"
 
+	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/audit"
 	"github.com/hashicorp/vault/logical"
 )
@@ -223,9 +225,17 @@ func TestAuditBroker_LogRequest(t *testing.T) {
 		Operation: logical.ReadOperation,
 		Path:      "sys/mounts",
 	}
+
+	// Create an identifier for the request to verify against
+	var err error
+	req.ID, err = uuid.GenerateUUID()
+	if err != nil {
+		t.Fatalf("failed to generate identifier for the request: path%s err: %v", req.Path, err)
+	}
+
 	reqErrs := errors.New("errs")
 
-	err := b.LogRequest(auth, req, reqErrs)
+	err = b.LogRequest(auth, req, reqErrs)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -250,7 +260,7 @@ func TestAuditBroker_LogRequest(t *testing.T) {
 
 	// Should FAIL work with both failing backends
 	a2.ReqErr = fmt.Errorf("failed")
-	if err := b.LogRequest(auth, req, nil); err.Error() != "no audit backend succeeded in logging the request" {
+	if err := b.LogRequest(auth, req, nil); !errwrap.Contains(err, "no audit backend succeeded in logging the request") {
 		t.Fatalf("err: %v", err)
 	}
 }

vault/core_test.go

@@ -1153,6 +1153,12 @@ func TestCore_StepDown(t *testing.T) {
 		Path:        "sys/step-down",
 	}
 
+	// Create an identifier for the request
+	req.ID, err = uuid.GenerateUUID()
+	if err != nil {
+		t.Fatalf("failed to generate identifier for the request: path: %s err: %v", req.Path, err)
+	}
+
 	// Step down core
 	err = core.StepDown(req)
 	if err != nil {

vault/request_handling.go

@@ -411,6 +411,9 @@ func (c *Core) wrapInCubbyhole(req *logical.Request, resp *logical.Response) (*l
 
 	httpResponse := logical.SanitizeResponse(resp)
 
+	// Add the unique identifier of the original request to the response
+	httpResponse.RequestID = req.ID
+
 	// Because of the way that JSON encodes (likely just in Go) we actually get
 	// mixed-up values for ints if we simply put this object in the response
 	// and encode the whole thing; so instead we marshal it first, then store

vault/router.go

@@ -248,11 +248,15 @@ func (r *Router) routeCommon(req *logical.Request, existenceCheck bool) (*logica
 	// Cache the pointer to the original connection object
 	originalConn := req.Connection
 
+	// Cache the identifier of the request
+	originalReqID := req.ID
+
 	// Reset the request before returning
 	defer func() {
 		req.Path = original
 		req.MountPoint = ""
 		req.Connection = originalConn
+		req.ID = originalReqID
 		req.Storage = nil
 		req.ClientToken = clientToken
 	}()

website/source/docs/audit/index.html.md

@@ -21,9 +21,10 @@ but also a second copy in case the first is tampered with.
 ## Sensitive Information
 
 The audit logs contain the full request and response objects for every
-interaction with Vault. The data in the request and the data in the
-response (including secrets and authentication tokens) will be hashed
-with a salt using HMAC-SHA256.
+interaction with Vault. The request and response can be matched utilizing a
+unique identifier assigned to each request. The data in the request and the
+data in the response (including secrets and authentication tokens) will be
+hashed with a salt using HMAC-SHA256.
 
 The purpose of the hash is so that secrets aren't in plaintext within your
 audit logs. However, you're still able to check the value of secrets by