diff --git a/api/secret.go b/api/secret.go index 8ac70ba25..14924f9d0 100644 --- a/api/secret.go +++ b/api/secret.go @@ -9,6 +9,9 @@ import ( // Secret is the structure returned for every secret within Vault. type Secret struct { + // The request ID that generated this response + RequestID string `json:"request_id"` + LeaseID string `json:"lease_id"` LeaseDuration int `json:"lease_duration"` Renewable bool `json:"renewable"` diff --git a/audit/format_json.go b/audit/format_json.go index bb60a882e..cd76f86e6 100644 --- a/audit/format_json.go +++ b/audit/format_json.go @@ -42,6 +42,7 @@ func (f *FormatJSON) FormatRequest( Request: JSONRequest{ ClientToken: req.ClientToken, + ID: req.ID, Operation: req.Operation, Path: req.Path, Data: req.Data, @@ -112,6 +113,7 @@ func (f *FormatJSON) FormatResponse( Request: JSONRequest{ ClientToken: req.ClientToken, + ID: req.ID, Operation: req.Operation, Path: req.Path, Data: req.Data, @@ -149,6 +151,7 @@ type JSONResponseEntry struct { } type JSONRequest struct { + ID string `json:"id"` Operation logical.Operation `json:"operation"` ClientToken string `json:"client_token"` Path string `json:"path"` diff --git a/http/logical.go b/http/logical.go index 94b434ec8..68832fa23 100644 --- a/http/logical.go +++ b/http/logical.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/hashicorp/errwrap" + "github.com/hashicorp/go-uuid" "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/vault" ) @@ -65,7 +66,13 @@ func buildLogicalRequest(w http.ResponseWriter, r *http.Request) (*logical.Reque } var err error + request_id, err := uuid.GenerateUUID() + if err != nil { + return nil, http.StatusBadRequest, errwrap.Wrapf("failed to generate identifier for the request: {{err}}", err) + } + req := requestAuth(r, &logical.Request{ + ID: request_id, Operation: op, Path: path, Data: data, @@ -135,11 +142,11 @@ func handleLogical(core *vault.Core, dataOnly bool, prepareRequestCallback Prepa } // Build the proper response - respondLogical(w, r, req.Path, dataOnly, resp) + respondLogical(w, r, req, dataOnly, resp) }) } -func respondLogical(w http.ResponseWriter, r *http.Request, path string, dataOnly bool, resp *logical.Response) { +func respondLogical(w http.ResponseWriter, r *http.Request, req *logical.Request, dataOnly bool, resp *logical.Response) { var httpResp interface{} if resp != nil { if resp.Redirect != "" { @@ -156,7 +163,7 @@ func respondLogical(w http.ResponseWriter, r *http.Request, path string, dataOnl // Check if this is a raw response if _, ok := resp.Data[logical.HTTPContentType]; ok { - respondRaw(w, r, path, resp) + respondRaw(w, r, req.Path, resp) return } @@ -170,7 +177,9 @@ func respondLogical(w http.ResponseWriter, r *http.Request, path string, dataOnl }, } } else { - httpResp = logical.SanitizeResponse(resp) + sanitizedHttp := logical.SanitizeResponse(resp) + sanitizedHttp.RequestID = req.ID + httpResp = sanitizedHttp } } diff --git a/http/logical_test.go b/http/logical_test.go index 177c7f8d6..51c686ca3 100644 --- a/http/logical_test.go +++ b/http/logical_test.go @@ -52,6 +52,7 @@ func TestLogical(t *testing.T) { testResponseStatus(t, resp, 200) testResponseBody(t, resp, &actual) delete(actual, "lease_id") + expected["request_id"] = actual["request_id"] if !reflect.DeepEqual(actual, expected) { t.Fatalf("bad:\nactual:\n%#v\nexpected:\n%#v", actual, expected) } @@ -157,6 +158,7 @@ func TestLogical_StandbyRedirect(t *testing.T) { delete(actualDataMap, "creation_time") delete(actualDataMap, "accessor") actual["data"] = actualDataMap + expected["request_id"] = actual["request_id"] delete(actual, "lease_id") if !reflect.DeepEqual(actual, expected) { t.Fatalf("bad: got %#v; expected %#v", actual, expected) @@ -198,6 +200,7 @@ func TestLogical_CreateToken(t *testing.T) { testResponseBody(t, resp, &actual) delete(actual["auth"].(map[string]interface{}), "client_token") delete(actual["auth"].(map[string]interface{}), "accessor") + expected["request_id"] = actual["request_id"] if !reflect.DeepEqual(actual, expected) { t.Fatalf("bad:\nexpected:\n%#v\nactual:\n%#v", expected, actual) } diff --git a/logical/request.go b/logical/request.go index 1dedb6ac4..5cd7f69a9 100644 --- a/logical/request.go +++ b/logical/request.go @@ -10,53 +10,56 @@ import ( // of a request being made to Vault. It is used to abstract // the details of the higher level request protocol from the handlers. type Request struct { + // Id is the uuid associated with each request + ID string `json:"id" structs:"id" mapstructure:"id"` + // Operation is the requested operation type - Operation Operation + Operation Operation `json:"operation" structs:"operation" mapstructure:"operation"` // Path is the part of the request path not consumed by the // routing. As an example, if the original request path is "prod/aws/foo" // and the AWS logical backend is mounted at "prod/aws/", then the // final path is "foo" since the mount prefix is trimmed. - Path string + Path string `json:"path" structs:"path" mapstructure:"path"` // Request data is an opaque map that must have string keys. - Data map[string]interface{} + Data map[string]interface{} `json:"map" structs:"data" mapstructure:"data"` // Storage can be used to durably store and retrieve state. - Storage Storage + Storage Storage `json:"storage" structs:"storage" mapstructure:"storage"` // Secret will be non-nil only for Revoke and Renew operations // to represent the secret that was returned prior. - Secret *Secret + Secret *Secret `json:"secret" structs:"secret" mapstructure:"secret"` // Auth will be non-nil only for Renew operations // to represent the auth that was returned prior. - Auth *Auth + Auth *Auth `json:"auth" structs:"auth" mapstructure:"auth"` // Connection will be non-nil only for credential providers to // inspect the connection information and potentially use it for // authentication/protection. - Connection *Connection + Connection *Connection `json:"connection" structs:"connection" mapstructure:"connection"` // ClientToken is provided to the core so that the identity // can be verified and ACLs applied. This value is passed // through to the logical backends but after being salted and // hashed. - ClientToken string + ClientToken string `json:"client_token" structs:"client_token" mapstructure:"client_token"` // DisplayName is provided to the logical backend to help associate // dynamic secrets with the source entity. This is not a sensitive // name, but is useful for operators. - DisplayName string + DisplayName string `json:"display_name" structs:"display_name" mapstructure:"display_name"` // MountPoint is provided so that a logical backend can generate // paths relative to itself. The `Path` is effectively the client // request path with the MountPoint trimmed off. - MountPoint string + MountPoint string `json:"mount_point" structs:"mount_point" mapstructure:"mount_point"` // WrapTTL contains the requested TTL of the token used to wrap the // response in a cubbyhole. - WrapTTL time.Duration + WrapTTL time.Duration `json:"wrap_ttl" struct:"wrap_ttl" mapstructure:"wrap_ttl"` } // Get returns a data field and guards for nil Data diff --git a/logical/response.go b/logical/response.go index 02566d99b..395559dba 100644 --- a/logical/response.go +++ b/logical/response.go @@ -31,51 +31,51 @@ const ( type WrapInfo struct { // Setting to non-zero specifies that the response should be wrapped. // Specifies the desired TTL of the wrapping token. - TTL time.Duration + TTL time.Duration `json:"ttl" structs:"ttl" mapstructure:"ttl"` // The token containing the wrapped response - Token string + Token string `json:"token" structs:"token" mapstructure:"token"` // The creation time. This can be used with the TTL to figure out an // expected expiration. - CreationTime time.Time + CreationTime time.Time `json:"creation_time" structs:"creation_time" mapstructure:"cration_time"` // If the contained response is the output of a token creation call, the // created token's accessor will be accessible here - WrappedAccessor string + WrappedAccessor string `json:"wrapped_accessor" structs:"wrapped_accessor" mapstructure:"wrapped_accessor"` } // Response is a struct that stores the response of a request. // It is used to abstract the details of the higher level request protocol. type Response struct { // Secret, if not nil, denotes that this response represents a secret. - Secret *Secret + Secret *Secret `json:"secret" structs:"secret" mapstructure:"secret"` // Auth, if not nil, contains the authentication information for // this response. This is only checked and means something for // credential backends. - Auth *Auth + Auth *Auth `json:"auth" structs:"auth" mapstructure:"auth"` // Response data is an opaque map that must have string keys. For // secrets, this data is sent down to the user as-is. To store internal // data that you don't want the user to see, store it in // Secret.InternalData. - Data map[string]interface{} + Data map[string]interface{} `json:"data" structs:"data" mapstructure:"data"` // Redirect is an HTTP URL to redirect to for further authentication. // This is only valid for credential backends. This will be blanked // for any logical backend and ignored. - Redirect string + Redirect string `json:"redirect" structs:"redirect" mapstructure:"redirect"` // Warnings allow operations or backends to return warnings in response // to user actions without failing the action outright. // Making it private helps ensure that it is easy for various parts of // Vault (backend, core, etc.) to add warnings without accidentally // replacing what exists. - warnings []string + warnings []string `json:"warnings" structs:"warnings" mapstructure:"warnings"` // Information for wrapping the response in a cubbyhole - WrapInfo *WrapInfo + WrapInfo *WrapInfo `json:"wrap_info" structs:"wrap_info" mapstructure:"wrap_info"` } func init() { diff --git a/logical/sanitize.go b/logical/sanitize.go index 3ea28aa61..0b6c702c4 100644 --- a/logical/sanitize.go +++ b/logical/sanitize.go @@ -35,6 +35,7 @@ func SanitizeResponse(input *Response) *HTTPResponse { } type HTTPResponse struct { + RequestID string `json:"request_id"` LeaseID string `json:"lease_id"` Renewable bool `json:"renewable"` LeaseDuration int `json:"lease_duration"` diff --git a/vault/audit.go b/vault/audit.go index 9688b0367..0ce56cb11 100644 --- a/vault/audit.go +++ b/vault/audit.go @@ -11,6 +11,7 @@ import ( "time" "github.com/armon/go-metrics" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-uuid" "github.com/hashicorp/vault/audit" "github.com/hashicorp/vault/helper/jsonutil" @@ -351,17 +352,24 @@ func (a *AuditBroker) GetHash(name string, input string) (string, error) { // LogRequest is used to ensure all the audit backends have an opportunity to // log the given request and that *at least one* succeeds. -func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) (reterr error) { +func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) (retErr error) { defer metrics.MeasureSince([]string{"audit", "log_request"}, time.Now()) a.l.RLock() defer a.l.RUnlock() defer func() { if r := recover(); r != nil { a.logger.Printf("[ERR] audit: panic logging: req path: %s", req.Path) - reterr = fmt.Errorf("panic generating audit log") + retErr = multierror.Append(retErr, fmt.Errorf("panic generating audit log")) } }() + // All logged requests must have an identifier + //if req.ID == "" { + // a.logger.Printf("[ERR] audit: missing identifier in request object: %s", req.Path) + // retErr = multierror.Append(retErr, fmt.Errorf("missing identifier in request object: %s", req.Path)) + // return + //} + // Ensure at least one backend logs anyLogged := false for name, be := range a.backends { @@ -375,7 +383,8 @@ func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outer } } if !anyLogged && len(a.backends) > 0 { - return fmt.Errorf("no audit backend succeeded in logging the request") + retErr = multierror.Append(retErr, fmt.Errorf("no audit backend succeeded in logging the request")) + return } return nil } diff --git a/vault/audit_test.go b/vault/audit_test.go index f673e0a26..0adc33a70 100644 --- a/vault/audit_test.go +++ b/vault/audit_test.go @@ -10,6 +10,8 @@ import ( "errors" + "github.com/hashicorp/errwrap" + "github.com/hashicorp/go-uuid" "github.com/hashicorp/vault/audit" "github.com/hashicorp/vault/logical" ) @@ -223,9 +225,17 @@ func TestAuditBroker_LogRequest(t *testing.T) { Operation: logical.ReadOperation, Path: "sys/mounts", } + + // Create an identifier for the request to verify against + var err error + req.ID, err = uuid.GenerateUUID() + if err != nil { + t.Fatalf("failed to generate identifier for the request: path%s err: %v", req.Path, err) + } + reqErrs := errors.New("errs") - err := b.LogRequest(auth, req, reqErrs) + err = b.LogRequest(auth, req, reqErrs) if err != nil { t.Fatalf("err: %v", err) } @@ -250,7 +260,7 @@ func TestAuditBroker_LogRequest(t *testing.T) { // Should FAIL work with both failing backends a2.ReqErr = fmt.Errorf("failed") - if err := b.LogRequest(auth, req, nil); err.Error() != "no audit backend succeeded in logging the request" { + if err := b.LogRequest(auth, req, nil); !errwrap.Contains(err, "no audit backend succeeded in logging the request") { t.Fatalf("err: %v", err) } } diff --git a/vault/core_test.go b/vault/core_test.go index 4692f1c83..2363ba51b 100644 --- a/vault/core_test.go +++ b/vault/core_test.go @@ -1153,6 +1153,12 @@ func TestCore_StepDown(t *testing.T) { Path: "sys/step-down", } + // Create an identifier for the request + req.ID, err = uuid.GenerateUUID() + if err != nil { + t.Fatalf("failed to generate identifier for the request: path: %s err: %v", req.Path, err) + } + // Step down core err = core.StepDown(req) if err != nil { diff --git a/vault/request_handling.go b/vault/request_handling.go index c3e7e2cd5..d2f99ae92 100644 --- a/vault/request_handling.go +++ b/vault/request_handling.go @@ -411,6 +411,9 @@ func (c *Core) wrapInCubbyhole(req *logical.Request, resp *logical.Response) (*l httpResponse := logical.SanitizeResponse(resp) + // Add the unique identifier of the original request to the response + httpResponse.RequestID = req.ID + // Because of the way that JSON encodes (likely just in Go) we actually get // mixed-up values for ints if we simply put this object in the response // and encode the whole thing; so instead we marshal it first, then store diff --git a/vault/router.go b/vault/router.go index 64f63bc3f..d1c074676 100644 --- a/vault/router.go +++ b/vault/router.go @@ -248,11 +248,15 @@ func (r *Router) routeCommon(req *logical.Request, existenceCheck bool) (*logica // Cache the pointer to the original connection object originalConn := req.Connection + // Cache the identifier of the request + originalReqID := req.ID + // Reset the request before returning defer func() { req.Path = original req.MountPoint = "" req.Connection = originalConn + req.ID = originalReqID req.Storage = nil req.ClientToken = clientToken }() diff --git a/website/source/docs/audit/index.html.md b/website/source/docs/audit/index.html.md index 764618407..74f8699c4 100644 --- a/website/source/docs/audit/index.html.md +++ b/website/source/docs/audit/index.html.md @@ -21,9 +21,10 @@ but also a second copy in case the first is tampered with. ## Sensitive Information The audit logs contain the full request and response objects for every -interaction with Vault. The data in the request and the data in the -response (including secrets and authentication tokens) will be hashed -with a salt using HMAC-SHA256. +interaction with Vault. The request and response can be matched utilizing a +unique identifier assigned to each request. The data in the request and the +data in the response (including secrets and authentication tokens) will be +hashed with a salt using HMAC-SHA256. The purpose of the hash is so that secrets aren't in plaintext within your audit logs. However, you're still able to check the value of secrets by