Reintroduce the ability to look up obfuscated values in the audit log

with a new endpoint '/sys/audit-hash', which returns the given input
string hashed with the given audit backend's hash function and salt
(currently, always HMAC-SHA256 and a backend-specific salt).

In the process of adding the HTTP handler, this also removes the custom
HTTP handlers for the other audit endpoints, which were simply
forwarding to the logical system backend. This means that the various
audit functions will now redirect correctly from a standby to master.
(Tests all pass.)

Fixes #784

CHANGELOG.md

@@ -39,6 +39,9 @@ IMPROVEMENTS:
  * audit: HMAC-SHA256'd client tokens are now stored with each request entry.
    Previously they were only displayed at creation time; this allows much
    better traceability of client actions. [GH-713]
+ * audit: There is now a `sys/audit-hash` endpoint that can be used to generate
+   an HMAC-SHA256'd value from provided data using the given audit backend's
+   salt [GH-784]
  * core: The physical storage read cache can now be disabled via
    "disable_cache" [GH-674]
  * core: The unsealing process can now be reset midway through (this feature

api/sys_audit.go

@@ -4,6 +4,31 @@ import (
 	"fmt"
 )
 
+func (c *Sys) AuditHash(path string, input string) (string, error) {
+	body := map[string]interface{}{
+		"input": input,
+	}
+
+	r := c.c.NewRequest("PUT", fmt.Sprintf("/v1/sys/audit-hash/%s", path))
+	if err := r.SetJSONBody(body); err != nil {
+		return "", err
+	}
+
+	resp, err := c.c.RawRequest(r)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	type d struct {
+		Hash string
+	}
+
+	var result d
+	err = resp.DecodeJSON(&result)
+	return result.Hash, err
+}
+
 func (c *Sys) ListAudit() (map[string]*Audit, error) {
 	r := c.c.NewRequest("GET", "/v1/sys/audit")
 	resp, err := c.c.RawRequest(r)

audit/audit.go

@@ -21,6 +21,11 @@ type Backend interface {
 	// MUST not be modified in anyway. They should be deep copied if this is
 	// a possibility.
 	LogResponse(*logical.Auth, *logical.Request, *logical.Response, error) error
+
+	// GetHash is used to return the given data with the backend's hash,
+	// so that a caller can determine if a value in the audit log matches
+	// an expected plaintext value
+	GetHash(string) string
 }
 
 type BackendConfig struct {

audit/hashstructure.go

@@ -10,6 +10,11 @@ import (
 	"github.com/mitchellh/reflectwalk"
 )
 
+// HashString hashes the given opaque string and returns it
+func HashString(salter *salt.Salt, data string) string {
+	return salter.GetIdentifiedHMAC(data)
+}
+
 // Hash will hash the given type. This has built-in support for auth,
 // requests, and responses. If it is a type that isn't recognized, then
 // it will be passed through.

audit/hashstructure_test.go

@@ -81,6 +81,25 @@ func TestCopy_response(t *testing.T) {
 	}
 }
 
+func TestHashString(t *testing.T) {
+	inmemStorage := &logical.InmemStorage{}
+	inmemStorage.Put(&logical.StorageEntry{
+		Key:   "salt",
+		Value: []byte("foo"),
+	})
+	localSalt, err := salt.NewSalt(inmemStorage, &salt.Config{
+		HMAC:     sha256.New,
+		HMACType: "hmac-sha256",
+	})
+	if err != nil {
+		t.Fatalf("Error instantiating salt: %s", err)
+	}
+	out := HashString(localSalt, "foo")
+	if out != "hmac-sha256:08ba357e274f528065766c770a639abf6809b39ccfd37c2a3157c7f51954da0a" {
+		t.Fatalf("err: HashString output did not match expected")
+	}
+}
+
 func TestHash(t *testing.T) {
 	now := time.Now().UTC()
 

builtin/audit/file/backend.go

@@ -63,6 +63,10 @@ type Backend struct {
 	f    *os.File
 }
 
+func (b *Backend) GetHash(data string) string {
+	return audit.HashString(b.salt, data)
+}
+
 func (b *Backend) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) error {
 	if err := b.open(); err != nil {
 		return err

builtin/audit/syslog/backend.go

@@ -60,6 +60,10 @@ type Backend struct {
 	salt   *salt.Salt
 }
 
+func (b *Backend) GetHash(data string) string {
+	return audit.HashString(b.salt, data)
+}
+
 func (b *Backend) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) error {
 	if !b.logRaw {
 		// Before we copy the structure we must nil out some data

helper/salt/salt.go

@@ -110,18 +110,18 @@ func (s *Salt) SaltID(id string) string {
 	return SaltID(s.salt, id, s.config.HashFunc)
 }
 
-// GetHMAC is used to apply a salt and hash function to an ID to make sure
-// it is not reversible, with an additional HMAC
-func (s *Salt) GetHMAC(id string) string {
+// GetHMAC is used to apply a salt and hash function to data to make sure it is
+// not reversible, with an additional HMAC
+func (s *Salt) GetHMAC(data string) string {
 	hm := hmac.New(s.config.HMAC, []byte(s.salt))
-	hm.Write([]byte(id))
+	hm.Write([]byte(data))
 	return hex.EncodeToString(hm.Sum(nil))
 }
 
-// GetIdentifiedHMAC is used to apply a salt and hash function to an ID to make sure
-// it is not reversible, with an additional HMAC, and ID prepended
-func (s *Salt) GetIdentifiedHMAC(id string) string {
-	return s.hmacType + ":" + s.GetHMAC(id)
+// GetIdentifiedHMAC is used to apply a salt and hash function to data to make
+// sure it is not reversible, with an additional HMAC, and ID prepended
+func (s *Salt) GetIdentifiedHMAC(data string) string {
+	return s.hmacType + ":" + s.GetHMAC(data)
 }
 
 // DidGenerate returns if the underlying salt value was generated

http/handler.go

@@ -34,8 +34,9 @@ func Handler(core *vault.Core) http.Handler {
 	mux.Handle("/v1/sys/revoke-prefix/", proxySysRequest(core))
 	mux.Handle("/v1/sys/auth", proxySysRequest(core))
 	mux.Handle("/v1/sys/auth/", proxySysRequest(core))
-	mux.Handle("/v1/sys/audit", handleSysListAudit(core))
-	mux.Handle("/v1/sys/audit/", handleSysAudit(core))
+	mux.Handle("/v1/sys/audit-hash/", proxySysRequest(core))
+	mux.Handle("/v1/sys/audit", proxySysRequest(core))
+	mux.Handle("/v1/sys/audit/", proxySysRequest(core))
 	mux.Handle("/v1/sys/leader", handleSysLeader(core))
 	mux.Handle("/v1/sys/health", handleSysHealth(core))
 	mux.Handle("/v1/sys/rotate", proxySysRequest(core))

http/sys_audit.go

@@ -1,114 +0,0 @@
-package http
-
-import (
-	"net/http"
-	"strings"
-
-	"github.com/hashicorp/vault/logical"
-	"github.com/hashicorp/vault/vault"
-)
-
-func handleSysListAudit(core *vault.Core) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != "GET" {
-			respondError(w, http.StatusMethodNotAllowed, nil)
-			return
-		}
-
-		resp, ok := request(core, w, r, requestAuth(r, &logical.Request{
-			Operation:  logical.ReadOperation,
-			Path:       "sys/audit",
-			Connection: getConnection(r),
-		}))
-		if !ok {
-			return
-		}
-
-		respondOk(w, resp.Data)
-	})
-}
-
-func handleSysAudit(core *vault.Core) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		switch r.Method {
-		case "POST":
-			fallthrough
-		case "PUT":
-			handleSysEnableAudit(core, w, r)
-		case "DELETE":
-			handleSysDisableAudit(core, w, r)
-		default:
-			respondError(w, http.StatusMethodNotAllowed, nil)
-			return
-		}
-	})
-}
-
-func handleSysDisableAudit(core *vault.Core, w http.ResponseWriter, r *http.Request) {
-	// Determine the path...
-	prefix := "/v1/sys/audit/"
-	if !strings.HasPrefix(r.URL.Path, prefix) {
-		respondError(w, http.StatusNotFound, nil)
-		return
-	}
-	path := r.URL.Path[len(prefix):]
-	if path == "" {
-		respondError(w, http.StatusNotFound, nil)
-		return
-	}
-
-	_, ok := request(core, w, r, requestAuth(r, &logical.Request{
-		Operation:  logical.DeleteOperation,
-		Path:       "sys/audit/" + path,
-		Connection: getConnection(r),
-	}))
-	if !ok {
-		return
-	}
-
-	respondOk(w, nil)
-}
-
-func handleSysEnableAudit(core *vault.Core, w http.ResponseWriter, r *http.Request) {
-	// Determine the path...
-	prefix := "/v1/sys/audit/"
-	if !strings.HasPrefix(r.URL.Path, prefix) {
-		respondError(w, http.StatusNotFound, nil)
-		return
-	}
-
-	path := r.URL.Path[len(prefix):]
-	if path == "" {
-		respondError(w, http.StatusNotFound, nil)
-		return
-	}
-
-	// Parse the request if we can
-	var req enableAuditRequest
-	if err := parseRequest(r, &req); err != nil {
-		respondError(w, http.StatusBadRequest, err)
-		return
-	}
-
-	_, ok := request(core, w, r, requestAuth(r, &logical.Request{
-		Operation:  logical.WriteOperation,
-		Path:       "sys/audit/" + path,
-		Connection: getConnection(r),
-		Data: map[string]interface{}{
-			"type":        req.Type,
-			"description": req.Description,
-			"options":     req.Options,
-		},
-	}))
-	if !ok {
-		return
-	}
-
-	respondOk(w, nil)
-}
-
-type enableAuditRequest struct {
-	Type        string            `json:"type"`
-	Description string            `json:"description"`
-	Options     map[string]string `json:"options"`
-}

http/sys_audit_test.go

@@ -59,3 +59,29 @@ func TestSysDisableAudit(t *testing.T) {
 		t.Fatalf("bad: %#v", actual)
 	}
 }
+
+func TestSysAuditHash(t *testing.T) {
+	core, _, token := vault.TestCoreUnsealed(t)
+	ln, addr := TestServer(t, core)
+	defer ln.Close()
+	TestServerAuth(t, addr, token)
+
+	resp := testHttpPost(t, token, addr+"/v1/sys/audit/noop", map[string]interface{}{
+		"type": "noop",
+	})
+	testResponseStatus(t, resp, 204)
+
+	resp = testHttpPost(t, token, addr+"/v1/sys/audit-hash/noop", map[string]interface{}{
+		"input": "bar",
+	})
+
+	var actual map[string]interface{}
+	expected := map[string]interface{}{
+		"hash": "hmac-sha256:f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317",
+	}
+	testResponseStatus(t, resp, 200)
+	testResponseBody(t, resp, &actual)
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatalf("bad: expected:\n%#v\n, got:\n%#v\n", expected, actual)
+	}
+}

vault/audit.go

@@ -287,6 +287,18 @@ func (a *AuditBroker) IsRegistered(name string) bool {
 	return ok
 }
 
+// GetHash returns a hash using the salt of the given backend
+func (a *AuditBroker) GetHash(name string, input string) (string, error) {
+	a.l.RLock()
+	defer a.l.RUnlock()
+	be, ok := a.backends[name]
+	if !ok {
+		return "", fmt.Errorf("unknown audit backend %s", name)
+	}
+
+	return be.backend.GetHash(input), nil
+}
+
 // LogRequest is used to ensure all the audit backends have an opportunity to
 // log the given request and that *at least one* succeeds.
 func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) (reterr error) {

vault/audit_test.go

@@ -43,6 +43,10 @@ func (n *NoopAudit) LogResponse(a *logical.Auth, r *logical.Request, re *logical
 	return n.RespErr
 }
 
+func (n *NoopAudit) GetHash(data string) string {
+	return n.Config.Salt.GetIdentifiedHMAC(data)
+}
+
 func TestCore_EnableAudit(t *testing.T) {
 	c, key, _ := TestCoreUnsealed(t)
 	c.auditBackends["noop"] = func(config *audit.BackendConfig) (audit.Backend, error) {

vault/logical_system.go

@@ -263,6 +263,28 @@ func NewSystemBackend(core *Core, config *logical.BackendConfig) logical.Backend
 				HelpDescription: strings.TrimSpace(sysHelp["policy"][1]),
 			},
 
+			&framework.Path{
+				Pattern: "audit-hash/(?P<path>.+)",
+
+				Fields: map[string]*framework.FieldSchema{
+					"path": &framework.FieldSchema{
+						Type:        framework.TypeString,
+						Description: strings.TrimSpace(sysHelp["audit_path"][0]),
+					},
+
+					"input": &framework.FieldSchema{
+						Type: framework.TypeString,
+					},
+				},
+
+				Callbacks: map[logical.Operation]framework.OperationFunc{
+					logical.WriteOperation: b.handleAuditHash,
+				},
+
+				HelpSynopsis:    strings.TrimSpace(sysHelp["audit-hash"][0]),
+				HelpDescription: strings.TrimSpace(sysHelp["audit-hash"][1]),
+			},
+
 			&framework.Path{
 				Pattern: "audit$",
 
@@ -822,6 +844,32 @@ func (b *SystemBackend) handleAuditTable(
 	return resp, nil
 }
 
+// handleAuditHash is used to fetch the hash of the given input data with the
+// specified audit backend's salt
+func (b *SystemBackend) handleAuditHash(
+	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	path := data.Get("path").(string)
+	input := data.Get("input").(string)
+	if input == "" {
+		return logical.ErrorResponse("the \"input\" parameter is empty"), nil
+	}
+
+	if !strings.HasSuffix(path, "/") {
+		path += "/"
+	}
+
+	hash, err := b.Core.auditBroker.GetHash(path, input)
+	if err != nil {
+		return logical.ErrorResponse(err.Error()), nil
+	}
+
+	return &logical.Response{
+		Data: map[string]interface{}{
+			"hash": hash,
+		},
+	}, nil
+}
+
 // handleEnableAudit is used to enable a new audit backend
 func (b *SystemBackend) handleEnableAudit(
 	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
@@ -1167,6 +1215,11 @@ or delete a policy.
 		"",
 	},
 
+	"audit-hash": {
+		"The hash of the given string via the given audit backend",
+		"",
+	},
+
 	"audit-table": {
 		"List the currently enabled audit backends.",
 		`

vault/logical_system_test.go

@@ -1,12 +1,14 @@
 package vault
 
 import (
+	"crypto/sha256"
 	"reflect"
 	"testing"
 	"time"
 
 	"github.com/fatih/structs"
 	"github.com/hashicorp/vault/audit"
+	"github.com/hashicorp/vault/helper/salt"
 	"github.com/hashicorp/vault/logical"
 )
 
@@ -543,6 +545,57 @@ func TestSystemBackend_enableAudit(t *testing.T) {
 	}
 }
 
+func TestSystemBackend_auditHash(t *testing.T) {
+	c, b, _ := testCoreSystemBackend(t)
+	c.auditBackends["noop"] = func(config *audit.BackendConfig) (audit.Backend, error) {
+		view := &logical.InmemStorage{}
+		view.Put(&logical.StorageEntry{
+			Key:   "salt",
+			Value: []byte("foo"),
+		})
+		var err error
+		config.Salt, err = salt.NewSalt(view, &salt.Config{
+			HMAC:     sha256.New,
+			HMACType: "hmac-sha256",
+		})
+		if err != nil {
+			t.Fatal("error getting new salt: %v", err)
+		}
+		return &NoopAudit{
+			Config: config,
+		}, nil
+	}
+
+	req := logical.TestRequest(t, logical.WriteOperation, "audit/foo")
+	req.Data["type"] = "noop"
+
+	resp, err := b.HandleRequest(req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if resp != nil {
+		t.Fatalf("bad: %v", resp)
+	}
+
+	req = logical.TestRequest(t, logical.WriteOperation, "audit-hash/foo")
+	req.Data["input"] = "bar"
+
+	resp, err = b.HandleRequest(req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if resp == nil || resp.Data == nil {
+		t.Fatalf("response or its data was nil")
+	}
+	hash, ok := resp.Data["hash"]
+	if !ok {
+		t.Fatalf("did not get hash back in response, response was %#v", resp.Data)
+	}
+	if hash.(string) != "hmac-sha256:f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317" {
+		t.Fatalf("bad hash back: %s", hash.(string))
+	}
+}
+
 func TestSystemBackend_enableAudit_invalid(t *testing.T) {
 	b := testSystemBackend(t)
 	req := logical.TestRequest(t, logical.WriteOperation, "audit/foo")

vault/testing.go

@@ -2,6 +2,7 @@ package vault
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"fmt"
 	"net"
 	"os/exec"
@@ -11,6 +12,7 @@ import (
 	"golang.org/x/crypto/ssh"
 
 	"github.com/hashicorp/vault/audit"
+	"github.com/hashicorp/vault/helper/salt"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/logical/framework"
 	"github.com/hashicorp/vault/physical"
@@ -58,6 +60,19 @@ oOyBJU/HMVvBfv4g+OVFLVgSwwm6owwsouZ0+D/LasbuHqYyqYqdyPJQYzWA2Y+F
 func TestCore(t *testing.T) *Core {
 	noopAudits := map[string]audit.Factory{
 		"noop": func(config *audit.BackendConfig) (audit.Backend, error) {
+			view := &logical.InmemStorage{}
+			view.Put(&logical.StorageEntry{
+				Key:   "salt",
+				Value: []byte("foo"),
+			})
+			var err error
+			config.Salt, err = salt.NewSalt(view, &salt.Config{
+				HMAC:     sha256.New,
+				HMACType: "hmac-sha256",
+			})
+			if err != nil {
+				t.Fatal("error getting new salt: %v", err)
+			}
 			return &noopAudit{
 				Config: config,
 			}, nil
@@ -247,6 +262,10 @@ type noopAudit struct {
 	Config *audit.BackendConfig
 }
 
+func (n *noopAudit) GetHash(data string) string {
+	return n.Config.Salt.GetIdentifiedHMAC(data)
+}
+
 func (n *noopAudit) LogRequest(a *logical.Auth, r *logical.Request, e error) error {
 	return nil
 }

website/source/docs/audit/index.html.md

@@ -25,11 +25,11 @@ interaction with Vault. The data in the request and the data in the
 response (including secrets and authentication tokens) will be hashed
 with a salt using HMAC-SHA256.
 
-<!---
-The purpose of the hash is so that secrets aren't in plaintext within
-your audit logs. However, you're still able to check the value of
-secrets by SHA-ing it yourself.
---->
+The purpose of the hash is so that secrets aren't in plaintext within your
+audit logs. However, you're still able to check the value of secrets by
+generating HMACs yourself; this can be done with the audit backend's hash
+function and salt by using the `/sys/audit-hash` API endpoint (see the
+documentation for more details).
 
 ## Enabling/Disabling Audit Backends
 

website/source/docs/http/sys-audit-hash.html.md

@@ -0,0 +1,53 @@
+---
+layout: "http"
+page_title: "HTTP API: /sys/audit-hash"
+sidebar_current: "docs-http-audits-hash"
+description: |-
+  The `/sys/audit-hash` endpoint is used to hash data using an audit backend's hash function and salt.
+---
+
+# /sys/audit-hash
+
+## POST
+
+<dl>
+  <dt>Description</dt>
+  <dd>
+    Hash the given input data with the specified audit backend's hash function
+    and salt. This endpoint can be used to discover whether a given plaintext
+    string (the `input` parameter) appears in the audit log in obfuscated form.
+    Note that the audit log records requests and responses; since the Vault API
+    is JSON-based, any binary data returned from an API call (such as a
+    DER-format certificate) is base64-encoded by the Vault server in the
+    response, and as a result such information should also be base64-encoded to
+    supply into the `input` parameter.
+  </dd>
+
+  <dt>Method</dt>
+  <dd>POST</dd>
+
+  <dt>URL</dt>
+  <dd>`/sys/audit-hash/<name>`</dd>
+
+  <dt>Parameters</dt>
+  <dd>
+    <ul>
+      <li>
+        <span class="param">input</span>
+        <span class="param-flags">required</span>
+        The input string to hash.
+      </li>
+    </ul>
+  </dd>
+
+  <dt>Returns</dt>
+  <dd>
+
+    ```javascript
+    {
+      "hash": "hmac-sha256:08ba357e274f528065766c770a639abf6809b39ccfd37c2a3157c7f51954da0a"
+    }
+    ```
+
+  </dd>
+</dl>

website/source/layouts/http.erb

@@ -74,6 +74,9 @@
 						<li<%= sidebar_current("docs-http-audits-audits") %>>
 							<a href="/docs/http/sys-audit.html">/sys/audit</a>
 						</li>
+						<li<%= sidebar_current("docs-http-audits-hash") %>>
+							<a href="/docs/http/sys-audit-hash.html">/sys/audit-hash</a>
+						</li>
 					</ul>
 				</li>