Merge pull request #1676 from hashicorp/accessor-listing

Add accessor list function to token store

vault/logical_system.go

@@ -572,12 +572,12 @@ func (b *SystemBackend) handleCapabilitiesAccessor(req *logical.Request, d *fram
 		return logical.ErrorResponse("missing accessor"), nil
 	}
 
-	token, err := b.Core.tokenStore.lookupByAccessor(accessor)
+	aEntry, err := b.Core.tokenStore.lookupByAccessor(accessor)
 	if err != nil {
 		return nil, err
 	}
 
-	capabilities, err := b.Core.Capabilities(token, d.Get("path").(string))
+	capabilities, err := b.Core.Capabilities(aEntry.TokenID, d.Get("path").(string))
 	if err != nil {
 		return nil, err
 	}

vault/token_store.go

@@ -107,6 +107,7 @@ func NewTokenStore(c *Core, config *logical.BackendConfig) (*TokenStore, error)
 		PathsSpecial: &logical.Paths{
 			Root: []string{
 				"revoke-orphan/*",
+				"accessors*",
 			},
 		},
 
@@ -122,6 +123,17 @@ func NewTokenStore(c *Core, config *logical.BackendConfig) (*TokenStore, error)
 				HelpDescription: tokenListRolesHelp,
 			},
 
+			&framework.Path{
+				Pattern: "accessors/?$",
+
+				Callbacks: map[logical.Operation]framework.OperationFunc{
+					logical.ListOperation: t.tokenStoreAccessorList,
+				},
+
+				HelpSynopsis:    tokenListAccessorsHelp,
+				HelpDescription: tokenListAccessorsHelp,
+			},
+
 			&framework.Path{
 				Pattern: "roles/" + framework.GenericNameRegex("role_name"),
 				Fields: map[string]*framework.FieldSchema{
@@ -484,6 +496,11 @@ type tsRoleEntry struct {
 	ExplicitMaxTTL time.Duration `json:"explicit_max_ttl" mapstructure:"explicit_max_ttl" structs:"explicit_max_ttl"`
 }
 
+type accessorEntry struct {
+	TokenID    string `json:"token_id"`
+	AccessorID string `json:"accessor_id"`
+}
+
 // SetExpirationManager is used to provide the token store with
 // an expiration manager. This is used to manage prefix based revocation
 // of tokens and to cleanup entries when removed from the token store.
@@ -510,6 +527,35 @@ func (ts *TokenStore) rootToken() (*TokenEntry, error) {
 	return te, nil
 }
 
+func (ts *TokenStore) tokenStoreAccessorList(
+	req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+	entries, err := ts.view.List(accessorPrefix)
+	if err != nil {
+		return nil, err
+	}
+
+	resp := &logical.Response{}
+
+	ret := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		aEntry, err := ts.lookupBySaltedAccessor(entry)
+		if err != nil {
+			resp.AddWarning("Found an accessor entry that could not be successfully decoded")
+			continue
+		}
+		if aEntry.TokenID == "" {
+			resp.AddWarning(fmt.Sprintf("Found an accessor entry missing a token: %v", aEntry.AccessorID))
+		} else {
+			ret = append(ret, aEntry.AccessorID)
+		}
+	}
+
+	resp.Data = map[string]interface{}{
+		"keys": ret,
+	}
+	return resp, nil
+}
+
 // createAccessor is used to create an identifier for the token ID.
 // A storage index, mapping the accessor to the token ID is also created.
 func (ts *TokenStore) createAccessor(entry *TokenEntry) error {
@@ -524,7 +570,17 @@ func (ts *TokenStore) createAccessor(entry *TokenEntry) error {
 
 	// Create index entry, mapping the accessor to the token ID
 	path := accessorPrefix + ts.SaltID(entry.Accessor)
-	le := &logical.StorageEntry{Key: path, Value: []byte(entry.ID)}
+
+	aEntry := &accessorEntry{
+		TokenID:    entry.ID,
+		AccessorID: entry.Accessor,
+	}
+	aEntryBytes, err := jsonutil.EncodeJSON(aEntry)
+	if err != nil {
+		return fmt.Errorf("failed to marshal accessor index entry: %v", err)
+	}
+
+	le := &logical.StorageEntry{Key: path, Value: aEntryBytes}
 	if err := ts.view.Put(le); err != nil {
 		return fmt.Errorf("failed to persist accessor index entry: %v", err)
 	}
@@ -845,16 +901,40 @@ func (ts *TokenStore) handleCreateAgainstRole(
 	return ts.handleCreateCommon(req, d, false, roleEntry)
 }
 
-func (ts *TokenStore) lookupByAccessor(accessor string) (string, error) {
-	entry, err := ts.view.Get(accessorPrefix + ts.SaltID(accessor))
+func (ts *TokenStore) lookupByAccessor(accessor string) (accessorEntry, error) {
+	return ts.lookupBySaltedAccessor(ts.SaltID(accessor))
+}
+
+func (ts *TokenStore) lookupBySaltedAccessor(saltedAccessor string) (accessorEntry, error) {
+	entry, err := ts.view.Get(accessorPrefix + saltedAccessor)
+	var aEntry accessorEntry
+
 	if err != nil {
-		return "", fmt.Errorf("failed to read index using accessor: %s", err)
+		return aEntry, fmt.Errorf("failed to read index using accessor: %s", err)
 	}
 	if entry == nil {
-		return "", &StatusBadRequest{Err: "invalid accessor"}
+		return aEntry, &StatusBadRequest{Err: "invalid accessor"}
 	}
 
-	return string(entry.Value), nil
+	err = jsonutil.DecodeJSON(entry.Value, &aEntry)
+	// If we hit an error, assume it's a pre-struct straight token ID
+	if err != nil {
+		aEntry.TokenID = string(entry.Value)
+		te, err := ts.lookupSalted(ts.SaltID(aEntry.TokenID))
+		if err != nil {
+			return accessorEntry{}, fmt.Errorf("failed to look up token using accessor index: %s", err)
+		}
+		// It's hard to reason about what to do here -- it may be that the
+		// token was revoked async, or that it's an old accessor index entry
+		// that was somehow not cleared up, or or or. A nonexistent token entry
+		// on lookup is nil, not an error, so we keep that behavior here to be
+		// safe...the token ID is simply not filled in.
+		if te != nil {
+			aEntry.AccessorID = te.Accessor
+		}
+	}
+
+	return aEntry, nil
 }
 
 // handleUpdateLookupAccessor handles the auth/token/lookup-accessor path for returning
@@ -868,7 +948,7 @@ func (ts *TokenStore) handleUpdateLookupAccessor(req *logical.Request, data *fra
 		}
 	}
 
-	tokenID, err := ts.lookupByAccessor(accessor)
+	aEntry, err := ts.lookupByAccessor(accessor)
 	if err != nil {
 		return nil, err
 	}
@@ -876,7 +956,7 @@ func (ts *TokenStore) handleUpdateLookupAccessor(req *logical.Request, data *fra
 	// Prepare the field data required for a lookup call
 	d := &framework.FieldData{
 		Raw: map[string]interface{}{
-			"token": tokenID,
+			"token": aEntry.TokenID,
 		},
 		Schema: map[string]*framework.FieldSchema{
 			"token": &framework.FieldSchema{
@@ -916,13 +996,13 @@ func (ts *TokenStore) handleUpdateRevokeAccessor(req *logical.Request, data *fra
 		}
 	}
 
-	tokenID, err := ts.lookupByAccessor(accessor)
+	aEntry, err := ts.lookupByAccessor(accessor)
 	if err != nil {
 		return nil, err
 	}
 
 	// Revoke the token and its children
-	if err := ts.RevokeTree(tokenID); err != nil {
+	if err := ts.RevokeTree(aEntry.TokenID); err != nil {
 		return logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest
 	}
 	return nil, nil
@@ -1700,4 +1780,10 @@ no effect on the token being renewed.`
 	tokenRenewableHelp = `Tokens created via this role will be
 renewable or not according to this value.
 Defaults to "true".`
+	tokenListAccessorsHelp = `List token accessors, which can then be
+be used to iterate and discover their properities
+or revoke them. Because this can be used to
+cause a denial of service, this endpoint
+requires 'sudo' capability in addition to
+'list'.`
 )

vault/token_store_test.go

@@ -70,14 +70,14 @@ func TestTokenStore_AccessorIndex(t *testing.T) {
 		t.Fatalf("bad: %#v", out)
 	}
 
-	token, err := ts.lookupByAccessor(out.Accessor)
+	aEntry, err := ts.lookupByAccessor(out.Accessor)
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
 
 	// Verify that the value returned from the index matches the token ID
-	if token != ent.ID {
-		t.Fatalf("bad: got\n%s\nexpected\n%s\n", token, ent.ID)
+	if aEntry.TokenID != ent.ID {
+		t.Fatalf("bad: got\n%s\nexpected\n%s\n", aEntry.TokenID, ent.ID)
 	}
 }
 
@@ -112,6 +112,83 @@ func TestTokenStore_HandleRequest_LookupAccessor(t *testing.T) {
 	}
 }
 
+func TestTokenStore_HandleRequest_ListAccessors(t *testing.T) {
+	_, ts, _, root := TestCoreWithTokenStore(t)
+
+	testKeys := []string{"token1", "token2", "token3", "token4"}
+	for _, key := range testKeys {
+		testMakeToken(t, ts, root, key, "", []string{"foo"})
+	}
+
+	// Revoke root to make the number of accessors match
+	ts.revokeSalted(ts.SaltID(root))
+
+	req := logical.TestRequest(t, logical.ListOperation, "accessors")
+
+	resp, err := ts.HandleRequest(req)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if resp.Data == nil {
+		t.Fatalf("response should contain data")
+	}
+	if resp.Data["keys"] == nil {
+		t.Fatalf("keys should not be empty")
+	}
+	keys := resp.Data["keys"].([]string)
+	if len(keys) != len(testKeys) {
+		t.Fatalf("wrong number of accessors found")
+	}
+	if len(resp.Warnings()) != 0 {
+		t.Fatalf("got warnings:\n%#v", resp.Warnings())
+	}
+
+	// Test upgrade from old struct method of accessor storage (of token id)
+	for _, accessor := range keys {
+		aEntry, err := ts.lookupByAccessor(accessor)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if aEntry.TokenID == "" || aEntry.AccessorID == "" {
+			t.Fatalf("error, accessor entry looked up is empty, but no error thrown")
+		}
+		path := accessorPrefix + ts.SaltID(accessor)
+		le := &logical.StorageEntry{Key: path, Value: []byte(aEntry.TokenID)}
+		if err := ts.view.Put(le); err != nil {
+			t.Fatalf("failed to persist accessor index entry: %v", err)
+		}
+	}
+
+	// Do the lookup again, should get same result
+	resp, err = ts.HandleRequest(req)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if resp.Data == nil {
+		t.Fatalf("response should contain data")
+	}
+	if resp.Data["keys"] == nil {
+		t.Fatalf("keys should not be empty")
+	}
+	keys2 := resp.Data["keys"].([]string)
+	if len(keys) != len(testKeys) {
+		t.Fatalf("wrong number of accessors found")
+	}
+	if len(resp.Warnings()) != 0 {
+		t.Fatalf("got warnings:\n%#v", resp.Warnings())
+	}
+
+	for _, accessor := range keys2 {
+		aEntry, err := ts.lookupByAccessor(accessor)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if aEntry.TokenID == "" || aEntry.AccessorID == "" {
+			t.Fatalf("error, accessor entry looked up is empty, but no error thrown")
+		}
+	}
+}
+
 func TestTokenStore_HandleRequest_RevokeAccessor(t *testing.T) {
 	_, ts, _, root := TestCoreWithTokenStore(t)
 	testMakeToken(t, ts, root, "tokenid", "", []string{"foo"})

website/source/docs/auth/token.html.md

@@ -40,6 +40,44 @@ of the header should be "X-Vault-Token" and the value should be the token.
 
 ## API
 
+### /auth/token/accessors
+#### LIST or GET 
+
+<dl class="api">
+  <dt>Description</dt>
+  <dd>
+    Lists token accessors. This requires `sudo` capability, and access to it
+    should be tightly controlled as the accessors can be used to revoke very
+    large numbers of tokens and their associated leases at once.
+  </dd>
+
+  <dt>Method</dt>
+  <dd>LIST or GET</dd>
+
+  <dt>URL</dt>
+  <dd>`/auth/token/accessors` (LIST)<dd>
+  <dd>`/auth/token/accessors?list=true` (GET)<dd>
+
+  <dt>Parameters</dt>
+  <dd>
+    None
+  </dd>
+
+  <dt>Returns</dt>
+  <dd>
+
+    ```javascript
+    {
+      "data": {
+        "keys": ["476ea048-ded5-4d07-eeea-938c6b4e43ec", "bb00c093-b7d3-b0e9-69cc-c4d85081165b"]
+      }
+    }
+    ```
+
+  </dd>
+</dl>
+
+
 ### /auth/token/create
 ### /auth/token/create-orphan
 ### /auth/token/create/[role_name]