Vault 1979: Query API for Irrevocable Leases (#11607)

* build out lease count (not fully working), start lease list

* build out irrevocable lease list

* bookkeeping

* test irrevocable lease counts for API/CLI

* fix listIrrevocableLeases, test listIrrevocableLeases, cleanup

* test expiration API limit

* namespace tweaks, test force flag on lease list

* integration test leases/count API, plenty of fixes and improvements

* test lease list API, fixes and improvements

* test force flag for irrevocable lease list API

* i guess this wasn't saved on the last refactor...

* fixes and improvements found during my review

* better test error msg

* Update vault/logical_system_paths.go

Co-authored-by: Brian Kassouf <briankassouf@users.noreply.github.com>

* Update vault/logical_system_paths.go

Co-authored-by: Brian Kassouf <briankassouf@users.noreply.github.com>

* return warning with data if more than default leases to list without force flag

* make api doc more generalized

* list leases in general, not by mount point

* change force flag to include_large_results

* sort leases by LeaseID for consistent API response

* switch from bool flag for API limit to string value

* sort first by leaseID, then stable sort by expiration

* move some utils to be in oss and ent

* improve sort efficiency for API response

Co-authored-by: Brian Kassouf <briankassouf@users.noreply.github.com>

changelog/11607.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core: add irrevocable lease list and count apis
+```

vault/expiration.go

@@ -8,6 +8,7 @@ import (
 	"math/rand"
 	"os"
 	"path"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -73,6 +74,12 @@ const (
 	genericIrrevocableErrorMessage = "unknown"
 
 	outOfRetriesMessage = "out of retries"
+
+	// maximum number of irrevocable leases we return to the irrevocable lease
+	// list API **without** the `force` flag set
+	MaxIrrevocableLeasesToReturn = 10000
+
+	MaxIrrevocableLeasesWarning = "Command halted because many irrevocable leases were found. To emit the entire list, re-run the command with force set true."
 )
 
 type pendingInfo struct {
@@ -261,18 +268,7 @@ func (r *revocationJob) OnFailure(err error) {
 func expireLeaseStrategyFairsharing(ctx context.Context, m *ExpirationManager, leaseID string, ns *namespace.Namespace) {
 	nsCtx := namespace.ContextWithNamespace(ctx, ns)
 
-	var mountAccessor string
+	mountAccessor := m.getLeaseMountAccessor(ctx, leaseID)
-	m.coreStateLock.RLock()
-	mount := m.core.router.MatchingMountEntry(nsCtx, leaseID)
-	m.coreStateLock.RUnlock()
-
-	if mount == nil {
-		// figure out what this means - if we couldn't find the mount, can we automatically revoke
-		m.logger.Debug("could not find lease path", "lease_id", leaseID)
-		mountAccessor = "mount-accessor-not-found"
-	} else {
-		mountAccessor = mount.Accessor
-	}
 
 	job, err := newRevocationJob(nsCtx, leaseID, ns, m)
 	if err != nil {
@@ -2418,6 +2414,155 @@ func (m *ExpirationManager) markLeaseIrrevocable(ctx context.Context, le *leaseE
 	m.nonexpiring.Delete(le.LeaseID)
 }
 
+func (m *ExpirationManager) getNamespaceFromLeaseID(ctx context.Context, leaseID string) (*namespace.Namespace, error) {
+	_, nsID := namespace.SplitIDFromString(leaseID)
+
+	// avoid re-declaring leaseNS and err with scope inside the if
+	leaseNS := namespace.RootNamespace
+	var err error
+	if nsID != "" {
+		leaseNS, err = NamespaceByID(ctx, nsID, m.core)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return leaseNS, nil
+}
+
+func (m *ExpirationManager) getLeaseMountAccessor(ctx context.Context, leaseID string) string {
+	m.coreStateLock.RLock()
+	mount := m.core.router.MatchingMountEntry(ctx, leaseID)
+	m.coreStateLock.RUnlock()
+
+	var mountAccessor string
+	if mount == nil {
+		mountAccessor = "mount-accessor-not-found"
+	} else {
+		mountAccessor = mount.Accessor
+	}
+
+	return mountAccessor
+}
+
+// TODO SW if keep counts as a map, should update the RFC
+func (m *ExpirationManager) getIrrevocableLeaseCounts(ctx context.Context, includeChildNamespaces bool) (map[string]interface{}, error) {
+	requestNS, err := namespace.FromContext(ctx)
+	if err != nil {
+		m.logger.Error("could not get namespace from context", "error", err)
+		return nil, err
+	}
+
+	numMatchingLeasesPerMount := make(map[string]int)
+	numMatchingLeases := 0
+	m.irrevocable.Range(func(k, v interface{}) bool {
+		leaseID := k.(string)
+		leaseNS, err := m.getNamespaceFromLeaseID(ctx, leaseID)
+		if err != nil {
+			// We should probably note that an error occured, but continue counting
+			m.logger.Warn("could not get lease namespace from ID", "error", err)
+			return true
+		}
+
+		leaseMatches := (leaseNS == requestNS) || (includeChildNamespaces && leaseNS.HasParent(requestNS))
+		if !leaseMatches {
+			// the lease doesn't meet our criteria, so keep looking
+			return true
+		}
+
+		mountAccessor := m.getLeaseMountAccessor(ctx, leaseID)
+
+		if _, ok := numMatchingLeasesPerMount[mountAccessor]; !ok {
+			numMatchingLeasesPerMount[mountAccessor] = 0
+		}
+
+		numMatchingLeases++
+		numMatchingLeasesPerMount[mountAccessor]++
+
+		return true
+	})
+
+	resp := make(map[string]interface{})
+	resp["lease_count"] = numMatchingLeases
+	resp["counts"] = numMatchingLeasesPerMount
+
+	return resp, nil
+}
+
+type leaseResponse struct {
+	LeaseID    string `json:"lease_id"`
+	MountID    string `json:"mount_id"`
+	ErrMsg     string `json:"error"`
+	expireTime time.Time
+}
+
+// returns a warning string, if applicable
+// limit specifies how many results to return, and must be >0
+// includeAll specifies if all results should be returned, regardless of limit
+func (m *ExpirationManager) listIrrevocableLeases(ctx context.Context, includeChildNamespaces, returnAll bool, limit int) (map[string]interface{}, string, error) {
+	requestNS, err := namespace.FromContext(ctx)
+	if err != nil {
+		m.logger.Error("could not get namespace from context", "error", err)
+		return nil, "", err
+	}
+
+	// map of mount point : lease info
+	matchingLeases := make([]*leaseResponse, 0)
+	numMatchingLeases := 0
+	var warning string
+	m.irrevocable.Range(func(k, v interface{}) bool {
+		leaseID := k.(string)
+		leaseInfo := v.(*leaseEntry)
+
+		leaseNS, err := m.getNamespaceFromLeaseID(ctx, leaseID)
+		if err != nil {
+			// We probably want to track that an error occured, but continue counting
+			m.logger.Warn("could not get lease namespace from ID", "error", err)
+			return true
+		}
+
+		leaseMatches := (leaseNS == requestNS) || (includeChildNamespaces && leaseNS.HasParent(requestNS))
+		if !leaseMatches {
+			// the lease doesn't meet our criteria, so keep looking
+			return true
+		}
+
+		if !returnAll && (numMatchingLeases >= limit) {
+			m.logger.Warn("hit max irrevocable leases without force flag set")
+			warning = MaxIrrevocableLeasesWarning
+			return false
+		}
+
+		mountAccessor := m.getLeaseMountAccessor(ctx, leaseID)
+
+		numMatchingLeases++
+		matchingLeases = append(matchingLeases, &leaseResponse{
+			LeaseID:    leaseID,
+			MountID:    mountAccessor,
+			ErrMsg:     leaseInfo.RevokeErr,
+			expireTime: leaseInfo.ExpireTime,
+		})
+
+		return true
+	})
+
+	// sort the results for consistent API response. we primarily sort on
+	// increasing expire time, and break ties with increasing lease id
+	sort.Slice(matchingLeases, func(i, j int) bool {
+		if !matchingLeases[i].expireTime.Equal(matchingLeases[j].expireTime) {
+			return matchingLeases[i].expireTime.Before(matchingLeases[j].expireTime)
+		}
+
+		return matchingLeases[i].LeaseID < matchingLeases[j].LeaseID
+	})
+
+	resp := make(map[string]interface{})
+	resp["lease_count"] = numMatchingLeases
+	resp["leases"] = matchingLeases
+
+	return resp, warning, nil
+}
+
 // leaseEntry is used to structure the values the expiration
 // manager stores. This is used to handle renew and revocation.
 type leaseEntry struct {

vault/expiration_test.go

@@ -3005,3 +3005,207 @@ func TestExpiration_unrecoverableErrorMakesIrrevocable(t *testing.T) {
 		}
 	}
 }
+
+// set up multiple mounts, and return a mapping of the path to the mount accessor
+func mountNoopBackends(t *testing.T, c *Core, paths []string) map[string]string {
+	t.Helper()
+
+	// enable the noop backend
+	c.logicalBackends["noop"] = func(ctx context.Context, config *logical.BackendConfig) (logical.Backend, error) {
+		return &NoopBackend{}, nil
+	}
+
+	pathToMount := make(map[string]string)
+	for _, path := range paths {
+		me := &MountEntry{
+			Table: mountTableType,
+			Path:  path,
+			Type:  "noop",
+		}
+		err := c.mount(namespace.RootContext(nil), me)
+		if err != nil {
+			t.Fatalf("err mounting backend %s: %v", path, err)
+		}
+
+		mount := c.router.MatchingMountEntry(namespace.RootContext(nil), path)
+		if mount == nil {
+			t.Fatalf("couldn't find mount for path %s", path)
+		}
+		pathToMount[path] = mount.Accessor
+	}
+
+	return pathToMount
+}
+
+func TestExpiration_getIrrevocableLeaseCounts(t *testing.T) {
+	c, _, _ := TestCoreUnsealed(t)
+
+	mountPrefixes := []string{"foo/bar/1/", "foo/bar/2/", "foo/bar/3/"}
+	pathToMount := mountNoopBackends(t, c, mountPrefixes)
+
+	exp := c.expiration
+
+	expectedPerMount := 10
+	for i := 0; i < expectedPerMount; i++ {
+		for _, mountPrefix := range mountPrefixes {
+			addIrrevocableLease(t, exp, mountPrefix, namespace.RootNamespace)
+		}
+	}
+
+	out, err := exp.getIrrevocableLeaseCounts(namespace.RootContext(nil), false)
+	if err != nil {
+		t.Fatalf("error getting irrevocable lease counts: %v", err)
+	}
+
+	countRaw, ok := out["lease_count"]
+	if !ok {
+		t.Fatal("no lease count")
+	}
+
+	countPerMountRaw, ok := out["counts"]
+	if !ok {
+		t.Fatal("no count per mount")
+	}
+
+	count := countRaw.(int)
+	countPerMount := countPerMountRaw.(map[string]int)
+
+	expectedCount := len(mountPrefixes) * expectedPerMount
+	if count != expectedCount {
+		t.Errorf("bad count. expected %d, got %d", expectedCount, count)
+	}
+
+	if len(countPerMount) != len(mountPrefixes) {
+		t.Fatalf("bad mounts. got %#v, expected %v", countPerMount, mountPrefixes)
+	}
+
+	for _, mountPrefix := range mountPrefixes {
+		mountCount := countPerMount[pathToMount[mountPrefix]]
+		if mountCount != expectedPerMount {
+			t.Errorf("bad count for prefix %q. expected %d, got %d", mountPrefix, expectedPerMount, mountCount)
+		}
+	}
+}
+
+func TestExpiration_listIrrevocableLeases(t *testing.T) {
+	c, _, _ := TestCoreUnsealed(t)
+
+	mountPrefixes := []string{"foo/bar/1/", "foo/bar/2/", "foo/bar/3/"}
+	pathToMount := mountNoopBackends(t, c, mountPrefixes)
+
+	exp := c.expiration
+
+	expectedLeases := make([]*basicLeaseTestInfo, 0)
+	expectedPerMount := 10
+	for i := 0; i < expectedPerMount; i++ {
+		for _, mountPrefix := range mountPrefixes {
+			le := addIrrevocableLease(t, exp, mountPrefix, namespace.RootNamespace)
+			expectedLeases = append(expectedLeases, &basicLeaseTestInfo{
+				id:     le.id,
+				mount:  pathToMount[mountPrefix],
+				expire: le.expire,
+			})
+		}
+	}
+
+	out, warn, err := exp.listIrrevocableLeases(namespace.RootContext(nil), false, false, MaxIrrevocableLeasesToReturn)
+	if err != nil {
+		t.Fatalf("error listing irrevocable leases: %v", err)
+	}
+	if warn != "" {
+		t.Errorf("expected no warning, got %q", warn)
+	}
+
+	countRaw, ok := out["lease_count"]
+	if !ok {
+		t.Fatal("no lease count")
+	}
+
+	leasesRaw, ok := out["leases"]
+	if !ok {
+		t.Fatal("no leases")
+	}
+
+	count := countRaw.(int)
+	leases := leasesRaw.([]*leaseResponse)
+
+	expectedCount := len(mountPrefixes) * expectedPerMount
+	if count != expectedCount {
+		t.Errorf("bad count. expected %d, got %d", expectedCount, count)
+	}
+	if len(leases) != len(expectedLeases) {
+		t.Errorf("bad lease results. expected %d, got %d with values %v", len(expectedLeases), len(leases), leases)
+	}
+
+	// `leases` is already sorted by lease ID
+	sort.Slice(expectedLeases, func(i, j int) bool {
+		return expectedLeases[i].id < expectedLeases[j].id
+	})
+	sort.SliceStable(expectedLeases, func(i, j int) bool {
+		return expectedLeases[i].expire.Before(expectedLeases[j].expire)
+	})
+
+	for i, lease := range expectedLeases {
+		if lease.id != leases[i].LeaseID {
+			t.Errorf("bad lease id. expected %q, got %q", lease.id, leases[i].LeaseID)
+		}
+		if lease.mount != leases[i].MountID {
+			t.Errorf("bad mount id. expected %q, got %q", lease.mount, leases[i].MountID)
+		}
+	}
+}
+
+func TestExpiration_listIrrevocableLeases_includeAll(t *testing.T) {
+	c, _, _ := TestCoreUnsealed(t)
+	exp := c.expiration
+
+	expectedNumLeases := MaxIrrevocableLeasesToReturn + 10
+	for i := 0; i < expectedNumLeases; i++ {
+		addIrrevocableLease(t, exp, "foo/", namespace.RootNamespace)
+	}
+
+	dataRaw, warn, err := exp.listIrrevocableLeases(namespace.RootContext(nil), false, false, MaxIrrevocableLeasesToReturn)
+	if err != nil {
+		t.Fatalf("expected no error, got: %v", err)
+	}
+	if warn != MaxIrrevocableLeasesWarning {
+		t.Errorf("expected warning %q, got %q", MaxIrrevocableLeasesWarning, warn)
+	}
+	if dataRaw == nil {
+		t.Fatal("expected partial data, got nil")
+	}
+
+	leaseListLength := len(dataRaw["leases"].([]*leaseResponse))
+	if leaseListLength != MaxIrrevocableLeasesToReturn {
+		t.Fatalf("expected %d results, got %d", MaxIrrevocableLeasesToReturn, leaseListLength)
+	}
+
+	dataRaw, warn, err = exp.listIrrevocableLeases(namespace.RootContext(nil), false, true, 0)
+	if err != nil {
+		t.Fatalf("got error when using limit=none: %v", err)
+	}
+	if warn != "" {
+		t.Errorf("expected no warning, got %q", warn)
+	}
+	if dataRaw == nil {
+		t.Fatalf("got nil data when using limit=none")
+	}
+
+	leaseListLength = len(dataRaw["leases"].([]*leaseResponse))
+	if leaseListLength != expectedNumLeases {
+		t.Fatalf("expected %d results, got %d", MaxIrrevocableLeasesToReturn, expectedNumLeases)
+	}
+
+	numLeasesRaw, ok := dataRaw["lease_count"]
+	if !ok {
+		t.Fatalf("lease count data not present")
+	}
+	if numLeasesRaw == nil {
+		t.Fatalf("nil lease count")
+	}
+
+	numLeases := numLeasesRaw.(int)
+	if numLeases != expectedNumLeases {
+		t.Errorf("bad lease count. expected %d, got %d", expectedNumLeases, numLeases)
+	}
+}

vault/expiration_util_common.go

@@ -0,0 +1,81 @@
+package vault
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	uuid "github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/vault/helper/namespace"
+)
+
+type basicLeaseTestInfo struct {
+	id     string
+	mount  string
+	expire time.Time
+}
+
+// add an irrevocable lease for test purposes
+// returns the lease ID and expire time
+func addIrrevocableLease(t *testing.T, m *ExpirationManager, pathPrefix string, ns *namespace.Namespace) *basicLeaseTestInfo {
+	t.Helper()
+
+	uuid, err := uuid.GenerateUUID()
+	if err != nil {
+		t.Fatalf("error generating uuid: %v", err)
+	}
+
+	if ns == nil {
+		ns = namespace.RootNamespace
+	}
+
+	nsSuffix := ""
+	if ns != namespace.RootNamespace {
+		nsSuffix = fmt.Sprintf("/blah.%s", ns.ID)
+	}
+
+	randomTimeDelta := time.Duration(rand.Int31n(24))
+	le := &leaseEntry{
+		LeaseID:    pathPrefix + "lease" + uuid + nsSuffix,
+		Path:       pathPrefix + nsSuffix,
+		namespace:  ns,
+		IssueTime:  time.Now(),
+		ExpireTime: time.Now().Add(randomTimeDelta * time.Hour),
+		RevokeErr:  "some error message",
+	}
+
+	m.pendingLock.Lock()
+	defer m.pendingLock.Unlock()
+
+	if err := m.persistEntry(context.Background(), le); err != nil {
+		t.Fatalf("error persisting irrevocable lease: %v", err)
+	}
+
+	m.updatePendingInternal(le)
+
+	return &basicLeaseTestInfo{
+		id:     le.LeaseID,
+		expire: le.ExpireTime,
+	}
+}
+
+// InjectIrrevocableLeases injects `count` irrevocable leases (currently to a
+// single mount).
+// It returns a map of the mount accessor to the number of leases stored there
+func (c *Core) InjectIrrevocableLeases(t *testing.T, ctx context.Context, count int) map[string]int {
+	out := make(map[string]int)
+	for i := 0; i < count; i++ {
+		le := addIrrevocableLease(t, c.expiration, "foo/", namespace.RootNamespace)
+
+		mountAccessor := c.expiration.getLeaseMountAccessor(ctx, le.id)
+		if _, ok := out[mountAccessor]; !ok {
+			out[mountAccessor] = 0
+		}
+
+		out[mountAccessor]++
+	}
+
+	return out
+}

vault/external_tests/expiration/expiration_test.go

@@ -0,0 +1,293 @@
+package expiration
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/hashicorp/vault/helper/namespace"
+	vaulthttp "github.com/hashicorp/vault/http"
+	"github.com/hashicorp/vault/vault"
+)
+
+func TestExpiration_irrevocableLeaseCountsAPI(t *testing.T) {
+	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
+		HandlerFunc: vaulthttp.Handler,
+		NumCores:    1,
+	})
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	client := cluster.Cores[0].Client
+	core := cluster.Cores[0].Core
+
+	params := make(map[string][]string)
+	params["type"] = []string{"irrevocable"}
+	resp, err := client.Logical().ReadWithData("sys/leases/count", params)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("response is nil")
+	}
+
+	if len(resp.Warnings) > 0 {
+		t.Errorf("expected no warnings, got: %v", resp.Warnings)
+	}
+
+	totalLeaseCountRaw, ok := resp.Data["lease_count"]
+	if !ok {
+		t.Fatalf("expected 'lease_count' response, got: %#v", resp.Data)
+	}
+
+	totalLeaseCount, err := totalLeaseCountRaw.(json.Number).Int64()
+	if err != nil {
+		t.Fatalf("error extracting lease count: %v", err)
+	}
+	if totalLeaseCount != 0 {
+		t.Errorf("expected no leases, got %d", totalLeaseCount)
+	}
+
+	countPerMountRaw, ok := resp.Data["counts"]
+	if !ok {
+		t.Fatalf("expected 'counts' response, got %#v", resp.Data)
+	}
+	countPerMount := countPerMountRaw.(map[string]interface{})
+	if len(countPerMount) != 0 {
+		t.Errorf("expected no mounts with counts, got %#v", countPerMount)
+	}
+
+	expectedNumLeases := 50
+	expectedCountPerMount := core.InjectIrrevocableLeases(t, namespace.RootContext(nil), expectedNumLeases)
+
+	resp, err = client.Logical().ReadWithData("sys/leases/count", params)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("response is nil")
+	}
+
+	if len(resp.Warnings) > 0 {
+		t.Errorf("expected no warnings, got: %v", resp.Warnings)
+	}
+
+	totalLeaseCountRaw, ok = resp.Data["lease_count"]
+	if !ok {
+		t.Fatalf("expected 'lease_count' response, got: %#v", resp.Data)
+	}
+
+	totalLeaseCount, err = totalLeaseCountRaw.(json.Number).Int64()
+	if err != nil {
+		t.Fatalf("error extracting lease count: %v", err)
+	}
+	if totalLeaseCount != int64(expectedNumLeases) {
+		t.Errorf("expected %d leases, got %d", expectedNumLeases, totalLeaseCount)
+	}
+
+	countPerMountRaw, ok = resp.Data["counts"]
+	if !ok {
+		t.Fatalf("expected 'counts' response, got %#v", resp.Data)
+	}
+
+	countPerMount = countPerMountRaw.(map[string]interface{})
+	if len(countPerMount) != len(expectedCountPerMount) {
+		t.Fatalf("expected %d mounts, got %d: %#v", len(expectedCountPerMount), len(countPerMount), countPerMount)
+	}
+
+	for mount, expectedCount := range expectedCountPerMount {
+		gotCountRaw, ok := countPerMount[mount]
+		if !ok {
+			t.Errorf("missing mount %q", mount)
+			continue
+		}
+
+		gotCount, err := gotCountRaw.(json.Number).Int64()
+		if err != nil {
+			t.Errorf("error extracting lease count for mount %q: %v", mount, err)
+			continue
+		}
+		if gotCount != int64(expectedCount) {
+			t.Errorf("bad count for mount %q: expected: %d, got: %d", mount, expectedCount, gotCount)
+		}
+	}
+}
+
+func TestExpiration_irrevocableLeaseListAPI(t *testing.T) {
+	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
+		HandlerFunc: vaulthttp.Handler,
+		NumCores:    1,
+	})
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	client := cluster.Cores[0].Client
+	core := cluster.Cores[0].Core
+
+	params := make(map[string][]string)
+	params["type"] = []string{"irrevocable"}
+	resp, err := client.Logical().ReadWithData("sys/leases", params)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("response is nil")
+	}
+
+	if len(resp.Warnings) > 0 {
+		t.Errorf("expected no warnings, got: %v", resp.Warnings)
+	}
+
+	totalLeaseCountRaw, ok := resp.Data["lease_count"]
+	if !ok {
+		t.Fatalf("expected 'lease_count' response, got: %#v", resp.Data)
+	}
+
+	totalLeaseCount, err := totalLeaseCountRaw.(json.Number).Int64()
+	if err != nil {
+		t.Fatalf("error extracting lease count: %v", err)
+	}
+	if totalLeaseCount != 0 {
+		t.Errorf("expected no leases, got %d", totalLeaseCount)
+	}
+
+	leasesRaw, ok := resp.Data["leases"]
+	if !ok {
+		t.Fatalf("expected 'leases' response, got %#v", resp.Data)
+	}
+	leases := leasesRaw.([]interface{})
+	if len(leases) != 0 {
+		t.Errorf("expected no mounts with leases, got %#v", leases)
+	}
+
+	// test with a low enough number to not give an error without limit set to none
+	expectedNumLeases := 50
+	expectedCountPerMount := core.InjectIrrevocableLeases(t, namespace.RootContext(nil), expectedNumLeases)
+
+	resp, err = client.Logical().ReadWithData("sys/leases", params)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("response is nil")
+	}
+
+	if len(resp.Warnings) > 0 {
+		t.Errorf("expected no warnings, got: %v", resp.Warnings)
+	}
+
+	totalLeaseCountRaw, ok = resp.Data["lease_count"]
+	if !ok {
+		t.Fatalf("expected 'lease_count' response, got: %#v", resp.Data)
+	}
+
+	totalLeaseCount, err = totalLeaseCountRaw.(json.Number).Int64()
+	if err != nil {
+		t.Fatalf("error extracting lease count: %v", err)
+	}
+	if totalLeaseCount != int64(expectedNumLeases) {
+		t.Errorf("expected %d leases, got %d", expectedNumLeases, totalLeaseCount)
+	}
+
+	leasesRaw, ok = resp.Data["leases"]
+	if !ok {
+		t.Fatalf("expected 'leases' response, got %#v", resp.Data)
+	}
+
+	leases = leasesRaw.([]interface{})
+	countPerMount := make(map[string]int)
+	for _, leaseRaw := range leases {
+		lease := leaseRaw.(map[string]interface{})
+		mount := lease["mount_id"].(string)
+
+		if _, ok := countPerMount[mount]; !ok {
+			countPerMount[mount] = 0
+		}
+
+		countPerMount[mount]++
+	}
+
+	if !reflect.DeepEqual(countPerMount, expectedCountPerMount) {
+		t.Errorf("bad mount count. expected %v, got %v", expectedCountPerMount, countPerMount)
+	}
+}
+
+func TestExpiration_irrevocableLeaseListAPI_includeAll(t *testing.T) {
+	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
+		HandlerFunc: vaulthttp.Handler,
+		NumCores:    1,
+	})
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	client := cluster.Cores[0].Client
+	core := cluster.Cores[0].Core
+
+	// test with a low enough number to not give an error with the default limit
+	expectedNumLeases := vault.MaxIrrevocableLeasesToReturn + 50
+	expectedCountPerMount := core.InjectIrrevocableLeases(t, namespace.RootContext(nil), expectedNumLeases)
+
+	params := make(map[string][]string)
+	params["type"] = []string{"irrevocable"}
+
+	resp, err := client.Logical().ReadWithData("sys/leases", params)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if resp == nil {
+		t.Fatal("unexpected nil response")
+	}
+
+	if len(resp.Warnings) != 1 {
+		t.Errorf("expected one warning (%q), got: %v", vault.MaxIrrevocableLeasesWarning, resp.Warnings)
+	}
+
+	// now try it with the no limit on return size - we expect no errors and many results
+	params["limit"] = []string{"none"}
+	resp, err = client.Logical().ReadWithData("sys/leases", params)
+	if err != nil {
+		t.Fatalf("unexpected error when using limit=none: %v", err)
+	}
+	if resp == nil {
+		t.Fatal("response is nil")
+	}
+
+	if len(resp.Warnings) > 0 {
+		t.Errorf("expected no warnings, got: %v", resp.Warnings)
+	}
+
+	totalLeaseCountRaw, ok := resp.Data["lease_count"]
+	if !ok {
+		t.Fatalf("expected 'lease_count' response, got: %#v", resp.Data)
+	}
+
+	totalLeaseCount, err := totalLeaseCountRaw.(json.Number).Int64()
+	if err != nil {
+		t.Fatalf("error extracting lease count: %v", err)
+	}
+	if totalLeaseCount != int64(expectedNumLeases) {
+		t.Errorf("expected %d leases, got %d", expectedNumLeases, totalLeaseCount)
+	}
+
+	leasesRaw, ok := resp.Data["leases"]
+	if !ok {
+		t.Fatalf("expected 'leases' response, got %#v", resp.Data)
+	}
+
+	leases := leasesRaw.([]interface{})
+	countPerMount := make(map[string]int)
+	for _, leaseRaw := range leases {
+		lease := leaseRaw.(map[string]interface{})
+		mount := lease["mount_id"].(string)
+
+		if _, ok := countPerMount[mount]; !ok {
+			countPerMount[mount] = 0
+		}
+
+		countPerMount[mount]++
+	}
+
+	if !reflect.DeepEqual(countPerMount, expectedCountPerMount) {
+		t.Errorf("bad mount count. expected %v, got %v", expectedCountPerMount, countPerMount)
+	}
+}

vault/logical_system.go

@@ -285,6 +285,84 @@ func (b *SystemBackend) handleTidyLeases(ctx context.Context, req *logical.Reque
 	return logical.RespondWithStatusCode(resp, req, http.StatusAccepted)
 }
 
+func (b *SystemBackend) handleLeaseCount(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+	typeRaw, ok := d.GetOk("type")
+	if !ok || strings.ToLower(typeRaw.(string)) != "irrevocable" {
+		return nil, nil
+	}
+
+	includeChildNamespacesRaw, ok := d.GetOk("include_child_namespaces")
+	includeChildNamespaces := ok && includeChildNamespacesRaw.(bool)
+
+	resp, err := b.Core.expiration.getIrrevocableLeaseCounts(ctx, includeChildNamespaces)
+	if err != nil {
+		return nil, err
+	}
+
+	return &logical.Response{
+		Data: resp,
+	}, nil
+}
+
+func processLimit(d *framework.FieldData) (bool, int, error) {
+	limitStr := ""
+	limitRaw, ok := d.GetOk("limit")
+	if ok {
+		limitStr = limitRaw.(string)
+	}
+
+	includeAll := false
+	maxResults := MaxIrrevocableLeasesToReturn
+	if limitStr == "" {
+		// use the defaults
+	} else if strings.ToLower(limitStr) == "none" {
+		includeAll = true
+	} else {
+		// not having a valid, positive int here is an error
+		limit, err := strconv.Atoi(limitStr)
+		if err != nil {
+			return false, 0, fmt.Errorf("invalid 'limit' provided: %w", err)
+		}
+
+		if limit < 1 {
+			return false, 0, fmt.Errorf("limit must be 'none' or a positive integer")
+		}
+
+		maxResults = limit
+	}
+
+	return includeAll, maxResults, nil
+}
+
+func (b *SystemBackend) handleLeaseList(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+	typeRaw, ok := d.GetOk("type")
+	if !ok || strings.ToLower(typeRaw.(string)) != "irrevocable" {
+		return nil, nil
+	}
+
+	includeChildNamespacesRaw, ok := d.GetOk("include_child_namespaces")
+	includeChildNamespaces := ok && includeChildNamespacesRaw.(bool)
+
+	includeAll, maxResults, err := processLimit(d)
+	if err != nil {
+		return nil, err
+	}
+
+	leases, warning, err := b.Core.expiration.listIrrevocableLeases(ctx, includeChildNamespaces, includeAll, maxResults)
+	if err != nil {
+		return nil, err
+	}
+
+	resp := &logical.Response{
+		Data: leases,
+	}
+	if warning != "" {
+		resp.AddWarning(warning)
+	}
+
+	return resp, nil
+}
+
 func (b *SystemBackend) handlePluginCatalogTypedList(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	pluginType, err := consts.ParsePluginType(d.Get("type").(string))
 	if err != nil {
@@ -4688,4 +4766,12 @@ This path responds to the following HTTP methods.
 		"Control the collection and reporting of client counts.",
 		"Control the collection and reporting of client counts.",
 	},
+	"count-leases": {
+		"Count of leases associated with this Vault cluster",
+		"Count of leases associated with this Vault cluster",
+	},
+	"list-leases": {
+		"List leases associated with this Vault cluster",
+		"List leases associated with this Vault cluster",
+	},
 }

vault/logical_system_paths.go

@@ -1210,6 +1210,59 @@ func (b *SystemBackend) leasePaths() []*framework.Path {
 			HelpSynopsis:    strings.TrimSpace(sysHelp["tidy_leases"][0]),
 			HelpDescription: strings.TrimSpace(sysHelp["tidy_leases"][1]),
 		},
+
+		{
+			Pattern: "leases/count$",
+			Fields: map[string]*framework.FieldSchema{
+				"type": {
+					Type:        framework.TypeString,
+					Required:    true,
+					Description: "Type of leases to get counts for (currently only supporting irrevocable).",
+				},
+				"include_child_namespaces": {
+					Type:        framework.TypeBool,
+					Default:     false,
+					Description: "Set true if you want counts for this namespace and its children.",
+				},
+			},
+
+			Callbacks: map[logical.Operation]framework.OperationFunc{
+				// currently only works for irrevocable leases with param: type=irrevocable
+				logical.ReadOperation: b.handleLeaseCount,
+			},
+
+			HelpSynopsis:    strings.TrimSpace(sysHelp["count-leases"][0]),
+			HelpDescription: strings.TrimSpace(sysHelp["count-leases"][1]),
+		},
+
+		{
+			Pattern: "leases(/)?$",
+			Fields: map[string]*framework.FieldSchema{
+				"type": {
+					Type:        framework.TypeString,
+					Required:    true,
+					Description: "Type of leases to retrieve (currently only supporting irrevocable).",
+				},
+				"include_child_namespaces": {
+					Type:        framework.TypeBool,
+					Default:     false,
+					Description: "Set true if you want leases for this namespace and its children.",
+				},
+				"limit": {
+					Type:        framework.TypeString,
+					Default:     "",
+					Description: "Set to a positive integer of the maximum number of entries to return. If you want all results, set to 'none'. If not set, you will get a maximum of 10,000 results returned.",
+				},
+			},
+
+			Callbacks: map[logical.Operation]framework.OperationFunc{
+				// currently only works for irrevocable leases with param: type=irrevocable
+				logical.ReadOperation: b.handleLeaseList,
+			},
+
+			HelpSynopsis:    strings.TrimSpace(sysHelp["list-leases"][0]),
+			HelpDescription: strings.TrimSpace(sysHelp["list-leases"][1]),
+		},
 	}
 }
 

vault/logical_system_test.go

@@ -3580,3 +3580,89 @@ func makeStorage(t *testing.T, entries ...*logical.StorageEntry) *logical.InmemS
 
 	return store
 }
+
+func leaseLimitFieldData(limit string) *framework.FieldData {
+	raw := make(map[string]interface{})
+	raw["limit"] = limit
+	return &framework.FieldData{
+		Raw: raw,
+		Schema: map[string]*framework.FieldSchema{
+			"limit": {
+				Type:        framework.TypeString,
+				Default:     "",
+				Description: "limit return results",
+			},
+		},
+	}
+}
+
+func TestProcessLimit(t *testing.T) {
+	testCases := []struct {
+		d               *framework.FieldData
+		expectReturnAll bool
+		expectLimit     int
+		expectErr       bool
+	}{
+		{
+			d:               leaseLimitFieldData("500"),
+			expectReturnAll: false,
+			expectLimit:     500,
+			expectErr:       false,
+		},
+		{
+			d:               leaseLimitFieldData(""),
+			expectReturnAll: false,
+			expectLimit:     MaxIrrevocableLeasesToReturn,
+			expectErr:       false,
+		},
+		{
+			d:               leaseLimitFieldData("none"),
+			expectReturnAll: true,
+			expectLimit:     10000,
+			expectErr:       false,
+		},
+		{
+			d:               leaseLimitFieldData("NoNe"),
+			expectReturnAll: true,
+			expectLimit:     10000,
+			expectErr:       false,
+		},
+		{
+			d:               leaseLimitFieldData("hello_world"),
+			expectReturnAll: false,
+			expectLimit:     0,
+			expectErr:       true,
+		},
+		{
+			d:               leaseLimitFieldData("0"),
+			expectReturnAll: false,
+			expectLimit:     0,
+			expectErr:       true,
+		},
+		{
+			d:               leaseLimitFieldData("-1"),
+			expectReturnAll: false,
+			expectLimit:     0,
+			expectErr:       true,
+		},
+	}
+
+	for i, tc := range testCases {
+		returnAll, limit, err := processLimit(tc.d)
+
+		if returnAll != tc.expectReturnAll {
+			t.Errorf("bad return all for test case %d. expected %t, got %t", i, tc.expectReturnAll, returnAll)
+		}
+		if limit != tc.expectLimit {
+			t.Errorf("bad limit for test case %d. expected %d, got %d", i, tc.expectLimit, limit)
+		}
+
+		haveErr := err != nil
+		if haveErr != tc.expectErr {
+			t.Errorf("bad error status for test case %d. expected error: %t, got error: %t", i, tc.expectErr, haveErr)
+			if err != nil {
+				t.Errorf("error was: %v", err)
+			}
+		}
+	}
+}

website/content/api-docs/system/leases.mdx

@@ -243,3 +243,68 @@ $ curl \
     --request POST \
     http://127.0.0.1:8200/v1/sys/leases/tidy
 ```
+
+## Lease Counts
+
+This endpoint returns the total count of a `type` of lease, as well as a count
+per mount point. Note that it currently only supports type "irrevocable".
+
+This can help determine if particular endpoints are disproportionately
+resulting in irrevocable leases.
+
+This endpoint was added in Vault 1.8.
+
+### Parameters
+
+- `type` (string: <required>) - Specifies the type of lease.
+- `include_child_namespaces` (bool: false) - Specifies if leases in child
+    namespaces should be included in the result.
+
+| Method | Path               |
+| :----- | :----------------- |
+| `GET`  | `/sys/leases/count`|
+
+### Sample Request
+
+```shell-session
+$ curl \
+    --header "X-Vault-Token: ..." \
+    --request GET \
+    http://127.0.0.1:8200/v1/sys/leases/count \
+    -d type=irrevocable
+```
+
+## Leases List
+
+This endpoint returns the total count of a `type` of lease, as well as a list
+of leases per mount point. Note that it currently only supports type
+"irrevocable".
+
+This can help determine if particular endpoints or causes are disproportionately
+resulting in irrevocable leases.
+
+This endpoint was added in Vault 1.8.
+
+### Parameters
+
+- `type` (string: <required>) - Specifies the type of lease.
+- `include_child_namespaces` (bool: false) - Specifies if leases in child
+   namespaces should be included in the result
+- `limit` (string: "") - Specifies the maximum number of leases to return in a
+   request. To return all results, set to `none`. If not set, this API will
+   return a maximum of 10,000 leases. If not set to `none` and there exist more
+   leases than `limit`, the response will include a warning.
+
+| Method | Path          |
+| :----- | :------------ |
+| `GET`  | `/sys/leases` |
+
+### Sample Request
+
+```shell-session
+$ curl \
+    --header "X-Vault-Token: ..." \
+    --request GET \
+    http://127.0.0.1:8200/v1/sys/leases \
+    -d type=irrevocable
+```