Add Partial Month Client Count API for Activity Log (#11022)

* sketch out partial month activity log client API

* unit test partialMonthClientCount

* cleanup api

* add api doc, fix test, update api nomenclature to match existing

* cleanup

* add PR changelog file

* integration test for API

* report entities and tokens separately

changelog/11022.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core: add partial month client count api
+```

vault/activity_log.go

@@ -1771,3 +1771,29 @@ func (c *Core) activeEntityGaugeCollector(ctx context.Context) ([]metricsutil.Ga
 	}
 	return a.PartialMonthMetrics(ctx)
 }
+
+// partialMonthClientCount returns the number of clients used so far this month.
+// If activity log is not enabled, the response will be nil
+func (a *ActivityLog) partialMonthClientCount(ctx context.Context) map[string]interface{} {
+	a.fragmentLock.RLock()
+	defer a.fragmentLock.RUnlock()
+
+	if !a.enabled {
+		// nothing to count
+		return nil
+	}
+
+	entityCount := len(a.activeEntities)
+	var tokenCount int
+	for _, countByNS := range a.currentSegment.tokenCount.CountByNamespaceID {
+		tokenCount += int(countByNS)
+	}
+	clientCount := entityCount + tokenCount
+
+	responseData := make(map[string]interface{})
+	responseData["distinct_entities"] = entityCount
+	responseData["non_entity_tokens"] = tokenCount
+	responseData["clients"] = clientCount
+
+	return responseData
+}

vault/activity_log_test.go

@@ -2435,3 +2435,56 @@ func TestActivityLog_Deletion(t *testing.T) {
 	checkPresent(21)
 
 }
+
+func TestActivityLog_partialMonthClientCount(t *testing.T) {
+	timeutil.SkipAtEndOfMonth(t)
+
+	ctx := context.Background()
+	now := time.Now().UTC()
+	a, entities, tokenCounts := setupActivityRecordsInStorage(t, timeutil.StartOfMonth(now), true, true)
+
+	a.SetEnable(true)
+	var wg sync.WaitGroup
+	err := a.refreshFromStoredLog(ctx, &wg, now)
+	if err != nil {
+		t.Fatalf("error loading clients: %v", err)
+	}
+	wg.Wait()
+
+	// entities[0] is from a previous month
+	partialMonthEntityCount := len(entities[1:])
+	var partialMonthTokenCount int
+	for _, countByNS := range tokenCounts {
+		partialMonthTokenCount += int(countByNS)
+	}
+
+	expectedClientCount := partialMonthEntityCount + partialMonthTokenCount
+
+	results := a.partialMonthClientCount(ctx)
+	if results == nil {
+		t.Fatal("no results to test")
+	}
+
+	entityCount, ok := results["distinct_entities"]
+	if !ok {
+		t.Fatalf("malformed results. got %v", results)
+	}
+	if entityCount != partialMonthEntityCount {
+		t.Errorf("bad entity count. expected %d, got %d", partialMonthEntityCount, entityCount)
+	}
+
+	tokenCount, ok := results["non_entity_tokens"]
+	if !ok {
+		t.Fatalf("malformed results. got %v", results)
+	}
+	if tokenCount != partialMonthTokenCount {
+		t.Errorf("bad token count. expected %d, got %d", partialMonthTokenCount, tokenCount)
+	}
+	clientCount, ok := results["clients"]
+	if !ok {
+		t.Fatalf("malformed results. got %v", results)
+	}
+	if clientCount != expectedClientCount {
+		t.Errorf("bad client count. expected %d, got %d", expectedClientCount, clientCount)
+	}
+}

vault/activity_log_testing_util.go

@@ -8,6 +8,34 @@ import (
 	"github.com/hashicorp/vault/vault/activity"
 )
 
+// InjectActivityLogDataThisMonth populates the in-memory client store
+// with some entities and tokens, overriding what was already there
+// It is currently used for API integration tests
+func (c *Core) InjectActivityLogDataThisMonth(t *testing.T) (map[string]struct{}, map[string]uint64) {
+	t.Helper()
+
+	activeEntities := map[string]struct{}{
+		"entity0": struct{}{},
+		"entity1": struct{}{},
+		"entity2": struct{}{},
+	}
+	tokens := map[string]uint64{
+		"ns0": 5,
+		"ns1": 1,
+		"ns2": 10,
+	}
+
+	c.activityLog.l.Lock()
+	defer c.activityLog.l.Unlock()
+	c.activityLog.fragmentLock.Lock()
+	defer c.activityLog.fragmentLock.Unlock()
+
+	c.activityLog.activeEntities = activeEntities
+	c.activityLog.currentSegment.tokenCount.CountByNamespaceID = tokens
+
+	return activeEntities, tokens
+}
+
 // Return the in-memory activeEntities from an activity log
 func (c *Core) GetActiveEntities() map[string]struct{} {
 	out := make(map[string]struct{})
@@ -171,3 +199,9 @@ func (a *ActivityLog) GetEnabled() bool {
 	defer a.fragmentLock.RUnlock()
 	return a.enabled
 }
+
+// GetActivityLog returns a pointer to the (private) activity log on a core
+// Note: you must do the usual locking scheme when modifying the ActivityLog
+func (c *Core) GetActivityLog() *ActivityLog {
+	return c.activityLog
+}

vault/external_tests/activity/activity_test.go

@@ -0,0 +1,116 @@
+package activity
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/hashicorp/vault/api"
+	"github.com/hashicorp/vault/builtin/credential/userpass"
+	"github.com/hashicorp/vault/helper/timeutil"
+	vaulthttp "github.com/hashicorp/vault/http"
+	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/hashicorp/vault/vault"
+)
+
+func validateClientCounts(t *testing.T, resp *api.Secret, expectedEntities, expectedTokens int) {
+	if resp == nil {
+		t.Fatal("nil response")
+	}
+	if resp.Data == nil {
+		t.Fatal("no data")
+	}
+
+	expectedClients := expectedEntities + expectedTokens
+
+	entityCountJSON, ok := resp.Data["distinct_entities"]
+	if !ok {
+		t.Fatalf("no entity count: %v", resp.Data)
+	}
+	entityCount, err := entityCountJSON.(json.Number).Int64()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if entityCount != int64(expectedEntities) {
+		t.Errorf("bad entity count. expected %v, got %v", expectedEntities, entityCount)
+	}
+
+	tokenCountJSON, ok := resp.Data["non_entity_tokens"]
+	if !ok {
+		t.Fatalf("no token count: %v", resp.Data)
+	}
+	tokenCount, err := tokenCountJSON.(json.Number).Int64()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tokenCount != int64(expectedTokens) {
+		t.Errorf("bad token count. expected %v, got %v", expectedTokens, tokenCount)
+	}
+
+	clientCountJSON, ok := resp.Data["clients"]
+	if !ok {
+		t.Fatalf("no client count: %v", resp.Data)
+	}
+	clientCount, err := clientCountJSON.(json.Number).Int64()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if clientCount != int64(expectedClients) {
+		t.Errorf("bad client count. expected %v, got %v", expectedClients, clientCount)
+	}
+}
+
+func TestActivityLog_MonthlyActivityApi(t *testing.T) {
+	timeutil.SkipAtEndOfMonth(t)
+
+	coreConfig := &vault.CoreConfig{
+		CredentialBackends: map[string]logical.Factory{
+			"userpass": userpass.Factory,
+		},
+		ActivityLogConfig: vault.ActivityLogCoreConfig{
+			ForceEnable: true,
+		},
+	}
+	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
+		HandlerFunc: vaulthttp.Handler,
+	})
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	client := cluster.Cores[0].Client
+	core := cluster.Cores[0].Core
+
+	resp, err := client.Logical().Read("sys/internal/counters/activity/monthly")
+	if err != nil {
+		t.Fatal(err)
+	}
+	validateClientCounts(t, resp, 0, 0)
+
+	// inject some data and query the API
+	entities, tokens := core.InjectActivityLogDataThisMonth(t)
+	expectedEntities := len(entities)
+	var expectedTokens int
+	for _, tokenCount := range tokens {
+		expectedTokens += int(tokenCount)
+	}
+
+	resp, err = client.Logical().Read("sys/internal/counters/activity/monthly")
+	if err != nil {
+		t.Fatal(err)
+	}
+	validateClientCounts(t, resp, expectedEntities, expectedTokens)
+
+	// we expect a 204 if activity log is disabled
+	core.GetActivityLog().SetEnable(false)
+	req := client.NewRequest("GET", "/v1/sys/internal/counters/activity/monthly")
+	rawResp, err := client.RawRequest(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rawResp == nil {
+		t.Error("nil response")
+	}
+	if rawResp.StatusCode != http.StatusNoContent {
+		t.Errorf("expected status code %v, got %v", http.StatusNoContent, rawResp.StatusCode)
+	}
+}

vault/logical_system.go

@@ -4670,6 +4670,10 @@ This path responds to the following HTTP methods.
 		"Query the historical count of clients.",
 		"Query the historical count of clients.",
 	},
+	"activity-monthly": {
+		"Count of active clients so far this month.",
+		"Count of active clients so far this month.",
+	},
 	"activity-config": {
 		"Control the collection and reporting of client counts.",
 		"Control the collection and reporting of client counts.",

vault/logical_system_activity.go

@@ -38,10 +38,26 @@ func (b *SystemBackend) activityQueryPath() *framework.Path {
 	}
 }
 
+// monthlyActivityCountPath is available in every namespace
+func (b *SystemBackend) monthlyActivityCountPath() *framework.Path {
+	return &framework.Path{
+		Pattern:         "internal/counters/activity/monthly",
+		HelpSynopsis:    strings.TrimSpace(sysHelp["activity-monthly"][0]),
+		HelpDescription: strings.TrimSpace(sysHelp["activity-monthly"][1]),
+		Operations: map[logical.Operation]framework.OperationHandler{
+			logical.ReadOperation: &framework.PathOperation{
+				Callback: b.handleMonthlyActivityCount,
+				Summary:  "Report the number of clients for this month, for this namespace and all child namespaces.",
+			},
+		},
+	}
+}
+
 // rootActivityPaths are available only in the root namespace
 func (b *SystemBackend) rootActivityPaths() []*framework.Path {
 	return []*framework.Path{
 		b.activityQueryPath(),
+		b.monthlyActivityCountPath(),
 		{
 			Pattern: "internal/counters/config$",
 			Fields: map[string]*framework.FieldSchema{
@@ -120,6 +136,22 @@ func (b *SystemBackend) handleClientMetricQuery(ctx context.Context, req *logica
 	}, nil
 }
 
+func (b *SystemBackend) handleMonthlyActivityCount(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+	a := b.Core.activityLog
+	if a == nil {
+		return logical.ErrorResponse("no activity log present"), nil
+	}
+
+	results := a.partialMonthClientCount(ctx)
+	if results == nil {
+		return logical.RespondWithStatusCode(nil, req, http.StatusNoContent)
+	}
+
+	return &logical.Response{
+		Data: results,
+	}, nil
+}
+
 func (b *SystemBackend) handleActivityConfigRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	a := b.Core.activityLog
 	if a == nil {

website/content/api-docs/system/internal-counters.mdx

@@ -233,6 +233,51 @@ $ curl \
     http://127.0.0.1:8200/v1/sys/internal/counters/activity?end_time=2020-06-30T00%3A00%3A00Z&start_time=2020-06-01T00%3A00%3A00Z
 ```
 
+## Partial Month Client Count
+
+This endpoint returns the number of clients for the current month, as the sum of active entities and non-entity tokens.
+An "active entity" is a distinct entity that has created one or more tokens in the given time period.
+A "non-entity token" is a token with no attached entity ID.
+
+The time period is from the start of the current month, up until the time that this request was made.
+
+Note: the client count may be inaccurate in the moments following a Vault reboot, or leadership change.
+The estimate will stabilize when background loading of client data has completed.
+
+This endpoint was added in Vault 1.6.4.
+
+| Method | Path                              |
+| :----- | :-------------------------------- |
+| `GET`  | `/sys/internal/counters/activity/monthly` |
+
+### Sample Request
+
+```shell-session
+$ curl \
+    --header "X-Vault-Token: ..." \
+    --request GET \
+    http://127.0.0.1:8200/v1/sys/internal/counters/activity/monthly
+```
+
+### Sample Response
+
+```json
+{
+  "request_id": "26be5ab9-dcac-9237-ec12-269a8ca64742",
+  "lease_id": "",
+  "renewable": false,
+  "lease_duration": 0,
+  "data": {
+    "distinct_entities": 100,
+    "non_entity_tokens": 120,
+    "clients": 220,
+  },
+  "wrap_info": null,
+  "warnings": null,
+  "auth": null
+}
+```
+
 ## Update the Client Count Configuration
 
 The `/sys/internal/counters/config` endpoint is used to configure logging of active clients.