220ca06201
This endpoint shows total services, connect service instances and billable service instances in the local datacenter or globally. Billable instances = total service instances - connect services - consul server instances.
523 lines
17 KiB
Go
523 lines
17 KiB
Go
package state
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/hashicorp/go-memdb"
|
|
|
|
"github.com/hashicorp/consul/agent/structs"
|
|
)
|
|
|
|
const (
|
|
serviceNamesUsageTable = "service-names"
|
|
kvUsageTable = "kv-entries"
|
|
connectNativeInstancesTable = "connect-native"
|
|
connectPrefix = "connect-mesh"
|
|
|
|
tableUsage = "usage"
|
|
)
|
|
|
|
var allConnectKind = []string{
|
|
string(structs.ServiceKindConnectProxy),
|
|
string(structs.ServiceKindIngressGateway),
|
|
string(structs.ServiceKindMeshGateway),
|
|
string(structs.ServiceKindTerminatingGateway),
|
|
connectNativeInstancesTable,
|
|
}
|
|
|
|
// usageTableSchema returns a new table schema used for tracking various indexes
|
|
// for the Raft log.
|
|
func usageTableSchema() *memdb.TableSchema {
|
|
return &memdb.TableSchema{
|
|
Name: tableUsage,
|
|
Indexes: map[string]*memdb.IndexSchema{
|
|
indexID: {
|
|
Name: indexID,
|
|
AllowMissing: false,
|
|
Unique: true,
|
|
Indexer: &memdb.StringFieldIndex{
|
|
Field: "ID",
|
|
Lowercase: true,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// UsageEntry represents a count of some arbitrary identifier within the
|
|
// state store, along with the last seen index.
|
|
type UsageEntry struct {
|
|
ID string
|
|
Index uint64
|
|
Count int
|
|
}
|
|
|
|
// NodeUsage contains all of the usage data related to nodes
|
|
type NodeUsage struct {
|
|
Nodes int
|
|
EnterpriseNodeUsage
|
|
}
|
|
|
|
// PeeringUsage contains all of the usage data related to peerings.
|
|
type PeeringUsage struct {
|
|
// Number of peerings.
|
|
Peerings int
|
|
EnterprisePeeringUsage
|
|
}
|
|
|
|
type KVUsage struct {
|
|
KVCount int
|
|
EnterpriseKVUsage
|
|
}
|
|
|
|
type ConfigEntryUsage struct {
|
|
ConfigByKind map[string]int
|
|
EnterpriseConfigEntryUsage
|
|
}
|
|
|
|
type uniqueServiceState int
|
|
|
|
const (
|
|
NoChange uniqueServiceState = 0
|
|
Deleted uniqueServiceState = 1
|
|
Created uniqueServiceState = 2
|
|
)
|
|
|
|
// updateUsage takes a set of memdb changes and computes a delta for specific
|
|
// usage metrics that we track.
|
|
func updateUsage(tx WriteTxn, changes Changes) error {
|
|
usageDeltas := make(map[string]int)
|
|
serviceNameChanges := make(map[structs.ServiceName]int)
|
|
for _, change := range changes.Changes {
|
|
var delta int
|
|
if change.Created() {
|
|
delta = 1
|
|
} else if change.Deleted() {
|
|
delta = -1
|
|
}
|
|
|
|
switch change.Table {
|
|
case tableNodes:
|
|
node := changeObject(change).(*structs.Node)
|
|
if node.PeerName != "" {
|
|
// TODO(peering) track peered nodes separately. For now not tracking to avoid double billing.
|
|
continue
|
|
}
|
|
usageDeltas[change.Table] += delta
|
|
addEnterpriseNodeUsage(usageDeltas, change)
|
|
|
|
case tablePeering:
|
|
usageDeltas[change.Table] += delta
|
|
addEnterprisePeeringUsage(usageDeltas, change)
|
|
|
|
case tableServices:
|
|
svc := changeObject(change).(*structs.ServiceNode)
|
|
if svc.PeerName != "" {
|
|
// TODO(peering) track peered services separately. For now not tracking to avoid double billing.
|
|
continue
|
|
}
|
|
usageDeltas[change.Table] += delta
|
|
addEnterpriseServiceInstanceUsage(usageDeltas, change)
|
|
|
|
connectDeltas(change, usageDeltas, delta)
|
|
billableServiceInstancesDeltas(change, usageDeltas, delta)
|
|
|
|
// Construct a mapping of all of the various service names that were
|
|
// changed, in order to compare it with the finished memdb state.
|
|
// Make sure to account for the fact that services can change their names.
|
|
if serviceNameChanged(change) {
|
|
serviceNameChanges[svc.CompoundServiceName()] += 1
|
|
before := change.Before.(*structs.ServiceNode)
|
|
serviceNameChanges[before.CompoundServiceName()] -= 1
|
|
} else {
|
|
serviceNameChanges[svc.CompoundServiceName()] += delta
|
|
}
|
|
|
|
case "kvs":
|
|
usageDeltas[change.Table] += delta
|
|
addEnterpriseKVUsage(usageDeltas, change)
|
|
case tableConfigEntries:
|
|
entry := changeObject(change).(structs.ConfigEntry)
|
|
usageDeltas[configEntryUsageTableName(entry.GetKind())] += delta
|
|
addEnterpriseConfigEntryUsage(usageDeltas, change)
|
|
}
|
|
}
|
|
|
|
serviceStates, err := updateServiceNameUsage(tx, usageDeltas, serviceNameChanges)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
addEnterpriseServiceUsage(usageDeltas, serviceStates)
|
|
|
|
idx := changes.Index
|
|
// This will happen when restoring from a snapshot, just take the max index
|
|
// of the tables we are tracking.
|
|
if idx == 0 {
|
|
// TODO(partitions? namespaces?)
|
|
idx = maxIndexTxn(tx, tableNodes, tableServices, "kvs")
|
|
}
|
|
|
|
return writeUsageDeltas(tx, idx, usageDeltas)
|
|
}
|
|
|
|
func updateServiceNameUsage(tx WriteTxn, usageDeltas map[string]int, serviceNameChanges map[structs.ServiceName]int) (map[structs.ServiceName]uniqueServiceState, error) {
|
|
serviceStates := make(map[structs.ServiceName]uniqueServiceState, len(serviceNameChanges))
|
|
for svc, delta := range serviceNameChanges {
|
|
q := Query{
|
|
Value: svc.Name,
|
|
EnterpriseMeta: svc.EnterpriseMeta,
|
|
}
|
|
serviceIter, err := tx.Get(tableServices, indexService, q)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Count the number of service instances associated with the given service
|
|
// name at the end of this transaction, and compare that with how many were
|
|
// added/removed during the transaction. This allows us to handle a single
|
|
// transaction committing multiple changes related to a single service
|
|
// name.
|
|
var count int
|
|
for service := serviceIter.Next(); service != nil; service = serviceIter.Next() {
|
|
count += 1
|
|
}
|
|
|
|
var serviceState uniqueServiceState
|
|
switch {
|
|
case count == 0:
|
|
// If no services exist, we know we deleted the last service
|
|
// instance.
|
|
serviceState = Deleted
|
|
usageDeltas[serviceNamesUsageTable] -= 1
|
|
case count == delta:
|
|
// If the current number of service instances equals the number added,
|
|
// than we know we created a new service name.
|
|
serviceState = Created
|
|
usageDeltas[serviceNamesUsageTable] += 1
|
|
default:
|
|
serviceState = NoChange
|
|
}
|
|
|
|
serviceStates[svc] = serviceState
|
|
}
|
|
|
|
return serviceStates, nil
|
|
}
|
|
|
|
// serviceNameChanged returns a boolean that indicates whether the
|
|
// provided change resulted in an update to the service's service name.
|
|
func serviceNameChanged(change memdb.Change) bool {
|
|
if change.Updated() {
|
|
before := change.Before.(*structs.ServiceNode)
|
|
after := change.After.(*structs.ServiceNode)
|
|
return before.ServiceName != after.ServiceName
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// connectUsageTableEntry is a convenience function to make prefix addition in 1 place
|
|
func connectUsageTableName(kind string) string {
|
|
return fmt.Sprintf("%s-%s", connectPrefix, kind)
|
|
}
|
|
|
|
// configEntryUsageTableName is a convenience function to easily get the prefix + config entry kind in 1 place
|
|
func configEntryUsageTableName(kind string) string {
|
|
return fmt.Sprintf("%s-%s", tableConfigEntries, kind)
|
|
}
|
|
|
|
func connectDeltas(change memdb.Change, usageDeltas map[string]int, delta int) {
|
|
// Connect metrics for updated services are more complicated. Check for:
|
|
// 1. Did ServiceKind change?
|
|
// 2. Is before ServiceKind typical? don't remove from old service kind
|
|
// 3. Is After ServiceKind typical? don't add to new service kind
|
|
// 4. Add and remove to both ServiceKind's
|
|
if change.Updated() {
|
|
before := change.Before.(*structs.ServiceNode)
|
|
after := change.After.(*structs.ServiceNode)
|
|
if before.ServiceKind != structs.ServiceKindTypical {
|
|
usageDeltas[connectUsageTableName(string(before.ServiceKind))] -= 1
|
|
addEnterpriseConnectServiceInstanceUsage(usageDeltas, before, -1)
|
|
}
|
|
if after.ServiceKind != structs.ServiceKindTypical {
|
|
usageDeltas[connectUsageTableName(string(after.ServiceKind))] += 1
|
|
addEnterpriseConnectServiceInstanceUsage(usageDeltas, after, 1)
|
|
}
|
|
|
|
if before.ServiceConnect.Native != after.ServiceConnect.Native {
|
|
if before.ServiceConnect.Native {
|
|
usageDeltas[connectUsageTableName(string(connectNativeInstancesTable))] -= 1
|
|
addEnterpriseConnectServiceInstanceUsage(usageDeltas, before, -1)
|
|
} else {
|
|
usageDeltas[connectUsageTableName(connectNativeInstancesTable)] += 1
|
|
addEnterpriseConnectServiceInstanceUsage(usageDeltas, after, 1)
|
|
}
|
|
}
|
|
} else {
|
|
svc := changeObject(change).(*structs.ServiceNode)
|
|
if svc.ServiceKind != structs.ServiceKindTypical {
|
|
usageDeltas[connectUsageTableName(string(svc.ServiceKind))] += delta
|
|
}
|
|
if svc.ServiceConnect.Native {
|
|
usageDeltas[connectUsageTableName(connectNativeInstancesTable)] += delta
|
|
}
|
|
addEnterpriseConnectServiceInstanceUsage(usageDeltas, svc, delta)
|
|
}
|
|
}
|
|
|
|
// billableServiceInstancesDeltas calculates deltas for the billable services. Billable services
|
|
// are of "typical" service kind (i.e. non-connect or connect-native), excluding the "consul" service.
|
|
func billableServiceInstancesDeltas(change memdb.Change, usageDeltas map[string]int, delta int) {
|
|
// Billable service instances = # of typical service instances (i.e. non-connect) + connect-native service instances.
|
|
// Specifically, it should exclude "consul" service instances from the count.
|
|
//
|
|
// If the service has been updated, then we check
|
|
// 1. If the service name changed to or from "consul" and update deltas such that we exclude consul server service instances.
|
|
// This case is a bit contrived because we don't expect consul service to change once it's registered (beyond changing its instance count).
|
|
// a) If changed to "consul" -> decrement deltas by one
|
|
// b) If changed from "consul" and it's not a "connect" service -> increase deltas by one
|
|
// 2. If the service kind changed to or from "typical", we need to we need to update deltas so that we only account
|
|
// for non-connect or connect-native instances.
|
|
if change.Updated() {
|
|
// When there's an update, the delta arg passed to this function is 0, and so we need to explicitly increment
|
|
// or decrement by 1 depending on the situation.
|
|
before := change.Before.(*structs.ServiceNode)
|
|
after := change.After.(*structs.ServiceNode)
|
|
// Service name changed away from "consul" means we now need to account for this service instances unless it's a "connect" service.
|
|
if before.ServiceName == structs.ConsulServiceName && after.ServiceName != structs.ConsulServiceName {
|
|
if after.ServiceKind == structs.ServiceKindTypical {
|
|
usageDeltas[billableServiceInstancesTableName()] += 1
|
|
addEnterpriseBillableServiceInstanceUsage(usageDeltas, after, 1)
|
|
}
|
|
}
|
|
if before.ServiceName != structs.ConsulServiceName && after.ServiceName == structs.ConsulServiceName {
|
|
usageDeltas[billableServiceInstancesTableName()] -= 1
|
|
addEnterpriseBillableServiceInstanceUsage(usageDeltas, before, -1)
|
|
}
|
|
|
|
if before.ServiceKind != structs.ServiceKindTypical && after.ServiceKind == structs.ServiceKindTypical {
|
|
usageDeltas[billableServiceInstancesTableName()] += 1
|
|
addEnterpriseBillableServiceInstanceUsage(usageDeltas, after, 1)
|
|
} else if before.ServiceKind == structs.ServiceKindTypical && after.ServiceKind != structs.ServiceKindTypical {
|
|
usageDeltas[billableServiceInstancesTableName()] -= 1
|
|
addEnterpriseBillableServiceInstanceUsage(usageDeltas, before, -1)
|
|
}
|
|
} else {
|
|
svc := changeObject(change).(*structs.ServiceNode)
|
|
// If it's not an update, only update delta if it's a typical service and not the "consul" service.
|
|
if svc.ServiceKind == structs.ServiceKindTypical && svc.ServiceName != structs.ConsulServiceName {
|
|
usageDeltas[billableServiceInstancesTableName()] += delta
|
|
addEnterpriseBillableServiceInstanceUsage(usageDeltas, svc, delta)
|
|
}
|
|
}
|
|
}
|
|
|
|
// writeUsageDeltas will take in a map of IDs to deltas and update each
|
|
// entry accordingly, checking for integer underflow. The index that is
|
|
// passed in will be recorded on the entry as well.
|
|
func writeUsageDeltas(tx WriteTxn, idx uint64, usageDeltas map[string]int) error {
|
|
for id, delta := range usageDeltas {
|
|
u, err := tx.First(tableUsage, indexID, id)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to retrieve existing usage entry: %s", err)
|
|
}
|
|
|
|
if u == nil {
|
|
if delta < 0 {
|
|
// Don't return an error here, since we don't want to block updates
|
|
// from happening to the state store. But, set the delta to 0 so that
|
|
// we do not accidentally underflow the uint64 and begin reporting
|
|
// large numbers.
|
|
delta = 0
|
|
}
|
|
err = tx.Insert(tableUsage, &UsageEntry{
|
|
ID: id,
|
|
Count: delta,
|
|
Index: idx,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to update usage entry: %s", err)
|
|
}
|
|
} else if cur, ok := u.(*UsageEntry); ok {
|
|
updated := cur.Count + delta
|
|
if updated < 0 {
|
|
// Don't return an error here, since we don't want to block updates
|
|
// from happening to the state store. But, set the delta to 0 so that
|
|
// we do not accidentally underflow the uint64 and begin reporting
|
|
// large numbers.
|
|
updated = 0
|
|
}
|
|
err := tx.Insert(tableUsage, &UsageEntry{
|
|
ID: id,
|
|
Count: updated,
|
|
Index: idx,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to update usage entry: %s", err)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// NodeUsage returns the latest seen Raft index, a compiled set of node usage
|
|
// data, and any errors.
|
|
func (s *Store) NodeUsage() (uint64, NodeUsage, error) {
|
|
tx := s.db.ReadTxn()
|
|
defer tx.Abort()
|
|
|
|
nodes, err := firstUsageEntry(nil, tx, tableNodes)
|
|
if err != nil {
|
|
return 0, NodeUsage{}, fmt.Errorf("failed nodes lookup: %s", err)
|
|
}
|
|
|
|
usage := NodeUsage{
|
|
Nodes: nodes.Count,
|
|
}
|
|
results, err := compileEnterpriseNodeUsage(tx, usage)
|
|
if err != nil {
|
|
return 0, NodeUsage{}, fmt.Errorf("failed nodes lookup: %s", err)
|
|
}
|
|
|
|
return nodes.Index, results, nil
|
|
}
|
|
|
|
// PeeringUsage returns the latest seen Raft index, a compiled set of peering usage
|
|
// data, and any errors.
|
|
func (s *Store) PeeringUsage() (uint64, PeeringUsage, error) {
|
|
tx := s.db.ReadTxn()
|
|
defer tx.Abort()
|
|
|
|
peerings, err := firstUsageEntry(nil, tx, tablePeering)
|
|
if err != nil {
|
|
return 0, PeeringUsage{}, fmt.Errorf("failed peerings lookup: %s", err)
|
|
}
|
|
|
|
usage := PeeringUsage{
|
|
Peerings: peerings.Count,
|
|
}
|
|
results, err := compileEnterprisePeeringUsage(tx, usage)
|
|
if err != nil {
|
|
return 0, PeeringUsage{}, fmt.Errorf("failed peerings lookup: %s", err)
|
|
}
|
|
|
|
return peerings.Index, results, nil
|
|
}
|
|
|
|
// ServiceUsage returns the latest seen Raft index, a compiled set of service
|
|
// usage data, and any errors.
|
|
func (s *Store) ServiceUsage(ws memdb.WatchSet) (uint64, structs.ServiceUsage, error) {
|
|
tx := s.db.ReadTxn()
|
|
defer tx.Abort()
|
|
|
|
serviceInstances, err := firstUsageEntry(ws, tx, tableServices)
|
|
if err != nil {
|
|
return 0, structs.ServiceUsage{}, fmt.Errorf("failed services lookup: %s", err)
|
|
}
|
|
|
|
services, err := firstUsageEntry(ws, tx, serviceNamesUsageTable)
|
|
if err != nil {
|
|
return 0, structs.ServiceUsage{}, fmt.Errorf("failed services lookup: %s", err)
|
|
}
|
|
|
|
serviceKindInstances := make(map[string]int)
|
|
for _, kind := range allConnectKind {
|
|
usage, err := firstUsageEntry(ws, tx, connectUsageTableName(kind))
|
|
if err != nil {
|
|
return 0, structs.ServiceUsage{}, fmt.Errorf("failed services lookup: %s", err)
|
|
}
|
|
serviceKindInstances[kind] = usage.Count
|
|
}
|
|
|
|
billableServiceInstances, err := firstUsageEntry(ws, tx, billableServiceInstancesTableName())
|
|
if err != nil {
|
|
return 0, structs.ServiceUsage{}, fmt.Errorf("failed billable services lookup: %s", err)
|
|
}
|
|
|
|
usage := structs.ServiceUsage{
|
|
ServiceInstances: serviceInstances.Count,
|
|
Services: services.Count,
|
|
ConnectServiceInstances: serviceKindInstances,
|
|
BillableServiceInstances: billableServiceInstances.Count,
|
|
}
|
|
results, err := compileEnterpriseServiceUsage(ws, tx, usage)
|
|
if err != nil {
|
|
return 0, structs.ServiceUsage{}, fmt.Errorf("failed services lookup: %s", err)
|
|
}
|
|
|
|
return serviceInstances.Index, results, nil
|
|
}
|
|
|
|
func (s *Store) KVUsage() (uint64, KVUsage, error) {
|
|
tx := s.db.ReadTxn()
|
|
defer tx.Abort()
|
|
|
|
kvs, err := firstUsageEntry(nil, tx, "kvs")
|
|
if err != nil {
|
|
return 0, KVUsage{}, fmt.Errorf("failed kvs lookup: %s", err)
|
|
}
|
|
|
|
usage := KVUsage{
|
|
KVCount: kvs.Count,
|
|
}
|
|
results, err := compileEnterpriseKVUsage(tx, usage)
|
|
if err != nil {
|
|
return 0, KVUsage{}, fmt.Errorf("failed kvs lookup: %s", err)
|
|
}
|
|
|
|
return kvs.Index, results, nil
|
|
}
|
|
|
|
func (s *Store) ConfigEntryUsage() (uint64, ConfigEntryUsage, error) {
|
|
tx := s.db.ReadTxn()
|
|
defer tx.Abort()
|
|
|
|
configEntries := make(map[string]int)
|
|
var maxIdx uint64
|
|
for _, kind := range structs.AllConfigEntryKinds {
|
|
configEntry, err := firstUsageEntry(nil, tx, configEntryUsageTableName(kind))
|
|
if configEntry.Index > maxIdx {
|
|
maxIdx = configEntry.Index
|
|
}
|
|
if err != nil {
|
|
return 0, ConfigEntryUsage{}, fmt.Errorf("failed config entry usage lookup: %s", err)
|
|
}
|
|
configEntries[kind] = configEntry.Count
|
|
}
|
|
usage := ConfigEntryUsage{
|
|
ConfigByKind: configEntries,
|
|
}
|
|
results, err := compileEnterpriseConfigEntryUsage(tx, usage)
|
|
if err != nil {
|
|
return 0, ConfigEntryUsage{}, fmt.Errorf("failed config entry usage lookup: %s", err)
|
|
}
|
|
|
|
return maxIdx, results, nil
|
|
}
|
|
|
|
func firstUsageEntry(ws memdb.WatchSet, tx ReadTxn, id string) (*UsageEntry, error) {
|
|
watch, usage, err := tx.FirstWatch(tableUsage, indexID, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ws.Add(watch)
|
|
|
|
// If no elements have been inserted, the usage entry will not exist. We
|
|
// return a valid value so that can be certain the return value is not nil
|
|
// when no error has occurred.
|
|
if usage == nil {
|
|
return &UsageEntry{ID: id, Count: 0}, nil
|
|
}
|
|
|
|
realUsage, ok := usage.(*UsageEntry)
|
|
if !ok {
|
|
return nil, fmt.Errorf("failed usage lookup: type %T is not *UsageEntry", usage)
|
|
}
|
|
|
|
return realUsage, nil
|
|
}
|
|
|
|
func billableServiceInstancesTableName() string {
|
|
return fmt.Sprintf("billable-%s", tableServices)
|
|
}
|