From bff8931640955d5e73b3188acd678d6beb6ef211 Mon Sep 17 00:00:00 2001 From: miagilepner Date: Tue, 23 May 2023 11:58:51 +0200 Subject: [PATCH] VAULT-14735: generate mock clients for activity log (#20252) * first part of segment client generation * fix imports * initial pr fixes * refactor and fix * update comments * assign client type --- vault/activity/generation/generate_data.pb.go | 72 ++++---- vault/activity/generation/generate_data.proto | 11 +- .../logical_system_activity_write_testonly.go | 125 ++++++++++++++ ...cal_system_activity_write_testonly_test.go | 163 ++++++++++++++++++ 4 files changed, 324 insertions(+), 47 deletions(-) diff --git a/vault/activity/generation/generate_data.pb.go b/vault/activity/generation/generate_data.pb.go index 404ed373a..522a5e360 100644 --- a/vault/activity/generation/generate_data.pb.go +++ b/vault/activity/generation/generate_data.pb.go @@ -439,12 +439,11 @@ type Client struct { Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` Count int32 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"` - TimesSeen int32 `protobuf:"varint,3,opt,name=times_seen,json=timesSeen,proto3" json:"times_seen,omitempty"` - Repeated bool `protobuf:"varint,4,opt,name=repeated,proto3" json:"repeated,omitempty"` - RepeatedFromMonth int32 `protobuf:"varint,5,opt,name=repeated_from_month,json=repeatedFromMonth,proto3" json:"repeated_from_month,omitempty"` - Namespace string `protobuf:"bytes,6,opt,name=namespace,proto3" json:"namespace,omitempty"` - Mount string `protobuf:"bytes,7,opt,name=mount,proto3" json:"mount,omitempty"` - NonEntity bool `protobuf:"varint,8,opt,name=non_entity,json=nonEntity,proto3" json:"non_entity,omitempty"` + Repeated bool `protobuf:"varint,3,opt,name=repeated,proto3" json:"repeated,omitempty"` + RepeatedFromMonth int32 `protobuf:"varint,4,opt,name=repeated_from_month,json=repeatedFromMonth,proto3" json:"repeated_from_month,omitempty"` + Namespace string `protobuf:"bytes,5,opt,name=namespace,proto3" json:"namespace,omitempty"` + Mount string `protobuf:"bytes,6,opt,name=mount,proto3" json:"mount,omitempty"` + NonEntity bool `protobuf:"varint,7,opt,name=non_entity,json=nonEntity,proto3" json:"non_entity,omitempty"` } func (x *Client) Reset() { @@ -493,13 +492,6 @@ func (x *Client) GetCount() int32 { return 0 } -func (x *Client) GetTimesSeen() int32 { - if x != nil { - return x.TimesSeen - } - return 0 -} - func (x *Client) GetRepeated() bool { if x != nil { return x.Repeated @@ -584,36 +576,34 @@ var file_vault_activity_generation_generate_data_proto_rawDesc = []byte{ 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x07, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x07, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, - 0x22, 0xec, 0x01, 0x0a, 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x22, 0xcd, 0x01, 0x0a, 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x53, 0x65, 0x65, 0x6e, - 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2e, 0x0a, 0x13, - 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x6d, 0x6f, - 0x6e, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x72, 0x65, 0x70, 0x65, 0x61, - 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x4d, 0x6f, 0x6e, 0x74, 0x68, 0x12, 0x1c, 0x0a, 0x09, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, - 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x75, 0x6e, 0x74, - 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x6f, 0x6e, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6e, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2a, - 0xa0, 0x01, 0x0a, 0x0c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x11, 0x0a, 0x0d, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, - 0x4e, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, - 0x43, 0x4f, 0x4d, 0x50, 0x55, 0x54, 0x45, 0x44, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x49, 0x45, 0x53, - 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x44, 0x49, 0x53, 0x54, - 0x49, 0x4e, 0x43, 0x54, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x53, 0x10, 0x02, 0x12, 0x12, - 0x0a, 0x0e, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x49, 0x45, 0x53, - 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45, - 0x43, 0x54, 0x5f, 0x54, 0x4f, 0x4b, 0x45, 0x4e, 0x53, 0x10, 0x04, 0x12, 0x15, 0x0a, 0x11, 0x57, - 0x52, 0x49, 0x54, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x54, 0x5f, 0x4c, 0x4f, 0x47, 0x53, - 0x10, 0x05, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, - 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2f, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2e, 0x0a, + 0x13, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x72, 0x65, 0x70, 0x65, + 0x61, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x4d, 0x6f, 0x6e, 0x74, 0x68, 0x12, 0x1c, 0x0a, + 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x75, 0x6e, + 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x6f, 0x6e, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6e, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x2a, 0xa0, 0x01, 0x0a, 0x0c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x12, 0x11, 0x0a, 0x0d, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, + 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x50, 0x52, + 0x45, 0x43, 0x4f, 0x4d, 0x50, 0x55, 0x54, 0x45, 0x44, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x49, 0x45, + 0x53, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x44, 0x49, 0x53, + 0x54, 0x49, 0x4e, 0x43, 0x54, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x53, 0x10, 0x02, 0x12, + 0x12, 0x0a, 0x0e, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x49, 0x45, + 0x53, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x44, 0x49, 0x52, + 0x45, 0x43, 0x54, 0x5f, 0x54, 0x4f, 0x4b, 0x45, 0x4e, 0x53, 0x10, 0x04, 0x12, 0x15, 0x0a, 0x11, + 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x54, 0x5f, 0x4c, 0x4f, 0x47, + 0x53, 0x10, 0x05, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, 0x61, 0x75, 0x6c, + 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, + 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( diff --git a/vault/activity/generation/generate_data.proto b/vault/activity/generation/generate_data.proto index b62209a21..c23a4141d 100644 --- a/vault/activity/generation/generate_data.proto +++ b/vault/activity/generation/generate_data.proto @@ -48,10 +48,9 @@ message Clients { message Client { string id = 1; int32 count = 2; - int32 times_seen = 3; - bool repeated = 4; - int32 repeated_from_month = 5; - string namespace = 6; - string mount = 7; - bool non_entity = 8; + bool repeated = 3; + int32 repeated_from_month = 4; + string namespace = 5; + string mount = 6; + bool non_entity = 7; } diff --git a/vault/logical_system_activity_write_testonly.go b/vault/logical_system_activity_write_testonly.go index 5d289793d..0d60b3667 100644 --- a/vault/logical_system_activity_write_testonly.go +++ b/vault/logical_system_activity_write_testonly.go @@ -7,9 +7,14 @@ package vault import ( "context" + "errors" + "fmt" + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/vault/activity" "github.com/hashicorp/vault/vault/activity/generation" "google.golang.org/protobuf/encoding/protojson" ) @@ -51,3 +56,123 @@ func (b *SystemBackend) handleActivityWriteData(ctx context.Context, request *lo } return nil, nil } + +// singleMonthActivityClients holds a single month's client IDs, in the order they were seen +type singleMonthActivityClients struct { + // clients are indexed by ID + clients []*activity.EntityRecord +} + +// multipleMonthsActivityClients holds multiple month's data +type multipleMonthsActivityClients struct { + // months are in order, with month 0 being the current month and index 1 being 1 month ago + months []*singleMonthActivityClients +} + +// addNewClients generates clients according to the given parameters, and adds them to the month +// the client will always have the mountAccessor as its mount accessor +func (s *singleMonthActivityClients) addNewClients(c *generation.Client, mountAccessor string) error { + count := 1 + if c.Count > 1 { + count = int(c.Count) + } + clientType := entityActivityType + if c.NonEntity { + clientType = nonEntityTokenActivityType + } + for i := 0; i < count; i++ { + record := &activity.EntityRecord{ + ClientID: c.Id, + NamespaceID: c.Namespace, + NonEntity: c.NonEntity, + MountAccessor: mountAccessor, + ClientType: clientType, + } + if record.ClientID == "" { + var err error + record.ClientID, err = uuid.GenerateUUID() + if err != nil { + return err + } + } + s.clients = append(s.clients, record) + } + return nil +} + +// processMonth populates a month of client data +func (m *multipleMonthsActivityClients) processMonth(ctx context.Context, core *Core, month *generation.Data) error { + if month.GetAll() == nil { + return errors.New("segmented monthly data is not yet supported") + } + + // default to using the root namespace and the first mount on the root namespace + mounts, err := core.ListMounts() + if err != nil { + return err + } + defaultMountAccessorRootNS := "" + for _, mount := range mounts { + if mount.NamespaceID == namespace.RootNamespaceID { + defaultMountAccessorRootNS = mount.Accessor + break + } + } + addingTo := m.months[month.GetMonthsAgo()] + + for _, clients := range month.GetAll().Clients { + if clients.Repeated || clients.RepeatedFromMonth > 0 { + return errors.New("repeated clients are not yet supported") + } + + if clients.Namespace == "" { + clients.Namespace = namespace.RootNamespaceID + } + + // verify that the namespace exists + ns, err := core.NamespaceByID(ctx, clients.Namespace) + if err != nil { + return err + } + + // verify that the mount exists + if clients.Mount != "" { + nctx := namespace.ContextWithNamespace(ctx, ns) + mountEntry := core.router.MatchingMountEntry(nctx, clients.Mount) + if mountEntry == nil { + return fmt.Errorf("unable to find matching mount in namespace %s", clients.Namespace) + } + } + + mountAccessor := defaultMountAccessorRootNS + if clients.Namespace != namespace.RootNamespaceID && clients.Mount == "" { + // if we're not using the root namespace, find a mount on the namespace that we are using + found := false + for _, mount := range mounts { + if mount.NamespaceID == clients.Namespace { + mountAccessor = mount.Accessor + found = true + break + } + } + if !found { + return fmt.Errorf("unable to find matching mount in namespace %s", clients.Namespace) + } + } + err = addingTo.addNewClients(clients, mountAccessor) + if err != nil { + return err + } + } + return nil +} + +func newMultipleMonthsActivityClients(numberOfMonths int) *multipleMonthsActivityClients { + m := &multipleMonthsActivityClients{ + months: make([]*singleMonthActivityClients, numberOfMonths), + } + for i := 0; i < numberOfMonths; i++ { + m.months[i] = new(singleMonthActivityClients) + } + return m +} diff --git a/vault/logical_system_activity_write_testonly_test.go b/vault/logical_system_activity_write_testonly_test.go index 57a4b8d41..2a0f44c5b 100644 --- a/vault/logical_system_activity_write_testonly_test.go +++ b/vault/logical_system_activity_write_testonly_test.go @@ -6,10 +6,12 @@ package vault import ( + "context" "testing" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/vault/activity/generation" "github.com/stretchr/testify/require" ) @@ -82,3 +84,164 @@ func TestSystemBackend_handleActivityWriteData(t *testing.T) { }) } } + +// Test_singleMonthActivityClients_addNewClients verifies that new clients are +// created correctly, adhering to the requested parameters. The clients should +// use the inputted mount and a generated ID if one is not supplied. The new +// client should be added to the month's `clients` slice +func Test_singleMonthActivityClients_addNewClients(t *testing.T) { + tests := []struct { + name string + mount string + clients *generation.Client + wantNamespace string + wantMount string + wantID string + }{ + { + name: "default mount is used", + mount: "default_mount", + wantMount: "default_mount", + clients: &generation.Client{}, + }, + { + name: "record namespace is used, default mount is used", + mount: "default_mount", + wantNamespace: "ns", + wantMount: "default_mount", + clients: &generation.Client{ + Namespace: "ns", + Mount: "mount", + }, + }, + { + name: "predefined ID is used", + clients: &generation.Client{ + Id: "client_id", + }, + wantID: "client_id", + }, + { + name: "non zero count", + clients: &generation.Client{ + Count: 5, + }, + }, + { + name: "non entity client", + clients: &generation.Client{ + NonEntity: true, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := &singleMonthActivityClients{} + err := m.addNewClients(tt.clients, tt.mount) + require.NoError(t, err) + numNew := tt.clients.Count + if numNew == 0 { + numNew = 1 + } + require.Len(t, m.clients, int(numNew)) + for _, rec := range m.clients { + require.NotNil(t, rec) + require.Equal(t, tt.wantNamespace, rec.NamespaceID) + require.Equal(t, tt.wantMount, rec.MountAccessor) + require.Equal(t, tt.clients.NonEntity, rec.NonEntity) + if tt.wantID != "" { + require.Equal(t, tt.wantID, rec.ClientID) + } else { + require.NotEqual(t, "", rec.ClientID) + } + } + }) + } +} + +// Test_multipleMonthsActivityClients_processMonth verifies that a month of data +// is added correctly. The test checks that default values are handled correctly +// for mounts and namespaces. +func Test_multipleMonthsActivityClients_processMonth(t *testing.T) { + core, _, _ := TestCoreUnsealed(t) + tests := []struct { + name string + clients *generation.Data + wantError bool + numMonths int + }{ + { + name: "specified namespace and mount exist", + clients: &generation.Data{ + Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{ + Namespace: namespace.RootNamespaceID, + Mount: "identity/", + }}}}, + }, + numMonths: 1, + }, + { + name: "specified namespace exists, mount empty", + clients: &generation.Data{ + Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{ + Namespace: namespace.RootNamespaceID, + }}}}, + }, + numMonths: 1, + }, + { + name: "empty namespace and mount", + clients: &generation.Data{ + Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{}}}}, + }, + numMonths: 1, + }, + { + name: "namespace doesn't exist", + clients: &generation.Data{ + Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{ + Namespace: "abcd", + }}}}, + }, + wantError: true, + numMonths: 1, + }, + { + name: "namespace exists, mount doesn't exist", + clients: &generation.Data{ + Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{ + Namespace: namespace.RootNamespaceID, + Mount: "mount", + }}}}, + }, + wantError: true, + numMonths: 1, + }, + { + name: "older month", + clients: &generation.Data{ + Month: &generation.Data_MonthsAgo{MonthsAgo: 4}, + Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{}}}}, + }, + numMonths: 5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := newMultipleMonthsActivityClients(tt.numMonths) + err := m.processMonth(context.Background(), core, tt.clients) + if tt.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Len(t, m.months[tt.clients.GetMonthsAgo()].clients, len(tt.clients.GetAll().Clients)) + for _, month := range m.months { + for _, c := range month.clients { + require.NotEmpty(t, c.NamespaceID) + require.NotEmpty(t, c.MountAccessor) + } + } + } + }) + } +}