Storage packer V1 updates (#6531)

* spv1 updates

* fix tests
This commit is contained in:
Vishal Nayak 2019-05-07 15:29:51 -04:00 committed by GitHub
parent 4cab0047a1
commit 550f2a52bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 136 additions and 192 deletions

View File

@ -46,11 +46,9 @@ type Group struct {
// ModifyIndex tracks the number of updates to the group. It is useful to detect
// updates to the groups.
ModifyIndex uint64 `sentinel:"" protobuf:"varint,9,opt,name=modify_index,json=modifyIndex,proto3" json:"modify_index,omitempty"`
// BucketKeyHash is the MD5 hash of the storage bucket key into which this
// group is stored in the underlying storage. This is useful to find all
// the groups belonging to a particular bucket during invalidation of the
// storage key.
BucketKeyHash string `sentinel:"" protobuf:"bytes,10,opt,name=bucket_key_hash,json=bucketKeyHash,proto3" json:"bucket_key_hash,omitempty"`
// BucketKey is the path of the storage packer key into which this group is
// stored.
BucketKey string `sentinel:"" protobuf:"bytes,10,opt,name=bucket_key,json=bucketKey,proto3" json:"bucket_key,omitempty"`
// Alias is used to mark this group as an internal mapping of a group that
// is external to the identity store. Alias can only be set if the 'type'
// is set to 'external'.
@ -157,9 +155,9 @@ func (m *Group) GetModifyIndex() uint64 {
return 0
}
func (m *Group) GetBucketKeyHash() string {
func (m *Group) GetBucketKey() string {
if m != nil {
return m.BucketKeyHash
return m.BucketKey
}
return ""
}
@ -221,11 +219,9 @@ type Entity struct {
MergedEntityIDs []string `sentinel:"" protobuf:"bytes,7,rep,name=merged_entity_ids,json=mergedEntityIDs,proto3" json:"merged_entity_ids,omitempty"`
// Policies the entity is entitled to
Policies []string `sentinel:"" protobuf:"bytes,8,rep,name=policies,proto3" json:"policies,omitempty"`
// BucketKeyHash is the MD5 hash of the storage bucket key into which this
// entity is stored in the underlying storage. This is useful to find all
// the entities belonging to a particular bucket during invalidation of the
// storage key.
BucketKeyHash string `sentinel:"" protobuf:"bytes,9,opt,name=bucket_key_hash,json=bucketKeyHash,proto3" json:"bucket_key_hash,omitempty"`
// BucketKey is the path of the storage packer key into which this entity is
// stored.
BucketKey string `sentinel:"" protobuf:"bytes,9,opt,name=bucket_key,json=bucketKey,proto3" json:"bucket_key,omitempty"`
// MFASecrets holds the MFA secrets indexed by the identifier of the MFA
// method configuration.
MFASecrets map[string]*mfa.Secret `sentinel:"" protobuf:"bytes,10,rep,name=mfa_secrets,json=mfaSecrets,proto3" json:"mfa_secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
@ -322,9 +318,9 @@ func (m *Entity) GetPolicies() []string {
return nil
}
func (m *Entity) GetBucketKeyHash() string {
func (m *Entity) GetBucketKey() string {
if m != nil {
return m.BucketKeyHash
return m.BucketKey
}
return ""
}
@ -738,59 +734,60 @@ func init() {
func init() { proto.RegisterFile("helper/identity/types.proto", fileDescriptor_319efdc71a5d7416) }
var fileDescriptor_319efdc71a5d7416 = []byte{
// 861 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x96, 0xcf, 0x8f, 0xdb, 0x44,
0x14, 0xc7, 0x95, 0x1f, 0x4e, 0xec, 0xe7, 0xfc, 0xd8, 0x0e, 0xa8, 0x32, 0x59, 0x95, 0x66, 0x2b,
0x15, 0xa5, 0xab, 0xca, 0x91, 0xb6, 0x07, 0x68, 0x39, 0xa0, 0x05, 0xb6, 0x10, 0x50, 0xa5, 0xca,
0x2d, 0x17, 0x2e, 0xd6, 0xc4, 0x9e, 0x24, 0xa3, 0xb5, 0x3d, 0x96, 0x67, 0x5c, 0x91, 0xff, 0x80,
0x23, 0x17, 0xfe, 0x24, 0xfe, 0x28, 0x6e, 0x68, 0x66, 0x6c, 0xc7, 0x8d, 0xd3, 0xa5, 0x2b, 0x22,
0x04, 0x52, 0x6f, 0xf6, 0x77, 0xde, 0xbc, 0x3c, 0xbf, 0xf7, 0x79, 0x5f, 0x05, 0x4e, 0x37, 0x24,
0x4a, 0x49, 0x36, 0xa7, 0x21, 0x49, 0x04, 0x15, 0xdb, 0xb9, 0xd8, 0xa6, 0x84, 0xbb, 0x69, 0xc6,
0x04, 0x43, 0x66, 0xa9, 0x4e, 0xee, 0xaf, 0x19, 0x5b, 0x47, 0x64, 0xae, 0xf4, 0x65, 0xbe, 0x9a,
0x0b, 0x1a, 0x13, 0x2e, 0x70, 0x9c, 0xea, 0xd0, 0xc9, 0xfd, 0xfd, 0x3c, 0xf1, 0x0a, 0xd7, 0x73,
0x3d, 0xf8, 0xa3, 0x0b, 0xc6, 0x77, 0x19, 0xcb, 0x53, 0x34, 0x82, 0x36, 0x0d, 0x9d, 0xd6, 0xb4,
0x35, 0xb3, 0xbc, 0x36, 0x0d, 0x11, 0x82, 0x6e, 0x82, 0x63, 0xe2, 0xb4, 0x95, 0xa2, 0x9e, 0xd1,
0x04, 0xcc, 0x94, 0x45, 0x34, 0xa0, 0x84, 0x3b, 0x9d, 0x69, 0x67, 0x66, 0x79, 0xd5, 0x3b, 0x9a,
0xc1, 0x49, 0x8a, 0x33, 0x92, 0x08, 0x7f, 0x2d, 0xf3, 0xf9, 0x34, 0xe4, 0x4e, 0x57, 0xc5, 0x8c,
0xb4, 0xae, 0x7e, 0x66, 0x11, 0x72, 0x74, 0x0e, 0x77, 0x62, 0x12, 0x2f, 0x49, 0xe6, 0xeb, 0xa2,
0x54, 0xa8, 0xa1, 0x42, 0xc7, 0xfa, 0xe0, 0x4a, 0xe9, 0x32, 0xf6, 0x29, 0x98, 0x31, 0x11, 0x38,
0xc4, 0x02, 0x3b, 0xbd, 0x69, 0x67, 0x66, 0x5f, 0xdc, 0x73, 0xcb, 0x8f, 0x71, 0x55, 0x46, 0xf7,
0x45, 0x71, 0x7e, 0x95, 0x88, 0x6c, 0xeb, 0x55, 0xe1, 0xe8, 0x2b, 0x18, 0x06, 0x19, 0xc1, 0x82,
0xb2, 0xc4, 0x97, 0x7d, 0x71, 0xfa, 0xd3, 0xd6, 0xcc, 0xbe, 0x98, 0xb8, 0xba, 0x69, 0x6e, 0xd9,
0x34, 0xf7, 0x75, 0xd9, 0x34, 0x6f, 0x50, 0x5e, 0x90, 0x12, 0xfa, 0x16, 0x4e, 0x22, 0xcc, 0x85,
0x9f, 0xa7, 0x21, 0x16, 0x44, 0xe7, 0x30, 0xff, 0x36, 0xc7, 0x48, 0xde, 0xf9, 0x49, 0x5d, 0x51,
0x59, 0xce, 0x60, 0x10, 0xb3, 0x90, 0xae, 0xb6, 0x3e, 0x4d, 0x42, 0xf2, 0x8b, 0x63, 0x4d, 0x5b,
0xb3, 0xae, 0x67, 0x6b, 0x6d, 0x21, 0x25, 0xf4, 0x19, 0x8c, 0x97, 0x79, 0x70, 0x4d, 0x84, 0x7f,
0x4d, 0xb6, 0xfe, 0x06, 0xf3, 0x8d, 0x03, 0xaa, 0xeb, 0x43, 0x2d, 0xff, 0x48, 0xb6, 0xdf, 0x63,
0xbe, 0x41, 0x0f, 0xc1, 0xc0, 0x11, 0xc5, 0xdc, 0xb1, 0x55, 0x15, 0xe3, 0x5d, 0x27, 0x2e, 0xa5,
0xec, 0xe9, 0x53, 0x39, 0x39, 0x39, 0x62, 0x67, 0xa0, 0x27, 0x27, 0x9f, 0x65, 0x15, 0x72, 0x82,
0x3c, 0xc5, 0x01, 0xf1, 0x69, 0xe8, 0x0c, 0xd5, 0x99, 0x5d, 0x69, 0x8b, 0x70, 0xf2, 0x25, 0x0c,
0xdf, 0x6a, 0x25, 0x3a, 0x81, 0xce, 0x35, 0xd9, 0x16, 0x48, 0xc8, 0x47, 0xf4, 0x31, 0x18, 0x6f,
0x70, 0x94, 0x97, 0x50, 0xe8, 0x97, 0x67, 0xed, 0x2f, 0x5a, 0x0f, 0x7e, 0x37, 0xa0, 0xa7, 0xa7,
0x86, 0x1e, 0x41, 0x5f, 0xd5, 0x41, 0xb8, 0xd3, 0x52, 0x13, 0x6b, 0xd4, 0x59, 0x9e, 0x17, 0xcc,
0xb5, 0x1b, 0xcc, 0x75, 0x6a, 0xcc, 0x3d, 0xab, 0x11, 0xd0, 0x55, 0xf9, 0x3e, 0xdd, 0xe5, 0xd3,
0x3f, 0xf9, 0xfe, 0x08, 0x18, 0x47, 0x40, 0xa0, 0x77, 0x6b, 0x04, 0x14, 0xf0, 0xd9, 0x9a, 0x84,
0x75, 0xe0, 0xfb, 0x25, 0xf0, 0xf2, 0x60, 0x07, 0x7c, 0x7d, 0xc5, 0xcc, 0xbd, 0x15, 0x3b, 0xc0,
0x89, 0x75, 0x88, 0x93, 0x4b, 0xb0, 0xe3, 0x15, 0xf6, 0x39, 0x09, 0x32, 0x22, 0xb8, 0x03, 0xaa,
0x6b, 0xd3, 0x66, 0xd7, 0x56, 0xf8, 0x95, 0x0e, 0xd1, 0x7d, 0x83, 0xb8, 0x12, 0x64, 0x19, 0x21,
0xe5, 0x78, 0x19, 0x91, 0x50, 0xd1, 0x66, 0x7a, 0xd5, 0x7b, 0x83, 0xa5, 0xc1, 0x71, 0x59, 0x9a,
0xfc, 0x00, 0xe3, 0xbd, 0xd2, 0x0e, 0x5c, 0x3f, 0xab, 0x5f, 0xb7, 0x2f, 0x6c, 0x37, 0x5e, 0x61,
0x57, 0xdf, 0xa9, 0x73, 0xf9, 0x5b, 0x17, 0x0c, 0x05, 0x5d, 0xc3, 0xdf, 0xce, 0x60, 0x10, 0xe0,
0x84, 0x25, 0x34, 0xc0, 0x91, 0x5f, 0x51, 0x68, 0x57, 0xda, 0x22, 0x44, 0xf7, 0x00, 0x62, 0x96,
0x27, 0xc2, 0x57, 0xeb, 0xa4, 0xa1, 0xb4, 0x94, 0xf2, 0x5a, 0xee, 0xd4, 0x43, 0x18, 0xe9, 0x63,
0x1c, 0x04, 0x84, 0x73, 0x96, 0x39, 0x5d, 0x3d, 0x0d, 0xa5, 0x5e, 0x16, 0xe2, 0x2e, 0x4b, 0x8a,
0xc5, 0x46, 0x11, 0x58, 0x66, 0x79, 0x89, 0xc5, 0xe6, 0x66, 0x87, 0x53, 0xa5, 0xbf, 0x13, 0xef,
0x72, 0x5d, 0xfa, 0xb5, 0x75, 0x69, 0x20, 0x6f, 0x1e, 0x01, 0x79, 0xeb, 0xd6, 0xc8, 0x3f, 0x85,
0x4f, 0x0a, 0xe4, 0x57, 0x19, 0x8b, 0xfd, 0x7a, 0xa7, 0x35, 0x90, 0x96, 0x77, 0x57, 0x07, 0x3c,
0xcf, 0x58, 0xfc, 0xcd, 0xae, 0xe9, 0xbc, 0x81, 0x97, 0x7d, 0x64, 0xab, 0xfa, 0xd5, 0x00, 0xa4,
0x37, 0xe0, 0x95, 0x60, 0x19, 0x5e, 0x13, 0x9d, 0xe2, 0x73, 0x30, 0x53, 0x92, 0x71, 0x96, 0xe0,
0xd2, 0xb7, 0x4e, 0x77, 0x73, 0x78, 0xa9, 0x4f, 0x94, 0x5d, 0x17, 0x53, 0x28, 0x83, 0xdf, 0xcb,
0xc4, 0x9e, 0x37, 0x4c, 0xec, 0x7c, 0x7f, 0x1d, 0xeb, 0xc5, 0x7c, 0x30, 0xb4, 0xb7, 0x0d, 0xed,
0xc5, 0x21, 0x43, 0x7b, 0x7c, 0x73, 0x07, 0xdf, 0x6d, 0x6e, 0xff, 0x1d, 0x77, 0xfa, 0xb3, 0x03,
0x77, 0x1a, 0x68, 0x35, 0x9c, 0xea, 0x14, 0xac, 0xaa, 0xcd, 0x45, 0x3d, 0x26, 0x29, 0xfa, 0xfb,
0xef, 0x78, 0xd4, 0x55, 0xc3, 0xa3, 0x1e, 0xdd, 0xb0, 0x1b, 0xff, 0x47, 0xbf, 0x7a, 0x02, 0x77,
0xeb, 0x7e, 0x55, 0xc3, 0x5a, 0x9b, 0xd5, 0x47, 0x3b, 0xb3, 0xaa, 0xd0, 0xfe, 0x47, 0x1c, 0x7d,
0xfd, 0xf8, 0xe7, 0xf3, 0x35, 0x15, 0x9b, 0x7c, 0xe9, 0x06, 0x2c, 0x9e, 0x4b, 0xf6, 0x69, 0xc0,
0xb2, 0x74, 0xfe, 0x06, 0xe7, 0x91, 0x98, 0xef, 0xfd, 0x6f, 0x5f, 0xf6, 0xd4, 0x37, 0x3c, 0xf9,
0x2b, 0x00, 0x00, 0xff, 0xff, 0x66, 0xa2, 0xa4, 0x7e, 0x19, 0x0c, 0x00, 0x00,
// 867 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x56, 0x5d, 0x8f, 0xdb, 0x44,
0x14, 0x55, 0x3e, 0x9c, 0xd8, 0xd7, 0xf9, 0xd8, 0x0e, 0xa8, 0x32, 0x59, 0x95, 0x66, 0x2b, 0x15,
0xa5, 0xab, 0xca, 0x91, 0xb6, 0x0f, 0xd0, 0xf2, 0x80, 0x16, 0xd8, 0x42, 0x40, 0x95, 0x2a, 0xb7,
0xbc, 0xf0, 0x62, 0x4d, 0xec, 0x49, 0x32, 0xaa, 0xed, 0xb1, 0x3c, 0xe3, 0x8a, 0xfc, 0x03, 0x1e,
0x78, 0xe0, 0x07, 0xf1, 0xc7, 0x78, 0x43, 0x33, 0xe3, 0xaf, 0xc6, 0xed, 0xd2, 0x15, 0x11, 0x02,
0xf5, 0xcd, 0x73, 0xee, 0x9d, 0x3b, 0x37, 0xe7, 0x9e, 0x7b, 0x14, 0x38, 0xdd, 0x91, 0x28, 0x25,
0xd9, 0x92, 0x86, 0x24, 0x11, 0x54, 0xec, 0x97, 0x62, 0x9f, 0x12, 0xee, 0xa6, 0x19, 0x13, 0x0c,
0x99, 0x25, 0x3a, 0xbb, 0xbb, 0x65, 0x6c, 0x1b, 0x91, 0xa5, 0xc2, 0xd7, 0xf9, 0x66, 0x29, 0x68,
0x4c, 0xb8, 0xc0, 0x71, 0xaa, 0x53, 0x67, 0x77, 0x0f, 0xeb, 0xc4, 0x1b, 0xdc, 0xac, 0x75, 0xef,
0x8f, 0x3e, 0x18, 0xdf, 0x65, 0x2c, 0x4f, 0xd1, 0x04, 0xba, 0x34, 0x74, 0x3a, 0xf3, 0xce, 0xc2,
0xf2, 0xba, 0x34, 0x44, 0x08, 0xfa, 0x09, 0x8e, 0x89, 0xd3, 0x55, 0x88, 0xfa, 0x46, 0x33, 0x30,
0x53, 0x16, 0xd1, 0x80, 0x12, 0xee, 0xf4, 0xe6, 0xbd, 0x85, 0xe5, 0x55, 0x67, 0xb4, 0x80, 0x93,
0x14, 0x67, 0x24, 0x11, 0xfe, 0x56, 0xd6, 0xf3, 0x69, 0xc8, 0x9d, 0xbe, 0xca, 0x99, 0x68, 0x5c,
0x3d, 0xb3, 0x0a, 0x39, 0x3a, 0x87, 0x5b, 0x31, 0x89, 0xd7, 0x24, 0xf3, 0x75, 0x53, 0x2a, 0xd5,
0x50, 0xa9, 0x53, 0x1d, 0xb8, 0x52, 0xb8, 0xcc, 0x7d, 0x0c, 0x66, 0x4c, 0x04, 0x0e, 0xb1, 0xc0,
0xce, 0x60, 0xde, 0x5b, 0xd8, 0x17, 0x77, 0xdc, 0xf2, 0xc7, 0xb8, 0xaa, 0xa2, 0xfb, 0xac, 0x88,
0x5f, 0x25, 0x22, 0xdb, 0x7b, 0x55, 0x3a, 0xfa, 0x0a, 0xc6, 0x41, 0x46, 0xb0, 0xa0, 0x2c, 0xf1,
0x25, 0x2f, 0xce, 0x70, 0xde, 0x59, 0xd8, 0x17, 0x33, 0x57, 0x93, 0xe6, 0x96, 0xa4, 0xb9, 0x2f,
0x4b, 0xd2, 0xbc, 0x51, 0x79, 0x41, 0x42, 0xe8, 0x5b, 0x38, 0x89, 0x30, 0x17, 0x7e, 0x9e, 0x86,
0x58, 0x10, 0x5d, 0xc3, 0xfc, 0xdb, 0x1a, 0x13, 0x79, 0xe7, 0x27, 0x75, 0x45, 0x55, 0x39, 0x83,
0x51, 0xcc, 0x42, 0xba, 0xd9, 0xfb, 0x34, 0x09, 0xc9, 0x2f, 0x8e, 0x35, 0xef, 0x2c, 0xfa, 0x9e,
0xad, 0xb1, 0x95, 0x84, 0xd0, 0x1d, 0x80, 0x75, 0x1e, 0xbc, 0x22, 0xc2, 0x7f, 0x45, 0xf6, 0x0e,
0x28, 0xc2, 0x2d, 0x8d, 0xfc, 0x48, 0xf6, 0xe8, 0x3e, 0x18, 0x38, 0xa2, 0x98, 0x3b, 0xb6, 0x7a,
0x7c, 0x5a, 0x13, 0x70, 0x29, 0x61, 0x4f, 0x47, 0xe5, 0xc0, 0xe4, 0x64, 0x9d, 0x91, 0x1e, 0x98,
0xfc, 0x96, 0x8f, 0xcb, 0xc1, 0xf1, 0x14, 0x07, 0xc4, 0xa7, 0xa1, 0x33, 0x56, 0x31, 0xbb, 0xc2,
0x56, 0xe1, 0xec, 0x4b, 0x18, 0xbf, 0xc1, 0x20, 0x3a, 0x81, 0x9e, 0x6c, 0x43, 0x2b, 0x41, 0x7e,
0xa2, 0x8f, 0xc1, 0x78, 0x8d, 0xa3, 0xbc, 0xd4, 0x82, 0x3e, 0x3c, 0xe9, 0x7e, 0xd1, 0xb9, 0xf7,
0x9b, 0x01, 0x03, 0x3d, 0x2c, 0xf4, 0x00, 0x86, 0xaa, 0x0f, 0xc2, 0x9d, 0x8e, 0x1a, 0x54, 0xab,
0xcf, 0x32, 0x5e, 0x48, 0xad, 0xdb, 0x92, 0x5a, 0xaf, 0x21, 0xb5, 0x27, 0x8d, 0xc1, 0xf7, 0x55,
0xbd, 0x4f, 0xeb, 0x7a, 0xfa, 0xc9, 0xf7, 0x9f, 0xbc, 0x71, 0x84, 0xc9, 0x0f, 0x6e, 0x3c, 0x79,
0xa5, 0xf3, 0x6c, 0x4b, 0xc2, 0xa6, 0xce, 0x87, 0xa5, 0xce, 0x65, 0xa0, 0xd6, 0x79, 0x73, 0xb3,
0xcc, 0x83, 0xcd, 0x7a, 0x53, 0x1e, 0xd6, 0xa1, 0x3c, 0x2e, 0xc1, 0x8e, 0x37, 0xd8, 0xe7, 0x24,
0xc8, 0x88, 0xe0, 0x0e, 0x28, 0xb2, 0xe6, 0x6d, 0xb2, 0x36, 0xf8, 0x85, 0x4e, 0xd1, 0x74, 0x41,
0x5c, 0x01, 0xf2, 0xf5, 0x90, 0x72, 0xbc, 0x8e, 0x48, 0xa8, 0x44, 0x66, 0x7a, 0xd5, 0xb9, 0x25,
0xa1, 0xd1, 0x71, 0x25, 0x34, 0xfb, 0x01, 0xa6, 0x07, 0xad, 0xbd, 0xe5, 0xfa, 0x59, 0xf3, 0xba,
0x7d, 0x61, 0xbb, 0xf1, 0x06, 0xbb, 0xfa, 0x4e, 0x53, 0x8e, 0xbf, 0xf7, 0xc1, 0x50, 0x5a, 0x6b,
0xb9, 0xd9, 0x19, 0x8c, 0x02, 0x9c, 0xb0, 0x84, 0x06, 0x38, 0xf2, 0x2b, 0xf1, 0xd9, 0x15, 0xb6,
0x0a, 0x25, 0xcd, 0x31, 0xcb, 0x13, 0xe1, 0xab, 0x2d, 0xd2, 0x5a, 0xb4, 0x14, 0xf2, 0x52, 0xae,
0xd2, 0x7d, 0x98, 0xe8, 0x30, 0x0e, 0x02, 0xc2, 0x39, 0xcb, 0x9c, 0xbe, 0x4a, 0x19, 0x2b, 0xf4,
0xb2, 0x00, 0xeb, 0x2a, 0x29, 0x16, 0x3b, 0x25, 0xbc, 0xb2, 0xca, 0x73, 0x2c, 0x76, 0xd7, 0xfb,
0x99, 0x6a, 0xfd, 0x9d, 0xaa, 0x2e, 0xb7, 0x64, 0xd8, 0xd8, 0x92, 0x96, 0xd2, 0xcd, 0x23, 0x28,
0xdd, 0xba, 0xb1, 0xd2, 0x1f, 0xc3, 0x27, 0x85, 0xd2, 0x37, 0x19, 0x8b, 0xfd, 0x26, 0xd3, 0x5a,
0x90, 0x96, 0x77, 0x5b, 0x27, 0x3c, 0xcd, 0x58, 0xfc, 0x4d, 0x4d, 0x3a, 0x6f, 0xc9, 0xcb, 0x3e,
0xb2, 0x43, 0xfd, 0x6a, 0x00, 0xd2, 0x1b, 0xf0, 0x42, 0xb0, 0x0c, 0x6f, 0x89, 0x2e, 0xf1, 0x39,
0x98, 0x29, 0xc9, 0x38, 0x4b, 0x70, 0x69, 0x57, 0xa7, 0xf5, 0x1c, 0x9e, 0xeb, 0x88, 0x32, 0xe7,
0x62, 0x0a, 0x65, 0xf2, 0x7b, 0x79, 0xd7, 0xd3, 0x96, 0x77, 0x9d, 0x1f, 0xae, 0x63, 0xb3, 0x99,
0x0f, 0xc5, 0xc7, 0x3e, 0x83, 0x69, 0xed, 0x63, 0xfe, 0x0e, 0xf3, 0x5d, 0x61, 0x66, 0xe3, 0xca,
0xcc, 0xbe, 0xc7, 0x7c, 0x87, 0x9e, 0xbd, 0xcd, 0xd0, 0x1e, 0x5e, 0xcf, 0xe0, 0xbb, 0xcd, 0xed,
0xbf, 0xe3, 0x4e, 0x7f, 0xf6, 0xe0, 0x56, 0x4b, 0x5a, 0x2d, 0xa7, 0x3a, 0x05, 0xab, 0xa2, 0xb9,
0xe8, 0xc7, 0x24, 0x05, 0xbf, 0xff, 0x8e, 0x47, 0x5d, 0xb5, 0x3c, 0xea, 0xc1, 0x35, 0xbb, 0xf1,
0x7f, 0xf4, 0xab, 0x47, 0x70, 0xbb, 0xe9, 0x57, 0x0d, 0x59, 0x6b, 0xb3, 0xfa, 0xa8, 0x36, 0xab,
0x4a, 0xda, 0xff, 0x48, 0x47, 0x5f, 0x3f, 0xfc, 0xf9, 0x7c, 0x4b, 0xc5, 0x2e, 0x5f, 0xbb, 0x01,
0x8b, 0x97, 0x52, 0xfb, 0x34, 0x60, 0x59, 0xba, 0x7c, 0x8d, 0xf3, 0x48, 0x2c, 0x0f, 0xfe, 0xa5,
0xaf, 0x07, 0xea, 0x37, 0x3c, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x54, 0x4e, 0x98, 0x4a, 0x07,
0x0c, 0x00, 0x00,
}

View File

@ -40,11 +40,9 @@ message Group {
// updates to the groups.
uint64 modify_index = 9;
// BucketKeyHash is the MD5 hash of the storage bucket key into which this
// group is stored in the underlying storage. This is useful to find all
// the groups belonging to a particular bucket during invalidation of the
// storage key.
string bucket_key_hash = 10;
// BucketKey is the path of the storage packer key into which this group is
// stored.
string bucket_key = 10;
// Alias is used to mark this group as an internal mapping of a group that
// is external to the identity store. Alias can only be set if the 'type'
@ -107,11 +105,9 @@ message Entity {
// Policies the entity is entitled to
repeated string policies = 8;
// BucketKeyHash is the MD5 hash of the storage bucket key into which this
// entity is stored in the underlying storage. This is useful to find all
// the entities belonging to a particular bucket during invalidation of the
// storage key.
string bucket_key_hash = 9;
// BucketKey is the path of the storage packer key into which this entity is
// stored.
string bucket_key = 9;
// MFASecrets holds the MFA secrets indexed by the identifier of the MFA
// method configuration.

View File

@ -3,7 +3,6 @@ package storagepacker
import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"strconv"
"strings"
@ -21,11 +20,9 @@ const (
StoragePackerBucketsPrefix = "packer/buckets/"
)
// StoragePacker packs the objects into a specific number of buckets by hashing
// its ID and indexing it. Currently this supports only 256 bucket entries and
// hence relies on the first byte of the hash value for indexing. The items
// that gets inserted into the packer should implement StorageBucketItem
// interface.
// StoragePacker packs items into a specific number of buckets by hashing
// its identifier and indexing on it. Currently this supports only 256 bucket entries and
// hence relies on the first byte of the hash value for indexing.
type StoragePacker struct {
view logical.Storage
logger log.Logger
@ -33,25 +30,6 @@ type StoragePacker struct {
viewPrefix string
}
// BucketPath returns the storage entry key for a given bucket key
func (s *StoragePacker) BucketPath(bucketKey string) string {
return s.viewPrefix + bucketKey
}
// BucketKeyHash returns the MD5 hash of the bucket storage key in which
// the item will be stored. The choice of MD5 is only for hash performance
// reasons since its value is not used for any security sensitive operation.
func (s *StoragePacker) BucketKeyHashByItemID(itemID string) string {
return s.BucketKeyHashByKey(s.BucketPath(s.BucketKey(itemID)))
}
// BucketKeyHashByKey returns the MD5 hash of the bucket storage key
func (s *StoragePacker) BucketKeyHashByKey(bucketKey string) string {
hf := md5.New()
hf.Write([]byte(bucketKey))
return hex.EncodeToString(hf.Sum(nil))
}
// View returns the storage view configured to be used by the packer
func (s *StoragePacker) View() logical.Storage {
return s.view
@ -67,7 +45,7 @@ func (s *StoragePacker) GetBucket(key string) (*Bucket, error) {
lock.RLock()
defer lock.RUnlock()
// Read from the underlying view
// Read from storage
storageEntry, err := s.view.Get(context.Background(), key)
if err != nil {
return nil, errwrap.Wrapf("failed to read packed storage entry: {{err}}", err)
@ -108,8 +86,8 @@ func (s *Bucket) upsert(item *Item) error {
return fmt.Errorf("missing item ID")
}
// Look for an item with matching key and don't modify the collection
// while iterating
// Look for an item with matching key and don't modify the collection while
// iterating
foundIdx := -1
for itemIdx, bucketItems := range s.Items {
if bucketItems.ID == item.ID {
@ -128,34 +106,25 @@ func (s *Bucket) upsert(item *Item) error {
return nil
}
// BucketIndex returns the bucket key index for a given storage key
func (s *StoragePacker) BucketIndex(key string) uint8 {
hf := md5.New()
hf.Write([]byte(key))
return uint8(hf.Sum(nil)[0])
}
// BucketKey returns the bucket key for a given item ID
// BucketKey returns the storage key of the bucket where the given item will be
// stored.
func (s *StoragePacker) BucketKey(itemID string) string {
return strconv.Itoa(int(s.BucketIndex(itemID)))
hf := md5.New()
hf.Write([]byte(itemID))
index := uint8(hf.Sum(nil)[0])
return s.viewPrefix + strconv.Itoa(int(index))
}
// DeleteItem removes the storage entry which the given key refers to from its
// corresponding bucket.
// DeleteItem removes the item from the respective bucket
func (s *StoragePacker) DeleteItem(_ context.Context, itemID string) error {
if itemID == "" {
return fmt.Errorf("empty item ID")
}
// Get the bucket key
bucketKey := s.BucketKey(itemID)
// Prepend the view prefix
bucketPath := s.BucketPath(bucketKey)
// Read from underlying view
storageEntry, err := s.view.Get(context.Background(), bucketPath)
// Read from storage
storageEntry, err := s.view.Get(context.Background(), bucketKey)
if err != nil {
return errwrap.Wrapf("failed to read packed storage value: {{err}}", err)
}
@ -192,7 +161,7 @@ func (s *StoragePacker) DeleteItem(_ context.Context, itemID string) error {
bucket.Items = append(bucket.Items[:foundIdx], bucket.Items[foundIdx+1:]...)
// Persist bucket entry only if there is an update
err = s.PutBucket(&bucket)
err = s.putBucket(&bucket)
if err != nil {
return err
}
@ -201,8 +170,7 @@ func (s *StoragePacker) DeleteItem(_ context.Context, itemID string) error {
return nil
}
// Put stores a packed bucket entry
func (s *StoragePacker) PutBucket(bucket *Bucket) error {
func (s *StoragePacker) putBucket(bucket *Bucket) error {
if bucket == nil {
return fmt.Errorf("nil bucket entry")
}
@ -247,10 +215,9 @@ func (s *StoragePacker) GetItem(itemID string) (*Item, error) {
}
bucketKey := s.BucketKey(itemID)
bucketPath := s.BucketPath(bucketKey)
// Fetch the bucket entry
bucket, err := s.GetBucket(bucketPath)
bucket, err := s.GetBucket(bucketKey)
if err != nil {
return nil, errwrap.Wrapf("failed to read packed storage item: {{err}}", err)
}
@ -268,7 +235,7 @@ func (s *StoragePacker) GetItem(itemID string) (*Item, error) {
return nil, nil
}
// PutItem stores a storage entry in its corresponding bucket
// PutItem stores the given item in its respective bucket
func (s *StoragePacker) PutItem(_ context.Context, item *Item) error {
if item == nil {
return fmt.Errorf("nil item")
@ -280,21 +247,20 @@ func (s *StoragePacker) PutItem(_ context.Context, item *Item) error {
var err error
bucketKey := s.BucketKey(item.ID)
bucketPath := s.BucketPath(bucketKey)
bucket := &Bucket{
Key: bucketPath,
Key: bucketKey,
}
// In this case, we persist the storage entry regardless of the read
// storageEntry below is nil or not. Hence, directly acquire write lock
// even to read the entry.
lock := locksutil.LockForKey(s.storageLocks, bucketPath)
lock := locksutil.LockForKey(s.storageLocks, bucketKey)
lock.Lock()
defer lock.Unlock()
// Check if there is an existing bucket for a given key
storageEntry, err := s.view.Get(context.Background(), bucketPath)
storageEntry, err := s.view.Get(context.Background(), bucketKey)
if err != nil {
return errwrap.Wrapf("failed to read packed storage bucket entry: {{err}}", err)
}
@ -325,8 +291,7 @@ func (s *StoragePacker) PutItem(_ context.Context, item *Item) error {
}
}
// Persist the result
return s.PutBucket(bucket)
return s.putBucket(bucket)
}
// NewStoragePacker creates a new storage packer for a given view

View File

@ -109,13 +109,6 @@ func (i *IdentityStore) Invalidate(ctx context.Context, key string) {
switch {
// Check if the key is a storage entry key for an entity bucket
case strings.HasPrefix(key, storagepacker.StoragePackerBucketsPrefix):
// Get the hash value of the storage bucket entry key
bucketKeyHash := i.entityPacker.BucketKeyHashByKey(key)
if len(bucketKeyHash) == 0 {
i.logger.Error("failed to get the bucket entry key hash")
return
}
// Create a MemDB transaction
txn := i.db.Txn(true)
defer txn.Abort()
@ -124,9 +117,9 @@ func (i *IdentityStore) Invalidate(ctx context.Context, key string) {
// entry key of the entity bucket. Fetch all the entities that
// belong to this bucket using the hash value. Remove these entities
// from MemDB along with all the aliases of each entity.
entitiesFetched, err := i.MemDBEntitiesByBucketEntryKeyHashInTxn(txn, string(bucketKeyHash))
entitiesFetched, err := i.MemDBEntitiesByBucketKeyInTxn(txn, key)
if err != nil {
i.logger.Error("failed to fetch entities using the bucket entry key hash", "bucket_entry_key_hash", bucketKeyHash)
i.logger.Error("failed to fetch entities using the bucket key", "key", key)
return
}
@ -183,20 +176,13 @@ func (i *IdentityStore) Invalidate(ctx context.Context, key string) {
// Check if the key is a storage entry key for an group bucket
case strings.HasPrefix(key, groupBucketsPrefix):
// Get the hash value of the storage bucket entry key
bucketKeyHash := i.groupPacker.BucketKeyHashByKey(key)
if len(bucketKeyHash) == 0 {
i.logger.Error("failed to get the bucket entry key hash")
return
}
// Create a MemDB transaction
txn := i.db.Txn(true)
defer txn.Abort()
groupsFetched, err := i.MemDBGroupsByBucketEntryKeyHashInTxn(txn, string(bucketKeyHash))
groupsFetched, err := i.MemDBGroupsByBucketKeyInTxn(txn, key)
if err != nil {
i.logger.Error("failed to fetch groups using the bucket entry key hash", "bucket_entry_key_hash", bucketKeyHash)
i.logger.Error("failed to fetch groups using the bucket key", "key", key)
return
}
@ -294,7 +280,7 @@ func (i *IdentityStore) parseEntityFromBucketItem(ctx context.Context, item *sto
entity.LastUpdateTime = oldEntity.LastUpdateTime
entity.MergedEntityIDs = oldEntity.MergedEntityIDs
entity.Policies = oldEntity.Policies
entity.BucketKeyHash = oldEntity.BucketKeyHash
entity.BucketKey = oldEntity.BucketKeyHash
entity.MFASecrets = oldEntity.MFASecrets
// Copy each alias individually since the format of aliases were
// also different

View File

@ -220,7 +220,7 @@ func TestIdentityStore_MemDBAliasIndexes(t *testing.T) {
Name: "testentityname",
}
entity.BucketKeyHash = is.entityPacker.BucketKeyHashByItemID(entity.ID)
entity.BucketKey = is.entityPacker.BucketKey(entity.ID)
txn := is.db.Txn(true)
defer txn.Abort()

View File

@ -479,7 +479,7 @@ func TestIdentityStore_MemDBImmutability(t *testing.T) {
},
}
entity.BucketKeyHash = is.entityPacker.BucketKeyHashByItemID(entity.ID)
entity.BucketKey = is.entityPacker.BucketKey(entity.ID)
txn := is.db.Txn(true)
defer txn.Abort()
@ -733,7 +733,7 @@ func TestIdentityStore_MemDBEntityIndexes(t *testing.T) {
},
}
entity.BucketKeyHash = is.entityPacker.BucketKeyHashByItemID(entity.ID)
entity.BucketKey = is.entityPacker.BucketKey(entity.ID)
txn := is.db.Txn(true)
defer txn.Abort()
@ -764,7 +764,7 @@ func TestIdentityStore_MemDBEntityIndexes(t *testing.T) {
}
txn = is.db.Txn(false)
entitiesFetched, err := is.MemDBEntitiesByBucketEntryKeyHashInTxn(txn, entity.BucketKeyHash)
entitiesFetched, err := is.MemDBEntitiesByBucketKeyInTxn(txn, entity.BucketKey)
if err != nil {
t.Fatal(err)
}

View File

@ -358,7 +358,7 @@ func TestIdentityStore_GroupAliases_MemDBIndexes(t *testing.T) {
ParentGroupIDs: []string{"testparentgroupid1", "testparentgroupid2"},
MemberEntityIDs: []string{"testentityid1", "testentityid2"},
Policies: []string{"testpolicy1", "testpolicy2"},
BucketKeyHash: i.groupPacker.BucketKeyHashByItemID("testgroupid"),
BucketKey: i.groupPacker.BucketKey("testgroupid"),
}
txn := i.db.Txn(true)

View File

@ -498,7 +498,7 @@ func TestIdentityStore_MemDBGroupIndexes(t *testing.T) {
ParentGroupIDs: []string{"testparentgroupid1", "testparentgroupid2"},
MemberEntityIDs: []string{"testentityid1", "testentityid2"},
Policies: []string{"testpolicy1", "testpolicy2"},
BucketKeyHash: i.groupPacker.BucketKeyHashByItemID("testgroupid"),
BucketKey: i.groupPacker.BucketKey("testgroupid"),
}
// Insert it into memdb
@ -521,7 +521,7 @@ func TestIdentityStore_MemDBGroupIndexes(t *testing.T) {
ParentGroupIDs: []string{"testparentgroupid2", "testparentgroupid3"},
MemberEntityIDs: []string{"testentityid2", "testentityid3"},
Policies: []string{"testpolicy2", "testpolicy3"},
BucketKeyHash: i.groupPacker.BucketKeyHashByItemID("testgroupid2"),
BucketKey: i.groupPacker.BucketKey("testgroupid2"),
}
// Insert it into memdb

View File

@ -106,10 +106,10 @@ func entitiesTableSchema(lowerCaseName bool) *memdb.TableSchema {
Field: "MergedEntityIDs",
},
},
"bucket_key_hash": &memdb.IndexSchema{
Name: "bucket_key_hash",
"bucket_key": &memdb.IndexSchema{
Name: "bucket_key",
Indexer: &memdb.StringFieldIndex{
Field: "BucketKeyHash",
Field: "BucketKey",
},
},
"namespace_id": &memdb.IndexSchema{
@ -162,10 +162,10 @@ func groupsTableSchema(lowerCaseName bool) *memdb.TableSchema {
Field: "ParentGroupIDs",
},
},
"bucket_key_hash": &memdb.IndexSchema{
Name: "bucket_key_hash",
"bucket_key": &memdb.IndexSchema{
Name: "bucket_key",
Indexer: &memdb.StringFieldIndex{
Field: "BucketKeyHash",
Field: "BucketKey",
},
},
"namespace_id": &memdb.IndexSchema{

View File

@ -51,7 +51,7 @@ func TestIdentityStore_UnsealingWhenConflictingAliasNames(t *testing.T) {
},
NamespaceID: namespace.RootNamespaceID,
}
entity.BucketKeyHash = c.identityStore.entityPacker.BucketKeyHashByItemID(entity.ID)
entity.BucketKey = c.identityStore.entityPacker.BucketKey(entity.ID)
err = c.identityStore.upsertEntity(namespace.RootContext(nil), entity, nil, true)
if err != nil {
@ -74,7 +74,7 @@ func TestIdentityStore_UnsealingWhenConflictingAliasNames(t *testing.T) {
},
NamespaceID: namespace.RootNamespaceID,
}
entity2.BucketKeyHash = c.identityStore.entityPacker.BucketKeyHashByItemID(entity2.ID)
entity2.BucketKey = c.identityStore.entityPacker.BucketKey(entity2.ID)
// Persist the second entity directly without the regular flow. This will skip
// merging of these enties.
@ -505,7 +505,7 @@ func TestIdentityStore_MergeConflictingAliases(t *testing.T) {
},
NamespaceID: namespace.RootNamespaceID,
}
entity.BucketKeyHash = c.identityStore.entityPacker.BucketKeyHashByItemID(entity.ID)
entity.BucketKey = c.identityStore.entityPacker.BucketKey(entity.ID)
err = c.identityStore.upsertEntity(namespace.RootContext(nil), entity, nil, true)
if err != nil {
t.Fatal(err)
@ -528,7 +528,7 @@ func TestIdentityStore_MergeConflictingAliases(t *testing.T) {
NamespaceID: namespace.RootNamespaceID,
}
entity2.BucketKeyHash = c.identityStore.entityPacker.BucketKeyHashByItemID(entity2.ID)
entity2.BucketKey = c.identityStore.entityPacker.BucketKey(entity2.ID)
err = c.identityStore.upsertEntity(namespace.RootContext(nil), entity2, nil, true)
if err != nil {

View File

@ -87,7 +87,7 @@ func (i *IdentityStore) loadGroups(ctx context.Context) error {
i.logger.Debug("groups collected", "num_existing", len(existing))
for _, key := range existing {
bucket, err := i.groupPacker.GetBucket(i.groupPacker.BucketPath(key))
bucket, err := i.groupPacker.GetBucket(groupBucketsPrefix + key)
if err != nil {
return err
}
@ -203,13 +203,13 @@ func (i *IdentityStore) loadEntities(ctx context.Context) error {
for {
select {
case bucketKey, ok := <-broker:
case key, ok := <-broker:
// broker has been closed, we are done
if !ok {
return
}
bucket, err := i.entityPacker.GetBucket(i.entityPacker.BucketPath(bucketKey))
bucket, err := i.entityPacker.GetBucket(storagepacker.StoragePackerBucketsPrefix + key)
if err != nil {
errs <- err
continue
@ -230,7 +230,7 @@ func (i *IdentityStore) loadEntities(ctx context.Context) error {
wg.Add(1)
go func() {
defer wg.Done()
for j, bucketKey := range existing {
for j, key := range existing {
if j%500 == 0 {
i.logger.Debug("entities loading", "progress", j)
}
@ -240,7 +240,7 @@ func (i *IdentityStore) loadEntities(ctx context.Context) error {
return
default:
broker <- bucketKey
broker <- key
}
}
@ -774,16 +774,16 @@ func (i *IdentityStore) MemDBEntityByNameInTxn(ctx context.Context, txn *memdb.T
return entity, nil
}
func (i *IdentityStore) MemDBEntitiesByBucketEntryKeyHashInTxn(txn *memdb.Txn, hashValue string) ([]*identity.Entity, error) {
func (i *IdentityStore) MemDBEntitiesByBucketKeyInTxn(txn *memdb.Txn, bucketKey string) ([]*identity.Entity, error) {
if txn == nil {
return nil, fmt.Errorf("nil txn")
}
if hashValue == "" {
return nil, fmt.Errorf("empty hash value")
if bucketKey == "" {
return nil, fmt.Errorf("empty bucket key")
}
entitiesIter, err := txn.Get(entitiesTable, "bucket_key_hash", hashValue)
entitiesIter, err := txn.Get(entitiesTable, "bucket_key", bucketKey)
if err != nil {
return nil, errwrap.Wrapf("failed to lookup entities using bucket entry key hash: {{err}}", err)
}
@ -971,8 +971,8 @@ func (i *IdentityStore) sanitizeEntity(ctx context.Context, entity *identity.Ent
return fmt.Errorf("failed to generate entity id")
}
// Set the hash value of the storage bucket key in entity
entity.BucketKeyHash = i.entityPacker.BucketKeyHashByItemID(entity.ID)
// Set the storage bucket key in entity
entity.BucketKey = i.entityPacker.BucketKey(entity.ID)
}
ns, err := namespace.FromContext(ctx)
@ -1032,7 +1032,7 @@ func (i *IdentityStore) sanitizeAndUpsertGroup(ctx context.Context, group *ident
}
// Set the hash value of the storage bucket key in group
group.BucketKeyHash = i.groupPacker.BucketKeyHashByItemID(group.ID)
group.BucketKey = i.groupPacker.BucketKey(group.ID)
}
if group.NamespaceID == "" {
@ -1792,16 +1792,16 @@ OUTER:
return name, nil
}
func (i *IdentityStore) MemDBGroupsByBucketEntryKeyHashInTxn(txn *memdb.Txn, hashValue string) ([]*identity.Group, error) {
func (i *IdentityStore) MemDBGroupsByBucketKeyInTxn(txn *memdb.Txn, bucketKey string) ([]*identity.Group, error) {
if txn == nil {
return nil, fmt.Errorf("nil txn")
}
if hashValue == "" {
return nil, fmt.Errorf("empty hash value")
if bucketKey == "" {
return nil, fmt.Errorf("empty bucket key")
}
groupsIter, err := txn.Get(groupsTable, "bucket_key_hash", hashValue)
groupsIter, err := txn.Get(groupsTable, "bucket_key", bucketKey)
if err != nil {
return nil, errwrap.Wrapf("failed to lookup groups using bucket entry key hash: {{err}}", err)
}

View File

@ -2603,9 +2603,9 @@ func TestSystemBackend_PathWildcardPreflight(t *testing.T) {
}
if err := core.identityStore.upsertEntity(ctx, &identity.Entity{
ID: "abcd",
Name: "abcd",
BucketKeyHash: "abcd",
ID: "abcd",
Name: "abcd",
BucketKey: "abcd",
}, nil, false); err != nil {
t.Fatal(err)
}