luxolus
/
open-vault
2
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

Fix issue with rotateCredentials deadlocking with itself (#7518)

This commit is contained in:
ncabatoff 2019-10-03 12:28:29 -04:00 committed by GitHub
parent a9b208793c
commit 71cb7cbf18
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 184 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
builtin/logical/database/rotation.go
View File

@ -127,25 +127,29 @@ type setCredentialsWAL struct {
// This method loops through the priority queue, popping the highest priority
// item until it encounters the first item that does not yet need rotation,
// based on the current time.
func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Storage) error {
for {
func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Storage) {
for b.rotateCredential(ctx, s) {
}
}
func (b *databaseBackend) rotateCredential(ctx context.Context, s logical.Storage) bool {
// Quit rotating credentials if shutdown has started
select {
case <-ctx.Done():
return nil
return false
default:
}
item, err := b.popFromRotationQueue()
if err != nil {
if err == queue.ErrEmpty {
return nil
if err != queue.ErrEmpty {
b.logger.Error("error popping item from queue", "err", err)
}
return err
return false
}
// Guard against possible nil item
if item == nil {
return nil
return false
}
// Grab the exclusive lock for this Role, to make sure we don't incur and
@ -162,11 +166,11 @@ func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Stora
if err := b.pushItem(item); err != nil {
b.logger.Error("unable to push item on to queue", "error", err)
}
continue
return true
}
if role == nil {
b.logger.Warn("role not found", "role", item.Key, "error", err)
continue
return true
}
// If "now" is less than the Item priority, then this item does not need to
@ -176,7 +180,7 @@ func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Stora
b.logger.Error("unable to push item on to queue", "error", err)
}
// Break out of the for loop
break
return false
}
input := &setStaticAccountInput{
@ -217,7 +221,7 @@ func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Stora
b.logger.Error("unable to push item on to queue", "error", err)
}
// Go to next item
continue
return true
}
lvr := resp.RotationTime
@ -231,8 +235,7 @@ func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Stora
if err := b.pushItem(item); err != nil {
b.logger.Warn("unable to push item on to queue", "error", err)
}
}
return nil
return true
}
// findStaticWAL loads a WAL entry by ID. If found, only return the WAL if it

91
builtin/logical/database/rotation_test.go
View File

@ -18,9 +18,12 @@ import (
"github.com/lib/pq"
)
const dbUser = "vaultstatictest"
const (
dbUser = "vaultstatictest"
dbUserDefaultPassword = "password"
const testMongoDBRole = `{ "db": "admin", "roles": [ { "role": "readWrite" } ] }`
testMongoDBRole = `{ "db": "admin", "roles": [ { "role": "readWrite" } ] }`
)
func TestBackend_StaticRole_Rotate_basic(t *testing.T) {
cluster, sys := getCluster(t)
@ -44,9 +47,9 @@ func TestBackend_StaticRole_Rotate_basic(t *testing.T) {
defer cleanup()
// create the database user
createTestPGUser(t, connURL, dbUser, "password", testRoleStaticCreate)
createTestPGUser(t, connURL, dbUser, dbUserDefaultPassword, testRoleStaticCreate)
verifyPgConn(t, dbUser, "password", connURL)
verifyPgConn(t, dbUser, dbUserDefaultPassword, connURL)
// Configure a connection
data := map[string]interface{}{
@ -192,7 +195,7 @@ func TestBackend_StaticRole_Rotate_NonStaticError(t *testing.T) {
defer cleanup()
// create the database user
createTestPGUser(t, connURL, dbUser, "password", testRoleStaticCreate)
createTestPGUser(t, connURL, dbUser, dbUserDefaultPassword, testRoleStaticCreate)
// Configure a connection
data := map[string]interface{}{
@ -254,7 +257,7 @@ func TestBackend_StaticRole_Rotate_NonStaticError(t *testing.T) {
}
// Verify username/password
verifyPgConn(t, dbUser, "password", connURL)
verifyPgConn(t, dbUser, dbUserDefaultPassword, connURL)
// Trigger rotation
data = map[string]interface{}{"name": "plugin-role-test"}
req = &logical.Request{
@ -296,7 +299,7 @@ func TestBackend_StaticRole_Revoke_user(t *testing.T) {
defer cleanup()
// create the database user
createTestPGUser(t, connURL, dbUser, "password", testRoleStaticCreate)
createTestPGUser(t, connURL, dbUser, dbUserDefaultPassword, testRoleStaticCreate)
// Configure a connection
data := map[string]interface{}{
@ -521,7 +524,7 @@ func TestBackend_Static_QueueWAL_discard_role_newer_rotation_date(t *testing.T)
defer cleanup()
// create the database user
createTestPGUser(t, connURL, dbUser, "password", testRoleStaticCreate)
createTestPGUser(t, connURL, dbUser, dbUserDefaultPassword, testRoleStaticCreate)
// Configure a connection
data := map[string]interface{}{
@ -706,7 +709,7 @@ func TestBackend_StaticRole_Rotations_PostgreSQL(t *testing.T) {
testCases := []string{"65", "130", "5400"}
// Create database users ahead
for _, tc := range testCases {
createTestPGUser(t, connURL, dbUser+tc, "password", testRoleStaticCreate)
createTestPGUser(t, connURL, dbUser+tc, dbUserDefaultPassword, testRoleStaticCreate)
}
// Configure a connection
@ -953,6 +956,76 @@ func TestBackend_StaticRole_Rotations_MongoDB(t *testing.T) {
}
}
// Demonstrates a bug fix for the credential rotation not releasing locks
func TestBackend_StaticRole_LockRegression(t *testing.T) {
cluster, sys := getCluster(t)
defer cluster.Cleanup()
config := logical.TestBackendConfig()
config.StorageView = &logical.InmemStorage{}
config.System = sys
lb, err := Factory(context.Background(), config)
if err != nil {
t.Fatal(err)
}
b, ok := lb.(*databaseBackend)
if !ok {
t.Fatal("could not convert to db backend")
}
defer b.Cleanup(context.Background())
cleanup, connURL := preparePostgresTestContainer(t, config.StorageView, b)
defer cleanup()
// Configure a connection
data := map[string]interface{}{
"connection_url": connURL,
"plugin_name": "postgresql-database-plugin",
"verify_connection": false,
"allowed_roles": []string{"*"},
"name": "plugin-test",
}
req := &logical.Request{
Operation: logical.UpdateOperation,
Path: "config/plugin-test",
Storage: config.StorageView,
Data: data,
}
resp, err := b.HandleRequest(namespace.RootContext(nil), req)
if err != nil || (resp != nil && resp.IsError()) {
t.Fatalf("err:%s resp:%#v\n", err, resp)
}
createTestPGUser(t, connURL, dbUser, dbUserDefaultPassword, testRoleStaticCreate)
for i := 0; i < 25; i++ {
data := map[string]interface{}{
"name": "plugin-role-test",
"db_name": "plugin-test",
"rotation_statements": testRoleStaticUpdate,
"username": dbUser,
"rotation_period": "7s",
}
req = &logical.Request{
Operation: logical.UpdateOperation,
Path: "static-roles/plugin-role-test",
Storage: config.StorageView,
Data: data,
}
resp, err = b.HandleRequest(namespace.RootContext(nil), req)
if err != nil || (resp != nil && resp.IsError()) {
t.Fatalf("err:%s resp:%#v\n", err, resp)
}
// sleeping is needed to trigger the deadlock, otherwise things are
// processed too quickly to trigger the rotation lock on so few roles
time.Sleep(500 * time.Millisecond)
}
}
// capturePasswords captures the current passwords at the time of calling, and
// returns a map of username / passwords building off of the input map
func capturePasswords(t *testing.T, b logical.Backend, config *logical.BackendConfig, testCases []string, pws map[string][]string) map[string][]string {