Add AWS Secret Engine Root Credential Rotation (#5140)

* Add AWS Secret Engine Root Credential Rotation

This allows the AWS Secret Engine to rotate its credentials used to
access AWS. This will only work when the AWS Secret Engine has been
provided explicit IAM credentials via the config/root endpoint, and
further, when the IAM credentials provided are the only access key on
the IAM user associated wtih the access key (because AWS allows a
maximum of 2 access keys per user).

Fixes #4385

* Add test for AWS root credential rotation

Also fix a typo in the root credential rotation code

* Add docs for AWS root rotation

* Add locks around reading and writing config/root

And wire the backend up in a bunch of places so the config can get the
lock

* Respond to PR feedback

* Fix casing in error messages

* Fix merge errors

* Fix locking bugs

builtin/logical/aws/backend.go

@@ -36,6 +36,7 @@ func Backend() *backend {
 
 		Paths: []*framework.Path{
 			pathConfigRoot(&b),
+			pathConfigRotateRoot(&b),
 			pathConfigLease(&b),
 			pathRoles(&b),
 			pathListRoles(&b),
@@ -60,7 +61,7 @@ type backend struct {
 	// Mutex to protect access to reading and writing policies
 	roleMutex sync.RWMutex
 
-	// Mutex to protect access to iam/sts clients
+	// Mutex to protect access to iam/sts clients and client configs
 	clientMutex sync.RWMutex
 
 	// iamClient and stsClient hold configured iam and sts clients for reuse, and
@@ -99,7 +100,7 @@ func (b *backend) clientIAM(ctx context.Context, s logical.Storage) (iamiface.IA
 		return b.iamClient, nil
 	}
 
-	iamClient, err := clientIAM(ctx, s)
+	iamClient, err := nonCachedClientIAM(ctx, s)
 	if err != nil {
 		return nil, err
 	}
@@ -126,7 +127,7 @@ func (b *backend) clientSTS(ctx context.Context, s logical.Storage) (stsiface.ST
 		return b.stsClient, nil
 	}
 
-	stsClient, err := clientSTS(ctx, s)
+	stsClient, err := nonCachedClientSTS(ctx, s)
 	if err != nil {
 		return nil, err
 	}

builtin/logical/aws/backend_test.go

@@ -80,6 +80,7 @@ func TestBackend_basicSTS(t *testing.T) {
 		Backend: getBackend(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepConfigWithCreds(t, accessKey),
+			testAccStepRotateRoot(accessKey),
 			testAccStepWritePolicy(t, "test", testDynamoPolicy),
 			testAccStepRead(t, "sts", "test", []credentialTestFunc{listDynamoTablesTest}),
 			testAccStepWriteArnPolicyRef(t, "test", ec2PolicyArn),
@@ -436,6 +437,44 @@ func testAccStepConfigWithCreds(t *testing.T, accessKey *awsAccessKey) logicalte
 	}
 }
 
+func testAccStepRotateRoot(oldAccessKey *awsAccessKey) logicaltest.TestStep {
+	return logicaltest.TestStep{
+		Operation: logical.UpdateOperation,
+		Path:      "config/rotate-root",
+		Check: func(resp *logical.Response) error {
+			if resp == nil {
+				return fmt.Errorf("received nil response from config/rotate-root")
+			}
+			newAccessKeyID := resp.Data["access_key"].(string)
+			if newAccessKeyID == oldAccessKey.AccessKeyID {
+				return fmt.Errorf("rotate-root didn't rotate access key")
+			}
+			awsConfig := &aws.Config{
+				Region:      aws.String("us-east-1"),
+				HTTPClient:  cleanhttp.DefaultClient(),
+				Credentials: credentials.NewStaticCredentials(oldAccessKey.AccessKeyID, oldAccessKey.SecretAccessKey, ""),
+			}
+			// sigh....
+			oldAccessKey.AccessKeyID = newAccessKeyID
+			log.Println("[WARN] Sleeping for 10 seconds waiting for AWS...")
+			time.Sleep(10 * time.Second)
+			svc := sts.New(session.New(awsConfig))
+			params := &sts.GetCallerIdentityInput{}
+			_, err := svc.GetCallerIdentity(params)
+			if err == nil {
+				return fmt.Errorf("bad: old credentials succeeded after rotate")
+			}
+			if aerr, ok := err.(awserr.Error); ok {
+				if aerr.Code() != "InvalidClientTokenId" {
+					return fmt.Errorf("Unknown error returned from AWS: %#v", aerr)
+				}
+				return nil
+			}
+			return err
+		},
+	}
+}
+
 func testAccStepRead(t *testing.T, path, name string, credentialTests []credentialTestFunc) logicaltest.TestStep {
 	return logicaltest.TestStep{
 		Operation: logical.ReadOperation,

builtin/logical/aws/client.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/vault/logical"
 )
 
+// NOTE: The caller is required to ensure that b.clientMutex is at least read locked
 func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*aws.Config, error) {
 	credsConfig := &awsutil.CredentialsConfig{}
 	var endpoint string
@@ -68,7 +69,7 @@ func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*
 	}, nil
 }
 
-func clientIAM(ctx context.Context, s logical.Storage) (*iam.IAM, error) {
+func nonCachedClientIAM(ctx context.Context, s logical.Storage) (*iam.IAM, error) {
 	awsConfig, err := getRootConfig(ctx, s, "iam")
 	if err != nil {
 		return nil, err
@@ -82,7 +83,7 @@ func clientIAM(ctx context.Context, s logical.Storage) (*iam.IAM, error) {
 	return client, nil
 }
 
-func clientSTS(ctx context.Context, s logical.Storage) (*sts.STS, error) {
+func nonCachedClientSTS(ctx context.Context, s logical.Storage) (*sts.STS, error) {
 	awsConfig, err := getRootConfig(ctx, s, "sts")
 	if err != nil {
 		return nil, err

builtin/logical/aws/path_config_root.go

@@ -56,6 +56,9 @@ func (b *backend) pathConfigRootWrite(ctx context.Context, req *logical.Request,
 	stsendpoint := data.Get("sts_endpoint").(string)
 	maxretries := data.Get("max_retries").(int)
 
+	b.clientMutex.Lock()
+	defer b.clientMutex.Unlock()
+
 	entry, err := logical.StorageEntryJSON("config/root", rootConfig{
 		AccessKey:   data.Get("access_key").(string),
 		SecretKey:   data.Get("secret_key").(string),
@@ -74,8 +77,6 @@ func (b *backend) pathConfigRootWrite(ctx context.Context, req *logical.Request,
 
 	// clear possible cached IAM / STS clients after successfully updating
 	// config/root
-	b.clientMutex.Lock()
-	defer b.clientMutex.Unlock()
 	b.iamClient = nil
 	b.stsClient = nil
 

builtin/logical/aws/path_config_rotate_root.go

@@ -0,0 +1,124 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/vault/logical"
+	"github.com/hashicorp/vault/logical/framework"
+)
+
+func pathConfigRotateRoot(b *backend) *framework.Path {
+	return &framework.Path{
+		Pattern: "config/rotate-root",
+		Callbacks: map[logical.Operation]framework.OperationFunc{
+			logical.UpdateOperation: b.pathConfigRotateRootUpdate,
+		},
+
+		HelpSynopsis:    pathConfigRotateRootHelpSyn,
+		HelpDescription: pathConfigRotateRootHelpDesc,
+	}
+}
+
+func (b *backend) pathConfigRotateRootUpdate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	// have to get the client config first because that takes out a read lock
+	client, err := b.clientIAM(ctx, req.Storage)
+	if err != nil {
+		return nil, err
+	}
+	if client == nil {
+		return nil, fmt.Errorf("nil IAM client")
+	}
+
+	b.clientMutex.Lock()
+	defer b.clientMutex.Unlock()
+
+	rawRootConfig, err := req.Storage.Get(ctx, "config/root")
+	if err != nil {
+		return nil, err
+	}
+	if rawRootConfig == nil {
+		return nil, fmt.Errorf("no configuration found for config/root")
+	}
+	var config rootConfig
+	if err := rawRootConfig.DecodeJSON(&config); err != nil {
+		return nil, errwrap.Wrapf("error reading root configuration: {{err}}", err)
+	}
+
+	if config.AccessKey == "" || config.SecretKey == "" {
+		return logical.ErrorResponse("Cannot call config/rotate-root when either access_key or secret_key is empty"), nil
+	}
+
+	var getUserInput iam.GetUserInput // empty input means get current user
+	getUserRes, err := client.GetUser(&getUserInput)
+	if err != nil {
+		return nil, errwrap.Wrapf("error calling GetUser: {{err}}", err)
+	}
+	if getUserRes == nil {
+		return nil, fmt.Errorf("nil response from GetUser")
+	}
+	if getUserRes.User == nil {
+		return nil, fmt.Errorf("nil user returned from GetUser")
+	}
+	if getUserRes.User.UserName == nil {
+		return nil, fmt.Errorf("nil UserName returned from GetUser")
+	}
+
+	createAccessKeyInput := iam.CreateAccessKeyInput{
+		UserName: getUserRes.User.UserName,
+	}
+	createAccessKeyRes, err := client.CreateAccessKey(&createAccessKeyInput)
+	if err != nil {
+		return nil, errwrap.Wrapf("error calling CreateAccessKey: {{err}}", err)
+	}
+	if createAccessKeyRes.AccessKey == nil {
+		return nil, fmt.Errorf("nil response from CreateAccessKey")
+	}
+	if createAccessKeyRes.AccessKey.AccessKeyId == nil || createAccessKeyRes.AccessKey.SecretAccessKey == nil {
+		return nil, fmt.Errorf("nil AccessKeyId or SecretAccessKey returned from CreateAccessKey")
+	}
+
+	oldAccessKey := config.AccessKey
+
+	config.AccessKey = *createAccessKeyRes.AccessKey.AccessKeyId
+	config.SecretKey = *createAccessKeyRes.AccessKey.SecretAccessKey
+
+	newEntry, err := logical.StorageEntryJSON("config/root", config)
+	if err != nil {
+		return nil, errwrap.Wrapf("error generating new config/root JSON: {{err}}", err)
+	}
+	if err := req.Storage.Put(ctx, newEntry); err != nil {
+		return nil, errwrap.Wrapf("error saving new config/root: {{err}}", err)
+	}
+
+	b.iamClient = nil
+	b.stsClient = nil
+
+	deleteAccessKeyInput := iam.DeleteAccessKeyInput{
+		AccessKeyId: aws.String(oldAccessKey),
+		UserName:    getUserRes.User.UserName,
+	}
+	_, err = client.DeleteAccessKey(&deleteAccessKeyInput)
+	if err != nil {
+		return nil, errwrap.Wrapf("error deleting old access key: {{err}}", err)
+	}
+
+	return &logical.Response{
+		Data: map[string]interface{}{
+			"access_key": config.AccessKey,
+		},
+	}, nil
+}
+
+const pathConfigRotateRootHelpSyn = `
+Request to rotate the AWS credentials used by Vault
+`
+
+const pathConfigRotateRootHelpDesc = `
+This path attempts to rotate the AWS credentials used by Vault for this mount.
+It is only valid if Vault has been configured to use AWS IAM credentials via the
+config/root endpoint.
+`

builtin/logical/aws/path_user.go

@@ -111,7 +111,7 @@ func (b *backend) pathCredsRead(ctx context.Context, req *logical.Request, d *fr
 	}
 }
 
-func pathUserRollback(ctx context.Context, req *logical.Request, _kind string, data interface{}) error {
+func (b *backend) pathUserRollback(ctx context.Context, req *logical.Request, _kind string, data interface{}) error {
 	var entry walUser
 	if err := mapstructure.Decode(data, &entry); err != nil {
 		return err
@@ -119,7 +119,7 @@ func pathUserRollback(ctx context.Context, req *logical.Request, _kind string, d
 	username := entry.UserName
 
 	// Get the client
-	client, err := clientIAM(ctx, req.Storage)
+	client, err := b.clientIAM(ctx, req.Storage)
 	if err != nil {
 		return err
 	}

builtin/logical/aws/rollback.go

@@ -9,11 +9,11 @@ import (
 	"github.com/hashicorp/vault/logical/framework"
 )
 
-var walRollbackMap = map[string]framework.WALRollbackFunc{
-	"user": pathUserRollback,
-}
-
 func (b *backend) walRollback(ctx context.Context, req *logical.Request, kind string, data interface{}) error {
+	walRollbackMap := map[string]framework.WALRollbackFunc{
+		"user": b.pathUserRollback,
+	}
+
 	if !b.System().LocalMount() && b.System().ReplicationState().HasState(consts.ReplicationPerformancePrimary) {
 		return nil
 	}

builtin/logical/aws/secret_access_keys.go

@@ -68,7 +68,6 @@ func genUsername(displayName, policyName, userType string) (ret string, warning
 func (b *backend) secretTokenCreate(ctx context.Context, s logical.Storage,
 	displayName, policyName, policy string,
 	lifeTimeInSeconds int64) (*logical.Response, error) {
-
 	stsClient, err := b.clientSTS(ctx, s)
 	if err != nil {
 		return logical.ErrorResponse(err.Error()), nil
@@ -114,7 +113,6 @@ func (b *backend) secretTokenCreate(ctx context.Context, s logical.Storage,
 func (b *backend) assumeRole(ctx context.Context, s logical.Storage,
 	displayName, roleName, roleArn, policy string,
 	lifeTimeInSeconds int64) (*logical.Response, error) {
-
 	stsClient, err := b.clientSTS(ctx, s)
 	if err != nil {
 		return logical.ErrorResponse(err.Error()), nil
@@ -164,7 +162,6 @@ func (b *backend) secretAccessKeysCreate(
 	ctx context.Context,
 	s logical.Storage,
 	displayName, policyName string, role *awsRoleEntry) (*logical.Response, error) {
-
 	iamClient, err := b.clientIAM(ctx, s)
 	if err != nil {
 		return logical.ErrorResponse(err.Error()), nil
@@ -313,7 +310,7 @@ func (b *backend) secretAccessKeysRevoke(ctx context.Context, req *logical.Reque
 	}
 
 	// Use the user rollback mechanism to delete this user
-	err := pathUserRollback(ctx, req, "user", map[string]interface{}{
+	err := b.pathUserRollback(ctx, req, "user", map[string]interface{}{
 		"username": username,
 	})
 	if err != nil {

website/source/api/secret/aws/index.html.md

@@ -80,6 +80,47 @@ $ curl \
     http://127.0.0.1:8200/v1/aws/config/root
 ```
 
+## Rotate Root IAM Credentials
+
+When you have configured Vault with static credentials, you can use this
+endpoint to have Vault rotate the access key it used. Note that, due to AWS
+eventual consistency, after calling this endpoint, subsequent calls from Vault
+to AWS may fail for a few seconds until AWS becomes consistent again.
+
+
+In order to call this endpoint, Vault's AWS access key MUST be the only access
+key on the IAM user; otherwise, generation of a new access key will fail. Once
+this method is called, Vault will now be the only entity that knows the AWS
+secret key is used to access AWS.
+
+| Method   | Path                         | Produces               |
+| :------- | :--------------------------- | :--------------------- |
+| `POST`   | `/aws/config/rotate-root`    | `200 application/json` |
+
+### Parameters
+
+There are no parameters to this operation.
+
+### Sample Request
+
+```$ curl \
+    --header "X-Vault-Token: ..." \
+    --request POST \
+    http://127.0.0.1:8211/v1/aws/config/rotate-root
+```
+
+### Sample Response
+
+```json
+{
+  "data": {
+    "access_key": "AKIA..."
+  }
+}
+```
+
+The new access key Vault uses is returned by this operation.
+
 ## Configure Lease
 
 This endpoint configures lease settings for the AWS secrets engine. It is