secrets/consul: Add support to auto-bootstrap Consul ACL system (#10751)

* Automatically bootstraps the Consul ACL system if no management token is given on the access config

builtin/logical/consul/backend_test.go

@@ -22,16 +22,24 @@ func TestBackend_Config_Access(t *testing.T) {
 		t.Parallel()
 		t.Run("pre-1.4.0", func(t *testing.T) {
 			t.Parallel()
-			testBackendConfigAccess(t, "1.3.1")
+			testBackendConfigAccess(t, "1.3.1", true)
 		})
 		t.Run("post-1.4.0", func(t *testing.T) {
 			t.Parallel()
-			testBackendConfigAccess(t, "")
+			testBackendConfigAccess(t, "", true)
+		})
+		t.Run("pre-1.4.0 automatic-bootstrap", func(t *testing.T) {
+			t.Parallel()
+			testBackendConfigAccess(t, "1.3.1", false)
+		})
+		t.Run("post-1.4.0 automatic-bootstrap", func(t *testing.T) {
+			t.Parallel()
+			testBackendConfigAccess(t, "", false)
 		})
 	})
 }
 
-func testBackendConfigAccess(t *testing.T, version string) {
+func testBackendConfigAccess(t *testing.T, version string, bootstrap bool) {
 	config := logical.TestBackendConfig()
 	config.StorageView = &logical.InmemStorage{}
 	b, err := Factory(context.Background(), config)
@@ -39,7 +47,7 @@ func testBackendConfigAccess(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, bootstrap)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -104,7 +112,7 @@ func testBackendRenewRevoke(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -209,7 +217,7 @@ func testBackendRenewRevoke14(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -321,7 +329,7 @@ func TestBackend_LocalToken(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, "", false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, "", false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -466,7 +474,7 @@ func testBackendManagement(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -511,7 +519,7 @@ func testBackendBasic(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -713,7 +721,7 @@ func TestBackend_Roles(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, "", false)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, "", false, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -842,7 +850,7 @@ func testBackendEntNamespace(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, "", true)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, "", true, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -962,7 +970,7 @@ func testBackendEntPartition(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, consulConfig := consul.PrepareTestContainer(t, "", true)
+	cleanup, consulConfig := consul.PrepareTestContainer(t, "", true, true)
 	defer cleanup()
 
 	connData := map[string]interface{}{

builtin/logical/consul/client.go

@@ -20,14 +20,7 @@ func (b *backend) client(ctx context.Context, s logical.Storage) (*api.Client, e
 		return nil, nil, fmt.Errorf("no error received but no configuration found")
 	}
 
-	consulConf := api.DefaultNonPooledConfig()
+	consulConf := conf.NewConfig()
-	consulConf.Address = conf.Address
-	consulConf.Scheme = conf.Scheme
-	consulConf.Token = conf.Token
-	consulConf.TLSConfig.CAPem = []byte(conf.CACert)
-	consulConf.TLSConfig.CertPEM = []byte(conf.ClientCert)
-	consulConf.TLSConfig.KeyPEM = []byte(conf.ClientKey)
-
 	client, err := api.NewClient(consulConf)
 	return client, nil, err
 }

builtin/logical/consul/path_config.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/logical"
 )
@@ -96,14 +97,31 @@ func (b *backend) pathConfigAccessRead(ctx context.Context, req *logical.Request
 }
 
 func (b *backend) pathConfigAccessWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
-	entry, err := logical.StorageEntryJSON("config/access", accessConfig{
+	config := accessConfig{
 		Address:    data.Get("address").(string),
 		Scheme:     data.Get("scheme").(string),
 		Token:      data.Get("token").(string),
 		CACert:     data.Get("ca_cert").(string),
 		ClientCert: data.Get("client_cert").(string),
 		ClientKey:  data.Get("client_key").(string),
-	})
+	}
+
+	// If a token has not been given by the user, we try to boostrap the ACL
+	// support
+	if config.Token == "" {
+		consulConf := config.NewConfig()
+		client, err := api.NewClient(consulConf)
+		if err != nil {
+			return nil, err
+		}
+		token, _, err := client.ACL().Bootstrap()
+		if err != nil {
+			return logical.ErrorResponse("Token not provided and failed to bootstrap ACLs"), err
+		}
+		config.Token = token.SecretID
+	}
+
+	entry, err := logical.StorageEntryJSON("config/access", config)
 	if err != nil {
 		return nil, err
 	}
@@ -123,3 +141,15 @@ type accessConfig struct {
 	ClientCert string `json:"client_cert"`
 	ClientKey  string `json:"client_key"`
 }
+
+func (conf *accessConfig) NewConfig() *api.Config {
+	consulConf := api.DefaultNonPooledConfig()
+	consulConf.Address = conf.Address
+	consulConf.Scheme = conf.Scheme
+	consulConf.Token = conf.Token
+	consulConf.TLSConfig.CAPem = []byte(conf.CACert)
+	consulConf.TLSConfig.CertPEM = []byte(conf.ClientCert)
+	consulConf.TLSConfig.KeyPEM = []byte(conf.ClientKey)
+
+	return consulConf
+}

changelog/10751.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+secrets/consul: Vault is now able to automatically bootstrap the Consul ACL system.
+```

helper/testhelpers/consul/consulhelper.go

@@ -27,7 +27,7 @@ func (c *Config) APIConfig() *consulapi.Config {
 // the Consul version used will be given by the environment variable
 // CONSUL_DOCKER_VERSION, or if that's empty, whatever we've hardcoded as the
 // the latest Consul version.
-func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func(), *Config) {
+func PrepareTestContainer(t *testing.T, version string, isEnterprise bool, bootstrap bool) (func(), *Config) {
 	t.Helper()
 
 	if retAddress := os.Getenv("CONSUL_HTTP_ADDR"); retAddress != "" {
@@ -94,6 +94,11 @@ func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func
 			return nil, err
 		}
 
+		// Make sure Consul is up
+		if _, err = consul.Status().Leader(); err != nil {
+			return nil, err
+		}
+
 		// For version of Consul < 1.4
 		if strings.HasPrefix(version, "1.3") {
 			consulToken := "test"
@@ -113,101 +118,104 @@ func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func
 		}
 
 		// New default behavior
-		aclbootstrap, _, err := consul.ACL().Bootstrap()
+		var consulToken string
-		if err != nil {
+		if bootstrap {
-			return nil, err
+			aclbootstrap, _, err := consul.ACL().Bootstrap()
-		}
-		consulToken := aclbootstrap.SecretID
-		policy := &consulapi.ACLPolicy{
-			Name:        "test",
-			Description: "test",
-			Rules: `node_prefix "" {
-               policy = "write"
-				}
-
-            service_prefix "" {
-               policy = "read"
-            }`,
-		}
-		q := &consulapi.WriteOptions{
-			Token: consulToken,
-		}
-		_, _, err = consul.ACL().PolicyCreate(policy, q)
-		if err != nil {
-			return nil, err
-		}
-
-		// Create a Consul role that contains the test policy, for Consul 1.5 and newer
-		currVersion, _ := goversion.NewVersion(version)
-		roleVersion, _ := goversion.NewVersion("1.5")
-		if currVersion.GreaterThanOrEqual(roleVersion) {
-			ACLList := []*consulapi.ACLLink{{Name: "test"}}
-
-			role := &consulapi.ACLRole{
-				Name:        "role-test",
-				Description: "consul roles test",
-				Policies:    ACLList,
-			}
-
-			_, _, err = consul.ACL().RoleCreate(role, q)
 			if err != nil {
 				return nil, err
 			}
-		}
+			consulToken = aclbootstrap.SecretID
-
+			policy := &consulapi.ACLPolicy{
-		// Configure a namespace and parition if testing enterprise Consul
+				Name:        "test",
-		if isEnterprise {
+				Description: "test",
-			// Namespaces require Consul 1.7 or newer
+				Rules: `node_prefix "" {
-			namespaceVersion, _ := goversion.NewVersion("1.7")
+					policy = "write"
-			if currVersion.GreaterThanOrEqual(namespaceVersion) {
-				namespace := &consulapi.Namespace{
-					Name:        "ns1",
-					Description: "ns1 test",
 				}
 
-				_, _, err = consul.Namespaces().Create(namespace, q)
+				service_prefix "" {
-				if err != nil {
+					policy = "read"
-					return nil, err
+				}`,
+			}
+			q := &consulapi.WriteOptions{
+				Token: consulToken,
+			}
+			_, _, err = consul.ACL().PolicyCreate(policy, q)
+			if err != nil {
+				return nil, err
+			}
+
+			// Create a Consul role that contains the test policy, for Consul 1.5 and newer
+			currVersion, _ := goversion.NewVersion(version)
+			roleVersion, _ := goversion.NewVersion("1.5")
+			if currVersion.GreaterThanOrEqual(roleVersion) {
+				ACLList := []*consulapi.ACLLink{{Name: "test"}}
+
+				role := &consulapi.ACLRole{
+					Name:        "role-test",
+					Description: "consul roles test",
+					Policies:    ACLList,
 				}
 
-				nsPolicy := &consulapi.ACLPolicy{
+				_, _, err = consul.ACL().RoleCreate(role, q)
-					Name:        "ns-test",
-					Description: "namespace test",
-					Namespace:   "ns1",
-					Rules: `service_prefix "" {
-							policy = "read"
-						}`,
-				}
-				_, _, err = consul.ACL().PolicyCreate(nsPolicy, q)
 				if err != nil {
 					return nil, err
 				}
 			}
 
-			// Partitions require Consul 1.11 or newer
+			// Configure a namespace and parition if testing enterprise Consul
-			partitionVersion, _ := goversion.NewVersion("1.11")
+			if isEnterprise {
-			if currVersion.GreaterThanOrEqual(partitionVersion) {
+				// Namespaces require Consul 1.7 or newer
-				partition := &consulapi.Partition{
+				namespaceVersion, _ := goversion.NewVersion("1.7")
-					Name:        "part1",
+				if currVersion.GreaterThanOrEqual(namespaceVersion) {
-					Description: "part1 test",
+					namespace := &consulapi.Namespace{
-				}
+						Name:        "ns1",
+						Description: "ns1 test",
+					}
 
-				_, _, err = consul.Partitions().Create(ctx, partition, q)
+					_, _, err = consul.Namespaces().Create(namespace, q)
-				if err != nil {
+					if err != nil {
-					return nil, err
+						return nil, err
-				}
+					}
 
-				partPolicy := &consulapi.ACLPolicy{
+					nsPolicy := &consulapi.ACLPolicy{
-					Name:        "part-test",
+						Name:        "ns-test",
-					Description: "partition test",
+						Description: "namespace test",
-					Partition:   "part1",
+						Namespace:   "ns1",
-					Rules: `service_prefix "" {
+						Rules: `service_prefix "" {
 							policy = "read"
 						}`,
+					}
+					_, _, err = consul.ACL().PolicyCreate(nsPolicy, q)
+					if err != nil {
+						return nil, err
+					}
 				}
-				_, _, err = consul.ACL().PolicyCreate(partPolicy, q)
+
-				if err != nil {
+				// Partitions require Consul 1.11 or newer
-					return nil, err
+				partitionVersion, _ := goversion.NewVersion("1.11")
+				if currVersion.GreaterThanOrEqual(partitionVersion) {
+					partition := &consulapi.Partition{
+						Name:        "part1",
+						Description: "part1 test",
+					}
+
+					_, _, err = consul.Partitions().Create(ctx, partition, q)
+					if err != nil {
+						return nil, err
+					}
+
+					partPolicy := &consulapi.ACLPolicy{
+						Name:        "part-test",
+						Description: "partition test",
+						Partition:   "part1",
+						Rules: `service_prefix "" {
+							policy = "read"
+						}`,
+					}
+					_, _, err = consul.ACL().PolicyCreate(partPolicy, q)
+					if err != nil {
+						return nil, err
+					}
 				}
 			}
 		}

helper/testhelpers/teststorage/consul/consul.go

@@ -12,7 +12,7 @@ import (
 )
 
 func MakeConsulBackend(t testing.T, logger hclog.Logger) *vault.PhysicalBackendBundle {
-	cleanup, config := consul.PrepareTestContainer(t.(*realtesting.T), "", false)
+	cleanup, config := consul.PrepareTestContainer(t.(*realtesting.T), "", false, true)
 
 	consulConf := map[string]string{
 		"address":      config.Address(),

physical/consul/consul_test.go

@@ -157,7 +157,7 @@ func TestConsul_newConsulBackend(t *testing.T) {
 }
 
 func TestConsulBackend(t *testing.T) {
-	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false)
+	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true)
 	defer cleanup()
 
 	client, err := api.NewClient(config.APIConfig())
@@ -187,7 +187,7 @@ func TestConsulBackend(t *testing.T) {
 }
 
 func TestConsul_TooLarge(t *testing.T) {
-	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false)
+	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true)
 	defer cleanup()
 
 	client, err := api.NewClient(config.APIConfig())
@@ -212,7 +212,7 @@ func TestConsul_TooLarge(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	zeros := make([]byte, 600000, 600000)
+	zeros := make([]byte, 600000)
 	n, err := rand.Read(zeros)
 	if n != 600000 {
 		t.Fatalf("expected 500k zeros, read %d", n)
@@ -250,7 +250,7 @@ func TestConsul_TooLarge(t *testing.T) {
 }
 
 func TestConsulHABackend(t *testing.T) {
-	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false)
+	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true)
 	defer cleanup()
 
 	client, err := api.NewClient(config.APIConfig())

serviceregistration/consul/consul_service_registration_test.go

@@ -50,7 +50,7 @@ func testConsulServiceRegistrationConfig(t *testing.T, conf *consulConf) *servic
 // TestConsul_ServiceRegistration tests whether consul ServiceRegistration works
 func TestConsul_ServiceRegistration(t *testing.T) {
 	// Prepare a docker-based consul instance
-	cleanup, config := consul.PrepareTestContainer(t, "", false)
+	cleanup, config := consul.PrepareTestContainer(t, "", false, true)
 	defer cleanup()
 
 	// Create a consul client

website/content/api-docs/secret/consul.mdx

@@ -31,8 +31,9 @@ Consul tokens.
 
 - `scheme` `(string: "http")` – Specifies the URL scheme to use.
 
-- `token` `(string: <required>)` – Specifies the Consul ACL token to use. This
+- `token` `(string: "")` – Specifies the Consul ACL token to use. This
-  must be a management type token.
+  must be a management type token. If this is not provided, Vault will try to
+  bootstrap the ACL system of the Consul cluster.
 
 - `ca_cert` `(string: "")` - CA certificate to use when verifying Consul server certificate,
   must be x509 PEM encoded.

website/content/docs/secrets/consul.mdx

@@ -25,41 +25,52 @@ management tool.
     By default, the secrets engine will mount at the name of the engine. To
     enable the secrets engine at a different path, use the `-path` argument.
 
-1.  Bootstrap the Consul ACL system if not already done. To begin configuring the secrets engine, we must give Vault
+1. Vault can bootstrap the ACL system of your Consul cluster if it has
-    the necessary credentials to manage Consul.
+   not already been done. In this case, you only need the address of your
+   Consul cluster to configure the Consul secret engine:
 
-    In Consul versions below 1.4, acquire a [management token][consul-mgmt-token] from Consul using the
+    ```text
-    `acl_master_token` from your Consul configuration file, or another management token:
+    $ vault write consul/config/access \
-
+        address=127.0.0.1:8500
-    ```shell-session
+    Success! Data written to: consul/config/access
-    $ curl \
-        --header "X-Consul-Token: my-management-token" \
-        --request POST \
-        --data '{"Name": "sample", "Type": "management"}' \
-        https://consul.rocks/v1/acl/create
     ```
 
-    Vault must have a "management" type token so that it can create and revoke ACL
+    If you have already bootstrapped the ACL system of your Consul cluster, you
-    tokens. The response will return a new token:
+    will need to give Vault a management token:
 
-    ```json
+     - In Consul versions below 1.4, acquire a [management token][consul-mgmt-token] from Consul, using the
-    {
+       `acl_master_token` from your Consul configuration file or another management
-      "ID": "7652ba4c-0f6e-8e75-5724-5e083d72cfe4"
+       token:
-    }
-    ```
 
-    For Consul 1.4 and above, use the command line to generate a token with the appropriate policy:
+        ```sh
+        $ curl \
+            --header "X-Consul-Token: my-management-token" \
+            --request PUT \
+            --data '{"Name": "sample", "Type": "management"}' \
+            https://consul.rocks/v1/acl/create
+        ```
 
-    ```shell-session
+        Vault must have a management type token so that it can create and revoke ACL
-    $ CONSUL_HTTP_TOKEN="<management-token>" consul acl token create -policy-name="global-management"
+        tokens. The response will return a new token:
-    AccessorID:   865dc5e9-e585-3180-7b49-4ddc0fc45135
+
-    SecretID:     ef35f0f1-885b-0cab-573c-7c91b65a7a7e
+        ```json
-    Description:
+        {
-    Local:        false
+        "ID": "7652ba4c-0f6e-8e75-5724-5e083d72cfe4"
-    Create Time:  2018-10-22 17:40:24.128188 -0700 PDT
+        }
-    Policies:
+        ```
-        00000000-0000-0000-0000-000000000001 - global-management
+
-    ```
+     - For Consul 1.4 and above, use the command line to generate a token with the appropriate policy:
+
+        ```shell-session
+        $ CONSUL_HTTP_TOKEN="<management-token>" consul acl token create -policy-name="global-management"
+        AccessorID:   865dc5e9-e585-3180-7b49-4ddc0fc45135
+        SecretID:     ef35f0f1-885b-0cab-573c-7c91b65a7a7e
+        Description:
+        Local:        false
+        Create Time:  2018-10-22 17:40:24.128188 -0700 PDT
+        Policies:
+            00000000-0000-0000-0000-000000000001 - global-management
+        ```
 
 1.  Configure Vault to connect and authenticate to Consul: