From bf4c4595f3cb7145ffd93b0eb332e1d89fa48a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Thu, 21 Apr 2022 00:16:15 +0200 Subject: [PATCH] secrets/consul: Add support to auto-bootstrap Consul ACL system (#10751) * Automatically bootstraps the Consul ACL system if no management token is given on the access config --- builtin/logical/consul/backend_test.go | 32 ++-- builtin/logical/consul/client.go | 9 +- builtin/logical/consul/path_config.go | 34 +++- changelog/10751.txt | 3 + helper/testhelpers/consul/consulhelper.go | 166 +++++++++--------- .../testhelpers/teststorage/consul/consul.go | 2 +- physical/consul/consul_test.go | 8 +- .../consul_service_registration_test.go | 2 +- website/content/api-docs/secret/consul.mdx | 5 +- website/content/docs/secrets/consul.mdx | 69 +++++--- 10 files changed, 192 insertions(+), 138 deletions(-) create mode 100644 changelog/10751.txt diff --git a/builtin/logical/consul/backend_test.go b/builtin/logical/consul/backend_test.go index 843338bf4..5474f7cb5 100644 --- a/builtin/logical/consul/backend_test.go +++ b/builtin/logical/consul/backend_test.go @@ -22,16 +22,24 @@ func TestBackend_Config_Access(t *testing.T) { t.Parallel() t.Run("pre-1.4.0", func(t *testing.T) { t.Parallel() - testBackendConfigAccess(t, "1.3.1") + testBackendConfigAccess(t, "1.3.1", true) }) t.Run("post-1.4.0", func(t *testing.T) { t.Parallel() - testBackendConfigAccess(t, "") + testBackendConfigAccess(t, "", true) + }) + t.Run("pre-1.4.0 automatic-bootstrap", func(t *testing.T) { + t.Parallel() + testBackendConfigAccess(t, "1.3.1", false) + }) + t.Run("post-1.4.0 automatic-bootstrap", func(t *testing.T) { + t.Parallel() + testBackendConfigAccess(t, "", false) }) }) } -func testBackendConfigAccess(t *testing.T, version string) { +func testBackendConfigAccess(t *testing.T, version string, bootstrap bool) { config := logical.TestBackendConfig() config.StorageView = &logical.InmemStorage{} b, err := Factory(context.Background(), config) @@ -39,7 +47,7 @@ func testBackendConfigAccess(t *testing.T, version string) { t.Fatal(err) } - cleanup, consulConfig := consul.PrepareTestContainer(t, version, false) + cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, bootstrap) defer cleanup() connData := map[string]interface{}{ @@ -104,7 +112,7 @@ func testBackendRenewRevoke(t *testing.T, version string) { t.Fatal(err) } - cleanup, consulConfig := consul.PrepareTestContainer(t, version, false) + cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true) defer cleanup() connData := map[string]interface{}{ @@ -209,7 +217,7 @@ func testBackendRenewRevoke14(t *testing.T, version string) { t.Fatal(err) } - cleanup, consulConfig := consul.PrepareTestContainer(t, version, false) + cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true) defer cleanup() connData := map[string]interface{}{ @@ -321,7 +329,7 @@ func TestBackend_LocalToken(t *testing.T) { t.Fatal(err) } - cleanup, consulConfig := consul.PrepareTestContainer(t, "", false) + cleanup, consulConfig := consul.PrepareTestContainer(t, "", false, true) defer cleanup() connData := map[string]interface{}{ @@ -466,7 +474,7 @@ func testBackendManagement(t *testing.T, version string) { t.Fatal(err) } - cleanup, consulConfig := consul.PrepareTestContainer(t, version, false) + cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true) defer cleanup() connData := map[string]interface{}{ @@ -511,7 +519,7 @@ func testBackendBasic(t *testing.T, version string) { t.Fatal(err) } - cleanup, consulConfig := consul.PrepareTestContainer(t, version, false) + cleanup, consulConfig := consul.PrepareTestContainer(t, version, false, true) defer cleanup() connData := map[string]interface{}{ @@ -713,7 +721,7 @@ func TestBackend_Roles(t *testing.T) { t.Fatal(err) } - cleanup, consulConfig := consul.PrepareTestContainer(t, "", false) + cleanup, consulConfig := consul.PrepareTestContainer(t, "", false, true) defer cleanup() connData := map[string]interface{}{ @@ -842,7 +850,7 @@ func testBackendEntNamespace(t *testing.T) { t.Fatal(err) } - cleanup, consulConfig := consul.PrepareTestContainer(t, "", true) + cleanup, consulConfig := consul.PrepareTestContainer(t, "", true, true) defer cleanup() connData := map[string]interface{}{ @@ -962,7 +970,7 @@ func testBackendEntPartition(t *testing.T) { t.Fatal(err) } - cleanup, consulConfig := consul.PrepareTestContainer(t, "", true) + cleanup, consulConfig := consul.PrepareTestContainer(t, "", true, true) defer cleanup() connData := map[string]interface{}{ diff --git a/builtin/logical/consul/client.go b/builtin/logical/consul/client.go index 3815b1002..fd54830a4 100644 --- a/builtin/logical/consul/client.go +++ b/builtin/logical/consul/client.go @@ -20,14 +20,7 @@ func (b *backend) client(ctx context.Context, s logical.Storage) (*api.Client, e return nil, nil, fmt.Errorf("no error received but no configuration found") } - consulConf := api.DefaultNonPooledConfig() - consulConf.Address = conf.Address - consulConf.Scheme = conf.Scheme - consulConf.Token = conf.Token - consulConf.TLSConfig.CAPem = []byte(conf.CACert) - consulConf.TLSConfig.CertPEM = []byte(conf.ClientCert) - consulConf.TLSConfig.KeyPEM = []byte(conf.ClientKey) - + consulConf := conf.NewConfig() client, err := api.NewClient(consulConf) return client, nil, err } diff --git a/builtin/logical/consul/path_config.go b/builtin/logical/consul/path_config.go index 8eaf0c437..1fd60e30e 100644 --- a/builtin/logical/consul/path_config.go +++ b/builtin/logical/consul/path_config.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/hashicorp/consul/api" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/logical" ) @@ -96,14 +97,31 @@ func (b *backend) pathConfigAccessRead(ctx context.Context, req *logical.Request } func (b *backend) pathConfigAccessWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - entry, err := logical.StorageEntryJSON("config/access", accessConfig{ + config := accessConfig{ Address: data.Get("address").(string), Scheme: data.Get("scheme").(string), Token: data.Get("token").(string), CACert: data.Get("ca_cert").(string), ClientCert: data.Get("client_cert").(string), ClientKey: data.Get("client_key").(string), - }) + } + + // If a token has not been given by the user, we try to boostrap the ACL + // support + if config.Token == "" { + consulConf := config.NewConfig() + client, err := api.NewClient(consulConf) + if err != nil { + return nil, err + } + token, _, err := client.ACL().Bootstrap() + if err != nil { + return logical.ErrorResponse("Token not provided and failed to bootstrap ACLs"), err + } + config.Token = token.SecretID + } + + entry, err := logical.StorageEntryJSON("config/access", config) if err != nil { return nil, err } @@ -123,3 +141,15 @@ type accessConfig struct { ClientCert string `json:"client_cert"` ClientKey string `json:"client_key"` } + +func (conf *accessConfig) NewConfig() *api.Config { + consulConf := api.DefaultNonPooledConfig() + consulConf.Address = conf.Address + consulConf.Scheme = conf.Scheme + consulConf.Token = conf.Token + consulConf.TLSConfig.CAPem = []byte(conf.CACert) + consulConf.TLSConfig.CertPEM = []byte(conf.ClientCert) + consulConf.TLSConfig.KeyPEM = []byte(conf.ClientKey) + + return consulConf +} diff --git a/changelog/10751.txt b/changelog/10751.txt new file mode 100644 index 000000000..6fb7c3185 --- /dev/null +++ b/changelog/10751.txt @@ -0,0 +1,3 @@ +```release-note:improvement +secrets/consul: Vault is now able to automatically bootstrap the Consul ACL system. +``` diff --git a/helper/testhelpers/consul/consulhelper.go b/helper/testhelpers/consul/consulhelper.go index 77aad05b0..e88149079 100644 --- a/helper/testhelpers/consul/consulhelper.go +++ b/helper/testhelpers/consul/consulhelper.go @@ -27,7 +27,7 @@ func (c *Config) APIConfig() *consulapi.Config { // the Consul version used will be given by the environment variable // CONSUL_DOCKER_VERSION, or if that's empty, whatever we've hardcoded as the // the latest Consul version. -func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func(), *Config) { +func PrepareTestContainer(t *testing.T, version string, isEnterprise bool, bootstrap bool) (func(), *Config) { t.Helper() if retAddress := os.Getenv("CONSUL_HTTP_ADDR"); retAddress != "" { @@ -94,6 +94,11 @@ func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func return nil, err } + // Make sure Consul is up + if _, err = consul.Status().Leader(); err != nil { + return nil, err + } + // For version of Consul < 1.4 if strings.HasPrefix(version, "1.3") { consulToken := "test" @@ -113,101 +118,104 @@ func PrepareTestContainer(t *testing.T, version string, isEnterprise bool) (func } // New default behavior - aclbootstrap, _, err := consul.ACL().Bootstrap() - if err != nil { - return nil, err - } - consulToken := aclbootstrap.SecretID - policy := &consulapi.ACLPolicy{ - Name: "test", - Description: "test", - Rules: `node_prefix "" { - policy = "write" - } - - service_prefix "" { - policy = "read" - }`, - } - q := &consulapi.WriteOptions{ - Token: consulToken, - } - _, _, err = consul.ACL().PolicyCreate(policy, q) - if err != nil { - return nil, err - } - - // Create a Consul role that contains the test policy, for Consul 1.5 and newer - currVersion, _ := goversion.NewVersion(version) - roleVersion, _ := goversion.NewVersion("1.5") - if currVersion.GreaterThanOrEqual(roleVersion) { - ACLList := []*consulapi.ACLLink{{Name: "test"}} - - role := &consulapi.ACLRole{ - Name: "role-test", - Description: "consul roles test", - Policies: ACLList, - } - - _, _, err = consul.ACL().RoleCreate(role, q) + var consulToken string + if bootstrap { + aclbootstrap, _, err := consul.ACL().Bootstrap() if err != nil { return nil, err } - } - - // Configure a namespace and parition if testing enterprise Consul - if isEnterprise { - // Namespaces require Consul 1.7 or newer - namespaceVersion, _ := goversion.NewVersion("1.7") - if currVersion.GreaterThanOrEqual(namespaceVersion) { - namespace := &consulapi.Namespace{ - Name: "ns1", - Description: "ns1 test", + consulToken = aclbootstrap.SecretID + policy := &consulapi.ACLPolicy{ + Name: "test", + Description: "test", + Rules: `node_prefix "" { + policy = "write" } - _, _, err = consul.Namespaces().Create(namespace, q) - if err != nil { - return nil, err + service_prefix "" { + policy = "read" + }`, + } + q := &consulapi.WriteOptions{ + Token: consulToken, + } + _, _, err = consul.ACL().PolicyCreate(policy, q) + if err != nil { + return nil, err + } + + // Create a Consul role that contains the test policy, for Consul 1.5 and newer + currVersion, _ := goversion.NewVersion(version) + roleVersion, _ := goversion.NewVersion("1.5") + if currVersion.GreaterThanOrEqual(roleVersion) { + ACLList := []*consulapi.ACLLink{{Name: "test"}} + + role := &consulapi.ACLRole{ + Name: "role-test", + Description: "consul roles test", + Policies: ACLList, } - nsPolicy := &consulapi.ACLPolicy{ - Name: "ns-test", - Description: "namespace test", - Namespace: "ns1", - Rules: `service_prefix "" { - policy = "read" - }`, - } - _, _, err = consul.ACL().PolicyCreate(nsPolicy, q) + _, _, err = consul.ACL().RoleCreate(role, q) if err != nil { return nil, err } } - // Partitions require Consul 1.11 or newer - partitionVersion, _ := goversion.NewVersion("1.11") - if currVersion.GreaterThanOrEqual(partitionVersion) { - partition := &consulapi.Partition{ - Name: "part1", - Description: "part1 test", - } + // Configure a namespace and parition if testing enterprise Consul + if isEnterprise { + // Namespaces require Consul 1.7 or newer + namespaceVersion, _ := goversion.NewVersion("1.7") + if currVersion.GreaterThanOrEqual(namespaceVersion) { + namespace := &consulapi.Namespace{ + Name: "ns1", + Description: "ns1 test", + } - _, _, err = consul.Partitions().Create(ctx, partition, q) - if err != nil { - return nil, err - } + _, _, err = consul.Namespaces().Create(namespace, q) + if err != nil { + return nil, err + } - partPolicy := &consulapi.ACLPolicy{ - Name: "part-test", - Description: "partition test", - Partition: "part1", - Rules: `service_prefix "" { + nsPolicy := &consulapi.ACLPolicy{ + Name: "ns-test", + Description: "namespace test", + Namespace: "ns1", + Rules: `service_prefix "" { policy = "read" }`, + } + _, _, err = consul.ACL().PolicyCreate(nsPolicy, q) + if err != nil { + return nil, err + } } - _, _, err = consul.ACL().PolicyCreate(partPolicy, q) - if err != nil { - return nil, err + + // Partitions require Consul 1.11 or newer + partitionVersion, _ := goversion.NewVersion("1.11") + if currVersion.GreaterThanOrEqual(partitionVersion) { + partition := &consulapi.Partition{ + Name: "part1", + Description: "part1 test", + } + + _, _, err = consul.Partitions().Create(ctx, partition, q) + if err != nil { + return nil, err + } + + partPolicy := &consulapi.ACLPolicy{ + Name: "part-test", + Description: "partition test", + Partition: "part1", + Rules: `service_prefix "" { + policy = "read" + }`, + } + _, _, err = consul.ACL().PolicyCreate(partPolicy, q) + if err != nil { + return nil, err + } } } } diff --git a/helper/testhelpers/teststorage/consul/consul.go b/helper/testhelpers/teststorage/consul/consul.go index 13c4f15d8..47ec99f29 100644 --- a/helper/testhelpers/teststorage/consul/consul.go +++ b/helper/testhelpers/teststorage/consul/consul.go @@ -12,7 +12,7 @@ import ( ) func MakeConsulBackend(t testing.T, logger hclog.Logger) *vault.PhysicalBackendBundle { - cleanup, config := consul.PrepareTestContainer(t.(*realtesting.T), "", false) + cleanup, config := consul.PrepareTestContainer(t.(*realtesting.T), "", false, true) consulConf := map[string]string{ "address": config.Address(), diff --git a/physical/consul/consul_test.go b/physical/consul/consul_test.go index 973c8d464..60a185287 100644 --- a/physical/consul/consul_test.go +++ b/physical/consul/consul_test.go @@ -157,7 +157,7 @@ func TestConsul_newConsulBackend(t *testing.T) { } func TestConsulBackend(t *testing.T) { - cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false) + cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true) defer cleanup() client, err := api.NewClient(config.APIConfig()) @@ -187,7 +187,7 @@ func TestConsulBackend(t *testing.T) { } func TestConsul_TooLarge(t *testing.T) { - cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false) + cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true) defer cleanup() client, err := api.NewClient(config.APIConfig()) @@ -212,7 +212,7 @@ func TestConsul_TooLarge(t *testing.T) { t.Fatalf("err: %s", err) } - zeros := make([]byte, 600000, 600000) + zeros := make([]byte, 600000) n, err := rand.Read(zeros) if n != 600000 { t.Fatalf("expected 500k zeros, read %d", n) @@ -250,7 +250,7 @@ func TestConsul_TooLarge(t *testing.T) { } func TestConsulHABackend(t *testing.T) { - cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false) + cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true) defer cleanup() client, err := api.NewClient(config.APIConfig()) diff --git a/serviceregistration/consul/consul_service_registration_test.go b/serviceregistration/consul/consul_service_registration_test.go index fcdd04179..21b2b2573 100644 --- a/serviceregistration/consul/consul_service_registration_test.go +++ b/serviceregistration/consul/consul_service_registration_test.go @@ -50,7 +50,7 @@ func testConsulServiceRegistrationConfig(t *testing.T, conf *consulConf) *servic // TestConsul_ServiceRegistration tests whether consul ServiceRegistration works func TestConsul_ServiceRegistration(t *testing.T) { // Prepare a docker-based consul instance - cleanup, config := consul.PrepareTestContainer(t, "", false) + cleanup, config := consul.PrepareTestContainer(t, "", false, true) defer cleanup() // Create a consul client diff --git a/website/content/api-docs/secret/consul.mdx b/website/content/api-docs/secret/consul.mdx index c65509d5e..eacc7e3f7 100644 --- a/website/content/api-docs/secret/consul.mdx +++ b/website/content/api-docs/secret/consul.mdx @@ -31,8 +31,9 @@ Consul tokens. - `scheme` `(string: "http")` – Specifies the URL scheme to use. -- `token` `(string: )` – Specifies the Consul ACL token to use. This - must be a management type token. +- `token` `(string: "")` – Specifies the Consul ACL token to use. This + must be a management type token. If this is not provided, Vault will try to + bootstrap the ACL system of the Consul cluster. - `ca_cert` `(string: "")` - CA certificate to use when verifying Consul server certificate, must be x509 PEM encoded. diff --git a/website/content/docs/secrets/consul.mdx b/website/content/docs/secrets/consul.mdx index 96a299144..738b7ea27 100644 --- a/website/content/docs/secrets/consul.mdx +++ b/website/content/docs/secrets/consul.mdx @@ -25,41 +25,52 @@ management tool. By default, the secrets engine will mount at the name of the engine. To enable the secrets engine at a different path, use the `-path` argument. -1. Bootstrap the Consul ACL system if not already done. To begin configuring the secrets engine, we must give Vault - the necessary credentials to manage Consul. +1. Vault can bootstrap the ACL system of your Consul cluster if it has + not already been done. In this case, you only need the address of your + Consul cluster to configure the Consul secret engine: - In Consul versions below 1.4, acquire a [management token][consul-mgmt-token] from Consul using the - `acl_master_token` from your Consul configuration file, or another management token: - - ```shell-session - $ curl \ - --header "X-Consul-Token: my-management-token" \ - --request POST \ - --data '{"Name": "sample", "Type": "management"}' \ - https://consul.rocks/v1/acl/create + ```text + $ vault write consul/config/access \ + address=127.0.0.1:8500 + Success! Data written to: consul/config/access ``` - Vault must have a "management" type token so that it can create and revoke ACL - tokens. The response will return a new token: + If you have already bootstrapped the ACL system of your Consul cluster, you + will need to give Vault a management token: - ```json - { - "ID": "7652ba4c-0f6e-8e75-5724-5e083d72cfe4" - } - ``` + - In Consul versions below 1.4, acquire a [management token][consul-mgmt-token] from Consul, using the + `acl_master_token` from your Consul configuration file or another management + token: - For Consul 1.4 and above, use the command line to generate a token with the appropriate policy: + ```sh + $ curl \ + --header "X-Consul-Token: my-management-token" \ + --request PUT \ + --data '{"Name": "sample", "Type": "management"}' \ + https://consul.rocks/v1/acl/create + ``` - ```shell-session - $ CONSUL_HTTP_TOKEN="" consul acl token create -policy-name="global-management" - AccessorID: 865dc5e9-e585-3180-7b49-4ddc0fc45135 - SecretID: ef35f0f1-885b-0cab-573c-7c91b65a7a7e - Description: - Local: false - Create Time: 2018-10-22 17:40:24.128188 -0700 PDT - Policies: - 00000000-0000-0000-0000-000000000001 - global-management - ``` + Vault must have a management type token so that it can create and revoke ACL + tokens. The response will return a new token: + + ```json + { + "ID": "7652ba4c-0f6e-8e75-5724-5e083d72cfe4" + } + ``` + + - For Consul 1.4 and above, use the command line to generate a token with the appropriate policy: + + ```shell-session + $ CONSUL_HTTP_TOKEN="" consul acl token create -policy-name="global-management" + AccessorID: 865dc5e9-e585-3180-7b49-4ddc0fc45135 + SecretID: ef35f0f1-885b-0cab-573c-7c91b65a7a7e + Description: + Local: false + Create Time: 2018-10-22 17:40:24.128188 -0700 PDT + Policies: + 00000000-0000-0000-0000-000000000001 - global-management + ``` 1. Configure Vault to connect and authenticate to Consul: