From 719eee8112082aa38034bf51b977aab6b4dcf7be Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Wed, 11 Jan 2023 09:34:28 -0600 Subject: [PATCH] consul: add client configuration for grpc_ca_file (#15701) * [no ci] first pass at plumbing grpc_ca_file * consul: add support for grpc_ca_file for tls grpc connections in consul 1.14+ This PR adds client config to Nomad for specifying consul.grpc_ca_file These changes combined with https://github.com/hashicorp/consul/pull/15913 should finally enable Nomad users to upgrade to Consul 1.14+ and use tls grpc connections. * consul: add cl entgry for grpc_ca_file * docs: mention grpc_tls changes due to Consul 1.14 --- .changelog/15701.txt | 3 + client/allocrunner/consul_grpc_sock_hook.go | 1 - .../taskrunner/envoy_bootstrap_hook.go | 70 ++++++++----------- .../taskrunner/envoy_bootstrap_hook_test.go | 16 +++-- nomad/structs/config/consul.go | 8 +++ nomad/structs/config/consul_test.go | 3 + website/content/docs/configuration/consul.mdx | 4 ++ .../docs/integrations/consul-connect.mdx | 22 ++++++ 8 files changed, 80 insertions(+), 47 deletions(-) create mode 100644 .changelog/15701.txt diff --git a/.changelog/15701.txt b/.changelog/15701.txt new file mode 100644 index 000000000..bc358d2b1 --- /dev/null +++ b/.changelog/15701.txt @@ -0,0 +1,3 @@ +```release-note:improvement +consul: add client configuration for grpc_ca_file +``` diff --git a/client/allocrunner/consul_grpc_sock_hook.go b/client/allocrunner/consul_grpc_sock_hook.go index a9e7c2ff1..ba68c2ad1 100644 --- a/client/allocrunner/consul_grpc_sock_hook.go +++ b/client/allocrunner/consul_grpc_sock_hook.go @@ -195,7 +195,6 @@ func (p *grpcSocketProxy) run(alloc *structs.Allocation) error { return fmt.Errorf("error parsing Consul address %q: %v", p.config.Addr, err) } - destAddr = net.JoinHostPort(host, p.consulGRPCFallbackPort) } diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go index 0445d3a2e..0da4c3d0d 100644 --- a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go +++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go @@ -44,27 +44,29 @@ const ( ) type consulTransportConfig struct { - HTTPAddr string // required - Auth string // optional, env CONSUL_HTTP_AUTH - SSL string // optional, env CONSUL_HTTP_SSL - VerifySSL string // optional, env CONSUL_HTTP_SSL_VERIFY - CAFile string // optional, arg -ca-file - CertFile string // optional, arg -client-cert - KeyFile string // optional, arg -client-key - Namespace string // optional, only consul Enterprise, env CONSUL_NAMESPACE + HTTPAddr string // required + Auth string // optional, env CONSUL_HTTP_AUTH + SSL string // optional, env CONSUL_HTTP_SSL + VerifySSL string // optional, env CONSUL_HTTP_SSL_VERIFY + GRPCCAFile string // optional, arg -grpc-ca-file + CAFile string // optional, arg -ca-file + CertFile string // optional, arg -client-cert + KeyFile string // optional, arg -client-key + Namespace string // optional, only consul Enterprise, env CONSUL_NAMESPACE // CAPath (dir) not supported by Nomad's config object } -func newConsulTransportConfig(consul *config.ConsulConfig) consulTransportConfig { +func newConsulTransportConfig(cc *config.ConsulConfig) consulTransportConfig { return consulTransportConfig{ - HTTPAddr: consul.Addr, - Auth: consul.Auth, - SSL: decodeTriState(consul.EnableSSL), - VerifySSL: decodeTriState(consul.VerifySSL), - CAFile: consul.CAFile, - CertFile: consul.CertFile, - KeyFile: consul.KeyFile, - Namespace: consul.Namespace, + HTTPAddr: cc.Addr, + Auth: cc.Auth, + SSL: decodeTriState(cc.EnableSSL), + VerifySSL: decodeTriState(cc.VerifySSL), + GRPCCAFile: cc.GRPCCAFile, + CAFile: cc.CAFile, + CertFile: cc.CertFile, + KeyFile: cc.KeyFile, + Namespace: cc.Namespace, } } @@ -125,7 +127,7 @@ type envoyBootstrapHook struct { // envoyBootstrapWaitTime is the total amount of time hook will wait for Consul envoyBootstrapWaitTime time.Duration - // envoyBootstrapInitialGap is the initial wait gap when retyring + // envoyBootstrapInitialGap is the initial wait gap when retrying envoyBoostrapInitialGap time.Duration // envoyBootstrapMaxJitter is the maximum amount of jitter applied to retries @@ -542,29 +544,19 @@ func (e envoyBootstrapArgs) args() []string { "-bootstrap", } - if v := e.gateway; v != "" { - arguments = append(arguments, "-gateway", v) + appendIfSet := func(param, value string) { + if value != "" { + arguments = append(arguments, param, value) + } } - if v := e.siToken; v != "" { - arguments = append(arguments, "-token", v) - } - - if v := e.consulConfig.CAFile; v != "" { - arguments = append(arguments, "-ca-file", v) - } - - if v := e.consulConfig.CertFile; v != "" { - arguments = append(arguments, "-client-cert", v) - } - - if v := e.consulConfig.KeyFile; v != "" { - arguments = append(arguments, "-client-key", v) - } - - if v := e.namespace; v != "" { - arguments = append(arguments, "-namespace", v) - } + appendIfSet("-gateway", e.gateway) + appendIfSet("-token", e.siToken) + appendIfSet("-grpc-ca-file", e.consulConfig.GRPCCAFile) + appendIfSet("-ca-file", e.consulConfig.CAFile) + appendIfSet("-client-cert", e.consulConfig.CertFile) + appendIfSet("-client-key", e.consulConfig.KeyFile) + appendIfSet("-namespace", e.namespace) return arguments } diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go index 70ebe47b5..4fcfb623b 100644 --- a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go +++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go @@ -105,13 +105,14 @@ var ( } consulTLSConfig = consulTransportConfig{ - HTTPAddr: "2.2.2.2", // arg - Auth: "user:password", // env - SSL: "true", // env - VerifySSL: "true", // env - CAFile: "/etc/tls/ca-file", // arg - CertFile: "/etc/tls/cert-file", // arg - KeyFile: "/etc/tls/key-file", // arg + HTTPAddr: "2.2.2.2", // arg + Auth: "user:password", // env + SSL: "true", // env + VerifySSL: "true", // env + GRPCCAFile: "/etc/tls/grpc-ca-file", // arg + CAFile: "/etc/tls/ca-file", // arg + CertFile: "/etc/tls/cert-file", // arg + KeyFile: "/etc/tls/key-file", // arg } ) @@ -175,6 +176,7 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) { "-address", "127.0.0.1:19100", "-proxy-id", "s1-sidecar-proxy", "-bootstrap", + "-grpc-ca-file", "/etc/tls/grpc-ca-file", "-ca-file", "/etc/tls/ca-file", "-client-cert", "/etc/tls/cert-file", "-client-key", "/etc/tls/key-file", diff --git a/nomad/structs/config/consul.go b/nomad/structs/config/consul.go index bddd7947f..cddbdb54a 100644 --- a/nomad/structs/config/consul.go +++ b/nomad/structs/config/consul.go @@ -102,6 +102,11 @@ type ConsulConfig struct { // Uses Consul's default and env var. VerifySSL *bool `hcl:"verify_ssl"` + // GRPCCAFile is the path to the ca certificate used for Consul gRPC communication. + // + // Uses Consul's default and env var. + GRPCCAFile string `hcl:"grpc_ca_file"` + // CAFile is the path to the ca certificate used for Consul communication. // // Uses Consul's default and env var. @@ -219,6 +224,9 @@ func (c *ConsulConfig) Merge(b *ConsulConfig) *ConsulConfig { if b.ShareSSL != nil { result.ShareSSL = pointer.Of(*b.ShareSSL) } + if b.GRPCCAFile != "" { + result.GRPCCAFile = b.GRPCCAFile + } if b.CAFile != "" { result.CAFile = b.CAFile } diff --git a/nomad/structs/config/consul_test.go b/nomad/structs/config/consul_test.go index 62f2fcd86..806b999d5 100644 --- a/nomad/structs/config/consul_test.go +++ b/nomad/structs/config/consul_test.go @@ -54,6 +54,7 @@ func TestConsulConfig_Merge(t *testing.T) { Auth: "1", EnableSSL: &no, VerifySSL: &no, + GRPCCAFile: "1", CAFile: "1", CertFile: "1", KeyFile: "1", @@ -81,6 +82,7 @@ func TestConsulConfig_Merge(t *testing.T) { Auth: "2", EnableSSL: &yes, VerifySSL: &yes, + GRPCCAFile: "2", CAFile: "2", CertFile: "2", KeyFile: "2", @@ -108,6 +110,7 @@ func TestConsulConfig_Merge(t *testing.T) { Auth: "2", EnableSSL: &yes, VerifySSL: &yes, + GRPCCAFile: "2", CAFile: "2", CertFile: "2", KeyFile: "2", diff --git a/website/content/docs/configuration/consul.mdx b/website/content/docs/configuration/consul.mdx index 3c3fc3d5c..9e9e3a867 100644 --- a/website/content/docs/configuration/consul.mdx +++ b/website/content/docs/configuration/consul.mdx @@ -61,6 +61,10 @@ configuring Nomad to talk to Consul via DNS such as consul.service.consul respective services, each tagged appropriately with either `http` or `rpc` tag. Nomad servers also advertise a `serf` tagged service. +- `grpc_ca_file` `(string: "")` - Specifies an optional path to the GRPC CA + certificate used for communication between Connect sidecar proxies and Consul + agents. Will default to the `CONSUL_GRPC_CACERT` environment variable if set. + - `ca_file` `(string: "")` - Specifies an optional path to the CA certificate used for Consul communication. This defaults to the system bundle if unspecified. Will default to the `CONSUL_CACERT` environment variable if set. diff --git a/website/content/docs/integrations/consul-connect.mdx b/website/content/docs/integrations/consul-connect.mdx index ea8490d4a..a970868f0 100644 --- a/website/content/docs/integrations/consul-connect.mdx +++ b/website/content/docs/integrations/consul-connect.mdx @@ -96,6 +96,27 @@ For JSON configurations: } ``` +#### Consul TLS + +~> **Note:** Consul 1.14+ made a [backwards incompatible change][consul_grpc_tls] +in how TLS enabled grpc listeners work. When using Consul 1.14 with TLS enabled users +will need to specify additional Nomad agent configuration to work with Connect. The +`consul.grpc_ca_file` value must now be configured (introduced in Nomad 1.4.4), +and `consul.grpc_address` will most likely need to be set to use the new standard +`grpc_tls` port of `8503`. + +```hcl +consul { + grpc_ca_file = "/etc/tls/consul-agent-ca.pem" + grpc_address = "127.0.0.1:8503" + ca_file = "/etc/tls/consul-agent-ca.pem" + cert_file = "/etc/tls/dc1-client-consul-0.pem" + key_file = "/etc/tls/dc1-client-consul-0-key.pem" + ssl = true + address = "127.0.0.1:8501" +} +``` + #### Consul ACLs ~> **Note:** Starting in Nomad v1.3.0, Consul Service Identity ACL tokens automatically @@ -356,3 +377,4 @@ dashes (`-`) are converted to underscores (`_`) in environment variables so [`Local`]: https://developer.hashicorp.com/consul/docs/security/acl/acl-tokens#token-attributes [anon_token]: https://developer.hashicorp.com/consul/docs/security/acl/acl-tokens#special-purpose-tokens [consul_ports]: https://developer.hashicorp.com/consul/docs/agent/config/config-files#ports +[consul_grpc_tls]: https://developer.hashicorp.com/consul/docs/upgrading/upgrade-specific#changes-to-grpc-tls-configuration \ No newline at end of file