diff --git a/nomad/job_endpoint_hook_connect.go b/nomad/job_endpoint_hook_connect.go index 1ac2b6cb5..d429360df 100644 --- a/nomad/job_endpoint_hook_connect.go +++ b/nomad/job_endpoint_hook_connect.go @@ -263,11 +263,16 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error { // inject the gateway task only if it does not yet already exist if !hasGatewayTaskForService(g, service.Name) { - // use the default envoy image, for now there is no support for a custom task task := newConnectGatewayTask(service.Name, netHost) g.Tasks = append(g.Tasks, task) + // the connect.sidecar_task stanza can also be used to configure + // a custom task to use as a gateway proxy + if service.Connect.SidecarTask != nil { + service.Connect.SidecarTask.MergeIntoTask(task) + } + task.Canonicalize(job, g) } } diff --git a/nomad/job_endpoint_hook_connect_test.go b/nomad/job_endpoint_hook_connect_test.go index 894c1b1ff..09c12f8c1 100644 --- a/nomad/job_endpoint_hook_connect_test.go +++ b/nomad/job_endpoint_hook_connect_test.go @@ -92,7 +92,7 @@ func TestJobEndpointConnect_groupConnectHook(t *testing.T) { tgExp.Services[0].Name = "backend" tgExp.Services[1].Name = "admin" - // Expect sidecar tasks to be properly canonicalized + // Expect sidecar tasks to be in canonical form. tgExp.Tasks[0].Canonicalize(job, tgExp) tgExp.Tasks[1].Canonicalize(job, tgExp) tgExp.Networks[0].DynamicPorts = []structs.Port{{ @@ -146,6 +146,75 @@ func TestJobEndpointConnect_groupConnectHook_IngressGateway(t *testing.T) { require.Exactly(t, expTG, job.TaskGroups[0]) } +func TestJobEndpointConnect_groupConnectHook_IngressGateway_CustomTask(t *testing.T) { + t.Parallel() + + // Test that the connect gateway task is inserted if a gateway service exists + // and since this is a bridge network, will rewrite the default gateway proxy + // block with correct configuration. + job := mock.ConnectIngressGatewayJob("bridge", false) + + job.Meta = map[string]string{ + "gateway_name": "my-gateway", + } + + job.TaskGroups[0].Services[0].Name = "${NOMAD_META_gateway_name}" + job.TaskGroups[0].Services[0].Connect.SidecarTask = &structs.SidecarTask{ + Driver: "raw_exec", + User: "sidecars", + Config: map[string]interface{}{ + "command": "/bin/sidecar", + "args": []string{"a", "b"}, + }, + Resources: &structs.Resources{ + CPU: 400, + // Memory: inherit 128 + }, + KillSignal: "SIGHUP", + } + + expTG := job.TaskGroups[0].Copy() + expTG.Tasks = []*structs.Task{ + // inject merged gateway task + { + Name: "connect-ingress-my-gateway", + Kind: structs.NewTaskKind(structs.ConnectIngressPrefix, "my-gateway"), + Driver: "raw_exec", + User: "sidecars", + Config: map[string]interface{}{ + "command": "/bin/sidecar", + "args": []string{"a", "b"}, + }, + Resources: &structs.Resources{ + CPU: 400, + MemoryMB: 128, + }, + LogConfig: &structs.LogConfig{ + MaxFiles: 2, + MaxFileSizeMB: 2, + }, + ShutdownDelay: 5 * time.Second, + KillSignal: "SIGHUP", + Constraints: structs.Constraints{ + connectGatewayVersionConstraint(), + }, + }, + } + expTG.Services[0].Name = "my-gateway" + expTG.Tasks[0].Canonicalize(job, expTG) + expTG.Networks[0].Canonicalize() + + // rewrite the service gateway proxy configuration + expTG.Services[0].Connect.Gateway.Proxy = gatewayProxyForBridge(expTG.Services[0].Connect.Gateway) + + require.NoError(t, groupConnectHook(job, job.TaskGroups[0])) + require.Exactly(t, expTG, job.TaskGroups[0]) + + // Test that the hook is idempotent + require.NoError(t, groupConnectHook(job, job.TaskGroups[0])) + require.Exactly(t, expTG, job.TaskGroups[0]) +} + // TestJobEndpoint_ConnectInterpolation asserts that when a Connect sidecar // proxy task is being created for a group service with an interpolated name, // the service name is interpolated *before the task is created. @@ -330,7 +399,7 @@ func TestJobEndpointConnect_gatewayProxyIsDefault(t *testing.T) { t.Run("bind-addresses set", func(t *testing.T) { result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{ EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{ - "listener1": &structs.ConsulGatewayBindAddress{ + "listener1": { Address: "1.1.1.1", Port: 9000, }, @@ -362,7 +431,7 @@ func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) { }}, }) require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{ - "service1": &structs.ConsulGatewayBindAddress{ + "service1": { Address: "0.0.0.0", Port: 3000, }, @@ -388,15 +457,15 @@ func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) { }}, }) require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{ - "service1": &structs.ConsulGatewayBindAddress{ + "service1": { Address: "0.0.0.0", Port: 3000, }, - "service2": &structs.ConsulGatewayBindAddress{ + "service2": { Address: "0.0.0.0", Port: 3000, }, - "service3": &structs.ConsulGatewayBindAddress{ + "service3": { Address: "0.0.0.0", Port: 3001, }, diff --git a/website/pages/docs/job-specification/gateway.mdx b/website/pages/docs/job-specification/gateway.mdx index c109cc06d..8ada5aedb 100644 --- a/website/pages/docs/job-specification/gateway.mdx +++ b/website/pages/docs/job-specification/gateway.mdx @@ -199,12 +199,30 @@ make use of the envoy version interpolation, e.g. meta.connect.gateway_image = custom/envoy-${NOMAD_envoy_version}:latest ``` +### Custom gateway task + +The task created for the gateway can be configured manually using the +[`sidecar_task`][sidecar_task] stanza. + +``` +connect { + gateway { + # ... + } + + sidecar_task { + # ... + } +} +``` + [proxy]: /docs/job-specification/gateway#proxy-parameters [ingress]: /docs/job-specification/gateway#ingress-parameters [tls]: /docs/job-specification/gateway#tls-parameters [listener]: /docs/job-specification/gateway#listener-parameters [service]: /docs/job-specification/gateway#service-parameters [service-default]: https://www.consul.io/docs/agent/config-entries/service-defaults +[sidecar_task]: /docs/job-specification/sidecar_task [connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout [address]: /docs/job-specification/gateway#address-parameters [Advanced Configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration diff --git a/website/pages/docs/job-specification/sidecar_task.mdx b/website/pages/docs/job-specification/sidecar_task.mdx index 64c64ce0c..5b58ff895 100644 --- a/website/pages/docs/job-specification/sidecar_task.mdx +++ b/website/pages/docs/job-specification/sidecar_task.mdx @@ -12,7 +12,7 @@ description: |- The `sidecar_task` stanza allows configuring various options for the proxy -sidecar managed by Nomad for [Consul +sidecar or connect gateway managed by Nomad for the [Consul Connect](/docs/integrations/consul-connect) integration such as resource requirements, kill timeouts and more as defined below. It is valid only within the context of a [`connect`][connect] stanza. @@ -53,25 +53,30 @@ job "countdash" { } ``` -## Default Envoy proxy sidecar +## Default Envoy configuration -Nomad automatically includes a default Envoy proxy sidecar task whenever a -group service has a [`sidecar_service`][sidecar_service] stanza. +Nomad automatically launches and manages an Envoy task for use as a proxy sidecar +or connect gateway, when [`sidecar_service`][sidecar_service] or [`gateway`][gateway] +are configured. -The default sidecar task is equivalent to: +The default envoy task is equivalent to: ```hcl sidecar_task { name = "connect-proxy-" + # "connect-gateway-" when used as a gateway - lifecycle { + lifecycle { # absent when used as a gateway hook = "prestart" sidecar = true } driver = "docker" + config { image = "${meta.connect.sidecar_image}" + # "${meta.connect.gateway_image}" when used as a gateway + args = [ "-c", "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json", @@ -97,13 +102,16 @@ sidecar_task { } ``` -The `meta.connect.sidecar_image`, `meta.connect.log_level`, and -`meta.connect.proxy_concurrency` variables are [_client_ -configurable][nodemeta] variables with the following defaults: +The `meta.connect.sidecar_image`, `meta.connect.gateway_image`, `meta.connect.log_level`, +and `meta.connect.proxy_concurrency` variables are [client configurable][nodemeta] +variables with the following defaults: - `sidecar_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically by a query to Consul. +- `gateway_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official + upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically + by a query to Consul. - `log_level` - `(string: "info")` - Envoy sidecar log level. "`debug`" is useful for debugging Connect related issues. - `proxy_concurrency` - `(string: "1")` - The number of [worker threads][worker_threads] the Envoy @@ -118,8 +126,8 @@ meta.connect.sidecar_image = custom/envoy-${NOMAD_envoy_version}:latest ## `sidecar_task` Parameters -- `name` `(string: "connect-proxy-")` - Name of the task. Defaults to - including the name of the service it is a proxy for. +- `name` `(string: "connect-[proxy|gateway]-")` - Name of the task. Defaults to + including the name of the service the proxy or gateway is providing. - `driver` `(string: "docker")` - Driver used for the sidecar task. @@ -166,12 +174,13 @@ The following example configures resources for the sidecar task and other config ``` [connect]: /docs/job-specification/connect 'Nomad connect Job Specification' -[job]: /docs/job-specification/job 'Nomad job Job Specification' +[gateway]: /docs/job-specification/gateway [group]: /docs/job-specification/group 'Nomad group Job Specification' -[task]: /docs/job-specification/task 'Nomad task Job Specification' [interpolation]: /docs/runtime/interpolation 'Nomad interpolation' -[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification' -[resources]: /docs/job-specification/resources 'Nomad resources Job Specification' +[job]: /docs/job-specification/job 'Nomad job Job Specification' [logs]: /docs/job-specification/logs 'Nomad logs Job Specification' +[resources]: /docs/job-specification/resources 'Nomad resources Job Specification' +[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification' +[task]: /docs/job-specification/task 'Nomad task Job Specification' [nodemeta]: /docs/configuration/client#meta [worker_threads]: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-concurrency