consul/connect: dynamically select envoy sidecar at runtime

As newer versions of Consul are released, the minimum version of Envoy
it supports as a sidecar proxy also gets bumped. Starting with the upcoming
Consul v1.9.X series, Envoy v1.11.X will no longer be supported. Current
versions of Nomad hardcode a version of Envoy v1.11.2 to be used as the
default implementation of Connect sidecar proxy.

This PR introduces a change such that each Nomad Client will query its
local Consul for a list of Envoy proxies that it supports (https://github.com/hashicorp/consul/pull/8545)
and then launch the Connect sidecar proxy task using the latest supported version
of Envoy. If the `SupportedProxies` API component is not available from
Consul, Nomad will fallback to the old version of Envoy supported by old
versions of Consul.

Setting the meta configuration option `meta.connect.sidecar_image` or
setting the `connect.sidecar_task` stanza will take precedence as is
the current behavior for sidecar proxies.

Setting the meta configuration option `meta.connect.gateway_image`
will take precedence as is the current behavior for connect gateways.

`meta.connect.sidecar_image` and `meta.connect.gateway_image` may make
use of the special `${NOMAD_envoy_version}` variable interpolation, which
resolves to the newest version of Envoy supported by the Consul agent.

Addresses #8585 #7665

client/allocrunner/alloc_runner.go

@@ -64,6 +64,10 @@ type allocRunner struct {
 	// registering services and checks
 	consulClient consul.ConsulServiceAPI
 
+	// consulProxiesClient is the client used by the envoy version hook for
+	// looking up supported envoy versions of the consul agent.
+	consulProxiesClient consul.SupportedProxiesAPI
+
 	// sidsClient is the client used by the service identity hook for
 	// managing SI tokens
 	sidsClient consul.ServiceIdentityAPI
@@ -186,6 +190,7 @@ func NewAllocRunner(config *Config) (*allocRunner, error) {
 		alloc:                    alloc,
 		clientConfig:             config.ClientConfig,
 		consulClient:             config.Consul,
+		consulProxiesClient:      config.ConsulProxies,
 		sidsClient:               config.ConsulSI,
 		vaultClient:              config.Vault,
 		tasks:                    make(map[string]*taskrunner.TaskRunner, len(tg.Tasks)),
@@ -236,7 +241,7 @@ func NewAllocRunner(config *Config) (*allocRunner, error) {
 // initTaskRunners creates task runners but does *not* run them.
 func (ar *allocRunner) initTaskRunners(tasks []*structs.Task) error {
 	for _, task := range tasks {
-		config := &taskrunner.Config{
+		trConfig := &taskrunner.Config{
 			Alloc:                ar.alloc,
 			ClientConfig:         ar.clientConfig,
 			Task:                 task,
@@ -246,6 +251,7 @@ func (ar *allocRunner) initTaskRunners(tasks []*structs.Task) error {
 			StateUpdater:         ar,
 			DynamicRegistry:      ar.dynamicRegistry,
 			Consul:               ar.consulClient,
+			ConsulProxies:        ar.consulProxiesClient,
 			ConsulSI:             ar.sidsClient,
 			Vault:                ar.vaultClient,
 			DeviceStatsReporter:  ar.deviceStatsReporter,
@@ -257,7 +263,7 @@ func (ar *allocRunner) initTaskRunners(tasks []*structs.Task) error {
 		}
 
 		// Create, but do not Run, the task runner
-		tr, err := taskrunner.NewTaskRunner(config)
+		tr, err := taskrunner.NewTaskRunner(trConfig)
 		if err != nil {
 			return fmt.Errorf("failed creating runner for task %q: %v", task.Name, err)
 		}

client/allocrunner/config.go

@@ -32,6 +32,10 @@ type Config struct {
 	// Consul is the Consul client used to register task services and checks
 	Consul consul.ConsulServiceAPI
 
+	// ConsulProxies is the Consul client used to lookup supported envoy versions
+	// of the Consul agent.
+	ConsulProxies consul.SupportedProxiesAPI
+
 	// ConsulSI is the Consul client used to manage service identity tokens.
 	ConsulSI consul.ServiceIdentityAPI
 

client/allocrunner/taskrunner/envoy_bootstrap_hook.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/nomad/client/allocdir"
-	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces"
 	agentconsul "github.com/hashicorp/nomad/command/agent/consul"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -152,7 +152,7 @@ func (h *envoyBootstrapHook) lookupService(svcKind, svcName, tgName string) (*st
 // Prestart creates an envoy bootstrap config file.
 //
 // Must be aware of both launching envoy as a sidecar proxy, as well as a connect gateway.
-func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
+func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestartRequest, resp *ifs.TaskPrestartResponse) error {
 	if !req.Task.Kind.IsConnectProxy() && !req.Task.Kind.IsAnyConnectGateway() {
 		// Not a Connect proxy sidecar
 		resp.Done = true

client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go

@@ -499,7 +499,7 @@ func TestTaskRunner_EnvoyBootstrapHook_gateway_ok(t *testing.T) {
 	// Run the hook
 	require.NoError(t, h.Prestart(context.Background(), req, &resp))
 
-	// Assert the hook is done
+	// Assert the hook is Done
 	require.True(t, resp.Done)
 	require.NotNil(t, resp.Env)
 

client/allocrunner/taskrunner/envoy_version_hook.go

@@ -0,0 +1,170 @@
+package taskrunner
+
+import (
+	"context"
+	"strings"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-version"
+	ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	"github.com/hashicorp/nomad/client/consul"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/pkg/errors"
+)
+
+const (
+	// envoyVersionHookName is the name of this hook and appears in logs.
+	envoyVersionHookName = "envoy_version"
+
+	// envoyLegacyImage is used when the version of Consul is too old to support
+	// the SupportedProxies field in the self API.
+	//
+	// This is the version defaulted by Nomad before v0.13.0 and/or when using versions
+	// of Consul before v1.7.8, v1.8.5, and v1.9.0.
+	envoyLegacyImage = "envoyproxy/envoy:v1.11.2@sha256:a7769160c9c1a55bb8d07a3b71ce5d64f72b1f665f10d81aa1581bc3cf850d09"
+)
+
+type envoyVersionHookConfig struct {
+	alloc         *structs.Allocation
+	proxiesClient consul.SupportedProxiesAPI
+	logger        hclog.Logger
+}
+
+func newEnvoyVersionHookConfig(alloc *structs.Allocation, proxiesClient consul.SupportedProxiesAPI, logger hclog.Logger) *envoyVersionHookConfig {
+	return &envoyVersionHookConfig{
+		alloc:         alloc,
+		logger:        logger,
+		proxiesClient: proxiesClient,
+	}
+}
+
+// envoyVersionHook is used to determine and set the Docker image used for Consul
+// Connect sidecar proxy tasks. It will query Consul for a set of preferred Envoy
+// versions if the task image is unset or references ${NOMAD_envoy_version}. Nomad
+// will fallback the image to the previous default Envoy v1.11.2 if Consul is too old
+// to support the supported proxies API.
+type envoyVersionHook struct {
+	// alloc is the allocation with the envoy task being rewritten.
+	alloc *structs.Allocation
+
+	// proxiesClient is the subset of the Consul API for getting information
+	// from Consul about the versions of Envoy it supports.
+	proxiesClient consul.SupportedProxiesAPI
+
+	// logger is used to log things.
+	logger hclog.Logger
+}
+
+func newEnvoyVersionHook(c *envoyVersionHookConfig) *envoyVersionHook {
+	return &envoyVersionHook{
+		alloc:         c.alloc,
+		proxiesClient: c.proxiesClient,
+		logger:        c.logger.Named(envoyVersionHookName),
+	}
+}
+
+func (envoyVersionHook) Name() string {
+	return envoyVersionHookName
+}
+
+func (h *envoyVersionHook) Prestart(_ context.Context, request *ifs.TaskPrestartRequest, response *ifs.TaskPrestartResponse) error {
+	if h.skip(request) {
+		response.Done = true
+		return nil
+	}
+
+	// We either need to acquire Consul's preferred Envoy version or fallback
+	// to the legacy default. Query Consul and use the (possibly empty) result.
+	proxies, err := h.proxiesClient.Proxies()
+	if err != nil {
+		return errors.Wrap(err, "error retrieving supported Envoy versions from Consul")
+	}
+
+	// Determine the concrete Envoy image identifier by applying version string
+	// substitution (${NOMAD_envoy_version}).
+	image, err := h.tweakImage(h.taskImage(request.Task.Config), proxies)
+	if err != nil {
+		return errors.Wrap(err, "error interpreting desired Envoy version from Consul")
+	}
+
+	// Set the resulting image.
+	h.logger.Trace("setting task envoy image", "image", image)
+	request.Task.Config["image"] = image
+	response.Done = true
+	return nil
+}
+
+// skip will return true if the request does not contain a task that should have
+// its envoy proxy version resolved automatically.
+func (h *envoyVersionHook) skip(request *ifs.TaskPrestartRequest) bool {
+	switch {
+	case request.Task.Driver != "docker":
+		return true
+	case !request.Task.UsesConnectSidecar():
+		return true
+	case !h.needsVersion(request.Task.Config):
+		return true
+	}
+	return false
+}
+
+// getConfiguredImage extracts the configured config.image value from the request.
+// If the image is empty or not a string, Nomad will fallback to the normal
+// official Envoy image as if the setting was not configured. This is also what
+// Nomad would do if the sidecar_task was not set in the first place.
+func (_ *envoyVersionHook) taskImage(config map[string]interface{}) string {
+	value, exists := config["image"]
+	if !exists {
+		return structs.EnvoyImageFormat
+	}
+
+	image, ok := value.(string)
+	if !ok {
+		return structs.EnvoyImageFormat
+	}
+
+	return image
+}
+
+// needsVersion returns true if the docker.config.image is making use of the
+// ${NOMAD_envoy_version} faux environment variable.
+// Nomad does not need to query Consul to get the preferred Envoy version, etc.)
+func (h *envoyVersionHook) needsVersion(config map[string]interface{}) bool {
+	if len(config) == 0 {
+		return false
+	}
+
+	image := h.taskImage(config)
+
+	return strings.Contains(image, structs.EnvoyVersionVar)
+}
+
+// image determines the best Envoy version to use. If supported is nil or empty
+// Nomad will fallback to the legacy envoy image used before Nomad v0.13.
+func (_ *envoyVersionHook) tweakImage(configured string, supported map[string][]string) (string, error) {
+	versions := supported["envoy"]
+	if len(versions) == 0 {
+		return envoyLegacyImage, nil
+	}
+
+	latest, err := semver(versions[0])
+	if err != nil {
+		return "", err
+	}
+
+	return strings.ReplaceAll(configured, structs.EnvoyVersionVar, latest), nil
+}
+
+// semver sanitizes the envoy version string coming from Consul into the format
+// used by the Envoy project when publishing images (i.e. proper semver). This
+// resulting string value does NOT contain the 'v' prefix for 2 reasons:
+// 1) the version library does not include the 'v'
+// 2) its plausible unofficial images use the 3 numbers without the prefix for
+//    tagging their own images
+func semver(chosen string) (string, error) {
+	v, err := version.NewVersion(chosen)
+	if err != nil {
+		return "", errors.Wrap(err, "unexpected envoy version format")
+	}
+	return v.String(), nil
+}

client/allocrunner/taskrunner/envoy_version_hook_test.go

@@ -0,0 +1,376 @@
+package taskrunner
+
+import (
+	"context"
+	"testing"
+
+	"github.com/hashicorp/nomad/client/allocdir"
+	ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	"github.com/hashicorp/nomad/client/taskenv"
+	"github.com/hashicorp/nomad/command/agent/consul"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEnvoyVersionHook_semver(t *testing.T) {
+	t.Parallel()
+
+	t.Run("with v", func(t *testing.T) {
+		result, err := semver("v1.2.3")
+		require.NoError(t, err)
+		require.Equal(t, "1.2.3", result)
+	})
+
+	t.Run("without v", func(t *testing.T) {
+		result, err := semver("1.2.3")
+		require.NoError(t, err)
+		require.Equal(t, "1.2.3", result)
+	})
+
+	t.Run("unexpected", func(t *testing.T) {
+		_, err := semver("foo")
+		require.EqualError(t, err, "unexpected envoy version format: Malformed version: foo")
+	})
+}
+
+func TestEnvoyVersionHook_taskImage(t *testing.T) {
+	t.Parallel()
+
+	t.Run("absent", func(t *testing.T) {
+		result := (*envoyVersionHook)(nil).taskImage(map[string]interface{}{
+			// empty
+		})
+		require.Equal(t, structs.EnvoyImageFormat, result)
+	})
+
+	t.Run("not a string", func(t *testing.T) {
+		result := (*envoyVersionHook)(nil).taskImage(map[string]interface{}{
+			"image": 7, // not a string
+		})
+		require.Equal(t, structs.EnvoyImageFormat, result)
+	})
+
+	t.Run("normal", func(t *testing.T) {
+		result := (*envoyVersionHook)(nil).taskImage(map[string]interface{}{
+			"image": "custom/envoy:latest",
+		})
+		require.Equal(t, "custom/envoy:latest", result)
+	})
+}
+
+func TestEnvoyVersionHook_tweakImage(t *testing.T) {
+	t.Parallel()
+	image := structs.EnvoyImageFormat
+
+	t.Run("legacy", func(t *testing.T) {
+		result, err := (*envoyVersionHook)(nil).tweakImage(image, nil)
+		require.NoError(t, err)
+		require.Equal(t, envoyLegacyImage, result)
+	})
+
+	t.Run("unexpected", func(t *testing.T) {
+		_, err := (*envoyVersionHook)(nil).tweakImage(image, map[string][]string{
+			"envoy": {"foo", "bar", "baz"},
+		})
+		require.EqualError(t, err, "unexpected envoy version format: Malformed version: foo")
+	})
+
+	t.Run("standard envoy", func(t *testing.T) {
+		result, err := (*envoyVersionHook)(nil).tweakImage(image, map[string][]string{
+			"envoy": {"1.15.0", "1.14.4", "1.13.4", "1.12.6"},
+		})
+		require.NoError(t, err)
+		require.Equal(t, "envoyproxy/envoy:v1.15.0", result)
+	})
+
+	t.Run("custom image", func(t *testing.T) {
+		custom := "custom-${NOMAD_envoy_version}/envoy:${NOMAD_envoy_version}"
+		result, err := (*envoyVersionHook)(nil).tweakImage(custom, map[string][]string{
+			"envoy": {"1.15.0", "1.14.4", "1.13.4", "1.12.6"},
+		})
+		require.NoError(t, err)
+		require.Equal(t, "custom-1.15.0/envoy:1.15.0", result)
+	})
+}
+
+func TestEnvoyVersionHook_skip(t *testing.T) {
+	t.Parallel()
+
+	h := new(envoyVersionHook)
+
+	t.Run("not docker", func(t *testing.T) {
+		skip := h.skip(&ifs.TaskPrestartRequest{
+			Task: &structs.Task{
+				Driver: "exec",
+				Config: nil,
+			},
+		})
+		require.True(t, skip)
+	})
+
+	t.Run("not connect", func(t *testing.T) {
+		skip := h.skip(&ifs.TaskPrestartRequest{
+			Task: &structs.Task{
+				Driver: "docker",
+				Kind:   "",
+			},
+		})
+		require.True(t, skip)
+	})
+
+	t.Run("version not needed", func(t *testing.T) {
+		skip := h.skip(&ifs.TaskPrestartRequest{
+			Task: &structs.Task{
+				Driver: "docker",
+				Kind:   structs.NewTaskKind(structs.ConnectProxyPrefix, "task"),
+				Config: map[string]interface{}{
+					"image": "custom/envoy:latest",
+				},
+			},
+		})
+		require.True(t, skip)
+	})
+
+	t.Run("version needed custom", func(t *testing.T) {
+		skip := h.skip(&ifs.TaskPrestartRequest{
+			Task: &structs.Task{
+				Driver: "docker",
+				Kind:   structs.NewTaskKind(structs.ConnectProxyPrefix, "task"),
+				Config: map[string]interface{}{
+					"image": "custom/envoy:v${NOMAD_envoy_version}",
+				},
+			},
+		})
+		require.False(t, skip)
+	})
+
+	t.Run("version needed standard", func(t *testing.T) {
+		skip := h.skip(&ifs.TaskPrestartRequest{
+			Task: &structs.Task{
+				Driver: "docker",
+				Kind:   structs.NewTaskKind(structs.ConnectProxyPrefix, "task"),
+				Config: map[string]interface{}{
+					"image": structs.EnvoyImageFormat,
+				},
+			},
+		})
+		require.False(t, skip)
+	})
+}
+
+func TestTaskRunner_EnvoyVersionHook_Prestart_standard(t *testing.T) {
+	t.Parallel()
+
+	logger := testlog.HCLogger(t)
+
+	// Setup an Allocation
+	alloc := mock.ConnectAlloc()
+	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
+	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
+	defer cleanupDir()
+
+	// Setup a mock for Consul API
+	spAPI := consul.MockSupportedProxiesAPI{
+		Value: map[string][]string{
+			"envoy": {"1.15.0", "1.14.4"},
+		},
+		Error: nil,
+	}
+
+	// Run envoy_version hook
+	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
+
+	// Create a prestart request
+	request := &ifs.TaskPrestartRequest{
+		Task:    alloc.Job.TaskGroups[0].Tasks[0],
+		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
+		TaskEnv: taskenv.NewEmptyTaskEnv(),
+	}
+	require.NoError(t, request.TaskDir.Build(false, nil))
+
+	// Prepare a response
+	var response ifs.TaskPrestartResponse
+
+	// Run the hook
+	require.NoError(t, h.Prestart(context.Background(), request, &response))
+
+	// Assert the hook is Done
+	require.True(t, response.Done)
+
+	// Assert the Task.Config[image] is concrete
+	require.Equal(t, "envoyproxy/envoy:v1.15.0", request.Task.Config["image"])
+}
+
+func TestTaskRunner_EnvoyVersionHook_Prestart_custom(t *testing.T) {
+	t.Parallel()
+
+	logger := testlog.HCLogger(t)
+
+	// Setup an Allocation
+	alloc := mock.ConnectAlloc()
+	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
+	alloc.Job.TaskGroups[0].Tasks[0].Config["image"] = "custom-${NOMAD_envoy_version}:latest"
+	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
+	defer cleanupDir()
+
+	// Setup a mock for Consul API
+	spAPI := consul.MockSupportedProxiesAPI{
+		Value: map[string][]string{
+			"envoy": {"1.14.1", "1.13.3"},
+		},
+		Error: nil,
+	}
+
+	// Run envoy_version hook
+	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
+
+	// Create a prestart request
+	request := &ifs.TaskPrestartRequest{
+		Task:    alloc.Job.TaskGroups[0].Tasks[0],
+		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
+		TaskEnv: taskenv.NewEmptyTaskEnv(),
+	}
+	require.NoError(t, request.TaskDir.Build(false, nil))
+
+	// Prepare a response
+	var response ifs.TaskPrestartResponse
+
+	// Run the hook
+	require.NoError(t, h.Prestart(context.Background(), request, &response))
+
+	// Assert the hook is Done
+	require.True(t, response.Done)
+
+	// Assert the Task.Config[image] is concrete
+	require.Equal(t, "custom-1.14.1:latest", request.Task.Config["image"])
+}
+
+func TestTaskRunner_EnvoyVersionHook_Prestart_skip(t *testing.T) {
+	t.Parallel()
+
+	logger := testlog.HCLogger(t)
+
+	// Setup an Allocation
+	alloc := mock.ConnectAlloc()
+	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
+	alloc.Job.TaskGroups[0].Tasks[0].Driver = "exec"
+	alloc.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{
+		"command": "/sidecar",
+	}
+	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
+	defer cleanupDir()
+
+	// Setup a mock for Consul API
+	spAPI := consul.MockSupportedProxiesAPI{
+		Value: map[string][]string{
+			"envoy": {"1.14.1", "1.13.3"},
+		},
+		Error: nil,
+	}
+
+	// Run envoy_version hook
+	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
+
+	// Create a prestart request
+	request := &ifs.TaskPrestartRequest{
+		Task:    alloc.Job.TaskGroups[0].Tasks[0],
+		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
+		TaskEnv: taskenv.NewEmptyTaskEnv(),
+	}
+	require.NoError(t, request.TaskDir.Build(false, nil))
+
+	// Prepare a response
+	var response ifs.TaskPrestartResponse
+
+	// Run the hook
+	require.NoError(t, h.Prestart(context.Background(), request, &response))
+
+	// Assert the hook is Done
+	require.True(t, response.Done)
+
+	// Assert the Task.Config[image] does not get set
+	require.Empty(t, request.Task.Config["image"])
+}
+
+func TestTaskRunner_EnvoyVersionHook_Prestart_fallback(t *testing.T) {
+	t.Parallel()
+
+	logger := testlog.HCLogger(t)
+
+	// Setup an Allocation
+	alloc := mock.ConnectAlloc()
+	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
+	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
+	defer cleanupDir()
+
+	// Setup a mock for Consul API
+	spAPI := consul.MockSupportedProxiesAPI{
+		Value: nil, // old consul, no .xDS.SupportedProxies
+		Error: nil,
+	}
+
+	// Run envoy_version hook
+	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
+
+	// Create a prestart request
+	request := &ifs.TaskPrestartRequest{
+		Task:    alloc.Job.TaskGroups[0].Tasks[0],
+		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
+		TaskEnv: taskenv.NewEmptyTaskEnv(),
+	}
+	require.NoError(t, request.TaskDir.Build(false, nil))
+
+	// Prepare a response
+	var response ifs.TaskPrestartResponse
+
+	// Run the hook
+	require.NoError(t, h.Prestart(context.Background(), request, &response))
+
+	// Assert the hook is Done
+	require.True(t, response.Done)
+
+	// Assert the Task.Config[image] is the fallback image
+	require.Equal(t, "envoyproxy/envoy:v1.11.2@sha256:a7769160c9c1a55bb8d07a3b71ce5d64f72b1f665f10d81aa1581bc3cf850d09", request.Task.Config["image"])
+}
+
+func TestTaskRunner_EnvoyVersionHook_Prestart_error(t *testing.T) {
+	t.Parallel()
+
+	logger := testlog.HCLogger(t)
+
+	// Setup an Allocation
+	alloc := mock.ConnectAlloc()
+	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
+	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
+	defer cleanupDir()
+
+	// Setup a mock for Consul API
+	spAPI := consul.MockSupportedProxiesAPI{
+		Value: nil,
+		Error: errors.New("some consul error"),
+	}
+
+	// Run envoy_version hook
+	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
+
+	// Create a prestart request
+	request := &ifs.TaskPrestartRequest{
+		Task:    alloc.Job.TaskGroups[0].Tasks[0],
+		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
+		TaskEnv: taskenv.NewEmptyTaskEnv(),
+	}
+	require.NoError(t, request.TaskDir.Build(false, nil))
+
+	// Prepare a response
+	var response ifs.TaskPrestartResponse
+
+	// Run the hook, error should be recoverable
+	err := h.Prestart(context.Background(), request, &response)
+	require.EqualError(t, err, "error retrieving supported Envoy versions from Consul: some consul error")
+
+	// Assert the hook is not Done
+	require.False(t, response.Done)
+}

client/allocrunner/taskrunner/task_runner.go

@@ -158,7 +158,12 @@ type TaskRunner struct {
 
 	// consulClient is the client used by the consul service hook for
 	// registering services and checks
-	consulClient consul.ConsulServiceAPI
+	consulServiceClient consul.ConsulServiceAPI
+
+	// consulProxiesClient is the client used by the envoy version hook for
+	// asking consul what version of envoy nomad should inject into the connect
+	// sidecar or gateway task.
+	consulProxiesClient consul.SupportedProxiesAPI
 
 	// sidsClient is the client used by the service identity hook for managing
 	// service identity tokens
@@ -234,6 +239,10 @@ type Config struct {
 	// Consul is the client to use for managing Consul service registrations
 	Consul consul.ConsulServiceAPI
 
+	// ConsulProxies is the client to use for looking up supported envoy versions
+	// from Consul.
+	ConsulProxies consul.SupportedProxiesAPI
+
 	// ConsulSI is the client to use for managing Consul SI tokens
 	ConsulSI consul.ServiceIdentityAPI
 
@@ -302,7 +311,8 @@ func NewTaskRunner(config *Config) (*TaskRunner, error) {
 		taskLeader:           config.Task.Leader,
 		envBuilder:           envBuilder,
 		dynamicRegistry:      config.DynamicRegistry,
-		consulClient:         config.Consul,
+		consulServiceClient:  config.Consul,
+		consulProxiesClient:  config.ConsulProxies,
 		siClient:             config.ConsulSI,
 		vaultClient:          config.Vault,
 		state:                tstate,

client/allocrunner/taskrunner/task_runner_hooks.go

@@ -106,7 +106,7 @@ func (tr *TaskRunner) initHooks() {
 		tr.runnerHooks = append(tr.runnerHooks, newServiceHook(serviceHookConfig{
 			alloc:     tr.Alloc(),
 			task:      tr.Task(),
-			consul:    tr.consulClient,
+			consul:    tr.consulServiceClient,
 			restarter: tr,
 			logger:    hookLogger,
 		}))
@@ -127,10 +127,11 @@ func (tr *TaskRunner) initHooks() {
 			}))
 		}
 
-		if task.Kind.IsConnectProxy() || task.Kind.IsAnyConnectGateway() {
-			tr.runnerHooks = append(tr.runnerHooks, newEnvoyBootstrapHook(
-				newEnvoyBootstrapHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger),
-			))
+		if task.UsesConnectSidecar() {
+			tr.runnerHooks = append(tr.runnerHooks,
+				newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, tr.consulProxiesClient, hookLogger)),
+				newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger)),
+			)
 		} else if task.Kind.IsConnectNative() {
 			tr.runnerHooks = append(tr.runnerHooks, newConnectNativeHook(
 				newConnectNativeHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger),
@@ -142,7 +143,7 @@ func (tr *TaskRunner) initHooks() {
 	scriptCheckHook := newScriptCheckHook(scriptCheckHookConfig{
 		alloc:  tr.Alloc(),
 		task:   tr.Task(),
-		consul: tr.consulClient,
+		consul: tr.consulServiceClient,
 		logger: hookLogger,
 	})
 	tr.runnerHooks = append(tr.runnerHooks, scriptCheckHook)

client/client.go

@@ -224,6 +224,10 @@ type Client struct {
 	// and checks.
 	consulService consulApi.ConsulServiceAPI
 
+	// consulProxies is Nomad's custom Consul client for looking up supported
+	// envoy versions
+	consulProxies consulApi.SupportedProxiesAPI
+
 	// consulCatalog is the subset of Consul's Catalog API Nomad uses.
 	consulCatalog consul.CatalogAPI
 
@@ -306,7 +310,7 @@ var (
 )
 
 // NewClient is used to create a new client from the given configuration
-func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService consulApi.ConsulServiceAPI) (*Client, error) {
+func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxies consulApi.SupportedProxiesAPI, consulService consulApi.ConsulServiceAPI) (*Client, error) {
 	// Create the tls wrapper
 	var tlsWrap tlsutil.RegionWrapper
 	if cfg.TLSConfig.EnableRPC {
@@ -331,6 +335,7 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic
 	c := &Client{
 		config:               cfg,
 		consulCatalog:        consulCatalog,
+		consulProxies:        consulProxies,
 		consulService:        consulService,
 		start:                time.Now(),
 		connPool:             pool.NewPool(logger, clientRPCCache, clientMaxStreams, tlsWrap),
@@ -2384,6 +2389,7 @@ func (c *Client) addAlloc(alloc *structs.Allocation, migrateToken string) error
 		ClientConfig:        c.configCopy,
 		StateDB:             c.stateDB,
 		Consul:              c.consulService,
+		ConsulProxies:       c.consulProxies,
 		ConsulSI:            c.tokensClient,
 		Vault:               c.vaultClient,
 		StateUpdater:        c,

client/client_test.go

@@ -622,7 +622,7 @@ func TestClient_SaveRestoreState(t *testing.T) {
 	c1.config.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", c1.config.Options, nil)
 	c1.config.PluginSingletonLoader = singleton.NewSingletonLoader(logger, c1.config.PluginLoader)
 
-	c2, err := NewClient(c1.config, consulCatalog, mockService)
+	c2, err := NewClient(c1.config, consulCatalog, nil, mockService)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}

client/config/config.go

@@ -13,7 +13,7 @@ import (
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
 	"github.com/hashicorp/nomad/nomad/structs"
-	"github.com/hashicorp/nomad/nomad/structs/config"
+	structsc "github.com/hashicorp/nomad/nomad/structs/config"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/version"
 )
@@ -149,10 +149,10 @@ type Config struct {
 	Version *version.VersionInfo
 
 	// ConsulConfig is this Agent's Consul configuration
-	ConsulConfig *config.ConsulConfig
+	ConsulConfig *structsc.ConsulConfig
 
 	// VaultConfig is this Agent's Vault configuration
-	VaultConfig *config.VaultConfig
+	VaultConfig *structsc.VaultConfig
 
 	// StatsCollectionInterval is the interval at which the Nomad client
 	// collects resource usage stats
@@ -167,7 +167,7 @@ type Config struct {
 	PublishAllocationMetrics bool
 
 	// TLSConfig holds various TLS related configurations
-	TLSConfig *config.TLSConfig
+	TLSConfig *structsc.TLSConfig
 
 	// GCInterval is the time interval at which the client triggers garbage
 	// collection
@@ -308,12 +308,12 @@ func (c *Config) Copy() *Config {
 func DefaultConfig() *Config {
 	return &Config{
 		Version:                 version.GetVersion(),
-		VaultConfig:             config.DefaultVaultConfig(),
-		ConsulConfig:            config.DefaultConsulConfig(),
+		VaultConfig:             structsc.DefaultVaultConfig(),
+		ConsulConfig:            structsc.DefaultConsulConfig(),
 		LogOutput:               os.Stderr,
 		Region:                  "global",
 		StatsCollectionInterval: 1 * time.Second,
-		TLSConfig:               &config.TLSConfig{},
+		TLSConfig:               &structsc.TLSConfig{},
 		LogLevel:                "DEBUG",
 		GCInterval:              1 * time.Minute,
 		GCParallelDestroys:      2,

client/consul/consul.go

@@ -42,3 +42,11 @@ type ServiceIdentityAPI interface {
 	// identity tokens be generated for tasks in the allocation.
 	DeriveSITokens(alloc *structs.Allocation, tasks []string) (map[string]string, error)
 }
+
+// SupportedProxiesAPI is the interface the Nomad Client uses to request from
+// Consul the set of supported proxied to use for Consul Connect.
+//
+// No ACL requirements
+type SupportedProxiesAPI interface {
+	Proxies() (map[string][]string, error)
+}

client/testing.go

@@ -5,9 +5,9 @@ import (
 	"time"
 
 	"github.com/hashicorp/nomad/client/config"
-	consulApi "github.com/hashicorp/nomad/client/consul"
+	consulapi "github.com/hashicorp/nomad/client/consul"
 	"github.com/hashicorp/nomad/client/fingerprint"
-	"github.com/hashicorp/nomad/command/agent/consul"
+	agentconsul "github.com/hashicorp/nomad/command/agent/consul"
 	"github.com/hashicorp/nomad/helper/pluginutils/catalog"
 	"github.com/hashicorp/nomad/helper/pluginutils/singleton"
 	"github.com/hashicorp/nomad/helper/testlog"
@@ -44,9 +44,9 @@ func TestClient(t testing.T, cb func(c *config.Config)) (*Client, func() error)
 	if conf.PluginSingletonLoader == nil {
 		conf.PluginSingletonLoader = singleton.NewSingletonLoader(logger, conf.PluginLoader)
 	}
-	catalog := consul.NewMockCatalog(logger)
-	mockService := consulApi.NewMockConsulServiceClient(t, logger)
-	client, err := NewClient(conf, catalog, mockService)
+	mockCatalog := agentconsul.NewMockCatalog(logger)
+	mockService := consulapi.NewMockConsulServiceClient(t, logger)
+	client, err := NewClient(conf, mockCatalog, nil, mockService)
 	if err != nil {
 		cleanup()
 		t.Fatalf("err: %v", err)

command/agent/agent.go

@@ -15,7 +15,7 @@ import (
 	"time"
 
 	metrics "github.com/armon/go-metrics"
-	"github.com/hashicorp/consul/api"
+	consulapi "github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/lib"
 	log "github.com/hashicorp/go-hclog"
 	uuidparse "github.com/hashicorp/go-uuid"
@@ -74,10 +74,13 @@ type Agent struct {
 	// and checks.
 	consulService *consul.ServiceClient
 
+	// consulProxies is the subset of Consul's Agent API Nomad uses.
+	consulProxies *consul.ConnectProxies
+
 	// consulCatalog is the subset of Consul's Catalog API Nomad uses.
 	consulCatalog consul.CatalogAPI
 
-	// consulConfigEntries is the subset of Consul's Configuration Entires API Nomad uses.
+	// consulConfigEntries is the subset of Consul's Configuration Entries API Nomad uses.
 	consulConfigEntries consul.ConfigAPI
 
 	// consulACLs is Nomad's subset of Consul's ACL API Nomad uses.
@@ -849,11 +852,11 @@ func (a *Agent) setupClient() error {
 		conf.StateDBFactory = state.GetStateDBFactory(conf.DevMode)
 	}
 
-	client, err := client.NewClient(conf, a.consulCatalog, a.consulService)
+	nomadClient, err := client.NewClient(conf, a.consulCatalog, a.consulProxies, a.consulService)
 	if err != nil {
 		return fmt.Errorf("client setup failed: %v", err)
 	}
-	a.client = client
+	a.client = nomadClient
 
 	// Create the Nomad Client  services for Consul
 	if *a.config.Consul.AutoAdvertise {
@@ -1123,26 +1126,30 @@ func (a *Agent) setupConsul(consulConfig *config.ConsulConfig) error {
 	if err != nil {
 		return err
 	}
-	client, err := api.NewClient(apiConf)
+
+	consulClient, err := consulapi.NewClient(apiConf)
 	if err != nil {
 		return err
 	}
 
 	// Create Consul Catalog client for service discovery.
-	a.consulCatalog = client.Catalog()
+	a.consulCatalog = consulClient.Catalog()
 
 	// Create Consul ConfigEntries client for managing Config Entries.
-	a.consulConfigEntries = client.ConfigEntries()
+	a.consulConfigEntries = consulClient.ConfigEntries()
 
 	// Create Consul ACL client for managing tokens.
-	a.consulACLs = client.ACL()
+	a.consulACLs = consulClient.ACL()
 
 	// Create Consul Service client for service advertisement and checks.
 	isClient := false
 	if a.config.Client != nil && a.config.Client.Enabled {
 		isClient = true
 	}
-	a.consulService = consul.NewServiceClient(client.Agent(), a.logger, isClient)
+	// Create Consul Agent client for looking info about the agent.
+	consulAgentClient := consulClient.Agent()
+	a.consulService = consul.NewServiceClient(consulAgentClient, a.logger, isClient)
+	a.consulProxies = consul.NewConnectProxiesClient(consulAgentClient)
 
 	// Run the Consul service client's sync'ing main loop
 	go a.consulService.Run()

command/agent/consul/catalog_testing.go

@@ -95,6 +95,16 @@ func (c *MockAgent) Self() (map[string]map[string]interface{}, error) {
 				"build": "0.8.1:'e9ca44d",
 			},
 		},
+		"xDS": {
+			"SupportedProxies": map[string]interface{}{
+				"envoy": []interface{}{
+					"1.14.2",
+					"1.13.2",
+					"1.12.4",
+					"1.11.2",
+				},
+			},
+		},
 	}
 	return s, nil
 }

command/agent/consul/connect_proxies.go

@@ -0,0 +1,92 @@
+package consul
+
+import (
+	"errors"
+)
+
+// ConnectProxies implements SupportedProxiesAPI by using the Consul Agent API.
+type ConnectProxies struct {
+	agentAPI AgentAPI
+}
+
+func NewConnectProxiesClient(agentAPI AgentAPI) *ConnectProxies {
+	return &ConnectProxies{
+		agentAPI: agentAPI,
+	}
+}
+
+// Proxies returns a map of the supported proxies. The proxies are sorted from
+// Consul with the most preferred version as the 0th element.
+//
+// If Consul is of a version that does not support the API, a nil map is returned
+// with no error.
+//
+// If Consul cannot be reached an error is returned.
+func (c *ConnectProxies) Proxies() (map[string][]string, error) {
+	// Based on the Consul query:
+	// $ curl -s localhost:8500/v1/agent/self | jq .xDS
+	// {
+	//  "SupportedProxies": {
+	//    "envoy": [
+	//      "1.15.0",
+	//      "1.14.4",
+	//      "1.13.4",
+	//      "1.12.6"
+	//    ]
+	//  }
+	// }
+
+	self, err := c.agentAPI.Self()
+	if err != nil {
+		// this should not fail as long as we can reach consul
+		return nil, err
+	}
+
+	// If consul does not return a map of the supported consul proxies, it
+	// must be a version from before when the API was added in versions
+	// 1.9.0, 1.8.3, 1.7.7. Earlier versions in the same point release as well
+	// as all of 1.6.X support Connect, but not the supported proxies API.
+	// For these cases, we can simply fallback to the old version of Envoy
+	// that Nomad defaulted to back then - but not in this logic. Instead,
+	// return nil so we can choose what to do at the caller.
+
+	xds, xdsExists := self["xDS"]
+	if !xdsExists {
+		return nil, nil
+	}
+
+	proxies, proxiesExists := xds["SupportedProxies"]
+	if !proxiesExists {
+		return nil, nil
+	}
+
+	// convert interface{} to map[string]interface{}
+
+	intermediate, ok := proxies.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("unexpected SupportedProxies response format from Consul")
+	}
+
+	// convert map[string]interface{} to map[string][]string
+
+	result := make(map[string][]string, len(intermediate))
+	for k, v := range intermediate {
+
+		// convert interface{} to []interface{}
+
+		if si, ok := v.([]interface{}); ok {
+			ss := make([]string, 0, len(si))
+			for _, z := range si {
+
+				// convert interface{} to string
+
+				if s, ok := z.(string); ok {
+					ss = append(ss, s)
+				}
+			}
+			result[k] = ss
+		}
+	}
+
+	return result, nil
+}

command/agent/consul/connect_proxies_test.go

@@ -0,0 +1,18 @@
+package consul
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestConnectProxies_Proxies(t *testing.T) {
+	agentAPI := NewMockAgent()
+	pc := NewConnectProxiesClient(agentAPI)
+
+	proxies, err := pc.Proxies()
+	require.NoError(t, err)
+	require.Equal(t, map[string][]string{
+		"envoy": []string{"1.14.2", "1.13.2", "1.12.4", "1.11.2"},
+	}, proxies)
+}

command/agent/consul/connect_proxies_testing.go

@@ -0,0 +1,11 @@
+package consul
+
+// ConnectProxies implements SupportedProxiesAPI by mocking the Consul Agent API.
+type MockSupportedProxiesAPI struct {
+	Value map[string][]string
+	Error error
+}
+
+func (m MockSupportedProxiesAPI) Proxies() (map[string][]string, error) {
+	return m.Value, m.Error
+}

nomad/job_endpoint_hook_connect.go

@@ -24,7 +24,7 @@ var (
 	// connect proxy sidecar task.
 	connectSidecarDriverConfig = func() map[string]interface{} {
 		return map[string]interface{}{
-			"image": "${meta.connect.sidecar_image}",
+			"image": structs.EnvoyImageFormat,
 			"args": []interface{}{
 				"-c", structs.EnvoyBootstrapPath,
 				"-l", "${meta.connect.log_level}",
@@ -40,7 +40,7 @@ var (
 	// networking is being used the network_mode driver configuration is set here.
 	connectGatewayDriverConfig = func(hostNetwork bool) map[string]interface{} {
 		m := map[string]interface{}{
-			"image": "${meta.connect.gateway_image}",
+			"image": structs.EnvoyImageFormat,
 			"args": []interface{}{
 				"-c", structs.EnvoyBootstrapPath,
 				"-l", "${meta.connect.log_level}",

nomad/mock/mock.go

@@ -721,6 +721,23 @@ func ConnectIngressGatewayJob(mode string, inject bool) *structs.Job {
 	return job
 }
 
+func ConnectSidecarTask() *structs.Task {
+	return &structs.Task{
+		Name:   "mysidecar-sidecar-task",
+		Driver: "docker",
+		User:   "nobody",
+		Config: map[string]interface{}{
+			"image": structs.EnvoyImageFormat,
+		},
+		Env: nil,
+		Resources: &structs.Resources{
+			CPU:      150,
+			MemoryMB: 350,
+		},
+		Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "mysidecar"),
+	}
+}
+
 func BatchJob() *structs.Job {
 	job := &structs.Job{
 		Region:      "global",

nomad/structs/config/consul.go

@@ -17,6 +17,9 @@ import (
 // - Bootstrap this Nomad Client with the list of Nomad Servers registered
 //   with Consul
 //
+// - Establish how this Nomad Client will resolve Envoy Connect Sidecar
+//   images.
+//
 // Both the Agent and the executor need to be able to import ConsulConfig.
 type ConsulConfig struct {
 	// ServerServiceName is the name of the service that Nomad uses to register

nomad/structs/connect.go

@@ -0,0 +1,17 @@
+package structs
+
+const (
+	// envoyImageFormat is the default format string used for official envoy Docker
+	// images with the tag being the semver of the version of envoy. Nomad fakes
+	// interpolation of ${NOMAD_envoy_version} by replacing it with the version
+	// string for envoy that Consul reports as preferred.
+	//
+	// Folks wanting to build and use custom images while still having Nomad refer
+	// to specific versions as preferred by Consul would set meta.connect.sidecar_image
+	// to something like: "custom/envoy:${NOMAD_envoy_version}".
+	EnvoyImageFormat = "envoyproxy/envoy:v" + EnvoyVersionVar
+
+	// envoyVersionVar will be replaced with the Envoy version string when
+	// used in the meta.connect.sidecar_image variable.
+	EnvoyVersionVar = "${NOMAD_envoy_version}"
+)

nomad/structs/structs.go

@@ -6385,7 +6385,11 @@ type Task struct {
 // Task, which exports known types of Tasks. UsesConnect will be true if the
 // task is a connect proxy, connect native, or is a connect gateway.
 func (t *Task) UsesConnect() bool {
-	return t.Kind.IsConnectProxy() || t.Kind.IsConnectNative() || t.Kind.IsAnyConnectGateway()
+	return t.Kind.IsConnectNative() || t.UsesConnectSidecar()
+}
+
+func (t *Task) UsesConnectSidecar() bool {
+	return t.Kind.IsConnectProxy() || t.Kind.IsAnyConnectGateway()
 }
 
 func (t *Task) Copy() *Task {

website/pages/docs/job-specification/gateway.mdx

@@ -187,6 +187,18 @@ accessible to any workload running on the same Nomad client. The admin interface
 information about the proxy, including a Consul Service Identity token if Consul ACLs
 are enabled.
 
+### Specify Envoy image
+
+The Docker image used for Connect gateway tasks defaults to the official [Envoy
+Docker] image, `envoyproxy/envoy:v${NOMAD_envoy_version}`, where `${NOMAD_envoy_version}`
+is resolved automatically by a query to Consul. The image to use can be configured
+by setting `meta.connect.gateway_image` in the Nomad job. Custom images can still
+make use of the envoy version interpolation, e.g.
+
+```hcl
+meta.connect.gateway_image = custom/envoy-${NOMAD_envoy_version}:latest
+```
+
 [proxy]: /docs/job-specification/gateway#proxy-parameters
 [ingress]: /docs/job-specification/gateway#ingress-parameters
 [tls]: /docs/job-specification/gateway#tls-parameters
@@ -195,4 +207,6 @@ are enabled.
 [service-default]: https://www.consul.io/docs/agent/config-entries/service-defaults
 [connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout
 [address]: /docs/job-specification/gateway#address-parameters
-[advanced configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration
+[Advanced Configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration
+[Envoy Docker]: https://hub.docker.com/r/envoyproxy/envoy/tags
+

website/pages/docs/job-specification/sidecar_task.mdx

@@ -97,10 +97,20 @@ The default sidecar task is equivalent to:
 The `meta.connect.sidecar_image` and `meta.connect.log_level` are [_client_
 configurable][nodemeta] variables with the following defaults:
 
-- `sidecar_image` - `"envoyproxy/envoy:v1.11.2@sha256:a7769160c9c1a55bb8d07a3b71ce5d64f72b1f665f10d81aa1581bc3cf850d09"` - The official upstream Envoy Docker image.
-- `log_level` - `"info"` - Envoy sidecar log level. "`debug`" is useful for
+- `sidecar_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official
+   upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically
+   by a query to Consul.
+- `log_level` - `(string: "info")` - Envoy sidecar log level. "`debug`" is useful for
   debugging Connect related issues.
 
+`meta.connect.sidecar_image` can be configured at the job, group, or task level.
+Custom images can make use of Consul's preferred Envoy version by making use of
+Nomad's version interpolation, e.g.
+
+```hcl
+meta.connect.sidecar_image = custom/envoy-${NOMAD_envoy_version}:latest
+```
+
 ## `sidecar_task` Parameters
 
 - `name` `(string: "connect-proxy-<service>")` - Name of the task. Defaults to

website/pages/docs/upgrade/upgrade-specific.mdx

@@ -60,6 +60,30 @@ Nomad. The specific configuration values replaced are:
 * Client `template.function_blacklist` is replaced with `template.function_denylist`.
 * Docker driver `docker.caps.whitelist` is replaced with `docker.caps.allowlist`.
 
+### Envoy proxy versions
+
+Nomad 0.13.0 changes the behavior around the selection of Envoy version used
+for Connect sidecar proxies. Previously, Nomad always defaulted to Envoy v1.11.2
+if neither the `meta.connect.sidecar_image` parameter or `sidecar_task` stanza
+were explicitly configured. Likewise the same version of Envoy would be used for
+Connect ingress gateways if `meta.connect.gateway_image` was unset. Starting with
+Nomad 0.13.0, each Nomad Client will query Consul for a list of supported Envoy
+versions. Nomad will make use of the latest version of Envoy supported by the
+Consul agent when launching Envoy as a Connect sidecar proxy. If the version of
+the Consul agent is older than v1.7.8, v1.8.4, or v1.9.0, Nomad will fallback to
+the v1.11.2 version of Envoy. As before, if the `meta.connect.sidecar_image`,
+ `meta.connect.gateway_image`, or `sidecar_task` stanza are set, those settings
+take precedence.
+
+When upgrading Nomad Clients from a previous version to v0.13.0 and above, it is
+recommended to also upgrade the Consul agents to v1.7.8, 1.8.4, or v1.9.0 or newer.
+Upgrading Nomad and Consul to versions that support the new behaviour while also doing a
+full [node drain](https://www.nomadproject.io/docs/upgrade#5-upgrade-clients) at
+the time of the upgrade for each node will ensure Connect workloads are properly
+rescheduled onto nodes in such a way that the Nomad Clients, Consul agents, and
+Envoy sidecar tasks maintain compatibility with one another.
+
+
 ## Nomad 0.12.0
 
 ### `mbits` and Task Network Resource deprecation