luxolus/open-nomad
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

Merge pull request #8945 from hashicorp/f-auto-sidecar

Browse source 
consul/connect: dynamically select envoy sidecar at runtime
This commit is contained in:
Seth Hoenig 2020-10-13 09:41:16 -05:00 committed by GitHub
parent 98a70d789e ed13e5723f
commit cb1999b9df
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 850 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
client/allocrunner/alloc_runner.go
View file

@ -64,6 +64,10 @@ type allocRunner struct {
// registering services and checks
consulClient consul.ConsulServiceAPI
// consulProxiesClient is the client used by the envoy version hook for
// looking up supported envoy versions of the consul agent.
consulProxiesClient consul.SupportedProxiesAPI
// sidsClient is the client used by the service identity hook for
// managing SI tokens
sidsClient consul.ServiceIdentityAPI
@ -186,6 +190,7 @@ func NewAllocRunner(config *Config) (*allocRunner, error) {
alloc: alloc,
clientConfig: config.ClientConfig,
consulClient: config.Consul,
consulProxiesClient: config.ConsulProxies,
sidsClient: config.ConsulSI,
vaultClient: config.Vault,
tasks: make(map[string]*taskrunner.TaskRunner, len(tg.Tasks)),
@ -236,7 +241,7 @@ func NewAllocRunner(config *Config) (*allocRunner, error) {
// initTaskRunners creates task runners but does *not* run them.
func (ar *allocRunner) initTaskRunners(tasks []*structs.Task) error {
for _, task := range tasks {
config := &taskrunner.Config{
trConfig := &taskrunner.Config{
Alloc: ar.alloc,
ClientConfig: ar.clientConfig,
Task: task,
@ -246,6 +251,7 @@ func (ar *allocRunner) initTaskRunners(tasks []*structs.Task) error {
StateUpdater: ar,
DynamicRegistry: ar.dynamicRegistry,
Consul: ar.consulClient,
ConsulProxies: ar.consulProxiesClient,
ConsulSI: ar.sidsClient,
Vault: ar.vaultClient,
DeviceStatsReporter: ar.deviceStatsReporter,
@ -257,7 +263,7 @@ func (ar *allocRunner) initTaskRunners(tasks []*structs.Task) error {
}
// Create, but do not Run, the task runner
tr, err := taskrunner.NewTaskRunner(config)
tr, err := taskrunner.NewTaskRunner(trConfig)
if err != nil {
return fmt.Errorf("failed creating runner for task %q: %v", task.Name, err)
}

4
client/allocrunner/config.go
View file

@ -32,6 +32,10 @@ type Config struct {
// Consul is the Consul client used to register task services and checks
Consul consul.ConsulServiceAPI
// ConsulProxies is the Consul client used to lookup supported envoy versions
// of the Consul agent.
ConsulProxies consul.SupportedProxiesAPI
// ConsulSI is the Consul client used to manage service identity tokens.
ConsulSI consul.ServiceIdentityAPI

4
client/allocrunner/taskrunner/envoybootstrap_hook.go → client/allocrunner/taskrunner/envoy_bootstrap_hook.go
View file

@ -12,7 +12,7 @@ import (
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/allocrunner/interfaces"
ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces"
agentconsul "github.com/hashicorp/nomad/command/agent/consul"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/structs"
@ -152,7 +152,7 @@ func (h *envoyBootstrapHook) lookupService(svcKind, svcName, tgName string) (*st
// Prestart creates an envoy bootstrap config file.
//
// Must be aware of both launching envoy as a sidecar proxy, as well as a connect gateway.
func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestartRequest, resp *ifs.TaskPrestartResponse) error {
if !req.Task.Kind.IsConnectProxy() && !req.Task.Kind.IsAnyConnectGateway() {
// Not a Connect proxy sidecar
resp.Done = true

2
client/allocrunner/taskrunner/envoybootstrap_hook_test.go → client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go
View file

@ -499,7 +499,7 @@ func TestTaskRunner_EnvoyBootstrapHook_gateway_ok(t *testing.T) {
// Run the hook
require.NoError(t, h.Prestart(context.Background(), req, &resp))
// Assert the hook is done
// Assert the hook is Done
require.True(t, resp.Done)
require.NotNil(t, resp.Env)

170
client/allocrunner/taskrunner/envoy_version_hook.go Normal file
View file

@ -0,0 +1,170 @@
package taskrunner
import (
"context"
"strings"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-version"
ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces"
"github.com/hashicorp/nomad/client/consul"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/pkg/errors"
)
const (
// envoyVersionHookName is the name of this hook and appears in logs.
envoyVersionHookName = "envoy_version"
// envoyLegacyImage is used when the version of Consul is too old to support
// the SupportedProxies field in the self API.
//
// This is the version defaulted by Nomad before v0.13.0 and/or when using versions
// of Consul before v1.7.8, v1.8.5, and v1.9.0.
envoyLegacyImage = "envoyproxy/envoy:v1.11.2@sha256:a7769160c9c1a55bb8d07a3b71ce5d64f72b1f665f10d81aa1581bc3cf850d09"
)
type envoyVersionHookConfig struct {
alloc *structs.Allocation
proxiesClient consul.SupportedProxiesAPI
logger hclog.Logger
}
func newEnvoyVersionHookConfig(alloc *structs.Allocation, proxiesClient consul.SupportedProxiesAPI, logger hclog.Logger) *envoyVersionHookConfig {
return &envoyVersionHookConfig{
alloc: alloc,
logger: logger,
proxiesClient: proxiesClient,
}
}
// envoyVersionHook is used to determine and set the Docker image used for Consul
// Connect sidecar proxy tasks. It will query Consul for a set of preferred Envoy
// versions if the task image is unset or references ${NOMAD_envoy_version}. Nomad
// will fallback the image to the previous default Envoy v1.11.2 if Consul is too old
// to support the supported proxies API.
type envoyVersionHook struct {
// alloc is the allocation with the envoy task being rewritten.
alloc *structs.Allocation
// proxiesClient is the subset of the Consul API for getting information
// from Consul about the versions of Envoy it supports.
proxiesClient consul.SupportedProxiesAPI
// logger is used to log things.
logger hclog.Logger
}
func newEnvoyVersionHook(c *envoyVersionHookConfig) *envoyVersionHook {
return &envoyVersionHook{
alloc: c.alloc,
proxiesClient: c.proxiesClient,
logger: c.logger.Named(envoyVersionHookName),
}
}
func (envoyVersionHook) Name() string {
return envoyVersionHookName
}
func (h *envoyVersionHook) Prestart(_ context.Context, request *ifs.TaskPrestartRequest, response *ifs.TaskPrestartResponse) error {
if h.skip(request) {
response.Done = true
return nil
}
// We either need to acquire Consul's preferred Envoy version or fallback
// to the legacy default. Query Consul and use the (possibly empty) result.
proxies, err := h.proxiesClient.Proxies()
if err != nil {
return errors.Wrap(err, "error retrieving supported Envoy versions from Consul")
}
// Determine the concrete Envoy image identifier by applying version string
// substitution (${NOMAD_envoy_version}).
image, err := h.tweakImage(h.taskImage(request.Task.Config), proxies)
if err != nil {
return errors.Wrap(err, "error interpreting desired Envoy version from Consul")
}
// Set the resulting image.
h.logger.Trace("setting task envoy image", "image", image)
request.Task.Config["image"] = image
response.Done = true
return nil
}
// skip will return true if the request does not contain a task that should have
// its envoy proxy version resolved automatically.
func (h *envoyVersionHook) skip(request *ifs.TaskPrestartRequest) bool {
switch {
case request.Task.Driver != "docker":
return true
case !request.Task.UsesConnectSidecar():
return true
case !h.needsVersion(request.Task.Config):
return true
}
return false
}
// getConfiguredImage extracts the configured config.image value from the request.
// If the image is empty or not a string, Nomad will fallback to the normal
// official Envoy image as if the setting was not configured. This is also what
// Nomad would do if the sidecar_task was not set in the first place.
func (_ *envoyVersionHook) taskImage(config map[string]interface{}) string {
value, exists := config["image"]
if !exists {
return structs.EnvoyImageFormat
}
image, ok := value.(string)
if !ok {
return structs.EnvoyImageFormat
}
return image
}
// needsVersion returns true if the docker.config.image is making use of the
// ${NOMAD_envoy_version} faux environment variable.
// Nomad does not need to query Consul to get the preferred Envoy version, etc.)
func (h *envoyVersionHook) needsVersion(config map[string]interface{}) bool {
if len(config) == 0 {
return false
}
image := h.taskImage(config)
return strings.Contains(image, structs.EnvoyVersionVar)
}
// image determines the best Envoy version to use. If supported is nil or empty
// Nomad will fallback to the legacy envoy image used before Nomad v0.13.
func (_ *envoyVersionHook) tweakImage(configured string, supported map[string][]string) (string, error) {
versions := supported["envoy"]
if len(versions) == 0 {
return envoyLegacyImage, nil
}
latest, err := semver(versions[0])
if err != nil {
return "", err
}
return strings.ReplaceAll(configured, structs.EnvoyVersionVar, latest), nil
}
// semver sanitizes the envoy version string coming from Consul into the format
// used by the Envoy project when publishing images (i.e. proper semver). This
// resulting string value does NOT contain the 'v' prefix for 2 reasons:
// 1) the version library does not include the 'v'
// 2) its plausible unofficial images use the 3 numbers without the prefix for
// tagging their own images
func semver(chosen string) (string, error) {
v, err := version.NewVersion(chosen)
if err != nil {
return "", errors.Wrap(err, "unexpected envoy version format")
}
return v.String(), nil
}

376
client/allocrunner/taskrunner/envoy_version_hook_test.go Normal file
View file

@ -0,0 +1,376 @@
package taskrunner
import (
"context"
"testing"
"github.com/hashicorp/nomad/client/allocdir"
ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/command/agent/consul"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
)
func TestEnvoyVersionHook_semver(t *testing.T) {
t.Parallel()
t.Run("with v", func(t *testing.T) {
result, err := semver("v1.2.3")
require.NoError(t, err)
require.Equal(t, "1.2.3", result)
})
t.Run("without v", func(t *testing.T) {
result, err := semver("1.2.3")
require.NoError(t, err)
require.Equal(t, "1.2.3", result)
})
t.Run("unexpected", func(t *testing.T) {
_, err := semver("foo")
require.EqualError(t, err, "unexpected envoy version format: Malformed version: foo")
})
}
func TestEnvoyVersionHook_taskImage(t *testing.T) {
t.Parallel()
t.Run("absent", func(t *testing.T) {
result := (*envoyVersionHook)(nil).taskImage(map[string]interface{}{
// empty
})
require.Equal(t, structs.EnvoyImageFormat, result)
})
t.Run("not a string", func(t *testing.T) {
result := (*envoyVersionHook)(nil).taskImage(map[string]interface{}{
"image": 7, // not a string
})
require.Equal(t, structs.EnvoyImageFormat, result)
})
t.Run("normal", func(t *testing.T) {
result := (*envoyVersionHook)(nil).taskImage(map[string]interface{}{
"image": "custom/envoy:latest",
})
require.Equal(t, "custom/envoy:latest", result)
})
}
func TestEnvoyVersionHook_tweakImage(t *testing.T) {
t.Parallel()
image := structs.EnvoyImageFormat
t.Run("legacy", func(t *testing.T) {
result, err := (*envoyVersionHook)(nil).tweakImage(image, nil)
require.NoError(t, err)
require.Equal(t, envoyLegacyImage, result)
})
t.Run("unexpected", func(t *testing.T) {
_, err := (*envoyVersionHook)(nil).tweakImage(image, map[string][]string{
"envoy": {"foo", "bar", "baz"},
})
require.EqualError(t, err, "unexpected envoy version format: Malformed version: foo")
})
t.Run("standard envoy", func(t *testing.T) {
result, err := (*envoyVersionHook)(nil).tweakImage(image, map[string][]string{
"envoy": {"1.15.0", "1.14.4", "1.13.4", "1.12.6"},
})
require.NoError(t, err)
require.Equal(t, "envoyproxy/envoy:v1.15.0", result)
})
t.Run("custom image", func(t *testing.T) {
custom := "custom-${NOMAD_envoy_version}/envoy:${NOMAD_envoy_version}"
result, err := (*envoyVersionHook)(nil).tweakImage(custom, map[string][]string{
"envoy": {"1.15.0", "1.14.4", "1.13.4", "1.12.6"},
})
require.NoError(t, err)
require.Equal(t, "custom-1.15.0/envoy:1.15.0", result)
})
}
func TestEnvoyVersionHook_skip(t *testing.T) {
t.Parallel()
h := new(envoyVersionHook)
t.Run("not docker", func(t *testing.T) {
skip := h.skip(&ifs.TaskPrestartRequest{
Task: &structs.Task{
Driver: "exec",
Config: nil,
},
})
require.True(t, skip)
})
t.Run("not connect", func(t *testing.T) {
skip := h.skip(&ifs.TaskPrestartRequest{
Task: &structs.Task{
Driver: "docker",
Kind: "",
},
})
require.True(t, skip)
})
t.Run("version not needed", func(t *testing.T) {
skip := h.skip(&ifs.TaskPrestartRequest{
Task: &structs.Task{
Driver: "docker",
Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "task"),
Config: map[string]interface{}{
"image": "custom/envoy:latest",
},
},
})
require.True(t, skip)
})
t.Run("version needed custom", func(t *testing.T) {
skip := h.skip(&ifs.TaskPrestartRequest{
Task: &structs.Task{
Driver: "docker",
Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "task"),
Config: map[string]interface{}{
"image": "custom/envoy:v${NOMAD_envoy_version}",
},
},
})
require.False(t, skip)
})
t.Run("version needed standard", func(t *testing.T) {
skip := h.skip(&ifs.TaskPrestartRequest{
Task: &structs.Task{
Driver: "docker",
Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "task"),
Config: map[string]interface{}{
"image": structs.EnvoyImageFormat,
},
},
})
require.False(t, skip)
})
}
func TestTaskRunner_EnvoyVersionHook_Prestart_standard(t *testing.T) {
t.Parallel()
logger := testlog.HCLogger(t)
// Setup an Allocation
alloc := mock.ConnectAlloc()
alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
defer cleanupDir()
// Setup a mock for Consul API
spAPI := consul.MockSupportedProxiesAPI{
Value: map[string][]string{
"envoy": {"1.15.0", "1.14.4"},
},
Error: nil,
}
// Run envoy_version hook
h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
// Create a prestart request
request := &ifs.TaskPrestartRequest{
Task: alloc.Job.TaskGroups[0].Tasks[0],
TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, request.TaskDir.Build(false, nil))
// Prepare a response
var response ifs.TaskPrestartResponse
// Run the hook
require.NoError(t, h.Prestart(context.Background(), request, &response))
// Assert the hook is Done
require.True(t, response.Done)
// Assert the Task.Config[image] is concrete
require.Equal(t, "envoyproxy/envoy:v1.15.0", request.Task.Config["image"])
}
func TestTaskRunner_EnvoyVersionHook_Prestart_custom(t *testing.T) {
t.Parallel()
logger := testlog.HCLogger(t)
// Setup an Allocation
alloc := mock.ConnectAlloc()
alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
alloc.Job.TaskGroups[0].Tasks[0].Config["image"] = "custom-${NOMAD_envoy_version}:latest"
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
defer cleanupDir()
// Setup a mock for Consul API
spAPI := consul.MockSupportedProxiesAPI{
Value: map[string][]string{
"envoy": {"1.14.1", "1.13.3"},
},
Error: nil,
}
// Run envoy_version hook
h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
// Create a prestart request
request := &ifs.TaskPrestartRequest{
Task: alloc.Job.TaskGroups[0].Tasks[0],
TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, request.TaskDir.Build(false, nil))
// Prepare a response
var response ifs.TaskPrestartResponse
// Run the hook
require.NoError(t, h.Prestart(context.Background(), request, &response))
// Assert the hook is Done
require.True(t, response.Done)
// Assert the Task.Config[image] is concrete
require.Equal(t, "custom-1.14.1:latest", request.Task.Config["image"])
}
func TestTaskRunner_EnvoyVersionHook_Prestart_skip(t *testing.T) {
t.Parallel()
logger := testlog.HCLogger(t)
// Setup an Allocation
alloc := mock.ConnectAlloc()
alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
alloc.Job.TaskGroups[0].Tasks[0].Driver = "exec"
alloc.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{
"command": "/sidecar",
}
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
defer cleanupDir()
// Setup a mock for Consul API
spAPI := consul.MockSupportedProxiesAPI{
Value: map[string][]string{
"envoy": {"1.14.1", "1.13.3"},
},
Error: nil,
}
// Run envoy_version hook
h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
// Create a prestart request
request := &ifs.TaskPrestartRequest{
Task: alloc.Job.TaskGroups[0].Tasks[0],
TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, request.TaskDir.Build(false, nil))
// Prepare a response
var response ifs.TaskPrestartResponse
// Run the hook
require.NoError(t, h.Prestart(context.Background(), request, &response))
// Assert the hook is Done
require.True(t, response.Done)
// Assert the Task.Config[image] does not get set
require.Empty(t, request.Task.Config["image"])
}
func TestTaskRunner_EnvoyVersionHook_Prestart_fallback(t *testing.T) {
t.Parallel()
logger := testlog.HCLogger(t)
// Setup an Allocation
alloc := mock.ConnectAlloc()
alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
defer cleanupDir()
// Setup a mock for Consul API
spAPI := consul.MockSupportedProxiesAPI{
Value: nil, // old consul, no .xDS.SupportedProxies
Error: nil,
}
// Run envoy_version hook
h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
// Create a prestart request
request := &ifs.TaskPrestartRequest{
Task: alloc.Job.TaskGroups[0].Tasks[0],
TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, request.TaskDir.Build(false, nil))
// Prepare a response
var response ifs.TaskPrestartResponse
// Run the hook
require.NoError(t, h.Prestart(context.Background(), request, &response))
// Assert the hook is Done
require.True(t, response.Done)
// Assert the Task.Config[image] is the fallback image
require.Equal(t, "envoyproxy/envoy:v1.11.2@sha256:a7769160c9c1a55bb8d07a3b71ce5d64f72b1f665f10d81aa1581bc3cf850d09", request.Task.Config["image"])
}
func TestTaskRunner_EnvoyVersionHook_Prestart_error(t *testing.T) {
t.Parallel()
logger := testlog.HCLogger(t)
// Setup an Allocation
alloc := mock.ConnectAlloc()
alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook")
defer cleanupDir()
// Setup a mock for Consul API
spAPI := consul.MockSupportedProxiesAPI{
Value: nil,
Error: errors.New("some consul error"),
}
// Run envoy_version hook
h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPI, logger))
// Create a prestart request
request := &ifs.TaskPrestartRequest{
Task: alloc.Job.TaskGroups[0].Tasks[0],
TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, request.TaskDir.Build(false, nil))
// Prepare a response
var response ifs.TaskPrestartResponse
// Run the hook, error should be recoverable
err := h.Prestart(context.Background(), request, &response)
require.EqualError(t, err, "error retrieving supported Envoy versions from Consul: some consul error")
// Assert the hook is not Done
require.False(t, response.Done)
}

14
client/allocrunner/taskrunner/task_runner.go
View file

@ -158,7 +158,12 @@ type TaskRunner struct {
// consulClient is the client used by the consul service hook for
// registering services and checks
consulClient consul.ConsulServiceAPI
consulServiceClient consul.ConsulServiceAPI
// consulProxiesClient is the client used by the envoy version hook for
// asking consul what version of envoy nomad should inject into the connect
// sidecar or gateway task.
consulProxiesClient consul.SupportedProxiesAPI
// sidsClient is the client used by the service identity hook for managing
// service identity tokens
@ -234,6 +239,10 @@ type Config struct {
// Consul is the client to use for managing Consul service registrations
Consul consul.ConsulServiceAPI
// ConsulProxies is the client to use for looking up supported envoy versions
// from Consul.
ConsulProxies consul.SupportedProxiesAPI
// ConsulSI is the client to use for managing Consul SI tokens
ConsulSI consul.ServiceIdentityAPI
@ -302,7 +311,8 @@ func NewTaskRunner(config *Config) (*TaskRunner, error) {
taskLeader: config.Task.Leader,
envBuilder: envBuilder,
dynamicRegistry: config.DynamicRegistry,
consulClient: config.Consul,
consulServiceClient: config.Consul,
consulProxiesClient: config.ConsulProxies,
siClient: config.ConsulSI,
vaultClient: config.Vault,
state: tstate,

13
client/allocrunner/taskrunner/task_runner_hooks.go
View file

@ -106,7 +106,7 @@ func (tr *TaskRunner) initHooks() {
tr.runnerHooks = append(tr.runnerHooks, newServiceHook(serviceHookConfig{
alloc: tr.Alloc(),
task: tr.Task(),
consul: tr.consulClient,
consul: tr.consulServiceClient,
restarter: tr,
logger: hookLogger,
}))
@ -127,10 +127,11 @@ func (tr *TaskRunner) initHooks() {
}))
}
if task.Kind.IsConnectProxy() || task.Kind.IsAnyConnectGateway() {
tr.runnerHooks = append(tr.runnerHooks, newEnvoyBootstrapHook(
newEnvoyBootstrapHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger),
))
if task.UsesConnectSidecar() {
tr.runnerHooks = append(tr.runnerHooks,
newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, tr.consulProxiesClient, hookLogger)),
newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger)),
)
} else if task.Kind.IsConnectNative() {
tr.runnerHooks = append(tr.runnerHooks, newConnectNativeHook(
newConnectNativeHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger),
@ -142,7 +143,7 @@ func (tr *TaskRunner) initHooks() {
scriptCheckHook := newScriptCheckHook(scriptCheckHookConfig{
alloc: tr.Alloc(),
task: tr.Task(),
consul: tr.consulClient,
consul: tr.consulServiceClient,
logger: hookLogger,
})
tr.runnerHooks = append(tr.runnerHooks, scriptCheckHook)

8
client/client.go
View file

@ -224,6 +224,10 @@ type Client struct {
// and checks.
consulService consulApi.ConsulServiceAPI
// consulProxies is Nomad's custom Consul client for looking up supported
// envoy versions
consulProxies consulApi.SupportedProxiesAPI
// consulCatalog is the subset of Consul's Catalog API Nomad uses.
consulCatalog consul.CatalogAPI
@ -306,7 +310,7 @@ var (
)
// NewClient is used to create a new client from the given configuration
func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService consulApi.ConsulServiceAPI) (*Client, error) {
func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxies consulApi.SupportedProxiesAPI, consulService consulApi.ConsulServiceAPI) (*Client, error) {
// Create the tls wrapper
var tlsWrap tlsutil.RegionWrapper
if cfg.TLSConfig.EnableRPC {
@ -331,6 +335,7 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic
c := &Client{
config: cfg,
consulCatalog: consulCatalog,
consulProxies: consulProxies,
consulService: consulService,
start: time.Now(),
connPool: pool.NewPool(logger, clientRPCCache, clientMaxStreams, tlsWrap),
@ -2384,6 +2389,7 @@ func (c *Client) addAlloc(alloc *structs.Allocation, migrateToken string) error
ClientConfig: c.configCopy,
StateDB: c.stateDB,
Consul: c.consulService,
ConsulProxies: c.consulProxies,
ConsulSI: c.tokensClient,
Vault: c.vaultClient,
StateUpdater: c,

2
client/client_test.go
View file

@ -622,7 +622,7 @@ func TestClient_SaveRestoreState(t *testing.T) {
c1.config.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", c1.config.Options, nil)
c1.config.PluginSingletonLoader = singleton.NewSingletonLoader(logger, c1.config.PluginLoader)
c2, err := NewClient(c1.config, consulCatalog, mockService)
c2, err := NewClient(c1.config, consulCatalog, nil, mockService)
if err != nil {
t.Fatalf("err: %v", err)
}

14
client/config/config.go
View file

@ -13,7 +13,7 @@ import (
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/pluginutils/loader"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/nomad/structs/config"
structsc "github.com/hashicorp/nomad/nomad/structs/config"
"github.com/hashicorp/nomad/plugins/base"
"github.com/hashicorp/nomad/version"
)
@ -149,10 +149,10 @@ type Config struct {
Version *version.VersionInfo
// ConsulConfig is this Agent's Consul configuration
ConsulConfig *config.ConsulConfig
ConsulConfig *structsc.ConsulConfig
// VaultConfig is this Agent's Vault configuration
VaultConfig *config.VaultConfig
VaultConfig *structsc.VaultConfig
// StatsCollectionInterval is the interval at which the Nomad client
// collects resource usage stats
@ -167,7 +167,7 @@ type Config struct {
PublishAllocationMetrics bool
// TLSConfig holds various TLS related configurations
TLSConfig *config.TLSConfig
TLSConfig *structsc.TLSConfig
// GCInterval is the time interval at which the client triggers garbage
// collection
@ -308,12 +308,12 @@ func (c *Config) Copy() *Config {
func DefaultConfig() *Config {
return &Config{
Version: version.GetVersion(),
VaultConfig: config.DefaultVaultConfig(),
ConsulConfig: config.DefaultConsulConfig(),
VaultConfig: structsc.DefaultVaultConfig(),
ConsulConfig: structsc.DefaultConsulConfig(),
LogOutput: os.Stderr,
Region: "global",
StatsCollectionInterval: 1 * time.Second,
TLSConfig: &config.TLSConfig{},
TLSConfig: &structsc.TLSConfig{},
LogLevel: "DEBUG",
GCInterval: 1 * time.Minute,
GCParallelDestroys: 2,

8
client/consul/consul.go
View file

@ -42,3 +42,11 @@ type ServiceIdentityAPI interface {
// identity tokens be generated for tasks in the allocation.
DeriveSITokens(alloc *structs.Allocation, tasks []string) (map[string]string, error)
}
// SupportedProxiesAPI is the interface the Nomad Client uses to request from
// Consul the set of supported proxied to use for Consul Connect.
//
// No ACL requirements
type SupportedProxiesAPI interface {
Proxies() (map[string][]string, error)
}

10
client/testing.go
View file

@ -5,9 +5,9 @@ import (
"time"
"github.com/hashicorp/nomad/client/config"
consulApi "github.com/hashicorp/nomad/client/consul"
consulapi "github.com/hashicorp/nomad/client/consul"
"github.com/hashicorp/nomad/client/fingerprint"
"github.com/hashicorp/nomad/command/agent/consul"
agentconsul "github.com/hashicorp/nomad/command/agent/consul"
"github.com/hashicorp/nomad/helper/pluginutils/catalog"
"github.com/hashicorp/nomad/helper/pluginutils/singleton"
"github.com/hashicorp/nomad/helper/testlog"
@ -44,9 +44,9 @@ func TestClient(t testing.T, cb func(c *config.Config)) (*Client, func() error)
if conf.PluginSingletonLoader == nil {
conf.PluginSingletonLoader = singleton.NewSingletonLoader(logger, conf.PluginLoader)
}
catalog := consul.NewMockCatalog(logger)
mockService := consulApi.NewMockConsulServiceClient(t, logger)
client, err := NewClient(conf, catalog, mockService)
mockCatalog := agentconsul.NewMockCatalog(logger)
mockService := consulapi.NewMockConsulServiceClient(t, logger)
client, err := NewClient(conf, mockCatalog, nil, mockService)
if err != nil {
cleanup()
t.Fatalf("err: %v", err)

25
command/agent/agent.go
View file

@ -15,7 +15,7 @@ import (
"time"
metrics "github.com/armon/go-metrics"
"github.com/hashicorp/consul/api"
consulapi "github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/lib"
log "github.com/hashicorp/go-hclog"
uuidparse "github.com/hashicorp/go-uuid"
@ -74,10 +74,13 @@ type Agent struct {
// and checks.
consulService *consul.ServiceClient
// consulProxies is the subset of Consul's Agent API Nomad uses.
consulProxies *consul.ConnectProxies
// consulCatalog is the subset of Consul's Catalog API Nomad uses.
consulCatalog consul.CatalogAPI
// consulConfigEntries is the subset of Consul's Configuration Entires API Nomad uses.
// consulConfigEntries is the subset of Consul's Configuration Entries API Nomad uses.
consulConfigEntries consul.ConfigAPI
// consulACLs is Nomad's subset of Consul's ACL API Nomad uses.
@ -849,11 +852,11 @@ func (a *Agent) setupClient() error {
conf.StateDBFactory = state.GetStateDBFactory(conf.DevMode)
}
client, err := client.NewClient(conf, a.consulCatalog, a.consulService)
nomadClient, err := client.NewClient(conf, a.consulCatalog, a.consulProxies, a.consulService)
if err != nil {
return fmt.Errorf("client setup failed: %v", err)
}
a.client = client
a.client = nomadClient
// Create the Nomad Client services for Consul
if *a.config.Consul.AutoAdvertise {
@ -1123,26 +1126,30 @@ func (a *Agent) setupConsul(consulConfig *config.ConsulConfig) error {
if err != nil {
return err
}
client, err := api.NewClient(apiConf)
consulClient, err := consulapi.NewClient(apiConf)
if err != nil {
return err
}
// Create Consul Catalog client for service discovery.
a.consulCatalog = client.Catalog()
a.consulCatalog = consulClient.Catalog()
// Create Consul ConfigEntries client for managing Config Entries.
a.consulConfigEntries = client.ConfigEntries()
a.consulConfigEntries = consulClient.ConfigEntries()
// Create Consul ACL client for managing tokens.
a.consulACLs = client.ACL()
a.consulACLs = consulClient.ACL()
// Create Consul Service client for service advertisement and checks.
isClient := false
if a.config.Client != nil && a.config.Client.Enabled {
isClient = true
}
a.consulService = consul.NewServiceClient(client.Agent(), a.logger, isClient)
// Create Consul Agent client for looking info about the agent.
consulAgentClient := consulClient.Agent()
a.consulService = consul.NewServiceClient(consulAgentClient, a.logger, isClient)
a.consulProxies = consul.NewConnectProxiesClient(consulAgentClient)
// Run the Consul service client's sync'ing main loop
go a.consulService.Run()

10
command/agent/consul/catalog_testing.go
View file

@ -95,6 +95,16 @@ func (c *MockAgent) Self() (map[string]map[string]interface{}, error) {
"build": "0.8.1:'e9ca44d",
},
},
"xDS": {
"SupportedProxies": map[string]interface{}{
"envoy": []interface{}{
"1.14.2",
"1.13.2",
"1.12.4",
"1.11.2",
},
},
},
}
return s, nil
}

92
command/agent/consul/connect_proxies.go Normal file
View file

@ -0,0 +1,92 @@
package consul
import (
"errors"
)
// ConnectProxies implements SupportedProxiesAPI by using the Consul Agent API.
type ConnectProxies struct {
agentAPI AgentAPI
}
func NewConnectProxiesClient(agentAPI AgentAPI) *ConnectProxies {
return &ConnectProxies{
agentAPI: agentAPI,
}
}
// Proxies returns a map of the supported proxies. The proxies are sorted from
// Consul with the most preferred version as the 0th element.
//
// If Consul is of a version that does not support the API, a nil map is returned
// with no error.
//
// If Consul cannot be reached an error is returned.
func (c *ConnectProxies) Proxies() (map[string][]string, error) {
// Based on the Consul query:
// $ curl -s localhost:8500/v1/agent/self | jq .xDS
// {
// "SupportedProxies": {
// "envoy": [
// "1.15.0",
// "1.14.4",
// "1.13.4",
// "1.12.6"
// ]
// }
// }
self, err := c.agentAPI.Self()
if err != nil {
// this should not fail as long as we can reach consul
return nil, err
}
// If consul does not return a map of the supported consul proxies, it
// must be a version from before when the API was added in versions
// 1.9.0, 1.8.3, 1.7.7. Earlier versions in the same point release as well
// as all of 1.6.X support Connect, but not the supported proxies API.
// For these cases, we can simply fallback to the old version of Envoy
// that Nomad defaulted to back then - but not in this logic. Instead,
// return nil so we can choose what to do at the caller.
xds, xdsExists := self["xDS"]
if !xdsExists {
return nil, nil
}
proxies, proxiesExists := xds["SupportedProxies"]
if !proxiesExists {
return nil, nil
}
// convert interface{} to map[string]interface{}
intermediate, ok := proxies.(map[string]interface{})
if !ok {
return nil, errors.New("unexpected SupportedProxies response format from Consul")
}
// convert map[string]interface{} to map[string][]string
result := make(map[string][]string, len(intermediate))
for k, v := range intermediate {
// convert interface{} to []interface{}
if si, ok := v.([]interface{}); ok {
ss := make([]string, 0, len(si))
for _, z := range si {
// convert interface{} to string
if s, ok := z.(string); ok {
ss = append(ss, s)
}
}
result[k] = ss
}
}
return result, nil
}

18
command/agent/consul/connect_proxies_test.go Normal file
View file

@ -0,0 +1,18 @@
package consul
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestConnectProxies_Proxies(t *testing.T) {
agentAPI := NewMockAgent()
pc := NewConnectProxiesClient(agentAPI)
proxies, err := pc.Proxies()
require.NoError(t, err)
require.Equal(t, map[string][]string{
"envoy": []string{"1.14.2", "1.13.2", "1.12.4", "1.11.2"},
}, proxies)
}

11
command/agent/consul/connect_proxies_testing.go Normal file
View file

@ -0,0 +1,11 @@
package consul
// ConnectProxies implements SupportedProxiesAPI by mocking the Consul Agent API.
type MockSupportedProxiesAPI struct {
Value map[string][]string
Error error
}
func (m MockSupportedProxiesAPI) Proxies() (map[string][]string, error) {
return m.Value, m.Error
}

0
command/agent/consul/client.go → command/agent/consul/service_client.go
View file

0
command/agent/consul/client_test.go → command/agent/consul/service_client_test.go
View file

4
nomad/job_endpoint_hook_connect.go
View file

@ -24,7 +24,7 @@ var (
// connect proxy sidecar task.
connectSidecarDriverConfig = func() map[string]interface{} {
return map[string]interface{}{
"image": "${meta.connect.sidecar_image}",
"image": structs.EnvoyImageFormat,
"args": []interface{}{
"-c", structs.EnvoyBootstrapPath,
"-l", "${meta.connect.log_level}",
@ -40,7 +40,7 @@ var (
// networking is being used the network_mode driver configuration is set here.
connectGatewayDriverConfig = func(hostNetwork bool) map[string]interface{} {
m := map[string]interface{}{
"image": "${meta.connect.gateway_image}",
"image": structs.EnvoyImageFormat,
"args": []interface{}{
"-c", structs.EnvoyBootstrapPath,
"-l", "${meta.connect.log_level}",

17
nomad/mock/mock.go
View file

@ -721,6 +721,23 @@ func ConnectIngressGatewayJob(mode string, inject bool) *structs.Job {
return job
}
func ConnectSidecarTask() *structs.Task {
return &structs.Task{
Name: "mysidecar-sidecar-task",
Driver: "docker",
User: "nobody",
Config: map[string]interface{}{
"image": structs.EnvoyImageFormat,
},
Env: nil,
Resources: &structs.Resources{
CPU: 150,
MemoryMB: 350,
},
Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "mysidecar"),
}
}
func BatchJob() *structs.Job {
job := &structs.Job{
Region: "global",

3
nomad/structs/config/consul.go
View file

@ -17,6 +17,9 @@ import (
// - Bootstrap this Nomad Client with the list of Nomad Servers registered
// with Consul
//
// - Establish how this Nomad Client will resolve Envoy Connect Sidecar
// images.
//
// Both the Agent and the executor need to be able to import ConsulConfig.
type ConsulConfig struct {
// ServerServiceName is the name of the service that Nomad uses to register

17
nomad/structs/connect.go Normal file
View file

@ -0,0 +1,17 @@
package structs
const (
// envoyImageFormat is the default format string used for official envoy Docker
// images with the tag being the semver of the version of envoy. Nomad fakes
// interpolation of ${NOMAD_envoy_version} by replacing it with the version
// string for envoy that Consul reports as preferred.
//
// Folks wanting to build and use custom images while still having Nomad refer
// to specific versions as preferred by Consul would set meta.connect.sidecar_image
// to something like: "custom/envoy:${NOMAD_envoy_version}".
EnvoyImageFormat = "envoyproxy/envoy:v" + EnvoyVersionVar
// envoyVersionVar will be replaced with the Envoy version string when
// used in the meta.connect.sidecar_image variable.
EnvoyVersionVar = "${NOMAD_envoy_version}"
)

6
nomad/structs/structs.go
View file

@ -6385,7 +6385,11 @@ type Task struct {
// Task, which exports known types of Tasks. UsesConnect will be true if the
// task is a connect proxy, connect native, or is a connect gateway.
func (t *Task) UsesConnect() bool {
return t.Kind.IsConnectProxy() || t.Kind.IsConnectNative() || t.Kind.IsAnyConnectGateway()
return t.Kind.IsConnectNative() || t.UsesConnectSidecar()
}
func (t *Task) UsesConnectSidecar() bool {
return t.Kind.IsConnectProxy() || t.Kind.IsAnyConnectGateway()
}
func (t *Task) Copy() *Task {

16
website/pages/docs/job-specification/gateway.mdx
View file

@ -187,6 +187,18 @@ accessible to any workload running on the same Nomad client. The admin interface
information about the proxy, including a Consul Service Identity token if Consul ACLs
are enabled.
### Specify Envoy image
The Docker image used for Connect gateway tasks defaults to the official [Envoy
Docker] image, `envoyproxy/envoy:v${NOMAD_envoy_version}`, where `${NOMAD_envoy_version}`
is resolved automatically by a query to Consul. The image to use can be configured
by setting `meta.connect.gateway_image` in the Nomad job. Custom images can still
make use of the envoy version interpolation, e.g.
```hcl
meta.connect.gateway_image = custom/envoy-${NOMAD_envoy_version}:latest
```
[proxy]: /docs/job-specification/gateway#proxy-parameters
[ingress]: /docs/job-specification/gateway#ingress-parameters
[tls]: /docs/job-specification/gateway#tls-parameters
@ -195,4 +207,6 @@ are enabled.
[service-default]: https://www.consul.io/docs/agent/config-entries/service-defaults
[connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout
[address]: /docs/job-specification/gateway#address-parameters
[advanced configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration
[Advanced Configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration
[Envoy Docker]: https://hub.docker.com/r/envoyproxy/envoy/tags

14
website/pages/docs/job-specification/sidecar_task.mdx
View file

@ -97,10 +97,20 @@ The default sidecar task is equivalent to:
The `meta.connect.sidecar_image` and `meta.connect.log_level` are [_client_
configurable][nodemeta] variables with the following defaults:
- `sidecar_image` - `"envoyproxy/envoy:v1.11.2@sha256:a7769160c9c1a55bb8d07a3b71ce5d64f72b1f665f10d81aa1581bc3cf850d09"` - The official upstream Envoy Docker image.
- `log_level` - `"info"` - Envoy sidecar log level. "`debug`" is useful for
- `sidecar_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official
upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically
by a query to Consul.
- `log_level` - `(string: "info")` - Envoy sidecar log level. "`debug`" is useful for
debugging Connect related issues.
`meta.connect.sidecar_image` can be configured at the job, group, or task level.
Custom images can make use of Consul's preferred Envoy version by making use of
Nomad's version interpolation, e.g.
```hcl
meta.connect.sidecar_image = custom/envoy-${NOMAD_envoy_version}:latest
```
## `sidecar_task` Parameters
- `name` `(string: "connect-proxy-<service>")` - Name of the task. Defaults to

24
website/pages/docs/upgrade/upgrade-specific.mdx
View file

@ -60,6 +60,30 @@ Nomad. The specific configuration values replaced are:
* Client `template.function_blacklist` is replaced with `template.function_denylist`.
* Docker driver `docker.caps.whitelist` is replaced with `docker.caps.allowlist`.
### Envoy proxy versions
Nomad 0.13.0 changes the behavior around the selection of Envoy version used
for Connect sidecar proxies. Previously, Nomad always defaulted to Envoy v1.11.2
if neither the `meta.connect.sidecar_image` parameter or `sidecar_task` stanza
were explicitly configured. Likewise the same version of Envoy would be used for
Connect ingress gateways if `meta.connect.gateway_image` was unset. Starting with
Nomad 0.13.0, each Nomad Client will query Consul for a list of supported Envoy
versions. Nomad will make use of the latest version of Envoy supported by the
Consul agent when launching Envoy as a Connect sidecar proxy. If the version of
the Consul agent is older than v1.7.8, v1.8.4, or v1.9.0, Nomad will fallback to
the v1.11.2 version of Envoy. As before, if the `meta.connect.sidecar_image`,
`meta.connect.gateway_image`, or `sidecar_task` stanza are set, those settings
take precedence.
When upgrading Nomad Clients from a previous version to v0.13.0 and above, it is
recommended to also upgrade the Consul agents to v1.7.8, 1.8.4, or v1.9.0 or newer.
Upgrading Nomad and Consul to versions that support the new behaviour while also doing a
full [node drain](https://www.nomadproject.io/docs/upgrade#5-upgrade-clients) at
the time of the upgrade for each node will ensure Connect workloads are properly
rescheduled onto nodes in such a way that the Nomad Clients, Consul agents, and
Envoy sidecar tasks maintain compatibility with one another.
## Nomad 0.12.0
### `mbits` and Task Network Resource deprecation