e7fcd281ae
Fixes #6594 #6711 #6714 #7567 e2e testing is still TBD in #6502 Before, we only passed the Nomad agent's configured Consul HTTP address onto the `consul connect envoy ...` bootstrap command. This meant any Consul setup with TLS enabled would not work with Nomad's Connect integration. This change now sets CLI args and Environment Variables for configuring TLS options for communicating with Consul when doing the envoy bootstrap, as described in https://www.consul.io/docs/commands/connect/envoy.html#usage
549 lines
16 KiB
Go
549 lines
16 KiB
Go
// +build !windows
|
|
// todo(shoenig): Once Connect is supported on Windows, we'll need to make this
|
|
// set of tests work there too.
|
|
|
|
package taskrunner
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
consulapi "github.com/hashicorp/consul/api"
|
|
consultest "github.com/hashicorp/consul/sdk/testutil"
|
|
"github.com/hashicorp/nomad/client/allocdir"
|
|
"github.com/hashicorp/nomad/client/allocrunner/interfaces"
|
|
"github.com/hashicorp/nomad/client/taskenv"
|
|
"github.com/hashicorp/nomad/client/testutil"
|
|
agentconsul "github.com/hashicorp/nomad/command/agent/consul"
|
|
"github.com/hashicorp/nomad/helper"
|
|
"github.com/hashicorp/nomad/helper/args"
|
|
"github.com/hashicorp/nomad/helper/testlog"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/hashicorp/nomad/nomad/structs/config"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
var _ interfaces.TaskPrestartHook = (*envoyBootstrapHook)(nil)
|
|
|
|
func writeTmp(t *testing.T, s string, fm os.FileMode) string {
|
|
dir, err := ioutil.TempDir("", "envoy-")
|
|
require.NoError(t, err)
|
|
|
|
fPath := filepath.Join(dir, sidsTokenFile)
|
|
err = ioutil.WriteFile(fPath, []byte(s), fm)
|
|
require.NoError(t, err)
|
|
|
|
return dir
|
|
}
|
|
|
|
func TestEnvoyBootstrapHook_maybeLoadSIToken(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// This test fails when running as root because the test case for checking
|
|
// the error condition when the file is unreadable fails (root can read the
|
|
// file even though the permissions are set to 0200).
|
|
if unix.Geteuid() == 0 {
|
|
t.Skip("test only works as non-root")
|
|
}
|
|
|
|
t.Run("file does not exist", func(t *testing.T) {
|
|
h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
|
|
cfg, err := h.maybeLoadSIToken("task1", "/does/not/exist")
|
|
require.NoError(t, err) // absence of token is not an error
|
|
require.Equal(t, "", cfg)
|
|
})
|
|
|
|
t.Run("load token from file", func(t *testing.T) {
|
|
token := uuid.Generate()
|
|
f := writeTmp(t, token, 0440)
|
|
defer cleanupDir(t, f)
|
|
|
|
h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
|
|
cfg, err := h.maybeLoadSIToken("task1", f)
|
|
require.NoError(t, err)
|
|
require.Equal(t, token, cfg)
|
|
})
|
|
|
|
t.Run("file is unreadable", func(t *testing.T) {
|
|
token := uuid.Generate()
|
|
f := writeTmp(t, token, 0200)
|
|
defer cleanupDir(t, f)
|
|
|
|
h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
|
|
cfg, err := h.maybeLoadSIToken("task1", f)
|
|
require.Error(t, err)
|
|
require.False(t, os.IsNotExist(err))
|
|
require.Equal(t, "", cfg)
|
|
})
|
|
}
|
|
|
|
func TestEnvoyBootstrapHook_decodeTriState(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
require.Equal(t, "", decodeTriState(nil))
|
|
require.Equal(t, "true", decodeTriState(helper.BoolToPtr(true)))
|
|
require.Equal(t, "false", decodeTriState(helper.BoolToPtr(false)))
|
|
}
|
|
|
|
var (
|
|
consulPlainConfig = envoyBootstrapConsulConfig{
|
|
HTTPAddr: "2.2.2.2",
|
|
}
|
|
|
|
consulTLSConfig = envoyBootstrapConsulConfig{
|
|
HTTPAddr: "2.2.2.2", // arg
|
|
Auth: "user:password", // env
|
|
SSL: "true", // env
|
|
VerifySSL: "true", // env
|
|
CAFile: "/etc/tls/ca-file", // arg
|
|
CertFile: "/etc/tls/cert-file", // arg
|
|
KeyFile: "/etc/tls/key-file", // arg
|
|
}
|
|
)
|
|
|
|
func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("excluding SI token", func(t *testing.T) {
|
|
ebArgs := envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulPlainConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
}
|
|
result := ebArgs.args()
|
|
require.Equal(t, []string{"connect", "envoy",
|
|
"-grpc-addr", "1.1.1.1",
|
|
"-http-addr", "2.2.2.2",
|
|
"-admin-bind", "localhost:3333",
|
|
"-bootstrap",
|
|
"-sidecar-for", "s1",
|
|
}, result)
|
|
})
|
|
|
|
t.Run("including SI token", func(t *testing.T) {
|
|
token := uuid.Generate()
|
|
ebArgs := envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulPlainConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
siToken: token,
|
|
}
|
|
result := ebArgs.args()
|
|
require.Equal(t, []string{"connect", "envoy",
|
|
"-grpc-addr", "1.1.1.1",
|
|
"-http-addr", "2.2.2.2",
|
|
"-admin-bind", "localhost:3333",
|
|
"-bootstrap",
|
|
"-sidecar-for", "s1",
|
|
"-token", token,
|
|
}, result)
|
|
})
|
|
|
|
t.Run("including certificates", func(t *testing.T) {
|
|
ebArgs := envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulTLSConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
}
|
|
result := ebArgs.args()
|
|
require.Equal(t, []string{"connect", "envoy",
|
|
"-grpc-addr", "1.1.1.1",
|
|
"-http-addr", "2.2.2.2",
|
|
"-admin-bind", "localhost:3333",
|
|
"-bootstrap",
|
|
"-sidecar-for", "s1",
|
|
"-ca-file", "/etc/tls/ca-file",
|
|
"-client-cert", "/etc/tls/cert-file",
|
|
"-client-key", "/etc/tls/key-file",
|
|
}, result)
|
|
})
|
|
}
|
|
|
|
func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
environment := []string{"foo=bar", "baz=1"}
|
|
|
|
t.Run("plain consul config", func(t *testing.T) {
|
|
require.Equal(t, []string{
|
|
"foo=bar", "baz=1",
|
|
}, envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulPlainConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
}.env(environment))
|
|
})
|
|
|
|
t.Run("tls consul config", func(t *testing.T) {
|
|
require.Equal(t, []string{
|
|
"foo=bar", "baz=1",
|
|
"CONSUL_HTTP_AUTH=user:password",
|
|
"CONSUL_HTTP_SSL=true",
|
|
"CONSUL_HTTP_SSL_VERIFY=true",
|
|
}, envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulTLSConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
}.env(environment))
|
|
})
|
|
}
|
|
|
|
// dig through envoy config to look for consul token
|
|
type envoyConfig struct {
|
|
DynamicResources struct {
|
|
ADSConfig struct {
|
|
GRPCServices struct {
|
|
InitialMetadata []struct {
|
|
Key string `json:"key"`
|
|
Value string `json:"value"`
|
|
} `json:"initial_metadata"`
|
|
} `json:"grpc_services"`
|
|
} `json:"ads_config"`
|
|
} `json:"dynamic_resources"`
|
|
}
|
|
|
|
// TestEnvoyBootstrapHook_with_SI_token asserts the bootstrap file written for
|
|
// Envoy contains a Consul SI token.
|
|
func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) {
|
|
t.Parallel()
|
|
testutil.RequireConsul(t)
|
|
|
|
testconsul, err := consultest.NewTestServerConfig(func(c *consultest.TestServerConfig) {
|
|
// If -v wasn't specified squelch consul logging
|
|
if !testing.Verbose() {
|
|
c.Stdout = ioutil.Discard
|
|
c.Stderr = ioutil.Discard
|
|
}
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("error starting test consul server: %v", err)
|
|
}
|
|
defer testconsul.Stop()
|
|
|
|
alloc := mock.Alloc()
|
|
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
|
|
{
|
|
Mode: "bridge",
|
|
IP: "10.0.0.1",
|
|
DynamicPorts: []structs.Port{
|
|
{
|
|
Label: "connect-proxy-foo",
|
|
Value: 9999,
|
|
To: 9999,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
tg := alloc.Job.TaskGroups[0]
|
|
tg.Services = []*structs.Service{
|
|
{
|
|
Name: "foo",
|
|
PortLabel: "9999", // Just need a valid port, nothing will bind to it
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{},
|
|
},
|
|
},
|
|
}
|
|
sidecarTask := &structs.Task{
|
|
Name: "sidecar",
|
|
Kind: "connect-proxy:foo",
|
|
}
|
|
tg.Tasks = append(tg.Tasks, sidecarTask)
|
|
|
|
logger := testlog.HCLogger(t)
|
|
|
|
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
|
|
defer cleanup()
|
|
|
|
// Register Group Services
|
|
consulConfig := consulapi.DefaultConfig()
|
|
consulConfig.Address = testconsul.HTTPAddr
|
|
consulAPIClient, err := consulapi.NewClient(consulConfig)
|
|
require.NoError(t, err)
|
|
|
|
consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
|
|
go consulClient.Run()
|
|
defer consulClient.Shutdown()
|
|
require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
|
|
|
|
// Run Connect bootstrap Hook
|
|
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
|
|
Addr: consulConfig.Address,
|
|
}, logger))
|
|
req := &interfaces.TaskPrestartRequest{
|
|
Task: sidecarTask,
|
|
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
|
|
}
|
|
require.NoError(t, req.TaskDir.Build(false, nil))
|
|
|
|
// Insert service identity token in the secrets directory
|
|
token := uuid.Generate()
|
|
siTokenFile := filepath.Join(req.TaskDir.SecretsDir, sidsTokenFile)
|
|
err = ioutil.WriteFile(siTokenFile, []byte(token), 0440)
|
|
require.NoError(t, err)
|
|
|
|
resp := &interfaces.TaskPrestartResponse{}
|
|
|
|
// Run the hook
|
|
require.NoError(t, h.Prestart(context.Background(), req, resp))
|
|
|
|
// Assert it is Done
|
|
require.True(t, resp.Done)
|
|
|
|
// Ensure the default path matches
|
|
env := map[string]string{
|
|
taskenv.SecretsDir: req.TaskDir.SecretsDir,
|
|
}
|
|
f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
|
|
require.NoError(t, err)
|
|
defer f.Close()
|
|
|
|
// Assert bootstrap configuration is valid json
|
|
var out envoyConfig
|
|
require.NoError(t, json.NewDecoder(f).Decode(&out))
|
|
|
|
// Assert the SI token got set
|
|
key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
|
|
value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
|
|
require.Equal(t, "x-consul-token", key)
|
|
require.Equal(t, token, value)
|
|
}
|
|
|
|
// TestTaskRunner_EnvoyBootstrapHook_Prestart asserts the EnvoyBootstrapHook
|
|
// creates Envoy's bootstrap.json configuration based on Connect proxy sidecars
|
|
// registered for the task.
|
|
func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) {
|
|
t.Parallel()
|
|
testutil.RequireConsul(t)
|
|
|
|
testconsul, err := consultest.NewTestServerConfig(func(c *consultest.TestServerConfig) {
|
|
// If -v wasn't specified squelch consul logging
|
|
if !testing.Verbose() {
|
|
c.Stdout = ioutil.Discard
|
|
c.Stderr = ioutil.Discard
|
|
}
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("error starting test consul server: %v", err)
|
|
}
|
|
defer testconsul.Stop()
|
|
|
|
alloc := mock.Alloc()
|
|
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
|
|
{
|
|
Mode: "bridge",
|
|
IP: "10.0.0.1",
|
|
DynamicPorts: []structs.Port{
|
|
{
|
|
Label: "connect-proxy-foo",
|
|
Value: 9999,
|
|
To: 9999,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
tg := alloc.Job.TaskGroups[0]
|
|
tg.Services = []*structs.Service{
|
|
{
|
|
Name: "foo",
|
|
PortLabel: "9999", // Just need a valid port, nothing will bind to it
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{},
|
|
},
|
|
},
|
|
}
|
|
sidecarTask := &structs.Task{
|
|
Name: "sidecar",
|
|
Kind: "connect-proxy:foo",
|
|
}
|
|
tg.Tasks = append(tg.Tasks, sidecarTask)
|
|
|
|
logger := testlog.HCLogger(t)
|
|
|
|
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
|
|
defer cleanup()
|
|
|
|
// Register Group Services
|
|
consulConfig := consulapi.DefaultConfig()
|
|
consulConfig.Address = testconsul.HTTPAddr
|
|
consulAPIClient, err := consulapi.NewClient(consulConfig)
|
|
require.NoError(t, err)
|
|
|
|
consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
|
|
go consulClient.Run()
|
|
defer consulClient.Shutdown()
|
|
require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
|
|
|
|
// Run Connect bootstrap Hook
|
|
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
|
|
Addr: consulConfig.Address,
|
|
}, logger))
|
|
req := &interfaces.TaskPrestartRequest{
|
|
Task: sidecarTask,
|
|
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
|
|
}
|
|
require.NoError(t, req.TaskDir.Build(false, nil))
|
|
|
|
resp := &interfaces.TaskPrestartResponse{}
|
|
|
|
// Run the hook
|
|
require.NoError(t, h.Prestart(context.Background(), req, resp))
|
|
|
|
// Assert it is Done
|
|
require.True(t, resp.Done)
|
|
|
|
require.NotNil(t, resp.Env)
|
|
require.Equal(t, "localhost:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])
|
|
|
|
// Ensure the default path matches
|
|
env := map[string]string{
|
|
taskenv.SecretsDir: req.TaskDir.SecretsDir,
|
|
}
|
|
f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
|
|
require.NoError(t, err)
|
|
defer f.Close()
|
|
|
|
// Assert bootstrap configuration is valid json
|
|
var out envoyConfig
|
|
require.NoError(t, json.NewDecoder(f).Decode(&out))
|
|
|
|
// Assert no SI token got set
|
|
key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
|
|
value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
|
|
require.Equal(t, "x-consul-token", key)
|
|
require.Equal(t, "", value)
|
|
}
|
|
|
|
// TestTaskRunner_EnvoyBootstrapHook_Noop asserts that the Envoy bootstrap hook
|
|
// is a noop for non-Connect proxy sidecar tasks.
|
|
func TestTaskRunner_EnvoyBootstrapHook_Noop(t *testing.T) {
|
|
t.Parallel()
|
|
logger := testlog.HCLogger(t)
|
|
|
|
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
|
|
defer cleanup()
|
|
|
|
alloc := mock.Alloc()
|
|
task := alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks[0]
|
|
|
|
// Run Envoy bootstrap Hook. Use invalid Consul address as it should
|
|
// not get hit.
|
|
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
|
|
Addr: "http://127.0.0.2:1",
|
|
}, logger))
|
|
req := &interfaces.TaskPrestartRequest{
|
|
Task: task,
|
|
TaskDir: allocDir.NewTaskDir(task.Name),
|
|
}
|
|
require.NoError(t, req.TaskDir.Build(false, nil))
|
|
|
|
resp := &interfaces.TaskPrestartResponse{}
|
|
|
|
// Run the hook
|
|
require.NoError(t, h.Prestart(context.Background(), req, resp))
|
|
|
|
// Assert it is Done
|
|
require.True(t, resp.Done)
|
|
|
|
// Assert no file was written
|
|
_, err := os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
|
|
require.Error(t, err)
|
|
require.True(t, os.IsNotExist(err))
|
|
}
|
|
|
|
// TestTaskRunner_EnvoyBootstrapHook_RecoverableError asserts the Envoy
|
|
// bootstrap hook returns a Recoverable error if the bootstrap command runs but
|
|
// fails.
|
|
func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) {
|
|
t.Parallel()
|
|
testutil.RequireConsul(t)
|
|
|
|
testconsul, err := consultest.NewTestServerConfig(func(c *consultest.TestServerConfig) {
|
|
// If -v wasn't specified squelch consul logging
|
|
if !testing.Verbose() {
|
|
c.Stdout = ioutil.Discard
|
|
c.Stderr = ioutil.Discard
|
|
}
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("error starting test consul server: %v", err)
|
|
}
|
|
defer testconsul.Stop()
|
|
|
|
alloc := mock.Alloc()
|
|
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
|
|
{
|
|
Mode: "bridge",
|
|
IP: "10.0.0.1",
|
|
DynamicPorts: []structs.Port{
|
|
{
|
|
Label: "connect-proxy-foo",
|
|
Value: 9999,
|
|
To: 9999,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
tg := alloc.Job.TaskGroups[0]
|
|
tg.Services = []*structs.Service{
|
|
{
|
|
Name: "foo",
|
|
PortLabel: "9999", // Just need a valid port, nothing will bind to it
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{},
|
|
},
|
|
},
|
|
}
|
|
sidecarTask := &structs.Task{
|
|
Name: "sidecar",
|
|
Kind: "connect-proxy:foo",
|
|
}
|
|
tg.Tasks = append(tg.Tasks, sidecarTask)
|
|
|
|
logger := testlog.HCLogger(t)
|
|
|
|
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
|
|
defer cleanup()
|
|
|
|
// Unlike the successful test above, do NOT register the group services
|
|
// yet. This should cause a recoverable error similar to if Consul was
|
|
// not running.
|
|
|
|
// Run Connect bootstrap Hook
|
|
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
|
|
Addr: testconsul.HTTPAddr,
|
|
}, logger))
|
|
req := &interfaces.TaskPrestartRequest{
|
|
Task: sidecarTask,
|
|
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
|
|
}
|
|
require.NoError(t, req.TaskDir.Build(false, nil))
|
|
|
|
resp := &interfaces.TaskPrestartResponse{}
|
|
|
|
// Run the hook
|
|
err = h.Prestart(context.Background(), req, resp)
|
|
require.EqualError(t, err, "error creating bootstrap configuration for Connect proxy sidecar: exit status 1")
|
|
require.True(t, structs.IsRecoverable(err))
|
|
|
|
// Assert it is not Done
|
|
require.False(t, resp.Done)
|
|
|
|
// Assert no file was written
|
|
_, err = os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
|
|
require.Error(t, err)
|
|
require.True(t, os.IsNotExist(err))
|
|
}
|