open-nomad/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go

702 lines
20 KiB
Go

// +build !windows
// todo(shoenig): Once Connect is supported on Windows, we'll need to make this
// set of tests work there too.
package taskrunner
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
consulapi "github.com/hashicorp/consul/api"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/allocrunner/interfaces"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/client/testutil"
agentconsul "github.com/hashicorp/nomad/command/agent/consul"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/args"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/nomad/structs/config"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
var _ interfaces.TaskPrestartHook = (*envoyBootstrapHook)(nil)
func writeTmp(t *testing.T, s string, fm os.FileMode) string {
dir, err := ioutil.TempDir("", "envoy-")
require.NoError(t, err)
fPath := filepath.Join(dir, sidsTokenFile)
err = ioutil.WriteFile(fPath, []byte(s), fm)
require.NoError(t, err)
return dir
}
func TestEnvoyBootstrapHook_maybeLoadSIToken(t *testing.T) {
t.Parallel()
// This test fails when running as root because the test case for checking
// the error condition when the file is unreadable fails (root can read the
// file even though the permissions are set to 0200).
if unix.Geteuid() == 0 {
t.Skip("test only works as non-root")
}
t.Run("file does not exist", func(t *testing.T) {
h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
cfg, err := h.maybeLoadSIToken("task1", "/does/not/exist")
require.NoError(t, err) // absence of token is not an error
require.Equal(t, "", cfg)
})
t.Run("load token from file", func(t *testing.T) {
token := uuid.Generate()
f := writeTmp(t, token, 0440)
defer cleanupDir(t, f)
h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
cfg, err := h.maybeLoadSIToken("task1", f)
require.NoError(t, err)
require.Equal(t, token, cfg)
})
t.Run("file is unreadable", func(t *testing.T) {
token := uuid.Generate()
f := writeTmp(t, token, 0200)
defer cleanupDir(t, f)
h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
cfg, err := h.maybeLoadSIToken("task1", f)
require.Error(t, err)
require.False(t, os.IsNotExist(err))
require.Equal(t, "", cfg)
})
}
func TestEnvoyBootstrapHook_decodeTriState(t *testing.T) {
t.Parallel()
require.Equal(t, "", decodeTriState(nil))
require.Equal(t, "true", decodeTriState(helper.BoolToPtr(true)))
require.Equal(t, "false", decodeTriState(helper.BoolToPtr(false)))
}
var (
consulPlainConfig = consulTransportConfig{
HTTPAddr: "2.2.2.2",
}
consulTLSConfig = consulTransportConfig{
HTTPAddr: "2.2.2.2", // arg
Auth: "user:password", // env
SSL: "true", // env
VerifySSL: "true", // env
CAFile: "/etc/tls/ca-file", // arg
CertFile: "/etc/tls/cert-file", // arg
KeyFile: "/etc/tls/key-file", // arg
}
)
func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
t.Parallel()
t.Run("excluding SI token", func(t *testing.T) {
ebArgs := envoyBootstrapArgs{
sidecarFor: "s1",
grpcAddr: "1.1.1.1",
consulConfig: consulPlainConfig,
envoyAdminBind: "localhost:3333",
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-bootstrap",
"-sidecar-for", "s1",
}, result)
})
t.Run("including SI token", func(t *testing.T) {
token := uuid.Generate()
ebArgs := envoyBootstrapArgs{
sidecarFor: "s1",
grpcAddr: "1.1.1.1",
consulConfig: consulPlainConfig,
envoyAdminBind: "localhost:3333",
siToken: token,
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-bootstrap",
"-sidecar-for", "s1",
"-token", token,
}, result)
})
t.Run("including certificates", func(t *testing.T) {
ebArgs := envoyBootstrapArgs{
sidecarFor: "s1",
grpcAddr: "1.1.1.1",
consulConfig: consulTLSConfig,
envoyAdminBind: "localhost:3333",
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-bootstrap",
"-sidecar-for", "s1",
"-ca-file", "/etc/tls/ca-file",
"-client-cert", "/etc/tls/cert-file",
"-client-key", "/etc/tls/key-file",
}, result)
})
t.Run("ingress gateway", func(t *testing.T) {
ebArgs := envoyBootstrapArgs{
consulConfig: consulPlainConfig,
grpcAddr: "1.1.1.1",
envoyAdminBind: "localhost:3333",
gateway: "my-ingress-gateway",
proxyID: "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-ig-ig-8080",
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-bootstrap",
"-gateway", "my-ingress-gateway",
"-proxy-id", "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-ig-ig-8080",
}, result)
})
}
func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) {
t.Parallel()
environment := []string{"foo=bar", "baz=1"}
t.Run("plain consul config", func(t *testing.T) {
require.Equal(t, []string{
"foo=bar", "baz=1",
}, envoyBootstrapArgs{
sidecarFor: "s1",
grpcAddr: "1.1.1.1",
consulConfig: consulPlainConfig,
envoyAdminBind: "localhost:3333",
}.env(environment))
})
t.Run("tls consul config", func(t *testing.T) {
require.Equal(t, []string{
"foo=bar", "baz=1",
"CONSUL_HTTP_AUTH=user:password",
"CONSUL_HTTP_SSL=true",
"CONSUL_HTTP_SSL_VERIFY=true",
}, envoyBootstrapArgs{
sidecarFor: "s1",
grpcAddr: "1.1.1.1",
consulConfig: consulTLSConfig,
envoyAdminBind: "localhost:3333",
}.env(environment))
})
}
// envoyConfig is used to unmarshal an envoy bootstrap configuration file, so that
// we can inspect the contents in tests.
type envoyConfig struct {
Admin struct {
Address struct {
SocketAddress struct {
Address string `json:"address"`
Port int `json:"port_value"`
} `json:"socket_address"`
} `json:"address"`
} `json:"admin"`
Node struct {
Cluster string `json:"cluster"`
ID string `json:"id"`
Metadata struct {
Namespace string `json:"namespace"`
Version string `json:"envoy_version"`
}
}
DynamicResources struct {
ADSConfig struct {
GRPCServices struct {
InitialMetadata []struct {
Key string `json:"key"`
Value string `json:"value"`
} `json:"initial_metadata"`
} `json:"grpc_services"`
} `json:"ads_config"`
} `json:"dynamic_resources"`
}
// TestEnvoyBootstrapHook_with_SI_token asserts the bootstrap file written for
// Envoy contains a Consul SI token.
func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) {
t.Parallel()
testutil.RequireConsul(t)
testConsul := getTestConsul(t)
defer testConsul.Stop()
alloc := mock.ConnectAlloc()
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
{
Mode: "bridge",
IP: "10.0.0.1",
DynamicPorts: []structs.Port{
{
Label: "connect-proxy-foo",
Value: 9999,
To: 9999,
},
},
},
}
tg := alloc.Job.TaskGroups[0]
tg.Services = []*structs.Service{
{
Name: "foo",
PortLabel: "9999", // Just need a valid port, nothing will bind to it
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{},
},
},
}
sidecarTask := &structs.Task{
Name: "sidecar",
Kind: "connect-proxy:foo",
}
tg.Tasks = append(tg.Tasks, sidecarTask)
logger := testlog.HCLogger(t)
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
defer cleanup()
// Register Group Services
consulConfig := consulapi.DefaultConfig()
consulConfig.Address = testConsul.HTTPAddr
consulAPIClient, err := consulapi.NewClient(consulConfig)
require.NoError(t, err)
consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
go consulClient.Run()
defer consulClient.Shutdown()
require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
// Run Connect bootstrap Hook
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
Addr: consulConfig.Address,
}, logger))
req := &interfaces.TaskPrestartRequest{
Task: sidecarTask,
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, req.TaskDir.Build(false, nil))
// Insert service identity token in the secrets directory
token := uuid.Generate()
siTokenFile := filepath.Join(req.TaskDir.SecretsDir, sidsTokenFile)
err = ioutil.WriteFile(siTokenFile, []byte(token), 0440)
require.NoError(t, err)
resp := &interfaces.TaskPrestartResponse{}
// Run the hook
require.NoError(t, h.Prestart(context.Background(), req, resp))
// Assert it is Done
require.True(t, resp.Done)
// Ensure the default path matches
env := map[string]string{
taskenv.SecretsDir: req.TaskDir.SecretsDir,
}
f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
require.NoError(t, err)
defer f.Close()
// Assert bootstrap configuration is valid json
var out envoyConfig
require.NoError(t, json.NewDecoder(f).Decode(&out))
// Assert the SI token got set
key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
require.Equal(t, "x-consul-token", key)
require.Equal(t, token, value)
}
// TestTaskRunner_EnvoyBootstrapHook_sidecar_ok asserts the EnvoyBootstrapHook
// creates Envoy's bootstrap.json configuration based on Connect proxy sidecars
// registered for the task.
func TestTaskRunner_EnvoyBootstrapHook_sidecar_ok(t *testing.T) {
t.Parallel()
testutil.RequireConsul(t)
testConsul := getTestConsul(t)
defer testConsul.Stop()
alloc := mock.ConnectAlloc()
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
{
Mode: "bridge",
IP: "10.0.0.1",
DynamicPorts: []structs.Port{
{
Label: "connect-proxy-foo",
Value: 9999,
To: 9999,
},
},
},
}
tg := alloc.Job.TaskGroups[0]
tg.Services = []*structs.Service{
{
Name: "foo",
PortLabel: "9999", // Just need a valid port, nothing will bind to it
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{},
},
},
}
sidecarTask := &structs.Task{
Name: "sidecar",
Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"),
}
tg.Tasks = append(tg.Tasks, sidecarTask)
logger := testlog.HCLogger(t)
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
defer cleanup()
// Register Group Services
consulConfig := consulapi.DefaultConfig()
consulConfig.Address = testConsul.HTTPAddr
consulAPIClient, err := consulapi.NewClient(consulConfig)
require.NoError(t, err)
consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
go consulClient.Run()
defer consulClient.Shutdown()
require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
// Run Connect bootstrap Hook
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
Addr: consulConfig.Address,
}, logger))
req := &interfaces.TaskPrestartRequest{
Task: sidecarTask,
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, req.TaskDir.Build(false, nil))
resp := &interfaces.TaskPrestartResponse{}
// Run the hook
require.NoError(t, h.Prestart(context.Background(), req, resp))
// Assert it is Done
require.True(t, resp.Done)
require.NotNil(t, resp.Env)
require.Equal(t, "localhost:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])
// Ensure the default path matches
env := map[string]string{
taskenv.SecretsDir: req.TaskDir.SecretsDir,
}
f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
require.NoError(t, err)
defer f.Close()
// Assert bootstrap configuration is valid json
var out envoyConfig
require.NoError(t, json.NewDecoder(f).Decode(&out))
// Assert no SI token got set
key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
require.Equal(t, "x-consul-token", key)
require.Equal(t, "", value)
}
func TestTaskRunner_EnvoyBootstrapHook_gateway_ok(t *testing.T) {
t.Parallel()
logger := testlog.HCLogger(t)
testConsul := getTestConsul(t)
defer testConsul.Stop()
// Setup an Allocation
alloc := mock.ConnectIngressGatewayAlloc("bridge")
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyBootstrapIngressGateway")
defer cleanupDir()
// Get a Consul client
consulConfig := consulapi.DefaultConfig()
consulConfig.Address = testConsul.HTTPAddr
consulAPIClient, err := consulapi.NewClient(consulConfig)
require.NoError(t, err)
// Register Group Services
serviceClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
go serviceClient.Run()
defer serviceClient.Shutdown()
require.NoError(t, serviceClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
// Register Configuration Entry
ceClient := consulAPIClient.ConfigEntries()
set, _, err := ceClient.Set(&consulapi.IngressGatewayConfigEntry{
Kind: consulapi.IngressGateway,
Name: "gateway-service", // matches job
Listeners: []consulapi.IngressListener{{
Port: 2000,
Protocol: "tcp",
Services: []consulapi.IngressService{{
Name: "service1",
}},
}},
}, nil)
require.NoError(t, err)
require.True(t, set)
// Run Connect bootstrap hook
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
Addr: consulConfig.Address,
}, logger))
req := &interfaces.TaskPrestartRequest{
Task: alloc.Job.TaskGroups[0].Tasks[0],
TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, req.TaskDir.Build(false, nil))
var resp interfaces.TaskPrestartResponse
// Run the hook
require.NoError(t, h.Prestart(context.Background(), req, &resp))
// Assert the hook is Done
require.True(t, resp.Done)
require.NotNil(t, resp.Env)
// Read the Envoy Config file
env := map[string]string{
taskenv.SecretsDir: req.TaskDir.SecretsDir,
}
f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
require.NoError(t, err)
defer f.Close()
var out envoyConfig
require.NoError(t, json.NewDecoder(f).Decode(&out))
// The only interesting thing on bootstrap is the presence of the cluster,
// and its associated ID that Nomad sets. Everything is configured at runtime
// through xDS.
expID := fmt.Sprintf("_nomad-task-%s-group-web-my-ingress-service-9999", alloc.ID)
require.Equal(t, expID, out.Node.ID)
require.Equal(t, "ingress-gateway", out.Node.Cluster)
}
// TestTaskRunner_EnvoyBootstrapHook_Noop asserts that the Envoy bootstrap hook
// is a noop for non-Connect proxy sidecar / gateway tasks.
func TestTaskRunner_EnvoyBootstrapHook_Noop(t *testing.T) {
t.Parallel()
logger := testlog.HCLogger(t)
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
defer cleanup()
alloc := mock.Alloc()
task := alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks[0]
// Run Envoy bootstrap Hook. Use invalid Consul address as it should
// not get hit.
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
Addr: "http://127.0.0.2:1",
}, logger))
req := &interfaces.TaskPrestartRequest{
Task: task,
TaskDir: allocDir.NewTaskDir(task.Name),
}
require.NoError(t, req.TaskDir.Build(false, nil))
resp := &interfaces.TaskPrestartResponse{}
// Run the hook
require.NoError(t, h.Prestart(context.Background(), req, resp))
// Assert it is Done
require.True(t, resp.Done)
// Assert no file was written
_, err := os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
}
// TestTaskRunner_EnvoyBootstrapHook_RecoverableError asserts the Envoy
// bootstrap hook returns a Recoverable error if the bootstrap command runs but
// fails.
func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) {
t.Parallel()
testutil.RequireConsul(t)
testConsul := getTestConsul(t)
defer testConsul.Stop()
alloc := mock.ConnectAlloc()
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
{
Mode: "bridge",
IP: "10.0.0.1",
DynamicPorts: []structs.Port{
{
Label: "connect-proxy-foo",
Value: 9999,
To: 9999,
},
},
},
}
tg := alloc.Job.TaskGroups[0]
tg.Services = []*structs.Service{
{
Name: "foo",
PortLabel: "9999", // Just need a valid port, nothing will bind to it
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{},
},
},
}
sidecarTask := &structs.Task{
Name: "sidecar",
Kind: "connect-proxy:foo",
}
tg.Tasks = append(tg.Tasks, sidecarTask)
logger := testlog.HCLogger(t)
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
defer cleanup()
// Unlike the successful test above, do NOT register the group services
// yet. This should cause a recoverable error similar to if Consul was
// not running.
// Run Connect bootstrap Hook
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
Addr: testConsul.HTTPAddr,
}, logger))
req := &interfaces.TaskPrestartRequest{
Task: sidecarTask,
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, req.TaskDir.Build(false, nil))
resp := &interfaces.TaskPrestartResponse{}
// Run the hook
err := h.Prestart(context.Background(), req, resp)
require.EqualError(t, err, "error creating bootstrap configuration for Connect proxy sidecar: exit status 1")
require.True(t, structs.IsRecoverable(err))
// Assert it is not Done
require.False(t, resp.Done)
// Assert no file was written
_, err = os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
}
func TestTaskRunner_EnvoyBootstrapHook_extractNameAndKind(t *testing.T) {
t.Run("connect sidecar", func(t *testing.T) {
kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"),
)
require.Nil(t, err)
require.Equal(t, "connect-proxy", kind)
require.Equal(t, "foo", name)
})
t.Run("connect gateway", func(t *testing.T) {
kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
structs.NewTaskKind(structs.ConnectIngressPrefix, "foo"),
)
require.Nil(t, err)
require.Equal(t, "connect-ingress", kind)
require.Equal(t, "foo", name)
})
t.Run("connect native", func(t *testing.T) {
_, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
structs.NewTaskKind(structs.ConnectNativePrefix, "foo"),
)
require.EqualError(t, err, "envoy must be used as connect sidecar or gateway")
})
t.Run("normal task", func(t *testing.T) {
_, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
structs.TaskKind(""),
)
require.EqualError(t, err, "envoy must be used as connect sidecar or gateway")
})
}
func TestTaskRunner_EnvoyBootstrapHook_grpcAddress(t *testing.T) {
bridgeH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(
mock.ConnectIngressGatewayAlloc("bridge"),
new(config.ConsulConfig),
testlog.HCLogger(t),
))
hostH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(
mock.ConnectIngressGatewayAlloc("host"),
new(config.ConsulConfig),
testlog.HCLogger(t),
))
t.Run("environment", func(t *testing.T) {
env := map[string]string{
grpcConsulVariable: "1.2.3.4:9000",
}
require.Equal(t, "1.2.3.4:9000", bridgeH.grpcAddress(env))
require.Equal(t, "1.2.3.4:9000", hostH.grpcAddress(env))
})
t.Run("defaults", func(t *testing.T) {
require.Equal(t, "unix://alloc/tmp/consul_grpc.sock", bridgeH.grpcAddress(nil))
require.Equal(t, "127.0.0.1:8502", hostH.grpcAddress(nil))
})
}