5b072029f2
This PR adds initial support for running Consul Connect Ingress Gateways (CIGs) in Nomad. These gateways are declared as part of a task group level service definition within the connect stanza.
```hcl
service {
connect {
gateway {
proxy {
// envoy proxy configuration
}
ingress {
// ingress-gateway configuration entry
}
}
}
}
```
A gateway can be run in `bridge` or `host` networking mode, with the caveat that host networking necessitates manually specifying the Envoy admin listener (which cannot be disabled) via the service port value.
Currently Envoy is the only supported gateway implementation in Consul, and Nomad only supports running Envoy as a gateway using the docker driver.
Aims to address #8294 and tangentially #8647
695 lines
20 KiB
Go
695 lines
20 KiB
Go
// +build !windows
|
|
// todo(shoenig): Once Connect is supported on Windows, we'll need to make this
|
|
// set of tests work there too.
|
|
|
|
package taskrunner
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
consulapi "github.com/hashicorp/consul/api"
|
|
"github.com/hashicorp/nomad/client/allocdir"
|
|
"github.com/hashicorp/nomad/client/allocrunner/interfaces"
|
|
"github.com/hashicorp/nomad/client/taskenv"
|
|
"github.com/hashicorp/nomad/client/testutil"
|
|
agentconsul "github.com/hashicorp/nomad/command/agent/consul"
|
|
"github.com/hashicorp/nomad/helper"
|
|
"github.com/hashicorp/nomad/helper/args"
|
|
"github.com/hashicorp/nomad/helper/testlog"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/hashicorp/nomad/nomad/structs/config"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
var _ interfaces.TaskPrestartHook = (*envoyBootstrapHook)(nil)
|
|
|
|
func writeTmp(t *testing.T, s string, fm os.FileMode) string {
|
|
dir, err := ioutil.TempDir("", "envoy-")
|
|
require.NoError(t, err)
|
|
|
|
fPath := filepath.Join(dir, sidsTokenFile)
|
|
err = ioutil.WriteFile(fPath, []byte(s), fm)
|
|
require.NoError(t, err)
|
|
|
|
return dir
|
|
}
|
|
|
|
func TestEnvoyBootstrapHook_maybeLoadSIToken(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// This test fails when running as root because the test case for checking
|
|
// the error condition when the file is unreadable fails (root can read the
|
|
// file even though the permissions are set to 0200).
|
|
if unix.Geteuid() == 0 {
|
|
t.Skip("test only works as non-root")
|
|
}
|
|
|
|
t.Run("file does not exist", func(t *testing.T) {
|
|
h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
|
|
cfg, err := h.maybeLoadSIToken("task1", "/does/not/exist")
|
|
require.NoError(t, err) // absence of token is not an error
|
|
require.Equal(t, "", cfg)
|
|
})
|
|
|
|
t.Run("load token from file", func(t *testing.T) {
|
|
token := uuid.Generate()
|
|
f := writeTmp(t, token, 0440)
|
|
defer cleanupDir(t, f)
|
|
|
|
h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
|
|
cfg, err := h.maybeLoadSIToken("task1", f)
|
|
require.NoError(t, err)
|
|
require.Equal(t, token, cfg)
|
|
})
|
|
|
|
t.Run("file is unreadable", func(t *testing.T) {
|
|
token := uuid.Generate()
|
|
f := writeTmp(t, token, 0200)
|
|
defer cleanupDir(t, f)
|
|
|
|
h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
|
|
cfg, err := h.maybeLoadSIToken("task1", f)
|
|
require.Error(t, err)
|
|
require.False(t, os.IsNotExist(err))
|
|
require.Equal(t, "", cfg)
|
|
})
|
|
}
|
|
|
|
func TestEnvoyBootstrapHook_decodeTriState(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
require.Equal(t, "", decodeTriState(nil))
|
|
require.Equal(t, "true", decodeTriState(helper.BoolToPtr(true)))
|
|
require.Equal(t, "false", decodeTriState(helper.BoolToPtr(false)))
|
|
}
|
|
|
|
var (
|
|
consulPlainConfig = consulTransportConfig{
|
|
HTTPAddr: "2.2.2.2",
|
|
}
|
|
|
|
consulTLSConfig = consulTransportConfig{
|
|
HTTPAddr: "2.2.2.2", // arg
|
|
Auth: "user:password", // env
|
|
SSL: "true", // env
|
|
VerifySSL: "true", // env
|
|
CAFile: "/etc/tls/ca-file", // arg
|
|
CertFile: "/etc/tls/cert-file", // arg
|
|
KeyFile: "/etc/tls/key-file", // arg
|
|
}
|
|
)
|
|
|
|
func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("excluding SI token", func(t *testing.T) {
|
|
ebArgs := envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulPlainConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
}
|
|
result := ebArgs.args()
|
|
require.Equal(t, []string{"connect", "envoy",
|
|
"-grpc-addr", "1.1.1.1",
|
|
"-http-addr", "2.2.2.2",
|
|
"-admin-bind", "localhost:3333",
|
|
"-bootstrap",
|
|
"-sidecar-for", "s1",
|
|
}, result)
|
|
})
|
|
|
|
t.Run("including SI token", func(t *testing.T) {
|
|
token := uuid.Generate()
|
|
ebArgs := envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulPlainConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
siToken: token,
|
|
}
|
|
result := ebArgs.args()
|
|
require.Equal(t, []string{"connect", "envoy",
|
|
"-grpc-addr", "1.1.1.1",
|
|
"-http-addr", "2.2.2.2",
|
|
"-admin-bind", "localhost:3333",
|
|
"-bootstrap",
|
|
"-sidecar-for", "s1",
|
|
"-token", token,
|
|
}, result)
|
|
})
|
|
|
|
t.Run("including certificates", func(t *testing.T) {
|
|
ebArgs := envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulTLSConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
}
|
|
result := ebArgs.args()
|
|
require.Equal(t, []string{"connect", "envoy",
|
|
"-grpc-addr", "1.1.1.1",
|
|
"-http-addr", "2.2.2.2",
|
|
"-admin-bind", "localhost:3333",
|
|
"-bootstrap",
|
|
"-sidecar-for", "s1",
|
|
"-ca-file", "/etc/tls/ca-file",
|
|
"-client-cert", "/etc/tls/cert-file",
|
|
"-client-key", "/etc/tls/key-file",
|
|
}, result)
|
|
})
|
|
|
|
t.Run("ingress gateway", func(t *testing.T) {
|
|
ebArgs := envoyBootstrapArgs{
|
|
consulConfig: consulPlainConfig,
|
|
grpcAddr: "1.1.1.1",
|
|
envoyAdminBind: "localhost:3333",
|
|
gateway: "my-ingress-gateway",
|
|
}
|
|
result := ebArgs.args()
|
|
require.Equal(t, []string{"connect", "envoy",
|
|
"-grpc-addr", "1.1.1.1",
|
|
"-http-addr", "2.2.2.2",
|
|
"-admin-bind", "localhost:3333",
|
|
"-bootstrap",
|
|
"-gateway", "my-ingress-gateway",
|
|
}, result)
|
|
})
|
|
}
|
|
|
|
func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
environment := []string{"foo=bar", "baz=1"}
|
|
|
|
t.Run("plain consul config", func(t *testing.T) {
|
|
require.Equal(t, []string{
|
|
"foo=bar", "baz=1",
|
|
}, envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulPlainConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
}.env(environment))
|
|
})
|
|
|
|
t.Run("tls consul config", func(t *testing.T) {
|
|
require.Equal(t, []string{
|
|
"foo=bar", "baz=1",
|
|
"CONSUL_HTTP_AUTH=user:password",
|
|
"CONSUL_HTTP_SSL=true",
|
|
"CONSUL_HTTP_SSL_VERIFY=true",
|
|
}, envoyBootstrapArgs{
|
|
sidecarFor: "s1",
|
|
grpcAddr: "1.1.1.1",
|
|
consulConfig: consulTLSConfig,
|
|
envoyAdminBind: "localhost:3333",
|
|
}.env(environment))
|
|
})
|
|
}
|
|
|
|
// envoyConfig is used to unmarshal an envoy bootstrap configuration file, so that
|
|
// we can inspect the contents in tests.
|
|
type envoyConfig struct {
|
|
Admin struct {
|
|
Address struct {
|
|
SocketAddress struct {
|
|
Address string `json:"address"`
|
|
Port int `json:"port_value"`
|
|
} `json:"socket_address"`
|
|
} `json:"address"`
|
|
} `json:"admin"`
|
|
Node struct {
|
|
Cluster string `json:"cluster"`
|
|
Metadata struct {
|
|
Namespace string `json:"namespace"`
|
|
Version string `json:"envoy_version"`
|
|
}
|
|
}
|
|
DynamicResources struct {
|
|
ADSConfig struct {
|
|
GRPCServices struct {
|
|
InitialMetadata []struct {
|
|
Key string `json:"key"`
|
|
Value string `json:"value"`
|
|
} `json:"initial_metadata"`
|
|
} `json:"grpc_services"`
|
|
} `json:"ads_config"`
|
|
} `json:"dynamic_resources"`
|
|
}
|
|
|
|
// TestEnvoyBootstrapHook_with_SI_token asserts the bootstrap file written for
|
|
// Envoy contains a Consul SI token.
|
|
func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) {
|
|
t.Parallel()
|
|
testutil.RequireConsul(t)
|
|
|
|
testConsul := getTestConsul(t)
|
|
defer testConsul.Stop()
|
|
|
|
alloc := mock.ConnectAlloc()
|
|
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
|
|
{
|
|
Mode: "bridge",
|
|
IP: "10.0.0.1",
|
|
DynamicPorts: []structs.Port{
|
|
{
|
|
Label: "connect-proxy-foo",
|
|
Value: 9999,
|
|
To: 9999,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
tg := alloc.Job.TaskGroups[0]
|
|
tg.Services = []*structs.Service{
|
|
{
|
|
Name: "foo",
|
|
PortLabel: "9999", // Just need a valid port, nothing will bind to it
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{},
|
|
},
|
|
},
|
|
}
|
|
sidecarTask := &structs.Task{
|
|
Name: "sidecar",
|
|
Kind: "connect-proxy:foo",
|
|
}
|
|
tg.Tasks = append(tg.Tasks, sidecarTask)
|
|
|
|
logger := testlog.HCLogger(t)
|
|
|
|
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
|
|
defer cleanup()
|
|
|
|
// Register Group Services
|
|
consulConfig := consulapi.DefaultConfig()
|
|
consulConfig.Address = testConsul.HTTPAddr
|
|
consulAPIClient, err := consulapi.NewClient(consulConfig)
|
|
require.NoError(t, err)
|
|
|
|
consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
|
|
go consulClient.Run()
|
|
defer consulClient.Shutdown()
|
|
require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
|
|
|
|
// Run Connect bootstrap Hook
|
|
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
|
|
Addr: consulConfig.Address,
|
|
}, logger))
|
|
req := &interfaces.TaskPrestartRequest{
|
|
Task: sidecarTask,
|
|
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
|
|
TaskEnv: taskenv.NewEmptyTaskEnv(),
|
|
}
|
|
require.NoError(t, req.TaskDir.Build(false, nil))
|
|
|
|
// Insert service identity token in the secrets directory
|
|
token := uuid.Generate()
|
|
siTokenFile := filepath.Join(req.TaskDir.SecretsDir, sidsTokenFile)
|
|
err = ioutil.WriteFile(siTokenFile, []byte(token), 0440)
|
|
require.NoError(t, err)
|
|
|
|
resp := &interfaces.TaskPrestartResponse{}
|
|
|
|
// Run the hook
|
|
require.NoError(t, h.Prestart(context.Background(), req, resp))
|
|
|
|
// Assert it is Done
|
|
require.True(t, resp.Done)
|
|
|
|
// Ensure the default path matches
|
|
env := map[string]string{
|
|
taskenv.SecretsDir: req.TaskDir.SecretsDir,
|
|
}
|
|
f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
|
|
require.NoError(t, err)
|
|
defer f.Close()
|
|
|
|
// Assert bootstrap configuration is valid json
|
|
var out envoyConfig
|
|
require.NoError(t, json.NewDecoder(f).Decode(&out))
|
|
|
|
// Assert the SI token got set
|
|
key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
|
|
value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
|
|
require.Equal(t, "x-consul-token", key)
|
|
require.Equal(t, token, value)
|
|
}
|
|
|
|
// TestTaskRunner_EnvoyBootstrapHook_sidecar_ok asserts the EnvoyBootstrapHook
|
|
// creates Envoy's bootstrap.json configuration based on Connect proxy sidecars
|
|
// registered for the task.
|
|
func TestTaskRunner_EnvoyBootstrapHook_sidecar_ok(t *testing.T) {
|
|
t.Parallel()
|
|
testutil.RequireConsul(t)
|
|
|
|
testConsul := getTestConsul(t)
|
|
defer testConsul.Stop()
|
|
|
|
alloc := mock.ConnectAlloc()
|
|
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
|
|
{
|
|
Mode: "bridge",
|
|
IP: "10.0.0.1",
|
|
DynamicPorts: []structs.Port{
|
|
{
|
|
Label: "connect-proxy-foo",
|
|
Value: 9999,
|
|
To: 9999,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
tg := alloc.Job.TaskGroups[0]
|
|
tg.Services = []*structs.Service{
|
|
{
|
|
Name: "foo",
|
|
PortLabel: "9999", // Just need a valid port, nothing will bind to it
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{},
|
|
},
|
|
},
|
|
}
|
|
sidecarTask := &structs.Task{
|
|
Name: "sidecar",
|
|
Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"),
|
|
}
|
|
tg.Tasks = append(tg.Tasks, sidecarTask)
|
|
|
|
logger := testlog.HCLogger(t)
|
|
|
|
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
|
|
defer cleanup()
|
|
|
|
// Register Group Services
|
|
consulConfig := consulapi.DefaultConfig()
|
|
consulConfig.Address = testConsul.HTTPAddr
|
|
consulAPIClient, err := consulapi.NewClient(consulConfig)
|
|
require.NoError(t, err)
|
|
|
|
consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
|
|
go consulClient.Run()
|
|
defer consulClient.Shutdown()
|
|
require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
|
|
|
|
// Run Connect bootstrap Hook
|
|
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
|
|
Addr: consulConfig.Address,
|
|
}, logger))
|
|
req := &interfaces.TaskPrestartRequest{
|
|
Task: sidecarTask,
|
|
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
|
|
TaskEnv: taskenv.NewEmptyTaskEnv(),
|
|
}
|
|
require.NoError(t, req.TaskDir.Build(false, nil))
|
|
|
|
resp := &interfaces.TaskPrestartResponse{}
|
|
|
|
// Run the hook
|
|
require.NoError(t, h.Prestart(context.Background(), req, resp))
|
|
|
|
// Assert it is Done
|
|
require.True(t, resp.Done)
|
|
|
|
require.NotNil(t, resp.Env)
|
|
require.Equal(t, "localhost:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])
|
|
|
|
// Ensure the default path matches
|
|
env := map[string]string{
|
|
taskenv.SecretsDir: req.TaskDir.SecretsDir,
|
|
}
|
|
f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
|
|
require.NoError(t, err)
|
|
defer f.Close()
|
|
|
|
// Assert bootstrap configuration is valid json
|
|
var out envoyConfig
|
|
require.NoError(t, json.NewDecoder(f).Decode(&out))
|
|
|
|
// Assert no SI token got set
|
|
key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
|
|
value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
|
|
require.Equal(t, "x-consul-token", key)
|
|
require.Equal(t, "", value)
|
|
}
|
|
|
|
func TestTaskRunner_EnvoyBootstrapHook_gateway_ok(t *testing.T) {
|
|
t.Parallel()
|
|
logger := testlog.HCLogger(t)
|
|
|
|
testConsul := getTestConsul(t)
|
|
defer testConsul.Stop()
|
|
|
|
// Setup an Allocation
|
|
alloc := mock.ConnectIngressGatewayAlloc("bridge")
|
|
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyBootstrapIngressGateway")
|
|
defer cleanupDir()
|
|
|
|
// Get a Consul client
|
|
consulConfig := consulapi.DefaultConfig()
|
|
consulConfig.Address = testConsul.HTTPAddr
|
|
consulAPIClient, err := consulapi.NewClient(consulConfig)
|
|
require.NoError(t, err)
|
|
|
|
// Register Group Services
|
|
serviceClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
|
|
go serviceClient.Run()
|
|
defer serviceClient.Shutdown()
|
|
require.NoError(t, serviceClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
|
|
|
|
// Register Configuration Entry
|
|
ceClient := consulAPIClient.ConfigEntries()
|
|
set, _, err := ceClient.Set(&consulapi.IngressGatewayConfigEntry{
|
|
Kind: consulapi.IngressGateway,
|
|
Name: "gateway-service", // matches job
|
|
Listeners: []consulapi.IngressListener{{
|
|
Port: 2000,
|
|
Protocol: "tcp",
|
|
Services: []consulapi.IngressService{{
|
|
Name: "service1",
|
|
}},
|
|
}},
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
require.True(t, set)
|
|
|
|
// Run Connect bootstrap hook
|
|
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
|
|
Addr: consulConfig.Address,
|
|
}, logger))
|
|
|
|
req := &interfaces.TaskPrestartRequest{
|
|
Task: alloc.Job.TaskGroups[0].Tasks[0],
|
|
TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
|
|
TaskEnv: taskenv.NewEmptyTaskEnv(),
|
|
}
|
|
require.NoError(t, req.TaskDir.Build(false, nil))
|
|
|
|
var resp interfaces.TaskPrestartResponse
|
|
|
|
// Run the hook
|
|
require.NoError(t, h.Prestart(context.Background(), req, &resp))
|
|
|
|
// Assert the hook is done
|
|
require.True(t, resp.Done)
|
|
require.NotNil(t, resp.Env)
|
|
|
|
// Read the Envoy Config file
|
|
env := map[string]string{
|
|
taskenv.SecretsDir: req.TaskDir.SecretsDir,
|
|
}
|
|
f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
|
|
require.NoError(t, err)
|
|
defer f.Close()
|
|
|
|
var out envoyConfig
|
|
require.NoError(t, json.NewDecoder(f).Decode(&out))
|
|
|
|
// the only interesting thing on bootstrap is the presence of the cluster,
|
|
// everything is configured at runtime through xDS
|
|
require.Equal(t, "my-ingress-service", out.Node.Cluster)
|
|
}
|
|
|
|
// TestTaskRunner_EnvoyBootstrapHook_Noop asserts that the Envoy bootstrap hook
|
|
// is a noop for non-Connect proxy sidecar / gateway tasks.
|
|
func TestTaskRunner_EnvoyBootstrapHook_Noop(t *testing.T) {
|
|
t.Parallel()
|
|
logger := testlog.HCLogger(t)
|
|
|
|
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
|
|
defer cleanup()
|
|
|
|
alloc := mock.Alloc()
|
|
task := alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks[0]
|
|
|
|
// Run Envoy bootstrap Hook. Use invalid Consul address as it should
|
|
// not get hit.
|
|
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
|
|
Addr: "http://127.0.0.2:1",
|
|
}, logger))
|
|
req := &interfaces.TaskPrestartRequest{
|
|
Task: task,
|
|
TaskDir: allocDir.NewTaskDir(task.Name),
|
|
}
|
|
require.NoError(t, req.TaskDir.Build(false, nil))
|
|
|
|
resp := &interfaces.TaskPrestartResponse{}
|
|
|
|
// Run the hook
|
|
require.NoError(t, h.Prestart(context.Background(), req, resp))
|
|
|
|
// Assert it is Done
|
|
require.True(t, resp.Done)
|
|
|
|
// Assert no file was written
|
|
_, err := os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
|
|
require.Error(t, err)
|
|
require.True(t, os.IsNotExist(err))
|
|
}
|
|
|
|
// TestTaskRunner_EnvoyBootstrapHook_RecoverableError asserts the Envoy
|
|
// bootstrap hook returns a Recoverable error if the bootstrap command runs but
|
|
// fails.
|
|
func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) {
|
|
t.Parallel()
|
|
testutil.RequireConsul(t)
|
|
|
|
testConsul := getTestConsul(t)
|
|
defer testConsul.Stop()
|
|
|
|
alloc := mock.ConnectAlloc()
|
|
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
|
|
{
|
|
Mode: "bridge",
|
|
IP: "10.0.0.1",
|
|
DynamicPorts: []structs.Port{
|
|
{
|
|
Label: "connect-proxy-foo",
|
|
Value: 9999,
|
|
To: 9999,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
tg := alloc.Job.TaskGroups[0]
|
|
tg.Services = []*structs.Service{
|
|
{
|
|
Name: "foo",
|
|
PortLabel: "9999", // Just need a valid port, nothing will bind to it
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{},
|
|
},
|
|
},
|
|
}
|
|
sidecarTask := &structs.Task{
|
|
Name: "sidecar",
|
|
Kind: "connect-proxy:foo",
|
|
}
|
|
tg.Tasks = append(tg.Tasks, sidecarTask)
|
|
|
|
logger := testlog.HCLogger(t)
|
|
|
|
allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
|
|
defer cleanup()
|
|
|
|
// Unlike the successful test above, do NOT register the group services
|
|
// yet. This should cause a recoverable error similar to if Consul was
|
|
// not running.
|
|
|
|
// Run Connect bootstrap Hook
|
|
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
|
|
Addr: testConsul.HTTPAddr,
|
|
}, logger))
|
|
req := &interfaces.TaskPrestartRequest{
|
|
Task: sidecarTask,
|
|
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
|
|
TaskEnv: taskenv.NewEmptyTaskEnv(),
|
|
}
|
|
require.NoError(t, req.TaskDir.Build(false, nil))
|
|
|
|
resp := &interfaces.TaskPrestartResponse{}
|
|
|
|
// Run the hook
|
|
err := h.Prestart(context.Background(), req, resp)
|
|
require.EqualError(t, err, "error creating bootstrap configuration for Connect proxy sidecar: exit status 1")
|
|
require.True(t, structs.IsRecoverable(err))
|
|
|
|
// Assert it is not Done
|
|
require.False(t, resp.Done)
|
|
|
|
// Assert no file was written
|
|
_, err = os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
|
|
require.Error(t, err)
|
|
require.True(t, os.IsNotExist(err))
|
|
}
|
|
|
|
func TestTaskRunner_EnvoyBootstrapHook_extractNameAndKind(t *testing.T) {
|
|
t.Run("connect sidecar", func(t *testing.T) {
|
|
kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
|
|
structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"),
|
|
)
|
|
require.Nil(t, err)
|
|
require.Equal(t, "connect-proxy", kind)
|
|
require.Equal(t, "foo", name)
|
|
})
|
|
|
|
t.Run("connect gateway", func(t *testing.T) {
|
|
kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
|
|
structs.NewTaskKind(structs.ConnectIngressPrefix, "foo"),
|
|
)
|
|
require.Nil(t, err)
|
|
require.Equal(t, "connect-ingress", kind)
|
|
require.Equal(t, "foo", name)
|
|
})
|
|
|
|
t.Run("connect native", func(t *testing.T) {
|
|
_, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
|
|
structs.NewTaskKind(structs.ConnectNativePrefix, "foo"),
|
|
)
|
|
require.EqualError(t, err, "envoy must be used as connect sidecar or gateway")
|
|
})
|
|
|
|
t.Run("normal task", func(t *testing.T) {
|
|
_, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
|
|
structs.TaskKind(""),
|
|
)
|
|
require.EqualError(t, err, "envoy must be used as connect sidecar or gateway")
|
|
})
|
|
}
|
|
|
|
func TestTaskRunner_EnvoyBootstrapHook_grpcAddress(t *testing.T) {
|
|
bridgeH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(
|
|
mock.ConnectIngressGatewayAlloc("bridge"),
|
|
new(config.ConsulConfig),
|
|
testlog.HCLogger(t),
|
|
))
|
|
|
|
hostH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(
|
|
mock.ConnectIngressGatewayAlloc("host"),
|
|
new(config.ConsulConfig),
|
|
testlog.HCLogger(t),
|
|
))
|
|
|
|
t.Run("environment", func(t *testing.T) {
|
|
env := map[string]string{
|
|
grpcConsulVariable: "1.2.3.4:9000",
|
|
}
|
|
require.Equal(t, "1.2.3.4:9000", bridgeH.grpcAddress(env))
|
|
require.Equal(t, "1.2.3.4:9000", hostH.grpcAddress(env))
|
|
})
|
|
|
|
t.Run("defaults", func(t *testing.T) {
|
|
require.Equal(t, "unix://alloc/tmp/consul_grpc.sock", bridgeH.grpcAddress(nil))
|
|
require.Equal(t, "127.0.0.1:8502", hostH.grpcAddress(nil))
|
|
})
|
|
}
|