luxolus/open-nomad
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

Merge pull request #8709 from hashicorp/f-cc-ingress

Browse source 
consul/connect: add initial support for ingress gateways
This commit is contained in:
Seth Hoenig 2020-08-26 15:50:08 -05:00 committed by GitHub
parent 8c8b91e7b9 9f1f2a5673
commit 1dd2076263
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
43 changed files with 4615 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
.circleci/config.yml generated
View file

@ -58,10 +58,10 @@ jobs:
name: install protoc
- run:
command: |
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/consul.zip
rm -rf /tmp/consul*
name: Install Consul 1.6.4
name: Install Consul 1.8.3
- run:
command: |
set -x
@ -176,10 +176,10 @@ jobs:
name: install protoc
- run:
command: |
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/consul.zip
rm -rf /tmp/consul*
name: Install Consul 1.6.4
name: Install Consul 1.8.3
- run:
command: |
set -x
@ -294,10 +294,10 @@ jobs:
name: install protoc
- run:
command: |
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/consul.zip
rm -rf /tmp/consul*
name: Install Consul 1.6.4
name: Install Consul 1.8.3
- run:
command: |
set -x
@ -412,10 +412,10 @@ jobs:
name: install protoc
- run:
command: |
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/consul.zip
rm -rf /tmp/consul*
name: Install Consul 1.6.4
name: Install Consul 1.8.3
- run:
command: |
set -x
@ -635,10 +635,10 @@ jobs:
name: install protoc
- run:
command: |
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/consul.zip
rm -rf /tmp/consul*
name: Install Consul 1.6.4
name: Install Consul 1.8.3
- run:
command: |
set -x
@ -846,10 +846,10 @@ jobs:
name: install protoc
- run:
command: |
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/consul.zip
rm -rf /tmp/consul*
name: Install Consul 1.6.4
name: Install Consul 1.8.3
- run:
command: |
set -x
@ -938,10 +938,10 @@ jobs:
name: install protoc
- run:
command: |
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/consul.zip
rm -rf /tmp/consul*
name: Install Consul 1.6.4
name: Install Consul 1.8.3
- run:
command: |
set -x
@ -1118,10 +1118,10 @@ jobs:
name: install protoc
- run:
command: |
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/consul.zip
rm -rf /tmp/consul*
name: Install Consul 1.6.4
name: Install Consul 1.8.3
- run:
command: |
set -x
@ -1298,10 +1298,10 @@ jobs:
name: install protoc
- run:
command: |
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip
curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/consul.zip
rm -rf /tmp/consul*
name: Install Consul 1.6.4
name: Install Consul 1.8.3
- run:
command: |
set -x

2
.circleci/config/commands/install-consul.yml
View file

@ -1,7 +1,7 @@
parameters:
version:
type: string
default: 1.6.4
default: 1.8.3
steps:
- run:
name: Install Consul << parameters.version >>

262
api/services.go
View file

@ -152,6 +152,7 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) {
// ConsulConnect represents a Consul Connect jobspec stanza.
type ConsulConnect struct {
Native bool
Gateway *ConsulGateway
SidecarService *ConsulSidecarService `mapstructure:"sidecar_service"`
SidecarTask *SidecarTask `mapstructure:"sidecar_task"`
}
@ -163,6 +164,7 @@ func (cc *ConsulConnect) Canonicalize() {
cc.SidecarService.Canonicalize()
cc.SidecarTask.Canonicalize()
cc.Gateway.Canonicalize()
}
// ConsulSidecarService represents a Consul Connect SidecarService jobspec
@ -290,3 +292,263 @@ type ConsulExposePath struct {
LocalPathPort int `mapstructure:"local_path_port"`
ListenerPort string `mapstructure:"listener_port"`
}
// ConsulGateway is used to configure one of the Consul Connect Gateway types.
type ConsulGateway struct {
// Proxy is used to configure the Envoy instance acting as the gateway.
Proxy *ConsulGatewayProxy
// Ingress represents the Consul Configuration Entry for an Ingress Gateway.
Ingress *ConsulIngressConfigEntry
// Terminating is not yet supported.
// Terminating *ConsulTerminatingConfigEntry
// Mesh is not yet supported.
// Mesh *ConsulMeshConfigEntry
}
func (g *ConsulGateway) Canonicalize() {
if g == nil {
return
}
g.Proxy.Canonicalize()
g.Ingress.Canonicalize()
}
func (g *ConsulGateway) Copy() *ConsulGateway {
if g == nil {
return nil
}
return &ConsulGateway{
Proxy: g.Proxy.Copy(),
Ingress: g.Ingress.Copy(),
}
}
type ConsulGatewayBindAddress struct {
Address string `mapstructure:"address"`
Port int `mapstructure:"port"`
}
var (
defaultGatewayConnectTimeout = 5 * time.Second
)
// ConsulGatewayProxy is used to tune parameters of the proxy instance acting as
// one of the forms of Connect gateways that Consul supports.
//
// https://www.consul.io/docs/connect/proxies/envoy#gateway-options
type ConsulGatewayProxy struct {
ConnectTimeout *time.Duration `mapstructure:"connect_timeout"`
EnvoyGatewayBindTaggedAddresses bool `mapstructure:"envoy_gateway_bind_tagged_addresses"`
EnvoyGatewayBindAddresses map[string]*ConsulGatewayBindAddress `mapstructure:"envoy_gateway_bind_addresses"`
EnvoyGatewayNoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind"`
Config map[string]interface{} // escape hatch envoy config
}
func (p *ConsulGatewayProxy) Canonicalize() {
if p == nil {
return
}
if p.ConnectTimeout == nil {
// same as the default from consul
p.ConnectTimeout = timeToPtr(defaultGatewayConnectTimeout)
}
if len(p.EnvoyGatewayBindAddresses) == 0 {
p.EnvoyGatewayBindAddresses = nil
}
if len(p.Config) == 0 {
p.Config = nil
}
}
func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy {
if p == nil {
return nil
}
var binds map[string]*ConsulGatewayBindAddress = nil
if p.EnvoyGatewayBindAddresses != nil {
binds = make(map[string]*ConsulGatewayBindAddress, len(p.EnvoyGatewayBindAddresses))
for k, v := range p.EnvoyGatewayBindAddresses {
binds[k] = v
}
}
var config map[string]interface{} = nil
if p.Config != nil {
config = make(map[string]interface{}, len(p.Config))
for k, v := range p.Config {
config[k] = v
}
}
return &ConsulGatewayProxy{
ConnectTimeout: timeToPtr(*p.ConnectTimeout),
EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses,
EnvoyGatewayBindAddresses: binds,
EnvoyGatewayNoDefaultBind: p.EnvoyGatewayNoDefaultBind,
Config: config,
}
}
// ConsulGatewayTLSConfig is used to configure TLS for a gateway.
type ConsulGatewayTLSConfig struct {
Enabled bool
}
func (tc *ConsulGatewayTLSConfig) Canonicalize() {
}
func (tc *ConsulGatewayTLSConfig) Copy() *ConsulGatewayTLSConfig {
if tc == nil {
return nil
}
return &ConsulGatewayTLSConfig{
Enabled: tc.Enabled,
}
}
// ConsulIngressService is used to configure a service fronted by the ingress gateway.
type ConsulIngressService struct {
// Namespace is not yet supported.
// Namespace string
Name string
Hosts []string
}
func (s *ConsulIngressService) Canonicalize() {
if s == nil {
return
}
if len(s.Hosts) == 0 {
s.Hosts = nil
}
}
func (s *ConsulIngressService) Copy() *ConsulIngressService {
if s == nil {
return nil
}
var hosts []string = nil
if n := len(s.Hosts); n > 0 {
hosts = make([]string, n)
copy(hosts, s.Hosts)
}
return &ConsulIngressService{
Name: s.Name,
Hosts: hosts,
}
}
const (
defaultIngressListenerProtocol = "tcp"
)
// ConsulIngressListener is used to configure a listener on a Consul Ingress
// Gateway.
type ConsulIngressListener struct {
Port int
Protocol string
Services []*ConsulIngressService
}
func (l *ConsulIngressListener) Canonicalize() {
if l == nil {
return
}
if l.Protocol == "" {
// same as default from consul
l.Protocol = defaultIngressListenerProtocol
}
if len(l.Services) == 0 {
l.Services = nil
}
}
func (l *ConsulIngressListener) Copy() *ConsulIngressListener {
if l == nil {
return nil
}
var services []*ConsulIngressService = nil
if n := len(l.Services); n > 0 {
services = make([]*ConsulIngressService, n)
for i := 0; i < n; i++ {
services[i] = l.Services[i].Copy()
}
}
return &ConsulIngressListener{
Port: l.Port,
Protocol: l.Protocol,
Services: services,
}
}
// ConsulIngressConfigEntry represents the Consul Configuration Entry type for
// an Ingress Gateway.
//
// https://www.consul.io/docs/agent/config-entries/ingress-gateway#available-fields
type ConsulIngressConfigEntry struct {
// Namespace is not yet supported.
// Namespace string
TLS *ConsulGatewayTLSConfig
Listeners []*ConsulIngressListener
}
func (e *ConsulIngressConfigEntry) Canonicalize() {
if e == nil {
return
}
e.TLS.Canonicalize()
if len(e.Listeners) == 0 {
e.Listeners = nil
}
for _, listener := range e.Listeners {
listener.Canonicalize()
}
}
func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry {
if e == nil {
return nil
}
var listeners []*ConsulIngressListener = nil
if n := len(e.Listeners); n > 0 {
listeners = make([]*ConsulIngressListener, n)
for i := 0; i < n; i++ {
listeners[i] = e.Listeners[i].Copy()
}
}
return &ConsulIngressConfigEntry{
TLS: e.TLS.Copy(),
Listeners: listeners,
}
}
// ConsulTerminatingConfigEntry is not yet supported.
// type ConsulTerminatingConfigEntry struct {
// }
// ConsulMeshConfigEntry is not yet supported.
// type ConsulMeshConfigEntry struct {
// }

155
api/services_test.go
View file

@ -261,3 +261,158 @@ func TestService_Connect_SidecarTask_Canonicalize(t *testing.T) {
require.Equal(t, exp, st.Resources)
})
}
func TestService_ConsulGateway_Canonicalize(t *testing.T) {
t.Parallel()
t.Run("nil", func(t *testing.T) {
cg := (*ConsulGateway)(nil)
cg.Canonicalize()
require.Nil(t, cg)
})
t.Run("set defaults", func(t *testing.T) {
cg := &ConsulGateway{
Proxy: &ConsulGatewayProxy{
ConnectTimeout: nil,
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayBindAddresses: make(map[string]*ConsulGatewayBindAddress, 0),
EnvoyGatewayNoDefaultBind: true,
Config: make(map[string]interface{}, 0),
},
Ingress: &ConsulIngressConfigEntry{
TLS: &ConsulGatewayTLSConfig{
Enabled: false,
},
Listeners: make([]*ConsulIngressListener, 0),
},
}
cg.Canonicalize()
require.Equal(t, timeToPtr(5*time.Second), cg.Proxy.ConnectTimeout)
require.Nil(t, cg.Proxy.EnvoyGatewayBindAddresses)
require.Nil(t, cg.Proxy.Config)
require.Nil(t, cg.Ingress.Listeners)
})
}
func TestService_ConsulGateway_Copy(t *testing.T) {
t.Parallel()
t.Run("nil", func(t *testing.T) {
result := (*ConsulGateway)(nil).Copy()
require.Nil(t, result)
})
gateway := &ConsulGateway{
Proxy: &ConsulGatewayProxy{
ConnectTimeout: timeToPtr(3 * time.Second),
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{
"listener1": {Address: "10.0.0.1", Port: 2000},
"listener2": {Address: "10.0.0.1", Port: 2001},
},
EnvoyGatewayNoDefaultBind: true,
Config: map[string]interface{}{
"foo": "bar",
"baz": 3,
},
},
Ingress: &ConsulIngressConfigEntry{
TLS: &ConsulGatewayTLSConfig{
Enabled: true,
},
Listeners: []*ConsulIngressListener{{
Port: 3333,
Protocol: "tcp",
Services: []*ConsulIngressService{{
Name: "service1",
Hosts: []string{
"127.0.0.1", "127.0.0.1:3333",
}},
}},
},
},
}
t.Run("complete", func(t *testing.T) {
result := gateway.Copy()
require.Equal(t, gateway, result)
})
}
func TestService_ConsulIngressConfigEntry_Canonicalize(t *testing.T) {
t.Parallel()
t.Run("nil", func(t *testing.T) {
c := (*ConsulIngressConfigEntry)(nil)
c.Canonicalize()
require.Nil(t, c)
})
t.Run("empty fields", func(t *testing.T) {
c := &ConsulIngressConfigEntry{
TLS: nil,
Listeners: []*ConsulIngressListener{},
}
c.Canonicalize()
require.Nil(t, c.TLS)
require.Nil(t, c.Listeners)
})
t.Run("complete", func(t *testing.T) {
c := &ConsulIngressConfigEntry{
TLS: &ConsulGatewayTLSConfig{Enabled: true},
Listeners: []*ConsulIngressListener{{
Port: 9090,
Protocol: "http",
Services: []*ConsulIngressService{{
Name: "service1",
Hosts: []string{"1.1.1.1"},
}},
}},
}
c.Canonicalize()
require.Equal(t, &ConsulIngressConfigEntry{
TLS: &ConsulGatewayTLSConfig{Enabled: true},
Listeners: []*ConsulIngressListener{{
Port: 9090,
Protocol: "http",
Services: []*ConsulIngressService{{
Name: "service1",
Hosts: []string{"1.1.1.1"},
}},
}},
}, c)
})
}
func TestService_ConsulIngressConfigEntry_Copy(t *testing.T) {
t.Parallel()
t.Run("nil", func(t *testing.T) {
result := (*ConsulIngressConfigEntry)(nil).Copy()
require.Nil(t, result)
})
entry := &ConsulIngressConfigEntry{
TLS: &ConsulGatewayTLSConfig{
Enabled: true,
},
Listeners: []*ConsulIngressListener{{
Port: 1111,
Protocol: "http",
Services: []*ConsulIngressService{{
Name: "service1",
Hosts: []string{"1.1.1.1", "1.1.1.1:9000"},
}, {
Name: "service2",
Hosts: []string{"2.2.2.2"},
}},
}},
}
t.Run("complete", func(t *testing.T) {
result := entry.Copy()
require.Equal(t, entry, result)
})
}

2
client/allocrunner/consul_grpc_sock_hook.go
View file

@ -68,7 +68,7 @@ func (h *consulGRPCSocketHook) shouldRun() bool {
}
for _, s := range tg.Services {
if s.Connect.HasSidecar() {
if s.Connect.HasSidecar() || s.Connect.IsGateway() {
return true
}
}

181
client/allocrunner/taskrunner/envoybootstrap_hook.go
View file

@ -75,6 +75,11 @@ const (
envoyAdminBindEnvPrefix = "NOMAD_ENVOY_ADMIN_ADDR_"
)
const (
grpcConsulVariable = "CONSUL_GRPC_ADDR"
grpcDefaultAddress = "127.0.0.1:8502"
)
// envoyBootstrapHook writes the bootstrap config for the Connect Envoy proxy
// sidecar.
type envoyBootstrapHook struct {
@ -103,41 +108,73 @@ func (envoyBootstrapHook) Name() string {
return envoyBootstrapHookName
}
func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
if !req.Task.Kind.IsConnectProxy() {
// Not a Connect proxy sidecar
resp.Done = true
return nil
func (_ *envoyBootstrapHook) extractNameAndKind(kind structs.TaskKind) (string, string, error) {
serviceKind := kind.Name()
serviceName := kind.Value()
switch serviceKind {
case structs.ConnectProxyPrefix, structs.ConnectIngressPrefix:
default:
return "", "", errors.New("envoy must be used as connect sidecar or gateway")
}
serviceName := req.Task.Kind.Value()
if serviceName == "" {
return errors.New("connect proxy sidecar does not specify service name")
return "", "", errors.New("envoy must be configured with a service name")
}
return serviceKind, serviceName, nil
}
func (h *envoyBootstrapHook) lookupService(svcKind, svcName, tgName string) (*structs.Service, error) {
tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup)
var service *structs.Service
for _, s := range tg.Services {
if s.Name == serviceName {
if s.Name == svcName {
service = s
break
}
}
if service == nil {
return errors.New("connect proxy sidecar task exists but no services configured with a sidecar")
if svcKind == structs.ConnectProxyPrefix {
return nil, errors.New("connect proxy sidecar task exists but no services configured with a sidecar")
} else {
return nil, errors.New("connect gateway task exists but no service associated")
}
}
h.logger.Debug("bootstrapping Connect proxy sidecar", "task", req.Task.Name, "service", serviceName)
return service, nil
}
//TODO Should connect directly to Consul if the sidecar is running on the host netns.
grpcAddr := "unix://" + allocdir.AllocGRPCSocket
// Prestart creates an envoy bootstrap config file.
//
// Must be aware of both launching envoy as a sidecar proxy, as well as a connect gateway.
func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
if !req.Task.Kind.IsConnectProxy() && !req.Task.Kind.IsAnyConnectGateway() {
// Not a Connect proxy sidecar
resp.Done = true
return nil
}
// Envoy runs an administrative API on the loopback interface. If multiple sidecars
// are running, the bind addresses need to have unique ports.
// TODO: support running in host netns, using freeport to find available port
envoyAdminBind := buildEnvoyAdminBind(h.alloc, req.Task.Name)
serviceKind, serviceName, err := h.extractNameAndKind(req.Task.Kind)
if err != nil {
return err
}
service, err := h.lookupService(serviceKind, serviceName, h.alloc.TaskGroup)
if err != nil {
return err
}
grpcAddr := h.grpcAddress(req.TaskEnv.EnvMap)
h.logger.Debug("bootstrapping Consul "+serviceKind, "task", req.Task.Name, "service", serviceName)
// Envoy runs an administrative API on the loopback interface. There is no
// way to turn this feature off.
// https://github.com/envoyproxy/envoy/issues/1297
envoyAdminBind := buildEnvoyAdminBind(h.alloc, serviceName, req.Task.Name)
resp.Env = map[string]string{
helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_'): envoyAdminBind,
}
@ -146,10 +183,6 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP
// it to the secrets directory like Vault tokens.
bootstrapFilePath := filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json")
id := agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+tg.Name, service)
h.logger.Debug("bootstrapping envoy", "sidecar_for", service.Name, "bootstrap_file", bootstrapFilePath, "sidecar_for_id", id, "grpc_addr", grpcAddr, "admin_bind", envoyAdminBind)
siToken, err := h.maybeLoadSIToken(req.Task.Name, req.TaskDir.SecretsDir)
if err != nil {
h.logger.Error("failed to generate envoy bootstrap config", "sidecar_for", service.Name)
@ -157,16 +190,9 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP
}
h.logger.Debug("check for SI token for task", "task", req.Task.Name, "exists", siToken != "")
bootstrapBuilder := envoyBootstrapArgs{
consulConfig: h.consulConfig,
sidecarFor: id,
grpcAddr: grpcAddr,
envoyAdminBind: envoyAdminBind,
siToken: siToken,
}
bootstrapArgs := bootstrapBuilder.args()
bootstrapEnv := bootstrapBuilder.env(os.Environ())
bootstrap := h.newEnvoyBootstrapArgs(h.alloc.TaskGroup, service, grpcAddr, envoyAdminBind, siToken, bootstrapFilePath)
bootstrapArgs := bootstrap.args()
bootstrapEnv := bootstrap.env(os.Environ())
// Since Consul services are registered asynchronously with this task
// hook running, retry a small number of times with backoff.
@ -231,12 +257,32 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP
return nil
}
func buildEnvoyAdminBind(alloc *structs.Allocation, taskName string) string {
// buildEnvoyAdminBind determines a unique port for use by the envoy admin
// listener.
//
// In bridge mode, if multiple sidecars are running, the bind addresses need
// to be unique within the namespace, so we simply start at 19000 and increment
// by the index of the task.
//
// In host mode, use the port provided through the service definition, which can
// be a port chosen by Nomad.
func buildEnvoyAdminBind(alloc *structs.Allocation, serviceName, taskName string) string {
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
port := envoyBaseAdminPort
for idx, task := range alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks {
if task.Name == taskName {
port += idx
break
switch tg.Networks[0].Mode {
case "host":
for _, service := range tg.Services {
if service.Name == serviceName {
_, port = tg.Networks.Port(service.PortLabel)
break
}
}
default:
for idx, task := range tg.Tasks {
if task.Name == taskName {
port += idx
break
}
}
}
return fmt.Sprintf("localhost:%d", port)
@ -269,15 +315,69 @@ func (h *envoyBootstrapHook) execute(cmd *exec.Cmd) (string, error) {
return stdout.String(), nil
}
// grpcAddress determines the Consul gRPC endpoint address to use.
//
// In host networking this will default to 127.0.0.1:8502.
// In bridge/cni networking this will default to unix://<socket>.
// In either case, CONSUL_GRPC_ADDR will override the default.
func (h *envoyBootstrapHook) grpcAddress(env map[string]string) string {
if address := env[grpcConsulVariable]; address != "" {
return address
}
tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup)
switch tg.Networks[0].Mode {
case "host":
return grpcDefaultAddress
default:
return "unix://" + allocdir.AllocGRPCSocket
}
}
func (h *envoyBootstrapHook) newEnvoyBootstrapArgs(
tgName string,
service *structs.Service,
grpcAddr, envoyAdminBind, siToken, filepath string,
) envoyBootstrapArgs {
var (
sidecarForID string // sidecar only
gateway string // gateway only
)
if service.Connect.HasSidecar() {
sidecarForID = agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+tgName, service)
}
if service.Connect.IsGateway() {
gateway = "ingress" // more types in the future
}
h.logger.Debug("bootstrapping envoy",
"sidecar_for", service.Name, "bootstrap_file", filepath,
"sidecar_for_id", sidecarForID, "grpc_addr", grpcAddr,
"admin_bind", envoyAdminBind, "gateway", gateway,
)
return envoyBootstrapArgs{
consulConfig: h.consulConfig,
sidecarFor: sidecarForID,
grpcAddr: grpcAddr,
envoyAdminBind: envoyAdminBind,
siToken: siToken,
gateway: gateway,
}
}
// envoyBootstrapArgs is used to accumulate CLI arguments that will be passed
// along to the exec invocation of consul which will then generate the bootstrap
// configuration file for envoy.
type envoyBootstrapArgs struct {
consulConfig consulTransportConfig
sidecarFor string
sidecarFor string // sidecars only
grpcAddr string
envoyAdminBind string
siToken string
gateway string // gateways only
}
// args returns the CLI arguments consul needs in the correct order, with the
@ -290,7 +390,14 @@ func (e envoyBootstrapArgs) args() []string {
"-http-addr", e.consulConfig.HTTPAddr,
"-admin-bind", e.envoyAdminBind,
"-bootstrap",
"-sidecar-for", e.sidecarFor,
}
if v := e.sidecarFor; v != "" {
arguments = append(arguments, "-sidecar-for", e.sidecarFor)
}
if v := e.gateway; v != "" {
arguments = append(arguments, "-gateway", e.gateway)
}
if v := e.siToken; v != "" {

208
client/allocrunner/taskrunner/envoybootstrap_hook_test.go
View file

@ -166,6 +166,23 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
"-client-key", "/etc/tls/key-file",
}, result)
})
t.Run("ingress gateway", func(t *testing.T) {
ebArgs := envoyBootstrapArgs{
consulConfig: consulPlainConfig,
grpcAddr: "1.1.1.1",
envoyAdminBind: "localhost:3333",
gateway: "my-ingress-gateway",
}
result := ebArgs.args()
require.Equal(t, []string{"connect", "envoy",
"-grpc-addr", "1.1.1.1",
"-http-addr", "2.2.2.2",
"-admin-bind", "localhost:3333",
"-bootstrap",
"-gateway", "my-ingress-gateway",
}, result)
})
}
func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) {
@ -199,8 +216,24 @@ func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) {
})
}
// dig through envoy config to look for consul token
// envoyConfig is used to unmarshal an envoy bootstrap configuration file, so that
// we can inspect the contents in tests.
type envoyConfig struct {
Admin struct {
Address struct {
SocketAddress struct {
Address string `json:"address"`
Port int `json:"port_value"`
} `json:"socket_address"`
} `json:"address"`
} `json:"admin"`
Node struct {
Cluster string `json:"cluster"`
Metadata struct {
Namespace string `json:"namespace"`
Version string `json:"envoy_version"`
}
}
DynamicResources struct {
ADSConfig struct {
GRPCServices struct {
@ -219,10 +252,10 @@ func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) {
t.Parallel()
testutil.RequireConsul(t)
testconsul := getTestConsul(t)
defer testconsul.Stop()
testConsul := getTestConsul(t)
defer testConsul.Stop()
alloc := mock.Alloc()
alloc := mock.ConnectAlloc()
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
{
Mode: "bridge",
@ -259,7 +292,7 @@ func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) {
// Register Group Services
consulConfig := consulapi.DefaultConfig()
consulConfig.Address = testconsul.HTTPAddr
consulConfig.Address = testConsul.HTTPAddr
consulAPIClient, err := consulapi.NewClient(consulConfig)
require.NoError(t, err)
@ -275,6 +308,7 @@ func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) {
req := &interfaces.TaskPrestartRequest{
Task: sidecarTask,
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, req.TaskDir.Build(false, nil))
@ -311,17 +345,17 @@ func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) {
require.Equal(t, token, value)
}
// TestTaskRunner_EnvoyBootstrapHook_Prestart asserts the EnvoyBootstrapHook
// TestTaskRunner_EnvoyBootstrapHook_sidecar_ok asserts the EnvoyBootstrapHook
// creates Envoy's bootstrap.json configuration based on Connect proxy sidecars
// registered for the task.
func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) {
func TestTaskRunner_EnvoyBootstrapHook_sidecar_ok(t *testing.T) {
t.Parallel()
testutil.RequireConsul(t)
testconsul := getTestConsul(t)
defer testconsul.Stop()
testConsul := getTestConsul(t)
defer testConsul.Stop()
alloc := mock.Alloc()
alloc := mock.ConnectAlloc()
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
{
Mode: "bridge",
@ -347,7 +381,7 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) {
}
sidecarTask := &structs.Task{
Name: "sidecar",
Kind: "connect-proxy:foo",
Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"),
}
tg.Tasks = append(tg.Tasks, sidecarTask)
@ -358,7 +392,7 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) {
// Register Group Services
consulConfig := consulapi.DefaultConfig()
consulConfig.Address = testconsul.HTTPAddr
consulConfig.Address = testConsul.HTTPAddr
consulAPIClient, err := consulapi.NewClient(consulConfig)
require.NoError(t, err)
@ -374,6 +408,7 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) {
req := &interfaces.TaskPrestartRequest{
Task: sidecarTask,
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, req.TaskDir.Build(false, nil))
@ -407,8 +442,85 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) {
require.Equal(t, "", value)
}
func TestTaskRunner_EnvoyBootstrapHook_gateway_ok(t *testing.T) {
t.Parallel()
logger := testlog.HCLogger(t)
testConsul := getTestConsul(t)
defer testConsul.Stop()
// Setup an Allocation
alloc := mock.ConnectIngressGatewayAlloc("bridge")
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyBootstrapIngressGateway")
defer cleanupDir()
// Get a Consul client
consulConfig := consulapi.DefaultConfig()
consulConfig.Address = testConsul.HTTPAddr
consulAPIClient, err := consulapi.NewClient(consulConfig)
require.NoError(t, err)
// Register Group Services
serviceClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
go serviceClient.Run()
defer serviceClient.Shutdown()
require.NoError(t, serviceClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
// Register Configuration Entry
ceClient := consulAPIClient.ConfigEntries()
set, _, err := ceClient.Set(&consulapi.IngressGatewayConfigEntry{
Kind: consulapi.IngressGateway,
Name: "gateway-service", // matches job
Listeners: []consulapi.IngressListener{{
Port: 2000,
Protocol: "tcp",
Services: []consulapi.IngressService{{
Name: "service1",
}},
}},
}, nil)
require.NoError(t, err)
require.True(t, set)
// Run Connect bootstrap hook
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
Addr: consulConfig.Address,
}, logger))
req := &interfaces.TaskPrestartRequest{
Task: alloc.Job.TaskGroups[0].Tasks[0],
TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, req.TaskDir.Build(false, nil))
var resp interfaces.TaskPrestartResponse
// Run the hook
require.NoError(t, h.Prestart(context.Background(), req, &resp))
// Assert the hook is done
require.True(t, resp.Done)
require.NotNil(t, resp.Env)
// Read the Envoy Config file
env := map[string]string{
taskenv.SecretsDir: req.TaskDir.SecretsDir,
}
f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
require.NoError(t, err)
defer f.Close()
var out envoyConfig
require.NoError(t, json.NewDecoder(f).Decode(&out))
// the only interesting thing on bootstrap is the presence of the cluster,
// everything is configured at runtime through xDS
require.Equal(t, "my-ingress-service", out.Node.Cluster)
}
// TestTaskRunner_EnvoyBootstrapHook_Noop asserts that the Envoy bootstrap hook
// is a noop for non-Connect proxy sidecar tasks.
// is a noop for non-Connect proxy sidecar / gateway tasks.
func TestTaskRunner_EnvoyBootstrapHook_Noop(t *testing.T) {
t.Parallel()
logger := testlog.HCLogger(t)
@ -451,10 +563,10 @@ func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) {
t.Parallel()
testutil.RequireConsul(t)
testconsul := getTestConsul(t)
defer testconsul.Stop()
testConsul := getTestConsul(t)
defer testConsul.Stop()
alloc := mock.Alloc()
alloc := mock.ConnectAlloc()
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
{
Mode: "bridge",
@ -495,11 +607,12 @@ func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) {
// Run Connect bootstrap Hook
h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
Addr: testconsul.HTTPAddr,
Addr: testConsul.HTTPAddr,
}, logger))
req := &interfaces.TaskPrestartRequest{
Task: sidecarTask,
TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
TaskEnv: taskenv.NewEmptyTaskEnv(),
}
require.NoError(t, req.TaskDir.Build(false, nil))
@ -518,3 +631,64 @@ func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) {
require.Error(t, err)
require.True(t, os.IsNotExist(err))
}
func TestTaskRunner_EnvoyBootstrapHook_extractNameAndKind(t *testing.T) {
t.Run("connect sidecar", func(t *testing.T) {
kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"),
)
require.Nil(t, err)
require.Equal(t, "connect-proxy", kind)
require.Equal(t, "foo", name)
})
t.Run("connect gateway", func(t *testing.T) {
kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
structs.NewTaskKind(structs.ConnectIngressPrefix, "foo"),
)
require.Nil(t, err)
require.Equal(t, "connect-ingress", kind)
require.Equal(t, "foo", name)
})
t.Run("connect native", func(t *testing.T) {
_, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
structs.NewTaskKind(structs.ConnectNativePrefix, "foo"),
)
require.EqualError(t, err, "envoy must be used as connect sidecar or gateway")
})
t.Run("normal task", func(t *testing.T) {
_, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
structs.TaskKind(""),
)
require.EqualError(t, err, "envoy must be used as connect sidecar or gateway")
})
}
func TestTaskRunner_EnvoyBootstrapHook_grpcAddress(t *testing.T) {
bridgeH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(
mock.ConnectIngressGatewayAlloc("bridge"),
new(config.ConsulConfig),
testlog.HCLogger(t),
))
hostH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(
mock.ConnectIngressGatewayAlloc("host"),
new(config.ConsulConfig),
testlog.HCLogger(t),
))
t.Run("environment", func(t *testing.T) {
env := map[string]string{
grpcConsulVariable: "1.2.3.4:9000",
}
require.Equal(t, "1.2.3.4:9000", bridgeH.grpcAddress(env))
require.Equal(t, "1.2.3.4:9000", hostH.grpcAddress(env))
})
t.Run("defaults", func(t *testing.T) {
require.Equal(t, "unix://alloc/tmp/consul_grpc.sock", bridgeH.grpcAddress(nil))
require.Equal(t, "127.0.0.1:8502", hostH.grpcAddress(nil))
})
}

2
client/allocrunner/taskrunner/task_runner_hooks.go
View file

@ -127,7 +127,7 @@ func (tr *TaskRunner) initHooks() {
}))
}
if task.Kind.IsConnectProxy() {
if task.Kind.IsConnectProxy() || task.Kind.IsAnyConnectGateway() {
tr.runnerHooks = append(tr.runnerHooks, newEnvoyBootstrapHook(
newEnvoyBootstrapHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger),
))

4
client/allocrunner/taskrunner/template/template_test.go
View file

@ -154,7 +154,9 @@ func newTestHarness(t *testing.T, templates []*structs.Template, consul, vault b
harness.taskDir = d
if consul {
harness.consul, err = ctestutil.NewTestServerConfigT(t, nil)
harness.consul, err = ctestutil.NewTestServerConfigT(t, func(c *ctestutil.TestServerConfig) {
// defaults
})
if err != nil {
t.Fatalf("error starting test Consul server: %v", err)
}

7
client/client.go
View file

@ -101,6 +101,10 @@ const (
// Update sidecar_task.html when updating this.
defaultConnectSidecarImage = "envoyproxy/envoy:v1.11.2@sha256:a7769160c9c1a55bb8d07a3b71ce5d64f72b1f665f10d81aa1581bc3cf850d09"
// defaultConnectGatewayImage is the image set in the node meta by default
// to be used by Consul Connect Gateway tasks.
defaultConnectGatewayImage = defaultConnectSidecarImage
// defaultConnectLogLevel is the log level set in the node meta by default
// to be used by Consul Connect sidecar tasks
defaultConnectLogLevel = "info"
@ -1386,6 +1390,9 @@ func (c *Client) setupNode() error {
if _, ok := node.Meta["connect.sidecar_image"]; !ok {
node.Meta["connect.sidecar_image"] = defaultConnectSidecarImage
}
if _, ok := node.Meta["connect.gateway_image"]; !ok {
node.Meta["connect.gateway_image"] = defaultConnectGatewayImage
}
if _, ok := node.Meta["connect.log_level"]; !ok {
node.Meta["connect.log_level"] = defaultConnectLogLevel
}

8
command/agent/agent.go
View file

@ -77,6 +77,9 @@ type Agent struct {
// consulCatalog is the subset of Consul's Catalog API Nomad uses.
consulCatalog consul.CatalogAPI
// consulConfigEntries is the subset of Consul's Configuration Entires API Nomad uses.
consulConfigEntries consul.ConfigAPI
// consulACLs is Nomad's subset of Consul's ACL API Nomad uses.
consulACLs consul.ACLsAPI
@ -669,7 +672,7 @@ func (a *Agent) setupServer() error {
}
// Create the server
server, err := nomad.NewServer(conf, a.consulCatalog, a.consulACLs)
server, err := nomad.NewServer(conf, a.consulCatalog, a.consulConfigEntries, a.consulACLs)
if err != nil {
return fmt.Errorf("server setup failed: %v", err)
}
@ -1124,6 +1127,9 @@ func (a *Agent) setupConsul(consulConfig *config.ConsulConfig) error {
// Create Consul Catalog client for service discovery.
a.consulCatalog = client.Catalog()
// Create Consul ConfigEntries client for managing Config Entries.
a.consulConfigEntries = client.ConfigEntries()
// Create Consul ACL client for managing tokens.
a.consulACLs = client.ACL()

20
command/agent/consul/client.go
View file

@ -105,6 +105,15 @@ type AgentAPI interface {
UpdateTTL(id, output, status string) error
}
// ConfigAPI is the consul/api.ConfigEntries API subset used by Nomad Server.
//
// ACL requirements
// - operator:write (server only)
type ConfigAPI interface {
Set(entry api.ConfigEntry, w *api.WriteOptions) (bool, *api.WriteMeta, error)
// Delete(kind, name string, w *api.WriteOptions) (*api.WriteMeta, error) (not used)
}
// ACLsAPI is the consul/api.ACL API subset used by Nomad Server.
//
// ACL requirements
@ -835,6 +844,9 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w
return nil, fmt.Errorf("invalid Consul Connect configuration for service %q: %v", service.Name, err)
}
// newConnectGateway returns nil if there's no Connect gateway.
gateway := newConnectGateway(service.Name, service.Connect)
// Determine whether to use meta or canary_meta
var meta map[string]string
if workload.Canary && len(service.CanaryMeta) > 0 {
@ -852,8 +864,15 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w
// This enables the consul UI to show that Nomad registered this service
meta["external-source"] = "nomad"
// Explicitly set the service kind in case this service represents a Connect gateway.
kind := api.ServiceKindTypical
if service.Connect.IsGateway() {
kind = api.ServiceKindIngressGateway
}
// Build the Consul Service registration request
serviceReg := &api.AgentServiceRegistration{
Kind: kind,
ID: id,
Name: service.Name,
Tags: tags,
@ -862,6 +881,7 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w
Port: port,
Meta: meta,
Connect: connect, // will be nil if no Connect stanza
Proxy: gateway, // will be nil if no Connect Gateway stanza
}
ops.regServices = append(ops.regServices, serviceReg)

55
command/agent/consul/config_entries_testing.go Normal file
View file

@ -0,0 +1,55 @@
package consul
import (
"sync"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/go-hclog"
)
var _ ConfigAPI = (*MockConfigsAPI)(nil)
type MockConfigsAPI struct {
logger hclog.Logger
lock sync.Mutex
state struct {
error error
entries map[string]api.ConfigEntry
}
}
func NewMockConfigsAPI(l hclog.Logger) *MockConfigsAPI {
return &MockConfigsAPI{
logger: l.Named("mock_consul"),
state: struct {
error error
entries map[string]api.ConfigEntry
}{entries: make(map[string]api.ConfigEntry)},
}
}
// Set is a mock of ConfigAPI.Set
func (m *MockConfigsAPI) Set(entry api.ConfigEntry, w *api.WriteOptions) (bool, *api.WriteMeta, error) {
m.lock.Lock()
defer m.lock.Unlock()
if m.state.error != nil {
return false, nil, m.state.error
}
m.state.entries[entry.GetName()] = entry
return true, &api.WriteMeta{
RequestTime: 1,
}, nil
}
// SetError is a helper method for configuring an error that will be returned
// on future calls to mocked methods.
func (m *MockConfigsAPI) SetError(err error) {
m.lock.Lock()
defer m.lock.Unlock()
m.state.error = err
}

62
command/agent/consul/connect.go
View file

@ -12,24 +12,66 @@ import (
// Connect struct. If the nomad Connect struct is nil, nil will be returned to
// disable Connect for this service.
func newConnect(serviceName string, nc *structs.ConsulConnect, networks structs.Networks) (*api.AgentServiceConnect, error) {
if nc == nil {
switch {
case nc == nil:
// no connect stanza means there is no connect service to register
return nil, nil
}
if nc.IsNative() {
case nc.IsGateway():
// gateway settings are configured on the service block on the consul side
return nil, nil
case nc.IsNative():
// the service is connect native
return &api.AgentServiceConnect{Native: true}, nil
case nc.HasSidecar():
sidecarReg, err := connectSidecarRegistration(serviceName, nc.SidecarService, networks)
if err != nil {
return nil, err
}
return &api.AgentServiceConnect{SidecarService: sidecarReg}, nil
default:
return nil, fmt.Errorf("Connect configuration empty for service %s", serviceName)
}
}
// newConnectGateway creates a new Consul AgentServiceConnectProxyConfig struct based on
// a Nomad Connect struct. If the Nomad Connect struct does not contain a gateway, nil
// will be returned as this service is not a gateway.
func newConnectGateway(serviceName string, connect *structs.ConsulConnect) *api.AgentServiceConnectProxyConfig {
if !connect.IsGateway() {
return nil
}
sidecarReg, err := connectSidecarRegistration(serviceName, nc.SidecarService, networks)
if err != nil {
return nil, err
proxy := connect.Gateway.Proxy
envoyConfig := make(map[string]interface{})
if len(proxy.EnvoyGatewayBindAddresses) > 0 {
envoyConfig["envoy_gateway_bind_addresses"] = proxy.EnvoyGatewayBindAddresses
}
return &api.AgentServiceConnect{
Native: false,
SidecarService: sidecarReg,
}, nil
if proxy.EnvoyGatewayNoDefaultBind {
envoyConfig["envoy_gateway_no_default_bind"] = true
}
if proxy.EnvoyGatewayBindTaggedAddresses {
envoyConfig["envoy_gateway_bind_tagged_addresses"] = true
}
if proxy.ConnectTimeout != nil {
envoyConfig["connect_timeout_ms"] = proxy.ConnectTimeout.Milliseconds()
}
if len(proxy.Config) > 0 {
for k, v := range proxy.Config {
envoyConfig[k] = v
}
}
return &api.AgentServiceConnectProxyConfig{Config: envoyConfig}
}
func connectSidecarRegistration(serviceName string, css *structs.ConsulSidecarService, networks structs.Networks) (*api.AgentServiceRegistration, error) {

65
command/agent/consul/connect_test.go
View file

@ -2,8 +2,10 @@ package consul
import (
"testing"
"time"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/stretchr/testify/require"
)
@ -397,3 +399,66 @@ func TestConnect_getExposePathPort(t *testing.T) {
require.EqualError(t, err, "Connect only supported with exactly 1 network (found 2)")
})
}
func TestConnect_newConnectGateway(t *testing.T) {
t.Parallel()
t.Run("not a gateway", func(t *testing.T) {
result := newConnectGateway("s1", &structs.ConsulConnect{Native: true})
require.Nil(t, result)
})
t.Run("canonical empty", func(t *testing.T) {
result := newConnectGateway("s1", &structs.ConsulConnect{
Gateway: &structs.ConsulGateway{
Proxy: &structs.ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(1 * time.Second),
EnvoyGatewayBindTaggedAddresses: false,
EnvoyGatewayBindAddresses: nil,
EnvoyGatewayNoDefaultBind: false,
Config: nil,
},
},
})
require.Equal(t, &api.AgentServiceConnectProxyConfig{
Config: map[string]interface{}{
"connect_timeout_ms": int64(1000),
},
}, result)
})
t.Run("full", func(t *testing.T) {
result := newConnectGateway("s1", &structs.ConsulConnect{
Gateway: &structs.ConsulGateway{
Proxy: &structs.ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(1 * time.Second),
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{
"service1": &structs.ConsulGatewayBindAddress{
Address: "10.0.0.1",
Port: 2000,
},
},
EnvoyGatewayNoDefaultBind: true,
Config: map[string]interface{}{
"foo": 1,
},
},
},
})
require.Equal(t, &api.AgentServiceConnectProxyConfig{
Config: map[string]interface{}{
"connect_timeout_ms": int64(1000),
"envoy_gateway_bind_tagged_addresses": true,
"envoy_gateway_bind_addresses": map[string]*structs.ConsulGatewayBindAddress{
"service1": &structs.ConsulGatewayBindAddress{
Address: "10.0.0.1",
Port: 2000,
},
},
"envoy_gateway_no_default_bind": true,
"foo": 1,
},
}, result)
})
}

24
command/agent/consul/int_test.go
View file

@ -39,7 +39,7 @@ func TestConsul_Integration(t *testing.T) {
if testing.Short() {
t.Skip("-short set; skipping")
}
require := require.New(t)
r := require.New(t)
// Create an embedded Consul server
testconsul, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) {
@ -133,7 +133,7 @@ func TestConsul_Integration(t *testing.T) {
taskDir := allocDir.NewTaskDir(task.Name)
vclient := vaultclient.NewMockVaultClient()
consulClient, err := consulapi.NewClient(consulConfig)
require.Nil(err)
r.Nil(err)
serviceClient := consul.NewServiceClient(consulClient.Agent(), testlog.HCLogger(t), true)
defer serviceClient.Shutdown() // just-in-case cleanup
@ -165,7 +165,7 @@ func TestConsul_Integration(t *testing.T) {
}
tr, err := taskrunner.NewTaskRunner(config)
require.NoError(err)
r.NoError(err)
go tr.Run()
defer func() {
// Make sure we always shutdown task runner when the test exits
@ -180,7 +180,7 @@ func TestConsul_Integration(t *testing.T) {
// Block waiting for the service to appear
catalog := consulClient.Catalog()
res, meta, err := catalog.Service("httpd2", "test", nil)
require.Nil(err)
r.Nil(err)
for i := 0; len(res) == 0 && i < 10; i++ {
//Expected initial request to fail, do a blocking query
@ -189,7 +189,7 @@ func TestConsul_Integration(t *testing.T) {
t.Fatalf("error querying for service: %v", err)
}
}
require.Len(res, 1)
r.Len(res, 1)
// Truncate results
res = res[:]
@ -197,16 +197,16 @@ func TestConsul_Integration(t *testing.T) {
// Assert the service with the checks exists
for i := 0; len(res) == 0 && i < 10; i++ {
res, meta, err = catalog.Service("httpd", "http", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second})
require.Nil(err)
r.Nil(err)
}
require.Len(res, 1)
r.Len(res, 1)
// Assert the script check passes (mock_driver script checks always
// pass) after having time to run once
time.Sleep(2 * time.Second)
checks, _, err := consulClient.Health().Checks("httpd", nil)
require.Nil(err)
require.Len(checks, 2)
r.Nil(err)
r.Len(checks, 2)
for _, check := range checks {
if expected := "httpd"; check.ServiceName != expected {
@ -261,7 +261,7 @@ func TestConsul_Integration(t *testing.T) {
// Ensure Consul is clean
services, _, err := catalog.Services(nil)
require.Nil(err)
require.Len(services, 1)
require.Contains(services, "consul")
r.Nil(err)
r.Len(services, 1)
r.Contains(services, "consul")
}

107
command/agent/job_endpoint.go
View file

@ -1290,6 +1290,111 @@ func ApiConsulConnectToStructs(in *api.ConsulConnect) *structs.ConsulConnect {
Native: in.Native,
SidecarService: apiConnectSidecarServiceToStructs(in.SidecarService),
SidecarTask: apiConnectSidecarTaskToStructs(in.SidecarTask),
Gateway: apiConnectGatewayToStructs(in.Gateway),
}
}
func apiConnectGatewayToStructs(in *api.ConsulGateway) *structs.ConsulGateway {
if in == nil {
return nil
}
return &structs.ConsulGateway{
Proxy: apiConnectGatewayProxyToStructs(in.Proxy),
Ingress: apiConnectIngressGatewayToStructs(in.Ingress),
}
}
func apiConnectGatewayProxyToStructs(in *api.ConsulGatewayProxy) *structs.ConsulGatewayProxy {
if in == nil {
return nil
}
var bindAddresses map[string]*structs.ConsulGatewayBindAddress
if in.EnvoyGatewayBindAddresses != nil {
bindAddresses = make(map[string]*structs.ConsulGatewayBindAddress)
for k, v := range in.EnvoyGatewayBindAddresses {
bindAddresses[k] = &structs.ConsulGatewayBindAddress{
Address: v.Address,
Port: v.Port,
}
}
}
return &structs.ConsulGatewayProxy{
ConnectTimeout: in.ConnectTimeout,
EnvoyGatewayBindTaggedAddresses: in.EnvoyGatewayBindTaggedAddresses,
EnvoyGatewayBindAddresses: bindAddresses,
EnvoyGatewayNoDefaultBind: in.EnvoyGatewayNoDefaultBind,
Config: helper.CopyMapStringInterface(in.Config),
}
}
func apiConnectIngressGatewayToStructs(in *api.ConsulIngressConfigEntry) *structs.ConsulIngressConfigEntry {
if in == nil {
return nil
}
return &structs.ConsulIngressConfigEntry{
TLS: apiConnectGatewayTLSConfig(in.TLS),
Listeners: apiConnectIngressListenersToStructs(in.Listeners),
}
}
func apiConnectGatewayTLSConfig(in *api.ConsulGatewayTLSConfig) *structs.ConsulGatewayTLSConfig {
if in == nil {
return nil
}
return &structs.ConsulGatewayTLSConfig{
Enabled: in.Enabled,
}
}
func apiConnectIngressListenersToStructs(in []*api.ConsulIngressListener) []*structs.ConsulIngressListener {
if len(in) == 0 {
return nil
}
listeners := make([]*structs.ConsulIngressListener, len(in))
for i, listener := range in {
listeners[i] = apiConnectIngressListenerToStructs(listener)
}
return listeners
}
func apiConnectIngressListenerToStructs(in *api.ConsulIngressListener) *structs.ConsulIngressListener {
if in == nil {
return nil
}
return &structs.ConsulIngressListener{
Port: in.Port,
Protocol: in.Protocol,
Services: apiConnectIngressServicesToStructs(in.Services),
}
}
func apiConnectIngressServicesToStructs(in []*api.ConsulIngressService) []*structs.ConsulIngressService {
if len(in) == 0 {
return nil
}
services := make([]*structs.ConsulIngressService, len(in))
for i, service := range in {
services[i] = apiConnectIngressServiceToStructs(service)
}
return services
}
func apiConnectIngressServiceToStructs(in *api.ConsulIngressService) *structs.ConsulIngressService {
if in == nil {
return nil
}
return &structs.ConsulIngressService{
Name: in.Name,
Hosts: helper.CopySliceString(in.Hosts),
}
}
@ -1313,7 +1418,7 @@ func apiConnectSidecarServiceProxyToStructs(in *api.ConsulProxy) *structs.Consul
LocalServicePort: in.LocalServicePort,
Upstreams: apiUpstreamsToStructs(in.Upstreams),
Expose: apiConsulExposeConfigToStructs(in.ExposeConfig),
Config: in.Config,
Config: helper.CopyMapStringInterface(in.Config),
}
}

2
command/agent/job_endpoint_test.go
View file

@ -3004,7 +3004,7 @@ func TestConversion_apiConnectSidecarServiceProxyToStructs(t *testing.T) {
require.Equal(t, &structs.ConsulProxy{
LocalServiceAddress: "192.168.30.1",
LocalServicePort: 9000,
Config: config,
Config: nil,
Upstreams: []structs.ConsulUpstream{{
DestinationName: "upstream",
}},

30
go.sum
View file

@ -141,7 +141,6 @@ github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533 h1:8wZizuKuZVu5COB7Es
github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/container-storage-interface/spec v1.2.0-rc1.0.20191021210849-a33ece0a8a9f h1:m2LYF3fo9IPapVt5FGRVw5bJPmlWqWIezB0jkQh03Zo=
github.com/container-storage-interface/spec v1.2.0-rc1.0.20191021210849-a33ece0a8a9f/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s=
github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
github.com/containerd/console v1.0.0 h1:fU3UuQapBs+zLJu82NhR11Rif1ny2zfMMAyPJzSN5tQ=
@ -175,7 +174,6 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd/v22 v22.1.0 h1:kq/SbG2BCKLkDKkjQf5OWwKWUKj1lgs3lFI4PxnR5lg=
github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
@ -210,7 +208,6 @@ github.com/docker/docker-credential-helpers v0.6.2-0.20180719074751-73e5f5dbfea3
github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916 h1:yWHOI+vFjEsAakUTSrtqc/SAHrhSkmn48pqjidZX3QA=
github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
@ -231,9 +228,13 @@ github.com/endocrimes/go-winio v0.4.13-0.20190628114223-fb47a8b41948 h1:PgcXIRC4
github.com/endocrimes/go-winio v0.4.13-0.20190628114223-fb47a8b41948/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
<<<<<<< HEAD
github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s=
=======
github.com/envoyproxy/go-control-plane v0.9.5 h1:lRJIqDD8yjV1YyPRqecMdytjDLs2fTXq363aCib5xPU=
github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s=
github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
>>>>>>> master
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
@ -269,7 +270,10 @@ github.com/godbus/dbus v5.0.1+incompatible h1:fsDsnr/6MFSIm3kl6JJpq5pH+vO/rA5jUu
github.com/godbus/dbus v5.0.1+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
<<<<<<< HEAD
=======
github.com/gogo/googleapis v1.2.0 h1:Z0v3OJDotX9ZBpdz2V+AI7F4fITSZhVE5mg6GQppwMM=
>>>>>>> master
github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@ -335,12 +339,10 @@ github.com/hashicorp/consul v1.7.7/go.mod h1:urbfGaVZDmnXC6geg0LYPh/SRUk1E8nfmDH
github.com/hashicorp/consul-template v0.24.1 h1:96zTJ5YOq4HMTgtehXRvzGoQNEG2Z4jBYY5ofhq8/Cc=
github.com/hashicorp/consul-template v0.24.1/go.mod h1:KcTEopo2kCp7kww0d4oG7d3oX2Uou4hzb1Rs/wY9TVI=
github.com/hashicorp/consul/api v1.2.0/go.mod h1:1SIkFYi2ZTXUE5Kgt179+4hH33djo11+0Eo2XgTAtkw=
github.com/hashicorp/consul/api v1.4.0 h1:jfESivXnO5uLdH650JU/6AnjRoHrLhULq0FnC3Kp9EY=
github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU=
github.com/hashicorp/consul/api v1.6.0 h1:SZB2hQW8AcTOpfDmiVblQbijxzsRuiyy0JpHfabvHio=
github.com/hashicorp/consul/api v1.6.0/go.mod h1:1NSuaUUkFaJzMasbfq/11wKYWSR67Xn6r2DXKhuDNFg=
github.com/hashicorp/consul/sdk v0.2.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/consul/sdk v0.4.0 h1:zBtCfKJZcJDBvSCkQJch4ulp59m1rATFLKwNo/LYY30=
github.com/hashicorp/consul/sdk v0.4.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM=
github.com/hashicorp/consul/sdk v0.6.0 h1:FfhMEkwvQl57CildXJyGHnwGGM4HMODGyfjGwNM1Vdw=
github.com/hashicorp/consul/sdk v0.6.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM=
@ -480,7 +482,6 @@ github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 h1:BvV6PYcRz0yGnW
github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@ -535,7 +536,6 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/cli v1.1.0 h1:tEElEatulEHDeedTxwckzyYMA5c86fbmNIUL1hBIiTg=
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
@ -549,7 +549,6 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b h1:9+ke9YJ9KGWw5ANXK6ozjoK47uI3uNbXv4YVINBnGm8=
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/go-testing-interface v1.0.3 h1:gqwbsGvc0jbhAPW/26WfEoSiPANAVlR49AAVdvaTjI4=
github.com/mitchellh/go-testing-interface v1.0.3/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@ -573,10 +572,8 @@ github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx
github.com/moby/sys/mountinfo v0.1.3 h1:KIrhRO14+AkwKvG/g2yIpNMOUVZ02xNhOw8KY1WsLOI=
github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
@ -668,16 +665,21 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
<<<<<<< HEAD
=======
github.com/rboyer/safeio v0.2.1 h1:05xhhdRNAdS3apYm7JRjOqngf4xruaW959jmRxGDuSU=
>>>>>>> master
github.com/rboyer/safeio v0.2.1/go.mod h1:Cq/cEPK+YXFn622lsQ0K4KsPZSPtaptHHEldsy7Fmig=
github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03 h1:Wdi9nwnhFNAlseAOekn6B5G/+GMtks9UKbvRU/CMM/o=
github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03/go.mod h1:gRAiPF5C5Nd0eyyRdqIu9qTiFSoZzpTq727b5B8fkkU=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
<<<<<<< HEAD
=======
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
>>>>>>> master
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@ -714,7 +716,6 @@ github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
@ -750,7 +751,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@ -763,7 +763,6 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
@ -1007,7 +1006,10 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
<<<<<<< HEAD
=======
google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
>>>>>>> master
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=

23
helper/funcs.go
View file

@ -93,11 +93,19 @@ func StringToPtr(str string) *string {
return &str
}
// TimeToPtr returns the pointer to a time stamp
// TimeToPtr returns the pointer to a time.Duration.
func TimeToPtr(t time.Duration) *time.Duration {
return &t
}
// CompareTimePtrs return true if a is the same as b.
func CompareTimePtrs(a, b *time.Duration) bool {
if a == nil || b == nil {
return a == b
}
return *a == *b
}
// Float64ToPtr returns the pointer to an float64
func Float64ToPtr(f float64) *float64 {
return &f
@ -287,6 +295,19 @@ func CopyMapStringStruct(m map[string]struct{}) map[string]struct{} {
return c
}
func CopyMapStringInterface(m map[string]interface{}) map[string]interface{} {
l := len(m)
if l == 0 {
return nil
}
c := make(map[string]interface{}, l)
for k, v := range m {
c[k] = v
}
return c
}
func CopyMapStringInt(m map[string]int) map[string]int {
l := len(m)
if l == 0 {

33
helper/funcs_test.go
View file

@ -5,6 +5,7 @@ import (
"reflect"
"sort"
"testing"
"time"
"github.com/stretchr/testify/require"
)
@ -32,6 +33,25 @@ func TestSliceStringContains(t *testing.T) {
require.False(t, SliceStringContains(list, "d"))
}
func TestCompareTimePtrs(t *testing.T) {
t.Run("nil", func(t *testing.T) {
a := (*time.Duration)(nil)
b := (*time.Duration)(nil)
require.True(t, CompareTimePtrs(a, b))
c := TimeToPtr(3 * time.Second)
require.False(t, CompareTimePtrs(a, c))
require.False(t, CompareTimePtrs(c, a))
})
t.Run("not nil", func(t *testing.T) {
a := TimeToPtr(1 * time.Second)
b := TimeToPtr(1 * time.Second)
c := TimeToPtr(2 * time.Second)
require.True(t, CompareTimePtrs(a, b))
require.False(t, CompareTimePtrs(a, c))
})
}
func TestCompareSliceSetString(t *testing.T) {
cases := []struct {
A []string
@ -134,6 +154,19 @@ func TestCopyMapStringSliceString(t *testing.T) {
}
}
func TestCopyMapSliceInterface(t *testing.T) {
m := map[string]interface{}{
"foo": "bar",
"baz": 2,
}
c := CopyMapStringInterface(m)
require.True(t, reflect.DeepEqual(m, c))
m["foo"] = "zzz"
require.False(t, reflect.DeepEqual(m, c))
}
func TestClearEnvVar(t *testing.T) {
type testCase struct {
input string

343
jobspec/parse_service.go
View file

@ -146,6 +146,7 @@ func parseService(o *ast.ObjectItem) (*api.Service, error) {
func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) {
valid := []string{
"native",
"gateway",
"sidecar_service",
"sidecar_task",
}
@ -160,6 +161,7 @@ func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) {
return nil, err
}
delete(m, "gateway")
delete(m, "sidecar_service")
delete(m, "sidecar_task")
@ -174,8 +176,20 @@ func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) {
return nil, fmt.Errorf("connect should be an object")
}
// Parse the gateway
o := connectList.Filter("gateway")
if len(o.Items) > 1 {
return nil, fmt.Errorf("only one 'gateway' block allowed per task")
} else if len(o.Items) == 1 {
g, err := parseGateway(o.Items[0])
if err != nil {
return nil, fmt.Errorf("gateway, %v", err)
}
connect.Gateway = g
}
// Parse the sidecar_service
o := connectList.Filter("sidecar_service")
o = connectList.Filter("sidecar_service")
if len(o.Items) == 0 {
return &connect, nil
}
@ -207,6 +221,329 @@ func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) {
return &connect, nil
}
func parseGateway(o *ast.ObjectItem) (*api.ConsulGateway, error) {
valid := []string{
"proxy",
"ingress",
}
if err := helper.CheckHCLKeys(o.Val, valid); err != nil {
return nil, multierror.Prefix(err, "gateway ->")
}
var gateway api.ConsulGateway
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return nil, err
}
delete(m, "proxy")
delete(m, "ingress")
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
WeaklyTypedInput: true,
Result: &gateway,
})
if err != nil {
return nil, err
}
if err := dec.Decode(m); err != nil {
return nil, fmt.Errorf("gateway: %v", err)
}
// list of parameters
var listVal *ast.ObjectList
if ot, ok := o.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return nil, fmt.Errorf("proxy: should be an object")
}
// extract and parse the proxy block
po := listVal.Filter("proxy")
if len(po.Items) != 1 {
return nil, fmt.Errorf("must have one 'proxy' block")
}
proxy, err := parseGatewayProxy(po.Items[0])
if err != nil {
return nil, fmt.Errorf("proxy, %v", err)
}
gateway.Proxy = proxy
// extract and parse the ingress block
io := listVal.Filter("ingress")
if len(io.Items) != 1 {
// in the future, may be terminating or mesh block instead
return nil, fmt.Errorf("must have one 'ingress' block")
}
ingress, err := parseIngressConfigEntry(io.Items[0])
if err != nil {
return nil, fmt.Errorf("ingress, %v", err)
}
gateway.Ingress = ingress
return &gateway, nil
}
// parseGatewayProxy parses envoy gateway proxy options supported by Consul.
//
// consul.io/docs/connect/proxies/envoy#gateway-options
func parseGatewayProxy(o *ast.ObjectItem) (*api.ConsulGatewayProxy, error) {
valid := []string{
"connect_timeout",
"envoy_gateway_bind_tagged_addresses",
"envoy_gateway_bind_addresses",
"envoy_gateway_no_default_bind",
"envoy_dns_discovery_type",
"config",
}
if err := helper.CheckHCLKeys(o.Val, valid); err != nil {
return nil, multierror.Prefix(err, "proxy ->")
}
var proxy api.ConsulGatewayProxy
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return nil, err
}
delete(m, "config")
delete(m, "envoy_gateway_bind_addresses")
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
WeaklyTypedInput: true,
Result: &proxy,
})
if err != nil {
return nil, err
}
if err := dec.Decode(m); err != nil {
return nil, fmt.Errorf("proxy: %v", err)
}
var listVal *ast.ObjectList
if ot, ok := o.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return nil, fmt.Errorf("proxy: should be an object")
}
// need to parse envoy_gateway_bind_addresses if present
if ebo := listVal.Filter("envoy_gateway_bind_addresses"); len(ebo.Items) > 0 {
proxy.EnvoyGatewayBindAddresses = make(map[string]*api.ConsulGatewayBindAddress)
for _, listenerM := range ebo.Items { // object item, each listener object
listenerName := listenerM.Keys[0].Token.Value().(string)
var listenerListVal *ast.ObjectList
if ot, ok := listenerM.Val.(*ast.ObjectType); ok {
listenerListVal = ot.List
} else {
return nil, fmt.Errorf("listener: should be an object")
}
var bind api.ConsulGatewayBindAddress
if err := hcl.DecodeObject(&bind, listenerListVal); err != nil {
panic(err)
}
proxy.EnvoyGatewayBindAddresses[listenerName] = &bind
}
}
// need to parse the opaque config if present
if co := listVal.Filter("config"); len(co.Items) > 1 {
return nil, fmt.Errorf("only 1 meta object supported")
} else if len(co.Items) == 1 {
var mSlice []map[string]interface{}
if err := hcl.DecodeObject(&mSlice, co.Items[0].Val); err != nil {
return nil, err
}
if len(mSlice) > 1 {
return nil, fmt.Errorf("only 1 meta object supported")
}
m := mSlice[0]
if err := mapstructure.WeakDecode(m, &proxy.Config); err != nil {
return nil, err
}
proxy.Config = flattenMapSlice(proxy.Config)
}
return &proxy, nil
}
func parseConsulIngressService(o *ast.ObjectItem) (*api.ConsulIngressService, error) {
valid := []string{
"name",
"hosts",
}
if err := helper.CheckHCLKeys(o.Val, valid); err != nil {
return nil, multierror.Prefix(err, "service ->")
}
var service api.ConsulIngressService
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return nil, err
}
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
Result: &service,
})
if err != nil {
return nil, err
}
if err := dec.Decode(m); err != nil {
return nil, err
}
return &service, nil
}
func parseConsulIngressListener(o *ast.ObjectItem) (*api.ConsulIngressListener, error) {
valid := []string{
"port",
"protocol",
"service",
}
if err := helper.CheckHCLKeys(o.Val, valid); err != nil {
return nil, multierror.Prefix(err, "listener ->")
}
var listener api.ConsulIngressListener
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return nil, err
}
delete(m, "service")
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
Result: &listener,
})
if err != nil {
return nil, err
}
if err := dec.Decode(m); err != nil {
return nil, err
}
// Parse services
var listVal *ast.ObjectList
if ot, ok := o.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return nil, fmt.Errorf("listener: should be an object")
}
so := listVal.Filter("service")
if len(so.Items) > 0 {
listener.Services = make([]*api.ConsulIngressService, len(so.Items))
for i := range so.Items {
is, err := parseConsulIngressService(so.Items[i])
if err != nil {
return nil, err
}
listener.Services[i] = is
}
}
return &listener, nil
}
func parseConsulGatewayTLS(o *ast.ObjectItem) (*api.ConsulGatewayTLSConfig, error) {
valid := []string{
"enabled",
}
if err := helper.CheckHCLKeys(o.Val, valid); err != nil {
return nil, multierror.Prefix(err, "tls ->")
}
var tls api.ConsulGatewayTLSConfig
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return nil, err
}
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
Result: &tls,
})
if err != nil {
return nil, err
}
if err := dec.Decode(m); err != nil {
return nil, err
}
return &tls, nil
}
func parseIngressConfigEntry(o *ast.ObjectItem) (*api.ConsulIngressConfigEntry, error) {
valid := []string{
"tls",
"listener",
}
if err := helper.CheckHCLKeys(o.Val, valid); err != nil {
return nil, multierror.Prefix(err, "ingress ->")
}
var ingress api.ConsulIngressConfigEntry
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return nil, err
}
delete(m, "tls")
delete(m, "listener")
// Parse tls and listener(s)
var listVal *ast.ObjectList
if ot, ok := o.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return nil, fmt.Errorf("ingress: should be an object")
}
if to := listVal.Filter("tls"); len(to.Items) > 1 {
return nil, fmt.Errorf("only 1 tls object supported")
} else if len(to.Items) == 1 {
if tls, err := parseConsulGatewayTLS(to.Items[0]); err != nil {
return nil, err
} else {
ingress.TLS = tls
}
}
lo := listVal.Filter("listener")
if len(lo.Items) > 0 {
ingress.Listeners = make([]*api.ConsulIngressListener, len(lo.Items))
for i := range lo.Items {
listener, err := parseConsulIngressListener(lo.Items[i])
if err != nil {
return nil, err
}
ingress.Listeners[i] = listener
}
}
return &ingress, nil
}
func parseSidecarService(o *ast.ObjectItem) (*api.ConsulSidecarService, error) {
valid := []string{
"port",
@ -340,6 +677,8 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) {
return nil, fmt.Errorf("proxy: %v", err)
}
// Parse upstreams, expose, and config
var listVal *ast.ObjectList
if ot, ok := o.Val.(*ast.ObjectType); ok {
listVal = ot.List
@ -347,8 +686,6 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) {
return nil, fmt.Errorf("proxy: should be an object")
}
// Parse the proxy
uo := listVal.Filter("upstreams")
if len(uo.Items) > 0 {
proxy.Upstreams = make([]*api.ConsulUpstream, len(uo.Items))

89
jobspec/parse_test.go
View file

@ -690,6 +690,37 @@ func TestParse(t *testing.T) {
},
false,
},
{
"service-check-pass-fail.hcl",
&api.Job{
ID: helper.StringToPtr("check_pass_fail"),
Name: helper.StringToPtr("check_pass_fail"),
Type: helper.StringToPtr("service"),
TaskGroups: []*api.TaskGroup{{
Name: helper.StringToPtr("group"),
Count: helper.IntToPtr(1),
Tasks: []*api.Task{{
Name: "task",
Services: []*api.Service{{
Name: "service",
PortLabel: "http",
Checks: []api.ServiceCheck{{
Name: "check-name",
Type: "http",
Path: "/",
Interval: 10 * time.Second,
Timeout: 2 * time.Second,
InitialStatus: capi.HealthPassing,
Method: "POST",
SuccessBeforePassing: 3,
FailuresBeforeCritical: 4,
}},
}},
}},
}},
},
false,
},
{
"service-check-bad-header.hcl",
nil,
@ -1366,7 +1397,63 @@ func TestParse(t *testing.T) {
},
false,
},
{
"tg-service-connect-gateway-ingress.hcl",
&api.Job{
ID: helper.StringToPtr("connect_gateway_ingress"),
Name: helper.StringToPtr("connect_gateway_ingress"),
TaskGroups: []*api.TaskGroup{{
Name: helper.StringToPtr("group"),
Services: []*api.Service{{
Name: "ingress-gateway-service",
Connect: &api.ConsulConnect{
Gateway: &api.ConsulGateway{
Proxy: &api.ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(3 * time.Second),
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayBindAddresses: map[string]*api.ConsulGatewayBindAddress{
"listener1": {Address: "10.0.0.1", Port: 8888},
"listener2": {Address: "10.0.0.2", Port: 8889},
},
EnvoyGatewayNoDefaultBind: true,
Config: map[string]interface{}{"foo": "bar"},
},
Ingress: &api.ConsulIngressConfigEntry{
TLS: &api.ConsulGatewayTLSConfig{
Enabled: true,
},
Listeners: []*api.ConsulIngressListener{{
Port: 8001,
Protocol: "tcp",
Services: []*api.ConsulIngressService{{
Name: "service1",
Hosts: []string{
"127.0.0.1:8001",
"[::1]:8001",
}}, {
Name: "service2",
Hosts: []string{
"10.0.0.1:8001",
}},
}}, {
Port: 8080,
Protocol: "http",
Services: []*api.ConsulIngressService{{
Name: "nginx",
Hosts: []string{
"2.2.2.2:8080",
},
}},
},
},
},
},
},
}},
}},
},
false,
},
{
"tg-scaling-policy-minimal.hcl",
&api.Job{

55
jobspec/test-fixtures/tg-service-connect-gateway-ingress.hcl Normal file
View file

@ -0,0 +1,55 @@
job "connect_gateway_ingress" {
group "group" {
service {
name = "ingress-gateway-service"
connect {
gateway {
proxy {
connect_timeout = "3s"
envoy_gateway_bind_tagged_addresses = true
envoy_gateway_bind_addresses "listener1" {
address = "10.0.0.1"
port = 8888
}
envoy_gateway_bind_addresses "listener2" {
address = "10.0.0.2"
port = 8889
}
envoy_gateway_no_default_bind = true
config {
foo = "bar"
}
}
ingress {
tls {
enabled = true
}
listener {
port = 8001
protocol = "tcp"
service {
name = "service1"
hosts = ["127.0.0.1:8001", "[::1]:8001"]
}
service {
name = "service2"
hosts = ["10.0.0.1:8001"]
}
}
listener {
port = 8080
protocol = "http"
service {
name = "nginx"
hosts = ["2.2.2.2:8080"]
}
}
}
}
}
}
}
}

112
nomad/consul.go
View file

@ -11,6 +11,7 @@ import (
"github.com/hashicorp/consul/api"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/nomad/command/agent/consul"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -35,6 +36,13 @@ const (
siTokenRevocationInterval = 5 * time.Minute
)
const (
// configEntriesRequestRateLimit is the maximum number of requests per second
// Nomad will make against Consul for operations on global Configuration Entry
// objects.
configEntriesRequestRateLimit rate.Limit = 10
)
const (
// ConsulPolicyWrite is the literal text of the policy field of a Consul Policy
// Rule that we check when validating an Operator Consul token against the
@ -435,3 +443,107 @@ func (s *Server) purgeSITokenAccessors(accessors []*structs.SITokenAccessor) err
_, _, err := s.raftApply(structs.ServiceIdentityAccessorDeregisterRequestType, request)
return err
}
// ConsulConfigsAPI is an abstraction over the consul/api.ConfigEntries API used by
// Nomad Server.
//
// Nomad will only perform write operations on Consul Ingress Gateway Configuration Entries.
// Removing the entries is not particularly safe, given that multiple Nomad clusters
// may be writing to the same config entries, which are global in the Consul scope.
type ConsulConfigsAPI interface {
// SetIngressGatewayConfigEntry adds the given ConfigEntry to Consul, overwriting
// the previous entry if set.
SetIngressGatewayConfigEntry(ctx context.Context, service string, entry *structs.ConsulIngressConfigEntry) error
// Stop is used to stop additional creations of Configuration Entries. Intended to
// be used on Nomad Server shutdown.
Stop()
}
type consulConfigsAPI struct {
// configsClient is the API subset of the real Consul client we need for
// managing Configuration Entries.
configsClient consul.ConfigAPI
// limiter is used to rate limit requests to Consul
limiter *rate.Limiter
// logger is used to log messages
logger hclog.Logger
// lock protects the stopped flag, which prevents use of the consul configs API
// client after shutdown.
lock sync.Mutex
stopped bool
}
func NewConsulConfigsAPI(configsClient consul.ConfigAPI, logger hclog.Logger) *consulConfigsAPI {
return &consulConfigsAPI{
configsClient: configsClient,
limiter: rate.NewLimiter(configEntriesRequestRateLimit, int(configEntriesRequestRateLimit)),
logger: logger,
}
}
func (c *consulConfigsAPI) Stop() {
c.lock.Lock()
defer c.lock.Unlock()
c.stopped = true
}
func (c *consulConfigsAPI) SetIngressGatewayConfigEntry(ctx context.Context, service string, entry *structs.ConsulIngressConfigEntry) error {
configEntry := convertIngressGatewayConfig(service, entry)
return c.setConfigEntry(ctx, configEntry)
}
// setConfigEntry will set the Configuration Entry of any type Consul supports.
func (c *consulConfigsAPI) setConfigEntry(ctx context.Context, entry api.ConfigEntry) error {
defer metrics.MeasureSince([]string{"nomad", "consul", "create_config_entry"}, time.Now())
// make sure the background deletion goroutine has not been stopped
c.lock.Lock()
stopped := c.stopped
c.lock.Unlock()
if stopped {
return errors.New("client stopped and may not longer create config entries")
}
// ensure we are under our wait limit
if err := c.limiter.Wait(ctx); err != nil {
return err
}
_, _, err := c.configsClient.Set(entry, nil)
return err
}
func convertIngressGatewayConfig(service string, entry *structs.ConsulIngressConfigEntry) api.ConfigEntry {
var listeners []api.IngressListener = nil
for _, listener := range entry.Listeners {
var services []api.IngressService = nil
for _, service := range listener.Services {
services = append(services, api.IngressService{
Name: service.Name,
Hosts: helper.CopySliceString(service.Hosts),
})
}
listeners = append(listeners, api.IngressListener{
Port: listener.Port,
Protocol: listener.Protocol,
Services: services,
})
}
tlsEnabled := false
if entry.TLS != nil && entry.TLS.Enabled {
tlsEnabled = true
}
return &api.IngressGatewayConfigEntry{
Kind: api.IngressGateway,
Name: service,
TLS: api.GatewayTLSConfig{Enabled: tlsEnabled},
Listeners: listeners,
}
}

33
nomad/consul_test.go
View file

@ -18,6 +18,39 @@ import (
var _ ConsulACLsAPI = (*consulACLsAPI)(nil)
var _ ConsulACLsAPI = (*mockConsulACLsAPI)(nil)
var _ ConsulConfigsAPI = (*consulConfigsAPI)(nil)
func TestConsulConfigsAPI_SetIngressGatewayConfigEntry(t *testing.T) {
t.Parallel()
try := func(t *testing.T, expErr error) {
logger := testlog.HCLogger(t)
configsAPI := consul.NewMockConfigsAPI(logger) // agent
configsAPI.SetError(expErr)
c := NewConsulConfigsAPI(configsAPI, logger)
ctx := context.Background()
err := c.SetIngressGatewayConfigEntry(ctx, "service1", &structs.ConsulIngressConfigEntry{
TLS: nil,
Listeners: nil,
})
if expErr != nil {
require.Equal(t, expErr, err)
} else {
require.NoError(t, err)
}
}
t.Run("set ingress CE success", func(t *testing.T) {
try(t, nil)
})
t.Run("set ingress CE failure", func(t *testing.T) {
try(t, errors.New("consul broke"))
})
}
type revokeRequest struct {
accessorID string

16
nomad/job_endpoint.go
View file

@ -277,6 +277,22 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis
}
}
// Create or Update Consul Configuration Entries defined in the job. For now
// Nomad only supports Configuration Entries of type "ingress-gateway" for managing
// Consul Connect Ingress Gateway tasks derived from TaskGroup services.
//
// This is done as a blocking operation that prevents the job from being
// submitted if the configuration entries cannot be set in Consul.
//
// Every job update will re-write the Configuration Entry into Consul.
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
for service, entry := range args.Job.ConfigEntries() {
if err := j.srv.consulConfigEntries.SetIngressGatewayConfigEntry(ctx, service, entry); err != nil {
return err
}
}
// Enforce Sentinel policies. Pass a copy of the job to prevent
// sentinel from altering it.
policyWarnings, err := j.enforceSubmitJob(args.PolicyOverride, args.Job.Copy())

206
nomad/job_endpoint_hook_connect.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"time"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/pkg/errors"
@ -19,9 +20,9 @@ var (
}
}
// connectDriverConfig is the driver configuration used by the injected
// connect proxy sidecar task
connectDriverConfig = func() map[string]interface{} {
// connectSidecarDriverConfig is the driver configuration used by the injected
// connect proxy sidecar task.
connectSidecarDriverConfig = func() map[string]interface{} {
return map[string]interface{}{
"image": "${meta.connect.sidecar_image}",
"args": []interface{}{
@ -32,17 +33,51 @@ var (
}
}
// connectVersionConstraint is used when building the sidecar task to ensure
// connectGatewayDriverConfig is the Docker driver configuration used by the
// injected connect proxy sidecar task.
//
// A gateway may run in a group with bridge or host networking, and if host
// networking is being used the network_mode driver configuration is set here.
connectGatewayDriverConfig = func(hostNetwork bool) map[string]interface{} {
m := map[string]interface{}{
"image": "${meta.connect.gateway_image}",
"args": []interface{}{
"-c", structs.EnvoyBootstrapPath,
"-l", "${meta.connect.log_level}",
"--disable-hot-restart",
},
}
if hostNetwork {
m["network_mode"] = "host"
}
return m
}
// connectMinimalVersionConstraint is used when building the sidecar task to ensure
// the proper Consul version is used that supports the necessary Connect
// features. This includes bootstrapping envoy with a unix socket for Consul's
// gRPC xDS API.
connectVersionConstraint = func() *structs.Constraint {
connectMinimalVersionConstraint = func() *structs.Constraint {
return &structs.Constraint{
LTarget: "${attr.consul.version}",
RTarget: ">= 1.6.0-beta1",
Operand: structs.ConstraintSemver,
}
}
// connectGatewayVersionConstraint is used when building a connect gateway
// task to ensure proper Consul version is used that supports Connect Gateway
// features. This includes making use of Consul Configuration Entries of type
// {ingress,terminating,mesh}-gateway.
connectGatewayVersionConstraint = func() *structs.Constraint {
return &structs.Constraint{
LTarget: "${attr.consul.version}",
RTarget: ">= 1.8.0",
Operand: structs.ConstraintSemver,
}
}
)
// jobConnectHook implements a job Mutating and Validating admission controller
@ -97,7 +132,22 @@ func getSidecarTaskForService(tg *structs.TaskGroup, svc string) *structs.Task {
}
func isSidecarForService(t *structs.Task, svc string) bool {
return t.Kind == structs.TaskKind(fmt.Sprintf("%s:%s", structs.ConnectProxyPrefix, svc))
return t.Kind == structs.NewTaskKind(structs.ConnectProxyPrefix, svc)
}
func hasGatewayTaskForService(tg *structs.TaskGroup, svc string) bool {
for _, t := range tg.Tasks {
switch {
case isIngressGatewayForService(t, svc):
// also terminating and mesh in the future
return true
}
}
return false
}
func isIngressGatewayForService(t *structs.Task, svc string) bool {
return t.Kind == structs.NewTaskKind(structs.ConnectIngressPrefix, svc)
}
// getNamedTaskForNativeService retrieves the Task with the name specified in the
@ -123,7 +173,10 @@ func getNamedTaskForNativeService(tg *structs.TaskGroup, serviceName, taskName s
// qualify, configure a port for envoy to use to expose their paths.
func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error {
for _, service := range g.Services {
if service.Connect.HasSidecar() {
switch {
// mutate depending on what the connect block is being used for
case service.Connect.HasSidecar():
// Check to see if the sidecar task already exists
task := getSidecarTaskForService(g, service.Name)
@ -167,7 +220,8 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error {
// create a port for the sidecar task's proxy port
makePort(fmt.Sprintf("%s-%s", structs.ConnectProxyPrefix, service.Name))
} else if service.Connect.IsNative() {
case service.Connect.IsNative():
// find the task backing this connect native service and set the kind
nativeTaskName := service.TaskName
if t, err := getNamedTaskForNativeService(g, service.Name, nativeTaskName); err != nil {
@ -176,6 +230,23 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error {
t.Kind = structs.NewTaskKind(structs.ConnectNativePrefix, service.Name)
service.TaskName = t.Name // in case the task was inferred
}
case service.Connect.IsGateway():
netHost := g.Networks[0].Mode == "host"
if !netHost && service.Connect.Gateway.Ingress != nil {
// Modify the gateway proxy service configuration to automatically
// do the correct envoy bind address plumbing when inside a net
// namespace, but only if things are not explicitly configured.
service.Connect.Gateway.Proxy = gatewayProxyForBridge(service.Connect.Gateway)
}
// inject the gateway task only if it does not yet already exist
if !hasGatewayTaskForService(g, service.Name) {
// use the default envoy image, for now there is no support for a custom task
task := newConnectGatewayTask(service.Name, netHost)
g.Tasks = append(g.Tasks, task)
task.Canonicalize(job, g)
}
}
}
@ -184,13 +255,99 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error {
return nil
}
// gatewayProxyIsDefault returns false if any of these gateway proxy configuration
// have been modified from their default values, indicating the operator wants
// custom behavior. Otherwise, we assume the operator wants Nomad to do the Right
// Thing, setting the configuration automatically.
//
// - envoy_gateway_no_default_bind
// - envoy_gateway_bind_tagged_addresses
// - envoy_gateway_bind_addresses
func gatewayProxyIsDefault(proxy *structs.ConsulGatewayProxy) bool {
if proxy == nil {
return true
}
if !proxy.EnvoyGatewayNoDefaultBind &&
!proxy.EnvoyGatewayBindTaggedAddresses &&
len(proxy.EnvoyGatewayBindAddresses) == 0 {
return true
}
return false
}
// gatewayProxyForBridge scans an existing gateway proxy configuration and tweaks
// it given an associated configuration entry so that it works as intended from
// inside a network namespace.
func gatewayProxyForBridge(gateway *structs.ConsulGateway) *structs.ConsulGatewayProxy {
if gateway == nil {
return nil
}
// operator has supplied custom proxy configuration, just use that without
// modification
if !gatewayProxyIsDefault(gateway.Proxy) {
return gateway.Proxy
}
// copy over unrelated fields if proxy block exists
proxy := new(structs.ConsulGatewayProxy)
if gateway.Proxy != nil {
proxy.ConnectTimeout = gateway.Proxy.ConnectTimeout
proxy.Config = gateway.Proxy.Config
}
// magically set the fields where Nomad knows what to do
proxy.EnvoyGatewayNoDefaultBind = true
proxy.EnvoyGatewayBindTaggedAddresses = false
proxy.EnvoyGatewayBindAddresses = gatewayBindAddresses(gateway.Ingress)
return proxy
}
func gatewayBindAddresses(ingress *structs.ConsulIngressConfigEntry) map[string]*structs.ConsulGatewayBindAddress {
if ingress == nil || len(ingress.Listeners) == 0 {
return nil
}
addresses := make(map[string]*structs.ConsulGatewayBindAddress)
for _, listener := range ingress.Listeners {
port := listener.Port
for _, service := range listener.Services {
addresses[service.Name] = &structs.ConsulGatewayBindAddress{
Address: "0.0.0.0",
Port: port,
}
}
}
return addresses
}
func newConnectGatewayTask(serviceName string, netHost bool) *structs.Task {
return &structs.Task{
// Name is used in container name so must start with '[A-Za-z0-9]'
Name: fmt.Sprintf("%s-%s", structs.ConnectIngressPrefix, serviceName),
Kind: structs.NewTaskKind(structs.ConnectIngressPrefix, serviceName),
Driver: "docker",
Config: connectGatewayDriverConfig(netHost),
ShutdownDelay: 5 * time.Second,
LogConfig: &structs.LogConfig{
MaxFiles: 2,
MaxFileSizeMB: 2,
},
Resources: connectSidecarResources(),
Constraints: structs.Constraints{
connectGatewayVersionConstraint(),
},
}
}
func newConnectTask(serviceName string) *structs.Task {
task := &structs.Task{
return &structs.Task{
// Name is used in container name so must start with '[A-Za-z0-9]'
Name: fmt.Sprintf("%s-%s", structs.ConnectProxyPrefix, serviceName),
Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, serviceName),
Driver: "docker",
Config: connectDriverConfig(),
Config: connectSidecarDriverConfig(),
ShutdownDelay: 5 * time.Second,
LogConfig: &structs.LogConfig{
MaxFiles: 2,
@ -202,23 +359,26 @@ func newConnectTask(serviceName string) *structs.Task {
Sidecar: true,
},
Constraints: structs.Constraints{
connectVersionConstraint(),
connectMinimalVersionConstraint(),
},
}
return task
}
func groupConnectValidate(g *structs.TaskGroup) (warnings []error, err error) {
for _, s := range g.Services {
if s.Connect.HasSidecar() {
switch {
case s.Connect.HasSidecar():
if err := groupConnectSidecarValidate(g); err != nil {
return nil, err
}
} else if s.Connect.IsNative() {
case s.Connect.IsNative():
if err := groupConnectNativeValidate(g, s); err != nil {
return nil, err
}
case s.Connect.IsGateway():
if err := groupConnectGatewayValidate(g); err != nil {
return nil, err
}
}
}
return nil, nil
@ -243,3 +403,19 @@ func groupConnectNativeValidate(g *structs.TaskGroup, s *structs.Service) error
}
return nil
}
func groupConnectGatewayValidate(g *structs.TaskGroup) error {
// the group needs to be either bridge or host mode so we know how to configure
// the docker driver config
if n := len(g.Networks); n != 1 {
return fmt.Errorf("Consul Connect gateways require exactly 1 network, found %d in group %q", n, g.Name)
}
modes := []string{"bridge", "host"}
if !helper.SliceStringContains(modes, g.Networks[0].Mode) {
return fmt.Errorf(`Consul Connect Gateway service requires Task Group with network mode of type "bridge" or "host"`)
}
return nil
}

284
nomad/job_endpoint_hook_connect_test.go
View file

@ -3,7 +3,9 @@ package nomad
import (
"fmt"
"testing"
"time"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
@ -110,6 +112,33 @@ func TestJobEndpointConnect_groupConnectHook(t *testing.T) {
require.Exactly(t, tgOut, job.TaskGroups[0])
}
func TestJobEndpointConnect_groupConnectHook_IngressGateway(t *testing.T) {
t.Parallel()
// Test that the connect gateway task is inserted if a gateway service exists
// and since this is a bridge network, will rewrite the default gateway proxy
// block with correct configuration.
job := mock.ConnectIngressGatewayJob("bridge", false)
expTG := job.TaskGroups[0].Copy()
expTG.Tasks = []*structs.Task{
// inject the gateway task
newConnectGatewayTask(expTG.Services[0].Name, false),
}
expTG.Tasks[0].Canonicalize(job, expTG)
expTG.Networks[0].Canonicalize()
// rewrite the service gateway proxy configuration
expTG.Services[0].Connect.Gateway.Proxy = gatewayProxyForBridge(expTG.Services[0].Connect.Gateway)
require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
require.Exactly(t, expTG, job.TaskGroups[0])
// Test that the hook is idempotent
require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
require.Exactly(t, expTG, job.TaskGroups[0])
}
// TestJobEndpoint_ConnectInterpolation asserts that when a Connect sidecar
// proxy task is being created for a group service with an interpolated name,
// the service name is interpolated *before the task is created.
@ -132,6 +161,8 @@ func TestJobEndpointConnect_ConnectInterpolation(t *testing.T) {
}
func TestJobEndpointConnect_groupConnectSidecarValidate(t *testing.T) {
t.Parallel()
t.Run("sidecar 0 networks", func(t *testing.T) {
require.EqualError(t, groupConnectSidecarValidate(&structs.TaskGroup{
Name: "g1",
@ -159,6 +190,8 @@ func TestJobEndpointConnect_groupConnectSidecarValidate(t *testing.T) {
}
func TestJobEndpointConnect_getNamedTaskForNativeService(t *testing.T) {
t.Parallel()
t.Run("named exists", func(t *testing.T) {
task, err := getNamedTaskForNativeService(&structs.TaskGroup{
Name: "g1",
@ -195,3 +228,254 @@ func TestJobEndpointConnect_getNamedTaskForNativeService(t *testing.T) {
require.Nil(t, task)
})
}
func TestJobEndpointConnect_groupConnectGatewayValidate(t *testing.T) {
t.Parallel()
t.Run("no group network", func(t *testing.T) {
err := groupConnectGatewayValidate(&structs.TaskGroup{
Name: "g1",
Networks: nil,
})
require.EqualError(t, err, `Consul Connect gateways require exactly 1 network, found 0 in group "g1"`)
})
t.Run("bad network mode", func(t *testing.T) {
err := groupConnectGatewayValidate(&structs.TaskGroup{
Name: "g1",
Networks: structs.Networks{{
Mode: "",
}},
})
require.EqualError(t, err, `Consul Connect Gateway service requires Task Group with network mode of type "bridge" or "host"`)
})
}
func TestJobEndpointConnect_newConnectGatewayTask_host(t *testing.T) {
task := newConnectGatewayTask("service1", true)
require.Equal(t, "connect-ingress-service1", task.Name)
require.Equal(t, "connect-ingress:service1", string(task.Kind))
require.Equal(t, ">= 1.8.0", task.Constraints[0].RTarget)
require.Equal(t, "host", task.Config["network_mode"])
require.Nil(t, task.Lifecycle)
}
func TestJobEndpointConnect_newConnectGatewayTask_bridge(t *testing.T) {
task := newConnectGatewayTask("service1", false)
require.NotContains(t, task.Config, "network_mode")
}
func TestJobEndpointConnect_hasGatewayTaskForService(t *testing.T) {
t.Run("no gateway task", func(t *testing.T) {
result := hasGatewayTaskForService(&structs.TaskGroup{
Name: "group",
Tasks: []*structs.Task{{
Name: "task1",
Kind: "",
}},
}, "my-service")
require.False(t, result)
})
t.Run("has gateway task", func(t *testing.T) {
result := hasGatewayTaskForService(&structs.TaskGroup{
Name: "group",
Tasks: []*structs.Task{{
Name: "task1",
Kind: "",
}, {
Name: "ingress-gateway-my-service",
Kind: structs.NewTaskKind(structs.ConnectIngressPrefix, "my-service"),
}},
}, "my-service")
require.True(t, result)
})
}
func TestJobEndpointConnect_gatewayProxyIsDefault(t *testing.T) {
t.Run("nil", func(t *testing.T) {
result := gatewayProxyIsDefault(nil)
require.True(t, result)
})
t.Run("unrelated fields set", func(t *testing.T) {
result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(2 * time.Second),
Config: map[string]interface{}{"foo": 1},
})
require.True(t, result)
})
t.Run("no-bind set", func(t *testing.T) {
result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{
EnvoyGatewayNoDefaultBind: true,
})
require.False(t, result)
})
t.Run("bind-tagged set", func(t *testing.T) {
result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{
EnvoyGatewayBindTaggedAddresses: true,
})
require.False(t, result)
})
t.Run("bind-addresses set", func(t *testing.T) {
result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{
EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{
"listener1": &structs.ConsulGatewayBindAddress{
Address: "1.1.1.1",
Port: 9000,
},
},
})
require.False(t, result)
})
}
func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) {
t.Run("nil", func(t *testing.T) {
result := gatewayBindAddresses(nil)
require.Nil(t, result)
})
t.Run("no listeners", func(t *testing.T) {
result := gatewayBindAddresses(&structs.ConsulIngressConfigEntry{Listeners: nil})
require.Nil(t, result)
})
t.Run("simple", func(t *testing.T) {
result := gatewayBindAddresses(&structs.ConsulIngressConfigEntry{
Listeners: []*structs.ConsulIngressListener{{
Port: 3000,
Protocol: "tcp",
Services: []*structs.ConsulIngressService{{
Name: "service1",
}},
}},
})
require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{
"service1": &structs.ConsulGatewayBindAddress{
Address: "0.0.0.0",
Port: 3000,
},
}, result)
})
t.Run("complex", func(t *testing.T) {
result := gatewayBindAddresses(&structs.ConsulIngressConfigEntry{
Listeners: []*structs.ConsulIngressListener{{
Port: 3000,
Protocol: "tcp",
Services: []*structs.ConsulIngressService{{
Name: "service1",
}, {
Name: "service2",
}},
}, {
Port: 3001,
Protocol: "http",
Services: []*structs.ConsulIngressService{{
Name: "service3",
}},
}},
})
require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{
"service1": &structs.ConsulGatewayBindAddress{
Address: "0.0.0.0",
Port: 3000,
},
"service2": &structs.ConsulGatewayBindAddress{
Address: "0.0.0.0",
Port: 3000,
},
"service3": &structs.ConsulGatewayBindAddress{
Address: "0.0.0.0",
Port: 3001,
},
}, result)
})
}
func TestJobEndpointConnect_gatewayProxyForBridge(t *testing.T) {
t.Run("nil", func(t *testing.T) {
result := gatewayProxyForBridge(nil)
require.Nil(t, result)
})
t.Run("nil proxy", func(t *testing.T) {
result := gatewayProxyForBridge(&structs.ConsulGateway{
Ingress: &structs.ConsulIngressConfigEntry{
Listeners: []*structs.ConsulIngressListener{{
Port: 3000,
Protocol: "tcp",
Services: []*structs.ConsulIngressService{{
Name: "service1",
}},
}},
},
})
require.Equal(t, &structs.ConsulGatewayProxy{
EnvoyGatewayNoDefaultBind: true,
EnvoyGatewayBindTaggedAddresses: false,
EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{
"service1": {
Address: "0.0.0.0",
Port: 3000,
}},
}, result)
})
t.Run("fill in defaults", func(t *testing.T) {
result := gatewayProxyForBridge(&structs.ConsulGateway{
Proxy: &structs.ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(2 * time.Second),
Config: map[string]interface{}{"foo": 1},
},
Ingress: &structs.ConsulIngressConfigEntry{
Listeners: []*structs.ConsulIngressListener{{
Port: 3000,
Protocol: "tcp",
Services: []*structs.ConsulIngressService{{
Name: "service1",
}},
}},
},
})
require.Equal(t, &structs.ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(2 * time.Second),
Config: map[string]interface{}{"foo": 1},
EnvoyGatewayNoDefaultBind: true,
EnvoyGatewayBindTaggedAddresses: false,
EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{
"service1": {
Address: "0.0.0.0",
Port: 3000,
}},
}, result)
})
t.Run("leave as-is", func(t *testing.T) {
result := gatewayProxyForBridge(&structs.ConsulGateway{
Proxy: &structs.ConsulGatewayProxy{
Config: map[string]interface{}{"foo": 1},
EnvoyGatewayBindTaggedAddresses: true,
},
Ingress: &structs.ConsulIngressConfigEntry{
Listeners: []*structs.ConsulIngressListener{{
Port: 3000,
Protocol: "tcp",
Services: []*structs.ConsulIngressService{{
Name: "service1",
}},
}},
},
})
require.Equal(t, &structs.ConsulGatewayProxy{
Config: map[string]interface{}{"foo": 1},
EnvoyGatewayNoDefaultBind: false,
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayBindAddresses: nil,
}, result)
})
}

204
nomad/job_endpoint_test.go
View file

@ -237,6 +237,208 @@ func TestJobEndpoint_Register_Connect(t *testing.T) {
require.Exactly(sidecarTask, out.TaskGroups[0].Tasks[1])
}
func TestJobEndpoint_Register_ConnectIngressGateway_minimum(t *testing.T) {
t.Parallel()
r := require.New(t)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.NumSchedulers = 0 // Prevent automatic dequeue
})
defer cleanupS1()
codec := rpcClient(t, s1)
testutil.WaitForLeader(t, s1.RPC)
// job contains the minimalist possible gateway service definition
job := mock.ConnectIngressGatewayJob("host", false)
// Create the register request
req := &structs.JobRegisterRequest{
Job: job,
WriteRequest: structs.WriteRequest{
Region: "global",
Namespace: job.Namespace,
},
}
// Fetch the response
var resp structs.JobRegisterResponse
r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp))
r.NotZero(resp.Index)
// Check for the node in the FSM
state := s1.fsm.State()
ws := memdb.NewWatchSet()
out, err := state.JobByID(ws, job.Namespace, job.ID)
r.NoError(err)
r.NotNil(out)
r.Equal(resp.JobModifyIndex, out.CreateIndex)
// Check that the gateway task got injected
r.Len(out.TaskGroups[0].Tasks, 1)
task := out.TaskGroups[0].Tasks[0]
r.Equal("connect-ingress-my-ingress-service", task.Name)
r.Equal("connect-ingress:my-ingress-service", string(task.Kind))
r.Equal("docker", task.Driver)
r.NotNil(task.Config)
// Check the CE fields got set
service := out.TaskGroups[0].Services[0]
r.Equal(&structs.ConsulIngressConfigEntry{
TLS: nil,
Listeners: []*structs.ConsulIngressListener{{
Port: 2000,
Protocol: "tcp",
Services: []*structs.ConsulIngressService{{
Name: "service1",
}},
}},
}, service.Connect.Gateway.Ingress)
// Check that round-tripping does not inject a duplicate task
out.Meta["test"] = "abc"
req.Job = out
r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp))
r.NotZero(resp.Index)
// Check for the new node in the fsm
state = s1.fsm.State()
ws = memdb.NewWatchSet()
out, err = state.JobByID(ws, job.Namespace, job.ID)
r.NoError(err)
r.NotNil(out)
r.Equal(resp.JobModifyIndex, out.CreateIndex)
// Check we did not re-add the task that was added the first time
r.Len(out.TaskGroups[0].Tasks, 1)
}
func TestJobEndpoint_Register_ConnectIngressGateway_full(t *testing.T) {
t.Parallel()
r := require.New(t)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.NumSchedulers = 0 // Prevent automatic dequeue
})
defer cleanupS1()
codec := rpcClient(t, s1)
testutil.WaitForLeader(t, s1.RPC)
// reconfigure job to fill in all the possible fields
job := mock.ConnectIngressGatewayJob("bridge", false)
job.TaskGroups[0].Services[0].Connect = &structs.ConsulConnect{
Gateway: &structs.ConsulGateway{
Proxy: &structs.ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(1 * time.Second),
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{
"service1": {
Address: "10.0.0.1",
Port: 2001,
},
"service2": {
Address: "10.0.0.2",
Port: 2002,
},
},
EnvoyGatewayNoDefaultBind: true,
Config: map[string]interface{}{
"foo": 1,
"bar": "baz",
},
},
Ingress: &structs.ConsulIngressConfigEntry{
TLS: &structs.ConsulGatewayTLSConfig{
Enabled: true,
},
Listeners: []*structs.ConsulIngressListener{{
Port: 3000,
Protocol: "tcp",
Services: []*structs.ConsulIngressService{{
Name: "db",
}},
}, {
Port: 3001,
Protocol: "http",
Services: []*structs.ConsulIngressService{{
Name: "website",
Hosts: []string{"10.0.1.0", "10.0.1.0:3001"},
}},
}},
},
},
}
// Create the register request
req := &structs.JobRegisterRequest{
Job: job,
WriteRequest: structs.WriteRequest{
Region: "global",
Namespace: job.Namespace,
},
}
// Fetch the response
var resp structs.JobRegisterResponse
r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp))
r.NotZero(resp.Index)
// Check for the node in the FSM
state := s1.fsm.State()
ws := memdb.NewWatchSet()
out, err := state.JobByID(ws, job.Namespace, job.ID)
r.NoError(err)
r.NotNil(out)
r.Equal(resp.JobModifyIndex, out.CreateIndex)
// Check that the gateway task got injected
r.Len(out.TaskGroups[0].Tasks, 1)
task := out.TaskGroups[0].Tasks[0]
r.Equal("connect-ingress-my-ingress-service", task.Name)
r.Equal("connect-ingress:my-ingress-service", string(task.Kind))
r.Equal("docker", task.Driver)
r.NotNil(task.Config)
// Check that the ingress service is all set
service := out.TaskGroups[0].Services[0]
r.Equal("my-ingress-service", service.Name)
r.Equal(&structs.ConsulIngressConfigEntry{
TLS: &structs.ConsulGatewayTLSConfig{
Enabled: true,
},
Listeners: []*structs.ConsulIngressListener{{
Port: 3000,
Protocol: "tcp",
Services: []*structs.ConsulIngressService{{
Name: "db",
}},
}, {
Port: 3001,
Protocol: "http",
Services: []*structs.ConsulIngressService{{
Name: "website",
Hosts: []string{"10.0.1.0", "10.0.1.0:3001"},
}},
}},
}, service.Connect.Gateway.Ingress)
// Check that round-tripping does not inject a duplicate task
out.Meta["test"] = "abc"
req.Job = out
r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp))
r.NotZero(resp.Index)
// Check for the new node in the fsm
state = s1.fsm.State()
ws = memdb.NewWatchSet()
out, err = state.JobByID(ws, job.Namespace, job.ID)
r.NoError(err)
r.NotNil(out)
r.Equal(resp.JobModifyIndex, out.CreateIndex)
// Check we did not re-add the task that was added the first time
r.Len(out.TaskGroups[0].Tasks, 1)
}
func TestJobEndpoint_Register_ConnectExposeCheck(t *testing.T) {
t.Parallel()
r := require.New(t)
@ -421,7 +623,7 @@ func TestJobEndpoint_Register_ConnectWithSidecarTask(t *testing.T) {
require.Equal("test", sidecarTask.Meta["source"])
require.Equal(500, sidecarTask.Resources.CPU)
require.Equal(connectSidecarResources().MemoryMB, sidecarTask.Resources.MemoryMB)
cfg := connectDriverConfig()
cfg := connectSidecarDriverConfig()
cfg["labels"] = map[string]interface{}{
"foo": "bar",
}

2
nomad/leader_test.go
View file

@ -651,7 +651,7 @@ func TestLeader_revokeSITokenAccessorsOnRestore(t *testing.T) {
defer cleanupS1()
testutil.WaitForLeader(t, s1.RPC)
// replace consul ACLs api with a mock for tracking calls
// replace consul ACLs API with a mock for tracking calls in tests
var consulACLsAPI mockConsulACLsAPI
s1.consulACLs = &consulACLsAPI

62
nomad/mock/mock.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"time"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/nomad/structs"
psstructs "github.com/hashicorp/nomad/plugins/shared/structs"
@ -668,6 +669,56 @@ func ConnectNativeJob(mode string) *structs.Job {
return job
}
// ConnectIngressGatewayJob creates a structs.Job that contains the definition
// of a Consul Ingress Gateway service. The mode is the name of the network
// mode assumed by the task group. If inject is true, a corresponding Task is
// set on the group's Tasks (i.e. what the job would look like after job mutation).
func ConnectIngressGatewayJob(mode string, inject bool) *structs.Job {
job := Job()
tg := job.TaskGroups[0]
tg.Networks = []*structs.NetworkResource{{
Mode: mode,
}}
tg.Services = []*structs.Service{{
Name: "my-ingress-service",
PortLabel: "9999",
Connect: &structs.ConsulConnect{
Gateway: &structs.ConsulGateway{
Proxy: &structs.ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(3 * time.Second),
},
Ingress: &structs.ConsulIngressConfigEntry{
Listeners: []*structs.ConsulIngressListener{{
Port: 2000,
Protocol: "tcp",
Services: []*structs.ConsulIngressService{{
Name: "service1",
}},
}},
},
},
},
}}
// some tests need to assume the gateway proxy task has already been injected
if inject {
tg.Tasks = []*structs.Task{{
Name: fmt.Sprintf("%s-%s", structs.ConnectIngressPrefix, "my-ingress-service"),
Kind: structs.NewTaskKind(structs.ConnectIngressPrefix, "my-ingress-service"),
Driver: "docker",
Config: make(map[string]interface{}),
ShutdownDelay: 5 * time.Second,
LogConfig: &structs.LogConfig{
MaxFiles: 2,
MaxFileSizeMB: 2,
},
}}
} else {
// otherwise there are no tasks in the group yet
tg.Tasks = nil
}
return job
}
func BatchJob() *structs.Job {
job := &structs.Job{
Region: "global",
@ -932,6 +983,7 @@ func ConnectAlloc() *structs.Allocation {
return alloc
}
// ConnectNativeAlloc creates an alloc with a connect native task.
func ConnectNativeAlloc(mode string) *structs.Allocation {
alloc := Alloc()
alloc.Job = ConnectNativeJob(mode)
@ -942,6 +994,16 @@ func ConnectNativeAlloc(mode string) *structs.Allocation {
return alloc
}
func ConnectIngressGatewayAlloc(mode string) *structs.Allocation {
alloc := Alloc()
alloc.Job = ConnectIngressGatewayJob(mode, true)
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{{
Mode: mode,
IP: "10.0.0.1",
}}
return alloc
}
func BatchConnectJob() *structs.Job {
job := &structs.Job{
Region: "global",

13
nomad/server.go
View file

@ -216,6 +216,9 @@ type Server struct {
// consulCatalog is used for discovering other Nomad Servers via Consul
consulCatalog consul.CatalogAPI
// consulConfigEntries is used for managing Consul Configuration Entries.
consulConfigEntries ConsulConfigsAPI
// consulACLs is used for managing Consul Service Identity tokens.
consulACLs ConsulACLsAPI
@ -283,7 +286,7 @@ type endpoints struct {
// NewServer is used to construct a new Nomad server from the
// configuration, potentially returning an error
func NewServer(config *Config, consulCatalog consul.CatalogAPI, consulACLs consul.ACLsAPI) (*Server, error) {
func NewServer(config *Config, consulCatalog consul.CatalogAPI, consulConfigEntries consul.ConfigAPI, consulACLs consul.ACLsAPI) (*Server, error) {
// Check the protocol version
if err := config.CheckVersion(); err != nil {
return nil, err
@ -362,7 +365,7 @@ func NewServer(config *Config, consulCatalog consul.CatalogAPI, consulACLs consu
s.statsFetcher = NewStatsFetcher(s.logger, s.connPool, s.config.Region)
// Setup Consul (more)
s.setupConsul(consulACLs)
s.setupConsul(consulConfigEntries, consulACLs)
// Setup Vault
if err := s.setupVaultClient(); err != nil {
@ -662,6 +665,9 @@ func (s *Server) Shutdown() error {
// Stop the Consul ACLs token revocations
s.consulACLs.Stop()
// Stop being able to set Configuration Entries
s.consulConfigEntries.Stop()
return nil
}
@ -1042,7 +1048,8 @@ func (s *Server) setupNodeDrainer() {
}
// setupConsul is used to setup Server specific consul components.
func (s *Server) setupConsul(consulACLs consul.ACLsAPI) {
func (s *Server) setupConsul(consulConfigEntries consul.ConfigAPI, consulACLs consul.ACLsAPI) {
s.consulConfigEntries = NewConsulConfigsAPI(consulConfigEntries, s.logger)
s.consulACLs = NewConsulACLsAPI(consulACLs, s.logger, s.purgeSITokenAccessors)
}

331
nomad/structs/diff.go
View file

@ -777,17 +777,344 @@ func connectDiffs(old, new *ConsulConnect, contextual bool) *ObjectDiff {
// Diff the primitive fields.
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
sidecarSvcDiff := connectSidecarServiceDiff(
old.SidecarService, new.SidecarService, contextual)
// Diff the object field SidecarService.
sidecarSvcDiff := connectSidecarServiceDiff(old.SidecarService, new.SidecarService, contextual)
if sidecarSvcDiff != nil {
diff.Objects = append(diff.Objects, sidecarSvcDiff)
}
// Diff the object field SidecarTask.
sidecarTaskDiff := sidecarTaskDiff(old.SidecarTask, new.SidecarTask, contextual)
if sidecarTaskDiff != nil {
diff.Objects = append(diff.Objects, sidecarTaskDiff)
}
// Diff the object field ConsulGateway.
gatewayDiff := connectGatewayDiff(old.Gateway, new.Gateway, contextual)
if gatewayDiff != nil {
diff.Objects = append(diff.Objects, gatewayDiff)
}
return diff
}
func connectGatewayDiff(prev, next *ConsulGateway, contextual bool) *ObjectDiff {
diff := &ObjectDiff{Type: DiffTypeNone, Name: "Gateway"}
var oldPrimitiveFlat, newPrimitiveFlat map[string]string
if reflect.DeepEqual(prev, next) {
return nil
} else if prev == nil {
prev = new(ConsulGateway)
diff.Type = DiffTypeAdded
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
} else if next == nil {
next = new(ConsulGateway)
diff.Type = DiffTypeDeleted
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
} else {
diff.Type = DiffTypeEdited
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
}
// Diff the primitive fields.
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
// Diff the ConsulGatewayProxy fields.
gatewayProxyDiff := connectGatewayProxyDiff(prev.Proxy, next.Proxy, contextual)
if gatewayProxyDiff != nil {
diff.Objects = append(diff.Objects, gatewayProxyDiff)
}
// Diff the ConsulGatewayIngress fields.
gatewayIngressDiff := connectGatewayIngressDiff(prev.Ingress, next.Ingress, contextual)
if gatewayIngressDiff != nil {
diff.Objects = append(diff.Objects, gatewayIngressDiff)
}
return diff
}
func connectGatewayIngressDiff(prev, next *ConsulIngressConfigEntry, contextual bool) *ObjectDiff {
diff := &ObjectDiff{Type: DiffTypeNone, Name: "Ingress"}
var oldPrimitiveFlat, newPrimitiveFlat map[string]string
if reflect.DeepEqual(prev, next) {
return nil
} else if prev == nil {
prev = new(ConsulIngressConfigEntry)
diff.Type = DiffTypeAdded
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
} else if next == nil {
next = new(ConsulIngressConfigEntry)
diff.Type = DiffTypeDeleted
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
} else {
diff.Type = DiffTypeEdited
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
}
// Diff the primitive fields.
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
// Diff the ConsulGatewayTLSConfig objects.
tlsConfigDiff := connectGatewayTLSConfigDiff(prev.TLS, next.TLS, contextual)
if tlsConfigDiff != nil {
diff.Objects = append(diff.Objects, tlsConfigDiff)
}
// Diff the Listeners lists.
gatewayIngressListenersDiff := connectGatewayIngressListenersDiff(prev.Listeners, next.Listeners, contextual)
if gatewayIngressListenersDiff != nil {
diff.Objects = append(diff.Objects, gatewayIngressListenersDiff...)
}
return diff
}
func connectGatewayTLSConfigDiff(prev, next *ConsulGatewayTLSConfig, contextual bool) *ObjectDiff {
diff := &ObjectDiff{Type: DiffTypeNone, Name: "TLS"}
var oldPrimitiveFlat, newPrimitiveFlat map[string]string
if reflect.DeepEqual(prev, next) {
return nil
} else if prev == nil {
diff.Type = DiffTypeAdded
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
} else if next == nil {
diff.Type = DiffTypeDeleted
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
} else {
diff.Type = DiffTypeEdited
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
}
// Diff the primitive field.
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
return diff
}
// connectGatewayIngressListenersDiff diffs are a set of listeners keyed by "protocol/port", which is
// a nifty workaround having slices instead of maps. Presumably such a key will be unique, because if
// it is not the config entry is not going to work anyway.
func connectGatewayIngressListenersDiff(prev, next []*ConsulIngressListener, contextual bool) []*ObjectDiff {
// create maps, diff the maps, keys are fields, keys are (port+protocol)
key := func(l *ConsulIngressListener) string {
return fmt.Sprintf("%s/%d", l.Protocol, l.Port)
}
prevMap := make(map[string]*ConsulIngressListener, len(prev))
nextMap := make(map[string]*ConsulIngressListener, len(next))
for _, l := range prev {
prevMap[key(l)] = l
}
for _, l := range next {
nextMap[key(l)] = l
}
var diffs []*ObjectDiff
for k, prevL := range prevMap {
// Diff the same, deleted, and edited
if diff := connectGatewayIngressListenerDiff(prevL, nextMap[k], contextual); diff != nil {
diffs = append(diffs, diff)
}
}
for k, nextL := range nextMap {
// Diff the added
if old, ok := prevMap[k]; !ok {
if diff := connectGatewayIngressListenerDiff(old, nextL, contextual); diff != nil {
diffs = append(diffs, diff)
}
}
}
sort.Sort(ObjectDiffs(diffs))
return diffs
}
func connectGatewayIngressListenerDiff(prev, next *ConsulIngressListener, contextual bool) *ObjectDiff {
diff := &ObjectDiff{Type: DiffTypeNone, Name: "Listener"}
var oldPrimitiveFlat, newPrimitiveFlat map[string]string
if reflect.DeepEqual(prev, next) {
return nil
} else if prev == nil {
prev = new(ConsulIngressListener)
diff.Type = DiffTypeAdded
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
} else if next == nil {
next = new(ConsulIngressListener)
diff.Type = DiffTypeDeleted
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
} else {
diff.Type = DiffTypeEdited
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
}
// Diff the primitive fields.
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
// Diff the Ingress Service objects.
if diffs := connectGatewayIngressServicesDiff(prev.Services, next.Services, contextual); diffs != nil {
diff.Objects = append(diff.Objects, diffs...)
}
return diff
}
// connectGatewayIngressServicesDiff diffs are a set of ingress services keyed by their service name, which
// is a workaround for having slices instead of maps. Presumably the service name is a unique key, because if
// no the config entry is not going to make sense anyway.
func connectGatewayIngressServicesDiff(prev, next []*ConsulIngressService, contextual bool) []*ObjectDiff {
prevMap := make(map[string]*ConsulIngressService, len(prev))
nextMap := make(map[string]*ConsulIngressService, len(next))
for _, s := range prev {
prevMap[s.Name] = s
}
for _, s := range next {
nextMap[s.Name] = s
}
var diffs []*ObjectDiff
for name, oldIS := range prevMap {
// Diff the same, deleted, and edited
if diff := connectGatewayIngressServiceDiff(oldIS, nextMap[name], contextual); diff != nil {
diffs = append(diffs, diff)
}
}
for name, newIS := range nextMap {
// Diff the added
if old, ok := prevMap[name]; !ok {
if diff := connectGatewayIngressServiceDiff(old, newIS, contextual); diff != nil {
diffs = append(diffs, diff)
}
}
}
sort.Sort(ObjectDiffs(diffs))
return diffs
}
func connectGatewayIngressServiceDiff(prev, next *ConsulIngressService, contextual bool) *ObjectDiff {
diff := &ObjectDiff{Type: DiffTypeNone, Name: "ConsulIngressService"}
var oldPrimitiveFlat, newPrimitiveFlat map[string]string
if reflect.DeepEqual(prev, next) {
return nil
} else if prev == nil {
prev = new(ConsulIngressService)
diff.Type = DiffTypeAdded
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
} else if next == nil {
next = new(ConsulIngressService)
diff.Type = DiffTypeDeleted
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
} else {
diff.Type = DiffTypeEdited
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
}
// Diff the primitive fields.
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
// Diff the hosts.
if hDiffs := stringSetDiff(prev.Hosts, next.Hosts, "Hosts", contextual); hDiffs != nil {
diff.Objects = append(diff.Objects, hDiffs)
}
return diff
}
func connectGatewayProxyDiff(prev, next *ConsulGatewayProxy, contextual bool) *ObjectDiff {
diff := &ObjectDiff{Type: DiffTypeNone, Name: "Proxy"}
var oldPrimitiveFlat, newPrimitiveFlat map[string]string
if reflect.DeepEqual(prev, next) {
return nil
} else if prev == nil {
prev = new(ConsulGatewayProxy)
diff.Type = DiffTypeAdded
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
} else if next == nil {
next = new(ConsulGatewayProxy)
diff.Type = DiffTypeDeleted
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
} else {
diff.Type = DiffTypeEdited
oldPrimitiveFlat = flatmap.Flatten(prev, nil, true)
newPrimitiveFlat = flatmap.Flatten(next, nil, true)
}
// Diff the ConnectTimeout field (dur ptr). (i.e. convert to string for comparison)
if oldPrimitiveFlat != nil && newPrimitiveFlat != nil {
if prev.ConnectTimeout == nil {
oldPrimitiveFlat["ConnectTimeout"] = ""
} else {
oldPrimitiveFlat["ConnectTimeout"] = fmt.Sprintf("%s", *prev.ConnectTimeout)
}
if next.ConnectTimeout == nil {
newPrimitiveFlat["ConnectTimeout"] = ""
} else {
newPrimitiveFlat["ConnectTimeout"] = fmt.Sprintf("%s", *next.ConnectTimeout)
}
}
// Diff the primitive fields.
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
// Diff the EnvoyGatewayBindAddresses map.
bindAddrsDiff := connectGatewayProxyEnvoyBindAddrsDiff(prev.EnvoyGatewayBindAddresses, next.EnvoyGatewayBindAddresses, contextual)
if bindAddrsDiff != nil {
diff.Objects = append(diff.Objects, bindAddrsDiff)
}
// Diff the opaque Config map.
if cDiff := configDiff(prev.Config, next.Config, contextual); cDiff != nil {
diff.Objects = append(diff.Objects, cDiff)
}
return diff
}
// connectGatewayProxyEnvoyBindAddrsDiff returns the diff of two maps. If contextual
// diff is enabled, all fields will be returned, even if no diff occurred.
func connectGatewayProxyEnvoyBindAddrsDiff(prev, next map[string]*ConsulGatewayBindAddress, contextual bool) *ObjectDiff {
diff := &ObjectDiff{Type: DiffTypeNone, Name: "EnvoyGatewayBindAddresses"}
if reflect.DeepEqual(prev, next) {
return nil
} else if len(prev) == 0 {
diff.Type = DiffTypeAdded
} else if len(next) == 0 {
diff.Type = DiffTypeDeleted
} else {
diff.Type = DiffTypeEdited
}
// convert to string representation
prevMap := make(map[string]string, len(prev))
nextMap := make(map[string]string, len(next))
for k, v := range prev {
prevMap[k] = fmt.Sprintf("%s:%d", v.Address, v.Port)
}
for k, v := range next {
nextMap[k] = fmt.Sprintf("%s:%d", v.Address, v.Port)
}
oldPrimitiveFlat := flatmap.Flatten(prevMap, nil, false)
newPrimitiveFlat := flatmap.Flatten(nextMap, nil, false)
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
return diff
}

216
nomad/structs/diff_test.go
View file

@ -2618,6 +2618,34 @@ func TestTaskGroupDiff(t *testing.T) {
"foo": "baz",
},
},
Gateway: &ConsulGateway{
Proxy: &ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(1 * time.Second),
EnvoyGatewayBindTaggedAddresses: false,
EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{
"service1": {
Address: "10.0.0.1",
Port: 2001,
},
},
EnvoyGatewayNoDefaultBind: false,
Config: map[string]interface{}{
"foo": 1,
},
},
Ingress: &ConsulIngressConfigEntry{
TLS: &ConsulGatewayTLSConfig{
Enabled: false,
},
Listeners: []*ConsulIngressListener{{
Port: 3001,
Protocol: "tcp",
Services: []*ConsulIngressService{{
Name: "listener1",
}},
}},
},
},
},
},
},
@ -2664,6 +2692,35 @@ func TestTaskGroupDiff(t *testing.T) {
},
},
},
Gateway: &ConsulGateway{
Proxy: &ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(2 * time.Second),
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{
"service1": {
Address: "10.0.0.2",
Port: 2002,
},
},
EnvoyGatewayNoDefaultBind: true,
Config: map[string]interface{}{
"foo": 2,
},
},
Ingress: &ConsulIngressConfigEntry{
TLS: &ConsulGatewayTLSConfig{
Enabled: true,
},
Listeners: []*ConsulIngressListener{{
Port: 3002,
Protocol: "http",
Services: []*ConsulIngressService{{
Name: "listener2",
Hosts: []string{"127.0.0.1", "127.0.0.1:3002"},
}},
}},
},
},
},
},
},
@ -2836,7 +2893,6 @@ func TestTaskGroupDiff(t *testing.T) {
},
},
},
{
Type: DiffTypeEdited,
Name: "ConsulConnect",
@ -2952,6 +3008,164 @@ func TestTaskGroupDiff(t *testing.T) {
},
},
},
{
Type: DiffTypeEdited,
Name: "Gateway",
Objects: []*ObjectDiff{
{
Type: DiffTypeEdited,
Name: "Proxy",
Fields: []*FieldDiff{
{
Type: DiffTypeEdited,
Name: "ConnectTimeout",
Old: "1s",
New: "2s",
},
{
Type: DiffTypeEdited,
Name: "EnvoyGatewayBindTaggedAddresses",
Old: "false",
New: "true",
},
{
Type: DiffTypeEdited,
Name: "EnvoyGatewayNoDefaultBind",
Old: "false",
New: "true",
},
},
Objects: []*ObjectDiff{
{
Type: DiffTypeEdited,
Name: "EnvoyGatewayBindAddresses",
Fields: []*FieldDiff{
{
Type: DiffTypeEdited,
Name: "service1",
Old: "10.0.0.1:2001",
New: "10.0.0.2:2002",
},
},
},
{
Type: DiffTypeEdited,
Name: "Config",
Fields: []*FieldDiff{
{
Type: DiffTypeEdited,
Name: "foo",
Old: "1",
New: "2",
},
},
},
},
},
{
Type: DiffTypeEdited,
Name: "Ingress",
Objects: []*ObjectDiff{
{
Type: DiffTypeEdited,
Name: "TLS",
Fields: []*FieldDiff{
{
Type: DiffTypeEdited,
Name: "Enabled",
Old: "false",
New: "true",
},
},
},
{
Type: DiffTypeAdded,
Name: "Listener",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "Port",
Old: "",
New: "3002",
},
{
Type: DiffTypeAdded,
Name: "Protocol",
Old: "",
New: "http",
},
},
Objects: []*ObjectDiff{
{
Type: DiffTypeAdded,
Name: "ConsulIngressService",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "Name",
Old: "",
New: "listener2",
},
},
Objects: []*ObjectDiff{
{
Type: DiffTypeAdded,
Name: "Hosts",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "Hosts",
Old: "",
New: "127.0.0.1",
},
{
Type: DiffTypeAdded,
Name: "Hosts",
Old: "",
New: "127.0.0.1:3002",
},
},
},
},
},
},
},
{
Type: DiffTypeDeleted,
Name: "Listener",
Fields: []*FieldDiff{
{
Type: DiffTypeDeleted,
Name: "Port",
Old: "3001",
New: "",
},
{
Type: DiffTypeDeleted,
Name: "Protocol",
Old: "tcp",
New: "",
},
},
Objects: []*ObjectDiff{
{
Type: DiffTypeDeleted,
Name: "ConsulIngressService",
Fields: []*FieldDiff{
{
Type: DiffTypeDeleted,
Name: "Name",
Old: "listener1",
New: "",
},
},
},
},
},
},
},
},
},
},
},
},

521
nomad/structs/services.go
View file

@ -677,6 +677,9 @@ type ConsulConnect struct {
// SidecarTask is non-nil if sidecar overrides are set
SidecarTask *SidecarTask
// Gateway is a Consul Connect Gateway Proxy.
Gateway *ConsulGateway
}
// Copy the stanza recursively. Returns nil if nil.
@ -689,6 +692,7 @@ func (c *ConsulConnect) Copy() *ConsulConnect {
Native: c.Native,
SidecarService: c.SidecarService.Copy(),
SidecarTask: c.SidecarTask.Copy(),
Gateway: c.Gateway.Copy(),
}
}
@ -702,32 +706,70 @@ func (c *ConsulConnect) Equals(o *ConsulConnect) bool {
return false
}
return c.SidecarService.Equals(o.SidecarService)
if !c.SidecarService.Equals(o.SidecarService) {
return false
}
// todo(shoenig) task has never been compared, should it be?
if !c.Gateway.Equals(o.Gateway) {
return false
}
return true
}
// HasSidecar checks if a sidecar task is needed
// HasSidecar checks if a sidecar task is configured.
func (c *ConsulConnect) HasSidecar() bool {
return c != nil && c.SidecarService != nil
}
// IsNative checks if the service is connect native.
func (c *ConsulConnect) IsNative() bool {
return c != nil && c.Native
}
// Validate that the Connect stanza has exactly one of Native or sidecar.
func (c *ConsulConnect) IsGateway() bool {
return c != nil && c.Gateway != nil
}
// Validate that the Connect block represents exactly one of:
// - Connect non-native service sidecar proxy
// - Connect native service
// - Connect gateway (any type)
func (c *ConsulConnect) Validate() error {
if c == nil {
return nil
}
if c.IsNative() && c.HasSidecar() {
return fmt.Errorf("Consul Connect must be native or use a sidecar service; not both")
// Count the number of things actually configured. If that number is not 1,
// the config is not valid.
count := 0
if c.HasSidecar() {
count++
}
if !c.IsNative() && !c.HasSidecar() {
return fmt.Errorf("Consul Connect must be native or use a sidecar service")
if c.IsNative() {
count++
}
if c.IsGateway() {
count++
}
if count != 1 {
return fmt.Errorf("Consul Connect must be exclusively native, make use of a sidecar, or represent a Gateway")
}
if c.IsGateway() {
if err := c.Gateway.Validate(); err != nil {
return err
}
}
// The Native and Sidecar cases are validated up at the service level.
return nil
}
@ -981,6 +1023,15 @@ func (p *ConsulProxy) Copy() *ConsulProxy {
return newP
}
// opaqueMapsEqual compares map[string]interface{} commonly used for opaque
// config blocks. Interprets nil and {} as the same.
func opaqueMapsEqual(a, b map[string]interface{}) bool {
if len(a) == 0 && len(b) == 0 {
return true
}
return reflect.DeepEqual(a, b)
}
// Equals returns true if the structs are recursively equal.
func (p *ConsulProxy) Equals(o *ConsulProxy) bool {
if p == nil || o == nil {
@ -1003,11 +1054,8 @@ func (p *ConsulProxy) Equals(o *ConsulProxy) bool {
return false
}
// Avoid nil vs {} differences
if len(p.Config) != 0 && len(o.Config) != 0 {
if !reflect.DeepEqual(p.Config, o.Config) {
return false
}
if !opaqueMapsEqual(p.Config, o.Config) {
return false
}
return true
@ -1112,3 +1160,452 @@ func (e *ConsulExposeConfig) Equals(o *ConsulExposeConfig) bool {
}
return exposePathsEqual(e.Paths, o.Paths)
}
// ConsulGateway is used to configure one of the Consul Connect Gateway types.
type ConsulGateway struct {
// Proxy is used to configure the Envoy instance acting as the gateway.
Proxy *ConsulGatewayProxy
// Ingress represents the Consul Configuration Entry for an Ingress Gateway.
Ingress *ConsulIngressConfigEntry
// Terminating is not yet supported.
// Terminating *ConsulTerminatingConfigEntry
// Mesh is not yet supported.
// Mesh *ConsulMeshConfigEntry
}
func (g *ConsulGateway) Copy() *ConsulGateway {
if g == nil {
return nil
}
return &ConsulGateway{
Proxy: g.Proxy.Copy(),
Ingress: g.Ingress.Copy(),
}
}
func (g *ConsulGateway) Equals(o *ConsulGateway) bool {
if g == nil || o == nil {
return g == o
}
if !g.Proxy.Equals(o.Proxy) {
return false
}
if !g.Ingress.Equals(o.Ingress) {
return false
}
return true
}
func (g *ConsulGateway) Validate() error {
if g == nil {
return nil
}
if g.Proxy != nil {
if err := g.Proxy.Validate(); err != nil {
return err
}
}
// eventually one of: ingress, terminating, mesh
if g.Ingress != nil {
return g.Ingress.Validate()
}
return fmt.Errorf("Consul Gateway ingress Configuration Entry must be set")
}
// ConsulGatewayBindAddress is equivalent to Consul's api/catalog.go ServiceAddress
// struct, as this is used to encode values to pass along to Envoy (i.e. via
// JSON encoding).
type ConsulGatewayBindAddress struct {
Address string
Port int
}
func (a *ConsulGatewayBindAddress) Equals(o *ConsulGatewayBindAddress) bool {
if a == nil || o == nil {
return a == o
}
if a.Address != o.Address {
return false
}
if a.Port != o.Port {
return false
}
return true
}
func (a *ConsulGatewayBindAddress) Copy() *ConsulGatewayBindAddress {
if a == nil {
return nil
}
return &ConsulGatewayBindAddress{
Address: a.Address,
Port: a.Port,
}
}
func (a *ConsulGatewayBindAddress) Validate() error {
if a == nil {
return nil
}
if a.Address == "" {
return fmt.Errorf("Consul Gateway Bind Address must be set")
}
if a.Port <= 0 {
return fmt.Errorf("Consul Gateway Bind Address must set valid Port")
}
return nil
}
// ConsulGatewayProxy is used to tune parameters of the proxy instance acting as
// one of the forms of Connect gateways that Consul supports.
//
// https://www.consul.io/docs/connect/proxies/envoy#gateway-options
type ConsulGatewayProxy struct {
ConnectTimeout *time.Duration
EnvoyGatewayBindTaggedAddresses bool
EnvoyGatewayBindAddresses map[string]*ConsulGatewayBindAddress
EnvoyGatewayNoDefaultBind bool
Config map[string]interface{}
}
func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy {
if p == nil {
return nil
}
bindAddresses := make(map[string]*ConsulGatewayBindAddress, len(p.EnvoyGatewayBindAddresses))
for k, v := range p.EnvoyGatewayBindAddresses {
bindAddresses[k] = v.Copy()
}
return &ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(*p.ConnectTimeout),
EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses,
EnvoyGatewayBindAddresses: bindAddresses,
EnvoyGatewayNoDefaultBind: p.EnvoyGatewayNoDefaultBind,
Config: helper.CopyMapStringInterface(p.Config),
}
}
func (p *ConsulGatewayProxy) equalBindAddresses(o map[string]*ConsulGatewayBindAddress) bool {
if len(p.EnvoyGatewayBindAddresses) != len(o) {
return false
}
for listener, addr := range p.EnvoyGatewayBindAddresses {
if !o[listener].Equals(addr) {
return false
}
}
return true
}
func (p *ConsulGatewayProxy) Equals(o *ConsulGatewayProxy) bool {
if p == nil || o == nil {
return p == o
}
if !helper.CompareTimePtrs(p.ConnectTimeout, o.ConnectTimeout) {
return false
}
if p.EnvoyGatewayBindTaggedAddresses != o.EnvoyGatewayBindTaggedAddresses {
return false
}
if !p.equalBindAddresses(o.EnvoyGatewayBindAddresses) {
return false
}
if p.EnvoyGatewayNoDefaultBind != o.EnvoyGatewayNoDefaultBind {
return false
}
if !opaqueMapsEqual(p.Config, o.Config) {
return false
}
return true
}
func (p *ConsulGatewayProxy) Validate() error {
if p == nil {
return nil
}
if p.ConnectTimeout == nil {
return fmt.Errorf("Consul Gateway Proxy connection_timeout must be set")
}
for _, bindAddr := range p.EnvoyGatewayBindAddresses {
if err := bindAddr.Validate(); err != nil {
return err
}
}
return nil
}
// ConsulGatewayTLSConfig is used to configure TLS for a gateway.
type ConsulGatewayTLSConfig struct {
Enabled bool
}
func (c *ConsulGatewayTLSConfig) Copy() *ConsulGatewayTLSConfig {
if c == nil {
return nil
}
return &ConsulGatewayTLSConfig{
Enabled: c.Enabled,
}
}
func (c *ConsulGatewayTLSConfig) Equals(o *ConsulGatewayTLSConfig) bool {
if c == nil || o == nil {
return c == o
}
return c.Enabled == o.Enabled
}
// ConsulIngressService is used to configure a service fronted by the ingress gateway.
type ConsulIngressService struct {
// Namespace is not yet supported.
// Namespace string
Name string
Hosts []string
}
func (s *ConsulIngressService) Copy() *ConsulIngressService {
if s == nil {
return nil
}
var hosts []string = nil
if n := len(s.Hosts); n > 0 {
hosts = make([]string, n)
copy(hosts, s.Hosts)
}
return &ConsulIngressService{
Name: s.Name,
Hosts: hosts,
}
}
func (s *ConsulIngressService) Equals(o *ConsulIngressService) bool {
if s == nil || o == nil {
return s == o
}
if s.Name != o.Name {
return false
}
return helper.CompareSliceSetString(s.Hosts, o.Hosts)
}
func (s *ConsulIngressService) Validate(isHTTP bool) error {
if s == nil {
return nil
}
if s.Name == "" {
return fmt.Errorf("Consul Ingress Service requires a name")
}
if isHTTP && len(s.Hosts) == 0 {
return fmt.Errorf("Consul Ingress Service requires one or more hosts when using HTTP protocol")
} else if !isHTTP && len(s.Hosts) > 0 {
return fmt.Errorf("Consul Ingress Service supports hosts only when using HTTP protocol")
}
return nil
}
// ConsulIngressListener is used to configure a listener on a Consul Ingress
// Gateway.
type ConsulIngressListener struct {
Port int
Protocol string
Services []*ConsulIngressService
}
func (l *ConsulIngressListener) Copy() *ConsulIngressListener {
if l == nil {
return nil
}
var services []*ConsulIngressService = nil
if n := len(l.Services); n > 0 {
services = make([]*ConsulIngressService, n)
for i := 0; i < n; i++ {
services[i] = l.Services[i].Copy()
}
}
return &ConsulIngressListener{
Port: l.Port,
Protocol: l.Protocol,
Services: services,
}
}
func (l *ConsulIngressListener) Equals(o *ConsulIngressListener) bool {
if l == nil || o == nil {
return l == o
}
if l.Port != o.Port {
return false
}
if l.Protocol != o.Protocol {
return false
}
return ingressServicesEqual(l.Services, o.Services)
}
func (l *ConsulIngressListener) Validate() error {
if l == nil {
return nil
}
if l.Port <= 0 {
return fmt.Errorf("Consul Ingress Listener requires valid Port")
}
protocols := []string{"http", "tcp"}
if !helper.SliceStringContains(protocols, l.Protocol) {
return fmt.Errorf(`Consul Ingress Listener requires protocol of "http" or "tcp", got %q`, l.Protocol)
}
if len(l.Services) == 0 {
return fmt.Errorf("Consul Ingress Listener requires one or more services")
}
for _, service := range l.Services {
if err := service.Validate(l.Protocol == "http"); err != nil {
return err
}
}
return nil
}
func ingressServicesEqual(servicesA, servicesB []*ConsulIngressService) bool {
if len(servicesA) != len(servicesB) {
return false
}
COMPARE: // order does not matter
for _, serviceA := range servicesA {
for _, serviceB := range servicesB {
if serviceA.Equals(serviceB) {
continue COMPARE
}
}
return false
}
return true
}
// ConsulIngressConfigEntry represents the Consul Configuration Entry type for
// an Ingress Gateway.
//
// https://www.consul.io/docs/agent/config-entries/ingress-gateway#available-fields
type ConsulIngressConfigEntry struct {
// Namespace is not yet supported.
// Namespace string
TLS *ConsulGatewayTLSConfig
Listeners []*ConsulIngressListener
}
func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry {
if e == nil {
return nil
}
var listeners []*ConsulIngressListener = nil
if n := len(e.Listeners); n > 0 {
listeners = make([]*ConsulIngressListener, n)
for i := 0; i < n; i++ {
listeners[i] = e.Listeners[i].Copy()
}
}
return &ConsulIngressConfigEntry{
TLS: e.TLS.Copy(),
Listeners: listeners,
}
}
func (e *ConsulIngressConfigEntry) Equals(o *ConsulIngressConfigEntry) bool {
if e == nil || o == nil {
return e == o
}
if !e.TLS.Equals(o.TLS) {
return false
}
return ingressListenersEqual(e.Listeners, o.Listeners)
}
func (e *ConsulIngressConfigEntry) Validate() error {
if e == nil {
return nil
}
if len(e.Listeners) == 0 {
return fmt.Errorf("Consul Ingress Gateway requires at least one listener")
}
for _, listener := range e.Listeners {
if err := listener.Validate(); err != nil {
return err
}
}
return nil
}
func ingressListenersEqual(listenersA, listenersB []*ConsulIngressListener) bool {
if len(listenersA) != len(listenersB) {
return false
}
COMPARE: // order does not matter
for _, listenerA := range listenersA {
for _, listenerB := range listenersB {
if listenerA.Equals(listenerB) {
continue COMPARE
}
}
return false
}
return true
}

419
nomad/structs/services_test.go
View file

@ -474,3 +474,422 @@ func TestConsulSidecarService_Copy(t *testing.T) {
}, result)
})
}
var (
consulIngressGateway1 = &ConsulGateway{
Proxy: &ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(1 * time.Second),
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{
"listener1": &ConsulGatewayBindAddress{Address: "10.0.0.1", Port: 2001},
"listener2": &ConsulGatewayBindAddress{Address: "10.0.0.1", Port: 2002},
},
EnvoyGatewayNoDefaultBind: true,
Config: map[string]interface{}{
"foo": 1,
},
},
Ingress: &ConsulIngressConfigEntry{
TLS: &ConsulGatewayTLSConfig{
Enabled: true,
},
Listeners: []*ConsulIngressListener{{
Port: 3000,
Protocol: "http",
Services: []*ConsulIngressService{{
Name: "service1",
Hosts: []string{"10.0.0.1", "10.0.0.1:3000"},
}, {
Name: "service2",
Hosts: []string{"10.0.0.2", "10.0.0.2:3000"},
}},
}, {
Port: 3001,
Protocol: "tcp",
Services: []*ConsulIngressService{{
Name: "service3",
}},
}},
},
}
)
func TestConsulGateway_Copy(t *testing.T) {
t.Parallel()
t.Run("nil", func(t *testing.T) {
g := (*ConsulGateway)(nil)
result := g.Copy()
require.Nil(t, result)
})
t.Run("as ingress", func(t *testing.T) {
result := consulIngressGateway1.Copy()
require.Equal(t, consulIngressGateway1, result)
require.True(t, result.Equals(consulIngressGateway1))
require.True(t, consulIngressGateway1.Equals(result))
})
}
func TestConsulGateway_Equals_ingress(t *testing.T) {
t.Parallel()
t.Run("nil", func(t *testing.T) {
a := (*ConsulGateway)(nil)
b := (*ConsulGateway)(nil)
require.True(t, a.Equals(b))
require.False(t, a.Equals(consulIngressGateway1))
require.False(t, consulIngressGateway1.Equals(a))
})
original := consulIngressGateway1.Copy()
type gway = ConsulGateway
type tweaker = func(g *gway)
t.Run("reflexive", func(t *testing.T) {
require.True(t, original.Equals(original))
})
try := func(t *testing.T, tweak tweaker) {
modifiable := original.Copy()
tweak(modifiable)
require.False(t, original.Equals(modifiable))
require.False(t, modifiable.Equals(original))
require.True(t, modifiable.Equals(modifiable))
}
// proxy stanza equality checks
t.Run("mod gateway timeout", func(t *testing.T) {
try(t, func(g *gway) { g.Proxy.ConnectTimeout = helper.TimeToPtr(9 * time.Second) })
})
t.Run("mod gateway envoy_gateway_bind_tagged_addresses", func(t *testing.T) {
try(t, func(g *gway) { g.Proxy.EnvoyGatewayBindTaggedAddresses = false })
})
t.Run("mod gateway envoy_gateway_bind_addresses", func(t *testing.T) {
try(t, func(g *gway) {
g.Proxy.EnvoyGatewayBindAddresses = map[string]*ConsulGatewayBindAddress{
"listener3": &ConsulGatewayBindAddress{Address: "9.9.9.9", Port: 9999},
}
})
})
t.Run("mod gateway envoy_gateway_no_default_bind", func(t *testing.T) {
try(t, func(g *gway) { g.Proxy.EnvoyGatewayNoDefaultBind = false })
})
t.Run("mod gateway config", func(t *testing.T) {
try(t, func(g *gway) {
g.Proxy.Config = map[string]interface{}{
"foo": 2,
}
})
})
// ingress config entry equality checks
t.Run("mod ingress tls", func(t *testing.T) {
try(t, func(g *gway) { g.Ingress.TLS = nil })
try(t, func(g *gway) { g.Ingress.TLS.Enabled = false })
})
t.Run("mod ingress listeners count", func(t *testing.T) {
try(t, func(g *gway) { g.Ingress.Listeners = g.Ingress.Listeners[:1] })
})
t.Run("mod ingress listeners port", func(t *testing.T) {
try(t, func(g *gway) { g.Ingress.Listeners[0].Port = 7777 })
})
t.Run("mod ingress listeners protocol", func(t *testing.T) {
try(t, func(g *gway) { g.Ingress.Listeners[0].Protocol = "tcp" })
})
t.Run("mod ingress listeners services count", func(t *testing.T) {
try(t, func(g *gway) { g.Ingress.Listeners[0].Services = g.Ingress.Listeners[0].Services[:1] })
})
t.Run("mod ingress listeners services name", func(t *testing.T) {
try(t, func(g *gway) { g.Ingress.Listeners[0].Services[0].Name = "serviceX" })
})
t.Run("mod ingress listeners services hosts count", func(t *testing.T) {
try(t, func(g *gway) { g.Ingress.Listeners[0].Services[0].Hosts = g.Ingress.Listeners[0].Services[0].Hosts[:1] })
})
t.Run("mod ingress listeners services hosts content", func(t *testing.T) {
try(t, func(g *gway) { g.Ingress.Listeners[0].Services[0].Hosts[0] = "255.255.255.255" })
})
}
func TestConsulGateway_ingressServicesEqual(t *testing.T) {
igs1 := []*ConsulIngressService{{
Name: "service1",
Hosts: []string{"host1", "host2"},
}, {
Name: "service2",
Hosts: []string{"host3"},
}}
require.False(t, ingressServicesEqual(igs1, nil))
reversed := []*ConsulIngressService{
igs1[1], igs1[0], // services reversed
}
require.True(t, ingressServicesEqual(igs1, reversed))
hostOrder := []*ConsulIngressService{{
Name: "service1",
Hosts: []string{"host2", "host1"}, // hosts reversed
}, {
Name: "service2",
Hosts: []string{"host3"},
}}
require.True(t, ingressServicesEqual(igs1, hostOrder))
}
func TestConsulGateway_ingressListenersEqual(t *testing.T) {
ils1 := []*ConsulIngressListener{{
Port: 2000,
Protocol: "http",
Services: []*ConsulIngressService{{
Name: "service1",
Hosts: []string{"host1", "host2"},
}},
}, {
Port: 2001,
Protocol: "tcp",
Services: []*ConsulIngressService{{
Name: "service2",
}},
}}
require.False(t, ingressListenersEqual(ils1, nil))
reversed := []*ConsulIngressListener{
ils1[1], ils1[0],
}
require.True(t, ingressListenersEqual(ils1, reversed))
}
func TestConsulGateway_Validate(t *testing.T) {
t.Run("bad proxy", func(t *testing.T) {
err := (&ConsulGateway{
Proxy: &ConsulGatewayProxy{
ConnectTimeout: nil,
},
Ingress: nil,
}).Validate()
require.EqualError(t, err, "Consul Gateway Proxy connection_timeout must be set")
})
t.Run("bad ingress config entry", func(t *testing.T) {
err := (&ConsulGateway{
Ingress: &ConsulIngressConfigEntry{
Listeners: nil,
},
}).Validate()
require.EqualError(t, err, "Consul Ingress Gateway requires at least one listener")
})
}
func TestConsulGatewayBindAddress_Validate(t *testing.T) {
t.Run("no address", func(t *testing.T) {
err := (&ConsulGatewayBindAddress{
Address: "",
Port: 2000,
}).Validate()
require.EqualError(t, err, "Consul Gateway Bind Address must be set")
})
t.Run("invalid port", func(t *testing.T) {
err := (&ConsulGatewayBindAddress{
Address: "10.0.0.1",
Port: 0,
}).Validate()
require.EqualError(t, err, "Consul Gateway Bind Address must set valid Port")
})
t.Run("ok", func(t *testing.T) {
err := (&ConsulGatewayBindAddress{
Address: "10.0.0.1",
Port: 2000,
}).Validate()
require.NoError(t, err)
})
}
func TestConsulGatewayProxy_Validate(t *testing.T) {
t.Run("no timeout", func(t *testing.T) {
err := (&ConsulGatewayProxy{
ConnectTimeout: nil,
}).Validate()
require.EqualError(t, err, "Consul Gateway Proxy connection_timeout must be set")
})
t.Run("invalid bind address", func(t *testing.T) {
err := (&ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(1 * time.Second),
EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{
"service1": {
Address: "10.0.0.1",
Port: 0,
}},
}).Validate()
require.EqualError(t, err, "Consul Gateway Bind Address must set valid Port")
})
t.Run("ok with nothing set", func(t *testing.T) {
err := (&ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(1 * time.Second),
}).Validate()
require.NoError(t, err)
})
t.Run("ok with everything set", func(t *testing.T) {
err := (&ConsulGatewayProxy{
ConnectTimeout: helper.TimeToPtr(1 * time.Second),
EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{
"service1": {
Address: "10.0.0.1",
Port: 2000,
}},
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayNoDefaultBind: true,
}).Validate()
require.NoError(t, err)
})
}
func TestConsulIngressService_Validate(t *testing.T) {
t.Run("invalid name", func(t *testing.T) {
err := (&ConsulIngressService{
Name: "",
}).Validate(true)
require.EqualError(t, err, "Consul Ingress Service requires a name")
})
t.Run("http missing hosts", func(t *testing.T) {
err := (&ConsulIngressService{
Name: "service1",
}).Validate(true)
require.EqualError(t, err, "Consul Ingress Service requires one or more hosts when using HTTP protocol")
})
t.Run("tcp extraneous hosts", func(t *testing.T) {
err := (&ConsulIngressService{
Name: "service1",
Hosts: []string{"host1"},
}).Validate(false)
require.EqualError(t, err, "Consul Ingress Service supports hosts only when using HTTP protocol")
})
t.Run("ok tcp", func(t *testing.T) {
err := (&ConsulIngressService{
Name: "service1",
}).Validate(false)
require.NoError(t, err)
})
t.Run("ok http", func(t *testing.T) {
err := (&ConsulIngressService{
Name: "service1",
Hosts: []string{"host1"},
}).Validate(true)
require.NoError(t, err)
})
}
func TestConsulIngressListener_Validate(t *testing.T) {
t.Run("invalid port", func(t *testing.T) {
err := (&ConsulIngressListener{
Port: 0,
Protocol: "tcp",
Services: []*ConsulIngressService{{
Name: "service1",
}},
}).Validate()
require.EqualError(t, err, "Consul Ingress Listener requires valid Port")
})
t.Run("invalid protocol", func(t *testing.T) {
err := (&ConsulIngressListener{
Port: 2000,
Protocol: "gopher",
Services: []*ConsulIngressService{{
Name: "service1",
}},
}).Validate()
require.EqualError(t, err, `Consul Ingress Listener requires protocol of "http" or "tcp", got "gopher"`)
})
t.Run("no services", func(t *testing.T) {
err := (&ConsulIngressListener{
Port: 2000,
Protocol: "tcp",
Services: nil,
}).Validate()
require.EqualError(t, err, "Consul Ingress Listener requires one or more services")
})
t.Run("invalid service", func(t *testing.T) {
err := (&ConsulIngressListener{
Port: 2000,
Protocol: "tcp",
Services: []*ConsulIngressService{{
Name: "",
}},
}).Validate()
require.EqualError(t, err, "Consul Ingress Service requires a name")
})
t.Run("ok", func(t *testing.T) {
err := (&ConsulIngressListener{
Port: 2000,
Protocol: "tcp",
Services: []*ConsulIngressService{{
Name: "service1",
}},
}).Validate()
require.NoError(t, err)
})
}
func TestConsulIngressConfigEntry_Validate(t *testing.T) {
t.Run("no listeners", func(t *testing.T) {
err := (&ConsulIngressConfigEntry{}).Validate()
require.EqualError(t, err, "Consul Ingress Gateway requires at least one listener")
})
t.Run("invalid listener", func(t *testing.T) {
err := (&ConsulIngressConfigEntry{
Listeners: []*ConsulIngressListener{{
Port: 9000,
Protocol: "tcp",
}},
}).Validate()
require.EqualError(t, err, "Consul Ingress Listener requires one or more services")
})
t.Run("full", func(t *testing.T) {
err := (&ConsulIngressConfigEntry{
TLS: &ConsulGatewayTLSConfig{
Enabled: true,
},
Listeners: []*ConsulIngressListener{{
Port: 9000,
Protocol: "tcp",
Services: []*ConsulIngressService{{
Name: "service1",
}},
}},
}).Validate()
require.NoError(t, err)
})
}

93
nomad/structs/structs.go
View file

@ -4241,9 +4241,9 @@ func (j *Job) ConnectTasks() map[string][]string {
m := make(map[string][]string)
for _, tg := range j.TaskGroups {
for _, task := range tg.Tasks {
if task.Kind.IsConnectProxy() {
// todo(shoenig): when we support native, probably need to check
// an additional TBD TaskKind as well.
if task.Kind.IsConnectProxy() ||
task.Kind.IsConnectNative() ||
task.Kind.IsAnyConnectGateway() {
m[tg.Name] = append(m[tg.Name], task.Name)
}
}
@ -4251,6 +4251,25 @@ func (j *Job) ConnectTasks() map[string][]string {
return m
}
// ConfigEntries accumulates the Consul Configuration Entries defined in task groups
// of j.
//
// Currently Nomad only supports entries for connect ingress gateways.
func (j *Job) ConfigEntries() map[string]*ConsulIngressConfigEntry {
igEntries := make(map[string]*ConsulIngressConfigEntry)
for _, tg := range j.TaskGroups {
for _, service := range tg.Services {
if service.Connect.IsGateway() {
if ig := service.Connect.Gateway.Ingress; ig != nil {
igEntries[service.Name] = ig
}
// imagine also accumulating other entry types in the future
}
}
}
return igEntries
}
// RequiredSignals returns a mapping of task groups to tasks to their required
// set of signals
func (j *Job) RequiredSignals() map[string]map[string][]string {
@ -5627,8 +5646,10 @@ func (tg *TaskGroup) Validate(j *Job) error {
mErr.Errors = append(mErr.Errors, errors.New("Task group count can't be negative"))
}
if len(tg.Tasks) == 0 {
// could be a lone consul gateway inserted by the connect mutator
mErr.Errors = append(mErr.Errors, errors.New("Missing tasks for task group"))
}
for idx, constr := range tg.Constraints {
if err := constr.Validate(); err != nil {
outer := fmt.Errorf("Constraint %d validation failed: %s", idx+1, err)
@ -5979,10 +6000,28 @@ func (tg *TaskGroup) LookupTask(name string) *Task {
return nil
}
// UsesConnect for convenience returns true if the TaskGroup contains at least
// one service that makes use of Consul Connect features.
//
// Currently used for validating that the task group contains one or more connect
// aware services before generating a service identity token.
func (tg *TaskGroup) UsesConnect() bool {
for _, service := range tg.Services {
if service.Connect != nil {
if service.Connect.IsNative() || service.Connect.HasSidecar() {
if service.Connect.IsNative() || service.Connect.HasSidecar() || service.Connect.IsGateway() {
return true
}
}
}
return false
}
// UsesConnectGateway for convenience returns true if the TaskGroup contains at
// least one service that makes use of Consul Connect Gateway features.
func (tg *TaskGroup) UsesConnectGateway() bool {
for _, service := range tg.Services {
if service.Connect != nil {
if service.Connect.IsGateway() {
return true
}
}
@ -6183,9 +6222,9 @@ type Task struct {
// UsesConnect is for conveniently detecting if the Task is able to make use
// of Consul Connect features. This will be indicated in the TaskKind of the
// Task, which exports known types of Tasks. UsesConnect will be true if the
// task is a connect proxy, or if the task is connect native.
// task is a connect proxy, connect native, or is a connect gateway.
func (t *Task) UsesConnect() bool {
return t.Kind.IsConnectProxy() || t.Kind.IsConnectNative()
return t.Kind.IsConnectProxy() || t.Kind.IsConnectNative() || t.Kind.IsAnyConnectGateway()
}
func (t *Task) Copy() *Task {
@ -6621,13 +6660,31 @@ func (k TaskKind) Value() string {
return ""
}
// IsConnectProxy returns true if the TaskKind is connect-proxy
func (k TaskKind) IsConnectProxy() bool {
return strings.HasPrefix(string(k), ConnectProxyPrefix+":") && len(k) > len(ConnectProxyPrefix)+1
func (k TaskKind) hasPrefix(prefix string) bool {
return strings.HasPrefix(string(k), prefix+":") && len(k) > len(prefix)+1
}
// IsConnectProxy returns true if the TaskKind is connect-proxy.
func (k TaskKind) IsConnectProxy() bool {
return k.hasPrefix(ConnectProxyPrefix)
}
// IsConnectNative returns true if the TaskKind is connect-native.
func (k TaskKind) IsConnectNative() bool {
return strings.HasPrefix(string(k), ConnectNativePrefix+":") && len(k) > len(ConnectNativePrefix)+1
return k.hasPrefix(ConnectNativePrefix)
}
func (k TaskKind) IsConnectIngress() bool {
return k.hasPrefix(ConnectIngressPrefix)
}
func (k TaskKind) IsAnyConnectGateway() bool {
switch {
case k.IsConnectIngress():
return true
default:
return false
}
}
const (
@ -6638,6 +6695,22 @@ const (
// ConnectNativePrefix is the prefix used for fields referencing a Connect
// Native Task
ConnectNativePrefix = "connect-native"
// ConnectIngressPrefix is the prefix used for fields referencing a Consul
// Connect Ingress Gateway Proxy.
ConnectIngressPrefix = "connect-ingress"
// ConnectTerminatingPrefix is the prefix used for fields referencing a Consul
// Connect Terminating Gateway Proxy.
//
// Not yet supported.
// ConnectTerminatingPrefix = "connect-terminating"
// ConnectMeshPrefix is the prefix used for fields referencing a Consul Connect
// Mesh Gateway Proxy.
//
// Not yet supported.
// ConnectMeshPrefix = "connect-mesh"
)
// ValidateConnectProxyService checks that the service that is being

19
nomad/structs/structs_test.go
View file

@ -828,6 +828,15 @@ func TestTask_UsesConnect(t *testing.T) {
usesConnect := task.UsesConnect()
require.True(t, usesConnect)
})
t.Run("ingress gateway", func(t *testing.T) {
task := &Task{
Name: "task1",
Kind: NewTaskKind(ConnectIngressPrefix, "task1"),
}
usesConnect := task.UsesConnect()
require.True(t, usesConnect)
})
}
func TestTaskGroup_UsesConnect(t *testing.T) {
@ -859,6 +868,16 @@ func TestTaskGroup_UsesConnect(t *testing.T) {
}, true)
})
t.Run("tg uses gateway", func(t *testing.T) {
try(t, &TaskGroup{
Services: []*Service{{
Connect: &ConsulConnect{
Gateway: consulIngressGateway1,
},
}},
}, true)
})
t.Run("tg does not use connect", func(t *testing.T) {
try(t, &TaskGroup{
Services: []*Service{

8
nomad/testing.go
View file

@ -99,9 +99,9 @@ func TestServer(t testing.T, cb func(*Config)) (*Server, func()) {
cb(config)
}
catalog := consul.NewMockCatalog(config.Logger)
acls := consul.NewMockACLsAPI(config.Logger)
cCatalog := consul.NewMockCatalog(config.Logger)
cConfigs := consul.NewMockConfigsAPI(config.Logger)
cACLs := consul.NewMockACLsAPI(config.Logger)
for i := 10; i >= 0; i-- {
// Get random ports, need to cleanup later
@ -114,7 +114,7 @@ func TestServer(t testing.T, cb func(*Config)) (*Server, func()) {
config.SerfConfig.MemberlistConfig.BindPort = ports[1]
// Create server
server, err := NewServer(config, catalog, acls)
server, err := NewServer(config, cCatalog, cConfigs, cACLs)
if err == nil {
return server, func() {
ch := make(chan error)

262
vendor/github.com/hashicorp/nomad/api/services.go generated vendored
View file

@ -152,6 +152,7 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) {
// ConsulConnect represents a Consul Connect jobspec stanza.
type ConsulConnect struct {
Native bool
Gateway *ConsulGateway
SidecarService *ConsulSidecarService `mapstructure:"sidecar_service"`
SidecarTask *SidecarTask `mapstructure:"sidecar_task"`
}
@ -163,6 +164,7 @@ func (cc *ConsulConnect) Canonicalize() {
cc.SidecarService.Canonicalize()
cc.SidecarTask.Canonicalize()
cc.Gateway.Canonicalize()
}
// ConsulSidecarService represents a Consul Connect SidecarService jobspec
@ -290,3 +292,263 @@ type ConsulExposePath struct {
LocalPathPort int `mapstructure:"local_path_port"`
ListenerPort string `mapstructure:"listener_port"`
}
// ConsulGateway is used to configure one of the Consul Connect Gateway types.
type ConsulGateway struct {
// Proxy is used to configure the Envoy instance acting as the gateway.
Proxy *ConsulGatewayProxy
// Ingress represents the Consul Configuration Entry for an Ingress Gateway.
Ingress *ConsulIngressConfigEntry
// Terminating is not yet supported.
// Terminating *ConsulTerminatingConfigEntry
// Mesh is not yet supported.
// Mesh *ConsulMeshConfigEntry
}
func (g *ConsulGateway) Canonicalize() {
if g == nil {
return
}
g.Proxy.Canonicalize()
g.Ingress.Canonicalize()
}
func (g *ConsulGateway) Copy() *ConsulGateway {
if g == nil {
return nil
}
return &ConsulGateway{
Proxy: g.Proxy.Copy(),
Ingress: g.Ingress.Copy(),
}
}
type ConsulGatewayBindAddress struct {
Address string `mapstructure:"address"`
Port int `mapstructure:"port"`
}
var (
defaultGatewayConnectTimeout = 5 * time.Second
)
// ConsulGatewayProxy is used to tune parameters of the proxy instance acting as
// one of the forms of Connect gateways that Consul supports.
//
// https://www.consul.io/docs/connect/proxies/envoy#gateway-options
type ConsulGatewayProxy struct {
ConnectTimeout *time.Duration `mapstructure:"connect_timeout"`
EnvoyGatewayBindTaggedAddresses bool `mapstructure:"envoy_gateway_bind_tagged_addresses"`
EnvoyGatewayBindAddresses map[string]*ConsulGatewayBindAddress `mapstructure:"envoy_gateway_bind_addresses"`
EnvoyGatewayNoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind"`
Config map[string]interface{} // escape hatch envoy config
}
func (p *ConsulGatewayProxy) Canonicalize() {
if p == nil {
return
}
if p.ConnectTimeout == nil {
// same as the default from consul
p.ConnectTimeout = timeToPtr(defaultGatewayConnectTimeout)
}
if len(p.EnvoyGatewayBindAddresses) == 0 {
p.EnvoyGatewayBindAddresses = nil
}
if len(p.Config) == 0 {
p.Config = nil
}
}
func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy {
if p == nil {
return nil
}
var binds map[string]*ConsulGatewayBindAddress = nil
if p.EnvoyGatewayBindAddresses != nil {
binds = make(map[string]*ConsulGatewayBindAddress, len(p.EnvoyGatewayBindAddresses))
for k, v := range p.EnvoyGatewayBindAddresses {
binds[k] = v
}
}
var config map[string]interface{} = nil
if p.Config != nil {
config = make(map[string]interface{}, len(p.Config))
for k, v := range p.Config {
config[k] = v
}
}
return &ConsulGatewayProxy{
ConnectTimeout: timeToPtr(*p.ConnectTimeout),
EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses,
EnvoyGatewayBindAddresses: binds,
EnvoyGatewayNoDefaultBind: p.EnvoyGatewayNoDefaultBind,
Config: config,
}
}
// ConsulGatewayTLSConfig is used to configure TLS for a gateway.
type ConsulGatewayTLSConfig struct {
Enabled bool
}
func (tc *ConsulGatewayTLSConfig) Canonicalize() {
}
func (tc *ConsulGatewayTLSConfig) Copy() *ConsulGatewayTLSConfig {
if tc == nil {
return nil
}
return &ConsulGatewayTLSConfig{
Enabled: tc.Enabled,
}
}
// ConsulIngressService is used to configure a service fronted by the ingress gateway.
type ConsulIngressService struct {
// Namespace is not yet supported.
// Namespace string
Name string
Hosts []string
}
func (s *ConsulIngressService) Canonicalize() {
if s == nil {
return
}
if len(s.Hosts) == 0 {
s.Hosts = nil
}
}
func (s *ConsulIngressService) Copy() *ConsulIngressService {
if s == nil {
return nil
}
var hosts []string = nil
if n := len(s.Hosts); n > 0 {
hosts = make([]string, n)
copy(hosts, s.Hosts)
}
return &ConsulIngressService{
Name: s.Name,
Hosts: hosts,
}
}
const (
defaultIngressListenerProtocol = "tcp"
)
// ConsulIngressListener is used to configure a listener on a Consul Ingress
// Gateway.
type ConsulIngressListener struct {
Port int
Protocol string
Services []*ConsulIngressService
}
func (l *ConsulIngressListener) Canonicalize() {
if l == nil {
return
}
if l.Protocol == "" {
// same as default from consul
l.Protocol = defaultIngressListenerProtocol
}
if len(l.Services) == 0 {
l.Services = nil
}
}
func (l *ConsulIngressListener) Copy() *ConsulIngressListener {
if l == nil {
return nil
}
var services []*ConsulIngressService = nil
if n := len(l.Services); n > 0 {
services = make([]*ConsulIngressService, n)
for i := 0; i < n; i++ {
services[i] = l.Services[i].Copy()
}
}
return &ConsulIngressListener{
Port: l.Port,
Protocol: l.Protocol,
Services: services,
}
}
// ConsulIngressConfigEntry represents the Consul Configuration Entry type for
// an Ingress Gateway.
//
// https://www.consul.io/docs/agent/config-entries/ingress-gateway#available-fields
type ConsulIngressConfigEntry struct {
// Namespace is not yet supported.
// Namespace string
TLS *ConsulGatewayTLSConfig
Listeners []*ConsulIngressListener
}
func (e *ConsulIngressConfigEntry) Canonicalize() {
if e == nil {
return
}
e.TLS.Canonicalize()
if len(e.Listeners) == 0 {
e.Listeners = nil
}
for _, listener := range e.Listeners {
listener.Canonicalize()
}
}
func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry {
if e == nil {
return nil
}
var listeners []*ConsulIngressListener = nil
if n := len(e.Listeners); n > 0 {
listeners = make([]*ConsulIngressListener, n)
for i := 0; i < n; i++ {
listeners[i] = e.Listeners[i].Copy()
}
}
return &ConsulIngressConfigEntry{
TLS: e.TLS.Copy(),
Listeners: listeners,
}
}
// ConsulTerminatingConfigEntry is not yet supported.
// type ConsulTerminatingConfigEntry struct {
// }
// ConsulMeshConfigEntry is not yet supported.
// type ConsulMeshConfigEntry struct {
// }

1
website/data/docs-navigation.js
View file

@ -166,6 +166,7 @@ export default [
'env',
'ephemeral_disk',
'expose',
'gateway',
'group',
'job',
'lifecycle',

207
website/pages/docs/job-specification/gateway.mdx Normal file
View file

@ -0,0 +1,207 @@
---
layout: docs
page_title: gateway Stanza - Job Specification
sidebar_title: gateway
description: |-
The "gateway" stanza allows specifying options for configuring Consul Gateways
used in the Consul Connect integration
---
# `gateway` Stanza
<Placement
groups={[
'job',
'group',
'service',
'connect',
'gateway',
]}
/>
The `gateway` stanza allows configuration of [Consul Connect Gateways](https://www.consul.io/docs/connect/gateways). Nomad will
automatically create the necessary Gateway [Configuration Entry](https://www.consul.io/docs/agent/config-entries)
as well as inject an Envoy proxy task into the Nomad job to serve as the Gateway.
The `gateway` configuration is valid within the context of a `connect` stanza.
Additional information about Gateway configurations can be found in Consul's
[Connect Gateways](https://www.consul.io/docs/connect/gateways) documentation.
~> **Note:** [Ingress Gateways](https://www.consul.io/docs/connect/gateways/ingress-gateway)
are generally intended for enabling access into a Consul service mesh from within the
same network. For public ingress products like [NGINX](https://learn.hashicorp.com/tutorials/nomad/load-balancing-nginx?in=nomad/load-balancing)
provide more suitable features.
```hcl
job "ingress-example" {
datacenters = ["dc1"]
group "ingress-group" {
network {
mode = "bridge"
port "inbound" {
static = 8080
}
}
service {
name = "ingress-service"
port = "8080"
connect {
gateway {
proxy {
// Consul Gateway Proxy configuration options
connect_timeout = "500ms"
}
ingress {
// Consul Ingress Gateway Configuration Entry
tls {
enabled = false
}
listener {
port = 8080
protocol = "http"
service {
name = "web"
hosts = ["example.com", "example.com:8080"]
}
}
listener {
port = 3306
protocol = "tcp"
service {
name = "database"
}
}
}
}
}
}
}
}
```
## `gateway` Parameters
- `proxy` <code>([proxy]: nil)</code> - Configuration of the Envoy proxy that will
be injected into the task group.
- `ingress` <code>([ingress]: nil)</code> - Configuration Entry of type `ingress-gateway`
that will be associated with the service.
### `proxy` Parameters
- `connect_timeout` `(string: "5s")` - The amount of time to allow when making upstream
connections before timing out. Defaults to 5 seconds. If the upstream service has
the configuration option <code>[connect_timeout_ms]</code> set for the `service-resolver`, that
timeout value will take precedence over this gateway proxy option.
- `envoy_gateway_bind_tagged_addresses` `(bool: false)` - Indicates that the gateway
services tagged addresses should be bound to listeners in addition to the default
listener address.
- `envoy_gateway_bind_addresses` <code>(map<string|[address]>: nil)</code> - A map of additional addresses to be bound.
The keys to this map are the same of the listeners to be created and the values are
a map with two keys - address and port, that combined make the address to bind the
listener to. These are bound in addition to the default address.
If `bridge` networking is in use, this map is automatically populated with additional
listeners enabling the Envoy proxy to work from inside the network namespace.
```
envoy_gateway_bind_addresses "<service>" {
address = "0.0.0.0"
port = <port>
}
```
- `envoy_gateway_no_default_bind` `(bool: false)` - Prevents binding to the default
address of the gateway service. This should be used with one of the other options
to configure the gateway's bind addresses. If `bridge` networking is in use, this
value will default to `true` since the Envoy proxy does not need to bind to the
service address from inside the network namespace.
- `config` `(map: nil)` - Escape hatch for [Advanced Configuration] of Envoy.
#### `address` Parameters
- `address` `(string: required)` - The address to bind to when combined with `port`.
- `port` `(int: required)` - The port to listen to.
### `ingress` Parameters
- `tls` <code>([tls]: nil)</code> - TLS configuration for this gateway.
- `listener` <code>(array<[listener]> : required)</code> - One or more listeners that the
ingress gateway should setup, uniquely identified by their port number.
#### `tls` Parameters
- `enabled` `(bool: false)` - Set this configuration to enable TLS for every listener
on the gateway. If TLS is enabled, then each host defined in the `host` field will
be added as a DNSSAN to the gateway's x509 certificate.
#### `listener` Parameters
- `port` `(int: required)` - The port that the listener should receive traffic on.
- `protocol` `(string: "tcp")` - The protocol associated with the listener. Either
`tcp` or `http`.
~> **Note:** If using `http`, preconfiguring a [service-default] in Consul to
set the [Protocol](https://www.consul.io/docs/agent/config-entries/service-defaults#protocol)
of the service to `http` is recommended.
- `service` <code>(array<[service]>: required)</code> - One or more services to be
exposed via this listener. For `tcp` listeners, only a single service is allowed.
#### `service` Parameters
- `name` `(string: required)` - The name of the service that should be exposed through
this listener. This can be either a service registered in the catalog, or a
service defined by other config entries, or a service that is going to be configured
by Nomad. If the wildcard specifier `*` is provided, then ALL services will be
exposed through this listener. This is not supported for a listener with protocol `tcp`.
- `hosts` `(array<string>: nil)` - A list of hosts that specify what requests will
match this service. This cannot be used with a `tcp` listener, and cannot be
specified alongside a wildcard (`*`) service name. If not specified, the default
domain `<service-name>.ingress.*` will be used to match services. Requests *must*
send the correct host to be routed to the defined service.
The wildcard specifier `*` can be used by itself to match all traffic coming to
the ingress gateway, if TLS is not enabled. This allows a user to route all traffic
to a single service without specifying a host, allowing simpler tests and demos.
Otherwise, the wildcard specifier can be used as part of the host to match
multiple hosts, but only in the leftmost DNS label. This ensures that all defined
hosts are valid DNS records. For example, `*.example.com` is valid while `example.*`
and `*-suffix.example.com` are not.
~> **Note:** If a well-known port is not used, i.e. a port other than 80 (http) or 443 (https),
then the port must be appended to the host to correctly match traffic. This is
defined in the [HTTP/1.1 RFC](https://tools.ietf.org/html/rfc2616#section-14.23).
If TLS is enabled, then the host **without** the port must be added to the `hosts`
field as well. TLS verification only matches against the hostname of the incoming
connection, and does not take into account the port.
### Gateway with host networking
Nomad supports running gateways using host networking. A static port must be allocated
for use by the [Envoy admin interface](https://www.envoyproxy.io/docs/envoy/latest/operations/admin)
and assigned to the proxy service definition.
!> **Warning:** There is no way to disable the Envoy admin interface, which will be
accessible to any workload running on the same Nomad client. The admin interface exposes
information about the proxy, including a Consul Service Identity token if Consul ACLs
are enabled.
[proxy]: /docs/job-specification/gateway#proxy-parameters
[ingress]: /docs/job-specification/gateway#ingress-parameters
[tls]: /docs/job-specification/gateway#tlsconfig-parameters
[listener]: /docs/job-specification/gateway#listener-parameters
[service]: /docs/job-specification/gateway#service-parameters
[service-default]: https://www.consul.io/docs/agent/config-entries/service-defaults
[connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout
[address]: /docs/job-specification/gateway#address-parameters
[Advanced Configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration