open-nomad/nomad/job_endpoint_hook_expose_ch...

656 lines
17 KiB
Go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package nomad
import (
"testing"
"github.com/hashicorp/nomad/ci"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/stretchr/testify/require"
)
func TestJobExposeCheckHook_Name(t *testing.T) {
ci.Parallel(t)
require.Equal(t, "expose-check", new(jobExposeCheckHook).Name())
}
func TestJobExposeCheckHook_tgUsesExposeCheck(t *testing.T) {
ci.Parallel(t)
t.Run("no check.expose", func(t *testing.T) {
require.False(t, tgUsesExposeCheck(&structs.TaskGroup{
Services: []*structs.Service{{
Checks: []*structs.ServiceCheck{{
Expose: false,
}},
}},
}))
})
t.Run("with check.expose", func(t *testing.T) {
require.True(t, tgUsesExposeCheck(&structs.TaskGroup{
Services: []*structs.Service{{
Checks: []*structs.ServiceCheck{{
Expose: false,
}, {
Expose: true,
}},
}},
}))
})
}
func TestJobExposeCheckHook_tgValidateUseOfBridgeMode(t *testing.T) {
ci.Parallel(t)
s1 := &structs.Service{
Name: "s1",
Checks: []*structs.ServiceCheck{{
Name: "s1-check1",
Type: "http",
PortLabel: "health",
Expose: true,
}},
}
t.Run("no networks but no use of expose", func(t *testing.T) {
require.Nil(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{
Networks: make(structs.Networks, 0),
}))
})
t.Run("no networks and uses expose", func(t *testing.T) {
require.EqualError(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{
Name: "g1",
Networks: make(structs.Networks, 0),
Services: []*structs.Service{s1},
}), `group "g1" must specify one bridge network for exposing service check(s)`)
})
t.Run("non-bridge network and uses expose", func(t *testing.T) {
require.EqualError(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{
Name: "g1",
Networks: structs.Networks{{
Mode: "host",
}},
Services: []*structs.Service{s1},
}), `group "g1" must use bridge network for exposing service check(s)`)
})
t.Run("bridge network uses expose", func(t *testing.T) {
require.Nil(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{
Name: "g1",
Networks: structs.Networks{{
Mode: "bridge",
}},
Services: []*structs.Service{s1},
}))
})
}
func TestJobExposeCheckHook_tgValidateUseOfCheckExpose(t *testing.T) {
ci.Parallel(t)
withCustomProxyTask := &structs.Service{
Name: "s1",
Connect: &structs.ConsulConnect{
SidecarTask: &structs.SidecarTask{Name: "custom"},
},
Checks: []*structs.ServiceCheck{{
Name: "s1-check1",
Type: "http",
PortLabel: "health",
Expose: true,
}},
}
t.Run("group-service uses custom proxy", func(t *testing.T) {
require.EqualError(t, tgValidateUseOfCheckExpose(&structs.TaskGroup{
Name: "g1",
Services: []*structs.Service{withCustomProxyTask},
}), `exposed service check g1->s1->s1-check1 requires use of sidecar_proxy`)
})
t.Run("group-service uses custom proxy but no expose", func(t *testing.T) {
withCustomProxyTaskNoExpose := *withCustomProxyTask
withCustomProxyTask.Checks[0].Expose = false
require.Nil(t, tgValidateUseOfCheckExpose(&structs.TaskGroup{
Name: "g1",
Services: []*structs.Service{&withCustomProxyTaskNoExpose},
}))
})
t.Run("task-service sets expose", func(t *testing.T) {
require.EqualError(t, tgValidateUseOfCheckExpose(&structs.TaskGroup{
Name: "g1",
Tasks: []*structs.Task{{
Name: "t1",
Services: []*structs.Service{{
Name: "s2",
Checks: []*structs.ServiceCheck{{
Name: "check1",
Type: "http",
Expose: true,
}},
}},
}},
}), `exposed service check g1[t1]->s2->check1 is not a task-group service`)
})
}
func TestJobExposeCheckHook_Validate(t *testing.T) {
ci.Parallel(t)
s1 := &structs.Service{
Name: "s1",
Checks: []*structs.ServiceCheck{{
Name: "s1-check1",
Type: "http",
Expose: true,
}},
}
t.Run("double network", func(t *testing.T) {
warnings, err := new(jobExposeCheckHook).Validate(&structs.Job{
TaskGroups: []*structs.TaskGroup{{
Name: "g1",
Networks: structs.Networks{{
Mode: "bridge",
}, {
Mode: "bridge",
}},
Services: []*structs.Service{s1},
}},
})
require.Empty(t, warnings)
require.EqualError(t, err, `group "g1" must specify one bridge network for exposing service check(s)`)
})
t.Run("expose in service check", func(t *testing.T) {
warnings, err := new(jobExposeCheckHook).Validate(&structs.Job{
TaskGroups: []*structs.TaskGroup{{
Name: "g1",
Networks: structs.Networks{{
Mode: "bridge",
}},
Tasks: []*structs.Task{{
Name: "t1",
Services: []*structs.Service{{
Name: "s2",
Checks: []*structs.ServiceCheck{{
Name: "s2-check1",
Type: "http",
Expose: true,
}},
}},
}},
}},
})
require.Empty(t, warnings)
require.EqualError(t, err, `exposed service check g1[t1]->s2->s2-check1 is not a task-group service`)
})
t.Run("ok", func(t *testing.T) {
warnings, err := new(jobExposeCheckHook).Validate(&structs.Job{
TaskGroups: []*structs.TaskGroup{{
Name: "g1",
Networks: structs.Networks{{
Mode: "bridge",
}},
Services: []*structs.Service{{
Name: "s1",
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{},
},
Checks: []*structs.ServiceCheck{{
Name: "check1",
Type: "http",
Expose: true,
}},
}},
Tasks: []*structs.Task{{
Name: "t1",
Services: []*structs.Service{{
Name: "s2",
Checks: []*structs.ServiceCheck{{
Name: "s2-check1",
Type: "http",
Expose: false,
}},
}},
}},
}},
})
require.Empty(t, warnings)
require.Nil(t, err)
})
}
func TestJobExposeCheckHook_exposePathForCheck(t *testing.T) {
ci.Parallel(t)
const checkIdx = 0
t.Run("not expose compatible", func(t *testing.T) {
c := &structs.ServiceCheck{
Type: "tcp", // not expose compatible
}
s := &structs.Service{
Checks: []*structs.ServiceCheck{c},
}
ePath, err := exposePathForCheck(&structs.TaskGroup{
Services: []*structs.Service{s},
}, s, c, checkIdx)
require.NoError(t, err)
require.Nil(t, ePath)
})
t.Run("direct port", func(t *testing.T) {
c := &structs.ServiceCheck{
Name: "check1",
Type: "http",
Path: "/health",
PortLabel: "hcPort",
}
s := &structs.Service{
Name: "service1",
PortLabel: "4000",
Checks: []*structs.ServiceCheck{c},
}
ePath, err := exposePathForCheck(&structs.TaskGroup{
Name: "group1",
Services: []*structs.Service{s},
}, s, c, checkIdx)
require.NoError(t, err)
require.Equal(t, &structs.ConsulExposePath{
Path: "/health",
Protocol: "", // often blank, consul does the Right Thing
LocalPathPort: 4000,
ListenerPort: "hcPort",
}, ePath)
})
t.Run("labeled port", func(t *testing.T) {
c := &structs.ServiceCheck{
Name: "check1",
Type: "http",
Path: "/health",
PortLabel: "hcPort",
}
s := &structs.Service{
Name: "service1",
PortLabel: "sPort", // port label indirection
Checks: []*structs.ServiceCheck{c},
}
ePath, err := exposePathForCheck(&structs.TaskGroup{
Name: "group1",
Services: []*structs.Service{s},
Networks: structs.Networks{{
Mode: "bridge",
DynamicPorts: []structs.Port{
{Label: "sPort", Value: 4000},
},
}},
}, s, c, checkIdx)
require.NoError(t, err)
require.Equal(t, &structs.ConsulExposePath{
Path: "/health",
Protocol: "",
LocalPathPort: 4000,
ListenerPort: "hcPort",
}, ePath)
})
t.Run("missing port", func(t *testing.T) {
c := &structs.ServiceCheck{
Name: "check1",
Type: "http",
Path: "/health",
PortLabel: "hcPort",
}
s := &structs.Service{
Name: "service1",
PortLabel: "sPort", // port label indirection
Checks: []*structs.ServiceCheck{c},
}
_, err := exposePathForCheck(&structs.TaskGroup{
Name: "group1",
Services: []*structs.Service{s},
Networks: structs.Networks{{
Mode: "bridge",
DynamicPorts: []structs.Port{
// service declares "sPort", but does not exist
},
}},
}, s, c, checkIdx)
require.EqualError(t, err, `unable to determine local service port for service check group1->service1->check1`)
})
t.Run("empty check port", func(t *testing.T) {
setup := func() (*structs.TaskGroup, *structs.Service, *structs.ServiceCheck) {
c := &structs.ServiceCheck{
Name: "check1",
Type: "http",
Path: "/health",
}
s := &structs.Service{
Name: "service1",
PortLabel: "9999",
Checks: []*structs.ServiceCheck{c},
}
tg := &structs.TaskGroup{
Name: "group1",
Services: []*structs.Service{s},
Networks: structs.Networks{{
Mode: "bridge",
DynamicPorts: []structs.Port{},
}},
}
return tg, s, c
}
tg, s, c := setup()
ePath, err := exposePathForCheck(tg, s, c, checkIdx)
require.NoError(t, err)
require.Len(t, tg.Networks[0].DynamicPorts, 1)
require.Equal(t, "default", tg.Networks[0].DynamicPorts[0].HostNetwork)
require.Equal(t, "svc_", tg.Networks[0].DynamicPorts[0].Label[0:4])
require.Equal(t, &structs.ConsulExposePath{
Path: "/health",
Protocol: "",
LocalPathPort: 9999,
ListenerPort: tg.Networks[0].DynamicPorts[0].Label,
}, ePath)
t.Run("deterministic generated port label", func(t *testing.T) {
tg2, s2, c2 := setup()
ePath2, err2 := exposePathForCheck(tg2, s2, c2, checkIdx)
require.NoError(t, err2)
require.Equal(t, ePath, ePath2)
})
t.Run("unique on check index", func(t *testing.T) {
tg3, s3, c3 := setup()
ePath3, err3 := exposePathForCheck(tg3, s3, c3, checkIdx+1)
require.NoError(t, err3)
require.NotEqual(t, ePath.ListenerPort, ePath3.ListenerPort)
})
})
t.Run("missing network with no service check port label", func(t *testing.T) {
// this test ensures we do not try to manipulate the group network
// to inject an expose port if the group network does not exist
c := &structs.ServiceCheck{
Name: "check1",
Type: "http",
Path: "/health",
PortLabel: "", // not set
Expose: true, // will require a service check port label
}
s := &structs.Service{
Name: "service1",
Checks: []*structs.ServiceCheck{c},
}
tg := &structs.TaskGroup{
Name: "group1",
Services: []*structs.Service{s},
Networks: nil, // not set, should cause validation error
}
ePath, err := exposePathForCheck(tg, s, c, checkIdx)
require.EqualError(t, err, `group "group1" must specify one bridge network for exposing service check(s)`)
require.Nil(t, ePath)
})
}
func TestJobExposeCheckHook_containsExposePath(t *testing.T) {
ci.Parallel(t)
t.Run("contains path", func(t *testing.T) {
require.True(t, containsExposePath([]structs.ConsulExposePath{{
Path: "/v2/health",
Protocol: "grpc",
LocalPathPort: 8080,
ListenerPort: "v2Port",
}, {
Path: "/health",
Protocol: "http",
LocalPathPort: 8080,
ListenerPort: "hcPort",
}}, structs.ConsulExposePath{
Path: "/health",
Protocol: "http",
LocalPathPort: 8080,
ListenerPort: "hcPort",
}))
})
t.Run("no such path", func(t *testing.T) {
require.False(t, containsExposePath([]structs.ConsulExposePath{{
Path: "/v2/health",
Protocol: "grpc",
LocalPathPort: 8080,
ListenerPort: "v2Port",
}, {
Path: "/health",
Protocol: "http",
LocalPathPort: 8080,
ListenerPort: "hcPort",
}}, structs.ConsulExposePath{
Path: "/v3/health",
Protocol: "http",
LocalPathPort: 8080,
ListenerPort: "hcPort",
}))
})
}
func TestJobExposeCheckHook_serviceExposeConfig(t *testing.T) {
ci.Parallel(t)
t.Run("proxy is nil", func(t *testing.T) {
require.NotNil(t, serviceExposeConfig(&structs.Service{
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{},
},
}))
})
t.Run("expose is nil", func(t *testing.T) {
require.NotNil(t, serviceExposeConfig(&structs.Service{
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{
Proxy: &structs.ConsulProxy{},
},
},
}))
})
t.Run("expose pre-existing", func(t *testing.T) {
exposeConfig := serviceExposeConfig(&structs.Service{
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{
Proxy: &structs.ConsulProxy{
Expose: &structs.ConsulExposeConfig{
Paths: []structs.ConsulExposePath{{
Path: "/health",
}},
},
},
},
},
})
require.NotNil(t, exposeConfig)
require.Equal(t, []structs.ConsulExposePath{{
Path: "/health",
}}, exposeConfig.Paths)
})
t.Run("append to paths is safe", func(t *testing.T) {
// double check that serviceExposeConfig(s).Paths can be appended to
// from a derived pointer without fear of the original underlying array
// pointer being lost
s := &structs.Service{
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{
Proxy: &structs.ConsulProxy{
Expose: &structs.ConsulExposeConfig{
Paths: []structs.ConsulExposePath{{
Path: "/one",
}},
},
},
},
},
}
exposeConfig := serviceExposeConfig(s)
exposeConfig.Paths = append(exposeConfig.Paths,
structs.ConsulExposePath{Path: "/two"},
structs.ConsulExposePath{Path: "/three"},
structs.ConsulExposePath{Path: "/four"},
structs.ConsulExposePath{Path: "/five"},
structs.ConsulExposePath{Path: "/six"},
structs.ConsulExposePath{Path: "/seven"},
structs.ConsulExposePath{Path: "/eight"},
structs.ConsulExposePath{Path: "/nine"},
)
// works, because exposeConfig.Paths gets re-assigned into exposeConfig
// which is a pointer, meaning the field is modified also from the
// service struct's perspective
require.Equal(t, 9, len(s.Connect.SidecarService.Proxy.Expose.Paths))
})
}
func TestJobExposeCheckHook_checkIsExposable(t *testing.T) {
ci.Parallel(t)
t.Run("grpc", func(t *testing.T) {
require.True(t, checkIsExposable(&structs.ServiceCheck{
Type: "grpc",
Path: "/health",
}))
require.True(t, checkIsExposable(&structs.ServiceCheck{
Type: "gRPC",
Path: "/health",
}))
})
t.Run("http", func(t *testing.T) {
require.True(t, checkIsExposable(&structs.ServiceCheck{
Type: "http",
Path: "/health",
}))
require.True(t, checkIsExposable(&structs.ServiceCheck{
Type: "HTTP",
Path: "/health",
}))
})
t.Run("tcp", func(t *testing.T) {
require.False(t, checkIsExposable(&structs.ServiceCheck{
Type: "tcp",
Path: "/health",
}))
})
t.Run("no path slash prefix", func(t *testing.T) {
require.False(t, checkIsExposable(&structs.ServiceCheck{
Type: "http",
Path: "health",
}))
})
}
func TestJobExposeCheckHook_Mutate(t *testing.T) {
ci.Parallel(t)
t.Run("typical", func(t *testing.T) {
result, warnings, err := new(jobExposeCheckHook).Mutate(&structs.Job{
TaskGroups: []*structs.TaskGroup{{
Name: "group0",
Networks: structs.Networks{{
Mode: "host",
}},
}, {
Name: "group1",
Networks: structs.Networks{{
Mode: "bridge",
}},
Services: []*structs.Service{{
Name: "service1",
PortLabel: "8000",
Checks: []*structs.ServiceCheck{{
Name: "check1",
Type: "tcp",
PortLabel: "8100",
}, {
Name: "check2",
Type: "http",
PortLabel: "health",
Path: "/health",
Expose: true,
}, {
Name: "check3",
Type: "grpc",
PortLabel: "health",
Path: "/v2/health",
Expose: true,
}},
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{
Proxy: &structs.ConsulProxy{
Expose: &structs.ConsulExposeConfig{
Paths: []structs.ConsulExposePath{{
Path: "/pre-existing",
Protocol: "http",
LocalPathPort: 9000,
ListenerPort: "otherPort",
}}}}}}}, {
Name: "service2",
PortLabel: "3000",
Checks: []*structs.ServiceCheck{{
Name: "check1",
Type: "grpc",
Protocol: "http2",
Path: "/ok",
PortLabel: "health",
Expose: true,
}},
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{
Proxy: &structs.ConsulProxy{},
},
},
}}}},
})
require.NoError(t, err)
require.Empty(t, warnings)
require.Equal(t, []structs.ConsulExposePath{{
Path: "/pre-existing",
LocalPathPort: 9000,
Protocol: "http",
ListenerPort: "otherPort",
}, {
Path: "/health",
LocalPathPort: 8000,
ListenerPort: "health",
}, {
Path: "/v2/health",
LocalPathPort: 8000,
ListenerPort: "health",
}}, result.TaskGroups[1].Services[0].Connect.SidecarService.Proxy.Expose.Paths)
require.Equal(t, []structs.ConsulExposePath{{
Path: "/ok",
LocalPathPort: 3000,
Protocol: "http2",
ListenerPort: "health",
}}, result.TaskGroups[1].Services[1].Connect.SidecarService.Proxy.Expose.Paths)
})
}