open-nomad/nomad/job_endpoint_hook_expose_check_test.go

package nomad

import (
	"testing"

	"github.com/hashicorp/nomad/ci"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/stretchr/testify/require"
)

func TestJobExposeCheckHook_Name(t *testing.T) {
	ci.Parallel(t)

	require.Equal(t, "expose-check", new(jobExposeCheckHook).Name())
}

func TestJobExposeCheckHook_tgUsesExposeCheck(t *testing.T) {
	ci.Parallel(t)

	t.Run("no check.expose", func(t *testing.T) {
		require.False(t, tgUsesExposeCheck(&structs.TaskGroup{
			Services: []*structs.Service{{
				Checks: []*structs.ServiceCheck{{
					Expose: false,
				}},
			}},
		}))
	})

	t.Run("with check.expose", func(t *testing.T) {
		require.True(t, tgUsesExposeCheck(&structs.TaskGroup{
			Services: []*structs.Service{{
				Checks: []*structs.ServiceCheck{{
					Expose: false,
				}, {
					Expose: true,
				}},
			}},
		}))
	})
}

func TestJobExposeCheckHook_tgValidateUseOfBridgeMode(t *testing.T) {
	ci.Parallel(t)

	s1 := &structs.Service{
		Name: "s1",
		Checks: []*structs.ServiceCheck{{
			Name:      "s1-check1",
			Type:      "http",
			PortLabel: "health",
			Expose:    true,
		}},
	}

	t.Run("no networks but no use of expose", func(t *testing.T) {
		require.Nil(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{
			Networks: make(structs.Networks, 0),
		}))
	})

	t.Run("no networks and uses expose", func(t *testing.T) {
		require.EqualError(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{
			Name:     "g1",
			Networks: make(structs.Networks, 0),
			Services: []*structs.Service{s1},
		}), `group "g1" must specify one bridge network for exposing service check(s)`)
	})

	t.Run("non-bridge network and uses expose", func(t *testing.T) {
		require.EqualError(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{
			Name: "g1",
			Networks: structs.Networks{{
				Mode: "host",
			}},
			Services: []*structs.Service{s1},
		}), `group "g1" must use bridge network for exposing service check(s)`)
	})

	t.Run("bridge network uses expose", func(t *testing.T) {
		require.Nil(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{
			Name: "g1",
			Networks: structs.Networks{{
				Mode: "bridge",
			}},
			Services: []*structs.Service{s1},
		}))
	})
}

func TestJobExposeCheckHook_tgValidateUseOfCheckExpose(t *testing.T) {
	ci.Parallel(t)

	withCustomProxyTask := &structs.Service{
		Name: "s1",
		Connect: &structs.ConsulConnect{
			SidecarTask: &structs.SidecarTask{Name: "custom"},
		},
		Checks: []*structs.ServiceCheck{{
			Name:      "s1-check1",
			Type:      "http",
			PortLabel: "health",
			Expose:    true,
		}},
	}

	t.Run("group-service uses custom proxy", func(t *testing.T) {
		require.EqualError(t, tgValidateUseOfCheckExpose(&structs.TaskGroup{
			Name:     "g1",
			Services: []*structs.Service{withCustomProxyTask},
		}), `exposed service check g1->s1->s1-check1 requires use of sidecar_proxy`)
	})

	t.Run("group-service uses custom proxy but no expose", func(t *testing.T) {
		withCustomProxyTaskNoExpose := *withCustomProxyTask
		withCustomProxyTask.Checks[0].Expose = false
		require.Nil(t, tgValidateUseOfCheckExpose(&structs.TaskGroup{
			Name:     "g1",
			Services: []*structs.Service{&withCustomProxyTaskNoExpose},
		}))
	})

	t.Run("task-service sets expose", func(t *testing.T) {
		require.EqualError(t, tgValidateUseOfCheckExpose(&structs.TaskGroup{
			Name: "g1",
			Tasks: []*structs.Task{{
				Name: "t1",
				Services: []*structs.Service{{
					Name: "s2",
					Checks: []*structs.ServiceCheck{{
						Name:   "check1",
						Type:   "http",
						Expose: true,
					}},
				}},
			}},
		}), `exposed service check g1[t1]->s2->check1 is not a task-group service`)
	})
}

func TestJobExposeCheckHook_Validate(t *testing.T) {
	ci.Parallel(t)

	s1 := &structs.Service{
		Name: "s1",
		Checks: []*structs.ServiceCheck{{
			Name:   "s1-check1",
			Type:   "http",
			Expose: true,
		}},
	}

	t.Run("double network", func(t *testing.T) {
		warnings, err := new(jobExposeCheckHook).Validate(&structs.Job{
			TaskGroups: []*structs.TaskGroup{{
				Name: "g1",
				Networks: structs.Networks{{
					Mode: "bridge",
				}, {
					Mode: "bridge",
				}},
				Services: []*structs.Service{s1},
			}},
		})
		require.Empty(t, warnings)
		require.EqualError(t, err, `group "g1" must specify one bridge network for exposing service check(s)`)
	})

	t.Run("expose in service check", func(t *testing.T) {
		warnings, err := new(jobExposeCheckHook).Validate(&structs.Job{
			TaskGroups: []*structs.TaskGroup{{
				Name: "g1",
				Networks: structs.Networks{{
					Mode: "bridge",
				}},
				Tasks: []*structs.Task{{
					Name: "t1",
					Services: []*structs.Service{{
						Name: "s2",
						Checks: []*structs.ServiceCheck{{
							Name:   "s2-check1",
							Type:   "http",
							Expose: true,
						}},
					}},
				}},
			}},
		})
		require.Empty(t, warnings)
		require.EqualError(t, err, `exposed service check g1[t1]->s2->s2-check1 is not a task-group service`)
	})

	t.Run("ok", func(t *testing.T) {
		warnings, err := new(jobExposeCheckHook).Validate(&structs.Job{
			TaskGroups: []*structs.TaskGroup{{
				Name: "g1",
				Networks: structs.Networks{{
					Mode: "bridge",
				}},
				Services: []*structs.Service{{
					Name: "s1",
					Connect: &structs.ConsulConnect{
						SidecarService: &structs.ConsulSidecarService{},
					},
					Checks: []*structs.ServiceCheck{{
						Name:   "check1",
						Type:   "http",
						Expose: true,
					}},
				}},
				Tasks: []*structs.Task{{
					Name: "t1",
					Services: []*structs.Service{{
						Name: "s2",
						Checks: []*structs.ServiceCheck{{
							Name:   "s2-check1",
							Type:   "http",
							Expose: false,
						}},
					}},
				}},
			}},
		})
		require.Empty(t, warnings)
		require.Nil(t, err)
	})
}

func TestJobExposeCheckHook_exposePathForCheck(t *testing.T) {
	ci.Parallel(t)

	const checkIdx = 0

	t.Run("not expose compatible", func(t *testing.T) {
		c := &structs.ServiceCheck{
			Type: "tcp", // not expose compatible
		}
		s := &structs.Service{
			Checks: []*structs.ServiceCheck{c},
		}
		ePath, err := exposePathForCheck(&structs.TaskGroup{
			Services: []*structs.Service{s},
		}, s, c, checkIdx)
		require.NoError(t, err)
		require.Nil(t, ePath)
	})

	t.Run("direct port", func(t *testing.T) {
		c := &structs.ServiceCheck{
			Name:      "check1",
			Type:      "http",
			Path:      "/health",
			PortLabel: "hcPort",
		}
		s := &structs.Service{
			Name:      "service1",
			PortLabel: "4000",
			Checks:    []*structs.ServiceCheck{c},
		}
		ePath, err := exposePathForCheck(&structs.TaskGroup{
			Name:     "group1",
			Services: []*structs.Service{s},
		}, s, c, checkIdx)
		require.NoError(t, err)
		require.Equal(t, &structs.ConsulExposePath{
			Path:          "/health",
			Protocol:      "", // often blank, consul does the Right Thing
			LocalPathPort: 4000,
			ListenerPort:  "hcPort",
		}, ePath)
	})

	t.Run("labeled port", func(t *testing.T) {
		c := &structs.ServiceCheck{
			Name:      "check1",
			Type:      "http",
			Path:      "/health",
			PortLabel: "hcPort",
		}
		s := &structs.Service{
			Name:      "service1",
			PortLabel: "sPort", // port label indirection
			Checks:    []*structs.ServiceCheck{c},
		}
		ePath, err := exposePathForCheck(&structs.TaskGroup{
			Name:     "group1",
			Services: []*structs.Service{s},
			Networks: structs.Networks{{
				Mode: "bridge",
				DynamicPorts: []structs.Port{
					{Label: "sPort", Value: 4000},
				},
			}},
		}, s, c, checkIdx)
		require.NoError(t, err)
		require.Equal(t, &structs.ConsulExposePath{
			Path:          "/health",
			Protocol:      "",
			LocalPathPort: 4000,
			ListenerPort:  "hcPort",
		}, ePath)
	})

	t.Run("missing port", func(t *testing.T) {
		c := &structs.ServiceCheck{
			Name:      "check1",
			Type:      "http",
			Path:      "/health",
			PortLabel: "hcPort",
		}
		s := &structs.Service{
			Name:      "service1",
			PortLabel: "sPort", // port label indirection
			Checks:    []*structs.ServiceCheck{c},
		}
		_, err := exposePathForCheck(&structs.TaskGroup{
			Name:     "group1",
			Services: []*structs.Service{s},
			Networks: structs.Networks{{
				Mode:         "bridge",
				DynamicPorts: []structs.Port{
					// service declares "sPort", but does not exist
				},
			}},
		}, s, c, checkIdx)
		require.EqualError(t, err, `unable to determine local service port for service check group1->service1->check1`)
	})

	t.Run("empty check port", func(t *testing.T) {
		setup := func() (*structs.TaskGroup, *structs.Service, *structs.ServiceCheck) {
			c := &structs.ServiceCheck{
				Name: "check1",
				Type: "http",
				Path: "/health",
			}
			s := &structs.Service{
				Name:      "service1",
				PortLabel: "9999",
				Checks:    []*structs.ServiceCheck{c},
			}
			tg := &structs.TaskGroup{
				Name:     "group1",
				Services: []*structs.Service{s},
				Networks: structs.Networks{{
					Mode:         "bridge",
					DynamicPorts: []structs.Port{},
				}},
			}
			return tg, s, c
		}

		tg, s, c := setup()
		ePath, err := exposePathForCheck(tg, s, c, checkIdx)
		require.NoError(t, err)
		require.Len(t, tg.Networks[0].DynamicPorts, 1)
		require.Equal(t, "default", tg.Networks[0].DynamicPorts[0].HostNetwork)
		require.Equal(t, "svc_", tg.Networks[0].DynamicPorts[0].Label[0:4])
		require.Equal(t, &structs.ConsulExposePath{
			Path:          "/health",
			Protocol:      "",
			LocalPathPort: 9999,
			ListenerPort:  tg.Networks[0].DynamicPorts[0].Label,
		}, ePath)

		t.Run("deterministic generated port label", func(t *testing.T) {
			tg2, s2, c2 := setup()
			ePath2, err2 := exposePathForCheck(tg2, s2, c2, checkIdx)
			require.NoError(t, err2)
			require.Equal(t, ePath, ePath2)
		})

		t.Run("unique on check index", func(t *testing.T) {
			tg3, s3, c3 := setup()
			ePath3, err3 := exposePathForCheck(tg3, s3, c3, checkIdx+1)
			require.NoError(t, err3)
			require.NotEqual(t, ePath.ListenerPort, ePath3.ListenerPort)
		})
	})

	t.Run("missing network with no service check port label", func(t *testing.T) {
		// this test ensures we do not try to manipulate the group network
		// to inject an expose port if the group network does not exist
		c := &structs.ServiceCheck{
			Name:      "check1",
			Type:      "http",
			Path:      "/health",
			PortLabel: "",   // not set
			Expose:    true, // will require a service check port label
		}
		s := &structs.Service{
			Name:   "service1",
			Checks: []*structs.ServiceCheck{c},
		}
		tg := &structs.TaskGroup{
			Name:     "group1",
			Services: []*structs.Service{s},
			Networks: nil, // not set, should cause validation error
		}
		ePath, err := exposePathForCheck(tg, s, c, checkIdx)
		require.EqualError(t, err, `group "group1" must specify one bridge network for exposing service check(s)`)
		require.Nil(t, ePath)
	})
}

func TestJobExposeCheckHook_containsExposePath(t *testing.T) {
	ci.Parallel(t)

	t.Run("contains path", func(t *testing.T) {
		require.True(t, containsExposePath([]structs.ConsulExposePath{{
			Path:          "/v2/health",
			Protocol:      "grpc",
			LocalPathPort: 8080,
			ListenerPort:  "v2Port",
		}, {
			Path:          "/health",
			Protocol:      "http",
			LocalPathPort: 8080,
			ListenerPort:  "hcPort",
		}}, structs.ConsulExposePath{
			Path:          "/health",
			Protocol:      "http",
			LocalPathPort: 8080,
			ListenerPort:  "hcPort",
		}))
	})

	t.Run("no such path", func(t *testing.T) {
		require.False(t, containsExposePath([]structs.ConsulExposePath{{
			Path:          "/v2/health",
			Protocol:      "grpc",
			LocalPathPort: 8080,
			ListenerPort:  "v2Port",
		}, {
			Path:          "/health",
			Protocol:      "http",
			LocalPathPort: 8080,
			ListenerPort:  "hcPort",
		}}, structs.ConsulExposePath{
			Path:          "/v3/health",
			Protocol:      "http",
			LocalPathPort: 8080,
			ListenerPort:  "hcPort",
		}))
	})
}

func TestJobExposeCheckHook_serviceExposeConfig(t *testing.T) {
	ci.Parallel(t)

	t.Run("proxy is nil", func(t *testing.T) {
		require.NotNil(t, serviceExposeConfig(&structs.Service{
			Connect: &structs.ConsulConnect{
				SidecarService: &structs.ConsulSidecarService{},
			},
		}))
	})

	t.Run("expose is nil", func(t *testing.T) {
		require.NotNil(t, serviceExposeConfig(&structs.Service{
			Connect: &structs.ConsulConnect{
				SidecarService: &structs.ConsulSidecarService{
					Proxy: &structs.ConsulProxy{},
				},
			},
		}))
	})

	t.Run("expose pre-existing", func(t *testing.T) {
		exposeConfig := serviceExposeConfig(&structs.Service{
			Connect: &structs.ConsulConnect{
				SidecarService: &structs.ConsulSidecarService{
					Proxy: &structs.ConsulProxy{
						Expose: &structs.ConsulExposeConfig{
							Paths: []structs.ConsulExposePath{{
								Path: "/health",
							}},
						},
					},
				},
			},
		})
		require.NotNil(t, exposeConfig)
		require.Equal(t, []structs.ConsulExposePath{{
			Path: "/health",
		}}, exposeConfig.Paths)
	})

	t.Run("append to paths is safe", func(t *testing.T) {
		// double check that serviceExposeConfig(s).Paths can be appended to
		// from a derived pointer without fear of the original underlying array
		// pointer being lost

		s := &structs.Service{
			Connect: &structs.ConsulConnect{
				SidecarService: &structs.ConsulSidecarService{
					Proxy: &structs.ConsulProxy{
						Expose: &structs.ConsulExposeConfig{
							Paths: []structs.ConsulExposePath{{
								Path: "/one",
							}},
						},
					},
				},
			},
		}

		exposeConfig := serviceExposeConfig(s)
		exposeConfig.Paths = append(exposeConfig.Paths,
			structs.ConsulExposePath{Path: "/two"},
			structs.ConsulExposePath{Path: "/three"},
			structs.ConsulExposePath{Path: "/four"},
			structs.ConsulExposePath{Path: "/five"},
			structs.ConsulExposePath{Path: "/six"},
			structs.ConsulExposePath{Path: "/seven"},
			structs.ConsulExposePath{Path: "/eight"},
			structs.ConsulExposePath{Path: "/nine"},
		)

		// works, because exposeConfig.Paths gets re-assigned into exposeConfig
		// which is a pointer, meaning the field is modified also from the
		// service struct's perspective
		require.Equal(t, 9, len(s.Connect.SidecarService.Proxy.Expose.Paths))
	})
}

func TestJobExposeCheckHook_checkIsExposable(t *testing.T) {
	ci.Parallel(t)

	t.Run("grpc", func(t *testing.T) {
		require.True(t, checkIsExposable(&structs.ServiceCheck{
			Type: "grpc",
			Path: "/health",
		}))
		require.True(t, checkIsExposable(&structs.ServiceCheck{
			Type: "gRPC",
			Path: "/health",
		}))
	})

	t.Run("http", func(t *testing.T) {
		require.True(t, checkIsExposable(&structs.ServiceCheck{
			Type: "http",
			Path: "/health",
		}))
		require.True(t, checkIsExposable(&structs.ServiceCheck{
			Type: "HTTP",
			Path: "/health",
		}))
	})

	t.Run("tcp", func(t *testing.T) {
		require.False(t, checkIsExposable(&structs.ServiceCheck{
			Type: "tcp",
			Path: "/health",
		}))
	})

	t.Run("no path slash prefix", func(t *testing.T) {
		require.False(t, checkIsExposable(&structs.ServiceCheck{
			Type: "http",
			Path: "health",
		}))
	})
}

func TestJobExposeCheckHook_Mutate(t *testing.T) {
	ci.Parallel(t)

	t.Run("typical", func(t *testing.T) {
		result, warnings, err := new(jobExposeCheckHook).Mutate(&structs.Job{
			TaskGroups: []*structs.TaskGroup{{
				Name: "group0",
				Networks: structs.Networks{{
					Mode: "host",
				}},
			}, {
				Name: "group1",
				Networks: structs.Networks{{
					Mode: "bridge",
				}},
				Services: []*structs.Service{{
					Name:      "service1",
					PortLabel: "8000",
					Checks: []*structs.ServiceCheck{{
						Name:      "check1",
						Type:      "tcp",
						PortLabel: "8100",
					}, {
						Name:      "check2",
						Type:      "http",
						PortLabel: "health",
						Path:      "/health",
						Expose:    true,
					}, {
						Name:      "check3",
						Type:      "grpc",
						PortLabel: "health",
						Path:      "/v2/health",
						Expose:    true,
					}},
					Connect: &structs.ConsulConnect{
						SidecarService: &structs.ConsulSidecarService{
							Proxy: &structs.ConsulProxy{
								Expose: &structs.ConsulExposeConfig{
									Paths: []structs.ConsulExposePath{{
										Path:          "/pre-existing",
										Protocol:      "http",
										LocalPathPort: 9000,
										ListenerPort:  "otherPort",
									}}}}}}}, {
					Name:      "service2",
					PortLabel: "3000",
					Checks: []*structs.ServiceCheck{{
						Name:      "check1",
						Type:      "grpc",
						Protocol:  "http2",
						Path:      "/ok",
						PortLabel: "health",
						Expose:    true,
					}},
					Connect: &structs.ConsulConnect{
						SidecarService: &structs.ConsulSidecarService{
							Proxy: &structs.ConsulProxy{},
						},
					},
				}}}},
		})

		require.NoError(t, err)
		require.Empty(t, warnings)
		require.Equal(t, []structs.ConsulExposePath{{
			Path:          "/pre-existing",
			LocalPathPort: 9000,
			Protocol:      "http",
			ListenerPort:  "otherPort",
		}, {
			Path:          "/health",
			LocalPathPort: 8000,
			ListenerPort:  "health",
		}, {
			Path:          "/v2/health",
			LocalPathPort: 8000,
			ListenerPort:  "health",
		}}, result.TaskGroups[1].Services[0].Connect.SidecarService.Proxy.Expose.Paths)
		require.Equal(t, []structs.ConsulExposePath{{
			Path:          "/ok",
			LocalPathPort: 3000,
			Protocol:      "http2",
			ListenerPort:  "health",
		}}, result.TaskGroups[1].Services[1].Connect.SidecarService.Proxy.Expose.Paths)
	})
}