diff --git a/.changelog/12951.txt b/.changelog/12951.txt new file mode 100644 index 000000000..8fa78b0dd --- /dev/null +++ b/.changelog/12951.txt @@ -0,0 +1,3 @@ +```release-note:improvement +consul: Enable setting custom tagged_addresses field +``` diff --git a/api/services.go b/api/services.go index 07464cb61..9cd9d8c5f 100644 --- a/api/services.go +++ b/api/services.go @@ -281,6 +281,18 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) { s.Provider = ServiceProviderConsul } + if len(s.Meta) == 0 { + s.Meta = nil + } + + if len(s.CanaryMeta) == 0 { + s.CanaryMeta = nil + } + + if len(s.TaggedAddresses) == 0 { + s.TaggedAddresses = nil + } + s.Connect.Canonicalize() // Canonicalize CheckRestart on Checks and merge Service.CheckRestart diff --git a/api/services_test.go b/api/services_test.go index c9ef884e6..5de82e17f 100644 --- a/api/services_test.go +++ b/api/services_test.go @@ -21,14 +21,15 @@ func TestServiceRegistrations_Delete(t *testing.T) { // TODO(jrasell) add tests once registration process is in place. } - func TestService_Canonicalize(t *testing.T) { testutil.Parallel(t) j := &Job{Name: stringToPtr("job")} tg := &TaskGroup{Name: stringToPtr("group")} task := &Task{Name: "task"} - s := &Service{} + s := &Service{ + TaggedAddresses: make(map[string]string), + } s.Canonicalize(task, tg, j) @@ -36,6 +37,9 @@ func TestService_Canonicalize(t *testing.T) { require.Equal(t, "auto", s.AddressMode) require.Equal(t, OnUpdateRequireHealthy, s.OnUpdate) require.Equal(t, ServiceProviderConsul, s.Provider) + require.Nil(t, s.Meta) + require.Nil(t, s.CanaryMeta) + require.Nil(t, s.TaggedAddresses) } func TestServiceCheck_Canonicalize(t *testing.T) { @@ -192,4 +196,4 @@ func TestService_Tags(t *testing.T) { r.True(service.EnableTagOverride) r.Equal([]string{"a", "b"}, service.Tags) r.Equal([]string{"c", "d"}, service.CanaryTags) -} \ No newline at end of file +} diff --git a/client/taskenv/services_test.go b/client/taskenv/services_test.go index 07d7a3e52..eadf49c50 100644 --- a/client/taskenv/services_test.go +++ b/client/taskenv/services_test.go @@ -27,6 +27,9 @@ func TestInterpolateServices(t *testing.T) { "canarymeta-key": "${canarymeta}", }, Address: "${address}", + TaggedAddresses: map[string]string{ + "${ta-key}": "${ta-address}", + }, Checks: []*structs.ServiceCheck{ { Name: "${checkname}", @@ -53,6 +56,8 @@ func TestInterpolateServices(t *testing.T) { "tags": "tags", "meta": "meta-value", "address": "example.com", + "ta-key": "public_wan", + "ta-address": "1.2.3.4", "canarymeta": "canarymeta-value", "checkname": "checkname", "checktype": "checktype", @@ -83,6 +88,9 @@ func TestInterpolateServices(t *testing.T) { "canarymeta-key": "canarymeta-value", }, Address: "example.com", + TaggedAddresses: map[string]string{ + "public_wan": "1.2.3.4", + }, Checks: []*structs.ServiceCheck{ { Name: "checkname", diff --git a/command/agent/consul/service_client.go b/command/agent/consul/service_client.go index 21a15db3a..679ab2b4d 100644 --- a/command/agent/consul/service_client.go +++ b/command/agent/consul/service_client.go @@ -1029,17 +1029,9 @@ func (c *ServiceClient) serviceRegs( } } - var taggedAddresses map[string]api.ServiceAddress - for k, v := range service.TaggedAddresses { - sa, err := parseAddress(v, port) - if err != nil { - c.logger.Warn("failed to parse advertise address", "name", k, "adrress", v) - continue - } - if taggedAddresses == nil { - taggedAddresses = make(map[string]api.ServiceAddress) - } - taggedAddresses[k] = sa + taggedAddresses, err := parseTaggedAddresses(service.TaggedAddresses, port) + if err != nil { + return nil, err } // Build the Consul Service registration request @@ -1708,3 +1700,16 @@ func parseAddress(raw string, port int) (api.ServiceAddress, error) { result.Port = port return result, nil } + +// morph the tagged_addresses map into the structure consul api wants +func parseTaggedAddresses(m map[string]string, port int) (map[string]api.ServiceAddress, error) { + result := make(map[string]api.ServiceAddress, len(m)) + for k, v := range m { + sa, err := parseAddress(v, port) + if err != nil { + return nil, err + } + result[k] = sa + } + return result, nil +} diff --git a/command/agent/consul/service_client_test.go b/command/agent/consul/service_client_test.go index 6890e2d8e..e269a9679 100644 --- a/command/agent/consul/service_client_test.go +++ b/command/agent/consul/service_client_test.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" + "github.com/shoenig/test/must" "github.com/stretchr/testify/require" ) @@ -28,6 +29,9 @@ func TestSyncLogic_agentServiceUpdateRequired(t *testing.T) { Address: "1.1.1.1", EnableTagOverride: true, Meta: map[string]string{"foo": "1"}, + TaggedAddresses: map[string]api.ServiceAddress{ + "public_wan": {Address: "1.2.3.4", Port: 8080}, + }, Connect: &api.AgentServiceConnect{ Native: false, SidecarService: &api.AgentServiceRegistration{ @@ -56,6 +60,9 @@ func TestSyncLogic_agentServiceUpdateRequired(t *testing.T) { Address: "1.1.1.1", EnableTagOverride: true, Meta: map[string]string{"foo": "1"}, + TaggedAddresses: map[string]api.ServiceAddress{ + "public_wan": {Address: "1.2.3.4", Port: 8080}, + }, } sidecar := &api.AgentService{ @@ -212,6 +219,15 @@ func TestSyncLogic_agentServiceUpdateRequired(t *testing.T) { }) }) + t.Run("different tagged addresses", func(t *testing.T) { + try(t, true, syncNewOps, func(w asr) *asr { + w.TaggedAddresses = map[string]api.ServiceAddress{ + "public_wan": {Address: "5.6.7.8", Port: 8080}, + } + return &w + }) + }) + // for remaining tests, EnableTagOverride = false existing.EnableTagOverride = false @@ -648,3 +664,44 @@ func TestSyncLogic_maybeSidecarProxyCheck(t *testing.T) { try("service:_nomad-task-2f5fb517-57d4-44ee-7780-dc1cb6e103cd-group-api-count-api-9001-sidecar-proxy: ", false) try("service", false) } + +func TestSyncLogic_parseTaggedAddresses(t *testing.T) { + ci.Parallel(t) + + t.Run("nil", func(t *testing.T) { + m, err := parseTaggedAddresses(nil, 0) + must.NoError(t, err) + must.MapEmpty(t, m) + }) + + t.Run("parse fail", func(t *testing.T) { + ta := map[string]string{ + "public_wan": "not an address", + } + result, err := parseTaggedAddresses(ta, 8080) + must.Error(t, err) + must.MapEmpty(t, result) + }) + + t.Run("parse address", func(t *testing.T) { + ta := map[string]string{ + "public_wan": "1.2.3.4", + } + result, err := parseTaggedAddresses(ta, 8080) + must.NoError(t, err) + must.MapEq(t, map[string]api.ServiceAddress{ + "public_wan": {Address: "1.2.3.4", Port: 8080}, + }, result) + }) + + t.Run("parse address and port", func(t *testing.T) { + ta := map[string]string{ + "public_wan": "1.2.3.4:9999", + } + result, err := parseTaggedAddresses(ta, 8080) + must.NoError(t, err) + must.MapEq(t, map[string]api.ServiceAddress{ + "public_wan": {Address: "1.2.3.4", Port: 9999}, + }, result) + }) +} diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 97e01fbfa..1741f13b1 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2518,6 +2518,9 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Meta: map[string]string{ "servicemeta": "foobar", }, + TaggedAddresses: map[string]string{ + "wan": "1.2.3.4", + }, CheckRestart: &api.CheckRestart{ Limit: 4, Grace: helper.TimeToPtr(11 * time.Second), @@ -2915,6 +2918,9 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Meta: map[string]string{ "servicemeta": "foobar", }, + TaggedAddresses: map[string]string{ + "wan": "1.2.3.4", + }, OnUpdate: "require_healthy", Checks: []*structs.ServiceCheck{ { diff --git a/go.mod b/go.mod index 34d534520..1600451c3 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/fsouza/go-dockerclient v1.6.5 github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.4 - github.com/google/go-cmp v0.5.6 + github.com/google/go-cmp v0.5.8 github.com/gorilla/handlers v1.5.1 github.com/gorilla/websocket v1.4.2 github.com/gosuri/uilive v0.0.4 @@ -109,6 +109,7 @@ require ( github.com/ryanuber/go-glob v1.0.0 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 github.com/shirou/gopsutil/v3 v3.21.12 + github.com/shoenig/test v0.2.5 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c github.com/stretchr/testify v1.7.1 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 @@ -263,7 +264,7 @@ require ( golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect + golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/api v0.60.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 95810200e..f48b82b59 100644 --- a/go.sum +++ b/go.sum @@ -582,8 +582,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -1166,6 +1167,8 @@ github.com/shirou/gopsutil v0.0.0-20181107111621-48177ef5f880/go.mod h1:5b4v6he4 github.com/shirou/gopsutil/v3 v3.21.12 h1:VoGxEW2hpmz0Vt3wUvHIl9fquzYLNpVpgNNB7pGJimA= github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shoenig/test v0.2.5 h1:CfxxPAhW9sJt9nVI39cOHrb4krmHd28SmU66oCXi6sY= +github.com/shoenig/test v0.2.5/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -1657,8 +1660,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= -golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/helper/funcs.go b/helper/funcs.go index 3493f413f..a6512d7ea 100644 --- a/helper/funcs.go +++ b/helper/funcs.go @@ -401,8 +401,6 @@ func CopyMapStringFloat64(m map[string]float64) map[string]float64 { return c } -// CopyMapStringSliceString copies a map of strings to string slices such as -// http.Header func CopyMapStringSliceString(m map[string][]string) map[string][]string { l := len(m) if l == 0 { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 1f2deadea..861e73a07 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -846,6 +846,28 @@ func TestParse(t *testing.T) { }, false, }, + { + "service-tagged-address.hcl", + &api.Job{ + ID: stringToPtr("service_tagged_address"), + Name: stringToPtr("service_tagged_address"), + Type: stringToPtr("service"), + TaskGroups: []*api.TaskGroup{ + { + Name: stringToPtr("group"), + Services: []*api.Service{ + { + Name: "service1", + TaggedAddresses: map[string]string{ + "public_wan": "1.2.3.4", + }, + }, + }, + }, + }, + }, + false, + }, { "service-check-driver-address.hcl", &api.Job{ diff --git a/jobspec/test-fixtures/service-tagged-address.hcl b/jobspec/test-fixtures/service-tagged-address.hcl new file mode 100644 index 000000000..0cdb47659 --- /dev/null +++ b/jobspec/test-fixtures/service-tagged-address.hcl @@ -0,0 +1,12 @@ +job "service_tagged_address" { + type = "service" + + group "group" { + service { + name = "service1" + tagged_addresses { + public_wan = "1.2.3.4" + } + } + } +} diff --git a/nomad/structs/services.go b/nomad/structs/services.go index 552d96d67..52251f23e 100644 --- a/nomad/structs/services.go +++ b/nomad/structs/services.go @@ -472,6 +472,8 @@ type Service struct { Meta map[string]string // Consul service meta CanaryMeta map[string]string // Consul service meta when it is a canary + // The values to set for tagged_addresses in Consul service registration. + // Does not affect Nomad networking, these are for Consul service discovery. TaggedAddresses map[string]string // The consul namespace in which this service will be registered. Namespace @@ -537,6 +539,9 @@ func (s *Service) Canonicalize(job, taskGroup, task, jobNamespace string) { if len(s.Checks) == 0 { s.Checks = nil } + if len(s.TaggedAddresses) == 0 { + s.TaggedAddresses = nil + } s.Name = args.ReplaceEnv(s.Name, map[string]string{ "JOB": job, diff --git a/nomad/structs/services_test.go b/nomad/structs/services_test.go index 2ee2f62d3..1044352c1 100644 --- a/nomad/structs/services_test.go +++ b/nomad/structs/services_test.go @@ -1601,7 +1601,7 @@ func TestService_Validate(t *testing.T) { } } -func TestService_Advertise(t *testing.T) { +func TestService_Validate_Address(t *testing.T) { try := func(mode, advertise string, exp error) { s := &Service{Name: "s1", Provider: "consul", AddressMode: mode, Address: advertise} result := s.Validate() @@ -1632,7 +1632,8 @@ func TestService_Equals(t *testing.T) { ci.Parallel(t) s := Service{ - Name: "testservice", + Name: "testservice", + TaggedAddresses: make(map[string]string), } s.Canonicalize("testjob", "testgroup", "testtask", "default") @@ -1679,6 +1680,9 @@ func TestService_Equals(t *testing.T) { o.Provider = "nomad" assertDiff() + + o.TaggedAddresses = map[string]string{"foo": "bar"} + assertDiff() } func TestService_validateNomadService(t *testing.T) { diff --git a/website/content/docs/job-specification/service.mdx b/website/content/docs/job-specification/service.mdx index a176b49f3..f9a8e4424 100644 --- a/website/content/docs/job-specification/service.mdx +++ b/website/content/docs/job-specification/service.mdx @@ -146,6 +146,9 @@ Connect][connect] integration. mode. Useful with interpolation - for example to advertise the public IP address of an AWS EC2 instance set this to `${attr.unique.platform.aws.public-ipv4}`. +- `tagged_addresses` `(map` - Specifies custom [tagged addresses][tagged_addresses] to + advertise in the Consul service registration. Only available where `provider = "consul"`. + - `address_mode` `(string: "auto")` - Specifies which address (host, alloc or driver-specific) this service should advertise. See [below for examples.](#using-driver-address-mode) Valid options are: @@ -833,3 +836,4 @@ advertise and check directly since Nomad isn't managing any port assignments. [service_task]: /docs/job-specification/service#task-1 [network_mode]: /docs/job-specification/network#mode [on_update]: /docs/job-specification/service#on_update +[tagged_addresses]: https://www.consul.io/docs/discovery/services#tagged-addresses \ No newline at end of file