docs: add docs and tests for tagged_addresses

.changelog/12951.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+consul: Enable setting custom tagged_addresses field
+```

api/services.go

@@ -281,6 +281,18 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) {
 		s.Provider = ServiceProviderConsul
 	}
 
+	if len(s.Meta) == 0 {
+		s.Meta = nil
+	}
+
+	if len(s.CanaryMeta) == 0 {
+		s.CanaryMeta = nil
+	}
+
+	if len(s.TaggedAddresses) == 0 {
+		s.TaggedAddresses = nil
+	}
+
 	s.Connect.Canonicalize()
 
 	// Canonicalize CheckRestart on Checks and merge Service.CheckRestart

api/services_test.go

@@ -21,14 +21,15 @@ func TestServiceRegistrations_Delete(t *testing.T) {
 	// TODO(jrasell) add tests once registration process is in place.
 }
 
-
 func TestService_Canonicalize(t *testing.T) {
 	testutil.Parallel(t)
 
 	j := &Job{Name: stringToPtr("job")}
 	tg := &TaskGroup{Name: stringToPtr("group")}
 	task := &Task{Name: "task"}
-	s := &Service{}
+	s := &Service{
+		TaggedAddresses: make(map[string]string),
+	}
 
 	s.Canonicalize(task, tg, j)
 
@@ -36,6 +37,9 @@ func TestService_Canonicalize(t *testing.T) {
 	require.Equal(t, "auto", s.AddressMode)
 	require.Equal(t, OnUpdateRequireHealthy, s.OnUpdate)
 	require.Equal(t, ServiceProviderConsul, s.Provider)
+	require.Nil(t, s.Meta)
+	require.Nil(t, s.CanaryMeta)
+	require.Nil(t, s.TaggedAddresses)
 }
 
 func TestServiceCheck_Canonicalize(t *testing.T) {

client/taskenv/services_test.go

@@ -27,6 +27,9 @@ func TestInterpolateServices(t *testing.T) {
 				"canarymeta-key": "${canarymeta}",
 			},
 			Address: "${address}",
+			TaggedAddresses: map[string]string{
+				"${ta-key}": "${ta-address}",
+			},
 			Checks: []*structs.ServiceCheck{
 				{
 					Name:          "${checkname}",
@@ -53,6 +56,8 @@ func TestInterpolateServices(t *testing.T) {
 			"tags":         "tags",
 			"meta":         "meta-value",
 			"address":      "example.com",
+			"ta-key":       "public_wan",
+			"ta-address":   "1.2.3.4",
 			"canarymeta":   "canarymeta-value",
 			"checkname":    "checkname",
 			"checktype":    "checktype",
@@ -83,6 +88,9 @@ func TestInterpolateServices(t *testing.T) {
 				"canarymeta-key": "canarymeta-value",
 			},
 			Address: "example.com",
+			TaggedAddresses: map[string]string{
+				"public_wan": "1.2.3.4",
+			},
 			Checks: []*structs.ServiceCheck{
 				{
 					Name:          "checkname",

command/agent/consul/service_client.go

@@ -1029,17 +1029,9 @@ func (c *ServiceClient) serviceRegs(
 		}
 	}
 
-	var taggedAddresses map[string]api.ServiceAddress
+	taggedAddresses, err := parseTaggedAddresses(service.TaggedAddresses, port)
-	for k, v := range service.TaggedAddresses {
+	if err != nil {
-		sa, err := parseAddress(v, port)
+		return nil, err
-		if err != nil {
-			c.logger.Warn("failed to parse advertise address", "name", k, "adrress", v)
-			continue
-		}
-		if taggedAddresses == nil {
-			taggedAddresses = make(map[string]api.ServiceAddress)
-		}
-		taggedAddresses[k] = sa
 	}
 
 	// Build the Consul Service registration request
@@ -1708,3 +1700,16 @@ func parseAddress(raw string, port int) (api.ServiceAddress, error) {
 	result.Port = port
 	return result, nil
 }
+
+// morph the tagged_addresses map into the structure consul api wants
+func parseTaggedAddresses(m map[string]string, port int) (map[string]api.ServiceAddress, error) {
+	result := make(map[string]api.ServiceAddress, len(m))
+	for k, v := range m {
+		sa, err := parseAddress(v, port)
+		if err != nil {
+			return nil, err
+		}
+		result[k] = sa
+	}
+	return result, nil
+}

command/agent/consul/service_client_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/shoenig/test/must"
 	"github.com/stretchr/testify/require"
 )
 
@@ -28,6 +29,9 @@ func TestSyncLogic_agentServiceUpdateRequired(t *testing.T) {
 			Address:           "1.1.1.1",
 			EnableTagOverride: true,
 			Meta:              map[string]string{"foo": "1"},
+			TaggedAddresses: map[string]api.ServiceAddress{
+				"public_wan": {Address: "1.2.3.4", Port: 8080},
+			},
 			Connect: &api.AgentServiceConnect{
 				Native: false,
 				SidecarService: &api.AgentServiceRegistration{
@@ -56,6 +60,9 @@ func TestSyncLogic_agentServiceUpdateRequired(t *testing.T) {
 		Address:           "1.1.1.1",
 		EnableTagOverride: true,
 		Meta:              map[string]string{"foo": "1"},
+		TaggedAddresses: map[string]api.ServiceAddress{
+			"public_wan": {Address: "1.2.3.4", Port: 8080},
+		},
 	}
 
 	sidecar := &api.AgentService{
@@ -212,6 +219,15 @@ func TestSyncLogic_agentServiceUpdateRequired(t *testing.T) {
 		})
 	})
 
+	t.Run("different tagged addresses", func(t *testing.T) {
+		try(t, true, syncNewOps, func(w asr) *asr {
+			w.TaggedAddresses = map[string]api.ServiceAddress{
+				"public_wan": {Address: "5.6.7.8", Port: 8080},
+			}
+			return &w
+		})
+	})
+
 	// for remaining tests, EnableTagOverride = false
 	existing.EnableTagOverride = false
 
@@ -648,3 +664,44 @@ func TestSyncLogic_maybeSidecarProxyCheck(t *testing.T) {
 	try("service:_nomad-task-2f5fb517-57d4-44ee-7780-dc1cb6e103cd-group-api-count-api-9001-sidecar-proxy: ", false)
 	try("service", false)
 }
+
+func TestSyncLogic_parseTaggedAddresses(t *testing.T) {
+	ci.Parallel(t)
+
+	t.Run("nil", func(t *testing.T) {
+		m, err := parseTaggedAddresses(nil, 0)
+		must.NoError(t, err)
+		must.MapEmpty(t, m)
+	})
+
+	t.Run("parse fail", func(t *testing.T) {
+		ta := map[string]string{
+			"public_wan": "not an address",
+		}
+		result, err := parseTaggedAddresses(ta, 8080)
+		must.Error(t, err)
+		must.MapEmpty(t, result)
+	})
+
+	t.Run("parse address", func(t *testing.T) {
+		ta := map[string]string{
+			"public_wan": "1.2.3.4",
+		}
+		result, err := parseTaggedAddresses(ta, 8080)
+		must.NoError(t, err)
+		must.MapEq(t, map[string]api.ServiceAddress{
+			"public_wan": {Address: "1.2.3.4", Port: 8080},
+		}, result)
+	})
+
+	t.Run("parse address and port", func(t *testing.T) {
+		ta := map[string]string{
+			"public_wan": "1.2.3.4:9999",
+		}
+		result, err := parseTaggedAddresses(ta, 8080)
+		must.NoError(t, err)
+		must.MapEq(t, map[string]api.ServiceAddress{
+			"public_wan": {Address: "1.2.3.4", Port: 9999},
+		}, result)
+	})
+}

command/agent/job_endpoint_test.go

@@ -2518,6 +2518,9 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 						Meta: map[string]string{
 							"servicemeta": "foobar",
 						},
+						TaggedAddresses: map[string]string{
+							"wan": "1.2.3.4",
+						},
 						CheckRestart: &api.CheckRestart{
 							Limit: 4,
 							Grace: helper.TimeToPtr(11 * time.Second),
@@ -2915,6 +2918,9 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 						Meta: map[string]string{
 							"servicemeta": "foobar",
 						},
+						TaggedAddresses: map[string]string{
+							"wan": "1.2.3.4",
+						},
 						OnUpdate: "require_healthy",
 						Checks: []*structs.ServiceCheck{
 							{

go.mod

@@ -37,7 +37,7 @@ require (
 	github.com/fsouza/go-dockerclient v1.6.5
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4
-	github.com/google/go-cmp v0.5.6
+	github.com/google/go-cmp v0.5.8
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/websocket v1.4.2
 	github.com/gosuri/uilive v0.0.4
@@ -109,6 +109,7 @@ require (
 	github.com/ryanuber/go-glob v1.0.0
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529
 	github.com/shirou/gopsutil/v3 v3.21.12
+	github.com/shoenig/test v0.2.5
 	github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c
 	github.com/stretchr/testify v1.7.1
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
@@ -263,7 +264,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect
+	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/api v0.60.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -582,8 +582,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -1166,6 +1167,8 @@ github.com/shirou/gopsutil v0.0.0-20181107111621-48177ef5f880/go.mod h1:5b4v6he4
 github.com/shirou/gopsutil/v3 v3.21.12 h1:VoGxEW2hpmz0Vt3wUvHIl9fquzYLNpVpgNNB7pGJimA=
 github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shoenig/test v0.2.5 h1:CfxxPAhW9sJt9nVI39cOHrb4krmHd28SmU66oCXi6sY=
+github.com/shoenig/test v0.2.5/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
@@ -1657,8 +1660,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI=
+golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
-golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

helper/funcs.go

@@ -401,8 +401,6 @@ func CopyMapStringFloat64(m map[string]float64) map[string]float64 {
 	return c
 }
 
-// CopyMapStringSliceString copies a map of strings to string slices such as
-// http.Header
 func CopyMapStringSliceString(m map[string][]string) map[string][]string {
 	l := len(m)
 	if l == 0 {

jobspec/parse_test.go

@@ -846,6 +846,28 @@ func TestParse(t *testing.T) {
 			},
 			false,
 		},
+		{
+			"service-tagged-address.hcl",
+			&api.Job{
+				ID:   stringToPtr("service_tagged_address"),
+				Name: stringToPtr("service_tagged_address"),
+				Type: stringToPtr("service"),
+				TaskGroups: []*api.TaskGroup{
+					{
+						Name: stringToPtr("group"),
+						Services: []*api.Service{
+							{
+								Name: "service1",
+								TaggedAddresses: map[string]string{
+									"public_wan": "1.2.3.4",
+								},
+							},
+						},
+					},
+				},
+			},
+			false,
+		},
 		{
 			"service-check-driver-address.hcl",
 			&api.Job{

jobspec/test-fixtures/service-tagged-address.hcl

@@ -0,0 +1,12 @@
+job "service_tagged_address" {
+  type = "service"
+
+  group "group" {
+    service {
+      name = "service1"
+      tagged_addresses {
+        public_wan = "1.2.3.4"
+      }
+    }
+  }
+}

nomad/structs/services.go

@@ -472,6 +472,8 @@ type Service struct {
 	Meta       map[string]string // Consul service meta
 	CanaryMeta map[string]string // Consul service meta when it is a canary
 
+	// The values to set for tagged_addresses in Consul service registration.
+	// Does not affect Nomad networking, these are for Consul service discovery.
 	TaggedAddresses map[string]string
 
 	// The consul namespace in which this service will be registered. Namespace
@@ -537,6 +539,9 @@ func (s *Service) Canonicalize(job, taskGroup, task, jobNamespace string) {
 	if len(s.Checks) == 0 {
 		s.Checks = nil
 	}
+	if len(s.TaggedAddresses) == 0 {
+		s.TaggedAddresses = nil
+	}
 
 	s.Name = args.ReplaceEnv(s.Name, map[string]string{
 		"JOB":       job,

nomad/structs/services_test.go

@@ -1601,7 +1601,7 @@ func TestService_Validate(t *testing.T) {
 	}
 }
 
-func TestService_Advertise(t *testing.T) {
+func TestService_Validate_Address(t *testing.T) {
 	try := func(mode, advertise string, exp error) {
 		s := &Service{Name: "s1", Provider: "consul", AddressMode: mode, Address: advertise}
 		result := s.Validate()
@@ -1632,7 +1632,8 @@ func TestService_Equals(t *testing.T) {
 	ci.Parallel(t)
 
 	s := Service{
-		Name: "testservice",
+		Name:            "testservice",
+		TaggedAddresses: make(map[string]string),
 	}
 
 	s.Canonicalize("testjob", "testgroup", "testtask", "default")
@@ -1679,6 +1680,9 @@ func TestService_Equals(t *testing.T) {
 
 	o.Provider = "nomad"
 	assertDiff()
+
+	o.TaggedAddresses = map[string]string{"foo": "bar"}
+	assertDiff()
 }
 
 func TestService_validateNomadService(t *testing.T) {

website/content/docs/job-specification/service.mdx

@@ -146,6 +146,9 @@ Connect][connect] integration.
   mode. Useful with interpolation - for example to advertise the public IP address
   of an AWS EC2 instance set this to `${attr.unique.platform.aws.public-ipv4}`.
 
+- `tagged_addresses` `(map<string|string>` - Specifies custom [tagged addresses][tagged_addresses] to
+  advertise in the Consul service registration. Only available where `provider = "consul"`.
+
 - `address_mode` `(string: "auto")` - Specifies which address (host, alloc or
   driver-specific) this service should advertise. See [below for
   examples.](#using-driver-address-mode) Valid options are:
@@ -833,3 +836,4 @@ advertise and check directly since Nomad isn't managing any port assignments.
 [service_task]: /docs/job-specification/service#task-1
 [network_mode]: /docs/job-specification/network#mode
 [on_update]: /docs/job-specification/service#on_update
+[tagged_addresses]: https://www.consul.io/docs/discovery/services#tagged-addresses