luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Enable gateways to resolve hostnames to IPv4 addresses (#7999) 

The DNS resolution will be handled by Envoy and defaults to LOGICAL_DNS. This discovery type can be overridden on a per-gateway basis with the envoy_dns_discovery_type Gateway Option.

If a service contains an instance with a hostname as an address we set the Envoy cluster to use DNS as the discovery type rather than EDS. Since both mesh gateways and terminating gateways route to clusters using SNI, whenever there is a mix of hostnames and IP addresses associated with a service we use the hostname + CDS rather than the IPs + EDS.

Note that we detect hostnames by attempting to parse the service instance's address as an IP. If it is not a valid IP we assume it is a hostname.
This commit is contained in:
Freddy 2020-06-03 15:28:45 -06:00 committed by GitHub
parent 5ce7c8d76d
commit f759a48726
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 1342 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
agent/proxycfg/snapshot.go
View File

@ -100,6 +100,10 @@ type configSnapshotTerminatingGateway struct {
// between the gateway and a service. TLS configuration stored here is
// used for TLS origination from the gateway to the linked service.
GatewayServices map[structs.ServiceID]structs.GatewayService
// HostnameServices is a map of service id to service instances with a hostname as the address.
// If hostnames are configured they must be provided to Envoy via CDS not EDS.
HostnameServices map[structs.ServiceID]structs.CheckServiceNodes
}
func (c *configSnapshotTerminatingGateway) IsEmpty() bool {
@ -113,7 +117,8 @@ func (c *configSnapshotTerminatingGateway) IsEmpty() bool {
len(c.WatchedServices) == 0 &&
len(c.ServiceResolvers) == 0 &&
len(c.WatchedResolvers) == 0 &&
len(c.GatewayServices) == 0
len(c.GatewayServices) == 0 &&
len(c.HostnameServices) == 0
}
type configSnapshotMeshGateway struct {
@ -155,6 +160,10 @@ type configSnapshotMeshGateway struct {
// ConsulServers is the list of consul servers in this datacenter.
ConsulServers structs.CheckServiceNodes
// HostnameDatacenters is a map of datacenters to mesh gateway instances with a hostname as the address.
// If hostnames are configured they must be provided to Envoy via CDS not EDS.
HostnameDatacenters map[string]structs.CheckServiceNodes
}
func (c *configSnapshotMeshGateway) Datacenters() []string {
@ -188,7 +197,8 @@ func (c *configSnapshotMeshGateway) IsEmpty() bool {
len(c.ServiceResolvers) == 0 &&
len(c.GatewayGroups) == 0 &&
len(c.FedStateGateways) == 0 &&
len(c.ConsulServers) == 0
len(c.ConsulServers) == 0 &&
len(c.HostnameDatacenters) == 0
}
type configSnapshotIngressGateway struct {

63
agent/proxycfg/state.go
View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"net"
"reflect"
"strings"
"time"
@ -552,12 +553,14 @@ func (s *state) initialConfigSnapshot() ConfigSnapshot {
snap.TerminatingGateway.ServiceGroups = make(map[structs.ServiceID]structs.CheckServiceNodes)
snap.TerminatingGateway.ServiceResolvers = make(map[structs.ServiceID]*structs.ServiceResolverConfigEntry)
snap.TerminatingGateway.GatewayServices = make(map[structs.ServiceID]structs.GatewayService)
snap.TerminatingGateway.HostnameServices = make(map[structs.ServiceID]structs.CheckServiceNodes)
case structs.ServiceKindMeshGateway:
snap.MeshGateway.WatchedServices = make(map[structs.ServiceID]context.CancelFunc)
snap.MeshGateway.WatchedDatacenters = make(map[string]context.CancelFunc)
snap.MeshGateway.ServiceGroups = make(map[structs.ServiceID]structs.CheckServiceNodes)
snap.MeshGateway.GatewayGroups = make(map[string]structs.CheckServiceNodes)
snap.MeshGateway.ServiceResolvers = make(map[structs.ServiceID]*structs.ServiceResolverConfigEntry)
snap.MeshGateway.HostnameDatacenters = make(map[string]structs.CheckServiceNodes)
// there is no need to initialize the map of service resolvers as we
// fully rebuild it every time we get updates
case structs.ServiceKindIngressGateway:
@ -1032,6 +1035,13 @@ func (s *state) handleUpdateTerminatingGateway(u cache.UpdateEvent, snap *Config
}
}
// Clean up services with hostname mapping for services that were not in the update
for sid, _ := range snap.TerminatingGateway.HostnameServices {
if _, ok := svcMap[sid]; !ok {
delete(snap.TerminatingGateway.HostnameServices, sid)
}
}
// Cancel service instance watches for services that were not in the update
for sid, cancelFn := range snap.TerminatingGateway.WatchedServices {
if _, ok := svcMap[sid]; !ok {
@ -1081,11 +1091,12 @@ func (s *state) handleUpdateTerminatingGateway(u cache.UpdateEvent, snap *Config
}
sid := structs.ServiceIDFromString(strings.TrimPrefix(u.CorrelationID, externalServiceIDPrefix))
delete(snap.TerminatingGateway.ServiceGroups, sid)
delete(snap.TerminatingGateway.HostnameServices, sid)
if len(resp.Nodes) > 0 {
snap.TerminatingGateway.ServiceGroups[sid] = resp.Nodes
} else if _, ok := snap.TerminatingGateway.ServiceGroups[sid]; ok {
delete(snap.TerminatingGateway.ServiceGroups, sid)
snap.TerminatingGateway.HostnameServices[sid] = s.hostnameEndpoints(logging.TerminatingGateway, snap.Datacenter, resp.Nodes)
}
// Store leaf cert for watched service
@ -1141,6 +1152,17 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
return fmt.Errorf("invalid type for response: %T", u.Result)
}
snap.MeshGateway.FedStateGateways = dcIndexedNodes.DatacenterNodes
for dc, nodes := range dcIndexedNodes.DatacenterNodes {
snap.MeshGateway.HostnameDatacenters[dc] = s.hostnameEndpoints(logging.MeshGateway, snap.Datacenter, nodes)
}
for dc, _ := range snap.MeshGateway.HostnameDatacenters {
if _, ok := dcIndexedNodes.DatacenterNodes[dc]; !ok {
delete(snap.MeshGateway.HostnameDatacenters, dc)
}
}
case serviceListWatchID:
services, ok := u.Result.(*structs.IndexedServiceList)
if !ok {
@ -1297,11 +1319,12 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
}
dc := strings.TrimPrefix(u.CorrelationID, "mesh-gateway:")
delete(snap.MeshGateway.GatewayGroups, dc)
delete(snap.MeshGateway.HostnameDatacenters, dc)
if len(resp.Nodes) > 0 {
snap.MeshGateway.GatewayGroups[dc] = resp.Nodes
} else if _, ok := snap.MeshGateway.GatewayGroups[dc]; ok {
delete(snap.MeshGateway.GatewayGroups, dc)
snap.MeshGateway.HostnameDatacenters[dc] = s.hostnameEndpoints(logging.MeshGateway, snap.Datacenter, resp.Nodes)
}
default:
// do nothing for now
@ -1518,3 +1541,35 @@ func (s *state) Changed(ns *structs.NodeService, token string) bool {
!reflect.DeepEqual(s.proxyCfg, proxyCfg) ||
s.token != token
}
// hostnameEndpoints returns all CheckServiceNodes that have hostnames instead of IPs as the address.
// Envoy cannot resolve hostnames provided through EDS, so we exclusively use CDS for these clusters.
// If there is a mix of hostnames and addresses we exclusively use the hostnames, since clusters cannot discover
// services with both EDS and DNS.
func (s *state) hostnameEndpoints(loggerName string, localDC string, nodes structs.CheckServiceNodes) structs.CheckServiceNodes {
var (
hasIP bool
hasHostname bool
resp structs.CheckServiceNodes
)
for _, n := range nodes {
addr, _ := n.BestAddress(localDC != n.Node.Datacenter)
if net.ParseIP(addr) != nil {
hasIP = true
continue
}
hasHostname = true
resp = append(resp, n)
}
if hasHostname && hasIP {
dc := nodes[0].Node.Datacenter
sid := nodes[0].Service.CompoundServiceName()
s.logger.Named(loggerName).
Warn("service contains instances with mix of hostnames and IP addresses; only hostnames will be passed to Envoy.",
"dc", dc, "service", sid.String())
}
return resp
}

207
agent/proxycfg/state_test.go
View File

@ -599,6 +599,9 @@ func TestState_WatchesAndUpdates(t *testing.T) {
db := structs.NewServiceID("db", nil)
dbStr := db.String()
api := structs.NewServiceID("api", nil)
apiStr := api.String()
cases := map[string]testCase{
"initial-gateway": testCase{
ns: structs.NodeService{
@ -714,6 +717,96 @@ func TestState_WatchesAndUpdates(t *testing.T) {
require.True(t, snap.MeshGateway.WatchedServicesSet)
},
},
verificationStage{
events: []cache.UpdateEvent{
cache.UpdateEvent{
CorrelationID: "mesh-gateway:dc4",
Result: &structs.IndexedCheckServiceNodes{
Nodes: TestGatewayNodesDC4Hostname(t),
},
Err: nil,
},
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid(), "gateway with service list is valid")
require.Len(t, snap.MeshGateway.WatchedServices, 2)
require.True(t, snap.MeshGateway.WatchedServicesSet)
expect := structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.30.1.2",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.2", 8443,
structs.ServiceAddress{Address: "10.30.1.2", Port: 8443},
structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}),
},
}
require.Equal(t, snap.MeshGateway.HostnameDatacenters["dc4"], expect)
},
},
verificationStage{
events: []cache.UpdateEvent{
cache.UpdateEvent{
CorrelationID: federationStateListGatewaysWatchID,
Result: &structs.DatacenterIndexedCheckServiceNodes{
DatacenterNodes: map[string]structs.CheckServiceNodes{
"dc5": TestGatewayNodesDC5Hostname(t),
},
},
Err: nil,
},
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid(), "gateway with service list is valid")
require.Len(t, snap.MeshGateway.WatchedServices, 2)
require.True(t, snap.MeshGateway.WatchedServicesSet)
expect := structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc5",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.30.1.2",
Datacenter: "dc5",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.2", 8443,
structs.ServiceAddress{Address: "10.30.1.2", Port: 8443},
structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}),
},
}
require.Equal(t, snap.MeshGateway.HostnameDatacenters["dc5"], expect)
},
},
},
},
"ingress-gateway": testCase{
@ -1039,6 +1132,10 @@ func TestState_WatchesAndUpdates(t *testing.T) {
Service: structs.NewServiceID("billing", nil),
Gateway: structs.NewServiceID("terminating-gateway", nil),
},
{
Service: structs.NewServiceID("api", nil),
Gateway: structs.NewServiceID("terminating-gateway", nil),
},
},
},
Err: nil,
@ -1047,27 +1144,33 @@ func TestState_WatchesAndUpdates(t *testing.T) {
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
db := structs.NewServiceID("db", nil)
billing := structs.NewServiceID("billing", nil)
api := structs.NewServiceID("api", nil)
require.True(t, snap.Valid(), "gateway with service list is valid")
require.Len(t, snap.TerminatingGateway.WatchedServices, 2)
require.Len(t, snap.TerminatingGateway.WatchedServices, 3)
require.Contains(t, snap.TerminatingGateway.WatchedServices, db)
require.Contains(t, snap.TerminatingGateway.WatchedServices, billing)
require.Contains(t, snap.TerminatingGateway.WatchedServices, api)
require.Len(t, snap.TerminatingGateway.WatchedIntentions, 2)
require.Len(t, snap.TerminatingGateway.WatchedIntentions, 3)
require.Contains(t, snap.TerminatingGateway.WatchedIntentions, db)
require.Contains(t, snap.TerminatingGateway.WatchedIntentions, billing)
require.Contains(t, snap.TerminatingGateway.WatchedIntentions, api)
require.Len(t, snap.TerminatingGateway.WatchedLeaves, 2)
require.Len(t, snap.TerminatingGateway.WatchedLeaves, 3)
require.Contains(t, snap.TerminatingGateway.WatchedLeaves, db)
require.Contains(t, snap.TerminatingGateway.WatchedLeaves, billing)
require.Contains(t, snap.TerminatingGateway.WatchedLeaves, api)
require.Len(t, snap.TerminatingGateway.WatchedResolvers, 2)
require.Len(t, snap.TerminatingGateway.WatchedResolvers, 3)
require.Contains(t, snap.TerminatingGateway.WatchedResolvers, db)
require.Contains(t, snap.TerminatingGateway.WatchedResolvers, billing)
require.Contains(t, snap.TerminatingGateway.WatchedResolvers, api)
require.Len(t, snap.TerminatingGateway.GatewayServices, 2)
require.Len(t, snap.TerminatingGateway.GatewayServices, 3)
require.Contains(t, snap.TerminatingGateway.GatewayServices, db)
require.Contains(t, snap.TerminatingGateway.GatewayServices, billing)
require.Contains(t, snap.TerminatingGateway.GatewayServices, api)
},
},
verificationStage{
@ -1112,6 +1215,97 @@ func TestState_WatchesAndUpdates(t *testing.T) {
)
},
},
verificationStage{
requiredWatches: map[string]verifyWatchRequest{
"external-service:" + apiStr: genVerifyServiceWatch("api", "", "dc1", false),
},
events: []cache.UpdateEvent{
cache.UpdateEvent{
CorrelationID: "external-service:" + apiStr,
Result: &structs.IndexedCheckServiceNodes{
Nodes: structs.CheckServiceNodes{
{
Node: &structs.Node{
Node: "node1",
Address: "10.0.1.1",
},
Service: &structs.NodeService{
ID: "api",
Service: "api",
Address: "api.mydomain",
},
},
{
Node: &structs.Node{
Node: "node2",
Address: "10.0.1.2",
},
Service: &structs.NodeService{
ID: "api",
Service: "api",
Address: "api.altdomain",
},
},
{
Node: &structs.Node{
Node: "node3",
Address: "10.0.1.3",
},
Service: &structs.NodeService{
ID: "api",
Service: "api",
Address: "10.0.1.3",
},
},
},
},
Err: nil,
},
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.Len(t, snap.TerminatingGateway.ServiceGroups, 2)
expect := structs.CheckServiceNodes{
{
Node: &structs.Node{
Node: "node1",
Address: "10.0.1.1",
},
Service: &structs.NodeService{
ID: "api",
Service: "api",
Address: "api.mydomain",
},
},
{
Node: &structs.Node{
Node: "node2",
Address: "10.0.1.2",
},
Service: &structs.NodeService{
ID: "api",
Service: "api",
Address: "api.altdomain",
},
},
{
Node: &structs.Node{
Node: "node3",
Address: "10.0.1.3",
},
Service: &structs.NodeService{
ID: "api",
Service: "api",
Address: "10.0.1.3",
},
},
}
sid := structs.NewServiceID("api", nil)
require.Equal(t, snap.TerminatingGateway.ServiceGroups[sid], expect)
// The instance in node3 should not be present in HostnameDatacenters because it has a valid IP
require.ElementsMatch(t, snap.TerminatingGateway.HostnameServices[sid], expect[:2])
},
},
verificationStage{
requiredWatches: map[string]verifyWatchRequest{
"service-leaf:" + dbStr: genVerifyLeafWatch("db", "dc1"),
@ -1198,10 +1392,11 @@ func TestState_WatchesAndUpdates(t *testing.T) {
require.Len(t, snap.TerminatingGateway.GatewayServices, 1)
require.Contains(t, snap.TerminatingGateway.GatewayServices, billing)
// There was no update event for billing's leaf/endpoints, so length is 0
// There was no update event for billing's leaf/endpoints/resolvers, so length is 0
require.Len(t, snap.TerminatingGateway.ServiceGroups, 0)
require.Len(t, snap.TerminatingGateway.ServiceLeaves, 0)
require.Len(t, snap.TerminatingGateway.ServiceResolvers, 0)
require.Len(t, snap.TerminatingGateway.HostnameServices, 0)
},
},
},

172
agent/proxycfg/testing.go
View File

@ -374,6 +374,88 @@ func TestGatewayNodesDC3(t testing.T) structs.CheckServiceNodes {
}
}
func TestGatewayNodesDC4Hostname(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.30.1.2",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.2", 8443,
structs.ServiceAddress{Address: "10.30.1.2", Port: 8443},
structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-3",
Node: "mesh-gateway",
Address: "10.30.1.3",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.3", 8443,
structs.ServiceAddress{Address: "10.30.1.3", Port: 8443},
structs.ServiceAddress{Address: "198.38.1.1", Port: 443}),
},
}
}
func TestGatewayNodesDC5Hostname(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc5",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.30.1.2",
Datacenter: "dc5",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.2", 8443,
structs.ServiceAddress{Address: "10.30.1.2", Port: 8443},
structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-3",
Node: "mesh-gateway",
Address: "10.30.1.3",
Datacenter: "dc5",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.3", 8443,
structs.ServiceAddress{Address: "10.30.1.3", Port: 8443},
structs.ServiceAddress{Address: "198.38.1.1", Port: 443}),
},
}
}
func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
@ -1305,11 +1387,41 @@ func testConfigSnapshotMeshGateway(t testing.T, populateServices bool, useFedera
},
GatewayGroups: map[string]structs.CheckServiceNodes{
"dc2": TestGatewayNodesDC2(t),
"dc4": TestGatewayNodesDC4Hostname(t),
},
HostnameDatacenters: map[string]structs.CheckServiceNodes{
"dc4": {
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.30.1.2",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.2", 8443,
structs.ServiceAddress{Address: "10.30.1.2", Port: 8443},
structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}),
},
},
},
}
if useFederationStates {
snap.MeshGateway.FedStateGateways = map[string]structs.CheckServiceNodes{
"dc2": TestGatewayNodesDC2(t),
"dc4": TestGatewayNodesDC4Hostname(t),
}
delete(snap.MeshGateway.GatewayGroups, "dc2")
@ -1505,10 +1617,59 @@ func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *C
}
api := structs.NewServiceID("api", nil)
apiNodes := TestUpstreamNodes(t)
for i := 0; i < len(apiNodes); i++ {
apiNodes[i].Service.Service = "api"
apiNodes[i].Service.Port = 8081
apiNodes := structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "api",
Node: "test1",
Address: "10.10.1.1",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "api",
Address: "api.mydomain",
Port: 8081,
},
Checks: structs.HealthChecks{
{
Status: "critical",
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test2",
Node: "test2",
Address: "10.10.1.2",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "api",
Address: "api.altdomain",
Port: 8081,
Meta: map[string]string{
"domain": "alt",
},
},
Checks: structs.HealthChecks{
{
Status: "passing",
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test3",
Node: "test3",
Address: "10.10.1.3",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "api",
Address: "10.10.1.3",
Port: 8081,
},
},
}
snap.TerminatingGateway = configSnapshotTerminatingGateway{
@ -1528,6 +1689,9 @@ func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *C
KeyFile: "api.key.pem",
},
},
HostnameServices: map[structs.ServiceID]structs.CheckServiceNodes{
api: {apiNodes[0], apiNodes[1]},
},
}
snap.TerminatingGateway.ServiceLeaves = map[structs.ServiceID]*structs.IssuedCert{
structs.NewServiceID("web", nil): {

177
agent/xds/clusters.go
View File

@ -133,35 +133,41 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho
if dc == cfgSnap.Datacenter {
continue // skip local
}
clusterName := connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain)
cluster, err := s.makeGatewayCluster(cfgSnap, clusterName)
if err != nil {
return nil, err
opts := gatewayClusterOpts{
name: connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain),
hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[dc],
isRemote: dc != cfgSnap.Datacenter,
}
cluster := s.makeGatewayCluster(cfgSnap, opts)
clusters = append(clusters, cluster)
}
if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil {
// Add all of the remote wildcard datacenter mappings for servers.
for _, dc := range datacenters {
clusterName := cfgSnap.ServerSNIFn(dc, "")
hostnameEndpoints := cfgSnap.MeshGateway.HostnameDatacenters[dc]
cluster, err := s.makeGatewayCluster(cfgSnap, clusterName)
if err != nil {
return nil, err
// If the DC is our current DC then this cluster is for traffic from a remote DC to a local server.
// HostnameDatacenters is populated with gateway addresses, so it does not apply here.
if dc == cfgSnap.Datacenter {
hostnameEndpoints = nil
}
opts := gatewayClusterOpts{
name: cfgSnap.ServerSNIFn(dc, ""),
hostnameEndpoints: hostnameEndpoints,
isRemote: dc != cfgSnap.Datacenter,
}
cluster := s.makeGatewayCluster(cfgSnap, opts)
clusters = append(clusters, cluster)
}
// And for the current datacenter, send all flavors appropriately.
for _, srv := range cfgSnap.MeshGateway.ConsulServers {
clusterName := cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node)
cluster, err := s.makeGatewayCluster(cfgSnap, clusterName)
if err != nil {
return nil, err
opts := gatewayClusterOpts{
name: cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node),
}
cluster := s.makeGatewayCluster(cfgSnap, opts)
clusters = append(clusters, cluster)
}
}
@ -179,6 +185,7 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho
func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
var services map[structs.ServiceID]structs.CheckServiceNodes
var resolvers map[structs.ServiceID]*structs.ServiceResolverConfigEntry
var hostnameEndpoints structs.CheckServiceNodes
switch cfgSnap.Kind {
case structs.ServiceKindTerminatingGateway:
@ -197,33 +204,44 @@ func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([
clusterName := connect.ServiceSNI(svc.ID, "", svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
resolver, hasResolver := resolvers[svc]
// Create the cluster for default/unnamed services
var cluster *envoy.Cluster
var err error
if !hasResolver {
// Use a zero value resolver with no timeout and no subsets
resolver = &structs.ServiceResolverConfigEntry{}
}
cluster, err = s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, resolver.ConnectTimeout)
if err != nil {
return nil, fmt.Errorf("failed to make %s cluster: %v", cfgSnap.Kind, err)
// When making service clusters we only pass endpoints with hostnames if the kind is a terminating gateway
// This is because the services a mesh gateway will route to are not external services and are not addressed by a hostname.
if cfgSnap.Kind == structs.ServiceKindTerminatingGateway {
hostnameEndpoints = cfgSnap.TerminatingGateway.HostnameServices[svc]
}
opts := gatewayClusterOpts{
name: clusterName,
hostnameEndpoints: hostnameEndpoints,
connectTimeout: resolver.ConnectTimeout,
}
cluster := s.makeGatewayCluster(cfgSnap, opts)
if cfgSnap.Kind == structs.ServiceKindTerminatingGateway {
injectTerminatingGatewayTLSContext(cfgSnap, cluster, svc)
}
clusters = append(clusters, cluster)
// If there is a service-resolver for this service then also setup a cluster for each subset
for subsetName := range resolver.Subsets {
clusterName := connect.ServiceSNI(svc.ID, subsetName, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
cluster, err := s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, resolver.ConnectTimeout)
for name, subset := range resolver.Subsets {
subsetHostnameEndpoints, err := s.filterSubsetEndpoints(&subset, hostnameEndpoints)
if err != nil {
return nil, fmt.Errorf("failed to make %s cluster: %v", cfgSnap.Kind, err)
return nil, err
}
opts := gatewayClusterOpts{
name: connect.ServiceSNI(svc.ID, name, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain),
hostnameEndpoints: subsetHostnameEndpoints,
onlyPassing: subset.OnlyPassing,
connectTimeout: resolver.ConnectTimeout,
}
cluster := s.makeGatewayCluster(cfgSnap, opts)
if cfgSnap.Kind == structs.ServiceKindTerminatingGateway {
injectTerminatingGatewayTLSContext(cfgSnap, cluster, svc)
}
@ -547,43 +565,86 @@ func makeClusterFromUserConfig(configJSON string) (*envoy.Cluster, error) {
return &c, err
}
func (s *Server) makeGatewayCluster(cfgSnap *proxycfg.ConfigSnapshot, clusterName string) (*envoy.Cluster, error) {
return s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, 0)
type gatewayClusterOpts struct {
// name for the cluster
name string
// isRemote determines whether the cluster is in a remote DC and we should prefer a WAN address
isRemote bool
// onlyPassing determines whether endpoints that do not have a passing status should be considered unhealthy
onlyPassing bool
// connectTimeout is the timeout for new network connections to hosts in the cluster
connectTimeout time.Duration
// hostnameEndpoints is a list of endpoints with a hostname as their address
hostnameEndpoints structs.CheckServiceNodes
}
// makeGatewayClusterWithConnectTimeout initializes a gateway cluster
// with the specified connect timeout. If the timeout is 0, the connect timeout
// defaults to use the configured gateway timeout.
func (s *Server) makeGatewayClusterWithConnectTimeout(cfgSnap *proxycfg.ConfigSnapshot,
clusterName string, connectTimeout time.Duration) (*envoy.Cluster, error) {
cfg, err := ParseGatewayConfig(cfgSnap.Proxy.Config)
func (s *Server) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, opts gatewayClusterOpts) *envoy.Cluster {
cfg, err := ParseGatewayConfig(snap.Proxy.Config)
if err != nil {
// Don't hard fail on a config typo, just warn. The parse func returns
// default config if there is an error so it's safe to continue.
s.Logger.Warn("failed to parse gateway config", "error", err)
}
if connectTimeout <= 0 {
connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond
if opts.connectTimeout <= 0 {
opts.connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond
}
cluster := envoy.Cluster{
Name: clusterName,
ConnectTimeout: connectTimeout,
ClusterDiscoveryType: &envoy.Cluster_Type{Type: envoy.Cluster_EDS},
EdsClusterConfig: &envoy.Cluster_EdsClusterConfig{
cluster := &envoy.Cluster{
Name: opts.name,
ConnectTimeout: opts.connectTimeout,
// Having an empty config enables outlier detection with default config.
OutlierDetection: &envoycluster.OutlierDetection{},
}
useEDS := true
if len(opts.hostnameEndpoints) > 0 {
useEDS = false
}
// If none of the service instances are addressed by a hostname we provide the endpoint IP addresses via EDS
if useEDS {
cluster.ClusterDiscoveryType = &envoy.Cluster_Type{Type: envoy.Cluster_EDS}
cluster.EdsClusterConfig = &envoy.Cluster_EdsClusterConfig{
EdsConfig: &envoycore.ConfigSource{
ConfigSourceSpecifier: &envoycore.ConfigSource_Ads{
Ads: &envoycore.AggregatedConfigSource{},
},
},
},
// Having an empty config enables outlier detection with default config.
OutlierDetection: &envoycluster.OutlierDetection{},
}
return cluster
}
return &cluster, nil
// When a service instance is addressed by a hostname we have Envoy do the DNS resolution
// by setting a DNS cluster type and passing the hostname endpoints via CDS.
rate := 10 * time.Second
cluster.DnsRefreshRate = &rate
cluster.DnsLookupFamily = envoy.Cluster_V4_ONLY
discoveryType := envoy.Cluster_Type{Type: envoy.Cluster_LOGICAL_DNS}
if cfg.DNSDiscoveryType == "strict_dns" {
discoveryType.Type = envoy.Cluster_STRICT_DNS
}
cluster.ClusterDiscoveryType = &discoveryType
endpoints := make([]envoyendpoint.LbEndpoint, 0, len(opts.hostnameEndpoints))
for _, e := range opts.hostnameEndpoints {
endpoints = append(endpoints, makeLbEndpoint(e, opts.isRemote, opts.onlyPassing))
}
cluster.LoadAssignment = &envoy.ClusterLoadAssignment{
ClusterName: cluster.Name,
Endpoints: []envoyendpoint.LocalityLbEndpoints{
{
LbEndpoints: endpoints,
},
},
}
return cluster
}
// injectTerminatingGatewayTLSContext adds an UpstreamTlsContext to a cluster for TLS origination
@ -621,3 +682,27 @@ func makeThresholdsIfNeeded(limits UpstreamLimits) []*envoycluster.CircuitBreake
return []*envoycluster.CircuitBreakers_Thresholds{threshold}
}
func makeLbEndpoint(csn structs.CheckServiceNode, isRemote, onlyPassing bool) envoyendpoint.LbEndpoint {
health, weight := calculateEndpointHealthAndWeight(csn, onlyPassing)
addr, port := csn.BestAddress(isRemote)
return envoyendpoint.LbEndpoint{
HostIdentifier: &envoyendpoint.LbEndpoint_Endpoint{
Endpoint: &envoyendpoint.Endpoint{
Address: &envoycore.Address{
Address: &envoycore.Address_SocketAddress{
SocketAddress: &envoycore.SocketAddress{
Address: addr,
PortSpecifier: &envoycore.SocketAddress_PortValue{
PortValue: uint32(port),
},
},
},
},
},
},
HealthStatus: health,
LoadBalancingWeight: makeUint32Value(weight),
}
}

17
agent/xds/clusters_test.go
View File

@ -449,6 +449,23 @@ func TestClustersFromSnapshot(t *testing.T) {
}
},
},
{
name: "terminating-gateway-hostname-service-subsets",
create: proxycfg.TestConfigSnapshotTerminatingGateway,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.TerminatingGateway.ServiceResolvers = map[structs.ServiceID]*structs.ServiceResolverConfigEntry{
structs.NewServiceID("api", nil): {
Kind: structs.ServiceResolver,
Name: "api",
Subsets: map[string]structs.ServiceResolverSubset{
"alt": {
Filter: "Service.Meta.domain == alt",
},
},
},
}
},
},
{
name: "terminating-gateway-ignore-extra-resolvers",
create: proxycfg.TestConfigSnapshotTerminatingGateway,

7
agent/xds/config.go
View File

@ -88,6 +88,10 @@ type GatewayConfig struct {
// gateway service
NoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind" alias:"envoy_mesh_gateway_no_default_bind"`
// DNSDiscoveryType indicates the DNS service discovery type.
// See: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/service_discovery#arch-overview-service-discovery-types
DNSDiscoveryType string `mapstructure:"envoy_dns_discovery_type"`
// ConnectTimeoutMs is the number of milliseconds to timeout making a new
// connection to this upstream. Defaults to 5000 (5 seconds) if not set.
ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"`
@ -113,6 +117,9 @@ func ParseGatewayConfig(m map[string]interface{}) (GatewayConfig, error) {
if cfg.ConnectTimeoutMs < 1 {
cfg.ConnectTimeoutMs = 5000
}
cfg.DNSDiscoveryType = strings.ToLower(cfg.DNSDiscoveryType)
return cfg, err
}

2
agent/xds/config_test.go
View File

@ -308,6 +308,7 @@ func TestParseGatewayConfig(t *testing.T) {
"envoy_gateway_bind_tagged_addresses": true,
"envoy_gateway_bind_addresses": map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}},
"envoy_gateway_no_default_bind": true,
"envoy_dns_discovery_type": "StRiCt_DnS",
"connect_timeout_ms": 10,
},
want: GatewayConfig{
@ -315,6 +316,7 @@ func TestParseGatewayConfig(t *testing.T) {
BindTaggedAddresses: true,
NoDefaultBind: true,
BindAddresses: map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}},
DNSDiscoveryType: "strict_dns",
},
},
{

17
agent/xds/endpoints.go
View File

@ -3,7 +3,6 @@ package xds
import (
"errors"
"fmt"
envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2"
envoycore "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
envoyendpoint "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
@ -119,9 +118,12 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh
// generate the endpoints for the gateways in the remote datacenters
for _, dc := range datacenters {
if dc == cfgSnap.Datacenter {
continue // skip local
// Skip creating endpoints for mesh gateways in local DC and gateways in remote DCs with a hostname as their address
// EDS cannot resolve hostnames so we provide them through CDS instead
if dc == cfgSnap.Datacenter || len(cfgSnap.MeshGateway.HostnameDatacenters[dc]) > 0 {
continue
}
endpoints, ok := cfgSnap.MeshGateway.GatewayGroups[dc]
if !ok {
endpoints, ok = cfgSnap.MeshGateway.FedStateGateways[dc]
@ -158,9 +160,8 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh
}
}
// generate endpoints for our servers if WAN federation is enabled
if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil {
// generate endpoints for our servers
var allServersLbEndpoints []envoyendpoint.LbEndpoint
for _, srv := range cfgSnap.MeshGateway.ConsulServers {
@ -217,6 +218,12 @@ func (s *Server) endpointsFromServicesAndResolvers(
// generate the endpoints for the linked service groups
for svc, endpoints := range services {
// Skip creating endpoints for services that have hostnames as addresses
// EDS cannot resolve hostnames so we provide them through CDS instead
if cfgSnap.Kind == structs.ServiceKindTerminatingGateway && len(cfgSnap.TerminatingGateway.HostnameServices[svc]) > 0 {
continue
}
clusterEndpoints := make(map[string][]loadAssignmentEndpointGroup)
clusterEndpoints[UnnamedSubset] = []loadAssignmentEndpointGroup{{Endpoints: endpoints, OnlyPassing: false}}

6
agent/xds/listeners_test.go
View File

@ -461,10 +461,10 @@ func TestListenersFromSnapshot(t *testing.T) {
})
// For terminating gateways we create filter chain matches for services/subsets from the ServiceGroups map
if snap.Kind == structs.ServiceKindTerminatingGateway {
for i := 0; i < len(listeners); i++ {
l := listeners[i].(*envoy.Listener)
for i := 0; i < len(listeners); i++ {
l := listeners[i].(*envoy.Listener)
if l.FilterChains != nil {
// Sort chains by the matched name with the exception of the last one
// The last chain is a fallback and does not have a FilterChainMatch
sort.Slice(l.FilterChains[:len(l.FilterChains)-1], func(i, j int) bool {

44
agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden vendored
View File

@ -33,6 +33,50 @@
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "123.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",

44
agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden vendored
View File

@ -33,6 +33,50 @@
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "123.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",

44
agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden vendored
View File

@ -33,6 +33,50 @@
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "123.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",

44
agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden vendored
View File

@ -33,6 +33,50 @@
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "123.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",

44
agent/xds/testdata/clusters/mesh-gateway.golden vendored
View File

@ -33,6 +33,50 @@
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "123.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",

155
agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden vendored Normal file
View File

@ -0,0 +1,155 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "alt.api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "alt.api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.altdomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"filename": "api.cert.pem"
},
"privateKey": {
"filename": "api.key.pem"
}
}
],
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.altdomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"filename": "api.cert.pem"
},
"privateKey": {
"filename": "api.key.pem"
}
}
],
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
}
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Cluster",
"nonce": "00000001"
}

44
agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden vendored
View File

@ -4,15 +4,41 @@
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
}
}
},
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.altdomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
@ -35,6 +61,8 @@
}
}
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}

44
agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden vendored
View File

@ -4,15 +4,41 @@
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
}
}
},
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.altdomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
@ -35,6 +61,8 @@
}
}
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}

44
agent/xds/testdata/clusters/terminating-gateway.golden vendored
View File

@ -4,15 +4,41 @@
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
}
}
},
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.altdomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
@ -35,6 +61,8 @@
}
}
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
}

2
agent/xds/testdata/endpoints/connect-proxy-with-default-chain-and-custom-cluster.golden vendored
View File

@ -38,4 +38,4 @@
],
"typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
"nonce": "00000001"
}
}

34
agent/xds/testdata/endpoints/terminating-gateway-default-service-subset.golden vendored
View File

@ -1,40 +1,6 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.1",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.2",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
"clusterName": "v1.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",

34
agent/xds/testdata/endpoints/terminating-gateway-service-subsets.golden vendored
View File

@ -1,40 +1,6 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.1",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.2",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
"clusterName": "v1.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",

34
agent/xds/testdata/endpoints/terminating-gateway.golden vendored
View File

@ -1,40 +1,6 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.1",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.2",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
"clusterName": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",

64
agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden vendored
View File

@ -27,6 +27,22 @@
}
]
},
{
"filterChainMatch": {
"serverNames": [
"*.dc4.internal.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"stat_prefix": "mesh_gateway_remote_bar_dc4_tcp"
}
}
]
},
{
"filters": [
{
@ -74,6 +90,22 @@
}
]
},
{
"filterChainMatch": {
"serverNames": [
"*.dc4.internal.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"stat_prefix": "mesh_gateway_remote_baz_dc4_tcp"
}
}
]
},
{
"filters": [
{
@ -121,6 +153,22 @@
}
]
},
{
"filterChainMatch": {
"serverNames": [
"*.dc4.internal.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"stat_prefix": "mesh_gateway_remote_default_dc4_tcp"
}
}
]
},
{
"filters": [
{
@ -168,6 +216,22 @@
}
]
},
{
"filterChainMatch": {
"serverNames": [
"*.dc4.internal.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"stat_prefix": "mesh_gateway_remote_foo_dc4_tcp"
}
}
]
},
{
"filters": [
{

32
agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden vendored
View File

@ -27,6 +27,22 @@
}
]
},
{
"filterChainMatch": {
"serverNames": [
"*.dc4.internal.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"stat_prefix": "mesh_gateway_remote_lan_dc4_tcp"
}
}
]
},
{
"filters": [
{
@ -74,6 +90,22 @@
}
]
},
{
"filterChainMatch": {
"serverNames": [
"*.dc4.internal.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"stat_prefix": "mesh_gateway_remote_wan_dc4_tcp"
}
}
]
},
{
"filters": [
{

16
agent/xds/testdata/listeners/mesh-gateway-using-federation-states.golden vendored
View File

@ -27,6 +27,22 @@
}
]
},
{
"filterChainMatch": {
"serverNames": [
"*.dc4.internal.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"stat_prefix": "mesh_gateway_remote_default_dc4_tcp"
}
}
]
},
{
"filters": [
{

16
agent/xds/testdata/listeners/mesh-gateway.golden vendored
View File

@ -27,6 +27,22 @@
}
]
},
{
"filterChainMatch": {
"serverNames": [
"*.dc4.internal.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"stat_prefix": "mesh_gateway_remote_default_dc4_tcp"
}
}
]
},
{
"filters": [
{

4
test/integration/connect/envoy/case-terminating-gateway-hostnames/capture.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
snapshot_envoy_admin localhost:20000 terminating-gateway primary || true
snapshot_envoy_admin localhost:19000 s1 primary || true

14
test/integration/connect/envoy/case-terminating-gateway-hostnames/config_entries.hcl Normal file
View File

@ -0,0 +1,14 @@
enable_central_service_config = true
config_entries {
bootstrap {
kind = "terminating-gateway"
name = "terminating-gateway"
services = [
{
name = "s4"
}
]
}
}

5
test/integration/connect/envoy/case-terminating-gateway-hostnames/gateway.hcl Normal file
View File

@ -0,0 +1,5 @@
services {
name = "terminating-gateway"
kind = "terminating-gateway"
port = 8443
}

16
test/integration/connect/envoy/case-terminating-gateway-hostnames/s1.hcl Normal file
View File

@ -0,0 +1,16 @@
services {
name = "s1"
port = 8080
connect {
sidecar_service {
proxy {
upstreams = [
{
destination_name = "s4"
local_bind_port = 5000
}
]
}
}
}
}

7
test/integration/connect/envoy/case-terminating-gateway-hostnames/s4.hcl Normal file
View File

@ -0,0 +1,7 @@
services {
name = "s4"
// EDS cannot resolve localhost to an IP address
address = "localhost"
port = 8382
}

9
test/integration/connect/envoy/case-terminating-gateway-hostnames/setup.sh Normal file
View File

@ -0,0 +1,9 @@
#!/bin/bash
set -euo pipefail
# wait for bootstrap to apply config entries
wait_for_config_entry terminating-gateway terminating-gateway
gen_envoy_bootstrap terminating-gateway 20000 primary true
gen_envoy_bootstrap s1 19000

4
test/integration/connect/envoy/case-terminating-gateway-hostnames/vars.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
# There is no sidecar proxy for s2, since the terminating gateway acts as the proxy
export REQUIRED_SERVICES="s1 s1-sidecar-proxy s4 terminating-gateway-primary"

33
test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats Normal file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env bats
load helpers
@test "terminating proxy admin is up on :20000" {
retry_default curl -f -s localhost:20000/stats -o /dev/null
}
@test "s1 proxy admin is up on :19000" {
retry_default curl -f -s localhost:19000/stats -o /dev/null
}
@test "terminating-gateway-primary listener is up on :8443" {
retry_default nc -z localhost:8443
}
@test "terminating-gateway should have healthy endpoints for s4" {
assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s4 HEALTHY 1
}
@test "s1 upstream should have healthy endpoints for s4" {
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s4.default.primary HEALTHY 1
}
@test "s1 upstream should be able to connect to s4" {
run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ]
[ "$output" = "hello" ]
}
@test "terminating-gateway is used for the upstream connection" {
assert_envoy_metric_at_least 127.0.0.1:20000 "s4.default.primary.*cx_total" 1
}

16
test/integration/connect/envoy/docker-compose.yml
View File

@ -172,6 +172,22 @@ services:
- "disabled"
network_mode: service:consul-primary
s4:
depends_on:
- consul-primary
image: "fortio/fortio"
environment:
- "FORTIO_NAME=s4"
command:
- "server"
- "-http-port"
- ":8382"
- "-grpc-port"
- ":8281"
- "-redirect-port"
- "disabled"
network_mode: service:consul-primary
s1-sidecar-proxy:
depends_on:
- consul-primary

2
test/integration/connect/envoy/helpers.bash
View File

@ -425,7 +425,7 @@ function docker_consul {
function docker_wget {
local DC=$1
shift 1
docker run -ti --rm --network container:envoy_consul-${DC}_1 alpine:3.9 wget "$@"
docker run --rm --network container:envoy_consul-${DC}_1 alpine:3.9 wget "$@"
}
function docker_curl {

1
test/integration/connect/envoy/main_test.go
View File

@ -38,6 +38,7 @@ func TestEnvoy(t *testing.T) {
"case-prometheus",
"case-statsd-udp",
"case-stats-proxy",
"case-terminating-gateway-hostnames",
"case-terminating-gateway-simple",
"case-terminating-gateway-subsets",
"case-terminating-gateway-without-services",

7
website/pages/docs/connect/proxies/envoy.mdx
View File

@ -319,6 +319,13 @@ will continue to be supported.
- `envoy_gateway_no_default_bind` - Prevents binding to the default address
of the gateway service. This should be used with one of the other options
to configure the gateway's bind addresses.
- `envoy_dns_discovery_type` - Determines how Envoy will resolve hostnames. Defaults to `LOGICAL_DNS`.
Must be one of `STRICT_DNS` or `LOGICAL_DNS`. Details for each type are available in
the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/v1.14.1/intro/arch_overview/upstream/service_discovery).
This option applies to terminating gateways that route to services
addressed by a hostname, such as a managed databased. It also applies to mesh gateways,
such as when gateways in other Consul datacenters are behind a load balancer that is addressed by a hostname.
## Advanced Configuration

3
website/pages/docs/connect/terminating-gateway.mdx
View File

@ -24,8 +24,7 @@ services outside the mesh, review the [terminating gateway guide](https://learn.
~> **Beta limitations:** Terminating Gateways currently do not support targeting service subsets with
[L7 configuration](/docs/connect/l7-traffic-management). They route to all instances of a service with no capabilities
for filtering by instance. Terminating Gateways also currently do not support routing to services with a hostname
defined as a their address. The service address registered with Consul, that the gateway will route traffic to, **must** be a resolved IP address.
for filtering by instance.
## Security Considerations