luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

TLS Origination for Terminating Gateways (#7671)

This commit is contained in:
Freddy 2020-04-27 16:25:37 -06:00 committed by GitHub
parent 69266b720e
commit f5c1e5268b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 448 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
agent/cache-types/gateway_services_test.go
View File

@ -34,6 +34,7 @@ func TestGatewayServices(t *testing.T) {
CAFile: "api/ca.crt",
CertFile: "api/client.crt",
KeyFile: "api/client.key",
SNI: "my-domain",
},
}
reply := args.Get(2).(*structs.IndexedGatewayServices)

4
agent/consul/internal_endpoint_test.go
View File

@ -731,6 +731,7 @@ func TestInternal_TerminatingGatewayServices(t *testing.T) {
CAFile: "api/ca.crt",
CertFile: "api/client.crt",
KeyFile: "api/client.key",
SNI: "my-domain",
},
{
Name: "db",
@ -740,6 +741,7 @@ func TestInternal_TerminatingGatewayServices(t *testing.T) {
CAFile: "ca.crt",
CertFile: "client.crt",
KeyFile: "client.key",
SNI: "my-alt-domain",
},
},
},
@ -766,6 +768,7 @@ func TestInternal_TerminatingGatewayServices(t *testing.T) {
CAFile: "api/ca.crt",
CertFile: "api/client.crt",
KeyFile: "api/client.key",
SNI: "my-domain",
},
{
Service: structs.NewServiceID("db", nil),
@ -782,6 +785,7 @@ func TestInternal_TerminatingGatewayServices(t *testing.T) {
CAFile: "ca.crt",
CertFile: "client.crt",
KeyFile: "client.key",
SNI: "my-alt-domain",
FromWildcard: true,
},
}

31
agent/consul/state/catalog.go
View File

@ -1037,12 +1037,16 @@ func (s *Store) serviceNodes(ws memdb.WatchSet, serviceName string, connect bool
// Gateways are tracked in a separate table, and we append them to the result set.
// We append rather than replace since it allows users to migrate a service
// to the mesh with a mix of sidecars and gateways until all its instances have a sidecar.
var idx uint64
if connect {
// Look up gateway nodes associated with the service
_, nodes, chs, err := s.serviceGatewayNodes(tx, ws, serviceName, structs.ServiceKindTerminatingGateway, entMeta)
gwIdx, nodes, chs, err := s.serviceGatewayNodes(tx, ws, serviceName, structs.ServiceKindTerminatingGateway, entMeta)
if err != nil {
return 0, nil, fmt.Errorf("failed gateway nodes lookup: %v", err)
}
if idx < gwIdx {
idx = gwIdx
}
for _, ch := range chs {
ws.Add(ch)
@ -1059,7 +1063,10 @@ func (s *Store) serviceNodes(ws memdb.WatchSet, serviceName string, connect bool
}
// Get the table index.
idx := s.maxIndexForService(tx, serviceName, len(results) > 0, false, entMeta)
svcIdx := s.maxIndexForService(tx, serviceName, len(results) > 0, false, entMeta)
if idx < svcIdx {
idx = svcIdx
}
return idx, results, nil
}
@ -2035,12 +2042,16 @@ func (s *Store) checkServiceNodesTxn(tx *memdb.Txn, ws memdb.WatchSet, serviceNa
// Gateways are tracked in a separate table, and we append them to the result set.
// We append rather than replace since it allows users to migrate a service
// to the mesh with a mix of sidecars and gateways until all its instances have a sidecar.
var idx uint64
if connect {
// Look up gateway nodes associated with the service
_, nodes, _, err := s.serviceGatewayNodes(tx, ws, serviceName, structs.ServiceKindTerminatingGateway, entMeta)
gwIdx, nodes, _, err := s.serviceGatewayNodes(tx, ws, serviceName, structs.ServiceKindTerminatingGateway, entMeta)
if err != nil {
return 0, nil, fmt.Errorf("failed gateway nodes lookup: %v", err)
}
if idx < gwIdx {
idx = gwIdx
}
for i := 0; i < len(nodes); i++ {
results = append(results, nodes[i])
serviceNames[nodes[i].ServiceName] = struct{}{}
@ -2056,7 +2067,6 @@ func (s *Store) checkServiceNodesTxn(tx *memdb.Txn, ws memdb.WatchSet, serviceNa
// (~682 service instances). See
// https://github.com/hashicorp/consul/issues/4984
watchOptimized := false
idx := uint64(0)
if len(serviceNames) > 0 {
// Assume optimization will work since it really should at this point. For
// safety we'll sanity check this below for each service name.
@ -2527,6 +2537,7 @@ func (s *Store) terminatingConfigGatewayServices(tx *memdb.Txn, gateway structs.
KeyFile: svc.KeyFile,
CertFile: svc.CertFile,
CAFile: svc.CAFile,
SNI: svc.SNI,
}
gatewayServices = append(gatewayServices, mapping)
@ -2684,10 +2695,22 @@ func (s *Store) serviceGatewayNodes(tx *memdb.Txn, ws memdb.WatchSet, service st
if err != nil {
return 0, nil, nil, fmt.Errorf("failed service lookup: %s", err)
}
var exists bool
for svc := gwServices.Next(); svc != nil; svc = gwServices.Next() {
sn := svc.(*structs.ServiceNode)
ret = append(ret, sn)
// Tracking existence to know whether we should check extinction index for service
exists = true
}
// This prevents the index from sliding back in case all instances of the service are deregistered
svcIdx := s.maxIndexForService(tx, mapping.Gateway.ID, exists, false, &mapping.Service.EnterpriseMeta)
if maxIdx < svcIdx {
maxIdx = svcIdx
}
watchChans = append(watchChans, gwServices.WatchCh())
}
return maxIdx, ret, watchChans, nil

49
agent/consul/state/catalog_test.go
View File

@ -2168,7 +2168,7 @@ func TestStateStore_ConnectServiceNodes_Gateways(t *testing.T) {
ws = memdb.NewWatchSet()
idx, nodes, err = s.ConnectServiceNodes(ws, "db", nil)
assert.Nil(err)
assert.Equal(idx, uint64(14))
assert.Equal(idx, uint64(17))
assert.Len(nodes, 2)
// Check sidecar
@ -2191,12 +2191,12 @@ func TestStateStore_ConnectServiceNodes_Gateways(t *testing.T) {
assert.True(watchFired(ws))
// Watch should fire when a gateway instance is de-registered
assert.Nil(s.DeleteService(29, "bar", "gateway", nil))
assert.Nil(s.DeleteService(19, "bar", "gateway", nil))
assert.True(watchFired(ws))
idx, nodes, err = s.ConnectServiceNodes(ws, "db", nil)
assert.Nil(err)
assert.Equal(idx, uint64(14))
assert.Equal(idx, uint64(19))
assert.Len(nodes, 2)
// Check the new gateway
@ -2205,6 +2205,22 @@ func TestStateStore_ConnectServiceNodes_Gateways(t *testing.T) {
assert.Equal("gateway", nodes[1].ServiceName)
assert.Equal("gateway-2", nodes[1].ServiceID)
assert.Equal(443, nodes[1].ServicePort)
// Index should not slide back after deleting all instances of the gateway
assert.Nil(s.DeleteService(20, "foo", "gateway-2", nil))
assert.True(watchFired(ws))
idx, nodes, err = s.ConnectServiceNodes(ws, "db", nil)
assert.Nil(err)
assert.Equal(idx, uint64(20))
assert.Len(nodes, 1)
// Ensure that remaining node is the proxy and not a gateway
assert.Equal(structs.ServiceKindConnectProxy, nodes[0].ServiceKind)
assert.Equal("foo", nodes[0].Node)
assert.Equal("proxy", nodes[0].ServiceName)
assert.Equal("proxy", nodes[0].ServiceID)
assert.Equal(8000, nodes[0].ServicePort)
}
func TestStateStore_Service_Snapshot(t *testing.T) {
@ -3622,6 +3638,11 @@ func TestStateStore_CheckConnectServiceNodes_Gateways(t *testing.T) {
assert.Nil(s.EnsureService(22, "foo", &structs.NodeService{Kind: structs.ServiceKindTerminatingGateway, ID: "gateway-2", Service: "gateway", Port: 443}))
assert.True(watchFired(ws))
idx, nodes, err = s.CheckConnectServiceNodes(ws, "db", nil)
assert.Nil(err)
assert.Equal(idx, uint64(22))
assert.Len(nodes, 3)
// Watch should fire when a gateway instance is de-registered
assert.Nil(s.DeleteService(23, "bar", "gateway", nil))
assert.True(watchFired(ws))
@ -3637,6 +3658,22 @@ func TestStateStore_CheckConnectServiceNodes_Gateways(t *testing.T) {
assert.Equal("gateway", nodes[1].Service.Service)
assert.Equal("gateway-2", nodes[1].Service.ID)
assert.Equal(443, nodes[1].Service.Port)
// Index should not slide back after deleting all instances of the gateway
assert.Nil(s.DeleteService(24, "foo", "gateway-2", nil))
assert.True(watchFired(ws))
idx, nodes, err = s.CheckConnectServiceNodes(ws, "db", nil)
assert.Nil(err)
assert.Equal(idx, uint64(24))
assert.Len(nodes, 1)
// Ensure that remaining node is the proxy and not a gateway
assert.Equal(structs.ServiceKindConnectProxy, nodes[0].Service.Kind)
assert.Equal("foo", nodes[0].Node.Node)
assert.Equal("proxy", nodes[0].Service.Service)
assert.Equal("proxy", nodes[0].Service.ID)
assert.Equal(8000, nodes[0].Service.Port)
}
func BenchmarkCheckServiceNodes(b *testing.B) {
@ -4484,6 +4521,7 @@ func TestStateStore_GatewayServices_Terminating(t *testing.T) {
CAFile: "api/ca.crt",
CertFile: "api/client.crt",
KeyFile: "api/client.key",
SNI: "my-domain",
},
{
Name: "db",
@ -4493,6 +4531,7 @@ func TestStateStore_GatewayServices_Terminating(t *testing.T) {
CAFile: "ca.crt",
CertFile: "client.crt",
KeyFile: "client.key",
SNI: "my-alt-domain",
},
},
}, nil))
@ -4513,6 +4552,7 @@ func TestStateStore_GatewayServices_Terminating(t *testing.T) {
CAFile: "api/ca.crt",
CertFile: "api/client.crt",
KeyFile: "api/client.key",
SNI: "my-domain",
RaftIndex: structs.RaftIndex{
CreateIndex: 22,
ModifyIndex: 22,
@ -4547,6 +4587,7 @@ func TestStateStore_GatewayServices_Terminating(t *testing.T) {
CAFile: "api/ca.crt",
CertFile: "api/client.crt",
KeyFile: "api/client.key",
SNI: "my-domain",
RaftIndex: structs.RaftIndex{
CreateIndex: 22,
ModifyIndex: 22,
@ -4568,6 +4609,7 @@ func TestStateStore_GatewayServices_Terminating(t *testing.T) {
CAFile: "ca.crt",
CertFile: "client.crt",
KeyFile: "client.key",
SNI: "my-alt-domain",
FromWildcard: true,
RaftIndex: structs.RaftIndex{
CreateIndex: 23,
@ -4594,6 +4636,7 @@ func TestStateStore_GatewayServices_Terminating(t *testing.T) {
CAFile: "api/ca.crt",
CertFile: "api/client.crt",
KeyFile: "api/client.key",
SNI: "my-domain",
RaftIndex: structs.RaftIndex{
CreateIndex: 22,
ModifyIndex: 22,

8
agent/proxycfg/snapshot.go
View File

@ -93,6 +93,11 @@ type configSnapshotTerminatingGateway struct {
// ServiceGroups is a map of service id to the service instances of that
// service in the local datacenter.
ServiceGroups map[structs.ServiceID]structs.CheckServiceNodes
// GatewayServices is a map of service id to the config entry association
// between the gateway and a service. TLS configuration stored here is
// used for TLS origination from the gateway to the linked service.
GatewayServices map[structs.ServiceID]structs.GatewayService
}
func (c *configSnapshotTerminatingGateway) IsEmpty() bool {
@ -105,7 +110,8 @@ func (c *configSnapshotTerminatingGateway) IsEmpty() bool {
len(c.ServiceGroups) == 0 &&
len(c.WatchedServices) == 0 &&
len(c.ServiceResolvers) == 0 &&
len(c.WatchedResolvers) == 0
len(c.WatchedResolvers) == 0 &&
len(c.GatewayServices) == 0
}
type configSnapshotMeshGateway struct {

11
agent/proxycfg/state.go
View File

@ -543,6 +543,7 @@ func (s *state) initialConfigSnapshot() ConfigSnapshot {
snap.TerminatingGateway.ServiceLeaves = make(map[structs.ServiceID]*structs.IssuedCert)
snap.TerminatingGateway.ServiceGroups = make(map[structs.ServiceID]structs.CheckServiceNodes)
snap.TerminatingGateway.ServiceResolvers = make(map[structs.ServiceID]*structs.ServiceResolverConfigEntry)
snap.TerminatingGateway.GatewayServices = make(map[structs.ServiceID]structs.GatewayService)
case structs.ServiceKindMeshGateway:
snap.MeshGateway.WatchedServices = make(map[structs.ServiceID]context.CancelFunc)
snap.MeshGateway.WatchedDatacenters = make(map[string]context.CancelFunc)
@ -914,6 +915,9 @@ func (s *state) handleUpdateTerminatingGateway(u cache.UpdateEvent, snap *Config
// Make sure to add every service to this map, we use it to cancel watches below.
svcMap[svc.Service] = struct{}{}
// Store the gateway <-> service mapping for TLS origination
snap.TerminatingGateway.GatewayServices[svc.Service] = *svc
// Watch the health endpoint to discover endpoints for the service
if _, ok := snap.TerminatingGateway.WatchedServices[svc.Service]; !ok {
ctx, cancel := context.WithCancel(s.ctx)
@ -1013,6 +1017,13 @@ func (s *state) handleUpdateTerminatingGateway(u cache.UpdateEvent, snap *Config
}
}
// Delete gateway service mapping for services that were not in the update
for sid, _ := range snap.TerminatingGateway.GatewayServices {
if _, ok := svcMap[sid]; !ok {
delete(snap.TerminatingGateway.GatewayServices, sid)
}
}
// Cancel service instance watches for services that were not in the update
for sid, cancelFn := range snap.TerminatingGateway.WatchedServices {
if _, ok := svcMap[sid]; !ok {

7
agent/proxycfg/state_test.go
View File

@ -921,6 +921,10 @@ func TestState_WatchesAndUpdates(t *testing.T) {
require.Len(t, snap.TerminatingGateway.WatchedResolvers, 2)
require.Contains(t, snap.TerminatingGateway.WatchedResolvers, db)
require.Contains(t, snap.TerminatingGateway.WatchedResolvers, billing)
require.Len(t, snap.TerminatingGateway.GatewayServices, 2)
require.Contains(t, snap.TerminatingGateway.GatewayServices, db)
require.Contains(t, snap.TerminatingGateway.GatewayServices, billing)
},
},
verificationStage{
@ -1048,6 +1052,9 @@ func TestState_WatchesAndUpdates(t *testing.T) {
require.Len(t, snap.TerminatingGateway.WatchedResolvers, 1)
require.Contains(t, snap.TerminatingGateway.WatchedResolvers, billing)
require.Len(t, snap.TerminatingGateway.GatewayServices, 1)
require.Contains(t, snap.TerminatingGateway.GatewayServices, billing)
// There was no update event for billing's leaf/endpoints, so length is 0
require.Len(t, snap.TerminatingGateway.ServiceGroups, 0)
require.Len(t, snap.TerminatingGateway.ServiceLeaves, 0)

12
agent/proxycfg/testing.go
View File

@ -1496,6 +1496,18 @@ func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *C
web: webNodes,
api: apiNodes,
},
GatewayServices: map[structs.ServiceID]structs.GatewayService{
web: {
Service: web,
CAFile: "ca.cert.pem",
},
api: {
Service: api,
CAFile: "ca.cert.pem",
CertFile: "api.cert.pem",
KeyFile: "api.key.pem",
},
},
}
snap.TerminatingGateway.ServiceLeaves = map[structs.ServiceID]*structs.IssuedCert{
structs.NewServiceID("web", nil): {

31
agent/structs/config_entry_gateways.go
View File

@ -200,6 +200,9 @@ type LinkedService struct {
// from the gateway to the linked service
KeyFile string `json:",omitempty"`
// SNI is the optional name to specify during the TLS handshake with a linked service
SNI string `json:",omitempty"`
EnterpriseMeta `hcl:",squash" mapstructure:",squash"`
}
@ -250,8 +253,9 @@ func (e *TerminatingGatewayConfigEntry) Validate() error {
}
seen[cid] = true
// If any TLS config flag was specified, all must be
if (svc.CAFile != "" || svc.CertFile != "" || svc.KeyFile != "") &&
// If either client cert config file was specified then the CA file, client cert, and key file must be specified
// Specifying only a CAFile is allowed for one-way TLS
if (svc.CertFile != "" || svc.KeyFile != "") &&
!(svc.CAFile != "" && svc.CertFile != "" && svc.KeyFile != "") {
return fmt.Errorf("Service %q must have a CertFile, CAFile, and KeyFile specified for TLS origination", svc.Name)
@ -299,6 +303,7 @@ type GatewayService struct {
CAFile string
CertFile string
KeyFile string
SNI string
FromWildcard bool
RaftIndex
}
@ -312,18 +317,22 @@ func (g *GatewayService) IsSame(o *GatewayService) bool {
g.Port == o.Port &&
g.CAFile == o.CAFile &&
g.CertFile == o.CertFile &&
g.KeyFile == o.KeyFile
g.KeyFile == o.KeyFile &&
g.SNI == o.SNI &&
g.FromWildcard == o.FromWildcard
}
func (g *GatewayService) Clone() *GatewayService {
return &GatewayService{
Gateway: g.Gateway,
Service: g.Service,
GatewayKind: g.GatewayKind,
Port: g.Port,
CAFile: g.CAFile,
CertFile: g.CertFile,
KeyFile: g.KeyFile,
RaftIndex: g.RaftIndex,
Gateway: g.Gateway,
Service: g.Service,
GatewayKind: g.GatewayKind,
Port: g.Port,
CAFile: g.CAFile,
CertFile: g.CertFile,
KeyFile: g.KeyFile,
SNI: g.SNI,
FromWildcard: g.FromWildcard,
RaftIndex: g.RaftIndex,
}
}

76
agent/structs/config_entry_gateways_test.go
View File

@ -315,8 +315,8 @@ func TestTerminatingConfigEntry_Validate(t *testing.T) {
Name: "terminating-gw-west",
Services: []LinkedService{
{
Name: "web",
CAFile: "ca.crt",
Name: "web",
CertFile: "client.crt",
},
},
},
@ -324,20 +324,6 @@ func TestTerminatingConfigEntry_Validate(t *testing.T) {
},
{
name: "not all TLS options provided-2",
entry: TerminatingGatewayConfigEntry{
Kind: "terminating-gateway",
Name: "terminating-gw-west",
Services: []LinkedService{
{
Name: "web",
CertFile: "client.crt",
},
},
},
expectErr: "must have a CertFile, CAFile, and KeyFile",
},
{
name: "not all TLS options provided-3",
entry: TerminatingGatewayConfigEntry{
Kind: "terminating-gateway",
Name: "terminating-gw-west",
@ -350,51 +336,6 @@ func TestTerminatingConfigEntry_Validate(t *testing.T) {
},
expectErr: "must have a CertFile, CAFile, and KeyFile",
},
{
name: "not all TLS options provided-4",
entry: TerminatingGatewayConfigEntry{
Kind: "terminating-gateway",
Name: "terminating-gw-west",
Services: []LinkedService{
{
Name: "web",
CAFile: "ca.crt",
KeyFile: "tls.key",
},
},
},
expectErr: "must have a CertFile, CAFile, and KeyFile",
},
{
name: "not all TLS options provided-5",
entry: TerminatingGatewayConfigEntry{
Kind: "terminating-gateway",
Name: "terminating-gw-west",
Services: []LinkedService{
{
Name: "web",
CAFile: "ca.crt",
CertFile: "client.crt",
},
},
},
expectErr: "must have a CertFile, CAFile, and KeyFile",
},
{
name: "not all TLS options provided-6",
entry: TerminatingGatewayConfigEntry{
Kind: "terminating-gateway",
Name: "terminating-gw-west",
Services: []LinkedService{
{
Name: "web",
KeyFile: "tls.key",
CertFile: "client.crt",
},
},
},
expectErr: "must have a CertFile, CAFile, and KeyFile",
},
{
name: "all TLS options provided",
entry: TerminatingGatewayConfigEntry{
@ -410,6 +351,19 @@ func TestTerminatingConfigEntry_Validate(t *testing.T) {
},
},
},
{
name: "only providing ca file is allowed",
entry: TerminatingGatewayConfigEntry{
Kind: "terminating-gateway",
Name: "terminating-gw-west",
Services: []LinkedService{
{
Name: "web",
CAFile: "ca.crt",
},
},
},
},
}
for _, test := range cases {

6
agent/structs/config_entry_test.go
View File

@ -658,12 +658,14 @@ func TestDecodeConfigEntry(t *testing.T) {
ca_file = "/etc/payments/ca.pem",
cert_file = "/etc/payments/cert.pem",
key_file = "/etc/payments/tls.key",
sni = "mydomain",
},
{
name = "*",
ca_file = "/etc/all/ca.pem",
cert_file = "/etc/all/cert.pem",
key_file = "/etc/all/tls.key",
sni = "my-alt-domain",
},
]
`,
@ -676,12 +678,14 @@ func TestDecodeConfigEntry(t *testing.T) {
CAFile = "/etc/payments/ca.pem",
CertFile = "/etc/payments/cert.pem",
KeyFile = "/etc/payments/tls.key",
SNI = "mydomain",
},
{
Name = "*",
CAFile = "/etc/all/ca.pem",
CertFile = "/etc/all/cert.pem",
KeyFile = "/etc/all/tls.key",
SNI = "my-alt-domain",
},
]
`,
@ -694,12 +698,14 @@ func TestDecodeConfigEntry(t *testing.T) {
CAFile: "/etc/payments/ca.pem",
CertFile: "/etc/payments/cert.pem",
KeyFile: "/etc/payments/tls.key",
SNI: "mydomain",
},
{
Name: "*",
CAFile: "/etc/all/ca.pem",
CertFile: "/etc/all/cert.pem",
KeyFile: "/etc/all/tls.key",
SNI: "my-alt-domain",
},
},
},

32
agent/structs/structs_test.go
View File

@ -2139,22 +2139,28 @@ func TestGatewayService_IsSame(t *testing.T) {
ca := "ca.pem"
cert := "client.pem"
key := "tls.key"
sni := "mydomain"
wildcard := false
g := &GatewayService{
Gateway: gateway,
Service: svc,
GatewayKind: kind,
CAFile: ca,
CertFile: cert,
KeyFile: key,
Gateway: gateway,
Service: svc,
GatewayKind: kind,
CAFile: ca,
CertFile: cert,
KeyFile: key,
SNI: sni,
FromWildcard: wildcard,
}
other := &GatewayService{
Gateway: gateway,
Service: svc,
GatewayKind: kind,
CAFile: ca,
CertFile: cert,
KeyFile: key,
Gateway: gateway,
Service: svc,
GatewayKind: kind,
CAFile: ca,
CertFile: cert,
KeyFile: key,
SNI: sni,
FromWildcard: wildcard,
}
check := func(twiddle, restore func()) {
t.Helper()
@ -2178,6 +2184,8 @@ func TestGatewayService_IsSame(t *testing.T) {
check(func() { other.CAFile = "/certs/cert.pem" }, func() { other.CAFile = ca })
check(func() { other.CertFile = "/certs/cert.pem" }, func() { other.CertFile = cert })
check(func() { other.KeyFile = "/certs/cert.pem" }, func() { other.KeyFile = key })
check(func() { other.SNI = "alt-domain" }, func() { other.SNI = sni })
check(func() { other.FromWildcard = true }, func() { other.FromWildcard = wildcard })
if !g.IsSame(other) {
t.Fatalf("should be equal, was %#v VS %#v", g, other)

99
agent/xds/clusters.go
View File

@ -31,7 +31,7 @@ func (s *Server) clustersFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot, _ string
case structs.ServiceKindConnectProxy:
return s.clustersFromSnapshotConnectProxy(cfgSnap)
case structs.ServiceKindTerminatingGateway:
return s.clustersFromSnapshotTerminatingGateway(cfgSnap)
return s.makeGatewayServiceClusters(cfgSnap)
case structs.ServiceKindMeshGateway:
return s.clustersFromSnapshotMeshGateway(cfgSnap)
case structs.ServiceKindIngressGateway:
@ -119,12 +119,6 @@ func makeExposeClusterName(destinationPort int) string {
return fmt.Sprintf("exposed_cluster_%d", destinationPort)
}
// clustersFromSnapshotTerminatingGateway returns the xDS API representation of the "clusters"
// for a terminating gateway. This will include 1 cluster per service and service subset.
func (s *Server) clustersFromSnapshotTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
return s.clustersFromServicesAndResolvers(cfgSnap, cfgSnap.TerminatingGateway.ServiceGroups, cfgSnap.TerminatingGateway.ServiceResolvers)
}
// clustersFromSnapshotMeshGateway returns the xDS API representation of the "clusters"
// for a mesh gateway. This will include 1 cluster per remote datacenter as well as
// 1 cluster for each service subset.
@ -141,7 +135,7 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho
}
clusterName := connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain)
cluster, err := s.makeGatewayCluster(clusterName, cfgSnap)
cluster, err := s.makeGatewayCluster(cfgSnap, clusterName)
if err != nil {
return nil, err
}
@ -153,7 +147,7 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho
for _, dc := range datacenters {
clusterName := cfgSnap.ServerSNIFn(dc, "")
cluster, err := s.makeGatewayCluster(clusterName, cfgSnap)
cluster, err := s.makeGatewayCluster(cfgSnap, clusterName)
if err != nil {
return nil, err
}
@ -164,7 +158,7 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho
for _, srv := range cfgSnap.MeshGateway.ConsulServers {
clusterName := cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node)
cluster, err := s.makeGatewayCluster(clusterName, cfgSnap)
cluster, err := s.makeGatewayCluster(cfgSnap, clusterName)
if err != nil {
return nil, err
}
@ -173,7 +167,7 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho
}
// generate the per-service/subset clusters
c, err := s.clustersFromServicesAndResolvers(cfgSnap, cfgSnap.MeshGateway.ServiceGroups, cfgSnap.MeshGateway.ServiceResolvers)
c, err := s.makeGatewayServiceClusters(cfgSnap)
if err != nil {
return nil, err
}
@ -182,10 +176,20 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho
return clusters, nil
}
func (s *Server) clustersFromServicesAndResolvers(
cfgSnap *proxycfg.ConfigSnapshot,
services map[structs.ServiceID]structs.CheckServiceNodes,
resolvers map[structs.ServiceID]*structs.ServiceResolverConfigEntry) ([]proto.Message, error) {
func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
var services map[structs.ServiceID]structs.CheckServiceNodes
var resolvers map[structs.ServiceID]*structs.ServiceResolverConfigEntry
switch cfgSnap.Kind {
case structs.ServiceKindTerminatingGateway:
services = cfgSnap.TerminatingGateway.ServiceGroups
resolvers = cfgSnap.TerminatingGateway.ServiceResolvers
case structs.ServiceKindMeshGateway:
services = cfgSnap.MeshGateway.ServiceGroups
resolvers = cfgSnap.MeshGateway.ServiceResolvers
default:
return nil, fmt.Errorf("unsupported gateway kind %q", cfgSnap.Kind)
}
clusters := make([]proto.Message, 0, len(services))
@ -196,28 +200,34 @@ func (s *Server) clustersFromServicesAndResolvers(
// Create the cluster for default/unnamed services
var cluster *envoy.Cluster
var err error
if hasResolver {
cluster, err = s.makeGatewayClusterWithConnectTimeout(clusterName, cfgSnap, resolver.ConnectTimeout)
} else {
cluster, err = s.makeGatewayCluster(clusterName, cfgSnap)
if !hasResolver {
// Use a zero value resolver with no timeout and no subsets
resolver = &structs.ServiceResolverConfigEntry{}
}
cluster, err = s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, resolver.ConnectTimeout)
if err != nil {
return nil, fmt.Errorf("failed to make %s cluster: %v", cfgSnap.Kind, err)
}
if cfgSnap.Kind == structs.ServiceKindTerminatingGateway {
injectTerminatingGatewayTLSContext(cfgSnap, cluster, svc)
}
clusters = append(clusters, cluster)
// if there is a service-resolver for this service then also setup subset clusters for it
if hasResolver {
// generate 1 cluster for each service subset
for subsetName := range resolver.Subsets {
clusterName := connect.ServiceSNI(svc.ID, subsetName, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
// If there is a service-resolver for this service then also setup a cluster for each subset
for subsetName := range resolver.Subsets {
clusterName := connect.ServiceSNI(svc.ID, subsetName, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
cluster, err := s.makeGatewayClusterWithConnectTimeout(clusterName, cfgSnap, resolver.ConnectTimeout)
if err != nil {
return nil, fmt.Errorf("failed to make %s cluster: %v", cfgSnap.Kind, err)
}
clusters = append(clusters, cluster)
cluster, err := s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, resolver.ConnectTimeout)
if err != nil {
return nil, fmt.Errorf("failed to make %s cluster: %v", cfgSnap.Kind, err)
}
if cfgSnap.Kind == structs.ServiceKindTerminatingGateway {
injectTerminatingGatewayTLSContext(cfgSnap, cluster, svc)
}
clusters = append(clusters, cluster)
}
}
@ -349,7 +359,7 @@ func (s *Server) makeUpstreamClusterForPreparedQuery(upstream structs.Upstream,
// Enable TLS upstream with the configured client certificate.
c.TlsContext = &envoyauth.UpstreamTlsContext{
CommonTlsContext: makeCommonTLSContext(cfgSnap, cfgSnap.Leaf()),
CommonTlsContext: makeCommonTLSContextFromLeaf(cfgSnap, cfgSnap.Leaf()),
Sni: sni,
}
@ -460,7 +470,7 @@ func (s *Server) makeUpstreamClustersForDiscoveryChain(
// Enable TLS upstream with the configured client certificate.
c.TlsContext = &envoyauth.UpstreamTlsContext{
CommonTlsContext: makeCommonTLSContext(cfgSnap, cfgSnap.Leaf()),
CommonTlsContext: makeCommonTLSContextFromLeaf(cfgSnap, cfgSnap.Leaf()),
Sni: sni,
}
@ -528,15 +538,16 @@ func makeClusterFromUserConfig(configJSON string) (*envoy.Cluster, error) {
return &c, err
}
func (s *Server) makeGatewayCluster(clusterName string, cfgSnap *proxycfg.ConfigSnapshot) (*envoy.Cluster, error) {
return s.makeGatewayClusterWithConnectTimeout(clusterName, cfgSnap, 0)
func (s *Server) makeGatewayCluster(cfgSnap *proxycfg.ConfigSnapshot, clusterName string) (*envoy.Cluster, error) {
return s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, 0)
}
// makeGatewayClusterWithConnectTimeout initializes a gateway cluster
// with the specified connect timeout. If the timeout is 0, the connect timeout
// defaults to use the configured gateway timeout.
func (s *Server) makeGatewayClusterWithConnectTimeout(clusterName string, cfgSnap *proxycfg.ConfigSnapshot,
connectTimeout time.Duration) (*envoy.Cluster, error) {
func (s *Server) makeGatewayClusterWithConnectTimeout(cfgSnap *proxycfg.ConfigSnapshot,
clusterName string, connectTimeout time.Duration) (*envoy.Cluster, error) {
cfg, err := ParseGatewayConfig(cfgSnap.Proxy.Config)
if err != nil {
// Don't hard fail on a config typo, just warn. The parse func returns
@ -548,7 +559,7 @@ func (s *Server) makeGatewayClusterWithConnectTimeout(clusterName string, cfgSna
connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond
}
return &envoy.Cluster{
cluster := envoy.Cluster{
Name: clusterName,
ConnectTimeout: connectTimeout,
ClusterDiscoveryType: &envoy.Cluster_Type{Type: envoy.Cluster_EDS},
@ -561,7 +572,21 @@ func (s *Server) makeGatewayClusterWithConnectTimeout(clusterName string, cfgSna
},
// Having an empty config enables outlier detection with default config.
OutlierDetection: &envoycluster.OutlierDetection{},
}, nil
}
return &cluster, nil
}
// injectTerminatingGatewayTLSContext adds an UpstreamTlsContext to a cluster for TLS origination
func injectTerminatingGatewayTLSContext(cfgSnap *proxycfg.ConfigSnapshot, cluster *envoy.Cluster, service structs.ServiceID) {
if mapping, ok := cfgSnap.TerminatingGateway.GatewayServices[service]; ok && mapping.CAFile != "" {
cluster.TlsContext = &envoyauth.UpstreamTlsContext{
CommonTlsContext: makeCommonTLSContextFromFiles(mapping.CAFile, mapping.CertFile, mapping.KeyFile),
// TODO (gateways) (freddy) If mapping.SNI is empty, does Envoy behave any differently if TlsContext.Sni is excluded?
Sni: mapping.SNI,
}
}
}
func makeThresholdsIfNeeded(limits UpstreamLimits) []*envoycluster.CircuitBreakers_Thresholds {

49
agent/xds/listeners.go
View File

@ -367,7 +367,7 @@ func makeListenerFromUserConfig(configJSON string) (*envoy.Listener, error) {
// specify custom listener params in config but still get our certs delivered
// dynamically and intentions enforced without coming up with some complicated
// templating/merging solution.
func injectConnectFilters(cfgSnap *proxycfg.ConfigSnapshot, token string, listener *envoy.Listener) error {
func injectConnectFilters(cfgSnap *proxycfg.ConfigSnapshot, token string, listener *envoy.Listener, setTLS bool) error {
authFilter, err := makeExtAuthFilter(token)
if err != nil {
return err
@ -378,7 +378,7 @@ func injectConnectFilters(cfgSnap *proxycfg.ConfigSnapshot, token string, listen
append([]envoylistener.Filter{authFilter}, listener.FilterChains[idx].Filters...)
listener.FilterChains[idx].TlsContext = &envoyauth.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContext(cfgSnap, cfgSnap.Leaf()),
CommonTlsContext: makeCommonTLSContextFromLeaf(cfgSnap, cfgSnap.Leaf()),
RequireClientCertificate: &types.BoolValue{Value: true},
}
}
@ -439,7 +439,7 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri
}
}
err = injectConnectFilters(cfgSnap, token, l)
err = injectConnectFilters(cfgSnap, token, l, true)
return l, err
}
@ -642,7 +642,7 @@ func (s *Server) sniFilterChainTerminatingGateway(listener, cluster, token strin
tcpProxy,
},
TlsContext: &envoyauth.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContext(cfgSnap, cfgSnap.TerminatingGateway.ServiceLeaves[service]),
CommonTlsContext: makeCommonTLSContextFromLeaf(cfgSnap, cfgSnap.TerminatingGateway.ServiceLeaves[service]),
RequireClientCertificate: &types.BoolValue{Value: true},
},
}, err
@ -1011,7 +1011,7 @@ func makeFilter(name string, cfg proto.Message) (envoylistener.Filter, error) {
}, nil
}
func makeCommonTLSContext(cfgSnap *proxycfg.ConfigSnapshot, leaf *structs.IssuedCert) *envoyauth.CommonTlsContext {
func makeCommonTLSContextFromLeaf(cfgSnap *proxycfg.ConfigSnapshot, leaf *structs.IssuedCert) *envoyauth.CommonTlsContext {
// Concatenate all the root PEMs into one.
// TODO(banks): verify this actually works with Envoy (docs are not clear).
rootPEMS := ""
@ -1050,3 +1050,42 @@ func makeCommonTLSContext(cfgSnap *proxycfg.ConfigSnapshot, leaf *structs.Issued
},
}
}
func makeCommonTLSContextFromFiles(caFile, certFile, keyFile string) *envoyauth.CommonTlsContext {
ctx := envoyauth.CommonTlsContext{
TlsParams: &envoyauth.TlsParameters{},
}
// Verify certificate of peer if caFile is specified
if caFile != "" {
ctx.ValidationContextType = &envoyauth.CommonTlsContext_ValidationContext{
ValidationContext: &envoyauth.CertificateValidationContext{
TrustedCa: &envoycore.DataSource{
Specifier: &envoycore.DataSource_Filename{
Filename: caFile,
},
},
},
}
}
// Present certificate for mTLS if cert and key files are specified
if certFile != "" && keyFile != "" {
ctx.TlsCertificates = []*envoyauth.TlsCertificate{
{
CertificateChain: &envoycore.DataSource{
Specifier: &envoycore.DataSource_Filename{
Filename: certFile,
},
},
PrivateKey: &envoycore.DataSource{
Specifier: &envoycore.DataSource_Filename{
Filename: keyFile,
},
},
},
}
}
return &ctx
}

58
agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden vendored
View File

@ -13,6 +13,28 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"filename": "api.cert.pem"
},
"privateKey": {
"filename": "api.key.pem"
}
}
],
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}
@ -29,6 +51,18 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}
@ -45,6 +79,18 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}
@ -61,6 +107,18 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}

58
agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden vendored
View File

@ -13,6 +13,28 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"filename": "api.cert.pem"
},
"privateKey": {
"filename": "api.key.pem"
}
}
],
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}
@ -29,6 +51,18 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}
@ -45,6 +79,18 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}
@ -61,6 +107,18 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}

34
agent/xds/testdata/clusters/terminating-gateway.golden vendored
View File

@ -13,6 +13,28 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"filename": "api.cert.pem"
},
"privateKey": {
"filename": "api.key.pem"
}
}
],
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}
@ -29,6 +51,18 @@
}
},
"connectTimeout": "5s",
"tlsContext": {
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
},
"outlierDetection": {
}

3
api/config_entry_gateways.go
View File

@ -129,6 +129,9 @@ type LinkedService struct {
// KeyFile is the optional path to a private key to use for TLS connections
// from the gateway to the linked service
KeyFile string `json:",omitempty"`
// SNI is the optional name to specify during the TLS handshake with a linked service
SNI string `json:",omitempty"`
}
func (g *TerminatingGatewayConfigEntry) GetKind() string {

2
api/config_entry_gateways_test.go
View File

@ -185,6 +185,7 @@ func TestAPI_ConfigEntries_TerminatingGateway(t *testing.T) {
CAFile: "/etc/web/ca.crt",
CertFile: "/etc/web/client.crt",
KeyFile: "/etc/web/tls.key",
SNI: "mydomain",
},
}
@ -212,6 +213,7 @@ func TestAPI_ConfigEntries_TerminatingGateway(t *testing.T) {
CAFile: "/etc/certs/ca.crt",
CertFile: "/etc/certs/client.crt",
KeyFile: "/etc/certs/tls.key",
SNI: "mydomain",
},
}
_, wm, err = configEntries.Set(terminating2, nil)

4
api/config_entry_test.go
View File

@ -686,7 +686,8 @@ func TestDecodeConfigEntry(t *testing.T) {
"Name": "web",
"CAFile": "/etc/ca.pem",
"CertFile": "/etc/cert.pem",
"KeyFile": "/etc/tls.key"
"KeyFile": "/etc/tls.key",
"SNI": "mydomain"
},
{
"Name": "api"
@ -707,6 +708,7 @@ func TestDecodeConfigEntry(t *testing.T) {
CAFile: "/etc/ca.pem",
CertFile: "/etc/cert.pem",
KeyFile: "/etc/tls.key",
SNI: "mydomain",
},
{
Name: "api",

10
command/config/write/config_write_test.go
View File

@ -258,6 +258,7 @@ func TestParseConfigEntry(t *testing.T) {
ca_file = "/etc/ca.crt"
cert_file = "/etc/client.crt"
key_file = "/etc/tls.key"
sni = "mydomain"
},
{
name = "*"
@ -276,6 +277,7 @@ func TestParseConfigEntry(t *testing.T) {
CAFile = "/etc/ca.crt"
CertFile = "/etc/client.crt"
KeyFile = "/etc/tls.key"
SNI = "mydomain"
},
{
Name = "*"
@ -294,7 +296,8 @@ func TestParseConfigEntry(t *testing.T) {
"namespace": "biz",
"ca_file": "/etc/ca.crt",
"cert_file": "/etc/client.crt",
"key_file": "/etc/tls.key"
"key_file": "/etc/tls.key",
"sni": "mydomain"
},
{
"name": "*",
@ -314,7 +317,8 @@ func TestParseConfigEntry(t *testing.T) {
"Namespace": "biz",
"CAFile": "/etc/ca.crt",
"CertFile": "/etc/client.crt",
"KeyFile": "/etc/tls.key"
"KeyFile": "/etc/tls.key",
"SNI": "mydomain"
},
{
"Name": "*",
@ -334,6 +338,7 @@ func TestParseConfigEntry(t *testing.T) {
CAFile: "/etc/ca.crt",
CertFile: "/etc/client.crt",
KeyFile: "/etc/tls.key",
SNI: "mydomain",
},
{
Name: "*",
@ -352,6 +357,7 @@ func TestParseConfigEntry(t *testing.T) {
CAFile: "/etc/ca.crt",
CertFile: "/etc/client.crt",
KeyFile: "/etc/tls.key",
SNI: "mydomain",
},
{
Name: "*",