luxolus/open-consul
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

agent: allow mesh gateways to initialize even if there are no connect services registered yet (#6576)

Browse source 
Fixes #6543

Also improved some of the proxycfg tests to cover snapshot validity
better.
This commit is contained in:
R.B. Boyer 2019-10-17 16:46:49 -05:00 committed by GitHub
parent 7e8c4c0804
commit b091647090
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 203 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
agent/proxycfg/snapshot.go
View file

@ -14,19 +14,46 @@ type configSnapshotConnectProxy struct {
WatchedUpstreamEndpoints map[string]map[string]structs.CheckServiceNodes
WatchedGateways map[string]map[string]context.CancelFunc
WatchedGatewayEndpoints map[string]map[string]structs.CheckServiceNodes
WatchedServiceChecks map[string][]structs.CheckType
WatchedServiceChecks map[string][]structs.CheckType // TODO: missing garbage collection
UpstreamEndpoints map[string]structs.CheckServiceNodes // DEPRECATED:see:WatchedUpstreamEndpoints
}
func (c *configSnapshotConnectProxy) IsEmpty() bool {
if c == nil {
return true
}
return c.Leaf == nil &&
len(c.DiscoveryChain) == 0 &&
len(c.WatchedUpstreams) == 0 &&
len(c.WatchedUpstreamEndpoints) == 0 &&
len(c.WatchedGateways) == 0 &&
len(c.WatchedGatewayEndpoints) == 0 &&
len(c.WatchedServiceChecks) == 0 &&
len(c.UpstreamEndpoints) == 0
}
type configSnapshotMeshGateway struct {
WatchedServices map[string]context.CancelFunc
WatchedServicesSet bool
WatchedDatacenters map[string]context.CancelFunc
ServiceGroups map[string]structs.CheckServiceNodes
ServiceResolvers map[string]*structs.ServiceResolverConfigEntry
GatewayGroups map[string]structs.CheckServiceNodes
}
func (c *configSnapshotMeshGateway) IsEmpty() bool {
if c == nil {
return true
}
return len(c.WatchedServices) == 0 &&
!c.WatchedServicesSet &&
len(c.WatchedDatacenters) == 0 &&
len(c.ServiceGroups) == 0 &&
len(c.ServiceResolvers) == 0 &&
len(c.GatewayGroups) == 0
}
// ConfigSnapshot captures all the resulting config needed for a proxy instance.
// It is meant to be point-in-time coherent and is used to deliver the current
// config state to observers who need it to be pushed in (e.g. XDS server).
@ -54,11 +81,9 @@ type ConfigSnapshot struct {
func (s *ConfigSnapshot) Valid() bool {
switch s.Kind {
case structs.ServiceKindConnectProxy:
// TODO(rb): sanity check discovery chain things here?
return s.Roots != nil && s.ConnectProxy.Leaf != nil
case structs.ServiceKindMeshGateway:
// TODO (mesh-gateway) - what happens if all the connect services go away
return s.Roots != nil && len(s.MeshGateway.ServiceGroups) > 0
return s.Roots != nil && (s.MeshGateway.WatchedServicesSet || len(s.MeshGateway.WatchedServices) > 0)
default:
return false
}

2
agent/proxycfg/state.go
View file

@ -726,6 +726,8 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
cancelFn()
}
}
snap.MeshGateway.WatchedServicesSet = true
case datacentersWatchID:
datacentersRaw, ok := u.Result.(*[]string)
if !ok {

88
agent/proxycfg/state_test.go
View file

@ -290,6 +290,16 @@ func genVerifyServiceWatch(expectedService, expectedFilter, expectedDatacenter s
func TestState_WatchesAndUpdates(t *testing.T) {
t.Parallel()
indexedRoots, issuedCert := TestCerts(t)
rootWatchEvent := func() cache.UpdateEvent {
return cache.UpdateEvent{
CorrelationID: rootsWatchID,
Result: indexedRoots,
Err: nil,
}
}
type verificationStage struct {
requiredWatches map[string]verifyWatchRequest
events []cache.UpdateEvent
@ -417,6 +427,12 @@ func TestState_WatchesAndUpdates(t *testing.T) {
}),
},
events: []cache.UpdateEvent{
rootWatchEvent(),
cache.UpdateEvent{
CorrelationID: leafWatchID,
Result: issuedCert,
Err: nil,
},
cache.UpdateEvent{
CorrelationID: "discovery-chain:api",
Result: &structs.DiscoveryChainResponse{
@ -477,6 +493,21 @@ func TestState_WatchesAndUpdates(t *testing.T) {
Err: nil,
},
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid())
require.True(t, snap.MeshGateway.IsEmpty())
require.Equal(t, indexedRoots, snap.Roots)
require.Equal(t, issuedCert, snap.ConnectProxy.Leaf)
require.Len(t, snap.ConnectProxy.DiscoveryChain, 5, "%+v", snap.ConnectProxy.DiscoveryChain)
require.Len(t, snap.ConnectProxy.WatchedUpstreams, 5, "%+v", snap.ConnectProxy.WatchedUpstreams)
require.Len(t, snap.ConnectProxy.WatchedUpstreamEndpoints, 5, "%+v", snap.ConnectProxy.WatchedUpstreamEndpoints)
require.Len(t, snap.ConnectProxy.WatchedGateways, 5, "%+v", snap.ConnectProxy.WatchedGateways)
require.Len(t, snap.ConnectProxy.WatchedGatewayEndpoints, 5, "%+v", snap.ConnectProxy.WatchedGatewayEndpoints)
require.Len(t, snap.ConnectProxy.WatchedServiceChecks, 0, "%+v", snap.ConnectProxy.WatchedServiceChecks)
require.Len(t, snap.ConnectProxy.UpstreamEndpoints, 0, "%+v", snap.ConnectProxy.UpstreamEndpoints)
},
}
stage1 := verificationStage{
@ -488,6 +519,21 @@ func TestState_WatchesAndUpdates(t *testing.T) {
"mesh-gateway:dc2:api-failover-remote?dc=dc2": genVerifyGatewayWatch("dc2"),
"mesh-gateway:dc1:api-failover-local?dc=dc2": genVerifyGatewayWatch("dc1"),
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid())
require.True(t, snap.MeshGateway.IsEmpty())
require.Equal(t, indexedRoots, snap.Roots)
require.Equal(t, issuedCert, snap.ConnectProxy.Leaf)
require.Len(t, snap.ConnectProxy.DiscoveryChain, 5, "%+v", snap.ConnectProxy.DiscoveryChain)
require.Len(t, snap.ConnectProxy.WatchedUpstreams, 5, "%+v", snap.ConnectProxy.WatchedUpstreams)
require.Len(t, snap.ConnectProxy.WatchedUpstreamEndpoints, 5, "%+v", snap.ConnectProxy.WatchedUpstreamEndpoints)
require.Len(t, snap.ConnectProxy.WatchedGateways, 5, "%+v", snap.ConnectProxy.WatchedGateways)
require.Len(t, snap.ConnectProxy.WatchedGatewayEndpoints, 5, "%+v", snap.ConnectProxy.WatchedGatewayEndpoints)
require.Len(t, snap.ConnectProxy.WatchedServiceChecks, 0, "%+v", snap.ConnectProxy.WatchedServiceChecks)
require.Len(t, snap.ConnectProxy.UpstreamEndpoints, 0, "%+v", snap.ConnectProxy.UpstreamEndpoints)
},
}
if meshGatewayProxyConfigValue == structs.MeshGatewayModeLocal {
@ -518,6 +564,48 @@ func TestState_WatchesAndUpdates(t *testing.T) {
serviceListWatchID: genVerifyListServicesWatch("dc1"),
datacentersWatchID: verifyDatacentersWatch,
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.False(t, snap.Valid(), "gateway without root is not valid")
require.True(t, snap.ConnectProxy.IsEmpty())
},
},
verificationStage{
events: []cache.UpdateEvent{
rootWatchEvent(),
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.False(t, snap.Valid(), "gateway without services is valid")
require.True(t, snap.ConnectProxy.IsEmpty())
require.Equal(t, indexedRoots, snap.Roots)
require.Empty(t, snap.MeshGateway.WatchedServices)
require.False(t, snap.MeshGateway.WatchedServicesSet)
require.Empty(t, snap.MeshGateway.WatchedDatacenters)
require.Empty(t, snap.MeshGateway.ServiceGroups)
require.Empty(t, snap.MeshGateway.ServiceResolvers)
require.Empty(t, snap.MeshGateway.GatewayGroups)
},
},
verificationStage{
events: []cache.UpdateEvent{
cache.UpdateEvent{
CorrelationID: serviceListWatchID,
Result: &structs.IndexedServices{
Services: make(structs.Services),
},
Err: nil,
},
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid(), "gateway with empty service list is vaild")
require.True(t, snap.ConnectProxy.IsEmpty())
require.Equal(t, indexedRoots, snap.Roots)
require.Empty(t, snap.MeshGateway.WatchedServices)
require.True(t, snap.MeshGateway.WatchedServicesSet)
require.Empty(t, snap.MeshGateway.WatchedDatacenters)
require.Empty(t, snap.MeshGateway.ServiceGroups)
require.Empty(t, snap.MeshGateway.ServiceResolvers)
require.Empty(t, snap.MeshGateway.GatewayGroups)
},
},
},
},

21
agent/proxycfg/testing.go
View file

@ -967,8 +967,16 @@ func testConfigSnapshotDiscoveryChain(t testing.T, variation string, additionalE
}
func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotMeshGateway(t, true)
}
func TestConfigSnapshotMeshGatewayNoServices(t testing.T) *ConfigSnapshot {
return testConfigSnapshotMeshGateway(t, false)
}
func testConfigSnapshotMeshGateway(t testing.T, populateServices bool) *ConfigSnapshot {
roots, _ := TestCerts(t)
return &ConfigSnapshot{
snap := &ConfigSnapshot{
Kind: structs.ServiceKindMeshGateway,
Service: "mesh-gateway",
ProxyID: "mesh-gateway",
@ -990,10 +998,17 @@ func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot {
Roots: roots,
Datacenter: "dc1",
MeshGateway: configSnapshotMeshGateway{
WatchedServicesSet: true,
},
}
if populateServices {
snap.MeshGateway = configSnapshotMeshGateway{
WatchedServices: map[string]context.CancelFunc{
"foo": nil,
"bar": nil,
},
WatchedServicesSet: true,
WatchedDatacenters: map[string]context.CancelFunc{
"dc2": nil,
},
@ -1004,8 +1019,10 @@ func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot {
GatewayGroups: map[string]structs.CheckServiceNodes{
"dc2": TestGatewayNodesDC2(t),
},
},
}
}
return snap
}
func TestConfigSnapshotExposeConfig(t testing.T) *ConfigSnapshot {

5
agent/xds/clusters_test.go
View file

@ -197,6 +197,11 @@ func TestClustersFromSnapshot(t *testing.T) {
create: proxycfg.TestConfigSnapshotMeshGateway,
setup: nil,
},
{
name: "mesh-gateway-no-services",
create: proxycfg.TestConfigSnapshotMeshGatewayNoServices,
setup: nil,
},
{
name: "mesh-gateway-service-subsets",
create: proxycfg.TestConfigSnapshotMeshGateway,

4
agent/xds/endpoints_test.go
View file

@ -238,6 +238,10 @@ func Test_endpointsFromSnapshot(t *testing.T) {
create: proxycfg.TestConfigSnapshotMeshGateway,
setup: nil,
},
{
name: "mesh-gateway-no-services",
create: proxycfg.TestConfigSnapshotMeshGatewayNoServices,
},
{
name: "connect-proxy-with-chain",
create: proxycfg.TestConfigSnapshotDiscoveryChain,

4
agent/xds/listeners_test.go
View file

@ -224,6 +224,10 @@ func TestListenersFromSnapshot(t *testing.T) {
name: "mesh-gateway",
create: proxycfg.TestConfigSnapshotMeshGateway,
},
{
name: "mesh-gateway-no-services",
create: proxycfg.TestConfigSnapshotMeshGatewayNoServices,
},
{
name: "mesh-gateway-tagged-addresses",
create: proxycfg.TestConfigSnapshotMeshGateway,

7
agent/xds/testdata/clusters/mesh-gateway-no-services.golden vendored Normal file
View file

@ -0,0 +1,7 @@
{
"versionInfo": "00000001",
"resources": [
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Cluster",
"nonce": "00000001"
}

7
agent/xds/testdata/endpoints/mesh-gateway-no-services.golden vendored Normal file
View file

@ -0,0 +1,7 @@
{
"versionInfo": "00000001",
"resources": [
],
"typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
"nonce": "00000001"
}

38
agent/xds/testdata/listeners/mesh-gateway-no-services.golden vendored Normal file
View file

@ -0,0 +1,38 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "default:1.2.3.4:8443",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 8443
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.sni_cluster"
},
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "",
"stat_prefix": "mesh_gateway_local_default_tcp"
}
}
]
}
],
"listenerFilters": [
{
"name": "envoy.listener.tls_inspector"
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Listener",
"nonce": "00000001"
}