agent: allow mesh gateways to initialize even if there are no connect services registered yet (#6576)

Fixes #6543

Also improved some of the proxycfg tests to cover snapshot validity
better.
This commit is contained in:
R.B. Boyer 2019-10-17 16:46:49 -05:00 committed by GitHub
parent 7e8c4c0804
commit b091647090
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 203 additions and 6 deletions

View File

@ -14,19 +14,46 @@ type configSnapshotConnectProxy struct {
WatchedUpstreamEndpoints map[string]map[string]structs.CheckServiceNodes WatchedUpstreamEndpoints map[string]map[string]structs.CheckServiceNodes
WatchedGateways map[string]map[string]context.CancelFunc WatchedGateways map[string]map[string]context.CancelFunc
WatchedGatewayEndpoints map[string]map[string]structs.CheckServiceNodes WatchedGatewayEndpoints map[string]map[string]structs.CheckServiceNodes
WatchedServiceChecks map[string][]structs.CheckType WatchedServiceChecks map[string][]structs.CheckType // TODO: missing garbage collection
UpstreamEndpoints map[string]structs.CheckServiceNodes // DEPRECATED:see:WatchedUpstreamEndpoints UpstreamEndpoints map[string]structs.CheckServiceNodes // DEPRECATED:see:WatchedUpstreamEndpoints
} }
func (c *configSnapshotConnectProxy) IsEmpty() bool {
if c == nil {
return true
}
return c.Leaf == nil &&
len(c.DiscoveryChain) == 0 &&
len(c.WatchedUpstreams) == 0 &&
len(c.WatchedUpstreamEndpoints) == 0 &&
len(c.WatchedGateways) == 0 &&
len(c.WatchedGatewayEndpoints) == 0 &&
len(c.WatchedServiceChecks) == 0 &&
len(c.UpstreamEndpoints) == 0
}
type configSnapshotMeshGateway struct { type configSnapshotMeshGateway struct {
WatchedServices map[string]context.CancelFunc WatchedServices map[string]context.CancelFunc
WatchedServicesSet bool
WatchedDatacenters map[string]context.CancelFunc WatchedDatacenters map[string]context.CancelFunc
ServiceGroups map[string]structs.CheckServiceNodes ServiceGroups map[string]structs.CheckServiceNodes
ServiceResolvers map[string]*structs.ServiceResolverConfigEntry ServiceResolvers map[string]*structs.ServiceResolverConfigEntry
GatewayGroups map[string]structs.CheckServiceNodes GatewayGroups map[string]structs.CheckServiceNodes
} }
func (c *configSnapshotMeshGateway) IsEmpty() bool {
if c == nil {
return true
}
return len(c.WatchedServices) == 0 &&
!c.WatchedServicesSet &&
len(c.WatchedDatacenters) == 0 &&
len(c.ServiceGroups) == 0 &&
len(c.ServiceResolvers) == 0 &&
len(c.GatewayGroups) == 0
}
// ConfigSnapshot captures all the resulting config needed for a proxy instance. // ConfigSnapshot captures all the resulting config needed for a proxy instance.
// It is meant to be point-in-time coherent and is used to deliver the current // It is meant to be point-in-time coherent and is used to deliver the current
// config state to observers who need it to be pushed in (e.g. XDS server). // config state to observers who need it to be pushed in (e.g. XDS server).
@ -54,11 +81,9 @@ type ConfigSnapshot struct {
func (s *ConfigSnapshot) Valid() bool { func (s *ConfigSnapshot) Valid() bool {
switch s.Kind { switch s.Kind {
case structs.ServiceKindConnectProxy: case structs.ServiceKindConnectProxy:
// TODO(rb): sanity check discovery chain things here?
return s.Roots != nil && s.ConnectProxy.Leaf != nil return s.Roots != nil && s.ConnectProxy.Leaf != nil
case structs.ServiceKindMeshGateway: case structs.ServiceKindMeshGateway:
// TODO (mesh-gateway) - what happens if all the connect services go away return s.Roots != nil && (s.MeshGateway.WatchedServicesSet || len(s.MeshGateway.WatchedServices) > 0)
return s.Roots != nil && len(s.MeshGateway.ServiceGroups) > 0
default: default:
return false return false
} }

View File

@ -726,6 +726,8 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
cancelFn() cancelFn()
} }
} }
snap.MeshGateway.WatchedServicesSet = true
case datacentersWatchID: case datacentersWatchID:
datacentersRaw, ok := u.Result.(*[]string) datacentersRaw, ok := u.Result.(*[]string)
if !ok { if !ok {

View File

@ -290,6 +290,16 @@ func genVerifyServiceWatch(expectedService, expectedFilter, expectedDatacenter s
func TestState_WatchesAndUpdates(t *testing.T) { func TestState_WatchesAndUpdates(t *testing.T) {
t.Parallel() t.Parallel()
indexedRoots, issuedCert := TestCerts(t)
rootWatchEvent := func() cache.UpdateEvent {
return cache.UpdateEvent{
CorrelationID: rootsWatchID,
Result: indexedRoots,
Err: nil,
}
}
type verificationStage struct { type verificationStage struct {
requiredWatches map[string]verifyWatchRequest requiredWatches map[string]verifyWatchRequest
events []cache.UpdateEvent events []cache.UpdateEvent
@ -417,6 +427,12 @@ func TestState_WatchesAndUpdates(t *testing.T) {
}), }),
}, },
events: []cache.UpdateEvent{ events: []cache.UpdateEvent{
rootWatchEvent(),
cache.UpdateEvent{
CorrelationID: leafWatchID,
Result: issuedCert,
Err: nil,
},
cache.UpdateEvent{ cache.UpdateEvent{
CorrelationID: "discovery-chain:api", CorrelationID: "discovery-chain:api",
Result: &structs.DiscoveryChainResponse{ Result: &structs.DiscoveryChainResponse{
@ -477,6 +493,21 @@ func TestState_WatchesAndUpdates(t *testing.T) {
Err: nil, Err: nil,
}, },
}, },
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid())
require.True(t, snap.MeshGateway.IsEmpty())
require.Equal(t, indexedRoots, snap.Roots)
require.Equal(t, issuedCert, snap.ConnectProxy.Leaf)
require.Len(t, snap.ConnectProxy.DiscoveryChain, 5, "%+v", snap.ConnectProxy.DiscoveryChain)
require.Len(t, snap.ConnectProxy.WatchedUpstreams, 5, "%+v", snap.ConnectProxy.WatchedUpstreams)
require.Len(t, snap.ConnectProxy.WatchedUpstreamEndpoints, 5, "%+v", snap.ConnectProxy.WatchedUpstreamEndpoints)
require.Len(t, snap.ConnectProxy.WatchedGateways, 5, "%+v", snap.ConnectProxy.WatchedGateways)
require.Len(t, snap.ConnectProxy.WatchedGatewayEndpoints, 5, "%+v", snap.ConnectProxy.WatchedGatewayEndpoints)
require.Len(t, snap.ConnectProxy.WatchedServiceChecks, 0, "%+v", snap.ConnectProxy.WatchedServiceChecks)
require.Len(t, snap.ConnectProxy.UpstreamEndpoints, 0, "%+v", snap.ConnectProxy.UpstreamEndpoints)
},
} }
stage1 := verificationStage{ stage1 := verificationStage{
@ -488,6 +519,21 @@ func TestState_WatchesAndUpdates(t *testing.T) {
"mesh-gateway:dc2:api-failover-remote?dc=dc2": genVerifyGatewayWatch("dc2"), "mesh-gateway:dc2:api-failover-remote?dc=dc2": genVerifyGatewayWatch("dc2"),
"mesh-gateway:dc1:api-failover-local?dc=dc2": genVerifyGatewayWatch("dc1"), "mesh-gateway:dc1:api-failover-local?dc=dc2": genVerifyGatewayWatch("dc1"),
}, },
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid())
require.True(t, snap.MeshGateway.IsEmpty())
require.Equal(t, indexedRoots, snap.Roots)
require.Equal(t, issuedCert, snap.ConnectProxy.Leaf)
require.Len(t, snap.ConnectProxy.DiscoveryChain, 5, "%+v", snap.ConnectProxy.DiscoveryChain)
require.Len(t, snap.ConnectProxy.WatchedUpstreams, 5, "%+v", snap.ConnectProxy.WatchedUpstreams)
require.Len(t, snap.ConnectProxy.WatchedUpstreamEndpoints, 5, "%+v", snap.ConnectProxy.WatchedUpstreamEndpoints)
require.Len(t, snap.ConnectProxy.WatchedGateways, 5, "%+v", snap.ConnectProxy.WatchedGateways)
require.Len(t, snap.ConnectProxy.WatchedGatewayEndpoints, 5, "%+v", snap.ConnectProxy.WatchedGatewayEndpoints)
require.Len(t, snap.ConnectProxy.WatchedServiceChecks, 0, "%+v", snap.ConnectProxy.WatchedServiceChecks)
require.Len(t, snap.ConnectProxy.UpstreamEndpoints, 0, "%+v", snap.ConnectProxy.UpstreamEndpoints)
},
} }
if meshGatewayProxyConfigValue == structs.MeshGatewayModeLocal { if meshGatewayProxyConfigValue == structs.MeshGatewayModeLocal {
@ -518,6 +564,48 @@ func TestState_WatchesAndUpdates(t *testing.T) {
serviceListWatchID: genVerifyListServicesWatch("dc1"), serviceListWatchID: genVerifyListServicesWatch("dc1"),
datacentersWatchID: verifyDatacentersWatch, datacentersWatchID: verifyDatacentersWatch,
}, },
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.False(t, snap.Valid(), "gateway without root is not valid")
require.True(t, snap.ConnectProxy.IsEmpty())
},
},
verificationStage{
events: []cache.UpdateEvent{
rootWatchEvent(),
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.False(t, snap.Valid(), "gateway without services is valid")
require.True(t, snap.ConnectProxy.IsEmpty())
require.Equal(t, indexedRoots, snap.Roots)
require.Empty(t, snap.MeshGateway.WatchedServices)
require.False(t, snap.MeshGateway.WatchedServicesSet)
require.Empty(t, snap.MeshGateway.WatchedDatacenters)
require.Empty(t, snap.MeshGateway.ServiceGroups)
require.Empty(t, snap.MeshGateway.ServiceResolvers)
require.Empty(t, snap.MeshGateway.GatewayGroups)
},
},
verificationStage{
events: []cache.UpdateEvent{
cache.UpdateEvent{
CorrelationID: serviceListWatchID,
Result: &structs.IndexedServices{
Services: make(structs.Services),
},
Err: nil,
},
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid(), "gateway with empty service list is vaild")
require.True(t, snap.ConnectProxy.IsEmpty())
require.Equal(t, indexedRoots, snap.Roots)
require.Empty(t, snap.MeshGateway.WatchedServices)
require.True(t, snap.MeshGateway.WatchedServicesSet)
require.Empty(t, snap.MeshGateway.WatchedDatacenters)
require.Empty(t, snap.MeshGateway.ServiceGroups)
require.Empty(t, snap.MeshGateway.ServiceResolvers)
require.Empty(t, snap.MeshGateway.GatewayGroups)
},
}, },
}, },
}, },

View File

@ -967,8 +967,16 @@ func testConfigSnapshotDiscoveryChain(t testing.T, variation string, additionalE
} }
func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot { func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotMeshGateway(t, true)
}
func TestConfigSnapshotMeshGatewayNoServices(t testing.T) *ConfigSnapshot {
return testConfigSnapshotMeshGateway(t, false)
}
func testConfigSnapshotMeshGateway(t testing.T, populateServices bool) *ConfigSnapshot {
roots, _ := TestCerts(t) roots, _ := TestCerts(t)
return &ConfigSnapshot{ snap := &ConfigSnapshot{
Kind: structs.ServiceKindMeshGateway, Kind: structs.ServiceKindMeshGateway,
Service: "mesh-gateway", Service: "mesh-gateway",
ProxyID: "mesh-gateway", ProxyID: "mesh-gateway",
@ -990,10 +998,17 @@ func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot {
Roots: roots, Roots: roots,
Datacenter: "dc1", Datacenter: "dc1",
MeshGateway: configSnapshotMeshGateway{ MeshGateway: configSnapshotMeshGateway{
WatchedServicesSet: true,
},
}
if populateServices {
snap.MeshGateway = configSnapshotMeshGateway{
WatchedServices: map[string]context.CancelFunc{ WatchedServices: map[string]context.CancelFunc{
"foo": nil, "foo": nil,
"bar": nil, "bar": nil,
}, },
WatchedServicesSet: true,
WatchedDatacenters: map[string]context.CancelFunc{ WatchedDatacenters: map[string]context.CancelFunc{
"dc2": nil, "dc2": nil,
}, },
@ -1004,8 +1019,10 @@ func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot {
GatewayGroups: map[string]structs.CheckServiceNodes{ GatewayGroups: map[string]structs.CheckServiceNodes{
"dc2": TestGatewayNodesDC2(t), "dc2": TestGatewayNodesDC2(t),
}, },
}, }
} }
return snap
} }
func TestConfigSnapshotExposeConfig(t testing.T) *ConfigSnapshot { func TestConfigSnapshotExposeConfig(t testing.T) *ConfigSnapshot {

View File

@ -197,6 +197,11 @@ func TestClustersFromSnapshot(t *testing.T) {
create: proxycfg.TestConfigSnapshotMeshGateway, create: proxycfg.TestConfigSnapshotMeshGateway,
setup: nil, setup: nil,
}, },
{
name: "mesh-gateway-no-services",
create: proxycfg.TestConfigSnapshotMeshGatewayNoServices,
setup: nil,
},
{ {
name: "mesh-gateway-service-subsets", name: "mesh-gateway-service-subsets",
create: proxycfg.TestConfigSnapshotMeshGateway, create: proxycfg.TestConfigSnapshotMeshGateway,

View File

@ -238,6 +238,10 @@ func Test_endpointsFromSnapshot(t *testing.T) {
create: proxycfg.TestConfigSnapshotMeshGateway, create: proxycfg.TestConfigSnapshotMeshGateway,
setup: nil, setup: nil,
}, },
{
name: "mesh-gateway-no-services",
create: proxycfg.TestConfigSnapshotMeshGatewayNoServices,
},
{ {
name: "connect-proxy-with-chain", name: "connect-proxy-with-chain",
create: proxycfg.TestConfigSnapshotDiscoveryChain, create: proxycfg.TestConfigSnapshotDiscoveryChain,

View File

@ -224,6 +224,10 @@ func TestListenersFromSnapshot(t *testing.T) {
name: "mesh-gateway", name: "mesh-gateway",
create: proxycfg.TestConfigSnapshotMeshGateway, create: proxycfg.TestConfigSnapshotMeshGateway,
}, },
{
name: "mesh-gateway-no-services",
create: proxycfg.TestConfigSnapshotMeshGatewayNoServices,
},
{ {
name: "mesh-gateway-tagged-addresses", name: "mesh-gateway-tagged-addresses",
create: proxycfg.TestConfigSnapshotMeshGateway, create: proxycfg.TestConfigSnapshotMeshGateway,

View File

@ -0,0 +1,7 @@
{
"versionInfo": "00000001",
"resources": [
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Cluster",
"nonce": "00000001"
}

View File

@ -0,0 +1,7 @@
{
"versionInfo": "00000001",
"resources": [
],
"typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
"nonce": "00000001"
}

View File

@ -0,0 +1,38 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "default:1.2.3.4:8443",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 8443
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.sni_cluster"
},
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "",
"stat_prefix": "mesh_gateway_local_default_tcp"
}
}
]
}
],
"listenerFilters": [
{
"name": "envoy.listener.tls_inspector"
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Listener",
"nonce": "00000001"
}