package proxycfg import ( "context" "fmt" "sort" "strings" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/proxycfg/internal/watch" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/proto/pbpeering" ) // TODO(ingress): Can we think of a better for this bag of data? // A shared data structure that contains information about discovered upstreams type ConfigSnapshotUpstreams struct { Leaf *structs.IssuedCert MeshConfig *structs.MeshConfigEntry MeshConfigSet bool // DiscoveryChain is a map of UpstreamID -> CompiledDiscoveryChain's, and // is used to determine what services could be targeted by this upstream. // We then instantiate watches for those targets. DiscoveryChain map[UpstreamID]*structs.CompiledDiscoveryChain // WatchedDiscoveryChains is a map of UpstreamID -> CancelFunc's // in order to cancel any watches when the proxy's configuration is // changed. Ingress gateways and transparent proxies need this because // discovery chain watches are added and removed through the lifecycle // of a single proxycfg state instance. WatchedDiscoveryChains map[UpstreamID]context.CancelFunc // WatchedUpstreams is a map of UpstreamID -> (map of TargetID -> // CancelFunc's) in order to cancel any watches when the configuration is // changed. WatchedUpstreams map[UpstreamID]map[string]context.CancelFunc // WatchedUpstreamEndpoints is a map of UpstreamID -> (map of // TargetID -> CheckServiceNodes) and is used to determine the backing // endpoints of an upstream. WatchedUpstreamEndpoints map[UpstreamID]map[string]structs.CheckServiceNodes // UpstreamPeerTrustBundles is a map of (PeerName -> PeeringTrustBundle). // It is used to store trust bundles for upstream TLS transport sockets. UpstreamPeerTrustBundles watch.Map[PeerName, *pbpeering.PeeringTrustBundle] // WatchedGateways is a map of UpstreamID -> (map of GatewayKey.String() -> // CancelFunc) in order to cancel watches for mesh gateways WatchedGateways map[UpstreamID]map[string]context.CancelFunc // WatchedGatewayEndpoints is a map of UpstreamID -> (map of // GatewayKey.String() -> CheckServiceNodes) and is used to determine the // backing endpoints of a mesh gateway. WatchedGatewayEndpoints map[UpstreamID]map[string]structs.CheckServiceNodes // WatchedLocalGWEndpoints is used to store the backing endpoints of // a local mesh gateway. Currently, this is used by peered upstreams // configured with local mesh gateway mode so that they can watch for // gateway endpoints. // // Note that the string form of GatewayKey is used as the key so empty // fields can be normalized in OSS. // GatewayKey.String() -> structs.CheckServiceNodes WatchedLocalGWEndpoints watch.Map[string, structs.CheckServiceNodes] // UpstreamConfig is a map to an upstream's configuration. UpstreamConfig map[UpstreamID]*structs.Upstream // PassthroughEndpoints is a map of: UpstreamID -> (map of TargetID -> // (set of IP addresses)). It contains the upstream endpoints that // can be dialed directly by a transparent proxy. PassthroughUpstreams map[UpstreamID]map[string]map[string]struct{} // PassthroughIndices is a map of: address -> indexedTarget. // It is used to track the modify index associated with a passthrough address. // Tracking this index helps break ties when a single address is shared by // more than one upstream due to a race. PassthroughIndices map[string]indexedTarget // IntentionUpstreams is a set of upstreams inferred from intentions. // // This list only applies to proxies registered in 'transparent' mode. IntentionUpstreams map[UpstreamID]struct{} // PeeredUpstreams is a set of all upstream targets in a local partition. // // This list only applies to proxies registered in 'transparent' mode. PeeredUpstreams map[UpstreamID]struct{} // PeerUpstreamEndpoints is a map of UpstreamID -> (set of IP addresses) // and used to determine the backing endpoints of an upstream in another // peer. PeerUpstreamEndpoints watch.Map[UpstreamID, structs.CheckServiceNodes] PeerUpstreamEndpointsUseHostnames map[UpstreamID]struct{} } // indexedTarget is used to associate the Raft modify index of a resource // with the corresponding upstream target. type indexedTarget struct { upstreamID UpstreamID targetID string idx uint64 } type GatewayKey struct { Datacenter string Partition string } func (k GatewayKey) String() string { resp := k.Datacenter if !acl.IsDefaultPartition(k.Partition) { resp = k.Partition + "." + resp } return resp } func (k GatewayKey) IsEmpty() bool { return k.Partition == "" && k.Datacenter == "" } func (k GatewayKey) Matches(dc, partition string) bool { return acl.EqualPartitions(k.Partition, partition) && k.Datacenter == dc } func gatewayKeyFromString(s string) GatewayKey { split := strings.SplitN(s, ".", 2) if len(split) == 1 { return GatewayKey{Datacenter: split[0], Partition: acl.DefaultPartitionName} } return GatewayKey{Partition: split[0], Datacenter: split[1]} } type configSnapshotConnectProxy struct { ConfigSnapshotUpstreams InboundPeerTrustBundlesSet bool InboundPeerTrustBundles []*pbpeering.PeeringTrustBundle WatchedServiceChecks map[structs.ServiceID][]structs.CheckType // TODO: missing garbage collection PreparedQueryEndpoints map[UpstreamID]structs.CheckServiceNodes // DEPRECATED:see:WatchedUpstreamEndpoints // NOTE: Intentions stores a list of lists as returned by the Intentions // Match RPC. So far we only use the first list as the list of matching // intentions. Intentions structs.Intentions IntentionsSet bool DestinationsUpstream watch.Map[UpstreamID, *structs.ServiceConfigEntry] DestinationGateways watch.Map[UpstreamID, structs.CheckServiceNodes] } // isEmpty is a test helper func (c *configSnapshotConnectProxy) isEmpty() bool { if c == nil { return true } return c.Leaf == nil && !c.IntentionsSet && len(c.DiscoveryChain) == 0 && len(c.WatchedDiscoveryChains) == 0 && len(c.WatchedUpstreams) == 0 && len(c.WatchedUpstreamEndpoints) == 0 && c.UpstreamPeerTrustBundles.Len() == 0 && len(c.WatchedGateways) == 0 && len(c.WatchedGatewayEndpoints) == 0 && len(c.WatchedServiceChecks) == 0 && len(c.PreparedQueryEndpoints) == 0 && len(c.UpstreamConfig) == 0 && len(c.PassthroughUpstreams) == 0 && len(c.IntentionUpstreams) == 0 && c.DestinationGateways.Len() == 0 && c.DestinationsUpstream.Len() == 0 && len(c.PeeredUpstreams) == 0 && !c.InboundPeerTrustBundlesSet && !c.MeshConfigSet && c.PeerUpstreamEndpoints.Len() == 0 && len(c.PeerUpstreamEndpointsUseHostnames) == 0 } func (c *configSnapshotConnectProxy) IsImplicitUpstream(uid UpstreamID) bool { _, intentionImplicit := c.IntentionUpstreams[uid] _, peeringImplicit := c.PeeredUpstreams[uid] return intentionImplicit || peeringImplicit } func (c *configSnapshotConnectProxy) GetUpstream(uid UpstreamID, entMeta *acl.EnterpriseMeta) (*structs.Upstream, bool) { upstream, found := c.UpstreamConfig[uid] // We should fallback to the wildcard defaults generated from service-defaults + proxy-defaults // whenever we don't find the upstream config. if !found { wildcardUID := NewWildcardUID(entMeta) upstream = c.UpstreamConfig[wildcardUID] } explicit := upstream != nil && upstream.HasLocalPortOrSocket() implicit := c.IsImplicitUpstream(uid) return upstream, !implicit && !explicit } type configSnapshotTerminatingGateway struct { MeshConfig *structs.MeshConfigEntry MeshConfigSet bool // WatchedServices is a map of service name to a cancel function. This cancel // function is tied to the watch of linked service instances for the given // id. If the linked services watch would indicate the removal of // a service altogether we then cancel watching that service for its endpoints. WatchedServices map[structs.ServiceName]context.CancelFunc // WatchedIntentions is a map of service name to a cancel function. // This cancel function is tied to the watch of intentions for linked services. // As with WatchedServices, intention watches will be cancelled when services // are no longer linked to the gateway. WatchedIntentions map[structs.ServiceName]context.CancelFunc // NOTE: Intentions stores a map of list of lists as returned by the Intentions // Match RPC. So far we only use the first list as the list of matching // intentions. // // A key being present implies that we have gotten at least one watch reply for the // service. This is logically the same as ConnectProxy.IntentionsSet==true Intentions map[structs.ServiceName]structs.Intentions // WatchedLeaves is a map of ServiceName to a cancel function. // This cancel function is tied to the watch of leaf certs for linked services. // As with WatchedServices, leaf watches will be cancelled when services // are no longer linked to the gateway. WatchedLeaves map[structs.ServiceName]context.CancelFunc // ServiceLeaves is a map of ServiceName to a leaf cert. // Terminating gateways will present different certificates depending // on the service that the caller is trying to reach. ServiceLeaves map[structs.ServiceName]*structs.IssuedCert // WatchedConfigs is a map of ServiceName to a cancel function. This cancel // function is tied to the watch of service configs for linked services. As // with WatchedServices, service config watches will be cancelled when // services are no longer linked to the gateway. WatchedConfigs map[structs.ServiceName]context.CancelFunc // ServiceConfigs is a map of service name to the resolved service config // for that service. ServiceConfigs map[structs.ServiceName]*structs.ServiceConfigResponse // WatchedResolvers is a map of ServiceName to a cancel function. // This cancel function is tied to the watch of resolvers for linked services. // As with WatchedServices, resolver watches will be cancelled when services // are no longer linked to the gateway. WatchedResolvers map[structs.ServiceName]context.CancelFunc // ServiceResolvers is a map of service name to an associated // service-resolver config entry for that service. ServiceResolvers map[structs.ServiceName]*structs.ServiceResolverConfigEntry ServiceResolversSet map[structs.ServiceName]bool // ServiceGroups is a map of service name to the service instances of that // service in the local datacenter. ServiceGroups map[structs.ServiceName]structs.CheckServiceNodes // GatewayServices is a map of service name to the config entry association // between the gateway and a service. TLS configuration stored here is // used for TLS origination from the gateway to the linked service. // This map does not include GatewayServices that represent Endpoints to external // destinations. GatewayServices map[structs.ServiceName]structs.GatewayService // DestinationServices is a map of service name to GatewayServices that represent // a destination to an external destination of the service mesh. DestinationServices map[structs.ServiceName]structs.GatewayService // HostnameServices is a map of service name to service instances with a hostname as the address. // If hostnames are configured they must be provided to Envoy via CDS not EDS. HostnameServices map[structs.ServiceName]structs.CheckServiceNodes } // ValidServices returns the list of service keys that have enough data to be emitted. func (c *configSnapshotTerminatingGateway) ValidServices() []structs.ServiceName { out := make([]structs.ServiceName, 0, len(c.ServiceGroups)) for svc := range c.ServiceGroups { // It only counts if ALL of our watches have come back (with data or not). // Skip the service if we don't know if there is a resolver or not. if _, ok := c.ServiceResolversSet[svc]; !ok { continue } // Skip the service if we don't have a cert to present for mTLS. if cert, ok := c.ServiceLeaves[svc]; !ok || cert == nil { continue } // Skip the service if we haven't gotten our intentions yet. if _, intentionsSet := c.Intentions[svc]; !intentionsSet { continue } // Skip the service if we haven't gotten our service config yet to know // the protocol. if _, ok := c.ServiceConfigs[svc]; !ok { continue } out = append(out, svc) } return out } // ValidDestinations returns the list of service keys (that represent exclusively endpoints) that have enough data to be emitted. func (c *configSnapshotTerminatingGateway) ValidDestinations() []structs.ServiceName { out := make([]structs.ServiceName, 0, len(c.DestinationServices)) for svc := range c.DestinationServices { // It only counts if ALL of our watches have come back (with data or not). // Skip the service if we don't have a cert to present for mTLS. if cert, ok := c.ServiceLeaves[svc]; !ok || cert == nil { continue } // Skip the service if we haven't gotten our intentions yet. if _, intentionsSet := c.Intentions[svc]; !intentionsSet { continue } // Skip the service if we haven't gotten our service config yet to know // the protocol. if conf, ok := c.ServiceConfigs[svc]; !ok || len(conf.Destination.Addresses) == 0 { continue } out = append(out, svc) } return out } // isEmpty is a test helper func (c *configSnapshotTerminatingGateway) isEmpty() bool { if c == nil { return true } return len(c.ServiceLeaves) == 0 && len(c.WatchedLeaves) == 0 && len(c.WatchedIntentions) == 0 && len(c.Intentions) == 0 && len(c.ServiceGroups) == 0 && len(c.WatchedServices) == 0 && len(c.ServiceResolvers) == 0 && len(c.ServiceResolversSet) == 0 && len(c.WatchedResolvers) == 0 && len(c.ServiceConfigs) == 0 && len(c.WatchedConfigs) == 0 && len(c.GatewayServices) == 0 && len(c.DestinationServices) == 0 && len(c.HostnameServices) == 0 && !c.MeshConfigSet } type PeerServersValue struct { Addresses []structs.ServiceAddress Index uint64 UseCDS bool } type PeeringServiceValue struct { Nodes structs.CheckServiceNodes UseCDS bool } type configSnapshotMeshGateway struct { // WatchedServices is a map of service name to a cancel function. This cancel // function is tied to the watch of connect enabled services for the given // id. If the main datacenter services watch would indicate the removal of // a service altogether we then cancel watching that service for its // connect endpoints. WatchedServices map[structs.ServiceName]context.CancelFunc // WatchedServicesSet indicates that the watch on the datacenters services // has completed. Even when there are no connect services, this being set // (and the Connect roots being available) will be enough for the config // snapshot to be considered valid. In the case of Envoy, this allows it to // start its listeners even when no services would be proxied and allow its // health check to pass. WatchedServicesSet bool // WatchedGateways is a map of GatewayKeys to a cancel function. // This cancel function is tied to the watch of mesh-gateway services in // that datacenter/partition. WatchedGateways map[string]context.CancelFunc // ServiceGroups is a map of service name to the service instances of that // service in the local datacenter. ServiceGroups map[structs.ServiceName]structs.CheckServiceNodes // PeeringServices is a map of peer name -> (map of // service name -> CheckServiceNodes) and is used to determine the backing // endpoints of a service on a peer. PeeringServices map[string]map[structs.ServiceName]PeeringServiceValue // WatchedPeeringServices is a map of peer name -> (map of service name -> // cancel function) and is used to track watches on services within a peer. WatchedPeeringServices map[string]map[structs.ServiceName]context.CancelFunc // WatchedPeers is a map of peer name -> cancel functions. It is used to // track watches on peers. WatchedPeers map[string]context.CancelFunc // ServiceResolvers is a map of service name to an associated // service-resolver config entry for that service. ServiceResolvers map[structs.ServiceName]*structs.ServiceResolverConfigEntry // GatewayGroups is a map of datacenter names to services of kind // mesh-gateway in that datacenter. GatewayGroups map[string]structs.CheckServiceNodes // FedStateGateways is a map of datacenter names to mesh gateways in that // datacenter. FedStateGateways map[string]structs.CheckServiceNodes // WatchedLocalServers is a map of (structs.ConsulServiceName -> structs.CheckServiceNodes)` // Mesh gateways can spin up watches for local servers both for // WAN federation and for peering. This map ensures we only have one // watch at a time. WatchedLocalServers watch.Map[string, structs.CheckServiceNodes] // HostnameDatacenters is a map of datacenters to mesh gateway instances with a hostname as the address. // If hostnames are configured they must be provided to Envoy via CDS not EDS. HostnameDatacenters map[string]structs.CheckServiceNodes // ExportedServicesSlice is a sorted slice of services that are exported to // connected peers. ExportedServicesSlice []structs.ServiceName // ExportedServicesWithPeers is a map of exported service name to a sorted // slice of peers that they are exported to. ExportedServicesWithPeers map[structs.ServiceName][]string // ExportedServicesSet indicates that the watch on the list of // peer-exported services has completed at least once. ExportedServicesSet bool // DiscoveryChain is a map of the peer-exported service names to their // local compiled discovery chain. This will be populated regardless of // L4/L7 status of the chain. DiscoveryChain map[structs.ServiceName]*structs.CompiledDiscoveryChain // WatchedDiscoveryChains is a map of peer-exported service names to a // cancel function. WatchedDiscoveryChains map[structs.ServiceName]context.CancelFunc // MeshConfig is the mesh config entry that should be used for services // fronted by this mesh gateway. MeshConfig *structs.MeshConfigEntry // MeshConfigSet indicates that the watch on the mesh config entry has // completed at least once. MeshConfigSet bool // Leaf is the leaf cert to be used by this mesh gateway. Leaf *structs.IssuedCert // LeafCertWatchCancel is a CancelFunc to use when refreshing this gateway's // leaf cert watch with different parameters. LeafCertWatchCancel context.CancelFunc // PeerServers is the map of peering server names to their addresses. PeerServers map[string]PeerServersValue // PeerServersWatchCancel is a CancelFunc to use when resetting the watch // on all peerings as it is enabled/disabled. PeerServersWatchCancel context.CancelFunc // PeeringTrustBundles is the list of trust bundles for peers where // services have been exported to using this mesh gateway. PeeringTrustBundles []*pbpeering.PeeringTrustBundle // PeeringTrustBundlesSet indicates that the watch on the peer trust // bundles has completed at least once. PeeringTrustBundlesSet bool } // MeshGatewayValidExportedServices ensures that the following data is present // if it exists for a service before it returns that in the set of services to // expose. // // - peering info // - discovery chain func (c *ConfigSnapshot) MeshGatewayValidExportedServices() []structs.ServiceName { out := make([]structs.ServiceName, 0, len(c.MeshGateway.ExportedServicesSlice)) for _, svc := range c.MeshGateway.ExportedServicesSlice { if _, ok := c.MeshGateway.ExportedServicesWithPeers[svc]; !ok { continue // not possible } if _, ok := c.MeshGateway.ServiceGroups[svc]; !ok { continue // unregistered services } chain, ok := c.MeshGateway.DiscoveryChain[svc] if !ok { continue // ignore; not ready } if structs.IsProtocolHTTPLike(chain.Protocol) { if c.MeshGateway.Leaf == nil { continue // ignore; not ready } } out = append(out, svc) } return out } func (c *ConfigSnapshot) GetMeshGatewayEndpoints(key GatewayKey) structs.CheckServiceNodes { // Mesh gateways in remote DCs are discovered in two ways: // // 1. Via an Internal.ServiceDump RPC in the remote DC (GatewayGroups). // 2. In the federation state that is replicated from the primary DC (FedStateGateways). // // We determine which set to use based on whichever contains the highest // raft ModifyIndex (and is therefore most up-to-date). // // Previously, GatewayGroups was always given presedence over FedStateGateways // but this was problematic when using mesh gateways for WAN federation. // // Consider the following example: // // - Primary and Secondary DCs are WAN Federated via local mesh gateways. // // - Secondary DC's mesh gateway is running on an ephemeral compute instance // and is abruptly terminated and rescheduled with a *new IP address*. // // - Primary DC's mesh gateway is no longer able to connect to the Secondary // DC as its proxy is configured with the old IP address. Therefore any RPC // from the Primary to the Secondary DC will fail (including the one to // discover the gateway's new IP address). // // - Secondary DC performs its regular anti-entropy of federation state data // to the Primary DC (this succeeds as there is still connectivity in this // direction). // // - At this point the Primary DC's mesh gateway should observe the new IP // address and reconfigure its proxy, however as we always prioritised // GatewayGroups this didn't happen and the connection remained severed. maxModifyIndex := func(vals structs.CheckServiceNodes) uint64 { var max uint64 for _, v := range vals { if i := v.Service.RaftIndex.ModifyIndex; i > max { max = i } } return max } endpoints := c.MeshGateway.GatewayGroups[key.String()] fedStateEndpoints := c.MeshGateway.FedStateGateways[key.String()] if maxModifyIndex(fedStateEndpoints) > maxModifyIndex(endpoints) { return fedStateEndpoints } return endpoints } func (c *configSnapshotMeshGateway) IsServiceExported(svc structs.ServiceName) bool { if c == nil || len(c.ExportedServicesWithPeers) == 0 { return false } _, ok := c.ExportedServicesWithPeers[svc] return ok } func (c *configSnapshotMeshGateway) GatewayKeys() []GatewayKey { sz1, sz2 := len(c.GatewayGroups), len(c.FedStateGateways) sz := sz1 if sz2 > sz1 { sz = sz2 } keys := make([]GatewayKey, 0, sz) for key := range c.FedStateGateways { keys = append(keys, gatewayKeyFromString(key)) } for key := range c.GatewayGroups { gk := gatewayKeyFromString(key) if _, ok := c.FedStateGateways[gk.Datacenter]; !ok { keys = append(keys, gk) } } // Always sort the results to ensure we generate deterministic things over // xDS, such as mesh-gateway listener filter chains. sort.Slice(keys, func(i, j int) bool { if keys[i].Datacenter != keys[j].Datacenter { return keys[i].Datacenter < keys[j].Datacenter } return keys[i].Partition < keys[j].Partition }) return keys } // isEmpty is a test helper func (c *configSnapshotMeshGateway) isEmpty() bool { if c == nil { return true } return len(c.WatchedServices) == 0 && !c.WatchedServicesSet && len(c.WatchedGateways) == 0 && len(c.ServiceGroups) == 0 && len(c.ServiceResolvers) == 0 && len(c.GatewayGroups) == 0 && len(c.FedStateGateways) == 0 && len(c.HostnameDatacenters) == 0 && c.WatchedLocalServers.Len() == 0 && c.isEmptyPeering() } // isEmptyPeering is a test helper func (c *configSnapshotMeshGateway) isEmptyPeering() bool { if c == nil { return true } return len(c.ExportedServicesSlice) == 0 && len(c.ExportedServicesWithPeers) == 0 && !c.ExportedServicesSet && len(c.DiscoveryChain) == 0 && len(c.WatchedDiscoveryChains) == 0 && !c.MeshConfigSet && c.LeafCertWatchCancel == nil && c.Leaf == nil && len(c.PeeringTrustBundles) == 0 && !c.PeeringTrustBundlesSet } type upstreamIDSet map[UpstreamID]struct{} func (u upstreamIDSet) add(uid UpstreamID) { u[uid] = struct{}{} } type routeUpstreamSet map[structs.ResourceReference]upstreamIDSet func (r routeUpstreamSet) hasUpstream(uid UpstreamID) bool { for _, set := range r { if _, ok := set[uid]; ok { return true } } return false } func (r routeUpstreamSet) set(route structs.ResourceReference, set upstreamIDSet) { r[route] = set } func (r routeUpstreamSet) delete(route structs.ResourceReference) { delete(r, route) } type listenerUpstreamMap map[APIGatewayListenerKey]structs.Upstreams type listenerRouteUpstreams map[structs.ResourceReference]listenerUpstreamMap func (l listenerRouteUpstreams) set(route structs.ResourceReference, listener APIGatewayListenerKey, upstreams structs.Upstreams) { if _, ok := l[route]; !ok { l[route] = make(listenerUpstreamMap) } l[route][listener] = upstreams } func (l listenerRouteUpstreams) delete(route structs.ResourceReference) { delete(l, route) } func (l listenerRouteUpstreams) toUpstreams() map[IngressListenerKey]structs.Upstreams { listeners := make(map[IngressListenerKey]structs.Upstreams, len(l)) for _, listenerMap := range l { for listener, set := range listenerMap { listeners[listener] = append(listeners[listener], set...) } } return listeners } type configSnapshotAPIGateway struct { ConfigSnapshotUpstreams TLSConfig structs.GatewayTLSConfig // GatewayConfigLoaded is used to determine if we have received the initial // api-gateway config entry yet. GatewayConfigLoaded bool GatewayConfig *structs.APIGatewayConfigEntry // BoundGatewayConfigLoaded is used to determine if we have received the initial // bound-api-gateway config entry yet. BoundGatewayConfigLoaded bool BoundGatewayConfig *structs.BoundAPIGatewayConfigEntry // Hosts is the list of extra host entries to add to our leaf cert's DNS SANs Hosts []string AreHostsSet bool // LeafCertWatchCancel is a CancelFunc to use when refreshing this gateway's // leaf cert watch with different parameters. //LeafCertWatchCancel context.CancelFunc // Upstreams is a list of upstreams this ingress gateway should serve traffic // to. This is constructed from the ingress-gateway config entry, and uses // the GatewayServices RPC to retrieve them. // TODO Determine if this is updated "for free" or not. If not, we might need // to do some work to populate it in handlerAPIGateway Upstreams listenerRouteUpstreams // UpstreamsSet is the unique set of UpstreamID the gateway routes to. UpstreamsSet routeUpstreamSet HTTPRoutes watch.Map[structs.ResourceReference, *structs.HTTPRouteConfigEntry] TCPRoutes watch.Map[structs.ResourceReference, *structs.TCPRouteConfigEntry] Certificates watch.Map[structs.ResourceReference, *structs.InlineCertificateConfigEntry] // LeafCertWatchCancel is a CancelFunc to use when refreshing this gateway's // leaf cert watch with different parameters. LeafCertWatchCancel context.CancelFunc // Listeners is the original listener config from the api-gateway config // entry to save us trying to pass fields through Upstreams Listeners map[string]structs.APIGatewayListener // this acts as an intermediary for inlining certificates ListenerCertificates map[IngressListenerKey][]structs.InlineCertificateConfigEntry BoundListeners map[string]structs.BoundAPIGatewayListener } // ToIngress converts a configSnapshotAPIGateway to a configSnapshotIngressGateway. // This is temporary, for the sake of re-using existing codepaths when integrating // Consul API Gateway into Consul core. // // FUTURE: Remove when API gateways have custom snapshot generation func (c *configSnapshotAPIGateway) ToIngress(datacenter string) (configSnapshotIngressGateway, error) { // Convert API Gateway Listeners to Ingress Listeners. ingressListeners := make(map[IngressListenerKey]structs.IngressListener, len(c.Listeners)) ingressUpstreams := make(map[IngressListenerKey]structs.Upstreams, len(c.Listeners)) synthesizedChains := map[UpstreamID]*structs.CompiledDiscoveryChain{} watchedUpstreamEndpoints := make(map[UpstreamID]map[string]structs.CheckServiceNodes) watchedGatewayEndpoints := make(map[UpstreamID]map[string]structs.CheckServiceNodes) // reset the cached certificates c.ListenerCertificates = make(map[IngressListenerKey][]structs.InlineCertificateConfigEntry) for name, listener := range c.Listeners { boundListener, ok := c.BoundListeners[name] if !ok { // Skip any listeners that don't have a bound listener. Once the bound listener is created, this will be run again. continue } if !c.GatewayConfig.ListenerIsReady(name) { // skip any listeners that might be in an invalid state continue } ingressListener := structs.IngressListener{ Port: listener.Port, Protocol: string(listener.Protocol), } // Create a synthesized discovery chain for each service. services, upstreams, compiled, err := c.synthesizeChains(datacenter, listener, boundListener) if err != nil { return configSnapshotIngressGateway{}, err } if len(upstreams) == 0 { // skip if we can't construct any upstreams continue } ingressListener.Services = services for i, service := range services { id := NewUpstreamIDFromServiceName(structs.NewServiceName(service.Name, &service.EnterpriseMeta)) upstreamEndpoints := make(map[string]structs.CheckServiceNodes) gatewayEndpoints := make(map[string]structs.CheckServiceNodes) // add the watched endpoints and gateway endpoints under the new upstream for _, endpoints := range c.WatchedUpstreamEndpoints { for targetID, endpoint := range endpoints { upstreamEndpoints[targetID] = endpoint } } for _, endpoints := range c.WatchedGatewayEndpoints { for targetID, endpoint := range endpoints { gatewayEndpoints[targetID] = endpoint } } synthesizedChains[id] = compiled[i] watchedUpstreamEndpoints[id] = upstreamEndpoints watchedGatewayEndpoints[id] = gatewayEndpoints } key := IngressListenerKey{ Port: listener.Port, Protocol: string(listener.Protocol), } // Configure TLS for the ingress listener tls, err := c.toIngressTLS(key, listener, boundListener) if err != nil { return configSnapshotIngressGateway{}, err } ingressListener.TLS = tls ingressListeners[key] = ingressListener ingressUpstreams[key] = upstreams } snapshotUpstreams := c.DeepCopy().ConfigSnapshotUpstreams snapshotUpstreams.DiscoveryChain = synthesizedChains snapshotUpstreams.WatchedUpstreamEndpoints = watchedUpstreamEndpoints snapshotUpstreams.WatchedGatewayEndpoints = watchedGatewayEndpoints return configSnapshotIngressGateway{ Upstreams: ingressUpstreams, ConfigSnapshotUpstreams: snapshotUpstreams, GatewayConfigLoaded: true, Listeners: ingressListeners, }, nil } func (c *configSnapshotAPIGateway) synthesizeChains(datacenter string, listener structs.APIGatewayListener, boundListener structs.BoundAPIGatewayListener) ([]structs.IngressService, structs.Upstreams, []*structs.CompiledDiscoveryChain, error) { chains := []*structs.CompiledDiscoveryChain{} trustDomain := "" DOMAIN_LOOP: for _, chain := range c.DiscoveryChain { for _, target := range chain.Targets { if !target.External { trustDomain = connect.TrustDomainForTarget(*target) if trustDomain != "" { break DOMAIN_LOOP } } } } synthesizer := discoverychain.NewGatewayChainSynthesizer(datacenter, trustDomain, listener.Name, c.GatewayConfig) synthesizer.SetHostname(listener.GetHostname()) for _, routeRef := range boundListener.Routes { switch routeRef.Kind { case structs.HTTPRoute: route, ok := c.HTTPRoutes.Get(routeRef) if !ok || listener.Protocol != structs.ListenerProtocolHTTP { continue } synthesizer.AddHTTPRoute(*route) for _, service := range route.GetServices() { id := NewUpstreamIDFromServiceName(structs.NewServiceName(service.Name, &service.EnterpriseMeta)) if chain := c.DiscoveryChain[id]; chain != nil { chains = append(chains, chain) } } case structs.TCPRoute: route, ok := c.TCPRoutes.Get(routeRef) if !ok || listener.Protocol != structs.ListenerProtocolTCP { continue } synthesizer.AddTCPRoute(*route) for _, service := range route.GetServices() { id := NewUpstreamIDFromServiceName(structs.NewServiceName(service.Name, &service.EnterpriseMeta)) if chain := c.DiscoveryChain[id]; chain != nil { chains = append(chains, chain) } } default: return nil, nil, nil, fmt.Errorf("unknown route kind %q", routeRef.Kind) } } if len(chains) == 0 { return nil, nil, nil, nil } services, compiled, err := synthesizer.Synthesize(chains...) if err != nil { return nil, nil, nil, err } // reconstruct the upstreams upstreams := make([]structs.Upstream, 0, len(services)) for _, service := range services { upstreams = append(upstreams, structs.Upstream{ DestinationName: service.Name, DestinationNamespace: service.NamespaceOrDefault(), DestinationPartition: service.PartitionOrDefault(), IngressHosts: service.Hosts, LocalBindPort: listener.Port, Config: map[string]interface{}{ "protocol": string(listener.Protocol), }, }) } return services, upstreams, compiled, err } func (c *configSnapshotAPIGateway) toIngressTLS(key IngressListenerKey, listener structs.APIGatewayListener, bound structs.BoundAPIGatewayListener) (*structs.GatewayTLSConfig, error) { if len(listener.TLS.Certificates) == 0 { return nil, nil } for _, certRef := range bound.Certificates { cert, ok := c.Certificates.Get(certRef) if !ok { continue } c.ListenerCertificates[key] = append(c.ListenerCertificates[key], *cert) } return &structs.GatewayTLSConfig{ Enabled: true, TLSMinVersion: listener.TLS.MinVersion, TLSMaxVersion: listener.TLS.MaxVersion, CipherSuites: listener.TLS.CipherSuites, }, nil } type configSnapshotIngressGateway struct { ConfigSnapshotUpstreams // TLSConfig is the gateway-level TLS configuration. Listener/service level // config is preserved in the Listeners map below. TLSConfig structs.GatewayTLSConfig // GatewayConfigLoaded is used to determine if we have received the initial // ingress-gateway config entry yet. GatewayConfigLoaded bool // Hosts is the list of extra host entries to add to our leaf cert's DNS SANs. Hosts []string HostsSet bool // LeafCertWatchCancel is a CancelFunc to use when refreshing this gateway's // leaf cert watch with different parameters. LeafCertWatchCancel context.CancelFunc // Upstreams is a list of upstreams this ingress gateway should serve traffic // to. This is constructed from the ingress-gateway config entry, and uses // the GatewayServices RPC to retrieve them. Upstreams map[IngressListenerKey]structs.Upstreams // UpstreamsSet is the unique set of UpstreamID the gateway routes to. UpstreamsSet map[UpstreamID]struct{} // Listeners is the original listener config from the ingress-gateway config // entry to save us trying to pass fields through Upstreams Listeners map[IngressListenerKey]structs.IngressListener // Defaults is the default configuration for upstream service instances Defaults structs.IngressServiceConfig } // isEmpty is a test helper func (c *configSnapshotIngressGateway) isEmpty() bool { if c == nil { return true } return len(c.Upstreams) == 0 && len(c.UpstreamsSet) == 0 && len(c.DiscoveryChain) == 0 && len(c.WatchedUpstreams) == 0 && len(c.WatchedUpstreamEndpoints) == 0 && !c.MeshConfigSet } type APIGatewayListenerKey = IngressListenerKey type IngressListenerKey struct { Protocol string Port int } func (k *IngressListenerKey) RouteName() string { return fmt.Sprintf("%d", k.Port) } func IngressListenerKeyFromGWService(s structs.GatewayService) IngressListenerKey { return IngressListenerKey{Protocol: s.Protocol, Port: s.Port} } func IngressListenerKeyFromListener(l structs.IngressListener) IngressListenerKey { return IngressListenerKey{Protocol: l.Protocol, Port: l.Port} } // ConfigSnapshot captures all the resulting config needed for a proxy instance. // It is meant to be point-in-time coherent and is used to deliver the current // config state to observers who need it to be pushed in (e.g. XDS server). type ConfigSnapshot struct { Kind structs.ServiceKind Service string ProxyID ProxyID Address string Port int ServiceMeta map[string]string TaggedAddresses map[string]structs.ServiceAddress Proxy structs.ConnectProxyConfig Datacenter string IntentionDefaultAllow bool Locality GatewayKey ServerSNIFn ServerSNIFunc Roots *structs.IndexedCARoots // connect-proxy specific ConnectProxy configSnapshotConnectProxy // terminating-gateway specific TerminatingGateway configSnapshotTerminatingGateway // mesh-gateway specific MeshGateway configSnapshotMeshGateway // ingress-gateway specific IngressGateway configSnapshotIngressGateway // api-gateway specific APIGateway configSnapshotAPIGateway } // Valid returns whether or not the snapshot has all required fields filled yet. func (s *ConfigSnapshot) Valid() bool { switch s.Kind { case structs.ServiceKindConnectProxy: if s.Proxy.Mode == structs.ProxyModeTransparent && !s.ConnectProxy.MeshConfigSet { return false } return s.Roots != nil && s.ConnectProxy.Leaf != nil && s.ConnectProxy.IntentionsSet && s.ConnectProxy.MeshConfigSet case structs.ServiceKindTerminatingGateway: return s.Roots != nil && s.TerminatingGateway.MeshConfigSet case structs.ServiceKindMeshGateway: if s.MeshGateway.WatchedLocalServers.Len() == 0 { if s.ServiceMeta[structs.MetaWANFederationKey] == "1" { return false } if cfg := s.MeshConfig(); cfg.PeerThroughMeshGateways() { return false } } return s.Roots != nil && (s.MeshGateway.WatchedServicesSet || len(s.MeshGateway.ServiceGroups) > 0) && s.MeshGateway.ExportedServicesSet && s.MeshGateway.MeshConfigSet && s.MeshGateway.PeeringTrustBundlesSet case structs.ServiceKindIngressGateway: return s.Roots != nil && s.IngressGateway.Leaf != nil && s.IngressGateway.GatewayConfigLoaded && s.IngressGateway.HostsSet && s.IngressGateway.MeshConfigSet case structs.ServiceKindAPIGateway: // TODO Is this the proper set of things to validate? return s.Roots != nil && s.APIGateway.Leaf != nil && s.APIGateway.GatewayConfigLoaded && s.APIGateway.BoundGatewayConfigLoaded && s.APIGateway.AreHostsSet && s.APIGateway.MeshConfigSet default: return false } } // Clone makes a deep copy of the snapshot we can send to other goroutines // without worrying that they will racily read or mutate shared maps etc. func (s *ConfigSnapshot) Clone() *ConfigSnapshot { snap := s.DeepCopy() // nil these out as anything receiving one of these clones does not need them and should never "cancel" our watches switch s.Kind { case structs.ServiceKindConnectProxy: // common with connect-proxy and ingress-gateway snap.ConnectProxy.WatchedUpstreams = nil snap.ConnectProxy.WatchedGateways = nil snap.ConnectProxy.WatchedDiscoveryChains = nil case structs.ServiceKindTerminatingGateway: snap.TerminatingGateway.WatchedServices = nil snap.TerminatingGateway.WatchedIntentions = nil snap.TerminatingGateway.WatchedLeaves = nil snap.TerminatingGateway.WatchedConfigs = nil snap.TerminatingGateway.WatchedResolvers = nil case structs.ServiceKindMeshGateway: snap.MeshGateway.WatchedGateways = nil snap.MeshGateway.WatchedServices = nil case structs.ServiceKindIngressGateway: // common with connect-proxy and ingress-gateway snap.IngressGateway.WatchedUpstreams = nil snap.IngressGateway.WatchedGateways = nil snap.IngressGateway.WatchedDiscoveryChains = nil // only ingress-gateway snap.IngressGateway.LeafCertWatchCancel = nil case structs.ServiceKindAPIGateway: // common with connect-proxy and api-gateway snap.APIGateway.WatchedUpstreams = nil snap.APIGateway.WatchedGateways = nil snap.APIGateway.WatchedDiscoveryChains = nil // only api-gateway //snap.APIGateway.LeafCertWatchCancel = nil //snap.APIGateway. } return snap } func (s *ConfigSnapshot) Leaf() *structs.IssuedCert { switch s.Kind { case structs.ServiceKindConnectProxy: return s.ConnectProxy.Leaf case structs.ServiceKindIngressGateway: return s.IngressGateway.Leaf case structs.ServiceKindAPIGateway: return s.APIGateway.Leaf case structs.ServiceKindMeshGateway: return s.MeshGateway.Leaf default: return nil } } func (s *ConfigSnapshot) PeeringTrustBundles() []*pbpeering.PeeringTrustBundle { switch s.Kind { case structs.ServiceKindConnectProxy: return s.ConnectProxy.InboundPeerTrustBundles case structs.ServiceKindMeshGateway: return s.MeshGateway.PeeringTrustBundles default: return nil } } // RootPEMs returns all PEM-encoded public certificates for the root CA. func (s *ConfigSnapshot) RootPEMs() string { var rootPEMs string for _, root := range s.Roots.Roots { rootPEMs += lib.EnsureTrailingNewline(root.RootCert) } return rootPEMs } func (s *ConfigSnapshot) MeshConfig() *structs.MeshConfigEntry { switch s.Kind { case structs.ServiceKindConnectProxy: return s.ConnectProxy.MeshConfig case structs.ServiceKindIngressGateway: return s.IngressGateway.MeshConfig case structs.ServiceKindAPIGateway: return s.APIGateway.MeshConfig case structs.ServiceKindTerminatingGateway: return s.TerminatingGateway.MeshConfig case structs.ServiceKindMeshGateway: return s.MeshGateway.MeshConfig default: return nil } } func (s *ConfigSnapshot) MeshConfigTLSIncoming() *structs.MeshDirectionalTLSConfig { mesh := s.MeshConfig() if mesh == nil || mesh.TLS == nil { return nil } return mesh.TLS.Incoming } func (s *ConfigSnapshot) MeshConfigTLSOutgoing() *structs.MeshDirectionalTLSConfig { mesh := s.MeshConfig() if mesh == nil || mesh.TLS == nil { return nil } return mesh.TLS.Outgoing } func (s *ConfigSnapshot) ToConfigSnapshotUpstreams() (*ConfigSnapshotUpstreams, error) { switch s.Kind { case structs.ServiceKindConnectProxy: return &s.ConnectProxy.ConfigSnapshotUpstreams, nil case structs.ServiceKindIngressGateway: return &s.IngressGateway.ConfigSnapshotUpstreams, nil case structs.ServiceKindAPIGateway: return &s.APIGateway.ConfigSnapshotUpstreams, nil default: // This is a coherence check and should never fail return nil, fmt.Errorf("No upstream snapshot for gateway mode %q", s.Kind) } } func (u *ConfigSnapshotUpstreams) UpstreamPeerMeta(uid UpstreamID) (structs.PeeringServiceMeta, bool) { nodes, _ := u.PeerUpstreamEndpoints.Get(uid) if len(nodes) == 0 { return structs.PeeringServiceMeta{}, false } // In agent/rpc/peering/subscription_manager.go we denormalize the // PeeringServiceMeta data onto each replicated service instance to convey // this information back to the importing side of the peering. // // This data is guaranteed (subject to any eventual consistency lag around // updates) to be the same across all instances, so we only need to take // the first item. // // TODO(peering): consider replicating this "common to all instances" data // using a different replication type and persist it separately in the // catalog to avoid this weird construction. csn := nodes[0] if csn.Service == nil { return structs.PeeringServiceMeta{}, false } return *csn.Service.Connect.PeerMeta, true } // PeeredUpstreamIDs returns a slice of peered UpstreamIDs from explicit config entries // and implicit imported services. // Upstreams whose trust bundles have not been stored in the snapshot are ignored. func (u *ConfigSnapshotUpstreams) PeeredUpstreamIDs() []UpstreamID { out := make([]UpstreamID, 0, u.PeerUpstreamEndpoints.Len()) u.PeerUpstreamEndpoints.ForEachKey(func(uid UpstreamID) bool { if _, ok := u.PeerUpstreamEndpoints.Get(uid); !ok { // uid might exist in the map but if Set hasn't been called, skip for now. return true } if _, ok := u.UpstreamPeerTrustBundles.Get(uid.Peer); !ok { // The trust bundle for this upstream is not available yet, skip for now. return true } out = append(out, uid) return true }) return out }