// Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package gateways import ( "context" "errors" "fmt" "sync" "time" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-memdb" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/consul/controller" "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/consul/fsm" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/consul/stream" "github.com/hashicorp/consul/agent/structs" ) var ( errServiceDoesNotExist = errors.New("service does not exist") errInvalidProtocol = errors.New("route protocol does not match targeted service protocol") ) // Updater is a thin wrapper around a set of callbacks used for updating // and deleting config entries via raft operations. type Updater struct { UpdateWithStatus func(entry structs.ControlledConfigEntry) error Update func(entry structs.ConfigEntry) error Delete func(entry structs.ConfigEntry) error } // apiGatewayReconciler is the monolithic reconciler used for reconciling // all of our routes and gateways into bound gateway state. type apiGatewayReconciler struct { fsm *fsm.FSM logger hclog.Logger updater *Updater controller controller.Controller } // Reconcile is the main reconciliation function for the gateway reconciler, it // delegates each reconciliation request to functions designated for a // particular type of config entry. func (r *apiGatewayReconciler) Reconcile(ctx context.Context, req controller.Request) error { // We do this in a single threaded way to avoid race conditions around setting // shared state. In our current out-of-repo code, this is handled via a global // lock on our shared store, but this makes it so we don't have to deal with lock // contention, and instead just work with a single control loop. switch req.Kind { case structs.APIGateway: return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileGateway, r.cleanupGateway) case structs.BoundAPIGateway: return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileBoundGateway, r.cleanupBoundGateway) case structs.HTTPRoute: return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileHTTPRoute, r.cleanupRoute) case structs.TCPRoute: return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileTCPRoute, r.cleanupRoute) case structs.InlineCertificate: return r.enqueueCertificateReferencedGateways(r.fsm.State(), ctx, req) default: return nil } } // reconcileEntry converts the controller request into a config entry that we then pass // along to either a cleanup function if the entry no longer exists (it's been deleted), // or a reconciler if the entry has been updated or created. func reconcileEntry[T structs.ControlledConfigEntry](store *state.Store, logger hclog.Logger, ctx context.Context, req controller.Request, reconciler func(ctx context.Context, req controller.Request, store *state.Store, entry T) error, cleaner func(ctx context.Context, req controller.Request, store *state.Store) error) error { _, entry, err := store.ConfigEntry(nil, req.Kind, req.Name, req.Meta) if err != nil { requestLogger(logger, req).Warn("error fetching config entry for reconciliation request", "error", err) return err } if entry == nil { return cleaner(ctx, req, store) } return reconciler(ctx, req, store, entry.(T)) } // enqueueCertificateReferencedGateways retrieves all gateway objects, filters to those referencing // the provided certificate, and enqueues the gateways for reconciliation func (r *apiGatewayReconciler) enqueueCertificateReferencedGateways(store *state.Store, _ context.Context, req controller.Request) error { logger := certificateRequestLogger(r.logger, req) logger.Trace("certificate changed, enqueueing dependent gateways") defer logger.Trace("finished enqueuing gateways") _, entries, err := store.ConfigEntriesByKind(nil, structs.APIGateway, acl.WildcardEnterpriseMeta()) if err != nil { logger.Warn("error retrieving api gateways", "error", err) return err } requests := []controller.Request{} for _, entry := range entries { gateway := entry.(*structs.APIGatewayConfigEntry) for _, listener := range gateway.Listeners { for _, certificate := range listener.TLS.Certificates { if certificate.IsSame(&structs.ResourceReference{ Kind: req.Kind, Name: req.Name, EnterpriseMeta: *req.Meta, }) { requests = append(requests, controller.Request{ Kind: structs.APIGateway, Name: gateway.Name, Meta: &gateway.EnterpriseMeta, }) } } } } r.controller.Enqueue(requests...) return nil } // cleanupBoundGateway retrieves all routes from the store and removes the gateway from any // routes that are bound to it, updating their status appropriately func (r *apiGatewayReconciler) cleanupBoundGateway(_ context.Context, req controller.Request, store *state.Store) error { logger := gatewayRequestLogger(r.logger, req) logger.Trace("cleaning up bound gateway") defer logger.Trace("finished cleaning up bound gateway") routes, err := retrieveAllRoutesFromStore(store) if err != nil { logger.Warn("error retrieving routes", "error", err) return err } resource := requestToResourceRef(req) resource.Kind = structs.APIGateway for _, modifiedRoute := range removeGateway(resource, routes...) { routeLogger := routeLogger(logger, modifiedRoute) routeLogger.Trace("persisting route status") if err := r.updater.Update(modifiedRoute); err != nil { routeLogger.Warn("error removing gateway from route", "error", err) return err } } return nil } // reconcileBoundGateway mainly handles orphaned bound gateways at startup, it just checks // to make sure there's still an existing gateway, and if not, it deletes the bound gateway func (r *apiGatewayReconciler) reconcileBoundGateway(_ context.Context, req controller.Request, store *state.Store, bound *structs.BoundAPIGatewayConfigEntry) error { logger := gatewayRequestLogger(r.logger, req) logger.Trace("reconciling bound gateway") defer logger.Trace("finished reconciling bound gateway") _, gateway, err := store.ConfigEntry(nil, structs.APIGateway, req.Name, req.Meta) if err != nil { logger.Warn("error retrieving api gateway", "error", err) return err } if gateway == nil { // delete the bound gateway logger.Trace("deleting bound api gateway") if err := r.updater.Delete(bound); err != nil { logger.Warn("error deleting bound api gateway", "error", err) return err } } return nil } // cleanupGateway deletes the associated bound gateway state with the config entry, route // cleanup occurs when the bound gateway is re-reconciled or on the next reconciliation // pass for the route. func (r *apiGatewayReconciler) cleanupGateway(_ context.Context, req controller.Request, store *state.Store) error { logger := gatewayRequestLogger(r.logger, req) logger.Trace("cleaning up deleted gateway") defer logger.Trace("finished cleaning up deleted gateway") _, bound, err := store.ConfigEntry(nil, structs.BoundAPIGateway, req.Name, req.Meta) if err != nil { logger.Warn("error retrieving bound api gateway", "error", err) return err } logger.Trace("deleting bound api gateway") if err := r.updater.Delete(bound); err != nil { logger.Warn("error deleting bound api gateway", "error", err) return err } return nil } // reconcileGateway attempts to initialize or fetch the associated bound // gateway state, fetch all route references, validate the existence of any // referenced certificates, and then update the bound gateway with certificate // references and add or remove any routes that reference or previously // referenced this gateway. It then persists any status updates for the gateway, // the modified routes, and updates the bound gateway. func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controller.Request, store *state.Store, gateway *structs.APIGatewayConfigEntry) error { conditions := newGatewayConditionGenerator() logger := gatewayRequestLogger(r.logger, req) logger.Trace("started reconciling gateway") defer logger.Trace("finished reconciling gateway") updater := structs.NewStatusUpdater(gateway) // we clear out the initial status conditions since we're doing a full update // of this gateway's status updater.ClearConditions() routes, err := retrieveAllRoutesFromStore(store) if err != nil { logger.Warn("error retrieving routes", "error", err) return err } // construct the tuple we'll be working on to update state _, bound, err := store.ConfigEntry(nil, structs.BoundAPIGateway, req.Name, req.Meta) if err != nil { logger.Warn("error retrieving bound api gateway", "error", err) return err } meta := newGatewayMeta(gateway, bound) certificateErrors, err := meta.checkCertificates(store) if err != nil { logger.Warn("error checking gateway certificates", "error", err) return err } for ref, err := range certificateErrors { updater.SetCondition(conditions.invalidCertificate(ref, err)) } if len(certificateErrors) > 0 { updater.SetCondition(conditions.invalidCertificates()) } else { updater.SetCondition(conditions.gatewayAccepted()) } // now we bind all of the routes we can updatedRoutes := []structs.ControlledConfigEntry{} for _, route := range routes { routeUpdater := structs.NewStatusUpdater(route) _, boundRefs, bindErrors := bindRoutesToGateways(route, meta) // unset the old gateway binding in case it's stale for _, parent := range route.GetParents() { if parent.Kind == gateway.Kind && parent.Name == gateway.Name && parent.EnterpriseMeta.IsSame(&gateway.EnterpriseMeta) { routeUpdater.RemoveCondition(conditions.routeBound(parent)) } } // set the status for parents that have bound successfully for _, ref := range boundRefs { routeUpdater.SetCondition(conditions.routeBound(ref)) } // set the status for any parents that have errored trying to // bind for ref, err := range bindErrors { routeUpdater.SetCondition(conditions.routeUnbound(ref, err)) } // if we've updated any statuses, then store them as needing // to be updated if entry, updated := routeUpdater.UpdateEntry(); updated { updatedRoutes = append(updatedRoutes, entry) } } // first set any gateway conflict statuses meta.setConflicts(updater) // now check if we need to update the gateway status if modifiedGateway, shouldUpdate := updater.UpdateEntry(); shouldUpdate { logger.Trace("persisting gateway status") if err := r.updater.UpdateWithStatus(modifiedGateway); err != nil { logger.Warn("error persisting gateway status", "error", err) return err } } // next update route statuses for _, modifiedRoute := range updatedRoutes { routeLogger := routeLogger(logger, modifiedRoute) routeLogger.Trace("persisting route status") if err := r.updater.UpdateWithStatus(modifiedRoute); err != nil { routeLogger.Warn("error persisting route status", "error", err) return err } } // now update the bound state if it changed if bound == nil || !bound.(*structs.BoundAPIGatewayConfigEntry).IsSame(meta.BoundGateway) { logger.Trace("persisting bound api gateway") if err := r.updater.Update(meta.BoundGateway); err != nil { logger.Warn("error persisting bound api gateway", "error", err) return err } } return nil } // cleanupRoute fetches all gateways and removes any existing reference to // the route we're reconciling from them. func (r *apiGatewayReconciler) cleanupRoute(_ context.Context, req controller.Request, store *state.Store) error { logger := routeRequestLogger(r.logger, req) logger.Trace("cleaning up route") defer logger.Trace("finished cleaning up route") meta, err := getAllGatewayMeta(store) if err != nil { logger.Warn("error retrieving gateways", "error", err) return err } for _, modifiedGateway := range removeRoute(requestToResourceRef(req), meta...) { gatewayLogger := gatewayLogger(logger, modifiedGateway.BoundGateway) gatewayLogger.Trace("persisting bound gateway state") if err := r.updater.Update(modifiedGateway.BoundGateway); err != nil { gatewayLogger.Warn("error updating bound api gateway", "error", err) return err } } r.controller.RemoveTrigger(req) return nil } // reconcileRoute attempts to validate a route against its referenced service // discovery chain, it also fetches all gateways, and attempts to either remove // the route being reconciled from gateways containing either stale references // when this route no longer references them, or add the route to gateways that // it now references. It then updates any necessary route statuses, checks for // gateways that now have route conflicts, and updates all statuses and states // as necessary. func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller.Request, store *state.Store, route structs.BoundRoute) error { conditions := newGatewayConditionGenerator() logger := routeRequestLogger(r.logger, req) logger.Trace("reconciling route") defer logger.Trace("finished reconciling route") meta, err := getAllGatewayMeta(store) if err != nil { logger.Warn("error retrieving gateways", "error", err) return err } updater := structs.NewStatusUpdater(route) // we clear out the initial status conditions since we're doing a full update // of this route's status updater.ClearConditions() ws := memdb.NewWatchSet() ws.Add(store.AbandonCh()) finalize := func(modifiedGateways []*structs.BoundAPIGatewayConfigEntry) error { // first update any gateway statuses that are now in conflict for _, gateway := range meta { modifiedGateway, shouldUpdate := gateway.checkConflicts() if shouldUpdate { gatewayLogger := gatewayLogger(logger, modifiedGateway) gatewayLogger.Trace("persisting gateway status") if err := r.updater.UpdateWithStatus(modifiedGateway); err != nil { gatewayLogger.Warn("error persisting gateway", "error", err) return err } } } // next update the route status if modifiedRoute, shouldUpdate := updater.UpdateEntry(); shouldUpdate { r.logger.Trace("persisting route status") if err := r.updater.UpdateWithStatus(modifiedRoute); err != nil { r.logger.Warn("error persisting route", "error", err) return err } } // now update all of the bound gateways that have been modified for _, bound := range modifiedGateways { gatewayLogger := gatewayLogger(logger, bound) gatewayLogger.Trace("persisting bound api gateway") if err := r.updater.Update(bound); err != nil { gatewayLogger.Warn("error persisting bound api gateway", "error", err) return err } } return nil } var triggerOnce sync.Once for _, service := range route.GetServiceNames() { _, chainSet, err := store.ReadDiscoveryChainConfigEntries(ws, service.Name, pointerTo(service.EnterpriseMeta)) if err != nil { logger.Warn("error reading discovery chain", "error", err) return err } // trigger a watch since we now need to check when the discovery chain gets updated triggerOnce.Do(func() { r.controller.AddTrigger(req, ws.WatchCtx) }) // make sure that we can actually compile a discovery chain based on this route // the main check is to make sure that all of the protocols align chain, err := discoverychain.Compile(discoverychain.CompileRequest{ ServiceName: service.Name, EvaluateInNamespace: service.NamespaceOrDefault(), EvaluateInPartition: service.PartitionOrDefault(), EvaluateInDatacenter: "dc1", // just mock out a fake dc since we're just checking for compilation errors EvaluateInTrustDomain: "consul.domain", // just mock out a fake trust domain since we're just checking for compilation errors Entries: chainSet, }) if err != nil { updater.SetCondition(conditions.routeInvalidDiscoveryChain(err)) continue } if chain.Protocol != string(route.GetProtocol()) { updater.SetCondition(conditions.routeInvalidDiscoveryChain(errInvalidProtocol)) continue } updater.SetCondition(conditions.routeAccepted()) } // if we have no upstream targets, then set the route as invalid // this should already happen in the validation check on write, but // we'll do it here too just in case if len(route.GetServiceNames()) == 0 { updater.SetCondition(conditions.routeNoUpstreams()) } // the route is valid, attempt to bind it to all gateways r.logger.Trace("binding routes to gateway") modifiedGateways, boundRefs, bindErrors := bindRoutesToGateways(route, meta...) // set the status of the references that are bound for _, ref := range boundRefs { updater.SetCondition(conditions.routeBound(ref)) } // set any binding errors for ref, err := range bindErrors { updater.SetCondition(conditions.routeUnbound(ref, err)) } // set any refs that haven't been bound or explicitly errored PARENT_LOOP: for _, ref := range route.GetParents() { for _, boundRef := range boundRefs { if ref.IsSame(&boundRef) { continue PARENT_LOOP } } if _, ok := bindErrors[ref]; ok { continue PARENT_LOOP } updater.SetCondition(conditions.gatewayNotFound(ref)) } return finalize(modifiedGateways) } // reconcileHTTPRoute is a thin wrapper around recnocileRoute for a HTTPRoutes func (r *apiGatewayReconciler) reconcileHTTPRoute(ctx context.Context, req controller.Request, store *state.Store, route *structs.HTTPRouteConfigEntry) error { return r.reconcileRoute(ctx, req, store, route) } // reconcileTCPRoute is a thin wrapper around recnocileRoute for a TCPRoutes func (r *apiGatewayReconciler) reconcileTCPRoute(ctx context.Context, req controller.Request, store *state.Store, route *structs.TCPRouteConfigEntry) error { return r.reconcileRoute(ctx, req, store, route) } // NewAPIGatewayController initializes a controller that reconciles all APIGateway objects func NewAPIGatewayController(fsm *fsm.FSM, publisher state.EventPublisher, updater *Updater, logger hclog.Logger) controller.Controller { reconciler := &apiGatewayReconciler{ fsm: fsm, logger: logger, updater: updater, } reconciler.controller = controller.New(publisher, reconciler) return reconciler.controller.Subscribe( &stream.SubscribeRequest{ Topic: state.EventTopicAPIGateway, Subject: stream.SubjectWildcard, }, ).Subscribe( &stream.SubscribeRequest{ Topic: state.EventTopicHTTPRoute, Subject: stream.SubjectWildcard, }, ).Subscribe( &stream.SubscribeRequest{ Topic: state.EventTopicTCPRoute, Subject: stream.SubjectWildcard, }, ).Subscribe( &stream.SubscribeRequest{ Topic: state.EventTopicBoundAPIGateway, Subject: stream.SubjectWildcard, }, ).Subscribe( &stream.SubscribeRequest{ Topic: state.EventTopicInlineCertificate, Subject: stream.SubjectWildcard, }) } // gatewayMeta embeds both a BoundAPIGateway and its corresponding APIGateway. // This is used for binding routes to a gateway, because the binding logic // requires correlation between fields on a gateway and a route, while persisting // the state onto the corresponding subfields of a BoundAPIGateway. For example, // when binding we need to validate that a route's protocol (e.g. http) // matches the protocol of the listener it wants to bind to. type gatewayMeta struct { // BoundGateway is the bound-api-gateway config entry for a given gateway. BoundGateway *structs.BoundAPIGatewayConfigEntry // Gateway is the api-gateway config entry for the gateway. Gateway *structs.APIGatewayConfigEntry // listeners is a map of gateway listeners by name for fast access // the map values are pointers so that we can update them directly // and have the changes propagate back to the container gateways. listeners map[string]*structs.APIGatewayListener // boundListeners is a map of gateway listeners by name for fast access // the map values are pointers so that we can update them directly // and have the changes propagate back to the container gateways. boundListeners map[string]*structs.BoundAPIGatewayListener generator *gatewayConditionGenerator } // getAllGatewayMeta returns a pre-constructed list of all valid gateway and state // tuples based on the state coming from the store. Any gateway that does not have // a corresponding bound-api-gateway config entry will be filtered out. func getAllGatewayMeta(store *state.Store) ([]*gatewayMeta, error) { _, gateways, err := store.ConfigEntriesByKind(nil, structs.APIGateway, acl.WildcardEnterpriseMeta()) if err != nil { return nil, err } _, boundGateways, err := store.ConfigEntriesByKind(nil, structs.BoundAPIGateway, acl.WildcardEnterpriseMeta()) if err != nil { return nil, err } meta := make([]*gatewayMeta, 0, len(boundGateways)) for _, b := range boundGateways { bound := b.(*structs.BoundAPIGatewayConfigEntry) for _, g := range gateways { gateway := g.(*structs.APIGatewayConfigEntry) if bound.IsInitializedForGateway(gateway) { meta = append(meta, (&gatewayMeta{ BoundGateway: bound, Gateway: gateway, }).initialize()) break } } } return meta, nil } // updateRouteBinding takes a BoundRoute and modifies the listeners on the // BoundAPIGateway config entry in GatewayMeta to reflect the binding of the // route to the gateway. // // The return values correspond to: // 1. whether the underlying BoundAPIGateway was actually modified // 2. what references from the BoundRoute actually bound to the Gateway successfully // 3. any errors that occurred while attempting to bind a particular reference to the Gateway func (g *gatewayMeta) updateRouteBinding(route structs.BoundRoute) (bool, []structs.ResourceReference, map[structs.ResourceReference]error) { errors := make(map[structs.ResourceReference]error) boundRefs := []structs.ResourceReference{} listenerUnbound := make(map[string]bool, len(g.boundListeners)) listenerBound := make(map[string]bool, len(g.boundListeners)) routeRef := structs.ResourceReference{ Kind: route.GetKind(), Name: route.GetName(), EnterpriseMeta: *route.GetEnterpriseMeta(), } // first attempt to unbind all of the routes from the listeners in case they're // stale g.eachListener(func(listener *structs.APIGatewayListener, bound *structs.BoundAPIGatewayListener) error { listenerUnbound[listener.Name] = bound.UnbindRoute(routeRef) return nil }) // now try and bind all of the route's current refs for _, ref := range route.GetParents() { if !g.shouldBindRoute(ref) { continue } if len(g.boundListeners) == 0 { errors[ref] = fmt.Errorf("route cannot bind because gateway has no listeners") continue } // try to bind to all listeners refDidBind := false g.eachListener(func(listener *structs.APIGatewayListener, bound *structs.BoundAPIGatewayListener) error { didBind, err := g.bindRoute(listener, bound, route, ref) if err != nil { errors[ref] = err } if didBind { refDidBind = true listenerBound[listener.Name] = true } return nil }) // double check that the wildcard ref actually bound to something if !refDidBind && errors[ref] == nil { errors[ref] = fmt.Errorf("failed to bind route %s to gateway %s with listener '%s'", route.GetName(), g.Gateway.Name, ref.SectionName) } if refDidBind { boundRefs = append(boundRefs, ref) } } didUpdate := false for name, didUnbind := range listenerUnbound { didBind := listenerBound[name] if didBind != didUnbind { didUpdate = true break } } return didUpdate, boundRefs, errors } // shouldBindRoute returns whether a Route's parent reference references the Gateway // that we wrap. func (g *gatewayMeta) shouldBindRoute(ref structs.ResourceReference) bool { return (ref.Kind == structs.APIGateway || ref.Kind == "") && g.Gateway.Name == ref.Name && g.Gateway.EnterpriseMeta.IsSame(&ref.EnterpriseMeta) } // shouldBindRouteToListener returns whether a Route's parent reference should attempt // to bind to the given listener because it is either explicitly named or the Route // is attempting to wildcard bind to the listener. func (g *gatewayMeta) shouldBindRouteToListener(l *structs.BoundAPIGatewayListener, ref structs.ResourceReference) bool { return l.Name == ref.SectionName || ref.SectionName == "" } // bindRoute takes a particular listener that a Route is attempting to bind to with a given reference // and returns whether the Route successfully bound to the listener or if it errored in the process. func (g *gatewayMeta) bindRoute(listener *structs.APIGatewayListener, bound *structs.BoundAPIGatewayListener, route structs.BoundRoute, ref structs.ResourceReference) (bool, error) { if !g.shouldBindRouteToListener(bound, ref) { return false, nil } // check to make sure we're not binding to an invalid gateway if !g.Gateway.Status.MatchesConditionStatus(g.generator.gatewayAccepted()) { return false, fmt.Errorf("failed to bind route to gateway %s: gateway has not been accepted", g.Gateway.Name) } // check to make sure we're not binding to an invalid route status := route.GetStatus() if !status.MatchesConditionStatus(g.generator.routeAccepted()) { return false, fmt.Errorf("failed to bind route to gateway %s: route has not been accepted", g.Gateway.Name) } if route, ok := route.(*structs.HTTPRouteConfigEntry); ok { // check our hostnames hostnames := route.FilteredHostnames(listener.GetHostname()) if len(hostnames) == 0 { return false, fmt.Errorf("failed to bind route to gateway %s: listener %s is does not have any hostnames that match the route", g.Gateway.Name, listener.Name) } } if listener.Protocol == route.GetProtocol() && bound.BindRoute(structs.ResourceReference{ Kind: route.GetKind(), Name: route.GetName(), EnterpriseMeta: *route.GetEnterpriseMeta(), }) { return true, nil } if ref.SectionName != "" { return false, fmt.Errorf("failed to bind route %s to gateway %s: listener %s is not a %s listener", route.GetName(), g.Gateway.Name, bound.Name, route.GetProtocol()) } return false, nil } // unbindRoute takes a route and unbinds it from all of the listeners on a gateway. // It returns true if the route was unbound and false if it was not. func (g *gatewayMeta) unbindRoute(route structs.ResourceReference) bool { didUnbind := false for _, listener := range g.boundListeners { if listener.UnbindRoute(route) { didUnbind = true } } return didUnbind } // eachListener iterates over all of the listeners for our underlying Gateway, it takes // a callback function that can return an error, if an error is returned it halts execution // and immediately returns the error. func (g *gatewayMeta) eachListener(fn func(listener *structs.APIGatewayListener, bound *structs.BoundAPIGatewayListener) error) error { for name, listener := range g.listeners { if err := fn(listener, g.boundListeners[name]); err != nil { return err } } return nil } // checkCertificates verifies that all certificates referenced by the listeners on the gateway // exist and collects them onto the bound gateway func (g *gatewayMeta) checkCertificates(store *state.Store) (map[structs.ResourceReference]error, error) { certificateErrors := map[structs.ResourceReference]error{} err := g.eachListener(func(listener *structs.APIGatewayListener, bound *structs.BoundAPIGatewayListener) error { for _, ref := range listener.TLS.Certificates { _, certificate, err := store.ConfigEntry(nil, ref.Kind, ref.Name, &ref.EnterpriseMeta) if err != nil { return err } if certificate == nil { certificateErrors[ref] = errors.New("certificate not found") } else { bound.Certificates = append(bound.Certificates, ref) } } return nil }) if err != nil { return nil, err } return certificateErrors, nil } // checkConflicts returns whether a gateway status needs to be updated with // conflicting route statuses func (g *gatewayMeta) checkConflicts() (structs.ControlledConfigEntry, bool) { updater := structs.NewStatusUpdater(g.Gateway) g.setConflicts(updater) return updater.UpdateEntry() } // setConflicts ensures that no TCP listener has more than the one allowed route and // assigns an appropriate status func (g *gatewayMeta) setConflicts(updater *structs.StatusUpdater) { conditions := newGatewayConditionGenerator() g.eachListener(func(listener *structs.APIGatewayListener, bound *structs.BoundAPIGatewayListener) error { ref := structs.ResourceReference{ Kind: structs.APIGateway, Name: g.Gateway.Name, SectionName: listener.Name, EnterpriseMeta: g.Gateway.EnterpriseMeta, } switch listener.Protocol { case structs.ListenerProtocolTCP: if len(bound.Routes) > 1 { updater.SetCondition(conditions.gatewayListenerConflicts(ref)) return nil } } updater.SetCondition(conditions.gatewayListenerNoConflicts(ref)) return nil }) } // initialize sets up the listener maps that we use for quickly indexing the listeners in our binding logic func (g *gatewayMeta) initialize() *gatewayMeta { g.generator = newGatewayConditionGenerator() // set up the maps for fast access g.boundListeners = make(map[string]*structs.BoundAPIGatewayListener, len(g.BoundGateway.Listeners)) for i, listener := range g.BoundGateway.Listeners { g.boundListeners[listener.Name] = &g.BoundGateway.Listeners[i] } g.listeners = make(map[string]*structs.APIGatewayListener, len(g.Gateway.Listeners)) for i, listener := range g.Gateway.Listeners { g.listeners[listener.Name] = &g.Gateway.Listeners[i] } return g } // newGatewayMeta returns an object that wraps the given APIGateway and BoundAPIGateway func newGatewayMeta(gateway *structs.APIGatewayConfigEntry, bound structs.ConfigEntry) *gatewayMeta { var b *structs.BoundAPIGatewayConfigEntry if bound == nil { b = &structs.BoundAPIGatewayConfigEntry{ Kind: structs.BoundAPIGateway, Name: gateway.Name, EnterpriseMeta: gateway.EnterpriseMeta, } } else { b = bound.(*structs.BoundAPIGatewayConfigEntry).DeepCopy() } // we just clear out the bound state here since we recalculate it entirely // in the gateway control loop listeners := make([]structs.BoundAPIGatewayListener, 0, len(gateway.Listeners)) for _, listener := range gateway.Listeners { listeners = append(listeners, structs.BoundAPIGatewayListener{ Name: listener.Name, }) } b.Listeners = listeners return (&gatewayMeta{ BoundGateway: b, Gateway: gateway, }).initialize() } // gatewayConditionGenerator is a simple struct used for isolating // the status conditions that we generate for our components type gatewayConditionGenerator struct { now *time.Time } // newGatewayConditionGenerator initializes a status conditions generator func newGatewayConditionGenerator() *gatewayConditionGenerator { return &gatewayConditionGenerator{ now: pointerTo(time.Now().UTC()), } } // invalidCertificate returns a condition used when a gateway references a // certificate that does not exist. It takes a ref used to scope the condition // to a given APIGateway listener. func (g *gatewayConditionGenerator) invalidCertificate(ref structs.ResourceReference, err error) structs.Condition { return structs.Condition{ Type: "Accepted", Status: "False", Reason: "InvalidCertificate", Message: err.Error(), Resource: pointerTo(ref), LastTransitionTime: g.now, } } // invalidCertificates is used to set the overall condition of the APIGateway // to invalid due to missing certificates that it references. func (g *gatewayConditionGenerator) invalidCertificates() structs.Condition { return structs.Condition{ Type: "Accepted", Status: "False", Reason: "InvalidCertificates", Message: "gateway references invalid certificates", LastTransitionTime: g.now, } } // gatewayAccepted marks the APIGateway as valid. func (g *gatewayConditionGenerator) gatewayAccepted() structs.Condition { return structs.Condition{ Type: "Accepted", Status: "True", Reason: "Accepted", Message: "gateway is valid", LastTransitionTime: g.now, } } // routeBound marks a Route as bound to the referenced APIGateway func (g *gatewayConditionGenerator) routeBound(ref structs.ResourceReference) structs.Condition { return structs.Condition{ Type: "Bound", Status: "True", Reason: "Bound", Resource: pointerTo(ref), Message: "successfully bound route", LastTransitionTime: g.now, } } // routeAccepted marks the Route as valid func (g *gatewayConditionGenerator) routeAccepted() structs.Condition { return structs.Condition{ Type: "Accepted", Status: "True", Reason: "Accepted", Message: "route is valid", LastTransitionTime: g.now, } } // routeUnbound marks the route as having failed to bind to the referenced APIGateway func (g *gatewayConditionGenerator) routeUnbound(ref structs.ResourceReference, err error) structs.Condition { return structs.Condition{ Type: "Bound", Status: "False", Reason: "FailedToBind", Resource: pointerTo(ref), Message: err.Error(), LastTransitionTime: g.now, } } // routeInvalidDiscoveryChain marks the route as invalid due to an error while validating its referenced // discovery chian func (g *gatewayConditionGenerator) routeInvalidDiscoveryChain(err error) structs.Condition { return structs.Condition{ Type: "Accepted", Status: "False", Reason: "InvalidDiscoveryChain", Message: err.Error(), LastTransitionTime: g.now, } } // routeNoUpstreams marks the route as invalid because it has no upstreams that it targets func (g *gatewayConditionGenerator) routeNoUpstreams() structs.Condition { return structs.Condition{ Type: "Accepted", Status: "False", Reason: "NoUpstreamServicesTargeted", Message: "route must target at least one upstream service", LastTransitionTime: g.now, } } // gatewayListenerConflicts marks an APIGateway listener as having bound routes that conflict with each other // and make the listener, therefore invalid func (g *gatewayConditionGenerator) gatewayListenerConflicts(ref structs.ResourceReference) structs.Condition { return structs.Condition{ Type: "Conflicted", Status: "True", Reason: "RouteConflict", Resource: pointerTo(ref), Message: "TCP-based listeners currently only support binding a single route", LastTransitionTime: g.now, } } // gatewayListenerNoConflicts marks an APIGateway listener as having no conflicts within its // bound routes func (g *gatewayConditionGenerator) gatewayListenerNoConflicts(ref structs.ResourceReference) structs.Condition { return structs.Condition{ Type: "Conflicted", Status: "False", Reason: "NoConflict", Resource: pointerTo(ref), Message: "listener has no route conflicts", LastTransitionTime: g.now, } } // gatewayNotFound marks a Route as having failed to bind to a referenced APIGateway due to // the Gateway not existing (or having not been reconciled yet) func (g *gatewayConditionGenerator) gatewayNotFound(ref structs.ResourceReference) structs.Condition { return structs.Condition{ Type: "Bound", Status: "False", Reason: "GatewayNotFound", Resource: pointerTo(ref), Message: "gateway was not found", LastTransitionTime: g.now, } } // bindRoutesToGateways takes a route variadic number of gateways. // It iterates over the parent references for the route. These parents are gateways the // route should be bound to. If the parent matches a bound gateway, the route is bound to the // gateway. Otherwise, the route is unbound from the gateway if it was previously bound. // // The function returns a list of references to the modified BoundAPIGatewayConfigEntry objects, // a list of parent references on the route that were successfully used to bind the route, and // a map of resource references to errors that occurred when they were attempted to be // bound to a gateway. func bindRoutesToGateways(route structs.BoundRoute, gateways ...*gatewayMeta) ([]*structs.BoundAPIGatewayConfigEntry, []structs.ResourceReference, map[structs.ResourceReference]error) { boundRefs := []structs.ResourceReference{} modified := make([]*structs.BoundAPIGatewayConfigEntry, 0, len(gateways)) // errored stores the errors from events where a resource reference failed to bind to a gateway. errored := make(map[structs.ResourceReference]error) // Iterate over all BoundAPIGateway config entries and try to bind them to the route if they are a parent. for _, gateway := range gateways { didUpdate, bound, errors := gateway.updateRouteBinding(route) if didUpdate { modified = append(modified, gateway.BoundGateway) } for ref, err := range errors { errored[ref] = err } boundRefs = append(boundRefs, bound...) } return modified, boundRefs, errored } // removeGateway sets the route's status appropriately when the gateway that it's // attempting to bind to does not exist func removeGateway(gateway structs.ResourceReference, entries ...structs.BoundRoute) []structs.ControlledConfigEntry { conditions := newGatewayConditionGenerator() modified := []structs.ControlledConfigEntry{} for _, route := range entries { updater := structs.NewStatusUpdater(route) for _, parent := range route.GetParents() { if parent.Kind == gateway.Kind && parent.Name == gateway.Name && parent.EnterpriseMeta.IsSame(&gateway.EnterpriseMeta) { updater.SetCondition(conditions.gatewayNotFound(parent)) } } if toUpdate, shouldUpdate := updater.UpdateEntry(); shouldUpdate { modified = append(modified, toUpdate) } } return modified } // removeRoute unbinds the route from the given gateways, returning the list of gateways that were modified. func removeRoute(route structs.ResourceReference, entries ...*gatewayMeta) []*gatewayMeta { modified := []*gatewayMeta{} for _, entry := range entries { if entry.unbindRoute(route) { modified = append(modified, entry) } } return modified } // requestToResourceRef constructs a resource reference from the given controller request func requestToResourceRef(req controller.Request) structs.ResourceReference { ref := structs.ResourceReference{ Kind: req.Kind, Name: req.Name, } if req.Meta != nil { ref.EnterpriseMeta = *req.Meta } return ref } // retrieveAllRoutesFromStore retrieves all HTTP and TCP routes from the given store func retrieveAllRoutesFromStore(store *state.Store) ([]structs.BoundRoute, error) { _, httpRoutes, err := store.ConfigEntriesByKind(nil, structs.HTTPRoute, acl.WildcardEnterpriseMeta()) if err != nil { return nil, err } _, tcpRoutes, err := store.ConfigEntriesByKind(nil, structs.TCPRoute, acl.WildcardEnterpriseMeta()) if err != nil { return nil, err } routes := make([]structs.BoundRoute, 0, len(tcpRoutes)+len(httpRoutes)) for _, route := range httpRoutes { routes = append(routes, route.(*structs.HTTPRouteConfigEntry)) } for _, route := range tcpRoutes { routes = append(routes, route.(*structs.TCPRouteConfigEntry)) } return routes, nil } // pointerTo returns a pointer to the value passed as an argument func pointerTo[T any](value T) *T { return &value } // requestLogger returns a logger that adds some request-specific fields to the given logger func requestLogger(logger hclog.Logger, request controller.Request) hclog.Logger { meta := request.Meta return logger.With("kind", request.Kind, "name", request.Name, "namespace", meta.NamespaceOrDefault(), "partition", meta.PartitionOrDefault()) } // certificateRequestLogger returns a logger that adds some certificate-specific fields to the given logger func certificateRequestLogger(logger hclog.Logger, request controller.Request) hclog.Logger { meta := request.Meta return logger.With("inline-certificate", request.Name, "namespace", meta.NamespaceOrDefault(), "partition", meta.PartitionOrDefault()) } // gatewayRequestLogger returns a logger that adds some gateway-specific fields to the given logger func gatewayRequestLogger(logger hclog.Logger, request controller.Request) hclog.Logger { meta := request.Meta return logger.With("gateway", request.Name, "namespace", meta.NamespaceOrDefault(), "partition", meta.PartitionOrDefault()) } // gatewayLogger returns a logger that adds some gateway-specific fields to the given logger, // it should be used when logging info about a gateway resource being modified from a non-gateway // reconciliation funciton func gatewayLogger(logger hclog.Logger, gateway structs.ConfigEntry) hclog.Logger { meta := gateway.GetEnterpriseMeta() return logger.With("gateway.name", gateway.GetName(), "gateway.namespace", meta.NamespaceOrDefault(), "gateway.partition", meta.PartitionOrDefault()) } // routeRequestLogger returns a logger that adds some route-specific fields to the given logger func routeRequestLogger(logger hclog.Logger, request controller.Request) hclog.Logger { meta := request.Meta return logger.With("kind", request.Kind, "route", request.Name, "namespace", meta.NamespaceOrDefault(), "partition", meta.PartitionOrDefault()) } // routeLogger returns a logger that adds some route-specific fields to the given logger, // it should be used when logging info about a route resource being modified from a non-route // reconciliation funciton func routeLogger(logger hclog.Logger, route structs.ConfigEntry) hclog.Logger { meta := route.GetEnterpriseMeta() return logger.With("route.kind", route.GetKind(), "route.name", route.GetName(), "route.namespace", meta.NamespaceOrDefault(), "route.partition", meta.PartitionOrDefault()) }