open-consul/agent/proxycfg-sources/catalog/config_source.go

package catalog

import (
	"context"
	"errors"
	"sync"

	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-memdb"

	"github.com/hashicorp/consul/acl"
	"github.com/hashicorp/consul/agent/configentry"
	"github.com/hashicorp/consul/agent/grpc-external/limiter"
	"github.com/hashicorp/consul/agent/local"
	"github.com/hashicorp/consul/agent/proxycfg"
	"github.com/hashicorp/consul/agent/structs"
)

const source proxycfg.ProxySource = "catalog"

// ConfigSource wraps a proxycfg.Manager to register services with it, from the
// catalog, when they are requested by the xDS server.
type ConfigSource struct {
	Config

	mu      sync.Mutex
	watches map[proxycfg.ProxyID]*watch

	shutdownCh chan struct{}
}

type watch struct {
	numWatchers int // guarded by ConfigSource.mu.
	closeCh     chan chan struct{}
}

// NewConfigSource creates a ConfigSource with the given configuration.
func NewConfigSource(cfg Config) *ConfigSource {
	return &ConfigSource{
		Config:     cfg,
		watches:    make(map[proxycfg.ProxyID]*watch),
		shutdownCh: make(chan struct{}),
	}
}

// Watch wraps the underlying proxycfg.Manager and dynamically registers
// services from the catalog with it when requested by the xDS server.
func (m *ConfigSource) Watch(serviceID structs.ServiceID, nodeName string, token string) (<-chan *proxycfg.ConfigSnapshot, limiter.SessionTerminatedChan, proxycfg.CancelFunc, error) {
	// If the service is registered to the local agent, use the LocalConfigSource
	// rather than trying to configure it from the catalog.
	if nodeName == m.NodeName && m.LocalState.ServiceExists(serviceID) {
		return m.LocalConfigSource.Watch(serviceID, nodeName, token)
	}

	// Begin a session with the xDS session concurrency limiter.
	//
	// We do this here rather than in the xDS server because we don't want to apply
	// the limit to services from the LocalConfigSource.
	//
	// See: https://github.com/hashicorp/consul/issues/15753
	session, err := m.SessionLimiter.BeginSession()
	if err != nil {
		return nil, nil, nil, err
	}

	proxyID := proxycfg.ProxyID{
		ServiceID: serviceID,
		NodeName:  nodeName,
		Token:     token,
	}

	// Start the watch on the real proxycfg Manager.
	snapCh, cancelWatch := m.Manager.Watch(proxyID)

	// Wrap the cancelWatch function with our bookkeeping. m.mu must be held when calling.
	var cancelOnce sync.Once
	cancel := func() {
		cancelOnce.Do(func() {
			cancelWatch()
			m.cleanup(proxyID)
			session.End()
		})
	}

	m.mu.Lock()
	defer m.mu.Unlock()

	w, ok := m.watches[proxyID]
	if ok {
		w.numWatchers++
	} else {
		w = &watch{closeCh: make(chan chan struct{}), numWatchers: 1}
		m.watches[proxyID] = w

		if err := m.startSync(w.closeCh, proxyID); err != nil {
			delete(m.watches, proxyID)
			cancelWatch()
			session.End()
			return nil, nil, nil, err
		}
	}

	return snapCh, session.Terminated(), cancel, nil
}

func (m *ConfigSource) Shutdown() {
	close(m.shutdownCh)
}

// startSync fetches a service from the state store's catalog tables and
// registers it with the proxycfg Manager. It spawns a goroutine to watch
// and re-register the service whenever it changes - this goroutine will
// run until a signal is sent on closeCh (at which point the service will
// be deregistered).
//
// If the first attempt to fetch and register the service fails, startSync
// will return an error (and no goroutine will be started).
func (m *ConfigSource) startSync(closeCh <-chan chan struct{}, proxyID proxycfg.ProxyID) error {
	logger := m.Logger.With(
		"proxy_service_id", proxyID.ServiceID.String(),
		"node", proxyID.NodeName,
	)

	logger.Trace("syncing catalog service")

	fetchAndRegister := func() (memdb.WatchSet, error) {
		store := m.GetStore()
		ws := memdb.NewWatchSet()

		// Add the store's AbandonCh to the WatchSet so that if the store is abandoned
		// during a snapshot restore we'll unblock and re-register the service.
		ws.Add(store.AbandonCh())

		_, ns, err := store.NodeService(ws, proxyID.NodeName, proxyID.ID, &proxyID.EnterpriseMeta, structs.DefaultPeerKeyword)
		switch {
		case err != nil:
			logger.Error("failed to read service from state store", "error", err.Error())
			return nil, err
		case ns == nil:
			m.Manager.Deregister(proxyID, source)
			logger.Trace("service does not exist in catalog, de-registering it with proxycfg manager")
			return ws, nil
		case !ns.Kind.IsProxy():
			err := errors.New("service must be a sidecar proxy or gateway")
			logger.Error(err.Error())
			return nil, err
		}

		_, ns, err = configentry.MergeNodeServiceWithCentralConfig(ws, store, ns, logger)
		if err != nil {
			logger.Error("failed to merge with central config", "error", err.Error())
			return nil, err
		}

		if err = m.Manager.Register(proxyID, ns, source, proxyID.Token, false); err != nil {
			logger.Error("failed to register service", "error", err.Error())
			return nil, err
		}

		return ws, nil
	}

	syncLoop := func(ws memdb.WatchSet) {
		// Cancel the context on return to clean up the goroutine started by WatchCh.
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		for {
			select {
			case <-ws.WatchCh(ctx):
				// Something changed, unblock and re-run the query.
				//
				// It is expected that all other branches of this select will return and
				// cancel the context given to WatchCh (to clean up its goroutine).
			case doneCh := <-closeCh:
				// All watchers of this service (xDS streams) have gone away, so it's time
				// to free its resources.
				//
				// TODO(agentless): we should probably wait for a short grace period before
				// de-registering the service to allow clients to reconnect after a network
				// blip.
				logger.Trace("de-registering service with proxycfg manager because all watchers have gone away")
				m.Manager.Deregister(proxyID, source)
				close(doneCh)
				return
			case <-m.shutdownCh:
				// Manager is shutting down, stop the goroutine.
				return
			}

			var err error
			ws, err = fetchAndRegister()
			if err != nil {
				return
			}
		}
	}

	ws, err := fetchAndRegister()
	if err != nil {
		// Currently, only the first attempt's error is returned to the xDS server,
		// which terminates the stream immediately.
		//
		// We don't (yet) have a way to surface subsequent errors to the xDS server.
		//
		// We could wrap ConfigSnapshot in a sum type (i.e. a struct that contains
		// either a snapshot or an error) but given the relative unlikelihood of a
		// query that succeeds once failing in the future, it doesn't seem worth it.
		//
		// Instead, we log the error and leave any watchers hanging. Perhaps another
		// solution would be to close any watch channels when de-registering a service?
		return err
	}
	go syncLoop(ws)

	return nil
}

// cleanup decrements the watchers counter for the given proxy, and if it has
// reached zero, stops the sync goroutine (and de-registers the service).
func (m *ConfigSource) cleanup(id proxycfg.ProxyID) {
	m.mu.Lock()
	defer m.mu.Unlock()

	h := m.watches[id]
	h.numWatchers--

	if h.numWatchers == 0 {
		// We wait for doneCh to be closed by the sync goroutine, so that the lock is
		// held until after the service is de-registered - this prevents a potential
		// race where another sync goroutine is started for the service and we undo
		// its call to register the service.
		//
		// This cannot deadlock because closeCh is unbuffered. Sending will only
		// succeed if the sync goroutine is ready to receive (which always closes
		// doneCh).
		doneCh := make(chan struct{})
		select {
		case h.closeCh <- doneCh:
			<-doneCh
		case <-m.shutdownCh:
			// ConfigSource is shutting down, so the goroutine will be stopped anyway.
		}

		delete(m.watches, id)
	}
}

type Config struct {
	// NodeName is the name of the local agent node.
	NodeName string

	// Manager is the proxycfg Manager with which proxy services will be registered.
	Manager ConfigManager

	// State is the agent's local state that will be used to check if a proxy is
	// registered locally.
	LocalState *local.State

	// LocalConfigSource is used to configure proxies registered in the agent's
	// local state.
	LocalConfigSource Watcher

	// GetStore is used to access the server's state store.
	GetStore func() Store

	// Logger will be used to write log messages.
	Logger hclog.Logger

	// SessionLimiter is used to enforce xDS concurrency limits.
	SessionLimiter SessionLimiter
}

//go:generate mockery --name ConfigManager --inpackage
type ConfigManager interface {
	Watch(req proxycfg.ProxyID) (<-chan *proxycfg.ConfigSnapshot, proxycfg.CancelFunc)
	Register(proxyID proxycfg.ProxyID, service *structs.NodeService, source proxycfg.ProxySource, token string, overwrite bool) error
	Deregister(proxyID proxycfg.ProxyID, source proxycfg.ProxySource)
}

type Store interface {
	NodeService(ws memdb.WatchSet, nodeName string, serviceID string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, *structs.NodeService, error)
	ReadResolvedServiceConfigEntries(ws memdb.WatchSet, serviceName string, entMeta *acl.EnterpriseMeta, upstreamIDs []structs.ServiceID, proxyMode structs.ProxyMode) (uint64, *configentry.ResolvedServiceConfigSet, error)
	AbandonCh() <-chan struct{}
}

//go:generate mockery --name Watcher --inpackage
type Watcher interface {
	Watch(proxyID structs.ServiceID, nodeName string, token string) (<-chan *proxycfg.ConfigSnapshot, limiter.SessionTerminatedChan, proxycfg.CancelFunc, error)
}

//go:generate mockery --name SessionLimiter --inpackage
type SessionLimiter interface {
	BeginSession() (limiter.Session, error)
}