424 lines
13 KiB
Go
424 lines
13 KiB
Go
package consul
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"math/rand"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/hashicorp/consul/agent/consul/state"
|
|
"github.com/hashicorp/consul/agent/structs"
|
|
"github.com/hashicorp/consul/api"
|
|
"github.com/hashicorp/consul/ipaddr"
|
|
"github.com/hashicorp/consul/lib/stringslice"
|
|
"github.com/hashicorp/consul/logging"
|
|
"github.com/hashicorp/go-hclog"
|
|
memdb "github.com/hashicorp/go-memdb"
|
|
)
|
|
|
|
// GatewayLocator assists in selecting an appropriate mesh gateway when wan
|
|
// federation via mesh gateways is enabled.
|
|
//
|
|
// This is exclusively used by the consul server itself when it needs to tunnel
|
|
// RPC or gossip through a mesh gateway to reach its ultimate destination.
|
|
//
|
|
// During secondary datacenter bootstrapping there is a phase where it is
|
|
// impossible for mesh gateways in the secondary datacenter to register
|
|
// themselves into the catalog to be discovered by the servers, so the servers
|
|
// maintain references for the mesh gateways in the primary in addition to its
|
|
// own local mesh gateways.
|
|
//
|
|
// After initial datacenter federation the primary mesh gateways are only used
|
|
// in extreme fallback situations (basically re-bootstrapping).
|
|
//
|
|
// For all other operations a consul server will ALWAYS contact a local mesh
|
|
// gateway to ultimately forward the request through a remote mesh gateway to
|
|
// reach its destination.
|
|
type GatewayLocator struct {
|
|
logger hclog.Logger
|
|
srv serverDelegate
|
|
datacenter string // THIS dc
|
|
primaryDatacenter string
|
|
|
|
// these ONLY contain ones that have the wanfed:1 meta
|
|
gatewaysLock sync.Mutex
|
|
primaryGateways []string // WAN addrs
|
|
localGateways []string // LAN addrs
|
|
|
|
// primaryMeshGatewayDiscoveredAddresses is the current fallback addresses
|
|
// for the mesh gateways in the primary datacenter.
|
|
primaryMeshGatewayDiscoveredAddresses []string
|
|
primaryMeshGatewayDiscoveredAddressesLock sync.Mutex
|
|
|
|
// This will be closed the FIRST time we get some gateways populated
|
|
primaryGatewaysReadyCh chan struct{}
|
|
primaryGatewaysReadyOnce sync.Once
|
|
|
|
// these are a collection of measurements that factor into deciding if we
|
|
// should directly dial the primary's mesh gateways or if we should try to
|
|
// route through our local gateways (if they are up).
|
|
lastReplLock sync.Mutex
|
|
lastReplSuccess time.Time
|
|
lastReplFailure time.Time
|
|
lastReplSuccesses uint64
|
|
lastReplFailures uint64
|
|
}
|
|
|
|
// SetLastFederationStateReplicationError is used to indicate if the federation
|
|
// state replication loop has succeeded (nil) or failed during the last
|
|
// execution.
|
|
//
|
|
// Rather than introduce a completely new mechanism to periodically probe that
|
|
// our chosen mesh-gateway configuration can reach the primary's servers (like
|
|
// a ping or status RPC) we cheat and use the federation state replicator
|
|
// goroutine's success or failure as a proxy.
|
|
func (g *GatewayLocator) SetLastFederationStateReplicationError(err error) {
|
|
g.lastReplLock.Lock()
|
|
defer g.lastReplLock.Unlock()
|
|
oldChoice := g.dialPrimaryThroughLocalGateway()
|
|
if err == nil {
|
|
g.lastReplSuccess = time.Now().UTC()
|
|
g.lastReplSuccesses++
|
|
g.lastReplFailures = 0
|
|
} else {
|
|
g.lastReplFailure = time.Now().UTC()
|
|
g.lastReplFailures++
|
|
g.lastReplSuccesses = 0
|
|
}
|
|
newChoice := g.dialPrimaryThroughLocalGateway()
|
|
if oldChoice != newChoice {
|
|
g.logPrimaryDialingMessage(newChoice)
|
|
}
|
|
}
|
|
|
|
func (g *GatewayLocator) logPrimaryDialingMessage(useLocal bool) {
|
|
if g.datacenter == g.primaryDatacenter {
|
|
// These messages are useless when the server is in the primary
|
|
// datacenter.
|
|
return
|
|
}
|
|
if useLocal {
|
|
g.logger.Info("will dial the primary datacenter using our local mesh gateways if possible")
|
|
} else {
|
|
g.logger.Info("will dial the primary datacenter through its mesh gateways")
|
|
}
|
|
}
|
|
|
|
// DialPrimaryThroughLocalGateway determines if we should dial the primary's
|
|
// mesh gateways directly or use our local mesh gateways (if they are up).
|
|
//
|
|
// Generally the system has three states:
|
|
//
|
|
// 1. Servers dial primary MGWs using fallback addresses from the agent config.
|
|
// 2. Servers dial primary MGWs using replicated federation state data.
|
|
// 3. Servers dial primary MGWs indirectly through local MGWs.
|
|
//
|
|
// After initial bootstrapping most communication should go through (3). If the
|
|
// local mesh gateways are not coming up for chicken/egg problems (mostly the
|
|
// kind that arise from secondary datacenter bootstrapping) then (2) is useful
|
|
// to solve the chicken/egg problem and get back to (3). In the worst case
|
|
// where we completely lost communication with the primary AND all of their old
|
|
// mesh gateway addresses are changed then we need to go all the way back to
|
|
// square one and re-bootstrap via (1).
|
|
//
|
|
// Since both (1) and (2) are meant to be temporary we simplify things and make
|
|
// the system only consider two overall configurations: (1+2, with the
|
|
// addresses being unioned) or (3).
|
|
//
|
|
// This method returns true if in state (3) and false if in state (1+2).
|
|
func (g *GatewayLocator) DialPrimaryThroughLocalGateway() bool {
|
|
if g.datacenter == g.primaryDatacenter {
|
|
return false // not important
|
|
}
|
|
g.lastReplLock.Lock()
|
|
defer g.lastReplLock.Unlock()
|
|
return g.dialPrimaryThroughLocalGateway()
|
|
}
|
|
|
|
const localFederationStateReplicatorFailuresBeforeDialingDirectly = 3
|
|
|
|
func (g *GatewayLocator) dialPrimaryThroughLocalGateway() bool {
|
|
if g.lastReplSuccess.IsZero() && g.lastReplFailure.IsZero() {
|
|
return false // no data yet
|
|
}
|
|
|
|
if g.lastReplSuccess.After(g.lastReplFailure) {
|
|
return true // we have viable data
|
|
}
|
|
|
|
if g.lastReplFailures < localFederationStateReplicatorFailuresBeforeDialingDirectly {
|
|
return true // maybe it's just a little broken
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// PrimaryMeshGatewayAddressesReadyCh returns a channel that will be closed
|
|
// when federation state replication ships back at least one primary mesh
|
|
// gateway (not via fallback config).
|
|
func (g *GatewayLocator) PrimaryMeshGatewayAddressesReadyCh() <-chan struct{} {
|
|
return g.primaryGatewaysReadyCh
|
|
}
|
|
|
|
// PickGateway returns the address for a gateway suitable for reaching the
|
|
// provided datacenter.
|
|
func (g *GatewayLocator) PickGateway(dc string) string {
|
|
item := g.pickGateway(dc == g.primaryDatacenter)
|
|
g.logger.Trace("picking gateway for transit", "gateway", item, "source_datacenter", g.datacenter, "dest_datacenter", dc)
|
|
return item
|
|
}
|
|
|
|
func (g *GatewayLocator) pickGateway(primary bool) string {
|
|
addrs := g.listGateways(primary)
|
|
return getRandomItem(addrs)
|
|
}
|
|
|
|
func (g *GatewayLocator) listGateways(primary bool) []string {
|
|
g.gatewaysLock.Lock()
|
|
defer g.gatewaysLock.Unlock()
|
|
|
|
var addrs []string
|
|
if primary {
|
|
if g.datacenter == g.primaryDatacenter {
|
|
addrs = g.primaryGateways
|
|
} else if g.DialPrimaryThroughLocalGateway() && len(g.localGateways) > 0 {
|
|
addrs = g.localGateways
|
|
} else {
|
|
// Note calling StringSliceMergeSorted only works because both
|
|
// inputs are pre-sorted. If for some reason one of the lists has
|
|
// *duplicates* (which shouldn't happen) it's not great but it
|
|
// won't break anything other than biasing our eventual random
|
|
// choice a little bit.
|
|
addrs = stringslice.MergeSorted(g.primaryGateways, g.PrimaryGatewayFallbackAddresses())
|
|
}
|
|
} else {
|
|
addrs = g.localGateways
|
|
}
|
|
|
|
return addrs
|
|
}
|
|
|
|
// RefreshPrimaryGatewayFallbackAddresses is used to update the list of current
|
|
// fallback addresses for locating mesh gateways in the primary datacenter.
|
|
func (g *GatewayLocator) RefreshPrimaryGatewayFallbackAddresses(addrs []string) {
|
|
sort.Strings(addrs)
|
|
|
|
g.primaryMeshGatewayDiscoveredAddressesLock.Lock()
|
|
defer g.primaryMeshGatewayDiscoveredAddressesLock.Unlock()
|
|
|
|
if !stringslice.Equal(addrs, g.primaryMeshGatewayDiscoveredAddresses) {
|
|
g.primaryMeshGatewayDiscoveredAddresses = addrs
|
|
g.logger.Info("updated fallback list of primary mesh gateways", "mesh_gateways", addrs)
|
|
}
|
|
}
|
|
|
|
// PrimaryGatewayFallbackAddresses returns the current set of discovered
|
|
// fallback addresses for the mesh gateways in the primary datacenter.
|
|
func (g *GatewayLocator) PrimaryGatewayFallbackAddresses() []string {
|
|
g.primaryMeshGatewayDiscoveredAddressesLock.Lock()
|
|
defer g.primaryMeshGatewayDiscoveredAddressesLock.Unlock()
|
|
|
|
out := make([]string, len(g.primaryMeshGatewayDiscoveredAddresses))
|
|
copy(out, g.primaryMeshGatewayDiscoveredAddresses)
|
|
return out
|
|
}
|
|
|
|
func getRandomItem(items []string) string {
|
|
switch len(items) {
|
|
case 0:
|
|
return ""
|
|
case 1:
|
|
return items[0]
|
|
default:
|
|
idx := int(rand.Int31n(int32(len(items))))
|
|
return items[idx]
|
|
}
|
|
}
|
|
|
|
type serverDelegate interface {
|
|
blockingQuery(queryOpts structs.QueryOptionsCompat, queryMeta structs.QueryMetaCompat, fn queryFn) error
|
|
IsLeader() bool
|
|
LeaderLastContact() time.Time
|
|
}
|
|
|
|
func NewGatewayLocator(
|
|
logger hclog.Logger,
|
|
srv serverDelegate,
|
|
datacenter string,
|
|
primaryDatacenter string,
|
|
) *GatewayLocator {
|
|
g := &GatewayLocator{
|
|
logger: logger.Named(logging.GatewayLocator),
|
|
srv: srv,
|
|
datacenter: datacenter,
|
|
primaryDatacenter: primaryDatacenter,
|
|
primaryGatewaysReadyCh: make(chan struct{}),
|
|
}
|
|
g.logPrimaryDialingMessage(g.DialPrimaryThroughLocalGateway())
|
|
return g
|
|
}
|
|
|
|
var errGatewayLocalStateNotInitialized = errors.New("local state not initialized")
|
|
|
|
func (g *GatewayLocator) Run(ctx context.Context) {
|
|
var lastFetchIndex uint64
|
|
retryLoopBackoff(ctx, func() error {
|
|
idx, err := g.runOnce(lastFetchIndex)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
lastFetchIndex = idx
|
|
|
|
return nil
|
|
}, func(err error) {
|
|
if !errors.Is(err, errGatewayLocalStateNotInitialized) {
|
|
g.logger.Error("error tracking primary and local mesh gateways", "error", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func (g *GatewayLocator) runOnce(lastFetchIndex uint64) (uint64, error) {
|
|
if err := g.checkLocalStateIsReady(); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
// NOTE: we can't do RPC here because we won't have a token so we'll just
|
|
// mostly assume that our FSM is caught up enough to answer locally. If
|
|
// this has drifted it's no different than a cache that drifts or an
|
|
// inconsistent read.
|
|
queryOpts := &structs.QueryOptions{
|
|
MinQueryIndex: lastFetchIndex,
|
|
RequireConsistent: false,
|
|
}
|
|
|
|
var (
|
|
results []*structs.FederationState
|
|
queryMeta structs.QueryMeta
|
|
)
|
|
err := g.srv.blockingQuery(
|
|
queryOpts,
|
|
&queryMeta,
|
|
func(ws memdb.WatchSet, state *state.Store) error {
|
|
// Get the existing stored version of this config that has replicated down.
|
|
// We could phone home to get this but that would incur extra WAN traffic
|
|
// when we already have enough information locally to figure it out
|
|
// (assuming that our replicator is still functioning).
|
|
idx, all, err := state.FederationStateList(ws)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
queryMeta.Index = idx
|
|
results = all
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
g.updateFromState(results)
|
|
|
|
return queryMeta.Index, nil
|
|
}
|
|
|
|
// checkLocalStateIsReady is inlined a bit from (*Server).ForwardRPC(). We need to
|
|
// wait until our own state machine is safe to read from.
|
|
func (g *GatewayLocator) checkLocalStateIsReady() error {
|
|
// Check if we can allow a stale read, ensure our local DB is initialized
|
|
if !g.srv.LeaderLastContact().IsZero() {
|
|
return nil // the raft leader talked to us
|
|
}
|
|
|
|
if g.srv.IsLeader() {
|
|
return nil // we are the leader
|
|
}
|
|
|
|
return errGatewayLocalStateNotInitialized
|
|
}
|
|
|
|
func (g *GatewayLocator) updateFromState(results []*structs.FederationState) {
|
|
var (
|
|
local structs.CheckServiceNodes
|
|
primary structs.CheckServiceNodes
|
|
)
|
|
for _, config := range results {
|
|
retained := retainGateways(config.MeshGateways)
|
|
if config.Datacenter == g.datacenter {
|
|
local = retained
|
|
}
|
|
// NOT else-if because conditionals are not mutually exclusive
|
|
if config.Datacenter == g.primaryDatacenter {
|
|
primary = retained
|
|
}
|
|
}
|
|
|
|
primaryAddrs := renderGatewayAddrs(primary, true)
|
|
localAddrs := renderGatewayAddrs(local, false)
|
|
|
|
g.gatewaysLock.Lock()
|
|
defer g.gatewaysLock.Unlock()
|
|
|
|
changed := false
|
|
primaryReady := false
|
|
if !stringslice.Equal(g.primaryGateways, primaryAddrs) {
|
|
g.primaryGateways = primaryAddrs
|
|
primaryReady = len(g.primaryGateways) > 0
|
|
changed = true
|
|
}
|
|
if !stringslice.Equal(g.localGateways, localAddrs) {
|
|
g.localGateways = localAddrs
|
|
changed = true
|
|
}
|
|
|
|
if changed {
|
|
g.logger.Info(
|
|
"new cached locations of mesh gateways",
|
|
"primary", primaryAddrs,
|
|
"local", localAddrs,
|
|
)
|
|
}
|
|
|
|
if primaryReady {
|
|
g.primaryGatewaysReadyOnce.Do(func() {
|
|
close(g.primaryGatewaysReadyCh)
|
|
})
|
|
}
|
|
}
|
|
|
|
func retainGateways(full structs.CheckServiceNodes) structs.CheckServiceNodes {
|
|
out := make([]structs.CheckServiceNode, 0, len(full))
|
|
for _, csn := range full {
|
|
if csn.Service.Meta[structs.MetaWANFederationKey] != "1" {
|
|
continue
|
|
}
|
|
|
|
// only keep healthy ones
|
|
ok := true
|
|
for _, chk := range csn.Checks {
|
|
if chk.Status == api.HealthCritical {
|
|
ok = false
|
|
}
|
|
}
|
|
|
|
if ok {
|
|
out = append(out, csn)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func renderGatewayAddrs(gateways structs.CheckServiceNodes, wan bool) []string {
|
|
out := make([]string, 0, len(gateways))
|
|
for _, csn := range gateways {
|
|
addr, port := csn.BestAddress(wan)
|
|
completeAddr := ipaddr.FormatAddressPort(addr, port)
|
|
out = append(out, completeAddr)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|