open-consul/agent/sidecar_service.go

229 lines
8.5 KiB
Go
Raw Normal View History

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package agent
import (
"fmt"
"strings"
"time"
"github.com/hashicorp/consul/ipaddr"
"github.com/hashicorp/consul/agent/structs"
)
const sidecarIDSuffix = "-sidecar-proxy"
func sidecarIDFromServiceID(serviceID string) string {
return serviceID + sidecarIDSuffix
}
// reverses the sidecarIDFromServiceID operation
func serviceIDFromSidecarID(sidecarID string) string {
return strings.TrimSuffix(sidecarID, sidecarIDSuffix)
}
// sidecarServiceFromNodeService returns a *structs.NodeService representing a
// sidecar service with all defaults populated based on the current agent
// config.
//
// It assumes the ns has been validated already which means the nested
// SidecarService is also already validated. It also assumes that any check
// definitions within the sidecar service definition have been validated if
// necessary. If no sidecar service is defined in ns, then nil is returned with
// nil error.
//
// The second return argument is a list of CheckTypes to register along with the
// service.
//
// The third return argument is the effective Token to use for the sidecar
// registration. This will be the same as the token parameter passed unless the
// SidecarService definition contains a distinct one.
// TODO: return AddServiceRequest
func sidecarServiceFromNodeService(ns *structs.NodeService, token string) (*structs.NodeService, []*structs.CheckType, string, error) {
if ns.Connect.SidecarService == nil {
return nil, nil, "", nil
}
// for now at least these must be identical
ns.Connect.SidecarService.EnterpriseMeta = ns.EnterpriseMeta
// Start with normal conversion from service definition
sidecar := ns.Connect.SidecarService.NodeService()
// Override the ID which must always be consistent for a given outer service
// ID. We rely on this for lifecycle management of the nested definition.
sidecar.ID = sidecarIDFromServiceID(ns.ID)
// Set some meta we can use to disambiguate between service instances we added
// later and are responsible for deregistering.
if sidecar.Meta != nil {
// Meta is non-nil validate it before we add the special key so we can
// enforce that user cannot add a consul- prefix one.
wan federation via mesh gateways (#6884) This is like a Möbius strip of code due to the fact that low-level components (serf/memberlist) are connected to high-level components (the catalog and mesh-gateways) in a twisty maze of references which make it hard to dive into. With that in mind here's a high level summary of what you'll find in the patch: There are several distinct chunks of code that are affected: * new flags and config options for the server * retry join WAN is slightly different * retry join code is shared to discover primary mesh gateways from secondary datacenters * because retry join logic runs in the *agent* and the results of that operation for primary mesh gateways are needed in the *server* there are some methods like `RefreshPrimaryGatewayFallbackAddresses` that must occur at multiple layers of abstraction just to pass the data down to the right layer. * new cache type `FederationStateListMeshGatewaysName` for use in `proxycfg/xds` layers * the function signature for RPC dialing picked up a new required field (the node name of the destination) * several new RPCs for manipulating a FederationState object: `FederationState:{Apply,Get,List,ListMeshGateways}` * 3 read-only internal APIs for debugging use to invoke those RPCs from curl * raft and fsm changes to persist these FederationStates * replication for FederationStates as they are canonically stored in the Primary and replicated to the Secondaries. * a special derivative of anti-entropy that runs in secondaries to snapshot their local mesh gateway `CheckServiceNodes` and sync them into their upstream FederationState in the primary (this works in conjunction with the replication to distribute addresses for all mesh gateways in all DCs to all other DCs) * a "gateway locator" convenience object to make use of this data to choose the addresses of gateways to use for any given RPC or gossip operation to a remote DC. This gets data from the "retry join" logic in the agent and also directly calls into the FSM. * RPC (`:8300`) on the server sniffs the first byte of a new connection to determine if it's actually doing native TLS. If so it checks the ALPN header for protocol determination (just like how the existing system uses the type-byte marker). * 2 new kinds of protocols are exclusively decoded via this native TLS mechanism: one for ferrying "packet" operations (udp-like) from the gossip layer and one for "stream" operations (tcp-like). The packet operations re-use sockets (using length-prefixing) to cut down on TLS re-negotiation overhead. * the server instances specially wrap the `memberlist.NetTransport` when running with gateway federation enabled (in a `wanfed.Transport`). The general gist is that if it tries to dial a node in the SAME datacenter (deduced by looking at the suffix of the node name) there is no change. If dialing a DIFFERENT datacenter it is wrapped up in a TLS+ALPN blob and sent through some mesh gateways to eventually end up in a server's :8300 port. * a new flag when launching a mesh gateway via `consul connect envoy` to indicate that the servers are to be exposed. This sets a special service meta when registering the gateway into the catalog. * `proxycfg/xds` notice this metadata blob to activate additional watches for the FederationState objects as well as the location of all of the consul servers in that datacenter. * `xds:` if the extra metadata is in place additional clusters are defined in a DC to bulk sink all traffic to another DC's gateways. For the current datacenter we listen on a wildcard name (`server.<dc>.consul`) that load balances all servers as well as one mini-cluster per node (`<node>.server.<dc>.consul`) * the `consul tls cert create` command got a new flag (`-node`) to help create an additional SAN in certs that can be used with this flavor of federation.
2020-03-09 20:59:02 +00:00
if err := structs.ValidateServiceMetadata(sidecar.Kind, sidecar.Meta, false); err != nil {
return nil, nil, "", err
}
}
// Copy the service metadata from the original service if no other meta was provided
if len(sidecar.Meta) == 0 && len(ns.Meta) > 0 {
if sidecar.Meta == nil {
sidecar.Meta = make(map[string]string)
}
for k, v := range ns.Meta {
sidecar.Meta[k] = v
}
}
// Copy the tags from the original service if no other tags were specified
if len(sidecar.Tags) == 0 && len(ns.Tags) > 0 {
sidecar.Tags = append(sidecar.Tags, ns.Tags...)
}
// Flag this as a sidecar - this is not persisted in catalog but only needed
// in local agent state to disambiguate lineage when deregistering the parent
// service later.
sidecar.LocallyRegisteredAsSidecar = true
// See if there is a more specific token for the sidecar registration
if ns.Connect.SidecarService.Token != "" {
token = ns.Connect.SidecarService.Token
}
// Setup some reasonable connect proxy defaults.
if sidecar.Kind == "" {
sidecar.Kind = structs.ServiceKindConnectProxy
}
if sidecar.Service == "" {
sidecar.Service = ns.Service + "-sidecar-proxy"
}
if sidecar.Address == "" {
// Inherit address from the service if it's provided
sidecar.Address = ns.Address
}
// Proxy defaults
if sidecar.Proxy.DestinationServiceName == "" {
sidecar.Proxy.DestinationServiceName = ns.Service
}
if sidecar.Proxy.DestinationServiceID == "" {
sidecar.Proxy.DestinationServiceID = ns.ID
}
// Fill defaults from NodeService if none of the address components are present.
// This really argues for a refactoring to a more generalized 'address' concept.
if sidecar.Proxy.LocalServiceSocketPath == "" && (sidecar.Proxy.LocalServiceAddress == "" || sidecar.Proxy.LocalServicePort < 1) {
if ns.SocketPath != "" {
sidecar.Proxy.LocalServiceSocketPath = ns.SocketPath
} else {
if sidecar.Proxy.LocalServiceAddress == "" {
sidecar.Proxy.LocalServiceAddress = "127.0.0.1"
}
if sidecar.Proxy.LocalServicePort < 1 {
sidecar.Proxy.LocalServicePort = ns.Port
}
}
}
// Setup checks
checks, err := ns.Connect.SidecarService.CheckTypes()
if err != nil {
return nil, nil, "", err
}
return sidecar, checks, token, nil
}
// sidecarPortFromServiceIDLocked is used to allocate a unique port for a sidecar proxy.
// This is called immediately before registration to avoid value collisions. This function assumes the state lock is already held.
func (a *Agent) sidecarPortFromServiceIDLocked(sidecarCompoundServiceID structs.ServiceID) (int, error) {
sidecarPort := 0
// Allocate port if needed (min and max inclusive).
rangeLen := a.config.ConnectSidecarMaxPort - a.config.ConnectSidecarMinPort + 1
if sidecarPort < 1 && a.config.ConnectSidecarMinPort > 0 && rangeLen > 0 {
// This did pick at random which was simpler but consul reload would assign
// new ports to all the sidecars since it unloads all state and
// re-populates. It also made this more difficult to test (have to pin the
// range to one etc.). Instead we assign sequentially, but rather than N^2
// lookups, just iterated services once and find the set of used ports in
// allocation range. We could maintain this state permanently in agent but
// it doesn't seem to be necessary - even with thousands of services this is
// not expensive to compute.
usedPorts := make(map[int]struct{})
for _, otherNS := range a.State.AllServices() {
// Check if other port is in auto-assign range
if otherNS.Port >= a.config.ConnectSidecarMinPort &&
otherNS.Port <= a.config.ConnectSidecarMaxPort {
if otherNS.CompoundServiceID() == sidecarCompoundServiceID {
// This sidecar is already registered with an auto-port and is just
// being updated so pick the same port as before rather than allocate
// a new one.
sidecarPort = otherNS.Port
break
}
usedPorts[otherNS.Port] = struct{}{}
}
// Note that the proxy might already be registered with a port that was
// not in the auto range or the auto range has moved. In either case we
// want to allocate a new one so it's no different from ignoring that it
// already exists as we do now.
}
// Check we still need to assign a port and didn't find we already had one
// allocated.
if sidecarPort < 1 {
// Iterate until we find lowest unused port
for p := a.config.ConnectSidecarMinPort; p <= a.config.ConnectSidecarMaxPort; p++ {
_, used := usedPorts[p]
if !used {
sidecarPort = p
break
}
}
}
}
// If no ports left (or auto ports disabled) fail
if sidecarPort < 1 {
// If ports are set to zero explicitly, config builder switches them to
// `-1`. In this case don't show the actual values since we don't know what
// was actually in config (zero or negative) and it might be confusing, we
// just know they explicitly disabled auto assignment.
if a.config.ConnectSidecarMinPort < 1 || a.config.ConnectSidecarMaxPort < 1 {
return 0, fmt.Errorf("no port provided for sidecar_service " +
"and auto-assignment disabled in config")
}
return 0, fmt.Errorf("no port provided for sidecar_service and none "+
"left in the configured range [%d, %d]", a.config.ConnectSidecarMinPort,
a.config.ConnectSidecarMaxPort)
}
return sidecarPort, nil
}
func sidecarDefaultChecks(sidecarID string, sidecarAddress string, proxyServiceAddress string, port int) []*structs.CheckType {
// The check should use the sidecar's address because it makes a request to the sidecar.
// If the sidecar's address is empty, we fall back to the address of the local service, as set in
// sidecar.Proxy.LocalServiceAddress, in the hope that the proxy is also accessible on that address
// (which in most cases it is because it's running as a sidecar in the same network).
// We could instead fall back to the address of the service as set by (ns.Address), but I've kept it using
// sidecar.Proxy.LocalServiceAddress so as to not change things too much in the
// process of fixing #14433.
checkAddress := sidecarAddress
if checkAddress == "" {
checkAddress = proxyServiceAddress
}
serviceID := serviceIDFromSidecarID(sidecarID)
return []*structs.CheckType{
{
Name: "Connect Sidecar Listening",
TCP: ipaddr.FormatAddressPort(checkAddress, port),
Interval: 10 * time.Second,
},
{
Name: "Connect Sidecar Aliasing " + serviceID,
AliasService: serviceID,
},
}
}