295 lines
8.5 KiB
Go
295 lines
8.5 KiB
Go
package xds
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
|
|
envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2"
|
|
envoycore "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
|
|
envoyendpoint "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
|
|
"github.com/gogo/protobuf/proto"
|
|
|
|
"github.com/hashicorp/consul/agent/proxycfg"
|
|
"github.com/hashicorp/consul/agent/structs"
|
|
"github.com/hashicorp/consul/api"
|
|
|
|
bexpr "github.com/hashicorp/go-bexpr"
|
|
)
|
|
|
|
// endpointsFromSnapshot returns the xDS API representation of the "endpoints"
|
|
func (s *Server) endpointsFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) {
|
|
if cfgSnap == nil {
|
|
return nil, errors.New("nil config given")
|
|
}
|
|
|
|
switch cfgSnap.Kind {
|
|
case structs.ServiceKindConnectProxy:
|
|
return s.endpointsFromSnapshotConnectProxy(cfgSnap, token)
|
|
case structs.ServiceKindMeshGateway:
|
|
return s.endpointsFromSnapshotMeshGateway(cfgSnap, token)
|
|
default:
|
|
return nil, fmt.Errorf("Invalid service kind: %v", cfgSnap.Kind)
|
|
}
|
|
}
|
|
|
|
// endpointsFromSnapshotConnectProxy returns the xDS API representation of the "endpoints"
|
|
// (upstream instances) in the snapshot.
|
|
func (s *Server) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) {
|
|
// TODO(rb): this sizing is a low bound.
|
|
resources := make([]proto.Message, 0, len(cfgSnap.ConnectProxy.UpstreamEndpoints))
|
|
|
|
// TODO(rb): should naming from 1.5 -> 1.6 for clusters remain unchanged?
|
|
|
|
for _, u := range cfgSnap.Proxy.Upstreams {
|
|
id := u.Identifier()
|
|
|
|
var chain *structs.CompiledDiscoveryChain
|
|
if u.DestinationType != structs.UpstreamDestTypePreparedQuery {
|
|
chain = cfgSnap.ConnectProxy.DiscoveryChain[id]
|
|
}
|
|
|
|
if chain == nil {
|
|
// We ONLY want this branch for prepared queries.
|
|
|
|
clusterName := UpstreamSNI(&u, "", cfgSnap)
|
|
endpoints, ok := cfgSnap.ConnectProxy.UpstreamEndpoints[id]
|
|
if ok {
|
|
la := makeLoadAssignment(
|
|
clusterName,
|
|
0,
|
|
[]loadAssignmentEndpointGroup{
|
|
{Endpoints: endpoints},
|
|
},
|
|
cfgSnap.Datacenter,
|
|
)
|
|
resources = append(resources, la)
|
|
}
|
|
|
|
} else {
|
|
// Newfangled discovery chain plumbing.
|
|
|
|
chainEndpointMap, ok := cfgSnap.ConnectProxy.WatchedUpstreamEndpoints[id]
|
|
if !ok {
|
|
continue // skip the upstream (should not happen)
|
|
}
|
|
|
|
for target, node := range chain.GroupResolverNodes {
|
|
groupResolver := node.GroupResolver
|
|
failover := groupResolver.Failover
|
|
|
|
endpoints, ok := chainEndpointMap[target]
|
|
if !ok {
|
|
continue // skip the cluster (should not happen)
|
|
}
|
|
|
|
var (
|
|
endpointGroups []loadAssignmentEndpointGroup
|
|
overprovisioningFactor int
|
|
)
|
|
|
|
primaryGroup := loadAssignmentEndpointGroup{
|
|
Endpoints: endpoints,
|
|
OnlyPassing: chain.SubsetDefinitionForTarget(target).OnlyPassing,
|
|
}
|
|
|
|
if failover != nil && len(failover.Targets) > 0 {
|
|
endpointGroups = make([]loadAssignmentEndpointGroup, 0, len(failover.Targets)+1)
|
|
|
|
endpointGroups = append(endpointGroups, primaryGroup)
|
|
|
|
if failover.Definition.OverprovisioningFactor > 0 {
|
|
overprovisioningFactor = failover.Definition.OverprovisioningFactor
|
|
}
|
|
if overprovisioningFactor <= 0 {
|
|
// We choose such a large value here that the failover math should
|
|
// in effect not happen until zero instances are healthy.
|
|
overprovisioningFactor = 100000
|
|
}
|
|
|
|
for _, failTarget := range failover.Targets {
|
|
failEndpoints, ok := chainEndpointMap[failTarget]
|
|
if !ok {
|
|
continue // skip the failover target (should not happen)
|
|
}
|
|
|
|
endpointGroups = append(endpointGroups, loadAssignmentEndpointGroup{
|
|
Endpoints: failEndpoints,
|
|
OnlyPassing: chain.SubsetDefinitionForTarget(failTarget).OnlyPassing,
|
|
})
|
|
}
|
|
} else {
|
|
endpointGroups = append(endpointGroups, primaryGroup)
|
|
}
|
|
|
|
sni := TargetSNI(target, cfgSnap)
|
|
clusterName := CustomizeClusterName(sni, chain)
|
|
|
|
la := makeLoadAssignment(
|
|
clusterName,
|
|
overprovisioningFactor,
|
|
endpointGroups,
|
|
cfgSnap.Datacenter,
|
|
)
|
|
resources = append(resources, la)
|
|
}
|
|
}
|
|
}
|
|
|
|
return resources, nil
|
|
}
|
|
|
|
func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) {
|
|
resources := make([]proto.Message, 0, len(cfgSnap.MeshGateway.GatewayGroups)+len(cfgSnap.MeshGateway.ServiceGroups))
|
|
|
|
// generate the endpoints for the gateways in the remote datacenters
|
|
for dc, endpoints := range cfgSnap.MeshGateway.GatewayGroups {
|
|
clusterName := DatacenterSNI(dc, cfgSnap)
|
|
la := makeLoadAssignment(
|
|
clusterName,
|
|
0,
|
|
[]loadAssignmentEndpointGroup{
|
|
{Endpoints: endpoints},
|
|
},
|
|
cfgSnap.Datacenter,
|
|
)
|
|
resources = append(resources, la)
|
|
}
|
|
|
|
// generate the endpoints for the local service groups
|
|
for svc, endpoints := range cfgSnap.MeshGateway.ServiceGroups {
|
|
clusterName := ServiceSNI(svc, "", "default", cfgSnap.Datacenter, cfgSnap)
|
|
la := makeLoadAssignment(
|
|
clusterName,
|
|
0,
|
|
[]loadAssignmentEndpointGroup{
|
|
{Endpoints: endpoints},
|
|
},
|
|
cfgSnap.Datacenter,
|
|
)
|
|
resources = append(resources, la)
|
|
}
|
|
|
|
// generate the endpoints for the service subsets
|
|
for svc, resolver := range cfgSnap.MeshGateway.ServiceResolvers {
|
|
for subsetName, subset := range resolver.Subsets {
|
|
clusterName := ServiceSNI(svc, subsetName, "default", cfgSnap.Datacenter, cfgSnap)
|
|
|
|
endpoints := cfgSnap.MeshGateway.ServiceGroups[svc]
|
|
|
|
// locally execute the subsets filter
|
|
if subset.Filter != "" {
|
|
filter, err := bexpr.CreateFilter(subset.Filter, nil, endpoints)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
raw, err := filter.Execute(endpoints)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
endpoints = raw.(structs.CheckServiceNodes)
|
|
}
|
|
|
|
la := makeLoadAssignment(
|
|
clusterName,
|
|
0,
|
|
[]loadAssignmentEndpointGroup{
|
|
{
|
|
Endpoints: endpoints,
|
|
OnlyPassing: subset.OnlyPassing,
|
|
},
|
|
},
|
|
cfgSnap.Datacenter,
|
|
)
|
|
resources = append(resources, la)
|
|
}
|
|
}
|
|
|
|
return resources, nil
|
|
}
|
|
|
|
func makeEndpoint(clusterName, host string, port int) envoyendpoint.LbEndpoint {
|
|
return envoyendpoint.LbEndpoint{
|
|
HostIdentifier: &envoyendpoint.LbEndpoint_Endpoint{
|
|
Endpoint: &envoyendpoint.Endpoint{
|
|
Address: makeAddressPtr(host, port),
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
type loadAssignmentEndpointGroup struct {
|
|
Endpoints structs.CheckServiceNodes
|
|
OnlyPassing bool
|
|
}
|
|
|
|
func makeLoadAssignment(
|
|
clusterName string,
|
|
overprovisioningFactor int,
|
|
endpointGroups []loadAssignmentEndpointGroup,
|
|
localDatacenter string,
|
|
) *envoy.ClusterLoadAssignment {
|
|
cla := &envoy.ClusterLoadAssignment{
|
|
ClusterName: clusterName,
|
|
Endpoints: make([]envoyendpoint.LocalityLbEndpoints, 0, len(endpointGroups)),
|
|
}
|
|
if overprovisioningFactor > 0 {
|
|
cla.Policy = &envoy.ClusterLoadAssignment_Policy{
|
|
OverprovisioningFactor: makeUint32Value(overprovisioningFactor),
|
|
}
|
|
}
|
|
|
|
for priority, endpointGroup := range endpointGroups {
|
|
endpoints := endpointGroup.Endpoints
|
|
es := make([]envoyendpoint.LbEndpoint, 0, len(endpoints))
|
|
|
|
for _, ep := range endpoints {
|
|
// TODO (mesh-gateway) - should we respect the translate_wan_addrs configuration here or just always use the wan for cross-dc?
|
|
addr, port := ep.BestAddress(localDatacenter != ep.Node.Datacenter)
|
|
healthStatus := envoycore.HealthStatus_HEALTHY
|
|
weight := 1
|
|
if ep.Service.Weights != nil {
|
|
weight = ep.Service.Weights.Passing
|
|
}
|
|
|
|
for _, chk := range ep.Checks {
|
|
if chk.Status == api.HealthCritical {
|
|
healthStatus = envoycore.HealthStatus_UNHEALTHY
|
|
}
|
|
if endpointGroup.OnlyPassing && chk.Status != api.HealthPassing {
|
|
healthStatus = envoycore.HealthStatus_UNHEALTHY
|
|
}
|
|
if chk.Status == api.HealthWarning && ep.Service.Weights != nil {
|
|
weight = ep.Service.Weights.Warning
|
|
}
|
|
}
|
|
// Make weights fit Envoy's limits. A zero weight means that either Warning
|
|
// (likely) or Passing (weirdly) weight has been set to 0 effectively making
|
|
// this instance unhealthy and should not be sent traffic.
|
|
if weight < 1 {
|
|
healthStatus = envoycore.HealthStatus_UNHEALTHY
|
|
weight = 1
|
|
}
|
|
if weight > 128 {
|
|
weight = 128
|
|
}
|
|
es = append(es, envoyendpoint.LbEndpoint{
|
|
HostIdentifier: &envoyendpoint.LbEndpoint_Endpoint{
|
|
Endpoint: &envoyendpoint.Endpoint{
|
|
Address: makeAddressPtr(addr, port),
|
|
},
|
|
},
|
|
HealthStatus: healthStatus,
|
|
LoadBalancingWeight: makeUint32Value(weight),
|
|
})
|
|
}
|
|
|
|
cla.Endpoints = append(cla.Endpoints, envoyendpoint.LocalityLbEndpoints{
|
|
Priority: uint32(priority),
|
|
LbEndpoints: es,
|
|
})
|
|
}
|
|
|
|
return cla
|
|
}
|