troubleshoot: service to service validation (#16096)

* Add Tproxy support to Envoy Extensions (this is needed for service to service validation)

* Add validation for Envoy configuration for an upstream service

* Use both /config_dump and /cluster to validate Envoy configuration
This is because of a bug in Envoy where the EndpointsConfigDump does not
include a cluster_name, making it impossible to match an endpoint to
verify it exists.

This removes endpoints support for builtin extensions since only the
validate plugin was using it, and it is no longer used. It also removes
test cases for endpoint validation. Endpoints validation now only occurs
in the top level test from config_dump and clusters json files.

Co-authored-by: Eric <eric@haberkorn.co>
This commit is contained in:
Nitya Dhanushkodi 2023-01-27 09:43:16 -10:00 committed by GitHub
parent 7e3c6c92c4
commit f820bfe53a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 4486 additions and 46 deletions

View File

@ -116,28 +116,47 @@ func TestConfigSnapshotTransparentProxy(t testing.T) *ConfigSnapshot {
})
}
func TestConfigSnapshotTransparentProxyHTTPUpstream(t testing.T) *ConfigSnapshot {
func TestConfigSnapshotTransparentProxyHTTPUpstream(t testing.T, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot {
// Set default service protocol to HTTP
entries := append(additionalEntries, &structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
})
// DiscoveryChain without an UpstreamConfig should yield a
// filter chain when in transparent proxy mode
var (
google = structs.NewServiceName("google", nil)
googleUID = NewUpstreamIDFromServiceName(google)
googleChain = discoverychain.TestCompileConfigEntries(t, "google", "default", "default", "dc1", connect.TestClusterID+".consul", nil,
// Set default service protocol to HTTP
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
entries...,
)
noEndpoints = structs.NewServiceName("no-endpoints", nil)
noEndpointsUID = NewUpstreamIDFromServiceName(noEndpoints)
noEndpointsChain = discoverychain.TestCompileConfigEntries(t, "no-endpoints", "default", "default", "dc1", connect.TestClusterID+".consul", nil)
db = structs.NewServiceName("db", nil)
db = structs.NewServiceName("db", nil)
nodes = []structs.CheckServiceNode{
{
Node: &structs.Node{
Address: "8.8.8.8",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "google",
Address: "9.9.9.9",
Port: 9090,
TaggedAddresses: map[string]structs.ServiceAddress{
"virtual": {Address: "10.0.0.1"},
structs.TaggedAddressVirtualIP: {Address: "240.0.0.1"},
},
},
},
}
)
return TestConfigSnapshot(t, func(ns *structs.NodeService) {
@ -174,26 +193,22 @@ func TestConfigSnapshotTransparentProxyHTTPUpstream(t testing.T) *ConfigSnapshot
Chain: noEndpointsChain,
},
},
{
CorrelationID: "upstream-target:v1.google.default.default.dc1:" + googleUID.String(),
Result: &structs.IndexedCheckServiceNodes{
Nodes: nodes,
},
},
{
CorrelationID: "upstream-target:v2.google.default.default.dc1:" + googleUID.String(),
Result: &structs.IndexedCheckServiceNodes{
Nodes: nodes,
},
},
{
CorrelationID: "upstream-target:google.default.default.dc1:" + googleUID.String(),
Result: &structs.IndexedCheckServiceNodes{
Nodes: []structs.CheckServiceNode{
{
Node: &structs.Node{
Address: "8.8.8.8",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "google",
Address: "9.9.9.9",
Port: 9090,
TaggedAddresses: map[string]structs.ServiceAddress{
"virtual": {Address: "10.0.0.1"},
structs.TaggedAddressVirtualIP: {Address: "240.0.0.1"},
},
},
},
},
Nodes: nodes,
},
},
{

View File

@ -93,6 +93,14 @@ end`,
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil, makeLambdaServiceDefaults(false))
},
},
{
name: "lambda-connect-proxy-tproxy",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
extra := makeLambdaServiceDefaults(false)
extra.Name = "google"
return proxycfg.TestConfigSnapshotTransparentProxyHTTPUpstream(t, extra)
},
},
// Make sure that if the upstream type is different from ExtensionConfiguration.Kind is, that the resources are not patched.
{
name: "lambda-connect-proxy-with-terminating-gateway-upstream",

View File

@ -0,0 +1,295 @@
package validate
import (
"fmt"
envoy_admin_v3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
envoy_aggregate_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
envoy_resource_v3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3"
"github.com/hashicorp/go-multierror"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/hashicorp/consul/agent/xds/builtinextensiontemplate"
"github.com/hashicorp/consul/agent/xds/xdscommon"
)
const builtinValidateExtension = "builtin/proxy/validate"
// Validate contains input information about which proxy resources to validate and output information about resources it
// has validated.
type Validate struct {
// envoyID is an argument to the Validate plugin and identifies which listener to begin the validation with.
envoyID string
// snis is all of the upstream SNIs for this proxy. It is set via ExtensionConfiguration.
snis map[string]struct{}
// listener specifies if the service's listener has been seen.
listener bool
// usesRDS determines if the listener's outgoing filter uses RDS.
usesRDS bool
// listener specifies if the service's route has been seen.
route bool
// resources is a mapping from SNI to the expected resources
// for that SNI. It is populated based on the cluster names on routes
// (whether they are specified on listener filters or routes).
resources map[string]*resource
}
type resource struct {
// required determines if the resource is required for the given upstream.
required bool
// cluster specifies if the cluster has been seen.
cluster bool
// aggregateCluster determines if the resource is an aggregate cluster.
aggregateCluster bool
// aggregateClusterChildren is a list of SNIs to identify the child clusters of this aggregate cluster.
aggregateClusterChildren []string
// parentCluster is empty if this is a top level cluster, and has a value if this is a child of an aggregate
// cluster.
parentCluster string
// loadAssignment specifies if the load assignment has been seen.
loadAssignment bool
// usesEDS specifies if the cluster has EDS configured.
usesEDS bool
// The number of endpoints for the cluster or load assignment.
endpoints int
}
var _ builtinextensiontemplate.Plugin = (*Validate)(nil)
// EndpointValidator allows us to inject a different function for tests.
type EndpointValidator func(*resource, string, *envoy_admin_v3.Clusters)
// MakeValidate is a builtinextensiontemplate.PluginConstructor for a builtinextensiontemplate.EnvoyExtension.
func MakeValidate(ext xdscommon.ExtensionConfiguration) (builtinextensiontemplate.Plugin, error) {
var resultErr error
var plugin Validate
if name := ext.EnvoyExtension.Name; name != builtinValidateExtension {
return nil, fmt.Errorf("expected extension name 'builtin/proxy/validate' but got %q", name)
}
envoyID, _ := ext.EnvoyExtension.Arguments["envoyID"]
mainEnvoyID, _ := envoyID.(string)
if len(mainEnvoyID) == 0 {
return nil, fmt.Errorf("envoyID is required")
}
plugin.envoyID = mainEnvoyID
plugin.snis = ext.Upstreams[ext.ServiceName].SNI
plugin.resources = make(map[string]*resource)
return &plugin, resultErr
}
// Errors returns the error based only on Validate's state.
func (v *Validate) Errors(validateEndpoints bool, endpointValidator EndpointValidator, clusters *envoy_admin_v3.Clusters) error {
var resultErr error
if !v.listener {
resultErr = multierror.Append(resultErr, fmt.Errorf("no listener"))
}
if v.usesRDS && !v.route {
resultErr = multierror.Append(resultErr, fmt.Errorf("no route"))
}
numRequiredResources := 0
// Resources will be marked as required in PatchFilter or PatchRoute because the listener or route will determine
// which clusters/endpoints to validate.
for sni, resource := range v.resources {
if !resource.required {
continue
}
numRequiredResources += 1
_, ok := v.snis[sni]
if !ok || !resource.cluster {
resultErr = multierror.Append(resultErr, fmt.Errorf("no cluster for sni %s", sni))
continue
}
if validateEndpoints {
// If resource is a top-level cluster (any cluster that is an aggregate cluster or not a child of an aggregate
// cluster), it will have an empty parent. If resource is a child cluster, it will have a nonempty parent.
if resource.parentCluster == "" && resource.aggregateCluster {
// Aggregate cluster case: do endpoint verification by checking each child cluster. We need at least one
// child cluster to have healthy endpoints.
oneClusterHasEndpoints := false
for _, childCluster := range resource.aggregateClusterChildren {
endpointValidator(v.resources[childCluster], childCluster, clusters)
if v.resources[childCluster].endpoints > 0 {
oneClusterHasEndpoints = true
}
}
if !oneClusterHasEndpoints {
resultErr = multierror.Append(resultErr, fmt.Errorf("zero healthy endpoints for aggregate cluster %s", sni))
}
} else if resource.parentCluster == "" {
// Top-level non-aggregate cluster case: check for load assignment and healthy endpoints.
endpointValidator(resource, sni, clusters)
if resource.usesEDS && !resource.loadAssignment {
resultErr = multierror.Append(resultErr, fmt.Errorf("no cluster load assignment for cluster %s", sni))
}
if resource.endpoints == 0 {
resultErr = multierror.Append(resultErr, fmt.Errorf("zero healthy endpoints for cluster %s", sni))
}
} else {
// Child cluster case: skip, since it'll be verified by the parent aggregate cluster.
continue
}
}
}
if numRequiredResources == 0 {
resultErr = multierror.Append(resultErr, fmt.Errorf("no clusters found on route or listener"))
}
return resultErr
}
// DoEndpointValidation implements the EndpointVerifier function type.
func DoEndpointValidation(r *resource, sni string, clusters *envoy_admin_v3.Clusters) {
clusterStatuses := clusters.GetClusterStatuses()
if clusterStatuses == nil {
return
}
status := &envoy_admin_v3.ClusterStatus{}
r.loadAssignment = false
for _, s := range clusterStatuses {
if s.Name == sni {
status = s
r.loadAssignment = true
break
}
}
healthyEndpoints := 0
hostStatuses := status.GetHostStatuses()
if r.loadAssignment && hostStatuses != nil {
for _, h := range hostStatuses {
health := h.GetHealthStatus()
if health != nil {
if health.EdsHealthStatus == envoy_core_v3.HealthStatus_HEALTHY && health.FailedOutlierCheck == false {
healthyEndpoints += 1
}
}
}
}
r.endpoints = healthyEndpoints
}
// CanApply determines if the extension can apply to the given extension configuration.
func (p *Validate) CanApply(config xdscommon.ExtensionConfiguration) bool {
return true
}
func (p *Validate) PatchRoute(route *envoy_route_v3.RouteConfiguration) (*envoy_route_v3.RouteConfiguration, bool, error) {
// Route name on connect proxies will be the envoy ID. We are only validating routes for the specific upstream with
// the envoyID configured.
if route.Name != p.envoyID {
return route, false, nil
}
p.route = true
for sni := range builtinextensiontemplate.RouteClusterNames(route) {
if _, ok := p.resources[sni]; ok {
continue
}
p.resources[sni] = &resource{required: true}
}
return route, false, nil
}
func (p *Validate) PatchCluster(c *envoy_cluster_v3.Cluster) (*envoy_cluster_v3.Cluster, bool, error) {
v, ok := p.resources[c.Name]
if !ok {
v = &resource{}
p.resources[c.Name] = v
}
v.cluster = true
// If it's an aggregate cluster, add the child clusters to p.resources if they are not already there.
aggregateCluster, ok := isAggregateCluster(c)
if ok {
// Mark this as an aggregate cluster, so we know we do not need to validate its endpoints directly.
v.aggregateCluster = true
for _, clusterName := range aggregateCluster.Clusters {
r, ok := p.resources[clusterName]
if !ok {
r = &resource{}
p.resources[clusterName] = r
}
if v.aggregateClusterChildren == nil {
v.aggregateClusterChildren = []string{}
}
// On the parent cluster, add the children.
v.aggregateClusterChildren = append(v.aggregateClusterChildren, clusterName)
// On the child cluster, set the parent.
r.parentCluster = c.Name
// The child clusters of an aggregate cluster will be required if the parent cluster is.
r.required = v.required
}
return c, false, nil
}
if c.EdsClusterConfig != nil {
v.usesEDS = true
} else {
la := c.LoadAssignment
if la == nil {
return c, false, nil
}
v.endpoints = len(la.Endpoints) + len(la.NamedEndpoints)
}
return c, false, nil
}
func (p *Validate) PatchFilter(filter *envoy_listener_v3.Filter) (*envoy_listener_v3.Filter, bool, error) {
// If a single filter exists for a listener we say it exists.
p.listener = true
if config := envoy_resource_v3.GetHTTPConnectionManager(filter); config != nil {
// If the http filter uses RDS, then the clusters we need to validate exist in the route, and there's nothing
// else we need to do with the filter.
if config.GetRds() != nil {
p.usesRDS = true
return filter, true, nil
}
}
// FilterClusterNames handles the filter being an http or tcp filter.
for sni := range builtinextensiontemplate.FilterClusterNames(filter) {
// Mark any clusters we see as required resources.
if r, ok := p.resources[sni]; ok {
r.required = true
} else {
p.resources[sni] = &resource{required: true}
}
}
return filter, true, nil
}
func isAggregateCluster(c *envoy_cluster_v3.Cluster) (*envoy_aggregate_cluster_v3.ClusterConfig, bool) {
aggregateCluster := &envoy_aggregate_cluster_v3.ClusterConfig{}
cdt, ok := c.ClusterDiscoveryType.(*envoy_cluster_v3.Cluster_ClusterType)
if ok {
cct := cdt.ClusterType.TypedConfig
if cct != nil {
err := anypb.UnmarshalTo(cct, aggregateCluster, proto.UnmarshalOptions{})
if err == nil {
return aggregateCluster, true
}
}
}
return nil, false
}

View File

@ -0,0 +1,304 @@
package validate
import (
"testing"
envoy_admin_v3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_aggregate_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
"github.com/hashicorp/consul/agent/xds/xdscommon"
"github.com/hashicorp/consul/api"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/types/known/anypb"
)
func TestErrors(t *testing.T) {
cases := map[string]struct {
validate func() *Validate
endpointValidator EndpointValidator
err string
}{
"success": {
validate: func() *Validate {
return &Validate{
envoyID: "db",
snis: map[string]struct{}{
"db-sni": {},
},
listener: true,
usesRDS: true,
route: true,
resources: map[string]*resource{
"db-sni": {
required: true,
cluster: true,
},
},
}
},
endpointValidator: func(r *resource, s string, clusters *envoy_admin_v3.Clusters) {
r.loadAssignment = true
r.endpoints = 1
},
},
"no clusters for listener or route": {
validate: func() *Validate {
return &Validate{
envoyID: "db",
snis: map[string]struct{}{
"db-sni": {},
},
listener: true,
usesRDS: true,
route: true,
resources: map[string]*resource{
"db-sni": {
required: false,
cluster: true,
},
},
}
},
endpointValidator: func(r *resource, s string, clusters *envoy_admin_v3.Clusters) {
r.loadAssignment = true
r.endpoints = 1
},
err: "no clusters found on route or listener",
},
"no healthy endpoints": {
validate: func() *Validate {
return &Validate{
envoyID: "db",
snis: map[string]struct{}{
"db-sni": {},
},
listener: true,
usesRDS: true,
route: true,
resources: map[string]*resource{
"db-sni": {
required: true,
cluster: true,
},
},
}
},
endpointValidator: func(r *resource, s string, clusters *envoy_admin_v3.Clusters) {
r.loadAssignment = true
},
err: "zero healthy endpoints",
},
"success: aggregate cluster with one target with endpoints": {
validate: func() *Validate {
return &Validate{
envoyID: "db",
snis: map[string]struct{}{
"db-sni": {},
"db-fail-1-sni": {},
"db-fail-2-sni": {},
},
listener: true,
usesRDS: true,
route: true,
resources: map[string]*resource{
"db-sni": {
required: true,
cluster: true,
aggregateCluster: true,
aggregateClusterChildren: []string{
"db-fail-1-sni",
"db-fail-2-sni",
},
},
"db-fail-1-sni": {
required: true,
cluster: true,
parentCluster: "db-sni",
// This doesn't usually get set here, but this tests that at least one child cluster has
// healthy endpoints case.
endpoints: 1,
},
"db-fail-2-sni": {
required: true,
cluster: true,
parentCluster: "db-sni",
},
},
}
},
endpointValidator: func(r *resource, s string, clusters *envoy_admin_v3.Clusters) {
r.loadAssignment = true
},
},
"aggregate cluster no healthy endpoints": {
validate: func() *Validate {
return &Validate{
envoyID: "db",
snis: map[string]struct{}{
"db-sni": {},
"db-fail-1-sni": {},
"db-fail-2-sni": {},
},
listener: true,
usesRDS: true,
route: true,
resources: map[string]*resource{
"db-sni": {
required: true,
cluster: true,
aggregateCluster: true,
aggregateClusterChildren: []string{
"db-fail-1-sni",
"db-fail-2-sni",
},
},
"db-fail-1-sni": {
required: true,
cluster: true,
parentCluster: "db-sni",
},
"db-fail-2-sni": {
required: true,
cluster: true,
parentCluster: "db-sni",
},
},
}
},
endpointValidator: func(r *resource, s string, clusters *envoy_admin_v3.Clusters) {
r.loadAssignment = true
r.endpoints = 0
},
err: "zero healthy endpoints for aggregate cluster",
},
}
for n, tc := range cases {
t.Run(n, func(t *testing.T) {
v := tc.validate()
err := v.Errors(true, tc.endpointValidator, nil)
if len(tc.err) == 0 {
require.NoError(t, err)
} else {
require.Error(t, err)
require.Contains(t, err.Error(), tc.err)
}
})
}
}
func TestIsAggregateCluster(t *testing.T) {
aggregateClusterConfig, err := anypb.New(&envoy_aggregate_cluster_v3.ClusterConfig{
Clusters: []string{"c1", "c2"},
})
require.NoError(t, err)
cases := map[string]struct {
input *envoy_cluster_v3.Cluster
expectedAggregateCluster *envoy_aggregate_cluster_v3.ClusterConfig
expectedOk bool
}{
"non-aggregate cluster": {
input: &envoy_cluster_v3.Cluster{
Name: "foo",
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_LOGICAL_DNS},
},
expectedOk: false,
},
"valid aggregate cluster": {
input: &envoy_cluster_v3.Cluster{
Name: "foo",
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_ClusterType{
ClusterType: &envoy_cluster_v3.Cluster_CustomClusterType{
Name: "foo",
TypedConfig: aggregateClusterConfig,
},
},
},
expectedOk: true,
expectedAggregateCluster: &envoy_aggregate_cluster_v3.ClusterConfig{Clusters: []string{"c1", "c2"}},
},
}
for n, tc := range cases {
t.Run(n, func(t *testing.T) {
ac, ok := isAggregateCluster(tc.input)
require.Equal(t, tc.expectedOk, ok)
if tc.expectedOk {
require.Equal(t, tc.expectedAggregateCluster.Clusters, ac.Clusters)
}
})
}
}
func TestMakeValidate(t *testing.T) {
cases := map[string]struct {
extensionName string
arguments map[string]interface{}
expected *Validate
snis map[string]struct{}
ok bool
}{
"with no arguments": {
arguments: nil,
ok: false,
},
"with an invalid name": {
arguments: map[string]interface{}{
"envoyID": "id",
},
extensionName: "bad",
ok: false,
},
"empty envoy ID": {
arguments: map[string]interface{}{"envoyID": ""},
ok: false,
},
"valid everything": {
arguments: map[string]interface{}{
"envoyID": "id",
},
snis: map[string]struct{}{
"sni1": {},
"sni2": {},
},
expected: &Validate{
envoyID: "id",
resources: map[string]*resource{},
},
ok: true,
},
}
for n, tc := range cases {
t.Run(n, func(t *testing.T) {
extensionName := builtinValidateExtension
if tc.extensionName != "" {
extensionName = tc.extensionName
}
svc := api.CompoundServiceName{Name: "svc"}
ext := xdscommon.ExtensionConfiguration{
ServiceName: svc,
EnvoyExtension: api.EnvoyExtension{
Name: extensionName,
Arguments: tc.arguments,
},
}
patcher, err := MakeValidate(ext)
if tc.ok {
require.NoError(t, err)
require.Equal(t, tc.expected, patcher)
} else {
require.Error(t, err)
}
})
}
}

View File

@ -7,8 +7,11 @@ import (
envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
envoy_tcp_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
envoy_resource_v3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3"
"github.com/hashicorp/go-multierror"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/hashicorp/consul/agent/xds/xdscommon"
"github.com/hashicorp/consul/api"
@ -16,7 +19,7 @@ import (
type EnvoyExtension struct {
Constructor PluginConstructor
plugin Plugin
Plugin Plugin
ready bool
}
@ -27,7 +30,7 @@ var _ xdscommon.EnvoyExtension = (*EnvoyExtension)(nil)
func (envoyExtension *EnvoyExtension) Validate(config xdscommon.ExtensionConfiguration) error {
plugin, err := envoyExtension.Constructor(config)
envoyExtension.plugin = plugin
envoyExtension.Plugin = plugin
envoyExtension.ready = err == nil
return err
@ -36,7 +39,7 @@ func (envoyExtension *EnvoyExtension) Validate(config xdscommon.ExtensionConfigu
// Extend updates indexed xDS structures to include patches for
// built-in extensions. It is responsible for applying Plugins to
// the the appropriate xDS resources. If any portion of this function fails,
// it will attempt continue and return an error. The caller can then determine
// it will attempt to continue and return an error. The caller can then determine
// if it is better to use a partially applied extension or error out.
func (envoyExtension *EnvoyExtension) Extend(resources *xdscommon.IndexedResources, config xdscommon.ExtensionConfiguration) (*xdscommon.IndexedResources, error) {
if !envoyExtension.ready {
@ -51,14 +54,14 @@ func (envoyExtension *EnvoyExtension) Extend(resources *xdscommon.IndexedResourc
return resources, nil
}
if !envoyExtension.plugin.CanApply(config) {
if !envoyExtension.Plugin.CanApply(config) {
return resources, nil
}
for _, indexType := range []string{
xdscommon.ClusterType,
xdscommon.ListenerType,
xdscommon.RouteType,
xdscommon.ClusterType,
} {
for nameOrSNI, msg := range resources.Index[indexType] {
switch resource := msg.(type) {
@ -75,7 +78,7 @@ func (envoyExtension *EnvoyExtension) Extend(resources *xdscommon.IndexedResourc
continue
}
newCluster, patched, err := envoyExtension.plugin.PatchCluster(resource)
newCluster, patched, err := envoyExtension.Plugin.PatchCluster(resource)
if err != nil {
resultErr = multierror.Append(resultErr, fmt.Errorf("error patching cluster: %w", err))
continue
@ -96,8 +99,9 @@ func (envoyExtension *EnvoyExtension) Extend(resources *xdscommon.IndexedResourc
case *envoy_route_v3.RouteConfiguration:
// If the Envoy extension configuration is for an upstream service, the route's
// name must match the upstream service's SNI.
if config.IsUpstream() && !config.MatchesUpstreamServiceSNI(nameOrSNI) {
// name must match the upstream service's Envoy ID.
matchesEnvoyID := config.EnvoyID() == nameOrSNI
if config.IsUpstream() && !config.MatchesUpstreamServiceSNI(nameOrSNI) && !matchesEnvoyID {
continue
}
@ -106,7 +110,7 @@ func (envoyExtension *EnvoyExtension) Extend(resources *xdscommon.IndexedResourc
continue
}
newRoute, patched, err := envoyExtension.plugin.PatchRoute(resource)
newRoute, patched, err := envoyExtension.Plugin.PatchRoute(resource)
if err != nil {
resultErr = multierror.Append(resultErr, fmt.Errorf("error patching route: %w", err))
continue
@ -114,7 +118,6 @@ func (envoyExtension *EnvoyExtension) Extend(resources *xdscommon.IndexedResourc
if patched {
resources.Index[xdscommon.RouteType][nameOrSNI] = newRoute
}
default:
resultErr = multierror.Append(resultErr, fmt.Errorf("unsupported type was skipped: %T", resource))
}
@ -157,7 +160,7 @@ func (envoyExtension EnvoyExtension) patchTerminatingGatewayListener(config xdsc
var filters []*envoy_listener_v3.Filter
for _, filter := range filterChain.Filters {
newFilter, ok, err := envoyExtension.plugin.PatchFilter(filter)
newFilter, ok, err := envoyExtension.Plugin.PatchFilter(filter)
if err != nil {
resultErr = multierror.Append(resultErr, fmt.Errorf("error patching listener filter: %w", err))
@ -182,8 +185,12 @@ func (envoyExtension EnvoyExtension) patchConnectProxyListener(config xdscommon.
envoyID = l.Name[:i]
}
if config.IsUpstream() && envoyID == xdscommon.OutboundListenerName {
return envoyExtension.patchTProxyListener(config, l)
}
// If the Envoy extension configuration is for an upstream service, the listener's
// name must match the upstream service's EnvoyID.
// name must match the upstream service's EnvoyID or be the outbound listener.
if config.IsUpstream() && envoyID != config.EnvoyID() {
return l, false, nil
}
@ -200,7 +207,7 @@ func (envoyExtension EnvoyExtension) patchConnectProxyListener(config xdscommon.
var filters []*envoy_listener_v3.Filter
for _, filter := range filterChain.Filters {
newFilter, ok, err := envoyExtension.plugin.PatchFilter(filter)
newFilter, ok, err := envoyExtension.Plugin.PatchFilter(filter)
if err != nil {
resultErr = multierror.Append(resultErr, fmt.Errorf("error patching listener filter: %w", err))
filters = append(filters, filter)
@ -217,6 +224,115 @@ func (envoyExtension EnvoyExtension) patchConnectProxyListener(config xdscommon.
return l, patched, resultErr
}
func (envoyExtension EnvoyExtension) patchTProxyListener(config xdscommon.ExtensionConfiguration, l *envoy_listener_v3.Listener) (proto.Message, bool, error) {
var resultErr error
patched := false
vip := config.Upstreams[config.ServiceName].VIP
for _, filterChain := range l.FilterChains {
var filters []*envoy_listener_v3.Filter
match := filterChainTProxyMatch(vip, filterChain)
if !match {
continue
}
for _, filter := range filterChain.Filters {
newFilter, ok, err := envoyExtension.Plugin.PatchFilter(filter)
if err != nil {
resultErr = multierror.Append(resultErr, fmt.Errorf("error patching listener filter: %w", err))
filters = append(filters, filter)
}
if ok {
filters = append(filters, newFilter)
patched = true
}
}
filterChain.Filters = filters
}
return l, patched, resultErr
}
func filterChainTProxyMatch(vip string, filterChain *envoy_listener_v3.FilterChain) bool {
for _, prefixRange := range filterChain.FilterChainMatch.PrefixRanges {
// Since we always set the address prefix as the full VIP (rather than a prefix), we can just check if they are
// equal to find the matching filter chain.
if vip == prefixRange.AddressPrefix {
return true
}
}
return false
}
func FilterClusterNames(filter *envoy_listener_v3.Filter) map[string]struct{} {
clusterNames := make(map[string]struct{})
if filter == nil {
return clusterNames
}
if config := envoy_resource_v3.GetHTTPConnectionManager(filter); config != nil {
// If it's using RDS, the cluster names will be in the route, rather than in the http filter's route config, so
// we don't return any cluster names in this case. They can be gathered from the route.
if config.GetRds() != nil {
return clusterNames
}
cfg := config.GetRouteConfig()
clusterNames = RouteClusterNames(cfg)
}
if config := GetTCPProxy(filter); config != nil {
clusterNames[config.GetCluster()] = struct{}{}
}
return clusterNames
}
func RouteClusterNames(route *envoy_route_v3.RouteConfiguration) map[string]struct{} {
if route == nil {
return nil
}
clusterNames := make(map[string]struct{})
for _, virtualHost := range route.VirtualHosts {
for _, route := range virtualHost.Routes {
r := route.GetRoute()
if r == nil {
continue
}
if c := r.GetCluster(); c != "" {
clusterNames[r.GetCluster()] = struct{}{}
}
if wc := r.GetWeightedClusters(); wc != nil {
for _, c := range wc.GetClusters() {
if c.Name != "" {
clusterNames[c.Name] = struct{}{}
}
}
}
}
}
return clusterNames
}
func GetTCPProxy(filter *envoy_listener_v3.Filter) *envoy_tcp_proxy_v3.TcpProxy {
if typedConfig := filter.GetTypedConfig(); typedConfig != nil {
config := &envoy_tcp_proxy_v3.TcpProxy{}
if err := anypb.UnmarshalTo(typedConfig, config, proto.UnmarshalOptions{}); err == nil {
return config
}
}
return nil
}
func getSNI(chain *envoy_listener_v3.FilterChain) string {
var sni string

View File

@ -97,7 +97,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
}
opts := makeListenerOpts{
name: OutboundListenerName,
name: xdscommon.OutboundListenerName,
accessLogs: cfgSnap.Proxy.AccessLogs,
addr: "127.0.0.1",
port: port,

View File

@ -814,8 +814,10 @@ func TestListenersFromSnapshot(t *testing.T) {
create: proxycfg.TestConfigSnapshotIngressGateway_GWTLSListener_MixedHTTP2gRPC,
},
{
name: "transparent-proxy-http-upstream",
create: proxycfg.TestConfigSnapshotTransparentProxyHTTPUpstream,
name: "transparent-proxy-http-upstream",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTransparentProxyHTTPUpstream(t)
},
},
{
name: "transparent-proxy-with-resolver-redirect-upstream",

View File

@ -52,9 +52,6 @@ var (
type ADSStream = envoy_discovery_v3.AggregatedDiscoveryService_StreamAggregatedResourcesServer
const (
// OutboundListenerName is the name we give the outbound Envoy listener when transparent proxy mode is enabled.
OutboundListenerName = "outbound_listener"
// LocalAgentClusterName is the name we give the local agent "cluster" in
// Envoy config. Note that all cluster names may collide with service names
// since we want cluster names and service names to match to enable nice

View File

@ -0,0 +1,250 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"circuitBreakers": {
},
"outlierDetection": {
},
"commonLbConfig": {
"healthyPanicThreshold": {
}
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
},
"matchSubjectAltNames": [
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db"
}
]
}
},
"sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"circuitBreakers": {
},
"outlierDetection": {
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
},
"matchSubjectAltNames": [
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target"
},
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target"
}
]
}
},
"sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul"
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "lambda.us-east-1.amazonaws.com",
"portValue": 443
}
}
}
}
]
}
]
},
"dnsLookupFamily": "V4_ONLY",
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"sni": "*.amazonaws.com"
}
},
"metadata": {
"filterMetadata": {
"com.amazonaws.lambda": {
"egress_gateway": true
}
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "local_app",
"type": "STATIC",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "local_app",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 8080
}
}
}
}
]
}
]
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "no-endpoints.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"altStatName": "no-endpoints.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"circuitBreakers": {
},
"outlierDetection": {
},
"commonLbConfig": {
"healthyPanicThreshold": {
}
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
},
"matchSubjectAltNames": [
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/no-endpoints"
}
]
}
},
"sni": "no-endpoints.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "original-destination",
"type": "ORIGINAL_DST",
"connectTimeout": "5s",
"lbPolicy": "CLUSTER_PROVIDED"
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}

View File

@ -0,0 +1,106 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.1",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.2",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.1",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.20.1.2",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "9.9.9.9",
"portValue": 9090
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "no-endpoints.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"nonce": "00000001"
}

View File

@ -0,0 +1,218 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "db:127.0.0.1:9191",
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 9191
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.db.default.default.dc1",
"cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "outbound_listener:127.0.0.1:15001",
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 15001
}
},
"filterChains": [
{
"filterChainMatch": {
"prefixRanges": [
{
"addressPrefix": "10.0.0.1",
"prefixLen": 32
},
{
"addressPrefix": "240.0.0.1",
"prefixLen": 32
}
]
},
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"statPrefix": "upstream.google.default.default.dc1",
"routeConfig": {
"name": "google",
"virtualHosts": [
{
"name": "google.default.default.dc1",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
]
},
"httpFilters": [
{
"name": "envoy.filters.http.aws_lambda",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.aws_lambda.v3.Config",
"arn": "arn:aws:lambda:us-east-1:111111111111:function:lambda-1234",
"payloadPassthrough": true
}
},
{
"name": "envoy.filters.http.router",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
],
"tracing": {
"randomSampling": {
}
},
"stripAnyHostPort": true
}
}
]
}
],
"defaultFilterChain": {
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.original-destination",
"cluster": "original-destination"
}
}
]
},
"listenerFilters": [
{
"name": "envoy.filters.listener.original_dst",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.listener.original_dst.v3.OriginalDst"
}
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "prepared_query:geo-cache:127.10.10.10:8181",
"address": {
"socketAddress": {
"address": "127.10.10.10",
"portValue": 8181
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.prepared_query_geo-cache",
"cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "public_listener:0.0.0.0:9999",
"address": {
"socketAddress": {
"address": "0.0.0.0",
"portValue": 9999
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.rbac",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC",
"rules": {
},
"statPrefix": "connect_authz"
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "public_listener",
"cluster": "local_app"
}
}
],
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
}
},
"requireClientCertificate": true
}
}
}
],
"trafficDirection": "INBOUND"
}
],
"typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
"nonce": "00000001"
}

View File

@ -0,0 +1,5 @@
{
"versionInfo": "00000001",
"typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,277 @@
{
"cluster_statuses": [
{
"name": "consul-dataplane",
"host_statuses": [
{
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 37595
}
},
"stats": [
{
"name": "cx_connect_fail"
},
{
"value": "1",
"name": "cx_total"
},
{
"name": "rq_error"
},
{
"name": "rq_success"
},
{
"name": "rq_timeout"
},
{
"value": "1",
"name": "rq_total"
},
{
"type": "GAUGE",
"value": "1",
"name": "cx_active"
},
{
"type": "GAUGE",
"value": "1",
"name": "rq_active"
}
],
"health_status": {
"eds_health_status": "HEALTHY"
},
"weight": 1,
"locality": {}
}
],
"circuit_breakers": {
"thresholds": [
{
"max_connections": 1024,
"max_pending_requests": 1024,
"max_requests": 1024,
"max_retries": 3
},
{
"priority": "HIGH",
"max_connections": 1024,
"max_pending_requests": 1024,
"max_requests": 1024,
"max_retries": 3
}
]
},
"observability_name": "consul-dataplane"
},
{
"name": "local_app",
"added_via_api": true,
"host_statuses": [
{
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 9090
}
},
"stats": [
{
"name": "cx_connect_fail"
},
{
"value": "39",
"name": "cx_total"
},
{
"name": "rq_error"
},
{
"name": "rq_success"
},
{
"name": "rq_timeout"
},
{
"value": "17",
"name": "rq_total"
},
{
"type": "GAUGE",
"name": "cx_active"
},
{
"type": "GAUGE",
"name": "rq_active"
}
],
"health_status": {
"eds_health_status": "HEALTHY"
},
"weight": 1,
"locality": {}
}
],
"circuit_breakers": {
"thresholds": [
{
"max_connections": 1024,
"max_pending_requests": 1024,
"max_requests": 1024,
"max_retries": 3
},
{
"priority": "HIGH",
"max_connections": 1024,
"max_pending_requests": 1024,
"max_requests": 1024,
"max_retries": 3
}
]
},
"observability_name": "local_app"
},
{
"name": "backend.default.dc1.internal.7838b4bd-58b3-8117-3df1-60584910541b.consul",
"added_via_api": true,
"host_statuses": [
{
"address": {
"socket_address": {
"address": "10.0.3.11",
"port_value": 20000
}
},
"stats": [
{
"name": "cx_connect_fail"
},
{
"value": "2",
"name": "cx_total"
},
{
"name": "rq_error"
},
{
"value": "14",
"name": "rq_success"
},
{
"name": "rq_timeout"
},
{
"value": "14",
"name": "rq_total"
},
{
"type": "GAUGE",
"value": "2",
"name": "cx_active"
},
{
"type": "GAUGE",
"name": "rq_active"
}
],
"health_status": {
"eds_health_status": "HEALTHY"
},
"weight": 1,
"locality": {}
}
],
"circuit_breakers": {
"thresholds": [
{
"max_connections": 1024,
"max_pending_requests": 1024,
"max_requests": 1024,
"max_retries": 3
},
{
"priority": "HIGH",
"max_connections": 1024,
"max_pending_requests": 1024,
"max_requests": 1024,
"max_retries": 3
}
]
},
"observability_name": "backend.default.dc1.internal.7838b4bd-58b3-8117-3df1-60584910541b.consul"
},
{
"name": "backend2.default.dc1.internal.7838b4bd-58b3-8117-3df1-60584910541b.consul",
"added_via_api": true,
"host_statuses": [
{
"address": {
"socket_address": {
"address": "10.0.0.11",
"port_value": 20000
}
},
"stats": [
{
"name": "cx_connect_fail"
},
{
"value": "4",
"name": "cx_total"
},
{
"name": "rq_error"
},
{
"value": "12",
"name": "rq_success"
},
{
"name": "rq_timeout"
},
{
"value": "12",
"name": "rq_total"
},
{
"type": "GAUGE",
"value": "2",
"name": "cx_active"
},
{
"type": "GAUGE",
"name": "rq_active"
}
],
"health_status": {
"eds_health_status": "HEALTHY"
},
"weight": 1,
"locality": {}
}
],
"circuit_breakers": {
"thresholds": [
{
"max_connections": 1024,
"max_pending_requests": 1024,
"max_requests": 1024,
"max_retries": 3
},
{
"priority": "HIGH",
"max_connections": 1024,
"max_pending_requests": 1024,
"max_requests": 1024,
"max_retries": 3
}
]
},
"observability_name": "backend2.default.dc1.internal.7838b4bd-58b3-8117-3df1-60584910541b.consul"
}
]
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,237 @@
package xds
import (
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/proxycfg"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/agent/xds/builtinextensions/validate"
"github.com/hashicorp/consul/agent/xds/builtinextensiontemplate"
"github.com/hashicorp/consul/agent/xds/xdscommon"
"github.com/hashicorp/consul/api"
envoy_admin_v3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
)
const (
listenersType string = "type.googleapis.com/envoy.admin.v3.ListenersConfigDump"
clustersType string = "type.googleapis.com/envoy.admin.v3.ClustersConfigDump"
routesType string = "type.googleapis.com/envoy.admin.v3.RoutesConfigDump"
endpointsType string = "type.googleapis.com/envoy.admin.v3.EndpointsConfigDump"
)
func ParseConfigDump(rawConfig []byte) (*xdscommon.IndexedResources, error) {
config := &envoy_admin_v3.ConfigDump{}
unmarshal := &protojson.UnmarshalOptions{
DiscardUnknown: true,
}
err := unmarshal.Unmarshal(rawConfig, config)
if err != nil {
return nil, err
}
return ProxyConfigDumpToIndexedResources(config)
}
func ParseClusters(rawClusters []byte) (*envoy_admin_v3.Clusters, error) {
clusters := &envoy_admin_v3.Clusters{}
unmarshal := &protojson.UnmarshalOptions{
DiscardUnknown: true,
}
err := unmarshal.Unmarshal(rawClusters, clusters)
if err != nil {
return nil, err
}
return clusters, nil
}
// Validate validates the Envoy resources (indexedResources) for a given upstream service, peer, and vip. The peer
// should be "" for an upstream not on a remote peer. The vip is required for a transparent proxy upstream.
func Validate(indexedResources *xdscommon.IndexedResources, service api.CompoundServiceName, peer string, vip string, validateEndpoints bool, clusters *envoy_admin_v3.Clusters) error {
em := acl.NewEnterpriseMetaWithPartition(service.Partition, service.Namespace)
svc := structs.NewServiceName(service.Name, &em)
// The envoyID is used to identify which listener and filter matches the upstream service.
var envoyID string
psn := structs.PeeredServiceName{
ServiceName: svc,
Peer: peer,
}
uid := proxycfg.NewUpstreamIDFromPeeredServiceName(psn)
envoyID = uid.EnvoyID()
// Get all SNIs from the clusters in the configuration. Not all SNIs will need to be validated, but this ensures we
// capture SNIs which aren't directly identical to the upstream service name, but are still used for that upstream
// service. For example, in the case of having a splitter/redirect or another L7 config entry, the upstream service
// name could be "db" but due to a redirect SNI would be something like
// "redis.default.dc1.internal.<trustdomain>.consul". The envoyID will be used to limit which SNIs we actually
// validate.
snis := map[string]struct{}{}
for s := range indexedResources.Index[xdscommon.ClusterType] {
snis[s] = struct{}{}
}
// Build an ExtensionConfiguration for Validate plugin.
extConfig := xdscommon.ExtensionConfiguration{
EnvoyExtension: api.EnvoyExtension{
Name: "builtin/proxy/validate",
Arguments: map[string]interface{}{
"envoyID": envoyID,
},
},
ServiceName: service,
Upstreams: map[api.CompoundServiceName]xdscommon.UpstreamData{
service: {
VIP: vip,
// Even though snis are under the upstream service name we're validating, it actually contains all
// the cluster SNIs configured on this proxy, not just the upstream being validated. This means the
// PatchCluster function in the Validate plugin will be run on all clusters, but errors will only
// surface for clusters related to the upstream being validated.
SNI: snis,
EnvoyID: envoyID,
},
},
Kind: api.ServiceKindConnectProxy,
}
extension := builtinextensiontemplate.EnvoyExtension{Constructor: validate.MakeValidate}
err := extension.Validate(extConfig)
if err != nil {
return err
}
_, err = extension.Extend(indexedResources, extConfig)
if err != nil {
return err
}
v, ok := extension.Plugin.(*validate.Validate)
if !ok {
panic("validate plugin was not correctly created")
}
return v.Errors(validateEndpoints, validate.DoEndpointValidation, clusters)
}
func ProxyConfigDumpToIndexedResources(config *envoy_admin_v3.ConfigDump) (*xdscommon.IndexedResources, error) {
indexedResources := xdscommon.EmptyIndexedResources()
unmarshal := &proto.UnmarshalOptions{
DiscardUnknown: true,
}
for _, cfg := range config.Configs {
switch cfg.TypeUrl {
case listenersType:
lcd := &envoy_admin_v3.ListenersConfigDump{}
err := unmarshal.Unmarshal(cfg.GetValue(), lcd)
if err != nil {
return indexedResources, err
}
for _, listener := range lcd.GetDynamicListeners() {
// TODO We should care about these:
// listener.GetErrorState()
// listener.GetDrainingState()
// listener.GetWarmingState()
r := indexedResources.Index[xdscommon.ListenerType]
if r == nil {
r = make(map[string]proto.Message)
}
as := listener.GetActiveState()
if as == nil {
continue
}
l := &envoy_listener_v3.Listener{}
unmarshal.Unmarshal(as.Listener.GetValue(), l)
if err != nil {
return indexedResources, err
}
r[listener.Name] = l
indexedResources.Index[xdscommon.ListenerType] = r
}
case clustersType:
ccd := &envoy_admin_v3.ClustersConfigDump{}
err := unmarshal.Unmarshal(cfg.GetValue(), ccd)
if err != nil {
return indexedResources, err
}
// TODO we should care about ccd.GetDynamicWarmingClusters()
for _, cluster := range ccd.GetDynamicActiveClusters() {
r := indexedResources.Index[xdscommon.ClusterType]
if r == nil {
r = make(map[string]proto.Message)
}
c := &envoy_cluster_v3.Cluster{}
unmarshal.Unmarshal(cluster.GetCluster().Value, c)
if err != nil {
return indexedResources, err
}
r[c.Name] = c
indexedResources.Index[xdscommon.ClusterType] = r
}
case routesType:
rcd := &envoy_admin_v3.RoutesConfigDump{}
err := unmarshal.Unmarshal(cfg.GetValue(), rcd)
if err != nil {
return indexedResources, err
}
for _, route := range rcd.GetDynamicRouteConfigs() {
r := indexedResources.Index[xdscommon.RouteType]
if r == nil {
r = make(map[string]proto.Message)
}
rc := &envoy_route_v3.RouteConfiguration{}
unmarshal.Unmarshal(route.GetRouteConfig().Value, rc)
if err != nil {
return indexedResources, err
}
r[rc.Name] = rc
indexedResources.Index[xdscommon.RouteType] = r
}
case endpointsType:
ecd := &envoy_admin_v3.EndpointsConfigDump{}
err := unmarshal.Unmarshal(cfg.GetValue(), ecd)
if err != nil {
return indexedResources, err
}
for _, route := range ecd.GetDynamicEndpointConfigs() {
r := indexedResources.Index[xdscommon.EndpointType]
if r == nil {
r = make(map[string]proto.Message)
}
rc := &envoy_endpoint_v3.ClusterLoadAssignment{}
err := unmarshal.Unmarshal(route.EndpointConfig.GetValue(), rc)
if err != nil {
return indexedResources, err
}
r[rc.ClusterName] = rc
indexedResources.Index[xdscommon.EndpointType] = r
}
}
}
return indexedResources, nil
}

View File

@ -0,0 +1,357 @@
package xds
import (
"io"
"os"
"testing"
envoy_admin_v3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
"github.com/hashicorp/consul/agent/structs"
testinf "github.com/mitchellh/go-testing-interface"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/agent/proxycfg"
"github.com/hashicorp/consul/agent/xds/proxysupport"
"github.com/hashicorp/consul/agent/xds/xdscommon"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/sdk/testutil"
)
// TestValidateUpstreams only tests validation for listeners, routes, and clusters. Endpoints validation is done in a
// top level test that can parse the output of the /clusters endpoint.
func TestValidateUpstreams(t *testing.T) {
sni := "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
listenerName := "db:127.0.0.1:9191"
httpServiceDefaults := &structs.ServiceConfigEntry{
Kind: structs.ServiceDefaults,
Name: "db",
Protocol: "http",
}
dbUID := proxycfg.NewUpstreamID(&structs.Upstream{
DestinationName: "db",
LocalBindPort: 9191,
})
nodes := proxycfg.TestUpstreamNodes(t, "db")
tests := []struct {
name string
create func(t testinf.T) *proxycfg.ConfigSnapshot
patcher func(*xdscommon.IndexedResources) *xdscommon.IndexedResources
err string
peer string
serviceName *api.CompoundServiceName
vip string
}{
{
name: "tcp-success",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil)
},
},
{
name: "tcp-missing-listener",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil)
},
patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources {
delete(ir.Index[xdscommon.ListenerType], listenerName)
return ir
},
err: "no listener",
},
{
name: "tcp-missing-cluster",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil)
},
patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources {
delete(ir.Index[xdscommon.ClusterType], sni)
return ir
},
err: "no cluster",
},
{
name: "http-success",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil, httpServiceDefaults)
},
},
{
name: "http-rds-success",
// RDS, Envoy's Route Discovery Service, is only used for HTTP services with a customized discovery chain, so we
// need to use the test snapshot and add L7 config entries.
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, []proxycfg.UpdateEvent{
// The events ensure there are endpoints for the v1 and v2 subsets.
{
CorrelationID: "upstream-target:v1.db.default.default.dc1:" + dbUID.String(),
Result: &structs.IndexedCheckServiceNodes{
Nodes: nodes,
},
},
{
CorrelationID: "upstream-target:v2.db.default.default.dc1:" + dbUID.String(),
Result: &structs.IndexedCheckServiceNodes{
Nodes: nodes,
},
},
}, configEntriesForDBSplits()...)
},
patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources {
return ir
},
},
{
name: "http-rds-missing-route",
// RDS, Envoy's Route Discovery Service, is only used for HTTP services with a customized discovery chain, so we
// need to use the test snapshot and add L7 config entries.
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, []proxycfg.UpdateEvent{
// The events ensure there are endpoints for the v1 and v2 subsets.
{
CorrelationID: "upstream-target:v1.db.default.default.dc1:" + dbUID.String(),
Result: &structs.IndexedCheckServiceNodes{
Nodes: nodes,
},
},
{
CorrelationID: "upstream-target:v2.db.default.default.dc1:" + dbUID.String(),
Result: &structs.IndexedCheckServiceNodes{
Nodes: nodes,
},
},
}, configEntriesForDBSplits()...)
},
patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources {
delete(ir.Index[xdscommon.RouteType], "db")
return ir
},
err: "no route",
},
{
name: "redirect",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "redirect-to-cluster-peer", nil, nil)
},
},
{
name: "failover",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil)
},
},
{
name: "failover-to-cluster-peer",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", nil, nil)
},
},
{
name: "non-eds",
create: proxycfg.TestConfigSnapshotPeering,
serviceName: &api.CompoundServiceName{Name: "payments"},
peer: "cloud",
},
{
name: "tproxy-success",
vip: "240.0.0.1",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTransparentProxyHTTPUpstream(t)
},
patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources {
return ir
},
},
{
name: "tproxy-http-missing-cluster",
vip: "240.0.0.1",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTransparentProxyHTTPUpstream(t)
},
patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources {
sni := "google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
delete(ir.Index[xdscommon.ClusterType], sni)
return ir
},
err: "no cluster",
},
{
name: "tproxy-http-redirect-success",
vip: "240.0.0.1",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTransparentProxyHTTPUpstream(t, configEntriesForGoogleRedirect()...)
},
serviceName: &api.CompoundServiceName{
Name: "google",
},
patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources {
return ir
},
},
{
name: "tproxy-http-split-success",
vip: "240.0.0.1",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTransparentProxyHTTPUpstream(t, configEntriesForGoogleSplits()...)
},
serviceName: &api.CompoundServiceName{
Name: "google",
},
patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources {
return ir
},
},
}
latestEnvoyVersion := proxysupport.EnvoyVersions[0]
sf, err := determineSupportedProxyFeaturesFromString(latestEnvoyVersion)
require.NoError(t, err)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Sanity check default with no overrides first
snap := tt.create(t)
// We need to replace the TLS certs with deterministic ones to make golden
// files workable. Note we don't update these otherwise they'd change
// golden files for every test case and so not be any use!
setupTLSRootsAndLeaf(t, snap)
g := newResourceGenerator(testutil.Logger(t), nil, false)
g.ProxyFeatures = sf
res, err := g.allResourcesFromSnapshot(snap)
require.NoError(t, err)
indexedResources := indexResources(g.Logger, res)
if tt.patcher != nil {
indexedResources = tt.patcher(indexedResources)
}
serviceName := tt.serviceName
if serviceName == nil {
serviceName = &api.CompoundServiceName{
Name: "db",
}
}
peer := tt.peer
// This only tests validation for listeners, routes, and clusters. Endpoints validation is done in a top
// level test that can parse the output of the /clusters endpoint. So for this test, we set clusters to nil.
err = Validate(indexedResources, *serviceName, peer, tt.vip, false, nil)
if len(tt.err) == 0 {
require.NoError(t, err)
} else {
require.Error(t, err)
require.Contains(t, err.Error(), tt.err)
}
})
}
}
// TODO make config.json and clusters.json use an http upstream with L7 config entries for more confidence.
func TestValidate(t *testing.T) {
indexedResources := getConfig(t)
clusters := getClusters(t)
err := Validate(indexedResources, service, "", "", true, clusters)
require.NoError(t, err)
}
// TODO: Manually inspect the config and clusters files and hardcode the list of expected resource names for higher
// confidence in these functions.
func getConfig(t *testing.T) *xdscommon.IndexedResources {
file, err := os.Open("testdata/validateupstream/config.json")
require.NoError(t, err)
jsonBytes, err := io.ReadAll(file)
require.NoError(t, err)
indexedResources, err := ParseConfigDump(jsonBytes)
require.NoError(t, err)
return indexedResources
}
func getClusters(t *testing.T) *envoy_admin_v3.Clusters {
file, err := os.Open("testdata/validateupstream/clusters.json")
require.NoError(t, err)
jsonBytes, err := io.ReadAll(file)
require.NoError(t, err)
clusters, err := ParseClusters(jsonBytes)
require.NoError(t, err)
return clusters
}
var service = api.CompoundServiceName{
Name: "backend",
}
func configEntriesForDBSplits() []structs.ConfigEntry {
httpServiceDefaults := &structs.ServiceConfigEntry{
Kind: structs.ServiceDefaults,
Name: "db",
Protocol: "http",
}
splitter := &structs.ServiceSplitterConfigEntry{
Kind: structs.ServiceSplitter,
Name: "db",
Splits: []structs.ServiceSplit{
{
Weight: 50,
Service: "db",
ServiceSubset: "v1",
},
{
Weight: 50,
Service: "db",
ServiceSubset: "v2",
},
},
}
resolver := &structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
Subsets: map[string]structs.ServiceResolverSubset{
"v1": {Filter: "Service.Meta.version == v1"},
"v2": {Filter: "Service.Meta.version == v2"},
},
}
return []structs.ConfigEntry{httpServiceDefaults, splitter, resolver}
}
func configEntriesForGoogleSplits() []structs.ConfigEntry {
splitter := &structs.ServiceSplitterConfigEntry{
Kind: structs.ServiceSplitter,
Name: "google",
Splits: []structs.ServiceSplit{
{
Weight: 50,
Service: "google",
ServiceSubset: "v1",
},
{
Weight: 50,
Service: "google",
ServiceSubset: "v2",
},
},
}
resolver := &structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "google",
Subsets: map[string]structs.ServiceResolverSubset{
"v1": {Filter: "Service.Meta.version == v1"},
"v2": {Filter: "Service.Meta.version == v2"},
},
}
return []structs.ConfigEntry{splitter, resolver}
}
func configEntriesForGoogleRedirect() []structs.ConfigEntry {
redirectGoogle := &structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "google",
Redirect: &structs.ServiceResolverRedirect{
Service: "google-v2",
},
}
return []structs.ConfigEntry{redirectGoogle}
}

View File

@ -43,6 +43,9 @@ const (
// We should probably just make it configurable if anyone actually has
// services named "local_app" in the future.
LocalAppClusterName = "local_app"
// OutboundListenerName is the name we give the outbound Envoy listener when transparent proxy mode is enabled.
OutboundListenerName = "outbound_listener"
)
type EnvoyExtension interface {
@ -106,6 +109,8 @@ type ExtensionConfiguration struct {
// UpstreamData has the SNI, EnvoyID, and OutgoingProxyKind of the upstream services for the local proxy and this data
// is used to choose which Envoy resources to patch.
type UpstreamData struct {
// VIP is the tproxy virtual IP used to reach an upstream service.
VIP string
// SNI is the SNI header used to reach an upstream service.
SNI map[string]struct{}
// EnvoyID is the envoy ID of an upstream service, structured <service> or <partition>/<ns>/<service> when using a
@ -152,6 +157,7 @@ func GetExtensionConfigurations(cfgSnap *proxycfg.ConfigSnapshot) map[api.Compou
case structs.ServiceKindConnectProxy:
kind = api.ServiceKindConnectProxy
outgoingKindByService := make(map[api.CompoundServiceName]api.ServiceKind)
vipForService := make(map[api.CompoundServiceName]string)
for uid, upstreamData := range cfgSnap.ConnectProxy.WatchedUpstreamEndpoints {
sn := upstreamIDToCompoundServiceName(uid)
@ -160,6 +166,12 @@ func GetExtensionConfigurations(cfgSnap *proxycfg.ConfigSnapshot) map[api.Compou
if serviceNode.Service == nil {
continue
}
vip := serviceNode.Service.TaggedAddresses[structs.TaggedAddressVirtualIP].Address
if vip != "" {
if _, ok := vipForService[sn]; !ok {
vipForService[sn] = vip
}
}
// Store the upstream's kind, and for ServiceKindTypical we don't do anything because we'll default
// any unset upstreams to ServiceKindConnectProxy below.
switch serviceNode.Service.Kind {
@ -188,6 +200,7 @@ func GetExtensionConfigurations(cfgSnap *proxycfg.ConfigSnapshot) map[api.Compou
upstreamMap[compoundServiceName] = UpstreamData{
SNI: map[string]struct{}{sni: {}},
VIP: vipForService[compoundServiceName],
EnvoyID: uid.EnvoyID(),
OutgoingProxyKind: outgoingKind,
}