Add support for ingress-gateway in CLI command (#7618)

* Add support for ingress-gateway in CLI command

- Supports -register command
- Creates a static Envoy listener that exposes only the /ready API so
that we can register a TCP healthcheck against the ingress gateway
itself
- Updates ServiceAddressValue.String() to be more in line with Value()
This commit is contained in:
Chris Piraino 2020-04-14 09:48:02 -05:00 committed by GitHub
parent 778ba5310a
commit 0ab9aa9489
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 588 additions and 39 deletions

View File

@ -32,6 +32,11 @@ const (
// ServiceKindTerminatingGateway is a Terminating Gateway for the Connect
// feature. This service will proxy connections to services outside the mesh.
ServiceKindTerminatingGateway ServiceKind = "terminating-gateway"
// ServiceKindIngressGateway is an Ingress Gateway for the Connect feature.
// This service will ingress connections based of configuration defined in
// the ingress-gateway config entry.
ServiceKindIngressGateway ServiceKind = "ingress-gateway"
)
// UpstreamDestType is the type of upstream discovery mechanism.

View File

@ -65,6 +65,15 @@ type BootstrapConfig struct {
// a 404.
StatsBindAddr string `mapstructure:"envoy_stats_bind_addr"`
// ReadyBindAddr configures an <ip>:<port> on which Envoy will listen and
// expose a single /ready HTTP endpoint. This is useful for checking the
// liveness of an Envoy instance when no other listeners are garaunteed to be
// configured, as is the case with ingress gateways.
//
// Not that we do not allow this to be configured via the service
// definition config map currently.
ReadyBindAddr string `mapstructure:"-"`
// OverrideJSONTpl allows replacing the base template used to render the
// bootstrap. This is an "escape hatch" allowing arbitrary control over the
// proxy's configuration but will the most effort to maintain and correctly
@ -196,13 +205,19 @@ func (c *BootstrapConfig) ConfigureArgs(args *BootstrapTplArgs) error {
}
// Setup prometheus if needed. This MUST happen after the Static*JSON is set above
if c.PrometheusBindAddr != "" {
if err := c.generateMetricsListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", "/metrics", "/stats/prometheus"); err != nil {
if err := c.generateListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", "/metrics", "/stats/prometheus"); err != nil {
return err
}
}
// Setup /stats proxy listener if needed. This MUST happen after the Static*JSON is set above
if c.StatsBindAddr != "" {
if err := c.generateMetricsListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats"); err != nil {
if err := c.generateListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats"); err != nil {
return err
}
}
// Setup /ready proxy listener if needed. This MUST happen after the Static*JSON is set above
if c.ReadyBindAddr != "" {
if err := c.generateListenerConfig(args, c.ReadyBindAddr, "envoy_ready", "path", "/ready", "/ready"); err != nil {
return err
}
}
@ -383,7 +398,7 @@ func (c *BootstrapConfig) generateStatsConfig(args *BootstrapTplArgs) error {
return nil
}
func (c *BootstrapConfig) generateMetricsListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite string) error {
func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite string) error {
host, port, err := net.SplitHostPort(bindAddr)
if err != nil {
return fmt.Errorf("invalid %s bind address: %s", name, err)

View File

@ -9,6 +9,20 @@ import (
)
const (
expectedSelfAdminCluster = `{
"name": "self_admin",
"connect_timeout": "5s",
"type": "STATIC",
"http_protocol_options": {},
"hosts": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
}
]
}`
expectedPromListener = `{
"name": "envoy_prometheus_metrics_listener",
"address": {
@ -66,20 +80,6 @@ const (
}
]
}`
expectedPromCluster = `{
"name": "self_admin",
"connect_timeout": "5s",
"type": "STATIC",
"http_protocol_options": {},
"hosts": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
}
]
}`
expectedStatsListener = `{
"name": "envoy_metrics_listener",
"address": {
@ -137,17 +137,60 @@ const (
}
]
}`
expectedStatsCluster = `{
"name": "self_admin",
"connect_timeout": "5s",
"type": "STATIC",
"http_protocol_options": {},
"hosts": [
expectedReadyListener = `{
"name": "envoy_ready_listener",
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 4444
}
},
"filter_chains": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"stat_prefix": "envoy_ready",
"codec_type": "HTTP1",
"route_config": {
"name": "self_admin_route",
"virtual_hosts": [
{
"name": "self_admin",
"domains": [
"*"
],
"routes": [
{
"match": {
"path": "/ready"
},
"route": {
"cluster": "self_admin",
"prefix_rewrite": "/ready"
}
},
{
"match": {
"prefix": "/"
},
"direct_response": {
"status": 404
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.router"
}
]
}
}
]
}
]
}`
@ -392,7 +435,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
// Should add a static cluster for the self-proxy to admin
StaticClustersJSON: expectedPromCluster,
StaticClustersJSON: expectedSelfAdminCluster,
// Should add a static http listener too
StaticListenersJSON: expectedPromListener,
StatsConfigJSON: defaultStatsConfigJSON,
@ -414,7 +457,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
// Should add a static cluster for the self-proxy to admin
StaticClustersJSON: `{"foo":"bar"},` + expectedPromCluster,
StaticClustersJSON: `{"foo":"bar"},` + expectedSelfAdminCluster,
// Should add a static http listener too
StaticListenersJSON: `{"baz":"qux"},` + expectedPromListener,
StatsConfigJSON: defaultStatsConfigJSON,
@ -434,7 +477,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
// Should add a static cluster for the self-proxy to admin
StaticClustersJSON: expectedStatsCluster,
StaticClustersJSON: expectedSelfAdminCluster,
// Should add a static http listener too
StaticListenersJSON: expectedStatsListener,
StatsConfigJSON: defaultStatsConfigJSON,
@ -456,7 +499,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
// Should add a static cluster for the self-proxy to admin
StaticClustersJSON: `{"foo":"bar"},` + expectedStatsCluster,
StaticClustersJSON: `{"foo":"bar"},` + expectedSelfAdminCluster,
// Should add a static http listener too
StaticListenersJSON: `{"baz":"qux"},` + expectedStatsListener,
StatsConfigJSON: defaultStatsConfigJSON,
@ -513,6 +556,48 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
},
wantErr: true,
},
{
name: "ready-bind-addr",
input: BootstrapConfig{
ReadyBindAddr: "0.0.0.0:4444",
},
baseArgs: BootstrapTplArgs{
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
},
wantArgs: BootstrapTplArgs{
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
// Should add a static cluster for the self-proxy to admin
StaticClustersJSON: expectedSelfAdminCluster,
// Should add a static http listener too
StaticListenersJSON: expectedReadyListener,
StatsConfigJSON: defaultStatsConfigJSON,
},
wantErr: false,
},
{
name: "ready-bind-addr-with-overrides",
input: BootstrapConfig{
ReadyBindAddr: "0.0.0.0:4444",
StaticClustersJSON: `{"foo":"bar"}`,
StaticListenersJSON: `{"baz":"qux"}`,
},
baseArgs: BootstrapTplArgs{
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
},
wantArgs: BootstrapTplArgs{
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
// Should add a static cluster for the self-proxy to admin
StaticClustersJSON: `{"foo":"bar"},` + expectedSelfAdminCluster,
// Should add a static http listener too
StaticListenersJSON: `{"baz":"qux"},` + expectedReadyListener,
StatsConfigJSON: defaultStatsConfigJSON,
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

View File

@ -76,6 +76,7 @@ const (
var supportedGateways = map[string]api.ServiceKind{
"mesh": api.ServiceKindMeshGateway,
"terminating": api.ServiceKindTerminatingGateway,
"ingress": api.ServiceKindIngressGateway,
}
func (c *cmd) init() {
@ -216,6 +217,7 @@ func (c *cmd) run(args []string) int {
c.UI.Error("The mesh-gateway flag is deprecated and cannot be used alongside the gateway flag")
return 1
}
if c.meshGateway {
c.gateway = meshGatewayVal
}
@ -235,7 +237,7 @@ func (c *cmd) run(args []string) int {
if c.gateway != "" {
kind, ok := supportedGateways[c.gateway]
if !ok {
c.UI.Error("Gateway must be one of: terminating or mesh")
c.UI.Error("Gateway must be one of: terminating, mesh, or ingress")
return 1
}
c.gatewayKind = kind
@ -464,6 +466,17 @@ func (c *cmd) generateConfig() ([]byte, error) {
var bsCfg BootstrapConfig
// Setup ready listener for ingress gateway to pass healthcheck
if c.gatewayKind == api.ServiceKindIngressGateway {
lanAddr := c.lanAddress.String()
// Deal with possibility of address not being specified and defaulting to
// ":443"
if strings.HasPrefix(lanAddr, ":") {
lanAddr = "127.0.0.1" + lanAddr
}
bsCfg.ReadyBindAddr = lanAddr
}
if !c.disableCentralConfig {
// Fetch any customization from the registration
svc, _, err := c.client.Agent().Service(c.proxyID, nil)

View File

@ -3,12 +3,6 @@ package envoy
import (
"encoding/json"
"flag"
"github.com/hashicorp/consul/agent"
"github.com/hashicorp/consul/agent/xds"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/mitchellh/cli"
"github.com/stretchr/testify/require"
"io/ioutil"
"net"
"net/http"
@ -17,6 +11,13 @@ import (
"path/filepath"
"strings"
"testing"
"github.com/hashicorp/consul/agent"
"github.com/hashicorp/consul/agent/xds"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/mitchellh/cli"
"github.com/stretchr/testify/require"
)
var update = flag.Bool("update", false, "update golden files")
@ -583,6 +584,40 @@ func TestGenerateConfig(t *testing.T) {
LocalAgentClusterName: xds.LocalAgentClusterName,
},
},
{
Name: "ingress-gateway",
Flags: []string{"-proxy-id", "ingress-gateway", "-gateway", "ingress"},
WantArgs: BootstrapTplArgs{
EnvoyVersion: defaultEnvoyVersion,
ProxyCluster: "ingress-gateway",
ProxyID: "ingress-gateway",
GRPC: GRPC{
AgentAddress: "127.0.0.1",
AgentPort: "8502",
},
AdminAccessLogPath: "/dev/null",
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
LocalAgentClusterName: xds.LocalAgentClusterName,
},
},
{
Name: "ingress-gateway-address-specified",
Flags: []string{"-proxy-id", "ingress-gateway", "-gateway", "ingress", "-address", "1.2.3.4:7777"},
WantArgs: BootstrapTplArgs{
EnvoyVersion: defaultEnvoyVersion,
ProxyCluster: "ingress-gateway",
ProxyID: "ingress-gateway",
GRPC: GRPC{
AgentAddress: "127.0.0.1",
AgentPort: "8502",
},
AdminAccessLogPath: "/dev/null",
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
LocalAgentClusterName: xds.LocalAgentClusterName,
},
},
}
copyAndReplaceAll := func(s []string, old, new string) []string {

View File

@ -20,7 +20,7 @@ type ServiceAddressValue struct {
}
func (s *ServiceAddressValue) String() string {
if s == nil {
if s == nil || (s.value.Port == 0 && s.value.Address == "") {
return fmt.Sprintf(":%d", defaultGatewayPort)
}
return fmt.Sprintf("%v:%d", s.value.Address, s.value.Port)

View File

@ -28,6 +28,24 @@ func TestServiceAddressValue_Value(t *testing.T) {
})
}
func TestServiceAddressValue_String(t *testing.T) {
t.Run("nil receiver", func(t *testing.T) {
var addr *ServiceAddressValue
require.Equal(t, addr.String(), ":443")
})
t.Run("default value", func(t *testing.T) {
addr := &ServiceAddressValue{}
require.Equal(t, addr.String(), ":443")
})
t.Run("set value", func(t *testing.T) {
addr := &ServiceAddressValue{}
require.NoError(t, addr.Set("localhost:3333"))
require.Equal(t, addr.String(), "localhost:3333")
})
}
func TestServiceAddressValue_Set(t *testing.T) {
var testcases = []struct {
name string

View File

@ -0,0 +1,189 @@
{
"admin": {
"access_log_path": "/dev/null",
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
}
},
"node": {
"cluster": "ingress-gateway",
"id": "ingress-gateway",
"metadata": {
"namespace": "default",
"envoy_version": "1.14.1"
}
},
"static_resources": {
"clusters": [
{
"name": "local_agent",
"connect_timeout": "1s",
"type": "STATIC",
"http2_protocol_options": {},
"hosts": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 8502
}
}
]
},
{
"name": "self_admin",
"connect_timeout": "5s",
"type": "STATIC",
"http_protocol_options": {},
"hosts": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
}
]
}
],
"listeners": [
{
"name": "envoy_ready_listener",
"address": {
"socket_address": {
"address": "1.2.3.4",
"port_value": 7777
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"stat_prefix": "envoy_ready",
"codec_type": "HTTP1",
"route_config": {
"name": "self_admin_route",
"virtual_hosts": [
{
"name": "self_admin",
"domains": [
"*"
],
"routes": [
{
"match": {
"path": "/ready"
},
"route": {
"cluster": "self_admin",
"prefix_rewrite": "/ready"
}
},
{
"match": {
"prefix": "/"
},
"direct_response": {
"status": 404
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.router"
}
]
}
}
]
}
]
}
]
},
"stats_config": {
"stats_tags": [
{
"regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.custom_hash"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.service_subset"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.service"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.namespace"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.datacenter"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
"tag_name": "consul.routing_type"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
"tag_name": "consul.trust_domain"
},
{
"regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.target"
},
{
"regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
"tag_name": "consul.full_target"
},
{
"tag_name": "local_cluster",
"fixed_value": "ingress-gateway"
}
],
"use_all_default_tags": true
},
"dynamic_resources": {
"lds_config": {
"ads": {}
},
"cds_config": {
"ads": {}
},
"ads_config": {
"api_type": "GRPC",
"grpc_services": {
"initial_metadata": [
{
"key": "x-consul-token",
"value": ""
}
],
"envoy_grpc": {
"cluster_name": "local_agent"
}
}
}
},
"layered_runtime": {
"layers": [
{
"name": "static_layer",
"static_layer": {
"envoy.deprecated_features:envoy.api.v2.Cluster.tls_context": true,
"envoy.deprecated_features:envoy.config.trace.v2.ZipkinConfig.HTTP_JSON_V1": true,
"envoy.deprecated_features:envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager.Tracing.operation_name": true
}
}
]
}
}

View File

@ -0,0 +1,189 @@
{
"admin": {
"access_log_path": "/dev/null",
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
}
},
"node": {
"cluster": "ingress-gateway",
"id": "ingress-gateway",
"metadata": {
"namespace": "default",
"envoy_version": "1.14.1"
}
},
"static_resources": {
"clusters": [
{
"name": "local_agent",
"connect_timeout": "1s",
"type": "STATIC",
"http2_protocol_options": {},
"hosts": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 8502
}
}
]
},
{
"name": "self_admin",
"connect_timeout": "5s",
"type": "STATIC",
"http_protocol_options": {},
"hosts": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
}
]
}
],
"listeners": [
{
"name": "envoy_ready_listener",
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 443
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"stat_prefix": "envoy_ready",
"codec_type": "HTTP1",
"route_config": {
"name": "self_admin_route",
"virtual_hosts": [
{
"name": "self_admin",
"domains": [
"*"
],
"routes": [
{
"match": {
"path": "/ready"
},
"route": {
"cluster": "self_admin",
"prefix_rewrite": "/ready"
}
},
{
"match": {
"prefix": "/"
},
"direct_response": {
"status": 404
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.router"
}
]
}
}
]
}
]
}
]
},
"stats_config": {
"stats_tags": [
{
"regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.custom_hash"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.service_subset"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.service"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.namespace"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.datacenter"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
"tag_name": "consul.routing_type"
},
{
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
"tag_name": "consul.trust_domain"
},
{
"regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
"tag_name": "consul.target"
},
{
"regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
"tag_name": "consul.full_target"
},
{
"tag_name": "local_cluster",
"fixed_value": "ingress-gateway"
}
],
"use_all_default_tags": true
},
"dynamic_resources": {
"lds_config": {
"ads": {}
},
"cds_config": {
"ads": {}
},
"ads_config": {
"api_type": "GRPC",
"grpc_services": {
"initial_metadata": [
{
"key": "x-consul-token",
"value": ""
}
],
"envoy_grpc": {
"cluster_name": "local_agent"
}
}
}
},
"layered_runtime": {
"layers": [
{
"name": "static_layer",
"static_layer": {
"envoy.deprecated_features:envoy.api.v2.Cluster.tls_context": true,
"envoy.deprecated_features:envoy.config.trace.v2.ZipkinConfig.HTTP_JSON_V1": true,
"envoy.deprecated_features:envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager.Tracing.operation_name": true
}
}
]
}
}