diff --git a/command/connect/envoy/bootstrap_config.go b/command/connect/envoy/bootstrap_config.go index 75bdb2dd3..27286373e 100644 --- a/command/connect/envoy/bootstrap_config.go +++ b/command/connect/envoy/bootstrap_config.go @@ -57,6 +57,14 @@ type BootstrapConfig struct { // be fixed in a future Consul version as Envoy 1.10 reaches stable release. PrometheusBindAddr string `mapstructure:"envoy_prometheus_bind_addr"` + // StatsBindAddr configures an : on which the Envoy will listen + // and expose the /stats HTTP path prefix for any agent to access. It + // does this by proxying that path prefix to the internal admin server + // which allows exposing metrics on the network without the security + // risk of exposing the full admin server API. Any other URL requested will be + // a 404. + StatsBindAddr string `mapstructure:"envoy_stats_bind_addr"` + // OverrideJSONTpl allows replacing the base template used to render the // bootstrap. This is an "escape hatch" allowing arbitrary control over the // proxy's configuration but will the most effort to maintain and correctly @@ -188,7 +196,13 @@ func (c *BootstrapConfig) ConfigureArgs(args *BootstrapTplArgs) error { } // Setup prometheus if needed. This MUST happen after the Static*JSON is set above if c.PrometheusBindAddr != "" { - if err := c.generatePrometheusConfig(args); err != nil { + if err := c.generateMetricsListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", "/metrics", "/stats/prometheus"); err != nil { + return err + } + } + // Setup /stats proxy listener if needed. This MUST happen after the Static*JSON is set above + if c.StatsBindAddr != "" { + if err := c.generateMetricsListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats"); err != nil { return err } } @@ -369,10 +383,10 @@ func (c *BootstrapConfig) generateStatsConfig(args *BootstrapTplArgs) error { return nil } -func (c *BootstrapConfig) generatePrometheusConfig(args *BootstrapTplArgs) error { - host, port, err := net.SplitHostPort(c.PrometheusBindAddr) +func (c *BootstrapConfig) generateMetricsListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite string) error { + host, port, err := net.SplitHostPort(bindAddr) if err != nil { - return fmt.Errorf("invalid prometheus_bind_addr: %s", err) + return fmt.Errorf("invalid %s bind address: %s", name, err) } clusterJSON := `{ @@ -390,7 +404,7 @@ func (c *BootstrapConfig) generatePrometheusConfig(args *BootstrapTplArgs) error ] }` listenerJSON := `{ - "name": "envoy_prometheus_metrics_listener", + "name": "` + name + `_listener", "address": { "socket_address": { "address": "` + host + `", @@ -403,7 +417,7 @@ func (c *BootstrapConfig) generatePrometheusConfig(args *BootstrapTplArgs) error { "name": "envoy.http_connection_manager", "config": { - "stat_prefix": "envoy_prometheus_metrics", + "stat_prefix": "` + name + `", "codec_type": "HTTP1", "route_config": { "name": "self_admin_route", @@ -416,11 +430,11 @@ func (c *BootstrapConfig) generatePrometheusConfig(args *BootstrapTplArgs) error "routes": [ { "match": { - "path": "/metrics" + "` + matchType + `": "` + matchValue + `" }, "route": { "cluster": "self_admin", - "prefix_rewrite": "/stats/prometheus" + "prefix_rewrite": "` + prefixRewrite + `" } }, { diff --git a/command/connect/envoy/bootstrap_config_test.go b/command/connect/envoy/bootstrap_config_test.go index d8db181c1..1b9411f3c 100644 --- a/command/connect/envoy/bootstrap_config_test.go +++ b/command/connect/envoy/bootstrap_config_test.go @@ -80,6 +80,77 @@ const ( } ] }` + expectedStatsListener = `{ + "name": "envoy_metrics_listener", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9000 + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "stat_prefix": "envoy_metrics", + "codec_type": "HTTP1", + "route_config": { + "name": "self_admin_route", + "virtual_hosts": [ + { + "name": "self_admin", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/stats" + }, + "route": { + "cluster": "self_admin", + "prefix_rewrite": "/stats" + } + }, + { + "match": { + "prefix": "/" + }, + "direct_response": { + "status": 404 + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.router" + } + ] + } + } + ] + } + ] + }` + expectedStatsCluster = `{ + "name": "self_admin", + "connect_timeout": "5s", + "type": "STATIC", + "http_protocol_options": {}, + "hosts": [ + { + "socket_address": { + "address": "127.0.0.1", + "port_value": 19000 + } + } + ] + }` ) func TestBootstrapConfig_ConfigureArgs(t *testing.T) { @@ -350,6 +421,48 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantErr: false, }, + { + name: "stats-bind-addr", + input: BootstrapConfig{ + StatsBindAddr: "0.0.0.0:9000", + }, + baseArgs: BootstrapTplArgs{ + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + }, + wantArgs: BootstrapTplArgs{ + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + // Should add a static cluster for the self-proxy to admin + StaticClustersJSON: expectedStatsCluster, + // Should add a static http listener too + StaticListenersJSON: expectedStatsListener, + StatsConfigJSON: defaultStatsConfigJSON, + }, + wantErr: false, + }, + { + name: "stats-bind-addr-with-overrides", + input: BootstrapConfig{ + StatsBindAddr: "0.0.0.0:9000", + StaticClustersJSON: `{"foo":"bar"}`, + StaticListenersJSON: `{"baz":"qux"}`, + }, + baseArgs: BootstrapTplArgs{ + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + }, + wantArgs: BootstrapTplArgs{ + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + // Should add a static cluster for the self-proxy to admin + StaticClustersJSON: `{"foo":"bar"},` + expectedStatsCluster, + // Should add a static http listener too + StaticListenersJSON: `{"baz":"qux"},` + expectedStatsListener, + StatsConfigJSON: defaultStatsConfigJSON, + }, + wantErr: false, + }, { name: "stats-flush-interval", input: BootstrapConfig{ @@ -379,6 +492,13 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantErr: true, }, + { + name: "err-bad-stats-addr", + input: BootstrapConfig{ + StatsBindAddr: "asdasdsad", + }, + wantErr: true, + }, { name: "err-bad-statsd-addr", input: BootstrapConfig{ diff --git a/test/integration/connect/envoy/case-stats-proxy/s1.hcl b/test/integration/connect/envoy/case-stats-proxy/s1.hcl new file mode 100644 index 000000000..7c12077ea --- /dev/null +++ b/test/integration/connect/envoy/case-stats-proxy/s1.hcl @@ -0,0 +1,23 @@ +services { + name = "s1" + port = 8080 + connect { + sidecar_service { + proxy { + upstreams = [ + { + destination_name = "s2" + local_bind_port = 5000 + config { + protocol = "http" + } + } + ] + config { + protocol = "http" + envoy_stats_bind_addr = "0.0.0.0:1239" + } + } + } + } +} diff --git a/test/integration/connect/envoy/case-stats-proxy/s2.hcl b/test/integration/connect/envoy/case-stats-proxy/s2.hcl new file mode 100644 index 000000000..c215cc235 --- /dev/null +++ b/test/integration/connect/envoy/case-stats-proxy/s2.hcl @@ -0,0 +1,13 @@ +services { + name = "s2" + port = 8181 + connect { + sidecar_service { + proxy { + config { + protocol = "http" + } + } + } + } +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-stats-proxy/setup.sh b/test/integration/connect/envoy/case-stats-proxy/setup.sh new file mode 100644 index 000000000..0655ef2a2 --- /dev/null +++ b/test/integration/connect/envoy/case-stats-proxy/setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -eEuo pipefail + +gen_envoy_bootstrap s1 19000 primary +gen_envoy_bootstrap s2 19001 primary \ No newline at end of file diff --git a/test/integration/connect/envoy/case-stats-proxy/verify.bats b/test/integration/connect/envoy/case-stats-proxy/verify.bats new file mode 100644 index 000000000..af89d47aa --- /dev/null +++ b/test/integration/connect/envoy/case-stats-proxy/verify.bats @@ -0,0 +1,62 @@ +#!/usr/bin/env bats + +load helpers + +@test "s1 proxy admin is up on :19000" { + retry_default curl -f -s localhost:19000/stats -o /dev/null +} + +@test "s2 proxy admin is up on :19001" { + retry_default curl -f -s localhost:19001/stats -o /dev/null +} + +@test "s1 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21000 s1 +} + +@test "s2 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21001 s2 +} + +@test "s2 proxy should be healthy" { + assert_service_has_healthy_instances s2 1 +} + +@test "s1 upstream should have healthy endpoints for s2" { + # protocol is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 1a47f6e1~s2.default.primary HEALTHY 1 +} + +@test "s1 upstream should be able to connect to s2 with http/1.1" { + run retry_default curl --http1.1 -s -f -d hello localhost:5000 + [ "$status" -eq 0 ] + [ "$output" = "hello" ] +} + +@test "s1 proxy should be exposing the /stats prefix" { + # Should have http metrics. This is just a sample one. Require the metric to + # be present not just found in a comment (anchor the regexp). + retry_default \ + must_match_in_stats_proxy_response localhost:1239 \ + 'stats' '^http.envoy_metrics.downstream_rq_active' + + # Response should include the the local cluster request. + retry_default \ + must_match_in_stats_proxy_response localhost:1239 \ + 'stats' 'cluster.local_agent.upstream_rq_active' + + # Response should include the http public listener. + retry_default \ + must_match_in_stats_proxy_response localhost:1239 \ + 'stats' 'http.public_listener_http' + + # /stats/prometheus should also be reachable and labelling the local cluster. + retry_default \ + must_match_in_stats_proxy_response localhost:1239 \ + 'stats/prometheus' '[\{,]local_cluster="s1"[,}]' + + # /stats/prometheus should also be reachable and exposing metrics. + retry_default \ + must_match_in_stats_proxy_response localhost:1239 \ + 'stats/prometheus' 'envoy_http_downstream_rq_active' +} diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index ef60136f5..a5cfdfdff 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -324,7 +324,7 @@ function get_healthy_service_count { local SERVICE_NAME=$1 local DC=$2 local NS=$3 - + run retry_default curl -s -f ${HEADERS} "127.0.0.1:8500/v1/health/connect/${SERVICE_NAME}?dc=${DC}&passing&ns=${NS}" [ "$status" -eq 0 ] echo "$output" | jq --raw-output '. | length' @@ -354,21 +354,21 @@ function assert_service_has_healthy_instances { function check_intention { local SOURCE=$1 local DESTINATION=$2 - + curl -s -f "localhost:8500/v1/connect/intentions/check?source=${SOURCE}&destination=${DESTINATION}" | jq ".Allowed" } function assert_intention_allowed { local SOURCE=$1 local DESTINATION=$2 - + [ "$(check_intention "${SOURCE}" "${DESTINATION}")" == "true" ] } function assert_intention_denied { local SOURCE=$1 local DESTINATION=$2 - + [ "$(check_intention "${SOURCE}" "${DESTINATION}")" == "false" ] } @@ -451,6 +451,18 @@ function must_match_in_prometheus_response { [ "$COUNT" -gt "0" ] } +function must_match_in_stats_proxy_response { + run curl -f -s $1/$2 + COUNT=$( echo "$output" | grep -Ec $3 ) + + echo "OUTPUT head -n 10" + echo "$output" | head -n 10 + echo "COUNT of '$3' matches: $COUNT" + + [ "$status" == 0 ] + [ "$COUNT" -gt "0" ] +} + # must_fail_tcp_connection checks that a request made through an upstream fails, # probably due to authz being denied if all other tests passed already. Although # we are using curl, this only works as expected for TCP upstreams as we are @@ -542,12 +554,12 @@ function get_intention_target_namespace { function get_intention_by_targets { local SOURCE=$1 local DESTINATION=$2 - + local SOURCE_NS=$(get_intention_target_namespace <<< "${SOURCE}") local SOURCE_NAME=$(get_intention_target_name <<< "${SOURCE}") local DESTINATION_NS=$(get_intention_target_namespace <<< "${DESTINATION}") local DESTINATION_NAME=$(get_intention_target_name <<< "${DESTINATION}") - + existing=$(list_intentions | jq ".[] | select(.SourceNS == \"$SOURCE_NS\" and .SourceName == \"$SOURCE_NAME\" and .DestinationNS == \"$DESTINATION_NS\" and .DestinationName == \"$DESTINATION_NAME\")") if test -z "$existing" then @@ -561,16 +573,16 @@ function update_intention { local SOURCE=$1 local DESTINATION=$2 local ACTION=$3 - + intention=$(get_intention_by_targets "${SOURCE}" "${DESTINATION}") if test $? -ne 0 then return 1 fi - - id=$(jq -r .ID <<< "${intention}") + + id=$(jq -r .ID <<< "${intention}") updated=$(jq ".Action = \"$ACTION\"" <<< "${intention}") - + curl -s -X PUT "http://localhost:8500/v1/connect/intentions/${id}" -d "${updated}" return $? } diff --git a/website/source/docs/connect/proxies/envoy.md b/website/source/docs/connect/proxies/envoy.md index 4c4747427..d8ac9fdb8 100644 --- a/website/source/docs/connect/proxies/envoy.md +++ b/website/source/docs/connect/proxies/envoy.md @@ -138,6 +138,11 @@ configuration entry](/docs/agent/config-entries/proxy-defaults.html). The env va -> **Note:** Envoy versions prior to 1.10 do not export timing histograms using the internal Prometheus endpoint. +- `envoy_stats_bind_addr` - Specifies that the proxy should expose the /stats prefix + to the _public_ network. It must be supplied in the form `ip:port` and + the ip/port combination must be free within the network namespace the proxy runs. + Typically the IP would be `0.0.0.0` to bind to all available interfaces or a pod IP address. + - `envoy_stats_tags` - Specifies one or more static tags that will be added to all metrics produced by the proxy. @@ -170,7 +175,7 @@ and `proxy.upstreams[*].config` fields of the [proxy service definition](/docs/connect/registration/service-registration.html) that is actually registered. -To learn about other options that can be configured centrally see the +To learn about other options that can be configured centrally see the [Configuration Entries](/docs/agent/config_entries.html) docs. ### Proxy Config Options