From e87d2bb24f042e7f9f4c8c041b6058d7a4770096 Mon Sep 17 00:00:00 2001 From: "R.B. Boyer" Date: Fri, 19 Feb 2021 14:38:43 -0600 Subject: [PATCH] xds: only try to create an ipv6 expose checks listener if ipv6 is supported by the kernel (#9765) Fixes #9311 This only fails if the kernel has ipv6 hard-disabled. It is not sufficient to merely not provide an ipv6 address for a network interface. --- .changelog/9765.txt | 3 + agent/xds/listeners.go | 20 +++- agent/xds/listeners_test.go | 54 +++++++++ agent/xds/net_fallback.go | 7 ++ agent/xds/net_linux.go | 35 ++++++ .../expose-checks.envoy-1-13-x.golden | 109 ++++++++++++++++++ .../expose-checks.envoy-1-14-x.golden | 109 ++++++++++++++++++ .../expose-checks.envoy-1-15-x.golden | 109 ++++++++++++++++++ .../expose-checks.envoy-1-16-x.golden | 109 ++++++++++++++++++ .../envoy/case-expose-checks/capture.sh | 4 + .../envoy/case-expose-checks/service_s1.hcl | 16 +++ .../envoy/case-expose-checks/service_s2.hcl | 22 ++++ .../connect/envoy/case-expose-checks/setup.sh | 8 ++ .../envoy/case-expose-checks/verify.bats | 41 +++++++ test/integration/connect/envoy/helpers.bash | 33 ++++++ 15 files changed, 674 insertions(+), 5 deletions(-) create mode 100644 .changelog/9765.txt create mode 100644 agent/xds/net_fallback.go create mode 100644 agent/xds/net_linux.go create mode 100644 agent/xds/testdata/listeners/expose-checks.envoy-1-13-x.golden create mode 100644 agent/xds/testdata/listeners/expose-checks.envoy-1-14-x.golden create mode 100644 agent/xds/testdata/listeners/expose-checks.envoy-1-15-x.golden create mode 100644 agent/xds/testdata/listeners/expose-checks.envoy-1-16-x.golden create mode 100644 test/integration/connect/envoy/case-expose-checks/capture.sh create mode 100644 test/integration/connect/envoy/case-expose-checks/service_s1.hcl create mode 100644 test/integration/connect/envoy/case-expose-checks/service_s2.hcl create mode 100644 test/integration/connect/envoy/case-expose-checks/setup.sh create mode 100644 test/integration/connect/envoy/case-expose-checks/verify.bats diff --git a/.changelog/9765.txt b/.changelog/9765.txt new file mode 100644 index 000000000..88b3c2041 --- /dev/null +++ b/.changelog/9765.txt @@ -0,0 +1,3 @@ +```release-note:improvement +xds: only try to create an ipv6 expose checks listener if ipv6 is supported by the kernel +``` diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 4b9b6defa..4471bb727 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -690,12 +690,22 @@ func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, clus advertiseLen = 128 } + ranges := make([]*envoycore.CidrRange, 0, 3) + ranges = append(ranges, + &envoycore.CidrRange{AddressPrefix: "127.0.0.1", PrefixLen: &wrappers.UInt32Value{Value: 8}}, + &envoycore.CidrRange{AddressPrefix: advertise, PrefixLen: &wrappers.UInt32Value{Value: uint32(advertiseLen)}}, + ) + + if ok, err := kernelSupportsIPv6(); err != nil { + return nil, err + } else if ok { + ranges = append(ranges, + &envoycore.CidrRange{AddressPrefix: "::1", PrefixLen: &wrappers.UInt32Value{Value: 128}}, + ) + } + chain.FilterChainMatch = &envoylistener.FilterChainMatch{ - SourcePrefixRanges: []*envoycore.CidrRange{ - {AddressPrefix: "127.0.0.1", PrefixLen: &wrappers.UInt32Value{Value: 8}}, - {AddressPrefix: "::1", PrefixLen: &wrappers.UInt32Value{Value: 128}}, - {AddressPrefix: advertise, PrefixLen: &wrappers.UInt32Value{Value: uint32(advertiseLen)}}, - }, + SourcePrefixRanges: ranges, } } diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 2772a9966..c0d3cb04e 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -6,6 +6,7 @@ import ( "sort" "testing" "text/template" + "time" envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2" "github.com/envoyproxy/go-control-plane/pkg/wellknown" @@ -16,6 +17,7 @@ import ( "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/proxysupport" "github.com/hashicorp/consul/sdk/testutil" + "github.com/hashicorp/consul/types" ) func TestListenersFromSnapshot(t *testing.T) { @@ -31,6 +33,7 @@ func TestListenersFromSnapshot(t *testing.T) { // test input. setup func(snap *proxycfg.ConfigSnapshot) overrideGoldenName string + serverSetup func(*Server) }{ { name: "defaults", @@ -304,6 +307,38 @@ func TestListenersFromSnapshot(t *testing.T) { } }, }, + { + // NOTE: if IPv6 is not supported in the kernel per + // kernelSupportsIPv6() then this test will fail because the golden + // files were generated assuming ipv6 support was present + name: "expose-checks", + create: proxycfg.TestConfigSnapshotExposeConfig, + setup: func(snap *proxycfg.ConfigSnapshot) { + snap.Proxy.Expose = structs.ExposeConfig{ + Checks: true, + } + }, + serverSetup: func(s *Server) { + s.CfgFetcher = configFetcherFunc(func() string { + return "192.0.2.1" + }) + + s.CheckFetcher = httpCheckFetcherFunc(func(sid structs.ServiceID) []structs.CheckType { + if sid != structs.NewServiceID("web", nil) { + return nil + } + return []structs.CheckType{{ + CheckID: types.CheckID("http"), + Name: "http", + HTTP: "http://127.0.0.1:8181/debug", + ProxyHTTP: "http://:21500/debug", + Method: "GET", + Interval: 10 * time.Second, + Timeout: 1 * time.Second, + }} + }) + }, + }, { name: "mesh-gateway", create: proxycfg.TestConfigSnapshotMeshGateway, @@ -519,6 +554,9 @@ func TestListenersFromSnapshot(t *testing.T) { // Need server just for logger dependency s := Server{Logger: testutil.Logger(t)} + if tt.serverSetup != nil { + tt.serverSetup(&s) + } cInfo := connectionInfo{ Token: "my-token", @@ -777,3 +815,19 @@ func customHTTPListenerJSON(t *testing.T, opts customHTTPListenerJSONOptions) st require.NoError(t, customHTTPListenerJSONTemplate.Execute(&buf, opts)) return buf.String() } + +type httpCheckFetcherFunc func(serviceID structs.ServiceID) []structs.CheckType + +var _ HTTPCheckFetcher = (httpCheckFetcherFunc)(nil) + +func (f httpCheckFetcherFunc) ServiceHTTPBasedChecks(serviceID structs.ServiceID) []structs.CheckType { + return f(serviceID) +} + +type configFetcherFunc func() string + +var _ ConfigFetcher = (configFetcherFunc)(nil) + +func (f configFetcherFunc) AdvertiseAddrLAN() string { + return f() +} diff --git a/agent/xds/net_fallback.go b/agent/xds/net_fallback.go new file mode 100644 index 000000000..08857991b --- /dev/null +++ b/agent/xds/net_fallback.go @@ -0,0 +1,7 @@ +// +build !linux + +package xds + +func kernelSupportsIPv6() (bool, error) { + return true, nil +} diff --git a/agent/xds/net_linux.go b/agent/xds/net_linux.go new file mode 100644 index 000000000..743f756cb --- /dev/null +++ b/agent/xds/net_linux.go @@ -0,0 +1,35 @@ +// +build linux + +package xds + +import ( + "fmt" + "os" + "sync" +) + +const ipv6SupportProcFile = "/proc/net/if_inet6" + +var ( + ipv6SupportOnce sync.Once + ipv6Supported bool + ipv6SupportedErr error +) + +func kernelSupportsIPv6() (bool, error) { + ipv6SupportOnce.Do(func() { + ipv6Supported, ipv6SupportedErr = checkIfKernelSupportsIPv6() + }) + return ipv6Supported, ipv6SupportedErr +} + +func checkIfKernelSupportsIPv6() (bool, error) { + _, err := os.Stat(ipv6SupportProcFile) + if os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("error checking for ipv6 support file %s: %w", ipv6SupportProcFile, err) + } + + return true, nil +} diff --git a/agent/xds/testdata/listeners/expose-checks.envoy-1-13-x.golden b/agent/xds/testdata/listeners/expose-checks.envoy-1-13-x.golden new file mode 100644 index 000000000..32fd8f41b --- /dev/null +++ b/agent/xds/testdata/listeners/expose-checks.envoy-1-13-x.golden @@ -0,0 +1,109 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "exposed_path_debug:1.2.3.4:21500", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 21500 + } + }, + "filterChains": [ + { + "filterChainMatch": { + "sourcePrefixRanges": [ + { + "addressPrefix": "127.0.0.1", + "prefixLen": 8 + }, + { + "addressPrefix": "192.0.2.1", + "prefixLen": 32 + }, + { + "addressPrefix": "::1", + "prefixLen": 128 + } + ] + }, + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http_filters": [ + { + "name": "envoy.router" + } + ], + "route_config": { + "name": "exposed_path_filter_debug_21500", + "virtual_hosts": [ + { + "domains": [ + "*" + ], + "name": "exposed_path_filter_debug_21500", + "routes": [ + { + "match": { + "path": "/debug" + }, + "route": { + "cluster": "exposed_cluster_8181" + } + } + ] + } + ] + }, + "stat_prefix": "exposed_path_filter_debug_21500", + "tracing": { + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "public_listener:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "tlsContext": { + "requireClientCertificate": true + }, + "filters": [ + { + "name": "envoy.filters.network.rbac", + "config": { + "rules": { + }, + "stat_prefix": "connect_authz" + } + }, + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "local_app", + "stat_prefix": "public_listener" + } + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/expose-checks.envoy-1-14-x.golden b/agent/xds/testdata/listeners/expose-checks.envoy-1-14-x.golden new file mode 100644 index 000000000..32fd8f41b --- /dev/null +++ b/agent/xds/testdata/listeners/expose-checks.envoy-1-14-x.golden @@ -0,0 +1,109 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "exposed_path_debug:1.2.3.4:21500", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 21500 + } + }, + "filterChains": [ + { + "filterChainMatch": { + "sourcePrefixRanges": [ + { + "addressPrefix": "127.0.0.1", + "prefixLen": 8 + }, + { + "addressPrefix": "192.0.2.1", + "prefixLen": 32 + }, + { + "addressPrefix": "::1", + "prefixLen": 128 + } + ] + }, + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http_filters": [ + { + "name": "envoy.router" + } + ], + "route_config": { + "name": "exposed_path_filter_debug_21500", + "virtual_hosts": [ + { + "domains": [ + "*" + ], + "name": "exposed_path_filter_debug_21500", + "routes": [ + { + "match": { + "path": "/debug" + }, + "route": { + "cluster": "exposed_cluster_8181" + } + } + ] + } + ] + }, + "stat_prefix": "exposed_path_filter_debug_21500", + "tracing": { + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "public_listener:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "tlsContext": { + "requireClientCertificate": true + }, + "filters": [ + { + "name": "envoy.filters.network.rbac", + "config": { + "rules": { + }, + "stat_prefix": "connect_authz" + } + }, + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "local_app", + "stat_prefix": "public_listener" + } + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/expose-checks.envoy-1-15-x.golden b/agent/xds/testdata/listeners/expose-checks.envoy-1-15-x.golden new file mode 100644 index 000000000..32fd8f41b --- /dev/null +++ b/agent/xds/testdata/listeners/expose-checks.envoy-1-15-x.golden @@ -0,0 +1,109 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "exposed_path_debug:1.2.3.4:21500", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 21500 + } + }, + "filterChains": [ + { + "filterChainMatch": { + "sourcePrefixRanges": [ + { + "addressPrefix": "127.0.0.1", + "prefixLen": 8 + }, + { + "addressPrefix": "192.0.2.1", + "prefixLen": 32 + }, + { + "addressPrefix": "::1", + "prefixLen": 128 + } + ] + }, + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http_filters": [ + { + "name": "envoy.router" + } + ], + "route_config": { + "name": "exposed_path_filter_debug_21500", + "virtual_hosts": [ + { + "domains": [ + "*" + ], + "name": "exposed_path_filter_debug_21500", + "routes": [ + { + "match": { + "path": "/debug" + }, + "route": { + "cluster": "exposed_cluster_8181" + } + } + ] + } + ] + }, + "stat_prefix": "exposed_path_filter_debug_21500", + "tracing": { + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "public_listener:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "tlsContext": { + "requireClientCertificate": true + }, + "filters": [ + { + "name": "envoy.filters.network.rbac", + "config": { + "rules": { + }, + "stat_prefix": "connect_authz" + } + }, + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "local_app", + "stat_prefix": "public_listener" + } + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/expose-checks.envoy-1-16-x.golden b/agent/xds/testdata/listeners/expose-checks.envoy-1-16-x.golden new file mode 100644 index 000000000..32fd8f41b --- /dev/null +++ b/agent/xds/testdata/listeners/expose-checks.envoy-1-16-x.golden @@ -0,0 +1,109 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "exposed_path_debug:1.2.3.4:21500", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 21500 + } + }, + "filterChains": [ + { + "filterChainMatch": { + "sourcePrefixRanges": [ + { + "addressPrefix": "127.0.0.1", + "prefixLen": 8 + }, + { + "addressPrefix": "192.0.2.1", + "prefixLen": 32 + }, + { + "addressPrefix": "::1", + "prefixLen": 128 + } + ] + }, + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http_filters": [ + { + "name": "envoy.router" + } + ], + "route_config": { + "name": "exposed_path_filter_debug_21500", + "virtual_hosts": [ + { + "domains": [ + "*" + ], + "name": "exposed_path_filter_debug_21500", + "routes": [ + { + "match": { + "path": "/debug" + }, + "route": { + "cluster": "exposed_cluster_8181" + } + } + ] + } + ] + }, + "stat_prefix": "exposed_path_filter_debug_21500", + "tracing": { + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "public_listener:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "tlsContext": { + "requireClientCertificate": true + }, + "filters": [ + { + "name": "envoy.filters.network.rbac", + "config": { + "rules": { + }, + "stat_prefix": "connect_authz" + } + }, + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "local_app", + "stat_prefix": "public_listener" + } + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-expose-checks/capture.sh b/test/integration/connect/envoy/case-expose-checks/capture.sh new file mode 100644 index 000000000..af3245f34 --- /dev/null +++ b/test/integration/connect/envoy/case-expose-checks/capture.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:19000 s1 || true +snapshot_envoy_admin localhost:19001 s2 || true diff --git a/test/integration/connect/envoy/case-expose-checks/service_s1.hcl b/test/integration/connect/envoy/case-expose-checks/service_s1.hcl new file mode 100644 index 000000000..7f48b1c51 --- /dev/null +++ b/test/integration/connect/envoy/case-expose-checks/service_s1.hcl @@ -0,0 +1,16 @@ +services { + name = "s1" + port = 8080 + connect { + sidecar_service { + proxy { + upstreams = [ + { + destination_name = "s2" + local_bind_port = 5000 + } + ] + } + } + } +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-expose-checks/service_s2.hcl b/test/integration/connect/envoy/case-expose-checks/service_s2.hcl new file mode 100644 index 000000000..ce2fe46c8 --- /dev/null +++ b/test/integration/connect/envoy/case-expose-checks/service_s2.hcl @@ -0,0 +1,22 @@ +services { + name = "s2" + port = 8181 + connect { + sidecar_service { + proxy { + expose { + checks = true + } + } + } + } + checks = [ + { + name = "http" + http = "http://127.0.0.1:8181/debug" + method = "GET" + interval = "10s" + timeout = "1s" + }, + ] +} diff --git a/test/integration/connect/envoy/case-expose-checks/setup.sh b/test/integration/connect/envoy/case-expose-checks/setup.sh new file mode 100644 index 000000000..e8321972b --- /dev/null +++ b/test/integration/connect/envoy/case-expose-checks/setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eEuo pipefail + +register_services primary + +gen_envoy_bootstrap s1 19000 primary +gen_envoy_bootstrap s2 19001 primary diff --git a/test/integration/connect/envoy/case-expose-checks/verify.bats b/test/integration/connect/envoy/case-expose-checks/verify.bats new file mode 100644 index 000000000..0c4dcbb2c --- /dev/null +++ b/test/integration/connect/envoy/case-expose-checks/verify.bats @@ -0,0 +1,41 @@ +#!/usr/bin/env bats + +load helpers + +@test "s1 proxy is running correct version" { + assert_envoy_version 19000 +} + +@test "s1 proxy admin is up on :19000" { + retry_default curl -f -s localhost:19000/stats -o /dev/null +} + +@test "s2 proxy admin is up on :19001" { + retry_default curl -f -s localhost:19001/stats -o /dev/null +} + +@test "s1 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21000 s1 +} + +@test "s2 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21001 s2 +} + +@test "s2 proxy should be healthy" { + assert_service_has_healthy_instances s2 1 +} + +@test "s1 upstream should have healthy endpoints for s2" { + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 +} + +@test "s1 upstream should be able to connect to s2" { + run retry_default curl -s -f -d hello localhost:5000 + [ "$status" -eq 0 ] + [ "$output" = "hello" ] +} + +@test "s2 exposes checks on a new listener" { + assert_envoy_expose_checks_listener_count localhost:19001 /debug +} diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index 63577e0e2..3a593d1ae 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -152,6 +152,39 @@ function assert_envoy_version { echo $VERSION | grep "/$ENVOY_VERSION/" } +function assert_envoy_expose_checks_listener_count { + local HOSTPORT=$1 + local EXPECT_PATH=$2 + + # scrape this once + BODY=$(get_envoy_expose_checks_listener_once $HOSTPORT) + echo "BODY = $BODY" + + CHAINS=$(echo "$BODY" | jq '.active_state.listener.filter_chains | length') + echo "CHAINS = $CHAINS (expect 1)" + [ "${CHAINS:-0}" -eq 1 ] + + RANGES=$(echo "$BODY" | jq '.active_state.listener.filter_chains[0].filter_chain_match.source_prefix_ranges | length') + echo "RANGES = $RANGES (expect 3)" + # note: if IPv6 is not supported in the kernel per + # agent/xds:kernelSupportsIPv6() then this will only be 2 + [ "${RANGES:-0}" -eq 3 ] + + HCM=$(echo "$BODY" | jq '.active_state.listener.filter_chains[0].filters[0]') + HCM_NAME=$(echo "$HCM" | jq -r '.name') + HCM_PATH=$(echo "$HCM" | jq -r '.config.route_config.virtual_hosts[0].routes[0].match.path') + echo "HCM = $HCM" + [ "${HCM_NAME:-}" == "envoy.http_connection_manager" ] + [ "${HCM_PATH:-}" == "${EXPECT_PATH}" ] +} + +function get_envoy_expose_checks_listener_once { + local HOSTPORT=$1 + run curl -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output '.configs[] | select(.["@type"] == "type.googleapis.com/envoy.admin.v3.ListenersConfigDump") | .dynamic_listeners[] | select(.name | startswith("exposed_path_"))' +} + function assert_envoy_http_rbac_policy_count { local HOSTPORT=$1 local EXPECT_COUNT=$2