From a0bcf4b941f7a45883d373d79b5c23992521140a Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 10 Oct 2022 13:55:28 -0600 Subject: [PATCH] Add integ test for peering through gateways --- .../alpha/base.hcl | 5 ++ .../alpha/config_entries.hcl | 32 +++++++++ .../alpha/service_gateway.hcl | 5 ++ .../alpha/service_s1.hcl | 1 + .../alpha/service_s2.hcl | 7 ++ .../alpha/setup.sh | 12 ++++ .../alpha/verify.bats | 49 +++++++++++++ .../bind.hcl | 2 + .../capture.sh | 4 ++ .../primary/base.hcl | 3 + .../primary/config_entries.hcl | 18 +++++ .../primary/service_gateway.hcl | 5 ++ .../primary/service_s1.hcl | 20 ++++++ .../primary/service_s2.hcl | 1 + .../primary/setup.sh | 11 +++ .../primary/verify.bats | 69 +++++++++++++++++++ .../case-cross-peer-control-plane-mgw/vars.sh | 4 ++ 17 files changed, 248 insertions(+) create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/base.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/config_entries.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_gateway.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_s1.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_s2.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/setup.sh create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/verify.bats create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/bind.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/capture.sh create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/base.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/config_entries.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_gateway.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_s1.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_s2.hcl create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/setup.sh create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats create mode 100644 test/integration/connect/envoy/case-cross-peer-control-plane-mgw/vars.sh diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/base.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/base.hcl new file mode 100644 index 000000000..f81ab0edd --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/base.hcl @@ -0,0 +1,5 @@ +primary_datacenter = "alpha" +log_level = "trace" +peering { + enabled = true +} diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/config_entries.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/config_entries.hcl new file mode 100644 index 000000000..996df1d21 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/config_entries.hcl @@ -0,0 +1,32 @@ +config_entries { + bootstrap = [ + { + kind = "proxy-defaults" + name = "global" + + config { + protocol = "tcp" + } + }, + { + kind = "mesh" + peering { + peer_through_mesh_gateways = true + } + }, + { + kind = "exported-services" + name = "default" + services = [ + { + name = "s2" + consumers = [ + { + peer = "alpha-to-primary" + } + ] + } + ] + } + ] +} diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_gateway.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_gateway.hcl new file mode 100644 index 000000000..bcdcb2e8b --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_gateway.hcl @@ -0,0 +1,5 @@ +services { + name = "mesh-gateway" + kind = "mesh-gateway" + port = 4432 +} diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_s1.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_s1.hcl new file mode 100644 index 000000000..e97ec2366 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_s1.hcl @@ -0,0 +1 @@ +# We don't want an s1 service in this peer diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_s2.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_s2.hcl new file mode 100644 index 000000000..01d4505c6 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/service_s2.hcl @@ -0,0 +1,7 @@ +services { + name = "s2" + port = 8181 + connect { + sidecar_service {} + } +} diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/setup.sh b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/setup.sh new file mode 100644 index 000000000..6d341b20a --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +register_services alpha + +gen_envoy_bootstrap s2 19002 alpha +gen_envoy_bootstrap mesh-gateway 19003 alpha true + +wait_for_config_entry proxy-defaults global alpha +wait_for_config_entry exported-services default alpha +wait_for_config_entry mesh mesh alpha diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/verify.bats b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/verify.bats new file mode 100644 index 000000000..06314df90 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/alpha/verify.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats + +load helpers + +@test "s2 proxy is running correct version" { + assert_envoy_version 19002 +} + +@test "s2 proxy admin is up on :19002" { + retry_default curl -f -s localhost:19002/stats -o /dev/null +} + +@test "gateway-alpha proxy admin is up on :19003" { + retry_default curl -f -s localhost:19003/stats -o /dev/null +} + +@test "s2 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21000 s2 alpha +} + +@test "s2 proxy should be healthy" { + assert_service_has_healthy_instances s2 1 alpha +} + +@test "gateway-alpha should be up and listening" { + retry_long nc -z consul-alpha-client:4432 +} + +@test "s2 proxies should be healthy" { + assert_service_has_healthy_instances s2 1 alpha +} + +@test "dialer gateway-alpha should have healthy endpoints for alpha servers" { + assert_upstream_has_endpoints_in_status 127.0.0.1:19003 server.alpha.peering HEALTHY 1 +} + +@test "dialer gateway-alpha should have healthy endpoints for primary servers" { + assert_upstream_has_endpoints_in_status 127.0.0.1:19003 server.primary.peering HEALTHY 1 +} + +# Re-peering the clusters is a way to have alpha dial out through its own gateway +# since we know it is configured with endpoints for primary from the first time they peered. +@test "re-peer the two clusters together" { + create_peering primary alpha +} + +@test "alpha servers made connection to primary servers via alpha gateway" { + assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.server.primary.peering.*cx_total" 1 +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/bind.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/bind.hcl new file mode 100644 index 000000000..f54393f03 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/bind.hcl @@ -0,0 +1,2 @@ +bind_addr = "0.0.0.0" +advertise_addr = "{{ GetInterfaceIP \"eth0\" }}" \ No newline at end of file diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/capture.sh b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/capture.sh new file mode 100644 index 000000000..f7205ae51 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/capture.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:19001 mesh-gateway primary || true +snapshot_envoy_admin localhost:19003 mesh-gateway alpha || true diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/base.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/base.hcl new file mode 100644 index 000000000..c1e134d5a --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/base.hcl @@ -0,0 +1,3 @@ +peering { + enabled = true +} diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/config_entries.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/config_entries.hcl new file mode 100644 index 000000000..6baeb569f --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/config_entries.hcl @@ -0,0 +1,18 @@ +config_entries { + bootstrap = [ + { + kind = "proxy-defaults" + name = "global" + + config { + protocol = "tcp" + } + }, + { + kind = "mesh" + peering { + peer_through_mesh_gateways = true + } + } + ] +} diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_gateway.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_gateway.hcl new file mode 100644 index 000000000..831a70ff3 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_gateway.hcl @@ -0,0 +1,5 @@ +services { + name = "mesh-gateway" + kind = "mesh-gateway" + port = 4431 +} diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_s1.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_s1.hcl new file mode 100644 index 000000000..af0773c29 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_s1.hcl @@ -0,0 +1,20 @@ +services { + name = "s1" + port = 8080 + connect { + sidecar_service { + proxy { + upstreams = [ + { + destination_name = "s2" + destination_peer = "primary-to-alpha" + local_bind_port = 5000 + mesh_gateway { + mode = "local" + } + } + ] + } + } + } +} diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_s2.hcl b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_s2.hcl new file mode 100644 index 000000000..77164e722 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/service_s2.hcl @@ -0,0 +1 @@ +# We don't want an s2 service in the primary dc \ No newline at end of file diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/setup.sh b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/setup.sh new file mode 100644 index 000000000..3aa37f8cb --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euo pipefail + +register_services primary + +gen_envoy_bootstrap s1 19000 primary +gen_envoy_bootstrap mesh-gateway 19001 primary true + +wait_for_config_entry proxy-defaults global +wait_for_config_entry mesh mesh alpha diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats new file mode 100644 index 000000000..bc21d5ee3 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats @@ -0,0 +1,69 @@ +#!/usr/bin/env bats + +load helpers + +@test "s1 proxy is running correct version" { + assert_envoy_version 19000 +} + +@test "s1 proxy admin is up on :19000" { + retry_default curl -f -s localhost:19000/stats -o /dev/null +} + +@test "gateway-primary proxy admin is up on :19001" { + retry_default curl -f -s localhost:19001/stats -o /dev/null +} + +@test "s1 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21000 s1 +} + +@test "s2 proxies should be healthy in alpha" { + assert_service_has_healthy_instances s2 1 alpha +} + +@test "gateway-primary should be up and listening" { + retry_long nc -z consul-primary-client:4431 +} + +@test "gateway-alpha should be up and listening" { + retry_long nc -z consul-alpha-client:4432 +} + +@test "peer the two clusters together" { + create_peering primary alpha +} + +@test "acceptor gateway-primary should have healthy endpoints for primary servers" { + assert_upstream_has_endpoints_in_status 127.0.0.1:19001 server.primary.peering HEALTHY 1 +} + +@test "alpha servers made connection to primary servers via primary gateway" { + assert_envoy_metric_at_least 127.0.0.1:19001 "cluster.server.primary.peering.*cx_total" 1 +} + +@test "s2 alpha proxies should be healthy in primary" { + assert_service_has_healthy_instances s2 1 primary "" "" primary-to-alpha +} + +@test "gateway-alpha should have healthy endpoints for s2" { + assert_upstream_has_endpoints_in_status consul-alpha-client:19003 exported~s2.default.alpha HEALTHY 1 +} + +@test "s1 upstream should have healthy endpoints for s2" { + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary-to-alpha.external HEALTHY 1 +} + +@test "s1 upstream should be able to connect to s2" { + run retry_default curl -s -f -d hello localhost:5000 + [ "$status" -eq 0 ] + [ "$output" = "hello" ] +} + +@test "s1 upstream made 1 connection to s2" { + assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary-to-alpha.external.*cx_total" 1 +} + +@test "s1 upstream made 1 connection to s2 through the primary mesh gateway" { + assert_envoy_metric_at_least 127.0.0.1:19001 "cluster.s2.default.default.alpha-to-primary.external.*cx_total" 1 +} diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/vars.sh b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/vars.sh new file mode 100644 index 000000000..c856de173 --- /dev/null +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/vars.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +export REQUIRED_SERVICES="s1 s1-sidecar-proxy gateway-primary s2-alpha s2-sidecar-proxy-alpha gateway-alpha" +export REQUIRE_PEERS=1