Add peering integration tests (#14836)

Add peering integration tests.

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/alpha/base.hcl

@@ -0,0 +1,5 @@
+primary_datacenter = "alpha"
+log_level          = "trace"
+peering {
+  enabled = true
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/alpha/config_entries.hcl

@@ -0,0 +1,26 @@
+config_entries {
+  bootstrap = [
+    {
+      kind = "proxy-defaults"
+      name = "global"
+
+      config {
+        protocol = "tcp"
+      }
+    },
+    {
+      kind = "exported-services"
+      name = "default"
+      services = [
+        {
+          name = "s2"
+          consumers = [
+            {
+              peer_name = "alpha-to-primary"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/alpha/service_gateway.hcl

@@ -0,0 +1,5 @@
+services {
+  name = "mesh-gateway"
+  kind = "mesh-gateway"
+  port = 4432
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/alpha/service_s1.hcl

@@ -0,0 +1 @@
+# We don't want an s1 service in this peer

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/alpha/service_s2.hcl

@@ -0,0 +1,10 @@
+services {
+  name = "s2"
+  port = 8181
+  checks = []
+  connect {
+    sidecar_service {
+      checks = []
+    }
+  }
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/alpha/setup.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euo pipefail
+
+register_services alpha
+
+gen_envoy_bootstrap s2 19002 alpha
+gen_envoy_bootstrap mesh-gateway 19003 alpha true
+
+wait_for_config_entry proxy-defaults global alpha
+wait_for_config_entry exported-services default alpha

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/alpha/verify.bats

@@ -0,0 +1,27 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "s2 proxy is running correct version" {
+  assert_envoy_version 19002
+}
+
+@test "s2 proxy admin is up on :19002" {
+  retry_default curl -f -s localhost:19002/stats -o /dev/null
+}
+
+@test "gateway-alpha proxy admin is up on :19003" {
+  retry_default curl -f -s localhost:19003/stats -o /dev/null
+}
+
+@test "s2 proxy listener should be up and have right cert" {
+  assert_proxy_presents_cert_uri localhost:21000 s2 alpha
+}
+
+@test "s2 proxy should be healthy" {
+  assert_service_has_healthy_instances s2 1 alpha
+}
+
+@test "gateway-alpha should be up and listening" {
+  retry_long nc -z consul-alpha-client:4432
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/bind.hcl

@@ -0,0 +1,2 @@
+bind_addr = "0.0.0.0"
+advertise_addr = "{{ GetInterfaceIP \"eth0\" }}"

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/capture.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+snapshot_envoy_admin localhost:19000 s1 primary || true
+snapshot_envoy_admin localhost:19001 s2 primary || true
+snapshot_envoy_admin localhost:19002 s2 alpha || true
+snapshot_envoy_admin localhost:19003 mesh-gateway alpha || true

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/base.hcl

@@ -0,0 +1,3 @@
+peering {
+  enabled = true
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/config_entries.hcl

@@ -0,0 +1,53 @@
+config_entries {
+  bootstrap {
+    kind = "proxy-defaults"
+    name = "global"
+
+    config {
+      protocol = "http"
+    }
+  }
+
+  bootstrap {
+    kind = "service-splitter"
+    name = "split-s2"
+    splits = [
+      {
+        Weight = 50
+        Service = "local-s2"
+        ResponseHeaders {
+          Set {
+            "x-test-split" = "primary"
+          }
+        }
+      },
+      {
+        Weight = 50
+        Service = "peer-s2"
+        ResponseHeaders {
+          Set {
+            "x-test-split" = "alpha"
+          }
+        }
+      },
+    ]
+  }
+
+  bootstrap {
+    kind = "service-resolver"
+    name = "local-s2"
+    redirect = {
+      service = "s2"
+    }
+  }
+
+  bootstrap {
+    kind = "service-resolver"
+    name = "peer-s2"
+
+    redirect = {
+      service = "s2"
+      peer = "primary-to-alpha"
+    }
+  }
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/service_s1.hcl

@@ -0,0 +1,22 @@
+services {
+  name = "s1"
+  port = 8080
+  checks = []
+  connect {
+    sidecar_service {
+      checks = []
+      proxy {
+        upstreams = [
+          {
+            destination_name = "split-s2"
+            local_bind_port  = 5000
+          },
+          {
+            destination_name = "peer-s2"
+            local_bind_port  = 5001
+          }
+        ]
+      }
+    }
+  }
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/service_s2.hcl

@@ -0,0 +1,10 @@
+services {
+  name = "s2"
+  port = 8181
+  checks = []
+  connect {
+    sidecar_service {
+      checks = []
+    }
+  }
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/setup.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -euo pipefail
+
+register_services primary
+
+gen_envoy_bootstrap s1 19000 primary
+gen_envoy_bootstrap s2 19001 primary
+
+wait_for_config_entry proxy-defaults global

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/verify.bats

@@ -0,0 +1,74 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "s1 proxy is running correct version" {
+  assert_envoy_version 19000
+}
+
+@test "s1 proxy admin is up" {
+  retry_default curl -f -s localhost:19000/stats -o /dev/null
+}
+
+@test "s2 proxy admin is up" {
+  retry_default curl -f -s localhost:19001/stats -o /dev/null
+}
+
+@test "s1 proxy listener should be up and have right cert" {
+  assert_proxy_presents_cert_uri localhost:21000 s1
+}
+
+@test "s2 proxies should be healthy in primary" {
+  assert_service_has_healthy_instances s2 1 primary
+}
+
+@test "s2 proxies should be healthy in alpha" {
+  assert_service_has_healthy_instances s2 1 alpha
+}
+
+@test "gateway-alpha should be up and listening" {
+  retry_long nc -z consul-alpha-client:4432
+}
+
+@test "peer the two clusters together" {
+  create_peering primary alpha
+}
+
+@test "s2 alpha proxies should be healthy in primary" {
+  assert_service_has_healthy_instances s2 1 primary "" "" primary-to-alpha
+}
+
+@test "s1 upstream should have healthy endpoints for s2 primary and alpha" {
+  assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary.internal HEALTHY 1
+  assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary-to-alpha.external HEALTHY 1
+}
+
+@test "s1 upstream should be split between peer and local dc" {
+  retry_long assert_url_header "http://127.0.0.1:5000/" "x-test-split" "primary"
+  [ "$status" -eq 0 ]
+  retry_long assert_url_header "http://127.0.0.1:5000/" "x-test-split" "alpha"
+  [ "$status" -eq 0 ]
+  retry_long assert_expected_fortio_name s2 127.0.0.1 5000
+  retry_long assert_expected_fortio_name s2-alpha 127.0.0.1 5000
+}
+
+@test "s1 upstream made 2 connections to primary s2 split" {
+  retry_long assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.internal.*upstream_rq_total" 1
+}
+
+@test "s1 upstream made 2 connections to alpha s2 split" {
+  retry_long assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary-to-alpha.external.*upstream_rq_total" 1
+}
+
+@test "reset envoy statistics" {
+  reset_envoy_metrics 127.0.0.1:19000
+  retry_long assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary-to-alpha.external.*upstream_rq_total" 0
+}
+
+@test "s1 upstream should be able to connect to s2 via peer-s2" {
+  assert_expected_fortio_name s2-alpha 127.0.0.1 5001
+}
+
+@test "s1 upstream made 1 connection to s2 via peer-s2" {
+  retry_long assert_envoy_metric_at_least 127.0.0.1:19000 "http.upstream.peer-s2.default.default.primary.rq_total" 1
+}

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/vars.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export REQUIRED_SERVICES="s1 s1-sidecar-proxy s2 s2-sidecar-proxy s2-alpha s2-sidecar-proxy-alpha gateway-alpha"
+export REQUIRE_PEERS=1

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/alpha/base.hcl

@@ -0,0 +1,5 @@
+primary_datacenter = "alpha"
+log_level          = "trace"
+peering {
+  enabled = true
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/alpha/config_entries.hcl

@@ -0,0 +1,34 @@
+config_entries {
+  bootstrap = [
+    {
+      kind = "proxy-defaults"
+      name = "global"
+
+      config {
+        protocol = "tcp"
+      }
+    },
+    {
+      kind = "exported-services"
+      name = "default"
+      services = [
+        {
+          name = "s1"
+          consumers = [
+            {
+              peer_name = "alpha-to-primary"
+            }
+          ]
+        },
+        {
+          name = "s2"
+          consumers = [
+            {
+              peer_name = "alpha-to-primary"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/alpha/service_gateway.hcl

@@ -0,0 +1,5 @@
+services {
+  name = "mesh-gateway"
+  kind = "mesh-gateway"
+  port = 4432
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/alpha/service_s1.hcl

@@ -0,0 +1,10 @@
+services {
+  name = "s1"
+  port = 8080
+  checks = []
+  connect {
+    sidecar_service {
+      checks = []
+    }
+  }
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/alpha/service_s2.hcl

@@ -0,0 +1,10 @@
+services {
+  name = "s2"
+  port = 8181
+  checks = []
+  connect {
+    sidecar_service {
+      checks = []
+    }
+  }
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/alpha/setup.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -euo pipefail
+
+wait_for_config_entry proxy-defaults global alpha
+wait_for_config_entry exported-services default alpha
+
+register_services alpha
+
+gen_envoy_bootstrap s1 19001 alpha
+gen_envoy_bootstrap s2 19002 alpha
+gen_envoy_bootstrap mesh-gateway 19003 alpha true

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/alpha/verify.bats

@@ -0,0 +1,34 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "proxies are running correct version" {
+  assert_envoy_version 19001
+  assert_envoy_version 19002
+}
+
+@test "s1 proxy admin is up on :19001" {
+  retry_default curl -f -s localhost:19001/stats -o /dev/null
+}
+
+@test "s2 proxy admin is up on :19002" {
+  retry_default curl -f -s localhost:19002/stats -o /dev/null
+}
+
+@test "gateway-alpha proxy admin is up on :19003" {
+  retry_default curl -f -s localhost:19003/stats -o /dev/null
+}
+
+@test "proxy listeners should be up and have right cert" {
+  assert_proxy_presents_cert_uri localhost:21000 s1 alpha
+  assert_proxy_presents_cert_uri localhost:21001 s2 alpha
+}
+
+@test "s1 and s2 proxies should be healthy" {
+  assert_service_has_healthy_instances s1 1 alpha
+  assert_service_has_healthy_instances s2 1 alpha
+}
+
+@test "gateway-alpha should be up and listening" {
+  retry_long nc -z consul-alpha-client:4432
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/bind.hcl

@@ -0,0 +1,2 @@
+bind_addr = "0.0.0.0"
+advertise_addr = "{{ GetInterfaceIP \"eth0\" }}"

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/capture.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+snapshot_envoy_admin localhost:20000 ingress-gateway primary || true
+snapshot_envoy_admin localhost:19000 s1 primary || true
+snapshot_envoy_admin localhost:19001 s1 alpha || true
+snapshot_envoy_admin localhost:19002 s2 alpha || true
+snapshot_envoy_admin localhost:19003 mesh-gateway alpha || true

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/base.hcl

@@ -0,0 +1,3 @@
+peering {
+  enabled = true
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/config_entries.hcl

@@ -0,0 +1,88 @@
+config_entries {
+
+  bootstrap {
+    kind = "proxy-defaults"
+    name = "global"
+    config {
+      protocol = "http"
+    }
+  }
+
+  bootstrap {
+    kind = "ingress-gateway"
+    name = "ingress-gateway"
+    listeners = [
+      {
+        protocol = "http"
+        port     = 9999
+        services = [
+          {
+            name = "peer-s2"
+          }
+        ]
+      },
+      {
+        protocol = "http"
+        port     = 10000
+        services = [
+          {
+            name = "peer-s1"
+          }
+        ]
+      },
+      {
+        protocol = "http"
+        port     = 10001
+        services = [
+          {
+            name = "s1"
+          }
+        ]
+      },
+      {
+        protocol = "http"
+        port     = 10002
+        services = [
+          {
+            name = "split"
+          }
+        ]
+      }
+    ]
+  }
+
+  bootstrap {
+    kind = "service-resolver"
+    name = "peer-s1"
+
+    redirect = {
+      service = "s1"
+      peer = "primary-to-alpha"
+    }
+  }
+
+  bootstrap {
+    kind = "service-resolver"
+    name = "peer-s2"
+
+    redirect = {
+      service = "s2"
+      peer = "primary-to-alpha"
+    }
+  }
+
+  bootstrap {
+    kind = "service-splitter"
+    name = "split"
+    splits = [
+      {
+        Weight = 50
+        Service = "peer-s1"
+      },
+      {
+        Weight = 50
+        Service = "peer-s2"
+      },
+    ]
+  }
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/service_ingress.hcl

@@ -0,0 +1,4 @@
+services {
+  name = "ingress-gateway"
+  kind = "ingress-gateway"
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/setup.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -eEuo pipefail
+
+# wait for bootstrap to apply config entries
+wait_for_config_entry ingress-gateway ingress-gateway
+wait_for_config_entry proxy-defaults global
+wait_for_config_entry service-resolver peer-s1
+wait_for_config_entry service-resolver peer-s2
+
+register_services primary
+
+gen_envoy_bootstrap ingress-gateway 20000 primary true
+gen_envoy_bootstrap s1 19000 primary

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/verify.bats

@@ -0,0 +1,70 @@
+#!/usr/bin/env bats
+
+load helpers
+
+
+@test "ingress-primary proxy admin is up" {
+  retry_default curl -f -s localhost:20000/stats -o /dev/null
+}
+
+@test "s1 proxy admin is up" {
+  retry_default curl -f -s localhost:19000/stats -o /dev/null
+}
+
+@test "services should be healthy in primary" {
+  assert_service_has_healthy_instances s1 1 alpha
+}
+
+@test "services should be healthy in alpha" {
+  assert_service_has_healthy_instances s1 1 alpha
+  assert_service_has_healthy_instances s2 1 alpha
+}
+
+@test "mesh-gateway should have healthy endpoints" {
+  assert_upstream_has_endpoints_in_status consul-alpha-client:19003 s1 HEALTHY 1
+  assert_upstream_has_endpoints_in_status consul-alpha-client:19003 s2 HEALTHY 1
+}
+
+@test "peer the two clusters together" {
+  create_peering primary alpha
+}
+
+@test "s1 alpha proxies should be healthy in primary" {
+  assert_service_has_healthy_instances s1 1 primary "" "" primary-to-alpha
+}
+
+@test "s2 alpha proxies should be healthy in primary" {
+  assert_service_has_healthy_instances s2 1 primary "" "" primary-to-alpha
+}
+
+@test "ingress should have healthy endpoints to alpha s1 and s2" {
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1.default.primary-to-alpha.external HEALTHY 1
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2.default.primary-to-alpha.external HEALTHY 1
+}
+
+@test "requests through ingress should proxy to alpha" {
+  assert_expected_fortio_name s1-alpha peer-s1.ingress.consul 10000
+  assert_expected_fortio_name s2-alpha peer-s2.ingress.consul 9999
+}
+
+@test "ingress made 1 connection to alpha s1" {
+  assert_envoy_metric_at_least 127.0.0.1:20000 "cluster.s1.default.primary-to-alpha.external.*cx_total" 1
+}
+
+@test "ingress made 1 connection to alpha s2" {
+  assert_envoy_metric_at_least 127.0.0.1:20000 "cluster.s2.default.primary-to-alpha.external.*cx_total" 1
+}
+
+@test "no requests contacted primary s1" {
+  assert_envoy_metric 127.0.0.1:19000 "http.public_listener.rq_total" 0
+}
+
+@test "requests through ingress should proxy to primary s1" {
+  assert_expected_fortio_name s1 s1.ingress.consul 10001
+  assert_envoy_metric 127.0.0.1:19000 "http.public_listener.rq_total" 1
+}
+
+@test "requests through ingress to splitter should go to alpha" {
+  retry_long assert_expected_fortio_name s1-alpha split.ingress.consul 10002
+  retry_long assert_expected_fortio_name s2-alpha split.ingress.consul 10002
+}

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/vars.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export REQUIRED_SERVICES="s1 s1-sidecar-proxy s1-alpha s1-sidecar-proxy-alpha s2-alpha s2-sidecar-proxy-alpha gateway-alpha ingress-gateway-primary"
+export REQUIRE_PEERS=1

test/integration/connect/envoy/case-ingress-gateway-peering-failover/alpha/base.hcl

@@ -0,0 +1,5 @@
+primary_datacenter = "alpha"
+log_level          = "trace"
+peering {
+  enabled = true
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/alpha/config_entries.hcl

@@ -0,0 +1,26 @@
+config_entries {
+  bootstrap = [
+    {
+      kind = "proxy-defaults"
+      name = "global"
+
+      config {
+        protocol = "tcp"
+      }
+    },
+    {
+      kind = "exported-services"
+      name = "default"
+      services = [
+        {
+          name = "s2"
+          consumers = [
+            {
+              peer_name = "alpha-to-primary"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/alpha/service_gateway.hcl

@@ -0,0 +1,5 @@
+services {
+  name = "mesh-gateway"
+  kind = "mesh-gateway"
+  port = 4432
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/alpha/service_s1.hcl

@@ -0,0 +1 @@
+# We don't want an s1 service in this peer

test/integration/connect/envoy/case-ingress-gateway-peering-failover/alpha/service_s2.hcl

@@ -0,0 +1,10 @@
+services {
+  name = "s2"
+  port = 8181
+  checks = []
+  connect {
+    sidecar_service {
+      checks = []
+    }
+  }
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/alpha/setup.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euo pipefail
+
+register_services alpha
+
+gen_envoy_bootstrap s2 19002 alpha
+gen_envoy_bootstrap mesh-gateway 19003 alpha true
+
+wait_for_config_entry proxy-defaults global alpha
+wait_for_config_entry exported-services default alpha

test/integration/connect/envoy/case-ingress-gateway-peering-failover/alpha/verify.bats

@@ -0,0 +1,27 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "s2 proxy is running correct version" {
+  assert_envoy_version 19002
+}
+
+@test "s2 proxy admin is up on :19002" {
+  retry_default curl -f -s localhost:19002/stats -o /dev/null
+}
+
+@test "gateway-alpha proxy admin is up on :19003" {
+  retry_default curl -f -s localhost:19003/stats -o /dev/null
+}
+
+@test "s2 proxy listener should be up and have right cert" {
+  assert_proxy_presents_cert_uri localhost:21000 s2 alpha
+}
+
+@test "s2 proxy should be healthy" {
+  assert_service_has_healthy_instances s2 1 alpha
+}
+
+@test "gateway-alpha should be up and listening" {
+  retry_long nc -z consul-alpha-client:4432
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/bind.hcl

@@ -0,0 +1,2 @@
+bind_addr = "0.0.0.0"
+advertise_addr = "{{ GetInterfaceIP \"eth0\" }}"

test/integration/connect/envoy/case-ingress-gateway-peering-failover/capture.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+snapshot_envoy_admin localhost:20000 ingress-gateway primary || true
+snapshot_envoy_admin localhost:19001 s2 primary || true
+snapshot_envoy_admin localhost:19002 s2 alpha || true
+snapshot_envoy_admin localhost:19003 mesh-gateway alpha || true

test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/base.hcl

@@ -0,0 +1,3 @@
+peering {
+  enabled = true
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/config_entries.hcl

@@ -0,0 +1,47 @@
+config_entries {
+  bootstrap {
+    kind = "proxy-defaults"
+    name = "global"
+
+    config {
+      protocol = "tcp"
+    }
+  }
+
+  bootstrap {
+    kind = "ingress-gateway"
+    name = "ingress-gateway"
+    listeners = [
+      {
+        protocol = "tcp"
+        port     = 10000
+        services = [
+          {
+            name = "s2"
+          }
+        ]
+      }
+    ]
+  }
+
+  bootstrap {
+    kind = "service-resolver"
+    name = "s2"
+
+    failover = {
+      "*" = {
+        targets = [{peer = "primary-to-alpha"}]
+      }
+    }
+  }
+
+  bootstrap {
+    kind = "service-resolver"
+    name = "virtual-s2"
+
+    redirect = {
+      service = "s2"
+      peer = "primary-to-alpha"
+    }
+  }
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/service_ingress.hcl

@@ -0,0 +1,4 @@
+services {
+  name = "ingress-gateway"
+  kind = "ingress-gateway"
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/service_s1.hcl

@@ -0,0 +1 @@
+# We don't need an s1 because requests go through the gateway and not a sidecar.

test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/service_s2.hcl

@@ -0,0 +1,10 @@
+services {
+  name = "s2"
+  port = 8181
+  checks = []
+  connect {
+    sidecar_service {
+      checks = []
+    }
+  }
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/setup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+wait_for_config_entry ingress-gateway ingress-gateway
+wait_for_config_entry proxy-defaults global
+wait_for_config_entry service-resolver s2
+wait_for_config_entry service-resolver virtual-s2
+
+register_services primary
+
+gen_envoy_bootstrap ingress-gateway 20000 primary true
+gen_envoy_bootstrap s2 19001 primary

test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/verify.bats

@@ -0,0 +1,77 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "s2 proxy is running correct version" {
+  assert_envoy_version 19001
+}
+
+@test "ingress-primary proxy admin is up" {
+  retry_default curl -f -s localhost:20000/stats -o /dev/null
+}
+
+@test "services should be healthy in primary" {
+  assert_service_has_healthy_instances s2 1 alpha
+}
+
+@test "services should be healthy in alpha" {
+  assert_service_has_healthy_instances s2 1 alpha
+}
+
+@test "mesh-gateway should have healthy endpoints" {
+  assert_upstream_has_endpoints_in_status consul-alpha-client:19003 s2 HEALTHY 1
+}
+
+@test "peer the two clusters together" {
+  create_peering primary alpha
+}
+
+@test "s2 alpha proxies should be healthy in primary" {
+  assert_service_has_healthy_instances s2 1 primary "" "" primary-to-alpha
+}
+
+# Failover
+
+@test "s1 upstream should have healthy endpoints for s2 in both primary and failover" {
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary.internal HEALTHY 1
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
+}
+
+
+@test "ingress-gateway should be able to connect to s2" {
+  assert_expected_fortio_name s2 127.0.0.1 10000
+}
+
+@test "s1 upstream made 1 connection" {
+  assert_envoy_metric_at_least 127.0.0.1:20000 "cluster.failover-target~s2.default.primary.internal.*cx_total" 1
+}
+
+@test "terminate instance of s2 primary envoy which should trigger failover to s2 alpha when the tcp check fails" {
+  kill_envoy s2 primary
+}
+
+@test "s2 proxies should be unhealthy in primary" {
+  assert_service_has_healthy_instances s2 0 primary
+}
+
+
+@test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" {
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary.internal UNHEALTHY 1
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
+}
+
+@test "reset envoy statistics for failover" {
+  reset_envoy_metrics 127.0.0.1:20000
+}
+
+@test "gateway-alpha should have healthy endpoints for s2" {
+  assert_upstream_has_endpoints_in_status consul-alpha-client:19003 exported~s2.default.alpha HEALTHY 1
+}
+
+@test "s1 upstream should be able to connect to s2 in the failover cluster peer" {
+  assert_expected_fortio_name s2-alpha 127.0.0.1 10000
+}
+
+@test "s1 upstream made 1 connection to s2 through the cluster peer" {
+  assert_envoy_metric_at_least 127.0.0.1:20000 "cluster.failover-target~s2.default.primary-to-alpha.external.*cx_total" 1
+}

test/integration/connect/envoy/case-ingress-gateway-peering-failover/vars.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export REQUIRED_SERVICES="s2 s2-sidecar-proxy s2-alpha s2-sidecar-proxy-alpha gateway-alpha ingress-gateway-primary"
+export REQUIRE_PEERS=1

test/integration/connect/envoy/helpers.bash

@@ -997,3 +997,23 @@ function varsub {
     sed -i "s/\${$v}/${!v}/g" $file
   done
 }
+
+function get_url_header {
+  local URL=$1
+  local HEADER=$2
+  run curl -s -f -X GET -I "${URL}"
+  [ "$status" == 0 ]
+  RESP=$(echo "$output" | tr -d '\r')
+  RESP=$(echo "$RESP" | grep -E "^${HEADER}: ")
+  RESP=$(echo "$RESP" | sed "s/^${HEADER}: //g")
+  echo "$RESP"
+}
+
+function assert_url_header {
+  local URL=$1
+  local HEADER=$2
+  local VALUE=$3
+  run get_url_header "$URL" "$HEADER"
+  [ "$status" == 0 ]
+  [ "$VALUE" = "$output" ]
+}