Flakiness test: case-cfg-splitter-peering-ingress-gateways (#15707)

* integ-test: fix flaky test - case-cfg-splitter-peering-ingress-gateways

* add retry peering to all peering cases

Co-authored-by: Dan Stough <dan.stough@hashicorp.com>

agent/xds/routes.go

@@ -874,6 +874,10 @@ func (s *ResourceGenerator) makeRouteActionForSplitter(
 		clusters = append(clusters, cw)
 	}
 
+	if len(clusters) <= 0 {
+		return nil, fmt.Errorf("number of clusters in splitter must be > 0; got %d", len(clusters))
+	}
+
 	return &envoy_route_v3.Route_Route{
 		Route: &envoy_route_v3.RouteAction{
 			ClusterSpecifier: &envoy_route_v3.RouteAction_WeightedClusters{

test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats

@@ -35,7 +35,7 @@ load helpers
 }
 
 @test "peer the two clusters together" {
-  create_peering primary alpha
+  retry_default create_peering primary alpha
 }
 
 @test "s2 alpha proxies should be healthy in primary" {

test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/verify.bats

@@ -31,7 +31,7 @@ load helpers
 }
 
 @test "peer the two clusters together" {
-  create_peering primary alpha
+  retry_default create_peering primary alpha
 }
 
 @test "s2 alpha proxies should be healthy in primary" {

test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/verify.bats

@@ -2,7 +2,6 @@
 
 load helpers
 
-
 @test "ingress-primary proxy admin is up" {
   retry_default curl -f -s localhost:20000/stats -o /dev/null
 }
@@ -12,7 +11,7 @@ load helpers
 }
 
 @test "services should be healthy in primary" {
-  assert_service_has_healthy_instances s1 1 alpha
+  assert_service_has_healthy_instances s1 1 primary
 }
 
 @test "services should be healthy in alpha" {
@@ -26,7 +25,12 @@ load helpers
 }
 
 @test "peer the two clusters together" {
-  create_peering primary alpha
+  retry_long create_peering primary alpha
+}
+
+@test "s1, s2 alpha proxies should be imported to primary" {
+  retry_long assert_service_has_imported primary s1 primary-to-alpha
+  retry_long assert_service_has_imported primary s2 primary-to-alpha
 }
 
 @test "s1 alpha proxies should be healthy in primary" {
@@ -67,4 +71,5 @@ load helpers
 @test "requests through ingress to splitter should go to alpha" {
   retry_long assert_expected_fortio_name s1-alpha split.ingress.consul 10002
   retry_long assert_expected_fortio_name s2-alpha split.ingress.consul 10002
-}
+}
+

test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats

@@ -31,7 +31,12 @@ load helpers
 }
 
 @test "peer the two clusters together" {
-  create_peering primary alpha
+  retry_default create_peering primary alpha
+}
+
+@test "s2 alpha proxies should be imported in primary" {
+  retry_long assert_service_has_imported primary s2 primary-to-alpha
+  [ "$status" -eq 0 ]
 }
 
 @test "acceptor gateway-primary should have healthy endpoints for primary servers" {

test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats

@@ -35,7 +35,7 @@ load helpers
 }
 
 @test "peer the two clusters together" {
-  create_peering primary alpha
+  retry_default create_peering primary alpha
 }
 
 @test "s2 alpha proxies should be healthy in primary" {

test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats

@@ -31,7 +31,7 @@ load helpers
 }
 
 @test "peer the two clusters together" {
-  create_peering primary alpha
+  retry_default create_peering primary alpha
 }
 
 @test "s2 alpha proxies should be healthy in primary" {

test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats

@@ -31,7 +31,7 @@ load helpers
 }
 
 @test "peer the two clusters together" {
-  create_peering primary alpha
+  retry_default create_peering primary alpha
 }
 
 @test "s2 alpha proxies should be healthy in primary" {

test/integration/connect/envoy/case-cross-peers/primary/verify.bats

@@ -31,7 +31,7 @@ load helpers
 }
 
 @test "peer the two clusters together" {
-  create_peering primary alpha
+  retry_default create_peering primary alpha
 }
 
 @test "s2 alpha proxies should be healthy in primary" {

test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/verify.bats

@@ -23,7 +23,7 @@ load helpers
 }
 
 @test "peer the two clusters together" {
-  create_peering primary alpha
+  retry_default create_peering primary alpha
 }
 
 @test "s2 alpha proxies should be healthy in primary" {
@@ -37,7 +37,6 @@ load helpers
   assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
 }
 
-
 @test "ingress-gateway should be able to connect to s2" {
   assert_expected_fortio_name s2 127.0.0.1 10000
 }
@@ -54,7 +53,6 @@ load helpers
   assert_service_has_healthy_instances s2 0 primary
 }
 
-
 @test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" {
   assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary.internal UNHEALTHY 1
   assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1

test/integration/connect/envoy/helpers.bash

@@ -936,6 +936,30 @@ function create_peering {
   run curl -sLv -XPOST "http://consul-${ESTABLISH_PEER}-client:8500/v1/peering/establish" -d"{ \"PeerName\" : \"${ESTABLISH_PEER}-to-${GENERATE_PEER}\", \"PeeringToken\" : \"${token}\" }"
   # echo "$output" >&3
   [ "$status" == 0 ]
+
+  sleep 1
+  run curl -s -f "http://consul-${GENERATE_PEER}-client:8500/v1/peering/${GENERATE_PEER}-to-${ESTABLISH_PEER}"
+  state="$(echo "$output" | jq --raw-output .State)"
+
+  if [ "$state" != "ACTIVE" ]; then
+    echo "fail to peering: $output"
+    return 1
+  fi
+}
+
+function assert_service_has_imported {
+  local DC=${1:-primary}
+  local SERVICE_NAME=$2
+  local PEER_NAME=$3
+
+  run curl -s -f "http://consul-${DC}-client:8500/v1/peering/${PEER_NAME}"
+  [ "$status" == 0 ]
+
+  echo "$output" | jq --raw-output '.StreamStatus.ImportedServices' | grep -e "${SERVICE_NAME}"
+  if [ $? -ne 0 ]; then
+    echo "Error finding service: ${SERVICE_NAME}"
+    return 1
+  fi
 }
 
 function get_lambda_envoy_http_filter {