From df06ab418196f71930a7663c6e0e0c07046a14a4 Mon Sep 17 00:00:00 2001 From: cskh Date: Wed, 7 Dec 2022 20:19:34 -0500 Subject: [PATCH] Flakiness test: case-cfg-splitter-peering-ingress-gateways (#15707) * integ-test: fix flaky test - case-cfg-splitter-peering-ingress-gateways * add retry peering to all peering cases Co-authored-by: Dan Stough --- agent/xds/routes.go | 4 ++++ .../primary/verify.bats | 2 +- .../primary/verify.bats | 2 +- .../primary/verify.bats | 13 ++++++---- .../primary/verify.bats | 7 +++++- .../primary/verify.bats | 2 +- .../case-cross-peers-http/primary/verify.bats | 2 +- .../primary/verify.bats | 2 +- .../case-cross-peers/primary/verify.bats | 2 +- .../primary/verify.bats | 4 +--- test/integration/connect/envoy/helpers.bash | 24 +++++++++++++++++++ 11 files changed, 50 insertions(+), 14 deletions(-) diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 966daf983..2931e2d77 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -874,6 +874,10 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( clusters = append(clusters, cw) } + if len(clusters) <= 0 { + return nil, fmt.Errorf("number of clusters in splitter must be > 0; got %d", len(clusters)) + } + return &envoy_route_v3.Route_Route{ Route: &envoy_route_v3.RouteAction{ ClusterSpecifier: &envoy_route_v3.RouteAction_WeightedClusters{ diff --git a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats index 7c8d9ee8a..b673be8cf 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats +++ b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats @@ -35,7 +35,7 @@ load helpers } @test "peer the two clusters together" { - create_peering primary alpha + retry_default create_peering primary alpha } @test "s2 alpha proxies should be healthy in primary" { diff --git a/test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/verify.bats b/test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/verify.bats index 718dac6e7..e442e3073 100644 --- a/test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/verify.bats +++ b/test/integration/connect/envoy/case-cfg-splitter-cluster-peering/primary/verify.bats @@ -31,7 +31,7 @@ load helpers } @test "peer the two clusters together" { - create_peering primary alpha + retry_default create_peering primary alpha } @test "s2 alpha proxies should be healthy in primary" { diff --git a/test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/verify.bats b/test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/verify.bats index e12e7058e..fabbdf0f2 100644 --- a/test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/verify.bats +++ b/test/integration/connect/envoy/case-cfg-splitter-peering-ingress-gateways/primary/verify.bats @@ -2,7 +2,6 @@ load helpers - @test "ingress-primary proxy admin is up" { retry_default curl -f -s localhost:20000/stats -o /dev/null } @@ -12,7 +11,7 @@ load helpers } @test "services should be healthy in primary" { - assert_service_has_healthy_instances s1 1 alpha + assert_service_has_healthy_instances s1 1 primary } @test "services should be healthy in alpha" { @@ -26,7 +25,12 @@ load helpers } @test "peer the two clusters together" { - create_peering primary alpha + retry_long create_peering primary alpha +} + +@test "s1, s2 alpha proxies should be imported to primary" { + retry_long assert_service_has_imported primary s1 primary-to-alpha + retry_long assert_service_has_imported primary s2 primary-to-alpha } @test "s1 alpha proxies should be healthy in primary" { @@ -67,4 +71,5 @@ load helpers @test "requests through ingress to splitter should go to alpha" { retry_long assert_expected_fortio_name s1-alpha split.ingress.consul 10002 retry_long assert_expected_fortio_name s2-alpha split.ingress.consul 10002 -} \ No newline at end of file +} + diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats index f14e12dd8..784881098 100644 --- a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats @@ -31,7 +31,12 @@ load helpers } @test "peer the two clusters together" { - create_peering primary alpha + retry_default create_peering primary alpha +} + +@test "s2 alpha proxies should be imported in primary" { + retry_long assert_service_has_imported primary s2 primary-to-alpha + [ "$status" -eq 0 ] } @test "acceptor gateway-primary should have healthy endpoints for primary servers" { diff --git a/test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats b/test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats index c6b2d59d6..a11c12441 100644 --- a/test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats @@ -35,7 +35,7 @@ load helpers } @test "peer the two clusters together" { - create_peering primary alpha + retry_default create_peering primary alpha } @test "s2 alpha proxies should be healthy in primary" { diff --git a/test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats b/test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats index 9e59677b9..7ae3901d9 100644 --- a/test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats @@ -31,7 +31,7 @@ load helpers } @test "peer the two clusters together" { - create_peering primary alpha + retry_default create_peering primary alpha } @test "s2 alpha proxies should be healthy in primary" { diff --git a/test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats b/test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats index e2673269e..7524d32aa 100644 --- a/test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats @@ -31,7 +31,7 @@ load helpers } @test "peer the two clusters together" { - create_peering primary alpha + retry_default create_peering primary alpha } @test "s2 alpha proxies should be healthy in primary" { diff --git a/test/integration/connect/envoy/case-cross-peers/primary/verify.bats b/test/integration/connect/envoy/case-cross-peers/primary/verify.bats index 840db4021..3ded23af5 100644 --- a/test/integration/connect/envoy/case-cross-peers/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peers/primary/verify.bats @@ -31,7 +31,7 @@ load helpers } @test "peer the two clusters together" { - create_peering primary alpha + retry_default create_peering primary alpha } @test "s2 alpha proxies should be healthy in primary" { diff --git a/test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/verify.bats b/test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/verify.bats index 9429a9d60..630a4d4ec 100644 --- a/test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/verify.bats +++ b/test/integration/connect/envoy/case-ingress-gateway-peering-failover/primary/verify.bats @@ -23,7 +23,7 @@ load helpers } @test "peer the two clusters together" { - create_peering primary alpha + retry_default create_peering primary alpha } @test "s2 alpha proxies should be healthy in primary" { @@ -37,7 +37,6 @@ load helpers assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 } - @test "ingress-gateway should be able to connect to s2" { assert_expected_fortio_name s2 127.0.0.1 10000 } @@ -54,7 +53,6 @@ load helpers assert_service_has_healthy_instances s2 0 primary } - @test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" { assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary.internal UNHEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:20000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index 9991fb697..3612cdb5c 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -936,6 +936,30 @@ function create_peering { run curl -sLv -XPOST "http://consul-${ESTABLISH_PEER}-client:8500/v1/peering/establish" -d"{ \"PeerName\" : \"${ESTABLISH_PEER}-to-${GENERATE_PEER}\", \"PeeringToken\" : \"${token}\" }" # echo "$output" >&3 [ "$status" == 0 ] + + sleep 1 + run curl -s -f "http://consul-${GENERATE_PEER}-client:8500/v1/peering/${GENERATE_PEER}-to-${ESTABLISH_PEER}" + state="$(echo "$output" | jq --raw-output .State)" + + if [ "$state" != "ACTIVE" ]; then + echo "fail to peering: $output" + return 1 + fi +} + +function assert_service_has_imported { + local DC=${1:-primary} + local SERVICE_NAME=$2 + local PEER_NAME=$3 + + run curl -s -f "http://consul-${DC}-client:8500/v1/peering/${PEER_NAME}" + [ "$status" == 0 ] + + echo "$output" | jq --raw-output '.StreamStatus.ImportedServices' | grep -e "${SERVICE_NAME}" + if [ $? -ne 0 ]; then + echo "Error finding service: ${SERVICE_NAME}" + return 1 + fi } function get_lambda_envoy_http_filter {