tests: further reduce envoy integration test flakiness (#6112)

In addition to waiting until s2 shows up healthy in the Catalog, wait
until s2 endpoints show up healthy via EDS in the s1 upstream clusters.

test/integration/connect/envoy/case-badauthz/verify.bats

@@ -22,6 +22,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should NOT be able to connect to s2" {
   run retry_default must_fail_tcp_connection localhost:5000
 

test/integration/connect/envoy/case-basic/verify.bats

@@ -26,6 +26,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]

test/integration/connect/envoy/case-centralconf/verify.bats

@@ -22,6 +22,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]

test/integration/connect/envoy/case-dogstatsd-udp/verify.bats

@@ -14,6 +14,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
 

test/integration/connect/envoy/case-grpc/verify.bats

@@ -14,6 +14,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should be able to connect to s2 via grpc" {
   run fortio grpcping localhost:5000
 

test/integration/connect/envoy/case-http-badauthz/verify.bats

@@ -22,6 +22,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should NOT be able to connect to s2" {
   run retry_default must_fail_http_connection localhost:5000
 

test/integration/connect/envoy/case-http/verify.bats

@@ -22,6 +22,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]

test/integration/connect/envoy/case-http2/verify.bats

@@ -22,6 +22,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should be able to connect to s2 via http2" {
   # We use grpc here because it's the easiest way to test http2. The server
   # needs to support h2c since the proxy doesn't talk TLS to the local app.

test/integration/connect/envoy/case-prometheus/verify.bats

@@ -22,6 +22,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]

test/integration/connect/envoy/case-statsd-udp/verify.bats

@@ -14,6 +14,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" == 0 ]

test/integration/connect/envoy/case-zipkin/verify.bats

@@ -22,6 +22,10 @@ load helpers
   assert_service_has_healthy_instances s2 1
 }
 
+@test "s1 upstream should have healthy endpoints for s2" {
+  assert_upstream_has_healthy_endpoints 127.0.0.1:19000 s2 1
+}
+
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" == "0" ]

test/integration/connect/envoy/helpers.bash

@@ -117,7 +117,38 @@ function snapshot_envoy_admin {
   local ENVOY_NAME=$2
 
   docker_wget "http://${HOSTPORT}/config_dump" -q -O - > "./workdir/envoy/${ENVOY_NAME}-config_dump.json"
-  docker_wget "http://${HOSTPORT}/clusters" -q -O - > "./workdir/envoy/${ENVOY_NAME}-clusters.out"
+  docker_wget "http://${HOSTPORT}/clusters?format=json" -q -O - > "./workdir/envoy/${ENVOY_NAME}-clusters.json"
+}
+
+function get_healthy_upstream_endpoint_count {
+  local HOSTPORT=$1
+  local CLUSTER_NAME=$2
+  run retry_default curl -s -f "http://${HOSTPORT}/clusters?format=json"
+  [ "$status" -eq 0 ]
+  echo "$output" | jq --raw-output "
+.cluster_statuses[]
+| select(.name|startswith(\"${CLUSTER_NAME}.default.dc1.internal.\"))
+| .host_statuses[].health_status
+| select(.eds_health_status == \"HEALTHY\")
+| length"
+}
+
+function assert_upstream_has_healthy_endpoints_once {
+  local HOSTPORT=$1
+  local CLUSTER_NAME=$2
+  local EXPECT_COUNT=$3
+
+  GOT_COUNT=$(get_healthy_upstream_endpoint_count $HOSTPORT $CLUSTER_NAME)
+
+  [ "$GOT_COUNT" -eq $EXPECT_COUNT ]
+}
+
+function assert_upstream_has_healthy_endpoints {
+  local HOSTPORT=$1
+  local CLUSTER_NAME=$2
+  local EXPECT_COUNT=$3
+  run retry 30 1 assert_upstream_has_healthy_endpoints_once $HOSTPORT $CLUSTER_NAME $EXPECT_COUNT
+  [ "$status" -eq 0 ]
 }
 
 function get_healthy_service_count {
@@ -127,7 +158,7 @@ function get_healthy_service_count {
   echo "$output" | jq --raw-output '. | length'
 }
 
-function health_service_count_matches {
+function assert_service_has_healthy_instances_once {
   local SERVICE_NAME=$1
   local EXPECT_COUNT=$2
 
@@ -140,7 +171,7 @@ function assert_service_has_healthy_instances {
   local SERVICE_NAME=$1
   local EXPECT_COUNT=$2
 
-  run retry 30 2 health_service_count_matches $SERVICE_NAME $EXPECT_COUNT
+  run retry 30 1 assert_service_has_healthy_instances_once $SERVICE_NAME $EXPECT_COUNT
   [ "$status" -eq 0 ]
 }