tests: make envoy integration tests more tolerant of internal retries that may inflate counters (#6539)

This should remove false positives that look like:

    cluster.s2.default.primary.*cx_total - expected count: 2, actual count: 3

test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-none/primary/verify.bats

@@ -44,7 +44,7 @@ load helpers
 }
 
 @test "s1 upstream made 1 connection" {
-  assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
+  assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
 }
 
 ################
@@ -63,10 +63,14 @@ load helpers
   assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary UNHEALTHY 1
 }
 
+@test "reset envoy statistics" {
+  reset_envoy_metrics 127.0.0.1:19000
+}
+
 @test "s1 upstream should be able to connect to s2 in secondary now" {
   assert_expected_fortio_name s2-secondary
 }
 
-@test "s1 upstream made 2 connections" {
+@test "s1 upstream made 1 connection" {
-  assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 2
+  assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
 }

test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/primary/verify.bats

@@ -46,7 +46,7 @@ load helpers
 }
 
 @test "s1 upstream made 1 connection" {
-  assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
+  assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
 }
 
 ################
@@ -66,10 +66,14 @@ load helpers
   assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary UNHEALTHY 0
 }
 
+@test "reset envoy statistics" {
+  reset_envoy_metrics 127.0.0.1:19000
+}
+
 @test "s1 upstream should be able to connect to s2 in secondary now" {
   assert_expected_fortio_name s2-secondary
 }
 
-@test "s1 upstream made 2 connections" {
+@test "s1 upstream made 1 connection" {
-  assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 2
+  assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
 }

test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/secondary/verify.bats

@@ -23,5 +23,5 @@ load helpers
 }
 
 @test "gateway-secondary is used for the upstream connection" {
-  assert_envoy_metric 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
+  assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
 }

test/integration/connect/envoy/case-gateways-local/primary/verify.bats

@@ -34,9 +34,9 @@ load helpers
 }
 
 @test "s1 upstream made 1 connection" {
-  assert_envoy_metric 127.0.0.1:19000 "cluster.c225dc1c~s2.default.secondary.*cx_total" 1
+  assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.c225dc1c~s2.default.secondary.*cx_total" 1
 }
 
 @test "gateway-primary is used for the upstream connection" {
-  assert_envoy_metric 127.0.0.1:19002 "cluster.secondary.*cx_total" 1
+  assert_envoy_metric_at_least 127.0.0.1:19002 "cluster.secondary.*cx_total" 1
 }

test/integration/connect/envoy/case-gateways-local/secondary/verify.bats

@@ -23,5 +23,5 @@ load helpers
 }
 
 @test "gateway-secondary is used for the upstream connection" {
-  assert_envoy_metric 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
+  assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
 }

test/integration/connect/envoy/case-gateways-remote/primary/verify.bats

@@ -26,5 +26,5 @@ load helpers
 }
 
 @test "s1 upstream made 1 connection" {
-  assert_envoy_metric 127.0.0.1:19000 "cluster.dd412229~s2.default.secondary.*cx_total" 1
+  assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.dd412229~s2.default.secondary.*cx_total" 1
 }

test/integration/connect/envoy/case-gateways-remote/secondary/verify.bats

@@ -23,5 +23,5 @@ load helpers
 }
 
 @test "gateway-secondary is used for the upstream connection" {
-  assert_envoy_metric 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
+  assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
 }

test/integration/connect/envoy/helpers.bash

@@ -166,6 +166,12 @@ function snapshot_envoy_admin {
   docker_wget "$DC" "http://${HOSTPORT}/stats" -q -O - > "./workdir/${DC}/envoy/${ENVOY_NAME}-stats.txt"
 }
 
+function reset_envoy_metrics {
+  local HOSTPORT=$1
+  curl -s -f -XPOST $HOSTPORT/reset_counters
+  return $?
+}
+
 function get_all_envoy_metrics {
   local HOSTPORT=$1
   curl -s -f $HOSTPORT/stats
@@ -243,6 +249,34 @@ function assert_envoy_metric {
   fi
 }
 
+function assert_envoy_metric_at_least {
+  set -eEuo pipefail
+  local HOSTPORT=$1
+  local METRIC=$2
+  local EXPECT_COUNT=$3
+
+  METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
+
+  if [ -z "${METRICS}" ]
+  then
+    echo "Metric not found" 1>&2
+    return 1
+  fi
+
+  GOT_COUNT=$(awk -F: '{print $2}' <<< "$METRICS" | head -n 1 | tr -d ' ')
+
+  if [ -z "$GOT_COUNT" ]
+  then
+    echo "Couldn't parse metric count" 1>&2
+    return 1
+  fi
+
+  if [ $EXPECT_COUNT -gt $GOT_COUNT ]
+  then
+    echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
+    return 1
+  fi
+}
 
 function get_healthy_service_count {
   local SERVICE_NAME=$1