diff --git a/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-none/primary/verify.bats b/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-none/primary/verify.bats index 9d5c8e60d..2a3c7cd11 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-none/primary/verify.bats +++ b/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-none/primary/verify.bats @@ -44,7 +44,7 @@ load helpers } @test "s1 upstream made 1 connection" { - assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1 + assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1 } ################ @@ -63,10 +63,14 @@ load helpers assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary UNHEALTHY 1 } +@test "reset envoy statistics" { + reset_envoy_metrics 127.0.0.1:19000 +} + @test "s1 upstream should be able to connect to s2 in secondary now" { assert_expected_fortio_name s2-secondary } -@test "s1 upstream made 2 connections" { - assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 2 +@test "s1 upstream made 1 connection" { + assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1 } diff --git a/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/primary/verify.bats b/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/primary/verify.bats index db99c8ddf..151230a57 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/primary/verify.bats +++ b/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/primary/verify.bats @@ -46,7 +46,7 @@ load helpers } @test "s1 upstream made 1 connection" { - assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1 + assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1 } ################ @@ -66,10 +66,14 @@ load helpers assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary UNHEALTHY 0 } +@test "reset envoy statistics" { + reset_envoy_metrics 127.0.0.1:19000 +} + @test "s1 upstream should be able to connect to s2 in secondary now" { assert_expected_fortio_name s2-secondary } -@test "s1 upstream made 2 connections" { - assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 2 +@test "s1 upstream made 1 connection" { + assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1 } diff --git a/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/secondary/verify.bats b/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/secondary/verify.bats index 8bd0a91b6..8df8213e2 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/secondary/verify.bats +++ b/test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/secondary/verify.bats @@ -23,5 +23,5 @@ load helpers } @test "gateway-secondary is used for the upstream connection" { - assert_envoy_metric 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1 + assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1 } diff --git a/test/integration/connect/envoy/case-gateways-local/primary/verify.bats b/test/integration/connect/envoy/case-gateways-local/primary/verify.bats index aa156efa6..b7fe8488f 100644 --- a/test/integration/connect/envoy/case-gateways-local/primary/verify.bats +++ b/test/integration/connect/envoy/case-gateways-local/primary/verify.bats @@ -34,9 +34,9 @@ load helpers } @test "s1 upstream made 1 connection" { - assert_envoy_metric 127.0.0.1:19000 "cluster.c225dc1c~s2.default.secondary.*cx_total" 1 + assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.c225dc1c~s2.default.secondary.*cx_total" 1 } @test "gateway-primary is used for the upstream connection" { - assert_envoy_metric 127.0.0.1:19002 "cluster.secondary.*cx_total" 1 + assert_envoy_metric_at_least 127.0.0.1:19002 "cluster.secondary.*cx_total" 1 } diff --git a/test/integration/connect/envoy/case-gateways-local/secondary/verify.bats b/test/integration/connect/envoy/case-gateways-local/secondary/verify.bats index b4f92b07a..1fc00afe8 100644 --- a/test/integration/connect/envoy/case-gateways-local/secondary/verify.bats +++ b/test/integration/connect/envoy/case-gateways-local/secondary/verify.bats @@ -23,5 +23,5 @@ load helpers } @test "gateway-secondary is used for the upstream connection" { - assert_envoy_metric 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1 + assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1 } diff --git a/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats b/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats index 0b901781e..825872e57 100644 --- a/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats +++ b/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats @@ -26,5 +26,5 @@ load helpers } @test "s1 upstream made 1 connection" { - assert_envoy_metric 127.0.0.1:19000 "cluster.dd412229~s2.default.secondary.*cx_total" 1 + assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.dd412229~s2.default.secondary.*cx_total" 1 } diff --git a/test/integration/connect/envoy/case-gateways-remote/secondary/verify.bats b/test/integration/connect/envoy/case-gateways-remote/secondary/verify.bats index b4f92b07a..1fc00afe8 100644 --- a/test/integration/connect/envoy/case-gateways-remote/secondary/verify.bats +++ b/test/integration/connect/envoy/case-gateways-remote/secondary/verify.bats @@ -23,5 +23,5 @@ load helpers } @test "gateway-secondary is used for the upstream connection" { - assert_envoy_metric 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1 + assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1 } diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index 44bf1a237..c230b4bd6 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -166,6 +166,12 @@ function snapshot_envoy_admin { docker_wget "$DC" "http://${HOSTPORT}/stats" -q -O - > "./workdir/${DC}/envoy/${ENVOY_NAME}-stats.txt" } +function reset_envoy_metrics { + local HOSTPORT=$1 + curl -s -f -XPOST $HOSTPORT/reset_counters + return $? +} + function get_all_envoy_metrics { local HOSTPORT=$1 curl -s -f $HOSTPORT/stats @@ -243,6 +249,34 @@ function assert_envoy_metric { fi } +function assert_envoy_metric_at_least { + set -eEuo pipefail + local HOSTPORT=$1 + local METRIC=$2 + local EXPECT_COUNT=$3 + + METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") + + if [ -z "${METRICS}" ] + then + echo "Metric not found" 1>&2 + return 1 + fi + + GOT_COUNT=$(awk -F: '{print $2}' <<< "$METRICS" | head -n 1 | tr -d ' ') + + if [ -z "$GOT_COUNT" ] + then + echo "Couldn't parse metric count" 1>&2 + return 1 + fi + + if [ $EXPECT_COUNT -gt $GOT_COUNT ] + then + echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 + return 1 + fi +} function get_healthy_service_count { local SERVICE_NAME=$1