tests: make envoy integration tests more tolerant of internal retries that may inflate counters (#6539)
This should remove false positives that look like:
cluster.s2.default.primary.*cx_total - expected count: 2, actual count: 3
This commit is contained in:
R.B. Boyer
GitHub
parent
f1b9476cd8
commit
9adc39cce0
|
|
@ -44,7 +44,7 @@ load helpers
|
|||
}
|
||||
|
||||
@test "s1 upstream made 1 connection" {
|
||||
assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
|
||||
assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
|
||||
}
|
||||
|
||||
################
|
||||
|
|
@ -63,10 +63,14 @@ load helpers
|
|||
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary UNHEALTHY 1
|
||||
}
|
||||
|
||||
@test "reset envoy statistics" {
|
||||
reset_envoy_metrics 127.0.0.1:19000
|
||||
}
|
||||
|
||||
@test "s1 upstream should be able to connect to s2 in secondary now" {
|
||||
assert_expected_fortio_name s2-secondary
|
||||
}
|
||||
|
||||
@test "s1 upstream made 2 connections" {
|
||||
assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 2
|
||||
@test "s1 upstream made 1 connection" {
|
||||
assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ load helpers
|
|||
}
|
||||
|
||||
@test "s1 upstream made 1 connection" {
|
||||
assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
|
||||
assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
|
||||
}
|
||||
|
||||
################
|
||||
|
|
@ -66,10 +66,14 @@ load helpers
|
|||
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary UNHEALTHY 0
|
||||
}
|
||||
|
||||
@test "reset envoy statistics" {
|
||||
reset_envoy_metrics 127.0.0.1:19000
|
||||
}
|
||||
|
||||
@test "s1 upstream should be able to connect to s2 in secondary now" {
|
||||
assert_expected_fortio_name s2-secondary
|
||||
}
|
||||
|
||||
@test "s1 upstream made 2 connections" {
|
||||
assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 2
|
||||
@test "s1 upstream made 1 connection" {
|
||||
assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary.*cx_total" 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,5 +23,5 @@ load helpers
|
|||
}
|
||||
|
||||
@test "gateway-secondary is used for the upstream connection" {
|
||||
assert_envoy_metric 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
|
||||
assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,9 +34,9 @@ load helpers
|
|||
}
|
||||
|
||||
@test "s1 upstream made 1 connection" {
|
||||
assert_envoy_metric 127.0.0.1:19000 "cluster.c225dc1c~s2.default.secondary.*cx_total" 1
|
||||
assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.c225dc1c~s2.default.secondary.*cx_total" 1
|
||||
}
|
||||
|
||||
@test "gateway-primary is used for the upstream connection" {
|
||||
assert_envoy_metric 127.0.0.1:19002 "cluster.secondary.*cx_total" 1
|
||||
assert_envoy_metric_at_least 127.0.0.1:19002 "cluster.secondary.*cx_total" 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,5 +23,5 @@ load helpers
|
|||
}
|
||||
|
||||
@test "gateway-secondary is used for the upstream connection" {
|
||||
assert_envoy_metric 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
|
||||
assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,5 +26,5 @@ load helpers
|
|||
}
|
||||
|
||||
@test "s1 upstream made 1 connection" {
|
||||
assert_envoy_metric 127.0.0.1:19000 "cluster.dd412229~s2.default.secondary.*cx_total" 1
|
||||
assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.dd412229~s2.default.secondary.*cx_total" 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,5 +23,5 @@ load helpers
|
|||
}
|
||||
|
||||
@test "gateway-secondary is used for the upstream connection" {
|
||||
assert_envoy_metric 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
|
||||
assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -166,6 +166,12 @@ function snapshot_envoy_admin {
|
|||
docker_wget "$DC" "http://${HOSTPORT}/stats" -q -O - > "./workdir/${DC}/envoy/${ENVOY_NAME}-stats.txt"
|
||||
}
|
||||
|
||||
function reset_envoy_metrics {
|
||||
local HOSTPORT=$1
|
||||
curl -s -f -XPOST $HOSTPORT/reset_counters
|
||||
return $?
|
||||
}
|
||||
|
||||
function get_all_envoy_metrics {
|
||||
local HOSTPORT=$1
|
||||
curl -s -f $HOSTPORT/stats
|
||||
|
|
@ -243,6 +249,34 @@ function assert_envoy_metric {
|
|||
fi
|
||||
}
|
||||
|
||||
function assert_envoy_metric_at_least {
|
||||
set -eEuo pipefail
|
||||
local HOSTPORT=$1
|
||||
local METRIC=$2
|
||||
local EXPECT_COUNT=$3
|
||||
|
||||
METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
|
||||
|
||||
if [ -z "${METRICS}" ]
|
||||
then
|
||||
echo "Metric not found" 1>&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
GOT_COUNT=$(awk -F: '{print $2}' <<< "$METRICS" | head -n 1 | tr -d ' ')
|
||||
|
||||
if [ -z "$GOT_COUNT" ]
|
||||
then
|
||||
echo "Couldn't parse metric count" 1>&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ $EXPECT_COUNT -gt $GOT_COUNT ]
|
||||
then
|
||||
echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function get_healthy_service_count {
|
||||
local SERVICE_NAME=$1
|
||||
|
|
|
|||
Loading…
Reference in New Issue