integ test: reduce flakiness due to compound output from retry (#15233)

* integ test: avoid flakiness due to compound output from retry

* changelog

.changelog/15233.txt

@@ -0,0 +1,3 @@
+```release-note: improvement
+integ test: fix flakiness due to test condition from retry app endoint
+```

test/integration/connect/envoy/case-basic/verify.bats

@@ -33,7 +33,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should have been configured with one rbac listener filter at L4" {

test/integration/connect/envoy/case-centralconf/verify.bats

@@ -29,7 +29,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should be exposing metrics to prometheus from central config" {

test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats

@@ -49,11 +49,10 @@ load helpers
   assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
 }
 
-
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection" {
@@ -68,7 +67,6 @@ load helpers
   assert_service_has_healthy_instances s2 0 primary
 }
 
-
 @test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" {
   assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal UNHEALTHY 1
   assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
@@ -85,7 +83,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 in the failover cluster peer" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2 through the cluster peer" {
@@ -105,7 +103,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 via virtual-s2" {
   run retry_default curl -s -f -d hello localhost:5001
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2 via virtual-s2 through the cluster peer" {

test/integration/connect/envoy/case-cfg-resolver-svc-redirect-http/verify.bats

@@ -37,10 +37,9 @@ load helpers
 @test "s1 upstream should be able to connect to its upstream simply" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream should be able to connect to s3 via upstream s2" {
   assert_expected_fortio_name s3
 }
-

test/integration/connect/envoy/case-cfg-resolver-svc-redirect-tcp/verify.bats

@@ -37,10 +37,9 @@ load helpers
 @test "s1 upstream should be able to connect to its upstream simply" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream should be able to connect to s3 via upstream s2" {
   assert_expected_fortio_name s3
 }
-

test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats

@@ -57,7 +57,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2" {

test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats

@@ -57,7 +57,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream should be able to connect to s2 via s2" {

test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats

@@ -49,7 +49,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2" {

test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats

@@ -49,7 +49,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream should be able to connect to s3 via s2 due to redirect" {

test/integration/connect/envoy/case-cross-peers/primary/verify.bats

@@ -49,7 +49,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2" {

test/integration/connect/envoy/case-expose-checks/verify.bats

@@ -33,7 +33,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s2 exposes checks on a new listener" {

test/integration/connect/envoy/case-gateways-local/primary/verify.bats

@@ -34,7 +34,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection" {

test/integration/connect/envoy/case-gateways-remote/primary/verify.bats

@@ -22,7 +22,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection" {

test/integration/connect/envoy/case-http/verify.bats

@@ -30,7 +30,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should have been configured with http connection managers" {

test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats

@@ -42,5 +42,5 @@ load helpers
 @test "ingress should be able to connect to s1 via configured port" {
   run retry_default curl -s -f -d hello localhost:9999
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }

test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats

@@ -30,7 +30,7 @@ load helpers
     --resolve s1.ingress.consul:9998:127.0.0.1 \
     https://s1.ingress.consul:9998
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "should be able to connect to s1 through the TLS-enabled ingress port using the custom host" {
@@ -39,5 +39,5 @@ load helpers
     --resolve test.example.com:9999:127.0.0.1 \
     https://test.example.com:9999
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats

@@ -33,7 +33,7 @@ load helpers
 @test "ingress should be able to connect to s1" {
   run retry_default curl -s -f -d hello localhost:10000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "ingress made 1 connection to s1" {
@@ -47,7 +47,7 @@ load helpers
 @test "ingress should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:9999
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "ingress made 1 connection to s2" {

test/integration/connect/envoy/case-multidc-rsa-ca/primary/verify.bats

@@ -21,7 +21,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection" {

test/integration/connect/envoy/case-prometheus/verify.bats

@@ -30,7 +30,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should be exposing metrics to prometheus" {

test/integration/connect/envoy/case-stats-proxy/verify.bats

@@ -30,7 +30,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should be exposing the /stats prefix" {

test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats

@@ -25,7 +25,7 @@ load helpers
 @test "s1 upstream should be able to connect to s4" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "terminating-gateway is used for the upstream connection" {

test/integration/connect/envoy/case-terminating-gateway-simple/verify.bats

@@ -25,7 +25,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "terminating-gateway is used for the upstream connection" {

test/integration/connect/envoy/helpers.bash

@@ -11,18 +11,14 @@ function retry {
   shift
 
   local errtrace=0
-  if grep -q "errtrace" <<< "$SHELLOPTS"
-  then
+  if grep -q "errtrace" <<<"$SHELLOPTS"; then
     errtrace=1
     set +E
   fi
 
-  for ((i=1;i<=$max;i++))
-  do
-    if "$@"
-    then
-      if test $errtrace -eq 1
-      then
+  for ((i = 1; i <= $max; i++)); do
+    if "$@"; then
+      if test $errtrace -eq 1; then
         set -E
       fi
       return 0
@@ -32,8 +28,7 @@ function retry {
     fi
   done
 
-  if test $errtrace -eq 1
-  then
+  if test $errtrace -eq 1; then
     set -E
   fi
   return 1
@@ -365,22 +360,19 @@ function assert_envoy_metric {
 
   METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
 
-  if [ -z "${METRICS}" ]
-  then
+  if [ -z "${METRICS}" ]; then
     echo "Metric not found" 1>&2
     return 1
   fi
 
   GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ')
 
-  if [ -z "$GOT_COUNT" ]
-  then
+  if [ -z "$GOT_COUNT" ]; then
     echo "Couldn't parse metric count" 1>&2
     return 1
   fi
 
-  if [ $EXPECT_COUNT -ne $GOT_COUNT ]
-  then
+  if [ $EXPECT_COUNT -ne $GOT_COUNT ]; then
     echo "$METRIC - expected count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
     return 1
   fi
@@ -394,22 +386,19 @@ function assert_envoy_metric_at_least {
 
   METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
 
-  if [ -z "${METRICS}" ]
-  then
+  if [ -z "${METRICS}" ]; then
     echo "Metric not found" 1>&2
     return 1
   fi
 
   GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ')
 
-  if [ -z "$GOT_COUNT" ]
-  then
+  if [ -z "$GOT_COUNT" ]; then
     echo "Couldn't parse metric count" 1>&2
     return 1
   fi
 
-  if [ $EXPECT_COUNT -gt $GOT_COUNT ]
-  then
+  if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then
     echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
     return 1
   fi
@@ -423,22 +412,19 @@ function assert_envoy_aggregate_metric_at_least {
 
   METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
 
-  if [ -z "${METRICS}" ]
-  then
+  if [ -z "${METRICS}" ]; then
     echo "Metric not found" 1>&2
     return 1
   fi
 
   GOT_COUNT=$(awk '{ sum += $2 } END { print sum }' <<<"$METRICS")
 
-  if [ -z "$GOT_COUNT" ]
-  then
+  if [ -z "$GOT_COUNT" ]; then
     echo "Couldn't parse metric count" 1>&2
     return 1
   fi
 
-  if [ $EXPECT_COUNT -gt $GOT_COUNT ]
-  then
+  if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then
     echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
     return 1
   fi
@@ -557,8 +543,7 @@ function docker_curl {
 }
 
 function docker_exec {
-  if ! docker exec -i "$@"
-  then
+  if ! docker exec -i "$@"; then
     echo "Failed to execute: docker exec -i $@" 1>&2
     return 1
   fi
@@ -638,7 +623,7 @@ function must_pass_tcp_connection {
   echo "OUTPUT $output"
 
   [ "$status" == "0" ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 # must_fail_http_connection see must_fail_tcp_connection but this expects Envoy
@@ -669,8 +654,8 @@ function must_pass_http_request {
     extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
   fi
   case "$METHOD" in
-    GET)
-      ;;
+  GET) ;;
+
   DELETE)
     extra_args="$extra_args -X${METHOD}"
     ;;
@@ -702,8 +687,8 @@ function must_fail_http_request {
   HEAD)
     extra_args="$extra_args -I"
     ;;
-    GET)
-      ;;
+  GET) ;;
+
   DELETE)
     extra_args="$extra_args -X${METHOD}"
     ;;
@@ -731,8 +716,7 @@ function gen_envoy_bootstrap {
   EXTRA_ENVOY_BS_ARGS="${5-}"
 
   PROXY_ID="$SERVICE"
-  if ! is_set "$IS_GW"
-  then
+  if ! is_set "$IS_GW"; then
     PROXY_ID="$SERVICE-sidecar-proxy"
   fi
 
@@ -822,15 +806,16 @@ function set_ttl_check_state {
   local DC=${3:-primary}
 
   case "$CHECK_STATE" in
-    pass)
-      ;;
-    warn)
-      ;;
-    fail)
-      ;;
+  pass) ;;
+
+  warn) ;;
+
+  fail) ;;
+
   *)
     echo "invalid ttl check state '${CHECK_STATE}'" >&2
     return 1
+    ;;
   esac
 
   retry_default docker_curl "${DC}" -sL -XPUT "http://localhost:8500/v1/agent/check/warn/${CHECK_ID}"
@@ -959,13 +944,13 @@ function register_lambdas {
   local DC=${1:-primary}
   # register lambdas to the catalog
   for f in $(find workdir/${DC}/register -type f -name 'lambda_*.json'); do
-    retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/catalog/register" >/dev/null && \
+    retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/catalog/register" >/dev/null &&
       echo "Registered Lambda: $(jq -r .Service.Service $f)"
   done
   # write service-defaults config entries for lambdas
   for f in $(find workdir/${DC}/register -type f -name 'service_defaults_*.json'); do
     varsub ${f} AWS_LAMBDA_REGION AWS_LAMBDA_ARN
-    retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/config" >/dev/null && \
+    retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/config" >/dev/null &&
       echo "Wrote config: $(jq -r '.Kind + " / " + .Name' $f)"
   done
 }
@@ -992,7 +977,8 @@ function assert_lambda_envoy_dynamic_http_filter_exists {
 }
 
 function varsub {
-  local file=$1 ; shift
+  local file=$1
+  shift
   for v in "$@"; do
     sed -i "s/\${$v}/${!v}/g" $file
   done