integ test: reduce flakiness due to compound output from retry (#15233)

* integ test: avoid flakiness due to compound output from retry

* changelog
This commit is contained in:
cskh 2022-11-02 14:08:17 -04:00 committed by GitHub
parent f704e72f3e
commit 0a3dbb1c6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 147 additions and 162 deletions

3
.changelog/15233.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note: improvement
integ test: fix flakiness due to test condition from retry app endoint
```

View File

@ -33,7 +33,7 @@ load helpers
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 proxy should have been configured with one rbac listener filter at L4" { @test "s1 proxy should have been configured with one rbac listener filter at L4" {

View File

@ -29,7 +29,7 @@ load helpers
@test "s1 upstream should be able to connect to s2 with http/1.1" { @test "s1 upstream should be able to connect to s2 with http/1.1" {
run retry_default curl --http1.1 -s -f -d hello localhost:5000 run retry_default curl --http1.1 -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 proxy should be exposing metrics to prometheus from central config" { @test "s1 proxy should be exposing metrics to prometheus from central config" {

View File

@ -49,11 +49,10 @@ load helpers
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
} }
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream made 1 connection" { @test "s1 upstream made 1 connection" {
@ -68,7 +67,6 @@ load helpers
assert_service_has_healthy_instances s2 0 primary assert_service_has_healthy_instances s2 0 primary
} }
@test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" { @test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" {
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal UNHEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal UNHEALTHY 1
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
@ -85,7 +83,7 @@ load helpers
@test "s1 upstream should be able to connect to s2 in the failover cluster peer" { @test "s1 upstream should be able to connect to s2 in the failover cluster peer" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream made 1 connection to s2 through the cluster peer" { @test "s1 upstream made 1 connection to s2 through the cluster peer" {
@ -105,7 +103,7 @@ load helpers
@test "s1 upstream should be able to connect to s2 via virtual-s2" { @test "s1 upstream should be able to connect to s2 via virtual-s2" {
run retry_default curl -s -f -d hello localhost:5001 run retry_default curl -s -f -d hello localhost:5001
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream made 1 connection to s2 via virtual-s2 through the cluster peer" { @test "s1 upstream made 1 connection to s2 via virtual-s2 through the cluster peer" {

View File

@ -37,10 +37,9 @@ load helpers
@test "s1 upstream should be able to connect to its upstream simply" { @test "s1 upstream should be able to connect to its upstream simply" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream should be able to connect to s3 via upstream s2" { @test "s1 upstream should be able to connect to s3 via upstream s2" {
assert_expected_fortio_name s3 assert_expected_fortio_name s3
} }

View File

@ -37,10 +37,9 @@ load helpers
@test "s1 upstream should be able to connect to its upstream simply" { @test "s1 upstream should be able to connect to its upstream simply" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream should be able to connect to s3 via upstream s2" { @test "s1 upstream should be able to connect to s3 via upstream s2" {
assert_expected_fortio_name s3 assert_expected_fortio_name s3
} }

View File

@ -57,7 +57,7 @@ load helpers
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream made 1 connection to s2" { @test "s1 upstream made 1 connection to s2" {

View File

@ -57,7 +57,7 @@ load helpers
@test "s1 upstream should be able to connect to s2 with http/1.1" { @test "s1 upstream should be able to connect to s2 with http/1.1" {
run retry_default curl --http1.1 -s -f -d hello localhost:5000 run retry_default curl --http1.1 -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream should be able to connect to s2 via s2" { @test "s1 upstream should be able to connect to s2 via s2" {

View File

@ -49,7 +49,7 @@ load helpers
@test "s1 upstream should be able to connect to s2 with http/1.1" { @test "s1 upstream should be able to connect to s2 with http/1.1" {
run retry_default curl --http1.1 -s -f -d hello localhost:5000 run retry_default curl --http1.1 -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream made 1 connection to s2" { @test "s1 upstream made 1 connection to s2" {

View File

@ -49,7 +49,7 @@ load helpers
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream should be able to connect to s3 via s2 due to redirect" { @test "s1 upstream should be able to connect to s3 via s2 due to redirect" {

View File

@ -49,7 +49,7 @@ load helpers
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream made 1 connection to s2" { @test "s1 upstream made 1 connection to s2" {

View File

@ -33,7 +33,7 @@ load helpers
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s2 exposes checks on a new listener" { @test "s2 exposes checks on a new listener" {

View File

@ -34,7 +34,7 @@ load helpers
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream made 1 connection" { @test "s1 upstream made 1 connection" {

View File

@ -22,7 +22,7 @@ load helpers
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream made 1 connection" { @test "s1 upstream made 1 connection" {

View File

@ -30,7 +30,7 @@ load helpers
@test "s1 upstream should be able to connect to s2 with http/1.1" { @test "s1 upstream should be able to connect to s2 with http/1.1" {
run retry_default curl --http1.1 -s -f -d hello localhost:5000 run retry_default curl --http1.1 -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 proxy should have been configured with http connection managers" { @test "s1 proxy should have been configured with http connection managers" {

View File

@ -42,5 +42,5 @@ load helpers
@test "ingress should be able to connect to s1 via configured port" { @test "ingress should be able to connect to s1 via configured port" {
run retry_default curl -s -f -d hello localhost:9999 run retry_default curl -s -f -d hello localhost:9999
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }

View File

@ -30,7 +30,7 @@ load helpers
--resolve s1.ingress.consul:9998:127.0.0.1 \ --resolve s1.ingress.consul:9998:127.0.0.1 \
https://s1.ingress.consul:9998 https://s1.ingress.consul:9998
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "should be able to connect to s1 through the TLS-enabled ingress port using the custom host" { @test "should be able to connect to s1 through the TLS-enabled ingress port using the custom host" {
@ -39,5 +39,5 @@ load helpers
--resolve test.example.com:9999:127.0.0.1 \ --resolve test.example.com:9999:127.0.0.1 \
https://test.example.com:9999 https://test.example.com:9999
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }

View File

@ -33,7 +33,7 @@ load helpers
@test "ingress should be able to connect to s1" { @test "ingress should be able to connect to s1" {
run retry_default curl -s -f -d hello localhost:10000 run retry_default curl -s -f -d hello localhost:10000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "ingress made 1 connection to s1" { @test "ingress made 1 connection to s1" {
@ -47,7 +47,7 @@ load helpers
@test "ingress should be able to connect to s2" { @test "ingress should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:9999 run retry_default curl -s -f -d hello localhost:9999
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "ingress made 1 connection to s2" { @test "ingress made 1 connection to s2" {

View File

@ -21,7 +21,7 @@ load helpers
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 upstream made 1 connection" { @test "s1 upstream made 1 connection" {

View File

@ -30,7 +30,7 @@ load helpers
@test "s1 upstream should be able to connect to s2 with http/1.1" { @test "s1 upstream should be able to connect to s2 with http/1.1" {
run retry_default curl --http1.1 -s -f -d hello localhost:5000 run retry_default curl --http1.1 -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 proxy should be exposing metrics to prometheus" { @test "s1 proxy should be exposing metrics to prometheus" {

View File

@ -30,7 +30,7 @@ load helpers
@test "s1 upstream should be able to connect to s2 with http/1.1" { @test "s1 upstream should be able to connect to s2 with http/1.1" {
run retry_default curl --http1.1 -s -f -d hello localhost:5000 run retry_default curl --http1.1 -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "s1 proxy should be exposing the /stats prefix" { @test "s1 proxy should be exposing the /stats prefix" {

View File

@ -25,7 +25,7 @@ load helpers
@test "s1 upstream should be able to connect to s4" { @test "s1 upstream should be able to connect to s4" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "terminating-gateway is used for the upstream connection" { @test "terminating-gateway is used for the upstream connection" {

View File

@ -25,7 +25,7 @@ load helpers
@test "s1 upstream should be able to connect to s2" { @test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
@test "terminating-gateway is used for the upstream connection" { @test "terminating-gateway is used for the upstream connection" {

View File

@ -11,18 +11,14 @@ function retry {
shift shift
local errtrace=0 local errtrace=0
if grep -q "errtrace" <<< "$SHELLOPTS" if grep -q "errtrace" <<<"$SHELLOPTS"; then
then
errtrace=1 errtrace=1
set +E set +E
fi fi
for ((i=1;i<=$max;i++)) for ((i = 1; i <= $max; i++)); do
do if "$@"; then
if "$@" if test $errtrace -eq 1; then
then
if test $errtrace -eq 1
then
set -E set -E
fi fi
return 0 return 0
@ -32,8 +28,7 @@ function retry {
fi fi
done done
if test $errtrace -eq 1 if test $errtrace -eq 1; then
then
set -E set -E
fi fi
return 1 return 1
@ -365,22 +360,19 @@ function assert_envoy_metric {
METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
if [ -z "${METRICS}" ] if [ -z "${METRICS}" ]; then
then
echo "Metric not found" 1>&2 echo "Metric not found" 1>&2
return 1 return 1
fi fi
GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ') GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ')
if [ -z "$GOT_COUNT" ] if [ -z "$GOT_COUNT" ]; then
then
echo "Couldn't parse metric count" 1>&2 echo "Couldn't parse metric count" 1>&2
return 1 return 1
fi fi
if [ $EXPECT_COUNT -ne $GOT_COUNT ] if [ $EXPECT_COUNT -ne $GOT_COUNT ]; then
then
echo "$METRIC - expected count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 echo "$METRIC - expected count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
return 1 return 1
fi fi
@ -394,22 +386,19 @@ function assert_envoy_metric_at_least {
METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
if [ -z "${METRICS}" ] if [ -z "${METRICS}" ]; then
then
echo "Metric not found" 1>&2 echo "Metric not found" 1>&2
return 1 return 1
fi fi
GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ') GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ')
if [ -z "$GOT_COUNT" ] if [ -z "$GOT_COUNT" ]; then
then
echo "Couldn't parse metric count" 1>&2 echo "Couldn't parse metric count" 1>&2
return 1 return 1
fi fi
if [ $EXPECT_COUNT -gt $GOT_COUNT ] if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then
then
echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
return 1 return 1
fi fi
@ -423,22 +412,19 @@ function assert_envoy_aggregate_metric_at_least {
METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
if [ -z "${METRICS}" ] if [ -z "${METRICS}" ]; then
then
echo "Metric not found" 1>&2 echo "Metric not found" 1>&2
return 1 return 1
fi fi
GOT_COUNT=$(awk '{ sum += $2 } END { print sum }' <<<"$METRICS") GOT_COUNT=$(awk '{ sum += $2 } END { print sum }' <<<"$METRICS")
if [ -z "$GOT_COUNT" ] if [ -z "$GOT_COUNT" ]; then
then
echo "Couldn't parse metric count" 1>&2 echo "Couldn't parse metric count" 1>&2
return 1 return 1
fi fi
if [ $EXPECT_COUNT -gt $GOT_COUNT ] if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then
then
echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
return 1 return 1
fi fi
@ -557,8 +543,7 @@ function docker_curl {
} }
function docker_exec { function docker_exec {
if ! docker exec -i "$@" if ! docker exec -i "$@"; then
then
echo "Failed to execute: docker exec -i $@" 1>&2 echo "Failed to execute: docker exec -i $@" 1>&2
return 1 return 1
fi fi
@ -638,7 +623,7 @@ function must_pass_tcp_connection {
echo "OUTPUT $output" echo "OUTPUT $output"
[ "$status" == "0" ] [ "$status" == "0" ]
[ "$output" = "hello" ] [[ "$output" == *"hello"* ]]
} }
# must_fail_http_connection see must_fail_tcp_connection but this expects Envoy # must_fail_http_connection see must_fail_tcp_connection but this expects Envoy
@ -669,8 +654,8 @@ function must_pass_http_request {
extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
fi fi
case "$METHOD" in case "$METHOD" in
GET) GET) ;;
;;
DELETE) DELETE)
extra_args="$extra_args -X${METHOD}" extra_args="$extra_args -X${METHOD}"
;; ;;
@ -702,8 +687,8 @@ function must_fail_http_request {
HEAD) HEAD)
extra_args="$extra_args -I" extra_args="$extra_args -I"
;; ;;
GET) GET) ;;
;;
DELETE) DELETE)
extra_args="$extra_args -X${METHOD}" extra_args="$extra_args -X${METHOD}"
;; ;;
@ -731,8 +716,7 @@ function gen_envoy_bootstrap {
EXTRA_ENVOY_BS_ARGS="${5-}" EXTRA_ENVOY_BS_ARGS="${5-}"
PROXY_ID="$SERVICE" PROXY_ID="$SERVICE"
if ! is_set "$IS_GW" if ! is_set "$IS_GW"; then
then
PROXY_ID="$SERVICE-sidecar-proxy" PROXY_ID="$SERVICE-sidecar-proxy"
fi fi
@ -822,15 +806,16 @@ function set_ttl_check_state {
local DC=${3:-primary} local DC=${3:-primary}
case "$CHECK_STATE" in case "$CHECK_STATE" in
pass) pass) ;;
;;
warn) warn) ;;
;;
fail) fail) ;;
;;
*) *)
echo "invalid ttl check state '${CHECK_STATE}'" >&2 echo "invalid ttl check state '${CHECK_STATE}'" >&2
return 1 return 1
;;
esac esac
retry_default docker_curl "${DC}" -sL -XPUT "http://localhost:8500/v1/agent/check/warn/${CHECK_ID}" retry_default docker_curl "${DC}" -sL -XPUT "http://localhost:8500/v1/agent/check/warn/${CHECK_ID}"
@ -959,13 +944,13 @@ function register_lambdas {
local DC=${1:-primary} local DC=${1:-primary}
# register lambdas to the catalog # register lambdas to the catalog
for f in $(find workdir/${DC}/register -type f -name 'lambda_*.json'); do for f in $(find workdir/${DC}/register -type f -name 'lambda_*.json'); do
retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/catalog/register" >/dev/null && \ retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/catalog/register" >/dev/null &&
echo "Registered Lambda: $(jq -r .Service.Service $f)" echo "Registered Lambda: $(jq -r .Service.Service $f)"
done done
# write service-defaults config entries for lambdas # write service-defaults config entries for lambdas
for f in $(find workdir/${DC}/register -type f -name 'service_defaults_*.json'); do for f in $(find workdir/${DC}/register -type f -name 'service_defaults_*.json'); do
varsub ${f} AWS_LAMBDA_REGION AWS_LAMBDA_ARN varsub ${f} AWS_LAMBDA_REGION AWS_LAMBDA_ARN
retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/config" >/dev/null && \ retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/config" >/dev/null &&
echo "Wrote config: $(jq -r '.Kind + " / " + .Name' $f)" echo "Wrote config: $(jq -r '.Kind + " / " + .Name' $f)"
done done
} }
@ -992,7 +977,8 @@ function assert_lambda_envoy_dynamic_http_filter_exists {
} }
function varsub { function varsub {
local file=$1 ; shift local file=$1
shift
for v in "$@"; do for v in "$@"; do
sed -i "s/\${$v}/${!v}/g" $file sed -i "s/\${$v}/${!v}/g" $file
done done