2019-04-29 16:27:57 +00:00
|
|
|
#!/bin/bash
|
|
|
|
|
|
|
|
# retry based on
|
|
|
|
# https://github.com/fernandoacorreia/azure-docker-registry/blob/master/tools/scripts/create-registry-server
|
|
|
|
# under MIT license.
|
|
|
|
function retry {
|
|
|
|
local n=1
|
|
|
|
local max=$1
|
|
|
|
shift
|
|
|
|
local delay=$1
|
|
|
|
shift
|
2019-07-24 21:01:42 +00:00
|
|
|
|
|
|
|
local errtrace=0
|
|
|
|
if grep -q "errtrace" <<< "$SHELLOPTS"
|
|
|
|
then
|
|
|
|
errtrace=1
|
|
|
|
set +E
|
|
|
|
fi
|
|
|
|
|
|
|
|
for ((i=1;i<=$max;i++))
|
|
|
|
do
|
2020-01-24 15:04:58 +00:00
|
|
|
if "$@"
|
2019-07-24 21:01:42 +00:00
|
|
|
then
|
|
|
|
if test $errtrace -eq 1
|
|
|
|
then
|
|
|
|
set -E
|
2019-04-29 16:27:57 +00:00
|
|
|
fi
|
2019-07-24 21:01:42 +00:00
|
|
|
return 0
|
|
|
|
else
|
|
|
|
echo "Command failed. Attempt $i/$max:"
|
|
|
|
sleep $delay
|
|
|
|
fi
|
2019-04-29 16:27:57 +00:00
|
|
|
done
|
2019-07-24 21:01:42 +00:00
|
|
|
|
|
|
|
if test $errtrace -eq 1
|
|
|
|
then
|
|
|
|
set -E
|
|
|
|
fi
|
|
|
|
return 1
|
2019-04-29 16:27:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
function retry_default {
|
2019-07-24 21:01:42 +00:00
|
|
|
set +E
|
|
|
|
ret=0
|
2020-01-24 15:04:58 +00:00
|
|
|
retry 5 1 "$@" || ret=1
|
2019-07-24 21:01:42 +00:00
|
|
|
set -E
|
|
|
|
return $ret
|
2019-04-29 16:27:57 +00:00
|
|
|
}
|
|
|
|
|
2019-07-24 01:08:36 +00:00
|
|
|
function retry_long {
|
2020-01-24 15:04:58 +00:00
|
|
|
retry 30 1 "$@"
|
2019-07-24 01:08:36 +00:00
|
|
|
}
|
|
|
|
|
2019-04-29 16:27:57 +00:00
|
|
|
function echored {
|
|
|
|
tput setaf 1
|
|
|
|
tput bold
|
|
|
|
echo $@
|
|
|
|
tput sgr0
|
|
|
|
}
|
|
|
|
|
|
|
|
function echogreen {
|
|
|
|
tput setaf 2
|
|
|
|
tput bold
|
|
|
|
echo $@
|
|
|
|
tput sgr0
|
|
|
|
}
|
|
|
|
|
2019-05-02 11:53:06 +00:00
|
|
|
function echoyellow {
|
|
|
|
tput setaf 3
|
|
|
|
tput bold
|
|
|
|
echo $@
|
|
|
|
tput sgr0
|
|
|
|
}
|
|
|
|
|
|
|
|
function echoblue {
|
|
|
|
tput setaf 4
|
|
|
|
tput bold
|
|
|
|
echo $@
|
|
|
|
tput sgr0
|
|
|
|
}
|
|
|
|
|
2019-07-24 21:01:42 +00:00
|
|
|
function is_set {
|
|
|
|
# Arguments:
|
|
|
|
# $1 - string value to check its truthiness
|
|
|
|
#
|
|
|
|
# Return:
|
|
|
|
# 0 - is truthy (backwards I know but allows syntax like `if is_set <var>` to work)
|
|
|
|
# 1 - is not truthy
|
|
|
|
|
|
|
|
local val=$(tr '[:upper:]' '[:lower:]' <<< "$1")
|
|
|
|
case $val in
|
|
|
|
1 | t | true | y | yes)
|
|
|
|
return 0
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
return 1
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
}
|
|
|
|
|
2019-04-29 16:27:57 +00:00
|
|
|
function get_cert {
|
|
|
|
local HOSTPORT=$1
|
2020-04-29 09:58:35 +00:00
|
|
|
CERT=$(openssl s_client -connect $HOSTPORT -showcerts </dev/null)
|
2019-07-24 21:01:42 +00:00
|
|
|
openssl x509 -noout -text <<< "$CERT"
|
2019-04-29 16:27:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
function assert_proxy_presents_cert_uri {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
local SERVICENAME=$2
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=${3:-primary}
|
2020-01-24 15:04:58 +00:00
|
|
|
local NS=${4:-default}
|
2019-07-24 21:01:42 +00:00
|
|
|
|
2019-04-29 16:27:57 +00:00
|
|
|
CERT=$(retry_default get_cert $HOSTPORT)
|
|
|
|
|
2020-01-24 15:04:58 +00:00
|
|
|
echo "WANT SERVICE: ${NS}/${SERVICENAME}"
|
2019-04-29 16:27:57 +00:00
|
|
|
echo "GOT CERT:"
|
|
|
|
echo "$CERT"
|
|
|
|
|
2020-01-24 15:04:58 +00:00
|
|
|
echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
|
2019-04-29 16:27:57 +00:00
|
|
|
}
|
|
|
|
|
2020-04-29 09:58:35 +00:00
|
|
|
function assert_dnssan_in_cert {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
local DNSSAN=$2
|
|
|
|
|
|
|
|
CERT=$(retry_default get_cert $HOSTPORT)
|
|
|
|
|
|
|
|
echo "WANT DNSSAN: ${DNSSAN}"
|
|
|
|
echo "GOT CERT:"
|
|
|
|
echo "$CERT"
|
|
|
|
|
|
|
|
echo "$CERT" | grep -Eo "DNS:${DNSSAN}"
|
|
|
|
}
|
|
|
|
|
2019-06-21 15:06:25 +00:00
|
|
|
function assert_envoy_version {
|
|
|
|
local ADMINPORT=$1
|
|
|
|
run retry_default curl -f -s localhost:$ADMINPORT/server_info
|
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
# Envoy 1.8.0 returns a plain text line like
|
|
|
|
# envoy 5d25f466c3410c0dfa735d7d4358beb76b2da507/1.8.0/Clean/DEBUG live 3 3 0
|
|
|
|
# Later versions return JSON.
|
|
|
|
if (echo $output | grep '^envoy') ; then
|
|
|
|
VERSION=$(echo $output | cut -d ' ' -f 2)
|
|
|
|
else
|
|
|
|
VERSION=$(echo $output | jq -r '.version')
|
|
|
|
fi
|
|
|
|
echo "Status=$status"
|
|
|
|
echo "Output=$output"
|
|
|
|
echo "---"
|
|
|
|
echo "Got version=$VERSION"
|
|
|
|
echo "Want version=$ENVOY_VERSION"
|
|
|
|
echo $VERSION | grep "/$ENVOY_VERSION/"
|
|
|
|
}
|
|
|
|
|
2020-10-22 19:30:28 +00:00
|
|
|
function assert_envoy_http_rbac_policy_count {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
local EXPECT_COUNT=$2
|
|
|
|
|
|
|
|
GOT_COUNT=$(get_envoy_http_rbac_once $HOSTPORT | jq '.rules.policies | length')
|
2020-10-30 21:49:54 +00:00
|
|
|
echo "GOT_COUNT = $GOT_COUNT"
|
2020-10-22 19:30:28 +00:00
|
|
|
[ "${GOT_COUNT:-0}" -eq $EXPECT_COUNT ]
|
|
|
|
}
|
|
|
|
|
|
|
|
function get_envoy_http_rbac_once {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
run curl -s -f $HOSTPORT/config_dump
|
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
echo "$output" | jq --raw-output '.configs[2].dynamic_listeners[].active_state.listener.filter_chains[0].filters[0].config.http_filters[] | select(.name == "envoy.filters.http.rbac") | .config'
|
|
|
|
}
|
|
|
|
|
|
|
|
function assert_envoy_network_rbac_policy_count {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
local EXPECT_COUNT=$2
|
|
|
|
|
|
|
|
GOT_COUNT=$(get_envoy_network_rbac_once $HOSTPORT | jq '.rules.policies | length')
|
2020-10-30 21:49:54 +00:00
|
|
|
echo "GOT_COUNT = $GOT_COUNT"
|
2020-10-22 19:30:28 +00:00
|
|
|
[ "${GOT_COUNT:-0}" -eq $EXPECT_COUNT ]
|
|
|
|
}
|
|
|
|
|
|
|
|
function get_envoy_network_rbac_once {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
run curl -s -f $HOSTPORT/config_dump
|
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
echo "$output" | jq --raw-output '.configs[2].dynamic_listeners[].active_state.listener.filter_chains[0].filters[] | select(.name == "envoy.filters.network.rbac") | .config'
|
|
|
|
}
|
|
|
|
|
2019-04-29 16:27:57 +00:00
|
|
|
function get_envoy_listener_filters {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
run retry_default curl -s -f $HOSTPORT/config_dump
|
|
|
|
[ "$status" -eq 0 ]
|
2020-10-22 19:30:28 +00:00
|
|
|
echo "$output" | jq --raw-output '.configs[2].dynamic_listeners[].active_state.listener | "\(.name) \( .filter_chains[0].filters | map(.name) | join(","))"'
|
2019-04-29 16:27:57 +00:00
|
|
|
}
|
|
|
|
|
2020-08-27 17:20:58 +00:00
|
|
|
function get_envoy_http_filters {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
run retry_default curl -s -f $HOSTPORT/config_dump
|
|
|
|
[ "$status" -eq 0 ]
|
2020-10-22 19:30:28 +00:00
|
|
|
echo "$output" | jq --raw-output '.configs[2].dynamic_listeners[].active_state.listener | "\(.name) \( .filter_chains[0].filters[] | select(.name == "envoy.http_connection_manager") | .config.http_filters | map(.name) | join(","))"'
|
2020-08-27 17:20:58 +00:00
|
|
|
}
|
|
|
|
|
2020-05-05 20:17:24 +00:00
|
|
|
function get_envoy_cluster_config {
|
2019-12-03 20:13:33 +00:00
|
|
|
local HOSTPORT=$1
|
|
|
|
local CLUSTER_NAME=$2
|
|
|
|
run retry_default curl -s -f $HOSTPORT/config_dump
|
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
echo "$output" | jq --raw-output "
|
|
|
|
.configs[1].dynamic_active_clusters[]
|
|
|
|
| select(.cluster.name|startswith(\"${CLUSTER_NAME}\"))
|
2020-05-05 20:17:24 +00:00
|
|
|
| .cluster
|
2019-12-03 20:13:33 +00:00
|
|
|
"
|
|
|
|
}
|
|
|
|
|
2019-04-29 16:27:57 +00:00
|
|
|
function get_envoy_stats_flush_interval {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
run retry_default curl -s -f $HOSTPORT/config_dump
|
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
#echo "$output" > /workdir/s1_envoy_dump.json
|
|
|
|
echo "$output" | jq --raw-output '.configs[0].bootstrap.stats_flush_interval'
|
|
|
|
}
|
|
|
|
|
2019-07-02 03:10:51 +00:00
|
|
|
# snapshot_envoy_admin is meant to be used from a teardown scriptlet from the host.
|
|
|
|
function snapshot_envoy_admin {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
local ENVOY_NAME=$2
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=${3:-primary}
|
2020-03-05 21:01:10 +00:00
|
|
|
local OUTDIR="${LOG_DIR}/envoy-snapshots/${DC}/${ENVOY_NAME}"
|
2020-11-02 17:37:03 +00:00
|
|
|
|
2020-03-05 21:01:10 +00:00
|
|
|
mkdir -p "${OUTDIR}"
|
|
|
|
docker_wget "$DC" "http://${HOSTPORT}/config_dump" -q -O - > "${OUTDIR}/config_dump.json"
|
|
|
|
docker_wget "$DC" "http://${HOSTPORT}/clusters?format=json" -q -O - > "${OUTDIR}/clusters.json"
|
|
|
|
docker_wget "$DC" "http://${HOSTPORT}/stats" -q -O - > "${OUTDIR}/stats.txt"
|
2020-11-16 23:37:19 +00:00
|
|
|
docker_wget "$DC" "http://${HOSTPORT}/stats/prometheus" -q -O - > "${OUTDIR}/stats_prometheus.txt"
|
2019-07-24 21:01:42 +00:00
|
|
|
}
|
|
|
|
|
2019-09-25 14:08:42 +00:00
|
|
|
function reset_envoy_metrics {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
curl -s -f -XPOST $HOSTPORT/reset_counters
|
|
|
|
return $?
|
|
|
|
}
|
|
|
|
|
2019-07-24 21:01:42 +00:00
|
|
|
function get_all_envoy_metrics {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
curl -s -f $HOSTPORT/stats
|
|
|
|
return $?
|
|
|
|
}
|
|
|
|
|
|
|
|
function get_envoy_metrics {
|
|
|
|
local HOSTPORT=$1
|
|
|
|
local METRICS=$2
|
|
|
|
|
|
|
|
get_all_envoy_metrics $HOSTPORT | grep "$METRICS"
|
2019-07-12 16:12:56 +00:00
|
|
|
}
|
|
|
|
|
2019-07-24 01:08:36 +00:00
|
|
|
function get_upstream_endpoint_in_status_count {
|
2019-07-12 16:12:56 +00:00
|
|
|
local HOSTPORT=$1
|
|
|
|
local CLUSTER_NAME=$2
|
2019-07-24 01:08:36 +00:00
|
|
|
local HEALTH_STATUS=$3
|
2020-10-22 19:30:28 +00:00
|
|
|
run curl -s -f "http://${HOSTPORT}/clusters?format=json"
|
2019-07-12 16:12:56 +00:00
|
|
|
[ "$status" -eq 0 ]
|
2019-07-24 01:08:36 +00:00
|
|
|
# echo "$output" >&3
|
2019-07-12 16:12:56 +00:00
|
|
|
echo "$output" | jq --raw-output "
|
|
|
|
.cluster_statuses[]
|
2019-07-24 21:01:42 +00:00
|
|
|
| select(.name|startswith(\"${CLUSTER_NAME}\"))
|
2019-07-24 01:08:36 +00:00
|
|
|
| [.host_statuses[].health_status.eds_health_status]
|
|
|
|
| [select(.[] == \"${HEALTH_STATUS}\")]
|
2019-07-12 16:12:56 +00:00
|
|
|
| length"
|
|
|
|
}
|
|
|
|
|
2019-07-24 01:08:36 +00:00
|
|
|
function assert_upstream_has_endpoints_in_status_once {
|
2019-07-12 16:12:56 +00:00
|
|
|
local HOSTPORT=$1
|
|
|
|
local CLUSTER_NAME=$2
|
2019-07-24 01:08:36 +00:00
|
|
|
local HEALTH_STATUS=$3
|
|
|
|
local EXPECT_COUNT=$4
|
2019-07-12 16:12:56 +00:00
|
|
|
|
2019-07-24 01:08:36 +00:00
|
|
|
GOT_COUNT=$(get_upstream_endpoint_in_status_count $HOSTPORT $CLUSTER_NAME $HEALTH_STATUS)
|
2019-07-12 16:12:56 +00:00
|
|
|
|
|
|
|
[ "$GOT_COUNT" -eq $EXPECT_COUNT ]
|
|
|
|
}
|
|
|
|
|
2019-07-24 01:08:36 +00:00
|
|
|
function assert_upstream_has_endpoints_in_status {
|
2019-07-12 16:12:56 +00:00
|
|
|
local HOSTPORT=$1
|
|
|
|
local CLUSTER_NAME=$2
|
2019-07-24 01:08:36 +00:00
|
|
|
local HEALTH_STATUS=$3
|
|
|
|
local EXPECT_COUNT=$4
|
|
|
|
run retry_long assert_upstream_has_endpoints_in_status_once $HOSTPORT $CLUSTER_NAME $HEALTH_STATUS $EXPECT_COUNT
|
2019-07-12 16:12:56 +00:00
|
|
|
[ "$status" -eq 0 ]
|
2019-07-02 03:10:51 +00:00
|
|
|
}
|
|
|
|
|
2019-07-24 21:01:42 +00:00
|
|
|
function assert_envoy_metric {
|
|
|
|
set -eEuo pipefail
|
|
|
|
local HOSTPORT=$1
|
|
|
|
local METRIC=$2
|
|
|
|
local EXPECT_COUNT=$3
|
|
|
|
|
|
|
|
METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
|
|
|
|
|
|
|
|
if [ -z "${METRICS}" ]
|
|
|
|
then
|
|
|
|
echo "Metric not found" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
GOT_COUNT=$(awk -F: '{print $2}' <<< "$METRICS" | head -n 1 | tr -d ' ')
|
|
|
|
|
|
|
|
if [ -z "$GOT_COUNT" ]
|
|
|
|
then
|
|
|
|
echo "Couldn't parse metric count" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ $EXPECT_COUNT -ne $GOT_COUNT ]
|
|
|
|
then
|
|
|
|
echo "$METRIC - expected count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2019-09-25 14:08:42 +00:00
|
|
|
function assert_envoy_metric_at_least {
|
|
|
|
set -eEuo pipefail
|
|
|
|
local HOSTPORT=$1
|
|
|
|
local METRIC=$2
|
|
|
|
local EXPECT_COUNT=$3
|
|
|
|
|
|
|
|
METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
|
|
|
|
|
|
|
|
if [ -z "${METRICS}" ]
|
|
|
|
then
|
|
|
|
echo "Metric not found" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
GOT_COUNT=$(awk -F: '{print $2}' <<< "$METRICS" | head -n 1 | tr -d ' ')
|
|
|
|
|
|
|
|
if [ -z "$GOT_COUNT" ]
|
|
|
|
then
|
|
|
|
echo "Couldn't parse metric count" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ $EXPECT_COUNT -gt $GOT_COUNT ]
|
|
|
|
then
|
|
|
|
echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
}
|
2019-07-24 21:01:42 +00:00
|
|
|
|
2020-01-24 15:04:58 +00:00
|
|
|
function assert_envoy_aggregate_metric_at_least {
|
|
|
|
set -eEuo pipefail
|
|
|
|
local HOSTPORT=$1
|
|
|
|
local METRIC=$2
|
|
|
|
local EXPECT_COUNT=$3
|
|
|
|
|
|
|
|
METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
|
|
|
|
|
|
|
|
if [ -z "${METRICS}" ]
|
|
|
|
then
|
|
|
|
echo "Metric not found" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
GOT_COUNT=$(awk '{ sum += $2 } END { print sum }' <<< "$METRICS")
|
|
|
|
|
|
|
|
if [ -z "$GOT_COUNT" ]
|
|
|
|
then
|
|
|
|
echo "Couldn't parse metric count" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ $EXPECT_COUNT -gt $GOT_COUNT ]
|
|
|
|
then
|
|
|
|
echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2019-07-10 20:58:25 +00:00
|
|
|
function get_healthy_service_count {
|
|
|
|
local SERVICE_NAME=$1
|
2019-08-05 18:30:35 +00:00
|
|
|
local DC=$2
|
2020-01-24 15:04:58 +00:00
|
|
|
local NS=$3
|
2020-02-03 17:19:34 +00:00
|
|
|
|
2020-10-22 19:30:28 +00:00
|
|
|
run curl -s -f ${HEADERS} "127.0.0.1:8500/v1/health/connect/${SERVICE_NAME}?dc=${DC}&passing&ns=${NS}"
|
2019-07-10 20:58:25 +00:00
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
echo "$output" | jq --raw-output '. | length'
|
|
|
|
}
|
|
|
|
|
2020-03-09 20:59:02 +00:00
|
|
|
function assert_alive_wan_member_count {
|
|
|
|
local EXPECT_COUNT=$1
|
2020-04-13 14:02:42 +00:00
|
|
|
run retry_long assert_alive_wan_member_count_once $EXPECT_COUNT
|
2020-03-09 20:59:02 +00:00
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
}
|
|
|
|
|
|
|
|
function assert_alive_wan_member_count_once {
|
|
|
|
local EXPECT_COUNT=$1
|
|
|
|
|
|
|
|
GOT_COUNT=$(get_alive_wan_member_count)
|
|
|
|
|
|
|
|
[ "$GOT_COUNT" -eq "$EXPECT_COUNT" ]
|
|
|
|
}
|
|
|
|
|
|
|
|
function get_alive_wan_member_count {
|
|
|
|
run retry_default curl -sL -f "127.0.0.1:8500/v1/agent/members?wan=1"
|
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
# echo "$output" >&3
|
|
|
|
echo "$output" | jq '.[] | select(.Status == 1) | .Name' | wc -l
|
|
|
|
}
|
|
|
|
|
2019-07-12 16:12:56 +00:00
|
|
|
function assert_service_has_healthy_instances_once {
|
2019-07-10 20:58:25 +00:00
|
|
|
local SERVICE_NAME=$1
|
|
|
|
local EXPECT_COUNT=$2
|
2019-08-05 18:30:35 +00:00
|
|
|
local DC=${3:-primary}
|
2020-01-24 15:04:58 +00:00
|
|
|
local NS=$4
|
2019-07-10 20:58:25 +00:00
|
|
|
|
2020-01-24 15:04:58 +00:00
|
|
|
GOT_COUNT=$(get_healthy_service_count "$SERVICE_NAME" "$DC" "$NS")
|
2019-07-10 20:58:25 +00:00
|
|
|
|
|
|
|
[ "$GOT_COUNT" -eq $EXPECT_COUNT ]
|
|
|
|
}
|
|
|
|
|
|
|
|
function assert_service_has_healthy_instances {
|
|
|
|
local SERVICE_NAME=$1
|
|
|
|
local EXPECT_COUNT=$2
|
2019-08-05 18:30:35 +00:00
|
|
|
local DC=${3:-primary}
|
2020-01-24 15:04:58 +00:00
|
|
|
local NS=$4
|
2019-07-10 20:58:25 +00:00
|
|
|
|
2020-01-24 15:04:58 +00:00
|
|
|
run retry_long assert_service_has_healthy_instances_once "$SERVICE_NAME" "$EXPECT_COUNT" "$DC" "$NS"
|
2019-07-10 20:58:25 +00:00
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
}
|
|
|
|
|
2020-01-29 22:30:38 +00:00
|
|
|
function check_intention {
|
|
|
|
local SOURCE=$1
|
|
|
|
local DESTINATION=$2
|
2020-02-03 17:19:34 +00:00
|
|
|
|
2020-01-29 22:30:38 +00:00
|
|
|
curl -s -f "localhost:8500/v1/connect/intentions/check?source=${SOURCE}&destination=${DESTINATION}" | jq ".Allowed"
|
|
|
|
}
|
|
|
|
|
|
|
|
function assert_intention_allowed {
|
|
|
|
local SOURCE=$1
|
|
|
|
local DESTINATION=$2
|
2020-02-03 17:19:34 +00:00
|
|
|
|
2020-10-26 22:04:35 +00:00
|
|
|
run check_intention "${SOURCE}" "${DESTINATION}"
|
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
[ "$output" = "true" ]
|
2020-01-29 22:30:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
function assert_intention_denied {
|
|
|
|
local SOURCE=$1
|
|
|
|
local DESTINATION=$2
|
2020-02-03 17:19:34 +00:00
|
|
|
|
2020-10-26 22:04:35 +00:00
|
|
|
run check_intention "${SOURCE}" "${DESTINATION}"
|
|
|
|
[ "$status" -eq 0 ]
|
|
|
|
[ "$output" = "false" ]
|
2020-01-29 22:30:38 +00:00
|
|
|
}
|
|
|
|
|
2019-04-29 16:27:57 +00:00
|
|
|
function docker_consul {
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=$1
|
|
|
|
shift 1
|
|
|
|
docker run -i --rm --network container:envoy_consul-${DC}_1 consul-dev "$@"
|
2019-04-29 16:27:57 +00:00
|
|
|
}
|
|
|
|
|
2019-07-02 03:10:51 +00:00
|
|
|
function docker_wget {
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=$1
|
|
|
|
shift 1
|
2020-11-02 17:37:03 +00:00
|
|
|
docker run --rm --network container:envoy_consul-${DC}_1 docker.mirror.hashicorp.services/alpine:3.9 wget "$@"
|
2019-07-02 03:10:51 +00:00
|
|
|
}
|
|
|
|
|
2019-07-24 01:20:24 +00:00
|
|
|
function docker_curl {
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=$1
|
|
|
|
shift 1
|
2020-05-19 18:00:00 +00:00
|
|
|
docker run --rm --network container:envoy_consul-${DC}_1 --entrypoint curl consul-dev "$@"
|
2019-07-24 01:20:24 +00:00
|
|
|
}
|
|
|
|
|
2020-01-24 15:04:58 +00:00
|
|
|
function docker_exec {
|
|
|
|
if ! docker exec -i "$@"
|
|
|
|
then
|
|
|
|
echo "Failed to execute: docker exec -i $@" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
function docker_consul_exec {
|
|
|
|
local DC=$1
|
|
|
|
shift 1
|
|
|
|
docker_exec envoy_consul-${DC}_1 "$@"
|
|
|
|
}
|
|
|
|
|
2019-07-24 01:08:36 +00:00
|
|
|
function get_envoy_pid {
|
|
|
|
local BOOTSTRAP_NAME=$1
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=${2:-primary}
|
2019-07-24 01:08:36 +00:00
|
|
|
run ps aux
|
|
|
|
[ "$status" == 0 ]
|
2019-07-24 21:01:42 +00:00
|
|
|
echo "$output" 1>&2
|
|
|
|
PID="$(echo "$output" | grep "envoy -c /workdir/$DC/envoy/${BOOTSTRAP_NAME}-bootstrap.json" | awk '{print $1}')"
|
2019-07-24 01:08:36 +00:00
|
|
|
[ -n "$PID" ]
|
|
|
|
|
|
|
|
echo "$PID"
|
|
|
|
}
|
|
|
|
|
|
|
|
function kill_envoy {
|
|
|
|
local BOOTSTRAP_NAME=$1
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=${2:-primary}
|
2019-07-24 01:08:36 +00:00
|
|
|
|
2019-07-24 21:01:42 +00:00
|
|
|
PID="$(get_envoy_pid $BOOTSTRAP_NAME "$DC")"
|
2019-07-24 01:08:36 +00:00
|
|
|
echo "PID = $PID"
|
|
|
|
|
|
|
|
kill -TERM $PID
|
|
|
|
}
|
|
|
|
|
2019-04-29 16:27:57 +00:00
|
|
|
function must_match_in_statsd_logs {
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=${2:-primary}
|
|
|
|
|
|
|
|
run cat /workdir/${DC}/statsd/statsd.log
|
|
|
|
echo "$output"
|
2019-04-29 16:27:57 +00:00
|
|
|
COUNT=$( echo "$output" | grep -Ec $1 )
|
|
|
|
|
|
|
|
echo "COUNT of '$1' matches: $COUNT"
|
|
|
|
|
|
|
|
[ "$status" == 0 ]
|
|
|
|
[ "$COUNT" -gt "0" ]
|
|
|
|
}
|
|
|
|
|
|
|
|
function must_match_in_prometheus_response {
|
|
|
|
run curl -f -s $1/metrics
|
|
|
|
COUNT=$( echo "$output" | grep -Ec $2 )
|
|
|
|
|
2019-05-01 23:39:31 +00:00
|
|
|
echo "OUTPUT head -n 10"
|
|
|
|
echo "$output" | head -n 10
|
2019-04-29 16:27:57 +00:00
|
|
|
echo "COUNT of '$2' matches: $COUNT"
|
|
|
|
|
|
|
|
[ "$status" == 0 ]
|
|
|
|
[ "$COUNT" -gt "0" ]
|
|
|
|
}
|
|
|
|
|
2020-02-03 17:19:34 +00:00
|
|
|
function must_match_in_stats_proxy_response {
|
|
|
|
run curl -f -s $1/$2
|
|
|
|
COUNT=$( echo "$output" | grep -Ec $3 )
|
|
|
|
|
|
|
|
echo "OUTPUT head -n 10"
|
|
|
|
echo "$output" | head -n 10
|
|
|
|
echo "COUNT of '$3' matches: $COUNT"
|
|
|
|
|
|
|
|
[ "$status" == 0 ]
|
|
|
|
[ "$COUNT" -gt "0" ]
|
|
|
|
}
|
|
|
|
|
2019-04-29 16:27:57 +00:00
|
|
|
# must_fail_tcp_connection checks that a request made through an upstream fails,
|
|
|
|
# probably due to authz being denied if all other tests passed already. Although
|
|
|
|
# we are using curl, this only works as expected for TCP upstreams as we are
|
|
|
|
# checking TCP-level errors. HTTP upstreams will return a valid 503 generated by
|
|
|
|
# Envoy rather than a connection-level error.
|
|
|
|
function must_fail_tcp_connection {
|
|
|
|
# Attempt to curl through upstream
|
2020-10-26 22:04:35 +00:00
|
|
|
run curl --no-keepalive -s -v -f -d hello $1
|
2019-04-29 16:27:57 +00:00
|
|
|
|
|
|
|
echo "OUTPUT $output"
|
|
|
|
|
|
|
|
# Should fail during handshake and return "got nothing" error
|
|
|
|
[ "$status" == "52" ]
|
|
|
|
|
|
|
|
# Verbose output should enclude empty reply
|
|
|
|
echo "$output" | grep 'Empty reply from server'
|
|
|
|
}
|
|
|
|
|
2020-10-26 22:04:35 +00:00
|
|
|
function must_pass_tcp_connection {
|
|
|
|
run curl --no-keepalive -s -f -d hello $1
|
|
|
|
|
|
|
|
echo "OUTPUT $output"
|
|
|
|
|
|
|
|
[ "$status" == "0" ]
|
|
|
|
[ "$output" = "hello" ]
|
|
|
|
}
|
|
|
|
|
2019-04-29 16:27:57 +00:00
|
|
|
# must_fail_http_connection see must_fail_tcp_connection but this expects Envoy
|
|
|
|
# to generate a 503 response since the upstreams have refused connection.
|
|
|
|
function must_fail_http_connection {
|
|
|
|
# Attempt to curl through upstream
|
2020-10-26 22:04:35 +00:00
|
|
|
run curl --no-keepalive -s -i -d hello "$1"
|
2019-04-29 16:27:57 +00:00
|
|
|
|
|
|
|
echo "OUTPUT $output"
|
|
|
|
|
2020-10-22 19:30:28 +00:00
|
|
|
[ "$status" == "0" ]
|
|
|
|
|
2020-08-27 17:20:58 +00:00
|
|
|
local expect_response="${2:-403 Forbidden}"
|
2019-04-29 16:27:57 +00:00
|
|
|
# Should fail request with 503
|
2020-08-27 17:20:58 +00:00
|
|
|
echo "$output" | grep "${expect_response}"
|
2019-05-01 23:39:31 +00:00
|
|
|
}
|
|
|
|
|
2020-10-06 22:09:13 +00:00
|
|
|
# must_pass_http_request allows you to craft a specific http request to assert
|
|
|
|
# that envoy will NOT reject the request. Primarily of use for testing L7
|
|
|
|
# intentions.
|
|
|
|
function must_pass_http_request {
|
|
|
|
local METHOD=$1
|
|
|
|
local URL=$2
|
|
|
|
local DEBUG_HEADER_VALUE="${3:-""}"
|
|
|
|
|
|
|
|
local extra_args
|
|
|
|
if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then
|
|
|
|
extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
|
|
|
|
fi
|
|
|
|
case "$METHOD" in
|
|
|
|
GET)
|
|
|
|
;;
|
|
|
|
DELETE)
|
|
|
|
extra_args="$extra_args -X${METHOD}"
|
|
|
|
;;
|
|
|
|
PUT|POST)
|
|
|
|
extra_args="$extra_args -d'{}' -X${METHOD}"
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
return 1
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
2020-10-26 22:04:35 +00:00
|
|
|
run curl --no-keepalive -v -s -f $extra_args "$URL"
|
2020-10-06 22:09:13 +00:00
|
|
|
[ "$status" == 0 ]
|
|
|
|
}
|
|
|
|
|
|
|
|
# must_fail_http_request allows you to craft a specific http request to assert
|
|
|
|
# that envoy will reject the request. Primarily of use for testing L7
|
|
|
|
# intentions.
|
|
|
|
function must_fail_http_request {
|
|
|
|
local METHOD=$1
|
|
|
|
local URL=$2
|
|
|
|
local DEBUG_HEADER_VALUE="${3:-""}"
|
|
|
|
|
|
|
|
local extra_args
|
|
|
|
if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then
|
|
|
|
extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
|
|
|
|
fi
|
|
|
|
case "$METHOD" in
|
|
|
|
HEAD)
|
|
|
|
extra_args="$extra_args -I"
|
|
|
|
;;
|
|
|
|
GET)
|
|
|
|
;;
|
|
|
|
DELETE)
|
|
|
|
extra_args="$extra_args -X${METHOD}"
|
|
|
|
;;
|
|
|
|
PUT|POST)
|
|
|
|
extra_args="$extra_args -d'{}' -X${METHOD}"
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
return 1
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
|
|
|
# Attempt to curl through upstream
|
2020-10-26 22:04:35 +00:00
|
|
|
run curl --no-keepalive -s -i $extra_args "$URL"
|
2020-10-06 22:09:13 +00:00
|
|
|
|
|
|
|
echo "OUTPUT $output"
|
|
|
|
|
|
|
|
echo "$output" | grep "403 Forbidden"
|
|
|
|
}
|
|
|
|
|
2019-05-01 23:39:31 +00:00
|
|
|
function gen_envoy_bootstrap {
|
|
|
|
SERVICE=$1
|
|
|
|
ADMIN_PORT=$2
|
2019-07-24 21:01:42 +00:00
|
|
|
DC=${3:-primary}
|
2020-04-16 21:00:48 +00:00
|
|
|
IS_GW=${4:-0}
|
2020-01-24 15:04:58 +00:00
|
|
|
EXTRA_ENVOY_BS_ARGS="${5-}"
|
2019-07-24 21:01:42 +00:00
|
|
|
|
|
|
|
PROXY_ID="$SERVICE"
|
2020-04-16 21:00:48 +00:00
|
|
|
if ! is_set "$IS_GW"
|
2019-07-24 21:01:42 +00:00
|
|
|
then
|
|
|
|
PROXY_ID="$SERVICE-sidecar-proxy"
|
|
|
|
fi
|
2019-05-01 23:39:31 +00:00
|
|
|
|
2019-07-24 21:01:42 +00:00
|
|
|
if output=$(docker_consul "$DC" connect envoy -bootstrap \
|
|
|
|
-proxy-id $PROXY_ID \
|
2020-02-10 19:53:04 +00:00
|
|
|
-envoy-version "$ENVOY_VERSION" \
|
2020-01-24 15:04:58 +00:00
|
|
|
-admin-bind 0.0.0.0:$ADMIN_PORT ${EXTRA_ENVOY_BS_ARGS} 2>&1); then
|
2019-05-01 23:39:31 +00:00
|
|
|
|
|
|
|
# All OK, write config to file
|
2019-07-24 21:01:42 +00:00
|
|
|
echo "$output" > workdir/${DC}/envoy/$SERVICE-bootstrap.json
|
2019-05-01 23:39:31 +00:00
|
|
|
else
|
|
|
|
status=$?
|
|
|
|
# Command failed, instead of swallowing error (printed on stdout by docker
|
|
|
|
# it seems) by writing it to file, echo it
|
|
|
|
echo "$output"
|
|
|
|
return $status
|
|
|
|
fi
|
2019-05-07 14:13:07 +00:00
|
|
|
}
|
2019-07-12 17:21:25 +00:00
|
|
|
|
2019-07-24 01:08:36 +00:00
|
|
|
function read_config_entry {
|
|
|
|
local KIND=$1
|
|
|
|
local NAME=$2
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=${3:-primary}
|
|
|
|
|
|
|
|
docker_consul "$DC" config read -kind $KIND -name $NAME
|
2019-07-24 01:08:36 +00:00
|
|
|
}
|
|
|
|
|
2020-10-28 16:58:40 +00:00
|
|
|
function wait_for_namespace {
|
2020-11-02 20:31:19 +00:00
|
|
|
local NS="${1}"
|
|
|
|
local DC=${2:-primary}
|
|
|
|
retry_default docker_curl "$DC" -sLf "http://127.0.0.1:8500/v1/namespace/${NS}" >/dev/null
|
2020-10-28 16:58:40 +00:00
|
|
|
}
|
|
|
|
|
2019-07-24 01:08:36 +00:00
|
|
|
function wait_for_config_entry {
|
2019-07-24 21:01:42 +00:00
|
|
|
retry_default read_config_entry "$@" >/dev/null
|
2019-07-24 01:08:36 +00:00
|
|
|
}
|
|
|
|
|
2019-07-24 01:20:24 +00:00
|
|
|
function delete_config_entry {
|
|
|
|
local KIND=$1
|
|
|
|
local NAME=$2
|
|
|
|
retry_default curl -sL -XDELETE "http://127.0.0.1:8500/v1/config/${KIND}/${NAME}"
|
|
|
|
}
|
|
|
|
|
2020-11-09 19:59:46 +00:00
|
|
|
function register_services {
|
|
|
|
local DC=${1:-primary}
|
|
|
|
docker_consul_exec ${DC} sh -c "consul services register /workdir/${DC}/register/service_*.hcl"
|
|
|
|
}
|
|
|
|
|
2020-10-26 22:04:35 +00:00
|
|
|
function setup_upsert_l4_intention {
|
2020-01-29 22:30:38 +00:00
|
|
|
local SOURCE=$1
|
|
|
|
local DESTINATION=$2
|
2020-10-26 22:04:35 +00:00
|
|
|
local ACTION=$3
|
2020-02-03 17:19:34 +00:00
|
|
|
|
2020-10-26 22:04:35 +00:00
|
|
|
retry_default docker_curl primary -sL -XPUT "http://127.0.0.1:8500/v1/connect/intentions/exact?source=${SOURCE}&destination=${DESTINATION}" \
|
|
|
|
-d"{\"Action\": \"${ACTION}\"}" >/dev/null
|
2020-01-29 22:30:38 +00:00
|
|
|
}
|
|
|
|
|
2020-10-26 22:04:35 +00:00
|
|
|
function upsert_l4_intention {
|
2020-01-29 22:30:38 +00:00
|
|
|
local SOURCE=$1
|
|
|
|
local DESTINATION=$2
|
|
|
|
local ACTION=$3
|
2020-02-03 17:19:34 +00:00
|
|
|
|
2020-10-26 22:04:35 +00:00
|
|
|
retry_default curl -sL -XPUT "http://127.0.0.1:8500/v1/connect/intentions/exact?source=${SOURCE}&destination=${DESTINATION}" \
|
|
|
|
-d"{\"Action\": \"${ACTION}\"}" >/dev/null
|
2020-01-29 22:30:38 +00:00
|
|
|
}
|
|
|
|
|
2020-04-29 09:58:35 +00:00
|
|
|
function get_ca_root {
|
|
|
|
curl -s -f "http://localhost:8500/v1/connect/ca/roots" | jq -r ".Roots[0].RootCert"
|
|
|
|
}
|
|
|
|
|
2019-07-24 01:20:24 +00:00
|
|
|
function wait_for_agent_service_register {
|
|
|
|
local SERVICE_ID=$1
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=${2:-primary}
|
|
|
|
retry_default docker_curl "$DC" -sLf "http://127.0.0.1:8500/v1/agent/service/${SERVICE_ID}" >/dev/null
|
2019-07-24 01:20:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
function set_ttl_check_state {
|
|
|
|
local CHECK_ID=$1
|
|
|
|
local CHECK_STATE=$2
|
2019-07-24 21:01:42 +00:00
|
|
|
local DC=${3:-primary}
|
2019-07-24 01:20:24 +00:00
|
|
|
|
|
|
|
case "$CHECK_STATE" in
|
|
|
|
pass)
|
|
|
|
;;
|
|
|
|
warn)
|
|
|
|
;;
|
|
|
|
fail)
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
echo "invalid ttl check state '${CHECK_STATE}'" >&2
|
|
|
|
return 1
|
|
|
|
esac
|
|
|
|
|
2019-07-24 21:01:42 +00:00
|
|
|
retry_default docker_curl "${DC}" -sL -XPUT "http://localhost:8500/v1/agent/check/warn/${CHECK_ID}"
|
2019-07-24 01:20:24 +00:00
|
|
|
}
|
|
|
|
|
2019-07-12 17:21:25 +00:00
|
|
|
function get_upstream_fortio_name {
|
2020-04-23 15:06:19 +00:00
|
|
|
local HOST=$1
|
|
|
|
local PORT=$2
|
|
|
|
local PREFIX=$3
|
2020-07-09 22:04:51 +00:00
|
|
|
local DEBUG_HEADER_VALUE="${4:-""}"
|
|
|
|
local extra_args
|
|
|
|
if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then
|
|
|
|
extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
|
|
|
|
fi
|
|
|
|
run retry_default curl -v -s -f -H"Host: ${HOST}" $extra_args \
|
|
|
|
"localhost:${PORT}${PREFIX}/debug?env=dump"
|
2019-07-12 17:21:25 +00:00
|
|
|
[ "$status" == 0 ]
|
|
|
|
echo "$output" | grep -E "^FORTIO_NAME="
|
|
|
|
}
|
|
|
|
|
|
|
|
function assert_expected_fortio_name {
|
|
|
|
local EXPECT_NAME=$1
|
2020-04-23 15:06:19 +00:00
|
|
|
local HOST=${2:-"localhost"}
|
|
|
|
local PORT=${3:-5000}
|
|
|
|
local URL_PREFIX=${4:-""}
|
2020-07-09 22:04:51 +00:00
|
|
|
local DEBUG_HEADER_VALUE="${5:-""}"
|
2019-07-12 17:21:25 +00:00
|
|
|
|
2020-07-09 22:04:51 +00:00
|
|
|
GOT=$(get_upstream_fortio_name ${HOST} ${PORT} "${URL_PREFIX}" "${DEBUG_HEADER_VALUE}")
|
2019-07-12 17:21:25 +00:00
|
|
|
|
2019-08-05 18:30:35 +00:00
|
|
|
if [ "$GOT" != "FORTIO_NAME=${EXPECT_NAME}" ]; then
|
|
|
|
echo "expected name: $EXPECT_NAME, actual name: $GOT" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
2019-07-12 17:21:25 +00:00
|
|
|
}
|
2020-07-09 22:04:51 +00:00
|
|
|
|
|
|
|
function assert_expected_fortio_name_pattern {
|
|
|
|
local EXPECT_NAME_PATTERN=$1
|
|
|
|
local HOST=${2:-"localhost"}
|
|
|
|
local PORT=${3:-5000}
|
|
|
|
local URL_PREFIX=${4:-""}
|
|
|
|
local DEBUG_HEADER_VALUE="${5:-""}"
|
|
|
|
|
|
|
|
GOT=$(get_upstream_fortio_name ${HOST} ${PORT} "${URL_PREFIX}" "${DEBUG_HEADER_VALUE}")
|
|
|
|
|
|
|
|
if [[ "$GOT" =~ $EXPECT_NAME_PATTERN ]]; then
|
|
|
|
:
|
|
|
|
else
|
|
|
|
echo "expected name pattern: $EXPECT_NAME_PATTERN, actual name: $GOT" 1>&2
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
}
|