Add cross-partition integration test

This commit is contained in:
freddygv 2021-11-12 14:45:50 -07:00
parent 29cfc23a27
commit 42d9542ac3
3 changed files with 106 additions and 37 deletions

View File

@ -2,4 +2,5 @@
export DEFAULT_REQUIRED_SERVICES="s1 s1-sidecar-proxy s2 s2-sidecar-proxy"
export REQUIRED_SERVICES="${DEFAULT_REQUIRED_SERVICES}"
export REQUIRE_SECONDARY=0
export REQUIRE_SECONDARY=0
export REQUIRE_PARTITIONS=0

View File

@ -115,14 +115,19 @@ function assert_proxy_presents_cert_uri {
local SERVICENAME=$2
local DC=${3:-primary}
local NS=${4:-default}
local PARTITION=${5:default}
CERT=$(retry_default get_cert $HOSTPORT)
echo "WANT SERVICE: ${NS}/${SERVICENAME}"
echo "WANT SERVICE: ${PARTITION}/${NS}/${SERVICENAME}"
echo "GOT CERT:"
echo "$CERT"
echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
if [[ -z $PARTITION ]] || [[ $PARTITION = "default" ]]; then
echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
else
echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ap/${PARTITION}/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
fi
}
function assert_dnssan_in_cert {

View File

@ -40,39 +40,39 @@ function network_snippet {
}
function init_workdir {
local DC="$1"
local CLUSTER="$1"
if test -z "$DC"
if test -z "$CLUSTER"
then
DC=primary
CLUSTER=primary
fi
# Note, we use explicit set of dirs so we don't delete .gitignore. Also,
# don't wipe logs between runs as they are already split and we need them to
# upload as artifacts later.
rm -rf workdir/${DC}
mkdir -p workdir/${DC}/{consul,register,envoy,bats,statsd,data}
rm -rf workdir/${CLUSTER}
mkdir -p workdir/${CLUSTER}/{consul,register,envoy,bats,statsd,data}
# Reload consul config from defaults
cp consul-base-cfg/*.hcl workdir/${DC}/consul/
cp consul-base-cfg/*.hcl workdir/${CLUSTER}/consul/
# Add any overrides if there are any (no op if not)
find ${CASE_DIR} -maxdepth 1 -name '*.hcl' -type f -exec cp -f {} workdir/${DC}/consul \;
find ${CASE_DIR} -maxdepth 1 -name '*.hcl' -type f -exec cp -f {} workdir/${CLUSTER}/consul \;
# Copy all the test files
find ${CASE_DIR} -maxdepth 1 -name '*.bats' -type f -exec cp -f {} workdir/${DC}/bats \;
# Copy DC specific bats
cp helpers.bash workdir/${DC}/bats
find ${CASE_DIR} -maxdepth 1 -name '*.bats' -type f -exec cp -f {} workdir/${CLUSTER}/bats \;
# Copy CLUSTER specific bats
cp helpers.bash workdir/${CLUSTER}/bats
# Add any DC overrides
if test -d "${CASE_DIR}/${DC}"
# Add any CLUSTER overrides
if test -d "${CASE_DIR}/${CLUSTER}"
then
find ${CASE_DIR}/${DC} -type f -name '*.hcl' -exec cp -f {} workdir/${DC}/consul \;
find ${CASE_DIR}/${DC} -type f -name '*.bats' -exec cp -f {} workdir/${DC}/bats \;
find ${CASE_DIR}/${CLUSTER} -type f -name '*.hcl' -exec cp -f {} workdir/${CLUSTER}/consul \;
find ${CASE_DIR}/${CLUSTER} -type f -name '*.bats' -exec cp -f {} workdir/${CLUSTER}/bats \;
fi
# move all of the registration files OUT of the consul config dir now
find workdir/${DC}/consul -type f -name 'service_*.hcl' -exec mv -f {} workdir/${DC}/register \;
find workdir/${CLUSTER}/consul -type f -name 'service_*.hcl' -exec mv -f {} workdir/${CLUSTER}/register \;
# copy the ca-certs for SDS so we can verify the right ones are served
mkdir -p workdir/test-sds-server/certs
@ -80,7 +80,7 @@ function init_workdir {
if test -d "${CASE_DIR}/data"
then
cp -r ${CASE_DIR}/data/* workdir/${DC}/data
cp -r ${CASE_DIR}/data/* workdir/${CLUSTER}/data
fi
return 0
@ -157,13 +157,48 @@ function start_consul {
-client "0.0.0.0" >/dev/null
}
function start_partitioned_client {
local PARTITION=${1:-ap1}
# Start consul now as setup script needs it up
docker_kill_rm consul-${PARTITION}
license="${CONSUL_LICENSE:-}"
# load the consul license so we can pass it into the consul
# containers as an env var in the case that this is a consul
# enterprise test
if test -z "$license" -a -n "${CONSUL_LICENSE_PATH:-}"
then
license=$(cat $CONSUL_LICENSE_PATH)
fi
sh -c "rm -rf /workdir/${PARTITION}/data"
# Run consul and expose some ports to the host to make debugging locally a
# bit easier.
#
docker run -d --name envoy_consul-${PARTITION}_1 \
--net=envoy-tests \
$WORKDIR_SNIPPET \
--hostname "consul-${PARTITION}" \
--network-alias "consul-${PARTITION}" \
-e "CONSUL_LICENSE=$license" \
consul-dev agent \
-datacenter "primary" \
-retry-join "consul-primary" \
-grpc-port 8502 \
-data-dir "/tmp/consul" \
-config-dir "/workdir/${PARTITION}/consul" \
-client "0.0.0.0" >/dev/null
}
function pre_service_setup {
local DC=${1:-primary}
local CLUSTER=${1:-primary}
# Run test case setup (e.g. generating Envoy bootstrap, starting containers)
if [ -f "${CASE_DIR}/${DC}/setup.sh" ]
if [ -f "${CASE_DIR}/${CLUSTER}/setup.sh" ]
then
source ${CASE_DIR}/${DC}/setup.sh
source ${CASE_DIR}/${CLUSTER}/setup.sh
else
source ${CASE_DIR}/setup.sh
fi
@ -184,29 +219,29 @@ function start_services {
}
function verify {
local DC=$1
if test -z "$DC"; then
DC=primary
local CLUSTER="$1"
if test -z "$CLUSTER"; then
CLUSTER="primary"
fi
# Execute tests
res=0
# Nuke any previous case's verify container.
docker_kill_rm verify-${DC}
docker_kill_rm verify-${CLUSTER}
echo "Running ${DC} verification step for ${CASE_DIR}..."
echo "Running ${CLUSTER} verification step for ${CASE_DIR}..."
# need to tell the PID 1 inside of the container that it won't be actual PID
# 1 because we're using --pid=host so we use TINI_SUBREAPER
if docker run --name envoy_verify-${DC}_1 -t \
if docker run --name envoy_verify-${CLUSTER}_1 -t \
-e TINI_SUBREAPER=1 \
-e ENVOY_VERSION \
$WORKDIR_SNIPPET \
--pid=host \
$(network_snippet $DC) \
$(network_snippet $CLUSTER) \
bats-verify \
--pretty /workdir/${DC}/bats ; then
--pretty /workdir/${CLUSTER}/bats ; then
echogreen "✓ PASS"
else
echored " FAIL"
@ -228,6 +263,11 @@ function capture_logs {
then
services="$services consul-secondary"
fi
if is_set $REQUIRE_PARTITIONS
then
services="$services consul-ap1"
fi
if [ -f "${CASE_DIR}/capture.sh" ]
then
@ -247,7 +287,7 @@ function stop_services {
# Teardown
docker_kill_rm $REQUIRED_SERVICES
docker_kill_rm consul-primary consul-secondary
docker_kill_rm consul-primary consul-secondary consul-ap1
}
function init_vars {
@ -286,6 +326,10 @@ function run_tests {
then
init_workdir secondary
fi
if is_set $REQUIRE_PARTITIONS
then
init_workdir ap1
fi
global_setup
@ -307,6 +351,10 @@ function run_tests {
if is_set $REQUIRE_SECONDARY; then
start_consul secondary
fi
if is_set $REQUIRE_PARTITIONS; then
docker_consul "primary" admin-partition create -name ap1 > /dev/null
start_partitioned_client ap1
fi
echo "Setting up the primary datacenter"
pre_service_setup primary
@ -315,14 +363,20 @@ function run_tests {
echo "Setting up the secondary datacenter"
pre_service_setup secondary
fi
if is_set $REQUIRE_PARTITIONS; then
echo "Setting up the non-default partition"
pre_service_setup ap1
fi
echo "Starting services"
start_services
# Run the verify container and report on the output
echo "Verifying the primary datacenter"
verify primary
if is_set $REQUIRE_SECONDARY; then
echo "Verifying the secondary datacenter"
verify secondary
fi
}
@ -378,7 +432,7 @@ function suite_teardown {
docker_kill_rm $(grep "^function run_container_" $self_name | \
sed 's/^function run_container_\(.*\) {/\1/g')
docker_kill_rm consul-primary consul-secondary
docker_kill_rm consul-primary consul-secondary consul-ap1
if docker network inspect envoy-tests &>/dev/null ; then
echo -n "Deleting network 'envoy-tests'..."
@ -402,13 +456,13 @@ function run_container {
function common_run_container_service {
local service="$1"
local DC="$2"
local CLUSTER="$2"
local httpPort="$3"
local grpcPort="$4"
docker run -d --name $(container_name_prev) \
-e "FORTIO_NAME=${service}" \
$(network_snippet $DC) \
$(network_snippet $CLUSTER) \
"${HASHICORP_DOCKER_PROXY}/fortio/fortio" \
server \
-http-port ":$httpPort" \
@ -420,6 +474,10 @@ function run_container_s1 {
common_run_container_service s1 primary 8080 8079
}
function run_container_s1-ap1 {
common_run_container_service s1 ap1 8080 8079
}
function run_container_s2 {
common_run_container_service s2 primary 8181 8179
}
@ -457,7 +515,7 @@ function run_container_s2-secondary {
function common_run_container_sidecar_proxy {
local service="$1"
local DC="$2"
local CLUSTER="$2"
# Hot restart breaks since both envoys seem to interact with each other
# despite separate containers that don't share IPC namespace. Not quite
@ -465,10 +523,10 @@ function common_run_container_sidecar_proxy {
# location?
docker run -d --name $(container_name_prev) \
$WORKDIR_SNIPPET \
$(network_snippet $DC) \
$(network_snippet $CLUSTER) \
"${HASHICORP_DOCKER_PROXY}/envoyproxy/envoy:v${ENVOY_VERSION}" \
envoy \
-c /workdir/${DC}/envoy/${service}-bootstrap.json \
-c /workdir/${CLUSTER}/envoy/${service}-bootstrap.json \
-l debug \
--disable-hot-restart \
--drain-time-s 1 >/dev/null
@ -477,6 +535,11 @@ function common_run_container_sidecar_proxy {
function run_container_s1-sidecar-proxy {
common_run_container_sidecar_proxy s1 primary
}
function run_container_s1-ap1-sidecar-proxy {
common_run_container_sidecar_proxy s1 ap1
}
function run_container_s1-sidecar-proxy-consul-exec {
docker run -d --name $(container_name) \
$(network_snippet primary) \