#!/bin/bash set -eEuo pipefail # DEBUG=1 enables set -x for this script so echos every command run DEBUG=${DEBUG:-} # FILTER_TESTS="" skips any test whose CASENAME doesn't match the # pattern. CASENAME is combination of the name from the case- dir and the # envoy version for example: "http, envoy 1.8.0". The pattern is passed to grep # over that string. FILTER_TESTS=${FILTER_TESTS:-} # STOP_ON_FAIL exits after a case fails so the workdir state can be viewed and # the components interacted with to debug the failure. This is useful when tests # only fail when run as part of a whole suite but work in isolation. STOP_ON_FAIL=${STOP_ON_FAIL:-} # ENVOY_VERSIONS is the list of envoy versions to run each test against ENVOY_VERSIONS=${ENVOY_VERSIONS:-"1.10.0 1.9.1 1.8.0 1.11.2"} if [ ! -z "$DEBUG" ] ; then set -x fi DIR=$(cd -P -- "$(dirname -- "$0")" && pwd -P) cd $DIR FILTER_TESTS=${FILTER_TESTS:-} LEAVE_CONSUL_UP=${LEAVE_CONSUL_UP:-} PROXY_LOGS_ON_FAIL=${PROXY_LOGS_ON_FAIL:-} source helpers.bash RESULT=1 CLEANED_UP=0 function cleanup { local STATUS="$?" if [ "$CLEANED_UP" != 0 ] ; then return fi CLEANED_UP=1 if [ "$STATUS" -ne 0 ] then capture_logs fi docker-compose down -v --remove-orphans } trap cleanup EXIT function command_error { echo "ERR: command exited with status $1" 1>&2 echo " command: $2" 1>&2 echo " line: $3" 1>&2 echo " function: $4" 1>&2 echo " called at: $5" 1>&2 # printf '%s\n' "${FUNCNAME[@]}" # printf '%s\n' "${BASH_SOURCE[@]}" # printf '%s\n' "${BASH_LINENO[@]}" } trap 'command_error $? "${BASH_COMMAND}" "${LINENO}" "${FUNCNAME[0]:-main}" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}"' ERR # Cleanup from any previous unclean runs. docker-compose down -v --remove-orphans # Start the volume container docker-compose up -d workdir function init_workdir { local DC="$1" if test -z "$DC" then DC=primary fi # Note, we use explicit set of dirs so we don't delete .gitignore. Also, # don't wipe logs between runs as they are already split and we need them to # upload as artifacts later. rm -rf workdir/${DC} mkdir -p workdir/${DC}/{consul,envoy,bats,statsd,data} # Reload consul config from defaults cp consul-base-cfg/* workdir/${DC}/consul/ # Add any overrides if there are any (no op if not) find ${CASE_DIR} -maxdepth 1 -name '*.hcl' -type f -exec cp -f {} workdir/${DC}/consul \; # Copy all the test files find ${CASE_DIR} -maxdepth 1 -name '*.bats' -type f -exec cp -f {} workdir/${DC}/bats \; # Copy DC specific bats cp helpers.bash workdir/${DC}/bats # Add any DC overrides if test -d "${CASE_DIR}/${DC}" then find ${CASE_DIR}/${DC} -type f -name '*.hcl' -exec cp -f {} workdir/${DC}/consul \; find ${CASE_DIR}/${DC} -type f -name '*.bats' -exec cp -f {} workdir/${DC}/bats \; fi if test -d "${CASE_DIR}/data" then cp -r ${CASE_DIR}/data/* workdir/${DC}/data fi return 0 } function start_consul { local DC=${1:-primary} # Start consul now as setup script needs it up docker-compose rm -s -v -f consul-${DC} || true docker-compose up -d consul-${DC} } function pre_service_setup { local DC=${1:-primary} # Run test case setup (e.g. generating Envoy bootstrap, starting containers) if [ -f "${CASE_DIR}${DC}/setup.sh" ] then source ${CASE_DIR}${DC}/setup.sh else source ${CASE_DIR}setup.sh fi } function start_services { # Push the state to the shared docker volume (note this is because CircleCI # can't use shared volumes) docker cp workdir/. envoy_workdir_1:/workdir # Start containers required if [ ! -z "$REQUIRED_SERVICES" ] ; then docker-compose rm -s -v -f $REQUIRED_SERVICES || true docker-compose up --build -d $REQUIRED_SERVICES fi return 0 } function verify { local DC=$1 if test -z "$DC" then DC=primary fi # Execute tests res=0 echo "- - - - - - - - - - - - - - - - - - - - - - - -" echoblue -n "CASE $CASE_STR" echo -n ": " # Nuke any previous case's verify container. docker-compose rm -s -v -f verify-${DC} || true if docker-compose up --abort-on-container-exit --exit-code-from verify-${DC} verify-${DC} ; then echogreen "✓ PASS" else echored "⨯ FAIL" res=1 fi echo "================================================" return $res } function capture_logs { echo "Capturing Logs for $CASE_STR" mkdir -p "$LOG_DIR" services="$REQUIRED_SERVICES consul-primary" if is_set $REQUIRE_SECONDARY then services="$services consul-secondary" fi if [ -f "${CASE_DIR}capture.sh" ] then echo "Executing ${CASE_DIR}capture.sh" source ${CASE_DIR}capture.sh || true fi for cont in $services do echo "Capturing log for $cont" docker-compose logs --no-color "$cont" 2>&1 > "${LOG_DIR}/${cont}.log" done } function stop_services { # Teardown if [ -f "${CASE_DIR}teardown.sh" ] ; then source "${CASE_DIR}teardown.sh" fi docker-compose rm -s -v -f $REQUIRED_SERVICES || true } function initVars { source "defaults.sh" if [ -f "${CASE_DIR}vars.sh" ] ; then source "${CASE_DIR}vars.sh" fi } function runTest { initVars # Initialize the workdir init_workdir primary if is_set $REQUIRE_SECONDARY then init_workdir secondary fi # Wipe state docker-compose up wipe-volumes # Push the state to the shared docker volume (note this is because CircleCI # can't use shared volumes) docker cp workdir/. envoy_workdir_1:/workdir start_consul primary if [ $? -ne 0 ] then capture_logs return 1 fi if is_set $REQUIRE_SECONDARY then start_consul secondary if [ $? -ne 0 ] then capture_logs return 1 fi fi echo "Setting up the primary datacenter" pre_service_setup primary if [ $? -ne 0 ] then echo "Setting up the primary datacenter failed" capture_logs return 1 fi if is_set $REQUIRE_SECONDARY then echo "Setting up the secondary datacenter" pre_service_setup secondary if [ $? -ne 0 ] then echo "Setting up the secondary datacenter failed" capture_logs return 1 fi fi echo "Starting services" start_services if [ $? -ne 0 ] then capture_logs return 1 fi # Run the verify container and report on the output verify primary TESTRESULT=$? if is_set $REQUIRE_SECONDARY && test "$TESTRESULT" -eq 0 then verify secondary SECONDARYRESULT=$? if [ "$SECONDARYRESULT" -ne 0 ] then TESTRESULT=$SECONDARYRESULT fi fi if [ "$TESTRESULT" -ne 0 ] then capture_logs fi stop_services primary if is_set $REQUIRE_SECONDARY then stop_services secondary fi return $TESTRESULT } RESULT=0 for c in ./case-*/ ; do for ev in $ENVOY_VERSIONS ; do export CASE_DIR="${c}" export CASE_NAME=$( basename $c | cut -c6- ) export CASE_ENVOY_VERSION="envoy $ev" export CASE_STR="$CASE_NAME, $CASE_ENVOY_VERSION" export ENVOY_VERSION="${ev}" export LOG_DIR="workdir/logs/${CASE_DIR}/${ENVOY_VERSION}" echo "================================================" echoblue "CASE $CASE_STR" echo "- - - - - - - - - - - - - - - - - - - - - - - -" if [ ! -z "$FILTER_TESTS" ] && echo "$CASE_STR" | grep -v "$FILTER_TESTS" > /dev/null ; then echo " SKIPPED: doesn't match FILTER_TESTS=$FILTER_TESTS" continue 1 fi if ! runTest then RESULT=1 fi if [ $RESULT -ne 0 ] && [ ! -z "$STOP_ON_FAIL" ] ; then echo " => STOPPING because STOP_ON_FAIL set" break 2 fi done done cleanup if [ $RESULT -eq 0 ] ; then echogreen "✓ PASS" else echored "⨯ FAIL" exit 1 fi