From 5611d230b7b83dd0ffe77cf6650bb1cb94b2b819 Mon Sep 17 00:00:00 2001 From: "R.B. Boyer" Date: Thu, 22 Oct 2020 13:20:31 -0500 Subject: [PATCH] speed up envoy integration tests by removing docker-compose (#8982) This speeds up individual envoy integration test runs from ~23m to ~14m. It's also a pre-req for possibly switching to doing the tests entirely within Go (no shell-outs). --- .../connect/envoy/case-consul-exec/setup.sh | 5 - .../connect/envoy/docker-compose.yml | 680 ------------------ test/integration/connect/envoy/down.sh | 9 + test/integration/connect/envoy/run-tests.sh | 402 ++++++++++- 4 files changed, 376 insertions(+), 720 deletions(-) delete mode 100644 test/integration/connect/envoy/docker-compose.yml create mode 100755 test/integration/connect/envoy/down.sh diff --git a/test/integration/connect/envoy/case-consul-exec/setup.sh b/test/integration/connect/envoy/case-consul-exec/setup.sh index 341fca395..a2dd572e9 100644 --- a/test/integration/connect/envoy/case-consul-exec/setup.sh +++ b/test/integration/connect/envoy/case-consul-exec/setup.sh @@ -1,8 +1,3 @@ #!/bin/bash set -eEuo pipefail - -# Force rebuild of the exec container since this doesn't happen if only the -# version argument changed which means we end up testing the wrong version of -# Envoy. -docker-compose build s1-sidecar-proxy-consul-exec consul-primary diff --git a/test/integration/connect/envoy/docker-compose.yml b/test/integration/connect/envoy/docker-compose.yml deleted file mode 100644 index c214eea41..000000000 --- a/test/integration/connect/envoy/docker-compose.yml +++ /dev/null @@ -1,680 +0,0 @@ -version: '3.4' - -x-workdir: - &workdir-volume - type: volume - source: workdir - target: /workdir - volume: - nocopy: true - -volumes: - workdir: - -services: - # This is a dummy container that we use to create volume and keep it - # accessible while other containers are down. - workdir: - image: alpine - volumes: - - *workdir-volume - command: - - sleep - - "86400" - - consul-primary: - image: "consul-dev" - command: - - "agent" - - "-dev" - - "-datacenter" - - "primary" - - "-config-dir" - - "/workdir/primary/consul" - - "-client" - - "0.0.0.0" - volumes: - - *workdir-volume - ports: - # Exposing to host makes debugging locally a bit easier - - "8500:8500" - - "8502:8502" - # For zipkin which uses this containers network - - 9411:9411 - # Jaeger UI - - 16686:16686 - - s1: - depends_on: - - consul-primary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s1" - command: - - "server" - - "-http-port" - - ":8080" - - "-grpc-port" - - ":8079" - - "-redirect-port" - - "disabled" - network_mode: service:consul-primary - - s2: - depends_on: - - consul-primary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s2" - command: - - "server" - - "-http-port" - - ":8181" - - "-grpc-port" - - ":8179" - - "-redirect-port" - - "disabled" - network_mode: service:consul-primary - - s2-v1: - depends_on: - - consul-primary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s2-v1" - command: - - "server" - - "-http-port" - - ":8182" - - "-grpc-port" - - ":8178" - - "-redirect-port" - - "disabled" - network_mode: service:consul-primary - - s2-v2: - depends_on: - - consul-primary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s2-v2" - command: - - "server" - - "-http-port" - - ":8183" - - "-grpc-port" - - ":8177" - - "-redirect-port" - - "disabled" - network_mode: service:consul-primary - - s3: - depends_on: - - consul-primary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s3" - command: - - "server" - - "-http-port" - - ":8282" - - "-grpc-port" - - ":8279" - - "-redirect-port" - - "disabled" - network_mode: service:consul-primary - - s3-v1: - depends_on: - - consul-primary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s3-v1" - command: - - "server" - - "-http-port" - - ":8283" - - "-grpc-port" - - ":8278" - - "-redirect-port" - - "disabled" - network_mode: service:consul-primary - - s3-v2: - depends_on: - - consul-primary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s3-v2" - command: - - "server" - - "-http-port" - - ":8284" - - "-grpc-port" - - ":8277" - - "-redirect-port" - - "disabled" - network_mode: service:consul-primary - - s3-alt: - depends_on: - - consul-primary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s3-alt" - command: - - "server" - - "-http-port" - - ":8286" - - "-grpc-port" - - ":8280" - - "-redirect-port" - - "disabled" - network_mode: service:consul-primary - - s4: - depends_on: - - consul-primary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s4" - command: - - "server" - - "-http-port" - - ":8382" - - "-grpc-port" - - ":8281" - - "-redirect-port" - - "disabled" - network_mode: service:consul-primary - - s1-sidecar-proxy: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/s1-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - s2-sidecar-proxy: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/s2-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - s2-v1-sidecar-proxy: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/s2-v1-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - s2-v2-sidecar-proxy: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/s2-v2-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - s3-sidecar-proxy: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/s3-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - s3-v1-sidecar-proxy: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/s3-v1-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - s3-v2-sidecar-proxy: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/s3-v2-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - s3-alt-sidecar-proxy: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/s3-alt-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - verify-primary: - depends_on: - - consul-primary - build: - context: . - dockerfile: Dockerfile-bats - tty: true - environment: - - ENVOY_VERSION - command: - - "--pretty" - - "/workdir/primary/bats" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - s1-sidecar-proxy-consul-exec: - depends_on: - - consul-primary - build: - context: . - dockerfile: Dockerfile-consul-envoy - args: - ENVOY_VERSION: ${ENVOY_VERSION} - image: consul-dev-envoy:${ENVOY_VERSION} - command: - - "consul" - - "connect" - - "envoy" - - "-sidecar-for" - - "s1" - - "-envoy-version" - - ${ENVOY_VERSION} - - "--" - - "-l" - - "debug" - network_mode: service:consul-primary - - fake-statsd: - depends_on: - - consul-primary - image: "alpine/socat" - command: - - -u - - UDP-RECVFROM:8125,fork,reuseaddr - # This magic incantation is needed since Envoy doesn't add newlines and so - # we need each packet to be passed to echo to add a new line before - # appending. - - SYSTEM:'xargs -0 echo >> /workdir/primary/statsd/statsd.log' - volumes: - - *workdir-volume - network_mode: service:consul-primary - - wipe-volumes: - volumes: - - *workdir-volume - image: alpine - command: - - sh - - -c - - 'rm -rf /workdir/*' - - # This is a debugging tool run docker-compose up dump-volumes to see the - # current state. - dump-volumes: - volumes: - - *workdir-volume - - ./:/cwd - image: alpine - command: - - cp - - -r - - /workdir/. - - /cwd/workdir/ - - zipkin: - volumes: - - *workdir-volume - image: openzipkin/zipkin - network_mode: service:consul-primary - - jaeger: - volumes: - - *workdir-volume - image: jaegertracing/all-in-one:1.11 - network_mode: service:consul-primary - command: - - --collector.zipkin.http-port=9411 - - consul-secondary: - image: "consul-dev" - command: - - "agent" - - "-dev" - - "-datacenter" - - "secondary" - - "-config-dir" - - "/workdir/secondary/consul" - - "-client" - - "0.0.0.0" - volumes: - - *workdir-volume - ports: - # Exposing to host makes debugging locally a bit easier - - "9500:8500" - - "9502:8502" - - s1-secondary: - depends_on: - - consul-secondary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s1-secondary" - command: - - "server" - - "-http-port" - - ":8080" - - "-grpc-port" - - ":8079" - - "-redirect-port" - - "disabled" - network_mode: service:consul-secondary - - s2-secondary: - depends_on: - - consul-secondary - image: "fortio/fortio" - environment: - - "FORTIO_NAME=s2-secondary" - command: - - "server" - - "-http-port" - - ":8181" - - "-grpc-port" - - ":8179" - - "-redirect-port" - - "disabled" - network_mode: service:consul-secondary - - s1-sidecar-proxy-secondary: - depends_on: - - consul-secondary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/secondary/envoy/s1-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-secondary - - s2-sidecar-proxy-secondary: - depends_on: - - consul-secondary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/secondary/envoy/s2-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-secondary - - gateway-primary: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/mesh-gateway-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - gateway-secondary: - depends_on: - - consul-secondary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/secondary/envoy/mesh-gateway-bootstrap.json" - - "-l" - - "debug" - # Hot restart breaks since both envoys seem to interact with each other - # despite separate containers that don't share IPC namespace. Not quite - # sure how this happens but may be due to unix socket being in some shared - # location? - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-secondary - - ingress-gateway-primary: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/ingress-gateway-bootstrap.json" - - "-l" - - "debug" - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - terminating-gateway-primary: - depends_on: - - consul-primary - image: "envoyproxy/envoy:v${ENVOY_VERSION}" - command: - - "envoy" - - "-c" - - "/workdir/primary/envoy/terminating-gateway-bootstrap.json" - - "-l" - - "debug" - - "--disable-hot-restart" - - "--drain-time-s" - - "1" - volumes: - - *workdir-volume - network_mode: service:consul-primary - - verify-primary: - depends_on: - - consul-primary - build: - context: . - dockerfile: Dockerfile-bats - tty: true - environment: - - ENVOY_VERSION - command: - - "--pretty" - - "/workdir/primary/bats" - volumes: - - *workdir-volume - network_mode: service:consul-primary - pid: host - - verify-secondary: - depends_on: - - consul-secondary - build: - context: . - dockerfile: Dockerfile-bats - tty: true - environment: - - ENVOY_VERSION - command: - - "--pretty" - - "/workdir/secondary/bats" - volumes: - - *workdir-volume - network_mode: service:consul-secondary - pid: host - - tcpdump-primary: - depends_on: - - consul-primary - build: - context: . - dockerfile: Dockerfile-tcpdump - - # we cant do this in circle but its only here to temporarily enable. - volumes: - - type: bind - source: ./${LOG_DIR} - target: /data - command: -v -i any -w /data/primary.pcap - network_mode: service:consul-primary - - tcpdump-secondary: - depends_on: - - consul-secondary - build: - context: . - dockerfile: Dockerfile-tcpdump - - # we cant do this in circle but its only here to temporarily enable. - volumes: - - type: bind - source: ./${LOG_DIR} - target: /data - command: -v -i any -w /data/secondary.pcap - network_mode: service:consul-secondary - - diff --git a/test/integration/connect/envoy/down.sh b/test/integration/connect/envoy/down.sh new file mode 100755 index 000000000..c56700c2d --- /dev/null +++ b/test/integration/connect/envoy/down.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +cd "$(dirname "$0")" + +set -x +exec ./run-tests.sh suite_teardown + diff --git a/test/integration/connect/envoy/run-tests.sh b/test/integration/connect/envoy/run-tests.sh index 09bfb6e89..80c15565c 100755 --- a/test/integration/connect/envoy/run-tests.sh +++ b/test/integration/connect/envoy/run-tests.sh @@ -2,6 +2,8 @@ set -eEuo pipefail +readonly self_name="$0" + # DEBUG=1 enables set -x for this script so echos every command run DEBUG=${DEBUG:-} @@ -28,6 +30,13 @@ function command_error { trap 'command_error $? "${BASH_COMMAND}" "${LINENO}" "${FUNCNAME[0]:-main}" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}"' ERR +readonly WORKDIR_SNIPPET='-v envoy_workdir:/workdir' + +function network_snippet { + local DC="$1" + echo "--net container:envoy_consul-${DC}_1" +} + function init_workdir { local DC="$1" @@ -68,13 +77,65 @@ function init_workdir { return 0 } +function docker_kill_rm { + local name + local todo=() + for name in "$@"; do + name="envoy_${name}_1" + if docker container inspect $name &>/dev/null; then + if [[ "$name" == envoy_tcpdump-* ]]; then + echo -n "Gracefully stopping $name..." + docker stop $name &> /dev/null + echo "done" + fi + todo+=($name) + fi + done + + if [[ ${#todo[@]} -eq 0 ]]; then + return 0 + fi + + echo -n "Killing and removing: ${todo[@]}..." + docker rm -v -f ${todo[@]} &> /dev/null + echo "done" +} + function start_consul { local DC=${1:-primary} # Start consul now as setup script needs it up - docker-compose kill consul-${DC} || true - docker-compose rm -v -f consul-${DC} || true - docker-compose up -d consul-${DC} + docker_kill_rm consul-${DC} + + # 8500/8502 are for consul + # 9411 is for zipkin which shares the network with consul + # 16686 is for jaeger ui which also shares the network with consul + ports=( + '-p=8500:8500' + '-p=8502:8502' + '-p=9411:9411' + '-p=16686:16686' + ) + if [[ $DC == 'secondary' ]]; then + ports=( + '-p=9500:8500' + '-p=9502:8502' + ) + fi + + # Run consul and expose some ports to the host to make debugging locally a + # bit easier. + # + docker run -d --name envoy_consul-${DC}_1 \ + --net=envoy-tests \ + $WORKDIR_SNIPPET \ + --hostname "consul-${DC}" \ + --network-alias "consul-${DC}" \ + ${ports[@]} \ + consul-dev \ + agent -dev -datacenter "${DC}" \ + -config-dir "/workdir/${DC}/consul" \ + -client "0.0.0.0" >/dev/null } function pre_service_setup { @@ -93,12 +154,11 @@ function start_services { # Push the state to the shared docker volume (note this is because CircleCI # can't use shared volumes) docker cp workdir/. envoy_workdir_1:/workdir - + # Start containers required if [ ! -z "$REQUIRED_SERVICES" ] ; then - docker-compose kill $REQUIRED_SERVICES || true - docker-compose rm -v -f $REQUIRED_SERVICES || true - docker-compose up --build -d $REQUIRED_SERVICES + docker_kill_rm $REQUIRED_SERVICES + run_containers $REQUIRED_SERVICES fi return 0 @@ -114,10 +174,17 @@ function verify { res=0 # Nuke any previous case's verify container. - docker-compose kill verify-${DC} || true - docker-compose rm -v -f verify-${DC} || true + docker_kill_rm verify-${DC} - if docker-compose up --abort-on-container-exit --exit-code-from verify-${DC} verify-${DC} ; then + echo "Running ${DC} verification step for ${CASE_DIR}..." + + if docker run --name envoy_verify-${DC}_1 -t \ + -e ENVOY_VERSION \ + $WORKDIR_SNIPPET \ + --pid=host \ + $(network_snippet $DC) \ + bats-verify \ + --pretty /workdir/${DC}/bats ; then echogreen "✓ PASS" else echored "⨯ FAIL" @@ -128,8 +195,7 @@ function verify { } function capture_logs { - # exported to prevent docker-compose warning about unset var - export LOG_DIR="workdir/logs/${CASE_DIR}/${ENVOY_VERSION}" + local LOG_DIR="workdir/logs/${CASE_DIR}/${ENVOY_VERSION}" init_vars @@ -147,10 +213,11 @@ function capture_logs { source ${CASE_DIR}/capture.sh || true fi - for cont in $services - do + for cont in $services; do echo "Capturing log for $cont" - docker-compose logs --no-color "$cont" 2>&1 > "${LOG_DIR}/${cont}.log" + docker logs "envoy_${cont}_1" &> "${LOG_DIR}/${cont}.log" || { + echo "EXIT CODE $?" > "${LOG_DIR}/${cont}.log" + } done } @@ -159,8 +226,9 @@ function stop_services { if [ -f "${CASE_DIR}/teardown.sh" ] ; then source "${CASE_DIR}/teardown.sh" fi - docker-compose kill $REQUIRED_SERVICES || true - docker-compose rm -v -f $REQUIRED_SERVICES || true + docker_kill_rm $REQUIRED_SERVICES + + docker_kill_rm consul-primary consul-secondary } function init_vars { @@ -176,13 +244,19 @@ function global_setup { fi } +function wipe_volumes { + docker run --rm -i \ + $WORKDIR_SNIPPET \ + --net=none \ + alpine \ + sh -c 'rm -rf /workdir/*' +} + function run_tests { CASE_DIR="${CASE_DIR?CASE_DIR must be set to the path of the test case}" CASE_NAME=$( basename $CASE_DIR | cut -c6- ) export CASE_NAME - export LOG_DIR="workdir/logs/${CASE_DIR}/${ENVOY_VERSION}" - init_vars # Initialize the workdir @@ -196,7 +270,7 @@ function run_tests { global_setup # Wipe state - docker-compose up wipe-volumes + wipe_volumes # Push the state to the shared docker volume (note this is because CircleCI # can't use shared volumes) @@ -228,35 +302,293 @@ function run_tests { } function test_teardown { - # Set a log dir to prevent docker-compose warning about unset var - export LOG_DIR="workdir/logs/" - init_vars - stop_services primary - - if is_set $REQUIRE_SECONDARY; then - stop_services secondary - fi + stop_services } +function workdir_cleanup { + docker_kill_rm workdir + docker volume rm -f envoy_workdir &>/dev/null || true +} + + function suite_setup { - # Set a log dir to prevent docker-compose warning about unset var - export LOG_DIR="workdir/logs/" # Cleanup from any previous unclean runs. - docker-compose down --volumes --timeout 0 --remove-orphans + suite_teardown + + docker network create envoy-tests &>/dev/null # Start the volume container - docker-compose up -d workdir + # + # This is a dummy container that we use to create volume and keep it + # accessible while other containers are down. + docker volume create envoy_workdir &>/dev/null + docker run -d --name envoy_workdir_1 \ + $WORKDIR_SNIPPET \ + --net=none \ + google/pause &>/dev/null + + # pre-build the verify container + echo "Rebuilding 'bats-verify' image..." + docker build -t bats-verify -f Dockerfile-bats . + + # pre-build the consul+envoy container + echo "Rebuilding 'consul-dev-envoy:${ENVOY_VERSION}' image..." + docker build -t consul-dev-envoy:${ENVOY_VERSION} \ + --build-arg ENVOY_VERSION=${ENVOY_VERSION} \ + -f Dockerfile-consul-envoy . } function suite_teardown { - # Set a log dir to prevent docker-compose warning about unset var - export LOG_DIR="workdir/logs/" + docker_kill_rm verify-primary verify-secondary - docker-compose down --volumes --timeout 0 --remove-orphans + # this is some hilarious magic + docker_kill_rm $(grep "^function run_container_" $self_name | \ + sed 's/^function run_container_\(.*\) {/\1/g') + + docker_kill_rm consul-primary consul-secondary + + if docker network inspect envoy-tests &>/dev/null ; then + echo -n "Deleting network 'envoy-tests'..." + docker network rm envoy-tests + echo "done" + fi + + workdir_cleanup } +function run_containers { + for name in $@ ; do + run_container $name + done +} + +function run_container { + docker_kill_rm "$1" + "run_container_$1" +} + +function common_run_container_service { + local service="$1" + local DC="$2" + local httpPort="$3" + local grpcPort="$4" + + docker run -d --name $(container_name_prev) \ + -e "FORTIO_NAME=${service}" \ + $(network_snippet $DC) \ + fortio/fortio \ + server \ + -http-port ":$httpPort" \ + -grpc-port ":$grpcPort" \ + -redirect-port disabled >/dev/null +} + +function run_container_s1 { + common_run_container_service s1 primary 8080 8079 +} + +function run_container_s2 { + common_run_container_service s2 primary 8181 8179 +} +function run_container_s2-v1 { + common_run_container_service s2-v1 primary 8182 8178 +} +function run_container_s2-v2 { + common_run_container_service s2-v2 primary 8183 8177 +} + +function run_container_s3 { + common_run_container_service s3 primary 8282 8279 +} +function run_container_s3-v1 { + common_run_container_service s3-v1 primary 8283 8278 +} +function run_container_s3-v2 { + common_run_container_service s3-v2 primary 8284 8277 +} +function run_container_s3-alt { + common_run_container_service s3-alt primary 8286 8280 +} + +function run_container_s4 { + common_run_container_service s4 primary 8382 8281 +} + +function run_container_s1-secondary { + common_run_container_service s1-secondary secondary 8080 8079 +} + +function run_container_s2-secondary { + common_run_container_service s2-secondary secondary 8181 8179 +} + +function common_run_container_sidecar_proxy { + local service="$1" + local DC="$2" + + # Hot restart breaks since both envoys seem to interact with each other + # despite separate containers that don't share IPC namespace. Not quite + # sure how this happens but may be due to unix socket being in some shared + # location? + docker run -d --name $(container_name_prev) \ + $WORKDIR_SNIPPET \ + $(network_snippet $DC) \ + "envoyproxy/envoy:v${ENVOY_VERSION}" \ + envoy \ + -c /workdir/${DC}/envoy/${service}-bootstrap.json \ + -l debug \ + --disable-hot-restart \ + --drain-time-s 1 >/dev/null +} + +function run_container_s1-sidecar-proxy { + common_run_container_sidecar_proxy s1 primary +} +function run_container_s1-sidecar-proxy-consul-exec { + docker run -d --name $(container_name) \ + $(network_snippet primary) \ + consul-dev-envoy:${ENVOY_VERSION} \ + consul connect envoy -sidecar-for s1 \ + -envoy-version ${ENVOY_VERSION} \ + -- \ + -l debug >/dev/null +} + +function run_container_s2-sidecar-proxy { + common_run_container_sidecar_proxy s2 primary +} +function run_container_s2-v1-sidecar-proxy { + common_run_container_sidecar_proxy s2-v1 primary +} +function run_container_s2-v2-sidecar-proxy { + common_run_container_sidecar_proxy s2-v2 primary +} + +function run_container_s3-sidecar-proxy { + common_run_container_sidecar_proxy s3 primary +} +function run_container_s3-v1-sidecar-proxy { + common_run_container_sidecar_proxy s3-v1 primary +} +function run_container_s3-v2-sidecar-proxy { + common_run_container_sidecar_proxy s3-v2 primary +} + +function run_container_s3-alt-sidecar-proxy { + common_run_container_sidecar_proxy s3-alt primary +} + +function run_container_s1-sidecar-proxy-secondary { + common_run_container_sidecar_proxy s1 secondary +} +function run_container_s2-sidecar-proxy-secondary { + common_run_container_sidecar_proxy s2 secondary +} + +function common_run_container_gateway { + local name="$1" + local DC="$2" + + # Hot restart breaks since both envoys seem to interact with each other + # despite separate containers that don't share IPC namespace. Not quite + # sure how this happens but may be due to unix socket being in some shared + # location? + docker run -d --name $(container_name_prev) \ + $WORKDIR_SNIPPET \ + $(network_snippet $DC) \ + "envoyproxy/envoy:v${ENVOY_VERSION}" \ + envoy \ + -c /workdir/${DC}/envoy/${name}-bootstrap.json \ + -l debug \ + --disable-hot-restart \ + --drain-time-s 1 >/dev/null +} + +function run_container_gateway-primary { + common_run_container_gateway mesh-gateway primary +} +function run_container_gateway-secondary { + common_run_container_gateway mesh-gateway secondary +} + +function run_container_ingress-gateway-primary { + common_run_container_gateway ingress-gateway primary +} + +function run_container_terminating-gateway-primary { + common_run_container_gateway terminating-gateway primary +} + +function run_container_fake-statsd { + # This magic SYSTEM incantation is needed since Envoy doesn't add newlines and so + # we need each packet to be passed to echo to add a new line before + # appending. + docker run -d --name $(container_name) \ + $WORKDIR_SNIPPET \ + $(network_snippet primary) \ + alpine/socat \ + -u UDP-RECVFROM:8125,fork,reuseaddr \ + SYSTEM:'xargs -0 echo >> /workdir/primary/statsd/statsd.log' +} + +function run_container_zipkin { + docker run -d --name $(container_name) \ + $WORKDIR_SNIPPET \ + $(network_snippet primary) \ + openzipkin/zipkin +} + +function run_container_jaeger { + docker run -d --name $(container_name) \ + $WORKDIR_SNIPPET \ + $(network_snippet primary) \ + jaegertracing/all-in-one:1.11 \ + --collector.zipkin.http-port=9411 +} + +function container_name { + echo "envoy_${FUNCNAME[1]/#run_container_/}_1" +} +function container_name_prev { + echo "envoy_${FUNCNAME[2]/#run_container_/}_1" +} + +# This is a debugging tool. Run via './run-tests.sh debug_dump_volumes' +function debug_dump_volumes { + docker run --rm -it \ + $WORKDIR_SNIPPET \ + -v ./:/cwd \ + --net=none \ + alpine \ + cp -r /workdir/. /cwd/workdir/ +} + +function run_container_tcpdump-primary { + # To use add "tcpdump-primary" to REQUIRED_SERVICES + common_run_container_tcpdump primary +} +function run_container_tcpdump-secondary { + # To use add "tcpdump-secondary" to REQUIRED_SERVICES + common_run_container_tcpdump secondary +} + +function common_run_container_tcpdump { + local DC="$1" + + # we cant run this in circle but its only here to temporarily enable. + + docker build -t envoy-tcpdump -f Dockerfile-tcpdump . + + docker run -d --name $(container_name_prev) \ + $(network_snippet $DC) \ + -v $(pwd)/workdir/${DC}/envoy/:/data \ + --privileged \ + envoy-tcpdump \ + -v -i any \ + -w "/data/${DC}.pcap" +} case "${1-}" in "")