test: run Envoy integration tests against both servers and clients (#13610)

.circleci/config.yml

@@ -23,6 +23,10 @@ references:
     BASH_ENV: .circleci/bash_env.sh
     VAULT_BINARY_VERSION: 1.9.4
     GO_VERSION: 1.18.1
+  envoy-versions: &supported_envoy_versions
+    - &default_envoy_version "1.19.5"
+    - "1.20.4"
+    - "1.21.3"
   images:
     # When updating the Go version, remember to also update the versions in the
     # workflows section for go-test-lib jobs.
@@ -852,13 +856,23 @@ jobs:
           path: *TEST_RESULTS_DIR
       - run: *notify-slack-failure
 
-  envoy-integration-test-1_19_5: &ENVOY_TESTS
+  envoy-integration-test: &ENVOY_TESTS
     machine:
       image: *UBUNTU_CI_IMAGE
     parallelism: 4
     resource_class: medium
+    parameters:
+      envoy-version:
+        type: enum
+        enum: *supported_envoy_versions
+        default: *default_envoy_version
+      xds-target:
+        type: enum
+        enum: ["server", "client"]
+        default: "server"
     environment:
-      ENVOY_VERSION: "1.19.5"
+      ENVOY_VERSION: << parameters.envoy-version >>
+      XDS_TARGET: << parameters.xds-target >>
     steps: &ENVOY_INTEGRATION_TEST_STEPS
       - checkout
       # Get go binary from workspace
@@ -891,21 +905,6 @@ jobs:
           path: *TEST_RESULTS_DIR
       - run: *notify-slack-failure
 
-  envoy-integration-test-1_20_4:
-    <<: *ENVOY_TESTS
-    environment:
-      ENVOY_VERSION: "1.20.4"
-
-  envoy-integration-test-1_21_3:
-    <<: *ENVOY_TESTS
-    environment:
-      ENVOY_VERSION: "1.21.3"
-
-  envoy-integration-test-1_22_2:
-    <<: *ENVOY_TESTS
-    environment:
-      ENVOY_VERSION: "1.22.2"
-
   # run integration tests for the connect ca providers
   test-connect-ca-providers:
     docker:
@@ -1116,18 +1115,13 @@ workflows:
       - nomad-integration-0_8:
           requires:
             - dev-build
-      - envoy-integration-test-1_19_5:
+      - envoy-integration-test:
-          requires:
-            - dev-build
-      - envoy-integration-test-1_20_4:
-          requires:
-            - dev-build
-      - envoy-integration-test-1_21_3:
-          requires:
-            - dev-build
-      - envoy-integration-test-1_22_2:
           requires:
             - dev-build
+          matrix:
+            parameters:
+              envoy-version: *supported_envoy_versions
+              xds-target: ["server", "client"]
       - compatibility-integration-test:
           requires:
             - dev-build

test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-none/primary/verify.bats

@@ -31,7 +31,7 @@ load helpers
 }
 
 @test "gateway-secondary should be up and listening" {
-  retry_long nc -z consul-secondary:4432
+  retry_long nc -z consul-secondary-client:4432
 }
 
 ################

test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-none/secondary/join.hcl

@@ -1 +1 @@
-retry_join_wan = ["consul-primary"]
+retry_join_wan = ["consul-primary-server"]

test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/primary/verify.bats

@@ -31,7 +31,7 @@ load helpers
 }
 
 @test "gateway-secondary should be up and listening" {
-  retry_long nc -z consul-secondary:4432
+  retry_long nc -z consul-secondary-client:4432
 }
 
 @test "wait until the first cluster is configured" {

test/integration/connect/envoy/case-cfg-resolver-dc-failover-gateways-remote/secondary/join.hcl

@@ -1 +1 @@
-retry_join_wan = ["consul-primary"]
+retry_join_wan = ["consul-primary-server"]

test/integration/connect/envoy/case-cross-peers/alpha/verify.bats

@@ -23,7 +23,7 @@ load helpers
 }
 
 @test "gateway-alpha should be up and listening" {
-  retry_long nc -z consul-alpha:4432
+  retry_long nc -z consul-alpha-client:4432
 }
 
 @test "s2 proxies should be healthy" {

test/integration/connect/envoy/case-cross-peers/primary/verify.bats

@@ -23,11 +23,11 @@ load helpers
 }
 
 @test "gateway-primary should be up and listening" {
-  retry_long nc -z consul-primary:4431
+  retry_long nc -z consul-primary-client:4431
 }
 
 @test "gateway-alpha should be up and listening" {
-  retry_long nc -z consul-alpha:4432
+  retry_long nc -z consul-alpha-client:4432
 }
 
 @test "peer the two clusters together" {
@@ -39,7 +39,7 @@ load helpers
 }
 
 @test "gateway-alpha should have healthy endpoints for s2" {
-  assert_upstream_has_endpoints_in_status consul-alpha:19003 s2.default.alpha HEALTHY 1
+  assert_upstream_has_endpoints_in_status consul-alpha-client:19003 s2.default.alpha HEALTHY 1
 }
 
 @test "s1 upstream should have healthy endpoints for s2" {

test/integration/connect/envoy/case-gateways-local/primary/verify.bats

@@ -28,7 +28,7 @@ load helpers
 }
 
 @test "gateway-secondary should have healthy endpoints for s2" {
-   assert_upstream_has_endpoints_in_status consul-secondary:19003 s2 HEALTHY 1
+   assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1
 }
 
 @test "s1 upstream should be able to connect to s2" {

test/integration/connect/envoy/case-gateways-local/secondary/join.hcl

@@ -1 +1 @@
-retry_join_wan = ["consul-primary"]
+retry_join_wan = ["consul-primary-server"]

test/integration/connect/envoy/case-gateways-remote/secondary/join.hcl

@@ -1 +1 @@
-retry_join_wan = ["consul-primary"]
+retry_join_wan = ["consul-primary-server"]

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats

@@ -23,11 +23,11 @@ load helpers
 }
 
 @test "gateway-secondary should have healthy endpoints for s1" {
-   assert_upstream_has_endpoints_in_status consul-secondary:19003 s1 HEALTHY 1
+   assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s1 HEALTHY 1
 }
 
 @test "gateway-secondary should have healthy endpoints for s2" {
-   assert_upstream_has_endpoints_in_status consul-secondary:19003 s2 HEALTHY 1
+   assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1
 }
 
 @test "ingress should be able to connect to s1" {

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/join.hcl

@@ -1 +1 @@
-retry_join_wan = ["consul-primary"]
+retry_join_wan = ["consul-primary-server"]

test/integration/connect/envoy/case-multidc-rsa-ca/secondary/join.hcl

@@ -1 +1 @@
-retry_join_wan = ["consul-primary"]
+retry_join_wan = ["consul-primary-server"]

test/integration/connect/envoy/case-wanfed-gw/primary/common.hcl

@@ -0,0 +1,6 @@
+ca_file                = "/workdir/primary/tls/consul-agent-ca.pem"
+cert_file              = "/workdir/primary/tls/primary-server-consul-0.pem"
+key_file               = "/workdir/primary/tls/primary-server-consul-0-key.pem"
+verify_incoming        = true
+verify_outgoing        = true
+verify_server_hostname = true

test/integration/connect/envoy/case-wanfed-gw/primary/verify.bats

@@ -19,7 +19,7 @@ load helpers
 }
 
 @test "gateway-secondary should be up and listening" {
-  retry_long nc -z consul-secondary:4432
+  retry_long nc -z consul-secondary-client:4432
 }
 
 @test "primary should be able to rpc to the secondary" {
@@ -27,5 +27,5 @@ load helpers
 }
 
 @test "wan pool should show 2 healthy nodes" {
-  assert_alive_wan_member_count 2
+  assert_alive_wan_member_count primary 2
 }

test/integration/connect/envoy/case-wanfed-gw/secondary/common.hcl

@@ -0,0 +1,6 @@
+ca_file                   = "/workdir/secondary/tls/consul-agent-ca.pem"
+cert_file                 = "/workdir/secondary/tls/secondary-server-consul-0.pem"
+key_file                  = "/workdir/secondary/tls/secondary-server-consul-0-key.pem"
+verify_incoming           = true
+verify_outgoing           = true
+verify_server_hostname    = true

test/integration/connect/envoy/case-wanfed-gw/secondary/server.hcl

@@ -4,13 +4,7 @@ connect {
   enable_mesh_gateway_wan_federation = true
 }
 primary_gateways = [
-  "consul-primary:4431",
+  "consul-primary-client:4431",
 ]
 primary_gateways_interval = "5s"
 retry_interval_wan        = "5s"
-ca_file                   = "/workdir/secondary/tls/consul-agent-ca.pem"
-cert_file                 = "/workdir/secondary/tls/secondary-server-consul-0.pem"
-key_file                  = "/workdir/secondary/tls/secondary-server-consul-0-key.pem"
-verify_incoming           = true
-verify_outgoing           = true
-verify_server_hostname    = true

test/integration/connect/envoy/case-wanfed-gw/secondary/verify.bats

@@ -19,7 +19,7 @@ load helpers
 }
 
 @test "gateway-primary should be up and listening" {
-  retry_long nc -z consul-primary:4431
+  retry_long nc -z consul-primary-client:4431
 }
 
 @test "secondary should be able to rpc to the primary" {
@@ -27,5 +27,5 @@ load helpers
 }
 
 @test "wan pool should show 2 healthy nodes" {
-  assert_alive_wan_member_count 2
+  assert_alive_wan_member_count secondary 2
 }

test/integration/connect/envoy/helpers.bash

@@ -479,28 +479,31 @@ function get_healthy_service_count {
   local AP=$4
   local PEER_NAME=$5
 
-  run curl -s -f ${HEADERS} "consul-${DC}:8500/v1/health/connect/${SERVICE_NAME}?passing&ns=${NS}&partition=${AP}&peer=${PEER_NAME}"
+  run curl -s -f ${HEADERS} "consul-${DC}-client:8500/v1/health/connect/${SERVICE_NAME}?passing&ns=${NS}&partition=${AP}&peer=${PEER_NAME}"
 
   [ "$status" -eq 0 ]
   echo "$output" | jq --raw-output '. | length'
 }
 
 function assert_alive_wan_member_count {
-  local EXPECT_COUNT=$1
+  local DC=$1
-  run retry_long assert_alive_wan_member_count_once $EXPECT_COUNT
+  local EXPECT_COUNT=$2
+  run retry_long assert_alive_wan_member_count_once $DC $EXPECT_COUNT
   [ "$status" -eq 0 ]
 }
 
 function assert_alive_wan_member_count_once {
-  local EXPECT_COUNT=$1
+  local DC=$1
+  local EXPECT_COUNT=$2
 
-  GOT_COUNT=$(get_alive_wan_member_count)
+  GOT_COUNT=$(get_alive_wan_member_count $DC)
 
   [ "$GOT_COUNT" -eq "$EXPECT_COUNT" ]
 }
 
 function get_alive_wan_member_count {
-  run retry_default curl -sL -f "127.0.0.1:8500/v1/agent/members?wan=1"
+  local DC=$1
+  run retry_default curl -sL -f "consul-${DC}-server:8500/v1/agent/members?wan=1"
   [ "$status" -eq 0 ]
   # echo "$output" >&3
   echo "$output" | jq '.[] | select(.Status == 1) | .Name' | wc -l
@@ -951,7 +954,7 @@ function assert_expected_fortio_host_header {
 function create_peering {
   local GENERATE_PEER=$1
   local ESTABLISH_PEER=$2
-  run curl -sL -XPOST "http://consul-${GENERATE_PEER}:8500/v1/peering/token" -d"{ \"PeerName\" : \"${GENERATE_PEER}-to-${ESTABLISH_PEER}\" }"
+  run curl -sL -XPOST "http://consul-${GENERATE_PEER}-client:8500/v1/peering/token" -d"{ \"PeerName\" : \"${GENERATE_PEER}-to-${ESTABLISH_PEER}\" }"
   # echo "$output" >&3
   [ "$status" == 0 ]
 
@@ -959,8 +962,7 @@ function create_peering {
   token="$(echo "$output" | jq -r .PeeringToken)"
   [ -n "$token" ]
 
-  run curl -sLv -XPOST "http://consul-${ESTABLISH_PEER}:8500/v1/peering/establish" -d"{ \"PeerName\" : \"${ESTABLISH_PEER}-to-${GENERATE_PEER}\", \"PeeringToken\" : \"${token}\" }"
+  run curl -sLv -XPOST "http://consul-${ESTABLISH_PEER}-client:8500/v1/peering/establish" -d"{ \"PeerName\" : \"${ESTABLISH_PEER}-to-${GENERATE_PEER}\", \"PeeringToken\" : \"${token}\" }"
   # echo "$output" >&3
   [ "$status" == 0 ]
 }
-

test/integration/connect/envoy/run-tests.sh

@@ -9,6 +9,8 @@ readonly HASHICORP_DOCKER_PROXY="docker.mirror.hashicorp.services"
 # DEBUG=1 enables set -x for this script so echos every command run
 DEBUG=${DEBUG:-}
 
+XDS_TARGET=${XDS_TARGET:-server}
+
 # ENVOY_VERSION to run each test against
 ENVOY_VERSION=${ENVOY_VERSION:-"1.22.2"}
 export ENVOY_VERSION
@@ -53,7 +55,7 @@ function init_workdir {
   # don't wipe logs between runs as they are already split and we need them to
   # upload as artifacts later.
   rm -rf workdir/${CLUSTER}
-  mkdir -p workdir/${CLUSTER}/{consul,register,envoy,bats,statsd,data}
+  mkdir -p workdir/${CLUSTER}/{consul,consul-server,register,envoy,bats,statsd,data}
 
   # Reload consul config from defaults
   cp consul-base-cfg/*.hcl workdir/${CLUSTER}/consul/
@@ -76,6 +78,13 @@ function init_workdir {
   # move all of the registration files OUT of the consul config dir now
   find workdir/${CLUSTER}/consul -type f -name 'service_*.hcl' -exec mv -f {} workdir/${CLUSTER}/register \;
 
+  # move the server.hcl out of the consul dir so that it doesn't get picked up
+  # by the client agent (if we're running with XDS_TARGET=client).
+  if test -f "workdir/${CLUSTER}/consul/server.hcl"
+  then
+    mv workdir/${CLUSTER}/consul/server.hcl workdir/${CLUSTER}/consul-server/server.hcl
+  fi
+
   # copy the ca-certs for SDS so we can verify the right ones are served
   mkdir -p workdir/test-sds-server/certs
   cp test-sds-server/certs/ca-root.crt workdir/test-sds-server/certs/ca-root.crt
@@ -115,9 +124,6 @@ function docker_kill_rm {
 function start_consul {
   local DC=${1:-primary}
 
-  # Start consul now as setup script needs it up
-  docker_kill_rm consul-${DC}
-
   # 8500/8502 are for consul
   # 9411 is for zipkin which shares the network with consul
   # 16686 is for jaeger ui which also shares the network with consul
@@ -151,20 +157,91 @@ function start_consul {
     license=$(cat $CONSUL_LICENSE_PATH)
   fi
 
-  # Run consul and expose some ports to the host to make debugging locally a
+  # We currently run these integration tests in two modes: one in which Envoy's
-  # bit easier.
+  # xDS sessions are served directly by a Consul server, and another in which it
+  # goes through a client agent.
   #
-  docker run -d --name envoy_consul-${DC}_1 \
+  # This is nessasary because servers and clients source configuration data in
-    --net=envoy-tests \
+  # different ways (client agents use an RPC-backed cache and servers use their
-    $WORKDIR_SNIPPET \
+  # own local data) and we want to catch regressions in both.
-    --hostname "consul-${DC}" \
+  #
-    --network-alias "consul-${DC}" \
+  # In the future we should also expand these tests to register services to the
-    -e "CONSUL_LICENSE=$license" \
+  # catalog directly (agentless) rather than relying on the server also being
-    ${ports[@]} \
+  # an agent.
-    consul-dev \
+  #
-    agent -dev -datacenter "${DC}" \
+  # When XDS_TARGET=client we'll start a Consul server with its gRPC port
-    -config-dir "/workdir/${DC}/consul" \
+  # disabled, and a client agent with its gRPC port enabled.
-    -client "0.0.0.0" >/dev/null
+  #
+  # When XDS_TARGET=server (or anything else) we'll run a single Consul server
+  # with its gRPC port enabled.
+  #
+  # In either case, the hostname `consul-${DC}-server` should be used as a
+  # server address (e.g. for WAN joining) and `consul-${DC}-client` should be
+  # used as a client address (e.g. for interacting with the HTTP API).
+  #
+  # Both hostnames work in both modes because we set network aliases on the
+  # containers such that both hostnames will resolve to the same container when
+  # XDS_TARGET=server.
+  #
+  # We also join containers to the network `container:consul-${DC}_1` in many
+  # places (see: network_snippet) so that we can curl localhost etc. In both
+  # modes, you can assume that this name refers to the client's container.
+  #
+  # Any .hcl files in the case/cluster directory will be given to both clients
+  # and servers (via the -config-dir flag) *except for* server.hcl which will
+  # only be applied to the server (and service registrations which will be made
+  # against the client).
+  if [[ "$XDS_TARGET" == "client" ]]
+  then
+    docker_kill_rm consul-${DC}-server
+    docker_kill_rm consul-${DC}
+
+    docker run -d --name envoy_consul-${DC}-server_1 \
+      --net=envoy-tests \
+      $WORKDIR_SNIPPET \
+      --hostname "consul-${DC}-server" \
+      --network-alias "consul-${DC}-server" \
+      -e "CONSUL_LICENSE=$license" \
+      consul-dev \
+      agent -dev -datacenter "${DC}" \
+      -config-dir "/workdir/${DC}/consul" \
+      -config-dir "/workdir/${DC}/consul-server" \
+      -grpc-port -1 \
+      -client "0.0.0.0" \
+      -bind "0.0.0.0" >/dev/null
+
+    docker run -d --name envoy_consul-${DC}_1 \
+      --net=envoy-tests \
+      $WORKDIR_SNIPPET \
+      --hostname "consul-${DC}-client" \
+      --network-alias "consul-${DC}-client" \
+      -e "CONSUL_LICENSE=$license" \
+      ${ports[@]} \
+      consul-dev \
+      agent -datacenter "${DC}" \
+      -config-dir "/workdir/${DC}/consul" \
+      -data-dir "/tmp/consul" \
+      -client "0.0.0.0" \
+      -grpc-port 8502 \
+      -datacenter "${DC}" \
+      -retry-join "consul-${DC}-server" >/dev/null
+  else
+    docker_kill_rm consul-${DC}
+
+    docker run -d --name envoy_consul-${DC}_1 \
+      --net=envoy-tests \
+      $WORKDIR_SNIPPET \
+      --hostname "consul-${DC}" \
+      --network-alias "consul-${DC}-client" \
+      --network-alias "consul-${DC}-server" \
+      -e "CONSUL_LICENSE=$license" \
+      ${ports[@]} \
+      consul-dev \
+      agent -dev -datacenter "${DC}" \
+      -config-dir "/workdir/${DC}/consul" \
+      -config-dir "/workdir/${DC}/consul-server" \
+      -client "0.0.0.0" >/dev/null
+  fi
 }
 
 function start_partitioned_client {
@@ -190,12 +267,12 @@ function start_partitioned_client {
   docker run -d --name envoy_consul-${PARTITION}_1 \
     --net=envoy-tests \
     $WORKDIR_SNIPPET \
-    --hostname "consul-${PARTITION}" \
+    --hostname "consul-${PARTITION}-client" \
-    --network-alias "consul-${PARTITION}" \
+    --network-alias "consul-${PARTITION}-client" \
     -e "CONSUL_LICENSE=$license" \
     consul-dev agent \
     -datacenter "primary" \
-    -retry-join "consul-primary" \
+    -retry-join "consul-primary-server" \
     -grpc-port 8502 \
     -data-dir "/tmp/consul" \
     -config-dir "/workdir/${PARTITION}/consul" \
@@ -268,18 +345,35 @@ function capture_logs {
 
   echo "Capturing Logs"
   mkdir -p "$LOG_DIR"
+
   services="$REQUIRED_SERVICES consul-primary"
+  if [[ "$XDS_TARGET" == "client" ]]
+  then
+    services="$services consul-primary-server"
+  fi
+
   if is_set $REQUIRE_SECONDARY
   then
     services="$services consul-secondary"
+
+    if [[ "$XDS_TARGET" == "client" ]]
+    then
+      services="$services consul-secondary-server"
+    fi
   fi
+
   if is_set $REQUIRE_PARTITIONS
   then
     services="$services consul-ap1"
   fi
   if is_set $REQUIRE_PEERS
   then
-      services="$services consul-alpha"
+    services="$services consul-alpha"
+
+    if [[ "$XDS_TARGET" == "client" ]]
+    then
+      services="$services consul-alpha-server"
+    fi
   fi
 
   if [ -f "${CASE_DIR}/capture.sh" ]
@@ -300,7 +394,7 @@ function stop_services {
   # Teardown
   docker_kill_rm $REQUIRED_SERVICES
 
-  docker_kill_rm consul-primary consul-secondary consul-ap1 consul-alpha
+  docker_kill_rm consul-primary consul-primary-server consul-secondary consul-secondary-server consul-ap1 consul-alpha consul-alpha-server
 }
 
 function init_vars {
@@ -467,7 +561,7 @@ function suite_teardown {
     docker_kill_rm $(grep "^function run_container_" $self_name | \
         sed 's/^function run_container_\(.*\) {/\1/g')
 
-    docker_kill_rm consul-primary consul-secondary consul-ap1 consul-alpha
+    docker_kill_rm consul-primary consul-primary-server consul-secondary consul-secondary-server consul-ap1 consul-alpha consul-alpha-server
 
     if docker network inspect envoy-tests &>/dev/null ; then
         echo -n "Deleting network 'envoy-tests'..."
@@ -771,4 +865,3 @@ case "${1-}" in
   *)
     "$@" ;;
 esac
-