From 42d9542ac3c7d48f45dd5ab0aadfd17e1595f11e Mon Sep 17 00:00:00 2001
From: freddygv <freddy@hashicorp.com>
Date: Fri, 12 Nov 2021 14:45:50 -0700
Subject: [PATCH] Add cross-partition integration test

---
 test/integration/connect/envoy/defaults.sh  |   3 +-
 test/integration/connect/envoy/helpers.bash |   9 +-
 test/integration/connect/envoy/run-tests.sh | 131 +++++++++++++++-----
 3 files changed, 106 insertions(+), 37 deletions(-)

diff --git a/test/integration/connect/envoy/defaults.sh b/test/integration/connect/envoy/defaults.sh
index 2961ff8ae..ccf85824d 100644
--- a/test/integration/connect/envoy/defaults.sh
+++ b/test/integration/connect/envoy/defaults.sh
@@ -2,4 +2,5 @@
 
 export DEFAULT_REQUIRED_SERVICES="s1 s1-sidecar-proxy s2 s2-sidecar-proxy"
 export REQUIRED_SERVICES="${DEFAULT_REQUIRED_SERVICES}"
-export REQUIRE_SECONDARY=0
\ No newline at end of file
+export REQUIRE_SECONDARY=0
+export REQUIRE_PARTITIONS=0
\ No newline at end of file
diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash
index dcf184a6a..0c5f60681 100755
--- a/test/integration/connect/envoy/helpers.bash
+++ b/test/integration/connect/envoy/helpers.bash
@@ -115,14 +115,19 @@ function assert_proxy_presents_cert_uri {
   local SERVICENAME=$2
   local DC=${3:-primary}
   local NS=${4:-default}
+  local PARTITION=${5:default}
 
   CERT=$(retry_default get_cert $HOSTPORT)
 
-  echo "WANT SERVICE: ${NS}/${SERVICENAME}"
+  echo "WANT SERVICE: ${PARTITION}/${NS}/${SERVICENAME}"
   echo "GOT CERT:"
   echo "$CERT"
 
-  echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
+  if [[ -z $PARTITION ]] || [[ $PARTITION = "default" ]]; then
+    echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
+  else
+    echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ap/${PARTITION}/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
+  fi
 }
 
 function assert_dnssan_in_cert {
diff --git a/test/integration/connect/envoy/run-tests.sh b/test/integration/connect/envoy/run-tests.sh
index ea2f8429b..bbeb7c423 100755
--- a/test/integration/connect/envoy/run-tests.sh
+++ b/test/integration/connect/envoy/run-tests.sh
@@ -40,39 +40,39 @@ function network_snippet {
 }
 
 function init_workdir {
-  local DC="$1"
+  local CLUSTER="$1"
 
-  if test -z "$DC"
+  if test -z "$CLUSTER"
   then
-    DC=primary
+    CLUSTER=primary
   fi
 
   # Note, we use explicit set of dirs so we don't delete .gitignore. Also,
   # don't wipe logs between runs as they are already split and we need them to
   # upload as artifacts later.
-  rm -rf workdir/${DC}
-  mkdir -p workdir/${DC}/{consul,register,envoy,bats,statsd,data}
+  rm -rf workdir/${CLUSTER}
+  mkdir -p workdir/${CLUSTER}/{consul,register,envoy,bats,statsd,data}
 
   # Reload consul config from defaults
-  cp consul-base-cfg/*.hcl workdir/${DC}/consul/
+  cp consul-base-cfg/*.hcl workdir/${CLUSTER}/consul/
 
   # Add any overrides if there are any (no op if not)
-  find ${CASE_DIR} -maxdepth 1 -name '*.hcl' -type f -exec cp -f {} workdir/${DC}/consul \;
+  find ${CASE_DIR} -maxdepth 1 -name '*.hcl' -type f -exec cp -f {} workdir/${CLUSTER}/consul \;
 
   # Copy all the test files
-  find ${CASE_DIR} -maxdepth 1 -name '*.bats' -type f -exec cp -f {} workdir/${DC}/bats \;
-  # Copy DC specific bats
-  cp helpers.bash workdir/${DC}/bats
+  find ${CASE_DIR} -maxdepth 1 -name '*.bats' -type f -exec cp -f {} workdir/${CLUSTER}/bats \;
+  # Copy CLUSTER specific bats
+  cp helpers.bash workdir/${CLUSTER}/bats
 
-  # Add any DC overrides
-  if test -d "${CASE_DIR}/${DC}"
+  # Add any CLUSTER overrides
+  if test -d "${CASE_DIR}/${CLUSTER}"
   then
-    find ${CASE_DIR}/${DC} -type f -name '*.hcl' -exec cp -f {} workdir/${DC}/consul \;
-    find ${CASE_DIR}/${DC} -type f -name '*.bats' -exec cp -f {} workdir/${DC}/bats \;
+    find ${CASE_DIR}/${CLUSTER} -type f -name '*.hcl' -exec cp -f {} workdir/${CLUSTER}/consul \;
+    find ${CASE_DIR}/${CLUSTER} -type f -name '*.bats' -exec cp -f {} workdir/${CLUSTER}/bats \;
   fi
 
   # move all of the registration files OUT of the consul config dir now
-  find workdir/${DC}/consul -type f -name 'service_*.hcl' -exec mv -f {} workdir/${DC}/register \;
+  find workdir/${CLUSTER}/consul -type f -name 'service_*.hcl' -exec mv -f {} workdir/${CLUSTER}/register \;
 
   # copy the ca-certs for SDS so we can verify the right ones are served
   mkdir -p workdir/test-sds-server/certs
@@ -80,7 +80,7 @@ function init_workdir {
 
   if test -d "${CASE_DIR}/data"
   then
-    cp -r ${CASE_DIR}/data/* workdir/${DC}/data
+    cp -r ${CASE_DIR}/data/* workdir/${CLUSTER}/data
   fi
 
   return 0
@@ -157,13 +157,48 @@ function start_consul {
     -client "0.0.0.0" >/dev/null
 }
 
+function start_partitioned_client {
+  local PARTITION=${1:-ap1}
+
+  # Start consul now as setup script needs it up
+  docker_kill_rm consul-${PARTITION}
+
+  license="${CONSUL_LICENSE:-}"
+  # load the consul license so we can pass it into the consul
+  # containers as an env var in the case that this is a consul
+  # enterprise test
+  if test -z "$license" -a -n "${CONSUL_LICENSE_PATH:-}"
+  then
+    license=$(cat $CONSUL_LICENSE_PATH)
+  fi
+
+  sh -c "rm -rf /workdir/${PARTITION}/data"
+
+  # Run consul and expose some ports to the host to make debugging locally a
+  # bit easier.
+  #
+  docker run -d --name envoy_consul-${PARTITION}_1 \
+    --net=envoy-tests \
+    $WORKDIR_SNIPPET \
+    --hostname "consul-${PARTITION}" \
+    --network-alias "consul-${PARTITION}" \
+    -e "CONSUL_LICENSE=$license" \
+    consul-dev agent \
+    -datacenter "primary" \
+    -retry-join "consul-primary" \
+    -grpc-port 8502 \
+    -data-dir "/tmp/consul" \
+    -config-dir "/workdir/${PARTITION}/consul" \
+    -client "0.0.0.0" >/dev/null
+}
+
 function pre_service_setup {
-  local DC=${1:-primary}
+  local CLUSTER=${1:-primary}
 
   # Run test case setup (e.g. generating Envoy bootstrap, starting containers)
-  if [ -f "${CASE_DIR}/${DC}/setup.sh" ]
+  if [ -f "${CASE_DIR}/${CLUSTER}/setup.sh" ]
   then
-    source ${CASE_DIR}/${DC}/setup.sh
+    source ${CASE_DIR}/${CLUSTER}/setup.sh
   else
     source ${CASE_DIR}/setup.sh
   fi
@@ -184,29 +219,29 @@ function start_services {
 }
 
 function verify {
-  local DC=$1
-  if test -z "$DC"; then
-    DC=primary
+  local CLUSTER="$1"
+  if test -z "$CLUSTER"; then
+    CLUSTER="primary"
   fi
 
   # Execute tests
   res=0
 
   # Nuke any previous case's verify container.
-  docker_kill_rm verify-${DC}
+  docker_kill_rm verify-${CLUSTER}
 
-  echo "Running ${DC} verification step for ${CASE_DIR}..."
+  echo "Running ${CLUSTER} verification step for ${CASE_DIR}..."
 
   # need to tell the PID 1 inside of the container that it won't be actual PID
   # 1 because we're using --pid=host so we use TINI_SUBREAPER
-  if docker run --name envoy_verify-${DC}_1 -t \
+  if docker run --name envoy_verify-${CLUSTER}_1 -t \
     -e TINI_SUBREAPER=1 \
     -e ENVOY_VERSION \
     $WORKDIR_SNIPPET \
     --pid=host \
-    $(network_snippet $DC) \
+    $(network_snippet $CLUSTER) \
     bats-verify \
-    --pretty /workdir/${DC}/bats ; then
+    --pretty /workdir/${CLUSTER}/bats ; then
     echogreen "✓ PASS"
   else
     echored "⨯ FAIL"
@@ -228,6 +263,11 @@ function capture_logs {
   then
     services="$services consul-secondary"
   fi
+  if is_set $REQUIRE_PARTITIONS
+  then
+    services="$services consul-ap1"
+  fi
+
 
   if [ -f "${CASE_DIR}/capture.sh" ]
   then
@@ -247,7 +287,7 @@ function stop_services {
   # Teardown
   docker_kill_rm $REQUIRED_SERVICES
 
-  docker_kill_rm consul-primary consul-secondary
+  docker_kill_rm consul-primary consul-secondary consul-ap1
 }
 
 function init_vars {
@@ -286,6 +326,10 @@ function run_tests {
   then
     init_workdir secondary
   fi
+  if is_set $REQUIRE_PARTITIONS
+  then
+    init_workdir ap1
+  fi
 
   global_setup
 
@@ -307,6 +351,10 @@ function run_tests {
   if is_set $REQUIRE_SECONDARY; then
     start_consul secondary
   fi
+  if is_set $REQUIRE_PARTITIONS; then
+    docker_consul "primary" admin-partition create -name ap1 > /dev/null
+    start_partitioned_client ap1
+  fi
 
   echo "Setting up the primary datacenter"
   pre_service_setup primary
@@ -315,14 +363,20 @@ function run_tests {
     echo "Setting up the secondary datacenter"
     pre_service_setup secondary
   fi
+  if is_set $REQUIRE_PARTITIONS; then
+    echo "Setting up the non-default partition"
+    pre_service_setup ap1
+  fi
 
   echo "Starting services"
   start_services
 
   # Run the verify container and report on the output
+  echo "Verifying the primary datacenter"
   verify primary
 
   if is_set $REQUIRE_SECONDARY; then
+    echo "Verifying the secondary datacenter"
     verify secondary
   fi
 }
@@ -378,7 +432,7 @@ function suite_teardown {
     docker_kill_rm $(grep "^function run_container_" $self_name | \
         sed 's/^function run_container_\(.*\) {/\1/g')
 
-    docker_kill_rm consul-primary consul-secondary
+    docker_kill_rm consul-primary consul-secondary consul-ap1
 
     if docker network inspect envoy-tests &>/dev/null ; then
         echo -n "Deleting network 'envoy-tests'..."
@@ -402,13 +456,13 @@ function run_container {
 
 function common_run_container_service {
   local service="$1"
-  local DC="$2"
+  local CLUSTER="$2"
   local httpPort="$3"
   local grpcPort="$4"
 
   docker run -d --name $(container_name_prev) \
     -e "FORTIO_NAME=${service}" \
-    $(network_snippet $DC) \
+    $(network_snippet $CLUSTER) \
     "${HASHICORP_DOCKER_PROXY}/fortio/fortio" \
     server \
     -http-port ":$httpPort" \
@@ -420,6 +474,10 @@ function run_container_s1 {
   common_run_container_service s1 primary 8080 8079
 }
 
+function run_container_s1-ap1 {
+  common_run_container_service s1 ap1 8080 8079
+}
+
 function run_container_s2 {
   common_run_container_service s2 primary 8181 8179
 }
@@ -457,7 +515,7 @@ function run_container_s2-secondary {
 
 function common_run_container_sidecar_proxy {
   local service="$1"
-  local DC="$2"
+  local CLUSTER="$2"
 
   # Hot restart breaks since both envoys seem to interact with each other
   # despite separate containers that don't share IPC namespace. Not quite
@@ -465,10 +523,10 @@ function common_run_container_sidecar_proxy {
   # location?
   docker run -d --name $(container_name_prev) \
     $WORKDIR_SNIPPET \
-    $(network_snippet $DC) \
+    $(network_snippet $CLUSTER) \
     "${HASHICORP_DOCKER_PROXY}/envoyproxy/envoy:v${ENVOY_VERSION}" \
     envoy \
-    -c /workdir/${DC}/envoy/${service}-bootstrap.json \
+    -c /workdir/${CLUSTER}/envoy/${service}-bootstrap.json \
     -l debug \
     --disable-hot-restart \
     --drain-time-s 1 >/dev/null
@@ -477,6 +535,11 @@ function common_run_container_sidecar_proxy {
 function run_container_s1-sidecar-proxy {
   common_run_container_sidecar_proxy s1 primary
 }
+
+function run_container_s1-ap1-sidecar-proxy {
+  common_run_container_sidecar_proxy s1 ap1
+}
+
 function run_container_s1-sidecar-proxy-consul-exec {
   docker run -d --name $(container_name) \
     $(network_snippet primary) \