test: run Envoy integration tests against both servers and clients (#13610)

This commit is contained in:
Dan Upton 2022-06-28 13:15:45 +01:00 committed by GitHub
parent f6f41da860
commit 883ccc2a98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 181 additions and 86 deletions

View File

@ -23,6 +23,10 @@ references:
BASH_ENV: .circleci/bash_env.sh
VAULT_BINARY_VERSION: 1.9.4
GO_VERSION: 1.18.1
envoy-versions: &supported_envoy_versions
- &default_envoy_version "1.19.5"
- "1.20.4"
- "1.21.3"
images:
# When updating the Go version, remember to also update the versions in the
# workflows section for go-test-lib jobs.
@ -852,13 +856,23 @@ jobs:
path: *TEST_RESULTS_DIR
- run: *notify-slack-failure
envoy-integration-test-1_19_5: &ENVOY_TESTS
envoy-integration-test: &ENVOY_TESTS
machine:
image: *UBUNTU_CI_IMAGE
parallelism: 4
resource_class: medium
parameters:
envoy-version:
type: enum
enum: *supported_envoy_versions
default: *default_envoy_version
xds-target:
type: enum
enum: ["server", "client"]
default: "server"
environment:
ENVOY_VERSION: "1.19.5"
ENVOY_VERSION: << parameters.envoy-version >>
XDS_TARGET: << parameters.xds-target >>
steps: &ENVOY_INTEGRATION_TEST_STEPS
- checkout
# Get go binary from workspace
@ -891,21 +905,6 @@ jobs:
path: *TEST_RESULTS_DIR
- run: *notify-slack-failure
envoy-integration-test-1_20_4:
<<: *ENVOY_TESTS
environment:
ENVOY_VERSION: "1.20.4"
envoy-integration-test-1_21_3:
<<: *ENVOY_TESTS
environment:
ENVOY_VERSION: "1.21.3"
envoy-integration-test-1_22_2:
<<: *ENVOY_TESTS
environment:
ENVOY_VERSION: "1.22.2"
# run integration tests for the connect ca providers
test-connect-ca-providers:
docker:
@ -1116,18 +1115,13 @@ workflows:
- nomad-integration-0_8:
requires:
- dev-build
- envoy-integration-test-1_19_5:
requires:
- dev-build
- envoy-integration-test-1_20_4:
requires:
- dev-build
- envoy-integration-test-1_21_3:
requires:
- dev-build
- envoy-integration-test-1_22_2:
- envoy-integration-test:
requires:
- dev-build
matrix:
parameters:
envoy-version: *supported_envoy_versions
xds-target: ["server", "client"]
- compatibility-integration-test:
requires:
- dev-build

View File

@ -31,7 +31,7 @@ load helpers
}
@test "gateway-secondary should be up and listening" {
retry_long nc -z consul-secondary:4432
retry_long nc -z consul-secondary-client:4432
}
################

View File

@ -1 +1 @@
retry_join_wan = ["consul-primary"]
retry_join_wan = ["consul-primary-server"]

View File

@ -31,7 +31,7 @@ load helpers
}
@test "gateway-secondary should be up and listening" {
retry_long nc -z consul-secondary:4432
retry_long nc -z consul-secondary-client:4432
}
@test "wait until the first cluster is configured" {

View File

@ -1 +1 @@
retry_join_wan = ["consul-primary"]
retry_join_wan = ["consul-primary-server"]

View File

@ -23,7 +23,7 @@ load helpers
}
@test "gateway-alpha should be up and listening" {
retry_long nc -z consul-alpha:4432
retry_long nc -z consul-alpha-client:4432
}
@test "s2 proxies should be healthy" {

View File

@ -23,11 +23,11 @@ load helpers
}
@test "gateway-primary should be up and listening" {
retry_long nc -z consul-primary:4431
retry_long nc -z consul-primary-client:4431
}
@test "gateway-alpha should be up and listening" {
retry_long nc -z consul-alpha:4432
retry_long nc -z consul-alpha-client:4432
}
@test "peer the two clusters together" {
@ -39,7 +39,7 @@ load helpers
}
@test "gateway-alpha should have healthy endpoints for s2" {
assert_upstream_has_endpoints_in_status consul-alpha:19003 s2.default.alpha HEALTHY 1
assert_upstream_has_endpoints_in_status consul-alpha-client:19003 s2.default.alpha HEALTHY 1
}
@test "s1 upstream should have healthy endpoints for s2" {

View File

@ -28,7 +28,7 @@ load helpers
}
@test "gateway-secondary should have healthy endpoints for s2" {
assert_upstream_has_endpoints_in_status consul-secondary:19003 s2 HEALTHY 1
assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1
}
@test "s1 upstream should be able to connect to s2" {

View File

@ -1 +1 @@
retry_join_wan = ["consul-primary"]
retry_join_wan = ["consul-primary-server"]

View File

@ -1 +1 @@
retry_join_wan = ["consul-primary"]
retry_join_wan = ["consul-primary-server"]

View File

@ -23,11 +23,11 @@ load helpers
}
@test "gateway-secondary should have healthy endpoints for s1" {
assert_upstream_has_endpoints_in_status consul-secondary:19003 s1 HEALTHY 1
assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s1 HEALTHY 1
}
@test "gateway-secondary should have healthy endpoints for s2" {
assert_upstream_has_endpoints_in_status consul-secondary:19003 s2 HEALTHY 1
assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1
}
@test "ingress should be able to connect to s1" {

View File

@ -1 +1 @@
retry_join_wan = ["consul-primary"]
retry_join_wan = ["consul-primary-server"]

View File

@ -1 +1 @@
retry_join_wan = ["consul-primary"]
retry_join_wan = ["consul-primary-server"]

View File

@ -0,0 +1,6 @@
ca_file = "/workdir/primary/tls/consul-agent-ca.pem"
cert_file = "/workdir/primary/tls/primary-server-consul-0.pem"
key_file = "/workdir/primary/tls/primary-server-consul-0-key.pem"
verify_incoming = true
verify_outgoing = true
verify_server_hostname = true

View File

@ -19,7 +19,7 @@ load helpers
}
@test "gateway-secondary should be up and listening" {
retry_long nc -z consul-secondary:4432
retry_long nc -z consul-secondary-client:4432
}
@test "primary should be able to rpc to the secondary" {
@ -27,5 +27,5 @@ load helpers
}
@test "wan pool should show 2 healthy nodes" {
assert_alive_wan_member_count 2
assert_alive_wan_member_count primary 2
}

View File

@ -0,0 +1,6 @@
ca_file = "/workdir/secondary/tls/consul-agent-ca.pem"
cert_file = "/workdir/secondary/tls/secondary-server-consul-0.pem"
key_file = "/workdir/secondary/tls/secondary-server-consul-0-key.pem"
verify_incoming = true
verify_outgoing = true
verify_server_hostname = true

View File

@ -4,13 +4,7 @@ connect {
enable_mesh_gateway_wan_federation = true
}
primary_gateways = [
"consul-primary:4431",
"consul-primary-client:4431",
]
primary_gateways_interval = "5s"
retry_interval_wan = "5s"
ca_file = "/workdir/secondary/tls/consul-agent-ca.pem"
cert_file = "/workdir/secondary/tls/secondary-server-consul-0.pem"
key_file = "/workdir/secondary/tls/secondary-server-consul-0-key.pem"
verify_incoming = true
verify_outgoing = true
verify_server_hostname = true

View File

@ -19,7 +19,7 @@ load helpers
}
@test "gateway-primary should be up and listening" {
retry_long nc -z consul-primary:4431
retry_long nc -z consul-primary-client:4431
}
@test "secondary should be able to rpc to the primary" {
@ -27,5 +27,5 @@ load helpers
}
@test "wan pool should show 2 healthy nodes" {
assert_alive_wan_member_count 2
assert_alive_wan_member_count secondary 2
}

View File

@ -479,28 +479,31 @@ function get_healthy_service_count {
local AP=$4
local PEER_NAME=$5
run curl -s -f ${HEADERS} "consul-${DC}:8500/v1/health/connect/${SERVICE_NAME}?passing&ns=${NS}&partition=${AP}&peer=${PEER_NAME}"
run curl -s -f ${HEADERS} "consul-${DC}-client:8500/v1/health/connect/${SERVICE_NAME}?passing&ns=${NS}&partition=${AP}&peer=${PEER_NAME}"
[ "$status" -eq 0 ]
echo "$output" | jq --raw-output '. | length'
}
function assert_alive_wan_member_count {
local EXPECT_COUNT=$1
run retry_long assert_alive_wan_member_count_once $EXPECT_COUNT
local DC=$1
local EXPECT_COUNT=$2
run retry_long assert_alive_wan_member_count_once $DC $EXPECT_COUNT
[ "$status" -eq 0 ]
}
function assert_alive_wan_member_count_once {
local EXPECT_COUNT=$1
local DC=$1
local EXPECT_COUNT=$2
GOT_COUNT=$(get_alive_wan_member_count)
GOT_COUNT=$(get_alive_wan_member_count $DC)
[ "$GOT_COUNT" -eq "$EXPECT_COUNT" ]
}
function get_alive_wan_member_count {
run retry_default curl -sL -f "127.0.0.1:8500/v1/agent/members?wan=1"
local DC=$1
run retry_default curl -sL -f "consul-${DC}-server:8500/v1/agent/members?wan=1"
[ "$status" -eq 0 ]
# echo "$output" >&3
echo "$output" | jq '.[] | select(.Status == 1) | .Name' | wc -l
@ -951,7 +954,7 @@ function assert_expected_fortio_host_header {
function create_peering {
local GENERATE_PEER=$1
local ESTABLISH_PEER=$2
run curl -sL -XPOST "http://consul-${GENERATE_PEER}:8500/v1/peering/token" -d"{ \"PeerName\" : \"${GENERATE_PEER}-to-${ESTABLISH_PEER}\" }"
run curl -sL -XPOST "http://consul-${GENERATE_PEER}-client:8500/v1/peering/token" -d"{ \"PeerName\" : \"${GENERATE_PEER}-to-${ESTABLISH_PEER}\" }"
# echo "$output" >&3
[ "$status" == 0 ]
@ -959,8 +962,7 @@ function create_peering {
token="$(echo "$output" | jq -r .PeeringToken)"
[ -n "$token" ]
run curl -sLv -XPOST "http://consul-${ESTABLISH_PEER}:8500/v1/peering/establish" -d"{ \"PeerName\" : \"${ESTABLISH_PEER}-to-${GENERATE_PEER}\", \"PeeringToken\" : \"${token}\" }"
run curl -sLv -XPOST "http://consul-${ESTABLISH_PEER}-client:8500/v1/peering/establish" -d"{ \"PeerName\" : \"${ESTABLISH_PEER}-to-${GENERATE_PEER}\", \"PeeringToken\" : \"${token}\" }"
# echo "$output" >&3
[ "$status" == 0 ]
}

View File

@ -9,6 +9,8 @@ readonly HASHICORP_DOCKER_PROXY="docker.mirror.hashicorp.services"
# DEBUG=1 enables set -x for this script so echos every command run
DEBUG=${DEBUG:-}
XDS_TARGET=${XDS_TARGET:-server}
# ENVOY_VERSION to run each test against
ENVOY_VERSION=${ENVOY_VERSION:-"1.22.2"}
export ENVOY_VERSION
@ -53,7 +55,7 @@ function init_workdir {
# don't wipe logs between runs as they are already split and we need them to
# upload as artifacts later.
rm -rf workdir/${CLUSTER}
mkdir -p workdir/${CLUSTER}/{consul,register,envoy,bats,statsd,data}
mkdir -p workdir/${CLUSTER}/{consul,consul-server,register,envoy,bats,statsd,data}
# Reload consul config from defaults
cp consul-base-cfg/*.hcl workdir/${CLUSTER}/consul/
@ -76,6 +78,13 @@ function init_workdir {
# move all of the registration files OUT of the consul config dir now
find workdir/${CLUSTER}/consul -type f -name 'service_*.hcl' -exec mv -f {} workdir/${CLUSTER}/register \;
# move the server.hcl out of the consul dir so that it doesn't get picked up
# by the client agent (if we're running with XDS_TARGET=client).
if test -f "workdir/${CLUSTER}/consul/server.hcl"
then
mv workdir/${CLUSTER}/consul/server.hcl workdir/${CLUSTER}/consul-server/server.hcl
fi
# copy the ca-certs for SDS so we can verify the right ones are served
mkdir -p workdir/test-sds-server/certs
cp test-sds-server/certs/ca-root.crt workdir/test-sds-server/certs/ca-root.crt
@ -115,9 +124,6 @@ function docker_kill_rm {
function start_consul {
local DC=${1:-primary}
# Start consul now as setup script needs it up
docker_kill_rm consul-${DC}
# 8500/8502 are for consul
# 9411 is for zipkin which shares the network with consul
# 16686 is for jaeger ui which also shares the network with consul
@ -151,20 +157,91 @@ function start_consul {
license=$(cat $CONSUL_LICENSE_PATH)
fi
# Run consul and expose some ports to the host to make debugging locally a
# bit easier.
# We currently run these integration tests in two modes: one in which Envoy's
# xDS sessions are served directly by a Consul server, and another in which it
# goes through a client agent.
#
docker run -d --name envoy_consul-${DC}_1 \
--net=envoy-tests \
$WORKDIR_SNIPPET \
--hostname "consul-${DC}" \
--network-alias "consul-${DC}" \
-e "CONSUL_LICENSE=$license" \
${ports[@]} \
consul-dev \
agent -dev -datacenter "${DC}" \
-config-dir "/workdir/${DC}/consul" \
-client "0.0.0.0" >/dev/null
# This is nessasary because servers and clients source configuration data in
# different ways (client agents use an RPC-backed cache and servers use their
# own local data) and we want to catch regressions in both.
#
# In the future we should also expand these tests to register services to the
# catalog directly (agentless) rather than relying on the server also being
# an agent.
#
# When XDS_TARGET=client we'll start a Consul server with its gRPC port
# disabled, and a client agent with its gRPC port enabled.
#
# When XDS_TARGET=server (or anything else) we'll run a single Consul server
# with its gRPC port enabled.
#
# In either case, the hostname `consul-${DC}-server` should be used as a
# server address (e.g. for WAN joining) and `consul-${DC}-client` should be
# used as a client address (e.g. for interacting with the HTTP API).
#
# Both hostnames work in both modes because we set network aliases on the
# containers such that both hostnames will resolve to the same container when
# XDS_TARGET=server.
#
# We also join containers to the network `container:consul-${DC}_1` in many
# places (see: network_snippet) so that we can curl localhost etc. In both
# modes, you can assume that this name refers to the client's container.
#
# Any .hcl files in the case/cluster directory will be given to both clients
# and servers (via the -config-dir flag) *except for* server.hcl which will
# only be applied to the server (and service registrations which will be made
# against the client).
if [[ "$XDS_TARGET" == "client" ]]
then
docker_kill_rm consul-${DC}-server
docker_kill_rm consul-${DC}
docker run -d --name envoy_consul-${DC}-server_1 \
--net=envoy-tests \
$WORKDIR_SNIPPET \
--hostname "consul-${DC}-server" \
--network-alias "consul-${DC}-server" \
-e "CONSUL_LICENSE=$license" \
consul-dev \
agent -dev -datacenter "${DC}" \
-config-dir "/workdir/${DC}/consul" \
-config-dir "/workdir/${DC}/consul-server" \
-grpc-port -1 \
-client "0.0.0.0" \
-bind "0.0.0.0" >/dev/null
docker run -d --name envoy_consul-${DC}_1 \
--net=envoy-tests \
$WORKDIR_SNIPPET \
--hostname "consul-${DC}-client" \
--network-alias "consul-${DC}-client" \
-e "CONSUL_LICENSE=$license" \
${ports[@]} \
consul-dev \
agent -datacenter "${DC}" \
-config-dir "/workdir/${DC}/consul" \
-data-dir "/tmp/consul" \
-client "0.0.0.0" \
-grpc-port 8502 \
-datacenter "${DC}" \
-retry-join "consul-${DC}-server" >/dev/null
else
docker_kill_rm consul-${DC}
docker run -d --name envoy_consul-${DC}_1 \
--net=envoy-tests \
$WORKDIR_SNIPPET \
--hostname "consul-${DC}" \
--network-alias "consul-${DC}-client" \
--network-alias "consul-${DC}-server" \
-e "CONSUL_LICENSE=$license" \
${ports[@]} \
consul-dev \
agent -dev -datacenter "${DC}" \
-config-dir "/workdir/${DC}/consul" \
-config-dir "/workdir/${DC}/consul-server" \
-client "0.0.0.0" >/dev/null
fi
}
function start_partitioned_client {
@ -190,12 +267,12 @@ function start_partitioned_client {
docker run -d --name envoy_consul-${PARTITION}_1 \
--net=envoy-tests \
$WORKDIR_SNIPPET \
--hostname "consul-${PARTITION}" \
--network-alias "consul-${PARTITION}" \
--hostname "consul-${PARTITION}-client" \
--network-alias "consul-${PARTITION}-client" \
-e "CONSUL_LICENSE=$license" \
consul-dev agent \
-datacenter "primary" \
-retry-join "consul-primary" \
-retry-join "consul-primary-server" \
-grpc-port 8502 \
-data-dir "/tmp/consul" \
-config-dir "/workdir/${PARTITION}/consul" \
@ -268,18 +345,35 @@ function capture_logs {
echo "Capturing Logs"
mkdir -p "$LOG_DIR"
services="$REQUIRED_SERVICES consul-primary"
if [[ "$XDS_TARGET" == "client" ]]
then
services="$services consul-primary-server"
fi
if is_set $REQUIRE_SECONDARY
then
services="$services consul-secondary"
if [[ "$XDS_TARGET" == "client" ]]
then
services="$services consul-secondary-server"
fi
fi
if is_set $REQUIRE_PARTITIONS
then
services="$services consul-ap1"
fi
if is_set $REQUIRE_PEERS
then
services="$services consul-alpha"
services="$services consul-alpha"
if [[ "$XDS_TARGET" == "client" ]]
then
services="$services consul-alpha-server"
fi
fi
if [ -f "${CASE_DIR}/capture.sh" ]
@ -300,7 +394,7 @@ function stop_services {
# Teardown
docker_kill_rm $REQUIRED_SERVICES
docker_kill_rm consul-primary consul-secondary consul-ap1 consul-alpha
docker_kill_rm consul-primary consul-primary-server consul-secondary consul-secondary-server consul-ap1 consul-alpha consul-alpha-server
}
function init_vars {
@ -467,7 +561,7 @@ function suite_teardown {
docker_kill_rm $(grep "^function run_container_" $self_name | \
sed 's/^function run_container_\(.*\) {/\1/g')
docker_kill_rm consul-primary consul-secondary consul-ap1 consul-alpha
docker_kill_rm consul-primary consul-primary-server consul-secondary consul-secondary-server consul-ap1 consul-alpha consul-alpha-server
if docker network inspect envoy-tests &>/dev/null ; then
echo -n "Deleting network 'envoy-tests'..."
@ -771,4 +865,3 @@ case "${1-}" in
*)
"$@" ;;
esac