From 45889f9f55466323e7aeca2576623824ef335c7d Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 31 Jan 2021 21:32:18 -0500 Subject: [PATCH] e2e: segment e2e clusters Ensure that the e2e clusters are isolated and never attempt to autojoin with another e2e cluster. This ensures that each cluster servers have a unique `ConsulAutoJoin`, to be used for discovery. --- e2e/terraform/compute.tf | 6 +++--- e2e/terraform/nomad.tf | 4 ++++ .../packer/ubuntu-bionic-amd64/provision.sh | 14 ++++++++++++++ .../packer/windows-2016-amd64/provision.ps1 | 10 ++++++++++ e2e/terraform/provision-nomad/main.tf | 6 +++++- e2e/terraform/provision-nomad/variables.tf | 6 ++++++ 6 files changed, 42 insertions(+), 4 deletions(-) diff --git a/e2e/terraform/compute.tf b/e2e/terraform/compute.tf index 49d1c9bf6..7e4299e62 100644 --- a/e2e/terraform/compute.tf +++ b/e2e/terraform/compute.tf @@ -14,7 +14,7 @@ resource "aws_instance" "server" { # Instance tags tags = { Name = "${local.random_name}-server-${count.index}" - ConsulAutoJoin = "auto-join" + ConsulAutoJoin = "auto-join-${local.random_name}" SHA = var.nomad_sha User = data.aws_caller_identity.current.arn } @@ -32,7 +32,7 @@ resource "aws_instance" "client_ubuntu_bionic_amd64" { # Instance tags tags = { Name = "${local.random_name}-client-ubuntu-bionic-amd64-${count.index}" - ConsulAutoJoin = "auto-join" + ConsulAutoJoin = "auto-join-${local.random_name}" SHA = var.nomad_sha User = data.aws_caller_identity.current.arn } @@ -52,7 +52,7 @@ resource "aws_instance" "client_windows_2016_amd64" { # Instance tags tags = { Name = "${local.random_name}-client-windows-2016-${count.index}" - ConsulAutoJoin = "auto-join" + ConsulAutoJoin = "auto-join-${local.random_name}" SHA = var.nomad_sha User = data.aws_caller_identity.current.arn } diff --git a/e2e/terraform/nomad.tf b/e2e/terraform/nomad.tf index 12acd3062..f06826676 100644 --- a/e2e/terraform/nomad.tf +++ b/e2e/terraform/nomad.tf @@ -21,6 +21,7 @@ module "nomad_server" { nomad_enterprise = var.nomad_enterprise nomad_acls = var.nomad_acls + cluster_name = local.random_name connection = { type = "ssh" @@ -56,6 +57,7 @@ module "nomad_client_ubuntu_bionic_amd64" { nomad_enterprise = var.nomad_enterprise nomad_acls = false + cluster_name = local.random_name connection = { type = "ssh" @@ -92,6 +94,8 @@ module "nomad_client_windows_2016_amd64" { nomad_enterprise = var.nomad_enterprise nomad_acls = false + cluster_name = local.random_name + connection = { type = "ssh" diff --git a/e2e/terraform/packer/ubuntu-bionic-amd64/provision.sh b/e2e/terraform/packer/ubuntu-bionic-amd64/provision.sh index 837f56934..129d2db5b 100755 --- a/e2e/terraform/packer/ubuntu-bionic-amd64/provision.sh +++ b/e2e/terraform/packer/ubuntu-bionic-amd64/provision.sh @@ -19,6 +19,7 @@ Options for configuration: --nostart do not start or restart Nomad --enterprise if nomad_sha is passed, use the ENT version --nomad_acls write Nomad ACL configuration + --autojoin the AWS ConsulAutoJoin tag value EOF @@ -36,6 +37,7 @@ NOMAD_PROFILE= NOMAD_ROLE= NOMAD_INDEX= BUILD_FOLDER="builds-oss" +CONSUL_AUTOJOIN= ACLS=0 install_from_s3() { @@ -118,6 +120,9 @@ install_config_profile() { fi } +update_consul_autojoin() { + sudo sed -i'' -e "s|tag_key=ConsulAutoJoin tag_value=auto-join|tag_key=ConsulAutoJoin tag_value=${CONSUL_AUTOJOIN}|g" /etc/consul.d/*.json +} while [[ $# -gt 0 ]] do @@ -156,6 +161,11 @@ opt="$1" NOMAD_INDEX="$2" shift 2 ;; + --autojoin) + if [ -z "$2" ]; then ehco "Missing autojoin parameter"; usage; fi + CONSUL_AUTOJOIN="$2" + shift 2 + ;; --nostart) # for initial packer builds, we don't want to start Nomad START=0 @@ -181,6 +191,10 @@ if [ -n "$NOMAD_PROFILE" ]; then install_config_profile fi +if [ -n "$CONSUL_AUTOJOIN" ]; then + update_consul_autojoin +fi + if [ $START == "1" ]; then if [ "$NOMAD_ROLE" == "server" ]; then sudo systemctl restart vault diff --git a/e2e/terraform/packer/windows-2016-amd64/provision.ps1 b/e2e/terraform/packer/windows-2016-amd64/provision.ps1 index badcfd6ce..06ae84ba4 100755 --- a/e2e/terraform/packer/windows-2016-amd64/provision.ps1 +++ b/e2e/terraform/packer/windows-2016-amd64/provision.ps1 @@ -7,6 +7,7 @@ param( [string]$config_profile, [string]$role, [string]$index, + [string]$autojoin, [switch]$nostart = $false ) @@ -26,6 +27,7 @@ Options for configuration: -index INDEX count of instance, for profiles with per-instance config -nostart do not start or restart Nomad -enterprise if nomad_sha is passed, use the ENT version +--autojoin the AWS ConsulAutoJoin tag value "@ @@ -189,6 +191,11 @@ function InstallConfigProfile { } } +function UpdateConsulAutojoin { + (Get-Content C:\opt\consul.d\aws.json).replace("tag_key=ConsulAutoJoin tag_value=auto-join", "tag_key=ConsulAutoJoin tag_value=${autojoin}") | ` + Set-Content C:\opt\consul.d\aws.json +} + function CreateConsulService { New-Service ` -Name "Consul" ` @@ -229,6 +236,9 @@ if ( "" -ne $nomad_binary ) { if ( "" -ne $config_profile) { InstallConfigProfile } +if ( "" -ne $autojoin) { + UpdateConsulAutojoin +} if (!($nostart)) { CreateConsulService diff --git a/e2e/terraform/provision-nomad/main.tf b/e2e/terraform/provision-nomad/main.tf index 0bb9dae78..fbc273c92 100644 --- a/e2e/terraform/provision-nomad/main.tf +++ b/e2e/terraform/provision-nomad/main.tf @@ -41,7 +41,7 @@ resource "null_resource" "provision_nomad" { } data "template_file" "provision_script" { - template = "${local.provision_script}${data.template_file.arg_nomad_sha.rendered}${data.template_file.arg_nomad_version.rendered}${data.template_file.arg_nomad_binary.rendered}${data.template_file.arg_nomad_enterprise.rendered}${data.template_file.arg_nomad_acls.rendered}${data.template_file.arg_profile.rendered}${data.template_file.arg_role.rendered}${data.template_file.arg_index.rendered}" + template = "${local.provision_script}${data.template_file.arg_nomad_sha.rendered}${data.template_file.arg_nomad_version.rendered}${data.template_file.arg_nomad_binary.rendered}${data.template_file.arg_nomad_enterprise.rendered}${data.template_file.arg_nomad_acls.rendered}${data.template_file.arg_profile.rendered}${data.template_file.arg_role.rendered}${data.template_file.arg_index.rendered}${data.template_file.autojoin_tag.rendered}" } data "template_file" "arg_nomad_sha" { @@ -76,6 +76,10 @@ data "template_file" "arg_index" { template = var.index != "" ? " ${local._arg}index ${var.index}" : "" } +data "template_file" "autojoin_tag" { + template = var.cluster_name != "" ? " ${local._arg}autojoin auto-join-${var.cluster_name}" : "" +} + resource "null_resource" "upload_nomad_binary" { count = var.nomad_local_binary != "" ? 1 : 0 diff --git a/e2e/terraform/provision-nomad/variables.tf b/e2e/terraform/provision-nomad/variables.tf index 5b5f9d39f..aa8a1f95c 100644 --- a/e2e/terraform/provision-nomad/variables.tf +++ b/e2e/terraform/provision-nomad/variables.tf @@ -52,6 +52,12 @@ variable "index" { default = "" } +variable "cluster_name" { + type = string + description = "The random name assigned to the cluster" + default = "" +} + variable "connection" { type = object({ type = string