Load test, upgrade packer version, fix k6s installation (#13382)

- fix sg: need remote access to test server
- Give the load generator a name
- Update loadtest hcl filename in readme
- Add terraform init
- Disable access to the server machine by default

.circleci/terraform/load-test/main.tf

@@ -7,12 +7,6 @@ provider "aws" {
   assume_role {
     role_arn = var.role_arn
   }
-
-  default_tags {
-    tags = {
-      Environment = "ConsulLoadTest"
-    }
-  }
 }
 
 module "load-test" {
@@ -21,6 +15,7 @@ module "load-test" {
   vpc_az               = ["us-east-2a", "us-east-2b"]
   vpc_name             = var.vpc_name
   vpc_cidr             = "10.0.0.0/16"
+  vpc_allwed_ssh_cidr  = "0.0.0.0/0"
   public_subnet_cidrs  = ["10.0.1.0/24", "10.0.2.0/24"]
   private_subnet_cidrs = ["10.0.3.0/24"]
   test_public_ip       = true

test/load/packer/README.md

@@ -1,5 +1,6 @@
 # Terraform Consul Load Testing
 Packer will output AMI IDs when it completes - save these AMI IDs as Terraform will require them later.
+The generated ami will have common tag `Environment = "consul-load-test"`.
 
 ```
 ==> Builds finished. The artifacts of successful builds are:
@@ -12,7 +13,8 @@ us-east-1: ami-19601070
 Within the `consul-ami/` directory
 1) Retrieve your [Datadog API key]((https://docs.datadoghq.com/account_management/api-app-keys/#api-keys)), set this as an environment variable, ex: `export DD_API_KEY=$YOURDDAPIKEYHERE`
 2) Set the AWS_DEFAULT_REGION for Packer, ex: `export AWS_DEFAULT_REGION=us-east-1`
-3) Run `packer build consul.json`. 
+3) Set up AWS credentials, ex: `export AWS_ACCESS_KEY_ID=#YOURAWSKEYID & export AWS_SECRET_ACCESS_KEY=YOURAWSKEY`
+4) Run `packer build consul.pkr.hcl`.
 
 For additional customization you can add [tags](https://docs.datadoghq.com/getting_started/tagging/assigning_tags/?tab=noncontainerizedenvironments) within the `scripts/datadog.yaml` file. An example of a tag could be `"consul_version" : "consulent_175"`. These tags are searchable through the datadog dashboard. Another form of customization is changing the datacenter tag within `scripts/telemetry.json`, however it is defaulted to `us-east-1`.
 
@@ -22,6 +24,6 @@ For additional customization you can add [tags](https://docs.datadoghq.com/getti
 Within the `loadtest-ami/` directory
 
 1) Set the AWS_DEFAULT_REGION for Packer, ex: `export AWS_DEFAULT_REGION=us-east-1`
-2) Run the command `packer build loadtest.json` 
+2) Run the command `packer build loadtest.pkr.hcl`
 
 The script that k6 runs is found within `scripts/loadtest.js`. This script can be updated to send requests to more Consul endpoints. For additional information on k6 please check out their [guides](https://k6.io/docs/getting-started/running-k6). 

test/load/packer/consul-ami/consul.json

@@ -1,55 +0,0 @@
-{
-  "min_packer_version": "1.5.4",
-  "variables": {
-    "aws_region": "{{env `AWS_DEFAULT_REGION`}}",
-    "dd_api_key": "{{env `DD_API_KEY`}}"
-  },
-  "builders": [{
-    "name": "ubuntu18-ami",
-    "ami_name": "consul-ubuntu-{{isotime | clean_resource_name}}-{{uuid}}",
-    "ami_description": "An Ubuntu 18.04 AMI that has Consul installed.",
-    "instance_type": "t2.micro",
-    "region": "{{user `aws_region`}}",
-    "associate_public_ip_address": true,
-    "type": "amazon-ebs",
-    "source_ami_filter": {
-      "filters": {
-        "virtualization-type": "hvm",
-        "architecture": "x86_64",
-        "name": "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*",
-        "block-device-mapping.volume-type": "gp2",
-        "root-device-type": "ebs"
-      },
-      "owners": ["099720109477"],
-      "most_recent": true
-    },
-    "ssh_username": "ubuntu"
-  }],
-  "provisioners": [{
-    "type": "shell",
-    "inline": ["mkdir -p /home/ubuntu/scripts"]
-  },{
-    "type": "file",
-    "source": "{{template_dir}}/scripts",
-    "destination": "/home/ubuntu",
-    "pause_before": "30s"
-  },{
-    "type": "shell",
-    "inline": [
-      "/home/ubuntu/scripts/setup-systemd-resolved"
-    ],
-    "pause_before": "30s"
-  },{
-    "type": "shell",
-    "inline": [
-      "DD_AGENT_MAJOR_VERSION=7 DD_API_KEY={{user `dd_api_key`}} bash -c \"$(curl -L https://raw.githubusercontent.com/DataDog/datadog-agent/master/cmd/agent/install_script.sh)\""
-    ]
-  },{
-    "type": "shell",
-    "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'",
-    "environment_vars": [
-      "DD_API_KEY={{user `dd_api_key`}}"
-  ],
-    "script": "{{template_dir}}/scripts/move-files.sh"
-  }]
-}

test/load/packer/consul-ami/consul.pkr.hcl

@@ -0,0 +1,69 @@
+packer {
+  required_version = ">= 1.5.4"
+}
+
+variable "aws_region" {
+  type    = string
+  default = "${env("AWS_DEFAULT_REGION")}"
+}
+
+variable "dd_api_key" {
+  type    = string
+  default = "${env("DD_API_KEY")}"
+}
+
+data "amazon-ami" "autogenerated_1" {
+  filters = {
+    architecture                       = "x86_64"
+    "block-device-mapping.volume-type" = "gp2"
+    name                               = "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"
+    root-device-type                   = "ebs"
+    virtualization-type                = "hvm"
+  }
+  most_recent = true
+  owners      = ["099720109477"]
+  region      = "${var.aws_region}"
+}
+
+source "amazon-ebs" "ubuntu18-ami" {
+  ami_description             = "An Ubuntu 18.04 AMI that has Consul installed."
+  ami_name                    = "consul-ubuntu-${ formatdate("YYYY-MM-DD", timestamp()) }T${ formatdate("hh-mm", timestamp()) }-{{uuid}}"
+  associate_public_ip_address = true
+  instance_type               = "t2.micro"
+  region                      = "${var.aws_region}"
+  source_ami                  = "${data.amazon-ami.autogenerated_1.id}"
+  ssh_username                = "ubuntu"
+  tags = {
+      Environment = "consul-load-test"
+  }
+}
+
+build {
+  sources = ["source.amazon-ebs.ubuntu18-ami"]
+
+  provisioner "shell" {
+    inline = ["mkdir -p /home/ubuntu/scripts"]
+  }
+
+  provisioner "file" {
+    destination  = "/home/ubuntu"
+    pause_before = "30s"
+    source       = "${path.root}/scripts"
+  }
+
+  provisioner "shell" {
+    inline       = ["/home/ubuntu/scripts/setup-systemd-resolved"]
+    pause_before = "30s"
+  }
+
+  provisioner "shell" {
+    inline = ["DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=${var.dd_api_key} bash -c \"$(curl -L https://raw.githubusercontent.com/DataDog/datadog-agent/master/cmd/agent/install_script.sh)\""]
+  }
+
+  provisioner "shell" {
+    environment_vars = ["DD_API_KEY=${var.dd_api_key}"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    script           = "${path.root}/scripts/move-files.sh"
+  }
+
+}

test/load/packer/loadtest-ami/loadtest.json

@@ -1,41 +0,0 @@
-{
-  "min_packer_version": "1.5.4",
-  "variables": {
-    "aws_region": "{{env `AWS_DEFAULT_REGION`}}"
-  },
-  "builders": [{
-    "name": "ubuntu18-ami",
-    "ami_name": "consul-test-{{isotime | clean_resource_name}}-{{uuid}}",
-    "ami_description": "An Ubuntu 18.04 AMI that has hey installed.",
-    "instance_type": "t2.micro",
-    "region": "{{user `aws_region`}}",
-    "associate_public_ip_address": true,
-    "type": "amazon-ebs",
-    "source_ami_filter": {
-      "filters": {
-        "virtualization-type": "hvm",
-        "architecture": "x86_64",
-        "name": "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*",
-        "block-device-mapping.volume-type": "gp2",
-        "root-device-type": "ebs"
-      },
-      "owners": ["099720109477"],
-      "most_recent": true
-    },
-    "ssh_username": "ubuntu"
-  }],
-  "provisioners": [{
-    "type": "shell",
-    "inline": ["mkdir -p /home/ubuntu/scripts/"]
-  },{
-    "type": "file",
-    "source": "{{template_dir}}/scripts",
-    "destination": "/home/ubuntu",
-    "pause_before": "30s"
-  },{
-    "type": "shell",
-    "execute_command": "sudo -S sh -c '{{ .Vars }} {{ .Path }}'",
-    "script": "./scripts/install-k6.sh"
-    }]
-  
-}

test/load/packer/loadtest-ami/loadtest.pkr.hcl

@@ -0,0 +1,51 @@
+packer {
+  required_version = ">= 1.5.4"
+}
+
+variable "aws_region" {
+  type    = string
+  default = "${env("AWS_DEFAULT_REGION")}"
+}
+
+data "amazon-ami" "autogenerated_1" {
+  filters = {
+    architecture                       = "x86_64"
+    "block-device-mapping.volume-type" = "gp2"
+    name                               = "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"
+    root-device-type                   = "ebs"
+    virtualization-type                = "hvm"
+  }
+  most_recent = true
+  owners      = ["099720109477"]
+  region      = "${var.aws_region}"
+}
+
+source "amazon-ebs" "ubuntu18-ami" {
+  ami_description             = "An Ubuntu 18.04 AMI that has hey installed."
+  ami_name                    = "consul-test-${ formatdate("YYYY-MM-DD", timestamp()) }T${ formatdate("hh-mm", timestamp()) }-{{uuid}}"
+  associate_public_ip_address = true
+  instance_type               = "t2.micro"
+  region                      = "${var.aws_region}"
+  source_ami                  = "${data.amazon-ami.autogenerated_1.id}"
+  ssh_username                = "ubuntu"
+}
+
+build {
+  sources = ["source.amazon-ebs.ubuntu18-ami"]
+
+  provisioner "shell" {
+    inline = ["mkdir -p /home/ubuntu/scripts/"]
+  }
+
+  provisioner "file" {
+    destination  = "/home/ubuntu"
+    pause_before = "30s"
+    source       = "${path.root}/scripts"
+  }
+
+  provisioner "shell" {
+    execute_command = "sudo -S sh -c '{{ .Vars }} {{ .Path }}'"
+    script          = "./scripts/install-k6.sh"
+  }
+
+}

test/load/packer/loadtest-ami/scripts/install-k6.sh

@@ -6,8 +6,8 @@ ulimit -Sn 100000
 sysctl -p
 
 # download k6
-sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61
+sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
-echo "deb https://dl.bintray.com/loadimpact/deb stable main" | sudo tee -a /etc/apt/sources.list
+echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
 sudo apt-get update
 sudo apt-get install k6
 

test/load/terraform/README.md

@@ -6,14 +6,16 @@
 to pulling from latest.
 4. Set either `consul_version` or `consul_download_url`. If neither is set it will default to utilizing Consul 1.9.0
 5. AWS Variables are set off of environment variables. Make sure to export necessary variables [shown here](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#environment-variables).
-6. Run `terraform plan -var-file=vars.tfvars`, and then `terraform apply -var-file=vars.tfvars` when ready.
+6. Run `terraform init` once to setup the working directory.
-7. Upon completion k6 should run and push metrics to the desired Datadog dashboard.
+7. Run `terraform plan -var-file=vars.tfvars`, and then `terraform apply -var-file=vars.tfvars` when ready.
+8. Upon completion k6 should run and push metrics to the desired Datadog dashboard.
 
 An example of a `vars.tfvars` :
 
 ```
 vpc_name             = "consul-test-vpc"
 vpc_cidr             = "11.0.0.0/16"
+vpc_allwed_ssh_cidr  = "0.0.0.0/0"
 public_subnet_cidrs  = ["11.0.1.0/24", "11.0.3.0/24"]
 private_subnet_cidrs = ["11.0.2.0/24"]
 vpc_az               = ["us-east-2a", "us-east-2b"]
@@ -25,7 +27,10 @@ instance_type        = "t2.micro"
 ami_owners           = ["******"]
 consul_ami_id        = "ami-016d80ff5472346f0"
 ````
- 
+
+Note that `vpc_allwed_ssh_cidr` must be set to allowed the test server to be accessible from the
+machine running the load test, e.g., "0.0.0.0/0" (It is disabled by default).
+
 ## Customization
 All customization for infrastructure that is available can be found by looking through the `variables.tf` file.
  

test/load/terraform/test-servers.tf

@@ -37,13 +37,13 @@ resource "aws_security_group" "test-servers" {
     from_port   = 22
     to_port     = 22
     protocol    = "6"
-    cidr_blocks = [var.vpc_cidr]
+    cidr_blocks = [var.vpc_allwed_ssh_cidr]
   }
   egress {
     from_port   = 0
     to_port     = 0
     protocol    = "-1"
-    cidr_blocks = [var.vpc_cidr]
+    cidr_blocks = ["0.0.0.0/0"]
   }
 }
 
@@ -54,6 +54,9 @@ resource "aws_instance" "test-server" {
   vpc_security_group_ids      = toset([aws_security_group.test-servers.id])
   associate_public_ip_address = var.test_public_ip
   subnet_id                   = (module.vpc.public_subnets)[0]
+  tags = {
+    Name = "consul-load-generator-server-${local.random_name}"
+  }
   provisioner "remote-exec" {
     inline = [
       "export LB_ENDPOINT=${module.alb.this_lb_dns_name}",

test/load/terraform/variables.tf

@@ -59,6 +59,11 @@ variable "vpc_cidr" {
   description = "List of CIDR blocks for the VPC module"
 }
 
+variable "vpc_allwed_ssh_cidr" {
+  description = "List of CIDR blocks allowed to ssh to the test server; set to 0.0.0.0/0 to allow access from anywhere"
+  default     = "10.0.0.0/16"
+}
+
 variable "public_subnet_cidrs" {
   type        = list(string)
   description = "CIDR Block for the Public Subnet, must be within VPC CIDR range"