From 340a19489407cdd17cf5112b1f8a9a69de63c1eb Mon Sep 17 00:00:00 2001 From: cskh Date: Wed, 15 Jun 2022 09:29:38 -0400 Subject: [PATCH] Load test, upgrade packer version, fix k6s installation (#13382) - fix sg: need remote access to test server - Give the load generator a name - Update loadtest hcl filename in readme - Add terraform init - Disable access to the server machine by default --- .circleci/terraform/load-test/main.tf | 7 +- test/load/packer/README.md | 6 +- test/load/packer/consul-ami/consul.json | 55 --------------- test/load/packer/consul-ami/consul.pkr.hcl | 69 +++++++++++++++++++ test/load/packer/loadtest-ami/loadtest.json | 41 ----------- .../load/packer/loadtest-ami/loadtest.pkr.hcl | 51 ++++++++++++++ .../packer/loadtest-ami/scripts/install-k6.sh | 4 +- test/load/terraform/README.md | 11 ++- test/load/terraform/test-servers.tf | 7 +- test/load/terraform/variables.tf | 5 ++ 10 files changed, 145 insertions(+), 111 deletions(-) delete mode 100644 test/load/packer/consul-ami/consul.json create mode 100644 test/load/packer/consul-ami/consul.pkr.hcl delete mode 100644 test/load/packer/loadtest-ami/loadtest.json create mode 100644 test/load/packer/loadtest-ami/loadtest.pkr.hcl diff --git a/.circleci/terraform/load-test/main.tf b/.circleci/terraform/load-test/main.tf index b46645ef2..1a8865c06 100644 --- a/.circleci/terraform/load-test/main.tf +++ b/.circleci/terraform/load-test/main.tf @@ -7,12 +7,6 @@ provider "aws" { assume_role { role_arn = var.role_arn } - - default_tags { - tags = { - Environment = "ConsulLoadTest" - } - } } module "load-test" { @@ -21,6 +15,7 @@ module "load-test" { vpc_az = ["us-east-2a", "us-east-2b"] vpc_name = var.vpc_name vpc_cidr = "10.0.0.0/16" + vpc_allwed_ssh_cidr = "0.0.0.0/0" public_subnet_cidrs = ["10.0.1.0/24", "10.0.2.0/24"] private_subnet_cidrs = ["10.0.3.0/24"] test_public_ip = true diff --git a/test/load/packer/README.md b/test/load/packer/README.md index f615694ca..42ce0e749 100644 --- a/test/load/packer/README.md +++ b/test/load/packer/README.md @@ -1,5 +1,6 @@ # Terraform Consul Load Testing Packer will output AMI IDs when it completes - save these AMI IDs as Terraform will require them later. +The generated ami will have common tag `Environment = "consul-load-test"`. ``` ==> Builds finished. The artifacts of successful builds are: @@ -12,7 +13,8 @@ us-east-1: ami-19601070 Within the `consul-ami/` directory 1) Retrieve your [Datadog API key]((https://docs.datadoghq.com/account_management/api-app-keys/#api-keys)), set this as an environment variable, ex: `export DD_API_KEY=$YOURDDAPIKEYHERE` 2) Set the AWS_DEFAULT_REGION for Packer, ex: `export AWS_DEFAULT_REGION=us-east-1` -3) Run `packer build consul.json`. +3) Set up AWS credentials, ex: `export AWS_ACCESS_KEY_ID=#YOURAWSKEYID & export AWS_SECRET_ACCESS_KEY=YOURAWSKEY` +4) Run `packer build consul.pkr.hcl`. For additional customization you can add [tags](https://docs.datadoghq.com/getting_started/tagging/assigning_tags/?tab=noncontainerizedenvironments) within the `scripts/datadog.yaml` file. An example of a tag could be `"consul_version" : "consulent_175"`. These tags are searchable through the datadog dashboard. Another form of customization is changing the datacenter tag within `scripts/telemetry.json`, however it is defaulted to `us-east-1`. @@ -22,6 +24,6 @@ For additional customization you can add [tags](https://docs.datadoghq.com/getti Within the `loadtest-ami/` directory 1) Set the AWS_DEFAULT_REGION for Packer, ex: `export AWS_DEFAULT_REGION=us-east-1` -2) Run the command `packer build loadtest.json` +2) Run the command `packer build loadtest.pkr.hcl` The script that k6 runs is found within `scripts/loadtest.js`. This script can be updated to send requests to more Consul endpoints. For additional information on k6 please check out their [guides](https://k6.io/docs/getting-started/running-k6). diff --git a/test/load/packer/consul-ami/consul.json b/test/load/packer/consul-ami/consul.json deleted file mode 100644 index 93a5a5986..000000000 --- a/test/load/packer/consul-ami/consul.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "min_packer_version": "1.5.4", - "variables": { - "aws_region": "{{env `AWS_DEFAULT_REGION`}}", - "dd_api_key": "{{env `DD_API_KEY`}}" - }, - "builders": [{ - "name": "ubuntu18-ami", - "ami_name": "consul-ubuntu-{{isotime | clean_resource_name}}-{{uuid}}", - "ami_description": "An Ubuntu 18.04 AMI that has Consul installed.", - "instance_type": "t2.micro", - "region": "{{user `aws_region`}}", - "associate_public_ip_address": true, - "type": "amazon-ebs", - "source_ami_filter": { - "filters": { - "virtualization-type": "hvm", - "architecture": "x86_64", - "name": "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*", - "block-device-mapping.volume-type": "gp2", - "root-device-type": "ebs" - }, - "owners": ["099720109477"], - "most_recent": true - }, - "ssh_username": "ubuntu" - }], - "provisioners": [{ - "type": "shell", - "inline": ["mkdir -p /home/ubuntu/scripts"] - },{ - "type": "file", - "source": "{{template_dir}}/scripts", - "destination": "/home/ubuntu", - "pause_before": "30s" - },{ - "type": "shell", - "inline": [ - "/home/ubuntu/scripts/setup-systemd-resolved" - ], - "pause_before": "30s" - },{ - "type": "shell", - "inline": [ - "DD_AGENT_MAJOR_VERSION=7 DD_API_KEY={{user `dd_api_key`}} bash -c \"$(curl -L https://raw.githubusercontent.com/DataDog/datadog-agent/master/cmd/agent/install_script.sh)\"" - ] - },{ - "type": "shell", - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'", - "environment_vars": [ - "DD_API_KEY={{user `dd_api_key`}}" - ], - "script": "{{template_dir}}/scripts/move-files.sh" - }] -} diff --git a/test/load/packer/consul-ami/consul.pkr.hcl b/test/load/packer/consul-ami/consul.pkr.hcl new file mode 100644 index 000000000..cb136e73c --- /dev/null +++ b/test/load/packer/consul-ami/consul.pkr.hcl @@ -0,0 +1,69 @@ +packer { + required_version = ">= 1.5.4" +} + +variable "aws_region" { + type = string + default = "${env("AWS_DEFAULT_REGION")}" +} + +variable "dd_api_key" { + type = string + default = "${env("DD_API_KEY")}" +} + +data "amazon-ami" "autogenerated_1" { + filters = { + architecture = "x86_64" + "block-device-mapping.volume-type" = "gp2" + name = "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] + region = "${var.aws_region}" +} + +source "amazon-ebs" "ubuntu18-ami" { + ami_description = "An Ubuntu 18.04 AMI that has Consul installed." + ami_name = "consul-ubuntu-${ formatdate("YYYY-MM-DD", timestamp()) }T${ formatdate("hh-mm", timestamp()) }-{{uuid}}" + associate_public_ip_address = true + instance_type = "t2.micro" + region = "${var.aws_region}" + source_ami = "${data.amazon-ami.autogenerated_1.id}" + ssh_username = "ubuntu" + tags = { + Environment = "consul-load-test" + } +} + +build { + sources = ["source.amazon-ebs.ubuntu18-ami"] + + provisioner "shell" { + inline = ["mkdir -p /home/ubuntu/scripts"] + } + + provisioner "file" { + destination = "/home/ubuntu" + pause_before = "30s" + source = "${path.root}/scripts" + } + + provisioner "shell" { + inline = ["/home/ubuntu/scripts/setup-systemd-resolved"] + pause_before = "30s" + } + + provisioner "shell" { + inline = ["DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=${var.dd_api_key} bash -c \"$(curl -L https://raw.githubusercontent.com/DataDog/datadog-agent/master/cmd/agent/install_script.sh)\""] + } + + provisioner "shell" { + environment_vars = ["DD_API_KEY=${var.dd_api_key}"] + execute_command = "sudo sh -c '{{ .Vars }} {{ .Path }}'" + script = "${path.root}/scripts/move-files.sh" + } + +} diff --git a/test/load/packer/loadtest-ami/loadtest.json b/test/load/packer/loadtest-ami/loadtest.json deleted file mode 100644 index 7a41d26d2..000000000 --- a/test/load/packer/loadtest-ami/loadtest.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "min_packer_version": "1.5.4", - "variables": { - "aws_region": "{{env `AWS_DEFAULT_REGION`}}" - }, - "builders": [{ - "name": "ubuntu18-ami", - "ami_name": "consul-test-{{isotime | clean_resource_name}}-{{uuid}}", - "ami_description": "An Ubuntu 18.04 AMI that has hey installed.", - "instance_type": "t2.micro", - "region": "{{user `aws_region`}}", - "associate_public_ip_address": true, - "type": "amazon-ebs", - "source_ami_filter": { - "filters": { - "virtualization-type": "hvm", - "architecture": "x86_64", - "name": "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*", - "block-device-mapping.volume-type": "gp2", - "root-device-type": "ebs" - }, - "owners": ["099720109477"], - "most_recent": true - }, - "ssh_username": "ubuntu" - }], - "provisioners": [{ - "type": "shell", - "inline": ["mkdir -p /home/ubuntu/scripts/"] - },{ - "type": "file", - "source": "{{template_dir}}/scripts", - "destination": "/home/ubuntu", - "pause_before": "30s" - },{ - "type": "shell", - "execute_command": "sudo -S sh -c '{{ .Vars }} {{ .Path }}'", - "script": "./scripts/install-k6.sh" - }] - -} diff --git a/test/load/packer/loadtest-ami/loadtest.pkr.hcl b/test/load/packer/loadtest-ami/loadtest.pkr.hcl new file mode 100644 index 000000000..03987a7cf --- /dev/null +++ b/test/load/packer/loadtest-ami/loadtest.pkr.hcl @@ -0,0 +1,51 @@ +packer { + required_version = ">= 1.5.4" +} + +variable "aws_region" { + type = string + default = "${env("AWS_DEFAULT_REGION")}" +} + +data "amazon-ami" "autogenerated_1" { + filters = { + architecture = "x86_64" + "block-device-mapping.volume-type" = "gp2" + name = "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] + region = "${var.aws_region}" +} + +source "amazon-ebs" "ubuntu18-ami" { + ami_description = "An Ubuntu 18.04 AMI that has hey installed." + ami_name = "consul-test-${ formatdate("YYYY-MM-DD", timestamp()) }T${ formatdate("hh-mm", timestamp()) }-{{uuid}}" + associate_public_ip_address = true + instance_type = "t2.micro" + region = "${var.aws_region}" + source_ami = "${data.amazon-ami.autogenerated_1.id}" + ssh_username = "ubuntu" +} + +build { + sources = ["source.amazon-ebs.ubuntu18-ami"] + + provisioner "shell" { + inline = ["mkdir -p /home/ubuntu/scripts/"] + } + + provisioner "file" { + destination = "/home/ubuntu" + pause_before = "30s" + source = "${path.root}/scripts" + } + + provisioner "shell" { + execute_command = "sudo -S sh -c '{{ .Vars }} {{ .Path }}'" + script = "./scripts/install-k6.sh" + } + +} diff --git a/test/load/packer/loadtest-ami/scripts/install-k6.sh b/test/load/packer/loadtest-ami/scripts/install-k6.sh index ad3047671..2163f2de4 100644 --- a/test/load/packer/loadtest-ami/scripts/install-k6.sh +++ b/test/load/packer/loadtest-ami/scripts/install-k6.sh @@ -6,8 +6,8 @@ ulimit -Sn 100000 sysctl -p # download k6 -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 -echo "deb https://dl.bintray.com/loadimpact/deb stable main" | sudo tee -a /etc/apt/sources.list +sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 +echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list sudo apt-get update sudo apt-get install k6 diff --git a/test/load/terraform/README.md b/test/load/terraform/README.md index 52b7bdf7c..b6b41d925 100644 --- a/test/load/terraform/README.md +++ b/test/load/terraform/README.md @@ -6,14 +6,16 @@ to pulling from latest. 4. Set either `consul_version` or `consul_download_url`. If neither is set it will default to utilizing Consul 1.9.0 5. AWS Variables are set off of environment variables. Make sure to export necessary variables [shown here](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#environment-variables). -6. Run `terraform plan -var-file=vars.tfvars`, and then `terraform apply -var-file=vars.tfvars` when ready. -7. Upon completion k6 should run and push metrics to the desired Datadog dashboard. +6. Run `terraform init` once to setup the working directory. +7. Run `terraform plan -var-file=vars.tfvars`, and then `terraform apply -var-file=vars.tfvars` when ready. +8. Upon completion k6 should run and push metrics to the desired Datadog dashboard. An example of a `vars.tfvars` : ``` vpc_name = "consul-test-vpc" vpc_cidr = "11.0.0.0/16" +vpc_allwed_ssh_cidr = "0.0.0.0/0" public_subnet_cidrs = ["11.0.1.0/24", "11.0.3.0/24"] private_subnet_cidrs = ["11.0.2.0/24"] vpc_az = ["us-east-2a", "us-east-2b"] @@ -25,7 +27,10 @@ instance_type = "t2.micro" ami_owners = ["******"] consul_ami_id = "ami-016d80ff5472346f0" ```` - + +Note that `vpc_allwed_ssh_cidr` must be set to allowed the test server to be accessible from the +machine running the load test, e.g., "0.0.0.0/0" (It is disabled by default). + ## Customization All customization for infrastructure that is available can be found by looking through the `variables.tf` file. diff --git a/test/load/terraform/test-servers.tf b/test/load/terraform/test-servers.tf index ff40fcabc..157843751 100644 --- a/test/load/terraform/test-servers.tf +++ b/test/load/terraform/test-servers.tf @@ -37,13 +37,13 @@ resource "aws_security_group" "test-servers" { from_port = 22 to_port = 22 protocol = "6" - cidr_blocks = [var.vpc_cidr] + cidr_blocks = [var.vpc_allwed_ssh_cidr] } egress { from_port = 0 to_port = 0 protocol = "-1" - cidr_blocks = [var.vpc_cidr] + cidr_blocks = ["0.0.0.0/0"] } } @@ -54,6 +54,9 @@ resource "aws_instance" "test-server" { vpc_security_group_ids = toset([aws_security_group.test-servers.id]) associate_public_ip_address = var.test_public_ip subnet_id = (module.vpc.public_subnets)[0] + tags = { + Name = "consul-load-generator-server-${local.random_name}" + } provisioner "remote-exec" { inline = [ "export LB_ENDPOINT=${module.alb.this_lb_dns_name}", diff --git a/test/load/terraform/variables.tf b/test/load/terraform/variables.tf index c7b1f71ea..2cec534b8 100644 --- a/test/load/terraform/variables.tf +++ b/test/load/terraform/variables.tf @@ -59,6 +59,11 @@ variable "vpc_cidr" { description = "List of CIDR blocks for the VPC module" } +variable "vpc_allwed_ssh_cidr" { + description = "List of CIDR blocks allowed to ssh to the test server; set to 0.0.0.0/0 to allow access from anywhere" + default = "10.0.0.0/16" +} + variable "public_subnet_cidrs" { type = list(string) description = "CIDR Block for the Public Subnet, must be within VPC CIDR range"