diff --git a/test/load/README.md b/test/load/README.md new file mode 100644 index 000000000..d4aef3b23 --- /dev/null +++ b/test/load/README.md @@ -0,0 +1,2 @@ +## Terraform Consul Load Testing +Consul Load Testing is used to capture baseline performance metrics for Consul under stress. This will assist in ensuring there are no performance regressions during releases and substainal changes to Consul. Per the defaults, the test runs for 10 minutes with 25 virtual users spawned by k6. Instance sizes for test instance are `t2.small` and for the Consul cluster `m5n.large`. As long as the thresholds stay under 2 seconds, it passes. All metrics from Consul are pushed to a datadog dashboard for user review. diff --git a/test/load/packer/loadtest-ami/scripts/loadtest.js b/test/load/packer/loadtest-ami/scripts/loadtest.js index f39335bb4..556c9ef4b 100644 --- a/test/load/packer/loadtest-ami/scripts/loadtest.js +++ b/test/load/packer/loadtest-ami/scripts/loadtest.js @@ -1,8 +1,8 @@ import http from 'k6/http'; import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.0.0/index.js"; -let data = JSON.parse(open('service.json')); -let check = JSON.parse(open('service-check.json')); +import { check, fail } from 'k6'; +let data = JSON.parse(open('service.json')); export default function() { @@ -14,49 +14,35 @@ export default function() { const kv_address = `${ipaddress + kv_uri + key}` //Put valid K/V - let res = http.put(kv_address, JSON.stringify(value)); + let kvres = http.put(kv_address, JSON.stringify(value)); if ( - !check(res, { - 'kv status code MUST be 200': (res) => res.status == 200, + !check(kvres, { + 'kv status code MUST be 200': (kvres) => kvres.status == 200, }) ) { - fail('kv status code was *not* 200'); + fail(`registry check status code was *not* 200. error: ${kvres.error}. body: ${kvres.body}`) } //Register Service - data["ID"] = key; data["Name"] = key; const service_uri = '/v1/agent/service/register'; + const service_address = `${ipaddress + service_uri }` - let res = http.put(service_address, JSON.stringify(data)) + let servres = http.put(service_address, JSON.stringify(data)); if ( - !check(res, { - 'register service status code MUST be 200': (res) => res.status == 200, + !check(servres, { + 'register service status code MUST be 200': (servres) => servres.status == 200, }) ) { - fail('register service status code was *not* 200'); + fail(`registry check status code was *not* 200. error: ${servres.error}. body: ${servres.body}`) } - - //Register Check - check["ServiceID"] = key; - const check_uri = '/v1/agent/check/register'; - const check_address = `${ipaddress + check_uri }` - let res = http.put(check_address, JSON.stringify(check)) - if ( - !check(res, { - 'register check status code MUST be 200': (res) => res.status == 200, - }) - ) { - fail('register check status code was *not* 200'); - } - } export let options = { - // 1 virtual user - vus: 100, - // 1 minute - duration: "15m", - // 95% of requests must complete below 0.280s - thresholds: { http_req_duration: ["p(95)<280"] }, + // 25 virtual users + vus: 25, + // 10 minute + duration: "10m", + // 95% of requests must complete below 2s + thresholds: { http_req_duration: ["p(95)<2000"] }, }; diff --git a/test/load/packer/loadtest-ami/scripts/service-check.json b/test/load/packer/loadtest-ami/scripts/service-check.json deleted file mode 100644 index 9084054cd..000000000 --- a/test/load/packer/loadtest-ami/scripts/service-check.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "ID": "api", - "Name": "Invalid Check so Services Will Automagically Deregister", - "HTTP": "https://example.com:850040220", - "tls_skip_verify": true, - "Method": "POST", - "Header": {"Content-Type": ["application/json"]}, - "Body": "{\"method\":\"health\"}", - "Interval": "10s", - "Timeout": "5s", - "DeregisterCriticalServiceAfter": "2m" -} diff --git a/test/load/packer/loadtest-ami/scripts/service.json b/test/load/packer/loadtest-ami/scripts/service.json index 804de7116..d1a3301c4 100644 --- a/test/load/packer/loadtest-ami/scripts/service.json +++ b/test/load/packer/loadtest-ami/scripts/service.json @@ -1,8 +1,16 @@ { - "Tags": ["primary", "v1"], - "Address": "127.0.0.1", - "Port": 8000, - "Meta": { - "redis_version": "4.0" - } - } \ No newline at end of file + "Address": "127.0.0.1", + "Port": 8000, + "Meta": { + "redis_version": "4.0" + }, + "Check": { + "HTTP": "https://example.com:850040220", + "Method": "POST", + "Header": {"Content-Type": ["application/json"]}, + "Body": "{\"method\":\"health\"}", + "Timeout": "5s", + "DeregisterCriticalServiceAfter": "2m", + "Interval": "10s" + } +} \ No newline at end of file diff --git a/test/load/terraform/README.md b/test/load/terraform/README.md index 09f4544d8..a02c94807 100644 --- a/test/load/terraform/README.md +++ b/test/load/terraform/README.md @@ -30,7 +30,7 @@ If `consul_version` or `consul_download_url` is not set within the Terraform var 6. Upon completion k6 should run and push metrics to desired Datadog dashboard. # Customization -All customization for infrastructure that is available can be found by looking through the `variables.tf` file. However, if customization of tests is desired then the `start-k6.sh` leverages user-data to place a `puts_script.js` onto the loadtest servers for k6 to run. This can be customized. +All customization for infrastructure that is available can be found by looking through the `variables.tf` file. # How to SSH After `terraform apply` is ran Terraform should create a `keys/` directory which will give access to all instances created. diff --git a/test/load/terraform/consul.tf b/test/load/terraform/consul.tf index 871d4d45d..f4bf9a096 100644 --- a/test/load/terraform/consul.tf +++ b/test/load/terraform/consul.tf @@ -113,6 +113,13 @@ module "alb" { backend_protocol = "HTTP" backend_port = 8500 target_type = "instance" + health_check = { + interval = 5 + timeout = 3 + protocol = "HTTP" + healthy_threshold = 2 + path = "/v1/status/leader" + } } ] diff --git a/test/load/terraform/test-servers.tf b/test/load/terraform/test-servers.tf index a0feab8db..5db03d14a 100644 --- a/test/load/terraform/test-servers.tf +++ b/test/load/terraform/test-servers.tf @@ -57,7 +57,7 @@ resource "aws_instance" "test-server" { provisioner "remote-exec" { inline = [ "export LB_ENDPOINT=${module.alb.this_lb_dns_name}", - "k6 run /home/ubuntu/scripts/loadtest.js" + "k6 run -q /home/ubuntu/scripts/loadtest.js" ] connection { type = "ssh" diff --git a/test/load/terraform/variables.tf b/test/load/terraform/variables.tf index 6e48f4ea3..cb00c2c44 100644 --- a/test/load/terraform/variables.tf +++ b/test/load/terraform/variables.tf @@ -96,6 +96,7 @@ variable "test_server_ami" { variable "test_instance_type" { type = string description = "AWS Instance type for all test servers" + default = "t2.small" } variable "test_public_ip" { @@ -106,6 +107,7 @@ variable "test_public_ip" { variable "instance_type" { type = string description = "Instance Type for all instances in the Consul Cluster" + default = "m5n.large" } variable "ami_owners" {