Minor load test fixes (#9394)

test/load/README.md

@@ -0,0 +1,2 @@
+## Terraform Consul Load Testing
+Consul Load Testing is used to capture baseline performance metrics for Consul under stress. This will assist in ensuring there are no performance regressions during releases and substainal changes to Consul. Per the defaults, the test runs for 10 minutes with 25 virtual users spawned by k6. Instance sizes for test instance are `t2.small` and for the Consul cluster `m5n.large`.  As long as the thresholds stay under 2 seconds, it passes. All metrics from Consul are pushed to a datadog dashboard for user review.

test/load/packer/loadtest-ami/scripts/loadtest.js

@@ -1,8 +1,8 @@
 import http from 'k6/http';
 import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.0.0/index.js";
-let data = JSON.parse(open('service.json'));
+import { check, fail } from 'k6';
-let check = JSON.parse(open('service-check.json'));
 
+let data = JSON.parse(open('service.json'));
 
 
 export default function() {
@@ -14,49 +14,35 @@ export default function() {
   const kv_address = `${ipaddress + kv_uri + key}`
   
   //Put valid K/V
-  let res = http.put(kv_address, JSON.stringify(value));
+  let kvres = http.put(kv_address, JSON.stringify(value));
   if (
-    !check(res, {
+    !check(kvres, {
-      'kv status code MUST be 200': (res) => res.status == 200,
+      'kv status code MUST be 200': (kvres) => kvres.status == 200,
     })
   ) {
-    fail('kv status code was *not* 200');
+    fail(`registry check status code was *not* 200. error: ${kvres.error}. body: ${kvres.body}`)
   }
 
   //Register Service
-  data["ID"] = key;
   data["Name"] = key;
   const service_uri = '/v1/agent/service/register';
+
   const service_address = `${ipaddress + service_uri }`
-  let res = http.put(service_address, JSON.stringify(data))
+  let servres = http.put(service_address, JSON.stringify(data));
   if (
-    !check(res, {
+    !check(servres, {
-      'register service status code MUST be 200': (res) => res.status == 200,
+      'register service status code MUST be 200': (servres) => servres.status == 200,
     })
   ) {
-    fail('register service status code was *not* 200');
+    fail(`registry check status code was *not* 200. error: ${servres.error}. body: ${servres.body}`)
   }
-
-  //Register Check
-  check["ServiceID"] = key;
-  const check_uri = '/v1/agent/check/register';
-  const check_address = `${ipaddress + check_uri }`
-  let res = http.put(check_address, JSON.stringify(check))
-  if (
-    !check(res, {
-      'register check status code MUST be 200': (res) => res.status == 200,
-    })
-  ) {
-    fail('register check status code was *not* 200');
-  }
-
 }
 
 export let options = {
-  // 1 virtual user
+  // 25 virtual users
-  vus: 100,
+  vus: 25,
-  // 1 minute
+  // 10 minute
-  duration: "15m",
+  duration: "10m",
-  // 95% of requests must complete below 0.280s
+  // 95% of requests must complete below 2s
-  thresholds: { http_req_duration: ["p(95)<280"] },
+  thresholds: { http_req_duration: ["p(95)<2000"] },
 };

test/load/packer/loadtest-ami/scripts/service-check.json

@@ -1,12 +0,0 @@
-{
-    "ID": "api",
-    "Name": "Invalid Check so Services Will Automagically Deregister",
-    "HTTP": "https://example.com:850040220",
-    "tls_skip_verify": true,
-    "Method": "POST",
-    "Header": {"Content-Type": ["application/json"]},
-    "Body": "{\"method\":\"health\"}",
-    "Interval": "10s",
-    "Timeout": "5s",
-    "DeregisterCriticalServiceAfter": "2m"
-}

test/load/packer/loadtest-ami/scripts/service.json

@@ -1,8 +1,16 @@
 {
-    "Tags": ["primary", "v1"],
+  "Address": "127.0.0.1",
-    "Address": "127.0.0.1",
+  "Port": 8000,
-    "Port": 8000,
+  "Meta": {
-    "Meta": {
+    "redis_version": "4.0"
-      "redis_version": "4.0"
+  },
-    }
+  "Check": {
-  }
+    "HTTP": "https://example.com:850040220",
+    "Method": "POST",
+    "Header": {"Content-Type": ["application/json"]},
+    "Body": "{\"method\":\"health\"}",
+    "Timeout": "5s",
+    "DeregisterCriticalServiceAfter": "2m",
+    "Interval": "10s"
+  }
+}

test/load/terraform/README.md

@@ -30,7 +30,7 @@ If `consul_version` or `consul_download_url` is not set within the Terraform var
 6. Upon completion k6 should run and push metrics to desired Datadog dashboard. 
 
 # Customization 
-All customization for infrastructure that is available can be found by looking through the `variables.tf` file. However, if customization of tests is desired then the `start-k6.sh` leverages user-data to place a `puts_script.js` onto the loadtest servers for k6 to run. This can be customized.
+All customization for infrastructure that is available can be found by looking through the `variables.tf` file.
 
 # How to SSH
 After `terraform apply` is ran Terraform should create a `keys/` directory which will give access to all instances created. 

test/load/terraform/consul.tf

@@ -113,6 +113,13 @@ module "alb" {
       backend_protocol = "HTTP"
       backend_port     = 8500
       target_type      = "instance"
+      health_check = {
+        interval          = 5
+        timeout           = 3
+        protocol          = "HTTP"
+        healthy_threshold = 2
+        path              = "/v1/status/leader"
+      }
     }
   ]
 

test/load/terraform/test-servers.tf

@@ -57,7 +57,7 @@ resource "aws_instance" "test-server" {
   provisioner "remote-exec" {
     inline = [
       "export LB_ENDPOINT=${module.alb.this_lb_dns_name}",
-      "k6 run /home/ubuntu/scripts/loadtest.js"
+      "k6 run -q /home/ubuntu/scripts/loadtest.js"
     ]
     connection {
       type        = "ssh"

test/load/terraform/variables.tf

@@ -96,6 +96,7 @@ variable "test_server_ami" {
 variable "test_instance_type" {
   type        = string
   description = "AWS Instance type for all test servers"
+  default     = "t2.small"
 }
 
 variable "test_public_ip" {
@@ -106,6 +107,7 @@ variable "test_public_ip" {
 variable "instance_type" {
   type        = string
   description = "Instance Type for all instances in the Consul Cluster"
+  default     = "m5n.large"
 }
 
 variable "ami_owners" {