Adding an Enos test for undo logs (#17675)

* Adding an Enos test for undo logs

* fixing a typo

* feedback

* fixing typo

* running make fmt

* removing a dependency

* var name change

* fixing a variable

* fix builder

* fix product version

* adding required fields

* feedback

* add artifcat bundle back

* fmt check

* point to correct instance

* minor fix

* feedback

* feedback

enos/enos-modules.hcl

@@ -99,6 +99,13 @@ module "vault_verify_raft_auto_join_voter" {
   vault_instance_count = var.vault_instance_count
 }
 
+module "vault_verify_undo_logs" {
+  source = "./modules/vault_verify_undo_logs"
+
+  vault_install_dir    = var.vault_install_dir
+  vault_instance_count = var.vault_instance_count
+}
+
 module "vault_verify_replication" {
   source = "./modules/vault-verify-replication"
 

enos/enos-scenario-autopilot.hcl

@@ -1,11 +1,12 @@
 scenario "autopilot" {
   matrix {
-    arch            = ["amd64", "arm64"]
-    artifact_source = ["local", "crt", "artifactory"]
-    artifact_type   = ["bundle", "package"]
-    distro          = ["ubuntu", "rhel"]
-    edition         = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
-    seal            = ["awskms", "shamir"]
+    arch             = ["amd64", "arm64"]
+    artifact_source  = ["local", "crt", "artifactory"]
+    artifact_type    = ["bundle", "package"]
+    distro           = ["ubuntu", "rhel"]
+    edition          = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
+    seal             = ["awskms", "shamir"]
+    undo_logs_status = ["0", "1"]
   }
 
   terraform_cli = terraform_cli.default
@@ -38,6 +39,9 @@ scenario "autopilot" {
       amd64 = "t3a.small"
       arm64 = "t4g.small"
     }
+
+    enable_undo_logs = matrix.undo_logs_status == "1" && semverconstraint(var.vault_product_version, ">=1.12.0-0") ? true : false
+
     vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch])
     vault_license_path  = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic"))
     vault_install_dir_packages = {
@@ -175,6 +179,7 @@ scenario "autopilot" {
       vault_unseal_when_no_init   = matrix.seal == "shamir"
       vault_unseal_keys           = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null
       vpc_id                      = step.create_vpc.vpc_id
+      vault_environment           = { "VAULT_REPLICATION_USE_UNDO_LOGS" : local.enable_undo_logs }
     }
   }
 
@@ -230,6 +235,25 @@ scenario "autopilot" {
     }
   }
 
+  step "verify_undo_logs_status" {
+    skip_step = semverconstraint(var.vault_product_version, "<1.12.0-0")
+    module    = module.vault_verify_undo_logs
+    depends_on = [
+      step.upgrade_vault_cluster_with_autopilot,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
+      vault_undo_logs_status          = matrix.undo_logs_status
+      vault_instances                 = step.upgrade_vault_cluster_with_autopilot.vault_instances
+      vault_root_token                = step.create_vault_cluster.vault_root_token
+    }
+  }
+
   output "vault_cluster_instance_ids" {
     description = "The Vault cluster instance IDs"
     value       = step.create_vault_cluster.instance_ids

enos/modules/vault_verify_undo_logs/main.tf

@@ -0,0 +1,70 @@
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+variable "vault_install_dir" {
+  type        = string
+  description = "The directory where the Vault binary will be installed"
+}
+
+variable "vault_instance_count" {
+  type        = number
+  description = "How many vault instances are in the cluster"
+}
+
+variable "vault_instances" {
+  type = map(object({
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The vault cluster instances that were created"
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+
+variable "vault_autopilot_upgrade_version" {
+  type        = string
+  description = "The vault version to which autopilot upgraded Vault"
+  default     = null
+}
+
+variable "vault_undo_logs_status" {
+  type        = string
+  description = "An integer either 0 or 1 which indicates whether undo_logs are disabled or enabled"
+  default     = null
+}
+
+locals {
+  public_ips = {
+    for idx in range(var.vault_instance_count) : idx => {
+      public_ip  = values(var.vault_instances)[idx].public_ip
+      private_ip = values(var.vault_instances)[idx].private_ip
+    }
+  }
+}
+
+resource "enos_remote_exec" "smoke-verify-undo-logs" {
+  for_each = local.public_ips
+
+  environment = {
+    VAULT_TOKEN                     = var.vault_root_token
+    VAULT_ADDR                      = "http://localhost:8200"
+    vault_undo_logs_status          = var.vault_undo_logs_status
+    vault_autopilot_upgrade_version = var.vault_autopilot_upgrade_version
+  }
+
+  scripts = [abspath("${path.module}/scripts/smoke-verify-undo-logs.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}

enos/modules/vault_verify_undo_logs/scripts/smoke-verify-undo-logs.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+undo_logs_status="${vault_undo_logs_status}"
+
+function fail() {
+	echo "$1" 1>&2
+	exit 1
+}
+
+count=0
+retries=7
+while :; do
+    state=$(curl --header "X-Vault-Token: $VAULT_TOKEN" "$VAULT_ADDR/v1/sys/metrics"  | jq -r '.Gauges[] | select(.Name == "vault.core.replication.write_undo_logs")')
+    target_undo_logs_status="$(jq -r '.Value' <<< "$state")"
+
+    if [ "$undo_logs_status" = "$target_undo_logs_status" ]; then
+        exit 0
+    fi
+
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ "$count" -lt "$retries" ]; then
+        echo "$state"
+        sleep "$wait"
+    else
+        fail "Undo_logs did not get into the correct status"
+    fi
+done