backport of commit 30a84354990c6ad945629317637e31e340355893 (#23926)

Co-authored-by: Ryan Cragun <me@ryan.ec>

enos/enos-modules.hcl

@@ -279,3 +279,10 @@ module "vault_wait_for_seal_rewrap" {
   vault_install_dir    = var.vault_install_dir
   vault_instance_count = var.vault_instance_count
 }
+
+module "verify_seal_type" {
+  source = "./modules/verify_seal_type"
+
+  vault_install_dir    = var.vault_install_dir
+  vault_instance_count = var.vault_instance_count
+}

enos/enos-scenario-seal-ha.hcl

@@ -407,6 +407,7 @@ scenario "seal_ha" {
     }
   }
 
+  // Perform all of our standard verifications after we've enabled multiseal
   step "verify_vault_version" {
     module     = module.vault_verify_version
     depends_on = [step.wait_for_seal_rewrap]
@@ -457,6 +458,7 @@ scenario "seal_ha" {
     }
   }
 
+  // Make sure our data is still available
   step "verify_read_test_data" {
     module     = module.vault_verify_read_data
     depends_on = [step.wait_for_seal_rewrap]
@@ -484,6 +486,166 @@ scenario "seal_ha" {
     }
   }
 
+  // Make sure we have a "multiseal" seal type
+  step "verify_seal_type" {
+    // Don't run this on versions less than 1.16.0-beta1 until VAULT-21053 is fixed on prior branches.
+    skip_step  = semverconstraint(var.vault_product_version, "< 1.16.0-beta1")
+    module     = module.verify_seal_type
+    depends_on = [step.wait_for_seal_rewrap]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      seal_type         = "multiseal"
+    }
+  }
+
+  // Now we'll migrate away from our initial seal to our secondary seal
+
+  // Stop the vault service on all nodes before we restart with new seal config
+  step "stop_vault_for_migration" {
+    module = module.stop_vault
+    depends_on = [
+      step.wait_for_seal_rewrap,
+      step.verify_read_test_data,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      target_hosts = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  // Remove the "primary" seal from the cluster. Set our "secondary" seal to priority 1. We do this
+  // by restarting vault with the correct config.
+  step "remove_primary_seal" {
+    module     = module.start_vault
+    depends_on = [step.stop_vault_for_migration]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      cluster_name    = step.create_vault_cluster_targets.cluster_name
+      install_dir     = local.vault_install_dir
+      license         = matrix.edition != "ce" ? step.read_vault_license.license : null
+      manage_service  = local.manage_service
+      seal_alias      = "secondary"
+      seal_type       = matrix.secondary_seal
+      seal_key_name   = step.create_secondary_seal_key.resource_name
+      storage_backend = matrix.backend
+      target_hosts    = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  // Wait for our cluster to elect a leader after restarting vault with a new primary seal
+  step "wait_for_leader_after_migration" {
+    module     = module.vault_wait_for_leader
+    depends_on = [step.remove_primary_seal]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      timeout           = 120 # seconds
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Since we've restarted our cluster we might have a new leader and followers. Get the new IPs.
+  step "get_cluster_ips_after_migration" {
+    module     = module.vault_get_cluster_ips
+    depends_on = [step.wait_for_leader_after_migration]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Make sure we unsealed
+  step "verify_vault_unsealed_after_migration" {
+    module     = module.vault_verify_unsealed
+    depends_on = [step.wait_for_leader_after_migration]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      vault_instances   = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  // Wait for the seal rewrap to complete and verify that no entries failed
+  step "wait_for_seal_rewrap_after_migration" {
+    module = module.vault_wait_for_seal_rewrap
+    depends_on = [
+      step.wait_for_leader_after_migration,
+      step.verify_vault_unsealed_after_migration,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Make sure our data is still available after migration
+  step "verify_read_test_data_after_migration" {
+    module     = module.vault_verify_read_data
+    depends_on = [step.wait_for_seal_rewrap_after_migration]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      node_public_ips   = step.get_cluster_ips_after_migration.follower_public_ips
+      vault_install_dir = local.vault_install_dir
+    }
+  }
+
+  // Make sure we have our secondary seal type after migration
+  step "verify_seal_type_after_migration" {
+    // Don't run this on versions less than 1.16.0-beta1 until VAULT-21053 is fixed on prior branches.
+    skip_step  = semverconstraint(var.vault_product_version, "<= 1.16.0-beta1")
+    module     = module.verify_seal_type
+    depends_on = [step.wait_for_seal_rewrap_after_migration]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      seal_type         = matrix.secondary_seal
+    }
+  }
+
   output "audit_device_file_path" {
     description = "The file path for the file audit device, if enabled"
     value       = step.create_vault_cluster.audit_device_file_path

enos/modules/start_vault/main.tf

@@ -51,7 +51,8 @@ locals {
     "awskms" = {
       type = "awskms"
       attributes = {
-        name       = "primary"
+        name       = var.seal_alias
+        priority   = var.seal_priority
         kms_key_id = var.seal_key_name
       }
     }
@@ -65,7 +66,8 @@ locals {
     "awskms" = {
       type = "awskms"
       attributes = {
-        name       = "secondary"
+        name       = var.seal_alias_secondary
+        priority   = var.seal_priority_secondary
         kms_key_id = var.seal_key_name_secondary
       }
     }

enos/modules/start_vault/variables.tf

@@ -53,9 +53,21 @@ variable "seal_ha_beta" {
   default     = true
 }
 
+variable "seal_alias" {
+  type        = string
+  description = "The primary seal alias name"
+  default     = "primary"
+}
+
+variable "seal_alias_secondary" {
+  type        = string
+  description = "The secondary seal alias name"
+  default     = "secondary"
+}
+
 variable "seal_key_name" {
   type        = string
-  description = "The auto-unseal key name"
+  description = "The primary auto-unseal key name"
   default     = null
 }
 
@@ -65,6 +77,18 @@ variable "seal_key_name_secondary" {
   default     = null
 }
 
+variable "seal_priority" {
+  type        = string
+  description = "The primary seal priority"
+  default     = "1"
+}
+
+variable "seal_priority_secondary" {
+  type        = string
+  description = "The secondary seal priority"
+  default     = "2"
+}
+
 variable "seal_type" {
   type        = string
   description = "The method by which to unseal the Vault cluster"

enos/modules/verify_seal_type/main.tf

@@ -0,0 +1,52 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+variable "vault_install_dir" {
+  type        = string
+  description = "The directory where the Vault binary will be installed"
+}
+
+variable "vault_instance_count" {
+  type        = number
+  description = "How many vault instances are in the cluster"
+}
+
+variable "vault_hosts" {
+  type = map(object({
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The vault cluster instances that were created"
+}
+
+variable "seal_type" {
+  type        = string
+  description = "The expected seal type"
+  default     = "shamir"
+}
+
+resource "enos_remote_exec" "verify_seal_type" {
+  for_each = var.vault_hosts
+
+  scripts = [abspath("${path.module}/scripts/verify-seal-type.sh")]
+
+  environment = {
+    VAULT_ADDR         = "http://127.0.0.1:8200"
+    VAULT_INSTALL_DIR  = var.vault_install_dir
+    EXPECTED_SEAL_TYPE = var.seal_type
+  }
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}

enos/modules/verify_seal_type/scripts/verify-seal-type.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+set -e
+
+fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "$EXPECTED_SEAL_TYPE" ]] && fail "EXPECTED_SEAL_TYPE env variable has not been set"
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set"
+
+binpath=${VAULT_INSTALL_DIR}/vault
+test -x "$binpath" || fail "unable to locate vault binary at $binpath"
+
+count=0
+retries=2
+while :; do
+  if seal_status=$($binpath read sys/seal-status -format=json); then
+    if jq -Mer --arg expected "$EXPECTED_SEAL_TYPE" '.data.type == $expected' <<< "$seal_status" &> /dev/null; then
+      exit 0
+    fi
+  fi
+
+  wait=$((2 ** count))
+  count=$((count + 1))
+  if [ "$count" -lt "$retries" ]; then
+    sleep "$wait"
+  else
+    printf "Seal Status: %s\n" "$seal_status"
+    got=$(jq -Mer '.data.type' <<< "$seal_status")
+    fail "Expected seal type to be $EXPECTED_SEAL_TYPE, got: $got"
+  fi
+done