diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 4d1e5be21..12b54e7e8 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -279,3 +279,10 @@ module "vault_wait_for_seal_rewrap" { vault_install_dir = var.vault_install_dir vault_instance_count = var.vault_instance_count } + +module "verify_seal_type" { + source = "./modules/verify_seal_type" + + vault_install_dir = var.vault_install_dir + vault_instance_count = var.vault_instance_count +} diff --git a/enos/enos-scenario-seal-ha.hcl b/enos/enos-scenario-seal-ha.hcl index b3cd44df0..a0f62a06a 100644 --- a/enos/enos-scenario-seal-ha.hcl +++ b/enos/enos-scenario-seal-ha.hcl @@ -407,6 +407,7 @@ scenario "seal_ha" { } } + // Perform all of our standard verifications after we've enabled multiseal step "verify_vault_version" { module = module.vault_verify_version depends_on = [step.wait_for_seal_rewrap] @@ -457,6 +458,7 @@ scenario "seal_ha" { } } + // Make sure our data is still available step "verify_read_test_data" { module = module.vault_verify_read_data depends_on = [step.wait_for_seal_rewrap] @@ -484,6 +486,166 @@ scenario "seal_ha" { } } + // Make sure we have a "multiseal" seal type + step "verify_seal_type" { + // Don't run this on versions less than 1.16.0-beta1 until VAULT-21053 is fixed on prior branches. + skip_step = semverconstraint(var.vault_product_version, "< 1.16.0-beta1") + module = module.verify_seal_type + depends_on = [step.wait_for_seal_rewrap] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_install_dir = local.vault_install_dir + vault_hosts = step.create_vault_cluster_targets.hosts + seal_type = "multiseal" + } + } + + // Now we'll migrate away from our initial seal to our secondary seal + + // Stop the vault service on all nodes before we restart with new seal config + step "stop_vault_for_migration" { + module = module.stop_vault + depends_on = [ + step.wait_for_seal_rewrap, + step.verify_read_test_data, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + target_hosts = step.create_vault_cluster_targets.hosts + } + } + + // Remove the "primary" seal from the cluster. Set our "secondary" seal to priority 1. We do this + // by restarting vault with the correct config. + step "remove_primary_seal" { + module = module.start_vault + depends_on = [step.stop_vault_for_migration] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + cluster_name = step.create_vault_cluster_targets.cluster_name + install_dir = local.vault_install_dir + license = matrix.edition != "ce" ? step.read_vault_license.license : null + manage_service = local.manage_service + seal_alias = "secondary" + seal_type = matrix.secondary_seal + seal_key_name = step.create_secondary_seal_key.resource_name + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts + } + } + + // Wait for our cluster to elect a leader after restarting vault with a new primary seal + step "wait_for_leader_after_migration" { + module = module.vault_wait_for_leader + depends_on = [step.remove_primary_seal] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + timeout = 120 # seconds + vault_hosts = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + // Since we've restarted our cluster we might have a new leader and followers. Get the new IPs. + step "get_cluster_ips_after_migration" { + module = module.vault_get_cluster_ips + depends_on = [step.wait_for_leader_after_migration] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_hosts = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + // Make sure we unsealed + step "verify_vault_unsealed_after_migration" { + module = module.vault_verify_unsealed + depends_on = [step.wait_for_leader_after_migration] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_install_dir = local.vault_install_dir + vault_instances = step.create_vault_cluster_targets.hosts + } + } + + // Wait for the seal rewrap to complete and verify that no entries failed + step "wait_for_seal_rewrap_after_migration" { + module = module.vault_wait_for_seal_rewrap + depends_on = [ + step.wait_for_leader_after_migration, + step.verify_vault_unsealed_after_migration, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_hosts = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + // Make sure our data is still available after migration + step "verify_read_test_data_after_migration" { + module = module.vault_verify_read_data + depends_on = [step.wait_for_seal_rewrap_after_migration] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + node_public_ips = step.get_cluster_ips_after_migration.follower_public_ips + vault_install_dir = local.vault_install_dir + } + } + + // Make sure we have our secondary seal type after migration + step "verify_seal_type_after_migration" { + // Don't run this on versions less than 1.16.0-beta1 until VAULT-21053 is fixed on prior branches. + skip_step = semverconstraint(var.vault_product_version, "<= 1.16.0-beta1") + module = module.verify_seal_type + depends_on = [step.wait_for_seal_rewrap_after_migration] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_install_dir = local.vault_install_dir + vault_hosts = step.create_vault_cluster_targets.hosts + seal_type = matrix.secondary_seal + } + } + output "audit_device_file_path" { description = "The file path for the file audit device, if enabled" value = step.create_vault_cluster.audit_device_file_path diff --git a/enos/modules/start_vault/main.tf b/enos/modules/start_vault/main.tf index 9ed49815d..c7344c2b7 100644 --- a/enos/modules/start_vault/main.tf +++ b/enos/modules/start_vault/main.tf @@ -51,7 +51,8 @@ locals { "awskms" = { type = "awskms" attributes = { - name = "primary" + name = var.seal_alias + priority = var.seal_priority kms_key_id = var.seal_key_name } } @@ -65,7 +66,8 @@ locals { "awskms" = { type = "awskms" attributes = { - name = "secondary" + name = var.seal_alias_secondary + priority = var.seal_priority_secondary kms_key_id = var.seal_key_name_secondary } } diff --git a/enos/modules/start_vault/variables.tf b/enos/modules/start_vault/variables.tf index d5f37465b..7faae7014 100644 --- a/enos/modules/start_vault/variables.tf +++ b/enos/modules/start_vault/variables.tf @@ -53,9 +53,21 @@ variable "seal_ha_beta" { default = true } +variable "seal_alias" { + type = string + description = "The primary seal alias name" + default = "primary" +} + +variable "seal_alias_secondary" { + type = string + description = "The secondary seal alias name" + default = "secondary" +} + variable "seal_key_name" { type = string - description = "The auto-unseal key name" + description = "The primary auto-unseal key name" default = null } @@ -65,6 +77,18 @@ variable "seal_key_name_secondary" { default = null } +variable "seal_priority" { + type = string + description = "The primary seal priority" + default = "1" +} + +variable "seal_priority_secondary" { + type = string + description = "The secondary seal priority" + default = "2" +} + variable "seal_type" { type = string description = "The method by which to unseal the Vault cluster" diff --git a/enos/modules/verify_seal_type/main.tf b/enos/modules/verify_seal_type/main.tf new file mode 100644 index 000000000..816c0cc08 --- /dev/null +++ b/enos/modules/verify_seal_type/main.tf @@ -0,0 +1,52 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_instance_count" { + type = number + description = "How many vault instances are in the cluster" +} + +variable "vault_hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The vault cluster instances that were created" +} + +variable "seal_type" { + type = string + description = "The expected seal type" + default = "shamir" +} + +resource "enos_remote_exec" "verify_seal_type" { + for_each = var.vault_hosts + + scripts = [abspath("${path.module}/scripts/verify-seal-type.sh")] + + environment = { + VAULT_ADDR = "http://127.0.0.1:8200" + VAULT_INSTALL_DIR = var.vault_install_dir + EXPECTED_SEAL_TYPE = var.seal_type + } + + transport = { + ssh = { + host = each.value.public_ip + } + } +} diff --git a/enos/modules/verify_seal_type/scripts/verify-seal-type.sh b/enos/modules/verify_seal_type/scripts/verify-seal-type.sh new file mode 100644 index 000000000..73ce06fd9 --- /dev/null +++ b/enos/modules/verify_seal_type/scripts/verify-seal-type.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$EXPECTED_SEAL_TYPE" ]] && fail "EXPECTED_SEAL_TYPE env variable has not been set" +[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set" + +binpath=${VAULT_INSTALL_DIR}/vault +test -x "$binpath" || fail "unable to locate vault binary at $binpath" + +count=0 +retries=2 +while :; do + if seal_status=$($binpath read sys/seal-status -format=json); then + if jq -Mer --arg expected "$EXPECTED_SEAL_TYPE" '.data.type == $expected' <<< "$seal_status" &> /dev/null; then + exit 0 + fi + fi + + wait=$((2 ** count)) + count=$((count + 1)) + if [ "$count" -lt "$retries" ]; then + sleep "$wait" + else + printf "Seal Status: %s\n" "$seal_status" + got=$(jq -Mer '.data.type' <<< "$seal_status") + fail "Expected seal type to be $EXPECTED_SEAL_TYPE, got: $got" + fi +done