From eb1376dc1383c49dc7a782d5ab971902e2db9bbe Mon Sep 17 00:00:00 2001 From: hc-github-team-secure-vault-core <82990506+hc-github-team-secure-vault-core@users.noreply.github.com> Date: Thu, 26 Oct 2023 17:36:50 -0400 Subject: [PATCH] backport of commit a46def288f06cff8176399f239f87a2a49ba5dd9 (#23869) Co-authored-by: Ryan Cragun --- .github/workflows/enos-lint.yml | 1 - .github/workflows/enos-run-k8s.yml | 1 - .github/workflows/test-enos-scenario-ui.yml | 1 - .../test-run-enos-scenario-matrix.yml | 1 - enos/ci/service-user-iam/main.tf | 1 + enos/enos-modules.hcl | 30 + enos/enos-scenario-agent.hcl | 45 +- enos/enos-scenario-autopilot.hcl | 70 ++- enos/enos-scenario-proxy.hcl | 45 +- enos/enos-scenario-replication.hcl | 88 ++- enos/enos-scenario-seal-ha.hcl | 551 ++++++++++++++++++ enos/enos-scenario-smoke.hcl | 45 +- enos/enos-scenario-ui.hcl | 49 +- enos/enos-scenario-upgrade.hcl | 45 +- enos/modules/create_vpc/main.tf | 12 - enos/modules/create_vpc/outputs.tf | 15 +- enos/modules/create_vpc/variables.tf | 6 - enos/modules/seal_key_awskms/main.tf | 56 ++ enos/modules/seal_key_shamir/main.tf | 17 + enos/modules/start_vault/main.tf | 167 ++++++ enos/modules/start_vault/outputs.tf | 33 ++ enos/modules/start_vault/variables.tf | 125 ++++ enos/modules/stop_vault/main.tf | 38 ++ enos/modules/target_ec2_fleet/main.tf | 26 +- enos/modules/target_ec2_fleet/variables.tf | 12 +- enos/modules/target_ec2_instances/main.tf | 26 +- .../modules/target_ec2_instances/variables.tf | 12 +- enos/modules/target_ec2_shim/main.tf | 2 +- enos/modules/target_ec2_spot_fleet/main.tf | 26 +- .../target_ec2_spot_fleet/variables.tf | 12 +- enos/modules/vault_cluster/main.tf | 178 ++---- enos/modules/vault_cluster/variables.tf | 56 +- .../scripts/wait-for-leader.sh | 6 +- .../vault_wait_for_seal_rewrap/main.tf | 67 +++ .../scripts/wait-for-seal-rewrap.sh | 67 +++ 35 files changed, 1532 insertions(+), 400 deletions(-) create mode 100644 enos/enos-scenario-seal-ha.hcl create mode 100644 enos/modules/seal_key_awskms/main.tf create mode 100644 enos/modules/seal_key_shamir/main.tf create mode 100644 enos/modules/start_vault/main.tf create mode 100644 enos/modules/start_vault/outputs.tf create mode 100644 enos/modules/start_vault/variables.tf create mode 100644 enos/modules/stop_vault/main.tf create mode 100644 enos/modules/vault_wait_for_seal_rewrap/main.tf create mode 100644 enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh diff --git a/.github/workflows/enos-lint.yml b/.github/workflows/enos-lint.yml index c7729e7fd..616b3e029 100644 --- a/.github/workflows/enos-lint.yml +++ b/.github/workflows/enos-lint.yml @@ -41,7 +41,6 @@ jobs: - uses: hashicorp/setup-terraform@v2 with: terraform_wrapper: false - terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved - uses: hashicorp/action-setup-enos@v1 with: github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} diff --git a/.github/workflows/enos-run-k8s.yml b/.github/workflows/enos-run-k8s.yml index 04ea05995..adddba47d 100644 --- a/.github/workflows/enos-run-k8s.yml +++ b/.github/workflows/enos-run-k8s.yml @@ -38,7 +38,6 @@ jobs: # the Terraform wrapper will break Terraform execution in Enos because # it changes the output to text when we expect it to be JSON. terraform_wrapper: false - terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved - name: Set up Enos uses: hashicorp/action-setup-enos@v1 with: diff --git a/.github/workflows/test-enos-scenario-ui.yml b/.github/workflows/test-enos-scenario-ui.yml index 5cc17e16d..fc348ce60 100644 --- a/.github/workflows/test-enos-scenario-ui.yml +++ b/.github/workflows/test-enos-scenario-ui.yml @@ -85,7 +85,6 @@ jobs: with: cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} terraform_wrapper: false - terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved - name: Prepare scenario dependencies run: | mkdir -p ./enos/support/terraform-plugin-cache diff --git a/.github/workflows/test-run-enos-scenario-matrix.yml b/.github/workflows/test-run-enos-scenario-matrix.yml index 4f3b3a236..c05b42930 100644 --- a/.github/workflows/test-run-enos-scenario-matrix.yml +++ b/.github/workflows/test-run-enos-scenario-matrix.yml @@ -102,7 +102,6 @@ jobs: # the Terraform wrapper will break Terraform execution in Enos because # it changes the output to text when we expect it to be JSON. terraform_wrapper: false - terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved - uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }} diff --git a/enos/ci/service-user-iam/main.tf b/enos/ci/service-user-iam/main.tf index d64115b4a..0df9bcafc 100644 --- a/enos/ci/service-user-iam/main.tf +++ b/enos/ci/service-user-iam/main.tf @@ -216,6 +216,7 @@ data "aws_iam_policy_document" "enos_scenario" { "kms:ListKeys", "kms:ListResourceTags", "kms:ScheduleKeyDeletion", + "kms:TagResource", "servicequotas:ListServiceQuotas" ] diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 4a7552190..4d1e5be21 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -57,6 +57,18 @@ module "replication_data" { source = "./modules/replication_data" } +module "seal_key_awskms" { + source = "./modules/seal_key_awskms" + + common_tags = var.tags +} + +module "seal_key_shamir" { + source = "./modules/seal_key_shamir" + + common_tags = var.tags +} + module "shutdown_node" { source = "./modules/shutdown_node" } @@ -65,6 +77,17 @@ module "shutdown_multiple_nodes" { source = "./modules/shutdown_multiple_nodes" } +module "start_vault" { + source = "./modules/start_vault" + + install_dir = var.vault_install_dir + log_level = var.vault_log_level +} + +module "stop_vault" { + source = "./modules/stop_vault" +} + # create target instances using ec2:CreateFleet module "target_ec2_fleet" { source = "./modules/target_ec2_fleet" @@ -249,3 +272,10 @@ module "vault_wait_for_leader" { vault_install_dir = var.vault_install_dir vault_instance_count = var.vault_instance_count } + +module "vault_wait_for_seal_rewrap" { + source = "./modules/vault_wait_for_seal_rewrap" + + vault_install_dir = var.vault_install_dir + vault_instance_count = var.vault_instance_count +} diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index 5a613bf60..741e99224 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -11,6 +11,7 @@ scenario "agent" { distro = ["ubuntu", "rhel"] edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] seal = ["awskms", "shamir"] + seal_ha_beta = ["true", "false"] # Our local builder always creates bundles exclude { @@ -81,6 +82,15 @@ scenario "agent" { } } + step "create_seal_key" { + module = "seal_key_${matrix.seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + common_tags = global.tags + } + } + // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -110,11 +120,11 @@ scenario "agent" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -127,11 +137,11 @@ scenario "agent" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.backend_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -171,7 +181,6 @@ scenario "agent" { variables { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name backend_cluster_tag_key = global.backend_tag_key cluster_name = step.create_vault_cluster_targets.cluster_name @@ -186,9 +195,11 @@ scenario "agent" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_seal_key.resource_name + seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts - unseal_method = matrix.seal } } @@ -389,11 +400,6 @@ scenario "agent" { value = step.create_vault_cluster.audit_device_file_path } - output "awskms_unseal_key_arn" { - description = "The Vault cluster KMS key arn" - value = step.create_vpc.kms_key_arn - } - output "cluster_name" { description = "The Vault cluster name" value = step.create_vault_cluster.cluster_name @@ -434,6 +440,11 @@ scenario "agent" { value = step.create_vault_cluster.recovery_keys_hex } + output "seal_key_name" { + description = "The name of the cluster seal key" + value = step.create_seal_key.resource_name + } + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" value = step.create_vault_cluster.unseal_keys_b64 diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index ffbbdcba1..f4d78f533 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -12,6 +12,7 @@ scenario "autopilot" { // release branch's version. initial_version = ["1.11.12", "1.12.11", "1.13.6", "1.14.2"] seal = ["awskms", "shamir"] + seal_ha_beta = ["true", "false"] # Our local builder always creates bundles exclude { @@ -77,6 +78,15 @@ scenario "autopilot" { } } + step "create_seal_key" { + module = "seal_key_${matrix.seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + common_tags = global.tags + } + } + step "read_license" { module = module.read_license @@ -94,11 +104,11 @@ scenario "autopilot" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -114,22 +124,23 @@ scenario "autopilot" { } variables { - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_name = step.create_vault_cluster_targets.cluster_name - install_dir = local.vault_install_dir - license = matrix.edition != "ce" ? step.read_license.license : null - packages = concat(global.packages, global.distro_packages[matrix.distro]) + cluster_name = step.create_vault_cluster_targets.cluster_name + enable_audit_devices = var.vault_enable_audit_devices + install_dir = local.vault_install_dir + license = matrix.edition != "ce" ? step.read_license.license : null + packages = concat(global.packages, global.distro_packages[matrix.distro]) release = { edition = matrix.edition version = matrix.initial_version } + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_seal_key.resource_name + seal_type = matrix.seal storage_backend = "raft" storage_backend_addl_config = { autopilot_upgrade_version = matrix.initial_version } - target_hosts = step.create_vault_cluster_targets.hosts - unseal_method = matrix.seal - enable_audit_devices = var.vault_enable_audit_devices + target_hosts = step.create_vault_cluster_targets.hosts } } @@ -190,11 +201,11 @@ scenario "autopilot" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - common_tags = global.tags - cluster_name = step.create_vault_cluster_targets.cluster_name - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + common_tags = global.tags + cluster_name = step.create_vault_cluster_targets.cluster_name + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -213,7 +224,7 @@ scenario "autopilot" { variables { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - awskms_unseal_key_arn = step.create_vpc.kms_key_arn + enable_audit_devices = var.vault_enable_audit_devices cluster_name = step.create_vault_cluster_targets.cluster_name log_level = var.vault_log_level force_unseal = matrix.seal == "shamir" @@ -224,13 +235,14 @@ scenario "autopilot" { manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) root_token = step.create_vault_cluster.root_token + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_seal_key.resource_name + seal_type = matrix.seal shamir_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null storage_backend = "raft" storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config storage_node_prefix = "upgrade_node" target_hosts = step.create_vault_cluster_upgrade_targets.hosts - unseal_method = matrix.seal - enable_audit_devices = var.vault_enable_audit_devices } } @@ -498,9 +510,9 @@ scenario "autopilot" { } } - output "awskms_unseal_key_arn" { - description = "The Vault cluster KMS key arn" - value = step.create_vpc.kms_key_arn + output "audit_device_file_path" { + description = "The file path for the file audit device, if enabled" + value = step.create_vault_cluster.audit_device_file_path } output "cluster_name" { @@ -543,6 +555,11 @@ scenario "autopilot" { value = step.create_vault_cluster.recovery_keys_hex } + output "seal_key_name" { + description = "The Vault cluster seal key name" + value = step.create_seal_key.resource_name + } + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" value = step.create_vault_cluster.unseal_keys_b64 @@ -567,9 +584,4 @@ scenario "autopilot" { description = "The Vault cluster public IPs" value = step.upgrade_vault_cluster_with_autopilot.public_ips } - - output "vault_audit_device_file_path" { - description = "The file path for the file audit device, if enabled" - value = step.create_vault_cluster.audit_device_file_path - } } diff --git a/enos/enos-scenario-proxy.hcl b/enos/enos-scenario-proxy.hcl index 3cf662ffc..23020b65a 100644 --- a/enos/enos-scenario-proxy.hcl +++ b/enos/enos-scenario-proxy.hcl @@ -11,6 +11,7 @@ scenario "proxy" { distro = ["ubuntu", "rhel"] edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] seal = ["awskms", "shamir"] + seal_ha_beta = ["true", "false"] # Our local builder always creates bundles exclude { @@ -81,6 +82,15 @@ scenario "proxy" { } } + step "create_seal_key" { + module = "seal_key_${matrix.seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + common_tags = global.tags + } + } + // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -110,11 +120,11 @@ scenario "proxy" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -127,11 +137,11 @@ scenario "proxy" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.backend_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -171,7 +181,6 @@ scenario "proxy" { variables { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name backend_cluster_tag_key = global.backend_tag_key cluster_name = step.create_vault_cluster_targets.cluster_name @@ -186,9 +195,11 @@ scenario "proxy" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_seal_key.resource_name + seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts - unseal_method = matrix.seal } } @@ -361,11 +372,6 @@ scenario "proxy" { value = step.create_vault_cluster.audit_device_file_path } - output "awskms_unseal_key_arn" { - description = "The Vault cluster KMS key arn" - value = step.create_vpc.kms_key_arn - } - output "cluster_name" { description = "The Vault cluster name" value = step.create_vault_cluster.cluster_name @@ -406,6 +412,11 @@ scenario "proxy" { value = step.create_vault_cluster.recovery_keys_hex } + output "seal_key_name" { + description = "The Vault cluster seal key name" + value = step.create_seal_key.resource_name + } + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" value = step.create_vault_cluster.unseal_keys_b64 diff --git a/enos/enos-scenario-replication.hcl b/enos/enos-scenario-replication.hcl index 7834078c7..803e17d34 100644 --- a/enos/enos-scenario-replication.hcl +++ b/enos/enos-scenario-replication.hcl @@ -14,6 +14,7 @@ scenario "replication" { edition = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] primary_backend = ["raft", "consul"] primary_seal = ["awskms", "shamir"] + seal_ha_beta = ["true", "false"] secondary_backend = ["raft", "consul"] secondary_seal = ["awskms", "shamir"] @@ -86,6 +87,26 @@ scenario "replication" { } } + step "create_primary_seal_key" { + module = "seal_key_${matrix.primary_seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + cluster_meta = "primary" + common_tags = global.tags + } + } + + step "create_secondary_seal_key" { + module = "seal_key_${matrix.secondary_seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + cluster_meta = "secondary" + common_tags = global.tags + } + } + // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -117,11 +138,11 @@ scenario "replication" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_primary_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -136,11 +157,11 @@ scenario "replication" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.backend_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + seal_key_names = step.create_primary_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -156,12 +177,12 @@ scenario "replication" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_name = step.create_primary_cluster_targets.cluster_name - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_name = step.create_primary_cluster_targets.cluster_name + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_primary_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -174,11 +195,11 @@ scenario "replication" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_secondary_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -191,11 +212,11 @@ scenario "replication" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.backend_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + seal_key_names = step.create_secondary_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -235,7 +256,6 @@ scenario "replication" { variables { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_primary_cluster_backend_targets.cluster_name backend_cluster_tag_key = global.backend_tag_key consul_license = (matrix.primary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null @@ -250,9 +270,11 @@ scenario "replication" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_primary_seal_key.resource_name + seal_type = matrix.primary_seal storage_backend = matrix.primary_backend target_hosts = step.create_primary_cluster_targets.hosts - unseal_method = matrix.primary_seal } } @@ -292,7 +314,6 @@ scenario "replication" { variables { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_secondary_cluster_backend_targets.cluster_name backend_cluster_tag_key = global.backend_tag_key consul_license = (matrix.secondary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null @@ -307,9 +328,11 @@ scenario "replication" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_secondary_seal_key.resource_name + seal_type = matrix.secondary_seal storage_backend = matrix.secondary_backend target_hosts = step.create_secondary_cluster_targets.hosts - unseal_method = matrix.secondary_seal } } @@ -584,7 +607,6 @@ scenario "replication" { variables { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_primary_cluster_backend_targets.cluster_name backend_cluster_tag_key = global.backend_tag_key cluster_name = step.create_primary_cluster_targets.cluster_name @@ -602,11 +624,13 @@ scenario "replication" { manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) root_token = step.create_primary_cluster.root_token + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_primary_seal_key.resource_name + seal_type = matrix.primary_seal shamir_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null storage_backend = matrix.primary_backend storage_node_prefix = "newprimary_node" target_hosts = step.create_primary_cluster_additional_targets.hosts - unseal_method = matrix.primary_seal } } diff --git a/enos/enos-scenario-seal-ha.hcl b/enos/enos-scenario-seal-ha.hcl new file mode 100644 index 000000000..b3cd44df0 --- /dev/null +++ b/enos/enos-scenario-seal-ha.hcl @@ -0,0 +1,551 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +scenario "seal_ha" { + matrix { + arch = ["amd64", "arm64"] + artifact_source = ["local", "crt", "artifactory"] + artifact_type = ["bundle", "package"] + backend = ["consul", "raft"] + consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"] + distro = ["ubuntu", "rhel"] + edition = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] + primary_seal = ["awskms"] + secondary_seal = ["awskms"] + + # Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } + + # HSM and FIPS 140-2 are only supported on amd64 + exclude { + arch = ["arm64"] + edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] + } + } + + terraform_cli = terraform_cli.default + terraform = terraform.default + providers = [ + provider.aws.default, + provider.enos.ubuntu, + provider.enos.rhel + ] + + locals { + artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null + enos_provider = { + rhel = provider.enos.rhel + ubuntu = provider.enos.ubuntu + } + manage_service = matrix.artifact_type == "bundle" + vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro] + } + + step "get_local_metadata" { + skip_step = matrix.artifact_source != "local" + module = module.get_local_metadata + } + + step "build_vault" { + module = "build_${matrix.artifact_source}" + + variables { + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] + artifact_path = local.artifact_path + goarch = matrix.arch + goos = "linux" + artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null + artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null + artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null + artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null + arch = matrix.artifact_source == "artifactory" ? matrix.arch : null + product_version = var.vault_product_version + artifact_type = matrix.artifact_type + distro = matrix.artifact_source == "artifactory" ? matrix.distro : null + edition = matrix.artifact_source == "artifactory" ? matrix.edition : null + revision = var.vault_revision + } + } + + step "ec2_info" { + module = module.ec2_info + } + + step "create_vpc" { + module = module.create_vpc + + variables { + common_tags = global.tags + } + } + + step "create_primary_seal_key" { + module = "seal_key_${matrix.primary_seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + cluster_meta = "primary" + common_tags = global.tags + } + } + + step "create_secondary_seal_key" { + module = "seal_key_${matrix.secondary_seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + cluster_meta = "secondary" + common_tags = global.tags + other_resources = step.create_primary_seal_key.resource_names + } + } + + // This step reads the contents of the backend license if we're using a Consul backend and + // the edition is "ent". + step "read_backend_license" { + skip_step = matrix.backend == "raft" || var.backend_edition == "ce" + module = module.read_license + + variables { + file_name = global.backend_license_path + } + } + + step "read_vault_license" { + skip_step = matrix.edition == "ce" + module = module.read_license + + variables { + file_name = global.vault_license_path + } + } + + step "create_vault_cluster_targets" { + module = module.target_ec2_instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_secondary_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + + step "create_vault_cluster_backend_targets" { + module = matrix.backend == "consul" ? module.target_ec2_instances : module.target_ec2_shim + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + seal_key_names = step.create_secondary_seal_key.resource_names + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + vpc_id = step.create_vpc.id + } + } + + step "create_backend_cluster" { + module = "backend_${matrix.backend}" + depends_on = [ + step.create_vault_cluster_backend_targets + ] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_name = step.create_vault_cluster_backend_targets.cluster_name + cluster_tag_key = global.backend_tag_key + license = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null + release = { + edition = var.backend_edition + version = matrix.consul_version + } + target_hosts = step.create_vault_cluster_backend_targets.hosts + } + } + + step "create_vault_cluster" { + module = module.vault_cluster + depends_on = [ + step.create_backend_cluster, + step.build_vault, + step.create_vault_cluster_targets + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name + backend_cluster_tag_key = global.backend_tag_key + cluster_name = step.create_vault_cluster_targets.cluster_name + consul_license = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null + consul_release = matrix.backend == "consul" ? { + edition = var.backend_edition + version = matrix.consul_version + } : null + enable_audit_devices = var.vault_enable_audit_devices + install_dir = local.vault_install_dir + license = matrix.edition != "ce" ? step.read_vault_license.license : null + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = concat(global.packages, global.distro_packages[matrix.distro]) + // Only configure our primary seal during our initial cluster setup + seal_type = matrix.primary_seal + seal_key_name = step.create_primary_seal_key.resource_name + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts + } + } + + // Wait for our cluster to elect a leader + step "wait_for_leader" { + module = module.vault_wait_for_leader + depends_on = [step.create_vault_cluster] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + timeout = 120 # seconds + vault_hosts = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "get_vault_cluster_ips" { + module = module.vault_get_cluster_ips + depends_on = [step.wait_for_leader] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_hosts = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "verify_vault_unsealed" { + module = module.vault_verify_unsealed + depends_on = [step.wait_for_leader] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_install_dir = local.vault_install_dir + vault_instances = step.create_vault_cluster_targets.hosts + } + } + + // Write some test data before we create the new seal + step "verify_write_test_data" { + module = module.vault_verify_write_data + depends_on = [ + step.create_vault_cluster, + step.get_vault_cluster_ips, + step.verify_vault_unsealed, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + leader_private_ip = step.get_vault_cluster_ips.leader_private_ip + vault_instances = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + // Wait for the initial seal rewrap to complete before we add our HA seal. + step "wait_for_initial_seal_rewrap" { + module = module.vault_wait_for_seal_rewrap + depends_on = [ + step.verify_write_test_data, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_hosts = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + // Stop the vault service on all nodes before we restart with new seal config + step "stop_vault" { + module = module.stop_vault + depends_on = [ + step.create_vault_cluster, + step.verify_write_test_data, + step.wait_for_initial_seal_rewrap, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + target_hosts = step.create_vault_cluster_targets.hosts + } + } + + // Add the secondary seal to the cluster + step "add_ha_seal_to_cluster" { + module = module.start_vault + depends_on = [step.stop_vault] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + cluster_name = step.create_vault_cluster_targets.cluster_name + install_dir = local.vault_install_dir + license = matrix.edition != "ce" ? step.read_vault_license.license : null + manage_service = local.manage_service + seal_type = matrix.primary_seal + seal_key_name = step.create_primary_seal_key.resource_name + seal_type_secondary = matrix.secondary_seal + seal_key_name_secondary = step.create_secondary_seal_key.resource_name + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts + } + } + + // Wait for our cluster to elect a leader + step "wait_for_new_leader" { + module = module.vault_wait_for_leader + depends_on = [step.add_ha_seal_to_cluster] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + timeout = 120 # seconds + vault_hosts = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "get_updated_cluster_ips" { + module = module.vault_get_cluster_ips + depends_on = [step.wait_for_new_leader] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_hosts = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "verify_vault_unsealed_with_new_seal" { + module = module.vault_verify_unsealed + depends_on = [step.wait_for_new_leader] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_install_dir = local.vault_install_dir + vault_instances = step.create_vault_cluster_targets.hosts + } + } + + // Wait for the seal rewrap to complete and verify that no entries failed + step "wait_for_seal_rewrap" { + module = module.vault_wait_for_seal_rewrap + depends_on = [ + step.add_ha_seal_to_cluster, + step.verify_vault_unsealed_with_new_seal, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_hosts = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "verify_vault_version" { + module = module.vault_verify_version + depends_on = [step.wait_for_seal_rewrap] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.create_vault_cluster_targets.hosts + vault_edition = matrix.edition + vault_install_dir = local.vault_install_dir + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "verify_raft_auto_join_voter" { + skip_step = matrix.backend != "raft" + module = module.vault_verify_raft_auto_join_voter + depends_on = [step.wait_for_seal_rewrap] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_install_dir = local.vault_install_dir + vault_instances = step.create_vault_cluster_targets.hosts + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "verify_replication" { + module = module.vault_verify_replication + depends_on = [step.wait_for_seal_rewrap] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_edition = matrix.edition + vault_install_dir = local.vault_install_dir + vault_instances = step.create_vault_cluster_targets.hosts + } + } + + step "verify_read_test_data" { + module = module.vault_verify_read_data + depends_on = [step.wait_for_seal_rewrap] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + node_public_ips = step.get_updated_cluster_ips.follower_public_ips + vault_install_dir = local.vault_install_dir + } + } + + step "verify_ui" { + module = module.vault_verify_ui + depends_on = [step.wait_for_seal_rewrap] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.create_vault_cluster_targets.hosts + } + } + + output "audit_device_file_path" { + description = "The file path for the file audit device, if enabled" + value = step.create_vault_cluster.audit_device_file_path + } + + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name + } + + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts + } + + output "primary_seal_key_name" { + description = "The Vault cluster primary seal key name" + value = step.create_primary_seal_key.resource_name + } + + output "private_ips" { + description = "The Vault cluster private IPs" + value = step.create_vault_cluster.private_ips + } + + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips + } + + output "root_token" { + description = "The Vault cluster root token" + value = step.create_vault_cluster.root_token + } + + output "recovery_key_shares" { + description = "The Vault cluster recovery key shares" + value = step.create_vault_cluster.recovery_key_shares + } + + output "recovery_keys_b64" { + description = "The Vault cluster recovery keys b64" + value = step.create_vault_cluster.recovery_keys_b64 + } + + output "recovery_keys_hex" { + description = "The Vault cluster recovery keys hex" + value = step.create_vault_cluster.recovery_keys_hex + } + + output "secondary_seal_key_name" { + description = "The Vault cluster secondary seal key name" + value = step.create_secondary_seal_key.resource_name + } + + output "unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.unseal_keys_b64 + } + + output "unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.unseal_keys_hex + } +} diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index cb8e30e5b..4215942c1 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -11,6 +11,7 @@ scenario "smoke" { distro = ["ubuntu", "rhel"] edition = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] seal = ["awskms", "shamir"] + seal_ha_beta = ["true", "false"] # Our local builder always creates bundles exclude { @@ -81,6 +82,15 @@ scenario "smoke" { } } + step "create_seal_key" { + module = "seal_key_${matrix.seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + common_tags = global.tags + } + } + // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -110,11 +120,11 @@ scenario "smoke" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -127,11 +137,11 @@ scenario "smoke" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.backend_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -171,7 +181,6 @@ scenario "smoke" { variables { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name backend_cluster_tag_key = global.backend_tag_key cluster_name = step.create_vault_cluster_targets.cluster_name @@ -186,9 +195,11 @@ scenario "smoke" { local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro]) + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_seal_key.resource_name + seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts - unseal_method = matrix.seal } } @@ -352,11 +363,6 @@ scenario "smoke" { value = step.create_vault_cluster.audit_device_file_path } - output "awskms_unseal_key_arn" { - description = "The Vault cluster KMS key arn" - value = step.create_vpc.kms_key_arn - } - output "cluster_name" { description = "The Vault cluster name" value = step.create_vault_cluster.cluster_name @@ -397,6 +403,11 @@ scenario "smoke" { value = step.create_vault_cluster.recovery_keys_hex } + output "seal_key_name" { + description = "The Vault cluster seal key name" + value = step.create_seal_key.name + } + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" value = step.create_vault_cluster.unseal_keys_b64 diff --git a/enos/enos-scenario-ui.hcl b/enos/enos-scenario-ui.hcl index ea2e20f1d..e0b423572 100644 --- a/enos/enos-scenario-ui.hcl +++ b/enos/enos-scenario-ui.hcl @@ -3,8 +3,9 @@ scenario "ui" { matrix { - edition = ["ce", "ent"] - backend = ["consul", "raft"] + edition = ["ce", "ent"] + backend = ["consul", "raft"] + seal_ha_beta = ["true", "false"] } terraform_cli = terraform_cli.default @@ -68,6 +69,15 @@ scenario "ui" { } } + step "create_seal_key" { + module = "seal_key_${local.seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + common_tags = global.tags + } + } + // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -97,11 +107,11 @@ scenario "ui" { } variables { - ami_id = step.ec2_info.ami_ids[local.arch][local.distro][var.ubuntu_distro_version] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.vault_tag_key - common_tags = local.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[local.arch][local.distro][var.ubuntu_distro_version] + cluster_tag_key = local.vault_tag_key + common_tags = local.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -114,11 +124,11 @@ scenario "ui" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.backend_tag_key - common_tags = local.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + cluster_tag_key = local.backend_tag_key + common_tags = local.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -157,7 +167,6 @@ scenario "ui" { } variables { - awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name backend_cluster_tag_key = local.backend_tag_key cluster_name = step.create_vault_cluster_targets.cluster_name @@ -171,9 +180,11 @@ scenario "ui" { license = matrix.edition != "ce" ? step.read_vault_license.license : null local_artifact_path = local.bundle_path packages = global.distro_packages["ubuntu"] + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_seal_key.resource_name + seal_type = local.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts - unseal_method = local.seal } } @@ -212,11 +223,6 @@ scenario "ui" { value = step.create_vault_cluster.audit_device_file_path } - output "awskms_unseal_key_arn" { - description = "The Vault cluster KMS key arn" - value = step.create_vpc.kms_key_arn - } - output "cluster_name" { description = "The Vault cluster name" value = step.create_vault_cluster.cluster_name @@ -257,6 +263,11 @@ scenario "ui" { value = step.create_vault_cluster.root_token } + output "seal_key_name" { + description = "The Vault cluster seal key name" + value = step.create_seal_key.resource_name + } + output "ui_test_environment" { value = step.test_ui.ui_test_environment description = "The environment variables that are required in order to run the test:enos yarn target" diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index d5270a7f1..5e94c26b9 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -16,6 +16,7 @@ scenario "upgrade" { // those earlier versions. initial_version = ["1.11.12", "1.12.11", "1.13.6", "1.14.2"] seal = ["awskms", "shamir"] + seal_ha_beta = ["true", "false"] # Our local builder always creates bundles exclude { @@ -93,6 +94,15 @@ scenario "upgrade" { } } + step "create_seal_key" { + module = "seal_key_${matrix.seal}" + + variables { + cluster_id = step.create_vpc.cluster_id + common_tags = global.tags + } + } + // This step reads the contents of the backend license if we're using a Consul backend and // the edition is "ent". step "read_backend_license" { @@ -122,11 +132,11 @@ scenario "upgrade" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -139,11 +149,11 @@ scenario "upgrade" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] - awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = global.backend_tag_key - common_tags = global.tags - vpc_id = step.create_vpc.vpc_id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id } } @@ -182,7 +192,6 @@ scenario "upgrade" { } variables { - awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name backend_cluster_tag_key = global.backend_tag_key consul_license = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null @@ -199,9 +208,11 @@ scenario "upgrade" { edition = matrix.edition version = matrix.initial_version } + seal_ha_beta = matrix.seal_ha_beta + seal_key_name = step.create_seal_key.resource_name + seal_type = matrix.seal storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts - unseal_method = matrix.seal } } @@ -413,11 +424,6 @@ scenario "upgrade" { value = step.create_vault_cluster.audit_device_file_path } - output "awskms_unseal_key_arn" { - description = "The Vault cluster KMS key arn" - value = step.create_vpc.kms_key_arn - } - output "cluster_name" { description = "The Vault cluster name" value = step.create_vault_cluster.cluster_name @@ -458,6 +464,11 @@ scenario "upgrade" { value = step.create_vault_cluster.recovery_keys_hex } + output "seal_key_name" { + description = "The Vault cluster seal key name" + value = step.create_seal_key.resource_name + } + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" value = step.create_vault_cluster.unseal_keys_b64 diff --git a/enos/modules/create_vpc/main.tf b/enos/modules/create_vpc/main.tf index 40e039ea3..2e7c3ebfc 100644 --- a/enos/modules/create_vpc/main.tf +++ b/enos/modules/create_vpc/main.tf @@ -18,18 +18,6 @@ resource "random_string" "cluster_id" { special = false } -resource "aws_kms_key" "key" { - count = var.create_kms_key ? 1 : 0 - description = "vault-ci-kms-key" - deletion_window_in_days = 7 // 7 is the shortest allowed window -} - -resource "aws_kms_alias" "alias" { - count = var.create_kms_key ? 1 : 0 - name = "alias/enos_key-${random_string.cluster_id.result}" - target_key_id = aws_kms_key.key[0].key_id -} - resource "aws_vpc" "vpc" { cidr_block = var.cidr enable_dns_hostnames = true diff --git a/enos/modules/create_vpc/outputs.tf b/enos/modules/create_vpc/outputs.tf index eb16a1a60..a064644d3 100644 --- a/enos/modules/create_vpc/outputs.tf +++ b/enos/modules/create_vpc/outputs.tf @@ -1,22 +1,17 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 -output "vpc_id" { +output "id" { description = "Created VPC ID" value = aws_vpc.vpc.id } -output "vpc_cidr" { +output "cidr" { description = "CIDR for whole VPC" value = var.cidr } -output "kms_key_arn" { - description = "ARN of the generated KMS key" - value = try(aws_kms_key.key[0].arn, null) -} - -output "kms_key_alias" { - description = "Alias of the generated KMS key" - value = try(aws_kms_alias.alias[0].name, null) +output "cluster_id" { + description = "A unique string associated with the VPC" + value = random_string.cluster_id.result } diff --git a/enos/modules/create_vpc/variables.tf b/enos/modules/create_vpc/variables.tf index a9cefe13b..198613622 100644 --- a/enos/modules/create_vpc/variables.tf +++ b/enos/modules/create_vpc/variables.tf @@ -24,9 +24,3 @@ variable "common_tags" { type = map(string) default = { "Project" : "vault-ci" } } - -variable "create_kms_key" { - description = "Whether or not to create an key management service key" - type = bool - default = true -} diff --git a/enos/modules/seal_key_awskms/main.tf b/enos/modules/seal_key_awskms/main.tf new file mode 100644 index 000000000..88e0bf6ae --- /dev/null +++ b/enos/modules/seal_key_awskms/main.tf @@ -0,0 +1,56 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +variable "cluster_id" { + type = string +} + +variable "cluster_meta" { + type = string + default = null +} + +variable "common_tags" { + type = map(string) + default = null +} + +variable "other_resources" { + type = list(string) + default = [] +} + +locals { + cluster_name = var.cluster_meta == null ? var.cluster_id : "${var.cluster_id}-${var.cluster_meta}" +} + +resource "aws_kms_key" "key" { + description = "auto-unseal-key-${local.cluster_name}" + deletion_window_in_days = 7 // 7 is the shortest allowed window + tags = var.common_tags +} + +resource "aws_kms_alias" "alias" { + name = "alias/auto-unseal-key-${local.cluster_name}" + target_key_id = aws_kms_key.key.key_id +} + +output "alias" { + description = "The key alias name" + value = aws_kms_alias.alias.name +} + +output "id" { + description = "The key ID" + value = aws_kms_key.key.key_id +} + +output "resource_name" { + description = "The ARN" + value = aws_kms_key.key.arn +} + +output "resource_names" { + description = "The list of names" + value = compact(concat([aws_kms_key.key.arn], var.other_resources)) +} diff --git a/enos/modules/seal_key_shamir/main.tf b/enos/modules/seal_key_shamir/main.tf new file mode 100644 index 000000000..15f6d591c --- /dev/null +++ b/enos/modules/seal_key_shamir/main.tf @@ -0,0 +1,17 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# A shim unseal key module for shamir seal types + +variable "cluster_id" { default = null } +variable "cluster_meta" { default = null } +variable "common_tags" { default = null } +variable "names" { + type = list(string) + default = [] +} + +output "alias" { value = null } +output "id" { value = null } +output "resource_name" { value = null } +output "resource_names" { value = var.names } diff --git a/enos/modules/start_vault/main.tf b/enos/modules/start_vault/main.tf new file mode 100644 index 000000000..9ed49815d --- /dev/null +++ b/enos/modules/start_vault/main.tf @@ -0,0 +1,167 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + # We need to specify the provider source in each module until we publish it + # to the public registry + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.4.7" + } + } +} + +data "enos_environment" "localhost" {} + +locals { + bin_path = "${var.install_dir}/vault" + environment = local.seal_secondary == null ? var.environment : merge( + var.environment, + { VAULT_ENABLE_SEAL_HA_BETA : tobool(var.seal_ha_beta) }, + ) + // In order to get Terraform to plan we have to use collections with keys + // that are known at plan time. In order for our module to work our var.target_hosts + // must be a map with known keys at plan time. Here we're creating locals + // that keep track of index values that point to our target hosts. + followers = toset(slice(local.instances, 1, length(local.instances))) + instances = [for idx in range(length(var.target_hosts)) : tostring(idx)] + key_shares = { + "awskms" = null + "shamir" = 5 + } + key_threshold = { + "awskms" = null + "shamir" = 3 + } + leader = toset(slice(local.instances, 0, 1)) + recovery_shares = { + "awskms" = 5 + "shamir" = null + } + recovery_threshold = { + "awskms" = 3 + "shamir" = null + } + seals = local.seal_secondary.type == "none" ? { primary = local.seal_primary } : { + primary = local.seal_primary + secondary = local.seal_secondary + } + seals_primary = { + "awskms" = { + type = "awskms" + attributes = { + name = "primary" + kms_key_id = var.seal_key_name + } + } + "shamir" = { + type = "shamir" + attributes = null + } + } + seal_primary = local.seals_primary[var.seal_type] + seals_secondary = { + "awskms" = { + type = "awskms" + attributes = { + name = "secondary" + kms_key_id = var.seal_key_name_secondary + } + } + "none" = { + type = "none" + attributes = null + } + } + seal_secondary = local.seals_secondary[var.seal_type_secondary] + storage_config = [for idx, host in var.target_hosts : (var.storage_backend == "raft" ? + merge( + { + node_id = "${var.storage_node_prefix}_${idx}" + }, + var.storage_backend_attrs + ) : + { + address = "127.0.0.1:8500" + path = "vault" + }) + ] +} + +resource "enos_vault_start" "leader" { + for_each = local.leader + + bin_path = local.bin_path + config_dir = var.config_dir + environment = local.environment + config = { + api_addr = "http://${var.target_hosts[each.value].private_ip}:8200" + cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201" + cluster_name = var.cluster_name + listener = { + type = "tcp" + attributes = { + address = "0.0.0.0:8200" + tls_disable = "true" + } + } + log_level = var.log_level + storage = { + type = var.storage_backend + attributes = ({ for key, value in local.storage_config[each.key] : key => value }) + } + seals = local.seals + ui = true + } + license = var.license + manage_service = var.manage_service + username = var.service_username + unit_name = "vault" + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +resource "enos_vault_start" "followers" { + depends_on = [ + enos_vault_start.leader, + ] + for_each = local.followers + + bin_path = local.bin_path + config_dir = var.config_dir + environment = local.environment + config = { + api_addr = "http://${var.target_hosts[each.value].private_ip}:8200" + cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201" + cluster_name = var.cluster_name + listener = { + type = "tcp" + attributes = { + address = "0.0.0.0:8200" + tls_disable = "true" + } + } + log_level = var.log_level + storage = { + type = var.storage_backend + attributes = { for key, value in local.storage_config[each.key] : key => value } + } + seals = local.seals + ui = true + } + license = var.license + manage_service = var.manage_service + username = var.service_username + unit_name = "vault" + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} diff --git a/enos/modules/start_vault/outputs.tf b/enos/modules/start_vault/outputs.tf new file mode 100644 index 000000000..b3107bc9d --- /dev/null +++ b/enos/modules/start_vault/outputs.tf @@ -0,0 +1,33 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +output "cluster_name" { + description = "The Vault cluster name" + value = var.cluster_name +} + +output "followers" { + description = "The follower enos_vault_start resources" + value = enos_vault_start.followers +} + +output "leader" { + description = "The leader enos_vault_start resource" + value = enos_vault_start.leader +} + +output "private_ips" { + description = "Vault cluster target host private_ips" + value = [for host in var.target_hosts : host.private_ip] +} + +output "public_ips" { + description = "Vault cluster target host public_ips" + value = [for host in var.target_hosts : host.public_ip] +} + +output "target_hosts" { + description = "The vault cluster instances that were created" + + value = var.target_hosts +} diff --git a/enos/modules/start_vault/variables.tf b/enos/modules/start_vault/variables.tf new file mode 100644 index 000000000..d5f37465b --- /dev/null +++ b/enos/modules/start_vault/variables.tf @@ -0,0 +1,125 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +variable "cluster_name" { + type = string + description = "The Vault cluster name" +} + +variable "config_dir" { + type = string + description = "The directory to use for Vault configuration" + default = "/etc/vault.d" +} + +variable "environment" { + description = "Optional Vault configuration environment variables to set starting Vault" + type = map(string) + default = null +} + +variable "install_dir" { + type = string + description = "The directory where the vault binary will be installed" + default = "/opt/vault/bin" +} + +variable "license" { + type = string + sensitive = true + description = "The value of the Vault license" + default = null +} + +variable "log_level" { + type = string + description = "The vault service log level" + default = "info" + + validation { + condition = contains(["trace", "debug", "info", "warn", "error"], var.log_level) + error_message = "The log_level must be one of 'trace', 'debug', 'info', 'warn', or 'error'." + } +} + +variable "manage_service" { + type = bool + description = "Manage the Vault service users and systemd unit. Disable this to use configuration in RPM and Debian packages" + default = true +} + +variable "seal_ha_beta" { + description = "Enable using Seal HA on clusters that meet minimum version requirements and are enterprise editions" + default = true +} + +variable "seal_key_name" { + type = string + description = "The auto-unseal key name" + default = null +} + +variable "seal_key_name_secondary" { + type = string + description = "The secondary auto-unseal key name" + default = null +} + +variable "seal_type" { + type = string + description = "The method by which to unseal the Vault cluster" + default = "awskms" + + validation { + condition = contains(["awskms", "shamir"], var.seal_type) + error_message = "The seal_type must be either awskms or shamir. No other unseal methods are supported." + } +} + +variable "seal_type_secondary" { + type = string + description = "A secondary HA seal method. Only supported in Vault Enterprise >= 1.15" + default = "none" + + validation { + condition = contains(["awskms", "none"], var.seal_type_secondary) + error_message = "The secondary_seal_type must be 'awskms' or 'none'. No other secondary unseal methods are supported." + } +} + +variable "service_username" { + type = string + description = "The host username to own the vault service" + default = "vault" +} + +variable "storage_backend" { + type = string + description = "The storage backend to use" + default = "raft" + + validation { + condition = contains(["raft", "consul"], var.storage_backend) + error_message = "The storage_backend must be either raft or consul. No other storage backends are supported." + } +} + +variable "storage_backend_attrs" { + type = map(any) + description = "An optional set of key value pairs to inject into the storage block" + default = {} +} + +variable "storage_node_prefix" { + type = string + description = "A prefix to use for each node in the Vault storage configuration" + default = "node" +} + +variable "target_hosts" { + description = "The target machines host addresses to use for the Vault cluster" + type = map(object({ + private_ip = string + public_ip = string + })) +} diff --git a/enos/modules/stop_vault/main.tf b/enos/modules/stop_vault/main.tf new file mode 100644 index 000000000..7a2c25081 --- /dev/null +++ b/enos/modules/stop_vault/main.tf @@ -0,0 +1,38 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + # We need to specify the provider source in each module until we publish it + # to the public registry + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.4.0" + } + } +} + +variable "service_name" { + type = string + description = "The Vault systemd service name" + default = "vault" +} + +variable "target_hosts" { + description = "The target machines host addresses to use for the Vault cluster" + type = map(object({ + private_ip = string + public_ip = string + })) +} + +resource "enos_remote_exec" "shutdown_multiple_nodes" { + for_each = var.target_hosts + inline = ["sudo systemctl stop ${var.service_name}.service; sleep 5"] + + transport = { + ssh = { + host = each.value.public_ip + } + } +} diff --git a/enos/modules/target_ec2_fleet/main.tf b/enos/modules/target_ec2_fleet/main.tf index 5f4dca176..983f71cf4 100644 --- a/enos/modules/target_ec2_fleet/main.tf +++ b/enos/modules/target_ec2_fleet/main.tf @@ -23,10 +23,6 @@ data "aws_subnets" "vpc" { } } -data "aws_kms_key" "kms_key" { - key_id = var.awskms_unseal_key_arn -} - data "aws_iam_policy_document" "target" { statement { resources = ["*"] @@ -37,16 +33,20 @@ data "aws_iam_policy_document" "target" { ] } - statement { - resources = [var.awskms_unseal_key_arn] + dynamic "statement" { + for_each = var.seal_key_names - actions = [ - "kms:DescribeKey", - "kms:ListKeys", - "kms:Encrypt", - "kms:Decrypt", - "kms:GenerateDataKey" - ] + content { + resources = [statement.value] + + actions = [ + "kms:DescribeKey", + "kms:ListKeys", + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey" + ] + } } } diff --git a/enos/modules/target_ec2_fleet/variables.tf b/enos/modules/target_ec2_fleet/variables.tf index 7b8426f52..606cf5c29 100644 --- a/enos/modules/target_ec2_fleet/variables.tf +++ b/enos/modules/target_ec2_fleet/variables.tf @@ -6,12 +6,6 @@ variable "ami_id" { type = string } -variable "awskms_unseal_key_arn" { - type = string - description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key" - default = null -} - variable "cluster_name" { type = string description = "A unique cluster identifier" @@ -73,6 +67,12 @@ variable "project_name" { type = string } +variable "seal_key_names" { + type = list(string) + description = "The key management seal key names" + default = null +} + variable "ssh_allow_ips" { description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted" type = list(string) diff --git a/enos/modules/target_ec2_instances/main.tf b/enos/modules/target_ec2_instances/main.tf index db137b5be..fcc1b254f 100644 --- a/enos/modules/target_ec2_instances/main.tf +++ b/enos/modules/target_ec2_instances/main.tf @@ -53,10 +53,6 @@ data "aws_subnets" "vpc" { } } -data "aws_kms_key" "kms_key" { - key_id = var.awskms_unseal_key_arn -} - data "aws_iam_policy_document" "target" { statement { resources = ["*"] @@ -67,16 +63,20 @@ data "aws_iam_policy_document" "target" { ] } - statement { - resources = [var.awskms_unseal_key_arn] + dynamic "statement" { + for_each = var.seal_key_names - actions = [ - "kms:DescribeKey", - "kms:ListKeys", - "kms:Encrypt", - "kms:Decrypt", - "kms:GenerateDataKey" - ] + content { + resources = [statement.value] + + actions = [ + "kms:DescribeKey", + "kms:ListKeys", + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey" + ] + } } } diff --git a/enos/modules/target_ec2_instances/variables.tf b/enos/modules/target_ec2_instances/variables.tf index 524701068..cedae53e5 100644 --- a/enos/modules/target_ec2_instances/variables.tf +++ b/enos/modules/target_ec2_instances/variables.tf @@ -6,12 +6,6 @@ variable "ami_id" { type = string } -variable "awskms_unseal_key_arn" { - type = string - description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key" - default = null -} - variable "cluster_name" { type = string description = "A unique cluster identifier" @@ -53,6 +47,12 @@ variable "project_name" { type = string } +variable "seal_key_names" { + type = list(string) + description = "The key management seal key names" + default = null +} + variable "ssh_allow_ips" { description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted" type = list(string) diff --git a/enos/modules/target_ec2_shim/main.tf b/enos/modules/target_ec2_shim/main.tf index 745979cd6..13c6f1d4d 100644 --- a/enos/modules/target_ec2_shim/main.tf +++ b/enos/modules/target_ec2_shim/main.tf @@ -13,7 +13,6 @@ terraform { } variable "ami_id" { default = null } -variable "awskms_unseal_key_arn" { default = null } variable "cluster_name" { default = null } variable "cluster_tag_key" { default = null } variable "common_tags" { default = null } @@ -25,6 +24,7 @@ variable "instance_mem_min" { default = null } variable "instance_types" { default = null } variable "max_price" { default = null } variable "project_name" { default = null } +variable "seal_key_names" { default = null } variable "ssh_allow_ips" { default = null } variable "ssh_keypair" { default = null } variable "vpc_id" { default = null } diff --git a/enos/modules/target_ec2_spot_fleet/main.tf b/enos/modules/target_ec2_spot_fleet/main.tf index c89a6e9cf..9c14de85c 100644 --- a/enos/modules/target_ec2_spot_fleet/main.tf +++ b/enos/modules/target_ec2_spot_fleet/main.tf @@ -23,10 +23,6 @@ data "aws_subnets" "vpc" { } } -data "aws_kms_key" "kms_key" { - key_id = var.awskms_unseal_key_arn -} - data "aws_iam_policy_document" "target" { statement { resources = ["*"] @@ -37,16 +33,20 @@ data "aws_iam_policy_document" "target" { ] } - statement { - resources = [var.awskms_unseal_key_arn] + dynamic "statement" { + for_each = var.seal_key_names - actions = [ - "kms:DescribeKey", - "kms:ListKeys", - "kms:Encrypt", - "kms:Decrypt", - "kms:GenerateDataKey" - ] + content { + resources = [statement.value] + + actions = [ + "kms:DescribeKey", + "kms:ListKeys", + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey" + ] + } } } diff --git a/enos/modules/target_ec2_spot_fleet/variables.tf b/enos/modules/target_ec2_spot_fleet/variables.tf index 5fe51a9b8..c2f5bb609 100644 --- a/enos/modules/target_ec2_spot_fleet/variables.tf +++ b/enos/modules/target_ec2_spot_fleet/variables.tf @@ -6,12 +6,6 @@ variable "ami_id" { type = string } -variable "awskms_unseal_key_arn" { - type = string - description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key" - default = null -} - variable "cluster_name" { type = string description = "A unique cluster identifier" @@ -73,6 +67,12 @@ variable "max_price" { default = "0.0416" } +variable "seal_key_names" { + type = list(string) + description = "The key management seal key names" + default = null +} + variable "ssh_allow_ips" { description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted" type = list(string) diff --git a/enos/modules/vault_cluster/main.tf b/enos/modules/vault_cluster/main.tf index 3daf3eafd..c82295f59 100644 --- a/enos/modules/vault_cluster/main.tf +++ b/enos/modules/vault_cluster/main.tf @@ -42,30 +42,6 @@ locals { "awskms" = 3 "shamir" = null } - seal = { - "awskms" = { - type = "awskms" - attributes = { - kms_key_id = var.awskms_unseal_key_arn - } - } - "shamir" = { - type = "shamir" - attributes = null - } - } - storage_config = [for idx, host in var.target_hosts : (var.storage_backend == "raft" ? - merge( - { - node_id = "${var.storage_node_prefix}_${idx}" - }, - var.storage_backend_addl_config - ) : - { - address = "127.0.0.1:8500" - path = "vault" - }) - ] vault_service_user = "vault" } @@ -148,90 +124,35 @@ resource "enos_consul_start" "consul" { } } -resource "enos_vault_start" "leader" { +module "start_vault" { + source = "../start_vault" + depends_on = [ enos_consul_start.consul, enos_bundle_install.vault, ] - for_each = local.leader - bin_path = local.bin_path - config_dir = var.config_dir - environment = var.config_env_vars - config = { - api_addr = "http://${var.target_hosts[each.value].private_ip}:8200" - cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201" - cluster_name = var.cluster_name - listener = { - type = "tcp" - attributes = { - address = "0.0.0.0:8200" - tls_disable = "true" - } - } - log_level = var.log_level - storage = { - type = var.storage_backend - attributes = ({ for key, value in local.storage_config[each.key] : key => value }) - } - seal = local.seal[var.unseal_method] - ui = true - } - license = var.license - manage_service = var.manage_service - username = local.vault_service_user - unit_name = "vault" - - transport = { - ssh = { - host = var.target_hosts[each.value].public_ip - } - } -} - -resource "enos_vault_start" "followers" { - depends_on = [ - enos_vault_start.leader, - ] - for_each = local.followers - - bin_path = local.bin_path - config_dir = var.config_dir - environment = var.config_env_vars - config = { - api_addr = "http://${var.target_hosts[each.value].private_ip}:8200" - cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201" - cluster_name = var.cluster_name - listener = { - type = "tcp" - attributes = { - address = "0.0.0.0:8200" - tls_disable = "true" - } - } - log_level = var.log_level - storage = { - type = var.storage_backend - attributes = { for key, value in local.storage_config[each.key] : key => value } - } - seal = local.seal[var.unseal_method] - ui = true - } - license = var.license - manage_service = var.manage_service - username = local.vault_service_user - unit_name = "vault" - - transport = { - ssh = { - host = var.target_hosts[each.value].public_ip - } - } + cluster_name = var.cluster_name + config_dir = var.config_dir + install_dir = var.install_dir + license = var.license + log_level = var.log_level + manage_service = var.manage_service + seal_ha_beta = var.seal_ha_beta + seal_key_name = var.seal_key_name + seal_key_name_secondary = var.seal_key_name_secondary + seal_type = var.seal_type + seal_type_secondary = var.seal_type_secondary + service_username = local.vault_service_user + storage_backend = var.storage_backend + storage_backend_attrs = var.storage_backend_addl_config + storage_node_prefix = var.storage_node_prefix + target_hosts = var.target_hosts } resource "enos_vault_init" "leader" { depends_on = [ - enos_vault_start.followers, + module.start_vault, ] for_each = toset([ for idx, leader in local.leader : leader @@ -239,13 +160,13 @@ resource "enos_vault_init" "leader" { ]) bin_path = local.bin_path - vault_addr = enos_vault_start.leader[0].config.api_addr + vault_addr = module.start_vault.leader[0].config.api_addr - key_shares = local.key_shares[var.unseal_method] - key_threshold = local.key_threshold[var.unseal_method] + key_shares = local.key_shares[var.seal_type] + key_threshold = local.key_threshold[var.seal_type] - recovery_shares = local.recovery_shares[var.unseal_method] - recovery_threshold = local.recovery_threshold[var.unseal_method] + recovery_shares = local.recovery_shares[var.seal_type] + recovery_threshold = local.recovery_threshold[var.seal_type] transport = { ssh = { @@ -256,15 +177,15 @@ resource "enos_vault_init" "leader" { resource "enos_vault_unseal" "leader" { depends_on = [ - enos_vault_start.followers, + module.start_vault, enos_vault_init.leader, ] for_each = enos_vault_init.leader // only unseal the leader if we initialized it bin_path = local.bin_path - vault_addr = enos_vault_start.leader[each.key].config.api_addr - seal_type = var.unseal_method - unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex) + vault_addr = module.start_vault.leader[each.key].config.api_addr + seal_type = var.seal_type + unseal_keys = var.seal_type != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex) transport = { ssh = { @@ -282,13 +203,13 @@ resource "enos_vault_unseal" "followers" { // initialized the cluster for_each = toset([ for idx, follower in local.followers : follower - if var.unseal_method == "shamir" && var.initialize_cluster + if var.seal_type == "shamir" && var.initialize_cluster ]) bin_path = local.bin_path - vault_addr = enos_vault_start.followers[each.key].config.api_addr - seal_type = var.unseal_method - unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex) + vault_addr = module.start_vault.followers[each.key].config.api_addr + seal_type = var.seal_type + unseal_keys = var.seal_type != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex) transport = { ssh = { @@ -303,7 +224,7 @@ resource "enos_vault_unseal" "followers" { // force_unseal to true. resource "enos_vault_unseal" "maybe_force_unseal" { depends_on = [ - enos_vault_start.followers, + module.start_vault.followers, ] for_each = { for idx, host in var.target_hosts : idx => host @@ -312,7 +233,7 @@ resource "enos_vault_unseal" "maybe_force_unseal" { bin_path = local.bin_path vault_addr = "http://localhost:8200" - seal_type = var.unseal_method + seal_type = var.seal_type unseal_keys = coalesce( var.shamir_unseal_keys, try(enos_vault_init.leader[0].unseal_keys_hex, null), @@ -325,38 +246,11 @@ resource "enos_vault_unseal" "maybe_force_unseal" { } } -resource "enos_remote_exec" "vault_write_license" { - for_each = toset([ - for idx, leader in local.leader : leader - if var.initialize_cluster - ]) - - depends_on = [ - enos_vault_unseal.leader, - enos_vault_unseal.maybe_force_unseal, - ] - - environment = { - BIN_PATH = local.bin_path, - LICENSE = coalesce(var.license, "none") - VAULT_TOKEN = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none") - } - - scripts = [abspath("${path.module}/scripts/vault-write-license.sh")] - - transport = { - ssh = { - host = var.target_hosts[each.value].public_ip - } - } -} - # We need to ensure that the directory used for audit logs is present and accessible to the vault # user on all nodes, since logging will only happen on the leader. resource "enos_remote_exec" "create_audit_log_dir" { depends_on = [ - enos_vault_start.leader, - enos_vault_start.followers, + module.start_vault, enos_vault_unseal.leader, enos_vault_unseal.followers, enos_vault_unseal.maybe_force_unseal, diff --git a/enos/modules/vault_cluster/variables.tf b/enos/modules/vault_cluster/variables.tf index 83aa2a4f6..78bf87587 100644 --- a/enos/modules/vault_cluster/variables.tf +++ b/enos/modules/vault_cluster/variables.tf @@ -12,12 +12,6 @@ variable "artifactory_release" { default = null } -variable "awskms_unseal_key_arn" { - type = string - description = "The AWSKMS key ARN if using the awskms unseal method" - default = null -} - variable "backend_cluster_name" { type = string description = "The name of the backend cluster" @@ -171,6 +165,45 @@ variable "root_token" { default = null } +variable "seal_ha_beta" { + description = "Enable using Seal HA on clusters that meet minimum version requirements and are enterprise editions" + default = true +} + +variable "seal_key_name" { + type = string + description = "The auto-unseal key name" + default = null +} + +variable "seal_key_name_secondary" { + type = string + description = "The secondary auto-unseal key name" + default = null +} + +variable "seal_type" { + type = string + description = "The method by which to unseal the Vault cluster" + default = "awskms" + + validation { + condition = contains(["awskms", "shamir"], var.seal_type) + error_message = "The seal_type must be either awskms or shamir. No other unseal methods are supported." + } +} + +variable "seal_type_secondary" { + type = string + description = "A secondary HA seal method. Only supported in Vault Enterprise >= 1.15" + default = "none" + + validation { + condition = contains(["awskms", "none"], var.seal_type_secondary) + error_message = "The secondary_seal_type must be 'awskms' or 'none'. No other secondary unseal methods are supported." + } +} + variable "shamir_unseal_keys" { type = list(string) description = "Shamir unseal keys. Often only used adding additional nodes to an already initialized cluster." @@ -207,14 +240,3 @@ variable "target_hosts" { public_ip = string })) } - -variable "unseal_method" { - type = string - description = "The method by which to unseal the Vault cluster" - default = "awskms" - - validation { - condition = contains(["awskms", "shamir"], var.unseal_method) - error_message = "The unseal_method must be either awskms or shamir. No other unseal methods are supported." - } -} diff --git a/enos/modules/vault_wait_for_leader/scripts/wait-for-leader.sh b/enos/modules/vault_wait_for_leader/scripts/wait-for-leader.sh index aa4ccb8de..f993ae8b6 100644 --- a/enos/modules/vault_wait_for_leader/scripts/wait-for-leader.sh +++ b/enos/modules/vault_wait_for_leader/scripts/wait-for-leader.sh @@ -23,14 +23,14 @@ test -x "$binpath" || fail "unable to locate vault binary at $binpath" findLeaderInPrivateIPs() { # Find the leader private IP address local leader_private_ip - if ! leader_private_ip=$($binpath read sys/leader -format=json | jq -r '.data.leader_address | scan("[0-9]+.[0-9]+.[0-9]+.[0-9]+")') ; then + if ! leader_private_ip=$($binpath read sys/leader -format=json | jq -er '.data.leader_address | scan("[0-9]+.[0-9]+.[0-9]+.[0-9]+")') ; then # Some older versions of vault don't support reading sys/leader. Fallback to the cli status. - if leader_private_ip=$($binpath status -format json | jq '.leader_address | scan("[0-9]+.[0-9]+.[0-9]+.[0-9]+")'); then + if ! leader_private_ip=$($binpath status -format json | jq -er '.leader_address | scan("[0-9]+.[0-9]+.[0-9]+.[0-9]+")'); then return 1 fi fi - if isIn=$(jq -r --arg ip "$leader_private_ip" 'map(select(. == $ip)) | length == 1' <<< "$VAULT_INSTANCE_PRIVATE_IPS"); then + if isIn=$(jq -er --arg ip "$leader_private_ip" 'map(select(. == $ip)) | length == 1' <<< "$VAULT_INSTANCE_PRIVATE_IPS"); then if [[ "$isIn" == "true" ]]; then echo "$leader_private_ip" return 0 diff --git a/enos/modules/vault_wait_for_seal_rewrap/main.tf b/enos/modules/vault_wait_for_seal_rewrap/main.tf new file mode 100644 index 000000000..ba4fe6178 --- /dev/null +++ b/enos/modules/vault_wait_for_seal_rewrap/main.tf @@ -0,0 +1,67 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +variable "vault_instance_count" { + type = number + description = "The number of instances in the vault cluster" +} + +variable "vault_hosts" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The vault cluster hosts that can be expected as a leader" +} + +variable "timeout" { + type = number + description = "The max number of seconds to wait before timing out" + default = 60 +} + +variable "retry_interval" { + type = number + description = "How many seconds to wait between each retry" + default = 2 +} + +locals { + private_ips = [for k, v in values(tomap(var.vault_hosts)) : tostring(v["private_ip"])] +} + +resource "enos_remote_exec" "wait_for_seal_rewrap_to_be_completed" { + environment = { + RETRY_INTERVAL = var.retry_interval + TIMEOUT_SECONDS = var.timeout + VAULT_ADDR = "http://127.0.0.1:8200" + VAULT_TOKEN = var.vault_root_token + VAULT_INSTALL_DIR = var.vault_install_dir + } + + scripts = [abspath("${path.module}/scripts/wait-for-seal-rewrap.sh")] + + transport = { + ssh = { + host = var.vault_hosts[0].public_ip + } + } +} diff --git a/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh b/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh new file mode 100644 index 000000000..46a600d83 --- /dev/null +++ b/enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set" +[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set" +[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set" +[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set" + +binpath=${VAULT_INSTALL_DIR}/vault +test -x "$binpath" || fail "unable to locate vault binary at $binpath" + +getRewrapData() { + $binpath read sys/sealwrap/rewrap -format=json | jq -eMc '.data' +} + +waitForRewrap() { + local data + if ! data=$(getRewrapData); then + echo "failed getting /v1/sys/sealwrap/rewrap data" 1>&2 + return 1 + fi + + if ! jq -e '.is_running == false' <<< "$data" &> /dev/null; then + echo "rewrap is running" 1>&2 + return 1 + fi + + if ! jq -e '.entries.failed == 0' <<< "$data" &> /dev/null; then + local entries + entries=$(jq -Mc '.entries.failed' <<< "$data") + echo "rewrap has $entries failed entries" 1>&2 + return 1 + fi + + if ! jq -e '.entries.processed == .entries.succeeded' <<< "$data" &> /dev/null; then + local processed + local succeeded + processed=$(jq -Mc '.entries.processed' <<< "$data") + succeeded=$(jq -Mc '.entries.succeeded' <<< "$data") + echo "the number of processed entries ($processed) does not equal then number of succeeded ($succeeded)" 1>&2 + return 1 + fi + + return 0 +} + +begin_time=$(date +%s) +end_time=$((begin_time + TIMEOUT_SECONDS)) +while [ "$(date +%s)" -lt "$end_time" ]; do + if waitForRewrap; then + exit 0 + fi + + sleep "$RETRY_INTERVAL" +done + +fail "Timed out waiting for seal rewrap to be completed. Data:\n\t$(getRewrapData)"