diff --git a/.github/workflows/test-enos-scenario-ui.yml b/.github/workflows/test-enos-scenario-ui.yml new file mode 100644 index 000000000..c2c36ea83 --- /dev/null +++ b/.github/workflows/test-enos-scenario-ui.yml @@ -0,0 +1,147 @@ +--- +name: Vault UI Tests + +on: + workflow_call: + inputs: + test_filter: + type: string + description: "A filter to limit the ui tests to. Will be appended to the ember test command as '-f='" + required: false + storage_backend: + type: string + description: "The storage backend to use, either 'raft' or 'consul'" + default: raft + workflow_dispatch: + inputs: + test_filter: + type: string + description: "A filter to limit the ui tests to. Will be appended to the ember test command as '-f='" + required: false + storage_backend: + description: "The storage backend to use, either 'raft' or 'consul'" + required: true + default: raft + type: choice + options: + - raft + - consul + +jobs: + get-metadata: + name: Get metadata + runs-on: ubuntu-latest + outputs: + go-version: ${{ steps.get-metadata.outputs.go-version }} + node-version: ${{ steps.get-metadata.outputs.node-version }} + runs-on: ${{ steps.get-metadata.outputs.runs-on }} + vault_edition: ${{ steps.get-metadata.outputs.vault_edition }} + steps: + - uses: actions/checkout@v3 + - id: get-metadata + env: + IS_ENT: ${{ startsWith(github.event.repository.name, 'vault-enterprise' ) }} + run: | + echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT + echo "node-version=$(cat ./ui/.nvmrc)" >> $GITHUB_OUTPUT + echo "chrome-installed=$(chrome --version && echo true || echo false)" >> $GITHUB_OUTPUT + if ${IS_ENT} == true; then + echo "detected vault_edition=ent" + echo "runs-on=['self-hosted', 'ondemand', 'os=linux', 'type=m5d.4xlarge']" >> $GITHUB_OUTPUT + echo "vault_edition=ent" >> $GITHUB_OUTPUT + else + echo "detected vault_edition=oss" + echo "runs-on=custom-linux-xl-vault-latest" >> $GITHUB_OUTPUT + echo "vault_edition=oss" >> $GITHUB_OUTPUT + fi + + run-ui-tests: + name: Run UI Tests + needs: get-metadata + runs-on: ${{ fromJSON(needs.get-metadata.outputs.runs-on) }} + timeout-minutes: 90 + env: + GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + # Pass in enos variables + ENOS_VAR_aws_region: us-east-1 + ENOS_VAR_aws_ssh_keypair_name: ${{ github.event.repository.name }}-ci-ssh-key + ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem + ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }} + ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache + ENOS_VAR_vault_license_path: ./support/vault.hclic + GOPRIVATE: github.com/hashicorp + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set Up Go + uses: actions/setup-go@v3 + with: + go-version: ${{ needs.get-metadata.outputs.go-version }} + - uses: hashicorp/action-setup-enos@v1 + with: + github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + - name: Set Up Git + run: git config --global url."https://${{ secrets.elevated_github_token }}:@github.com".insteadOf "https://github.com" + - name: Set Up Node + uses: actions/setup-node@v3 + with: + node-version: ${{ needs.get-metadata.outputs.node-version }} + - name: Set Up Terraform + uses: hashicorp/setup-terraform@v2 + with: + cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} + terraform_wrapper: false + - name: Prepare scenario dependencies + run: | + mkdir -p ./enos/support/terraform-plugin-cache + echo "${{ secrets.SSH_KEY_PRIVATE_CI }}" > ./enos/support/private_key.pem + chmod 600 ./enos/support/private_key.pem + - name: Set Up Vault Enterprise License + if: contains(${{ github.event.repository.name }}, 'ent') + run: echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true + - name: Install Chrome Dependencies + if: ${{ !needs.get-metadata.outputs.chrome-installed }} + run: | + sudo apt update + sudo apt install -y libnss3-dev libgdk-pixbuf2.0-dev libgtk-3-dev libxss-dev libasound2 + - name: Install Chrome + if: ${{ !needs.get-metadata.outputs.chrome-installed }} + uses: browser-actions/setup-chrome@v1 + - run: | + chrome --version + - name: Configure AWS credentials from Test account + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_CI }} + aws-region: us-east-1 + role-to-assume: ${{ secrets.AWS_ROLE_ARN_CI }} + role-skip-session-tagging: true + role-duration-seconds: 3600 + - name: Set Up Cluster + id: setup_cluster + env: + ENOS_VAR_ui_run_tests: false + # Continue once and retry to handle occasional blips when creating infrastructure. + continue-on-error: true + run: enos scenario launch --timeout 60m0s --chdir ./enos ui edition:${{ needs.get-metadata.outputs.vault_edition }} backend:${{ inputs.storage_backend }} + - name: Retry Set Up Cluster + id: setup_cluster_retry + if: steps.setup_cluster.outcome == 'failure' + env: + ENOS_VAR_ui_run_tests: false + run: enos scenario launch --timeout 60m0s --chdir ./enos ui edition:${{ needs.get-metadata.outputs.vault_edition }} backend:${{ inputs.storage_backend }} + - name: Run UI Tests + id: run_ui_tests + env: + ENOS_VAR_ui_test_filter: "${{ inputs.test_filter }}" + run: enos scenario run --timeout 60m0s --chdir ./enos ui edition:${{ needs.get-metadata.outputs.vault_edition }} backend:${{ inputs.storage_backend }} + - name: Ensure scenario has been destroyed + if: ${{ always() }} + run: enos scenario destroy --timeout 60m0s --chdir ./enos ui edition:${{ needs.get-metadata.outputs.vault_edition }} backend:${{ inputs.storage_backend }} + - name: Clean up Enos runtime directories + if: ${{ always() }} + run: | + rm -rf /tmp/enos* + rm -rf ./enos/support + rm -rf ./enos/.enos diff --git a/enos/README.md b/enos/README.md index f0d6f23d4..a33f4abe1 100644 --- a/enos/README.md +++ b/enos/README.md @@ -135,6 +135,56 @@ The [`replication` scenario](./enos-scenario-replication.hcl) creates two 3-node This scenario verifies the performance replication status on both clusters to have their connection_status as "connected" and that the secondary cluster has known_primaries cluster addresses updated to the active nodes IP addresses of the primary Vault cluster. This scenario currently works around issues VAULT-12311 and VAULT-12309. The scenario fails when the primary storage backend is Consul due to issue VAULT-12332 +## UI Tests +The [`ui` scenario](./enos-scenario-ui.hcl) creates a Vault cluster (deployed to AWS) using a version +built from the current checkout of the project. Once the cluster is available the UI acceptance tests +are run in a headless browser. +### Variables +In addition to the required variables that must be set, as described in the [Scenario Variables](#Scenario Variables), +the `ui` scenario has two optional variables: + +**ui_test_filter** - An optional test filter to limit the tests that are run, i.e. `'!enterprise'`. +To set a filter export the variable as follows: +```shell +> export ENOS_VAR_ui_test_filter="some filter" +``` +**ui_run_tests** - An optional boolean variable to run or not run the tests. The default value is true. +Setting this value to false is useful in the case where you want to create a cluster, but run the tests +manually. The section [Running the Tests](#Running the Tests) describes the different ways to run the +'UI' acceptance tests. + +### Running the Tests +The UI tests can be run fully automated or manually. +#### Fully Automated +The following will deploy the cluster, run the tests, and subsequently tear down the cluster: +```shell +> export ENOS_VAR_ui_test_filter="some filter" # <-- optional +> cd enos +> enos scenario ui run edition:oss +``` +#### Manually +The UI tests can be run manually as follows: +```shell +> export ENOS_VAR_ui_test_filter="some filter" # <-- optional +> export ENOS_VAR_ui_run_tests=false +> cd enos +> enos scenario ui launch edition:oss +# once complete the scenario will output a set of environment variables that must be exported. The +# output will look as follows: +export TEST_FILTER='some filter>' \ +export VAULT_ADDR='http://:8200' \ +export VAULT_TOKEN='' \ +export VAULT_UNSEAL_KEYS='["","",""]' +# copy and paste the above into the terminal to export the values +> cd ../ui +> yarn test:enos # run headless +# or +> yarn test:enos -s # run manually in a web browser +# once testing is complete +> cd ../enos +> enos scenario ui destroy edition:oss +``` + # Variants Both scenarios support a matrix of variants. In order to achieve broad coverage while keeping test run time reasonable, the variants executed by the `enos-run` Github diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 817d97e07..26cb20dac 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -197,3 +197,9 @@ module "vault_raft_remove_peer" { source = "./modules/vault_raft_remove_peer" vault_install_dir = var.vault_install_dir } + +module "vault_test_ui" { + source = "./modules/vault_test_ui" + + ui_run_tests = var.ui_run_tests +} diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index 9b5283301..d9bb24b54 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -380,7 +380,7 @@ scenario "autopilot" { } step "verify_undo_logs_status" { - skip_step = semverconstraint(var.vault_product_version, "<1.13.0-0") + skip_step = try(semverconstraint(var.vault_product_version, "<1.13.0-0"), true) module = module.vault_verify_undo_logs depends_on = [ step.remove_old_nodes, diff --git a/enos/enos-scenario-ui.hcl b/enos/enos-scenario-ui.hcl new file mode 100644 index 000000000..36c67b8d9 --- /dev/null +++ b/enos/enos-scenario-ui.hcl @@ -0,0 +1,205 @@ +scenario "ui" { + matrix { + edition = ["oss", "ent"] + backend = ["consul", "raft"] + } + + terraform_cli = terraform_cli.default + terraform = terraform.default + providers = [ + provider.aws.default, + provider.enos.ubuntu + ] + + locals { + arch = "amd64" + distro = "ubuntu" + seal = "awskms" + artifact_type = "bundle" + consul_version = "1.14.2" + build_tags = { + "oss" = ["ui"] + "ent" = ["ui", "enterprise", "ent"] + } + bundle_path = abspath(var.vault_bundle_path) + tags = merge({ + "Project Name" : var.project_name + "Project" : "Enos", + "Environment" : "ci" + }, var.tags) + vault_instance_types = { + amd64 = "t3a.small" + arm64 = "t4g.small" + } + vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[local.arch]) + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) + vault_install_dir_packages = { + rhel = "/bin" + ubuntu = "/usr/bin" + } + vault_install_dir = var.vault_install_dir + ui_test_filter = var.ui_test_filter != null ? var.ui_test_filter : (matrix.edition == "oss") ? "!enterprise" : null + } + + step "get_local_metadata" { + module = module.get_local_metadata + } + + step "build_vault" { + module = module.build_local + + variables { + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] + bundle_path = local.bundle_path + goarch = local.arch + goos = "linux" + product_version = var.vault_product_version + artifact_type = local.artifact_type + revision = var.vault_revision + } + } + + step "find_azs" { + module = module.az_finder + + variables { + instance_type = [ + var.backend_instance_type, + local.vault_instance_type + ] + } + } + + step "create_vpc" { + module = module.create_vpc + + variables { + ami_architectures = [local.arch] + availability_zones = step.find_azs.availability_zones + common_tags = local.tags + } + } + + step "read_license" { + skip_step = matrix.edition == "oss" + module = module.read_license + + variables { + file_name = local.vault_license_path + } + } + + step "create_backend_cluster" { + module = "backend_${matrix.backend}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.create_vpc.ami_ids["ubuntu"]["amd64"] + common_tags = local.tags + consul_release = { + edition = var.backend_edition + version = local.consul_version + } + instance_type = var.backend_instance_type + kms_key_arn = step.create_vpc.kms_key_arn + vpc_id = step.create_vpc.vpc_id + } + } + + step "create_vault_cluster" { + module = module.vault_cluster + depends_on = [ + step.create_backend_cluster, + step.build_vault, + ] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.create_vpc.ami_ids[local.distro][local.arch] + common_tags = local.tags + consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag + instance_type = local.vault_instance_type + kms_key_arn = step.create_vpc.kms_key_arn + storage_backend = matrix.backend + unseal_method = local.seal + vault_local_artifact_path = local.bundle_path + vault_install_dir = local.vault_install_dir + vault_license = matrix.edition != "oss" ? step.read_license.license : null + vpc_id = step.create_vpc.vpc_id + } + } + + step "test_ui" { + module = module.vault_test_ui + + variables { + vault_addr = step.create_vault_cluster.instance_public_ips[0] + vault_root_token = step.create_vault_cluster.vault_root_token + vault_unseal_keys = step.create_vault_cluster.vault_recovery_keys_b64 + vault_recovery_threshold = step.create_vault_cluster.vault_recovery_threshold + ui_test_filter = local.ui_test_filter + } + } + + output "vault_cluster_instance_ids" { + description = "The Vault cluster instance IDs" + value = step.create_vault_cluster.instance_ids + } + + output "vault_cluster_pub_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.instance_public_ips + } + + output "vault_cluster_priv_ips" { + description = "The Vault cluster private IPs" + value = step.create_vault_cluster.instance_private_ips + } + + output "vault_cluster_key_id" { + description = "The Vault cluster Key ID" + value = step.create_vault_cluster.key_id + } + + output "vault_cluster_root_token" { + description = "The Vault cluster root token" + value = step.create_vault_cluster.vault_root_token + } + + output "vault_cluster_unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.vault_unseal_keys_b64 + } + + output "vault_cluster_unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.vault_unseal_keys_hex + } + + output "vault_cluster_tag" { + description = "The Vault cluster tag" + value = step.create_vault_cluster.vault_cluster_tag + } + + output "ui_test_stderr" { + description = "The stderr of the ui tests that ran" + value = step.test_ui.ui_test_stderr + } + + output "ui_test_stdout" { + description = "The stdout of the ui tests that ran" + value = step.test_ui.ui_test_stdout + } + + output "ui_test_environment" { + value = step.test_ui.ui_test_environment + description = "The environment variables that are required in order to run the test:enos yarn target" + } +} diff --git a/enos/enos-variables.hcl b/enos/enos-variables.hcl index cd246a459..e0dd65f19 100644 --- a/enos/enos-variables.hcl +++ b/enos/enos-variables.hcl @@ -179,3 +179,15 @@ variable "remove_vault_instances" { description = "The old vault nodes to be removed" } + +variable "ui_test_filter" { + type = string + description = "A test filter to limit the ui tests to execute. Will be appended to the ember test command as '-f=\"\"'" + default = null +} + +variable "ui_run_tests" { + type = bool + description = "Whether to run the UI tests or not. If set to false a cluster will be created but no tests will be run" + default = true +} diff --git a/enos/modules/vault_test_ui/main.tf b/enos/modules/vault_test_ui/main.tf new file mode 100644 index 000000000..2af426232 --- /dev/null +++ b/enos/modules/vault_test_ui/main.tf @@ -0,0 +1,31 @@ +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +locals { + # base test environment excludes the filter argument + ui_test_environment_base = { + VAULT_ADDR = "http://${var.vault_addr}:8200" + VAULT_TOKEN = var.vault_root_token + VAULT_UNSEAL_KEYS = jsonencode(slice(var.vault_unseal_keys, 0, var.vault_recovery_threshold)) + } + ui_test_environment = var.ui_test_filter == null || try(length(trimspace(var.ui_test_filter)) == 0, true) ? local.ui_test_environment_base : merge(local.ui_test_environment_base, { + TEST_FILTER = var.ui_test_filter + }) + # The environment variables need to be double escaped since the process of rendering them to the + # outputs eats the escaping. Therefore double escaping ensures that the values are rendered as + # properly escaped json, i.e. "[\"value\"]" suitable to be parsed as json. + escaped_ui_test_environment = [ + for key, value in local.ui_test_environment : "export ${key}='${value}'" + ] +} + +resource "enos_local_exec" "test_ui" { + count = var.ui_run_tests ? 1 : 0 + environment = local.ui_test_environment + scripts = ["${path.module}/scripts/test_ui.sh"] +} diff --git a/enos/modules/vault_test_ui/outputs.tf b/enos/modules/vault_test_ui/outputs.tf new file mode 100644 index 000000000..abe4924ce --- /dev/null +++ b/enos/modules/vault_test_ui/outputs.tf @@ -0,0 +1,12 @@ +output "ui_test_stderr" { + value = var.ui_run_tests ? enos_local_exec.test_ui[0].stderr : "No std out tests where not run" +} + +output "ui_test_stdout" { + value = var.ui_run_tests ? enos_local_exec.test_ui[0].stdout : "No std out tests where not run" +} + +output "ui_test_environment" { + value = join(" \\ \n", local.escaped_ui_test_environment) + description = "The environment variables that are required in order to run the test:enos yarn target" +} diff --git a/enos/modules/vault_test_ui/scripts/test_ui.sh b/enos/modules/vault_test_ui/scripts/test_ui.sh new file mode 100755 index 000000000..f84cb929f --- /dev/null +++ b/enos/modules/vault_test_ui/scripts/test_ui.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -eux -o pipefail + +project_root=$(git rev-parse --show-toplevel) +pushd "$project_root" > /dev/null + +echo "running test-ember-enos" +make test-ember-enos +popd > /dev/null diff --git a/enos/modules/vault_test_ui/variables.tf b/enos/modules/vault_test_ui/variables.tf new file mode 100644 index 000000000..807cf01ae --- /dev/null +++ b/enos/modules/vault_test_ui/variables.tf @@ -0,0 +1,31 @@ +variable "vault_addr" { + description = "The host address for the vault instance to test" + type = string +} + +variable "vault_root_token" { + description = "The vault root token" + type = string +} + +variable "ui_test_filter" { + type = string + description = "A test filter to limit the ui tests to execute. Will be appended to the ember test command as '-f='" + default = null +} + +variable "vault_unseal_keys" { + description = "Base64 encoded recovery keys to use for the seal/unseal test" + type = list(string) +} + +variable "vault_recovery_threshold" { + description = "The number of recovery keys to require when unsealing Vault" + type = string +} + +variable "ui_run_tests" { + type = bool + description = "Whether to run the UI tests or not. If set to false a cluster will be created but no tests will be run" + default = true +} diff --git a/ui/scripts/enos-test-ember.js b/ui/scripts/enos-test-ember.js index 1b711f0d5..6415a4891 100755 --- a/ui/scripts/enos-test-ember.js +++ b/ui/scripts/enos-test-ember.js @@ -45,9 +45,8 @@ const testHelper = require('./test-helper'); try { const testArgs = ['test', '-c', 'testem.enos.js']; - if (process.env.TEST_FILTERS) { - const filters = JSON.parse(process.env.TEST_FILTERS).map((filter) => '-f=' + filter); - testArgs.push(...filters); + if (process.env.TEST_FILTER && process.env.TEST_FILTER.length > 0) { + testArgs.push('-f=' + process.env.TEST_FILTER); } await testHelper.run('ember', [...testArgs, ...process.argv.slice(2)], false);