[QT-304] Add enos ui scenario (#18518)

* Add enos ui scenario
* Add github action for running the UI scenario
This commit is contained in:
Mike Baum 2023-02-03 09:55:06 -05:00 committed by GitHub
parent c74c057bdb
commit 3131c48501
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 507 additions and 4 deletions

View File

@ -0,0 +1,147 @@
---
name: Vault UI Tests
on:
workflow_call:
inputs:
test_filter:
type: string
description: "A filter to limit the ui tests to. Will be appended to the ember test command as '-f=<filter>'"
required: false
storage_backend:
type: string
description: "The storage backend to use, either 'raft' or 'consul'"
default: raft
workflow_dispatch:
inputs:
test_filter:
type: string
description: "A filter to limit the ui tests to. Will be appended to the ember test command as '-f=<filter>'"
required: false
storage_backend:
description: "The storage backend to use, either 'raft' or 'consul'"
required: true
default: raft
type: choice
options:
- raft
- consul
jobs:
get-metadata:
name: Get metadata
runs-on: ubuntu-latest
outputs:
go-version: ${{ steps.get-metadata.outputs.go-version }}
node-version: ${{ steps.get-metadata.outputs.node-version }}
runs-on: ${{ steps.get-metadata.outputs.runs-on }}
vault_edition: ${{ steps.get-metadata.outputs.vault_edition }}
steps:
- uses: actions/checkout@v3
- id: get-metadata
env:
IS_ENT: ${{ startsWith(github.event.repository.name, 'vault-enterprise' ) }}
run: |
echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT
echo "node-version=$(cat ./ui/.nvmrc)" >> $GITHUB_OUTPUT
echo "chrome-installed=$(chrome --version && echo true || echo false)" >> $GITHUB_OUTPUT
if ${IS_ENT} == true; then
echo "detected vault_edition=ent"
echo "runs-on=['self-hosted', 'ondemand', 'os=linux', 'type=m5d.4xlarge']" >> $GITHUB_OUTPUT
echo "vault_edition=ent" >> $GITHUB_OUTPUT
else
echo "detected vault_edition=oss"
echo "runs-on=custom-linux-xl-vault-latest" >> $GITHUB_OUTPUT
echo "vault_edition=oss" >> $GITHUB_OUTPUT
fi
run-ui-tests:
name: Run UI Tests
needs: get-metadata
runs-on: ${{ fromJSON(needs.get-metadata.outputs.runs-on) }}
timeout-minutes: 90
env:
GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
# Pass in enos variables
ENOS_VAR_aws_region: us-east-1
ENOS_VAR_aws_ssh_keypair_name: ${{ github.event.repository.name }}-ci-ssh-key
ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem
ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
ENOS_VAR_vault_license_path: ./support/vault.hclic
GOPRIVATE: github.com/hashicorp
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set Up Go
uses: actions/setup-go@v3
with:
go-version: ${{ needs.get-metadata.outputs.go-version }}
- uses: hashicorp/action-setup-enos@v1
with:
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
- name: Set Up Git
run: git config --global url."https://${{ secrets.elevated_github_token }}:@github.com".insteadOf "https://github.com"
- name: Set Up Node
uses: actions/setup-node@v3
with:
node-version: ${{ needs.get-metadata.outputs.node-version }}
- name: Set Up Terraform
uses: hashicorp/setup-terraform@v2
with:
cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
terraform_wrapper: false
- name: Prepare scenario dependencies
run: |
mkdir -p ./enos/support/terraform-plugin-cache
echo "${{ secrets.SSH_KEY_PRIVATE_CI }}" > ./enos/support/private_key.pem
chmod 600 ./enos/support/private_key.pem
- name: Set Up Vault Enterprise License
if: contains(${{ github.event.repository.name }}, 'ent')
run: echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true
- name: Install Chrome Dependencies
if: ${{ !needs.get-metadata.outputs.chrome-installed }}
run: |
sudo apt update
sudo apt install -y libnss3-dev libgdk-pixbuf2.0-dev libgtk-3-dev libxss-dev libasound2
- name: Install Chrome
if: ${{ !needs.get-metadata.outputs.chrome-installed }}
uses: browser-actions/setup-chrome@v1
- run: |
chrome --version
- name: Configure AWS credentials from Test account
uses: aws-actions/configure-aws-credentials@v1-node16
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_CI }}
aws-region: us-east-1
role-to-assume: ${{ secrets.AWS_ROLE_ARN_CI }}
role-skip-session-tagging: true
role-duration-seconds: 3600
- name: Set Up Cluster
id: setup_cluster
env:
ENOS_VAR_ui_run_tests: false
# Continue once and retry to handle occasional blips when creating infrastructure.
continue-on-error: true
run: enos scenario launch --timeout 60m0s --chdir ./enos ui edition:${{ needs.get-metadata.outputs.vault_edition }} backend:${{ inputs.storage_backend }}
- name: Retry Set Up Cluster
id: setup_cluster_retry
if: steps.setup_cluster.outcome == 'failure'
env:
ENOS_VAR_ui_run_tests: false
run: enos scenario launch --timeout 60m0s --chdir ./enos ui edition:${{ needs.get-metadata.outputs.vault_edition }} backend:${{ inputs.storage_backend }}
- name: Run UI Tests
id: run_ui_tests
env:
ENOS_VAR_ui_test_filter: "${{ inputs.test_filter }}"
run: enos scenario run --timeout 60m0s --chdir ./enos ui edition:${{ needs.get-metadata.outputs.vault_edition }} backend:${{ inputs.storage_backend }}
- name: Ensure scenario has been destroyed
if: ${{ always() }}
run: enos scenario destroy --timeout 60m0s --chdir ./enos ui edition:${{ needs.get-metadata.outputs.vault_edition }} backend:${{ inputs.storage_backend }}
- name: Clean up Enos runtime directories
if: ${{ always() }}
run: |
rm -rf /tmp/enos*
rm -rf ./enos/support
rm -rf ./enos/.enos

View File

@ -135,6 +135,56 @@ The [`replication` scenario](./enos-scenario-replication.hcl) creates two 3-node
This scenario verifies the performance replication status on both clusters to have their connection_status as "connected" and that the secondary cluster has known_primaries cluster addresses updated to the active nodes IP addresses of the primary Vault cluster. This scenario currently works around issues VAULT-12311 and VAULT-12309. The scenario fails when the primary storage backend is Consul due to issue VAULT-12332
## UI Tests
The [`ui` scenario](./enos-scenario-ui.hcl) creates a Vault cluster (deployed to AWS) using a version
built from the current checkout of the project. Once the cluster is available the UI acceptance tests
are run in a headless browser.
### Variables
In addition to the required variables that must be set, as described in the [Scenario Variables](#Scenario Variables),
the `ui` scenario has two optional variables:
**ui_test_filter** - An optional test filter to limit the tests that are run, i.e. `'!enterprise'`.
To set a filter export the variable as follows:
```shell
> export ENOS_VAR_ui_test_filter="some filter"
```
**ui_run_tests** - An optional boolean variable to run or not run the tests. The default value is true.
Setting this value to false is useful in the case where you want to create a cluster, but run the tests
manually. The section [Running the Tests](#Running the Tests) describes the different ways to run the
'UI' acceptance tests.
### Running the Tests
The UI tests can be run fully automated or manually.
#### Fully Automated
The following will deploy the cluster, run the tests, and subsequently tear down the cluster:
```shell
> export ENOS_VAR_ui_test_filter="some filter" # <-- optional
> cd enos
> enos scenario ui run edition:oss
```
#### Manually
The UI tests can be run manually as follows:
```shell
> export ENOS_VAR_ui_test_filter="some filter" # <-- optional
> export ENOS_VAR_ui_run_tests=false
> cd enos
> enos scenario ui launch edition:oss
# once complete the scenario will output a set of environment variables that must be exported. The
# output will look as follows:
export TEST_FILTER='some filter>' \
export VAULT_ADDR='http://<some ip address>:8200' \
export VAULT_TOKEN='<some token>' \
export VAULT_UNSEAL_KEYS='["<some key>","<some key>","<some key>"]'
# copy and paste the above into the terminal to export the values
> cd ../ui
> yarn test:enos # run headless
# or
> yarn test:enos -s # run manually in a web browser
# once testing is complete
> cd ../enos
> enos scenario ui destroy edition:oss
```
# Variants
Both scenarios support a matrix of variants. In order to achieve broad coverage while
keeping test run time reasonable, the variants executed by the `enos-run` Github

View File

@ -197,3 +197,9 @@ module "vault_raft_remove_peer" {
source = "./modules/vault_raft_remove_peer"
vault_install_dir = var.vault_install_dir
}
module "vault_test_ui" {
source = "./modules/vault_test_ui"
ui_run_tests = var.ui_run_tests
}

View File

@ -380,7 +380,7 @@ scenario "autopilot" {
}
step "verify_undo_logs_status" {
skip_step = semverconstraint(var.vault_product_version, "<1.13.0-0")
skip_step = try(semverconstraint(var.vault_product_version, "<1.13.0-0"), true)
module = module.vault_verify_undo_logs
depends_on = [
step.remove_old_nodes,

205
enos/enos-scenario-ui.hcl Normal file
View File

@ -0,0 +1,205 @@
scenario "ui" {
matrix {
edition = ["oss", "ent"]
backend = ["consul", "raft"]
}
terraform_cli = terraform_cli.default
terraform = terraform.default
providers = [
provider.aws.default,
provider.enos.ubuntu
]
locals {
arch = "amd64"
distro = "ubuntu"
seal = "awskms"
artifact_type = "bundle"
consul_version = "1.14.2"
build_tags = {
"oss" = ["ui"]
"ent" = ["ui", "enterprise", "ent"]
}
bundle_path = abspath(var.vault_bundle_path)
tags = merge({
"Project Name" : var.project_name
"Project" : "Enos",
"Environment" : "ci"
}, var.tags)
vault_instance_types = {
amd64 = "t3a.small"
arm64 = "t4g.small"
}
vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[local.arch])
vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic"))
vault_install_dir_packages = {
rhel = "/bin"
ubuntu = "/usr/bin"
}
vault_install_dir = var.vault_install_dir
ui_test_filter = var.ui_test_filter != null ? var.ui_test_filter : (matrix.edition == "oss") ? "!enterprise" : null
}
step "get_local_metadata" {
module = module.get_local_metadata
}
step "build_vault" {
module = module.build_local
variables {
build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition]
bundle_path = local.bundle_path
goarch = local.arch
goos = "linux"
product_version = var.vault_product_version
artifact_type = local.artifact_type
revision = var.vault_revision
}
}
step "find_azs" {
module = module.az_finder
variables {
instance_type = [
var.backend_instance_type,
local.vault_instance_type
]
}
}
step "create_vpc" {
module = module.create_vpc
variables {
ami_architectures = [local.arch]
availability_zones = step.find_azs.availability_zones
common_tags = local.tags
}
}
step "read_license" {
skip_step = matrix.edition == "oss"
module = module.read_license
variables {
file_name = local.vault_license_path
}
}
step "create_backend_cluster" {
module = "backend_${matrix.backend}"
depends_on = [step.create_vpc]
providers = {
enos = provider.enos.ubuntu
}
variables {
ami_id = step.create_vpc.ami_ids["ubuntu"]["amd64"]
common_tags = local.tags
consul_release = {
edition = var.backend_edition
version = local.consul_version
}
instance_type = var.backend_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" {
module = module.vault_cluster
depends_on = [
step.create_backend_cluster,
step.build_vault,
]
providers = {
enos = provider.enos.ubuntu
}
variables {
ami_id = step.create_vpc.ami_ids[local.distro][local.arch]
common_tags = local.tags
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = matrix.backend
unseal_method = local.seal
vault_local_artifact_path = local.bundle_path
vault_install_dir = local.vault_install_dir
vault_license = matrix.edition != "oss" ? step.read_license.license : null
vpc_id = step.create_vpc.vpc_id
}
}
step "test_ui" {
module = module.vault_test_ui
variables {
vault_addr = step.create_vault_cluster.instance_public_ips[0]
vault_root_token = step.create_vault_cluster.vault_root_token
vault_unseal_keys = step.create_vault_cluster.vault_recovery_keys_b64
vault_recovery_threshold = step.create_vault_cluster.vault_recovery_threshold
ui_test_filter = local.ui_test_filter
}
}
output "vault_cluster_instance_ids" {
description = "The Vault cluster instance IDs"
value = step.create_vault_cluster.instance_ids
}
output "vault_cluster_pub_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.instance_public_ips
}
output "vault_cluster_priv_ips" {
description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips
}
output "vault_cluster_key_id" {
description = "The Vault cluster Key ID"
value = step.create_vault_cluster.key_id
}
output "vault_cluster_root_token" {
description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token
}
output "vault_cluster_unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64
}
output "vault_cluster_unseal_keys_hex" {
description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex
}
output "vault_cluster_tag" {
description = "The Vault cluster tag"
value = step.create_vault_cluster.vault_cluster_tag
}
output "ui_test_stderr" {
description = "The stderr of the ui tests that ran"
value = step.test_ui.ui_test_stderr
}
output "ui_test_stdout" {
description = "The stdout of the ui tests that ran"
value = step.test_ui.ui_test_stdout
}
output "ui_test_environment" {
value = step.test_ui.ui_test_environment
description = "The environment variables that are required in order to run the test:enos yarn target"
}
}

View File

@ -179,3 +179,15 @@ variable "remove_vault_instances" {
description = "The old vault nodes to be removed"
}
variable "ui_test_filter" {
type = string
description = "A test filter to limit the ui tests to execute. Will be appended to the ember test command as '-f=\"<filter>\"'"
default = null
}
variable "ui_run_tests" {
type = bool
description = "Whether to run the UI tests or not. If set to false a cluster will be created but no tests will be run"
default = true
}

View File

@ -0,0 +1,31 @@
terraform {
required_providers {
enos = {
source = "app.terraform.io/hashicorp-qti/enos"
}
}
}
locals {
# base test environment excludes the filter argument
ui_test_environment_base = {
VAULT_ADDR = "http://${var.vault_addr}:8200"
VAULT_TOKEN = var.vault_root_token
VAULT_UNSEAL_KEYS = jsonencode(slice(var.vault_unseal_keys, 0, var.vault_recovery_threshold))
}
ui_test_environment = var.ui_test_filter == null || try(length(trimspace(var.ui_test_filter)) == 0, true) ? local.ui_test_environment_base : merge(local.ui_test_environment_base, {
TEST_FILTER = var.ui_test_filter
})
# The environment variables need to be double escaped since the process of rendering them to the
# outputs eats the escaping. Therefore double escaping ensures that the values are rendered as
# properly escaped json, i.e. "[\"value\"]" suitable to be parsed as json.
escaped_ui_test_environment = [
for key, value in local.ui_test_environment : "export ${key}='${value}'"
]
}
resource "enos_local_exec" "test_ui" {
count = var.ui_run_tests ? 1 : 0
environment = local.ui_test_environment
scripts = ["${path.module}/scripts/test_ui.sh"]
}

View File

@ -0,0 +1,12 @@
output "ui_test_stderr" {
value = var.ui_run_tests ? enos_local_exec.test_ui[0].stderr : "No std out tests where not run"
}
output "ui_test_stdout" {
value = var.ui_run_tests ? enos_local_exec.test_ui[0].stdout : "No std out tests where not run"
}
output "ui_test_environment" {
value = join(" \\ \n", local.escaped_ui_test_environment)
description = "The environment variables that are required in order to run the test:enos yarn target"
}

View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -eux -o pipefail
project_root=$(git rev-parse --show-toplevel)
pushd "$project_root" > /dev/null
echo "running test-ember-enos"
make test-ember-enos
popd > /dev/null

View File

@ -0,0 +1,31 @@
variable "vault_addr" {
description = "The host address for the vault instance to test"
type = string
}
variable "vault_root_token" {
description = "The vault root token"
type = string
}
variable "ui_test_filter" {
type = string
description = "A test filter to limit the ui tests to execute. Will be appended to the ember test command as '-f=<filter>'"
default = null
}
variable "vault_unseal_keys" {
description = "Base64 encoded recovery keys to use for the seal/unseal test"
type = list(string)
}
variable "vault_recovery_threshold" {
description = "The number of recovery keys to require when unsealing Vault"
type = string
}
variable "ui_run_tests" {
type = bool
description = "Whether to run the UI tests or not. If set to false a cluster will be created but no tests will be run"
default = true
}

View File

@ -45,9 +45,8 @@ const testHelper = require('./test-helper');
try {
const testArgs = ['test', '-c', 'testem.enos.js'];
if (process.env.TEST_FILTERS) {
const filters = JSON.parse(process.env.TEST_FILTERS).map((filter) => '-f=' + filter);
testArgs.push(...filters);
if (process.env.TEST_FILTER && process.env.TEST_FILTER.length > 0) {
testArgs.push('-f=' + process.env.TEST_FILTER);
}
await testHelper.run('ember', [...testArgs, ...process.argv.slice(2)], false);