diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 12e1f94f9..8cad99db6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,35 +12,31 @@ env: GO_TAGS: "ui" jobs: - get-product-version: + + product-metadata: runs-on: ubuntu-latest outputs: product-version: ${{ steps.get-product-version.outputs.product-version }} product-base-version: ${{ steps.get-product-version.outputs.product-base-version }} + build-date: ${{ steps.get-build-date.outputs.build-date }} steps: - - uses: actions/checkout@v2 - - name: get product version + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + - name: Get product version id: get-product-version run: | make version IFS="-" read BASE_VERSION _other <<< "$(make version)" echo "::set-output name=product-version::$(make version)" echo "::set-output name=product-base-version::${BASE_VERSION}" - - get-build-date: - runs-on: ubuntu-latest - outputs: - build-date: ${{ steps.get-build-date.outputs.build-date }} - steps: - - uses: actions/checkout@v2 - - name: get build date + - name: Get build date id: get-build-date run: | make build-date echo "::set-output name=build-date::$(make build-date)" generate-metadata-file: - needs: get-product-version + needs: product-metadata runs-on: ubuntu-latest outputs: filepath: ${{ steps.generate-metadata-file.outputs.filepath }} @@ -51,22 +47,20 @@ jobs: id: generate-metadata-file uses: hashicorp/actions-generate-metadata@v1 with: - version: ${{ needs.get-product-version.outputs.product-version }} + version: ${{ needs.product-metadata.outputs.product-version }} product: ${{ env.PKG_NAME }} - - uses: actions/upload-artifact@v2 with: name: metadata.json path: ${{ steps.generate-metadata-file.outputs.filepath }} build-other: - needs: [ get-product-version, get-build-date ] + needs: [ product-metadata ] runs-on: ubuntu-latest strategy: matrix: goos: [ freebsd, windows, netbsd, openbsd, solaris ] goarch: [ "386", "amd64", "arm" ] - go: [ "1.18.4" ] exclude: - goos: solaris goarch: 386 @@ -76,14 +70,14 @@ jobs: goarch: arm fail-fast: true - name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build + name: Go ${{ matrix.goos }} ${{ matrix.goarch }} build steps: - uses: actions/checkout@v2 - name: Setup go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: ${{ matrix.go }} + go-version-file: go.mod - name: Setup node and yarn uses: actions/setup-node@v2 with: @@ -104,32 +98,31 @@ jobs: CGO_ENABLED: 0 run: | mkdir dist out - GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.get-product-version.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.get-build-date.outputs.build-date }}" make build - zip -r -j out/${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ + GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.product-metadata.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.get-build-date.outputs.build-date }}" make build + zip -r -j out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ - uses: actions/upload-artifact@v2 with: - name: ${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - path: out/${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip + name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip + path: out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip build-linux: - needs: [ get-product-version, get-build-date ] + needs: [ product-metadata ] runs-on: ubuntu-latest strategy: matrix: goos: [linux] goarch: ["arm", "arm64", "386", "amd64"] - go: ["1.18.4"] fail-fast: true - name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build + name: Go ${{ matrix.goos }} ${{ matrix.goarch }} build steps: - uses: actions/checkout@v2 - name: Setup go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: ${{ matrix.go }} + go-version-file: go.mod - name: Setup node and yarn uses: actions/setup-node@v2 with: @@ -150,12 +143,12 @@ jobs: CGO_ENABLED: 0 run: | mkdir dist out - GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.get-product-version.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.get-build-date.outputs.build-date }}" make build - zip -r -j out/${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ + GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.product-metadata.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.get-build-date.outputs.build-date }}" make build + zip -r -j out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ - uses: actions/upload-artifact@v2 with: - name: ${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - path: out/${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip + name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip + path: out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - name: Package uses: hashicorp/actions-packaging-linux@v1 @@ -163,7 +156,7 @@ jobs: name: ${{ github.event.repository.name }} description: "Vault is a tool for secrets management, encryption as a service, and privileged access management." arch: ${{ matrix.goarch }} - version: ${{ needs.get-product-version.outputs.product-version }} + version: ${{ needs.product-metadata.outputs.product-version }} maintainer: "HashiCorp" homepage: "https://github.com/hashicorp/vault" license: "MPL-2.0" @@ -189,22 +182,21 @@ jobs: path: out/${{ env.DEB_PACKAGE }} build-darwin: - needs: [ get-product-version, get-build-date ] + needs: [ product-metadata ] runs-on: macos-latest strategy: matrix: goos: [ darwin ] goarch: [ "amd64", "arm64" ] - go: [ "1.18.4" ] fail-fast: true - name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build + name: Go ${{ matrix.goos }} ${{ matrix.goarch }} build steps: - uses: actions/checkout@v2 - name: Setup go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: ${{ matrix.go }} + go-version-file: go.mod - name: Setup node and yarn uses: actions/setup-node@v2 with: @@ -226,17 +218,17 @@ jobs: CGO_ENABLED: 0 run: | mkdir dist out - GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.get-product-version.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.get-build-date.outputs.build-date }}" make build - zip -r -j out/${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ + GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.product-metadata.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.get-build-date.outputs.build-date }}" make build + zip -r -j out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ - uses: actions/upload-artifact@v2 with: - name: ${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - path: out/${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip + name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip + path: out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip build-docker: name: Docker ${{ matrix.arch }} build needs: - - get-product-version + - product-metadata - build-linux runs-on: ubuntu-latest strategy: @@ -244,7 +236,7 @@ jobs: arch: ["arm", "arm64", "386", "amd64"] env: repo: ${{github.event.repository.name}} - version: ${{needs.get-product-version.outputs.product-version}} + version: ${{needs.product-metadata.outputs.product-version}} steps: - uses: actions/checkout@v2 - name: Docker Build (Action) @@ -253,7 +245,7 @@ jobs: version: ${{env.version}} target: default arch: ${{matrix.arch}} - zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_linux_${{ matrix.arch }}.zip + zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_linux_${{ matrix.arch }}.zip tags: | docker.io/hashicorp/${{env.repo}}:${{env.version}} public.ecr.aws/hashicorp/${{env.repo}}:${{env.version}} @@ -261,7 +253,7 @@ jobs: build-ubi: name: Red Hat UBI ${{ matrix.arch }} build needs: - - get-product-version + - product-metadata - build-linux runs-on: ubuntu-latest strategy: @@ -269,7 +261,7 @@ jobs: arch: ["amd64"] env: repo: ${{github.event.repository.name}} - version: ${{needs.get-product-version.outputs.product-version}} + version: ${{needs.product-metadata.outputs.product-version}} steps: - uses: actions/checkout@v2 - name: Docker Build (Action) @@ -278,5 +270,22 @@ jobs: version: ${{env.version}} target: ubi arch: ${{matrix.arch}} - zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_linux_${{ matrix.arch }}.zip + zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_linux_${{ matrix.arch }}.zip redhat_tag: scan.connect.redhat.com/ospid-f0a92725-d8c6-4023-9a87-ba785b94c3fd/${{env.repo}}:${{env.version}}-ubi + + enos: + name: Enos + # Only run the Enos workflow against branches that are created from the + # hashicorp/vault repository. This has the effect of limiting execution of + # Enos scenarios to branches that originate from authors that have write + # access to hashicorp/vault repository. This is required as Github Actions + # will not populate the required secrets for branches created by outside + # contributors in order to protect the secrets integrity. + if: "! github.event.pull_request.head.repo.fork" + needs: + - product-metadata + - build-linux + uses: ./.github/workflows/enos-run.yml + with: + artifact-name: "vault_${{ needs.product-metadata.outputs.product-version }}_linux_amd64.zip" + secrets: inherit diff --git a/.github/workflows/enos-fmt.yml b/.github/workflows/enos-fmt.yml new file mode 100644 index 000000000..c10395c8b --- /dev/null +++ b/.github/workflows/enos-fmt.yml @@ -0,0 +1,27 @@ +--- +name: enos_fmt + +on: + pull_request: + paths: + - enos/** + +jobs: + fmt_check: + # Only run this workflow on pull requests from hashicorp/vault branches + # as we need secrets to install enos. + if: "! github.event.pull_request.head.repo.fork" + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - uses: hashicorp/setup-terraform@v2 + with: + terraform_wrapper: false + - uses: hashicorp/action-setup-enos@v1 + with: + github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + - name: "check formatting" + working-directory: ./enos + run: make check-fmt diff --git a/.github/workflows/enos-run.yml b/.github/workflows/enos-run.yml new file mode 100644 index 000000000..26e63f646 --- /dev/null +++ b/.github/workflows/enos-run.yml @@ -0,0 +1,114 @@ +--- +name: enos + +on: + # Only trigger this working using workflow_call. It assumes that secrets are + # being inherited from the caller. + workflow_call: + inputs: + artifact-name: + required: true + type: string + +env: + PKG_NAME: vault + +jobs: + enos: + name: Integration + strategy: + matrix: + # Run four scenarios to get a maximal distribution of variants in as + # few jobs as possible. + include: + - scenario: "smoke backend:consul consul_version:1.12.3 distro:ubuntu seal:awskms" + aws_region: "us-west-1" + - scenario: "smoke backend:raft consul_version:1.12.3 distro:ubuntu seal:shamir" + aws_region: "us-west-2" + - scenario: "upgrade backend:raft consul_version:1.11.7 distro:rhel seal:shamir" + aws_region: "us-west-1" + - scenario: "upgrade backend:consul consul_version:1.11.7 distro:rhel seal:awskms" + aws_region: "us-west-2" + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Terraform + uses: hashicorp/setup-terraform@v2 + with: + # the Terraform wrapper will break Terraform execution in Enos because + # it changes the output to text when we expect it to be JSON. + terraform_wrapper: false + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ matrix.aws_region }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-skip-session-tagging: true + role-duration-seconds: 3600 + - name: Set up Enos + uses: hashicorp/action-setup-enos@v1 + with: + github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + - name: Set up AWS SSH private key + run: | + mkdir -p ./enos/support + echo "${{ secrets.ENOS_CI_SSH_KEY }}" > ./enos/support/private_key.pem + chmod 600 ./enos/support/private_key.pem + - name: Download Linux AMD64 Vault bundle + id: download + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.artifact-name }} + path: ./enos/support/downloads + - name: Prepare for scenario execution + run: | + unzip ${{steps.download.outputs.download-path}}/*.zip -d enos/support + mv ${{steps.download.outputs.download-path}}/*.zip enos/support/vault.zip + mkdir -p enos/support/terraform-plugin-cache + - name: Run Enos scenario + id: run + # Continue once and retry to handle occasional blips when creating + # infrastructure. + continue-on-error: true + env: + ENOS_VAR_aws_region: ${{ matrix.aws_region }} + ENOS_VAR_aws_ssh_keypair_name: enos-ci-ssh-key + ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem + ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }} + ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache + ENOS_VAR_vault_bundle_path: ./support/vault.zip + run: | + enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }} arch:amd64 builder:crt edition:oss + - name: Retry Enos scenario + id: run_retry + if: steps.run.outcome == 'failure' + env: + ENOS_VAR_aws_region: ${{ matrix.aws_region }} + ENOS_VAR_aws_ssh_keypair_name: enos-ci-ssh-key + ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem + ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }} + ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache + ENOS_VAR_vault_bundle_path: ./support/vault.zip + run: | + enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }} arch:amd64 builder:crt edition:oss + - name: Destroy Enos scenario + if: ${{ always() }} + env: + ENOS_VAR_aws_region: ${{ matrix.aws_region }} + ENOS_VAR_aws_ssh_keypair_name: enos-ci-ssh-key + ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem + ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }} + ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache + ENOS_VAR_vault_bundle_path: ./support/vault.zip + run: | + enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario }} arch:amd64 builder:crt edition:oss + - name: Output debug information on failure + if: ${{ failure() }} + run: | + env + find ./enos -name "scenario.tf" -exec cat {} \; diff --git a/.gitignore b/.gitignore index 75b7fec10..f5793c735 100644 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,11 @@ Vagrantfile !.release/linux/package/etc/vault.d/vault.hcl !command/agent/config/test-fixtures/*.hcl !command/server/test-fixtures/**/*.hcl +!enos/*.hcl +# Enos +enos/.enos +enos/support .DS_Store .idea diff --git a/enos/Makefile b/enos/Makefile new file mode 100644 index 000000000..ad27fb0ff --- /dev/null +++ b/enos/Makefile @@ -0,0 +1,24 @@ +.PHONY: default +default: check-fmt + +.PHONY: check-fmt +check-fmt: check-fmt-enos check-fmt-modules + +.PHONY: fmt +fmt: fmt-enos fmt-modules + +.PHONY: check-fmt-enos +check-fmt-enos: + enos fmt --check --diff . + +.PHONY: fmt-enos +fmt-enos: + enos fmt . + +.PHONY: check-fmt-modules +check-fmt-modules: + terraform fmt -check -diff -recursive ./modules + +.PHONY: fmt-modules +fmt-modules: + terraform fmt -diff -recursive ./modules diff --git a/enos/README.md b/enos/README.md new file mode 100644 index 000000000..7e165b0df --- /dev/null +++ b/enos/README.md @@ -0,0 +1,142 @@ +# Enos + +Enos is an quality testing framework that allows composing and executing quality +requirement scenarios as code. For Vault, it is currently used to perform +infrastructure integration testing using the artifacts that are created as part +of the `build` workflow. While intended to be executed via Github Actions using +the results of the `build` workflow, scenarios are also executable from a developer +machine that has the requisite dependencies and configuration. + +Refer to the [Enos documentation](https://github.com/hashicorp/Enos-Docs) +for further information regarding installation, execution or composing Enos scenarios. + +## When to use Enos +Determining whether to use `vault.NewTestCluster()` or Enos for testing a feature +or scenario is ultimately up to the author. Sometimes one, the other, or both +might be appropriate depending on the requirements. Generally, `vault.NewTestCluster()` +is going to give you faster feedback and execution time, whereas Enos is going +to give you a real-world execution and validation of the requirement. Consider +the following cases as examples of when one might opt for an Enos scenario: + +* The feature require third-party integrations. Whether that be networked + dependencies like a real Consul backend, a real KMS key to test awskms + auto-unseal, auto-join discovery using AWS tags, or Cloud hardware KMS's. +* The feature might behave differently under multiple configuration variants + and therefore should be tested with both combinations, e.g. auto-unseal and + manual shamir unseal or replication in HA mode with integrated storage or + Consul storage. +* The scenario requires coordination between multiple targets. For example, + consider the complex lifecycle event of migrating the seal type or storage, + or manually triggering a raft disaster scenario by partitioning the network + between the leader and follower nodes. Or perhaps an auto-pilot upgrade between + a stable version of Vault and our candidate version. +* The scenario has specific deployment strategy requirements. For example, + if we want to add a regression test for an issue that only arises when the + software is deployed in a certain manner. +* The scenario needs to use actual build artifacts that will be promoted + through the pipeline. + +## Requirements +* AWS access. HashiCorp Vault developers should use Doormat. +* Terraform >= 1.2 +* Enos >= v0.0.10. You can [install it from a release channel](https://github.com/hashicorp/Enos-Docs/blob/main/installation.md). +* Access to the QTI org in Terraform Cloud. HashiCorp Vault developers can + access a shared token in 1Password or request their own in #team-quality on + Slack. +* An SSH keypair in the AWS region you wish to run the scenario. You can use + Doormat to log in to the AWS console to create or upload an existing keypair. +* A Vault install bundle downloaded from releases.hashicorp.com or Artifactory + when using the `builder:crt` variants. When using the `builder:local` variants + Enos will build a Vault bundle from the current branch for you. + +## Scenario Variables +In CI, each scenario is executed via Github Actions and has been configured using +environment variable inputs that follow the `ENOS_VAR_varname` pattern. + +For local execution you can specify all the required variables using environment +variables, or you can update `enos.vars.hcl` with values and uncomment the lines. + +Variables that are required: +* `aws_ssh_keypair_name` +* `aws_ssh_private_key_path` +* `tfc_api_token` +* `vault_bundle_path` +* `vault_license_path` (only required for non-OSS editions) + +See [enos.vars.hcl](./enos.vars.hcl) or [enos-variables.hcl](./enos-variables.hcl) +for further descriptions of the variables. + +## Executing Scenarios +From the `enos` directory: + +```bash +# List all available scenarios +enos scenario list +# Run the smoke or upgrade scenario with an artifact that is built locally. Make sure +# the local machine has been configured as detailed in the requirements +# section. This will execute the scenario and clean up any resources if successful. +enos scenario run smoke builder:local +enos scenario run upgrade builder:local +# To run the same scenario variants that are run in CI, refer to the scenarios listed +# in .github/workflows/enos-run.yml under `jobs.enos.strategy.matrix.include`, +# adding `builder:local` to run locally. +enos scenario run smoke backend:consul consul_version:1.12.3 distro:ubuntu seal:awskms builder:local arch:amd64 edition:oss +# Launch an individual scenario but leave infrastructure up after execution +enos scenario launch smoke builder:local +# Check an individual scenario for validity. This is useful during scenario +# authoring and debugging. +enos scenario validate smoke builder:local +# If you've run the tests and desire to see the outputs, such as the URL or +# credentials, you can run the output command to see them. Please note that +# after "run" or destroy there will be no "outputs" as the infrastructure +# will have been destroyed and state cleared. +enos scenario output smoke builder:local +# Explicitly destroy all existing infrastructure +enos scenario destroy smoke builder:local +``` + +Refer to the [Enos documentation](https://github.com/hashicorp/Enos-Docs) +for further information regarding installation, execution or composing scenarios. + +# Scenarios +There are current two scenarios: `smoke` and `upgrade`. Both begin by building Vault +as specified by the selected `builder` variant (see Variants section below for more +information). + +## Smoke +The [`smoke` scenario](./enos-scenario-smoke.hcl) creates a Vault cluster using +the version from the current branch (either in CI or locally), with the backend +specified by the `backend` variant (`raft` or `consul`). Next, it unseals with the +appropriate method (`awskms` or `shamir`) and performs different verifications +depending on the backend and seal type. + +## Upgrade +The [`upgrade` scenario](./enos-scenario-upgrade.hcl) creates a Vault cluster using +the version specified in `vault_upgrade_initial_release`, with the backend specified +by the `backend` variant (`raft` or `consul`). Next, it upgrades the Vault binary +that is determined by the `builder` variant. After the upgrade, it verifies that +cluster is at the desired version, along with additional verifications. + + +## Autopilot +The [`autopilot` scenario](./enos-scenario-autopilot.hcl) creates a Vault cluster using +the version specified in `vault_upgrade_initial_release`. Next, it creates additional +nodes with the candiate version of Vault as determined by the `builder` variant. +The module uses AWS auto-join to handle discovery and unseals with auto-unseal +or Shamir depending on the `seal` variant. After the new nodes have joined and been +unsealed, it waits for Autopilot to upgrade the new nodes and demote the old nodes. + +# Variants +Both scenarios support a matrix of variants. In order to achieve broad coverage while +keeping test run time reasonable, the variants executed by the `enos-run` Github +Actions are tailored to maximize variant distribution per scenario. + +## `builder:crt` +This variant is designed for use in Github Actions. The `enos-run.yml` workflow +downloads the artifact built by the `build.yml` workflow, unzips it, and sets the +`vault_bundle_path` to the zip file and the `vault_local_binary_path` to the binary. + +## `builder:local` +This variant is for running the Enos scenario locally. It builds the Vault bundle +from the current branch, placing the bundle at the `vault_bundle_path` and the +unzipped Vault binary at the `vault_local_binary_path`. diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl new file mode 100644 index 000000000..5c4b8a3e7 --- /dev/null +++ b/enos/enos-modules.hcl @@ -0,0 +1,96 @@ +module "autopilot_upgrade_storageconfig" { + source = "./modules/autopilot_upgrade_storageconfig" +} + +module "az_finder" { + source = "./modules/az_finder" +} + +module "backend_consul" { + source = "app.terraform.io/hashicorp-qti/aws-consul/enos" + + project_name = var.project_name + environment = "ci" + common_tags = var.tags + ssh_aws_keypair = var.aws_ssh_keypair_name + + # Set this to a real license vault if using an Enterprise edition of Consul + consul_license = var.backend_license_path == null ? "none" : file(abspath(var.backend_license_path)) +} + +module "backend_raft" { + source = "./modules/backend_raft" +} + +module "build_crt" { + source = "./modules/build_crt" +} + +module "build_local" { + source = "./modules/build_local" +} + +module "create_vpc" { + source = "app.terraform.io/hashicorp-qti/aws-infra/enos" + + project_name = var.project_name + environment = "ci" + common_tags = var.tags + ami_architectures = ["amd64", "arm64"] +} + +module "get_local_version_from_make" { + source = "./modules/get_local_version_from_make" +} + +module "read_license" { + source = "./modules/read_license" +} + +module "vault_cluster" { + source = "app.terraform.io/hashicorp-qti/aws-vault/enos" + # source = "../../terraform-enos-aws-vault" + + common_tags = var.tags + environment = "ci" + instance_count = var.vault_instance_count + project_name = var.project_name + ssh_aws_keypair = var.aws_ssh_keypair_name + vault_install_dir = var.vault_install_dir +} + +module "vault_upgrade" { + source = "./modules/vault_upgrade" + + vault_install_dir = var.vault_install_dir + vault_instance_count = var.vault_instance_count +} + +module "vault_verify_autopilot" { + source = "./modules/vault_verify_autopilot" + + vault_autopilot_upgrade_status = "await-server-removal" + vault_install_dir = var.vault_install_dir + vault_instance_count = var.vault_instance_count +} + +module "vault_verify_raft_auto_join_voter" { + source = "./modules/vault_verify_raft_auto_join_voter" + + vault_install_dir = var.vault_install_dir + vault_instance_count = var.vault_instance_count +} + +module "vault_verify_unsealed" { + source = "./modules/vault_verify_unsealed" + + vault_install_dir = var.vault_install_dir + vault_instance_count = var.vault_instance_count +} + +module "vault_verify_version" { + source = "./modules/vault_verify_version" + + vault_install_dir = var.vault_install_dir + vault_instance_count = var.vault_instance_count +} diff --git a/enos/enos-providers.hcl b/enos/enos-providers.hcl new file mode 100644 index 000000000..9301b5503 --- /dev/null +++ b/enos/enos-providers.hcl @@ -0,0 +1,21 @@ +provider "aws" "default" { + region = var.aws_region +} + +provider "enos" "rhel" { + transport = { + ssh = { + user = "ec2-user" + private_key_path = abspath(var.aws_ssh_private_key_path) + } + } +} + +provider "enos" "ubuntu" { + transport = { + ssh = { + user = "ubuntu" + private_key_path = abspath(var.aws_ssh_private_key_path) + } + } +} diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl new file mode 100644 index 000000000..4e744ba20 --- /dev/null +++ b/enos/enos-scenario-autopilot.hcl @@ -0,0 +1,252 @@ +scenario "autopilot" { + matrix { + arch = ["amd64", "arm64"] + builder = ["local", "crt"] + distro = ["ubuntu", "rhel"] + edition = ["ent"] + seal = ["awskms", "shamir"] + } + + terraform_cli = terraform_cli.default + terraform = terraform.default + providers = [ + provider.aws.default, + provider.enos.ubuntu, + provider.enos.rhel + ] + + locals { + build_tags = { + "ent" = ["enterprise", "ent"] + } + bundle_path = abspath(var.vault_bundle_path) + dependencies_to_install = ["jq"] + enos_provider = { + rhel = provider.enos.rhel + ubuntu = provider.enos.ubuntu + } + tags = merge({ + "Project Name" : var.project_name + "Project" : "Enos", + "Environment" : "ci" + }, var.tags) + vault_instance_types = { + amd64 = "t3a.small" + arm64 = "t4g.small" + } + vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + } + + step "build_vault" { + module = matrix.builder == "crt" ? module.build_crt : module.build_local + + variables { + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] + bundle_path = local.bundle_path + goarch = matrix.arch + goos = "linux" + } + } + + step "find_azs" { + module = module.az_finder + + variables { + instance_type = [ + local.vault_instance_type + ] + } + } + + step "create_vpc" { + module = module.create_vpc + + variables { + ami_architectures = [matrix.arch] + } + } + + step "read_license" { + module = module.read_license + + variables { + file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + } + } + + step "create_vault_cluster" { + module = module.vault_cluster + depends_on = [ + step.create_vpc, + step.build_vault, + ] + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + common_tags = local.tags + dependencies_to_install = local.dependencies_to_install + instance_type = local.vault_instance_type + kms_key_arn = step.create_vpc.kms_key_arn + storage_backend = "raft" + storage_backend_addl_config = { + autopilot_upgrade_version = var.vault_autopilot_initial_release.version + } + unseal_method = matrix.seal + vault_release = var.vault_autopilot_initial_release + vault_license = step.read_license.license + vpc_id = step.create_vpc.vpc_id + } + } + + step "get_local_version" { + module = module.get_local_version_from_make + } + + step "create_autopilot_upgrade_storageconfig" { + module = module.autopilot_upgrade_storageconfig + depends_on = [step.get_local_version] + + variables { + vault_product_version = step.get_local_version.version + } + } + + step "upgrade_vault_cluster_with_autopilot" { + module = module.vault_cluster + depends_on = [ + step.create_vault_cluster, + step.create_autopilot_upgrade_storageconfig, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + common_tags = local.tags + dependencies_to_install = local.dependencies_to_install + instance_type = local.vault_instance_type + kms_key_arn = step.create_vpc.kms_key_arn + storage_backend = "raft" + storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config + unseal_method = matrix.seal + vault_cluster_tag = step.create_vault_cluster.vault_cluster_tag + vault_init = false + vault_license = step.read_license.license + vault_local_artifact_path = local.bundle_path + vault_node_prefix = "upgrade_node" + vault_root_token = step.create_vault_cluster.vault_root_token + vault_unseal_when_no_init = matrix.seal == "shamir" + vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null + vpc_id = step.create_vpc.vpc_id + } + } + + step "verify_autopilot_upgraded_vault_cluster" { + module = module.vault_verify_autopilot + depends_on = [step.upgrade_vault_cluster_with_autopilot] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_autopilot_upgrade_version = step.get_local_version.version + vault_instances = step.create_vault_cluster.vault_instances + vault_root_token = step.create_vault_cluster.vault_root_token + } + } + + step "verify_vault_unsealed" { + module = module.vault_verify_unsealed + depends_on = [ + step.create_vault_cluster, + step.upgrade_vault_cluster_with_autopilot, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.create_vault_cluster.vault_instances + vault_root_token = step.create_vault_cluster.vault_root_token + } + } + + step "verify_raft_auto_join_voter" { + module = module.vault_verify_raft_auto_join_voter + depends_on = [ + step.create_vault_cluster, + step.upgrade_vault_cluster_with_autopilot, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.create_vault_cluster.vault_instances + vault_root_token = step.create_vault_cluster.vault_root_token + } + } + + output "vault_cluster_instance_ids" { + description = "The Vault cluster instance IDs" + value = step.create_vault_cluster.instance_ids + } + + output "vault_cluster_pub_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.instance_public_ips + } + + output "vault_cluster_priv_ips" { + description = "The Vault cluster private IPs" + value = step.create_vault_cluster.instance_private_ips + } + + output "vault_cluster_key_id" { + description = "The Vault cluster Key ID" + value = step.create_vault_cluster.key_id + } + + output "vault_cluster_root_token" { + description = "The Vault cluster root token" + value = step.create_vault_cluster.vault_root_token + } + + output "vault_cluster_unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.vault_unseal_keys_b64 + } + + output "vault_cluster_unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.vault_unseal_keys_hex + } + + output "vault_cluster_tag" { + description = "The Vault cluster tag" + value = step.create_vault_cluster.vault_cluster_tag + } + + output "upgraded_vault_cluster_instance_ids" { + description = "The Vault cluster instance IDs" + value = step.upgrade_vault_cluster_with_autopilot.instance_ids + } + + output "upgraded_vault_cluster_pub_ips" { + description = "The Vault cluster public IPs" + value = step.upgrade_vault_cluster_with_autopilot.instance_public_ips + } + + output "upgraded_vault_cluster_priv_ips" { + description = "The Vault cluster private IPs" + value = step.upgrade_vault_cluster_with_autopilot.instance_private_ips + } +} diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl new file mode 100644 index 000000000..f14796c23 --- /dev/null +++ b/enos/enos-scenario-smoke.hcl @@ -0,0 +1,206 @@ +scenario "smoke" { + matrix { + arch = ["amd64", "arm64"] + backend = ["consul", "raft"] + builder = ["local", "crt"] + consul_version = ["1.12.3", "1.11.7", "1.10.12"] + distro = ["ubuntu", "rhel"] + edition = ["oss", "ent"] + seal = ["awskms", "shamir"] + } + + terraform_cli = terraform_cli.default + terraform = terraform.default + providers = [ + provider.aws.default, + provider.enos.ubuntu, + provider.enos.rhel + ] + + locals { + build_tags = { + "oss" = ["ui"] + "ent" = ["enterprise", "ent"] + } + bundle_path = abspath(var.vault_bundle_path) + dependencies_to_install = ["jq"] + enos_provider = { + rhel = provider.enos.rhel + ubuntu = provider.enos.ubuntu + } + tags = merge({ + "Project Name" : var.project_name + "Project" : "Enos", + "Environment" : "ci" + }, var.tags) + vault_instance_types = { + amd64 = "t3a.small" + arm64 = "t4g.small" + } + vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + } + + step "build_vault" { + module = matrix.builder == "crt" ? module.build_crt : module.build_local + + variables { + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] + bundle_path = local.bundle_path + goarch = matrix.arch + goos = "linux" + } + } + + step "find_azs" { + module = module.az_finder + + variables { + instance_type = [ + var.backend_instance_type, + local.vault_instance_type + ] + } + } + + step "create_vpc" { + module = module.create_vpc + + variables { + ami_architectures = [matrix.arch] + availability_zones = step.find_azs.availability_zones + common_tags = local.tags + } + } + + step "read_license" { + skip_step = matrix.edition == "oss" + module = module.read_license + + variables { + file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + } + } + + step "create_backend_cluster" { + module = "backend_${matrix.backend}" + depends_on = [ + step.create_vpc, + step.build_vault, + ] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.create_vpc.ami_ids["ubuntu"][matrix.arch] + common_tags = local.tags + consul_release = { + edition = var.backend_edition + version = matrix.consul_version + } + instance_type = var.backend_instance_type + kms_key_arn = step.create_vpc.kms_key_arn + vpc_id = step.create_vpc.vpc_id + } + } + + step "create_vault_cluster" { + module = module.vault_cluster + depends_on = [ + step.create_vpc, + step.create_backend_cluster, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + common_tags = local.tags + consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag + dependencies_to_install = local.dependencies_to_install + instance_type = local.vault_instance_type + kms_key_arn = step.create_vpc.kms_key_arn + storage_backend = matrix.backend + unseal_method = matrix.seal + vault_local_artifact_path = local.bundle_path + vault_license = matrix.edition != "oss" ? step.read_license.license : null + vpc_id = step.create_vpc.vpc_id + } + } + + step "verify_vault_unsealed" { + module = module.vault_verify_unsealed + depends_on = [ + step.create_vault_cluster, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.create_vault_cluster.vault_instances + vault_root_token = step.create_vault_cluster.vault_root_token + } + } + + step "verify_raft_auto_join_voter" { + skip_step = matrix.backend != "raft" + module = module.vault_verify_raft_auto_join_voter + depends_on = [ + step.create_vault_cluster, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.create_vault_cluster.vault_instances + vault_root_token = step.create_vault_cluster.vault_root_token + } + } + + output "vault_cluster_instance_ids" { + description = "The Vault cluster instance IDs" + value = step.create_vault_cluster.instance_ids + } + + output "vault_cluster_pub_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.instance_public_ips + } + + output "vault_cluster_priv_ips" { + description = "The Vault cluster private IPs" + value = step.create_vault_cluster.instance_private_ips + } + + output "vault_cluster_key_id" { + description = "The Vault cluster Key ID" + value = step.create_vault_cluster.key_id + } + + output "vault_cluster_root_token" { + description = "The Vault cluster root token" + value = step.create_vault_cluster.vault_root_token + } + + output "vault_cluster_unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.vault_unseal_keys_b64 + } + + output "vault_cluster_unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.vault_unseal_keys_hex + } + + output "vault_cluster_tag" { + description = "The Vault cluster tag" + value = step.create_vault_cluster.vault_cluster_tag + } +} diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl new file mode 100644 index 000000000..4193b06fe --- /dev/null +++ b/enos/enos-scenario-upgrade.hcl @@ -0,0 +1,243 @@ +scenario "upgrade" { + matrix { + arch = ["amd64", "arm64"] + backend = ["consul", "raft"] + builder = ["local", "crt"] + consul_version = ["1.12.3", "1.11.7", "1.10.12"] + distro = ["ubuntu", "rhel"] + edition = ["oss", "ent"] + seal = ["awskms", "shamir"] + } + + terraform_cli = terraform_cli.default + terraform = terraform.default + providers = [ + provider.aws.default, + provider.enos.ubuntu, + provider.enos.rhel + ] + + locals { + build_tags = { + "oss" = ["ui"] + "ent" = ["enterprise", "ent"] + } + bundle_path = abspath(var.vault_bundle_path) + dependencies_to_install = ["jq"] + enos_provider = { + rhel = provider.enos.rhel + ubuntu = provider.enos.ubuntu + } + tags = merge({ + "Project Name" : var.project_name + "Project" : "Enos", + "Environment" : "ci" + }, var.tags) + vault_instance_types = { + amd64 = "t3a.small" + arm64 = "t4g.small" + } + vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + } + + step "build_vault" { + module = matrix.builder == "crt" ? module.build_crt : module.build_local + + variables { + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] + bundle_path = local.bundle_path + goarch = matrix.arch + goos = "linux" + } + } + + step "find_azs" { + module = module.az_finder + + variables { + instance_type = [ + var.backend_instance_type, + local.vault_instance_type, + ] + } + } + + step "create_vpc" { + module = module.create_vpc + + variables { + ami_architectures = [matrix.arch] + availability_zones = step.find_azs.availability_zones + common_tags = local.tags + } + } + + step "read_license" { + skip_step = matrix.edition == "oss" + module = module.read_license + + variables { + file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + } + } + + step "create_backend_cluster" { + module = "backend_${matrix.backend}" + depends_on = [ + step.create_vpc, + step.build_vault, + ] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.create_vpc.ami_ids["ubuntu"][matrix.arch] + common_tags = local.tags + consul_release = { + edition = var.backend_edition + version = matrix.consul_version + } + instance_type = var.backend_instance_type + kms_key_arn = step.create_vpc.kms_key_arn + vpc_id = step.create_vpc.vpc_id + } + } + + step "create_vault_cluster" { + module = module.vault_cluster + depends_on = [ + step.create_vpc, + step.create_backend_cluster, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + common_tags = local.tags + consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag + dependencies_to_install = local.dependencies_to_install + instance_type = local.vault_instance_type + kms_key_arn = step.create_vpc.kms_key_arn + storage_backend = matrix.backend + unseal_method = matrix.seal + vault_release = var.vault_upgrade_initial_release + vault_license = matrix.edition != "oss" ? step.read_license.license : null + vpc_id = step.create_vpc.vpc_id + } + } + + step "upgrade_vault" { + module = module.vault_upgrade + depends_on = [ + step.create_vault_cluster, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_api_addr = "http://localhost:8200" + vault_instances = step.create_vault_cluster.vault_instances + vault_local_bundle_path = local.bundle_path + vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null + vault_seal_type = matrix.seal + } + } + + step "verify_vault_version" { + module = module.vault_verify_version + depends_on = [ + step.create_backend_cluster, + step.upgrade_vault, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.create_vault_cluster.vault_instances + } + } + + step "verify_vault_unsealed" { + module = module.vault_verify_unsealed + depends_on = [ + step.create_vault_cluster, + step.upgrade_vault, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.create_vault_cluster.vault_instances + vault_root_token = step.create_vault_cluster.vault_root_token + } + } + + step "verify_raft_auto_join_voter" { + skip_step = matrix.backend != "raft" + module = module.vault_verify_raft_auto_join_voter + depends_on = [ + step.create_backend_cluster, + step.upgrade_vault, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.create_vault_cluster.vault_instances + vault_root_token = step.create_vault_cluster.vault_root_token + } + } + + output "vault_cluster_instance_ids" { + description = "The Vault cluster instance IDs" + value = step.create_vault_cluster.instance_ids + } + + output "vault_cluster_pub_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.instance_public_ips + } + + output "vault_cluster_priv_ips" { + description = "The Vault cluster private IPs" + value = step.create_vault_cluster.instance_private_ips + } + + output "vault_cluster_key_id" { + description = "The Vault cluster Key ID" + value = step.create_vault_cluster.key_id + } + + output "vault_cluster_root_token" { + description = "The Vault cluster root token" + value = step.create_vault_cluster.vault_root_token + } + + output "vault_cluster_unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.vault_unseal_keys_b64 + } + + output "vault_cluster_unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.vault_unseal_keys_hex + } + + output "vault_cluster_tag" { + description = "The Vault cluster tag" + value = step.create_vault_cluster.vault_cluster_tag + } +} diff --git a/enos/enos-terraform.hcl b/enos/enos-terraform.hcl new file mode 100644 index 000000000..0c88cc791 --- /dev/null +++ b/enos/enos-terraform.hcl @@ -0,0 +1,30 @@ +terraform_cli "default" { + plugin_cache_dir = var.terraform_plugin_cache_dir != null ? abspath(var.terraform_plugin_cache_dir) : null + + credentials "app.terraform.io" { + token = var.tfc_api_token + } + + /* + provider_installation { + dev_overrides = { + "app.terraform.io/hashicorp-qti/enos" = abspath("../../enos-provider") + } + direct {} + } + */ +} + +terraform "default" { + required_version = ">= 1.2.0" + + required_providers { + aws = { + source = "hashicorp/aws" + } + + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} diff --git a/enos/enos-variables.hcl b/enos/enos-variables.hcl new file mode 100644 index 000000000..f62dfc12d --- /dev/null +++ b/enos/enos-variables.hcl @@ -0,0 +1,111 @@ +variable "aws_region" { + description = "The AWS region where we'll create infrastructure" + type = string + default = "us-west-1" +} + +variable "aws_ssh_keypair_name" { + description = "The AWS keypair to use for SSH" + type = string + default = "enos-ci-ssh-key" +} + +variable "aws_ssh_private_key_path" { + description = "The path to the AWS keypair private key" + type = string + default = "./support/private_key.pem" +} + +variable "backend_edition" { + description = "The backend release edition if applicable" + type = string + default = "oss" +} + +variable "backend_instance_type" { + description = "The instance type to use for the Vault backend" + type = string + default = "t3.small" +} + +variable "backend_license_path" { + description = "The license for the backend if applicable (Consul Enterprise)" + type = string + default = null +} + +variable "project_name" { + description = "The description of the project" + type = string + default = "vault-enos-integration" +} + +variable "tags" { + description = "Tags that will be applied to infrastructure resources that support tagging" + type = map(string) + default = null +} + +variable "terraform_plugin_cache_dir" { + description = "The directory to cache Terraform modules and providers" + type = string + default = null +} + +variable "tfc_api_token" { + description = "The Terraform Cloud QTI Organization API token." + type = string +} + +variable "vault_autopilot_initial_release" { + description = "The Vault release to deploy before upgrading with autopilot" + default = { + edition = "ent" + version = "1.11.0" + } +} + +variable "vault_bundle_path" { + description = "Path to CRT generated or local vault.zip bundle" + type = string + default = "/tmp/vault.zip" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the vault binary will be installed" + default = "/opt/vault/bin" +} + +variable "vault_instance_type" { + description = "The instance type to use for the Vault backend" + type = string + default = null +} + +variable "vault_instance_count" { + description = "How many instances to create for the Vault cluster" + type = number + default = 3 +} + +variable "vault_license_path" { + description = "The path to a valid Vault enterprise edition license. This is only required for non-oss editions" + type = string + default = null +} + +variable "vault_local_build_tags" { + description = "The build tags to pass to the Go compiler for builder:local variants" + type = list(string) + default = null +} + +variable "vault_upgrade_initial_release" { + description = "The Vault release to deploy before upgrading" + default = { + edition = "oss" + // vault 1.10.5 has a known issue with retry_join. + version = "1.10.4" + } +} diff --git a/enos/enos.vars.hcl b/enos/enos.vars.hcl new file mode 100644 index 000000000..d157c7b92 --- /dev/null +++ b/enos/enos.vars.hcl @@ -0,0 +1,48 @@ +# aws_region is the AWS region where we'll create infrastructure +# for the smoke scenario +# aws_region = "us-west-1" + +# aws_ssh_keypair_name is the AWS keypair to use for SSH +# aws_ssh_keypair_name = "enos-ci-ssh-key" + +# aws_ssh_private_key_path is the path to the AWS keypair private key +# aws_ssh_private_key_path = "./support/private_key.pem" + +# backend_instance_type is the instance type to use for the Vault backend +# backend_instance_type = "t3.small" + +# tags are a map of tags that will be applied to infrastructure resources that +# support tagging. +# tags = { "Project Name" : "Vault", "Something Cool" : "Value" } + +# terraform_plugin_cache_dir is the directory to cache Terraform modules and providers. +# It must exist. +# terraform_plugin_cache_dir = "/Users//.terraform/plugin-cache-dir + +# tfc_api_token is the Terraform Cloud QTI Organization API token. We need this +# to download the enos Terraform provider and the enos Terraform modules. +# tfc_api_token = "XXXXX.atlasv1.XXXXX..." + +# vault_bundle_path is the path to CRT generated or local vault.zip bundle. When +# using the "builder:local" variant a bundle will be built from the current branch. +# In CI it will use the output of the build workflow. +# vault_bundle_path = "./dist/vault.zip" + +# vault_install_dir is the directory where the vault binary will be installed on +# the remote machines. +# vault_install_dir = "/opt/vault/bin" + +# vault_local_binary_path is the path of the local binary that we're upgrading to. +# vault_local_binary_path = "./support/vault" + +# vault_instance_type is the instance type to use for the Vault backend +# vault_instance_type = "t3.small" + +# vault_instance_count is how many instances to create for the Vault cluster. +# vault_instance_count = 3 + +# vault_license_path is the path to a valid Vault enterprise edition license. +# This is only required for non-oss editions" +# vault_license_path = "./support/vault.hclic" + +# vault_upgrade_initial_release is the Vault release to deploy before upgrading. diff --git a/enos/modules/autopilot_upgrade_storageconfig/main.tf b/enos/modules/autopilot_upgrade_storageconfig/main.tf new file mode 100644 index 000000000..6093b8b10 --- /dev/null +++ b/enos/modules/autopilot_upgrade_storageconfig/main.tf @@ -0,0 +1,7 @@ +variable "vault_product_version" {} + +output "storage_addl_config" { + value = { + autopilot_upgrade_version = var.vault_product_version + } +} diff --git a/enos/modules/az_finder/main.tf b/enos/modules/az_finder/main.tf new file mode 100644 index 000000000..b55975578 --- /dev/null +++ b/enos/modules/az_finder/main.tf @@ -0,0 +1,25 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + } +} + +variable "instance_type" { + default = ["t3.small"] + type = list(string) +} + +data "aws_ec2_instance_type_offerings" "infra" { + filter { + name = "instance-type" + values = var.instance_type + } + + location_type = "availability-zone" +} + +output "availability_zones" { + value = data.aws_ec2_instance_type_offerings.infra.locations +} diff --git a/enos/modules/backend_raft/main.tf b/enos/modules/backend_raft/main.tf new file mode 100644 index 000000000..4cb8e58a5 --- /dev/null +++ b/enos/modules/backend_raft/main.tf @@ -0,0 +1,46 @@ +// Shim module to handle the fact that Vault doesn't actually need a backend module +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "ami_id" { + default = null +} +variable "common_tags" { + default = null +} +variable "consul_license" { + default = null +} +variable "consul_release" { + default = null +} +variable "environment" { + default = null +} +variable "instance_type" { + default = null +} +variable "kms_key_arn" { + default = null +} +variable "project_name" { + default = null +} +variable "ssh_aws_keypair" { + default = null +} +variable "vpc_id" { + default = null +} + +output "consul_cluster_tag" { + value = null +} diff --git a/enos/modules/build_crt/main.tf b/enos/modules/build_crt/main.tf new file mode 100644 index 000000000..6e54287d2 --- /dev/null +++ b/enos/modules/build_crt/main.tf @@ -0,0 +1,20 @@ +# Shim module since CRT provided things will use the crt_bundle_path variable +variable "bundle_path" { + default = "/tmp/vault.zip" +} + +variable "build_tags" { + default = ["ui"] +} + +variable "goarch" { + type = string + description = "The Go architecture target" + default = "amd64" +} + +variable "goos" { + type = string + description = "The Go OS target" + default = "linux" +} diff --git a/enos/modules/build_local/main.tf b/enos/modules/build_local/main.tf new file mode 100644 index 000000000..563e970f0 --- /dev/null +++ b/enos/modules/build_local/main.tf @@ -0,0 +1,38 @@ +terraform { + required_providers { + enos = { + source = "hashicorp.com/qti/enos" + } + } +} + +variable "bundle_path" { + type = string + default = "/tmp/vault.zip" +} + +variable "build_tags" { + type = list(string) + description = "The build tags to pass to the Go compiler" +} + +variable "goarch" { + type = string + description = "The Go architecture target" + default = "amd64" +} + +variable "goos" { + type = string + description = "The Go OS target" + default = "linux" +} + +resource "enos_local_exec" "build" { + content = templatefile("${path.module}/templates/build.sh", { + bundle_path = var.bundle_path, + build_tags = join(" ", var.build_tags) + goarch = var.goarch + goos = var.goos + }) +} diff --git a/enos/modules/build_local/templates/build.sh b/enos/modules/build_local/templates/build.sh new file mode 100755 index 000000000..661df529b --- /dev/null +++ b/enos/modules/build_local/templates/build.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -eux -o pipefail + +# Requirements +npm install --global yarn || true + +# Set up the environment for building Vault. +root_dir="$(git rev-parse --show-toplevel)" + +pushd "$root_dir" > /dev/null + +export GO_TAGS=${build_tags} +export CGO_ENABLED=0 + +IFS="-" read -r BASE_VERSION _other <<< "$(make version)" +export VAULT_VERSION=$BASE_VERSION + +build_date="$(make build-date)" +export VAULT_BUILD_DATE=$build_date + +revision="$(git rev-parse HEAD)" +export VAULT_REVISION=$revision +popd > /dev/null + +# Go to the UI directory of the Vault repo and build the UI +pushd "$root_dir/ui" > /dev/null +yarn install --ignore-optional +npm rebuild node-sass +yarn --verbose run build +popd > /dev/null + +# Build for linux/amd64 and create a bundle since we're deploying it to linux/amd64 +pushd "$root_dir" > /dev/null +export GOARCH=${goarch} +export GOOS=${goos} +make build + +zip -r -j ${bundle_path} dist/ +popd > /dev/null diff --git a/enos/modules/get_local_version_from_make/main.tf b/enos/modules/get_local_version_from_make/main.tf new file mode 100644 index 000000000..a7dcf6be7 --- /dev/null +++ b/enos/modules/get_local_version_from_make/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +resource "enos_local_exec" "get_version" { + scripts = ["${path.module}/scripts/version.sh"] +} + +output "version" { + value = trimspace(enos_local_exec.get_version.stdout) +} diff --git a/enos/modules/get_local_version_from_make/scripts/version.sh b/enos/modules/get_local_version_from_make/scripts/version.sh new file mode 100755 index 000000000..13970ead9 --- /dev/null +++ b/enos/modules/get_local_version_from_make/scripts/version.sh @@ -0,0 +1,10 @@ +#!/bin/env bash +set -eu -o pipefail + +# Set up the environment for building Vault. +root_dir="$(git rev-parse --show-toplevel)" + +pushd "$root_dir" > /dev/null + +IFS="-" read -r VAULT_VERSION _other <<< "$(make version)" +echo $VAULT_VERSION diff --git a/enos/modules/read_license/main.tf b/enos/modules/read_license/main.tf new file mode 100644 index 000000000..1b645272a --- /dev/null +++ b/enos/modules/read_license/main.tf @@ -0,0 +1,5 @@ +variable "file_name" {} + +output "license" { + value = file(var.file_name) +} diff --git a/enos/modules/vault_upgrade/main.tf b/enos/modules/vault_upgrade/main.tf new file mode 100644 index 000000000..375be70d6 --- /dev/null +++ b/enos/modules/vault_upgrade/main.tf @@ -0,0 +1,163 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "vault_api_addr" { + type = string + description = "The API address of the Vault cluster" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_instance_count" { + type = number + description = "How many vault instances are in the cluster" +} + +variable "vault_instances" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The vault cluster instances that were created" +} + +variable "vault_local_bundle_path" { + type = string + description = "The path to the local Vault (vault.zip) bundle" +} + +variable "vault_seal_type" { + type = string + description = "The Vault seal type" +} + +variable "vault_unseal_keys" { + type = list(string) + description = "The keys to use to unseal Vault when not using auto-unseal" + default = null +} + +locals { + instances = { + for idx in range(var.vault_instance_count) : idx => { + public_ip = values(var.vault_instances)[idx].public_ip + private_ip = values(var.vault_instances)[idx].private_ip + } + } + followers = toset([for idx in range(var.vault_instance_count - 1) : tostring(idx)]) + follower_ips = compact(split(" ", enos_remote_exec.get_follower_public_ips.stdout)) + vault_bin_path = "${var.vault_install_dir}/vault" +} + +resource "enos_bundle_install" "upgrade_vault_binary" { + for_each = local.instances + + destination = var.vault_install_dir + path = var.vault_local_bundle_path + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_remote_exec" "get_leader_public_ip" { + depends_on = [enos_bundle_install.upgrade_vault_binary] + + content = templatefile("${path.module}/templates/get-leader-public-ip.sh", { + vault_install_dir = var.vault_install_dir, + vault_instances = jsonencode(local.instances) + }) + + transport = { + ssh = { + host = local.instances[0].public_ip + } + } +} + +resource "enos_remote_exec" "get_follower_public_ips" { + depends_on = [enos_bundle_install.upgrade_vault_binary] + + content = templatefile("${path.module}/templates/get-follower-public-ips.sh", { + vault_install_dir = var.vault_install_dir, + vault_instances = jsonencode(local.instances) + }) + + transport = { + ssh = { + host = local.instances[0].public_ip + } + } +} + +resource "enos_remote_exec" "restart_followers" { + for_each = local.followers + depends_on = [enos_remote_exec.get_follower_public_ips] + + content = file("${path.module}/templates/restart-vault.sh") + + transport = { + ssh = { + host = trimspace(local.follower_ips[tonumber(each.key)]) + } + } +} + +resource "enos_vault_unseal" "followers" { + depends_on = [enos_remote_exec.restart_followers] + for_each = { + for idx, follower in local.followers : idx => follower + if var.vault_seal_type == "shamir" + } + bin_path = local.vault_bin_path + vault_addr = var.vault_api_addr + seal_type = var.vault_seal_type + unseal_keys = var.vault_unseal_keys + + transport = { + ssh = { + host = trimspace(local.follower_ips[each.key]) + } + } +} + +resource "enos_remote_exec" "restart_leader" { + depends_on = [enos_vault_unseal.followers] + + content = file("${path.module}/templates/restart-vault.sh") + + transport = { + ssh = { + host = trimspace(enos_remote_exec.get_leader_public_ip.stdout) + } + } +} + +resource "enos_vault_unseal" "leader" { + count = var.vault_seal_type == "shamir" ? 1 : 0 + depends_on = [enos_remote_exec.restart_leader] + + bin_path = local.vault_bin_path + vault_addr = var.vault_api_addr + seal_type = var.vault_seal_type + unseal_keys = var.vault_unseal_keys + + transport = { + ssh = { + host = trimspace(enos_remote_exec.get_leader_public_ip.stdout) + } + } +} diff --git a/enos/modules/vault_upgrade/templates/get-follower-public-ips.sh b/enos/modules/vault_upgrade/templates/get-follower-public-ips.sh new file mode 100644 index 000000000..e424aa444 --- /dev/null +++ b/enos/modules/vault_upgrade/templates/get-follower-public-ips.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +binpath=${vault_install_dir}/vault +export VAULT_ADDR="http://localhost:8200" + +instances='${vault_instances}' + +# Find the leader +leader_address=$($binpath status -format json | jq '.leader_address | rtrimstr(":8200") | ltrimstr("http://")') + +# Get the public ip addresses of the followers +follower_ips=$(jq ".[] | select(.private_ip!=$leader_address) | .public_ip" <<< "$instances") + +echo "$follower_ips" | sed 's/\"//g' | tr '\n' ' ' diff --git a/enos/modules/vault_upgrade/templates/get-leader-public-ip.sh b/enos/modules/vault_upgrade/templates/get-leader-public-ip.sh new file mode 100644 index 000000000..5c36dae33 --- /dev/null +++ b/enos/modules/vault_upgrade/templates/get-leader-public-ip.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +binpath=${vault_install_dir}/vault +export VAULT_ADDR="http://localhost:8200" + +instances='${vault_instances}' + +# Find the leader +leader_address=$($binpath status -format json | jq '.leader_address | rtrimstr(":8200") | ltrimstr("http://")') + +# Get the public ip address of the leader +leader_public=$(jq ".[] | select(.private_ip==$leader_address) | .public_ip" <<< "$instances") +echo "$leader_public" | sed 's/\"//g' diff --git a/enos/modules/vault_upgrade/templates/restart-vault.sh b/enos/modules/vault_upgrade/templates/restart-vault.sh new file mode 100644 index 000000000..aa6853643 --- /dev/null +++ b/enos/modules/vault_upgrade/templates/restart-vault.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eux + +sudo systemctl restart vault diff --git a/enos/modules/vault_verify_autopilot/main.tf b/enos/modules/vault_verify_autopilot/main.tf new file mode 100644 index 000000000..af9a673e6 --- /dev/null +++ b/enos/modules/vault_verify_autopilot/main.tf @@ -0,0 +1,68 @@ +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_instance_count" { + type = number + description = "How many vault instances are in the cluster" +} + +variable "vault_instances" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The vault cluster instances that were created" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +variable "vault_autopilot_upgrade_version" { + type = string + description = "The directory where the Vault binary will be installed" + default = null +} + +variable "vault_autopilot_upgrade_status" { + type = string + description = "The directory where the Vault binary will be installed" + default = null +} + +locals { + public_ips = { + for idx in range(var.vault_instance_count) : idx => { + public_ip = values(var.vault_instances)[idx].public_ip + private_ip = values(var.vault_instances)[idx].private_ip + } + } +} + +resource "enos_remote_exec" "smoke-verify-autopilot" { + for_each = local.public_ips + + content = templatefile("${path.module}/templates/smoke-verify-autopilot.sh", { + vault_install_dir = var.vault_install_dir + vault_token = var.vault_root_token + vault_autopilot_upgrade_status = var.vault_autopilot_upgrade_status, + vault_autopilot_upgrade_version = var.vault_autopilot_upgrade_version, + }) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} diff --git a/enos/modules/vault_verify_autopilot/templates/smoke-verify-autopilot.sh b/enos/modules/vault_verify_autopilot/templates/smoke-verify-autopilot.sh new file mode 100755 index 000000000..1dd5d9014 --- /dev/null +++ b/enos/modules/vault_verify_autopilot/templates/smoke-verify-autopilot.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +token="${vault_token}" +autopilot_version="${vault_autopilot_upgrade_version}" +autopilot_status="${vault_autopilot_upgrade_status}" + +export VAULT_ADDR="http://localhost:8200" +export VAULT_TOKEN="$token" + +function fail() { + echo "$1" 1>&2 + exit 1 +} + +count=0 +retries=7 +while :; do + state=$(${vault_install_dir}/vault read -format=json sys/storage/raft/autopilot/state) + status="$(jq -r '.data.upgrade_info.status' <<< "$state")" + target_version="$(jq -r '.data.upgrade_info.target_version' <<< "$state")" + + if [ "$status" = "$autopilot_status" ] && [ "$target_version" = "$autopilot_version" ]; then + exit 0 + fi + + wait=$((2 ** count)) + count=$((count + 1)) + if [ "$count" -lt "$retries" ]; then + echo "$state" + sleep "$wait" + else + fail "Autopilot did not get into the correct status" + fi +done diff --git a/enos/modules/vault_verify_raft_auto_join_voter/main.tf b/enos/modules/vault_verify_raft_auto_join_voter/main.tf new file mode 100644 index 000000000..ded9c3cc7 --- /dev/null +++ b/enos/modules/vault_verify_raft_auto_join_voter/main.tf @@ -0,0 +1,62 @@ +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "vault_cluster_addr_port" { + description = "The Raft cluster address port" + type = string + default = "8201" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_instance_count" { + type = number + description = "How many vault instances are in the cluster" +} + +variable "vault_instances" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The vault cluster instances that were created" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +locals { + instances = { + for idx in range(var.vault_instance_count) : idx => { + public_ip = values(var.vault_instances)[idx].public_ip + private_ip = values(var.vault_instances)[idx].private_ip + } + } +} + +resource "enos_remote_exec" "verify_raft_auto_join_voter" { + for_each = local.instances + + content = templatefile("${path.module}/templates/verify-raft-auto-join-voter.sh", { + vault_cluster_addr = "${each.value.private_ip}:${var.vault_cluster_addr_port}" + vault_install_dir = var.vault_install_dir + vault_local_binary_path = "${var.vault_install_dir}/vault" + vault_token = var.vault_root_token + }) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} diff --git a/enos/modules/vault_verify_raft_auto_join_voter/templates/verify-raft-auto-join-voter.sh b/enos/modules/vault_verify_raft_auto_join_voter/templates/verify-raft-auto-join-voter.sh new file mode 100644 index 000000000..e1172d715 --- /dev/null +++ b/enos/modules/vault_verify_raft_auto_join_voter/templates/verify-raft-auto-join-voter.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -e + +binpath=${vault_install_dir}/vault + +fail() { + echo "$1" 2>&1 + return 1 +} + +retry() { + local retries=$1 + shift + local count=0 + + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ "$count" -lt "$retries" ]; then + sleep "$wait" + echo "retry $count" + else + return "$exit" + fi + done + + return 0 +} + +check_voter_status() { + voter_status=$($binpath operator raft list-peers -format json | jq -Mr --argjson expected "true" '.data.config.servers[] | select(.address=="${vault_cluster_addr}") | .voter == $expected') + + if [[ "$voter_status" != 'true' ]]; then + fail "expected ${vault_cluster_addr} to be raft voter, got raft status for node: $($binpath operator raft list-peers -format json | jq '.data.config.servers[] | select(.address==${vault_cluster_addr})')" + fi +} + +test -x "$binpath" || fail "unable to locate vault binary at $binpath" + +export VAULT_ADDR='http://127.0.0.1:8200' +export VAULT_TOKEN='${vault_token}' + +# Retry a few times because it can take some time for things to settle after +# all the nodes are unsealed +retry 5 check_voter_status diff --git a/enos/modules/vault_verify_unsealed/main.tf b/enos/modules/vault_verify_unsealed/main.tf new file mode 100644 index 000000000..ead8d0181 --- /dev/null +++ b/enos/modules/vault_verify_unsealed/main.tf @@ -0,0 +1,62 @@ +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "vault_cluster_addr_port" { + description = "The Raft cluster address port" + type = string + default = "8201" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_instance_count" { + type = number + description = "How many vault instances are in the cluster" +} + +variable "vault_instances" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The vault cluster instances that were created" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +locals { + instances = { + for idx in range(var.vault_instance_count) : idx => { + public_ip = values(var.vault_instances)[idx].public_ip + private_ip = values(var.vault_instances)[idx].private_ip + } + } +} + +resource "enos_remote_exec" "verify_raft_auto_join_voter" { + for_each = local.instances + + content = templatefile("${path.module}/templates/verify-vault-node-unsealed.sh", { + vault_cluster_addr = "${each.value.private_ip}:${var.vault_cluster_addr_port}" + vault_install_dir = var.vault_install_dir + vault_local_binary_path = "${var.vault_install_dir}/vault" + vault_token = var.vault_root_token + }) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} diff --git a/enos/modules/vault_verify_unsealed/templates/verify-vault-node-unsealed.sh b/enos/modules/vault_verify_unsealed/templates/verify-vault-node-unsealed.sh new file mode 100644 index 000000000..de3edd648 --- /dev/null +++ b/enos/modules/vault_verify_unsealed/templates/verify-vault-node-unsealed.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +binpath=${vault_install_dir}/vault + +fail() { + echo "$1" 1>&2 + return 1 +} + +test -x "$binpath" || fail "unable to locate vault binary at $binpath" + +export VAULT_ADDR='http://127.0.0.1:8200' +export VAULT_TOKEN='${vault_token}' + +unseal_status=$($binpath status -format json | jq -Mr --argjson expected "false" '.sealed == $expected') +if [[ "$unseal_status" != 'true' ]]; then + fail "expected ${vault_cluster_addr} to be unsealed, got unseal status: $unseal_status" +fi diff --git a/enos/modules/vault_verify_version/main.tf b/enos/modules/vault_verify_version/main.tf new file mode 100644 index 000000000..fddb765bf --- /dev/null +++ b/enos/modules/vault_verify_version/main.tf @@ -0,0 +1,48 @@ +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_instance_count" { + type = number + description = "How many vault instances are in the cluster" +} + +variable "vault_instances" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The vault cluster instances that were created" +} + +locals { + instances = { + for idx in range(var.vault_instance_count) : idx => { + public_ip = values(var.vault_instances)[idx].public_ip + private_ip = values(var.vault_instances)[idx].private_ip + } + } +} + +resource "enos_remote_exec" "verify_all_nodes_have_updated_version" { + for_each = local.instances + + content = templatefile("${path.module}/templates/verify-cluster-version.sh", { + vault_install_dir = var.vault_install_dir, + }) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} diff --git a/enos/modules/vault_verify_version/templates/verify-cluster-version.sh b/enos/modules/vault_verify_version/templates/verify-cluster-version.sh new file mode 100644 index 000000000..402ca8235 --- /dev/null +++ b/enos/modules/vault_verify_version/templates/verify-cluster-version.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# The Vault smoke test to verify the Vault version installed + +set -e + +binpath=${vault_install_dir}/vault + +fail() { + echo "$1" 1>&2 + exit 1 +} + +test -x "$binpath" || fail "unable to locate vault binary at $binpath" + +binary_version_full=$($binpath version) +# Get the Vault build tag +binary_version=$(cut -d ' ' -f2 <<< $binary_version_full) +# Strip the leading v +semantic=$${binary_version:1} +# Get the build timestamp +build_date=$(cut -d ' ' -f5 <<< $binary_version_full) + +export VAULT_ADDR='http://127.0.0.1:8200' + +# Ensure that the cluster version and build time match the binary installed +vault_status=$("$binpath" status -format json) +result=$(jq -Mr \ + --arg version "$semantic" \ + --arg build_date "$build_date" \ + 'select(.version == $version) | .build_date == $build_date' \ + <<< $vault_status +) + +if [[ "$result" != "true" ]]; then + fail "expected version $binary_version with build_date $build_date, got status $vault_status" +fi