Refactor Enos scenario matrix generation (#17060)

* Refactor Enos scenario matrix generation * Generate scenario matrix based on artifact edition to test * Configure Vault license for testing Ent artifact * Run Autopilot scenario for Ent Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> Cleanup Enos runtime Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> Fix syntax Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> Use script to generate Enos scenario matrix Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> Checkout repo to generate matrix Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> Fix matrix syntax Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> Fix json format Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> Update Enos scenario license condition Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> * Address review feedback Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> * Fix syntax Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> * Update json format for scenario matrix Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> * Address review comments Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com> Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com>
2022-09-14 14:17:07 -04:00 · 2022-09-14 14:17:07 -04:00 · 2231f588a5
parent f6de4f75b4
commit 2231f588a5
3 changed files with 70 additions and 19 deletions
--- a/.github/enos-run-matrices/ent.json
+++ b/.github/enos-run-matrices/ent.json
@ -0,0 +1,24 @@
+{
+  "include": [
+    {
+      "scenario": "smoke backend:consul consul_version:1.12.3 distro:ubuntu seal:awskms arch:amd64 builder:crt edition:ent",
+      "aws_region": "us-west-1"
+    },
+    {
+      "scenario": "smoke backend:raft consul_version:1.12.3 distro:ubuntu seal:shamir arch:amd64 builder:crt edition:ent",
+      "aws_region": "us-west-2"
+    },
+    {
+      "scenario": "upgrade backend:raft consul_version:1.11.7 distro:rhel seal:shamir arch:amd64 builder:crt edition:ent",
+      "aws_region": "us-west-1"
+    },
+    {
+      "scenario": "upgrade backend:consul consul_version:1.11.7 distro:rhel seal:awskms arch:amd64 builder:crt edition:ent",
+      "aws_region": "us-west-2"
+    },
+    {
+      "scenario": "autopilot distro:ubuntu seal:shamir arch:amd64 builder:crt edition:ent",
+      "aws_region": "us-west-1"
+    }
+  ]
+}
--- a/.github/enos-run-matrices/oss.json
+++ b/.github/enos-run-matrices/oss.json
@ -0,0 +1,20 @@
+{
+  "include": [
+    {
+      "scenario": "smoke backend:consul consul_version:1.12.3 distro:ubuntu seal:awskms arch:amd64 builder:crt edition:oss",
+      "aws_region": "us-west-1"
+    },
+    {
+      "scenario": "smoke backend:raft consul_version:1.12.3 distro:ubuntu seal:shamir arch:amd64 builder:crt edition:oss",
+      "aws_region": "us-west-2"
+    },
+    {
+      "scenario": "upgrade backend:raft consul_version:1.11.7 distro:rhel seal:shamir arch:amd64 builder:crt edition:oss",
+      "aws_region": "us-west-1"
+    },
+    {
+      "scenario": "upgrade backend:consul consul_version:1.11.7 distro:rhel seal:awskms arch:amd64 builder:crt edition:oss",
+      "aws_region": "us-west-2"
+    }
+  ]
+}
--- a/.github/workflows/enos-run.yml
+++ b/.github/workflows/enos-run.yml
@ -12,24 +12,29 @@ on:

 env:
  PKG_NAME: vault
+  ARTIFACT_NAME: ${{ inputs.artifact-name }}

 jobs:
+  # Read Enos scenario matrix file based on artifact-name input to test
+  read-enos-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      enos-scenarios: ${{ steps.enos-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Create Enos scenario matrix
+        id: enos-matrix
+        run: |
+          [[ ${{ env.ARTIFACT_NAME }} == *"ent"* ]] && scenarioFile=$(cat ./.github/enos-run-matrices/ent.json |jq -c .) || scenarioFile=$(cat ./.github/enos-run-matrices/oss.json |jq -c .)
+          echo "::set-output name=matrix::$scenarioFile"
+  # Run Integration tests on Enos scenario matrix
  enos:
    name: Integration
+    needs: read-enos-matrix
    strategy:
      fail-fast: false # don't fail as that can skip required cleanup steps for jobs
-      matrix:
-        # Run four scenarios to get a maximal distribution of variants in as
-        # few jobs as possible.
-        include:
-          - scenario: "smoke backend:consul consul_version:1.12.3 distro:ubuntu seal:awskms"
-            aws_region: "us-west-1"
-          - scenario: "smoke backend:raft consul_version:1.12.3 distro:ubuntu seal:shamir"
-            aws_region: "us-west-2"
-          - scenario: "upgrade backend:raft consul_version:1.11.7 distro:rhel seal:shamir"
-            aws_region: "us-west-1"
-          - scenario: "upgrade backend:consul consul_version:1.11.7 distro:rhel seal:awskms"
-            aws_region: "us-west-2"
+      matrix: ${{ fromJson(needs.read-enos-matrix.outputs.enos-scenarios) }}
    runs-on: ubuntu-latest
    env:
      GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
@ -71,6 +76,7 @@ jobs:
          unzip ${{steps.download.outputs.download-path}}/*.zip -d enos/support
          mv ${{steps.download.outputs.download-path}}/*.zip enos/support/vault.zip
          mkdir -p enos/support/terraform-plugin-cache
+          [[ ${{ env.ARTIFACT_NAME }} == *"ent"* ]] && echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true
      - name: Run Enos scenario
        id: run
        # Continue once and retry to handle occasional blips when creating
@ -84,7 +90,7 @@ jobs:
          ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
          ENOS_VAR_vault_bundle_path: ./support/vault.zip
        run: |
-          enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }} arch:amd64 builder:crt edition:oss
+          enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }}
      - name: Retry Enos scenario
        id: run_retry
        if: steps.run.outcome == 'failure'
@ -96,7 +102,7 @@ jobs:
          ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
          ENOS_VAR_vault_bundle_path: ./support/vault.zip
        run: |
-          enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }} arch:amd64 builder:crt edition:oss
+          enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }}
      - name: Destroy Enos scenario
        if: ${{ always() }}
        env:
@ -107,9 +113,10 @@ jobs:
          ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
          ENOS_VAR_vault_bundle_path: ./support/vault.zip
        run: |
-          enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario }} arch:amd64 builder:crt edition:oss
-      - name: Output debug information on failure
-        if: ${{ failure() }}
+          enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario }}
+      - name: Cleanup Enos runtime directories
+        if: ${{ always() }}
        run: |
-          env
-          find ./enos -name "scenario.tf" -exec cat {} \;
+          rm -rf /tmp/enos*
+          rm -rf ./enos/support
+          rm -rf ./enos/.enos