Update Vault version verification integration test (#17377)

* Update Vault version verification integration test

Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com>

* Refactor to validate artifact build date

Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com>

* Address review comments

Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com>

* Fix capitalization in variable descriptions

Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com>

Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com>

.github/workflows/build.yml

@@ -16,12 +16,16 @@ jobs:
   product-metadata:
     runs-on: ubuntu-latest
     outputs:
+      product-revision: ${{ steps.get-product-revision.outputs.product-revision }}
       product-version: ${{ steps.get-product-version.outputs.product-version }}
       product-base-version: ${{ steps.get-product-version.outputs.product-base-version }}
       build-date: ${{ steps.get-build-date.outputs.build-date }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
+      - name: Get product revision
+        id: get-product-revision
+        run: echo "::set-output name=product-revision::$(git rev-parse HEAD)"
       - name: Get product version
         id: get-product-version
         run: |
@@ -287,5 +291,8 @@ jobs:
       - build-linux
     uses: ./.github/workflows/enos-run.yml
     with:
+      artifact-build-date: "${{needs.product-metadata.outputs.build-date}}"
       artifact-name: "vault_${{ needs.product-metadata.outputs.product-version }}_linux_amd64.zip"
+      artifact-revision: "${{needs.product-metadata.outputs.product-revision}}"
+      artifact-version: "${{needs.product-metadata.outputs.product-version}}"
     secrets: inherit

.github/workflows/enos-run.yml

@@ -6,13 +6,25 @@ on:
   # being inherited from the caller.
   workflow_call:
     inputs:
+      artifact-build-date:
+        required: false
+        type: string
       artifact-name:
         required: true
         type: string
+      artifact-revision:
+        required: true
+        type: string
+      artifact-version:
+        required: true
+        type: string
 
 env:
   PKG_NAME: vault
+  ARTIFACT_BUILD_DATE: ${{ inputs.artifact-build-date }}
   ARTIFACT_NAME: ${{ inputs.artifact-name }}
+  ARTIFACT_REVISION: ${{ inputs.artifact-revision }}
+  ARTIFACT_VERSION: ${{ inputs.artifact-version }}
 
 jobs:
   # Read Enos scenario matrix file based on artifact-name input to test
@@ -88,6 +100,9 @@ jobs:
           ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem
           ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
           ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
+          ENOS_VAR_vault_build_date: ${{ env.ARTIFACT_BUILD_DATE }}
+          ENOS_VAR_vault_product_version: ${{ env.ARTIFACT_VERSION }}
+          ENOS_VAR_vault_revision: ${{ env.ARTIFACT_REVISION }}
           ENOS_VAR_vault_bundle_path: ./support/vault.zip
         run: |
           enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }}
@@ -100,6 +115,9 @@ jobs:
           ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem
           ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
           ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
+          ENOS_VAR_vault_build_date: ${{ env.ARTIFACT_BUILD_DATE }}
+          ENOS_VAR_vault_product_version: ${{ env.ARTIFACT_VERSION }}
+          ENOS_VAR_vault_revision: ${{ env.ARTIFACT_REVISION }}
           ENOS_VAR_vault_bundle_path: ./support/vault.zip
         run: |
           enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }}
@@ -114,6 +132,9 @@ jobs:
           ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem
           ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
           ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
+          ENOS_VAR_vault_build_date: ${{ env.ARTIFACT_BUILD_DATE }}
+          ENOS_VAR_vault_product_version: ${{ env.ARTIFACT_VERSION }}
+          ENOS_VAR_vault_revision: ${{ env.ARTIFACT_REVISION }}
           ENOS_VAR_vault_bundle_path: ./support/vault.zip
         run: |
           enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario }}

enos/enos-modules.hcl

@@ -39,8 +39,8 @@ module "create_vpc" {
   ami_architectures = ["amd64", "arm64"]
 }
 
-module "get_local_version_from_make" {
-  source = "./modules/get_local_version_from_make"
+module "get_local_metadata" {
+  source = "./modules/get_local_metadata"
 }
 
 module "read_license" {

enos/enos-scenario-autopilot.hcl

@@ -104,16 +104,16 @@ scenario "autopilot" {
     }
   }
 
-  step "get_local_version" {
-    module = module.get_local_version_from_make
+  step "get_local_metadata" {
+    skip_step = matrix.builder != "local"
+    module    = module.get_local_metadata
   }
 
   step "create_autopilot_upgrade_storageconfig" {
-    module     = module.autopilot_upgrade_storageconfig
-    depends_on = [step.get_local_version]
+    module = module.autopilot_upgrade_storageconfig
 
     variables {
-      vault_product_version = step.get_local_version.version
+      vault_product_version = matrix.builder == "local" ? step.get_local_metadata.version : var.vault_product_version
     }
   }
 
@@ -158,7 +158,7 @@ scenario "autopilot" {
     }
 
     variables {
-      vault_autopilot_upgrade_version = step.get_local_version.version
+      vault_autopilot_upgrade_version = matrix.builder == "local" ? step.get_local_metadata.version : var.vault_product_version
       vault_instances                 = step.create_vault_cluster.vault_instances
       vault_root_token                = step.create_vault_cluster.vault_root_token
     }

enos/enos-scenario-smoke.hcl

@@ -105,6 +105,11 @@ scenario "smoke" {
     }
   }
 
+  step "get_local_metadata" {
+    skip_step = matrix.builder != "local"
+    module    = module.get_local_metadata
+  }
+
   step "create_vault_cluster" {
     module = module.vault_cluster
     depends_on = [
@@ -131,6 +136,26 @@ scenario "smoke" {
     }
   }
 
+  step "verify_vault_version" {
+    module = module.vault_verify_version
+    depends_on = [
+      step.create_vault_cluster,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_instances       = step.create_vault_cluster.vault_instances
+      vault_edition         = matrix.edition
+      vault_product_version = matrix.builder == "local" ? step.get_local_metadata.version : var.vault_product_version
+      vault_revision        = matrix.builder == "local" ? step.get_local_metadata.revision : var.vault_revision
+      vault_build_date      = matrix.builder == "local" ? step.get_local_metadata.build_date : var.vault_build_date
+      vault_root_token      = step.create_vault_cluster.vault_root_token
+    }
+  }
+
   step "verify_vault_unsealed" {
     module = module.vault_verify_unsealed
     depends_on = [

enos/enos-scenario-upgrade.hcl

@@ -81,6 +81,11 @@ scenario "upgrade" {
     }
   }
 
+  step "get_local_metadata" {
+    skip_step = matrix.builder != "local"
+    module    = module.get_local_metadata
+  }
+
   step "create_backend_cluster" {
     module = "backend_${matrix.backend}"
     depends_on = [
@@ -162,7 +167,12 @@ scenario "upgrade" {
     }
 
     variables {
-      vault_instances = step.create_vault_cluster.vault_instances
+      vault_instances       = step.create_vault_cluster.vault_instances
+      vault_edition         = matrix.edition
+      vault_product_version = matrix.builder == "local" ? step.get_local_metadata.version : var.vault_product_version
+      vault_revision        = matrix.builder == "local" ? step.get_local_metadata.revision : var.vault_revision
+      vault_build_date      = matrix.builder == "local" ? step.get_local_metadata.build_date : var.vault_build_date
+      vault_root_token      = step.create_vault_cluster.vault_root_token
     }
   }
 

enos/enos-variables.hcl

@@ -73,7 +73,7 @@ variable "vault_bundle_path" {
 
 variable "vault_install_dir" {
   type        = string
-  description = "The directory where the vault binary will be installed"
+  description = "The directory where the Vault binary will be installed"
   default     = "/opt/vault/bin"
 }
 
@@ -101,11 +101,29 @@ variable "vault_local_build_tags" {
   default     = null
 }
 
+variable "vault_build_date" {
+  description = "The build date for Vault artifact"
+  type        = string
+  default     = ""
+}
+
+variable "vault_product_version" {
+  description = "The version of Vault we are testing"
+  type        = string
+  default     = null
+}
+
+variable "vault_revision" {
+  description = "The git sha of Vault artifact we are testing"
+  type        = string
+  default     = null
+}
+
 variable "vault_upgrade_initial_release" {
   description = "The Vault release to deploy before upgrading"
   default = {
     edition = "oss"
-    // vault 1.10.5 has a known issue with retry_join.
+    // Vault 1.10.5 has a known issue with retry_join.
     version = "1.10.4"
   }
 }

enos/modules/get_local_metadata/main.tf

@@ -0,0 +1,31 @@
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+resource "enos_local_exec" "get_build_date" {
+  scripts = ["${path.module}/scripts/build_date.sh"]
+}
+
+output "build_date" {
+  value = trimspace(enos_local_exec.get_build_date.stdout)
+}
+
+resource "enos_local_exec" "get_version" {
+  scripts = ["${path.module}/scripts/version.sh"]
+}
+
+output "version" {
+  value = trimspace(enos_local_exec.get_version.stdout)
+}
+
+resource "enos_local_exec" "get_revision" {
+  inline = ["git rev-parse HEAD"]
+}
+
+output "revision" {
+  value = trimspace(enos_local_exec.get_revision.stdout)
+}

enos/modules/get_local_metadata/scripts/build_date.sh

@@ -0,0 +1,10 @@
+#!/bin/env bash
+set -eu -o pipefail
+
+# Set up the environment for building Vault.
+root_dir="$(git rev-parse --show-toplevel)"
+
+pushd "$root_dir" > /dev/null
+
+IFS="-" read -r VAULT_BUILD_DATE _other <<< "$(make build-date)"
+echo $VAULT_BUILD_DATE

enos/modules/get_local_version_from_make/main.tf

@@ -1,15 +0,0 @@
-terraform {
-  required_providers {
-    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
-    }
-  }
-}
-
-resource "enos_local_exec" "get_version" {
-  scripts = ["${path.module}/scripts/version.sh"]
-}
-
-output "version" {
-  value = trimspace(enos_local_exec.get_version.stdout)
-}

enos/modules/vault_verify_version/main.tf

@@ -6,6 +6,12 @@ terraform {
   }
 }
 
+variable "vault_build_date" {
+  type        = string
+  description = "The Vault artifact build date"
+  default     = null
+}
+
 variable "vault_install_dir" {
   type        = string
   description = "The directory where the Vault binary will be installed"
@@ -13,7 +19,7 @@ variable "vault_install_dir" {
 
 variable "vault_instance_count" {
   type        = number
-  description = "How many vault instances are in the cluster"
+  description = "How many Vault instances are in the cluster"
 }
 
 variable "vault_instances" {
@@ -21,7 +27,31 @@ variable "vault_instances" {
     private_ip = string
     public_ip  = string
   }))
-  description = "The vault cluster instances that were created"
+  description = "The Vault cluster instances that were created"
+}
+
+variable "vault_product_version" {
+  type        = string
+  description = "The Vault product version"
+  default     = null
+}
+
+variable "vault_edition" {
+  type        = string
+  description = "The Vault product edition"
+  default     = null
+}
+
+variable "vault_revision" {
+  type        = string
+  description = "The Vault product revision"
+  default     = null
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The Vault root token"
+  default     = null
 }
 
 locals {
@@ -38,6 +68,11 @@ resource "enos_remote_exec" "verify_all_nodes_have_updated_version" {
 
   content = templatefile("${path.module}/templates/verify-cluster-version.sh", {
     vault_install_dir = var.vault_install_dir,
+    vault_build_date  = var.vault_build_date,
+    vault_version     = var.vault_product_version,
+    vault_edition     = var.vault_edition,
+    vault_revision    = var.vault_revision,
+    vault_token       = var.vault_root_token,
   })
 
   transport = {

enos/modules/vault_verify_version/templates/verify-cluster-version.sh

@@ -5,6 +5,11 @@
 set -e
 
 binpath=${vault_install_dir}/vault
+edition=${vault_edition}
+version=${vault_version}
+sha=${vault_revision}
+builddate=${vault_build_date}
+release="$version+$edition"
 
 fail() {
 	echo "$1" 1>&2
@@ -13,25 +18,37 @@ fail() {
 
 test -x "$binpath" || fail "unable to locate vault binary at $binpath"
 
-binary_version_full=$($binpath version)
-# Get the Vault build tag
-binary_version=$(cut -d ' ' -f2 <<< $binary_version_full)
-# Strip the leading v
-semantic=$${binary_version:1}
-# Get the build timestamp
-build_date=$(cut -d ' ' -f5 <<< $binary_version_full)
-
 export VAULT_ADDR='http://127.0.0.1:8200'
+export VAULT_TOKEN='${vault_token}'
 
-# Ensure that the cluster version and build time match the binary installed
-vault_status=$("$binpath" status -format json)
-result=$(jq -Mr \
-  --arg version "$semantic" \
-  --arg build_date "$build_date" \
-  'select(.version == $version) | .build_date == $build_date' \
-  <<< $vault_status
-)
-
-if [[ "$result" != "true" ]]; then
-  fail "expected version $binary_version with build_date $build_date, got status $vault_status"
+if [[ "$builddate" != "" ]]; then
+  build_date=$builddate
+else
+  build_date=$("$binpath" status -format=json | jq -Mr .build_date)
+fi
+
+if [[ "$(echo $version |awk -F'.' '{print $2}')" -ge 11 ]]; then
+  version_expected="Vault v$release ($sha), built $build_date"
+else
+  version_expected="Vault v$release ($sha)"
+fi
+
+case "$release" in
+  *+oss) [[ "$(echo $version |awk -F'.' '{print $2}')" -ge 11 ]] && \
+    version_expected="Vault v$version ($sha), built $build_date" || \
+    version_expected="Vault v$version ($sha)" ;;
+  *+ent) ;;
+  *+ent.hsm) version_expected="$version_expected (cgo)";;
+  *+ent.fips1402) version_expected="$version_expected (cgo)" ;;
+  *+ent.hsm.fips1402) version_expected="$version_expected (cgo)" ;;
+  *) fail "($release) file doesn't match any known license types"
+esac
+
+version_expected_nosha=$(echo "$version_expected" | awk '!($3="")' | sed 's/  / /' | sed -e 's/[[:space:]]*$//')
+version_output=$("$binpath" version)
+
+if [[ "$version_output" == "$version_expected_nosha" ]] || [[ "$version_output" == "$version_expected" ]]; then
+  echo "Version verification succeeded!"
+else
+  fail "expected Version=$version_expected or $version_expected_nosha, got: $version_output"
 fi