From 4c4798417ff191b21acadbe860310f72f39437d1 Mon Sep 17 00:00:00 2001 From: Ryan Cragun Date: Fri, 11 Nov 2022 13:14:43 -0700 Subject: [PATCH] [QT-358] Unify CRT and local builder workflows (#17766) Here we make the following major changes: * Centralize CRT builder logic into a script utility so that we can share the logic for building artifacts in CI or locally. * Simplify the build workflow by calling a reusable workflow many times instead of repeating the contents. * Create a workflow that validates whether or not the build workflow and all child workflows have succeeded to allow for merge protection. Motivation * We need branch requirements for the build workflow and all subsequent integration tests (QT-353) * We need to ensure that the Enos local builder works (QT-558) * Debugging build failures can be difficult because one has to hand craft the steps to recreate the build * Merge conflicts between Vault OSS and Vault ENT build workflows are quite painful. As the build workflow must be the same file and name we'll reduce what is contained in each that is unique. Implementations of building will be unique per edition so we don't have to worry about conflict resolution. * Since we're going to be touching the build workflow to do the first two items we might as well try and improve those other issues at the same time to reduce the overhead of backports and conflicts. Considerations * Build logic for Vault OSS and Vault ENT differs * The Enos local builder was duplicating a lot of what we did in the CRT build workflow * Version and other artifact metadata has been an issue before. Debugging it has been tedious and error prone. * The build workflow is full of brittle copy and paste that is hard to understand, especially for all of the release editions in Vault Enterprise * Branch check requirements for workflows are incredibly painful to use for workflows that are dynamic or change often. The required workflows have to be configured in Github settings by administrators. They would also prevent us from having simple docs PRs since required integration workflows always have to run to satisfy branch requirements. * Doormat credentials requirements that are coming will require us to modify which event types trigger workflows. This changes those ahead of time since we're doing so much to build workflow. The only noticeable impact will be that the build workflow no longer runs on pushes to non-main or release branches. In order to test other branches it requires a workflow_dispatch from the Actions tab or a pull request. Solutions * Centralize the logic that determines build metadata and creates releasable Vault artifacts. Instead of cargo-culting logic multiple times in the build workflow and the Enos local modules, we now have a crt-builder script which determines build metadata and also handles building the UI, Vault, and the package bundle. There are make targets for all of the available sub-commands. Now what we use in the pipeline is the same thing as the local builder, and it can be executed locally by developers. The crt-builder script works in OSS and Enterprise so we will never have to deal with them being divergent or with special casing things in the build workflow. * Refactor the bulk of the Vault building into a reusable workflow that we can call multiple times. This allows us to define Vault builds in a much simpler manner and makes resolving merge conflicts much easier. * Rather than trying to maintain a list and manually configure the branch check requirements for build, we'll trigger a single workflow that uses the github event system to determine if the build workflow (all of the sub-workflows included) have passed. We'll then create branch restrictions on that single workflow down the line. Signed-off-by: Ryan Cragun me@ryan.ec --- .github/workflows/build-docs.yml | 20 + .github/workflows/build-ensure-success.yml | 32 ++ .github/workflows/build-vault-oss.yml | 109 +++++ .github/workflows/build.yml | 380 ++++++------------ .github/workflows/enos-fmt.yml | 2 +- .github/workflows/enos-run.yml | 2 +- .github/workflows/enos-verify-stable.yml | 8 +- Makefile | 56 ++- enos/enos-scenario-agent.hcl | 3 +- enos/enos-scenario-autopilot.hcl | 5 +- enos/enos-scenario-smoke.hcl | 5 +- enos/enos-scenario-upgrade.hcl | 5 +- enos/modules/build_local/main.tf | 14 +- enos/modules/build_local/scripts/build.sh | 12 + enos/modules/build_local/templates/build.sh | 39 -- .../get_local_metadata/scripts/build_date.sh | 10 +- .../get_local_metadata/scripts/version.sh | 10 +- scripts/build.sh | 4 +- scripts/build_date.sh | 6 - scripts/crt-builder.sh | 234 +++++++++++ scripts/version.sh | 12 - 21 files changed, 615 insertions(+), 353 deletions(-) create mode 100644 .github/workflows/build-docs.yml create mode 100644 .github/workflows/build-ensure-success.yml create mode 100644 .github/workflows/build-vault-oss.yml create mode 100755 enos/modules/build_local/scripts/build.sh delete mode 100755 enos/modules/build_local/templates/build.sh delete mode 100755 scripts/build_date.sh create mode 100755 scripts/crt-builder.sh delete mode 100755 scripts/version.sh diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml new file mode 100644 index 000000000..ee24bb9ab --- /dev/null +++ b/.github/workflows/build-docs.yml @@ -0,0 +1,20 @@ +--- +name: build-docs + +on: + push: + branches: + - docs/** + - backport/docs/** + +jobs: + # This is a workaround to allow docs PRs to fullfil the required branch + # protection rules which require that the 'build' workflow has + # succeeded before a PR is allowed to be merged. We validate that via the + # 'build-ensure-success' workflow that is triggered either by either the + # 'build' or 'build-docs' workflow. Without this trigger the required branch + # protection rules would never be met on docs branches. + trigger-build-success-workflow: + runs-on: ubuntu-latest + steps: + - run: exit 0 diff --git a/.github/workflows/build-ensure-success.yml b/.github/workflows/build-ensure-success.yml new file mode 100644 index 000000000..aa0c8e8dc --- /dev/null +++ b/.github/workflows/build-ensure-success.yml @@ -0,0 +1,32 @@ +--- +name: ensure_build_workflow_succeeded + +on: + # Github's branch protection rules for required workflow checks are a bit + # cumbersome when you have many jobs. The required checks have to be configured + # at the workflow job level. As child workflows of build.yml and Enos can be + # dynamic and change often, there's no way to create or maintain a valid list + # of workflow checks that need to succeeded in order to merge. + # + # To work around this problem we've chosen to trigger this workflow on + # completion of all the jobs that are triggered by build or build-docs. This + # workflow inspects the conclusion of the build workflow event and either passes + # or fails. This allows our branch protection rules for the build workflow to + # depend only on this workflow succeeding, which can only happen if all child + # workflows of the build job have succeeded. + workflow_run: + workflows: [build, build-docs] + types: [completed] + +jobs: + ensure-all-build-workflows-succeeded: + name: Ensure that all workflows spawned by the build workflow succeeded + runs-on: ubuntu-latest + steps: + - if: ${{ github.event.workflow_run.conclusion != 'success' }} + uses: actions/github-script@v6 + with: + script: | + core.setFailed('One or more workflows spawned by the build job did not succeed. All build job workflows are required to pass before merge') + - if: ${{ github.event.workflow_run.conclusion == 'success' }} + run: echo "All build and integration workflows have succeeded!" diff --git a/.github/workflows/build-vault-oss.yml b/.github/workflows/build-vault-oss.yml new file mode 100644 index 000000000..9dbc36336 --- /dev/null +++ b/.github/workflows/build-vault-oss.yml @@ -0,0 +1,109 @@ +--- +name: build_vault + +# This workflow is intended to be called by the build workflow for each Vault +# binary that needs to be built and packaged. The crt make targets that are +# utilized automatically determine build metadata and handle building and +# packing vault. + +on: + workflow_call: + inputs: + bundle-path: + required: false + type: string + cgo-enabled: + type: string + default: 0 + create-packages: + type: boolean + default: true + goos: + required: true + type: string + goarch: + required: true + type: string + go-tags: + type: string + go-version: + type: string + package-name: + type: string + default: vault + vault-version: + type: string + required: true + +jobs: + build: + runs-on: "ubuntu-latest" + name: Vault ${{ inputs.goos }} ${{ inputs.goarch }} v${{ inputs.vault-version }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: ${{ inputs.go-version }} + - name: Set up node and yarn + uses: actions/setup-node@v3 + with: + node-version: 14 + cache: yarn + cache-dependency-path: ui/yarn.lock + - name: Build UI + run: make crt-build-ui + - name: Build Vault + env: + CGO_ENABLED: ${{ inputs.cgo-enabled }} + GOARCH: ${{ inputs.goarch }} + GOOS: ${{ inputs.goos }} + GO_TAGS: ${{ inputs.go-tags }} + run: make crt-build + - name: Determine artifact basename + env: + GOARCH: ${{ inputs.goarch }} + GOOS: ${{ inputs.goos }} + run: echo "ARTIFACT_BASENAME=$(make crt-get-artifact-basename)" >> $GITHUB_ENV + - name: Bundle Vault + env: + BUNDLE_PATH: out/${{ env.ARTIFACT_BASENAME }}.zip + run: make crt-bundle + - uses: actions/upload-artifact@v3 + with: + name: ${{ env.ARTIFACT_BASENAME }}.zip + path: out/${{ env.ARTIFACT_BASENAME }}.zip + if-no-files-found: error + - if: ${{ inputs.create-packages }} + uses: hashicorp/actions-packaging-linux@v1 + with: + name: ${{ github.event.repository.name }} + description: Vault is a tool for secrets management, encryption as a service, and privileged access management. + arch: ${{ inputs.goarch }} + version: ${{ inputs.vault-version }} + maintainer: HashiCorp + homepage: https://github.com/hashicorp/vault + license: MPL-2.0 + binary: dist/${{ inputs.package-name }} + deb_depends: openssl + rpm_depends: openssl + config_dir: .release/linux/package/ + preinstall: .release/linux/preinst + postinstall: .release/linux/postinst + postremove: .release/linux/postrm + - if: ${{ inputs.create-packages }} + name: Determine package file names + run: | + echo "RPM_PACKAGE=$(basename out/*.rpm)" >> $GITHUB_ENV + echo "DEB_PACKAGE=$(basename out/*.deb)" >> $GITHUB_ENV + - if: ${{ inputs.create-packages }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.RPM_PACKAGE }} + path: out/${{ env.RPM_PACKAGE }} + if-no-files-found: error + - if: ${{ inputs.create-packages }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.DEB_PACKAGE }} + path: out/${{ env.DEB_PACKAGE }} + if-no-files-found: error diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0cf2cbb03..5d24f2884 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,69 +2,63 @@ name: build on: workflow_dispatch: - push: + pull_request: + types: + - opened + - reopened + - synchronize branches-ignore: - docs/** - backport/docs/** + push: + branches: + - main + - release/** env: - PKG_NAME: "vault" - GO_TAGS: "ui" + PKG_NAME: vault jobs: - product-metadata: runs-on: ubuntu-latest outputs: - product-revision: ${{ steps.get-product-revision.outputs.product-revision }} - product-version: ${{ steps.get-product-version.outputs.product-version }} - product-base-version: ${{ steps.get-product-version.outputs.product-base-version }} - build-date: ${{ steps.get-build-date.outputs.build-date }} + build-date: ${{ steps.get-metadata.outputs.build-date }} + filepath: ${{ steps.generate-metadata-file.outputs.filepath }} + go-version: ${{ steps.get-metadata.outputs.go-version }} + package-name: ${{ steps.get-metadata.outputs.package-name }} + vault-revision: ${{ steps.get-metadata.outputs.vault-revision }} + vault-version: ${{ steps.get-metadata.outputs.vault-version }} + vault-base-version: ${{ steps.get-metadata.outputs.vault-base-version }} steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 - - name: Get product revision - id: get-product-revision - run: echo "::set-output name=product-revision::$(git rev-parse HEAD)" - - name: Get product version - id: get-product-version + - name: Get metadata + id: get-metadata run: | - make version - IFS="-" read BASE_VERSION _other <<< "$(make version)" - echo "::set-output name=product-version::$(make version)" - echo "::set-output name=product-base-version::${BASE_VERSION}" - - name: Get build date - id: get-build-date - run: | - make build-date - echo "::set-output name=build-date::$(make build-date)" - - generate-metadata-file: - needs: product-metadata - runs-on: ubuntu-latest - outputs: - filepath: ${{ steps.generate-metadata-file.outputs.filepath }} - steps: - - name: 'Checkout directory' - uses: actions/checkout@v2 - - name: Generate metadata file + echo "build-date=$(make crt-get-date)" >> $GITHUB_OUTPUT + echo "package-name=${{ env.PKG_NAME }}" >> $GITHUB_OUTPUT + echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT + echo "vault-base-version=$(make crt-get-version-base)" >> $GITHUB_OUTPUT + echo "vault-revision=$(make crt-get-revision)" >> $GITHUB_OUTPUT + echo "vault-version=$(make crt-get-version)" >> $GITHUB_OUTPUT + - uses: hashicorp/actions-generate-metadata@v1 id: generate-metadata-file - uses: hashicorp/actions-generate-metadata@v1 with: - version: ${{ needs.product-metadata.outputs.product-version }} - product: ${{ env.PKG_NAME }} - - uses: actions/upload-artifact@v2 + version: ${{ steps.get-metadata.outputs.vault-version }} + product: ${{ steps.get-metadata.outputs.package-name }} + - uses: actions/upload-artifact@v3 with: name: metadata.json path: ${{ steps.generate-metadata-file.outputs.filepath }} + if-no-files-found: error build-other: - needs: [ product-metadata ] - runs-on: ubuntu-latest + name: Build Vault Other + needs: product-metadata strategy: matrix: - goos: [ freebsd, windows, netbsd, openbsd, solaris ] - goarch: [ "386", "amd64", "arm" ] + goos: [freebsd, windows, netbsd, openbsd, solaris] + goarch: [386, amd64, arm] exclude: - goos: solaris goarch: 386 @@ -73,229 +67,99 @@ jobs: - goos: windows goarch: arm fail-fast: true - - name: Go ${{ matrix.goos }} ${{ matrix.goarch }} build - - steps: - - uses: actions/checkout@v2 - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version-file: go.mod - - name: Setup node and yarn - uses: actions/setup-node@v2 - with: - node-version: '14' - cache: 'yarn' - cache-dependency-path: 'ui/yarn.lock' - - name: UI Build - run: | - cd ui - yarn install --ignore-optional - npm rebuild node-sass - yarn --verbose run build - cd .. - - name: Build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: 0 - run: | - mkdir dist out - GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.product-metadata.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.product-metadata.outputs.build-date }}" make build - zip -r -j out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - path: out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip + uses: ./.github/workflows/build-vault-oss.yml + with: + create-packages: false + goarch: ${{ matrix.goarch }} + goos: ${{ matrix.goos }} + go-tags: ui + go-version: ${{ needs.product-metadata.outputs.go-version }} + package-name: ${{ needs.product-metadata.outputs.package-name }} + vault-version: ${{ needs.product-metadata.outputs.vault-version }} + secrets: inherit build-linux: - needs: [ product-metadata ] - runs-on: ubuntu-latest + name: Build Vault Linux + needs: product-metadata strategy: matrix: goos: [linux] - goarch: ["arm", "arm64", "386", "amd64"] + goarch: [arm, arm64, 386, amd64] fail-fast: true - - name: Go ${{ matrix.goos }} ${{ matrix.goarch }} build - - steps: - - uses: actions/checkout@v2 - - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version-file: go.mod - - name: Setup node and yarn - uses: actions/setup-node@v2 - with: - node-version: '14' - cache: 'yarn' - cache-dependency-path: 'ui/yarn.lock' - - name: UI Build - run: | - cd ui - yarn install --ignore-optional - npm rebuild node-sass - yarn --verbose run build - cd .. - - name: Build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: 0 - run: | - mkdir dist out - GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.product-metadata.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.product-metadata.outputs.build-date }}" make build - zip -r -j out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - path: out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - - - name: Package - uses: hashicorp/actions-packaging-linux@v1 - with: - name: ${{ github.event.repository.name }} - description: "Vault is a tool for secrets management, encryption as a service, and privileged access management." - arch: ${{ matrix.goarch }} - version: ${{ needs.product-metadata.outputs.product-version }} - maintainer: "HashiCorp" - homepage: "https://github.com/hashicorp/vault" - license: "MPL-2.0" - binary: "dist/${{ env.PKG_NAME }}" - deb_depends: "openssl" - rpm_depends: "openssl" - config_dir: ".release/linux/package/" - preinstall: ".release/linux/preinst" - postinstall: ".release/linux/postinst" - postremove: ".release/linux/postrm" - - name: Add Package names to env - run: | - echo "RPM_PACKAGE=$(basename out/*.rpm)" >> $GITHUB_ENV - echo "DEB_PACKAGE=$(basename out/*.deb)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.RPM_PACKAGE }} - path: out/${{ env.RPM_PACKAGE }} - - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.DEB_PACKAGE }} - path: out/${{ env.DEB_PACKAGE }} - - build-darwin: - needs: [ product-metadata ] - runs-on: macos-latest - strategy: - matrix: - goos: [ darwin ] - goarch: [ "amd64", "arm64" ] - fail-fast: true - name: Go ${{ matrix.goos }} ${{ matrix.goarch }} build - steps: - - uses: actions/checkout@v2 - - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version-file: go.mod - - name: Setup node and yarn - uses: actions/setup-node@v2 - with: - node-version: '14' - cache: 'yarn' - cache-dependency-path: 'ui/yarn.lock' - - name: UI Build - run: | - cd ui - yarn install --ignore-optional - npm rebuild node-sass - yarn --verbose run build - cd .. - - name: Build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - GO_TAGS: "${{ env.GO_TAGS }} netcgo" - CGO_ENABLED: 0 - run: | - mkdir dist out - GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.product-metadata.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.product-metadata.outputs.build-date }}" make build - zip -r -j out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - path: out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - - build-docker: - name: Docker ${{ matrix.arch }} build - needs: - - product-metadata - - build-linux - runs-on: ubuntu-latest - strategy: - matrix: - arch: ["arm", "arm64", "386", "amd64"] - env: - repo: ${{github.event.repository.name}} - version: ${{needs.product-metadata.outputs.product-version}} - steps: - - uses: actions/checkout@v2 - - name: Docker Build (Action) - uses: hashicorp/actions-docker-build@v1 - with: - version: ${{env.version}} - target: default - arch: ${{matrix.arch}} - zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_linux_${{ matrix.arch }}.zip - tags: | - docker.io/hashicorp/${{env.repo}}:${{env.version}} - public.ecr.aws/hashicorp/${{env.repo}}:${{env.version}} - - enos-test-docker: - name: Enos Docker - # Only run the Enos workflow against branches that are created from the - # hashicorp/vault repository. This has the effect of limiting execution of - # Enos scenarios to branches that originate from authors that have write - # access to hashicorp/vault repository. This is required as Github Actions - # will not populate the required secrets for branches created by outside - # contributors in order to protect the secrets integrity. - if: "! github.event.pull_request.head.repo.fork" - needs: - - product-metadata - - build-docker - uses: ./.github/workflows/enos-run-k8s.yml + uses: ./.github/workflows/build-vault-oss.yml with: - artifact-build-date: "${{needs.product-metadata.outputs.build-date}}" - artifact-name: "${{github.event.repository.name}}_default_linux_amd64_${{needs.product-metadata.outputs.product-version}}_${{needs.product-metadata.outputs.product-revision}}.docker.tar" - artifact-revision: "${{needs.product-metadata.outputs.product-revision}}" - artifact-version: "${{needs.product-metadata.outputs.product-version}}" + goarch: ${{ matrix.goarch }} + goos: ${{ matrix.goos }} + go-tags: ui + go-version: ${{ needs.product-metadata.outputs.go-version }} + package-name: ${{ needs.product-metadata.outputs.package-name }} + vault-version: ${{ needs.product-metadata.outputs.vault-version }} secrets: inherit - build-ubi: - name: Red Hat UBI ${{ matrix.arch }} build + build-darwin: + name: Build Vault Darwin + needs: product-metadata + strategy: + matrix: + goos: [darwin] + goarch: [amd64, arm64] + fail-fast: true + uses: ./.github/workflows/build-vault-oss.yml + with: + create-packages: false + goarch: ${{ matrix.goarch }} + goos: ${{ matrix.goos }} + go-tags: ui netcgo + go-version: ${{ needs.product-metadata.outputs.go-version }} + package-name: ${{ needs.product-metadata.outputs.package-name }} + vault-version: ${{ needs.product-metadata.outputs.vault-version }} + secrets: inherit + + build-docker: + name: Build Vault Docker needs: - product-metadata - build-linux runs-on: ubuntu-latest strategy: matrix: - arch: ["amd64"] + arch: [arm, arm64, 386, amd64] env: - repo: ${{github.event.repository.name}} - version: ${{needs.product-metadata.outputs.product-version}} + repo: ${{ github.event.repository.name }} + version: ${{ needs.product-metadata.outputs.vault-version }} + steps: + - uses: actions/checkout@v3 + - uses: hashicorp/actions-docker-build@v1 + with: + version: ${{ env.version }} + target: default + arch: ${{ matrix.arch }} + zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.vault-version }}_linux_${{ matrix.arch }}.zip + tags: | + docker.io/hashicorp/${{ env.repo }}:${{ env.version }} + public.ecr.aws/hashicorp/${{ env.repo }}:${{ env.version }} + + build-ubi: + name: Build Vault Red Hat UBI + needs: + - product-metadata + - build-linux + runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64] + env: + repo: ${{ github.event.repository.name }} + version: ${{ needs.product-metadata.outputs.vault-version }} steps: - uses: actions/checkout@v2 - - name: Docker Build (Action) - uses: hashicorp/actions-docker-build@v1 + - uses: hashicorp/actions-docker-build@v1 with: - version: ${{env.version}} + version: ${{ env.version }} target: ubi - arch: ${{matrix.arch}} - zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_linux_${{ matrix.arch }}.zip - redhat_tag: quay.io/redhat-isv-containers/5f89bb5e0b94cf64cfeb500a:${{env.version}}-ubi + arch: ${{ matrix.arch }} + zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.vault-version }}_linux_${{ matrix.arch }}.zip + redhat_tag: quay.io/redhat-isv-containers/5f89bb5e0b94cf64cfeb500a:${{ env.version }}-ubi enos: name: Enos @@ -311,9 +175,29 @@ jobs: - build-linux uses: ./.github/workflows/enos-run.yml with: - artifact-build-date: "${{needs.product-metadata.outputs.build-date}}" - artifact-name: "vault_${{ needs.product-metadata.outputs.product-version }}_linux_amd64.zip" - artifact-revision: "${{needs.product-metadata.outputs.product-revision}}" - artifact-source: "crt" - artifact-version: "${{needs.product-metadata.outputs.product-version}}" + artifact-build-date: ${{ needs.product-metadata.outputs.build-date }} + artifact-name: vault_${{ needs.product-metadata.outputs.vault-version }}_linux_amd64.zip + artifact-revision: ${{ needs.product-metadata.outputs.vault-revision }} + artifact-source: crt + artifact-version: ${{ needs.product-metadata.outputs.vault-version }} + secrets: inherit + + enos-docker-k8s: + name: Enos Docker K8s + # Only run the Enos workflow against branches that are created from the + # hashicorp/vault repository. This has the effect of limiting execution of + # Enos scenarios to branches that originate from authors that have write + # access to hashicorp/vault repository. This is required as Github Actions + # will not populate the required secrets for branches created by outside + # contributors in order to protect the secrets integrity. + if: "! github.event.pull_request.head.repo.fork" + needs: + - product-metadata + - build-docker + uses: ./.github/workflows/enos-run-k8s.yml + with: + artifact-build-date: ${{ needs.product-metadata.outputs.build-date }} + artifact-name: ${{ github.event.repository.name }}_default_linux_amd64_${{ needs.product-metadata.outputs.vault-version }}_${{ needs.product-metadata.outputs.vault-revision }}.docker.tar + artifact-revision: ${{ needs.product-metadata.outputs.vault-revision }} + artifact-version: ${{ needs.product-metadata.outputs.vault-version }} secrets: inherit diff --git a/.github/workflows/enos-fmt.yml b/.github/workflows/enos-fmt.yml index c10395c8b..298b2dc18 100644 --- a/.github/workflows/enos-fmt.yml +++ b/.github/workflows/enos-fmt.yml @@ -22,6 +22,6 @@ jobs: - uses: hashicorp/action-setup-enos@v1 with: github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - - name: "check formatting" + - name: check formatting working-directory: ./enos run: make check-fmt diff --git a/.github/workflows/enos-run.yml b/.github/workflows/enos-run.yml index 3fbc3192e..d934b98b3 100644 --- a/.github/workflows/enos-run.yml +++ b/.github/workflows/enos-run.yml @@ -43,7 +43,7 @@ jobs: id: enos-matrix run: | [[ ${{ env.ARTIFACT_NAME }} == *"ent"* ]] && scenarioFile=$(cat ./.github/enos-run-matrices/${{ env.ARTIFACT_SOURCE }}-ent.json |jq -c .) || scenarioFile=$(cat ./.github/enos-run-matrices/${{ env.ARTIFACT_SOURCE }}-oss.json |jq -c .) - echo "::set-output name=matrix::$scenarioFile" + echo "matrix=$scenarioFile" >> $GITHUB_OUTPUT # Run Integration tests on Enos scenario matrix enos: name: Integration diff --git a/.github/workflows/enos-verify-stable.yml b/.github/workflows/enos-verify-stable.yml index c68836c0f..6c0bf19ba 100644 --- a/.github/workflows/enos-verify-stable.yml +++ b/.github/workflows/enos-verify-stable.yml @@ -12,8 +12,8 @@ jobs: if: ${{ startsWith(github.event.client_payload.payload.branch, 'release/') }} uses: ./.github/workflows/enos-run.yml with: - artifact-source: "artifactory" - artifact-name: "${{ github.event.client_payload.payload.product }}_${{ github.event.client_payload.payload.version }}_linux_amd64.zip" - artifact-revision: "${{ github.event.client_payload.payload.sha }}" - artifact-version: "${{ github.event.client_payload.payload.version }}" + artifact-source: artifactory + artifact-name: ${{ github.event.client_payload.payload.product }}_${{ github.event.client_payload.payload.version }}_linux_amd64.zip + artifact-revision: ${{ github.event.client_payload.payload.sha }} + artifact-version: ${{ github.event.client_payload.payload.version }} secrets: inherit diff --git a/Makefile b/Makefile index 0827093d0..b5f77760c 100644 --- a/Makefile +++ b/Makefile @@ -254,18 +254,48 @@ ci-verify: .NOTPARALLEL: ember-dist ember-dist-dev -.PHONY: build -# This is used for release builds by .github/workflows/build.yml -build: - @echo "--> Building Vault $(VAULT_VERSION)" - @go build -v -tags "$(GO_TAGS)" -ldflags " -s -w -X github.com/hashicorp/vault/sdk/version.Version=$(VAULT_VERSION) -X github.com/hashicorp/vault/sdk/version.GitCommit=$(VAULT_REVISION) -X github.com/hashicorp/vault/sdk/version.BuildDate=$(VAULT_BUILD_DATE)" -o dist/ +# These crt targets are used for release builds by .github/workflows/build.yml +# and for artifact_source:local Enos scenario variants. +.PHONY: crt-build +crt-build: + @$(CURDIR)/scripts/crt-builder.sh build -.PHONY: version -# This is used for release builds by .github/workflows/build.yml -version: - @$(CURDIR)/scripts/version.sh sdk/version/version_base.go +.PHONY: crt-build-ui +crt-build-ui: + @$(CURDIR)/scripts/crt-builder.sh build-ui -.PHONY: build-date -# This is used for release builds by .github/workflows/build.yml -build-date: - @$(CURDIR)/scripts/build_date.sh +.PHONY: crt-bundle +crt-bundle: + @$(CURDIR)/scripts/crt-builder.sh bundle + +.PHONY: crt-get-artifact-basename +crt-get-artifact-basename: + @$(CURDIR)/scripts/crt-builder.sh artifact-basename + +.PHONY: crt-get-date +crt-get-date: + @$(CURDIR)/scripts/crt-builder.sh date + +.PHONY: crt-get-revision +crt-get-revision: + @$(CURDIR)/scripts/crt-builder.sh revision + +.PHONY: crt-get-version +crt-get-version: + @$(CURDIR)/scripts/crt-builder.sh version + +.PHONY: crt-get-version-base +crt-get-version-base: + @$(CURDIR)/scripts/crt-builder.sh version-base + +.PHONY: crt-get-version-pre +crt-get-version-pre: + @$(CURDIR)/scripts/crt-builder.sh version-pre + +.PHONY: crt-get-version-meta +crt-get-version-meta: + @$(CURDIR)/scripts/crt-builder.sh version-meta + +.PHONY: crt-prepare-legal +crt-prepare-legal: + @$(CURDIR)/scripts/crt-builder.sh prepare-legal diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index 81cd12c4a..a785c6101 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -36,6 +36,7 @@ scenario "agent" { arm64 = "t4g.small" } vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) } step "get_local_metadata" { @@ -91,7 +92,7 @@ scenario "agent" { module = module.read_license variables { - file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + file_name = local.vault_license_path } } diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index f2e9db6e7..881613759 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -36,13 +36,14 @@ scenario "autopilot" { arm64 = "t4g.small" } vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) } step "build_vault" { module = "build_${matrix.artifact_source}" variables { - build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition]) + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] bundle_path = local.bundle_path goarch = matrix.arch goos = "linux" @@ -85,7 +86,7 @@ scenario "autopilot" { module = module.read_license variables { - file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + file_name = local.vault_license_path } } diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index 6091e66c4..56b53d006 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -39,6 +39,7 @@ scenario "smoke" { arm64 = "t4g.small" } vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) } step "get_local_metadata" { @@ -50,7 +51,7 @@ scenario "smoke" { module = "build_${matrix.artifact_source}" variables { - build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition]) + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] bundle_path = local.bundle_path goarch = matrix.arch goos = "linux" @@ -94,7 +95,7 @@ scenario "smoke" { module = module.read_license variables { - file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + file_name = local.vault_license_path } } diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index cc8507767..2baf11d2f 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -39,13 +39,14 @@ scenario "upgrade" { arm64 = "t4g.small" } vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) } step "build_vault" { module = "build_${matrix.artifact_source}" variables { - build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition]) + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] bundle_path = local.bundle_path goarch = matrix.arch goos = "linux" @@ -89,7 +90,7 @@ scenario "upgrade" { module = module.read_license variables { - file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + file_name = local.vault_license_path } } diff --git a/enos/modules/build_local/main.tf b/enos/modules/build_local/main.tf index 3dc189b0e..f8c12b1a2 100644 --- a/enos/modules/build_local/main.tf +++ b/enos/modules/build_local/main.tf @@ -58,10 +58,12 @@ variable "vault_product_version" { } resource "enos_local_exec" "build" { - content = templatefile("${path.module}/templates/build.sh", { - bundle_path = var.bundle_path, - build_tags = join(" ", var.build_tags) - goarch = var.goarch - goos = var.goos - }) + scripts = ["${path.module}/scripts/build.sh"] + + environment = { + BUNDLE_PATH = var.bundle_path, + GO_TAGS = join(" ", var.build_tags) + GOARCH = var.goarch + GOOS = var.goos + } } diff --git a/enos/modules/build_local/scripts/build.sh b/enos/modules/build_local/scripts/build.sh new file mode 100755 index 000000000..3478079c2 --- /dev/null +++ b/enos/modules/build_local/scripts/build.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -eux -o pipefail + +# Install yarn so we can build the UI +npm install --global yarn || true + +export CGO_ENABLED=0 + +root_dir="$(git rev-parse --show-toplevel)" +pushd "$root_dir" > /dev/null +make crt-build-ui crt-build crt-bundle +popd > /dev/null diff --git a/enos/modules/build_local/templates/build.sh b/enos/modules/build_local/templates/build.sh deleted file mode 100755 index 661df529b..000000000 --- a/enos/modules/build_local/templates/build.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -eux -o pipefail - -# Requirements -npm install --global yarn || true - -# Set up the environment for building Vault. -root_dir="$(git rev-parse --show-toplevel)" - -pushd "$root_dir" > /dev/null - -export GO_TAGS=${build_tags} -export CGO_ENABLED=0 - -IFS="-" read -r BASE_VERSION _other <<< "$(make version)" -export VAULT_VERSION=$BASE_VERSION - -build_date="$(make build-date)" -export VAULT_BUILD_DATE=$build_date - -revision="$(git rev-parse HEAD)" -export VAULT_REVISION=$revision -popd > /dev/null - -# Go to the UI directory of the Vault repo and build the UI -pushd "$root_dir/ui" > /dev/null -yarn install --ignore-optional -npm rebuild node-sass -yarn --verbose run build -popd > /dev/null - -# Build for linux/amd64 and create a bundle since we're deploying it to linux/amd64 -pushd "$root_dir" > /dev/null -export GOARCH=${goarch} -export GOOS=${goos} -make build - -zip -r -j ${bundle_path} dist/ -popd > /dev/null diff --git a/enos/modules/get_local_metadata/scripts/build_date.sh b/enos/modules/get_local_metadata/scripts/build_date.sh index fbb7f9eb5..d0528554e 100755 --- a/enos/modules/get_local_metadata/scripts/build_date.sh +++ b/enos/modules/get_local_metadata/scripts/build_date.sh @@ -1,10 +1,6 @@ #!/bin/env bash set -eu -o pipefail -# Set up the environment for building Vault. -root_dir="$(git rev-parse --show-toplevel)" - -pushd "$root_dir" > /dev/null - -IFS="-" read -r VAULT_BUILD_DATE _other <<< "$(make build-date)" -echo $VAULT_BUILD_DATE +pushd "$(git rev-parse --show-toplevel)" > /dev/null +make crt-get-date +popd > /dev/null diff --git a/enos/modules/get_local_metadata/scripts/version.sh b/enos/modules/get_local_metadata/scripts/version.sh index 13970ead9..ef0b91f37 100755 --- a/enos/modules/get_local_metadata/scripts/version.sh +++ b/enos/modules/get_local_metadata/scripts/version.sh @@ -1,10 +1,6 @@ #!/bin/env bash set -eu -o pipefail -# Set up the environment for building Vault. -root_dir="$(git rev-parse --show-toplevel)" - -pushd "$root_dir" > /dev/null - -IFS="-" read -r VAULT_VERSION _other <<< "$(make version)" -echo $VAULT_VERSION +pushd "$(git rev-parse --show-toplevel)" > /dev/null +make crt-get-version +popd > /dev/null diff --git a/scripts/build.sh b/scripts/build.sh index b2e1302cb..c041e0574 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -18,10 +18,10 @@ cd "$DIR" BUILD_TAGS="${BUILD_TAGS:-"vault"}" # Get the git commit -GIT_COMMIT="$(git rev-parse HEAD)" +GIT_COMMIT="$("$SOURCE_DIR"/crt-builder.sh revision)" GIT_DIRTY="$(test -n "`git status --porcelain`" && echo "+CHANGES" || true)" -BUILD_DATE=$("$SOURCE_DIR"/build_date.sh) +BUILD_DATE="$("$SOURCE_DIR"/crt-builder.sh date)" GOPATH=${GOPATH:-$(${GO_CMD} env GOPATH)} case $(uname) in diff --git a/scripts/build_date.sh b/scripts/build_date.sh deleted file mode 100755 index 56504595f..000000000 --- a/scripts/build_date.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# it's tricky to do an RFC3339 format in a cross platform way, so we hardcode UTC -DATE_FORMAT="%Y-%m-%dT%H:%M:%SZ" - -# we're using this for build date because it's stable across platform builds -git show --no-show-signature -s --format=%cd --date=format:"$DATE_FORMAT" HEAD diff --git a/scripts/crt-builder.sh b/scripts/crt-builder.sh new file mode 100755 index 000000000..f69797efc --- /dev/null +++ b/scripts/crt-builder.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash + +# The crt-builder is used to detemine build metadata and create Vault builds. +# We use it in build-vault.yml for building release artifacts with CRT. It is +# also used by Enos for artifact_source:local scenario variants. + +set -euo pipefail + +# We don't want to get stuck in some kind of interactive pager +export GIT_PAGER=cat + +# Get the full version information +function version() { + local version + local prerelease + local metadata + + version=$(version_base) + prerelease=$(version_pre) + metadata=$(version_metadata) + + if [ -n "$metadata" ] && [ -n "$prerelease" ]; then + echo "$version-$prerelease+$metadata" + elif [ -n "$metadata" ]; then + echo "$version+$metadata" + elif [ -n "$prerelease" ]; then + echo "$version-$prerelease" + else + echo "$version" + fi +} + +# Get the base version +function version_base() { + : "${VAULT_VERSION:=""}" + + if [ -n "$VAULT_VERSION" ]; then + echo "$VAULT_VERSION" + return + fi + + : "${VERSION_FILE:=$(repo_root)/sdk/version/version_base.go}" + awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "$VERSION_FILE" +} + +# Get the version pre-release +function version_pre() { + : "${VAULT_PRERELEASE:=""}" + + if [ -n "$VAULT_PRERELEASE" ]; then + echo "$VAULT_PRERELEASE" + return + fi + + : "${VERSION_FILE:=$(repo_root)/sdk/version/version_base.go}" + awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "$VERSION_FILE" +} + +# Get the version metadata, which is commonly the edition +function version_metadata() { + : "${VAULT_METADATA:=""}" + + if [ -n "$VAULT_METADATA" ]; then + echo "$VAULT_METADATA" + return + fi + + : "${VERSION_FILE:=$(repo_root)/sdk/version/version_base.go}" + awk '$1 == "VersionMetadata" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "$VERSION_FILE" +} + +# Get the build date from the latest commit since it can be used across all +# builds +function build_date() { + # It's tricky to do an RFC3339 format in a cross platform way, so we hardcode UTC + : "${DATE_FORMAT:="%Y-%m-%dT%H:%M:%SZ"}" + git show --no-show-signature -s --format=%cd --date=format:"$DATE_FORMAT" HEAD +} + +# Get the revision, which is the latest commit SHA +function build_revision() { + git rev-parse HEAD +} + +# Determine our repository by looking at our origin URL +function repo() { + basename -s .git "$(git config --get remote.origin.url)" +} + +# Determine the root directory of the repository +function repo_root() { + git rev-parse --show-toplevel +} + +# Determine the artifact basename based on metadata +function artifact_basename() { + : "${PKG_NAME:="vault"}" + : "${GOOS:=$(go env GOOS)}" + : "${GOARCH:=$(go env GOARCH)}" + + echo "${PKG_NAME}_$(version)_${GOOS}_${GOARCH}" +} + +# Build the UI +function build_ui() { + local repo_root + repo_root=$(repo_root) + + pushd "$repo_root" + mkdir -p http/web_ui + popd + pushd "$repo_root/ui" + yarn install --ignore-optional + npm rebuild node-sass + yarn --verbose run build + popd +} + +# Build Vault +function build() { + local version + local revision + local prerelease + local build_date + local ldflags + local msg + + # Get or set our basic build metadata + version=$(version_base) + revision=$(build_revision) + metadata=$(version_metadata) + prerelease=$(version_pre) + build_date=$(build_date) + : "${GO_TAGS:=""}" + : "${KEEP_SYMBOLS:=""}" + + # Build our ldflags + msg="--> Building Vault v$version, revision $revision, built $build_date" + + # Strip the symbol and dwarf information by default + if [ -n "$KEEP_SYMBOLS" ]; then + ldflags="" + else + ldflags="-s -w " + fi + + ldflags="${ldflags}-X github.com/hashicorp/vault/sdk/version.Version=$version -X github.com/hashicorp/vault/sdk/version.GitCommit=$revision -X github.com/hashicorp/vault/sdk/version.BuildDate=$build_date" + + if [ -n "$prerelease" ]; then + msg="${msg}, prerelease ${prerelease}" + ldflags="${ldflags} -X github.com/hashicorp/vault/sdk/version.VersionPrerelease=$prerelease" + fi + + if [ -n "$metadata" ]; then + msg="${msg}, metadata ${VAULT_METADATA}" + ldflags="${ldflags} -X github.com/hashicorp/vault/sdk/version.VersionMetadata=$metadata" + fi + + # Build vault + echo "$msg" + pushd "$(repo_root)" + mkdir -p dist + mkdir -p out + set -x + go build -v -tags "$GO_TAGS" -ldflags "$ldflags" -o dist/ + set +x + popd +} + +# Bundle the dist directory +function bundle() { + : "${BUNDLE_PATH:=$(repo_root)/vault.zip}" + echo "--> Bundling dist/* to $BUNDLE_PATH" + zip -r -j "$BUNDLE_PATH" dist/ +} + +# Prepare legal requirements for packaging +function prepare_legal() { + : "${PKG_NAME:="vault"}" + + pushd "$(repo_root)" + mkdir -p dist + curl -o dist/EULA.txt https://eula.hashicorp.com/EULA.txt + curl -o dist/TermsOfEvaluation.txt https://eula.hashicorp.com/TermsOfEvaluation.txt + mkdir -p ".release/linux/package/usr/share/doc/$PKG_NAME" + cp dist/EULA.txt ".release/linux/package/usr/share/doc/$PKG_NAME/EULA.txt" + cp dist/TermsOfEvaluation.txt ".release/linux/package/usr/share/doc/$PKG_NAME/TermsOfEvaluation.txt" + popd +} + +# Run the CRT Builder +function main() { + case $1 in + artifact-basename) + artifact_basename + ;; + build) + build + ;; + build-ui) + build_ui + ;; + bundle) + bundle + ;; + date) + build_date + ;; + prepare-legal) + prepare_legal + ;; + revision) + build_revision + ;; + version) + version + ;; + version-base) + version_base + ;; + version-pre) + version_pre + ;; + version-meta) + version_metadata + ;; + *) + echo "unknown sub-command" >&2 + exit 1 + ;; + esac +} + +main "$@" diff --git a/scripts/version.sh b/scripts/version.sh deleted file mode 100755 index 41177a45f..000000000 --- a/scripts/version.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -version_file=$1 -version=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "${version_file}") -prerelease=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "${version_file}") - -if [ -n "$prerelease" ]; then - echo "${version}-${prerelease}" -else - echo "${version}" -fi