From b03da5157ec92e85bcb19bc4880445d49e2603ce Mon Sep 17 00:00:00 2001 From: Mike Baum Date: Wed, 30 Nov 2022 12:44:02 -0500 Subject: [PATCH] [QT-318] Add Vault CI bootstrap scenarios (#17907) --- .github/workflows/enos-bootstrap-ci.yml | 67 +++++++++++ .gitignore | 5 + enos/README.md | 65 +++++++++++ enos/ci/bootstrap/main.tf | 66 +++++++++++ enos/ci/bootstrap/outputs.tf | 20 ++++ enos/ci/bootstrap/variables.tf | 4 + enos/ci/service-user-iam/main.tf | 145 ++++++++++++++++++++++++ enos/ci/service-user-iam/outputs.tf | 13 +++ enos/ci/service-user-iam/variables.tf | 8 ++ 9 files changed, 393 insertions(+) create mode 100644 .github/workflows/enos-bootstrap-ci.yml create mode 100644 enos/ci/bootstrap/main.tf create mode 100644 enos/ci/bootstrap/outputs.tf create mode 100644 enos/ci/bootstrap/variables.tf create mode 100644 enos/ci/service-user-iam/main.tf create mode 100644 enos/ci/service-user-iam/outputs.tf create mode 100644 enos/ci/service-user-iam/variables.tf diff --git a/.github/workflows/enos-bootstrap-ci.yml b/.github/workflows/enos-bootstrap-ci.yml new file mode 100644 index 000000000..c4889b9a7 --- /dev/null +++ b/.github/workflows/enos-bootstrap-ci.yml @@ -0,0 +1,67 @@ +name: enos-ci-bootstrap + +on: + pull_request: + branches: + - main + push: + branches: + - main + paths: + - enos/ci/** + - .github/workflows/enos-bootstrap-ci.yml + +jobs: + bootstrap-ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Terraform + uses: hashicorp/setup-terraform@v2 + - name: Prepare for Terraform execution + id: prepare_for_terraform + env: + IS_ENT: ${{ startsWith(github.event.repository.name, 'vault-enterprise' ) }} + run: | + if ${IS_ENT} == true; then + echo "aws_role=arn:aws:iam::505811019928:role/github_actions-vault-enterprise_ci" >> $GITHUB_OUTPUT + echo "aws role set to 'arn:aws:iam::505811019928:role/github_actions-vault-enterprise_ci'" + echo "product_line=vault-enterprise" >> $GITHUB_OUTPUT + echo "product line set to 'vault-enterprise'" + else + echo "aws_role=arn:aws:iam::040730498200:role/github_actions-vault_ci" >> $GITHUB_OUTPUT + echo "aws role set to 'arn:aws:iam::040730498200:role/github_actions-vault_ci'" + echo "product_line=vault" >> $GITHUB_OUTPUT + echo "product line set to 'vault'" + fi + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_CI }} + aws-region: us-east-1 + role-to-assume: ${{ steps.prepare_for_terraform.outputs.aws_role }} + role-skip-session-tagging: true + role-duration-seconds: 3600 + - name: Init Terraform + id: tf_init + run: | + export TF_WORKSPACE="${{ steps.prepare_for_terraform.outputs.product_line }}-ci-enos-bootstrap" + export TF_VAR_aws_ssh_public_key="${{ secrets.ENOS_CI_SSH_KEY }}" + export TF_TOKEN_app_terraform_io="${{ secrets.TF_API_TOKEN }}" + terraform -chdir=enos/ci/bootstrap init + - name: Plan Terraform + id: tf_plan + run: | + export TF_WORKSPACE="${{ steps.prepare_for_terraform.outputs.product_line }}-ci-enos-bootstrap" + export TF_VAR_aws_ssh_public_key="${{ secrets.ENOS_CI_SSH_KEY }}" + export TF_TOKEN_app_terraform_io="${{ secrets.TF_API_TOKEN }}" + terraform -chdir=enos/ci/bootstrap plan + - name: Apply Terraform + if: ${{ github.ref == 'refs/heads/main' }} + id: tf_apply + run: | + export TF_WORKSPACE="${{ steps.prepare_for_terraform.outputs.product_line }}-ci-enos-bootstrap" + export TF_VAR_aws_ssh_public_key="${{ secrets.ENOS_CI_SSH_KEY }}" + export TF_TOKEN_app_terraform_io="${{ secrets.TF_API_TOKEN }}" + terraform -chdir=enos/ci/bootstrap apply -auto-approve diff --git a/.gitignore b/.gitignore index c2a6a252e..961849caa 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,11 @@ Vagrantfile # Enos enos/.enos enos/support +# Enos local Terraform files +enos/.terraform/* +enos/.terraform.lock.hcl +enos/*.tfstate +enos/*.tfstate.* .DS_Store .idea diff --git a/enos/README.md b/enos/README.md index 7e165b0df..a38cbdecf 100644 --- a/enos/README.md +++ b/enos/README.md @@ -140,3 +140,68 @@ downloads the artifact built by the `build.yml` workflow, unzips it, and sets th This variant is for running the Enos scenario locally. It builds the Vault bundle from the current branch, placing the bundle at the `vault_bundle_path` and the unzipped Vault binary at the `vault_local_binary_path`. + +# CI Bootstrap +In order to execute any of the scenarios in this repository, it is first necessary to bootstrap the +CI AWS account with the required permissions and supporting AWS resources. There are two Terraform +modules which are used for this purpose, [service-user-iam](./ci/service-user-iam) for the account +permissions and [bootstrap](./ci/bootstrap) for the supporting resources. + +**Supported Regions** - enos scenarios are supported in the following regions: +`"us-east-1", "us-east-2", "us-west-1", "us-west-2"` + +## Bootstrap Process +These steps should be followed to bootstrap this repo for enos scenario execution: + +### Set up CI service user IAM role +The service user that is used when executing enos scenarios from any GitHub Action workflow must have +a properly configured IAM role granting the access required to create resources in AWS. The +[service-user-iam](./ci/service-user-iam) module contains the IAM Policy and Role for that grants +this access. This module should be updated whenever a new AWS resource type is required for a scenario. +Since this is persistent and cannot be created and destroyed each time a scenario is run, the Terraform +state will be managed by Terraform Cloud. Here are the steps to configure the GitHub Actions service user: + +#### Pre-requisites +- Access to the `hashicorp-qti` organization in Terraform Cloud. +- Full access to the CI AWS account is required. + +**Notes:** +- For help with access to Terraform Cloud and the CI Account, contact the QT team on Slack (#team-quality) + for an invite. After receiving an invite to Terraform Cloud, a personal access token can be created + by clicking `User Settings` --> `Tokens` --> `Create an API token`. +- Access to the AWS account can be done via Doormat, at: https://doormat.hashicorp.services/. + - For the vault repo the account is: `vault_ci` and for the vault-enterprise repo, the account is: + `vault-enterprise_ci`. + - Access can be requested by clicking: `Cloud Access` --> `AWS` --> `Request Account Access`. + +1. **Create the Terraform Cloud Workspace** - The name of the workspace to be created depends on the + repository for which it is being created, but the pattern is: `-ci-service-user-iam`, + e.g. `vault-ci-service-user-iam`. It is important that the execution mode for the workspace be set + to `local`. For help on setting up the workspace, contact the QT team on Slack (#team-quality) + + +2. **Execute the Terraform module** +```shell +> cd ./enos/ci/service-user-iam +> export TF_WORKSPACE=-ci-service-user-iam +> export TF_TOKEN_app_terraform_io= +> export TF_VAR_repository= +> terraform init +> terraform plan +> terraform apply -auto-approve +``` + +### Bootstrap the CI resources +Bootstrapping of the resources in the CI account is accomplished via the GitHub Actions workflow: +[enos-bootstrap-ci](../.github/workflows/enos-bootstrap-ci.yml). Before this workflow can be run a +workspace must be created as follows: + +1. **Create the Terraform Cloud Workspace** - The name workspace to be created depends on the repository + for which it is being created, but the pattern is: `-ci-bootstrap`, e.g. + `vault-ci-bootstrap`. It is important that the execution mode for the workspace be set to + `local`. For help on setting up the workspace, contact the QT team on Slack (#team-quality). + +Once the workspace has been created, changes to the bootstrap module will automatically be applied via +the GitHub PR workflow. Each time a PR is created for changes to files within that module the module +will be planned via the workflow described above. If the plan is ok and the PR is merged, the module +will automatically be applied via the same workflow. diff --git a/enos/ci/bootstrap/main.tf b/enos/ci/bootstrap/main.tf new file mode 100644 index 000000000..990ba3ea9 --- /dev/null +++ b/enos/ci/bootstrap/main.tf @@ -0,0 +1,66 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + } + + cloud { + hostname = "app.terraform.io" + organization = "hashicorp-qti" + // workspace must be exported in the environment as: TF_WORKSPACE=-ci-enos-boostrap + } +} + +provider "aws" { + region = "us-east-1" + alias = "us_east_1" +} + +provider "aws" { + region = "us-east-2" + alias = "us_east_2" +} + +provider "aws" { + region = "us-west-1" + alias = "us_west_1" +} + +provider "aws" { + region = "us-west-2" + alias = "us_west_2" +} + + +locals { + key_name = "enos-ci-ssh-key" +} + +resource "aws_key_pair" "enos_ci_key_us_east_1" { + key_name = local.key_name + public_key = var.aws_ssh_public_key + + provider = aws.us_east_1 +} + +resource "aws_key_pair" "enos_ci_key_us_east_2" { + key_name = local.key_name + public_key = var.aws_ssh_public_key + + provider = aws.us_east_2 +} + +resource "aws_key_pair" "enos_ci_key_us_west_1" { + key_name = local.key_name + public_key = var.aws_ssh_public_key + + provider = aws.us_west_1 +} + +resource "aws_key_pair" "enos_ci_key_us_west_2" { + key_name = local.key_name + public_key = var.aws_ssh_public_key + + provider = aws.us_west_2 +} diff --git a/enos/ci/bootstrap/outputs.tf b/enos/ci/bootstrap/outputs.tf new file mode 100644 index 000000000..858318e4c --- /dev/null +++ b/enos/ci/bootstrap/outputs.tf @@ -0,0 +1,20 @@ +output "keys" { + value = { + "us-east-1" = { + name = aws_key_pair.enos_ci_key_us_east_1.key_name + arn = aws_key_pair.enos_ci_key_us_east_1.arn + } + "us-east-2" = { + name = aws_key_pair.enos_ci_key_us_east_2.key_name + arn = aws_key_pair.enos_ci_key_us_east_2.arn + } + "us-west-1" = { + name = aws_key_pair.enos_ci_key_us_west_1.key_name + arn = aws_key_pair.enos_ci_key_us_west_1.arn + } + "us-west-2" = { + name = aws_key_pair.enos_ci_key_us_west_2.key_name + arn = aws_key_pair.enos_ci_key_us_west_2.arn + } + } +} diff --git a/enos/ci/bootstrap/variables.tf b/enos/ci/bootstrap/variables.tf new file mode 100644 index 000000000..fcba83777 --- /dev/null +++ b/enos/ci/bootstrap/variables.tf @@ -0,0 +1,4 @@ +variable "aws_ssh_public_key" { + description = "The public key to use for the ssh key" + type = string +} diff --git a/enos/ci/service-user-iam/main.tf b/enos/ci/service-user-iam/main.tf new file mode 100644 index 000000000..e4095e6ee --- /dev/null +++ b/enos/ci/service-user-iam/main.tf @@ -0,0 +1,145 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + } + + cloud { + hostname = "app.terraform.io" + organization = "hashicorp-qti" + // workspace must be exported in the environment as: TF_WORKSPACE=-ci-enos-service-user-iam + } +} + +provider "aws" { + region = "us-east-1" +} + +locals { + enterprise_repositories = ["vault-enterprise"] + is_ent = contains(local.enterprise_repositories, var.repository) + ci_account_prefix = local.is_ent ? "vault-enterprise" : "vault" + service_user = "github_actions-${local.ci_account_prefix}_ci" + aws_account_id = local.is_ent ? "505811019928" : "040730498200" +} + +resource "aws_iam_role" "role" { + name = local.service_user + assume_role_policy = data.aws_iam_policy_document.assume_role_policy_document.json +} + +data "aws_iam_policy_document" "assume_role_policy_document" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${local.aws_account_id}:user/${local.service_user}"] + } + } +} + +resource "aws_iam_role_policy" "role_policy" { + role = aws_iam_role.role.name + name = "${local.service_user}_policy" + policy = data.aws_iam_policy_document.iam_policy_document.json +} + +data "aws_iam_policy_document" "iam_policy_document" { + statement { + effect = "Allow" + actions = [ + "iam:ListRoles", + "iam:CreateRole", + "iam:GetRole", + "iam:DeleteRole", + "iam:ListInstanceProfiles", + "iam:ListInstanceProfilesForRole", + "iam:CreateInstanceProfile", + "iam:GetInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:ListPolicies", + "iam:CreatePolicy", + "iam:DeletePolicy", + "iam:ListRoles", + "iam:CreateRole", + "iam:AddRoleToInstanceProfile", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteRole", + "iam:ListRolePolicies", + "iam:ListAttachedRolePolicies", + "iam:AttachRolePolicy", + "iam:GetRolePolicy", + "iam:PutRolePolicy", + "iam:DetachRolePolicy", + "iam:DeleteRolePolicy", + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceCreditSpecifications", + "ec2:DescribeImages", + "ec2:DescribeTags", + "ec2:DescribeVpcClassicLink", + "ec2:DescribeVpcClassicLinkDnsSupport", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeSecurityGroups", + "ec2:CreateSecurityGroup", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:DeleteSecurityGroup", + "ec2:RevokeSecurityGroupIngress", + "ec2:RevokeSecurityGroupEgress", + "ec2:DescribeInstances", + "ec2:DescribeInstanceAttribute", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:ModifyInstanceAttribute", + "ec2:TerminateInstances", + "ec2:ResetInstanceAttribute", + "ec2:DeleteTags", + "ec2:DescribeVolumes", + "ec2:CreateVolume", + "ec2:DeleteVolume", + "ec2:DescribeVpcs", + "ec2:DescribeVpcAttribute", + "ec2:CreateVPC", + "ec2:ModifyVPCAttribute", + "ec2:DeleteVPC", + "ec2:DescribeSubnets", + "ec2:CreateSubnet", + "ec2:ModifySubnetAttribute", + "ec2:DeleteSubnet", + "ec2:DescribeInternetGateways", + "ec2:CreateInternetGateway", + "ec2:AttachInternetGateway", + "ec2:DetachInternetGateway", + "ec2:DeleteInternetGateway", + "ec2:DescribeRouteTables", + "ec2:CreateRoute", + "ec2:CreateRouteTable", + "ec2:AssociateRouteTable", + "ec2:DisassociateRouteTable", + "ec2:DeleteRouteTable", + "ec2:CreateKeyPair", + "ec2:ImportKeyPair", + "ec2:DeleteKeyPair", + "ec2:DescribeKeyPairs", + "kms:ListKeys", + "kms:ListResourceTags", + "kms:GetKeyPolicy", + "kms:GetKeyRotationStatus", + "kms:DescribeKey", + "kms:CreateKey", + "kms:Encrypt", + "kms:Decrypt", + "kms:ScheduleKeyDeletion", + "kms:ListAliases", + "kms:CreateAlias", + "kms:DeleteAlias", + ] + resources = ["*"] + } +} diff --git a/enos/ci/service-user-iam/outputs.tf b/enos/ci/service-user-iam/outputs.tf new file mode 100644 index 000000000..d4ba89910 --- /dev/null +++ b/enos/ci/service-user-iam/outputs.tf @@ -0,0 +1,13 @@ +output "ci_role" { + value = { + name = aws_iam_role.role.name + arn = aws_iam_role.role.arn + } +} + +output "ci_role_policy" { + value = { + name = aws_iam_role_policy.role_policy.name + policy = aws_iam_role_policy.role_policy.policy + } +} diff --git a/enos/ci/service-user-iam/variables.tf b/enos/ci/service-user-iam/variables.tf new file mode 100644 index 000000000..6cc7efd6b --- /dev/null +++ b/enos/ci/service-user-iam/variables.tf @@ -0,0 +1,8 @@ +variable "repository" { + description = "The GitHub repository, either vault or vault-enterprise" + type = string + validation { + condition = contains(["vault", "vault-enterprise"], var.repository) + error_message = "Invalid repository, only vault or vault-enterprise are supported" + } +}