e2e: add EBS and EFS volumes for testing CSI (#7266)

This changeset adds volumes but does not mount them to instances so
that we can test the mounting ("staging") via CSI plugins. The CSI
plugins themselves will be installed as Nomad jobs.

In order to ensure we can always mount the EFS volume, this changeset
pins the deployment of the cluster to a specific subnet. In future
work we should spread the cluster out among several AZs and test that
behavior explicitly.

e2e/terraform/compute.tf

@@ -4,6 +4,7 @@ resource "aws_instance" "server" {
   key_name               = module.keys.key_name
   vpc_security_group_ids = [aws_security_group.primary.id]
   count                  = var.server_count
+  availability_zone      = var.availability_zone
 
   # Instance tags
   tags = {
@@ -23,6 +24,7 @@ resource "aws_instance" "client_linux" {
   vpc_security_group_ids = [aws_security_group.primary.id]
   count                  = var.client_count
   depends_on             = [aws_instance.server]
+  availability_zone      = var.availability_zone
 
   # Instance tags
   tags = {
@@ -50,6 +52,7 @@ resource "aws_instance" "client_windows" {
   count                  = var.windows_client_count
   depends_on             = [aws_instance.server]
   iam_instance_profile   = aws_iam_instance_profile.instance_profile.name
+  availability_zone      = var.availability_zone
 
   # Instance tags
   tags = {

e2e/terraform/iam.tf

@@ -46,6 +46,8 @@ data "aws_iam_policy_document" "auto_discover_cluster" {
     actions = [
       "ec2:DescribeInstances",
       "ec2:DescribeTags",
+      "ec2:DescribeVolume*",
+      "ec2:AttachVolume",
       "autoscaling:DescribeAutoScalingGroups",
     ]
     resources = ["*"]
@@ -62,4 +64,3 @@ data "aws_iam_policy_document" "auto_discover_cluster" {
     resources = ["arn:aws:s3:::nomad-team-test-binary/*"]
   }
 }
-

e2e/terraform/main.tf

@@ -8,6 +8,11 @@ variable "region" {
   default     = "us-east-1"
 }
 
+variable "availability_zone" {
+  description = "The AWS availability zone to deploy to."
+  default     = "us-east-1a"
+}
+
 variable "indexed" {
   description = "Different configurations per client/server"
   default     = true

e2e/terraform/network.tf

@@ -2,6 +2,11 @@ data "aws_vpc" "default" {
   default = true
 }
 
+data "aws_subnet" "default" {
+  availability_zone = var.availability_zone
+  vpc_id            = data.aws_vpc.default.id
+}
+
 resource "aws_security_group" "primary" {
   name   = local.random_name
   vpc_id = data.aws_vpc.default.id
@@ -76,3 +81,14 @@ resource "aws_security_group" "primary" {
   }
 }
 
+resource "aws_security_group" "nfs" {
+  name   = "${local.random_name}-nfs"
+  vpc_id = data.aws_vpc.default.id
+
+  ingress {
+    from_port       = 2049
+    to_port         = 2049
+    protocol        = "tcp"
+    security_groups = [aws_security_group.primary.id]
+  }
+}

e2e/terraform/packer/linux/setup.sh

@@ -29,7 +29,7 @@ NOMADDIR=/opt/nomad
 # Dependencies
 sudo apt-get install -y software-properties-common
 sudo apt-get update
-sudo apt-get install -y unzip tree redis-tools jq curl tmux awscli
+sudo apt-get install -y unzip tree redis-tools jq curl tmux awscli nfs-common
 
 # Numpy (for Spark)
 sudo apt-get install -y python-setuptools

e2e/terraform/volumes.tf

@@ -0,0 +1,24 @@
+resource "aws_efs_file_system" "csi" {
+  creation_token = "CSI"
+
+  tags = {
+    Name = "${local.random_name}-efs"
+    User = data.aws_caller_identity.current.arn
+  }
+}
+
+resource "aws_efs_mount_target" "csi" {
+  file_system_id  = aws_efs_file_system.csi.id
+  subnet_id       = data.aws_subnet.default.id
+  security_groups = [aws_security_group.nfs.id]
+}
+
+resource "aws_ebs_volume" "csi" {
+  availability_zone = var.availability_zone
+  size              = 40
+
+  tags = {
+    Name = "${local.random_name}-ebs"
+    User = data.aws_caller_identity.current.arn
+  }
+}