e2e: add flag to opt-in to creating EBS/EFS volumes (#9082)

For everyday developer use, we don't need volumes for testing CSI. Providing a
flag to opt-in speeds up deploying dev clusters and slightly reduces infra costs.

Skip CSI test if missing volume specs.

e2e/csi/csi.go

@@ -36,6 +36,17 @@ func init() {
 
 func (tc *CSIVolumesTest) BeforeAll(f *framework.F) {
 	t := f.T()
+
+	_, err := os.Stat("csi/input/volume-ebs.hcl")
+	if err != nil {
+		t.Skip("skipping CSI test because EBS volume spec file missing:", err)
+	}
+
+	_, err = os.Stat("csi/input/volume-efs.hcl")
+	if err != nil {
+		t.Skip("skipping CSI test because EFS volume spec file missing:", err)
+	}
+
 	// Ensure cluster has leader and at least two client
 	// nodes in a ready state before running tests
 	e2eutil.WaitForLeader(t, tc.Nomad())

e2e/terraform/network.tf

@@ -90,6 +90,7 @@ resource "aws_security_group" "primary" {
 }
 
 resource "aws_security_group" "nfs" {
+  count  = var.volumes ? 1 : 0
   name   = "${local.random_name}-nfs"
   vpc_id = data.aws_vpc.default.id
 

e2e/terraform/terraform.tfvars

@@ -7,3 +7,4 @@ profile              = "full-cluster"
 nomad_enterprise     = true
 nomad_acls           = true
 vault                = true
+volumes              = true

e2e/terraform/terraform.tfvars.dev

@@ -7,6 +7,7 @@ profile              = "dev-cluster"
 nomad_acls           = false
 nomad_enterprise     = false
 vault                = true
+volumes              = false
 
 # Example overrides:
 # nomad_local_binary = "../../pkg/linux_amd/nomad"

e2e/terraform/variables.tf

@@ -91,6 +91,12 @@ variable "vault" {
   default     = false
 }
 
+variable "volumes" {
+  type        = bool
+  description = "Include external EBS and EFS volumes (for CSI)"
+  default     = false
+}
+
 # ----------------------------------------
 # If you want to deploy multiple versions you can use these variables to
 # provide a list of builds to override the values of nomad_sha, nomad_version,

e2e/terraform/volumes.tf

@@ -1,4 +1,5 @@
 resource "aws_efs_file_system" "csi" {
+  count          = var.volumes ? 1 : 0
   creation_token = "${local.random_name}-CSI"
 
   tags = {
@@ -8,12 +9,14 @@ resource "aws_efs_file_system" "csi" {
 }
 
 resource "aws_efs_mount_target" "csi" {
-  file_system_id  = aws_efs_file_system.csi.id
+  count           = var.volumes ? 1 : 0
+  file_system_id  = aws_efs_file_system.csi[0].id
   subnet_id       = data.aws_subnet.default.id
-  security_groups = [aws_security_group.nfs.id]
+  security_groups = [aws_security_group.nfs[0].id]
 }
 
 resource "aws_ebs_volume" "csi" {
+  count             = var.volumes ? 1 : 0
   availability_zone = var.availability_zone
   size              = 40
 
@@ -24,11 +27,12 @@ resource "aws_ebs_volume" "csi" {
 }
 
 data "template_file" "ebs_volume_hcl" {
+  count    = var.volumes ? 1 : 0
   template = <<EOT
 type = "csi"
 id = "ebs-vol0"
 name = "ebs-vol0"
-external_id = "${aws_ebs_volume.csi.id}"
+external_id = "${aws_ebs_volume.csi[0].id}"
 access_mode = "single-node-writer"
 attachment_mode = "file-system"
 plugin_id = "aws-ebs0"
@@ -36,11 +40,12 @@ EOT
 }
 
 data "template_file" "efs_volume_hcl" {
+  count    = var.volumes ? 1 : 0
   template = <<EOT
 type = "csi"
 id = "efs-vol0"
 name = "efs-vol0"
-external_id = "${aws_efs_file_system.csi.id}"
+external_id = "${aws_efs_file_system.csi[0].id}"
 access_mode = "single-node-writer"
 attachment_mode = "file-system"
 plugin_id = "aws-efs0"
@@ -48,13 +53,15 @@ EOT
 }
 
 resource "local_file" "ebs_volume_hcl" {
-  content         = data.template_file.ebs_volume_hcl.rendered
+  count           = var.volumes ? 1 : 0
+  content         = data.template_file.ebs_volume_hcl[0].rendered
   filename        = "${path.module}/../csi/input/volume-ebs.hcl"
   file_permission = "0664"
 }
 
 resource "local_file" "efs_volume_hcl" {
-  content         = data.template_file.efs_volume_hcl.rendered
+  count           = var.volumes ? 1 : 0
+  content         = data.template_file.efs_volume_hcl[0].rendered
   filename        = "${path.module}/../csi/input/volume-efs.hcl"
   file_permission = "0664"
 }