e2e: prevent Ubuntu startup race conditions (#9428)

The cloud-init configuration runs on boot, which can result in a race
condition between that and service startup. This has caused provisioning
failures because Nomad expects the userdata to have configured a host volume
directory. Diagnosing this was also compounded by a warning being fired by
systemd for the Nomad unit file.

* Update the location of the `StartLimitIntervalSec` field to it's
  post-systemd-230 location.
* Ensure that the weekly AMI build is up-to-date to reduce the risk of
  unexpected system software changes.
* Move the host volume to a directory we can set up at AMI build time rather
  than in userdata.

e2e/terraform/config/dev-cluster/nomad/client-linux/client.hcl

@@ -15,7 +15,7 @@ client {
   }
 
   host_volume "shared_data" {
-    path = "/tmp/data"
+    path = "/srv/data"
   }
 }
 

e2e/terraform/config/full-cluster/nomad/client-linux/indexed/client-0.hcl

@@ -12,7 +12,7 @@ client {
   }
 
   host_volume "shared_data" {
-    path = "/tmp/data"
+    path = "/srv/data"
   }
 }
 

e2e/terraform/packer/ubuntu-bionic-amd64/nomad.service

@@ -2,6 +2,7 @@
 Description=Nomad Agent
 Requires=network-online.target
 After=network-online.target
+StartLimitIntervalSec=10
 
 [Service]
 ExecReload=/bin/kill -HUP $MAINPID
@@ -13,9 +14,7 @@ LimitNPROC=infinity
 TasksMax=infinity
 Restart=on-failure
 RestartSec=2
-StartLimitIntervalSec=10
 StartLimitBurst=3
 
-
 [Install]
 WantedBy=multi-user.target

e2e/terraform/packer/ubuntu-bionic-amd64/setup.sh

@@ -24,9 +24,11 @@ sudo mkdir -p /ops/shared
 sudo chown -R ubuntu:ubuntu /ops/shared
 
 mkdir_for_root /opt
+mkdir_for_root /srv/data # for host volumes
 
 # Dependencies
 sudo apt-get update
+sudo apt-get upgrade -y
 sudo apt-get install -y \
      software-properties-common \
      dnsmasq unzip tree redis-tools jq curl tmux awscli nfs-common \

e2e/terraform/userdata/ubuntu-bionic.sh

@@ -14,9 +14,6 @@ nameserver $DOCKER_BRIDGE_IP_ADDRESS
 EOF
 sudo mv /tmp/resolv.conf /etc/resolv.conf
 
-# For host volume testing
-sudo mkdir -p /tmp/data
-
 # need to get the interface for dnsmasq config so that we can
 # accomodate both "predictable" and old-style interface names
 IFACE=$(/usr/local/bin/sockaddr eval 'GetDefaultInterfaces | attr "Name"')