e2e: setup nomad permissions correctly (client vs. server) (#16399)

This PR configures

- server nodes with a systemd unit running the agent as the nomad service user
- client nodes with a root owned nomad data directory

e2e/terraform/etc/nomad.d/nomad-client.service

@@ -1,11 +1,12 @@
 [Unit]
-Description=Nomad Agent
+Description=Nomad Client Agent
 Requires=network-online.target
 After=network-online.target
 StartLimitIntervalSec=0
 StartLimitBurst=3
 
 [Service]
+User=root
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d
 EnvironmentFile=-/etc/nomad.d/.environment

e2e/terraform/etc/nomad.d/nomad-server.service

@@ -1,11 +1,12 @@
 [Unit]
-Description=Nomad Agent
+Description=Nomad Server Agent
 Requires=network-online.target
 After=network-online.target
 StartLimitIntervalSec=0
 StartLimitBurst=3
 
 [Service]
+User=nomad
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d
 EnvironmentFile=-/etc/nomad.d/.environment

e2e/terraform/provision-nomad/install-linux.tf

@@ -58,6 +58,10 @@ resource "null_resource" "install_consul_configs_linux" {
   }
 }
 
+locals {
+  data_owner = var.role == "client" ? "root" : "nomad"
+}
+
 resource "null_resource" "install_nomad_configs_linux" {
   count = var.platform == "linux" ? 1 : 0
 
@@ -79,6 +83,7 @@ resource "null_resource" "install_nomad_configs_linux" {
       "mkdir -p /etc/nomad.d",
       "mkdir -p /opt/nomad/data",
       "sudo chmod 0700 /opt/nomad/data",
+      "sudo chown ${local.data_owner}:${local.data_owner} /opt/nomad/data",
       "sudo rm -rf /etc/nomad.d/*",
       "sudo mv /tmp/consul.hcl /etc/nomad.d/consul.hcl",
       "sudo mv /tmp/vault.hcl /etc/nomad.d/vault.hcl",