cli: upload var file(s) content on job submission (#17128)

This PR makes it so that the content of any -var-file files is uploaded
to Nomad on job run.

command/helpers.go

@@ -527,6 +527,9 @@ func (j *JobGetter) Get(jpath string) (*api.JobSubmission, *api.Job, error) {
 			Format: formatJSON,
 		}
 	default:
+		// we are parsing HCL2
+
+		// make a copy of the job file (or stdio)
 		if _, err = io.Copy(&source, jobfile); err != nil {
 			return nil, nil, fmt.Errorf("Failed to parse HCL job: %w", err)
 		}
@@ -542,9 +545,20 @@ func (j *JobGetter) Get(jpath string) (*api.JobSubmission, *api.Job, error) {
 			Strict:   j.Strict,
 		})
 
+		var varFileCat string
+		var readVarFileErr error
+		if err == nil {
+			// combine any -var-file data into one big blob
+			varFileCat, readVarFileErr = extractVarFiles([]string(j.VarFiles))
+			if readVarFileErr != nil {
+				return nil, nil, fmt.Errorf("Failed to read var file(s): %w", readVarFileErr)
+			}
+		}
+
 		// submit the job with the submission with content from -var flags
 		jobSubmission = &api.JobSubmission{
 			VariableFlags: extractVarFlags(j.Vars),
+			Variables:     varFileCat,
 			Source:        source.String(),
 			Format:        formatHCL2,
 		}
@@ -562,6 +576,21 @@ func (j *JobGetter) Get(jpath string) (*api.JobSubmission, *api.Job, error) {
 	return jobSubmission, jobStruct, nil
 }
 
+// extractVarFiles concatenates the content of each file in filenames and
+// returns it all as one big content blob
+func extractVarFiles(filenames []string) (string, error) {
+	var sb strings.Builder
+	for _, filename := range filenames {
+		b, err := os.ReadFile(filename)
+		if err != nil {
+			return "", err
+		}
+		sb.WriteString(string(b))
+		sb.WriteString("\n")
+	}
+	return sb.String(), nil
+}
+
 // extractVarFlags is used to parse the values of -var command line arguments
 // and turn them into a map to be used for submission. The result is never
 // nil for convenience.

command/helpers_test.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"testing"
@@ -16,6 +17,7 @@ import (
 
 	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client/testutil"
 	"github.com/hashicorp/nomad/helper/flatmap"
 	"github.com/hashicorp/nomad/helper/pointer"
 	"github.com/kr/pretty"
@@ -629,6 +631,41 @@ func TestUiErrorWriter(t *testing.T) {
 	require.Equal(t, expectedErr, errBuf.String())
 }
 
+func Test_extractVarFiles(t *testing.T) {
+	ci.Parallel(t)
+
+	t.Run("none", func(t *testing.T) {
+		result, err := extractVarFiles(nil)
+		must.NoError(t, err)
+		must.Eq(t, "", result)
+	})
+
+	t.Run("files", func(t *testing.T) {
+		d := t.TempDir()
+		fileOne := filepath.Join(d, "one.hcl")
+		fileTwo := filepath.Join(d, "two.hcl")
+
+		must.NoError(t, os.WriteFile(fileOne, []byte(`foo = "bar"`), 0o644))
+		must.NoError(t, os.WriteFile(fileTwo, []byte(`baz = 42`), 0o644))
+
+		result, err := extractVarFiles([]string{fileOne, fileTwo})
+		must.NoError(t, err)
+		must.Eq(t, "foo = \"bar\"\nbaz = 42\n", result)
+	})
+
+	t.Run("unreadble", func(t *testing.T) {
+		testutil.RequireNonRoot(t)
+
+		d := t.TempDir()
+		fileOne := filepath.Join(d, "one.hcl")
+
+		must.NoError(t, os.WriteFile(fileOne, []byte(`foo = "bar"`), 0o200))
+
+		_, err := extractVarFiles([]string{fileOne})
+		must.ErrorContains(t, err, "permission denied")
+	})
+}
+
 func Test_extractVarFlags(t *testing.T) {
 	ci.Parallel(t)
 

e2e/jobsubmissions/input/x.hcl

@@ -0,0 +1,4 @@
+# This is an HCL comment
+
+X = "my var file x value"
+

e2e/jobsubmissions/input/y.hcl

@@ -0,0 +1,3 @@
+
+Y = 700
+

e2e/jobsubmissions/input/z.hcl

@@ -0,0 +1,2 @@
+Z = true
+

e2e/jobsubmissions/jobsubapi_test.go

@@ -25,6 +25,7 @@ func TestJobSubmissionAPI(t *testing.T) {
 	t.Run("testSubmissionACL", testSubmissionACL)
 	t.Run("testMaxSize", testMaxSize)
 	t.Run("testReversion", testReversion)
+	t.Run("testVarFiles", testVarFiles)
 }
 
 func testParseAPI(t *testing.T) {
@@ -317,3 +318,39 @@ func testReversion(t *testing.T) {
 		must.Eq(t, expectY[version], sub.VariableFlags["Y"])
 	}
 }
+
+func testVarFiles(t *testing.T) {
+	nomad := e2eutil.NomadClient(t)
+
+	jobID := "job-sub-var-files-" + uuid.Short()
+	jobIDs := []string{jobID}
+	t.Cleanup(e2eutil.MaybeCleanupJobsAndGC(&jobIDs))
+
+	// register the xyz job using x.hcl y.hcl z.hcl var files
+	err := e2eutil.RegisterWithArgs(
+		jobID,
+		"input/xyz.hcl",
+		"-var-file=input/x.hcl",
+		"-var-file=input/y.hcl",
+		"-var-file=input/z.hcl",
+	)
+	must.NoError(t, err)
+
+	const version = 0
+
+	// find our alloc id
+	allocID := e2eutil.SingleAllocID(t, jobID, "", version)
+
+	// wait for alloc to complete
+	_ = e2eutil.WaitForAllocStopped(t, nomad, allocID)
+
+	// get submission
+	sub, _, err := nomad.Jobs().Submission(jobID, version, &api.QueryOptions{
+		Region:    "global",
+		Namespace: "default",
+	})
+	must.NoError(t, err)
+	must.StrContains(t, sub.Variables, `X = "my var file x value"`)
+	must.StrContains(t, sub.Variables, `Y = 700`)
+	must.StrContains(t, sub.Variables, `Z = true`)
+}