From 291dd3dfb1c7dba778cafa7e0be1207adc42ee33 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Tue, 3 Nov 2015 15:15:40 -0600 Subject: [PATCH 1/4] add getter package, a thin wrapper of go-getter --- client/getter/getter.go | 44 +++++++++++++++ client/getter/getter_test.go | 103 +++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 client/getter/getter.go create mode 100644 client/getter/getter_test.go diff --git a/client/getter/getter.go b/client/getter/getter.go new file mode 100644 index 000000000..a113dd1cb --- /dev/null +++ b/client/getter/getter.go @@ -0,0 +1,44 @@ +package getter + +import ( + "fmt" + "log" + "net/url" + "path" + "path/filepath" + "runtime" + "strings" + "syscall" + + gg "github.com/hashicorp/go-getter" +) + +func GetArtifact(destDir, source, checksum string, logger *log.Logger) (string, error) { + // We use go-getter to support a variety of protocols, but need to change + // file permissions of the resulted download to be executable + + u, err := url.Parse(source) + if err != nil { + return "", err + } + + // look for checksum, apply to URL + if checksum != "" { + source = strings.Join([]string{source, fmt.Sprintf("checksum=%s", checksum)}, "?") + logger.Printf("[DEBUG] Applying checksum to Artifact Source URL, new url: %s", source) + } + + artifactName := path.Base(u.Path) + artifactFile := filepath.Join(destDir, artifactName) + if err := gg.GetFile(artifactFile, source); err != nil { + return "", fmt.Errorf("Error downloading artifact: %s", err) + } + + // Add execution permissions to the newly downloaded artifact + if runtime.GOOS != "windows" { + if err := syscall.Chmod(artifactFile, 0755); err != nil { + logger.Printf("[ERR] driver.raw_exec: Error making artifact executable: %s", err) + } + } + return artifactFile, nil +} diff --git a/client/getter/getter_test.go b/client/getter/getter_test.go new file mode 100644 index 000000000..570b2a8c9 --- /dev/null +++ b/client/getter/getter_test.go @@ -0,0 +1,103 @@ +package getter + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "strings" + "testing" +) + +func TestGetArtifact_basic(t *testing.T) { + + logger := log.New(os.Stderr, "", log.LstdFlags) + + // TODO: Use http.TestServer to serve these files from fixtures dir + passing := []struct { + Source, Checksum string + }{ + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_darwin_amd64", + "sha256:66aa0f05fc0cfcf1e5ed8cc5307b5df51e33871d6b295a60e0f9f6dd573da977", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "sha256:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "md5:a9b14903a8942748e4f8474e11f795d3", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64?checksum=sha256:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", + "", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "", + }, + } + + for i, p := range passing { + destDir, err := ioutil.TempDir("", fmt.Sprintf("nomad-test-%d", i)) + if err != nil { + t.Fatalf("Error in TestGetArtifact_basic makeing TempDir: %s", err) + } + + path, err := GetArtifact(destDir, p.Source, p.Checksum, logger) + if err != nil { + t.Fatalf("TestGetArtifact_basic unexpected failure here: %s", err) + } + + if p.Checksum != "" { + if ok := strings.Contains(path, p.Checksum); ok { + t.Fatalf("path result should not contain the checksum, got: %s", path) + } + } + + // verify artifact exists + if _, err := os.Stat(path); err != nil { + t.Fatalf("source path error: %s", err) + } + } +} + +func TestGetArtifact_fails(t *testing.T) { + + logger := log.New(os.Stderr, "", log.LstdFlags) + + failing := []struct { + Source, Checksum string + }{ + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_darwin_amd64", + "sha256:66aa0f05fc0cfcf1e5ed8cc5307b5d", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "sha257:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", + }, + // malformed checksum + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", + }, + // 404 + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd86", + "", + }, + } + for i, p := range failing { + destDir, err := ioutil.TempDir("", fmt.Sprintf("nomad-test-%d", i)) + if err != nil { + t.Fatalf("Error in TestGetArtifact_basic makeing TempDir: %s", err) + } + + _, err = GetArtifact(destDir, p.Source, p.Checksum, logger) + if err == nil { + t.Fatalf("TestGetArtifact_basic expected failure, but got none") + } + } +} From 145a2e361087d7ae1cec5696fd25a46cc720c1e4 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Tue, 3 Nov 2015 15:16:17 -0600 Subject: [PATCH 2/4] refactor drivers to use new getter --- client/driver/exec.go | 37 ++-- client/driver/java.go | 27 ++- client/driver/java_test.go | 10 +- client/driver/qemu.go | 44 ++--- client/driver/qemu_test.go | 18 +- client/driver/raw_exec.go | 30 +--- client/getter/getter.go | 65 ++++--- client/getter/getter_test.go | 176 ++++++++++--------- website/source/docs/drivers/exec.html.md | 5 + website/source/docs/drivers/java.html.md | 25 ++- website/source/docs/drivers/qemu.html.md | 10 +- website/source/docs/drivers/raw_exec.html.md | 5 + 12 files changed, 224 insertions(+), 228 deletions(-) diff --git a/client/driver/exec.go b/client/driver/exec.go index e48604894..213bc574f 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -2,17 +2,15 @@ package driver import ( "fmt" - "log" - "path" "path/filepath" "runtime" "syscall" "time" - "github.com/hashicorp/go-getter" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/executor" + "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/nomad/structs" ) @@ -55,29 +53,24 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, fmt.Errorf("missing command for exec driver") } + // Create a location to download the artifact. + taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] + if !ok { + return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) + } + // Check if an artificat is specified and attempt to download it source, ok := task.Config["artifact_source"] if ok && source != "" { // Proceed to download an artifact to be executed. - // We use go-getter to support a variety of protocols, but need to change - // file permissions of the resulted download to be executable - - // Create a location to download the artifact. - taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] - if !ok { - return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) - } - destDir := filepath.Join(taskDir, allocdir.TaskLocal) - - artifactName := path.Base(source) - artifactFile := filepath.Join(destDir, artifactName) - if err := getter.GetFile(artifactFile, source); err != nil { - return nil, fmt.Errorf("Error downloading artifact for Exec driver: %s", err) - } - - // Add execution permissions to the newly downloaded artifact - if err := syscall.Chmod(artifactFile, 0755); err != nil { - log.Printf("[ERR] driver.exec: Error making artifact executable: %s", err) + _, err := getter.GetArtifact( + filepath.Join(taskDir, allocdir.TaskLocal), + task.Config["artifact_source"], + task.Config["checksum"], + d.logger, + ) + if err != nil { + return nil, err } } diff --git a/client/driver/java.go b/client/driver/java.go index e7563f6e2..808bdfe5b 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -4,17 +4,16 @@ import ( "bytes" "fmt" "os/exec" - "path" "path/filepath" "runtime" "strings" "syscall" "time" - "github.com/hashicorp/go-getter" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/executor" + "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/nomad/structs" ) @@ -89,26 +88,24 @@ func (d *JavaDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, } func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { - // Get the jar source - source, ok := task.Config["jar_source"] - if !ok || source == "" { - return nil, fmt.Errorf("missing jar source for Java Jar driver") - } - taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] if !ok { return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } - destDir := filepath.Join(taskDir, allocdir.TaskLocal) - - // Create a location to download the binary. - jarName := path.Base(source) - jarPath := filepath.Join(destDir, jarName) - if err := getter.GetFile(jarPath, source); err != nil { - return nil, fmt.Errorf("Error downloading source for Java driver: %s", err) + // Proceed to download an artifact to be executed. + path, err := getter.GetArtifact( + filepath.Join(taskDir, allocdir.TaskLocal), + task.Config["artifact_source"], + task.Config["checksum"], + d.logger, + ) + if err != nil { + return nil, err } + jarName := filepath.Base(path) + // Get the environment variables. envVars := TaskEnvironmentVariables(ctx, task) diff --git a/client/driver/java_test.go b/client/driver/java_test.go index b4f2f2e15..e525d4dce 100644 --- a/client/driver/java_test.go +++ b/client/driver/java_test.go @@ -97,10 +97,8 @@ func TestJavaDriver_Start_Wait(t *testing.T) { task := &structs.Task{ Name: "demo-app", Config: map[string]string{ - "jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", - // "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar", - // "args": "-d64", - "jvm_options": "-Xmx2048m -Xms256m", + "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", + "jvm_options": "-Xmx2048m -Xms256m", }, Resources: basicResources, } @@ -145,9 +143,7 @@ func TestJavaDriver_Start_Kill_Wait(t *testing.T) { task := &structs.Task{ Name: "demo-app", Config: map[string]string{ - "jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", - // "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar", - // "args": "-d64", + "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", }, Resources: basicResources, } diff --git a/client/driver/qemu.go b/client/driver/qemu.go index abf6d4dfa..b06c5dd47 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -2,11 +2,8 @@ package driver import ( "bytes" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" - "io" "log" "os" "os/exec" @@ -17,9 +14,9 @@ import ( "strings" "time" - "github.com/hashicorp/go-getter" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/nomad/structs" ) @@ -82,7 +79,7 @@ func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, // image and save it to the Drivers Allocation Dir func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { // Get the image source - source, ok := task.Config["image_source"] + source, ok := task.Config["artifact_source"] if !ok || source == "" { return nil, fmt.Errorf("Missing source image Qemu driver") } @@ -99,34 +96,19 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } - // Create a location to download the binary. - destDir := filepath.Join(taskDir, allocdir.TaskLocal) - vmID := fmt.Sprintf("qemu-vm-%s-%s", structs.GenerateUUID(), filepath.Base(source)) - vmPath := filepath.Join(destDir, vmID) - if err := getter.GetFile(vmPath, source); err != nil { - return nil, fmt.Errorf("Error downloading artifact for Qemu driver: %s", err) + // Proceed to download an artifact to be executed. + vmPath, err := getter.GetArtifact( + filepath.Join(taskDir, allocdir.TaskLocal), + task.Config["artifact_source"], + task.Config["checksum"], + d.logger, + ) + if err != nil { + d.logger.Printf("\n---\nDEBUG\n---\n error here: %s\n---\n", err) + return nil, err } - // compute and check checksum - if check, ok := task.Config["checksum"]; ok { - d.logger.Printf("[DEBUG] Running checksum on (%s)", vmID) - hasher := sha256.New() - file, err := os.Open(vmPath) - if err != nil { - return nil, fmt.Errorf("Failed to open file for checksum") - } - - defer file.Close() - io.Copy(hasher, file) - - sum := hex.EncodeToString(hasher.Sum(nil)) - if sum != check { - return nil, fmt.Errorf( - "Error in Qemu: checksums did not match.\nExpected (%s), got (%s)", - check, - sum) - } - } + vmID := filepath.Base(vmPath) // Parse configuration arguments // Create the base arguments diff --git a/client/driver/qemu_test.go b/client/driver/qemu_test.go index e9a9e5744..7b6ca1bb7 100644 --- a/client/driver/qemu_test.go +++ b/client/driver/qemu_test.go @@ -54,10 +54,10 @@ func TestQemuDriver_Start(t *testing.T) { task := &structs.Task{ Name: "linux", Config: map[string]string{ - "image_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", - "checksum": "a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", - "accelerator": "tcg", - "guest_ports": "22,8080", + "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", + "checksum": "sha256:a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", + "accelerator": "tcg", + "guest_ports": "22,8080", }, Resources: &structs.Resources{ MemoryMB: 512, @@ -103,11 +103,11 @@ func TestQemuDriver_RequiresMemory(t *testing.T) { task := &structs.Task{ Name: "linux", Config: map[string]string{ - "image_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", - "accelerator": "tcg", - "host_port": "8080", - "guest_port": "8081", - "checksum": "a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", + "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", + "accelerator": "tcg", + "host_port": "8080", + "guest_port": "8081", + "checksum": "sha256:a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", // ssh u/p would be here }, } diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index fd54e1b86..9a2716297 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -2,21 +2,18 @@ package driver import ( "fmt" - "log" "os" "os/exec" - "path" "path/filepath" "runtime" "strconv" "strings" - "syscall" "time" - "github.com/hashicorp/go-getter" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/args" + "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/nomad/structs" ) @@ -83,23 +80,14 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl source, ok := task.Config["artifact_source"] if ok && source != "" { // Proceed to download an artifact to be executed. - // We use go-getter to support a variety of protocols, but need to change - // file permissions of the resulted download to be executable - - // Create a location to download the artifact. - destDir := filepath.Join(taskDir, allocdir.TaskLocal) - - artifactName := path.Base(source) - artifactFile := filepath.Join(destDir, artifactName) - if err := getter.GetFile(artifactFile, source); err != nil { - return nil, fmt.Errorf("Error downloading artifact for Raw Exec driver: %s", err) - } - - // Add execution permissions to the newly downloaded artifact - if runtime.GOOS != "windows" { - if err := syscall.Chmod(artifactFile, 0755); err != nil { - log.Printf("[ERR] driver.raw_exec: Error making artifact executable: %s", err) - } + _, err := getter.GetArtifact( + filepath.Join(taskDir, allocdir.TaskLocal), + task.Config["artifact_source"], + task.Config["checksum"], + d.logger, + ) + if err != nil { + return nil, err } } diff --git a/client/getter/getter.go b/client/getter/getter.go index a113dd1cb..1a721c3f2 100644 --- a/client/getter/getter.go +++ b/client/getter/getter.go @@ -1,44 +1,43 @@ package getter import ( - "fmt" - "log" - "net/url" - "path" - "path/filepath" - "runtime" - "strings" - "syscall" + "fmt" + "log" + "net/url" + "path" + "path/filepath" + "runtime" + "strings" + "syscall" - gg "github.com/hashicorp/go-getter" + gg "github.com/hashicorp/go-getter" ) func GetArtifact(destDir, source, checksum string, logger *log.Logger) (string, error) { - // We use go-getter to support a variety of protocols, but need to change - // file permissions of the resulted download to be executable + if source == "" { + return "", fmt.Errorf("Source url is empty in Artifact Getter") + } + u, err := url.Parse(source) + if err != nil { + return "", err + } - u, err := url.Parse(source) - if err != nil { - return "", err - } + // if checksum is seperate, apply to source + if checksum != "" { + source = strings.Join([]string{source, fmt.Sprintf("checksum=%s", checksum)}, "?") + logger.Printf("[DEBUG] client.getter: Applying checksum to Artifact Source URL, new url: %s", source) + } - // look for checksum, apply to URL - if checksum != "" { - source = strings.Join([]string{source, fmt.Sprintf("checksum=%s", checksum)}, "?") - logger.Printf("[DEBUG] Applying checksum to Artifact Source URL, new url: %s", source) - } + artifactFile := filepath.Join(destDir, path.Base(u.Path)) + if err := gg.GetFile(artifactFile, source); err != nil { + return "", fmt.Errorf("Error downloading artifact: %s", err) + } - artifactName := path.Base(u.Path) - artifactFile := filepath.Join(destDir, artifactName) - if err := gg.GetFile(artifactFile, source); err != nil { - return "", fmt.Errorf("Error downloading artifact: %s", err) - } - - // Add execution permissions to the newly downloaded artifact - if runtime.GOOS != "windows" { - if err := syscall.Chmod(artifactFile, 0755); err != nil { - logger.Printf("[ERR] driver.raw_exec: Error making artifact executable: %s", err) - } - } - return artifactFile, nil + // Add execution permissions to the newly downloaded artifact + if runtime.GOOS != "windows" { + if err := syscall.Chmod(artifactFile, 0755); err != nil { + logger.Printf("[ERR] driver.raw_exec: Error making artifact executable: %s", err) + } + } + return artifactFile, nil } diff --git a/client/getter/getter_test.go b/client/getter/getter_test.go index 570b2a8c9..54eff20c6 100644 --- a/client/getter/getter_test.go +++ b/client/getter/getter_test.go @@ -1,103 +1,111 @@ package getter import ( - "fmt" - "io/ioutil" - "log" - "os" - "strings" - "testing" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + "testing" ) func TestGetArtifact_basic(t *testing.T) { - logger := log.New(os.Stderr, "", log.LstdFlags) + logger := log.New(os.Stderr, "", log.LstdFlags) - // TODO: Use http.TestServer to serve these files from fixtures dir - passing := []struct { - Source, Checksum string - }{ - { - "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_darwin_amd64", - "sha256:66aa0f05fc0cfcf1e5ed8cc5307b5df51e33871d6b295a60e0f9f6dd573da977", - }, - { - "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", - "sha256:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", - }, - { - "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", - "md5:a9b14903a8942748e4f8474e11f795d3", - }, - { - "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64?checksum=sha256:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", - "", - }, - { - "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", - "", - }, - } + // TODO: Use http.TestServer to serve these files from fixtures dir + passing := []struct { + Source, Checksum string + }{ + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_darwin_amd64", + "sha256:66aa0f05fc0cfcf1e5ed8cc5307b5df51e33871d6b295a60e0f9f6dd573da977", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "sha256:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "md5:a9b14903a8942748e4f8474e11f795d3", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64?checksum=sha256:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", + "", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "", + }, + } - for i, p := range passing { - destDir, err := ioutil.TempDir("", fmt.Sprintf("nomad-test-%d", i)) - if err != nil { - t.Fatalf("Error in TestGetArtifact_basic makeing TempDir: %s", err) - } + for i, p := range passing { + destDir, err := ioutil.TempDir("", fmt.Sprintf("nomad-test-%d", i)) + if err != nil { + t.Fatalf("Error in TestGetArtifact_basic makeing TempDir: %s", err) + } - path, err := GetArtifact(destDir, p.Source, p.Checksum, logger) - if err != nil { - t.Fatalf("TestGetArtifact_basic unexpected failure here: %s", err) - } + path, err := GetArtifact(destDir, p.Source, p.Checksum, logger) + if err != nil { + t.Fatalf("TestGetArtifact_basic unexpected failure here: %s", err) + } - if p.Checksum != "" { - if ok := strings.Contains(path, p.Checksum); ok { - t.Fatalf("path result should not contain the checksum, got: %s", path) - } - } + if p.Checksum != "" { + if ok := strings.Contains(path, p.Checksum); ok { + t.Fatalf("path result should not contain the checksum, got: %s", path) + } + } - // verify artifact exists - if _, err := os.Stat(path); err != nil { - t.Fatalf("source path error: %s", err) - } - } + // verify artifact exists + if _, err := os.Stat(path); err != nil { + t.Fatalf("source path error: %s", err) + } + } } func TestGetArtifact_fails(t *testing.T) { - logger := log.New(os.Stderr, "", log.LstdFlags) + logger := log.New(os.Stderr, "", log.LstdFlags) - failing := []struct { - Source, Checksum string - }{ - { - "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_darwin_amd64", - "sha256:66aa0f05fc0cfcf1e5ed8cc5307b5d", - }, - { - "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", - "sha257:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", - }, - // malformed checksum - { - "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", - "6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", - }, - // 404 - { - "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd86", - "", - }, - } - for i, p := range failing { - destDir, err := ioutil.TempDir("", fmt.Sprintf("nomad-test-%d", i)) - if err != nil { - t.Fatalf("Error in TestGetArtifact_basic makeing TempDir: %s", err) - } + failing := []struct { + Source, Checksum string + }{ + { + "", + "sha256:66aa0f05fc0cfcf1e5ed8cc5307b5d", + }, + { + "/u/47675/jar_thing/hi_darwin_amd64", + "sha256:66aa0f05fc0cfcf1e5ed8cc5307b5d", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_darwin_amd64", + "sha256:66aa0f05fc0cfcf1e5ed8cc5307b5d", + }, + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "sha257:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", + }, + // malformed checksum + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd64", + "6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c", + }, + // 404 + { + "https://dl.dropboxusercontent.com/u/47675/jar_thing/hi_linux_amd86", + "", + }, + } + for i, p := range failing { + destDir, err := ioutil.TempDir("", fmt.Sprintf("nomad-test-%d", i)) + if err != nil { + t.Fatalf("Error in TestGetArtifact_basic makeing TempDir: %s", err) + } - _, err = GetArtifact(destDir, p.Source, p.Checksum, logger) - if err == nil { - t.Fatalf("TestGetArtifact_basic expected failure, but got none") - } - } + _, err = GetArtifact(destDir, p.Source, p.Checksum, logger) + if err == nil { + t.Fatalf("TestGetArtifact_basic expected failure, but got none") + } + } } diff --git a/website/source/docs/drivers/exec.html.md b/website/source/docs/drivers/exec.html.md index e82aa1505..f897b1ea4 100644 --- a/website/source/docs/drivers/exec.html.md +++ b/website/source/docs/drivers/exec.html.md @@ -24,6 +24,10 @@ The `exec` driver supports the following configuration in the job spec: * `artifact_source` – (Optional) Source location of an executable artifact. Must be accessible from the Nomad client. If you specify an `artifact_source` to be executed, you must reference it in the `command` as show in the examples below +* `checksum` - **(Optional)** The checksum type and value for the `artifact_source` image. +The format is `type:value`, where type is any of `md5`, `sha1`, `sha256`, or `sha512`, +and the value is the computed checksum. If a checksum is supplied and does not +match the downloaded artifact, the driver will fail to start * `args` - The argument list to the command, space seperated. Optional. ## Client Requirements @@ -53,6 +57,7 @@ To execute a binary specified by `artifact_source`: ``` config { artifact_source = "https://dl.dropboxusercontent.com/u/1234/binary.bin" + checksum = "sha256:abd123445ds4555555555" command = "$NOMAD_TASK_DIR/binary.bin" } ``` diff --git a/website/source/docs/drivers/java.html.md b/website/source/docs/drivers/java.html.md index ecfddb645..f2bbd2b76 100644 --- a/website/source/docs/drivers/java.html.md +++ b/website/source/docs/drivers/java.html.md @@ -18,8 +18,12 @@ HTTP from the Nomad client. The `java` driver supports the following configuration in the job spec: -* `jar_source` - **(Required)** The hosted location of the source Jar file. Must be accessible +* `artifact_source` - **(Required)** The hosted location of the source Jar file. Must be accessible from the Nomad client +* `checksum` - **(Optional)** The checksum type and value for the `artifact_source` image. +The format is `type:value`, where type is any of `md5`, `sha1`, `sha256`, or `sha512`, +and the value is the computed checksum. If a checksum is supplied and does not +match the downloaded artifact, the driver will fail to start * `args` - **(Optional)** The argument list for the `java` command, space separated. @@ -29,10 +33,27 @@ from the Nomad client ## Client Requirements The `java` driver requires Java to be installed and in your systems `$PATH`. -The `jar_source` must be accessible by the node running Nomad. This can be an +The `artifact_source` must be accessible by the node running Nomad. This can be an internal source, private to your cluster, but it must be reachable by the client over HTTP. +## Examples + +A simple config block to run a Java Jar: + +```json +# Define a task to run +task "web" { + # Run a Java Jar + driver = "java" + + config { + artifact_source = "https://dl.dropboxusercontent.com/u/1234/hello.jar" + checksum = "md5:123445555555555" + jvm_options = "-Xmx2048m -Xms256m" + } +``` + ## Client Attributes The `java` driver will set the following client attributes: diff --git a/website/source/docs/drivers/qemu.html.md b/website/source/docs/drivers/qemu.html.md index 3e19076a0..403926b4c 100644 --- a/website/source/docs/drivers/qemu.html.md +++ b/website/source/docs/drivers/qemu.html.md @@ -23,10 +23,12 @@ The `Qemu` driver can execute any regular `qemu` image (e.g. `qcow`, `img`, The `Qemu` driver supports the following configuration in the job spec: -* `image_source` - **(Required)** The hosted location of the source Qemu image. Must be accessible +* `artifact_source` - **(Required)** The hosted location of the source Qemu image. Must be accessible from the Nomad client, via HTTP. -* `checksum` - **(Required)** The SHA256 checksum of the `qemu` image. If the -checksums do not match, the `Qemu` driver will fail to start the image +* `checksum` - **(Optional)** The checksum type and value for the `artifact_source` image. +The format is `type:value`, where type is any of `md5`, `sha1`, `sha256`, or `sha512`, +and the value is the computed checksum. If a checksum is supplied and does not +match the downloaded artifact, the driver will fail to start * `accelerator` - (Optional) The type of accelerator to use in the invocation. If the host machine has `Qemu` installed with KVM support, users can specify `kvm` for the `accelerator`. Default is `tcg` * `host_port` - **(Required)** Port on the host machine to forward to the guest @@ -38,7 +40,7 @@ in the `Task` specification ## Client Requirements The `Qemu` driver requires Qemu to be installed and in your system's `$PATH`. -The `image_source` must be accessible by the node running Nomad. This can be an +The `artifact_source` must be accessible by the node running Nomad. This can be an internal source, private to your cluster, but it must be reachable by the client over HTTP. diff --git a/website/source/docs/drivers/raw_exec.html.md b/website/source/docs/drivers/raw_exec.html.md index fa67129ba..2dc741887 100644 --- a/website/source/docs/drivers/raw_exec.html.md +++ b/website/source/docs/drivers/raw_exec.html.md @@ -22,6 +22,10 @@ The `raw_exec` driver supports the following configuration in the job spec: * `artifact_source` – (Optional) Source location of an executable artifact. Must be accessible from the Nomad client. If you specify an `artifact_source` to be executed, you must reference it in the `command` as show in the examples below +* `checksum` - **(Optional)** The checksum type and value for the `artifact_source` image. +The format is `type:value`, where type is any of `md5`, `sha1`, `sha256`, or `sha512`, +and the value is the computed checksum. If a checksum is supplied and does not +match the downloaded artifact, the driver will fail to start * `args` - The argument list to the command, space seperated. Optional. ## Client Requirements @@ -57,6 +61,7 @@ To execute a binary specified by `artifact_source`: ``` config { artifact_source = "https://dl.dropboxusercontent.com/u/1234/binary.bin" + checksum = "sha256:133jifjiofu9090fsadjofsdjlk" command = "$NOMAD_TASK_DIR/binary.bin" } ``` From 1a35649ea1e8e0e9452bb4cd8a312238b44cfb56 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Wed, 4 Nov 2015 10:16:22 -0600 Subject: [PATCH 3/4] remove debug line --- client/driver/qemu.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/driver/qemu.go b/client/driver/qemu.go index b06c5dd47..52b2e78e0 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -104,9 +104,8 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, d.logger, ) if err != nil { - d.logger.Printf("\n---\nDEBUG\n---\n error here: %s\n---\n", err) return nil, err - } + } vmID := filepath.Base(vmPath) From fb8066d8f318b31a3ada4668a45a548d81ba308b Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Thu, 5 Nov 2015 09:23:45 -0600 Subject: [PATCH 4/4] add checksums to tests, formatting after rebase --- client/driver/exec_test.go | 3 ++- client/driver/java_test.go | 7 ++++--- client/driver/qemu.go | 26 +++++++++++++------------- client/driver/qemu_test.go | 18 +++++++++--------- client/driver/raw_exec.go | 18 +++++++++--------- client/driver/raw_exec_test.go | 5 ++++- 6 files changed, 41 insertions(+), 36 deletions(-) diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go index 1bb4adf36..bc8323889 100644 --- a/client/driver/exec_test.go +++ b/client/driver/exec_test.go @@ -123,11 +123,12 @@ func TestExecDriver_Start_Wait(t *testing.T) { func TestExecDriver_Start_Artifact_basic(t *testing.T) { ctestutils.ExecCompatible(t) file := "hi_linux_amd64" + checksum := "sha256:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c" task := &structs.Task{ Name: "sleep", Config: map[string]string{ - "artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s", file), + "artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s?checksum=%s", file, checksum), "command": filepath.Join("$NOMAD_TASK_DIR", file), }, Resources: basicResources, diff --git a/client/driver/java_test.go b/client/driver/java_test.go index e525d4dce..206cc1c78 100644 --- a/client/driver/java_test.go +++ b/client/driver/java_test.go @@ -97,8 +97,9 @@ func TestJavaDriver_Start_Wait(t *testing.T) { task := &structs.Task{ Name: "demo-app", Config: map[string]string{ - "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", - "jvm_options": "-Xmx2048m -Xms256m", + "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", + "jvm_options": "-Xmx2048m -Xms256m", + "checksum": "sha256:58d6e8130308d32e197c5108edd4f56ddf1417408f743097c2e662df0f0b17c8", }, Resources: basicResources, } @@ -143,7 +144,7 @@ func TestJavaDriver_Start_Kill_Wait(t *testing.T) { task := &structs.Task{ Name: "demo-app", Config: map[string]string{ - "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", + "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", }, Resources: basicResources, } diff --git a/client/driver/qemu.go b/client/driver/qemu.go index 52b2e78e0..0eab4e659 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -16,7 +16,7 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" - "github.com/hashicorp/nomad/client/getter" + "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/nomad/structs" ) @@ -79,7 +79,7 @@ func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, // image and save it to the Drivers Allocation Dir func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { // Get the image source - source, ok := task.Config["artifact_source"] + source, ok := task.Config["artifact_source"] if !ok || source == "" { return nil, fmt.Errorf("Missing source image Qemu driver") } @@ -96,18 +96,18 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } - // Proceed to download an artifact to be executed. - vmPath, err := getter.GetArtifact( - filepath.Join(taskDir, allocdir.TaskLocal), - task.Config["artifact_source"], - task.Config["checksum"], - d.logger, - ) - if err != nil { - return nil, err - } + // Proceed to download an artifact to be executed. + vmPath, err := getter.GetArtifact( + filepath.Join(taskDir, allocdir.TaskLocal), + task.Config["artifact_source"], + task.Config["checksum"], + d.logger, + ) + if err != nil { + return nil, err + } - vmID := filepath.Base(vmPath) + vmID := filepath.Base(vmPath) // Parse configuration arguments // Create the base arguments diff --git a/client/driver/qemu_test.go b/client/driver/qemu_test.go index 7b6ca1bb7..dffdc7bf0 100644 --- a/client/driver/qemu_test.go +++ b/client/driver/qemu_test.go @@ -54,10 +54,10 @@ func TestQemuDriver_Start(t *testing.T) { task := &structs.Task{ Name: "linux", Config: map[string]string{ - "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", - "checksum": "sha256:a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", - "accelerator": "tcg", - "guest_ports": "22,8080", + "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", + "checksum": "sha256:a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", + "accelerator": "tcg", + "guest_ports": "22,8080", }, Resources: &structs.Resources{ MemoryMB: 512, @@ -103,11 +103,11 @@ func TestQemuDriver_RequiresMemory(t *testing.T) { task := &structs.Task{ Name: "linux", Config: map[string]string{ - "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", - "accelerator": "tcg", - "host_port": "8080", - "guest_port": "8081", - "checksum": "sha256:a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", + "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", + "accelerator": "tcg", + "host_port": "8080", + "guest_port": "8081", + "checksum": "sha256:a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", // ssh u/p would be here }, } diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index 9a2716297..856f2b7fc 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -13,7 +13,7 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/args" - "github.com/hashicorp/nomad/client/getter" + "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/nomad/structs" ) @@ -80,14 +80,14 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl source, ok := task.Config["artifact_source"] if ok && source != "" { // Proceed to download an artifact to be executed. - _, err := getter.GetArtifact( - filepath.Join(taskDir, allocdir.TaskLocal), - task.Config["artifact_source"], - task.Config["checksum"], - d.logger, - ) - if err != nil { - return nil, err + _, err := getter.GetArtifact( + filepath.Join(taskDir, allocdir.TaskLocal), + task.Config["artifact_source"], + task.Config["checksum"], + d.logger, + ) + if err != nil { + return nil, err } } diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go index 0a6133df9..1dda991c8 100644 --- a/client/driver/raw_exec_test.go +++ b/client/driver/raw_exec_test.go @@ -94,12 +94,14 @@ func TestRawExecDriver_StartOpen_Wait(t *testing.T) { } func TestRawExecDriver_Start_Artifact_basic(t *testing.T) { - var file string + var file, checksum string switch runtime.GOOS { case "darwin": file = "hi_darwin_amd64" + checksum = "md5:d7f2fdb13b36dcb7407721d78926b335" default: file = "hi_linux_amd64" + checksum = "md5:a9b14903a8942748e4f8474e11f795d3" } task := &structs.Task{ @@ -107,6 +109,7 @@ func TestRawExecDriver_Start_Artifact_basic(t *testing.T) { Config: map[string]string{ "artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s", file), "command": filepath.Join("$NOMAD_TASK_DIR", file), + "checksum": checksum, }, } driverCtx := testDriverContext(task.Name)