372 lines
10 KiB
Go
372 lines
10 KiB
Go
package getter
|
|
|
|
import (
|
|
"fmt"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/hashicorp/nomad/client/taskenv"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// fakeReplacer is a noop version of taskenv.TaskEnv.ReplaceEnv
|
|
type fakeReplacer struct{}
|
|
|
|
func (fakeReplacer) ReplaceEnv(s string) string {
|
|
return s
|
|
}
|
|
|
|
var taskEnv = fakeReplacer{}
|
|
|
|
func TestGetArtifact_FileAndChecksum(t *testing.T) {
|
|
// Create the test server hosting the file to download
|
|
ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/"))))
|
|
defer ts.Close()
|
|
|
|
// Create a temp directory to download into
|
|
taskDir, err := ioutil.TempDir("", "nomad-test")
|
|
if err != nil {
|
|
t.Fatalf("failed to make temp directory: %v", err)
|
|
}
|
|
defer os.RemoveAll(taskDir)
|
|
|
|
// Create the artifact
|
|
file := "test.sh"
|
|
artifact := &structs.TaskArtifact{
|
|
GetterSource: fmt.Sprintf("%s/%s", ts.URL, file),
|
|
GetterOptions: map[string]string{
|
|
"checksum": "md5:bce963762aa2dbfed13caf492a45fb72",
|
|
},
|
|
}
|
|
|
|
// Download the artifact
|
|
if err := GetArtifact(taskEnv, artifact, taskDir); err != nil {
|
|
t.Fatalf("GetArtifact failed: %v", err)
|
|
}
|
|
|
|
// Verify artifact exists
|
|
if _, err := os.Stat(filepath.Join(taskDir, file)); err != nil {
|
|
t.Fatalf("file not found: %s", err)
|
|
}
|
|
}
|
|
|
|
func TestGetArtifact_File_RelativeDest(t *testing.T) {
|
|
// Create the test server hosting the file to download
|
|
ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/"))))
|
|
defer ts.Close()
|
|
|
|
// Create a temp directory to download into
|
|
taskDir, err := ioutil.TempDir("", "nomad-test")
|
|
if err != nil {
|
|
t.Fatalf("failed to make temp directory: %v", err)
|
|
}
|
|
defer os.RemoveAll(taskDir)
|
|
|
|
// Create the artifact
|
|
file := "test.sh"
|
|
relative := "foo/"
|
|
artifact := &structs.TaskArtifact{
|
|
GetterSource: fmt.Sprintf("%s/%s", ts.URL, file),
|
|
GetterOptions: map[string]string{
|
|
"checksum": "md5:bce963762aa2dbfed13caf492a45fb72",
|
|
},
|
|
RelativeDest: relative,
|
|
}
|
|
|
|
// Download the artifact
|
|
if err := GetArtifact(taskEnv, artifact, taskDir); err != nil {
|
|
t.Fatalf("GetArtifact failed: %v", err)
|
|
}
|
|
|
|
// Verify artifact was downloaded to the correct path
|
|
if _, err := os.Stat(filepath.Join(taskDir, relative, file)); err != nil {
|
|
t.Fatalf("file not found: %s", err)
|
|
}
|
|
}
|
|
|
|
func TestGetGetterUrl_Interpolation(t *testing.T) {
|
|
// Create the artifact
|
|
artifact := &structs.TaskArtifact{
|
|
GetterSource: "${NOMAD_META_ARTIFACT}",
|
|
}
|
|
|
|
url := "foo.com"
|
|
alloc := mock.Alloc()
|
|
task := alloc.Job.TaskGroups[0].Tasks[0]
|
|
task.Meta = map[string]string{"artifact": url}
|
|
taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build()
|
|
|
|
act, err := getGetterUrl(taskEnv, artifact)
|
|
if err != nil {
|
|
t.Fatalf("getGetterUrl() failed: %v", err)
|
|
}
|
|
|
|
if act != url {
|
|
t.Fatalf("getGetterUrl() returned %q; want %q", act, url)
|
|
}
|
|
}
|
|
|
|
func TestGetArtifact_InvalidChecksum(t *testing.T) {
|
|
// Create the test server hosting the file to download
|
|
ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/"))))
|
|
defer ts.Close()
|
|
|
|
// Create a temp directory to download into
|
|
taskDir, err := ioutil.TempDir("", "nomad-test")
|
|
if err != nil {
|
|
t.Fatalf("failed to make temp directory: %v", err)
|
|
}
|
|
defer os.RemoveAll(taskDir)
|
|
|
|
// Create the artifact with an incorrect checksum
|
|
file := "test.sh"
|
|
artifact := &structs.TaskArtifact{
|
|
GetterSource: fmt.Sprintf("%s/%s", ts.URL, file),
|
|
GetterOptions: map[string]string{
|
|
"checksum": "md5:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
},
|
|
}
|
|
|
|
// Download the artifact and expect an error
|
|
if err := GetArtifact(taskEnv, artifact, taskDir); err == nil {
|
|
t.Fatalf("GetArtifact should have failed")
|
|
}
|
|
}
|
|
|
|
func createContents(basedir string, fileContents map[string]string, t *testing.T) {
|
|
for relPath, content := range fileContents {
|
|
folder := basedir
|
|
if strings.Index(relPath, "/") != -1 {
|
|
// Create the folder.
|
|
folder = filepath.Join(basedir, filepath.Dir(relPath))
|
|
if err := os.Mkdir(folder, 0777); err != nil {
|
|
t.Fatalf("failed to make directory: %v", err)
|
|
}
|
|
}
|
|
|
|
// Create a file in the existing folder.
|
|
file := filepath.Join(folder, filepath.Base(relPath))
|
|
if err := ioutil.WriteFile(file, []byte(content), 0777); err != nil {
|
|
t.Fatalf("failed to write data to file %v: %v", file, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func checkContents(basedir string, fileContents map[string]string, t *testing.T) {
|
|
for relPath, content := range fileContents {
|
|
path := filepath.Join(basedir, relPath)
|
|
actual, err := ioutil.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("failed to read file %q: %v", path, err)
|
|
}
|
|
|
|
if !reflect.DeepEqual(actual, []byte(content)) {
|
|
t.Fatalf("%q: expected %q; got %q", path, content, string(actual))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetArtifact_Archive(t *testing.T) {
|
|
// Create the test server hosting the file to download
|
|
ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/"))))
|
|
defer ts.Close()
|
|
|
|
// Create a temp directory to download into and create some of the same
|
|
// files that exist in the artifact to ensure they are overridden
|
|
taskDir, err := ioutil.TempDir("", "nomad-test")
|
|
if err != nil {
|
|
t.Fatalf("failed to make temp directory: %v", err)
|
|
}
|
|
defer os.RemoveAll(taskDir)
|
|
|
|
create := map[string]string{
|
|
"exist/my.config": "to be replaced",
|
|
"untouched": "existing top-level",
|
|
}
|
|
createContents(taskDir, create, t)
|
|
|
|
file := "archive.tar.gz"
|
|
artifact := &structs.TaskArtifact{
|
|
GetterSource: fmt.Sprintf("%s/%s", ts.URL, file),
|
|
GetterOptions: map[string]string{
|
|
"checksum": "sha1:20bab73c72c56490856f913cf594bad9a4d730f6",
|
|
},
|
|
}
|
|
|
|
if err := GetArtifact(taskEnv, artifact, taskDir); err != nil {
|
|
t.Fatalf("GetArtifact failed: %v", err)
|
|
}
|
|
|
|
// Verify the unarchiving overrode files properly.
|
|
expected := map[string]string{
|
|
"untouched": "existing top-level",
|
|
"exist/my.config": "hello world\n",
|
|
"new/my.config": "hello world\n",
|
|
"test.sh": "sleep 1\n",
|
|
}
|
|
checkContents(taskDir, expected, t)
|
|
}
|
|
|
|
func TestGetArtifact_Setuid(t *testing.T) {
|
|
// Create the test server hosting the file to download
|
|
ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/"))))
|
|
defer ts.Close()
|
|
|
|
// Create a temp directory to download into and create some of the same
|
|
// files that exist in the artifact to ensure they are overridden
|
|
taskDir, err := ioutil.TempDir("", "nomad-test")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(taskDir)
|
|
|
|
file := "setuid.tgz"
|
|
artifact := &structs.TaskArtifact{
|
|
GetterSource: fmt.Sprintf("%s/%s", ts.URL, file),
|
|
GetterOptions: map[string]string{
|
|
"checksum": "sha1:e892194748ecbad5d0f60c6c6b2db2bdaa384a90",
|
|
},
|
|
}
|
|
|
|
require.NoError(t, GetArtifact(taskEnv, artifact, taskDir))
|
|
|
|
var expected map[string]int
|
|
|
|
if runtime.GOOS == "windows" {
|
|
// windows doesn't support Chmod changing file permissions.
|
|
expected = map[string]int{
|
|
"public": 0666,
|
|
"private": 0666,
|
|
"setuid": 0666,
|
|
}
|
|
} else {
|
|
// Verify the unarchiving masked files properly.
|
|
expected = map[string]int{
|
|
"public": 0666,
|
|
"private": 0600,
|
|
"setuid": 0755,
|
|
}
|
|
}
|
|
|
|
for file, perm := range expected {
|
|
path := filepath.Join(taskDir, "setuid", file)
|
|
s, err := os.Stat(path)
|
|
require.NoError(t, err)
|
|
p := os.FileMode(perm)
|
|
o := s.Mode()
|
|
require.Equalf(t, p, o, "%s expected %o found %o", file, p, o)
|
|
}
|
|
}
|
|
|
|
func TestGetGetterUrl_Queries(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
artifact *structs.TaskArtifact
|
|
output string
|
|
}{
|
|
{
|
|
name: "adds query parameters",
|
|
artifact: &structs.TaskArtifact{
|
|
GetterSource: "https://foo.com?test=1",
|
|
GetterOptions: map[string]string{
|
|
"foo": "bar",
|
|
"bam": "boom",
|
|
},
|
|
},
|
|
output: "https://foo.com?bam=boom&foo=bar&test=1",
|
|
},
|
|
{
|
|
name: "git without http",
|
|
artifact: &structs.TaskArtifact{
|
|
GetterSource: "github.com/hashicorp/nomad",
|
|
GetterOptions: map[string]string{
|
|
"ref": "abcd1234",
|
|
},
|
|
},
|
|
output: "github.com/hashicorp/nomad?ref=abcd1234",
|
|
},
|
|
{
|
|
name: "git using ssh",
|
|
artifact: &structs.TaskArtifact{
|
|
GetterSource: "git@github.com:hashicorp/nomad?sshkey=1",
|
|
GetterOptions: map[string]string{
|
|
"ref": "abcd1234",
|
|
},
|
|
},
|
|
output: "git@github.com:hashicorp/nomad?ref=abcd1234&sshkey=1",
|
|
},
|
|
{
|
|
name: "s3 scheme 1",
|
|
artifact: &structs.TaskArtifact{
|
|
GetterSource: "s3::https://s3.amazonaws.com/bucket/foo",
|
|
GetterOptions: map[string]string{
|
|
"aws_access_key_id": "abcd1234",
|
|
},
|
|
},
|
|
output: "s3::https://s3.amazonaws.com/bucket/foo?aws_access_key_id=abcd1234",
|
|
},
|
|
{
|
|
name: "s3 scheme 2",
|
|
artifact: &structs.TaskArtifact{
|
|
GetterSource: "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo",
|
|
GetterOptions: map[string]string{
|
|
"aws_access_key_id": "abcd1234",
|
|
},
|
|
},
|
|
output: "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo?aws_access_key_id=abcd1234",
|
|
},
|
|
{
|
|
name: "s3 scheme 3",
|
|
artifact: &structs.TaskArtifact{
|
|
GetterSource: "bucket.s3.amazonaws.com/foo",
|
|
GetterOptions: map[string]string{
|
|
"aws_access_key_id": "abcd1234",
|
|
},
|
|
},
|
|
output: "bucket.s3.amazonaws.com/foo?aws_access_key_id=abcd1234",
|
|
},
|
|
{
|
|
name: "s3 scheme 4",
|
|
artifact: &structs.TaskArtifact{
|
|
GetterSource: "bucket.s3-eu-west-1.amazonaws.com/foo/bar",
|
|
GetterOptions: map[string]string{
|
|
"aws_access_key_id": "abcd1234",
|
|
},
|
|
},
|
|
output: "bucket.s3-eu-west-1.amazonaws.com/foo/bar?aws_access_key_id=abcd1234",
|
|
},
|
|
{
|
|
name: "gcs",
|
|
artifact: &structs.TaskArtifact{
|
|
GetterSource: "gcs::https://www.googleapis.com/storage/v1/b/d/f",
|
|
},
|
|
output: "gcs::https://www.googleapis.com/storage/v1/b/d/f",
|
|
},
|
|
{
|
|
name: "local file",
|
|
artifact: &structs.TaskArtifact{
|
|
GetterSource: "/foo/bar",
|
|
},
|
|
output: "/foo/bar",
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
act, err := getGetterUrl(taskEnv, c.artifact)
|
|
if err != nil {
|
|
t.Fatalf("want %q; got err %v", c.output, err)
|
|
} else if act != c.output {
|
|
t.Fatalf("want %q; got %q", c.output, act)
|
|
}
|
|
})
|
|
}
|
|
}
|