luxolus/open-nomad
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

client/driver: Remove package

Browse source
This commit is contained in:
Danielle Tomlinson 2018-11-29 16:45:19 +01:00
parent d4cbd608ff
commit b9295f0d56
51 changed files with 0 additions and 14361 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2194
client/driver/docker.go
View file

File diff suppressed because it is too large Load diff

409
client/driver/docker_coordinator.go
View file

@ -1,409 +0,0 @@
package driver
import (
"context"
"fmt"
"log"
"regexp"
"sync"
"time"
docker "github.com/fsouza/go-dockerclient"
"github.com/hashicorp/nomad/nomad/structs"
)
var (
// createCoordinator allows us to only create a single coordinator
createCoordinator sync.Once
// globalCoordinator is the shared coordinator and should only be retrieved
// using the GetDockerCoordinator() method.
globalCoordinator *dockerCoordinator
// imageNotFoundMatcher is a regex expression that matches the image not
// found error Docker returns.
imageNotFoundMatcher = regexp.MustCompile(`Error: image .+ not found`)
)
// pullFuture is a sharable future for retrieving a pulled images ID and any
// error that may have occurred during the pull.
type pullFuture struct {
waitCh chan struct{}
err error
imageID string
}
// newPullFuture returns a new pull future
func newPullFuture() *pullFuture {
return &pullFuture{
waitCh: make(chan struct{}),
}
}
// wait waits till the future has a result
func (p *pullFuture) wait() *pullFuture {
<-p.waitCh
return p
}
// result returns the results of the future and should only ever be called after
// wait returns.
func (p *pullFuture) result() (imageID string, err error) {
return p.imageID, p.err
}
// set is used to set the results and unblock any waiter. This may only be
// called once.
func (p *pullFuture) set(imageID string, err error) {
p.imageID = imageID
p.err = err
close(p.waitCh)
}
// DockerImageClient provides the methods required to do CRUD operations on the
// Docker images
type DockerImageClient interface {
PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
InspectImage(id string) (*docker.Image, error)
RemoveImage(id string) error
}
// dockerCoordinatorConfig is used to configure the Docker coordinator.
type dockerCoordinatorConfig struct {
// logger is the logger the coordinator should use
logger *log.Logger
// cleanup marks whether images should be deleting when the reference count
// is zero
cleanup bool
// client is the Docker client to use for communicating with Docker
client DockerImageClient
// removeDelay is the delay between an image's reference count going to
// zero and the image actually being deleted.
removeDelay time.Duration
}
// dockerCoordinator is used to coordinate actions against images to prevent
// racy deletions. It can be thought of as a reference counter on images.
type dockerCoordinator struct {
*dockerCoordinatorConfig
// imageLock is used to lock access to all images
imageLock sync.Mutex
// pullFutures is used to allow multiple callers to pull the same image but
// only have one request be sent to Docker
pullFutures map[string]*pullFuture
// pullLoggers is used to track the LogEventFn for each alloc pulling an image.
// If multiple alloc's are attempting to pull the same image, each will need
// to register its own LogEventFn with the coordinator.
pullLoggers map[string][]LogEventFn
// pullLoggerLock is used to sync access to the pullLoggers map
pullLoggerLock sync.RWMutex
// imageRefCount is the reference count of image IDs
imageRefCount map[string]map[string]struct{}
// deleteFuture is indexed by image ID and has a cancelable delete future
deleteFuture map[string]context.CancelFunc
}
// NewDockerCoordinator returns a new Docker coordinator
func NewDockerCoordinator(config *dockerCoordinatorConfig) *dockerCoordinator {
if config.client == nil {
return nil
}
return &dockerCoordinator{
dockerCoordinatorConfig: config,
pullFutures: make(map[string]*pullFuture),
pullLoggers: make(map[string][]LogEventFn),
imageRefCount: make(map[string]map[string]struct{}),
deleteFuture: make(map[string]context.CancelFunc),
}
}
// GetDockerCoordinator returns the shared dockerCoordinator instance
func GetDockerCoordinator(config *dockerCoordinatorConfig) *dockerCoordinator {
createCoordinator.Do(func() {
globalCoordinator = NewDockerCoordinator(config)
})
return globalCoordinator
}
// PullImage is used to pull an image. It returns the pulled imaged ID or an
// error that occurred during the pull
func (d *dockerCoordinator) PullImage(image string, authOptions *docker.AuthConfiguration, callerID string, emitFn LogEventFn) (imageID string, err error) {
// Get the future
d.imageLock.Lock()
future, ok := d.pullFutures[image]
d.registerPullLogger(image, emitFn)
if !ok {
// Make the future
future = newPullFuture()
d.pullFutures[image] = future
go d.pullImageImpl(image, authOptions, future)
}
d.imageLock.Unlock()
// We unlock while we wait since this can take a while
id, err := future.wait().result()
d.imageLock.Lock()
defer d.imageLock.Unlock()
// Delete the future since we don't need it and we don't want to cache an
// image being there if it has possibly been manually deleted (outside of
// Nomad).
if _, ok := d.pullFutures[image]; ok {
delete(d.pullFutures, image)
}
// If we are cleaning up, we increment the reference count on the image
if err == nil && d.cleanup {
d.incrementImageReferenceImpl(id, image, callerID)
}
return id, err
}
// pullImageImpl is the implementation of pulling an image. The results are
// returned via the passed future
func (d *dockerCoordinator) pullImageImpl(image string, authOptions *docker.AuthConfiguration, future *pullFuture) {
defer d.clearPullLogger(image)
// Parse the repo and tag
repo, tag := parseDockerImage(image)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
pm := newImageProgressManager(image, cancel, d.handlePullInactivity,
d.handlePullProgressReport, d.handleSlowPullProgressReport)
defer pm.stop()
pullOptions := docker.PullImageOptions{
Repository: repo,
Tag: tag,
OutputStream: pm,
RawJSONStream: true,
Context: ctx,
}
// Attempt to pull the image
var auth docker.AuthConfiguration
if authOptions != nil {
auth = *authOptions
}
err := d.client.PullImage(pullOptions, auth)
if ctxErr := ctx.Err(); ctxErr == context.DeadlineExceeded {
d.logger.Printf("[ERR] driver.docker: timeout pulling container %s", dockerImageRef(repo, tag))
future.set("", recoverablePullError(ctxErr, image))
return
}
if err != nil {
d.logger.Printf("[ERR] driver.docker: failed pulling container %s: %s", dockerImageRef(repo, tag), err)
future.set("", recoverablePullError(err, image))
return
}
d.logger.Printf("[DEBUG] driver.docker: docker pull %s succeeded", dockerImageRef(repo, tag))
dockerImage, err := d.client.InspectImage(image)
if err != nil {
d.logger.Printf("[ERR] driver.docker: failed getting image id for %q: %v", image, err)
future.set("", recoverableErrTimeouts(err))
return
}
future.set(dockerImage.ID, nil)
return
}
// IncrementImageReference is used to increment an image reference count
func (d *dockerCoordinator) IncrementImageReference(imageID, imageName, callerID string) {
d.imageLock.Lock()
defer d.imageLock.Unlock()
if d.cleanup {
d.incrementImageReferenceImpl(imageID, imageName, callerID)
}
}
// incrementImageReferenceImpl assumes the lock is held
func (d *dockerCoordinator) incrementImageReferenceImpl(imageID, imageName, callerID string) {
// Cancel any pending delete
if cancel, ok := d.deleteFuture[imageID]; ok {
d.logger.Printf("[DEBUG] driver.docker: cancelling removal of image %q", imageName)
cancel()
delete(d.deleteFuture, imageID)
}
// Increment the reference
references, ok := d.imageRefCount[imageID]
if !ok {
references = make(map[string]struct{})
d.imageRefCount[imageID] = references
}
if _, ok := references[callerID]; !ok {
references[callerID] = struct{}{}
d.logger.Printf("[DEBUG] driver.docker: image %q (%v) reference count incremented: %d", imageName, imageID, len(references))
}
}
// RemoveImage removes the given image. If there are any errors removing the
// image, the remove is retried internally.
func (d *dockerCoordinator) RemoveImage(imageID, callerID string) {
d.imageLock.Lock()
defer d.imageLock.Unlock()
if !d.cleanup {
return
}
references, ok := d.imageRefCount[imageID]
if !ok {
d.logger.Printf("[WARN] driver.docker: RemoveImage on non-referenced counted image id %q", imageID)
return
}
// Decrement the reference count
delete(references, callerID)
count := len(references)
d.logger.Printf("[DEBUG] driver.docker: image id %q reference count decremented: %d", imageID, count)
// Nothing to do
if count != 0 {
return
}
// This should never be the case but we safety guard so we don't leak a
// cancel.
if cancel, ok := d.deleteFuture[imageID]; ok {
d.logger.Printf("[ERR] driver.docker: image id %q has lingering delete future", imageID)
cancel()
}
// Setup a future to delete the image
ctx, cancel := context.WithCancel(context.Background())
d.deleteFuture[imageID] = cancel
go d.removeImageImpl(imageID, ctx)
// Delete the key from the reference count
delete(d.imageRefCount, imageID)
}
// removeImageImpl is used to remove an image. It wil wait the specified remove
// delay to remove the image. If the context is cancelled before that the image
// removal will be cancelled.
func (d *dockerCoordinator) removeImageImpl(id string, ctx context.Context) {
// Wait for the delay or a cancellation event
select {
case <-ctx.Done():
// We have been cancelled
return
case <-time.After(d.removeDelay):
}
// Ensure we are suppose to delete. Do a short check while holding the lock
// so there can't be interleaving. There is still the smallest chance that
// the delete occurs after the image has been pulled but before it has been
// incremented. For handling that we just treat it as a recoverable error in
// the docker driver.
d.imageLock.Lock()
select {
case <-ctx.Done():
d.imageLock.Unlock()
return
default:
}
d.imageLock.Unlock()
for i := 0; i < 3; i++ {
err := d.client.RemoveImage(id)
if err == nil {
break
}
if err == docker.ErrNoSuchImage {
d.logger.Printf("[DEBUG] driver.docker: unable to cleanup image %q: does not exist", id)
return
}
if derr, ok := err.(*docker.Error); ok && derr.Status == 409 {
d.logger.Printf("[DEBUG] driver.docker: unable to cleanup image %q: still in use", id)
return
}
// Retry on unknown errors
d.logger.Printf("[DEBUG] driver.docker: failed to remove image %q (attempt %d): %v", id, i+1, err)
select {
case <-ctx.Done():
// We have been cancelled
return
case <-time.After(3 * time.Second):
}
}
d.logger.Printf("[DEBUG] driver.docker: cleanup removed downloaded image: %q", id)
// Cleanup the future from the map and free the context by cancelling it
d.imageLock.Lock()
if cancel, ok := d.deleteFuture[id]; ok {
delete(d.deleteFuture, id)
cancel()
}
d.imageLock.Unlock()
}
func (d *dockerCoordinator) registerPullLogger(image string, logger LogEventFn) {
d.pullLoggerLock.Lock()
defer d.pullLoggerLock.Unlock()
if _, ok := d.pullLoggers[image]; !ok {
d.pullLoggers[image] = []LogEventFn{}
}
d.pullLoggers[image] = append(d.pullLoggers[image], logger)
}
func (d *dockerCoordinator) clearPullLogger(image string) {
d.pullLoggerLock.Lock()
defer d.pullLoggerLock.Unlock()
delete(d.pullLoggers, image)
}
func (d *dockerCoordinator) emitEvent(image, message string, args ...interface{}) {
d.pullLoggerLock.RLock()
defer d.pullLoggerLock.RUnlock()
for i := range d.pullLoggers[image] {
go d.pullLoggers[image][i](message, args...)
}
}
func (d *dockerCoordinator) handlePullInactivity(image, msg string, timestamp time.Time) {
d.logger.Printf("[ERR] driver.docker: image %s pull aborted due to inactivity, last message recevieved at [%s]: %s", image, timestamp.String(), msg)
}
func (d *dockerCoordinator) handlePullProgressReport(image, msg string, _ time.Time) {
d.logger.Printf("[DEBUG] driver.docker: image %s pull progress: %s", image, msg)
}
func (d *dockerCoordinator) handleSlowPullProgressReport(image, msg string, _ time.Time) {
d.emitEvent(image, "Docker image %s pull progress: %s", image, msg)
}
// recoverablePullError wraps the error gotten when trying to pull and image if
// the error is recoverable.
func recoverablePullError(err error, image string) error {
recoverable := true
if imageNotFoundMatcher.MatchString(err.Error()) {
recoverable = false
}
return structs.NewRecoverableError(fmt.Errorf("Failed to pull `%s`: %s", image, err), recoverable)
}

239
client/driver/docker_coordinator_test.go
View file

@ -1,239 +0,0 @@
package driver
import (
"fmt"
"testing"
"time"
docker "github.com/fsouza/go-dockerclient"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/testutil"
)
type mockImageClient struct {
pulled map[string]int
idToName map[string]string
removed map[string]int
pullDelay time.Duration
}
func newMockImageClient(idToName map[string]string, pullDelay time.Duration) *mockImageClient {
return &mockImageClient{
pulled: make(map[string]int),
removed: make(map[string]int),
idToName: idToName,
pullDelay: pullDelay,
}
}
func (m *mockImageClient) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error {
time.Sleep(m.pullDelay)
m.pulled[opts.Repository]++
return nil
}
func (m *mockImageClient) InspectImage(id string) (*docker.Image, error) {
return &docker.Image{
ID: m.idToName[id],
}, nil
}
func (m *mockImageClient) RemoveImage(id string) error {
m.removed[id]++
return nil
}
func TestDockerCoordinator_ConcurrentPulls(t *testing.T) {
t.Parallel()
image := "foo"
imageID := uuid.Generate()
mapping := map[string]string{imageID: image}
// Add a delay so we can get multiple queued up
mock := newMockImageClient(mapping, 10*time.Millisecond)
config := &dockerCoordinatorConfig{
logger: testlog.Logger(t),
cleanup: true,
client: mock,
removeDelay: 100 * time.Millisecond,
}
// Create a coordinator
coordinator := NewDockerCoordinator(config)
id := ""
for i := 0; i < 10; i++ {
go func() {
id, _ = coordinator.PullImage(image, nil, uuid.Generate(), nil)
}()
}
testutil.WaitForResult(func() (bool, error) {
p := mock.pulled[image]
if p >= 10 {
return false, fmt.Errorf("Wrong number of pulls: %d", p)
}
// Check the reference count
if references := coordinator.imageRefCount[id]; len(references) != 10 {
return false, fmt.Errorf("Got reference count %d; want %d", len(references), 10)
}
// Ensure there is no pull future
if len(coordinator.pullFutures) != 0 {
return false, fmt.Errorf("Pull future exists after pull finished")
}
return true, nil
}, func(err error) {
t.Fatalf("err: %v", err)
})
}
func TestDockerCoordinator_Pull_Remove(t *testing.T) {
t.Parallel()
image := "foo"
imageID := uuid.Generate()
mapping := map[string]string{imageID: image}
// Add a delay so we can get multiple queued up
mock := newMockImageClient(mapping, 10*time.Millisecond)
config := &dockerCoordinatorConfig{
logger: testlog.Logger(t),
cleanup: true,
client: mock,
removeDelay: 1 * time.Millisecond,
}
// Create a coordinator
coordinator := NewDockerCoordinator(config)
id := ""
callerIDs := make([]string, 10, 10)
for i := 0; i < 10; i++ {
callerIDs[i] = uuid.Generate()
id, _ = coordinator.PullImage(image, nil, callerIDs[i], nil)
}
// Check the reference count
if references := coordinator.imageRefCount[id]; len(references) != 10 {
t.Fatalf("Got reference count %d; want %d", len(references), 10)
}
// Remove some
for i := 0; i < 8; i++ {
coordinator.RemoveImage(id, callerIDs[i])
}
// Check the reference count
if references := coordinator.imageRefCount[id]; len(references) != 2 {
t.Fatalf("Got reference count %d; want %d", len(references), 2)
}
// Remove all
for i := 8; i < 10; i++ {
coordinator.RemoveImage(id, callerIDs[i])
}
// Check the reference count
if references := coordinator.imageRefCount[id]; len(references) != 0 {
t.Fatalf("Got reference count %d; want %d", len(references), 0)
}
// Check that only one delete happened
testutil.WaitForResult(func() (bool, error) {
removes := mock.removed[id]
return removes == 1, fmt.Errorf("Wrong number of removes: %d", removes)
}, func(err error) {
t.Fatalf("err: %v", err)
})
// Make sure there is no future still
if _, ok := coordinator.deleteFuture[id]; ok {
t.Fatal("Got delete future")
}
}
func TestDockerCoordinator_Remove_Cancel(t *testing.T) {
t.Parallel()
image := "foo"
imageID := uuid.Generate()
mapping := map[string]string{imageID: image}
mock := newMockImageClient(mapping, 1*time.Millisecond)
config := &dockerCoordinatorConfig{
logger: testlog.Logger(t),
cleanup: true,
client: mock,
removeDelay: 100 * time.Millisecond,
}
// Create a coordinator
coordinator := NewDockerCoordinator(config)
callerID := uuid.Generate()
// Pull image
id, _ := coordinator.PullImage(image, nil, callerID, nil)
// Check the reference count
if references := coordinator.imageRefCount[id]; len(references) != 1 {
t.Fatalf("Got reference count %d; want %d", len(references), 1)
}
// Remove image
coordinator.RemoveImage(id, callerID)
// Check the reference count
if references := coordinator.imageRefCount[id]; len(references) != 0 {
t.Fatalf("Got reference count %d; want %d", len(references), 0)
}
// Pull image again within delay
id, _ = coordinator.PullImage(image, nil, callerID, nil)
// Check the reference count
if references := coordinator.imageRefCount[id]; len(references) != 1 {
t.Fatalf("Got reference count %d; want %d", len(references), 1)
}
// Check that only no delete happened
if removes := mock.removed[id]; removes != 0 {
t.Fatalf("Image deleted when it shouldn't have")
}
}
func TestDockerCoordinator_No_Cleanup(t *testing.T) {
t.Parallel()
image := "foo"
imageID := uuid.Generate()
mapping := map[string]string{imageID: image}
mock := newMockImageClient(mapping, 1*time.Millisecond)
config := &dockerCoordinatorConfig{
logger: testlog.Logger(t),
cleanup: false,
client: mock,
removeDelay: 1 * time.Millisecond,
}
// Create a coordinator
coordinator := NewDockerCoordinator(config)
callerID := uuid.Generate()
// Pull image
id, _ := coordinator.PullImage(image, nil, callerID, nil)
// Check the reference count
if references := coordinator.imageRefCount[id]; len(references) != 0 {
t.Fatalf("Got reference count %d; want %d", len(references), 0)
}
// Remove image
coordinator.RemoveImage(id, callerID)
// Check that only no delete happened
if removes := mock.removed[id]; removes != 0 {
t.Fatalf("Image deleted when it shouldn't have")
}
}

36
client/driver/docker_default.go
View file

@ -1,36 +0,0 @@
//+build !windows
package driver
import (
docker "github.com/fsouza/go-dockerclient"
"github.com/moby/moby/daemon/caps"
)
const (
// Setting default network mode for non-windows OS as bridge
defaultNetworkMode = "bridge"
)
func getPortBinding(ip string, port string) []docker.PortBinding {
return []docker.PortBinding{{HostIP: ip, HostPort: port}}
}
func tweakCapabilities(basics, adds, drops []string) ([]string, error) {
// Moby mixes 2 different capabilities formats: prefixed with "CAP_"
// and not. We do the conversion here to have a consistent,
// non-prefixed format on the Nomad side.
for i, cap := range basics {
basics[i] = "CAP_" + cap
}
effectiveCaps, err := caps.TweakCapabilities(basics, adds, drops)
if err != nil {
return effectiveCaps, err
}
for i, cap := range effectiveCaps {
effectiveCaps[i] = cap[len("CAP_"):]
}
return effectiveCaps, nil
}

127
client/driver/docker_linux_test.go
View file

@ -1,127 +0,0 @@
package driver
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"time"
docker "github.com/fsouza/go-dockerclient"
"github.com/hashicorp/nomad/client/testutil"
tu "github.com/hashicorp/nomad/testutil"
"github.com/stretchr/testify/require"
)
func TestDockerDriver_authFromHelper(t *testing.T) {
dir, err := ioutil.TempDir("", "test-docker-driver_authfromhelper")
require.NoError(t, err)
defer os.RemoveAll(dir)
helperPayload := "{\"Username\":\"hashi\",\"Secret\":\"nomad\"}"
helperContent := []byte(fmt.Sprintf("#!/bin/sh\ncat > %s/helper-$1.out;echo '%s'", dir, helperPayload))
helperFile := filepath.Join(dir, "docker-credential-testnomad")
err = ioutil.WriteFile(helperFile, helperContent, 0777)
require.NoError(t, err)
path := os.Getenv("PATH")
os.Setenv("PATH", fmt.Sprintf("%s:%s", path, dir))
defer os.Setenv("PATH", path)
helper := authFromHelper("testnomad")
creds, err := helper("registry.local:5000/repo/image")
require.NoError(t, err)
require.NotNil(t, creds)
require.Equal(t, "hashi", creds.Username)
require.Equal(t, "nomad", creds.Password)
if _, err := os.Stat(filepath.Join(dir, "helper-get.out")); os.IsNotExist(err) {
t.Fatalf("Expected helper-get.out to exist")
}
content, err := ioutil.ReadFile(filepath.Join(dir, "helper-get.out"))
require.NoError(t, err)
require.Equal(t, []byte("https://registry.local:5000"), content)
}
func TestDockerDriver_PidsLimit(t *testing.T) {
if !tu.IsTravis() {
t.Parallel()
}
if !testutil.DockerIsConnected(t) {
t.Skip("Docker not connected")
}
task, _, _ := dockerTask(t)
task.Config["pids_limit"] = "1"
task.Config["command"] = "/bin/sh"
// this starts three processes in container: /bin/sh and two sleep
// while a single sleep suffices, our observation is that it's image dependent
// (i.e. using a single sleep here in alpine image doesn't trigger PID limit failure)
task.Config["args"] = []string{"-c", "sleep 2 & sleep 2"}
ctx := testDockerDriverContexts(t, task)
defer ctx.Destroy()
d := NewDockerDriver(ctx.DriverCtx)
// TODO: current log capture of docker driver is broken
// so we must fetch logs from docker daemon directly
// which works in Linux as well as Mac
d.(*DockerDriver).DriverContext.config.Options[dockerCleanupContainerConfigOption] = "false"
// Copy the image into the task's directory
copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar")
_, err := d.Prestart(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
h := resp.Handle.(*DockerHandle)
defer h.client.RemoveContainer(docker.RemoveContainerOptions{
ID: h.containerID,
RemoveVolumes: true,
Force: true,
})
defer resp.Handle.Kill()
select {
case res := <-resp.Handle.WaitCh():
if res.Successful() {
t.Fatalf("expected error, but container exited successful")
}
// /bin/sh exits with 2
if res.ExitCode != 2 {
t.Fatalf("expected exit code of 2 but found %v", res.ExitCode)
}
case <-time.After(time.Duration(tu.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
// XXX Logging doesn't work on OSX so just test on Linux
// Check that data was written to the directory.
var act bytes.Buffer
err = h.client.Logs(docker.LogsOptions{
Container: h.containerID,
Stderr: true,
ErrorStream: &act,
})
if err != nil {
t.Fatalf("error in fetching logs: %v", err)
}
exp := "can't fork"
if !strings.Contains(act.String(), exp) {
t.Fatalf("Expected failed fork: %q", act)
}
}

289
client/driver/docker_progress.go
View file

@ -1,289 +0,0 @@
package driver
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"strings"
"sync"
"time"
"github.com/docker/docker/pkg/jsonmessage"
units "github.com/docker/go-units"
)
const (
// dockerPullActivityDeadline is the default value set in the imageProgressManager
// when newImageProgressManager is called
dockerPullActivityDeadline = 2 * time.Minute
// dockerImageProgressReportInterval is the default value set in the
// imageProgressManager when newImageProgressManager is called
dockerImageProgressReportInterval = 10 * time.Second
// dockerImageSlowProgressReportInterval is the default value set in the
// imageProgressManager when newImageProgressManager is called
dockerImageSlowProgressReportInterval = 2 * time.Minute
)
// layerProgress tracks the state and downloaded bytes of a single layer within
// a docker image
type layerProgress struct {
id string
status layerProgressStatus
currentBytes int64
totalBytes int64
}
type layerProgressStatus int
const (
layerProgressStatusUnknown layerProgressStatus = iota
layerProgressStatusStarting
layerProgressStatusWaiting
layerProgressStatusDownloading
layerProgressStatusVerifying
layerProgressStatusDownloaded
layerProgressStatusExtracting
layerProgressStatusComplete
layerProgressStatusExists
)
func lpsFromString(status string) layerProgressStatus {
switch status {
case "Pulling fs layer":
return layerProgressStatusStarting
case "Waiting":
return layerProgressStatusWaiting
case "Downloading":
return layerProgressStatusDownloading
case "Verifying Checksum":
return layerProgressStatusVerifying
case "Download complete":
return layerProgressStatusDownloaded
case "Extracting":
return layerProgressStatusExtracting
case "Pull complete":
return layerProgressStatusComplete
case "Already exists":
return layerProgressStatusExists
default:
return layerProgressStatusUnknown
}
}
// imageProgress tracks the status of each child layer as its pulled from a
// docker image repo
type imageProgress struct {
sync.RWMutex
lastMessage *jsonmessage.JSONMessage
timestamp time.Time
layers map[string]*layerProgress
pullStart time.Time
}
// get returns a status message and the timestamp of the last status update
func (p *imageProgress) get() (string, time.Time) {
p.RLock()
defer p.RUnlock()
if p.lastMessage == nil {
return "No progress", p.timestamp
}
var pulled, pulling, waiting int
for _, l := range p.layers {
switch {
case l.status == layerProgressStatusStarting ||
l.status == layerProgressStatusWaiting:
waiting++
case l.status == layerProgressStatusDownloading ||
l.status == layerProgressStatusVerifying:
pulling++
case l.status >= layerProgressStatusDownloaded:
pulled++
}
}
elapsed := time.Now().Sub(p.pullStart)
cur := p.currentBytes()
total := p.totalBytes()
var est int64
if cur != 0 {
est = (elapsed.Nanoseconds() / cur * total) - elapsed.Nanoseconds()
}
var msg strings.Builder
fmt.Fprintf(&msg, "Pulled %d/%d (%s/%s) layers: %d waiting/%d pulling",
pulled, len(p.layers), units.BytesSize(float64(cur)), units.BytesSize(float64(total)),
waiting, pulling)
if est > 0 {
fmt.Fprintf(&msg, " - est %.1fs remaining", time.Duration(est).Seconds())
}
return msg.String(), p.timestamp
}
// set takes a status message received from the docker engine api during an image
// pull and updates the status of the corresponding layer
func (p *imageProgress) set(msg *jsonmessage.JSONMessage) {
p.Lock()
defer p.Unlock()
p.lastMessage = msg
p.timestamp = time.Now()
lps := lpsFromString(msg.Status)
if lps == layerProgressStatusUnknown {
return
}
layer, ok := p.layers[msg.ID]
if !ok {
layer = &layerProgress{id: msg.ID}
p.layers[msg.ID] = layer
}
layer.status = lps
if msg.Progress != nil && lps == layerProgressStatusDownloading {
layer.currentBytes = msg.Progress.Current
layer.totalBytes = msg.Progress.Total
} else if lps == layerProgressStatusDownloaded {
layer.currentBytes = layer.totalBytes
}
}
// currentBytes iterates through all image layers and sums the total of
// current bytes. The caller is responsible for acquiring a read lock on the
// imageProgress struct
func (p *imageProgress) currentBytes() int64 {
var b int64
for _, l := range p.layers {
b += l.currentBytes
}
return b
}
// totalBytes iterates through all image layers and sums the total of
// total bytes. The caller is responsible for acquiring a read lock on the
// imageProgress struct
func (p *imageProgress) totalBytes() int64 {
var b int64
for _, l := range p.layers {
b += l.totalBytes
}
return b
}
// progressReporterFunc defines the method for handling inactivity and report
// events from the imageProgressManager. The image name, current status message
// and timestamp of last received status update are passed in.
type progressReporterFunc func(image string, msg string, timestamp time.Time)
// imageProgressManager tracks the progress of pulling a docker image from an
// image repository.
// It also implemented the io.Writer interface so as to be passed to the docker
// client pull image method in order to receive status updates from the docker
// engine api.
type imageProgressManager struct {
imageProgress *imageProgress
image string
activityDeadline time.Duration
inactivityFunc progressReporterFunc
reportInterval time.Duration
reporter progressReporterFunc
slowReportInterval time.Duration
slowReporter progressReporterFunc
lastSlowReport time.Time
cancel context.CancelFunc
stopCh chan struct{}
buf bytes.Buffer
}
func newImageProgressManager(
image string, cancel context.CancelFunc,
inactivityFunc, reporter, slowReporter progressReporterFunc) *imageProgressManager {
pm := &imageProgressManager{
image: image,
activityDeadline: dockerPullActivityDeadline,
inactivityFunc: inactivityFunc,
reportInterval: dockerImageProgressReportInterval,
reporter: reporter,
slowReportInterval: dockerImageSlowProgressReportInterval,
slowReporter: slowReporter,
imageProgress: &imageProgress{
timestamp: time.Now(),
layers: make(map[string]*layerProgress),
},
cancel: cancel,
stopCh: make(chan struct{}),
}
pm.start()
return pm
}
// start intiates the ticker to trigger the inactivity and reporter handlers
func (pm *imageProgressManager) start() {
now := time.Now()
pm.imageProgress.pullStart = now
pm.lastSlowReport = now
go func() {
ticker := time.NewTicker(dockerImageProgressReportInterval)
for {
select {
case <-ticker.C:
msg, lastStatusTime := pm.imageProgress.get()
t := time.Now()
if t.Sub(lastStatusTime) > pm.activityDeadline {
pm.inactivityFunc(pm.image, msg, lastStatusTime)
pm.cancel()
return
}
if t.Sub(pm.lastSlowReport) > pm.slowReportInterval {
pm.slowReporter(pm.image, msg, lastStatusTime)
pm.lastSlowReport = t
}
pm.reporter(pm.image, msg, lastStatusTime)
case <-pm.stopCh:
return
}
}
}()
}
func (pm *imageProgressManager) stop() {
close(pm.stopCh)
}
func (pm *imageProgressManager) Write(p []byte) (n int, err error) {
n, err = pm.buf.Write(p)
var msg jsonmessage.JSONMessage
for {
line, err := pm.buf.ReadBytes('\n')
if err == io.EOF {
// Partial write of line; push back onto buffer and break until full line
pm.buf.Write(line)
break
}
if err != nil {
return n, err
}
err = json.Unmarshal(line, &msg)
if err != nil {
return n, err
}
if msg.Error != nil {
// error received from the docker engine api
return n, msg.Error
}
pm.imageProgress.set(&msg)
}
return
}

52
client/driver/docker_progress_test.go
View file

@ -1,52 +0,0 @@
package driver
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func Test_DockerImageProgressManager(t *testing.T) {
pm := &imageProgressManager{
imageProgress: &imageProgress{
timestamp: time.Now(),
layers: make(map[string]*layerProgress),
},
}
_, err := pm.Write([]byte(`{"status":"Pulling from library/golang","id":"1.9.5"}
{"status":"Pulling fs layer","progressDetail":{},"id":"c73ab1c6897b"}
{"status":"Pulling fs layer","progressDetail":{},"id":"1ab373b3deae"}
`))
require.NoError(t, err)
require.Equal(t, 2, len(pm.imageProgress.layers), "number of layers should be 2")
cur := pm.imageProgress.currentBytes()
require.Zero(t, cur)
tot := pm.imageProgress.totalBytes()
require.Zero(t, tot)
_, err = pm.Write([]byte(`{"status":"Pulling fs layer","progress`))
require.NoError(t, err)
require.Equal(t, 2, len(pm.imageProgress.layers), "number of layers should be 2")
_, err = pm.Write([]byte(`Detail":{},"id":"b542772b4177"}` + "\n"))
require.NoError(t, err)
require.Equal(t, 3, len(pm.imageProgress.layers), "number of layers should be 3")
_, err = pm.Write([]byte(`{"status":"Downloading","progressDetail":{"current":45800,"total":4335495},"progress":"[\u003e ] 45.8kB/4.335MB","id":"b542772b4177"}
{"status":"Downloading","progressDetail":{"current":113576,"total":11108010},"progress":"[\u003e ] 113.6kB/11.11MB","id":"1ab373b3deae"}
{"status":"Downloading","progressDetail":{"current":694257,"total":4335495},"progress":"[========\u003e ] 694.3kB/4.335MB","id":"b542772b4177"}` + "\n"))
require.NoError(t, err)
require.Equal(t, 3, len(pm.imageProgress.layers), "number of layers should be 3")
require.Equal(t, int64(807833), pm.imageProgress.currentBytes())
require.Equal(t, int64(15443505), pm.imageProgress.totalBytes())
_, err = pm.Write([]byte(`{"status":"Download complete","progressDetail":{},"id":"b542772b4177"}` + "\n"))
require.NoError(t, err)
require.Equal(t, 3, len(pm.imageProgress.layers), "number of layers should be 3")
require.Equal(t, int64(4449071), pm.imageProgress.currentBytes())
require.Equal(t, int64(15443505), pm.imageProgress.totalBytes())
}

2624
client/driver/docker_test.go
View file

File diff suppressed because it is too large Load diff

105
client/driver/docker_unix_test.go
View file

@ -1,105 +0,0 @@
// +build !windows
package driver
import (
"io/ioutil"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
"github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/nomad/structs"
tu "github.com/hashicorp/nomad/testutil"
)
func TestDockerDriver_Signal(t *testing.T) {
if !tu.IsTravis() {
t.Parallel()
}
if !testutil.DockerIsConnected(t) {
t.Skip("Docker not connected")
}
task := &structs.Task{
Name: "redis-demo",
Driver: "docker",
Config: map[string]interface{}{
"image": "busybox",
"load": "busybox.tar",
"command": "/bin/sh",
"args": []string{"local/test.sh"},
},
Resources: &structs.Resources{
MemoryMB: 256,
CPU: 512,
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
}
ctx := testDockerDriverContexts(t, task)
defer ctx.Destroy()
d := NewDockerDriver(ctx.DriverCtx)
// Copy the image into the task's directory
copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar")
testFile := filepath.Join(ctx.ExecCtx.TaskDir.LocalDir, "test.sh")
testData := []byte(`
at_term() {
echo 'Terminated.' > $NOMAD_TASK_DIR/output
exit 3
}
trap at_term INT
while true; do
echo 'sleeping'
sleep 0.2
done
`)
if err := ioutil.WriteFile(testFile, testData, 0777); err != nil {
t.Fatalf("Failed to write data: %v", err)
}
_, err := d.Prestart(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer resp.Handle.Kill()
waitForExist(t, resp.Handle.(*DockerHandle).client, resp.Handle.(*DockerHandle))
time.Sleep(1 * time.Second)
if err := resp.Handle.Signal(syscall.SIGINT); err != nil {
t.Fatalf("Signal returned an error: %v", err)
}
select {
case res := <-resp.Handle.WaitCh():
if res.Successful() {
t.Fatalf("should err: %v", res)
}
case <-time.After(time.Duration(tu.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
// Check the log file to see it exited because of the signal
outputFile := filepath.Join(ctx.ExecCtx.TaskDir.LocalDir, "output")
act, err := ioutil.ReadFile(outputFile)
if err != nil {
t.Fatalf("Couldn't read expected output: %v", err)
}
exp := "Terminated."
if strings.TrimSpace(string(act)) != exp {
t.Fatalf("Command outputted %v; want %v", act, exp)
}
}

17
client/driver/docker_windows.go
View file

@ -1,17 +0,0 @@
package driver
import docker "github.com/fsouza/go-dockerclient"
const (
// Default network mode for windows containers is nat
defaultNetworkMode = "nat"
)
//Currently Windows containers don't support host ip in port binding.
func getPortBinding(ip string, port string) []docker.PortBinding {
return []docker.PortBinding{{HostIP: "", HostPort: port}}
}
func tweakCapabilities(basics, adds, drops []string) ([]string, error) {
return nil, nil
}

361
client/driver/driver.go
View file

@ -1,361 +0,0 @@
package driver
import (
"context"
"crypto/md5"
"fmt"
"io"
"log"
"os"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/fingerprint"
"github.com/hashicorp/nomad/drivers/shared/env"
"github.com/hashicorp/nomad/nomad/structs"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
cstructs "github.com/hashicorp/nomad/client/structs"
)
var (
// BuiltinDrivers contains the built in registered drivers
// which are available for allocation handling
BuiltinDrivers = map[string]Factory{
"docker": NewDockerDriver,
"exec": NewExecDriver,
"raw_exec": NewRawExecDriver,
"java": NewJavaDriver,
"qemu": NewQemuDriver,
"rkt": NewRktDriver,
}
)
// NewDriver is used to instantiate and return a new driver
// given the name and a logger
func NewDriver(name string, ctx *DriverContext) (Driver, error) {
// Lookup the factory function
factory, ok := BuiltinDrivers[name]
if !ok {
return nil, fmt.Errorf("unknown driver '%s'", name)
}
// Instantiate the driver
d := factory(ctx)
return d, nil
}
// Factory is used to instantiate a new Driver
type Factory func(*DriverContext) Driver
// PrestartResponse is driver state returned by Driver.Prestart.
type PrestartResponse struct {
// CreatedResources by the driver.
CreatedResources *CreatedResources
// Network contains driver-specific network parameters such as the port
// map between the host and a container.
//
// Since the network configuration may not be fully populated by
// Prestart, it will only be used for creating an environment for
// Start. It will be overridden by the DriverNetwork returned by Start.
Network *cstructs.DriverNetwork
}
// NewPrestartResponse creates a new PrestartResponse with CreatedResources
// initialized.
func NewPrestartResponse() *PrestartResponse {
return &PrestartResponse{
CreatedResources: NewCreatedResources(),
}
}
// CreatedResources is a map of resources (eg downloaded images) created by a driver
// that must be cleaned up.
type CreatedResources struct {
Resources map[string][]string
}
func NewCreatedResources() *CreatedResources {
return &CreatedResources{Resources: make(map[string][]string)}
}
// Add a new resource if it doesn't already exist.
func (r *CreatedResources) Add(k, v string) {
if r.Resources == nil {
r.Resources = map[string][]string{k: {v}}
return
}
existing, ok := r.Resources[k]
if !ok {
// Key doesn't exist, create it
r.Resources[k] = []string{v}
return
}
for _, item := range existing {
if item == v {
// resource exists, return
return
}
}
// Resource type exists but value did not, append it
r.Resources[k] = append(existing, v)
return
}
// Remove a resource. Return true if removed, otherwise false.
//
// Removes the entire key if the needle is the last value in the list.
func (r *CreatedResources) Remove(k, needle string) bool {
haystack := r.Resources[k]
for i, item := range haystack {
if item == needle {
r.Resources[k] = append(haystack[:i], haystack[i+1:]...)
if len(r.Resources[k]) == 0 {
delete(r.Resources, k)
}
return true
}
}
return false
}
// Copy returns a new deep copy of CreatedResources.
func (r *CreatedResources) Copy() *CreatedResources {
if r == nil {
return nil
}
newr := CreatedResources{
Resources: make(map[string][]string, len(r.Resources)),
}
for k, v := range r.Resources {
newv := make([]string, len(v))
copy(newv, v)
newr.Resources[k] = newv
}
return &newr
}
// Merge another CreatedResources into this one. If the other CreatedResources
// is nil this method is a noop.
func (r *CreatedResources) Merge(o *CreatedResources) {
if o == nil {
return
}
for k, v := range o.Resources {
// New key
if len(r.Resources[k]) == 0 {
r.Resources[k] = v
continue
}
// Existing key
OUTER:
for _, item := range v {
for _, existing := range r.Resources[k] {
if item == existing {
// Found it, move on
continue OUTER
}
}
// New item, append it
r.Resources[k] = append(r.Resources[k], item)
}
}
}
func (r *CreatedResources) Hash() []byte {
h := md5.New()
for k, values := range r.Resources {
io.WriteString(h, k)
io.WriteString(h, "values")
for i, v := range values {
io.WriteString(h, fmt.Sprintf("%d-%v", i, v))
}
}
return h.Sum(nil)
}
// StartResponse is returned by Driver.Start.
type StartResponse struct {
// Handle to the driver's task executor for controlling the lifecycle
// of the task.
Handle DriverHandle
// Network contains driver-specific network parameters such as the port
// map between the host and a container.
//
// Network may be nil as not all drivers or configurations create
// networks.
Network *cstructs.DriverNetwork
}
// Driver is used for execution of tasks. This allows Nomad
// to support many pluggable implementations of task drivers.
// Examples could include LXC, Docker, Qemu, etc.
type Driver interface {
// Drivers must support the fingerprint interface for detection
fingerprint.Fingerprint
// Prestart prepares the task environment and performs expensive
// initialization steps like downloading images.
//
// CreatedResources may be non-nil even when an error occurs.
Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error)
// Start is used to begin task execution. If error is nil,
// StartResponse.Handle will be the handle to the task's executor.
// StartResponse.Network may be nil if the task doesn't configure a
// network.
Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error)
// Open is used to re-open a handle to a task
Open(ctx *ExecContext, handleID string) (DriverHandle, error)
// Cleanup is called to remove resources which were created for a task
// and no longer needed. Cleanup is not called if CreatedResources is
// nil.
//
// If Cleanup returns a recoverable error it may be retried. On retry
// it will be passed the same CreatedResources, so all successfully
// cleaned up resources should be removed or handled idempotently.
Cleanup(*ExecContext, *CreatedResources) error
// Drivers must validate their configuration
Validate(map[string]interface{}) error
// Abilities returns the abilities of the driver
Abilities() DriverAbilities
// FSIsolation returns the method of filesystem isolation used
FSIsolation() cstructs.FSIsolation
}
// DriverAbilities marks the abilities the driver has.
type DriverAbilities struct {
// SendSignals marks the driver as being able to send signals
SendSignals bool
// Exec marks the driver as being able to execute arbitrary commands
// such as health checks. Used by the ScriptExecutor interface.
Exec bool
}
// LogEventFn is a callback which allows Drivers to emit task events.
type LogEventFn func(message string, args ...interface{})
// DriverContext is a means to inject dependencies such as loggers, configs, and
// node attributes into a Driver without having to change the Driver interface
// each time we do it. Used in conjunction with Factory, above.
type DriverContext struct {
jobName string
taskGroupName string
taskName string
allocID string
config *config.Config
logger *log.Logger
node *structs.Node
emitEvent LogEventFn
}
// NewEmptyDriverContext returns a DriverContext with all fields set to their
// zero value.
func NewEmptyDriverContext() *DriverContext {
return &DriverContext{}
}
// NewDriverContext initializes a new DriverContext with the specified fields.
// This enables other packages to create DriverContexts but keeps the fields
// private to the driver. If we want to change this later we can gorename all of
// the fields in DriverContext.
func NewDriverContext(jobName, taskGroupName, taskName, allocID string,
config *config.Config, node *structs.Node,
logger *log.Logger, eventEmitter LogEventFn) *DriverContext {
return &DriverContext{
jobName: jobName,
taskGroupName: taskGroupName,
taskName: taskName,
allocID: allocID,
config: config,
node: node,
logger: logger,
emitEvent: eventEmitter,
}
}
// DriverHandle is an opaque handle into a driver used for task
// manipulation
type DriverHandle interface {
// Returns an opaque handle that can be used to re-open the handle
ID() string
// WaitCh is used to return a channel used wait for task completion
WaitCh() chan *dstructs.WaitResult
// Update is used to update the task if possible and update task related
// configurations.
Update(task *structs.Task) error
// Kill is used to stop the task
Kill() error
// Stats returns aggregated stats of the driver
Stats() (*cstructs.TaskResourceUsage, error)
// Signal is used to send a signal to the task
Signal(s os.Signal) error
// ScriptExecutor is an interface used to execute commands such as
// health check scripts in the a DriverHandle's context.
ScriptExecutor
// Network returns the driver's network or nil if the driver did not
// create a network.
Network() *cstructs.DriverNetwork
}
// ScriptExecutor is an interface that supports Exec()ing commands in the
// driver's context. Split out of DriverHandle to ease testing.
type ScriptExecutor interface {
Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error)
}
// ExecContext is a task's execution context
type ExecContext struct {
// TaskDir contains information about the task directory structure.
TaskDir *allocdir.TaskDir
// TaskEnv contains the task's environment variables.
TaskEnv *env.TaskEnv
// StdoutFifo is the path to the named pipe to write stdout to
StdoutFifo string
// StderrFifo is the path to the named pipe to write stderr to
StderrFifo string
}
// NewExecContext is used to create a new execution context
func NewExecContext(td *allocdir.TaskDir, te *env.TaskEnv) *ExecContext {
return &ExecContext{
TaskDir: td,
TaskEnv: te,
}
}
func mapMergeStrStr(maps ...map[string]string) map[string]string {
out := map[string]string{}
for _, in := range maps {
for key, val := range in {
out[key] = val
}
}
return out
}

478
client/driver/driver_test.go
View file

@ -1,478 +0,0 @@
package driver
import (
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"reflect"
"runtime"
"testing"
"time"
plugin "github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/logmon"
"github.com/hashicorp/nomad/drivers/shared/env"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/helper/testtask"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
)
var basicResources = &structs.Resources{
CPU: 250,
MemoryMB: 256,
DiskMB: 20,
}
func init() {
rand.Seed(49875)
}
func TestMain(m *testing.M) {
if !testtask.Run() {
os.Exit(m.Run())
}
}
// copyFile moves an existing file to the destination
func copyFile(src, dst string, t *testing.T) {
in, err := os.Open(src)
if err != nil {
t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
}
defer func() {
if err := out.Close(); err != nil {
t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
}
}()
if _, err = io.Copy(out, in); err != nil {
t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
}
if err := out.Sync(); err != nil {
t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
}
}
func testConfig(t *testing.T) *config.Config {
conf := config.DefaultConfig()
// Evaluate the symlinks so that the temp directory resolves correctly on
// Mac OS.
d1, err := ioutil.TempDir("", "TestStateDir")
if err != nil {
t.Fatal(err)
}
d2, err := ioutil.TempDir("", "TestAllocDir")
if err != nil {
t.Fatal(err)
}
p1, err := filepath.EvalSymlinks(d1)
if err != nil {
t.Fatal(err)
}
p2, err := filepath.EvalSymlinks(d2)
if err != nil {
t.Fatal(err)
}
// Give the directories access to everyone
if err := os.Chmod(p1, 0777); err != nil {
t.Fatal(err)
}
if err := os.Chmod(p2, 0777); err != nil {
t.Fatal(err)
}
conf.StateDir = p1
conf.AllocDir = p2
conf.MaxKillTimeout = 10 * time.Second
conf.Region = "global"
conf.Node = mock.Node()
conf.LogLevel = "DEBUG"
conf.LogOutput = testlog.NewWriter(t)
return conf
}
type testContext struct {
AllocDir *allocdir.AllocDir
DriverCtx *DriverContext
ExecCtx *ExecContext
EnvBuilder *env.Builder
logmon logmon.LogMon
logmonPlugin *plugin.Client
}
func (ctx *testContext) Destroy() {
ctx.AllocDir.Destroy()
ctx.logmon.Stop()
ctx.logmonPlugin.Kill()
}
// testDriverContext sets up an alloc dir, task dir, DriverContext, and ExecContext.
//
// It is up to the caller to call Destroy to cleanup.
func testDriverContexts(t *testing.T, task *structs.Task) *testContext {
cfg := testConfig(t)
cfg.Node = mock.Node()
alloc := mock.Alloc()
alloc.NodeID = cfg.Node.ID
allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(cfg.AllocDir, alloc.ID))
if err := allocDir.Build(); err != nil {
t.Fatalf("AllocDir.Build() failed: %v", err)
}
// Build a temp driver so we can call FSIsolation and build the task dir
tmpdrv, err := NewDriver(task.Driver, NewEmptyDriverContext())
if err != nil {
allocDir.Destroy()
t.Fatalf("NewDriver(%q, nil) failed: %v", task.Driver, err)
return nil
}
// Build the task dir
td := allocDir.NewTaskDir(task.Name)
if err := td.Build(false, config.DefaultChrootEnv, tmpdrv.FSIsolation()); err != nil {
allocDir.Destroy()
t.Fatalf("TaskDir.Build(%#v, %q) failed: %v", config.DefaultChrootEnv, tmpdrv.FSIsolation(), err)
return nil
}
eb := env.NewBuilder(cfg.Node, alloc, task, cfg.Region)
setEnvvars(eb, tmpdrv.FSIsolation(), td, cfg)
execCtx := NewExecContext(td, eb.Build())
logger := testlog.Logger(t)
hcLogger := testlog.HCLogger(t)
emitter := func(m string, args ...interface{}) {
hcLogger.Info(fmt.Sprintf("[EVENT] "+m, args...))
}
driverCtx := NewDriverContext(alloc.Job.Name, alloc.TaskGroup, task.Name, alloc.ID, cfg, cfg.Node, logger, emitter)
l, c, err := logmon.LaunchLogMon(hcLogger)
if err != nil {
allocDir.Destroy()
t.Fatalf("LaunchLogMon() failed: %v", err)
}
var stdoutFifo, stderrFifo string
if runtime.GOOS == "windows" {
id := uuid.Generate()[:8]
stdoutFifo = fmt.Sprintf("//./pipe/%s.stdout.%s", id, task.Name)
stderrFifo = fmt.Sprintf("//./pipe/%s.stderr.%s", id, task.Name)
} else {
stdoutFifo = filepath.Join(td.LogDir, fmt.Sprintf("%s.stdout", task.Name))
stderrFifo = filepath.Join(td.LogDir, fmt.Sprintf("%s.stderr", task.Name))
}
err = l.Start(&logmon.LogConfig{
LogDir: td.LogDir,
StdoutLogFile: fmt.Sprintf("%s.stdout", task.Name),
StderrLogFile: fmt.Sprintf("%s.stderr", task.Name),
StdoutFifo: stdoutFifo,
StderrFifo: stderrFifo,
MaxFiles: 10,
MaxFileSizeMB: 10,
})
if err != nil {
allocDir.Destroy()
t.Fatalf("LogMon.Start() failed: %v", err)
}
execCtx.StdoutFifo = stdoutFifo
execCtx.StderrFifo = stderrFifo
return &testContext{allocDir, driverCtx, execCtx, eb, l, c}
}
// setupTaskEnv creates a test env for GetTaskEnv testing. Returns task dir,
// expected env, and actual env.
func setupTaskEnv(t *testing.T, driver string) (*allocdir.TaskDir, map[string]string, map[string]string) {
task := &structs.Task{
Name: "Foo",
Driver: driver,
Env: map[string]string{
"HELLO": "world",
"lorem": "ipsum",
},
Resources: &structs.Resources{
CPU: 1000,
MemoryMB: 500,
Networks: []*structs.NetworkResource{
{
IP: "1.2.3.4",
ReservedPorts: []structs.Port{{Label: "one", Value: 80}, {Label: "two", Value: 443}},
DynamicPorts: []structs.Port{{Label: "admin", Value: 8081}, {Label: "web", Value: 8086}},
},
},
},
Meta: map[string]string{
"chocolate": "cake",
"strawberry": "icecream",
},
}
alloc := mock.Alloc()
alloc.Job.TaskGroups[0].Tasks[0] = task
alloc.Name = "Bar"
alloc.AllocatedResources.Tasks["web"].Networks[0].DynamicPorts[0].Value = 2000
conf := testConfig(t)
allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(conf.AllocDir, alloc.ID))
taskDir := allocDir.NewTaskDir(task.Name)
eb := env.NewBuilder(conf.Node, alloc, task, conf.Region)
tmpDriver, err := NewDriver(driver, NewEmptyDriverContext())
if err != nil {
t.Fatalf("unable to create driver %q: %v", driver, err)
}
setEnvvars(eb, tmpDriver.FSIsolation(), taskDir, conf)
exp := map[string]string{
"NOMAD_CPU_LIMIT": "1000",
"NOMAD_MEMORY_LIMIT": "500",
"NOMAD_ADDR_one": "1.2.3.4:80",
"NOMAD_IP_one": "1.2.3.4",
"NOMAD_PORT_one": "80",
"NOMAD_HOST_PORT_one": "80",
"NOMAD_ADDR_two": "1.2.3.4:443",
"NOMAD_IP_two": "1.2.3.4",
"NOMAD_PORT_two": "443",
"NOMAD_HOST_PORT_two": "443",
"NOMAD_ADDR_admin": "1.2.3.4:8081",
"NOMAD_ADDR_web_admin": "192.168.0.100:5000",
"NOMAD_ADDR_web_http": "192.168.0.100:2000",
"NOMAD_IP_web_admin": "192.168.0.100",
"NOMAD_IP_web_http": "192.168.0.100",
"NOMAD_PORT_web_http": "2000",
"NOMAD_PORT_web_admin": "5000",
"NOMAD_IP_admin": "1.2.3.4",
"NOMAD_PORT_admin": "8081",
"NOMAD_HOST_PORT_admin": "8081",
"NOMAD_ADDR_web": "1.2.3.4:8086",
"NOMAD_IP_web": "1.2.3.4",
"NOMAD_PORT_web": "8086",
"NOMAD_HOST_PORT_web": "8086",
"NOMAD_META_CHOCOLATE": "cake",
"NOMAD_META_STRAWBERRY": "icecream",
"NOMAD_META_ELB_CHECK_INTERVAL": "30s",
"NOMAD_META_ELB_CHECK_TYPE": "http",
"NOMAD_META_ELB_CHECK_MIN": "3",
"NOMAD_META_OWNER": "armon",
"NOMAD_META_chocolate": "cake",
"NOMAD_META_strawberry": "icecream",
"NOMAD_META_elb_check_interval": "30s",
"NOMAD_META_elb_check_type": "http",
"NOMAD_META_elb_check_min": "3",
"NOMAD_META_owner": "armon",
"HELLO": "world",
"lorem": "ipsum",
"NOMAD_ALLOC_ID": alloc.ID,
"NOMAD_ALLOC_INDEX": "0",
"NOMAD_ALLOC_NAME": alloc.Name,
"NOMAD_TASK_NAME": task.Name,
"NOMAD_GROUP_NAME": alloc.TaskGroup,
"NOMAD_JOB_NAME": alloc.Job.Name,
"NOMAD_DC": "dc1",
"NOMAD_REGION": "global",
}
act := eb.Build().Map()
return taskDir, exp, act
}
func TestDriver_GetTaskEnv_None(t *testing.T) {
t.Parallel()
taskDir, exp, act := setupTaskEnv(t, "raw_exec")
// raw_exec should use host alloc dir path
exp[env.AllocDir] = taskDir.SharedAllocDir
exp[env.TaskLocalDir] = taskDir.LocalDir
exp[env.SecretsDir] = taskDir.SecretsDir
// Since host env vars are included only ensure expected env vars are present
for expk, expv := range exp {
v, ok := act[expk]
if !ok {
t.Errorf("%q not found in task env", expk)
continue
}
if v != expv {
t.Errorf("Expected %s=%q but found %q", expk, expv, v)
}
}
// Make sure common host env vars are included.
for _, envvar := range [...]string{"PATH", "HOME", "USER"} {
if exp := os.Getenv(envvar); act[envvar] != exp {
t.Errorf("Expected envvar %s=%q != %q", envvar, exp, act[envvar])
}
}
}
func TestDriver_GetTaskEnv_Chroot(t *testing.T) {
t.Parallel()
_, exp, act := setupTaskEnv(t, "exec")
exp[env.AllocDir] = allocdir.SharedAllocContainerPath
exp[env.TaskLocalDir] = allocdir.TaskLocalContainerPath
exp[env.SecretsDir] = allocdir.TaskSecretsContainerPath
// Since host env vars are included only ensure expected env vars are present
for expk, expv := range exp {
v, ok := act[expk]
if !ok {
t.Errorf("%q not found in task env", expk)
continue
}
if v != expv {
t.Errorf("Expected %s=%q but found %q", expk, expv, v)
}
}
// Make sure common host env vars are included.
for _, envvar := range [...]string{"PATH", "HOME", "USER"} {
if exp := os.Getenv(envvar); act[envvar] != exp {
t.Errorf("Expected envvar %s=%q != %q", envvar, exp, act[envvar])
}
}
}
// TestDriver_TaskEnv_Image ensures host environment variables are not set
// for image based drivers. See #2211
func TestDriver_TaskEnv_Image(t *testing.T) {
t.Parallel()
_, exp, act := setupTaskEnv(t, "docker")
exp[env.AllocDir] = allocdir.SharedAllocContainerPath
exp[env.TaskLocalDir] = allocdir.TaskLocalContainerPath
exp[env.SecretsDir] = allocdir.TaskSecretsContainerPath
// Since host env vars are excluded expected and actual maps should be equal
for expk, expv := range exp {
v, ok := act[expk]
delete(act, expk)
if !ok {
t.Errorf("Env var %s missing. Expected %s=%q", expk, expk, expv)
continue
}
if v != expv {
t.Errorf("Env var %s=%q -- Expected %q", expk, v, expk)
}
}
// Any remaining env vars are unexpected
for actk, actv := range act {
t.Errorf("Env var %s=%q is unexpected", actk, actv)
}
}
func TestMapMergeStrStr(t *testing.T) {
t.Parallel()
a := map[string]string{
"cake": "chocolate",
"cookie": "caramel",
}
b := map[string]string{
"cake": "strawberry",
"pie": "apple",
}
c := mapMergeStrStr(a, b)
d := map[string]string{
"cake": "strawberry",
"cookie": "caramel",
"pie": "apple",
}
if !reflect.DeepEqual(c, d) {
t.Errorf("\nExpected\n%+v\nGot\n%+v\n", d, c)
}
}
func TestCreatedResources_AddMerge(t *testing.T) {
t.Parallel()
res1 := NewCreatedResources()
res1.Add("k1", "v1")
res1.Add("k1", "v2")
res1.Add("k1", "v1")
res1.Add("k2", "v1")
expected := map[string][]string{
"k1": {"v1", "v2"},
"k2": {"v1"},
}
if !reflect.DeepEqual(expected, res1.Resources) {
t.Fatalf("1. %#v != expected %#v", res1.Resources, expected)
}
// Make sure merging nil works
var res2 *CreatedResources
res1.Merge(res2)
if !reflect.DeepEqual(expected, res1.Resources) {
t.Fatalf("2. %#v != expected %#v", res1.Resources, expected)
}
// Make sure a normal merge works
res2 = NewCreatedResources()
res2.Add("k1", "v3")
res2.Add("k2", "v1")
res2.Add("k3", "v3")
res1.Merge(res2)
expected = map[string][]string{
"k1": {"v1", "v2", "v3"},
"k2": {"v1"},
"k3": {"v3"},
}
if !reflect.DeepEqual(expected, res1.Resources) {
t.Fatalf("3. %#v != expected %#v", res1.Resources, expected)
}
}
func TestCreatedResources_CopyRemove(t *testing.T) {
t.Parallel()
res1 := NewCreatedResources()
res1.Add("k1", "v1")
res1.Add("k1", "v2")
res1.Add("k1", "v3")
res1.Add("k2", "v1")
// Assert Copy creates a deep copy
res2 := res1.Copy()
if !reflect.DeepEqual(res1, res2) {
t.Fatalf("%#v != %#v", res1, res2)
}
// Assert removing v1 from k1 returns true and updates Resources slice
if removed := res2.Remove("k1", "v1"); !removed {
t.Fatalf("expected v1 to be removed: %#v", res2)
}
if expected := []string{"v2", "v3"}; !reflect.DeepEqual(expected, res2.Resources["k1"]) {
t.Fatalf("unexpected list for k1: %#v", res2.Resources["k1"])
}
// Assert removing the only value from a key removes the key
if removed := res2.Remove("k2", "v1"); !removed {
t.Fatalf("expected v1 to be removed from k2: %#v", res2.Resources)
}
if _, found := res2.Resources["k2"]; found {
t.Fatalf("k2 should have been removed from Resources: %#v", res2.Resources)
}
// Make sure res1 wasn't updated
if reflect.DeepEqual(res1, res2) {
t.Fatalf("res1 should not equal res2: #%v", res1)
}
}

310
client/driver/exec.go
View file

@ -1,310 +0,0 @@
package driver
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"time"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/allocdir"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
pexecutor "github.com/hashicorp/nomad/plugins/executor"
"github.com/mitchellh/mapstructure"
)
// ExecDriver fork/execs tasks using as many of the underlying OS's isolation
// features.
type ExecDriver struct {
DriverContext
// A tri-state boolean to know if the fingerprinting has happened and
// whether it has been successful
fingerprintSuccess *bool
}
type ExecDriverConfig struct {
Command string `mapstructure:"command"`
Args []string `mapstructure:"args"`
}
// execHandle is returned from Start/Open as a handle to the PID
type execHandle struct {
pluginClient *plugin.Client
executor executor.Executor
userPid int
taskShutdownSignal string
taskDir *allocdir.TaskDir
killTimeout time.Duration
maxKillTimeout time.Duration
logger *log.Logger
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
version string
}
// NewExecDriver is used to create a new exec driver
func NewExecDriver(ctx *DriverContext) Driver {
return &ExecDriver{DriverContext: *ctx}
}
// Validate is used to validate the driver configuration
func (d *ExecDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"command": {
Type: fields.TypeString,
Required: true,
},
"args": {
Type: fields.TypeArray,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *ExecDriver) Abilities() DriverAbilities {
return DriverAbilities{
SendSignals: true,
Exec: true,
}
}
func (d *ExecDriver) FSIsolation() cstructs.FSIsolation {
return cstructs.FSIsolationChroot
}
func (d *ExecDriver) Periodic() (bool, time.Duration) {
return true, 15 * time.Second
}
func (d *ExecDriver) Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error) {
return nil, nil
}
func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
var driverConfig ExecDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
// Get the command to be ran
command := driverConfig.Command
if err := validateCommand(command, "args"); err != nil {
return nil, err
}
pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out")
executorConfig := &pexecutor.ExecutorConfig{
LogFile: pluginLogFile,
LogLevel: d.config.LogLevel,
FSIsolation: true,
}
exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig)
if err != nil {
return nil, err
}
_, err = getTaskKillSignal(task.KillSignal)
if err != nil {
return nil, err
}
execCmd := &executor.ExecCommand{
Cmd: command,
Args: driverConfig.Args,
ResourceLimits: true,
User: getExecutorUser(task),
Resources: &executor.Resources{
CPU: task.Resources.CPU,
MemoryMB: task.Resources.MemoryMB,
IOPS: task.Resources.IOPS,
DiskMB: task.Resources.DiskMB,
},
Env: ctx.TaskEnv.List(),
TaskDir: ctx.TaskDir.Dir,
StdoutPath: ctx.StdoutFifo,
StderrPath: ctx.StderrFifo,
}
ps, err := exec.Launch(execCmd)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[DEBUG] driver.exec: started process via plugin with pid: %v", ps.Pid)
// Return a driver handle
maxKill := d.DriverContext.config.MaxKillTimeout
h := &execHandle{
pluginClient: pluginClient,
userPid: ps.Pid,
taskShutdownSignal: task.KillSignal,
executor: exec,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
logger: d.logger,
version: d.config.Version.VersionNumber(),
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
taskDir: ctx.TaskDir,
}
go h.run()
return &StartResponse{Handle: h}, nil
}
func (d *ExecDriver) Cleanup(*ExecContext, *CreatedResources) error { return nil }
type execId struct {
Version string
KillTimeout time.Duration
MaxKillTimeout time.Duration
UserPid int
PluginConfig *pexecutor.PluginReattachConfig
}
func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
id := &execId{}
if err := json.Unmarshal([]byte(handleID), id); err != nil {
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, client, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput)
if err != nil {
merrs := new(multierror.Error)
merrs.Errors = append(merrs.Errors, err)
d.logger.Println("[ERR] driver.exec: error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e))
}
return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil())
}
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.exec : version of executor: %v", ver.Version)
// Return a driver handle
h := &execHandle{
pluginClient: client,
executor: exec,
userPid: id.UserPid,
logger: d.logger,
version: id.Version,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
taskDir: ctx.TaskDir,
}
go h.run()
return h, nil
}
func (h *execHandle) ID() string {
id := execId{
Version: h.version,
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
PluginConfig: pexecutor.NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
UserPid: h.userPid,
}
data, err := json.Marshal(id)
if err != nil {
h.logger.Printf("[ERR] driver.exec: failed to marshal ID to JSON: %s", err)
}
return string(data)
}
func (h *execHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *execHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
h.executor.UpdateResources(&executor.Resources{
CPU: task.Resources.CPU,
MemoryMB: task.Resources.MemoryMB,
IOPS: task.Resources.IOPS,
DiskMB: task.Resources.DiskMB,
})
// Update is not possible
return nil
}
func (h *execHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
deadline, ok := ctx.Deadline()
if !ok {
// No deadline set on context; default to 1 minute
deadline = time.Now().Add(time.Minute)
}
return h.executor.Exec(deadline, cmd, args)
}
func (h *execHandle) Signal(s os.Signal) error {
return h.executor.Signal(s)
}
func (d *execHandle) Network() *cstructs.DriverNetwork {
return nil
}
func (h *execHandle) Kill() error {
if err := h.executor.Shutdown(h.taskShutdownSignal, h.killTimeout); err != nil {
if h.pluginClient.Exited() {
return nil
}
return fmt.Errorf("executor Kill failed: %v", err)
}
select {
case <-h.doneCh:
case <-time.After(h.killTimeout):
if h.pluginClient.Exited() {
break
}
if err := h.executor.Shutdown(h.taskShutdownSignal, h.killTimeout); err != nil {
return fmt.Errorf("executor Destroy failed: %v", err)
}
}
return nil
}
func (h *execHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return h.executor.Stats()
}
func (h *execHandle) run() {
ps, werr := h.executor.Wait()
close(h.doneCh)
// Destroy the executor
if err := h.executor.Shutdown(h.taskShutdownSignal, 0); err != nil {
h.logger.Printf("[ERR] driver.exec: error destroying executor: %v", err)
}
h.pluginClient.Kill()
// Send the results
h.waitCh <- dstructs.NewWaitResult(ps.ExitCode, ps.Signal, werr)
close(h.waitCh)
}

14
client/driver/exec_default.go
View file

@ -1,14 +0,0 @@
//+build darwin dragonfly freebsd netbsd openbsd solaris windows
package driver
import (
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper"
)
func (d *ExecDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
d.fingerprintSuccess = helper.BoolToPtr(false)
resp.Detected = false
return nil
}

42
client/driver/exec_linux.go
View file

@ -1,42 +0,0 @@
package driver
import (
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper"
"golang.org/x/sys/unix"
)
const (
// The key populated in Node Attributes to indicate the presence of the Exec
// driver
execDriverAttr = "driver.exec"
)
func (d *ExecDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
// The exec driver will be detected in every case
resp.Detected = true
// Only enable if cgroups are available and we are root
if !cgroupsMounted(req.Node) {
if d.fingerprintSuccess == nil || *d.fingerprintSuccess {
d.logger.Printf("[INFO] driver.exec: cgroups unavailable, disabling")
}
d.fingerprintSuccess = helper.BoolToPtr(false)
resp.RemoveAttribute(execDriverAttr)
return nil
} else if unix.Geteuid() != 0 {
if d.fingerprintSuccess == nil || *d.fingerprintSuccess {
d.logger.Printf("[DEBUG] driver.exec: must run as root user, disabling")
}
d.fingerprintSuccess = helper.BoolToPtr(false)
resp.RemoveAttribute(execDriverAttr)
return nil
}
if d.fingerprintSuccess == nil || !*d.fingerprintSuccess {
d.logger.Printf("[DEBUG] driver.exec: exec driver is enabled")
}
resp.AddAttribute(execDriverAttr, "1")
d.fingerprintSuccess = helper.BoolToPtr(true)
return nil
}

426
client/driver/exec_test.go
View file

@ -1,426 +0,0 @@
package driver
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"path/filepath"
"reflect"
"runtime"
"strings"
"testing"
"time"
"github.com/hashicorp/nomad/client/config"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/drivers/shared/env"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
ctestutils "github.com/hashicorp/nomad/client/testutil"
)
// Test that we do not enable exec on non-linux machines
func TestExecDriver_Fingerprint_NonLinux(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if runtime.GOOS == "linux" {
t.Skip("Test only available not on Linux")
}
d := NewExecDriver(&DriverContext{})
node := &structs.Node{}
request := &cstructs.FingerprintRequest{Config: &config.Config{}, Node: node}
var response cstructs.FingerprintResponse
err := d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
if response.Detected {
t.Fatalf("Should not be detected on non-linux platforms")
}
}
func TestExecDriver_Fingerprint(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "foo",
Driver: "exec",
Resources: structs.DefaultResources(),
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewExecDriver(ctx.DriverCtx)
node := &structs.Node{
Attributes: map[string]string{
"unique.cgroup.mountpoint": "/sys/fs/cgroup",
},
}
request := &cstructs.FingerprintRequest{Config: &config.Config{}, Node: node}
var response cstructs.FingerprintResponse
err := d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
if !response.Detected {
t.Fatalf("expected response to be applicable")
}
if response.Attributes == nil || response.Attributes["driver.exec"] == "" {
t.Fatalf("missing driver")
}
}
func TestExecDriver_StartOpen_Wait(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "sleep",
Driver: "exec",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": []string{"5"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Attempt to open
handle2, err := d.Open(ctx.ExecCtx, resp.Handle.ID())
if err != nil {
t.Fatalf("err: %v", err)
}
if handle2 == nil {
t.Fatalf("missing handle")
}
resp.Handle.Kill()
handle2.Kill()
}
func TestExecDriver_Start_Wait(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "sleep",
Driver: "exec",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": []string{"2"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Update should be a no-op
err = resp.Handle.Update(task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Task should terminate quickly
select {
case res := <-resp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
}
func TestExecDriver_Start_Wait_AllocDir(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.ExecCompatible(t)
exp := []byte{'w', 'i', 'n'}
file := "output.txt"
task := &structs.Task{
Name: "sleep",
Driver: "exec",
Config: map[string]interface{}{
"command": "/bin/bash",
"args": []string{
"-c",
fmt.Sprintf(`sleep 1; echo -n %s > ${%s}/%s`, string(exp), env.AllocDir, file),
},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Task should terminate quickly
select {
case res := <-resp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
// Check that data was written to the shared alloc directory.
outputFile := filepath.Join(ctx.AllocDir.SharedDir, file)
act, err := ioutil.ReadFile(outputFile)
if err != nil {
t.Fatalf("Couldn't read expected output: %v", err)
}
if !reflect.DeepEqual(act, exp) {
t.Fatalf("Command outputted %v; want %v", act, exp)
}
}
func TestExecDriver_Start_Kill_Wait(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "sleep",
Driver: "exec",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": []string{"100"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
KillTimeout: 10 * time.Second,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
go func() {
time.Sleep(100 * time.Millisecond)
err := resp.Handle.Kill()
if err != nil {
t.Fatalf("err: %v", err)
}
}()
// Task should terminate quickly
select {
case res := <-resp.Handle.WaitCh():
if res.Successful() {
t.Fatal("should err")
}
case <-time.After(time.Duration(testutil.TestMultiplier()*10) * time.Second):
t.Fatalf("timeout")
}
}
func TestExecDriverUser(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "sleep",
Driver: "exec",
User: "alice",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": []string{"100"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
KillTimeout: 10 * time.Second,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err == nil {
resp.Handle.Kill()
t.Fatalf("Should've failed")
}
msg := "user alice"
if !strings.Contains(err.Error(), msg) {
t.Fatalf("Expecting '%v' in '%v'", msg, err)
}
}
// TestExecDriver_HandlerExec ensures the exec driver's handle properly
// executes commands inside the container.
func TestExecDriver_HandlerExec(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "sleep",
Driver: "exec",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": []string{"9000"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
//defer ctx.Destroy()
d := NewExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
handle := resp.Handle
// Exec a command that should work and dump the environment
out, code, err := handle.Exec(context.Background(), "/bin/sh", []string{"-c", "env | grep ^NOMAD"})
if err != nil {
t.Fatalf("error exec'ing stat: %v", err)
}
if code != 0 {
t.Fatalf("expected `stat /alloc` to succeed but exit code was: %d", code)
}
// Assert exec'd commands are run in a task-like environment
scriptEnv := make(map[string]string)
for _, line := range strings.Split(string(out), "\n") {
if line == "" {
continue
}
parts := strings.SplitN(string(line), "=", 2)
if len(parts) != 2 {
t.Fatalf("Invalid env var: %q", line)
}
scriptEnv[parts[0]] = parts[1]
}
if v, ok := scriptEnv["NOMAD_SECRETS_DIR"]; !ok || v != "/secrets" {
t.Errorf("Expected NOMAD_SECRETS_DIR=/secrets but found=%t value=%q", ok, v)
}
if v, ok := scriptEnv["NOMAD_ALLOC_ID"]; !ok || v != ctx.DriverCtx.allocID {
t.Errorf("Expected NOMAD_SECRETS_DIR=%q but found=%t value=%q", ctx.DriverCtx.allocID, ok, v)
}
// Assert cgroup membership
out, code, err = handle.Exec(context.Background(), "/bin/cat", []string{"/proc/self/cgroup"})
if err != nil {
t.Fatalf("error exec'ing cat /proc/self/cgroup: %v", err)
}
if code != 0 {
t.Fatalf("expected `cat /proc/self/cgroup` to succeed but exit code was: %d", code)
}
found := false
for _, line := range strings.Split(string(out), "\n") {
// Every cgroup entry should be /nomad/$ALLOC_ID
if line == "" {
continue
}
if !strings.Contains(line, ":/nomad/") && !strings.Contains(line, ":name=") {
t.Errorf("Not a member of the alloc's cgroup: expected=...:/nomad/... -- found=%q", line)
continue
}
found = true
}
if !found {
t.Errorf("exec'd command isn't in the task's cgroup")
}
// Exec a command that should fail
out, code, err = handle.Exec(context.Background(), "/usr/bin/stat", []string{"lkjhdsaflkjshowaisxmcvnlia"})
if err != nil {
t.Fatalf("error exec'ing stat: %v", err)
}
if code == 0 {
t.Fatalf("expected `stat` to fail but exit code was: %d", code)
}
if expected := "No such file or directory"; !bytes.Contains(out, []byte(expected)) {
t.Fatalf("expected output to contain %q but found: %q", expected, out)
}
if err := handle.Kill(); err != nil {
t.Logf("Check allocdir: %x", ctx.AllocDir.AllocDir)
t.Fatalf("error killing exec handle: %v", err)
}
}

170
client/driver/exec_unix_test.go
View file

@ -1,170 +0,0 @@
// +build !windows
package driver
import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
ctestutils "github.com/hashicorp/nomad/client/testutil"
)
func TestExecDriver_KillUserPid_OnPluginReconnectFailure(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "sleep",
Driver: "exec",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": []string{"1000000"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer resp.Handle.Kill()
id := &execId{}
if err := json.Unmarshal([]byte(resp.Handle.ID()), id); err != nil {
t.Fatalf("Failed to parse handle '%s': %v", resp.Handle.ID(), err)
}
pluginPid := id.PluginConfig.Pid
proc, err := os.FindProcess(pluginPid)
if err != nil {
t.Fatalf("can't find plugin pid: %v", pluginPid)
}
if err := proc.Kill(); err != nil {
t.Fatalf("can't kill plugin pid: %v", err)
}
// Attempt to open
handle2, err := d.Open(ctx.ExecCtx, resp.Handle.ID())
if err == nil {
t.Fatalf("expected error")
}
if handle2 != nil {
handle2.Kill()
t.Fatalf("expected handle2 to be nil")
}
// Test if the userpid is still present
userProc, _ := os.FindProcess(id.UserPid)
for retry := 3; retry > 0; retry-- {
if err = userProc.Signal(syscall.Signal(0)); err != nil {
// Process is gone as expected; exit
return
}
// Killing processes is async; wait and check again
time.Sleep(time.Second)
}
if err = userProc.Signal(syscall.Signal(0)); err == nil {
t.Fatalf("expected user process to die")
}
}
func TestExecDriver_Signal(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "signal",
Driver: "exec",
Config: map[string]interface{}{
"command": "/bin/bash",
"args": []string{"test.sh"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
KillTimeout: 10 * time.Second,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewExecDriver(ctx.DriverCtx)
testFile := filepath.Join(ctx.ExecCtx.TaskDir.Dir, "test.sh")
testData := []byte(`
at_term() {
echo 'Terminated.'
exit 3
}
trap at_term USR1
while true; do
sleep 1
done
`)
if err := ioutil.WriteFile(testFile, testData, 0777); err != nil {
t.Fatalf("Failed to write data: %v", err)
}
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
go func() {
time.Sleep(100 * time.Millisecond)
err := resp.Handle.Signal(syscall.SIGUSR1)
if err != nil {
t.Fatalf("err: %v", err)
}
}()
// Task should terminate quickly
select {
case res := <-resp.Handle.WaitCh():
if res.Successful() {
t.Fatal("should err")
}
case <-time.After(time.Duration(testutil.TestMultiplier()*6) * time.Second):
t.Fatalf("timeout")
}
// Check the log file to see it exited because of the signal
outputFile := filepath.Join(ctx.ExecCtx.TaskDir.LogDir, "signal.stdout.0")
act, err := ioutil.ReadFile(outputFile)
if err != nil {
t.Fatalf("Couldn't read expected output: %v", err)
}
exp := "Terminated."
if strings.TrimSpace(string(act)) != exp {
t.Logf("Read from %v", outputFile)
t.Fatalf("Command outputted %v; want %v", act, exp)
}
}

453
client/driver/java.go
View file

@ -1,453 +0,0 @@
package driver
import (
"bytes"
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"time"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-plugin"
"github.com/mitchellh/mapstructure"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/client/fingerprint"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/drivers/shared/env"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
pexecutor "github.com/hashicorp/nomad/plugins/executor"
)
const (
// The key populated in Node Attributes to indicate presence of the Java
// driver
javaDriverAttr = "driver.java"
)
// JavaDriver is a simple driver to execute applications packaged in Jars.
// It literally just fork/execs tasks with the java command.
type JavaDriver struct {
DriverContext
fingerprint.StaticFingerprinter
// A tri-state boolean to know if the fingerprinting has happened and
// whether it has been successful
fingerprintSuccess *bool
}
type JavaDriverConfig struct {
Class string `mapstructure:"class"`
ClassPath string `mapstructure:"class_path"`
JarPath string `mapstructure:"jar_path"`
JvmOpts []string `mapstructure:"jvm_options"`
Args []string `mapstructure:"args"`
}
// javaHandle is returned from Start/Open as a handle to the PID
type javaHandle struct {
pluginClient *plugin.Client
userPid int
executor executor.Executor
taskDir string
killTimeout time.Duration
maxKillTimeout time.Duration
shutdownSignal string
version string
logger *log.Logger
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
}
// NewJavaDriver is used to create a new exec driver
func NewJavaDriver(ctx *DriverContext) Driver {
return &JavaDriver{DriverContext: *ctx}
}
// Validate is used to validate the driver configuration
func (d *JavaDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"class": {
Type: fields.TypeString,
},
"class_path": {
Type: fields.TypeString,
},
"jar_path": {
Type: fields.TypeString,
},
"jvm_options": {
Type: fields.TypeArray,
},
"args": {
Type: fields.TypeArray,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *JavaDriver) Abilities() DriverAbilities {
return DriverAbilities{
SendSignals: true,
Exec: true,
}
}
func (d *JavaDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
// Only enable if we are root and cgroups are mounted when running on linux systems.
if runtime.GOOS == "linux" && (syscall.Geteuid() != 0 || !cgroupsMounted(req.Node)) {
if d.fingerprintSuccess == nil || *d.fingerprintSuccess {
d.logger.Printf("[INFO] driver.java: root privileges and mounted cgroups required on linux, disabling")
}
d.fingerprintSuccess = helper.BoolToPtr(false)
resp.RemoveAttribute(javaDriverAttr)
resp.Detected = true
return nil
}
// Find java version
var out bytes.Buffer
var erOut bytes.Buffer
cmd := exec.Command("java", "-version")
cmd.Stdout = &out
cmd.Stderr = &erOut
err := cmd.Run()
if err != nil {
// assume Java wasn't found
d.fingerprintSuccess = helper.BoolToPtr(false)
resp.RemoveAttribute(javaDriverAttr)
return nil
}
// 'java -version' returns output on Stderr typically.
// Check stdout, but it's probably empty
var infoString string
if out.String() != "" {
infoString = out.String()
}
if erOut.String() != "" {
infoString = erOut.String()
}
if infoString == "" {
if d.fingerprintSuccess == nil || *d.fingerprintSuccess {
d.logger.Println("[WARN] driver.java: error parsing Java version information, aborting")
}
d.fingerprintSuccess = helper.BoolToPtr(false)
resp.RemoveAttribute(javaDriverAttr)
return nil
}
// Assume 'java -version' returns 3 lines:
// java version "1.6.0_36"
// OpenJDK Runtime Environment (IcedTea6 1.13.8) (6b36-1.13.8-0ubuntu1~12.04)
// OpenJDK 64-Bit Server VM (build 23.25-b01, mixed mode)
// Each line is terminated by \n
info := strings.Split(infoString, "\n")
versionString := info[0]
versionString = strings.TrimPrefix(versionString, "java version ")
versionString = strings.Trim(versionString, "\"")
resp.AddAttribute(javaDriverAttr, "1")
resp.AddAttribute("driver.java.version", versionString)
resp.AddAttribute("driver.java.runtime", info[1])
resp.AddAttribute("driver.java.vm", info[2])
d.fingerprintSuccess = helper.BoolToPtr(true)
resp.Detected = true
return nil
}
func (d *JavaDriver) Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error) {
return nil, nil
}
func NewJavaDriverConfig(task *structs.Task, env *env.TaskEnv) (*JavaDriverConfig, error) {
var driverConfig JavaDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
// Interpolate everything
driverConfig.Class = env.ReplaceEnv(driverConfig.Class)
driverConfig.ClassPath = env.ReplaceEnv(driverConfig.ClassPath)
driverConfig.JarPath = env.ReplaceEnv(driverConfig.JarPath)
driverConfig.JvmOpts = env.ParseAndReplace(driverConfig.JvmOpts)
driverConfig.Args = env.ParseAndReplace(driverConfig.Args)
// Validate
jarSpecified := driverConfig.JarPath != ""
classSpecified := driverConfig.Class != ""
if !jarSpecified && !classSpecified {
return nil, fmt.Errorf("jar_path or class must be specified")
}
return &driverConfig, nil
}
func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
driverConfig, err := NewJavaDriverConfig(task, ctx.TaskEnv)
if err != nil {
return nil, err
}
args := []string{}
// Look for jvm options
if len(driverConfig.JvmOpts) != 0 {
d.logger.Printf("[DEBUG] driver.java: found JVM options: %s", driverConfig.JvmOpts)
args = append(args, driverConfig.JvmOpts...)
}
// Add the classpath
if driverConfig.ClassPath != "" {
args = append(args, "-cp", driverConfig.ClassPath)
}
// Add the jar
if driverConfig.JarPath != "" {
args = append(args, "-jar", driverConfig.JarPath)
}
// Add the class
if driverConfig.Class != "" {
args = append(args, driverConfig.Class)
}
// Add any args
if len(driverConfig.Args) != 0 {
args = append(args, driverConfig.Args...)
}
pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out")
executorConfig := &pexecutor.ExecutorConfig{
LogFile: pluginLogFile,
LogLevel: d.config.LogLevel,
FSIsolation: true,
}
execIntf, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig)
if err != nil {
return nil, err
}
absPath, err := GetAbsolutePath("java")
if err != nil {
return nil, err
}
_, err = getTaskKillSignal(task.KillSignal)
if err != nil {
return nil, err
}
execCmd := &executor.ExecCommand{
Cmd: absPath,
Args: args,
ResourceLimits: true,
User: getExecutorUser(task),
Resources: &executor.Resources{
CPU: task.Resources.CPU,
MemoryMB: task.Resources.MemoryMB,
IOPS: task.Resources.IOPS,
DiskMB: task.Resources.DiskMB,
},
Env: ctx.TaskEnv.List(),
TaskDir: ctx.TaskDir.Dir,
StdoutPath: ctx.StdoutFifo,
StderrPath: ctx.StderrFifo,
}
ps, err := execIntf.Launch(execCmd)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[DEBUG] driver.java: started process with pid: %v", ps.Pid)
// Return a driver handle
maxKill := d.DriverContext.config.MaxKillTimeout
h := &javaHandle{
pluginClient: pluginClient,
executor: execIntf,
userPid: ps.Pid,
shutdownSignal: task.KillSignal,
taskDir: ctx.TaskDir.Dir,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
version: d.config.Version.VersionNumber(),
logger: d.logger,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
go h.run()
return &StartResponse{Handle: h}, nil
}
func (d *JavaDriver) Cleanup(*ExecContext, *CreatedResources) error { return nil }
type javaId struct {
Version string
KillTimeout time.Duration
MaxKillTimeout time.Duration
PluginConfig *pexecutor.PluginReattachConfig
TaskDir string
UserPid int
ShutdownSignal string
}
func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
id := &javaId{}
if err := json.Unmarshal([]byte(handleID), id); err != nil {
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput)
if err != nil {
merrs := new(multierror.Error)
merrs.Errors = append(merrs.Errors, err)
d.logger.Println("[ERR] driver.java: error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e))
}
return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil())
}
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.java: version of executor: %v", ver.Version)
// Return a driver handle
h := &javaHandle{
pluginClient: pluginClient,
executor: exec,
userPid: id.UserPid,
shutdownSignal: id.ShutdownSignal,
logger: d.logger,
version: id.Version,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
go h.run()
return h, nil
}
func (h *javaHandle) ID() string {
id := javaId{
Version: h.version,
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
PluginConfig: pexecutor.NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
UserPid: h.userPid,
TaskDir: h.taskDir,
ShutdownSignal: h.shutdownSignal,
}
data, err := json.Marshal(id)
if err != nil {
h.logger.Printf("[ERR] driver.java: failed to marshal ID to JSON: %s", err)
}
return string(data)
}
func (h *javaHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *javaHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
h.executor.UpdateResources(&executor.Resources{
CPU: task.Resources.CPU,
MemoryMB: task.Resources.MemoryMB,
IOPS: task.Resources.IOPS,
DiskMB: task.Resources.DiskMB,
})
// Update is not possible
return nil
}
func (h *javaHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
deadline, ok := ctx.Deadline()
if !ok {
// No deadline set on context; default to 1 minute
deadline = time.Now().Add(time.Minute)
}
return h.executor.Exec(deadline, cmd, args)
}
func (h *javaHandle) Signal(s os.Signal) error {
return h.executor.Signal(s)
}
func (d *javaHandle) Network() *cstructs.DriverNetwork {
return nil
}
func (h *javaHandle) Kill() error {
if err := h.executor.Shutdown(h.shutdownSignal, h.killTimeout); err != nil {
if h.pluginClient.Exited() {
return nil
}
return fmt.Errorf("executor Kill failed: %v", err)
}
select {
case <-h.doneCh:
case <-time.After(h.killTimeout):
if h.pluginClient.Exited() {
break
}
if err := h.executor.Shutdown(h.shutdownSignal, h.killTimeout); err != nil {
return fmt.Errorf("executor Destroy failed: %v", err)
}
}
return nil
}
func (h *javaHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return h.executor.Stats()
}
func (h *javaHandle) run() {
ps, werr := h.executor.Wait()
close(h.doneCh)
if ps.ExitCode == 0 && werr != nil {
if e := killProcess(h.userPid); e != nil {
h.logger.Printf("[ERR] driver.java: error killing user process: %v", e)
}
}
// Destroy the executor
h.executor.Shutdown(h.shutdownSignal, 0)
h.pluginClient.Kill()
// Send the results
h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: werr}
close(h.waitCh)
}

7
client/driver/java_linux.go
View file

@ -1,7 +0,0 @@
package driver
import cstructs "github.com/hashicorp/nomad/client/structs"
func (d *JavaDriver) FSIsolation() cstructs.FSIsolation {
return cstructs.FSIsolationChroot
}

523
client/driver/java_test.go
View file

@ -1,523 +0,0 @@
package driver
import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"testing"
"time"
"github.com/hashicorp/nomad/client/config"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
"github.com/stretchr/testify/assert"
ctestutils "github.com/hashicorp/nomad/client/testutil"
)
var (
osJavaDriverSupport = map[string]bool{
"linux": true,
}
)
// javaLocated checks whether java is installed so we can run java stuff.
func javaLocated() bool {
_, err := exec.Command("java", "-version").CombinedOutput()
return err == nil
}
// The fingerprinter test should always pass, even if Java is not installed.
func TestJavaDriver_Fingerprint(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "foo",
Driver: "java",
Resources: structs.DefaultResources(),
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewJavaDriver(ctx.DriverCtx)
node := &structs.Node{
Attributes: map[string]string{
"unique.cgroup.mountpoint": "/sys/fs/cgroups",
},
}
request := &cstructs.FingerprintRequest{Config: &config.Config{}, Node: node}
var response cstructs.FingerprintResponse
err := d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
if !response.Detected {
t.Fatalf("expected response to be applicable")
}
if response.Attributes["driver.java"] != "1" && javaLocated() {
if v, ok := osJavaDriverSupport[runtime.GOOS]; v && ok {
t.Fatalf("missing java driver")
} else {
t.Skipf("missing java driver, no OS support")
}
}
for _, key := range []string{"driver.java.version", "driver.java.runtime", "driver.java.vm"} {
if response.Attributes[key] == "" {
t.Fatalf("missing driver key (%s)", key)
}
}
}
func TestJavaDriver_StartOpen_Wait(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !javaLocated() {
t.Skip("Java not found; skipping")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Driver: "java",
Config: map[string]interface{}{
"jar_path": "demoapp.jar",
"jvm_options": []string{"-Xmx64m", "-Xms32m"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewJavaDriver(ctx.DriverCtx)
// Copy the test jar into the task's directory
dst := ctx.ExecCtx.TaskDir.Dir
copyFile("./test-resources/java/demoapp.jar", filepath.Join(dst, "demoapp.jar"), t)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Attempt to open
handle2, err := d.Open(ctx.ExecCtx, resp.Handle.ID())
if err != nil {
t.Fatalf("err: %v", err)
}
if handle2 == nil {
t.Fatalf("missing handle")
}
time.Sleep(2 * time.Second)
// There is a race condition between the handle waiting and killing. One
// will return an error.
resp.Handle.Kill()
handle2.Kill()
}
func TestJavaDriver_Start_Wait(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !javaLocated() {
t.Skip("Java not found; skipping")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Driver: "java",
Config: map[string]interface{}{
"jar_path": "demoapp.jar",
"args": []string{"1"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewJavaDriver(ctx.DriverCtx)
// Copy the test jar into the task's directory
dst := ctx.ExecCtx.TaskDir.Dir
copyFile("./test-resources/java/demoapp.jar", filepath.Join(dst, "demoapp.jar"), t)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Task should terminate after 1 seconds
select {
case res := <-resp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res.String())
}
case <-time.After(5 * time.Second):
t.Fatalf("timeout")
}
// Get the stdout of the process and assert that it's not empty
stdout := filepath.Join(ctx.ExecCtx.TaskDir.LogDir, "demo-app.stdout.0")
fInfo, err := os.Stat(stdout)
if err != nil {
t.Fatalf("failed to get stdout of process: %v", err)
}
if fInfo.Size() == 0 {
t.Fatalf("stdout of process is empty")
}
// need to kill long lived process
err = resp.Handle.Kill()
if err != nil {
t.Fatalf("Error: %s", err)
}
}
func TestJavaDriver_Start_Kill_Wait(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !javaLocated() {
t.Skip("Java not found; skipping")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Driver: "java",
Config: map[string]interface{}{
"jar_path": "demoapp.jar",
"args": []string{"5"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewJavaDriver(ctx.DriverCtx)
// Copy the test jar into the task's directory
dst := ctx.ExecCtx.TaskDir.Dir
copyFile("./test-resources/java/demoapp.jar", filepath.Join(dst, "demoapp.jar"), t)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
errCh := make(chan error, 1)
go func() {
time.Sleep(10 * time.Millisecond)
err := resp.Handle.Kill()
if err != nil {
errCh <- err
}
}()
// Task should terminate quickly
select {
case err := <-errCh:
t.Fatalf("err: %v", err)
case res := <-resp.Handle.WaitCh():
if res.Successful() {
t.Fatal("should err")
}
case <-time.After(time.Duration(testutil.TestMultiplier()*10) * time.Second):
t.Fatalf("timeout")
// Need to kill long lived process
if err = resp.Handle.Kill(); err != nil {
t.Fatalf("Error: %s", err)
}
}
}
func TestJavaDriver_Signal(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !javaLocated() {
t.Skip("Java not found; skipping")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Driver: "java",
Config: map[string]interface{}{
"jar_path": "demoapp.jar",
"args": []string{"5"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewJavaDriver(ctx.DriverCtx)
// Copy the test jar into the task's directory
dst := ctx.ExecCtx.TaskDir.Dir
copyFile("./test-resources/java/demoapp.jar", filepath.Join(dst, "demoapp.jar"), t)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
errCh := make(chan error, 1)
go func() {
time.Sleep(10 * time.Millisecond)
err := resp.Handle.Signal(syscall.SIGHUP)
if err != nil {
errCh <- err
}
}()
// Task should terminate quickly
select {
case err := <-errCh:
t.Fatalf("err: %v", err)
case res := <-resp.Handle.WaitCh():
if res.Successful() {
t.Fatal("should err")
}
case <-time.After(time.Duration(testutil.TestMultiplier()*10) * time.Second):
t.Fatalf("timeout")
// Need to kill long lived process
if err = resp.Handle.Kill(); err != nil {
t.Fatalf("Error: %s", err)
}
}
}
func TestJavaDriver_User(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !javaLocated() {
t.Skip("Java not found; skipping")
}
if runtime.GOOS != "linux" {
t.Skip("Linux only test")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Driver: "java",
User: "alice",
Config: map[string]interface{}{
"jar_path": "demoapp.jar",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewJavaDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err == nil {
resp.Handle.Kill()
t.Fatalf("Should've failed")
}
msg := "user alice"
if !strings.Contains(err.Error(), msg) {
t.Fatalf("Expecting '%v' in '%v'", msg, err)
}
}
func TestJavaDriver_Start_Wait_Class(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !javaLocated() {
t.Skip("Java not found; skipping")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Driver: "java",
Config: map[string]interface{}{
"class_path": "${NOMAD_TASK_DIR}",
"class": "Hello",
"args": []string{"1"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewJavaDriver(ctx.DriverCtx)
// Copy the test jar into the task's directory
dst := ctx.ExecCtx.TaskDir.LocalDir
copyFile("./test-resources/java/Hello.class", filepath.Join(dst, "Hello.class"), t)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Task should terminate after 1 seconds
select {
case res := <-resp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res.String())
}
case <-time.After(5 * time.Second):
t.Fatalf("timeout")
}
// Get the stdout of the process and assert that it's not empty
stdout := filepath.Join(ctx.ExecCtx.TaskDir.LogDir, "demo-app.stdout.0")
fInfo, err := os.Stat(stdout)
if err != nil {
t.Fatalf("failed to get stdout of process: %v", err)
}
if fInfo.Size() == 0 {
t.Fatalf("stdout of process is empty")
}
// need to kill long lived process
if err := resp.Handle.Kill(); err != nil {
t.Fatalf("Error: %s", err)
}
}
func TestJavaDriver_Start_Kill(t *testing.T) {
assert := assert.New(t)
if !testutil.IsTravis() {
t.Parallel()
}
if !javaLocated() {
t.Skip("Java not found; skipping")
}
// Test that a valid kill signal will successfully stop the process
{
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Driver: "java",
KillSignal: "SIGKILL",
Config: map[string]interface{}{
"jar_path": "demoapp.jar",
"args": []string{"5"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewJavaDriver(ctx.DriverCtx)
// Copy the test jar into the task's directory
dst := ctx.ExecCtx.TaskDir.Dir
copyFile("./test-resources/java/demoapp.jar", filepath.Join(dst, "demoapp.jar"), t)
_, err := d.Prestart(ctx.ExecCtx, task)
assert.Nil(err)
resp, err := d.Start(ctx.ExecCtx, task)
assert.Nil(err)
assert.NotNil(resp.Handle)
err = resp.Handle.Kill()
assert.Nil(err)
}
// Test that an unsupported kill signal will return an error
{
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Driver: "java",
KillSignal: "ABCDEF",
Config: map[string]interface{}{
"jar_path": "demoapp.jar",
"args": []string{"5"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewJavaDriver(ctx.DriverCtx)
// Copy the test jar into the task's directory
dst := ctx.ExecCtx.TaskDir.Dir
copyFile("./test-resources/java/demoapp.jar", filepath.Join(dst, "demoapp.jar"), t)
_, err := d.Prestart(ctx.ExecCtx, task)
assert.Nil(err)
_, err = d.Start(ctx.ExecCtx, task)
assert.NotNil(err)
assert.Contains(err.Error(), "Signal ABCDEF is not supported")
}
}

9
client/driver/java_universal.go
View file

@ -1,9 +0,0 @@
// +build !linux
package driver
import cstructs "github.com/hashicorp/nomad/client/structs"
func (d *JavaDriver) FSIsolation() cstructs.FSIsolation {
return cstructs.FSIsolationNone
}

598
client/driver/lxc.go
View file

@ -1,598 +0,0 @@
//+build linux,lxc
package driver
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/hashicorp/nomad/client/fingerprint"
"github.com/hashicorp/nomad/client/stats"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/mitchellh/mapstructure"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
cstructs "github.com/hashicorp/nomad/client/structs"
lxc "gopkg.in/lxc/go-lxc.v2"
)
const (
// lxcConfigOption is the key for enabling the LXC driver in the
// Config.Options map.
lxcConfigOption = "driver.lxc.enable"
// lxcVolumesConfigOption is the key for enabling the use of
// custom bind volumes to arbitrary host paths
lxcVolumesConfigOption = "lxc.volumes.enabled"
lxcVolumesConfigDefault = true
// containerMonitorIntv is the interval at which the driver checks if the
// container is still alive
containerMonitorIntv = 2 * time.Second
)
var (
LXCMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
LXCMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
)
// Add the lxc driver to the list of builtin drivers
func init() {
BuiltinDrivers["lxc"] = NewLxcDriver
}
// LxcDriver allows users to run LXC Containers
type LxcDriver struct {
DriverContext
fingerprint.StaticFingerprinter
}
// LxcDriverConfig is the configuration of the LXC Container
type LxcDriverConfig struct {
Template string
Distro string
Release string
Arch string
ImageVariant string `mapstructure:"image_variant"`
ImageServer string `mapstructure:"image_server"`
GPGKeyID string `mapstructure:"gpg_key_id"`
GPGKeyServer string `mapstructure:"gpg_key_server"`
DisableGPGValidation bool `mapstructure:"disable_gpg"`
FlushCache bool `mapstructure:"flush_cache"`
ForceCache bool `mapstructure:"force_cache"`
TemplateArgs []string `mapstructure:"template_args"`
LogLevel string `mapstructure:"log_level"`
Verbosity string
Volumes []string `mapstructure:"volumes"`
}
// NewLxcDriver returns a new instance of the LXC driver
func NewLxcDriver(ctx *DriverContext) Driver {
return &LxcDriver{DriverContext: *ctx}
}
// Validate validates the lxc driver configuration
func (d *LxcDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"template": {
Type: fields.TypeString,
Required: true,
},
"distro": {
Type: fields.TypeString,
Required: false,
},
"release": {
Type: fields.TypeString,
Required: false,
},
"arch": {
Type: fields.TypeString,
Required: false,
},
"image_variant": {
Type: fields.TypeString,
Required: false,
},
"image_server": {
Type: fields.TypeString,
Required: false,
},
"gpg_key_id": {
Type: fields.TypeString,
Required: false,
},
"gpg_key_server": {
Type: fields.TypeString,
Required: false,
},
"disable_gpg": {
Type: fields.TypeString,
Required: false,
},
"flush_cache": {
Type: fields.TypeString,
Required: false,
},
"force_cache": {
Type: fields.TypeString,
Required: false,
},
"template_args": {
Type: fields.TypeArray,
Required: false,
},
"log_level": {
Type: fields.TypeString,
Required: false,
},
"verbosity": {
Type: fields.TypeString,
Required: false,
},
"volumes": {
Type: fields.TypeArray,
Required: false,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
volumes, ok := fd.GetOk("volumes")
if ok {
for _, volDesc := range volumes.([]interface{}) {
volStr := volDesc.(string)
paths := strings.Split(volStr, ":")
if len(paths) != 2 {
return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr)
}
if len(paths[0]) == 0 || len(paths[1]) == 0 {
return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr)
}
if paths[1][0] == '/' {
return fmt.Errorf("unsupported absolute container mount point: '%s'", paths[1])
}
}
}
return nil
}
func (d *LxcDriver) Abilities() DriverAbilities {
return DriverAbilities{
SendSignals: false,
Exec: false,
}
}
func (d *LxcDriver) FSIsolation() cstructs.FSIsolation {
return cstructs.FSIsolationImage
}
// Fingerprint fingerprints the lxc driver configuration
func (d *LxcDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
cfg := req.Config
enabled := cfg.ReadBoolDefault(lxcConfigOption, true)
if !enabled && !cfg.DevMode {
return nil
}
version := lxc.Version()
if version == "" {
return nil
}
resp.AddAttribute("driver.lxc.version", version)
resp.AddAttribute("driver.lxc", "1")
resp.Detected = true
// Advertise if this node supports lxc volumes
if d.config.ReadBoolDefault(lxcVolumesConfigOption, lxcVolumesConfigDefault) {
resp.AddAttribute("driver."+lxcVolumesConfigOption, "1")
}
return nil
}
func (d *LxcDriver) Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error) {
return nil, nil
}
// Start starts the LXC Driver
func (d *LxcDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
sresp, err, errCleanup := d.startWithCleanup(ctx, task)
if err != nil {
if cleanupErr := errCleanup(); cleanupErr != nil {
d.logger.Printf("[ERR] error occurred while cleaning up from error in Start: %v", cleanupErr)
}
}
return sresp, err
}
func (d *LxcDriver) startWithCleanup(ctx *ExecContext, task *structs.Task) (*StartResponse, error, func() error) {
noCleanup := func() error { return nil }
var driverConfig LxcDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err, noCleanup
}
lxcPath := lxc.DefaultConfigPath()
if path := d.config.Read("driver.lxc.path"); path != "" {
lxcPath = path
}
containerName := fmt.Sprintf("%s-%s", task.Name, d.DriverContext.allocID)
c, err := lxc.NewContainer(containerName, lxcPath)
if err != nil {
return nil, fmt.Errorf("unable to initialize container: %v", err), noCleanup
}
var verbosity lxc.Verbosity
switch driverConfig.Verbosity {
case "verbose":
verbosity = lxc.Verbose
case "", "quiet":
verbosity = lxc.Quiet
default:
return nil, fmt.Errorf("lxc driver config 'verbosity' can only be either quiet or verbose"), noCleanup
}
c.SetVerbosity(verbosity)
var logLevel lxc.LogLevel
switch driverConfig.LogLevel {
case "trace":
logLevel = lxc.TRACE
case "debug":
logLevel = lxc.DEBUG
case "info":
logLevel = lxc.INFO
case "warn":
logLevel = lxc.WARN
case "", "error":
logLevel = lxc.ERROR
default:
return nil, fmt.Errorf("lxc driver config 'log_level' can only be trace, debug, info, warn or error"), noCleanup
}
c.SetLogLevel(logLevel)
logFile := filepath.Join(ctx.TaskDir.Dir, fmt.Sprintf("%v-lxc.log", task.Name))
c.SetLogFile(logFile)
options := lxc.TemplateOptions{
Template: driverConfig.Template,
Distro: driverConfig.Distro,
Release: driverConfig.Release,
Arch: driverConfig.Arch,
FlushCache: driverConfig.FlushCache,
DisableGPGValidation: driverConfig.DisableGPGValidation,
ExtraArgs: driverConfig.TemplateArgs,
}
if err := c.Create(options); err != nil {
return nil, fmt.Errorf("unable to create container: %v", err), noCleanup
}
// Set the network type to none
if err := c.SetConfigItem("lxc.network.type", "none"); err != nil {
return nil, fmt.Errorf("error setting network type configuration: %v", err), c.Destroy
}
// Bind mount the shared alloc dir and task local dir in the container
mounts := []string{
fmt.Sprintf("%s local none rw,bind,create=dir", ctx.TaskDir.LocalDir),
fmt.Sprintf("%s alloc none rw,bind,create=dir", ctx.TaskDir.SharedAllocDir),
fmt.Sprintf("%s secrets none rw,bind,create=dir", ctx.TaskDir.SecretsDir),
}
volumesEnabled := d.config.ReadBoolDefault(lxcVolumesConfigOption, lxcVolumesConfigDefault)
for _, volDesc := range driverConfig.Volumes {
// the format was checked in Validate()
paths := strings.Split(volDesc, ":")
if filepath.IsAbs(paths[0]) {
if !volumesEnabled {
return nil, fmt.Errorf("absolute bind-mount volume in config but '%v' is false", lxcVolumesConfigOption), c.Destroy
}
} else {
// Relative source paths are treated as relative to alloc dir
paths[0] = filepath.Join(ctx.TaskDir.Dir, paths[0])
}
mounts = append(mounts, fmt.Sprintf("%s %s none rw,bind,create=dir", paths[0], paths[1]))
}
for _, mnt := range mounts {
if err := c.SetConfigItem("lxc.mount.entry", mnt); err != nil {
return nil, fmt.Errorf("error setting bind mount %q error: %v", mnt, err), c.Destroy
}
}
// Start the container
if err := c.Start(); err != nil {
return nil, fmt.Errorf("unable to start container: %v", err), c.Destroy
}
stopAndDestroyCleanup := func() error {
if err := c.Stop(); err != nil {
return err
}
return c.Destroy()
}
// Set the resource limits
if err := c.SetMemoryLimit(lxc.ByteSize(task.Resources.MemoryMB) * lxc.MB); err != nil {
return nil, fmt.Errorf("unable to set memory limits: %v", err), stopAndDestroyCleanup
}
if err := c.SetCgroupItem("cpu.shares", strconv.Itoa(task.Resources.CPU)); err != nil {
return nil, fmt.Errorf("unable to set cpu shares: %v", err), stopAndDestroyCleanup
}
h := lxcDriverHandle{
container: c,
initPid: c.InitPid(),
lxcPath: lxcPath,
logger: d.logger,
killTimeout: GetKillTimeout(task.KillTimeout, d.DriverContext.config.MaxKillTimeout),
maxKillTimeout: d.DriverContext.config.MaxKillTimeout,
totalCpuStats: stats.NewCpuStats(),
userCpuStats: stats.NewCpuStats(),
systemCpuStats: stats.NewCpuStats(),
waitCh: make(chan *dstructs.WaitResult, 1),
doneCh: make(chan bool, 1),
}
go h.run()
return &StartResponse{Handle: &h}, nil, noCleanup
}
func (d *LxcDriver) Cleanup(*ExecContext, *CreatedResources) error { return nil }
// Open creates the driver to monitor an existing LXC container
func (d *LxcDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
pid := &lxcPID{}
if err := json.Unmarshal([]byte(handleID), pid); err != nil {
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
}
var container *lxc.Container
containers := lxc.Containers(pid.LxcPath)
for _, c := range containers {
if c.Name() == pid.ContainerName {
container = c
break
}
}
if container == nil {
return nil, fmt.Errorf("container %v not found", pid.ContainerName)
}
handle := lxcDriverHandle{
container: container,
initPid: container.InitPid(),
lxcPath: pid.LxcPath,
logger: d.logger,
killTimeout: pid.KillTimeout,
maxKillTimeout: d.DriverContext.config.MaxKillTimeout,
totalCpuStats: stats.NewCpuStats(),
userCpuStats: stats.NewCpuStats(),
systemCpuStats: stats.NewCpuStats(),
waitCh: make(chan *dstructs.WaitResult, 1),
doneCh: make(chan bool, 1),
}
go handle.run()
return &handle, nil
}
// lxcDriverHandle allows controlling the lifecycle of an lxc container
type lxcDriverHandle struct {
container *lxc.Container
initPid int
lxcPath string
logger *log.Logger
killTimeout time.Duration
maxKillTimeout time.Duration
totalCpuStats *stats.CpuStats
userCpuStats *stats.CpuStats
systemCpuStats *stats.CpuStats
waitCh chan *dstructs.WaitResult
doneCh chan bool
}
type lxcPID struct {
ContainerName string
InitPid int
LxcPath string
KillTimeout time.Duration
}
func (h *lxcDriverHandle) ID() string {
pid := lxcPID{
ContainerName: h.container.Name(),
InitPid: h.initPid,
LxcPath: h.lxcPath,
KillTimeout: h.killTimeout,
}
data, err := json.Marshal(pid)
if err != nil {
h.logger.Printf("[ERR] driver.lxc: failed to marshal lxc PID to JSON: %v", err)
}
return string(data)
}
func (h *lxcDriverHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *lxcDriverHandle) Update(task *structs.Task) error {
h.killTimeout = GetKillTimeout(task.KillTimeout, h.killTimeout)
return nil
}
func (h *lxcDriverHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
return nil, 0, fmt.Errorf("lxc driver cannot execute commands")
}
func (h *lxcDriverHandle) Kill() error {
name := h.container.Name()
h.logger.Printf("[INFO] driver.lxc: shutting down container %q", name)
if err := h.container.Shutdown(h.killTimeout); err != nil {
h.logger.Printf("[INFO] driver.lxc: shutting down container %q failed: %v", name, err)
if err := h.container.Stop(); err != nil {
h.logger.Printf("[ERR] driver.lxc: error stopping container %q: %v", name, err)
}
}
close(h.doneCh)
return nil
}
func (h *lxcDriverHandle) Signal(s os.Signal) error {
return fmt.Errorf("LXC does not support signals")
}
func (d *lxcDriverHandle) Network() *cstructs.DriverNetwork {
return nil
}
func (h *lxcDriverHandle) Stats() (*cstructs.TaskResourceUsage, error) {
cpuStats, err := h.container.CPUStats()
if err != nil {
return nil, nil
}
total, err := h.container.CPUTime()
if err != nil {
return nil, nil
}
t := time.Now()
// Get the cpu stats
system := cpuStats["system"]
user := cpuStats["user"]
cs := &cstructs.CpuStats{
SystemMode: h.systemCpuStats.Percent(float64(system)),
UserMode: h.systemCpuStats.Percent(float64(user)),
Percent: h.totalCpuStats.Percent(float64(total)),
TotalTicks: float64(user + system),
Measured: LXCMeasuredCpuStats,
}
// Get the Memory Stats
memData := map[string]uint64{
"rss": 0,
"cache": 0,
"swap": 0,
}
rawMemStats := h.container.CgroupItem("memory.stat")
for _, rawMemStat := range rawMemStats {
key, val, err := keysToVal(rawMemStat)
if err != nil {
h.logger.Printf("[ERR] driver.lxc: error getting stat for line %q", rawMemStat)
continue
}
if _, ok := memData[key]; ok {
memData[key] = val
}
}
ms := &cstructs.MemoryStats{
RSS: memData["rss"],
Cache: memData["cache"],
Swap: memData["swap"],
Measured: LXCMeasuredMemStats,
}
mu := h.container.CgroupItem("memory.max_usage_in_bytes")
for _, rawMemMaxUsage := range mu {
val, err := strconv.ParseUint(rawMemMaxUsage, 10, 64)
if err != nil {
h.logger.Printf("[ERR] driver.lxc: unable to get max memory usage: %v", err)
continue
}
ms.MaxUsage = val
}
ku := h.container.CgroupItem("memory.kmem.usage_in_bytes")
for _, rawKernelUsage := range ku {
val, err := strconv.ParseUint(rawKernelUsage, 10, 64)
if err != nil {
h.logger.Printf("[ERR] driver.lxc: unable to get kernel memory usage: %v", err)
continue
}
ms.KernelUsage = val
}
mku := h.container.CgroupItem("memory.kmem.max_usage_in_bytes")
for _, rawMaxKernelUsage := range mku {
val, err := strconv.ParseUint(rawMaxKernelUsage, 10, 64)
if err != nil {
h.logger.Printf("[ERR] driver.lxc: unable to get max kernel memory usage: %v", err)
continue
}
ms.KernelMaxUsage = val
}
taskResUsage := cstructs.TaskResourceUsage{
ResourceUsage: &cstructs.ResourceUsage{
CpuStats: cs,
MemoryStats: ms,
},
Timestamp: t.UTC().UnixNano(),
}
return &taskResUsage, nil
}
func (h *lxcDriverHandle) run() {
defer close(h.waitCh)
timer := time.NewTimer(containerMonitorIntv)
for {
select {
case <-timer.C:
process, err := os.FindProcess(h.initPid)
if err != nil {
h.waitCh <- &dstructs.WaitResult{Err: err}
return
}
if err := process.Signal(syscall.Signal(0)); err != nil {
h.waitCh <- &dstructs.WaitResult{}
return
}
timer.Reset(containerMonitorIntv)
case <-h.doneCh:
h.waitCh <- &dstructs.WaitResult{}
return
}
}
}
func keysToVal(line string) (string, uint64, error) {
tokens := strings.Split(line, " ")
if len(tokens) != 2 {
return "", 0, fmt.Errorf("line isn't a k/v pair")
}
key := tokens[0]
val, err := strconv.ParseUint(tokens[1], 10, 64)
return key, val, err
}

348
client/driver/lxc_test.go
View file

@ -1,348 +0,0 @@
//+build linux,lxc
package driver
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
"github.com/hashicorp/nomad/client/config"
cstructs "github.com/hashicorp/nomad/client/structs"
ctestutil "github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
lxc "gopkg.in/lxc/go-lxc.v2"
)
func TestLxcDriver_Fingerprint(t *testing.T) {
t.Parallel()
if !lxcPresent(t) {
t.Skip("lxc not present")
}
task := &structs.Task{
Name: "foo",
Driver: "lxc",
Resources: structs.DefaultResources(),
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewLxcDriver(ctx.DriverCtx)
node := &structs.Node{
Attributes: map[string]string{},
}
// test with an empty config
{
request := &cstructs.FingerprintRequest{Config: &config.Config{}, Node: node}
var response cstructs.FingerprintResponse
err := d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
}
// test when lxc is enable din the config
{
conf := &config.Config{Options: map[string]string{lxcConfigOption: "1"}}
request := &cstructs.FingerprintRequest{Config: conf, Node: node}
var response cstructs.FingerprintResponse
err := d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
if !response.Detected {
t.Fatalf("expected response to be applicable")
}
if response.Attributes["driver.lxc"] == "" {
t.Fatalf("missing driver")
}
}
}
func TestLxcDriver_Start_Wait(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !lxcPresent(t) {
t.Skip("lxc not present")
}
ctestutil.RequireRoot(t)
task := &structs.Task{
Name: "foo",
Driver: "lxc",
Config: map[string]interface{}{
"template": "/usr/share/lxc/templates/lxc-busybox",
"volumes": []string{"/tmp/:mnt/tmp"},
},
KillTimeout: 10 * time.Second,
Resources: structs.DefaultResources(),
}
testFileContents := []byte("this should be visible under /mnt/tmp")
tmpFile, err := ioutil.TempFile("/tmp", "testlxcdriver_start_wait")
if err != nil {
t.Fatalf("error writing temp file: %v", err)
}
defer os.Remove(tmpFile.Name())
if _, err := tmpFile.Write(testFileContents); err != nil {
t.Fatalf("error writing temp file: %v", err)
}
if err := tmpFile.Close(); err != nil {
t.Fatalf("error closing temp file: %v", err)
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewLxcDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
sresp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
lxcHandle, _ := sresp.Handle.(*lxcDriverHandle)
// Destroy the container after the test
defer func() {
lxcHandle.container.Stop()
lxcHandle.container.Destroy()
}()
testutil.WaitForResult(func() (bool, error) {
state := lxcHandle.container.State()
if state == lxc.RUNNING {
return true, nil
}
return false, fmt.Errorf("container in state: %v", state)
}, func(err error) {
t.Fatalf("err: %v", err)
})
// Look for mounted directories in their proper location
containerName := fmt.Sprintf("%s-%s", task.Name, ctx.DriverCtx.allocID)
for _, mnt := range []string{"alloc", "local", "secrets", "mnt/tmp"} {
fullpath := filepath.Join(lxcHandle.lxcPath, containerName, "rootfs", mnt)
stat, err := os.Stat(fullpath)
if err != nil {
t.Fatalf("err %v", err)
}
if !stat.IsDir() {
t.Fatalf("expected %q to be a dir", fullpath)
}
}
// Test that /mnt/tmp/$tempFile exists in the container:
mountedContents, err := exec.Command("lxc-attach", "-n", containerName, "--", "cat", filepath.Join("/mnt/", tmpFile.Name())).Output()
if err != nil {
t.Fatalf("err reading temp file in bind mount: %v", err)
}
if !bytes.Equal(mountedContents, testFileContents) {
t.Fatalf("contents of temp bind mounted file did not match, was '%s'", mountedContents)
}
// Destroy the container
if err := sresp.Handle.Kill(); err != nil {
t.Fatalf("err: %v", err)
}
select {
case res := <-sresp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
}
func TestLxcDriver_Open_Wait(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !lxcPresent(t) {
t.Skip("lxc not present")
}
ctestutil.RequireRoot(t)
task := &structs.Task{
Name: "foo",
Driver: "lxc",
Config: map[string]interface{}{
"template": "/usr/share/lxc/templates/lxc-busybox",
},
KillTimeout: 10 * time.Second,
Resources: structs.DefaultResources(),
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewLxcDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
sresp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Destroy the container after the test
lh := sresp.Handle.(*lxcDriverHandle)
defer func() {
lh.container.Stop()
lh.container.Destroy()
}()
handle2, err := d.Open(ctx.ExecCtx, lh.ID())
if err != nil {
t.Fatalf("err: %v", err)
}
if handle2 == nil {
t.Fatalf("missing handle on open")
}
lxcHandle, _ := handle2.(*lxcDriverHandle)
testutil.WaitForResult(func() (bool, error) {
state := lxcHandle.container.State()
if state == lxc.RUNNING {
return true, nil
}
return false, fmt.Errorf("container in state: %v", state)
}, func(err error) {
t.Fatalf("err: %v", err)
})
// Destroy the container
if err := handle2.Kill(); err != nil {
t.Fatalf("err: %v", err)
}
}
func lxcPresent(t *testing.T) bool {
return lxc.Version() != ""
}
func TestLxcDriver_Volumes_ConfigValidation(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !lxcPresent(t) {
t.Skip("lxc not present")
}
ctestutil.RequireRoot(t)
brokenVolumeConfigs := [][]string{
{
"foo:/var",
},
{
":",
},
{
"abc:",
},
{
":def",
},
{
"abc:def:ghi",
},
}
for _, bc := range brokenVolumeConfigs {
if err := testVolumeConfig(t, bc); err == nil {
t.Fatalf("error expected in validate for config %+v", bc)
}
}
if err := testVolumeConfig(t, []string{"abc:def"}); err != nil {
t.Fatalf("error in validate for syntactically valid config abc:def")
}
}
func testVolumeConfig(t *testing.T, volConfig []string) error {
task := &structs.Task{
Name: "voltest",
Driver: "lxc",
KillTimeout: 10 * time.Second,
Resources: structs.DefaultResources(),
Config: map[string]interface{}{
"template": "busybox",
},
}
task.Config["volumes"] = volConfig
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
driver := NewLxcDriver(ctx.DriverCtx)
err := driver.Validate(task.Config)
return err
}
func TestLxcDriver_Start_NoVolumes(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
if !lxcPresent(t) {
t.Skip("lxc not present")
}
ctestutil.RequireRoot(t)
task := &structs.Task{
Name: "foo",
Driver: "lxc",
Config: map[string]interface{}{
"template": "/usr/share/lxc/templates/lxc-busybox",
"volumes": []string{"/tmp/:mnt/tmp"},
},
KillTimeout: 10 * time.Second,
Resources: structs.DefaultResources(),
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
// set lxcVolumesConfigOption to false to disallow absolute paths as the source for the bind mount
ctx.DriverCtx.config.Options = map[string]string{lxcVolumesConfigOption: "false"}
d := NewLxcDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
// expect the "absolute bind-mount volume in config.. " error
_, err := d.Start(ctx.ExecCtx, task)
if err == nil {
t.Fatalf("expected error in start, got nil.")
}
// Because the container was created but not started before
// the expected error, we can test that the destroy-only
// cleanup is done here.
containerName := fmt.Sprintf("%s-%s", task.Name, ctx.DriverCtx.allocID)
if err := exec.Command("bash", "-c", fmt.Sprintf("lxc-ls -1 | grep -q %s", containerName)).Run(); err == nil {
t.Fatalf("error, container '%s' is still around", containerName)
}
}

484
client/driver/mock_driver.go
View file

@ -1,484 +0,0 @@
package driver
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"os"
"strconv"
"strings"
"time"
"github.com/mitchellh/mapstructure"
hclog "github.com/hashicorp/go-hclog"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/client/logmon/logging"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/nomad/structs"
)
const (
// shutdownPeriodicAfter is a config key that can be used during tests to
// "stop" a previously-functioning driver, allowing for testing of periodic
// drivers and fingerprinters
shutdownPeriodicAfter = "test.shutdown_periodic_after"
// shutdownPeriodicDuration is a config option that can be used during tests
// to "stop" a previously functioning driver after the specified duration
// (specified in seconds) for testing of periodic drivers and fingerprinters.
shutdownPeriodicDuration = "test.shutdown_periodic_duration"
mockDriverName = "driver.mock_driver"
)
// MockDriverConfig is the driver configuration for the MockDriver
type MockDriverConfig struct {
// StartErr specifies the error that should be returned when starting the
// mock driver.
StartErr string `mapstructure:"start_error"`
// StartErrRecoverable marks the error returned is recoverable
StartErrRecoverable bool `mapstructure:"start_error_recoverable"`
// StartBlockFor specifies a duration in which to block Start before
// returning. Useful for testing the behavior of tasks in pending.
StartBlockFor time.Duration `mapstructure:"start_block_for"`
// KillAfter is the duration after which the mock driver indicates the task
// has exited after getting the initial SIGINT signal
KillAfter time.Duration `mapstructure:"kill_after"`
// RunFor is the duration for which the fake task runs for. After this
// period the MockDriver responds to the task running indicating that the
// task has terminated
RunFor time.Duration `mapstructure:"run_for"`
// ExitCode is the exit code with which the MockDriver indicates the task
// has exited
ExitCode int `mapstructure:"exit_code"`
// ExitSignal is the signal with which the MockDriver indicates the task has
// been killed
ExitSignal int `mapstructure:"exit_signal"`
// ExitErrMsg is the error message that the task returns while exiting
ExitErrMsg string `mapstructure:"exit_err_msg"`
// SignalErr is the error message that the task returns if signalled
SignalErr string `mapstructure:"signal_error"`
// DriverIP will be returned as the DriverNetwork.IP from Start()
DriverIP string `mapstructure:"driver_ip"`
// DriverAdvertise will be returned as DriverNetwork.AutoAdvertise from
// Start().
DriverAdvertise bool `mapstructure:"driver_advertise"`
// DriverPortMap will parse a label:number pair and return it in
// DriverNetwork.PortMap from Start().
DriverPortMap string `mapstructure:"driver_port_map"`
// StdoutString is the string that should be sent to stdout
StdoutString string `mapstructure:"stdout_string"`
// StdoutRepeat is the number of times the output should be sent.
StdoutRepeat int `mapstructure:"stdout_repeat"`
// StdoutRepeatDur is the duration between repeated outputs.
StdoutRepeatDur time.Duration `mapstructure:"stdout_repeat_duration"`
}
// MockDriver is a driver which is used for testing purposes
type MockDriver struct {
DriverContext
cleanupFailNum int
// shutdownFingerprintTime is the time up to which the driver will be up
shutdownFingerprintTime time.Time
}
// NewMockDriver is a factory method which returns a new Mock Driver
func NewMockDriver(ctx *DriverContext) Driver {
md := &MockDriver{DriverContext: *ctx}
// if the shutdown configuration options are set, start the timer here.
// This config option defaults to false
if ctx.config != nil && ctx.config.ReadBoolDefault(shutdownPeriodicAfter, false) {
duration, err := ctx.config.ReadInt(shutdownPeriodicDuration)
if err != nil {
errMsg := fmt.Sprintf("unable to read config option for shutdown_periodic_duration %v, got err %s", duration, err.Error())
panic(errMsg)
}
md.shutdownFingerprintTime = time.Now().Add(time.Second * time.Duration(duration))
}
return md
}
func (d *MockDriver) Abilities() DriverAbilities {
return DriverAbilities{
SendSignals: false,
Exec: true,
}
}
func (d *MockDriver) FSIsolation() cstructs.FSIsolation {
return cstructs.FSIsolationNone
}
func (d *MockDriver) Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error) {
return nil, nil
}
// Start starts the mock driver
func (m *MockDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
var driverConfig MockDriverConfig
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
WeaklyTypedInput: true,
Result: &driverConfig,
})
if err != nil {
return nil, err
}
if err := dec.Decode(task.Config); err != nil {
return nil, err
}
if driverConfig.StartBlockFor != 0 {
time.Sleep(driverConfig.StartBlockFor)
}
if driverConfig.StartErr != "" {
return nil, structs.NewRecoverableError(errors.New(driverConfig.StartErr), driverConfig.StartErrRecoverable)
}
// Create the driver network
net := &cstructs.DriverNetwork{
IP: driverConfig.DriverIP,
AutoAdvertise: driverConfig.DriverAdvertise,
}
if raw := driverConfig.DriverPortMap; len(raw) > 0 {
parts := strings.Split(raw, ":")
if len(parts) != 2 {
return nil, fmt.Errorf("malformed port map: %q", raw)
}
port, err := strconv.Atoi(parts[1])
if err != nil {
return nil, fmt.Errorf("malformed port map: %q -- error: %v", raw, err)
}
net.PortMap = map[string]int{parts[0]: port}
}
h := mockDriverHandle{
ctx: ctx,
task: task,
taskName: task.Name,
runFor: driverConfig.RunFor,
killAfter: driverConfig.KillAfter,
killTimeout: task.KillTimeout,
exitCode: driverConfig.ExitCode,
exitSignal: driverConfig.ExitSignal,
stdoutString: driverConfig.StdoutString,
stdoutRepeat: driverConfig.StdoutRepeat,
stdoutRepeatDur: driverConfig.StdoutRepeatDur,
logger: m.logger,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if driverConfig.ExitErrMsg != "" {
h.exitErr = errors.New(driverConfig.ExitErrMsg)
}
if driverConfig.SignalErr != "" {
h.signalErr = fmt.Errorf(driverConfig.SignalErr)
}
m.logger.Printf("[DEBUG] driver.mock: starting task %q", task.Name)
go h.run()
return &StartResponse{Handle: &h, Network: net}, nil
}
// Cleanup deletes all keys except for Config.Options["cleanup_fail_on"] for
// Config.Options["cleanup_fail_num"] times. For failures it will return a
// recoverable error.
func (m *MockDriver) Cleanup(ctx *ExecContext, res *CreatedResources) error {
if res == nil {
panic("Cleanup should not be called with nil *CreatedResources")
}
var err error
failn, _ := strconv.Atoi(m.config.Options["cleanup_fail_num"])
failk := m.config.Options["cleanup_fail_on"]
for k := range res.Resources {
if k == failk && m.cleanupFailNum < failn {
m.cleanupFailNum++
err = structs.NewRecoverableError(fmt.Errorf("mock_driver failure on %q call %d/%d", k, m.cleanupFailNum, failn), true)
} else {
delete(res.Resources, k)
}
}
return err
}
// Validate validates the mock driver configuration
func (m *MockDriver) Validate(map[string]interface{}) error {
return nil
}
// Fingerprint fingerprints a node and returns if MockDriver is enabled
func (m *MockDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
switch {
// If the driver is configured to shut down after a period of time, and the
// current time is after the time which the node should shut down, simulate
// driver failure
case !m.shutdownFingerprintTime.IsZero() && time.Now().After(m.shutdownFingerprintTime):
resp.RemoveAttribute(mockDriverName)
default:
resp.AddAttribute(mockDriverName, "1")
resp.Detected = true
}
return nil
}
// When testing, poll for updates
func (m *MockDriver) Periodic() (bool, time.Duration) {
return true, 500 * time.Millisecond
}
// HealthCheck implements the interface for HealthCheck, and indicates the current
// health status of the mock driver.
func (m *MockDriver) HealthCheck(req *cstructs.HealthCheckRequest, resp *cstructs.HealthCheckResponse) error {
switch {
case !m.shutdownFingerprintTime.IsZero() && time.Now().After(m.shutdownFingerprintTime):
notHealthy := &structs.DriverInfo{
Healthy: false,
HealthDescription: "not running",
UpdateTime: time.Now(),
}
resp.AddDriverInfo("mock_driver", notHealthy)
return nil
default:
healthy := &structs.DriverInfo{
Healthy: true,
HealthDescription: "running",
UpdateTime: time.Now(),
}
resp.AddDriverInfo("mock_driver", healthy)
return nil
}
}
// GetHealthCheckInterval implements the interface for HealthCheck and indicates
// that mock driver should be checked periodically. Returns a boolean
// indicating if it should be checked, and the duration at which to do this
// check.
func (m *MockDriver) GetHealthCheckInterval(req *cstructs.HealthCheckIntervalRequest, resp *cstructs.HealthCheckIntervalResponse) error {
resp.Eligible = true
resp.Period = 1 * time.Second
return nil
}
// MockDriverHandle is a driver handler which supervises a mock task
type mockDriverHandle struct {
ctx *ExecContext
task *structs.Task
taskName string
runFor time.Duration
killAfter time.Duration
killTimeout time.Duration
exitCode int
exitSignal int
exitErr error
signalErr error
logger *log.Logger
stdoutString string
stdoutRepeat int
stdoutRepeatDur time.Duration
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
}
type mockDriverID struct {
TaskName string
RunFor time.Duration
KillAfter time.Duration
KillTimeout time.Duration
ExitCode int
ExitSignal int
ExitErr error
SignalErr error
}
func (h *mockDriverHandle) ID() string {
id := mockDriverID{
TaskName: h.taskName,
RunFor: h.runFor,
KillAfter: h.killAfter,
KillTimeout: h.killTimeout,
ExitCode: h.exitCode,
ExitSignal: h.exitSignal,
ExitErr: h.exitErr,
SignalErr: h.signalErr,
}
data, err := json.Marshal(id)
if err != nil {
h.logger.Printf("[ERR] driver.mock_driver: failed to marshal ID to JSON: %s", err)
}
return string(data)
}
// Open re-connects the driver to the running task
func (m *MockDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
id := &mockDriverID{}
if err := json.Unmarshal([]byte(handleID), id); err != nil {
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
}
h := mockDriverHandle{
taskName: id.TaskName,
runFor: id.RunFor,
killAfter: id.KillAfter,
killTimeout: id.KillTimeout,
exitCode: id.ExitCode,
exitSignal: id.ExitSignal,
exitErr: id.ExitErr,
signalErr: id.SignalErr,
logger: m.logger,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
go h.run()
return &h, nil
}
func (h *mockDriverHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *mockDriverHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
h.logger.Printf("[DEBUG] driver.mock: Exec(%q, %q)", cmd, args)
return []byte(fmt.Sprintf("Exec(%q, %q)", cmd, args)), 0, nil
}
// TODO Implement when we need it.
func (h *mockDriverHandle) Update(task *structs.Task) error {
h.killTimeout = task.KillTimeout
return nil
}
// TODO Implement when we need it.
func (d *mockDriverHandle) Network() *cstructs.DriverNetwork {
return nil
}
// TODO Implement when we need it.
func (h *mockDriverHandle) Signal(s os.Signal) error {
return h.signalErr
}
// Kill kills a mock task
func (h *mockDriverHandle) Kill() error {
h.logger.Printf("[DEBUG] driver.mock: killing task %q after %s or kill timeout: %v", h.taskName, h.killAfter, h.killTimeout)
select {
case <-h.doneCh:
case <-time.After(h.killAfter):
select {
case <-h.doneCh:
// already closed
default:
close(h.doneCh)
}
case <-time.After(h.killTimeout):
h.logger.Printf("[DEBUG] driver.mock: terminating task %q", h.taskName)
select {
case <-h.doneCh:
// already closed
default:
close(h.doneCh)
}
}
return nil
}
// TODO Implement when we need it.
func (h *mockDriverHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return nil, nil
}
// run waits for the configured amount of time and then indicates the task has
// terminated
func (h *mockDriverHandle) run() {
defer close(h.waitCh)
// Setup logging output
if h.stdoutString != "" {
go h.handleLogging()
}
timer := time.NewTimer(h.runFor)
defer timer.Stop()
for {
select {
case <-timer.C:
select {
case <-h.doneCh:
// already closed
default:
close(h.doneCh)
}
case <-h.doneCh:
h.logger.Printf("[DEBUG] driver.mock: finished running task %q", h.taskName)
h.waitCh <- dstructs.NewWaitResult(h.exitCode, h.exitSignal, h.exitErr)
return
}
}
}
// handleLogging handles logging stdout messages
func (h *mockDriverHandle) handleLogging() {
if h.stdoutString == "" {
return
}
// Setup a log rotator
logFileSize := int64(h.task.LogConfig.MaxFileSizeMB * 1024 * 1024)
lro, err := logging.NewFileRotator(h.ctx.TaskDir.LogDir, fmt.Sprintf("%v.stdout", h.taskName),
h.task.LogConfig.MaxFiles, logFileSize, hclog.Default()) //TODO: plumb hclog
if err != nil {
h.exitErr = err
close(h.doneCh)
h.logger.Printf("[ERR] mock_driver: failed to setup file rotator: %v", err)
return
}
defer lro.Close()
// Do initial write to stdout.
if _, err := io.WriteString(lro, h.stdoutString); err != nil {
h.exitErr = err
close(h.doneCh)
h.logger.Printf("[ERR] mock_driver: failed to write to stdout: %v", err)
return
}
for i := 0; i < h.stdoutRepeat; i++ {
select {
case <-h.doneCh:
return
case <-time.After(h.stdoutRepeatDur):
if _, err := io.WriteString(lro, h.stdoutString); err != nil {
h.exitErr = err
close(h.doneCh)
h.logger.Printf("[ERR] mock_driver: failed to write to stdout: %v", err)
return
}
}
}
}

8
client/driver/mock_driver_testing.go
View file

@ -1,8 +0,0 @@
// +build !release
package driver
// Add the mock driver to the list of builtin drivers
func init() {
BuiltinDrivers["mock_driver"] = NewMockDriver
}

543
client/driver/qemu.go
View file

@ -1,543 +0,0 @@
package driver
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"time"
"github.com/coreos/go-semver/semver"
plugin "github.com/hashicorp/go-plugin"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/client/fingerprint"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
pexecutor "github.com/hashicorp/nomad/plugins/executor"
"github.com/mitchellh/mapstructure"
)
var (
reQemuVersion = regexp.MustCompile(`version (\d[\.\d+]+)`)
// Prior to qemu 2.10.1, monitor socket paths are truncated to 108 bytes.
// We should consider this if driver.qemu.version is < 2.10.1 and the
// generated monitor path is too long.
//
// Relevant fix is here:
// https://github.com/qemu/qemu/commit/ad9579aaa16d5b385922d49edac2c96c79bcfb6
qemuVersionLongSocketPathFix = semver.New("2.10.1")
)
const (
// The key populated in Node Attributes to indicate presence of the Qemu driver
qemuDriverAttr = "driver.qemu"
qemuDriverVersionAttr = "driver.qemu.version"
// Represents an ACPI shutdown request to the VM (emulates pressing a physical power button)
// Reference: https://en.wikibooks.org/wiki/QEMU/Monitor
qemuGracefulShutdownMsg = "system_powerdown\n"
qemuMonitorSocketName = "qemu-monitor.sock"
// Maximum socket path length prior to qemu 2.10.1
qemuLegacyMaxMonitorPathLen = 108
)
// QemuDriver is a driver for running images via Qemu
// We attempt to chose sane defaults for now, with more configuration available
// planned in the future
type QemuDriver struct {
DriverContext
fingerprint.StaticFingerprinter
driverConfig *QemuDriverConfig
}
type QemuDriverConfig struct {
ImagePath string `mapstructure:"image_path"`
Accelerator string `mapstructure:"accelerator"`
GracefulShutdown bool `mapstructure:"graceful_shutdown"`
PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports.
Args []string `mapstructure:"args"` // extra arguments to qemu executable
}
// qemuHandle is returned from Start/Open as a handle to the PID
type qemuHandle struct {
pluginClient *plugin.Client
userPid int
executor executor.Executor
monitorPath string
shutdownSignal string
killTimeout time.Duration
maxKillTimeout time.Duration
logger *log.Logger
version string
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
}
// getMonitorPath is used to determine whether a qemu monitor socket can be
// safely created and accessed in the task directory by the version of qemu
// present on the host. If it is safe to use, the socket's full path is
// returned along with a nil error. Otherwise, an empty string is returned
// along with a descriptive error.
func (d *QemuDriver) getMonitorPath(dir string) (string, error) {
var longPathSupport bool
currentQemuVer := d.DriverContext.node.Attributes[qemuDriverVersionAttr]
currentQemuSemver := semver.New(currentQemuVer)
if currentQemuSemver.LessThan(*qemuVersionLongSocketPathFix) {
longPathSupport = false
d.logger.Printf("[DEBUG] driver.qemu: long socket paths are not available in this version of QEMU (%s)", currentQemuVer)
} else {
longPathSupport = true
d.logger.Printf("[DEBUG] driver.qemu: long socket paths available in this version of QEMU (%s)", currentQemuVer)
}
fullSocketPath := fmt.Sprintf("%s/%s", dir, qemuMonitorSocketName)
if len(fullSocketPath) > qemuLegacyMaxMonitorPathLen && longPathSupport == false {
return "", fmt.Errorf("monitor path is too long for this version of qemu")
}
return fullSocketPath, nil
}
// NewQemuDriver is used to create a new exec driver
func NewQemuDriver(ctx *DriverContext) Driver {
return &QemuDriver{DriverContext: *ctx}
}
// Validate is used to validate the driver configuration
func (d *QemuDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"image_path": {
Type: fields.TypeString,
Required: true,
},
"accelerator": {
Type: fields.TypeString,
},
"graceful_shutdown": {
Type: fields.TypeBool,
Required: false,
},
"port_map": {
Type: fields.TypeArray,
},
"args": {
Type: fields.TypeArray,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *QemuDriver) Abilities() DriverAbilities {
return DriverAbilities{
SendSignals: false,
Exec: false,
}
}
func (d *QemuDriver) FSIsolation() cstructs.FSIsolation {
return cstructs.FSIsolationImage
}
func (d *QemuDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
bin := "qemu-system-x86_64"
if runtime.GOOS == "windows" {
// On windows, the "qemu-system-x86_64" command does not respond to the
// version flag.
bin = "qemu-img"
}
outBytes, err := exec.Command(bin, "--version").Output()
if err != nil {
// return no error, as it isn't an error to not find qemu, it just means we
// can't use it.
return nil
}
out := strings.TrimSpace(string(outBytes))
matches := reQemuVersion.FindStringSubmatch(out)
if len(matches) != 2 {
resp.RemoveAttribute(qemuDriverAttr)
return fmt.Errorf("Unable to parse Qemu version string: %#v", matches)
}
currentQemuVersion := matches[1]
resp.AddAttribute(qemuDriverAttr, "1")
resp.AddAttribute(qemuDriverVersionAttr, currentQemuVersion)
resp.Detected = true
return nil
}
func (d *QemuDriver) Prestart(_ *ExecContext, task *structs.Task) (*PrestartResponse, error) {
var driverConfig QemuDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
if len(driverConfig.PortMap) > 1 {
return nil, fmt.Errorf("Only one port_map block is allowed in the qemu driver config")
}
d.driverConfig = &driverConfig
r := NewPrestartResponse()
if len(driverConfig.PortMap) == 1 {
r.Network = &cstructs.DriverNetwork{
PortMap: driverConfig.PortMap[0],
}
}
return r, nil
}
// Run an existing Qemu image. Start() will pull down an existing, valid Qemu
// image and save it to the Drivers Allocation Dir
func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
// Get the image source
vmPath := d.driverConfig.ImagePath
if vmPath == "" {
return nil, fmt.Errorf("image_path must be set")
}
vmID := filepath.Base(vmPath)
// Parse configuration arguments
// Create the base arguments
accelerator := "tcg"
if d.driverConfig.Accelerator != "" {
accelerator = d.driverConfig.Accelerator
}
if task.Resources.MemoryMB < 128 || task.Resources.MemoryMB > 4000000 {
return nil, fmt.Errorf("Qemu memory assignment out of bounds")
}
mem := fmt.Sprintf("%dM", task.Resources.MemoryMB)
absPath, err := GetAbsolutePath("qemu-system-x86_64")
if err != nil {
return nil, err
}
args := []string{
absPath,
"-machine", "type=pc,accel=" + accelerator,
"-name", vmID,
"-m", mem,
"-drive", "file=" + vmPath,
"-nographic",
}
var monitorPath string
if d.driverConfig.GracefulShutdown {
if runtime.GOOS == "windows" {
return nil, errors.New("QEMU graceful shutdown is unsupported on the Windows platform")
}
// This socket will be used to manage the virtual machine (for example,
// to perform graceful shutdowns)
monitorPath, err = d.getMonitorPath(ctx.TaskDir.Dir)
if err != nil {
d.logger.Printf("[ERR] driver.qemu: could not get qemu monitor path: %s", err)
return nil, err
}
d.logger.Printf("[DEBUG] driver.qemu: got monitor path OK: %s", monitorPath)
args = append(args, "-monitor", fmt.Sprintf("unix:%s,server,nowait", monitorPath))
}
// Add pass through arguments to qemu executable. A user can specify
// these arguments in driver task configuration. These arguments are
// passed directly to the qemu driver as command line options.
// For example, args = [ "-nodefconfig", "-nodefaults" ]
// This will allow a VM with embedded configuration to boot successfully.
args = append(args, d.driverConfig.Args...)
// Check the Resources required Networks to add port mappings. If no resources
// are required, we assume the VM is a purely compute job and does not require
// the outside world to be able to reach it. VMs ran without port mappings can
// still reach out to the world, but without port mappings it is effectively
// firewalled
protocols := []string{"udp", "tcp"}
if len(task.Resources.Networks) > 0 && len(d.driverConfig.PortMap) == 1 {
// Loop through the port map and construct the hostfwd string, to map
// reserved ports to the ports listenting in the VM
// Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080
var forwarding []string
taskPorts := task.Resources.Networks[0].PortLabels()
for label, guest := range d.driverConfig.PortMap[0] {
host, ok := taskPorts[label]
if !ok {
return nil, fmt.Errorf("Unknown port label %q", label)
}
for _, p := range protocols {
forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest))
}
}
if len(forwarding) != 0 {
args = append(args,
"-netdev",
fmt.Sprintf("user,id=user.0,%s", strings.Join(forwarding, ",")),
"-device", "virtio-net,netdev=user.0",
)
}
}
// If using KVM, add optimization args
if accelerator == "kvm" {
if runtime.GOOS == "windows" {
return nil, errors.New("KVM accelerator is unsupported on the Windows platform")
}
args = append(args,
"-enable-kvm",
"-cpu", "host",
// Do we have cores information available to the Driver?
// "-smp", fmt.Sprintf("%d", cores),
)
}
d.logger.Printf("[DEBUG] driver.qemu: starting QemuVM command: %q", strings.Join(args, " "))
pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out")
executorConfig := &pexecutor.ExecutorConfig{
LogFile: pluginLogFile,
LogLevel: d.config.LogLevel,
}
exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig)
if err != nil {
return nil, err
}
_, err = getTaskKillSignal(task.KillSignal)
if err != nil {
return nil, err
}
execCmd := &executor.ExecCommand{
Cmd: args[0],
Args: args[1:],
User: task.User,
TaskDir: ctx.TaskDir.Dir,
Env: ctx.TaskEnv.List(),
StdoutPath: ctx.StdoutFifo,
StderrPath: ctx.StderrFifo,
}
ps, err := exec.Launch(execCmd)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[INFO] driver.qemu: started new QemuVM: %s", vmID)
// Create and Return Handle
maxKill := d.DriverContext.config.MaxKillTimeout
h := &qemuHandle{
pluginClient: pluginClient,
executor: exec,
userPid: ps.Pid,
shutdownSignal: task.KillSignal,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
monitorPath: monitorPath,
version: d.config.Version.VersionNumber(),
logger: d.logger,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
go h.run()
resp := &StartResponse{Handle: h}
if len(d.driverConfig.PortMap) == 1 {
resp.Network = &cstructs.DriverNetwork{
PortMap: d.driverConfig.PortMap[0],
}
}
return resp, nil
}
type qemuId struct {
Version string
KillTimeout time.Duration
MaxKillTimeout time.Duration
UserPid int
PluginConfig *pexecutor.PluginReattachConfig
ShutdownSignal string
}
func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
id := &qemuId{}
if err := json.Unmarshal([]byte(handleID), id); err != nil {
return nil, fmt.Errorf("Failed to parse handle %q: %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput)
if err != nil {
d.logger.Printf("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid %d and user pid %d", id.PluginConfig.Pid, id.UserPid)
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
d.logger.Printf("[ERR] driver.qemu: error destroying plugin pid %d and userpid %d: %v", id.PluginConfig.Pid, id.UserPid, e)
}
return nil, fmt.Errorf("error connecting to plugin: %v", err)
}
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.qemu: version of executor: %v", ver.Version)
// Return a driver handle
h := &qemuHandle{
pluginClient: pluginClient,
executor: exec,
userPid: id.UserPid,
logger: d.logger,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
shutdownSignal: id.ShutdownSignal,
version: id.Version,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
go h.run()
return h, nil
}
func (d *QemuDriver) Cleanup(*ExecContext, *CreatedResources) error { return nil }
func (h *qemuHandle) ID() string {
id := qemuId{
Version: h.version,
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
PluginConfig: pexecutor.NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
UserPid: h.userPid,
ShutdownSignal: h.shutdownSignal,
}
data, err := json.Marshal(id)
if err != nil {
h.logger.Printf("[ERR] driver.qemu: failed to marshal ID to JSON: %s", err)
}
return string(data)
}
func (h *qemuHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *qemuHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
// Update is not possible
return nil
}
func (h *qemuHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
return nil, 0, fmt.Errorf("Qemu driver can't execute commands")
}
func (h *qemuHandle) Signal(s os.Signal) error {
return fmt.Errorf("Qemu driver can't send signals")
}
func (d *qemuHandle) Network() *cstructs.DriverNetwork {
return nil
}
func (h *qemuHandle) Kill() error {
gracefulShutdownSent := false
// Attempt a graceful shutdown only if it was configured in the job
if h.monitorPath != "" {
if err := sendQemuShutdown(h.logger, h.monitorPath, h.userPid); err == nil {
gracefulShutdownSent = true
} else {
h.logger.Printf("[DEBUG] driver.qemu: error sending graceful shutdown for user process pid %d: %s", h.userPid, err)
}
}
// If Nomad did not send a graceful shutdown signal, issue an interrupt to
// the qemu process as a last resort
if gracefulShutdownSent == false {
h.logger.Printf("[DEBUG] driver.qemu: graceful shutdown is not enabled, sending an interrupt signal to pid: %d", h.userPid)
if err := h.executor.Signal(os.Interrupt); err != nil {
if h.pluginClient.Exited() {
return nil
}
return fmt.Errorf("executor Shutdown failed: %v", err)
}
}
// If the qemu process exits before the kill timeout is reached, doneChan
// will close and we'll exit without an error. If it takes too long, the
// timer will fire and we'll attempt to kill the process.
select {
case <-h.doneCh:
return nil
case <-time.After(h.killTimeout):
h.logger.Printf("[DEBUG] driver.qemu: kill timeout of %s exceeded for user process pid %d", h.killTimeout.String(), h.userPid)
if h.pluginClient.Exited() {
return nil
}
if err := h.executor.Shutdown(h.shutdownSignal, h.killTimeout); err != nil {
return fmt.Errorf("executor Destroy failed: %v", err)
}
return nil
}
}
func (h *qemuHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return h.executor.Stats()
}
func (h *qemuHandle) run() {
ps, werr := h.executor.Wait()
if ps.ExitCode == 0 && werr != nil {
if e := killProcess(h.userPid); e != nil {
h.logger.Printf("[ERR] driver.qemu: error killing user process pid %d: %v", h.userPid, e)
}
}
close(h.doneCh)
// Destroy the executor
h.executor.Shutdown(h.shutdownSignal, 0)
h.pluginClient.Kill()
// Send the results
h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: werr}
close(h.waitCh)
}
// sendQemuShutdown attempts to issue an ACPI power-off command via the qemu
// monitor
func sendQemuShutdown(logger *log.Logger, monitorPath string, userPid int) error {
if monitorPath == "" {
return errors.New("monitorPath not set")
}
monitorSocket, err := net.Dial("unix", monitorPath)
if err != nil {
logger.Printf("[WARN] driver.qemu: could not connect to qemu monitor %q for user process pid %d: %s", monitorPath, userPid, err)
return err
}
defer monitorSocket.Close()
logger.Printf("[DEBUG] driver.qemu: sending graceful shutdown command to qemu monitor socket %q for user process pid %d", monitorPath, userPid)
_, err = monitorSocket.Write([]byte(qemuGracefulShutdownMsg))
if err != nil {
logger.Printf("[WARN] driver.qemu: failed to send shutdown message %q to monitor socket %q for user process pid %d: %s", qemuGracefulShutdownMsg, monitorPath, userPid, err)
}
return err
}

478
client/driver/qemu_test.go
View file

@ -1,478 +0,0 @@
package driver
import (
"bytes"
"fmt"
"net"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
"time"
"github.com/hashicorp/consul/lib/freeport"
"github.com/hashicorp/nomad/client/config"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
ctestutils "github.com/hashicorp/nomad/client/testutil"
)
// The fingerprinter test should always pass, even if QEMU is not installed.
func TestQemuDriver_Fingerprint(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.QemuCompatible(t)
task := &structs.Task{
Name: "foo",
Driver: "qemu",
Resources: structs.DefaultResources(),
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewQemuDriver(ctx.DriverCtx)
node := &structs.Node{
Attributes: make(map[string]string),
}
request := &cstructs.FingerprintRequest{Config: &config.Config{}, Node: node}
var response cstructs.FingerprintResponse
err := d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
if !response.Detected {
t.Fatalf("expected response to be applicable")
}
attributes := response.Attributes
if attributes == nil {
t.Fatalf("attributes should not be nil")
}
if attributes[qemuDriverAttr] == "" {
t.Fatalf("Missing Qemu driver")
}
if attributes[qemuDriverVersionAttr] == "" {
t.Fatalf("Missing Qemu driver version")
}
}
func TestQemuDriver_StartOpen_Wait(t *testing.T) {
logger := testlog.Logger(t)
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.QemuCompatible(t)
task := &structs.Task{
Name: "linux",
Driver: "qemu",
Config: map[string]interface{}{
"image_path": "linux-0.2.img",
"accelerator": "tcg",
"graceful_shutdown": false,
"port_map": []map[string]int{{
"main": 22,
"web": 8080,
}},
"args": []string{"-nodefconfig", "-nodefaults"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
CPU: 500,
MemoryMB: 512,
Networks: []*structs.NetworkResource{
{
ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
},
},
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewQemuDriver(ctx.DriverCtx)
// Copy the test image into the task's directory
dst := ctx.ExecCtx.TaskDir.Dir
copyFile("./test-resources/qemu/linux-0.2.img", filepath.Join(dst, "linux-0.2.img"), t)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("Prestart failed: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Ensure that sending a Signal returns an error
if err := resp.Handle.Signal(syscall.SIGINT); err == nil {
t.Fatalf("Expect an error when signalling")
}
// Attempt to open
handle2, err := d.Open(ctx.ExecCtx, resp.Handle.ID())
if err != nil {
t.Fatalf("err: %v", err)
}
if handle2 == nil {
t.Fatalf("missing handle")
}
// Clean up
if err := resp.Handle.Kill(); err != nil {
logger.Printf("Error killing Qemu test: %s", err)
}
}
func TestQemuDriver_GracefulShutdown(t *testing.T) {
testutil.SkipSlow(t)
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.QemuCompatible(t)
logger := testlog.Logger(t)
// Graceful shutdown may be really slow unfortunately
killTimeout := 3 * time.Minute
// Grab a free port so we can tell when the image has started
port := freeport.GetT(t, 1)[0]
task := &structs.Task{
Name: "alpine-shutdown-test",
Driver: "qemu",
Config: map[string]interface{}{
"image_path": "alpine.qcow2",
"graceful_shutdown": true,
"args": []string{"-nodefconfig", "-nodefaults"},
"port_map": []map[string]int{{
"ssh": 22,
}},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
CPU: 1000,
MemoryMB: 256,
Networks: []*structs.NetworkResource{
{
ReservedPorts: []structs.Port{{Label: "ssh", Value: port}},
},
},
},
KillTimeout: killTimeout,
}
ctx := testDriverContexts(t, task)
ctx.DriverCtx.config.MaxKillTimeout = killTimeout
defer ctx.Destroy()
d := NewQemuDriver(ctx.DriverCtx)
request := &cstructs.FingerprintRequest{Config: &config.Config{}, Node: ctx.DriverCtx.node}
var response cstructs.FingerprintResponse
err := d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
for name, value := range response.Attributes {
ctx.DriverCtx.node.Attributes[name] = value
}
dst := ctx.ExecCtx.TaskDir.Dir
copyFile("./test-resources/qemu/alpine.qcow2", filepath.Join(dst, "alpine.qcow2"), t)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("Prestart failed: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Clean up
defer func() {
select {
case <-resp.Handle.WaitCh():
// Already exited
return
default:
}
if err := resp.Handle.Kill(); err != nil {
logger.Printf("[TEST] Error killing Qemu test: %s", err)
}
}()
// Wait until sshd starts before attempting to do a graceful shutdown
testutil.WaitForResult(func() (bool, error) {
conn, err := net.Dial("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
if err != nil {
return false, err
}
// Since the connection will be accepted by the QEMU process
// before sshd actually starts, we need to block until we can
// read the "SSH" magic bytes
header := make([]byte, 3)
conn.SetReadDeadline(time.Now().Add(10 * time.Second))
_, err = conn.Read(header)
if err != nil {
return false, err
}
if !bytes.Equal(header, []byte{'S', 'S', 'H'}) {
return false, fmt.Errorf("expected 'SSH' but received: %q %v", string(header), header)
}
logger.Printf("[TEST] connected to sshd in VM")
conn.Close()
return true, nil
}, func(err error) {
t.Fatalf("failed to connect to sshd in VM: %v", err)
})
monitorPath := filepath.Join(ctx.AllocDir.AllocDir, task.Name, qemuMonitorSocketName)
// userPid supplied in sendQemuShutdown calls is bogus (it's used only
// for log output)
if err := sendQemuShutdown(ctx.DriverCtx.logger, "", 0); err == nil {
t.Fatalf("sendQemuShutdown should return an error if monitorPath parameter is empty")
}
if err := sendQemuShutdown(ctx.DriverCtx.logger, "/path/that/does/not/exist", 0); err == nil {
t.Fatalf("sendQemuShutdown should return an error if file does not exist at monitorPath")
}
if err := sendQemuShutdown(ctx.DriverCtx.logger, monitorPath, 0); err != nil {
t.Fatalf("unexpected error from sendQemuShutdown: %s", err)
}
select {
case <-resp.Handle.WaitCh():
logger.Printf("[TEST] VM exited gracefully as expected")
case <-time.After(killTimeout):
t.Fatalf("VM did not exit gracefully exit before timeout: %s", killTimeout)
}
}
func TestQemuDriverUser(t *testing.T) {
if !testutil.IsTravis() {
t.Parallel()
}
ctestutils.QemuCompatible(t)
tasks := []*structs.Task{
{
Name: "linux",
Driver: "qemu",
User: "alice",
Config: map[string]interface{}{
"image_path": "linux-0.2.img",
"accelerator": "tcg",
"graceful_shutdown": false,
"port_map": []map[string]int{{
"main": 22,
"web": 8080,
}},
"args": []string{"-nodefconfig", "-nodefaults"},
"msg": "unknown user alice",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
CPU: 500,
MemoryMB: 512,
Networks: []*structs.NetworkResource{
{
ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
},
},
},
},
{
Name: "linux",
Driver: "qemu",
User: "alice",
Config: map[string]interface{}{
"image_path": "linux-0.2.img",
"accelerator": "tcg",
"port_map": []map[string]int{{
"main": 22,
"web": 8080,
}},
"args": []string{"-nodefconfig", "-nodefaults"},
"msg": "Qemu memory assignment out of bounds",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
CPU: 500,
MemoryMB: -1,
Networks: []*structs.NetworkResource{
{
ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
},
},
},
},
}
for _, task := range tasks {
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewQemuDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("Prestart faild: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err == nil {
resp.Handle.Kill()
t.Fatalf("Should've failed")
}
msg := task.Config["msg"].(string)
if !strings.Contains(err.Error(), msg) {
t.Fatalf("Expecting '%v' in '%v'", msg, err)
}
}
}
func TestQemuDriverGetMonitorPathOldQemu(t *testing.T) {
task := &structs.Task{
Name: "linux",
Driver: "qemu",
Config: map[string]interface{}{
"image_path": "linux-0.2.img",
"accelerator": "tcg",
"graceful_shutdown": true,
"port_map": []map[string]int{{
"main": 22,
"web": 8080,
}},
"args": []string{"-nodefconfig", "-nodefaults"},
},
KillTimeout: time.Duration(1 * time.Second),
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
CPU: 500,
MemoryMB: 512,
Networks: []*structs.NetworkResource{
{
ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
},
},
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
// Simulate an older version of qemu which does not support long monitor socket paths
ctx.DriverCtx.node.Attributes[qemuDriverVersionAttr] = "2.0.0"
d := &QemuDriver{DriverContext: *ctx.DriverCtx}
shortPath := strings.Repeat("x", 10)
_, err := d.getMonitorPath(shortPath)
if err != nil {
t.Fatal("Should not have returned an error")
}
longPath := strings.Repeat("x", qemuLegacyMaxMonitorPathLen+100)
_, err = d.getMonitorPath(longPath)
if err == nil {
t.Fatal("Should have returned an error")
}
// Max length includes the '/' separator and socket name
maxLengthCount := qemuLegacyMaxMonitorPathLen - len(qemuMonitorSocketName) - 1
maxLengthLegacyPath := strings.Repeat("x", maxLengthCount)
_, err = d.getMonitorPath(maxLengthLegacyPath)
if err != nil {
t.Fatalf("Should not have returned an error: %s", err)
}
}
func TestQemuDriverGetMonitorPathNewQemu(t *testing.T) {
task := &structs.Task{
Name: "linux",
Driver: "qemu",
Config: map[string]interface{}{
"image_path": "linux-0.2.img",
"accelerator": "tcg",
"graceful_shutdown": true,
"port_map": []map[string]int{{
"main": 22,
"web": 8080,
}},
"args": []string{"-nodefconfig", "-nodefaults"},
},
KillTimeout: time.Duration(1 * time.Second),
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
CPU: 500,
MemoryMB: 512,
Networks: []*structs.NetworkResource{
{
ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
},
},
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
// Simulate a version of qemu which supports long monitor socket paths
ctx.DriverCtx.node.Attributes[qemuDriverVersionAttr] = "2.99.99"
d := &QemuDriver{DriverContext: *ctx.DriverCtx}
shortPath := strings.Repeat("x", 10)
_, err := d.getMonitorPath(shortPath)
if err != nil {
t.Fatal("Should not have returned an error")
}
longPath := strings.Repeat("x", qemuLegacyMaxMonitorPathLen+100)
_, err = d.getMonitorPath(longPath)
if err != nil {
t.Fatal("Should not have returned an error")
}
maxLengthCount := qemuLegacyMaxMonitorPathLen - len(qemuMonitorSocketName) - 1
maxLengthLegacyPath := strings.Repeat("x", maxLengthCount)
_, err = d.getMonitorPath(maxLengthLegacyPath)
if err != nil {
t.Fatal("Should not have returned an error")
}
}

337
client/driver/raw_exec.go
View file

@ -1,337 +0,0 @@
package driver
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"runtime"
"syscall"
"time"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/allocdir"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/client/fingerprint"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/drivers/shared/env"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
pexecutor "github.com/hashicorp/nomad/plugins/executor"
"github.com/mitchellh/mapstructure"
)
const (
// rawExecEnableOption is the option that enables this driver in the Config.Options map.
rawExecEnableOption = "driver.raw_exec.enable"
// rawExecNoCgroupOption forces no cgroups.
rawExecNoCgroupOption = "driver.raw_exec.no_cgroups"
// The key populated in Node Attributes to indicate presence of the Raw Exec
// driver
rawExecDriverAttr = "driver.raw_exec"
)
// The RawExecDriver is a privileged version of the exec driver. It provides no
// resource isolation and just fork/execs. The Exec driver should be preferred
// and this should only be used when explicitly needed.
type RawExecDriver struct {
DriverContext
fingerprint.StaticFingerprinter
// useCgroup tracks whether we should use a cgroup to manage the process
// tree
useCgroup bool
}
// rawExecHandle is returned from Start/Open as a handle to the PID
type rawExecHandle struct {
version string
pluginClient *plugin.Client
userPid int
executor executor.Executor
killTimeout time.Duration
maxKillTimeout time.Duration
shutdownSignal string
logger *log.Logger
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
taskEnv *env.TaskEnv
taskDir *allocdir.TaskDir
}
// NewRawExecDriver is used to create a new raw exec driver
func NewRawExecDriver(ctx *DriverContext) Driver {
return &RawExecDriver{DriverContext: *ctx}
}
// Validate is used to validate the driver configuration
func (d *RawExecDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"command": {
Type: fields.TypeString,
Required: true,
},
"args": {
Type: fields.TypeArray,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *RawExecDriver) Abilities() DriverAbilities {
return DriverAbilities{
SendSignals: true,
Exec: true,
}
}
func (d *RawExecDriver) FSIsolation() cstructs.FSIsolation {
return cstructs.FSIsolationNone
}
func (d *RawExecDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
// Check that the user has explicitly enabled this executor.
enabled := req.Config.ReadBoolDefault(rawExecEnableOption, false)
if enabled || req.Config.DevMode {
d.logger.Printf("[WARN] driver.raw_exec: raw exec is enabled. Only enable if needed")
resp.AddAttribute(rawExecDriverAttr, "1")
resp.Detected = true
return nil
}
resp.RemoveAttribute(rawExecDriverAttr)
return nil
}
func (d *RawExecDriver) Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error) {
// If we are on linux, running as root, cgroups are mounted, and cgroups
// aren't disabled by the operator use cgroups for pid management.
forceDisable := d.DriverContext.config.ReadBoolDefault(rawExecNoCgroupOption, false)
if !forceDisable && runtime.GOOS == "linux" &&
syscall.Geteuid() == 0 && cgroupsMounted(d.DriverContext.node) {
d.useCgroup = true
}
return nil, nil
}
func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
var driverConfig ExecDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
// Get the command to be ran
command := driverConfig.Command
if err := validateCommand(command, "args"); err != nil {
return nil, err
}
pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out")
executorConfig := &pexecutor.ExecutorConfig{
LogFile: pluginLogFile,
LogLevel: d.config.LogLevel,
}
exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig)
if err != nil {
return nil, err
}
_, err = getTaskKillSignal(task.KillSignal)
if err != nil {
return nil, err
}
execCmd := &executor.ExecCommand{
Cmd: command,
Args: ctx.TaskEnv.ParseAndReplace(driverConfig.Args),
User: task.User,
BasicProcessCgroup: d.useCgroup,
Env: ctx.TaskEnv.List(),
TaskDir: ctx.TaskDir.Dir,
StdoutPath: ctx.StdoutFifo,
StderrPath: ctx.StderrFifo,
}
ps, err := exec.Launch(execCmd)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[DEBUG] driver.raw_exec: started process with pid: %v", ps.Pid)
// Return a driver handle
maxKill := d.DriverContext.config.MaxKillTimeout
h := &rawExecHandle{
pluginClient: pluginClient,
executor: exec,
userPid: ps.Pid,
shutdownSignal: task.KillSignal,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
version: d.config.Version.VersionNumber(),
logger: d.logger,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
taskEnv: ctx.TaskEnv,
taskDir: ctx.TaskDir,
}
go h.run()
return &StartResponse{Handle: h}, nil
}
func (d *RawExecDriver) Cleanup(*ExecContext, *CreatedResources) error { return nil }
type rawExecId struct {
Version string
KillTimeout time.Duration
MaxKillTimeout time.Duration
UserPid int
PluginConfig *pexecutor.PluginReattachConfig
ShutdownSignal string
}
func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
id := &rawExecId{}
if err := json.Unmarshal([]byte(handleID), id); err != nil {
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput)
if err != nil {
merrs := new(multierror.Error)
merrs.Errors = append(merrs.Errors, err)
d.logger.Println("[ERR] driver.raw_exec: error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e))
}
return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil())
}
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.raw_exec: version of executor: %v", ver.Version)
// Return a driver handle
h := &rawExecHandle{
pluginClient: pluginClient,
executor: exec,
userPid: id.UserPid,
logger: d.logger,
shutdownSignal: id.ShutdownSignal,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
version: id.Version,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
taskEnv: ctx.TaskEnv,
taskDir: ctx.TaskDir,
}
go h.run()
return h, nil
}
func (h *rawExecHandle) ID() string {
id := rawExecId{
Version: h.version,
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
PluginConfig: pexecutor.NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
UserPid: h.userPid,
ShutdownSignal: h.shutdownSignal,
}
data, err := json.Marshal(id)
if err != nil {
h.logger.Printf("[ERR] driver.raw_exec: failed to marshal ID to JSON: %s", err)
}
return string(data)
}
func (h *rawExecHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *rawExecHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
// Update is not possible
return nil
}
func (h *rawExecHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
return executor.ExecScript(ctx, h.taskDir.Dir, h.taskEnv.List(), nil, h.taskEnv.ReplaceEnv(cmd), h.taskEnv.ParseAndReplace(args))
}
func (h *rawExecHandle) Signal(s os.Signal) error {
return h.executor.Signal(s)
}
func (d *rawExecHandle) Network() *cstructs.DriverNetwork {
return nil
}
func (h *rawExecHandle) Kill() error {
if err := h.executor.Signal(os.Interrupt); err != nil {
if h.pluginClient.Exited() {
return nil
}
return fmt.Errorf("executor Shutdown failed: %v", err)
}
select {
case <-h.doneCh:
return nil
case <-time.After(h.killTimeout):
if h.pluginClient.Exited() {
return nil
}
if err := h.executor.Shutdown(h.shutdownSignal, h.killTimeout); err != nil {
return fmt.Errorf("executor Exit failed: %v", err)
}
return nil
}
}
func (h *rawExecHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return h.executor.Stats()
}
func (h *rawExecHandle) run() {
ps, werr := h.executor.Wait()
close(h.doneCh)
if ps.ExitCode == 0 && werr != nil {
if e := killProcess(h.userPid); e != nil {
h.logger.Printf("[ERR] driver.raw_exec: error killing user process: %v", e)
}
}
// Destroy the executor
if err := h.executor.Shutdown(h.shutdownSignal, 0); err != nil {
h.logger.Printf("[ERR] driver.raw_exec: error killing executor: %v", err)
}
h.pluginClient.Kill()
// Send the results
h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: werr}
close(h.waitCh)
}

424
client/driver/raw_exec_test.go
View file

@ -1,424 +0,0 @@
package driver
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"strconv"
"syscall"
"testing"
"time"
"github.com/hashicorp/nomad/client/config"
cstructs "github.com/hashicorp/nomad/client/structs"
tu "github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/drivers/shared/env"
"github.com/hashicorp/nomad/helper/testtask"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
)
func TestRawExecDriver_Fingerprint(t *testing.T) {
t.Parallel()
task := &structs.Task{
Name: "foo",
Driver: "raw_exec",
Resources: structs.DefaultResources(),
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRawExecDriver(ctx.DriverCtx)
node := &structs.Node{
Attributes: make(map[string]string),
}
// Disable raw exec.
cfg := &config.Config{Options: map[string]string{rawExecEnableOption: "false"}}
request := &cstructs.FingerprintRequest{Config: cfg, Node: node}
var response cstructs.FingerprintResponse
err := d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
if response.Attributes["driver.raw_exec"] != "" {
t.Fatalf("driver incorrectly enabled")
}
// Enable raw exec.
request.Config.Options[rawExecEnableOption] = "true"
err = d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
if !response.Detected {
t.Fatalf("expected response to be applicable")
}
if response.Attributes["driver.raw_exec"] != "1" {
t.Fatalf("driver not enabled")
}
}
func TestRawExecDriver_StartOpen_Wait(t *testing.T) {
t.Parallel()
task := &structs.Task{
Name: "sleep",
Driver: "raw_exec",
Config: map[string]interface{}{
"command": testtask.Path(),
"args": []string{"sleep", "1s"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
testtask.SetTaskEnv(task)
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRawExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Attempt to open
handle2, err := d.Open(ctx.ExecCtx, resp.Handle.ID())
if err != nil {
t.Fatalf("err: %v", err)
}
if handle2 == nil {
t.Fatalf("missing handle")
}
// Task should terminate quickly
select {
case <-handle2.WaitCh():
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
resp.Handle.Kill()
handle2.Kill()
}
func TestRawExecDriver_Start_Wait(t *testing.T) {
t.Parallel()
task := &structs.Task{
Name: "sleep",
Driver: "raw_exec",
Config: map[string]interface{}{
"command": testtask.Path(),
"args": []string{"sleep", "1s"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
testtask.SetTaskEnv(task)
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRawExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Update should be a no-op
err = resp.Handle.Update(task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Task should terminate quickly
select {
case res := <-resp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
}
func TestRawExecDriver_Start_Wait_AllocDir(t *testing.T) {
t.Parallel()
exp := []byte("win")
file := "output.txt"
outPath := fmt.Sprintf(`${%s}/%s`, env.AllocDir, file)
task := &structs.Task{
Name: "sleep",
Driver: "raw_exec",
Config: map[string]interface{}{
"command": testtask.Path(),
"args": []string{
"sleep", "1s",
"write", string(exp), outPath,
},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
testtask.SetTaskEnv(task)
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRawExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Task should terminate quickly
select {
case res := <-resp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
// Check that data was written to the shared alloc directory.
outputFile := filepath.Join(ctx.AllocDir.SharedDir, file)
act, err := ioutil.ReadFile(outputFile)
if err != nil {
t.Fatalf("Couldn't read expected output: %v", err)
}
if !reflect.DeepEqual(act, exp) {
t.Fatalf("Command outputted %v; want %v", act, exp)
}
}
func TestRawExecDriver_Start_Kill_Wait(t *testing.T) {
t.Parallel()
task := &structs.Task{
Name: "sleep",
Driver: "raw_exec",
Config: map[string]interface{}{
"command": testtask.Path(),
"args": []string{"sleep", "45s"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
testtask.SetTaskEnv(task)
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRawExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
go func() {
time.Sleep(1 * time.Second)
err := resp.Handle.Kill()
// Can't rely on the ordering between wait and kill on travis...
if !testutil.IsTravis() && err != nil {
t.Fatalf("err: %v", err)
}
}()
// Task should terminate quickly
select {
case res := <-resp.Handle.WaitCh():
if res.Successful() {
t.Fatal("should err")
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
}
// This test creates a process tree such that without cgroups tracking the
// processes cleanup of the children would not be possible. Thus the test
// asserts that the processes get killed properly when using cgroups.
func TestRawExecDriver_Start_Kill_Wait_Cgroup(t *testing.T) {
tu.ExecCompatible(t)
t.Parallel()
pidFile := "pid"
task := &structs.Task{
Name: "sleep",
Driver: "raw_exec",
Config: map[string]interface{}{
"command": testtask.Path(),
"args": []string{"fork/exec", pidFile, "pgrp", "0", "sleep", "20s"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
User: "root",
}
testtask.SetTaskEnv(task)
ctx := testDriverContexts(t, task)
ctx.DriverCtx.node.Attributes["unique.cgroup.mountpoint"] = "foo" // Enable cgroups
defer ctx.Destroy()
d := NewRawExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Find the process
var pidData []byte
testutil.WaitForResult(func() (bool, error) {
var err error
pidData, err = ioutil.ReadFile(filepath.Join(ctx.AllocDir.AllocDir, "sleep", pidFile))
if err != nil {
return false, err
}
return true, nil
}, func(err error) {
t.Fatalf("err: %v", err)
})
pid, err := strconv.Atoi(string(pidData))
if err != nil {
t.Fatalf("failed to convert pid: %v", err)
}
// Check the pid is up
process, err := os.FindProcess(pid)
if err != nil {
t.Fatalf("failed to find process")
}
if err := process.Signal(syscall.Signal(0)); err != nil {
t.Fatalf("process doesn't exist: %v", err)
}
go func() {
time.Sleep(1 * time.Second)
err := resp.Handle.Kill()
// Can't rely on the ordering between wait and kill on travis...
if !testutil.IsTravis() && err != nil {
t.Fatalf("err: %v", err)
}
}()
// Task should terminate quickly
select {
case res := <-resp.Handle.WaitCh():
if res.Successful() {
t.Fatal("should err")
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
testutil.WaitForResult(func() (bool, error) {
if err := process.Signal(syscall.Signal(0)); err == nil {
return false, fmt.Errorf("process should not exist: %v", pid)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %v", err)
})
}
func TestRawExecDriver_HandlerExec(t *testing.T) {
t.Parallel()
task := &structs.Task{
Name: "sleep",
Driver: "raw_exec",
Config: map[string]interface{}{
"command": testtask.Path(),
"args": []string{"sleep", "9000s"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
testtask.SetTaskEnv(task)
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRawExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Exec a command that should work
out, code, err := resp.Handle.Exec(context.TODO(), "/usr/bin/stat", []string{"/tmp"})
if err != nil {
t.Fatalf("error exec'ing stat: %v", err)
}
if code != 0 {
t.Fatalf("expected `stat /alloc` to succeed but exit code was: %d", code)
}
if expected := 100; len(out) < expected {
t.Fatalf("expected at least %d bytes of output but found %d:\n%s", expected, len(out), out)
}
// Exec a command that should fail
out, code, err = resp.Handle.Exec(context.TODO(), "/usr/bin/stat", []string{"lkjhdsaflkjshowaisxmcvnlia"})
if err != nil {
t.Fatalf("error exec'ing stat: %v", err)
}
if code == 0 {
t.Fatalf("expected `stat` to fail but exit code was: %d", code)
}
if expected := "No such file or directory"; !bytes.Contains(out, []byte(expected)) {
t.Fatalf("expected output to contain %q but found: %q", expected, out)
}
select {
case res := <-resp.Handle.WaitCh():
t.Fatalf("Shouldn't be exited: %v", res.String())
default:
}
if err := resp.Handle.Kill(); err != nil {
t.Fatalf("error killing exec handle: %v", err)
}
}

137
client/driver/raw_exec_unix_test.go
View file

@ -1,137 +0,0 @@
// +build !windows
package driver
import (
"fmt"
"io/ioutil"
"path/filepath"
"runtime"
"strings"
"syscall"
"testing"
"time"
"github.com/hashicorp/nomad/helper/testtask"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
"github.com/stretchr/testify/require"
)
func TestRawExecDriver_User(t *testing.T) {
t.Parallel()
if runtime.GOOS != "linux" {
t.Skip("Linux only test")
}
task := &structs.Task{
Name: "sleep",
Driver: "raw_exec",
User: "alice",
Config: map[string]interface{}{
"command": testtask.Path(),
"args": []string{"sleep", "45s"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
testtask.SetTaskEnv(task)
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRawExecDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err == nil {
resp.Handle.Kill()
t.Fatalf("Should've failed")
}
msg := "unknown user alice"
if !strings.Contains(err.Error(), msg) {
t.Fatalf("Expecting '%v' in '%v'", msg, err)
}
}
func TestRawExecDriver_Signal(t *testing.T) {
t.Parallel()
task := &structs.Task{
Name: "signal",
Driver: "raw_exec",
Config: map[string]interface{}{
"command": "/bin/bash",
"args": []string{"test.sh"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
KillTimeout: 10 * time.Second,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRawExecDriver(ctx.DriverCtx)
testFile := filepath.Join(ctx.ExecCtx.TaskDir.Dir, "test.sh")
testData := []byte(`
at_term() {
echo 'Terminated.'
exit 3
}
trap at_term USR1
while true; do
sleep 1
done
`)
if err := ioutil.WriteFile(testFile, testData, 0777); err != nil {
t.Fatalf("Failed to write data: %v", err)
}
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("prestart err: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
go func() {
time.Sleep(100 * time.Millisecond)
err := resp.Handle.Signal(syscall.SIGUSR1)
if err != nil {
t.Fatalf("err: %v", err)
}
}()
// Task should terminate quickly
select {
case res := <-resp.Handle.WaitCh():
if res.Successful() {
t.Fatal("should err")
}
case <-time.After(time.Duration(testutil.TestMultiplier()*6) * time.Second):
t.Fatalf("timeout")
}
// Check the log file to see it exited because of the signal
outputFile := filepath.Join(ctx.ExecCtx.TaskDir.LogDir, "signal.stdout.0")
exp := "Terminated."
testutil.WaitForResult(func() (bool, error) {
act, err := ioutil.ReadFile(outputFile)
if err != nil {
return false, fmt.Errorf("Couldn't read expected output: %v", err)
}
if strings.TrimSpace(string(act)) != exp {
t.Logf("Read from %v", outputFile)
return false, fmt.Errorf("Command outputted %v; want %v", act, exp)
}
return true, nil
}, func(err error) { require.NoError(t, err) })
}

893
client/driver/rkt.go
View file

@ -1,893 +0,0 @@
// +build linux
package driver
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"math/rand"
"net"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"
"syscall"
"time"
appcschema "github.com/appc/spec/schema"
rktv1 "github.com/rkt/rkt/api/v1"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/go-version"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/drivers/shared/env"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
pexecutor "github.com/hashicorp/nomad/plugins/executor"
"github.com/mitchellh/mapstructure"
)
var (
reRktVersion = regexp.MustCompile(`rkt [vV]ersion[:]? (\d[.\d]+)`)
reAppcVersion = regexp.MustCompile(`appc [vV]ersion[:]? (\d[.\d]+)`)
)
const (
// minRktVersion is the earliest supported version of rkt. rkt added support
// for CPU and memory isolators in 0.14.0. We cannot support an earlier
// version to maintain an uniform interface across all drivers
minRktVersion = "1.27.0"
// The key populated in the Node Attributes to indicate the presence of the
// Rkt driver
rktDriverAttr = "driver.rkt"
// rktVolumesConfigOption is the key for enabling the use of custom
// bind volumes.
rktVolumesConfigOption = "rkt.volumes.enabled"
rktVolumesConfigDefault = true
// rktCmd is the command rkt is installed as.
rktCmd = "rkt"
// rktNetworkDeadline is how long to wait for container network to start
rktNetworkDeadline = 1 * time.Minute
)
// RktDriver is a driver for running images via Rkt
// We attempt to chose sane defaults for now, with more configuration available
// planned in the future
type RktDriver struct {
DriverContext
// A tri-state boolean to know if the fingerprinting has happened and
// whether it has been successful
fingerprintSuccess *bool
}
type RktDriverConfig struct {
ImageName string `mapstructure:"image"`
Command string `mapstructure:"command"`
Args []string `mapstructure:"args"`
TrustPrefix string `mapstructure:"trust_prefix"`
DNSServers []string `mapstructure:"dns_servers"` // DNS Server for containers
DNSSearchDomains []string `mapstructure:"dns_search_domains"` // DNS Search domains for containers
Net []string `mapstructure:"net"` // Networks for the containers
PortMapRaw []map[string]string `mapstructure:"port_map"` //
PortMap map[string]string `mapstructure:"-"` // A map of host port and the port name defined in the image manifest file
Volumes []string `mapstructure:"volumes"` // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container[:readOnly]
InsecureOptions []string `mapstructure:"insecure_options"` // list of args for --insecure-options
NoOverlay bool `mapstructure:"no_overlay"` // disable overlayfs for rkt run
Debug bool `mapstructure:"debug"` // Enable debug option for rkt command
Group string `mapstructure:"group"` // Group override for the container
}
// rktHandle is returned from Start/Open as a handle to the PID
type rktHandle struct {
uuid string
env *env.TaskEnv
taskDir *allocdir.TaskDir
pluginClient *plugin.Client
executorPid int
executor executor.Executor
logger *log.Logger
killTimeout time.Duration
maxKillTimeout time.Duration
shutdownSignal string
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
}
// rktPID is a struct to map the pid running the process to the vm image on
// disk
type rktPID struct {
UUID string
PluginConfig *pexecutor.PluginReattachConfig
ExecutorPid int
KillTimeout time.Duration
MaxKillTimeout time.Duration
ShutdownSignal string
}
// Retrieve pod status for the pod with the given UUID.
func rktGetStatus(uuid string, logger *log.Logger) (*rktv1.Pod, error) {
statusArgs := []string{
"status",
"--format=json",
uuid,
}
var outBuf, errBuf bytes.Buffer
cmd := exec.Command(rktCmd, statusArgs...)
cmd.Stdout = &outBuf
cmd.Stderr = &errBuf
if err := cmd.Run(); err != nil {
if outBuf.Len() > 0 {
logger.Printf("[DEBUG] driver.rkt: status output for UUID %s: %q", uuid, elide(outBuf))
}
if errBuf.Len() == 0 {
return nil, err
}
logger.Printf("[DEBUG] driver.rkt: status error output for UUID %s: %q", uuid, elide(errBuf))
return nil, fmt.Errorf("%s. stderr: %q", err, elide(errBuf))
}
var status rktv1.Pod
if err := json.Unmarshal(outBuf.Bytes(), &status); err != nil {
return nil, err
}
return &status, nil
}
// Retrieves a pod manifest
func rktGetManifest(uuid string) (*appcschema.PodManifest, error) {
statusArgs := []string{
"cat-manifest",
uuid,
}
var outBuf bytes.Buffer
cmd := exec.Command(rktCmd, statusArgs...)
cmd.Stdout = &outBuf
cmd.Stderr = ioutil.Discard
if err := cmd.Run(); err != nil {
return nil, err
}
var manifest appcschema.PodManifest
if err := json.Unmarshal(outBuf.Bytes(), &manifest); err != nil {
return nil, err
}
return &manifest, nil
}
func rktGetDriverNetwork(uuid string, driverConfigPortMap map[string]string, logger *log.Logger) (*cstructs.DriverNetwork, error) {
deadline := time.Now().Add(rktNetworkDeadline)
var lastErr error
try := 0
for time.Now().Before(deadline) {
try++
if status, err := rktGetStatus(uuid, logger); err == nil {
for _, net := range status.Networks {
if !net.IP.IsGlobalUnicast() {
continue
}
// Get the pod manifest so we can figure out which ports are exposed
var portmap map[string]int
manifest, err := rktGetManifest(uuid)
if err == nil {
portmap, err = rktManifestMakePortMap(manifest, driverConfigPortMap)
if err != nil {
lastErr = fmt.Errorf("could not create manifest-based portmap: %v", err)
return nil, lastErr
}
} else {
lastErr = fmt.Errorf("could not get pod manifest: %v", err)
return nil, lastErr
}
// This is a successful landing; log if its not the first attempt.
if try > 1 {
logger.Printf("[DEBUG] driver.rkt: retrieved network info for pod UUID %s on attempt %d", uuid, try)
}
return &cstructs.DriverNetwork{
PortMap: portmap,
IP: status.Networks[0].IP.String(),
}, nil
}
if len(status.Networks) == 0 {
lastErr = fmt.Errorf("no networks found")
} else {
lastErr = fmt.Errorf("no good driver networks out of %d returned", len(status.Networks))
}
} else {
lastErr = fmt.Errorf("getting status failed: %v", err)
}
waitTime := getJitteredNetworkRetryTime()
logger.Printf("[DEBUG] driver.rkt: failed getting network info for pod UUID %s attempt %d: %v. Sleeping for %v", uuid, try, lastErr, waitTime)
time.Sleep(waitTime)
}
return nil, fmt.Errorf("timed out, last error: %v", lastErr)
}
// Given a rkt/appc pod manifest and driver portmap configuration, create
// a driver portmap.
func rktManifestMakePortMap(manifest *appcschema.PodManifest, configPortMap map[string]string) (map[string]int, error) {
if len(manifest.Apps) == 0 {
return nil, fmt.Errorf("manifest has no apps")
}
if len(manifest.Apps) != 1 {
return nil, fmt.Errorf("manifest has multiple apps!")
}
app := manifest.Apps[0]
if app.App == nil {
return nil, fmt.Errorf("specified app has no App object")
}
portMap := make(map[string]int)
for svc, name := range configPortMap {
for _, port := range app.App.Ports {
if port.Name.String() == name {
portMap[svc] = int(port.Port)
}
}
}
return portMap, nil
}
// rktRemove pod after it has exited.
func rktRemove(uuid string) error {
errBuf := &bytes.Buffer{}
cmd := exec.Command(rktCmd, "rm", uuid)
cmd.Stdout = ioutil.Discard
cmd.Stderr = errBuf
if err := cmd.Run(); err != nil {
if msg := errBuf.String(); len(msg) > 0 {
return fmt.Errorf("error removing pod: %s", msg)
}
return err
}
return nil
}
// NewRktDriver is used to create a new rkt driver
func NewRktDriver(ctx *DriverContext) Driver {
return &RktDriver{DriverContext: *ctx}
}
func (d *RktDriver) FSIsolation() cstructs.FSIsolation {
return cstructs.FSIsolationImage
}
// Validate is used to validate the driver configuration
func (d *RktDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"image": {
Type: fields.TypeString,
Required: true,
},
"command": {
Type: fields.TypeString,
},
"args": {
Type: fields.TypeArray,
},
"trust_prefix": {
Type: fields.TypeString,
},
"dns_servers": {
Type: fields.TypeArray,
},
"dns_search_domains": {
Type: fields.TypeArray,
},
"net": {
Type: fields.TypeArray,
},
"port_map": {
Type: fields.TypeArray,
},
"debug": {
Type: fields.TypeBool,
},
"volumes": {
Type: fields.TypeArray,
},
"no_overlay": {
Type: fields.TypeBool,
},
"insecure_options": {
Type: fields.TypeArray,
},
"group": {
Type: fields.TypeString,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *RktDriver) Abilities() DriverAbilities {
return DriverAbilities{
SendSignals: false,
Exec: true,
}
}
func (d *RktDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
// Only enable if we are root when running on non-windows systems.
if runtime.GOOS != "windows" && syscall.Geteuid() != 0 {
if d.fingerprintSuccess == nil || *d.fingerprintSuccess {
d.logger.Printf("[DEBUG] driver.rkt: must run as root user, disabling")
}
d.fingerprintSuccess = helper.BoolToPtr(false)
resp.RemoveAttribute(rktDriverAttr)
return nil
}
outBytes, err := exec.Command(rktCmd, "version").Output()
if err != nil {
d.fingerprintSuccess = helper.BoolToPtr(false)
return nil
}
out := strings.TrimSpace(string(outBytes))
rktMatches := reRktVersion.FindStringSubmatch(out)
appcMatches := reAppcVersion.FindStringSubmatch(out)
if len(rktMatches) != 2 || len(appcMatches) != 2 {
d.fingerprintSuccess = helper.BoolToPtr(false)
resp.RemoveAttribute(rktDriverAttr)
return fmt.Errorf("Unable to parse Rkt version string: %#v", rktMatches)
}
minVersion, _ := version.NewVersion(minRktVersion)
currentVersion, _ := version.NewVersion(rktMatches[1])
if currentVersion.LessThan(minVersion) {
// Do not allow ancient rkt versions
if d.fingerprintSuccess == nil {
// Only log on first failure
d.logger.Printf("[WARN] driver.rkt: unsupported rkt version %s; please upgrade to >= %s",
currentVersion, minVersion)
}
d.fingerprintSuccess = helper.BoolToPtr(false)
resp.RemoveAttribute(rktDriverAttr)
return nil
}
// Output version information when the fingerprinter first sees rkt
if info, ok := req.Node.Drivers["rkt"]; ok && info != nil && !info.Detected {
d.logger.Printf("[DEBUG] driver.rkt: detect version: %s", strings.Replace(out, "\n", " ", -1))
}
resp.AddAttribute(rktDriverAttr, "1")
resp.AddAttribute("driver.rkt.version", rktMatches[1])
resp.AddAttribute("driver.rkt.appc.version", appcMatches[1])
resp.Detected = true
// Advertise if this node supports rkt volumes
if d.config.ReadBoolDefault(rktVolumesConfigOption, rktVolumesConfigDefault) {
resp.AddAttribute("driver."+rktVolumesConfigOption, "1")
}
d.fingerprintSuccess = helper.BoolToPtr(true)
return nil
}
func (d *RktDriver) Periodic() (bool, time.Duration) {
return true, 15 * time.Second
}
func (d *RktDriver) Prestart(ctx *ExecContext, task *structs.Task) (*PrestartResponse, error) {
return nil, nil
}
// Run an existing Rkt image.
func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
var driverConfig RktDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
driverConfig.PortMap = mapMergeStrStr(driverConfig.PortMapRaw...)
// ACI image
img := driverConfig.ImageName
// Global arguments given to both prepare and run-prepared
globalArgs := make([]string, 0, 50)
// Add debug option to rkt command.
debug := driverConfig.Debug
// Add the given trust prefix
trustPrefix := driverConfig.TrustPrefix
insecure := false
if trustPrefix != "" {
var outBuf, errBuf bytes.Buffer
cmd := exec.Command(rktCmd, "trust", "--skip-fingerprint-review=true", fmt.Sprintf("--prefix=%s", trustPrefix), fmt.Sprintf("--debug=%t", debug))
cmd.Stdout = &outBuf
cmd.Stderr = &errBuf
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("Error running rkt trust: %s\n\nOutput: %s\n\nError: %s",
err, outBuf.String(), errBuf.String())
}
d.logger.Printf("[DEBUG] driver.rkt: added trust prefix: %q", trustPrefix)
} else {
// Disable signature verification if the trust command was not run.
insecure = true
}
// if we have a selective insecure_options, prefer them
// insecure options are rkt's global argument, so we do this before the actual "run"
if len(driverConfig.InsecureOptions) > 0 {
globalArgs = append(globalArgs, fmt.Sprintf("--insecure-options=%s", strings.Join(driverConfig.InsecureOptions, ",")))
} else if insecure {
globalArgs = append(globalArgs, "--insecure-options=all")
}
// debug is rkt's global argument, so add it before the actual "run"
globalArgs = append(globalArgs, fmt.Sprintf("--debug=%t", debug))
prepareArgs := make([]string, 0, 50)
runArgs := make([]string, 0, 50)
prepareArgs = append(prepareArgs, globalArgs...)
prepareArgs = append(prepareArgs, "prepare")
runArgs = append(runArgs, globalArgs...)
runArgs = append(runArgs, "run-prepared")
// disable overlayfs
if driverConfig.NoOverlay {
prepareArgs = append(prepareArgs, "--no-overlay=true")
}
// Convert underscores to dashes in task names for use in volume names #2358
sanitizedName := strings.Replace(task.Name, "_", "-", -1)
// Mount /alloc
allocVolName := fmt.Sprintf("%s-%s-alloc", d.DriverContext.allocID, sanitizedName)
prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", allocVolName, ctx.TaskDir.SharedAllocDir))
prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", allocVolName, ctx.TaskEnv.EnvMap[env.AllocDir]))
// Mount /local
localVolName := fmt.Sprintf("%s-%s-local", d.DriverContext.allocID, sanitizedName)
prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", localVolName, ctx.TaskDir.LocalDir))
prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", localVolName, ctx.TaskEnv.EnvMap[env.TaskLocalDir]))
// Mount /secrets
secretsVolName := fmt.Sprintf("%s-%s-secrets", d.DriverContext.allocID, sanitizedName)
prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", secretsVolName, ctx.TaskDir.SecretsDir))
prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", secretsVolName, ctx.TaskEnv.EnvMap[env.SecretsDir]))
// Mount arbitrary volumes if enabled
if len(driverConfig.Volumes) > 0 {
if enabled := d.config.ReadBoolDefault(rktVolumesConfigOption, rktVolumesConfigDefault); !enabled {
return nil, fmt.Errorf("%s is false; cannot use rkt volumes: %+q", rktVolumesConfigOption, driverConfig.Volumes)
}
for i, rawvol := range driverConfig.Volumes {
parts := strings.Split(rawvol, ":")
readOnly := "false"
// job spec:
// volumes = ["/host/path:/container/path[:readOnly]"]
// the third parameter is optional, mount is read-write by default
if len(parts) == 3 {
if parts[2] == "readOnly" {
d.logger.Printf("[DEBUG] Mounting %s:%s as readOnly", parts[0], parts[1])
readOnly = "true"
} else {
d.logger.Printf("[WARN] Unknown volume parameter '%s' ignored for mount %s", parts[2], parts[0])
}
} else if len(parts) != 2 {
return nil, fmt.Errorf("invalid rkt volume: %q", rawvol)
}
volName := fmt.Sprintf("%s-%s-%d", d.DriverContext.allocID, sanitizedName, i)
prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s,readOnly=%s", volName, parts[0], readOnly))
prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", volName, parts[1]))
}
}
// Inject environment variables
for k, v := range ctx.TaskEnv.Map() {
prepareArgs = append(prepareArgs, fmt.Sprintf("--set-env=%s=%s", k, v))
}
// Image is set here, because the commands that follow apply to it
prepareArgs = append(prepareArgs, img)
// Check if the user has overridden the exec command.
if driverConfig.Command != "" {
prepareArgs = append(prepareArgs, fmt.Sprintf("--exec=%v", driverConfig.Command))
}
// Add memory isolator
prepareArgs = append(prepareArgs, fmt.Sprintf("--memory=%vM", int64(task.Resources.MemoryMB)))
// Add CPU isolator
prepareArgs = append(prepareArgs, fmt.Sprintf("--cpu=%vm", int64(task.Resources.CPU)))
// Add DNS servers
if len(driverConfig.DNSServers) == 1 && (driverConfig.DNSServers[0] == "host" || driverConfig.DNSServers[0] == "none") {
// Special case single item lists with the special values "host" or "none"
runArgs = append(runArgs, fmt.Sprintf("--dns=%s", driverConfig.DNSServers[0]))
} else {
for _, ip := range driverConfig.DNSServers {
if err := net.ParseIP(ip); err == nil {
msg := fmt.Errorf("invalid ip address for container dns server %q", ip)
d.logger.Printf("[DEBUG] driver.rkt: %v", msg)
return nil, msg
} else {
runArgs = append(runArgs, fmt.Sprintf("--dns=%s", ip))
}
}
}
// set DNS search domains
for _, domain := range driverConfig.DNSSearchDomains {
runArgs = append(runArgs, fmt.Sprintf("--dns-search=%s", domain))
}
// set network
network := strings.Join(driverConfig.Net, ",")
if network != "" {
runArgs = append(runArgs, fmt.Sprintf("--net=%s", network))
}
// Setup port mapping and exposed ports
if len(task.Resources.Networks) == 0 {
d.logger.Println("[DEBUG] driver.rkt: No network interfaces are available")
if len(driverConfig.PortMap) > 0 {
return nil, fmt.Errorf("Trying to map ports but no network interface is available")
}
} else if network == "host" {
// Port mapping is skipped when host networking is used.
d.logger.Println("[DEBUG] driver.rkt: Ignoring port_map when using --net=host")
} else {
// TODO add support for more than one network
network := task.Resources.Networks[0]
for _, port := range network.ReservedPorts {
var containerPort string
mapped, ok := driverConfig.PortMap[port.Label]
if !ok {
// If the user doesn't have a mapped port using port_map, driver stops running container.
return nil, fmt.Errorf("port_map is not set. When you defined port in the resources, you need to configure port_map.")
}
containerPort = mapped
hostPortStr := strconv.Itoa(port.Value)
d.logger.Printf("[DEBUG] driver.rkt: exposed port %s", containerPort)
// Add port option to rkt run arguments. rkt allows multiple port args
prepareArgs = append(prepareArgs, fmt.Sprintf("--port=%s:%s", containerPort, hostPortStr))
}
for _, port := range network.DynamicPorts {
// By default we will map the allocated port 1:1 to the container
var containerPort string
if mapped, ok := driverConfig.PortMap[port.Label]; ok {
containerPort = mapped
} else {
// If the user doesn't have mapped a port using port_map, driver stops running container.
return nil, fmt.Errorf("port_map is not set. When you defined port in the resources, you need to configure port_map.")
}
hostPortStr := strconv.Itoa(port.Value)
d.logger.Printf("[DEBUG] driver.rkt: exposed port %s", containerPort)
// Add port option to rkt run arguments. rkt allows multiple port args
prepareArgs = append(prepareArgs, fmt.Sprintf("--port=%s:%s", containerPort, hostPortStr))
}
}
// If a user has been specified for the task, pass it through to the user
if task.User != "" {
prepareArgs = append(prepareArgs, fmt.Sprintf("--user=%s", task.User))
}
// There's no task-level parameter for groups so check the driver
// config for a custom group
if driverConfig.Group != "" {
prepareArgs = append(prepareArgs, fmt.Sprintf("--group=%s", driverConfig.Group))
}
// Add user passed arguments.
if len(driverConfig.Args) != 0 {
parsed := ctx.TaskEnv.ParseAndReplace(driverConfig.Args)
// Need to start arguments with "--"
if len(parsed) > 0 {
prepareArgs = append(prepareArgs, "--")
}
for _, arg := range parsed {
prepareArgs = append(prepareArgs, fmt.Sprintf("%v", arg))
}
}
pluginLogFile := filepath.Join(ctx.TaskDir.Dir, fmt.Sprintf("%s-executor.out", task.Name))
executorConfig := &pexecutor.ExecutorConfig{
LogFile: pluginLogFile,
LogLevel: d.config.LogLevel,
}
execIntf, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig)
if err != nil {
return nil, err
}
absPath, err := GetAbsolutePath(rktCmd)
if err != nil {
return nil, err
}
var outBuf, errBuf bytes.Buffer
cmd := exec.Command(rktCmd, prepareArgs...)
cmd.Stdout = &outBuf
cmd.Stderr = &errBuf
d.logger.Printf("[DEBUG] driver.rkt: preparing pod %q for task %q with: %v", img, d.taskName, prepareArgs)
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("Error preparing rkt pod: %s\n\nOutput: %s\n\nError: %s",
err, outBuf.String(), errBuf.String())
}
uuid := strings.TrimSpace(outBuf.String())
d.logger.Printf("[DEBUG] driver.rkt: pod %q for task %q prepared. (UUID: %s)", img, d.taskName, uuid)
runArgs = append(runArgs, uuid)
// The task's environment is set via --set-env flags above, but the rkt
// command itself needs an evironment with PATH set to find iptables.
eb := env.NewEmptyBuilder()
filter := strings.Split(d.config.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
rktEnv := eb.SetHostEnvvars(filter).Build()
_, err = getTaskKillSignal(task.KillSignal)
if err != nil {
return nil, err
}
// Enable ResourceLimits to place the executor in a parent cgroup of
// the rkt container. This allows stats collection via the executor to
// work just like it does for exec.
execCmd := &executor.ExecCommand{
Cmd: absPath,
Args: runArgs,
ResourceLimits: true,
//
// Don't re-enable these flags as they would apply to the
// rkt CLI invocation rather than ultimate container running task
// Commenting them explicitly to protect against future changes
// that re-add them
//
// Resources: &executor.Resources{
// CPU: task.Resources.CPU,
// MemoryMB: task.Resources.MemoryMB,
// IOPS: task.Resources.IOPS,
// DiskMB: task.Resources.DiskMB,
// },
// Env: ctx.TaskEnv.List(),
// TaskDir: ctx.TaskDir.Dir,
// StdoutPath: ctx.StdoutFifo,
// StderrPath: ctx.StderrFifo,
}
ps, err := execIntf.Launch(execCmd)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[DEBUG] driver.rkt: started ACI %q (UUID: %s) for task %q with: %v", img, uuid, d.taskName, runArgs)
maxKill := d.DriverContext.config.MaxKillTimeout
h := &rktHandle{
uuid: uuid,
env: rktEnv,
taskDir: ctx.TaskDir,
pluginClient: pluginClient,
executor: execIntf,
executorPid: ps.Pid,
logger: d.logger,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
shutdownSignal: task.KillSignal,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
go h.run()
// Do not attempt to retrieve driver network if one won't exist:
// - "host" means the container itself has no networking metadata
// - "none" means no network is configured
// https://coreos.com/rkt/docs/latest/networking/overview.html#no-loopback-only-networking
var driverNetwork *cstructs.DriverNetwork
if network != "host" && network != "none" {
d.logger.Printf("[DEBUG] driver.rkt: retrieving network information for pod %q (UUID: %s) for task %q", img, uuid, d.taskName)
driverNetwork, err = rktGetDriverNetwork(uuid, driverConfig.PortMap, d.logger)
if err != nil && !pluginClient.Exited() {
d.logger.Printf("[WARN] driver.rkt: network status retrieval for pod %q (UUID: %s) for task %q failed. Last error: %v", img, uuid, d.taskName, err)
// If a portmap was given, this turns into a fatal error
if len(driverConfig.PortMap) != 0 {
pluginClient.Kill()
return nil, fmt.Errorf("Trying to map ports but driver could not determine network information")
}
}
}
return &StartResponse{Handle: h, Network: driverNetwork}, nil
}
func (d *RktDriver) Cleanup(*ExecContext, *CreatedResources) error { return nil }
func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
// Parse the handle
pidBytes := []byte(strings.TrimPrefix(handleID, "Rkt:"))
id := &rktPID{}
if err := json.Unmarshal(pidBytes, id); err != nil {
return nil, fmt.Errorf("failed to parse Rkt handle '%s': %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput)
if err != nil {
d.logger.Println("[ERR] driver.rkt: error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.ExecutorPid); e != nil {
d.logger.Printf("[ERR] driver.rkt: error destroying plugin and executor pid: %v", e)
}
return nil, fmt.Errorf("error connecting to plugin: %v", err)
}
// The task's environment is set via --set-env flags in Start, but the rkt
// command itself needs an evironment with PATH set to find iptables.
eb := env.NewEmptyBuilder()
filter := strings.Split(d.config.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
rktEnv := eb.SetHostEnvvars(filter).Build()
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.rkt: version of executor: %v", ver.Version)
// Return a driver handle
h := &rktHandle{
uuid: id.UUID,
env: rktEnv,
taskDir: ctx.TaskDir,
pluginClient: pluginClient,
executorPid: id.ExecutorPid,
executor: exec,
logger: d.logger,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
shutdownSignal: id.ShutdownSignal,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
go h.run()
return h, nil
}
func (h *rktHandle) ID() string {
// Return a handle to the PID
pid := &rktPID{
UUID: h.uuid,
PluginConfig: pexecutor.NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
ExecutorPid: h.executorPid,
ShutdownSignal: h.shutdownSignal,
}
data, err := json.Marshal(pid)
if err != nil {
h.logger.Printf("[ERR] driver.rkt: failed to marshal rkt PID to JSON: %s", err)
}
return fmt.Sprintf("Rkt:%s", string(data))
}
func (h *rktHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *rktHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
// Update is not possible
return nil
}
func (h *rktHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
if h.uuid == "" {
return nil, 0, fmt.Errorf("unable to find rkt pod UUID")
}
// enter + UUID + cmd + args...
enterArgs := make([]string, 3+len(args))
enterArgs[0] = "enter"
enterArgs[1] = h.uuid
enterArgs[2] = h.env.ReplaceEnv(cmd)
copy(enterArgs[3:], h.env.ParseAndReplace(args))
return executor.ExecScript(ctx, h.taskDir.Dir, h.env.List(), nil, rktCmd, enterArgs)
}
func (h *rktHandle) Signal(s os.Signal) error {
return fmt.Errorf("Rkt does not support signals")
}
//FIXME implement
func (d *rktHandle) Network() *cstructs.DriverNetwork {
return nil
}
// Kill is used to terminate the task. We send an Interrupt
// and then provide a 5 second grace period before doing a Kill.
func (h *rktHandle) Kill() error {
return h.executor.Shutdown(h.shutdownSignal, h.killTimeout)
}
func (h *rktHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return h.executor.Stats()
}
func (h *rktHandle) run() {
ps, werr := h.executor.Wait()
close(h.doneCh)
if ps.ExitCode == 0 && werr != nil {
if e := killProcess(h.executorPid); e != nil {
h.logger.Printf("[ERR] driver.rkt: error killing user process: %v", e)
}
}
// Destroy the executor
if err := h.executor.Shutdown(h.shutdownSignal, 0); err != nil {
h.logger.Printf("[ERR] driver.rkt: error killing executor: %v", err)
}
h.pluginClient.Kill()
// Remove the pod
if err := rktRemove(h.uuid); err != nil {
h.logger.Printf("[ERR] driver.rkt: error removing pod (UUID: %q) - must gc manually: %v", h.uuid, err)
} else {
h.logger.Printf("[DEBUG] driver.rkt: removed pod (UUID: %q)", h.uuid)
}
// Send the results
h.waitCh <- dstructs.NewWaitResult(ps.ExitCode, 0, werr)
close(h.waitCh)
}
// Create a time with a 0 to 100ms jitter for rktGetDriverNetwork retries
func getJitteredNetworkRetryTime() time.Duration {
return time.Duration(900+rand.Intn(100)) * time.Millisecond
}
// Conditionally elide a buffer to an arbitrary length
func elideToLen(inBuf bytes.Buffer, length int) bytes.Buffer {
if inBuf.Len() > length {
inBuf.Truncate(length)
inBuf.WriteString("...")
}
return inBuf
}
// Conditionally elide a buffer to an 80 character string
func elide(inBuf bytes.Buffer) string {
tempBuf := elideToLen(inBuf, 80)
return tempBuf.String()
}

54
client/driver/rkt_nonlinux.go
View file

@ -1,54 +0,0 @@
//+build !linux
package driver
import (
"time"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/nomad/structs"
)
// NewRktDriver returns an unimplemented driver that returns false during
// fingerprinting.
func NewRktDriver(*DriverContext) Driver {
return RktDriver{}
}
type RktDriver struct{}
func (RktDriver) Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error) {
panic("not implemented")
}
func (RktDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
panic("not implemented")
}
func (RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
panic("not implemented")
}
func (RktDriver) Cleanup(*ExecContext, *CreatedResources) error {
panic("not implemented")
}
func (RktDriver) Validate(map[string]interface{}) error {
panic("not implemented")
}
func (RktDriver) Abilities() DriverAbilities {
panic("not implemented")
}
func (RktDriver) FSIsolation() cstructs.FSIsolation {
panic("not implemented")
}
func (RktDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
return nil
}
func (RktDriver) Periodic() (bool, time.Duration) {
return false, 0
}

735
client/driver/rkt_test.go
View file

@ -1,735 +0,0 @@
// +build linux
package driver
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"syscall"
"testing"
"time"
"github.com/hashicorp/nomad/client/config"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
"github.com/stretchr/testify/assert"
ctestutil "github.com/hashicorp/nomad/client/testutil"
)
func TestRktVersionRegex(t *testing.T) {
ctestutil.RktCompatible(t)
t.Parallel()
inputRkt := "rkt version 0.8.1"
inputAppc := "appc version 1.2.0"
expectedRkt := "0.8.1"
expectedAppc := "1.2.0"
rktMatches := reRktVersion.FindStringSubmatch(inputRkt)
appcMatches := reAppcVersion.FindStringSubmatch(inputAppc)
if rktMatches[1] != expectedRkt {
fmt.Printf("Test failed; got %q; want %q\n", rktMatches[1], expectedRkt)
}
if appcMatches[1] != expectedAppc {
fmt.Printf("Test failed; got %q; want %q\n", appcMatches[1], expectedAppc)
}
}
// The fingerprinter test should always pass, even if rkt is not installed.
func TestRktDriver_Fingerprint(t *testing.T) {
ctestutil.RktCompatible(t)
t.Parallel()
ctx := testDriverContexts(t, &structs.Task{Name: "foo", Driver: "rkt"})
d := NewRktDriver(ctx.DriverCtx)
node := &structs.Node{
Attributes: make(map[string]string),
}
request := &cstructs.FingerprintRequest{Config: &config.Config{}, Node: node}
var response cstructs.FingerprintResponse
err := d.Fingerprint(request, &response)
if err != nil {
t.Fatalf("err: %v", err)
}
if !response.Detected {
t.Fatalf("expected response to be applicable")
}
attributes := response.Attributes
if attributes == nil {
t.Fatalf("expected attributes to not equal nil")
}
if attributes["driver.rkt"] != "1" {
t.Fatalf("Missing Rkt driver")
}
if attributes["driver.rkt.version"] == "" {
t.Fatalf("Missing Rkt driver version")
}
if attributes["driver.rkt.appc.version"] == "" {
t.Fatalf("Missing appc version for the Rkt driver")
}
}
func TestRktDriver_Start_DNS(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
task := &structs.Task{
Name: "etcd",
Driver: "rkt",
Config: map[string]interface{}{
"trust_prefix": "coreos.com/etcd",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"dns_servers": []string{"8.8.8.8", "8.8.4.4"},
"dns_search_domains": []string{"example.com", "example.org", "example.net"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer resp.Handle.Kill()
// Attempt to open
handle2, err := d.Open(ctx.ExecCtx, resp.Handle.ID())
if err != nil {
t.Fatalf("err: %v", err)
}
if handle2 == nil {
t.Fatalf("missing handle")
}
handle2.Kill()
}
func TestRktDriver_Start_Wait(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
task := &structs.Task{
Name: "etcd",
Driver: "rkt",
Config: map[string]interface{}{
"trust_prefix": "coreos.com/etcd",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"args": []string{"--version"},
// Disable networking to speed up test as it's not needed
"net": []string{"none"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
handle := resp.Handle.(*rktHandle)
defer handle.Kill()
// Update should be a no-op
if err := handle.Update(task); err != nil {
t.Fatalf("err: %v", err)
}
// Signal should be an error
if err := resp.Handle.Signal(syscall.SIGTERM); err == nil {
t.Fatalf("err: %v", err)
}
select {
case res := <-resp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
t.Fatalf("timeout")
}
// Make sure pod was removed #3561
var stderr bytes.Buffer
cmd := exec.Command(rktCmd, "status", handle.uuid)
cmd.Stdout = ioutil.Discard
cmd.Stderr = &stderr
if err := cmd.Run(); err == nil {
t.Fatalf("expected error running 'rkt status %s' on removed container", handle.uuid)
}
if out := stderr.String(); !strings.Contains(out, "no matches found") {
t.Fatalf("expected 'no matches found' but received: %s", out)
}
}
func TestRktDriver_Start_Wait_Skip_Trust(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
task := &structs.Task{
Name: "etcd",
Driver: "rkt",
Config: map[string]interface{}{
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"args": []string{"--version"},
// Disable networking to speed up test as it's not needed
"net": []string{"none"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer resp.Handle.Kill()
// Update should be a no-op
err = resp.Handle.Update(task)
if err != nil {
t.Fatalf("err: %v", err)
}
select {
case res := <-resp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
t.Fatalf("timeout")
}
}
func TestRktDriver_Start_Wait_AllocDir(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
exp := []byte{'w', 'i', 'n'}
file := "output.txt"
tmpvol, err := ioutil.TempDir("", "nomadtest_rktdriver_volumes")
if err != nil {
t.Fatalf("error creating temporary dir: %v", err)
}
defer os.RemoveAll(tmpvol)
hostpath := filepath.Join(tmpvol, file)
task := &structs.Task{
Name: "rkttest_alpine",
Driver: "rkt",
Config: map[string]interface{}{
"image": "docker://redis:3.2-alpine",
"command": "/bin/sh",
"args": []string{
"-c",
fmt.Sprintf("echo -n %s > /foo/%s", string(exp), file),
},
"net": []string{"none"},
"volumes": []string{fmt.Sprintf("%s:/foo", tmpvol)},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer resp.Handle.Kill()
select {
case res := <-resp.Handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
t.Fatalf("timeout")
}
// Check that data was written to the shared alloc directory.
act, err := ioutil.ReadFile(hostpath)
if err != nil {
t.Fatalf("Couldn't read expected output: %v", err)
}
if !reflect.DeepEqual(act, exp) {
t.Fatalf("Command output is %v; expected %v", act, exp)
}
}
// TestRktDriver_UserGroup asserts tasks may override the user and group of the
// rkt image.
func TestRktDriver_UserGroup(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
require := assert.New(t)
task := &structs.Task{
Name: "sleepy",
Driver: "rkt",
User: "nobody",
Config: map[string]interface{}{
"image": "docker://redis:3.2-alpine",
"group": "nogroup",
"command": "sleep",
"args": []string{"9000"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
tctx := testDriverContexts(t, task)
defer tctx.Destroy()
d := NewRktDriver(tctx.DriverCtx)
_, err := d.Prestart(tctx.ExecCtx, task)
require.NoError(err)
resp, err := d.Start(tctx.ExecCtx, task)
require.NoError(err)
defer resp.Handle.Kill()
ctx := context.Background()
// WaitUntil we can determine the user/group redis is running as
expected := []byte("\nnobody nogroup /bin/sleep 9000\n")
testutil.WaitForResult(func() (bool, error) {
raw, code, err := resp.Handle.Exec(ctx, "ps", []string{"-o", "user,group,args"})
if err != nil {
err = fmt.Errorf("original error: %v; code: %d; raw output: %s", err, code, string(raw))
return false, err
}
if code != 0 {
return false, fmt.Errorf("unexpected exit code: %d", code)
}
return bytes.Contains(raw, expected), fmt.Errorf("expected %q but found:\n%s", expected, raw)
}, func(err error) {
t.Fatalf("err: %v", err)
})
require.NoError(resp.Handle.Kill())
}
func TestRktTrustPrefix(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
task := &structs.Task{
Name: "etcd",
Driver: "rkt",
Config: map[string]interface{}{
"trust_prefix": "example.com/invalid",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"args": []string{"--version"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err == nil {
resp.Handle.Kill()
t.Fatalf("Should've failed")
}
msg := "Error running rkt trust"
if !strings.Contains(err.Error(), msg) {
t.Fatalf("Expecting '%v' in '%v'", msg, err)
}
}
func TestRktTaskValidate(t *testing.T) {
ctestutil.RktCompatible(t)
t.Parallel()
task := &structs.Task{
Name: "etcd",
Driver: "rkt",
Config: map[string]interface{}{
"trust_prefix": "coreos.com/etcd",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"args": []string{"--version"},
"dns_servers": []string{"8.8.8.8", "8.8.4.4"},
"dns_search_domains": []string{"example.com", "example.org", "example.net"},
},
Resources: basicResources,
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if err := d.Validate(task.Config); err != nil {
t.Fatalf("Validation error in TaskConfig : '%v'", err)
}
}
func TestRktDriver_PortMapping(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
task := &structs.Task{
Name: "redis",
Driver: "rkt",
Config: map[string]interface{}{
"image": "docker://redis:3.2-alpine",
"port_map": []map[string]string{
{
"main": "6379-tcp",
},
},
"debug": "true",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 256,
CPU: 512,
Networks: []*structs.NetworkResource{
{
IP: "127.0.0.1",
ReservedPorts: []structs.Port{{Label: "main", Value: 8080}},
},
},
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer resp.Handle.Kill()
if resp.Network == nil {
t.Fatalf("Expected driver to set a DriverNetwork, but it did not!")
}
failCh := make(chan error, 1)
go func() {
time.Sleep(1 * time.Second)
if err := resp.Handle.Kill(); err != nil {
failCh <- err
}
}()
select {
case err := <-failCh:
t.Fatalf("failed to kill handle: %v", err)
case <-resp.Handle.WaitCh():
case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
t.Fatalf("timeout")
}
}
// TestRktDriver_PortsMapping_Host asserts that port_map isn't required when
// host networking is used.
func TestRktDriver_PortsMapping_Host(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
task := &structs.Task{
Name: "redis",
Driver: "rkt",
Config: map[string]interface{}{
"image": "docker://redis:3.2-alpine",
"net": []string{"host"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 256,
CPU: 512,
Networks: []*structs.NetworkResource{
{
IP: "127.0.0.1",
ReservedPorts: []structs.Port{{Label: "main", Value: 8080}},
},
},
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer resp.Handle.Kill()
if resp.Network != nil {
t.Fatalf("No network should be returned with --net=host but found: %#v", resp.Network)
}
failCh := make(chan error, 1)
go func() {
time.Sleep(1 * time.Second)
if err := resp.Handle.Kill(); err != nil {
failCh <- err
}
}()
select {
case err := <-failCh:
t.Fatalf("failed to kill handle: %v", err)
case <-resp.Handle.WaitCh():
case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
t.Fatalf("timeout")
}
}
func TestRktDriver_HandlerExec(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
task := &structs.Task{
Name: "etcd",
Driver: "rkt",
Config: map[string]interface{}{
"trust_prefix": "coreos.com/etcd",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer resp.Handle.Kill()
// Exec a command that should work
testutil.WaitForResult(func() (bool, error) {
out, code, err := resp.Handle.Exec(context.TODO(), "/etcd", []string{"--version"})
if err != nil {
return false, fmt.Errorf("error exec'ing etcd --version: %v", err)
}
if code != 0 {
return false, fmt.Errorf("expected `etcd --version` to succeed but exit code was: %d\n%s", code, string(out))
}
if expected := []byte("etcd version "); !bytes.HasPrefix(out, expected) {
return false, fmt.Errorf("expected output to start with %q but found:\n%q", expected, out)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %v", err)
})
// Exec a command that should fail
out, code, err := resp.Handle.Exec(context.TODO(), "/etcd", []string{"--kaljdshf"})
if err != nil {
t.Fatalf("error exec'ing bad command: %v", err)
}
if code == 0 {
t.Fatalf("expected `stat` to fail but exit code was: %d", code)
}
if expected := "flag provided but not defined"; !bytes.Contains(out, []byte(expected)) {
t.Fatalf("expected output to contain %q but found: %q", expected, out)
}
if err := resp.Handle.Kill(); err != nil {
t.Fatalf("error killing handle: %v", err)
}
}
func TestRktDriver_Stats(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
task := &structs.Task{
Name: "etcd",
Driver: "rkt",
Config: map[string]interface{}{
"trust_prefix": "coreos.com/etcd",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
ctx := testDriverContexts(t, task)
defer ctx.Destroy()
d := NewRktDriver(ctx.DriverCtx)
if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("error in prestart: %v", err)
}
resp, err := d.Start(ctx.ExecCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer resp.Handle.Kill()
testutil.WaitForResult(func() (bool, error) {
stats, err := resp.Handle.Stats()
if err != nil {
return false, err
}
if stats == nil || stats.ResourceUsage == nil {
return false, fmt.Errorf("stats is nil")
}
if stats.ResourceUsage.CpuStats.TotalTicks == 0 {
return false, fmt.Errorf("cpu ticks unset")
}
if stats.ResourceUsage.MemoryStats.RSS == 0 {
return false, fmt.Errorf("rss stats unset")
}
return true, nil
}, func(err error) {
t.Fatalf("error: %v", err)
})
}
func TestRktDriver_Remove_Error(t *testing.T) {
ctestutil.RktCompatible(t)
if !testutil.IsTravis() {
t.Parallel()
}
// Removing a nonexistent pod should return an error
if err := rktRemove("00000000-0000-0000-0000-000000000000"); err == nil {
t.Fatalf("expected an error")
}
if err := rktRemove("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"); err == nil {
t.Fatalf("expected an error")
}
}

57
client/driver/structs/structs.go
View file

@ -1,57 +0,0 @@
package structs
import (
"fmt"
"time"
)
const (
// The default user that the executor uses to run tasks
DefaultUnprivilegedUser = "nobody"
// CheckBufSize is the size of the check output result
CheckBufSize = 4 * 1024
)
// WaitResult stores the result of a Wait operation.
type WaitResult struct {
ExitCode int
Signal int
Err error
}
func NewWaitResult(code, signal int, err error) *WaitResult {
return &WaitResult{
ExitCode: code,
Signal: signal,
Err: err,
}
}
func (r *WaitResult) Successful() bool {
return r.ExitCode == 0 && r.Signal == 0 && r.Err == nil
}
func (r *WaitResult) String() string {
return fmt.Sprintf("Wait returned exit code %v, signal %v, and error %v",
r.ExitCode, r.Signal, r.Err)
}
// CheckResult encapsulates the result of a check
type CheckResult struct {
// ExitCode is the exit code of the check
ExitCode int
// Output is the output of the check script
Output string
// Timestamp is the time at which the check was executed
Timestamp time.Time
// Duration is the time it took the check to run
Duration time.Duration
// Err is the error that a check returned
Err error
}

12
client/driver/structs/structs_default.go
View file

@ -1,12 +0,0 @@
// +build darwin dragonfly freebsd netbsd openbsd solaris windows
package structs
// IsolationConfig has information about the isolation mechanism the executor
// uses to put resource constraints and isolation on the user process. The
// default implementation is empty. Platforms that support resource isolation
// (e.g. Linux's Cgroups) should build their own platform-specific copy. This
// information is transmitted via RPC so it is not permissible to change the
// API.
type IsolationConfig struct {
}

4
client/driver/test-resources/docker/Dockerfile
View file

@ -1,4 +0,0 @@
FROM python
ADD main.py main.py
ENV PYTHONUNBUFFERED=1
CMD ["python", "main.py"]

13
client/driver/test-resources/docker/auth.json
View file

@ -1,13 +0,0 @@
{
"auths": {
"https://index.docker.io/v1/": {
"auth": "dGVzdDoxMjM0"
},
"quay.io": {
"auth": "dGVzdDo1Njc4"
},
"https://other.io/v1/": {
"auth": "dGVzdDphYmNk"
}
}
}

BIN
client/driver/test-resources/docker/busybox.tar (Stored with Git LFS)
View file

Binary file not shown.

BIN
client/driver/test-resources/docker/busybox_glibc.tar (Stored with Git LFS)
View file

Binary file not shown.

BIN
client/driver/test-resources/docker/busybox_musl.tar (Stored with Git LFS)
View file

Binary file not shown.

17
client/driver/test-resources/docker/main.py
View file

@ -1,17 +0,0 @@
import signal
import time
# Setup handler for sigterm so we can exit when docker stop is called.
def term(signum, stack_Frame):
exit(1)
signal.signal(signal.SIGTERM, term)
print ("Starting")
max = 3
for i in range(max):
time.sleep(1)
print("Heartbeat {0}/{1}".format(i + 1, max))
print("Exiting")

BIN
client/driver/test-resources/java/Hello.class (Stored with Git LFS)
View file

Binary file not shown.

14
client/driver/test-resources/java/Hello.java
View file

@ -1,14 +0,0 @@
public class Hello {
public static void main(String[] args) {
System.out.println("Hello");
int seconds = 5;
if (args.length != 0) {
seconds = Integer.parseInt(args[0]);
}
try {
Thread.sleep(1000*seconds); //1000 milliseconds is one second.
} catch(InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
}

BIN
client/driver/test-resources/java/demoapp.jar (Stored with Git LFS)
View file

Binary file not shown.

21
client/driver/test-resources/qemu/README.md
View file

@ -1,21 +0,0 @@
# QEMU Test Images
## `linux-0.2.img`
via https://en.wikibooks.org/wiki/QEMU/Images
Does not support graceful shutdown.
## Alpine
```
qemu-img create -fmt qcow2 alpine.qcow2 8G
# Download virtual x86_64 Alpine image https://alpinelinux.org/downloads/
qemu-system-x86_64 -cdrom path/to/alpine.iso -hda alpine.qcow2 -boot d -net nic -net user -m 256 -localtime
# In the guest run setup-alpine and exit when complete
# Boot again with:
qemu-system-x86_64 alpine.qcow2
```

BIN
client/driver/test-resources/qemu/alpine.qcow2 (Stored with Git LFS)
View file

Binary file not shown.

BIN
client/driver/test-resources/qemu/linux-0.2.img (Stored with Git LFS)
View file

Binary file not shown.

193
client/driver/utils.go
View file

@ -1,193 +0,0 @@
package driver
import (
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/hashicorp/consul-template/signals"
hclog "github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/config"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/helper/discover"
"github.com/hashicorp/nomad/nomad/structs"
pexecutor "github.com/hashicorp/nomad/plugins/executor"
)
// cgroupsMounted returns true if the cgroups are mounted on a system otherwise
// returns false
func cgroupsMounted(node *structs.Node) bool {
_, ok := node.Attributes["unique.cgroup.mountpoint"]
return ok
}
// createExecutor launches an executor plugin and returns an instance of the
// Executor interface
func createExecutor(w io.Writer, clientConfig *config.Config,
executorConfig *pexecutor.ExecutorConfig) (executor.Executor, *plugin.Client, error) {
c, err := json.Marshal(executorConfig)
if err != nil {
return nil, nil, fmt.Errorf("unable to create executor config: %v", err)
}
bin, err := discover.NomadExecutable()
if err != nil {
return nil, nil, fmt.Errorf("unable to find the nomad binary: %v", err)
}
config := &plugin.ClientConfig{
Cmd: exec.Command(bin, "executor", string(c)),
}
config.HandshakeConfig = pexecutor.HandshakeConfig
config.Plugins = pexecutor.GetPluginMap(w, hclog.LevelFromString(clientConfig.LogLevel), executorConfig.FSIsolation)
config.MaxPort = clientConfig.ClientMaxPort
config.MinPort = clientConfig.ClientMinPort
// setting the setsid of the plugin process so that it doesn't get signals sent to
// the nomad client.
if config.Cmd != nil {
isolateCommand(config.Cmd)
}
executorClient := plugin.NewClient(config)
rpcClient, err := executorClient.Client()
if err != nil {
return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
}
raw, err := rpcClient.Dispense("executor")
if err != nil {
return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
}
executorPlugin := raw.(executor.Executor)
return executorPlugin, executorClient, nil
}
func createExecutorWithConfig(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) {
config.HandshakeConfig = pexecutor.HandshakeConfig
// Setting this to DEBUG since the log level at the executor server process
// is already set, and this effects only the executor client.
config.Plugins = pexecutor.GetPluginMap(w, hclog.Debug, false)
executorClient := plugin.NewClient(config)
rpcClient, err := executorClient.Client()
if err != nil {
return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
}
raw, err := rpcClient.Dispense("executor")
if err != nil {
return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
}
executorPlugin, ok := raw.(*pexecutor.ExecutorRPC)
if !ok {
return nil, nil, fmt.Errorf("unexpected executor rpc type: %T", raw)
}
return executorPlugin, executorClient, nil
}
// killProcess kills a process with the given pid
func killProcess(pid int) error {
proc, err := os.FindProcess(pid)
if err != nil {
return err
}
return proc.Kill()
}
// destroyPlugin kills the plugin with the given pid and also kills the user
// process
func destroyPlugin(pluginPid int, userPid int) error {
var merr error
if err := killProcess(pluginPid); err != nil {
merr = multierror.Append(merr, err)
}
if err := killProcess(userPid); err != nil {
merr = multierror.Append(merr, err)
}
return merr
}
// validateCommand validates that the command only has a single value and
// returns a user friendly error message telling them to use the passed
// argField.
func validateCommand(command, argField string) error {
trimmed := strings.TrimSpace(command)
if len(trimmed) == 0 {
return fmt.Errorf("command empty: %q", command)
}
if len(trimmed) != len(command) {
return fmt.Errorf("command contains extra white space: %q", command)
}
return nil
}
// GetKillTimeout returns the kill timeout to use given the tasks desired kill
// timeout and the operator configured max kill timeout.
func GetKillTimeout(desired, max time.Duration) time.Duration {
maxNanos := max.Nanoseconds()
desiredNanos := desired.Nanoseconds()
// Make the minimum time between signal and kill, 1 second.
if desiredNanos <= 0 {
desiredNanos = (1 * time.Second).Nanoseconds()
}
// Protect against max not being set properly.
if maxNanos <= 0 {
maxNanos = (10 * time.Second).Nanoseconds()
}
if desiredNanos < maxNanos {
return time.Duration(desiredNanos)
}
return max
}
// GetAbsolutePath returns the absolute path of the passed binary by resolving
// it in the path and following symlinks.
func GetAbsolutePath(bin string) (string, error) {
lp, err := exec.LookPath(bin)
if err != nil {
return "", fmt.Errorf("failed to resolve path to %q executable: %v", bin, err)
}
return filepath.EvalSymlinks(lp)
}
// getExecutorUser returns the user of the task, defaulting to
// dstructs.DefaultUnprivilegedUser if none was given.
func getExecutorUser(task *structs.Task) string {
if task.User == "" {
return dstructs.DefaultUnprivilegedUser
}
return task.User
}
// getTaskKillSignal looks up the signal specified for the task if it has been
// specified. If it is not supported on the platform, returns an error.
func getTaskKillSignal(signal string) (os.Signal, error) {
if signal == "" {
return os.Interrupt, nil
}
taskKillSignal := signals.SignalLookup[signal]
if taskKillSignal == nil {
return nil, fmt.Errorf("Signal %s is not supported", signal)
}
return taskKillSignal, nil
}

58
client/driver/utils_test.go
View file

@ -1,58 +0,0 @@
package driver
import (
"os"
"runtime"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestDriver_KillTimeout(t *testing.T) {
t.Parallel()
expected := 1 * time.Second
max := 10 * time.Second
if actual := GetKillTimeout(expected, max); expected != actual {
t.Fatalf("GetKillTimeout() returned %v; want %v", actual, expected)
}
expected = 10 * time.Second
input := 11 * time.Second
if actual := GetKillTimeout(input, max); expected != actual {
t.Fatalf("KillTimeout() returned %v; want %v", actual, expected)
}
}
func TestDriver_getTaskKillSignal(t *testing.T) {
assert := assert.New(t)
t.Parallel()
if runtime.GOOS != "linux" {
t.Skip("Linux only test")
}
// Test that the default is SIGINT
{
sig, err := getTaskKillSignal("")
assert.Nil(err)
assert.Equal(sig, os.Interrupt)
}
// Test that unsupported signals return an error
{
_, err := getTaskKillSignal("ABCDEF")
assert.NotNil(err)
assert.Contains(err.Error(), "Signal ABCDEF is not supported")
}
// Test that supported signals return that signal
{
sig, err := getTaskKillSignal("SIGKILL")
assert.Nil(err)
assert.Equal(sig, syscall.SIGKILL)
}
}

18
client/driver/utils_unix.go
View file

@ -1,18 +0,0 @@
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
package driver
import (
"os/exec"
"syscall"
)
// isolateCommand sets the setsid flag in exec.Cmd to true so that the process
// becomes the process leader in a new session and doesn't receive signals that
// are sent to the parent process.
func isolateCommand(cmd *exec.Cmd) {
if cmd.SysProcAttr == nil {
cmd.SysProcAttr = &syscall.SysProcAttr{}
}
cmd.SysProcAttr.Setsid = true
}

9
client/driver/utils_windows.go
View file

@ -1,9 +0,0 @@
package driver
import (
"os/exec"
)
// TODO Figure out if this is needed in Wondows
func isolateCommand(cmd *exec.Cmd) {
}