Merge pull request #4918 from hashicorp/f-lxc-driver

Add LXC driver

client/driver/lxc.go

@@ -153,18 +153,20 @@ func (d *LxcDriver) Validate(config map[string]interface{}) error {
 		return err
 	}
 
-	volumes, _ := fd.GetOk("volumes")
+	volumes, ok := fd.GetOk("volumes")
-	for _, volDesc := range volumes.([]interface{}) {
+	if ok {
-		volStr := volDesc.(string)
+		for _, volDesc := range volumes.([]interface{}) {
-		paths := strings.Split(volStr, ":")
+			volStr := volDesc.(string)
-		if len(paths) != 2 {
+			paths := strings.Split(volStr, ":")
-			return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr)
+			if len(paths) != 2 {
-		}
+				return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr)
-		if len(paths[0]) == 0 || len(paths[1]) == 0 {
+			}
-			return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr)
+			if len(paths[0]) == 0 || len(paths[1]) == 0 {
-		}
+				return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr)
-		if paths[1][0] == '/' {
+			}
-			return fmt.Errorf("unsupported absolute container mount point: '%s'", paths[1])
+			if paths[1][0] == '/' {
+				return fmt.Errorf("unsupported absolute container mount point: '%s'", paths[1])
+			}
 		}
 	}
 
@@ -371,7 +373,7 @@ func (d *LxcDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error
 	containers := lxc.Containers(pid.LxcPath)
 	for _, c := range containers {
 		if c.Name() == pid.ContainerName {
-			container = &c
+			container = c
 			break
 		}
 	}

drivers/lxc/driver.go

@@ -0,0 +1,513 @@
+//+build linux,lxc
+
+package lxc
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad/client/stats"
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/drivers/shared/eventer"
+	"github.com/hashicorp/nomad/plugins/base"
+	"github.com/hashicorp/nomad/plugins/drivers"
+	"github.com/hashicorp/nomad/plugins/shared/hclspec"
+	"github.com/hashicorp/nomad/plugins/shared/loader"
+
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+const (
+	// pluginName is the name of the plugin
+	pluginName = "lxc"
+
+	// fingerprintPeriod is the interval at which the driver will send fingerprint responses
+	fingerprintPeriod = 30 * time.Second
+)
+
+var (
+	// PluginID is the rawexec plugin metadata registered in the plugin
+	// catalog.
+	PluginID = loader.PluginID{
+		Name:       pluginName,
+		PluginType: base.PluginTypeDriver,
+	}
+
+	// PluginConfig is the rawexec factory function registered in the
+	// plugin catalog.
+	PluginConfig = &loader.InternalPluginConfig{
+		Config:  map[string]interface{}{},
+		Factory: func(l hclog.Logger) interface{} { return NewLXCDriver(l) },
+	}
+)
+
+// PluginLoader maps pre-0.9 client driver options to post-0.9 plugin options.
+func PluginLoader(opts map[string]string) (map[string]interface{}, error) {
+	conf := map[string]interface{}{}
+	if v, err := strconv.ParseBool(opts["driver.lxc.enable"]); err == nil {
+		conf["enabled"] = v
+	}
+	if v, err := strconv.ParseBool(opts["lxc.volumes.enabled"]); err == nil {
+		conf["volumes"] = v
+	}
+	if v, ok := opts["driver.lxc.path"]; ok {
+		conf["path"] = v
+	}
+	return conf, nil
+}
+
+var (
+	// pluginInfo is the response returned for the PluginInfo RPC
+	pluginInfo = &base.PluginInfoResponse{
+		Type:             base.PluginTypeDriver,
+		PluginApiVersion: "0.0.1",
+		PluginVersion:    "0.1.0",
+		Name:             pluginName,
+	}
+
+	// configSpec is the hcl specification returned by the ConfigSchema RPC
+	configSpec = hclspec.NewObject(map[string]*hclspec.Spec{
+		"enabled": hclspec.NewDefault(
+			hclspec.NewAttr("enabled", "bool", false),
+			hclspec.NewLiteral("true"),
+		),
+		"volumes": hclspec.NewDefault(
+			hclspec.NewAttr("volumes", "bool", false),
+			hclspec.NewLiteral("true"),
+		),
+		"path": hclspec.NewDefault(
+			hclspec.NewAttr("path", "string", false),
+			hclspec.NewLiteral("\"\""),
+		),
+	})
+
+	// taskConfigSpec is the hcl specification for the driver config section of
+	// a task within a job. It is returned in the TaskConfigSchema RPC
+	taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{
+		"template":       hclspec.NewAttr("template", "string", true),
+		"distro":         hclspec.NewAttr("distro", "string", false),
+		"release":        hclspec.NewAttr("release", "string", false),
+		"arch":           hclspec.NewAttr("arch", "string", false),
+		"image_variant":  hclspec.NewAttr("image_variant", "string", false),
+		"image_server":   hclspec.NewAttr("image_server", "string", false),
+		"gpg_key_id":     hclspec.NewAttr("gpg_key_id", "string", false),
+		"gpg_key_server": hclspec.NewAttr("gpg_key_server", "string", false),
+		"disable_gpg":    hclspec.NewAttr("disable_gpg", "string", false),
+		"flush_cache":    hclspec.NewAttr("flush_cache", "string", false),
+		"force_cache":    hclspec.NewAttr("force_cache", "string", false),
+		"template_args":  hclspec.NewAttr("template_args", "list(string)", false),
+		"log_level":      hclspec.NewAttr("log_level", "string", false),
+		"verbosity":      hclspec.NewAttr("verbosity", "string", false),
+		"volumes":        hclspec.NewAttr("volumes", "list(string)", false),
+	})
+
+	// capabilities is returned by the Capabilities RPC and indicates what
+	// optional features this driver supports
+	capabilities = &drivers.Capabilities{
+		SendSignals: false,
+		Exec:        false,
+		FSIsolation: cstructs.FSIsolationImage,
+	}
+)
+
+// Driver is a driver for running LXC containers
+type Driver struct {
+	// eventer is used to handle multiplexing of TaskEvents calls such that an
+	// event can be broadcast to all callers
+	eventer *eventer.Eventer
+
+	// config is the driver configuration set by the SetConfig RPC
+	config *Config
+
+	// nomadConfig is the client config from nomad
+	nomadConfig *base.ClientDriverConfig
+
+	// tasks is the in memory datastore mapping taskIDs to rawExecDriverHandles
+	tasks *taskStore
+
+	// ctx is the context for the driver. It is passed to other subsystems to
+	// coordinate shutdown
+	ctx context.Context
+
+	// signalShutdown is called when the driver is shutting down and cancels the
+	// ctx passed to any subsystems
+	signalShutdown context.CancelFunc
+
+	// logger will log to the Nomad agent
+	logger hclog.Logger
+}
+
+// Config is the driver configuration set by the SetConfig RPC call
+type Config struct {
+	// Enabled is set to true to enable the lxc driver
+	Enabled bool `codec:"enabled"`
+
+	AllowVolumes bool `codec:"volumes"`
+
+	Path string `codec:"path"`
+}
+
+// TaskConfig is the driver configuration of a task within a job
+type TaskConfig struct {
+	Template             string   `codec:"template"`
+	Distro               string   `codec:"distro"`
+	Release              string   `codec:"release"`
+	Arch                 string   `codec:"arch"`
+	ImageVariant         string   `codec:"image_variant"`
+	ImageServer          string   `codec:"image_server"`
+	GPGKeyID             string   `codec:"gpg_key_id"`
+	GPGKeyServer         string   `codec:"gpg_key_server"`
+	DisableGPGValidation bool     `codec:"disable_gpg"`
+	FlushCache           bool     `codec:"flush_cache"`
+	ForceCache           bool     `codec:"force_cache"`
+	TemplateArgs         []string `codec:"template_args"`
+	LogLevel             string   `codec:"log_level"`
+	Verbosity            string   `codec:"verbosity"`
+	Volumes              []string `codec:"volumes"`
+}
+
+// TaskState is the state which is encoded in the handle returned in
+// StartTask. This information is needed to rebuild the task state and handler
+// during recovery.
+type TaskState struct {
+	TaskConfig    *drivers.TaskConfig
+	ContainerName string
+	StartedAt     time.Time
+}
+
+// NewLXCDriver returns a new DriverPlugin implementation
+func NewLXCDriver(logger hclog.Logger) drivers.DriverPlugin {
+	ctx, cancel := context.WithCancel(context.Background())
+	logger = logger.Named(pluginName)
+	return &Driver{
+		eventer:        eventer.NewEventer(ctx, logger),
+		config:         &Config{},
+		tasks:          newTaskStore(),
+		ctx:            ctx,
+		signalShutdown: cancel,
+		logger:         logger,
+	}
+}
+
+func (d *Driver) PluginInfo() (*base.PluginInfoResponse, error) {
+	return pluginInfo, nil
+}
+
+func (d *Driver) ConfigSchema() (*hclspec.Spec, error) {
+	return configSpec, nil
+}
+
+func (d *Driver) SetConfig(data []byte, cfg *base.ClientAgentConfig) error {
+	var config Config
+	if err := base.MsgPackDecode(data, &config); err != nil {
+		return err
+	}
+
+	d.config = &config
+	if cfg != nil {
+		d.nomadConfig = cfg.Driver
+	}
+
+	return nil
+}
+
+func (d *Driver) Shutdown(ctx context.Context) error {
+	d.signalShutdown()
+	return nil
+}
+
+func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
+	return taskConfigSpec, nil
+}
+
+func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
+	return capabilities, nil
+}
+
+func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) {
+	ch := make(chan *drivers.Fingerprint)
+	go d.handleFingerprint(ctx, ch)
+	return ch, nil
+}
+
+func (d *Driver) handleFingerprint(ctx context.Context, ch chan<- *drivers.Fingerprint) {
+	defer close(ch)
+	ticker := time.NewTimer(0)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-d.ctx.Done():
+			return
+		case <-ticker.C:
+			ticker.Reset(fingerprintPeriod)
+			ch <- d.buildFingerprint()
+		}
+	}
+}
+
+func (d *Driver) buildFingerprint() *drivers.Fingerprint {
+	var health drivers.HealthState
+	var desc string
+	attrs := map[string]string{}
+
+	lxcVersion := lxc.Version()
+
+	if d.config.Enabled && lxcVersion != "" {
+		health = drivers.HealthStateHealthy
+		desc = "ready"
+		attrs["driver.lxc"] = "1"
+		attrs["driver.lxc.version"] = lxcVersion
+	} else {
+		health = drivers.HealthStateUndetected
+		desc = "disabled"
+	}
+
+	if d.config.AllowVolumes {
+		attrs["driver.lxc.volumes.enabled"] = "1"
+	}
+
+	return &drivers.Fingerprint{
+		Attributes:        attrs,
+		Health:            health,
+		HealthDescription: desc,
+	}
+}
+
+func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error {
+	if handle == nil {
+		return fmt.Errorf("error: handle cannot be nil")
+	}
+
+	if _, ok := d.tasks.Get(handle.Config.ID); ok {
+		return nil
+	}
+
+	var taskState TaskState
+	if err := handle.GetDriverState(&taskState); err != nil {
+		return fmt.Errorf("failed to decode task state from handle: %v", err)
+	}
+
+	var driverConfig TaskConfig
+	if err := taskState.TaskConfig.DecodeDriverConfig(&driverConfig); err != nil {
+		return fmt.Errorf("failed to decode driver config: %v", err)
+	}
+
+	c, err := lxc.NewContainer(taskState.ContainerName, d.lxcPath())
+	if err != nil {
+		return fmt.Errorf("failed to create container ref: %v", err)
+	}
+
+	initPid := c.InitPid()
+	h := &taskHandle{
+		container:  c,
+		initPid:    initPid,
+		taskConfig: taskState.TaskConfig,
+		procState:  drivers.TaskStateRunning,
+		startedAt:  taskState.StartedAt,
+		exitResult: &drivers.ExitResult{},
+
+		totalCpuStats:  stats.NewCpuStats(),
+		userCpuStats:   stats.NewCpuStats(),
+		systemCpuStats: stats.NewCpuStats(),
+	}
+
+	d.tasks.Set(taskState.TaskConfig.ID, h)
+
+	go h.run()
+	return nil
+}
+
+func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstructs.DriverNetwork, error) {
+	if _, ok := d.tasks.Get(cfg.ID); ok {
+		return nil, nil, fmt.Errorf("task with ID %q already started", cfg.ID)
+	}
+
+	var driverConfig TaskConfig
+	if err := cfg.DecodeDriverConfig(&driverConfig); err != nil {
+		return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
+	}
+
+	d.logger.Info("starting lxc task", "driver_cfg", hclog.Fmt("%+v", driverConfig))
+	handle := drivers.NewTaskHandle(pluginName)
+	handle.Config = cfg
+
+	c, err := d.initializeContainer(cfg, driverConfig)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	opt := toLXCCreateOptions(driverConfig)
+	if err := c.Create(opt); err != nil {
+		return nil, nil, fmt.Errorf("unable to create container: %v", err)
+	}
+
+	cleanup := func() {
+		if err := c.Destroy(); err != nil {
+			d.logger.Error("failed to clean up from an error in Start", "error", err)
+		}
+	}
+
+	if err := d.configureContainerNetwork(c); err != nil {
+		cleanup()
+		return nil, nil, err
+	}
+
+	if err := d.mountVolumes(c, cfg, driverConfig); err != nil {
+		cleanup()
+		return nil, nil, err
+	}
+
+	if err := c.Start(); err != nil {
+		cleanup()
+		return nil, nil, fmt.Errorf("unable to start container: %v", err)
+	}
+
+	if err := d.setResourceLimits(c, cfg); err != nil {
+		cleanup()
+		return nil, nil, err
+	}
+
+	pid := c.InitPid()
+
+	h := &taskHandle{
+		container:  c,
+		initPid:    pid,
+		taskConfig: cfg,
+		procState:  drivers.TaskStateRunning,
+		startedAt:  time.Now().Round(time.Millisecond),
+		logger:     d.logger,
+
+		totalCpuStats:  stats.NewCpuStats(),
+		userCpuStats:   stats.NewCpuStats(),
+		systemCpuStats: stats.NewCpuStats(),
+	}
+
+	driverState := TaskState{
+		ContainerName: c.Name(),
+		TaskConfig:    cfg,
+		StartedAt:     h.startedAt,
+	}
+
+	if err := handle.SetDriverState(&driverState); err != nil {
+		d.logger.Error("failed to start task, error setting driver state", "error", err)
+		cleanup()
+		return nil, nil, fmt.Errorf("failed to set driver state: %v", err)
+	}
+
+	d.tasks.Set(cfg.ID, h)
+
+	go h.run()
+
+	return handle, nil, nil
+}
+
+func (d *Driver) WaitTask(ctx context.Context, taskID string) (<-chan *drivers.ExitResult, error) {
+	handle, ok := d.tasks.Get(taskID)
+	if !ok {
+		return nil, drivers.ErrTaskNotFound
+	}
+
+	ch := make(chan *drivers.ExitResult)
+	go d.handleWait(ctx, handle, ch)
+
+	return ch, nil
+}
+
+func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *drivers.ExitResult) {
+	defer close(ch)
+
+	//
+	// Wait for process completion by polling status from handler.
+	// We cannot use the following alternatives:
+	//   * Process.Wait() requires LXC container processes to be children
+	//     of self process; but LXC runs container in separate PID hierarchy
+	//     owned by PID 1.
+	//   * lxc.Container.Wait() holds a write lock on container and prevents
+	//     any other calls, including stats.
+	//
+	// Going with simplest approach of polling for handler to mark exit.
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-d.ctx.Done():
+			return
+		case <-ticker.C:
+			s := handle.TaskStatus()
+			if s.State == drivers.TaskStateExited {
+				ch <- handle.exitResult
+			}
+		}
+	}
+}
+
+func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) error {
+	handle, ok := d.tasks.Get(taskID)
+	if !ok {
+		return drivers.ErrTaskNotFound
+	}
+
+	if err := handle.shutdown(timeout); err != nil {
+		return fmt.Errorf("executor Shutdown failed: %v", err)
+	}
+
+	return nil
+}
+
+func (d *Driver) DestroyTask(taskID string, force bool) error {
+	handle, ok := d.tasks.Get(taskID)
+	if !ok {
+		return drivers.ErrTaskNotFound
+	}
+
+	if handle.IsRunning() && !force {
+		return fmt.Errorf("cannot destroy running task")
+	}
+
+	if handle.IsRunning() {
+		// grace period is chosen arbitrary here
+		if err := handle.shutdown(1 * time.Minute); err != nil {
+			handle.logger.Error("failed to destroy executor", "err", err)
+		}
+	}
+
+	d.tasks.Delete(taskID)
+	return nil
+}
+
+func (d *Driver) InspectTask(taskID string) (*drivers.TaskStatus, error) {
+	handle, ok := d.tasks.Get(taskID)
+	if !ok {
+		return nil, drivers.ErrTaskNotFound
+	}
+
+	return handle.TaskStatus(), nil
+}
+
+func (d *Driver) TaskStats(taskID string) (*cstructs.TaskResourceUsage, error) {
+	handle, ok := d.tasks.Get(taskID)
+	if !ok {
+		return nil, drivers.ErrTaskNotFound
+	}
+
+	return handle.stats()
+}
+
+func (d *Driver) TaskEvents(ctx context.Context) (<-chan *drivers.TaskEvent, error) {
+	return d.eventer.TaskEvents(ctx)
+}
+
+func (d *Driver) SignalTask(taskID string, signal string) error {
+	return fmt.Errorf("LXC driver does not support signals")
+}
+
+func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (*drivers.ExecTaskResult, error) {
+	return nil, fmt.Errorf("LXC driver does not support exec")
+}

drivers/lxc/driver_test.go

@@ -0,0 +1,271 @@
+// +build linux,lxc
+
+package lxc
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/hcl2/hcl"
+	ctestutil "github.com/hashicorp/nomad/client/testutil"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/plugins/drivers"
+	"github.com/hashicorp/nomad/plugins/shared"
+	"github.com/hashicorp/nomad/plugins/shared/hclspec"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/stretchr/testify/require"
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+func TestLXCDriver_Fingerprint(t *testing.T) {
+	t.Parallel()
+	requireLXC(t)
+
+	require := require.New(t)
+
+	d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
+	d.config.Enabled = true
+	harness := drivers.NewDriverHarness(t, d)
+
+	fingerCh, err := harness.Fingerprint(context.Background())
+	require.NoError(err)
+	select {
+	case finger := <-fingerCh:
+		require.Equal(drivers.HealthStateHealthy, finger.Health)
+		require.Equal("1", finger.Attributes["driver.lxc"])
+		require.NotEmpty(finger.Attributes["driver.lxc.version"])
+	case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
+		require.Fail("timeout receiving fingerprint")
+	}
+}
+
+func TestLXCDriver_FingerprintNotEnabled(t *testing.T) {
+	t.Parallel()
+	requireLXC(t)
+
+	require := require.New(t)
+
+	d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
+	d.config.Enabled = false
+	harness := drivers.NewDriverHarness(t, d)
+
+	fingerCh, err := harness.Fingerprint(context.Background())
+	require.NoError(err)
+	select {
+	case finger := <-fingerCh:
+		require.Equal(drivers.HealthStateUndetected, finger.Health)
+		require.Equal("", finger.Attributes["driver.lxc"])
+		require.Empty(finger.Attributes["driver.lxc.version"])
+	case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
+		require.Fail("timeout receiving fingerprint")
+	}
+}
+
+func TestLXCDriver_Start_Wait(t *testing.T) {
+	if !testutil.IsTravis() {
+		t.Parallel()
+	}
+	requireLXC(t)
+	ctestutil.RequireRoot(t)
+
+	require := require.New(t)
+
+	// prepare test file
+	testFileContents := []byte("this should be visible under /mnt/tmp")
+	tmpFile, err := ioutil.TempFile("/tmp", "testlxcdriver_start_wait")
+	if err != nil {
+		t.Fatalf("error writing temp file: %v", err)
+	}
+	defer os.Remove(tmpFile.Name())
+	if _, err := tmpFile.Write(testFileContents); err != nil {
+		t.Fatalf("error writing temp file: %v", err)
+	}
+	if err := tmpFile.Close(); err != nil {
+		t.Fatalf("error closing temp file: %v", err)
+	}
+
+	d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
+	d.config.Enabled = true
+	d.config.AllowVolumes = true
+
+	harness := drivers.NewDriverHarness(t, d)
+	task := &drivers.TaskConfig{
+		ID:   uuid.Generate(),
+		Name: "test",
+		Resources: &drivers.Resources{
+			NomadResources: &structs.Resources{
+				CPU:      1,
+				MemoryMB: 2,
+			},
+			LinuxResources: &drivers.LinuxResources{
+				CPUShares:        1024,
+				MemoryLimitBytes: 2 * 1024,
+			},
+		},
+	}
+	taskConfig := map[string]interface{}{
+		"template": "/usr/share/lxc/templates/lxc-busybox",
+		"volumes":  []string{"/tmp/:mnt/tmp"},
+	}
+	encodeDriverHelper(require, task, taskConfig)
+
+	cleanup := harness.MkAllocDir(task, false)
+	defer cleanup()
+
+	handle, _, err := harness.StartTask(task)
+	require.NoError(err)
+	require.NotNil(handle)
+
+	lxcHandle, ok := d.tasks.Get(task.ID)
+	require.True(ok)
+
+	container := lxcHandle.container
+
+	// Destroy container after test
+	defer func() {
+		container.Stop()
+		container.Destroy()
+	}()
+
+	// Test that container is running
+	testutil.WaitForResult(func() (bool, error) {
+		state := container.State()
+		if state == lxc.RUNNING {
+			return true, nil
+		}
+		return false, fmt.Errorf("container in state: %v", state)
+	}, func(err error) {
+		t.Fatalf("container failed to start: %v", err)
+	})
+
+	// Test that directories are mounted in their proper location
+	containerName := container.Name()
+	for _, mnt := range []string{"alloc", "local", "secrets", "mnt/tmp"} {
+		fullpath := filepath.Join(d.lxcPath(), containerName, "rootfs", mnt)
+		stat, err := os.Stat(fullpath)
+		require.NoError(err)
+		require.True(stat.IsDir())
+	}
+
+	// Test bind mount volumes exist in container:
+	mountedContents, err := exec.Command("lxc-attach",
+		"-n", containerName, "--",
+		"cat", filepath.Join("/mnt/", tmpFile.Name()),
+	).Output()
+	require.NoError(err)
+	require.Equal(string(testFileContents), string(mountedContents))
+
+	// Test that killing container marks container as stopped
+	require.NoError(container.Stop())
+
+	testutil.WaitForResult(func() (bool, error) {
+		status, err := d.InspectTask(task.ID)
+		if err == nil && status.State == drivers.TaskStateExited {
+			return true, nil
+		}
+		return false, fmt.Errorf("task in state: %v", status.State)
+	}, func(err error) {
+		t.Fatalf("task was not marked as stopped: %v", err)
+	})
+}
+
+func TestLXCDriver_Start_Stop(t *testing.T) {
+	if !testutil.IsTravis() {
+		t.Parallel()
+	}
+	requireLXC(t)
+	ctestutil.RequireRoot(t)
+
+	require := require.New(t)
+
+	d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
+	d.config.Enabled = true
+	d.config.AllowVolumes = true
+
+	harness := drivers.NewDriverHarness(t, d)
+	task := &drivers.TaskConfig{
+		ID:   uuid.Generate(),
+		Name: "test",
+		Resources: &drivers.Resources{
+			NomadResources: &structs.Resources{
+				CPU:      1,
+				MemoryMB: 2,
+			},
+			LinuxResources: &drivers.LinuxResources{
+				CPUShares:        1024,
+				MemoryLimitBytes: 2 * 1024,
+			},
+		},
+	}
+	taskConfig := map[string]interface{}{
+		"template": "/usr/share/lxc/templates/lxc-busybox",
+	}
+	encodeDriverHelper(require, task, taskConfig)
+
+	cleanup := harness.MkAllocDir(task, false)
+	defer cleanup()
+
+	handle, _, err := harness.StartTask(task)
+	require.NoError(err)
+	require.NotNil(handle)
+
+	lxcHandle, ok := d.tasks.Get(task.ID)
+	require.True(ok)
+
+	container := lxcHandle.container
+
+	// Destroy container after test
+	defer func() {
+		container.Stop()
+		container.Destroy()
+	}()
+
+	// Test that container is running
+	testutil.WaitForResult(func() (bool, error) {
+		state := container.State()
+		if state == lxc.RUNNING {
+			return true, nil
+		}
+		return false, fmt.Errorf("container in state: %v", state)
+	}, func(err error) {
+		t.Fatalf("container failed to start: %v", err)
+	})
+
+	require.NoError(d.StopTask(task.ID, 5*time.Second, "kill"))
+
+	testutil.WaitForResult(func() (bool, error) {
+		status, err := d.InspectTask(task.ID)
+		if err == nil && status.State == drivers.TaskStateExited {
+			return true, nil
+		}
+		return false, fmt.Errorf("task in state: %v", status.State)
+	}, func(err error) {
+		t.Fatalf("task was not marked as stopped: %v", err)
+	})
+}
+
+func requireLXC(t *testing.T) {
+	if lxc.Version() == "" {
+		t.Skip("skipping, lxc not present")
+	}
+}
+
+func encodeDriverHelper(require *require.Assertions, task *drivers.TaskConfig, taskConfig map[string]interface{}) {
+	evalCtx := &hcl.EvalContext{
+		Functions: shared.GetStdlibFuncs(),
+	}
+	spec, diag := hclspec.Convert(taskConfigSpec)
+	require.False(diag.HasErrors())
+	taskConfigCtyVal, diag := shared.ParseHclInterface(taskConfig, spec, evalCtx)
+	require.False(diag.HasErrors())
+	err := task.EncodeDriverConfig(taskConfigCtyVal)
+	require.Nil(err)
+}

drivers/lxc/handle.go

@@ -0,0 +1,200 @@
+//+build linux,lxc
+
+package lxc
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad/client/stats"
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/plugins/drivers"
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+type taskHandle struct {
+	container *lxc.Container
+	initPid   int
+	logger    hclog.Logger
+
+	totalCpuStats  *stats.CpuStats
+	userCpuStats   *stats.CpuStats
+	systemCpuStats *stats.CpuStats
+
+	// stateLock syncs access to all fields below
+	stateLock sync.RWMutex
+
+	taskConfig  *drivers.TaskConfig
+	procState   drivers.TaskState
+	startedAt   time.Time
+	completedAt time.Time
+	exitResult  *drivers.ExitResult
+}
+
+var (
+	LXCMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
+
+	LXCMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
+)
+
+func (h *taskHandle) TaskStatus() *drivers.TaskStatus {
+	h.stateLock.RLock()
+	defer h.stateLock.RUnlock()
+
+	return &drivers.TaskStatus{
+		ID:          h.taskConfig.ID,
+		Name:        h.taskConfig.Name,
+		State:       h.procState,
+		StartedAt:   h.startedAt,
+		CompletedAt: h.completedAt,
+		ExitResult:  h.exitResult,
+		DriverAttributes: map[string]string{
+			"pid": strconv.Itoa(h.initPid),
+		},
+	}
+}
+
+func (h *taskHandle) IsRunning() bool {
+	h.stateLock.RLock()
+	defer h.stateLock.RUnlock()
+	return h.procState == drivers.TaskStateRunning
+}
+
+func (h *taskHandle) run() {
+	h.stateLock.Lock()
+	if h.exitResult == nil {
+		h.exitResult = &drivers.ExitResult{}
+	}
+	h.stateLock.Unlock()
+
+	if ok, err := waitTillStopped(h.container); !ok {
+		h.logger.Error("failed to find container process", "error", err)
+		return
+	}
+
+	h.stateLock.Lock()
+	defer h.stateLock.Unlock()
+
+	h.procState = drivers.TaskStateExited
+	h.exitResult.ExitCode = 0
+	h.exitResult.Signal = 0
+	h.completedAt = time.Now()
+
+	// TODO: detect if the task OOMed
+}
+
+func (h *taskHandle) stats() (*cstructs.TaskResourceUsage, error) {
+	cpuStats, err := h.container.CPUStats()
+	if err != nil {
+		h.logger.Error("failed to get container cpu stats", "error", err)
+		return nil, nil
+	}
+	total, err := h.container.CPUTime()
+	if err != nil {
+		h.logger.Error("failed to get container cpu time", "error", err)
+		return nil, nil
+	}
+
+	t := time.Now()
+
+	// Get the cpu stats
+	system := cpuStats["system"]
+	user := cpuStats["user"]
+	cs := &cstructs.CpuStats{
+		SystemMode: h.systemCpuStats.Percent(float64(system)),
+		UserMode:   h.systemCpuStats.Percent(float64(user)),
+		Percent:    h.totalCpuStats.Percent(float64(total)),
+		TotalTicks: float64(user + system),
+		Measured:   LXCMeasuredCpuStats,
+	}
+
+	// Get the Memory Stats
+	memData := map[string]uint64{
+		"rss":   0,
+		"cache": 0,
+		"swap":  0,
+	}
+	rawMemStats := h.container.CgroupItem("memory.stat")
+	for _, rawMemStat := range rawMemStats {
+		key, val, err := keysToVal(rawMemStat)
+		if err != nil {
+			h.logger.Error("failed to get stat", "line", rawMemStat, "error", err)
+			continue
+		}
+		if _, ok := memData[key]; ok {
+			memData[key] = val
+
+		}
+	}
+	ms := &cstructs.MemoryStats{
+		RSS:      memData["rss"],
+		Cache:    memData["cache"],
+		Swap:     memData["swap"],
+		Measured: LXCMeasuredMemStats,
+	}
+
+	mu := h.container.CgroupItem("memory.max_usage_in_bytes")
+	for _, rawMemMaxUsage := range mu {
+		val, err := strconv.ParseUint(rawMemMaxUsage, 10, 64)
+		if err != nil {
+			h.logger.Error("failed to get max memory usage", "error", err)
+			continue
+		}
+		ms.MaxUsage = val
+	}
+	ku := h.container.CgroupItem("memory.kmem.usage_in_bytes")
+	for _, rawKernelUsage := range ku {
+		val, err := strconv.ParseUint(rawKernelUsage, 10, 64)
+		if err != nil {
+			h.logger.Error("failed to get kernel memory usage", "error", err)
+			continue
+		}
+		ms.KernelUsage = val
+	}
+
+	mku := h.container.CgroupItem("memory.kmem.max_usage_in_bytes")
+	for _, rawMaxKernelUsage := range mku {
+		val, err := strconv.ParseUint(rawMaxKernelUsage, 10, 64)
+		if err != nil {
+			h.logger.Error("failed tog get max kernel memory usage", "error", err)
+			continue
+		}
+		ms.KernelMaxUsage = val
+	}
+
+	taskResUsage := cstructs.TaskResourceUsage{
+		ResourceUsage: &cstructs.ResourceUsage{
+			CpuStats:    cs,
+			MemoryStats: ms,
+		},
+		Timestamp: t.UTC().UnixNano(),
+	}
+
+	return &taskResUsage, nil
+
+}
+
+func keysToVal(line string) (string, uint64, error) {
+	tokens := strings.Split(line, " ")
+	if len(tokens) != 2 {
+		return "", 0, fmt.Errorf("line isn't a k/v pair")
+	}
+	key := tokens[0]
+	val, err := strconv.ParseUint(tokens[1], 10, 64)
+	return key, val, err
+}
+
+// shutdown shuts down the container, with `timeout` grace period
+// before killing the container with SIGKILL.
+func (h *taskHandle) shutdown(timeout time.Duration) error {
+	err := h.container.Shutdown(timeout)
+	if err == nil {
+		return nil
+	}
+
+	return h.container.Stop()
+}

drivers/lxc/lxc.go

@@ -0,0 +1,170 @@
+//+build linux,lxc
+
+package lxc
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/hashicorp/nomad/plugins/drivers"
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+var (
+	verbosityLevels = map[string]lxc.Verbosity{
+		"":        lxc.Quiet,
+		"verbose": lxc.Verbose,
+		"quiet":   lxc.Quiet,
+	}
+
+	logLevels = map[string]lxc.LogLevel{
+		"":      lxc.ERROR,
+		"debug": lxc.DEBUG,
+		"error": lxc.ERROR,
+		"info":  lxc.INFO,
+		"trace": lxc.TRACE,
+		"warn":  lxc.WARN,
+	}
+)
+
+const (
+	// containerMonitorIntv is the interval at which the driver checks if the
+	// container is still alive
+	containerMonitorIntv = 2 * time.Second
+)
+
+func (d *Driver) lxcPath() string {
+	lxcPath := d.config.Path
+	if lxcPath == "" {
+		lxcPath = lxc.DefaultConfigPath()
+	}
+	return lxcPath
+
+}
+func (d *Driver) initializeContainer(cfg *drivers.TaskConfig, taskConfig TaskConfig) (*lxc.Container, error) {
+	lxcPath := d.lxcPath()
+
+	containerName := fmt.Sprintf("%s-%s", cfg.Name, cfg.AllocID)
+	c, err := lxc.NewContainer(containerName, lxcPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize container: %v", err)
+	}
+
+	if v, ok := verbosityLevels[taskConfig.Verbosity]; ok {
+		c.SetVerbosity(v)
+	} else {
+		return nil, fmt.Errorf("lxc driver config 'verbosity' can only be either quiet or verbose")
+	}
+
+	if v, ok := logLevels[taskConfig.LogLevel]; ok {
+		c.SetLogLevel(v)
+	} else {
+		return nil, fmt.Errorf("lxc driver config 'log_level' can only be trace, debug, info, warn or error")
+	}
+
+	logFile := filepath.Join(cfg.TaskDir().Dir, fmt.Sprintf("%v-lxc.log", cfg.Name))
+	c.SetLogFile(logFile)
+
+	return c, nil
+}
+
+func (d *Driver) configureContainerNetwork(c *lxc.Container) error {
+	// Set the network type to none
+	if err := c.SetConfigItem(networkTypeConfigKey(), "none"); err != nil {
+		return fmt.Errorf("error setting network type configuration: %v", err)
+	}
+	return nil
+}
+
+func networkTypeConfigKey() string {
+	if lxc.VersionAtLeast(2, 1, 0) {
+		return "lxc.net.0.type"
+	}
+
+	// prior to 2.1, network used
+	return "lxc.network.type"
+}
+
+func (d *Driver) mountVolumes(c *lxc.Container, cfg *drivers.TaskConfig, taskConfig TaskConfig) error {
+	// Bind mount the shared alloc dir and task local dir in the container
+	mounts := []string{
+		fmt.Sprintf("%s local none rw,bind,create=dir", cfg.TaskDir().LocalDir),
+		fmt.Sprintf("%s alloc none rw,bind,create=dir", cfg.TaskDir().SharedAllocDir),
+		fmt.Sprintf("%s secrets none rw,bind,create=dir", cfg.TaskDir().SecretsDir),
+	}
+
+	volumesEnabled := d.config.AllowVolumes
+
+	for _, volDesc := range taskConfig.Volumes {
+		// the format was checked in Validate()
+		paths := strings.Split(volDesc, ":")
+
+		if filepath.IsAbs(paths[0]) {
+			if !volumesEnabled {
+				return fmt.Errorf("absolute bind-mount volume in config but volumes are disabled")
+			}
+		} else {
+			// Relative source paths are treated as relative to alloc dir
+			paths[0] = filepath.Join(cfg.TaskDir().Dir, paths[0])
+		}
+
+		mounts = append(mounts, fmt.Sprintf("%s %s none rw,bind,create=dir", paths[0], paths[1]))
+	}
+
+	for _, mnt := range mounts {
+		if err := c.SetConfigItem("lxc.mount.entry", mnt); err != nil {
+			return fmt.Errorf("error setting bind mount %q error: %v", mnt, err)
+		}
+	}
+
+	return nil
+}
+
+func (d *Driver) setResourceLimits(c *lxc.Container, cfg *drivers.TaskConfig) error {
+	if err := c.SetMemoryLimit(lxc.ByteSize(cfg.Resources.NomadResources.MemoryMB) * lxc.MB); err != nil {
+		return fmt.Errorf("unable to set memory limits: %v", err)
+	}
+
+	if err := c.SetCgroupItem("cpu.shares", strconv.FormatInt(cfg.Resources.LinuxResources.CPUShares, 10)); err != nil {
+		return fmt.Errorf("unable to set cpu shares: %v", err)
+	}
+
+	return nil
+}
+
+func toLXCCreateOptions(taskConfig TaskConfig) lxc.TemplateOptions {
+	return lxc.TemplateOptions{
+		Template:             taskConfig.Template,
+		Distro:               taskConfig.Distro,
+		Release:              taskConfig.Release,
+		Arch:                 taskConfig.Arch,
+		FlushCache:           taskConfig.FlushCache,
+		DisableGPGValidation: taskConfig.DisableGPGValidation,
+		ExtraArgs:            taskConfig.TemplateArgs,
+	}
+}
+
+// waitTillStopped blocks and returns true when container stops;
+// returns false with an error message if the container processes cannot be identified.
+//
+// Use this in preference to c.Wait() - lxc Wait() function holds a write lock on the container
+// blocking any other operation on container, including looking up container stats
+func waitTillStopped(c *lxc.Container) (bool, error) {
+	ps, err := os.FindProcess(c.InitPid())
+	if err != nil {
+		return false, err
+	}
+
+	for {
+		if err := ps.Signal(syscall.Signal(0)); err != nil {
+			return true, nil
+		}
+
+		time.Sleep(containerMonitorIntv)
+	}
+}

drivers/lxc/state.go

@@ -0,0 +1,35 @@
+//+build linux,lxc
+
+package lxc
+
+import (
+	"sync"
+)
+
+type taskStore struct {
+	store map[string]*taskHandle
+	lock  sync.RWMutex
+}
+
+func newTaskStore() *taskStore {
+	return &taskStore{store: map[string]*taskHandle{}}
+}
+
+func (ts *taskStore) Set(id string, handle *taskHandle) {
+	ts.lock.Lock()
+	defer ts.lock.Unlock()
+	ts.store[id] = handle
+}
+
+func (ts *taskStore) Get(id string) (*taskHandle, bool) {
+	ts.lock.RLock()
+	defer ts.lock.RUnlock()
+	t, ok := ts.store[id]
+	return t, ok
+}
+
+func (ts *taskStore) Delete(id string) {
+	ts.lock.Lock()
+	defer ts.lock.Unlock()
+	delete(ts.store, id)
+}

plugins/shared/catalog/register_lxc.go

@@ -0,0 +1,14 @@
+//+build linux,lxc
+
+package catalog
+
+import (
+	"github.com/hashicorp/nomad/drivers/lxc"
+)
+
+// This file is where all builtin plugins should be registered in the catalog.
+// Plugins with build restrictions should be placed in the appropriate
+// register_XXX.go file.
+func init() {
+	RegisterDeferredConfig(lxc.PluginID, lxc.PluginConfig, lxc.PluginLoader)
+}

vendor/gopkg.in/lxc/go-lxc.v2/Makefile

@@ -45,4 +45,16 @@ escape-analysis:
 ctags:
 	@ctags -R --languages=c,go
 
+scope:
+	@echo "$(OK_COLOR)==> Exported container calls in container.go $(NO_COLOR)"
+	@/bin/grep -E "\bc+\.([A-Z])\w+" container.go || true
+
+setup-test-cgroup:
+	for d in /sys/fs/cgroup/*; do \
+	    [ -f $$d/cgroup.clone_children ] && echo 1 | sudo tee $$d/cgroup.clone_children; \
+	    [ -f $$d/cgroup.use_hierarchy ] && echo 1 | sudo tee $$d/cgroup.use_hierarchy; \
+	    sudo mkdir -p $$d/lxc; \
+	    sudo chown -R $$USER: $$d/lxc; \
+	done
+
 .PHONY: all format test doc vet lint ctags

vendor/gopkg.in/lxc/go-lxc.v2/error.go

@@ -16,6 +16,7 @@ var (
 	ErrAttachInterfaceFailed         = NewError("attaching specified netdev to the container failed")
 	ErrBlkioUsage                    = NewError("BlkioUsage for the container failed")
 	ErrCheckpointFailed              = NewError("checkpoint failed")
+	ErrClearingConfigItemFailed      = NewError("clearing config item for the container failed")
 	ErrClearingCgroupItemFailed      = NewError("clearing cgroup item for the container failed")
 	ErrCloneFailed                   = NewError("cloning the container failed")
 	ErrCloseAllFdsFailed             = NewError("setting close_all_fds flag for container failed")

vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.c

@@ -15,10 +15,9 @@
 
 #include "lxc-binding.h"
 
-#define VERSION_AT_LEAST(major, minor, micro)							\
+#ifndef LXC_DEVEL
-	(!(major > LXC_VERSION_MAJOR ||								\
+#define LXC_DEVEL 0
-	major == LXC_VERSION_MAJOR && minor > LXC_VERSION_MINOR ||				\
+#endif
-	major == LXC_VERSION_MAJOR && minor == LXC_VERSION_MINOR && micro > LXC_VERSION_MICRO))
 
 bool go_lxc_defined(struct lxc_container *c) {
 	return c->is_defined(c);
@@ -92,16 +91,24 @@ bool go_lxc_wait(struct lxc_container *c, const char *state, int timeout) {
 	return c->wait(c, state, timeout);
 }
 
-char* go_lxc_get_config_item(struct lxc_container *c, const char *key) {
+char *go_lxc_get_config_item(struct lxc_container *c, const char *key)
+{
+	char *value = NULL;
+
 	int len = c->get_config_item(c, key, NULL, 0);
-	if (len <= 0) {
+	if (len <= 0)
+		return NULL;
+
+again:
+	value = (char *)malloc(sizeof(char) * len + 1);
+	if (value == NULL)
+		goto again;
+
+	if (c->get_config_item(c, key, value, len + 1) != len) {
+		free(value);
 		return NULL;
 	}
 
-	char* value = (char*)malloc(sizeof(char)*len + 1);
-	if (c->get_config_item(c, key, value, len + 1) != len) {
-		return NULL;
-	}
 	return value;
 }
 
@@ -121,29 +128,45 @@ char* go_lxc_get_running_config_item(struct lxc_container *c, const char *key) {
 	return c->get_running_config_item(c, key);
 }
 
-char* go_lxc_get_keys(struct lxc_container *c, const char *key) {
+char *go_lxc_get_keys(struct lxc_container *c, const char *key)
+{
+	char *value = NULL;
+
 	int len = c->get_keys(c, key, NULL, 0);
-	if (len <= 0) {
+	if (len <= 0)
+		return NULL;
+
+again:
+	value = (char *)malloc(sizeof(char) * len + 1);
+	if (value == NULL)
+		goto again;
+
+	if (c->get_keys(c, key, value, len + 1) != len) {
+		free(value);
 		return NULL;
 	}
 
-	char* value = (char*)malloc(sizeof(char)*len + 1);
-	if (c->get_keys(c, key, value, len + 1) != len) {
-		return NULL;
-	}
 	return value;
 }
 
-char* go_lxc_get_cgroup_item(struct lxc_container *c, const char *key) {
+char *go_lxc_get_cgroup_item(struct lxc_container *c, const char *key)
+{
+	char *value = NULL;
+
 	int len = c->get_cgroup_item(c, key, NULL, 0);
-	if (len <= 0) {
+	if (len <= 0)
+		return NULL;
+
+again:
+	value = (char *)malloc(sizeof(char) * len + 1);
+	if (value == NULL)
+		goto again;
+
+	if (c->get_cgroup_item(c, key, value, len + 1) != len) {
+		free(value);
 		return NULL;
 	}
 
-	char* value = (char*)malloc(sizeof(char)*len + 1);
-	if (c->get_cgroup_item(c, key, value, len + 1) != len) {
-		return NULL;
-	}
 	return value;
 }
 
@@ -173,10 +196,12 @@ bool go_lxc_clone(struct lxc_container *c, const char *newname, const char *lxcp
 
 int go_lxc_console_getfd(struct lxc_container *c, int ttynum) {
 	int masterfd;
+	int ret = 0;
+
+	ret = c->console_getfd(c, &ttynum, &masterfd);
+	if (ret < 0)
+		return ret;
 
-	if (c->console_getfd(c, &ttynum, &masterfd) < 0) {
-		return -1;
-	}
 	return masterfd;
 }
 
@@ -212,6 +237,51 @@ again:
         return status;
 }
 
+int go_lxc_attach_no_wait(struct lxc_container *c,
+		bool clear_env,
+		int namespaces,
+		long personality,
+		uid_t uid, gid_t gid,
+		int stdinfd, int stdoutfd, int stderrfd,
+		char *initial_cwd,
+		char **extra_env_vars,
+		char **extra_keep_env,
+		const char * const argv[],
+		pid_t *attached_pid) {
+	int ret;
+
+	lxc_attach_options_t attach_options = LXC_ATTACH_OPTIONS_DEFAULT;
+	lxc_attach_command_t command = (lxc_attach_command_t){.program = NULL};
+
+	attach_options.env_policy = LXC_ATTACH_KEEP_ENV;
+	if (clear_env) {
+		attach_options.env_policy = LXC_ATTACH_CLEAR_ENV;
+	}
+
+	attach_options.namespaces = namespaces;
+	attach_options.personality = personality;
+
+	attach_options.uid = uid;
+	attach_options.gid = gid;
+
+	attach_options.stdin_fd = stdinfd;
+	attach_options.stdout_fd = stdoutfd;
+	attach_options.stderr_fd = stderrfd;
+
+	attach_options.initial_cwd = initial_cwd;
+	attach_options.extra_env_vars = extra_env_vars;
+	attach_options.extra_keep_env = extra_keep_env;
+
+	command.program = (char *)argv[0];
+	command.argv = (char **)argv;
+
+	ret = c->attach(c, lxc_attach_run_command, &command, &attach_options, attached_pid);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
 int go_lxc_attach(struct lxc_container *c,
 		bool clear_env,
 		int namespaces,
@@ -257,16 +327,16 @@ int go_lxc_attach(struct lxc_container *c,
 
 	ret = c->attach(c, lxc_attach_run_shell, NULL, &attach_options, &pid);
 	if (ret < 0)
-		return -1;
+		return ret;
 
 	ret = wait_for_pid_status(pid);
 	if (ret < 0)
-		return -1;
+		return ret;
 
 	if (WIFEXITED(ret))
 		return WEXITSTATUS(ret);
 
-	return -1;
+	return ret;
 }
 
 int go_lxc_attach_run_wait(struct lxc_container *c,
@@ -366,6 +436,9 @@ bool go_lxc_restore(struct lxc_container *c, char *directory, bool verbose) {
 }
 
 int go_lxc_migrate(struct lxc_container *c, unsigned int cmd, struct migrate_opts *opts, struct extra_migrate_opts *extras) {
+#if VERSION_AT_LEAST(3, 0, 0)
+	opts->features_to_check = extras->features_to_check;
+#endif
 #if VERSION_AT_LEAST(2, 0, 4)
 	opts->action_script = extras->action_script;
 	opts->ghost_limit = extras->ghost_limit;
@@ -397,3 +470,34 @@ bool go_lxc_detach_interface(struct lxc_container *c, const char *dev, const cha
 	return false;
 #endif
 }
+
+bool go_lxc_config_item_is_supported(const char *key)
+{
+#if VERSION_AT_LEAST(2, 1, 0)
+	return lxc_config_item_is_supported(key);
+#else
+	return false;
+#endif
+}
+
+int go_lxc_error_num(struct lxc_container *c)
+{
+	return c->error_num;
+}
+
+int go_lxc_console_log(struct lxc_container *c, struct lxc_console_log *log) {
+#if VERSION_AT_LEAST(3, 0, 0)
+	return c->console_log(c, log);
+#else
+	return false;
+#endif
+}
+
+bool go_lxc_has_api_extension(const char *extension)
+{
+#if VERSION_AT_LEAST(3, 1, 0)
+	return lxc_has_api_extension(extension);
+#else
+	return false;
+#endif
+}

vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.go

@@ -11,10 +11,16 @@ package lxc
 // #include <lxc/lxccontainer.h>
 // #include <lxc/version.h>
 // #include "lxc-binding.h"
+// #ifndef LXC_DEVEL
+// #define LXC_DEVEL 0
+// #endif
 import "C"
 
 import (
+	"fmt"
 	"runtime"
+	"strconv"
+	"strings"
 	"unsafe"
 )
 
@@ -54,12 +60,22 @@ func Release(c *Container) bool {
 	// http://golang.org/pkg/runtime/#SetFinalizer
 	runtime.SetFinalizer(c, nil)
 
+	// Go is bad at refcounting sometimes
+	c.mu.Lock()
+
 	return C.lxc_container_put(c.container) == 1
 }
 
 // Version returns the LXC version.
 func Version() string {
-	return C.GoString(C.lxc_get_version())
+	version := C.GoString(C.lxc_get_version())
+
+	// New liblxc versions append "-devel" when LXC_DEVEL is set.
+	if strings.HasSuffix(version, "-devel") {
+		return fmt.Sprintf("%s (devel)", version[:(len(version)-len("-devel"))])
+	}
+
+	return version
 }
 
 // GlobalConfigItem returns the value of the given global config key.
@@ -108,12 +124,12 @@ func ContainerNames(lxcpath ...string) []string {
 
 // Containers returns the defined and active containers on the system. Only
 // containers that could retrieved successfully are returned.
-func Containers(lxcpath ...string) []Container {
+func Containers(lxcpath ...string) []*Container {
-	var containers []Container
+	var containers []*Container
 
 	for _, v := range ContainerNames(lxcpath...) {
 		if container, err := NewContainer(v, lxcpath...); err == nil {
-			containers = append(containers, *container)
+			containers = append(containers, container)
 		}
 	}
 
@@ -143,12 +159,12 @@ func DefinedContainerNames(lxcpath ...string) []string {
 
 // DefinedContainers returns the defined containers on the system.  Only
 // containers that could retrieved successfully are returned.
-func DefinedContainers(lxcpath ...string) []Container {
+func DefinedContainers(lxcpath ...string) []*Container {
-	var containers []Container
+	var containers []*Container
 
 	for _, v := range DefinedContainerNames(lxcpath...) {
 		if container, err := NewContainer(v, lxcpath...); err == nil {
-			containers = append(containers, *container)
+			containers = append(containers, container)
 		}
 	}
 
@@ -178,18 +194,19 @@ func ActiveContainerNames(lxcpath ...string) []string {
 
 // ActiveContainers returns the active containers on the system. Only
 // containers that could retrieved successfully are returned.
-func ActiveContainers(lxcpath ...string) []Container {
+func ActiveContainers(lxcpath ...string) []*Container {
-	var containers []Container
+	var containers []*Container
 
 	for _, v := range ActiveContainerNames(lxcpath...) {
 		if container, err := NewContainer(v, lxcpath...); err == nil {
-			containers = append(containers, *container)
+			containers = append(containers, container)
 		}
 	}
 
 	return containers
 }
 
+// VersionNumber returns the LXC version.
 func VersionNumber() (major int, minor int) {
 	major = C.LXC_VERSION_MAJOR
 	minor = C.LXC_VERSION_MINOR
@@ -197,7 +214,12 @@ func VersionNumber() (major int, minor int) {
 	return
 }
 
+// VersionAtLeast returns true when the tested version >= current version.
 func VersionAtLeast(major int, minor int, micro int) bool {
+	if C.LXC_DEVEL == 1 {
+		return true
+	}
+
 	if major > C.LXC_VERSION_MAJOR {
 		return false
 	}
@@ -215,3 +237,90 @@ func VersionAtLeast(major int, minor int, micro int) bool {
 
 	return true
 }
+
+// IsSupportedConfigItem returns true if the key belongs to a supported config item.
+func IsSupportedConfigItem(key string) bool {
+	configItem := C.CString(key)
+	defer C.free(unsafe.Pointer(configItem))
+	return bool(C.go_lxc_config_item_is_supported(configItem))
+}
+
+// runtimeLiblxcVersionAtLeast checks if the system's liblxc matches the
+// provided version requirement
+func runtimeLiblxcVersionAtLeast(major int, minor int, micro int) bool {
+	version := Version()
+	version = strings.Replace(version, " (devel)", "-devel", 1)
+	parts := strings.Split(version, ".")
+	partsLen := len(parts)
+	if partsLen == 0 {
+		return false
+	}
+
+	develParts := strings.Split(parts[partsLen-1], "-")
+	if len(develParts) == 2 && develParts[1] == "devel" {
+		return true
+	}
+
+	maj := -1
+	min := -1
+	mic := -1
+
+	for i, v := range parts {
+		if i > 2 {
+			break
+		}
+
+		num, err := strconv.Atoi(v)
+		if err != nil {
+			return false
+		}
+
+		switch i {
+		case 0:
+			maj = num
+		case 1:
+			min = num
+		case 2:
+			mic = num
+		}
+	}
+
+	/* Major version is greater. */
+	if maj > major {
+		return true
+	}
+
+	if maj < major {
+		return false
+	}
+
+	/* Minor number is greater.*/
+	if min > minor {
+		return true
+	}
+
+	if min < minor {
+		return false
+	}
+
+	/* Patch number is greater. */
+	if mic > micro {
+		return true
+	}
+
+	if mic < micro {
+		return false
+	}
+
+	return true
+}
+
+// HasApiExtension returns true if the extension is supported.
+func HasApiExtension(extension string) bool {
+	if runtimeLiblxcVersionAtLeast(3, 1, 0) {
+		apiExtension := C.CString(extension)
+		defer C.free(unsafe.Pointer(apiExtension))
+		return bool(C.go_lxc_has_api_extension(apiExtension))
+	}
+	return false
+}

vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.h

@@ -2,6 +2,11 @@
 // Use of this source code is governed by a LGPLv2.1
 // license that can be found in the LICENSE file.
 
+#define VERSION_AT_LEAST(major, minor, micro)							\
+	((LXC_DEVEL == 1) || (!(major > LXC_VERSION_MAJOR ||					\
+	major == LXC_VERSION_MAJOR && minor > LXC_VERSION_MINOR ||				\
+	major == LXC_VERSION_MAJOR && minor == LXC_VERSION_MINOR && micro > LXC_VERSION_MICRO)))
+
 extern bool go_lxc_add_device_node(struct lxc_container *c, const char *src_path, const char *dest_path);
 extern void go_lxc_clear_config(struct lxc_container *c);
 extern bool go_lxc_clear_config_item(struct lxc_container *c, const char *key);
@@ -60,19 +65,32 @@ extern int go_lxc_attach(struct lxc_container *c,
 		char *initial_cwd,
 		char **extra_env_vars,
 		char **extra_keep_env);
+extern int go_lxc_attach_no_wait(struct lxc_container *c,
+		bool clear_env,
+		int namespaces,
+		long personality,
+		uid_t uid, gid_t gid,
+		int stdinfd, int stdoutfd, int stderrfd,
+		char *initial_cwd,
+		char **extra_env_vars,
+		char **extra_keep_env,
+		const char * const argv[],
+		pid_t *attached_pid);
 extern int go_lxc_console_getfd(struct lxc_container *c, int ttynum);
 extern int go_lxc_snapshot_list(struct lxc_container *c, struct lxc_snapshot **ret);
 extern int go_lxc_snapshot(struct lxc_container *c);
 extern pid_t go_lxc_init_pid(struct lxc_container *c);
 extern bool go_lxc_checkpoint(struct lxc_container *c, char *directory, bool stop, bool verbose);
 extern bool go_lxc_restore(struct lxc_container *c, char *directory, bool verbose);
+extern bool go_lxc_config_item_is_supported(const char *key);
+extern bool go_lxc_has_api_extension(const char *extension);
 
 /* n.b. that we're just adding the fields here to shorten the definition
  * of go_lxc_migrate; in the case where we don't have the ->migrate API call,
  * we don't want to have to pass all the arguments in to let conditional
  * compilation handle things, but the call will still fail
  */
-#if LXC_VERSION_MAJOR != 2
+#if !VERSION_AT_LEAST(2, 0, 0)
 struct migrate_opts {
 	char *directory;
 	bool verbose;
@@ -89,8 +107,21 @@ struct extra_migrate_opts {
 	bool preserves_inodes;
 	char *action_script;
 	uint64_t ghost_limit;
+	uint64_t features_to_check;
 };
 int go_lxc_migrate(struct lxc_container *c, unsigned int cmd, struct migrate_opts *opts, struct extra_migrate_opts *extras);
 
 extern bool go_lxc_attach_interface(struct lxc_container *c, const char *dev, const char *dst_dev);
 extern bool go_lxc_detach_interface(struct lxc_container *c, const char *dev, const char *dst_dev);
+
+#if !VERSION_AT_LEAST(3, 0, 0)
+struct lxc_console_log {
+	bool clear;
+	bool read;
+	uint64_t *read_max;
+	char *data;
+};
+#endif
+
+extern int go_lxc_console_log(struct lxc_container *c, struct lxc_console_log *log);
+extern int go_lxc_error_num(struct lxc_container *c);

vendor/gopkg.in/lxc/go-lxc.v2/options.go

@@ -142,7 +142,7 @@ type ConsoleOptions struct {
 	EscapeCharacter rune
 }
 
-// DefailtConsoleOptions is a convenient set of options to be used.
+// DefaultConsoleOptions is a convenient set of options to be used.
 var DefaultConsoleOptions = ConsoleOptions{
 	Tty:             -1,
 	StdinFd:         os.Stdin.Fd(),
@@ -175,25 +175,35 @@ var DefaultCloneOptions = CloneOptions{
 	Backend: Directory,
 }
 
-// CheckpointOptions type is used for defining checkpoint options for CRIU
+// CheckpointOptions type is used for defining checkpoint options for CRIU.
 type CheckpointOptions struct {
 	Directory string
 	Stop      bool
 	Verbose   bool
 }
 
-// RestoreOptions type is used for defining restore options for CRIU
+// RestoreOptions type is used for defining restore options for CRIU.
 type RestoreOptions struct {
 	Directory string
 	Verbose   bool
 }
 
+// MigrateOptions type is used for defining migrate options.
 type MigrateOptions struct {
 	Directory       string
+	PredumpDir      string
+	ActionScript    string
 	Verbose         bool
 	Stop            bool
-	PredumpDir      string
 	PreservesInodes bool
-	ActionScript    string
 	GhostLimit      uint64
+	FeaturesToCheck CriuFeatures
+}
+
+// ConsoleLogOptioins type is used for defining console log options.
+type ConsoleLogOptions struct {
+	ClearLog       bool
+	ReadLog        bool
+	ReadMax        uint64
+	WriteToLogFile bool
 }

vendor/gopkg.in/lxc/go-lxc.v2/type.go

@@ -260,7 +260,15 @@ const (
 )
 
 const (
-	MIGRATE_PRE_DUMP = 0
+	MIGRATE_PRE_DUMP      = 0
-	MIGRATE_DUMP     = 1
+	MIGRATE_DUMP          = 1
-	MIGRATE_RESTORE  = 2
+	MIGRATE_RESTORE       = 2
+	MIGRATE_FEATURE_CHECK = 3
+)
+
+type CriuFeatures uint64
+
+const (
+	FEATURE_MEM_TRACK CriuFeatures = 1 << iota
+	FEATURE_LAZY_PAGES
 )

vendor/gopkg.in/lxc/go-lxc.v2/util.go

@@ -12,7 +12,7 @@ package lxc
 
 static char** makeCharArray(size_t size) {
     // caller checks return value
-    return calloc(sizeof(char*), size);
+    return calloc(size, sizeof(char*));
 }
 
 static void setArrayString(char **array, char *string, size_t n) {

vendor/vendor.json

@@ -434,7 +434,7 @@
 		{"path":"google.golang.org/grpc/transport","checksumSHA1":"oFGr0JoquaPGVnV86fVL8MVTc3A=","revision":"0c41876308d45bc82e587965971e28be659a1aca","revisionTime":"2017-07-21T17:58:12Z"},
 		{"path":"gopkg.in/fsnotify.v1","checksumSHA1":"eIhF+hmL/XZhzTiAwhLD0M65vlY=","revision":"629574ca2a5df945712d3079857300b5e4da0236","revisionTime":"2016-10-11T02:33:12Z"},
 		{"path":"gopkg.in/inf.v0","checksumSHA1":"6f8MEU31llHM1sLM/GGH4/Qxu0A=","revision":"3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4","revisionTime":"2015-09-11T12:57:57Z"},
-		{"path":"gopkg.in/lxc/go-lxc.v2","checksumSHA1":"i97goLq3AIfUNB8l1hxGGMSW0+s=","revision":"f8a6938e600c634232eeef79dc04a1226f73a88b","revisionTime":"2016-08-03T16:52:18Z"},
+		{"path":"gopkg.in/lxc/go-lxc.v2","checksumSHA1":"oAflbBrzWC7OMmZQixkp9bnPQW8=","revision":"0aadfc37157c2e3f0e63bedd10f8615e66e91cad","revisionTime":"2018-11-01T16:03:35Z"},
 		{"path":"gopkg.in/tomb.v1","checksumSHA1":"TO8baX+t1Qs7EmOYth80MkbKzFo=","revision":"dd632973f1e7218eb1089048e0798ec9ae7dceb8","revisionTime":"2014-10-24T13:56:13Z"},
 		{"path":"gopkg.in/tomb.v2","checksumSHA1":"WiyCOMvfzRdymImAJ3ME6aoYUdM=","revision":"14b3d72120e8d10ea6e6b7f87f7175734b1faab8","revisionTime":"2014-06-26T14:46:23Z"},
 		{"path":"gopkg.in/yaml.v2","checksumSHA1":"12GqsW8PiRPnezDDy0v4brZrndM=","revision":"a5b47d31c556af34a302ce5d659e6fea44d90de0","revisionTime":"2016-09-28T15:37:09Z"}