remove lxc references
This commit is contained in:
Mahmood Ali
parent
426c981c34
commit
62a7f951c0
22
GNUmakefile
22
GNUmakefile
|
|
@ -16,14 +16,9 @@ ifeq (,$(findstring $(THIS_OS),Darwin Linux FreeBSD))
|
||||||
$(error Building Nomad is currently only supported on Darwin and Linux.)
|
$(error Building Nomad is currently only supported on Darwin and Linux.)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
# On Linux we build for Linux, Windows, and potentially Linux+LXC
|
# On Linux we build for Linux and Windows
|
||||||
ifeq (Linux,$(THIS_OS))
|
ifeq (Linux,$(THIS_OS))
|
||||||
|
|
||||||
# Detect if we have LXC on the path
|
|
||||||
ifeq (0,$(shell pkg-config --exists lxc; echo $$?))
|
|
||||||
HAS_LXC="true"
|
|
||||||
endif
|
|
||||||
|
|
||||||
ifeq ($(TRAVIS),true)
|
ifeq ($(TRAVIS),true)
|
||||||
$(info Running in Travis, verbose mode is disabled)
|
$(info Running in Travis, verbose mode is disabled)
|
||||||
else
|
else
|
||||||
|
|
@ -38,9 +33,6 @@ ALL_TARGETS += linux_386 \
|
||||||
windows_386 \
|
windows_386 \
|
||||||
windows_amd64
|
windows_amd64
|
||||||
|
|
||||||
ifeq ("true",$(HAS_LXC))
|
|
||||||
ALL_TARGETS += linux_amd64-lxc
|
|
||||||
endif
|
|
||||||
endif
|
endif
|
||||||
|
|
||||||
# On MacOS, we only build for MacOS
|
# On MacOS, we only build for MacOS
|
||||||
|
|
@ -122,14 +114,6 @@ pkg/windows_amd64/nomad: $(SOURCE_FILES) ## Build Nomad for windows/amd64
|
||||||
-tags "$(GO_TAGS)" \
|
-tags "$(GO_TAGS)" \
|
||||||
-o "$@.exe"
|
-o "$@.exe"
|
||||||
|
|
||||||
pkg/linux_amd64-lxc/nomad: $(SOURCE_FILES) ## Build Nomad+LXC for linux/amd64
|
|
||||||
@echo "==> Building $@ with tags $(GO_TAGS)..."
|
|
||||||
@CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \
|
|
||||||
go build \
|
|
||||||
-ldflags $(GO_LDFLAGS) \
|
|
||||||
-tags "$(GO_TAGS) lxc" \
|
|
||||||
-o "$@"
|
|
||||||
|
|
||||||
# Define package targets for each of the build targets we actually have on this system
|
# Define package targets for each of the build targets we actually have on this system
|
||||||
define makePackageTarget
|
define makePackageTarget
|
||||||
|
|
||||||
|
|
@ -222,7 +206,7 @@ changelogfmt:
|
||||||
dev: GOOS=$(shell go env GOOS)
|
dev: GOOS=$(shell go env GOOS)
|
||||||
dev: GOARCH=$(shell go env GOARCH)
|
dev: GOARCH=$(shell go env GOARCH)
|
||||||
dev: GOPATH=$(shell go env GOPATH)
|
dev: GOPATH=$(shell go env GOPATH)
|
||||||
dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)$(if $(HAS_LXC),-lxc)/nomad
|
dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)/nomad
|
||||||
dev: vendorfmt changelogfmt ## Build for the current development platform
|
dev: vendorfmt changelogfmt ## Build for the current development platform
|
||||||
@echo "==> Removing old development build..."
|
@echo "==> Removing old development build..."
|
||||||
@rm -f $(PROJECT_ROOT)/$(DEV_TARGET)
|
@rm -f $(PROJECT_ROOT)/$(DEV_TARGET)
|
||||||
|
|
@ -268,7 +252,7 @@ test-nomad: dev ## Run Nomad test suites
|
||||||
$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
|
$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
|
||||||
-cover \
|
-cover \
|
||||||
-timeout=15m \
|
-timeout=15m \
|
||||||
-tags="$(if $(HAS_LXC),lxc)" ./... $(if $(VERBOSE), >test.log ; echo $$? > exit-code)
|
./... $(if $(VERBOSE), >test.log ; echo $$? > exit-code)
|
||||||
@if [ $(VERBOSE) ] ; then \
|
@if [ $(VERBOSE) ] ; then \
|
||||||
bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \
|
bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -1,513 +0,0 @@
|
||||||
//+build linux,lxc
|
|
||||||
|
|
||||||
package lxc
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/hashicorp/go-hclog"
|
|
||||||
"github.com/hashicorp/nomad/client/stats"
|
|
||||||
"github.com/hashicorp/nomad/drivers/shared/eventer"
|
|
||||||
"github.com/hashicorp/nomad/plugins/base"
|
|
||||||
"github.com/hashicorp/nomad/plugins/drivers"
|
|
||||||
"github.com/hashicorp/nomad/plugins/shared/hclspec"
|
|
||||||
"github.com/hashicorp/nomad/plugins/shared/loader"
|
|
||||||
pstructs "github.com/hashicorp/nomad/plugins/shared/structs"
|
|
||||||
"gopkg.in/lxc/go-lxc.v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// pluginName is the name of the plugin
|
|
||||||
pluginName = "lxc"
|
|
||||||
|
|
||||||
// fingerprintPeriod is the interval at which the driver will send fingerprint responses
|
|
||||||
fingerprintPeriod = 30 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
// PluginID is the rawexec plugin metadata registered in the plugin
|
|
||||||
// catalog.
|
|
||||||
PluginID = loader.PluginID{
|
|
||||||
Name: pluginName,
|
|
||||||
PluginType: base.PluginTypeDriver,
|
|
||||||
}
|
|
||||||
|
|
||||||
// PluginConfig is the rawexec factory function registered in the
|
|
||||||
// plugin catalog.
|
|
||||||
PluginConfig = &loader.InternalPluginConfig{
|
|
||||||
Config: map[string]interface{}{},
|
|
||||||
Factory: func(l hclog.Logger) interface{} { return NewLXCDriver(l) },
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
// PluginLoader maps pre-0.9 client driver options to post-0.9 plugin options.
|
|
||||||
func PluginLoader(opts map[string]string) (map[string]interface{}, error) {
|
|
||||||
conf := map[string]interface{}{}
|
|
||||||
if v, err := strconv.ParseBool(opts["driver.lxc.enable"]); err == nil {
|
|
||||||
conf["enabled"] = v
|
|
||||||
}
|
|
||||||
if v, err := strconv.ParseBool(opts["lxc.volumes.enabled"]); err == nil {
|
|
||||||
conf["volumes"] = v
|
|
||||||
}
|
|
||||||
if v, ok := opts["driver.lxc.path"]; ok {
|
|
||||||
conf["path"] = v
|
|
||||||
}
|
|
||||||
return conf, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
// pluginInfo is the response returned for the PluginInfo RPC
|
|
||||||
pluginInfo = &base.PluginInfoResponse{
|
|
||||||
Type: base.PluginTypeDriver,
|
|
||||||
PluginApiVersions: []string{drivers.ApiVersion010},
|
|
||||||
PluginVersion: "0.1.0",
|
|
||||||
Name: pluginName,
|
|
||||||
}
|
|
||||||
|
|
||||||
// configSpec is the hcl specification returned by the ConfigSchema RPC
|
|
||||||
configSpec = hclspec.NewObject(map[string]*hclspec.Spec{
|
|
||||||
"enabled": hclspec.NewDefault(
|
|
||||||
hclspec.NewAttr("enabled", "bool", false),
|
|
||||||
hclspec.NewLiteral("true"),
|
|
||||||
),
|
|
||||||
"volumes": hclspec.NewDefault(
|
|
||||||
hclspec.NewAttr("volumes", "bool", false),
|
|
||||||
hclspec.NewLiteral("true"),
|
|
||||||
),
|
|
||||||
"path": hclspec.NewDefault(
|
|
||||||
hclspec.NewAttr("path", "string", false),
|
|
||||||
hclspec.NewLiteral("\"\""),
|
|
||||||
),
|
|
||||||
})
|
|
||||||
|
|
||||||
// taskConfigSpec is the hcl specification for the driver config section of
|
|
||||||
// a task within a job. It is returned in the TaskConfigSchema RPC
|
|
||||||
taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{
|
|
||||||
"template": hclspec.NewAttr("template", "string", true),
|
|
||||||
"distro": hclspec.NewAttr("distro", "string", false),
|
|
||||||
"release": hclspec.NewAttr("release", "string", false),
|
|
||||||
"arch": hclspec.NewAttr("arch", "string", false),
|
|
||||||
"image_variant": hclspec.NewAttr("image_variant", "string", false),
|
|
||||||
"image_server": hclspec.NewAttr("image_server", "string", false),
|
|
||||||
"gpg_key_id": hclspec.NewAttr("gpg_key_id", "string", false),
|
|
||||||
"gpg_key_server": hclspec.NewAttr("gpg_key_server", "string", false),
|
|
||||||
"disable_gpg": hclspec.NewAttr("disable_gpg", "string", false),
|
|
||||||
"flush_cache": hclspec.NewAttr("flush_cache", "string", false),
|
|
||||||
"force_cache": hclspec.NewAttr("force_cache", "string", false),
|
|
||||||
"template_args": hclspec.NewAttr("template_args", "list(string)", false),
|
|
||||||
"log_level": hclspec.NewAttr("log_level", "string", false),
|
|
||||||
"verbosity": hclspec.NewAttr("verbosity", "string", false),
|
|
||||||
"volumes": hclspec.NewAttr("volumes", "list(string)", false),
|
|
||||||
})
|
|
||||||
|
|
||||||
// capabilities is returned by the Capabilities RPC and indicates what
|
|
||||||
// optional features this driver supports
|
|
||||||
capabilities = &drivers.Capabilities{
|
|
||||||
SendSignals: false,
|
|
||||||
Exec: false,
|
|
||||||
FSIsolation: drivers.FSIsolationImage,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
// Driver is a driver for running LXC containers
|
|
||||||
type Driver struct {
|
|
||||||
// eventer is used to handle multiplexing of TaskEvents calls such that an
|
|
||||||
// event can be broadcast to all callers
|
|
||||||
eventer *eventer.Eventer
|
|
||||||
|
|
||||||
// config is the driver configuration set by the SetConfig RPC
|
|
||||||
config *Config
|
|
||||||
|
|
||||||
// nomadConfig is the client config from nomad
|
|
||||||
nomadConfig *base.ClientDriverConfig
|
|
||||||
|
|
||||||
// tasks is the in memory datastore mapping taskIDs to rawExecDriverHandles
|
|
||||||
tasks *taskStore
|
|
||||||
|
|
||||||
// ctx is the context for the driver. It is passed to other subsystems to
|
|
||||||
// coordinate shutdown
|
|
||||||
ctx context.Context
|
|
||||||
|
|
||||||
// signalShutdown is called when the driver is shutting down and cancels the
|
|
||||||
// ctx passed to any subsystems
|
|
||||||
signalShutdown context.CancelFunc
|
|
||||||
|
|
||||||
// logger will log to the Nomad agent
|
|
||||||
logger hclog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config is the driver configuration set by the SetConfig RPC call
|
|
||||||
type Config struct {
|
|
||||||
// Enabled is set to true to enable the lxc driver
|
|
||||||
Enabled bool `codec:"enabled"`
|
|
||||||
|
|
||||||
AllowVolumes bool `codec:"volumes"`
|
|
||||||
|
|
||||||
Path string `codec:"path"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// TaskConfig is the driver configuration of a task within a job
|
|
||||||
type TaskConfig struct {
|
|
||||||
Template string `codec:"template"`
|
|
||||||
Distro string `codec:"distro"`
|
|
||||||
Release string `codec:"release"`
|
|
||||||
Arch string `codec:"arch"`
|
|
||||||
ImageVariant string `codec:"image_variant"`
|
|
||||||
ImageServer string `codec:"image_server"`
|
|
||||||
GPGKeyID string `codec:"gpg_key_id"`
|
|
||||||
GPGKeyServer string `codec:"gpg_key_server"`
|
|
||||||
DisableGPGValidation bool `codec:"disable_gpg"`
|
|
||||||
FlushCache bool `codec:"flush_cache"`
|
|
||||||
ForceCache bool `codec:"force_cache"`
|
|
||||||
TemplateArgs []string `codec:"template_args"`
|
|
||||||
LogLevel string `codec:"log_level"`
|
|
||||||
Verbosity string `codec:"verbosity"`
|
|
||||||
Volumes []string `codec:"volumes"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// TaskState is the state which is encoded in the handle returned in
|
|
||||||
// StartTask. This information is needed to rebuild the task state and handler
|
|
||||||
// during recovery.
|
|
||||||
type TaskState struct {
|
|
||||||
TaskConfig *drivers.TaskConfig
|
|
||||||
ContainerName string
|
|
||||||
StartedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewLXCDriver returns a new DriverPlugin implementation
|
|
||||||
func NewLXCDriver(logger hclog.Logger) drivers.DriverPlugin {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
logger = logger.Named(pluginName)
|
|
||||||
return &Driver{
|
|
||||||
eventer: eventer.NewEventer(ctx, logger),
|
|
||||||
config: &Config{},
|
|
||||||
tasks: newTaskStore(),
|
|
||||||
ctx: ctx,
|
|
||||||
signalShutdown: cancel,
|
|
||||||
logger: logger,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) PluginInfo() (*base.PluginInfoResponse, error) {
|
|
||||||
return pluginInfo, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) ConfigSchema() (*hclspec.Spec, error) {
|
|
||||||
return configSpec, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) SetConfig(cfg *base.Config) error {
|
|
||||||
var config Config
|
|
||||||
if len(cfg.PluginConfig) != 0 {
|
|
||||||
if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
d.config = &config
|
|
||||||
if cfg.AgentConfig != nil {
|
|
||||||
d.nomadConfig = cfg.AgentConfig.Driver
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) Shutdown() {
|
|
||||||
d.signalShutdown()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
|
|
||||||
return taskConfigSpec, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
|
|
||||||
return capabilities, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) {
|
|
||||||
ch := make(chan *drivers.Fingerprint)
|
|
||||||
go d.handleFingerprint(ctx, ch)
|
|
||||||
return ch, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) handleFingerprint(ctx context.Context, ch chan<- *drivers.Fingerprint) {
|
|
||||||
defer close(ch)
|
|
||||||
ticker := time.NewTimer(0)
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-d.ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
ticker.Reset(fingerprintPeriod)
|
|
||||||
ch <- d.buildFingerprint()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) buildFingerprint() *drivers.Fingerprint {
|
|
||||||
var health drivers.HealthState
|
|
||||||
var desc string
|
|
||||||
attrs := map[string]*pstructs.Attribute{}
|
|
||||||
|
|
||||||
lxcVersion := lxc.Version()
|
|
||||||
|
|
||||||
if d.config.Enabled && lxcVersion != "" {
|
|
||||||
health = drivers.HealthStateHealthy
|
|
||||||
desc = "ready"
|
|
||||||
attrs["driver.lxc"] = pstructs.NewBoolAttribute(true)
|
|
||||||
attrs["driver.lxc.version"] = pstructs.NewStringAttribute(lxcVersion)
|
|
||||||
} else {
|
|
||||||
health = drivers.HealthStateUndetected
|
|
||||||
desc = "disabled"
|
|
||||||
}
|
|
||||||
|
|
||||||
if d.config.AllowVolumes {
|
|
||||||
attrs["driver.lxc.volumes.enabled"] = pstructs.NewBoolAttribute(true)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &drivers.Fingerprint{
|
|
||||||
Attributes: attrs,
|
|
||||||
Health: health,
|
|
||||||
HealthDescription: desc,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error {
|
|
||||||
if handle == nil {
|
|
||||||
return fmt.Errorf("error: handle cannot be nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, ok := d.tasks.Get(handle.Config.ID); ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var taskState TaskState
|
|
||||||
if err := handle.GetDriverState(&taskState); err != nil {
|
|
||||||
return fmt.Errorf("failed to decode task state from handle: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var driverConfig TaskConfig
|
|
||||||
if err := taskState.TaskConfig.DecodeDriverConfig(&driverConfig); err != nil {
|
|
||||||
return fmt.Errorf("failed to decode driver config: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
c, err := lxc.NewContainer(taskState.ContainerName, d.lxcPath())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create container ref: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
initPid := c.InitPid()
|
|
||||||
h := &taskHandle{
|
|
||||||
container: c,
|
|
||||||
initPid: initPid,
|
|
||||||
taskConfig: taskState.TaskConfig,
|
|
||||||
procState: drivers.TaskStateRunning,
|
|
||||||
startedAt: taskState.StartedAt,
|
|
||||||
exitResult: &drivers.ExitResult{},
|
|
||||||
|
|
||||||
totalCpuStats: stats.NewCpuStats(),
|
|
||||||
userCpuStats: stats.NewCpuStats(),
|
|
||||||
systemCpuStats: stats.NewCpuStats(),
|
|
||||||
}
|
|
||||||
|
|
||||||
d.tasks.Set(taskState.TaskConfig.ID, h)
|
|
||||||
|
|
||||||
go h.run()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) {
|
|
||||||
if _, ok := d.tasks.Get(cfg.ID); ok {
|
|
||||||
return nil, nil, fmt.Errorf("task with ID %q already started", cfg.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
var driverConfig TaskConfig
|
|
||||||
if err := cfg.DecodeDriverConfig(&driverConfig); err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
d.logger.Info("starting lxc task", "driver_cfg", hclog.Fmt("%+v", driverConfig))
|
|
||||||
handle := drivers.NewTaskHandle(pluginName)
|
|
||||||
handle.Config = cfg
|
|
||||||
|
|
||||||
c, err := d.initializeContainer(cfg, driverConfig)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
opt := toLXCCreateOptions(driverConfig)
|
|
||||||
if err := c.Create(opt); err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("unable to create container: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanup := func() {
|
|
||||||
if err := c.Destroy(); err != nil {
|
|
||||||
d.logger.Error("failed to clean up from an error in Start", "error", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := d.configureContainerNetwork(c); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := d.mountVolumes(c, cfg, driverConfig); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := c.Start(); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return nil, nil, fmt.Errorf("unable to start container: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := d.setResourceLimits(c, cfg); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
pid := c.InitPid()
|
|
||||||
|
|
||||||
h := &taskHandle{
|
|
||||||
container: c,
|
|
||||||
initPid: pid,
|
|
||||||
taskConfig: cfg,
|
|
||||||
procState: drivers.TaskStateRunning,
|
|
||||||
startedAt: time.Now().Round(time.Millisecond),
|
|
||||||
logger: d.logger,
|
|
||||||
|
|
||||||
totalCpuStats: stats.NewCpuStats(),
|
|
||||||
userCpuStats: stats.NewCpuStats(),
|
|
||||||
systemCpuStats: stats.NewCpuStats(),
|
|
||||||
}
|
|
||||||
|
|
||||||
driverState := TaskState{
|
|
||||||
ContainerName: c.Name(),
|
|
||||||
TaskConfig: cfg,
|
|
||||||
StartedAt: h.startedAt,
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := handle.SetDriverState(&driverState); err != nil {
|
|
||||||
d.logger.Error("failed to start task, error setting driver state", "error", err)
|
|
||||||
cleanup()
|
|
||||||
return nil, nil, fmt.Errorf("failed to set driver state: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
d.tasks.Set(cfg.ID, h)
|
|
||||||
|
|
||||||
go h.run()
|
|
||||||
|
|
||||||
return handle, nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) WaitTask(ctx context.Context, taskID string) (<-chan *drivers.ExitResult, error) {
|
|
||||||
handle, ok := d.tasks.Get(taskID)
|
|
||||||
if !ok {
|
|
||||||
return nil, drivers.ErrTaskNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
ch := make(chan *drivers.ExitResult)
|
|
||||||
go d.handleWait(ctx, handle, ch)
|
|
||||||
|
|
||||||
return ch, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *drivers.ExitResult) {
|
|
||||||
defer close(ch)
|
|
||||||
|
|
||||||
//
|
|
||||||
// Wait for process completion by polling status from handler.
|
|
||||||
// We cannot use the following alternatives:
|
|
||||||
// * Process.Wait() requires LXC container processes to be children
|
|
||||||
// of self process; but LXC runs container in separate PID hierarchy
|
|
||||||
// owned by PID 1.
|
|
||||||
// * lxc.Container.Wait() holds a write lock on container and prevents
|
|
||||||
// any other calls, including stats.
|
|
||||||
//
|
|
||||||
// Going with simplest approach of polling for handler to mark exit.
|
|
||||||
ticker := time.NewTicker(2 * time.Second)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-d.ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
s := handle.TaskStatus()
|
|
||||||
if s.State == drivers.TaskStateExited {
|
|
||||||
ch <- handle.exitResult
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) error {
|
|
||||||
handle, ok := d.tasks.Get(taskID)
|
|
||||||
if !ok {
|
|
||||||
return drivers.ErrTaskNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := handle.shutdown(timeout); err != nil {
|
|
||||||
return fmt.Errorf("executor Shutdown failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) DestroyTask(taskID string, force bool) error {
|
|
||||||
handle, ok := d.tasks.Get(taskID)
|
|
||||||
if !ok {
|
|
||||||
return drivers.ErrTaskNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
if handle.IsRunning() && !force {
|
|
||||||
return fmt.Errorf("cannot destroy running task")
|
|
||||||
}
|
|
||||||
|
|
||||||
if handle.IsRunning() {
|
|
||||||
// grace period is chosen arbitrary here
|
|
||||||
if err := handle.shutdown(1 * time.Minute); err != nil {
|
|
||||||
handle.logger.Error("failed to destroy executor", "err", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
d.tasks.Delete(taskID)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) InspectTask(taskID string) (*drivers.TaskStatus, error) {
|
|
||||||
handle, ok := d.tasks.Get(taskID)
|
|
||||||
if !ok {
|
|
||||||
return nil, drivers.ErrTaskNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
return handle.TaskStatus(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) TaskStats(taskID string) (*drivers.TaskResourceUsage, error) {
|
|
||||||
handle, ok := d.tasks.Get(taskID)
|
|
||||||
if !ok {
|
|
||||||
return nil, drivers.ErrTaskNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
return handle.stats()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) TaskEvents(ctx context.Context) (<-chan *drivers.TaskEvent, error) {
|
|
||||||
return d.eventer.TaskEvents(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) SignalTask(taskID string, signal string) error {
|
|
||||||
return fmt.Errorf("LXC driver does not support signals")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (*drivers.ExecTaskResult, error) {
|
|
||||||
return nil, fmt.Errorf("LXC driver does not support exec")
|
|
||||||
}
|
|
||||||
|
|
@ -1,280 +0,0 @@
|
||||||
// +build linux,lxc
|
|
||||||
|
|
||||||
package lxc
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/hashicorp/hcl2/hcl"
|
|
||||||
ctestutil "github.com/hashicorp/nomad/client/testutil"
|
|
||||||
"github.com/hashicorp/nomad/helper/testlog"
|
|
||||||
"github.com/hashicorp/nomad/helper/uuid"
|
|
||||||
"github.com/hashicorp/nomad/nomad/structs"
|
|
||||||
"github.com/hashicorp/nomad/plugins/drivers"
|
|
||||||
dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
|
|
||||||
"github.com/hashicorp/nomad/plugins/shared/hclspec"
|
|
||||||
"github.com/hashicorp/nomad/plugins/shared/hclutils"
|
|
||||||
"github.com/hashicorp/nomad/testutil"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
lxc "gopkg.in/lxc/go-lxc.v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestLXCDriver_Fingerprint(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
requireLXC(t)
|
|
||||||
|
|
||||||
require := require.New(t)
|
|
||||||
|
|
||||||
d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
|
|
||||||
d.config.Enabled = true
|
|
||||||
harness := dtestutil.NewDriverHarness(t, d)
|
|
||||||
|
|
||||||
fingerCh, err := harness.Fingerprint(context.Background())
|
|
||||||
require.NoError(err)
|
|
||||||
select {
|
|
||||||
case finger := <-fingerCh:
|
|
||||||
require.Equal(drivers.HealthStateHealthy, finger.Health)
|
|
||||||
require.True(finger.Attributes["driver.lxc"].GetBool())
|
|
||||||
require.NotEmpty(finger.Attributes["driver.lxc.version"].GetString())
|
|
||||||
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
|
|
||||||
require.Fail("timeout receiving fingerprint")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLXCDriver_FingerprintNotEnabled(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
requireLXC(t)
|
|
||||||
|
|
||||||
require := require.New(t)
|
|
||||||
|
|
||||||
d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
|
|
||||||
d.config.Enabled = false
|
|
||||||
harness := dtestutil.NewDriverHarness(t, d)
|
|
||||||
|
|
||||||
fingerCh, err := harness.Fingerprint(context.Background())
|
|
||||||
require.NoError(err)
|
|
||||||
select {
|
|
||||||
case finger := <-fingerCh:
|
|
||||||
require.Equal(drivers.HealthStateUndetected, finger.Health)
|
|
||||||
require.Empty(finger.Attributes["driver.lxc"])
|
|
||||||
require.Empty(finger.Attributes["driver.lxc.version"])
|
|
||||||
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
|
|
||||||
require.Fail("timeout receiving fingerprint")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLXCDriver_Start_Wait(t *testing.T) {
|
|
||||||
if !testutil.IsTravis() {
|
|
||||||
t.Parallel()
|
|
||||||
}
|
|
||||||
requireLXC(t)
|
|
||||||
ctestutil.RequireRoot(t)
|
|
||||||
|
|
||||||
require := require.New(t)
|
|
||||||
|
|
||||||
// prepare test file
|
|
||||||
testFileContents := []byte("this should be visible under /mnt/tmp")
|
|
||||||
tmpFile, err := ioutil.TempFile("/tmp", "testlxcdriver_start_wait")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("error writing temp file: %v", err)
|
|
||||||
}
|
|
||||||
defer os.Remove(tmpFile.Name())
|
|
||||||
if _, err := tmpFile.Write(testFileContents); err != nil {
|
|
||||||
t.Fatalf("error writing temp file: %v", err)
|
|
||||||
}
|
|
||||||
if err := tmpFile.Close(); err != nil {
|
|
||||||
t.Fatalf("error closing temp file: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
|
|
||||||
d.config.Enabled = true
|
|
||||||
d.config.AllowVolumes = true
|
|
||||||
|
|
||||||
harness := dtestutil.NewDriverHarness(t, d)
|
|
||||||
task := &drivers.TaskConfig{
|
|
||||||
ID: uuid.Generate(),
|
|
||||||
Name: "test",
|
|
||||||
Resources: &drivers.Resources{
|
|
||||||
NomadResources: &structs.AllocatedTaskResources{
|
|
||||||
Memory: structs.AllocatedMemoryResources{
|
|
||||||
MemoryMB: 2,
|
|
||||||
},
|
|
||||||
Cpu: structs.AllocatedCpuResources{
|
|
||||||
CpuShares: 1024,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
LinuxResources: &drivers.LinuxResources{
|
|
||||||
CPUShares: 1024,
|
|
||||||
MemoryLimitBytes: 2 * 1024,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
taskConfig := map[string]interface{}{
|
|
||||||
"template": "/usr/share/lxc/templates/lxc-busybox",
|
|
||||||
"volumes": []string{"/tmp/:mnt/tmp"},
|
|
||||||
}
|
|
||||||
encodeDriverHelper(require, task, taskConfig)
|
|
||||||
|
|
||||||
cleanup := harness.MkAllocDir(task, false)
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
handle, _, err := harness.StartTask(task)
|
|
||||||
require.NoError(err)
|
|
||||||
require.NotNil(handle)
|
|
||||||
|
|
||||||
lxcHandle, ok := d.tasks.Get(task.ID)
|
|
||||||
require.True(ok)
|
|
||||||
|
|
||||||
container := lxcHandle.container
|
|
||||||
|
|
||||||
// Destroy container after test
|
|
||||||
defer func() {
|
|
||||||
container.Stop()
|
|
||||||
container.Destroy()
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Test that container is running
|
|
||||||
testutil.WaitForResult(func() (bool, error) {
|
|
||||||
state := container.State()
|
|
||||||
if state == lxc.RUNNING {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
return false, fmt.Errorf("container in state: %v", state)
|
|
||||||
}, func(err error) {
|
|
||||||
t.Fatalf("container failed to start: %v", err)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test that directories are mounted in their proper location
|
|
||||||
containerName := container.Name()
|
|
||||||
for _, mnt := range []string{"alloc", "local", "secrets", "mnt/tmp"} {
|
|
||||||
fullpath := filepath.Join(d.lxcPath(), containerName, "rootfs", mnt)
|
|
||||||
stat, err := os.Stat(fullpath)
|
|
||||||
require.NoError(err)
|
|
||||||
require.True(stat.IsDir())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test bind mount volumes exist in container:
|
|
||||||
mountedContents, err := exec.Command("lxc-attach",
|
|
||||||
"-n", containerName, "--",
|
|
||||||
"cat", filepath.Join("/mnt/", tmpFile.Name()),
|
|
||||||
).Output()
|
|
||||||
require.NoError(err)
|
|
||||||
require.Equal(string(testFileContents), string(mountedContents))
|
|
||||||
|
|
||||||
// Test that killing container marks container as stopped
|
|
||||||
require.NoError(container.Stop())
|
|
||||||
|
|
||||||
testutil.WaitForResult(func() (bool, error) {
|
|
||||||
status, err := d.InspectTask(task.ID)
|
|
||||||
if err == nil && status.State == drivers.TaskStateExited {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
return false, fmt.Errorf("task in state: %v", status.State)
|
|
||||||
}, func(err error) {
|
|
||||||
t.Fatalf("task was not marked as stopped: %v", err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLXCDriver_Start_Stop(t *testing.T) {
|
|
||||||
if !testutil.IsTravis() {
|
|
||||||
t.Parallel()
|
|
||||||
}
|
|
||||||
requireLXC(t)
|
|
||||||
ctestutil.RequireRoot(t)
|
|
||||||
|
|
||||||
require := require.New(t)
|
|
||||||
|
|
||||||
d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
|
|
||||||
d.config.Enabled = true
|
|
||||||
d.config.AllowVolumes = true
|
|
||||||
|
|
||||||
harness := dtestutil.NewDriverHarness(t, d)
|
|
||||||
task := &drivers.TaskConfig{
|
|
||||||
ID: uuid.Generate(),
|
|
||||||
Name: "test",
|
|
||||||
Resources: &drivers.Resources{
|
|
||||||
NomadResources: &structs.AllocatedTaskResources{
|
|
||||||
Memory: structs.AllocatedMemoryResources{
|
|
||||||
MemoryMB: 2,
|
|
||||||
},
|
|
||||||
Cpu: structs.AllocatedCpuResources{
|
|
||||||
CpuShares: 1024,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
LinuxResources: &drivers.LinuxResources{
|
|
||||||
CPUShares: 1024,
|
|
||||||
MemoryLimitBytes: 2 * 1024,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
taskConfig := map[string]interface{}{
|
|
||||||
"template": "/usr/share/lxc/templates/lxc-busybox",
|
|
||||||
}
|
|
||||||
encodeDriverHelper(require, task, taskConfig)
|
|
||||||
|
|
||||||
cleanup := harness.MkAllocDir(task, false)
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
handle, _, err := harness.StartTask(task)
|
|
||||||
require.NoError(err)
|
|
||||||
require.NotNil(handle)
|
|
||||||
|
|
||||||
lxcHandle, ok := d.tasks.Get(task.ID)
|
|
||||||
require.True(ok)
|
|
||||||
|
|
||||||
container := lxcHandle.container
|
|
||||||
|
|
||||||
// Destroy container after test
|
|
||||||
defer func() {
|
|
||||||
container.Stop()
|
|
||||||
container.Destroy()
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Test that container is running
|
|
||||||
testutil.WaitForResult(func() (bool, error) {
|
|
||||||
state := container.State()
|
|
||||||
if state == lxc.RUNNING {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
return false, fmt.Errorf("container in state: %v", state)
|
|
||||||
}, func(err error) {
|
|
||||||
t.Fatalf("container failed to start: %v", err)
|
|
||||||
})
|
|
||||||
|
|
||||||
require.NoError(d.StopTask(task.ID, 5*time.Second, "kill"))
|
|
||||||
|
|
||||||
testutil.WaitForResult(func() (bool, error) {
|
|
||||||
status, err := d.InspectTask(task.ID)
|
|
||||||
if err == nil && status.State == drivers.TaskStateExited {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
return false, fmt.Errorf("task in state: %v", status.State)
|
|
||||||
}, func(err error) {
|
|
||||||
t.Fatalf("task was not marked as stopped: %v", err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireLXC(t *testing.T) {
|
|
||||||
if lxc.Version() == "" {
|
|
||||||
t.Skip("skipping, lxc not present")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func encodeDriverHelper(require *require.Assertions, task *drivers.TaskConfig, taskConfig map[string]interface{}) {
|
|
||||||
evalCtx := &hcl.EvalContext{
|
|
||||||
Functions: hclutils.GetStdlibFuncs(),
|
|
||||||
}
|
|
||||||
spec, diag := hclspec.Convert(taskConfigSpec)
|
|
||||||
require.False(diag.HasErrors())
|
|
||||||
taskConfigCtyVal, diag := hclutils.ParseHclInterface(taskConfig, spec, evalCtx)
|
|
||||||
require.False(diag.HasErrors())
|
|
||||||
err := task.EncodeDriverConfig(taskConfigCtyVal)
|
|
||||||
require.Nil(err)
|
|
||||||
}
|
|
||||||
|
|
@ -1,199 +0,0 @@
|
||||||
//+build linux,lxc
|
|
||||||
|
|
||||||
package lxc
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
hclog "github.com/hashicorp/go-hclog"
|
|
||||||
"github.com/hashicorp/nomad/client/stats"
|
|
||||||
"github.com/hashicorp/nomad/plugins/drivers"
|
|
||||||
lxc "gopkg.in/lxc/go-lxc.v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
type taskHandle struct {
|
|
||||||
container *lxc.Container
|
|
||||||
initPid int
|
|
||||||
logger hclog.Logger
|
|
||||||
|
|
||||||
totalCpuStats *stats.CpuStats
|
|
||||||
userCpuStats *stats.CpuStats
|
|
||||||
systemCpuStats *stats.CpuStats
|
|
||||||
|
|
||||||
// stateLock syncs access to all fields below
|
|
||||||
stateLock sync.RWMutex
|
|
||||||
|
|
||||||
taskConfig *drivers.TaskConfig
|
|
||||||
procState drivers.TaskState
|
|
||||||
startedAt time.Time
|
|
||||||
completedAt time.Time
|
|
||||||
exitResult *drivers.ExitResult
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
LXCMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
|
|
||||||
|
|
||||||
LXCMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
|
|
||||||
)
|
|
||||||
|
|
||||||
func (h *taskHandle) TaskStatus() *drivers.TaskStatus {
|
|
||||||
h.stateLock.RLock()
|
|
||||||
defer h.stateLock.RUnlock()
|
|
||||||
|
|
||||||
return &drivers.TaskStatus{
|
|
||||||
ID: h.taskConfig.ID,
|
|
||||||
Name: h.taskConfig.Name,
|
|
||||||
State: h.procState,
|
|
||||||
StartedAt: h.startedAt,
|
|
||||||
CompletedAt: h.completedAt,
|
|
||||||
ExitResult: h.exitResult,
|
|
||||||
DriverAttributes: map[string]string{
|
|
||||||
"pid": strconv.Itoa(h.initPid),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *taskHandle) IsRunning() bool {
|
|
||||||
h.stateLock.RLock()
|
|
||||||
defer h.stateLock.RUnlock()
|
|
||||||
return h.procState == drivers.TaskStateRunning
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *taskHandle) run() {
|
|
||||||
h.stateLock.Lock()
|
|
||||||
if h.exitResult == nil {
|
|
||||||
h.exitResult = &drivers.ExitResult{}
|
|
||||||
}
|
|
||||||
h.stateLock.Unlock()
|
|
||||||
|
|
||||||
if ok, err := waitTillStopped(h.container); !ok {
|
|
||||||
h.logger.Error("failed to find container process", "error", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.stateLock.Lock()
|
|
||||||
defer h.stateLock.Unlock()
|
|
||||||
|
|
||||||
h.procState = drivers.TaskStateExited
|
|
||||||
h.exitResult.ExitCode = 0
|
|
||||||
h.exitResult.Signal = 0
|
|
||||||
h.completedAt = time.Now()
|
|
||||||
|
|
||||||
// TODO: detect if the task OOMed
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *taskHandle) stats() (*drivers.TaskResourceUsage, error) {
|
|
||||||
cpuStats, err := h.container.CPUStats()
|
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("failed to get container cpu stats", "error", err)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
total, err := h.container.CPUTime()
|
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("failed to get container cpu time", "error", err)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
t := time.Now()
|
|
||||||
|
|
||||||
// Get the cpu stats
|
|
||||||
system := cpuStats["system"]
|
|
||||||
user := cpuStats["user"]
|
|
||||||
cs := &drivers.CpuStats{
|
|
||||||
SystemMode: h.systemCpuStats.Percent(float64(system)),
|
|
||||||
UserMode: h.systemCpuStats.Percent(float64(user)),
|
|
||||||
Percent: h.totalCpuStats.Percent(float64(total)),
|
|
||||||
TotalTicks: float64(user + system),
|
|
||||||
Measured: LXCMeasuredCpuStats,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the Memory Stats
|
|
||||||
memData := map[string]uint64{
|
|
||||||
"rss": 0,
|
|
||||||
"cache": 0,
|
|
||||||
"swap": 0,
|
|
||||||
}
|
|
||||||
rawMemStats := h.container.CgroupItem("memory.stat")
|
|
||||||
for _, rawMemStat := range rawMemStats {
|
|
||||||
key, val, err := keysToVal(rawMemStat)
|
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("failed to get stat", "line", rawMemStat, "error", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, ok := memData[key]; ok {
|
|
||||||
memData[key] = val
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ms := &drivers.MemoryStats{
|
|
||||||
RSS: memData["rss"],
|
|
||||||
Cache: memData["cache"],
|
|
||||||
Swap: memData["swap"],
|
|
||||||
Measured: LXCMeasuredMemStats,
|
|
||||||
}
|
|
||||||
|
|
||||||
mu := h.container.CgroupItem("memory.max_usage_in_bytes")
|
|
||||||
for _, rawMemMaxUsage := range mu {
|
|
||||||
val, err := strconv.ParseUint(rawMemMaxUsage, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("failed to get max memory usage", "error", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ms.MaxUsage = val
|
|
||||||
}
|
|
||||||
ku := h.container.CgroupItem("memory.kmem.usage_in_bytes")
|
|
||||||
for _, rawKernelUsage := range ku {
|
|
||||||
val, err := strconv.ParseUint(rawKernelUsage, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("failed to get kernel memory usage", "error", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ms.KernelUsage = val
|
|
||||||
}
|
|
||||||
|
|
||||||
mku := h.container.CgroupItem("memory.kmem.max_usage_in_bytes")
|
|
||||||
for _, rawMaxKernelUsage := range mku {
|
|
||||||
val, err := strconv.ParseUint(rawMaxKernelUsage, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("failed tog get max kernel memory usage", "error", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ms.KernelMaxUsage = val
|
|
||||||
}
|
|
||||||
|
|
||||||
taskResUsage := drivers.TaskResourceUsage{
|
|
||||||
ResourceUsage: &drivers.ResourceUsage{
|
|
||||||
CpuStats: cs,
|
|
||||||
MemoryStats: ms,
|
|
||||||
},
|
|
||||||
Timestamp: t.UTC().UnixNano(),
|
|
||||||
}
|
|
||||||
|
|
||||||
return &taskResUsage, nil
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func keysToVal(line string) (string, uint64, error) {
|
|
||||||
tokens := strings.Split(line, " ")
|
|
||||||
if len(tokens) != 2 {
|
|
||||||
return "", 0, fmt.Errorf("line isn't a k/v pair")
|
|
||||||
}
|
|
||||||
key := tokens[0]
|
|
||||||
val, err := strconv.ParseUint(tokens[1], 10, 64)
|
|
||||||
return key, val, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// shutdown shuts down the container, with `timeout` grace period
|
|
||||||
// before killing the container with SIGKILL.
|
|
||||||
func (h *taskHandle) shutdown(timeout time.Duration) error {
|
|
||||||
err := h.container.Shutdown(timeout)
|
|
||||||
if err == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return h.container.Stop()
|
|
||||||
}
|
|
||||||
|
|
@ -1,253 +0,0 @@
|
||||||
//+build linux,lxc
|
|
||||||
|
|
||||||
package lxc
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/hashicorp/nomad/plugins/drivers"
|
|
||||||
ldevices "github.com/opencontainers/runc/libcontainer/devices"
|
|
||||||
"gopkg.in/lxc/go-lxc.v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
verbosityLevels = map[string]lxc.Verbosity{
|
|
||||||
"": lxc.Quiet,
|
|
||||||
"verbose": lxc.Verbose,
|
|
||||||
"quiet": lxc.Quiet,
|
|
||||||
}
|
|
||||||
|
|
||||||
logLevels = map[string]lxc.LogLevel{
|
|
||||||
"": lxc.ERROR,
|
|
||||||
"debug": lxc.DEBUG,
|
|
||||||
"error": lxc.ERROR,
|
|
||||||
"info": lxc.INFO,
|
|
||||||
"trace": lxc.TRACE,
|
|
||||||
"warn": lxc.WARN,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// containerMonitorIntv is the interval at which the driver checks if the
|
|
||||||
// container is still alive
|
|
||||||
containerMonitorIntv = 2 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
func (d *Driver) lxcPath() string {
|
|
||||||
lxcPath := d.config.Path
|
|
||||||
if lxcPath == "" {
|
|
||||||
lxcPath = lxc.DefaultConfigPath()
|
|
||||||
}
|
|
||||||
return lxcPath
|
|
||||||
|
|
||||||
}
|
|
||||||
func (d *Driver) initializeContainer(cfg *drivers.TaskConfig, taskConfig TaskConfig) (*lxc.Container, error) {
|
|
||||||
lxcPath := d.lxcPath()
|
|
||||||
|
|
||||||
containerName := fmt.Sprintf("%s-%s", cfg.Name, cfg.AllocID)
|
|
||||||
c, err := lxc.NewContainer(containerName, lxcPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to initialize container: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if v, ok := verbosityLevels[taskConfig.Verbosity]; ok {
|
|
||||||
c.SetVerbosity(v)
|
|
||||||
} else {
|
|
||||||
return nil, fmt.Errorf("lxc driver config 'verbosity' can only be either quiet or verbose")
|
|
||||||
}
|
|
||||||
|
|
||||||
if v, ok := logLevels[taskConfig.LogLevel]; ok {
|
|
||||||
c.SetLogLevel(v)
|
|
||||||
} else {
|
|
||||||
return nil, fmt.Errorf("lxc driver config 'log_level' can only be trace, debug, info, warn or error")
|
|
||||||
}
|
|
||||||
|
|
||||||
logFile := filepath.Join(cfg.TaskDir().Dir, fmt.Sprintf("%v-lxc.log", cfg.Name))
|
|
||||||
c.SetLogFile(logFile)
|
|
||||||
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) configureContainerNetwork(c *lxc.Container) error {
|
|
||||||
// Set the network type to none
|
|
||||||
if err := c.SetConfigItem(networkTypeConfigKey(), "none"); err != nil {
|
|
||||||
return fmt.Errorf("error setting network type configuration: %v", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func networkTypeConfigKey() string {
|
|
||||||
if lxc.VersionAtLeast(2, 1, 0) {
|
|
||||||
return "lxc.net.0.type"
|
|
||||||
}
|
|
||||||
|
|
||||||
// prior to 2.1, network used
|
|
||||||
return "lxc.network.type"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) mountVolumes(c *lxc.Container, cfg *drivers.TaskConfig, taskConfig TaskConfig) error {
|
|
||||||
mounts, err := d.mountEntries(cfg, taskConfig)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
devCgroupAllows, err := d.devicesCgroupEntries(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, mnt := range mounts {
|
|
||||||
if err := c.SetConfigItem("lxc.mount.entry", mnt); err != nil {
|
|
||||||
return fmt.Errorf("error setting bind mount %q error: %v", mnt, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, cgroupDev := range devCgroupAllows {
|
|
||||||
if err := c.SetConfigItem("lxc.cgroup.devices.allow", cgroupDev); err != nil {
|
|
||||||
return fmt.Errorf("error setting cgroup permission %q error: %v", cgroupDev, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mountEntries compute the mount entries to be set on the container
|
|
||||||
func (d *Driver) mountEntries(cfg *drivers.TaskConfig, taskConfig TaskConfig) ([]string, error) {
|
|
||||||
// Bind mount the shared alloc dir and task local dir in the container
|
|
||||||
mounts := []string{
|
|
||||||
fmt.Sprintf("%s local none rw,bind,create=dir", cfg.TaskDir().LocalDir),
|
|
||||||
fmt.Sprintf("%s alloc none rw,bind,create=dir", cfg.TaskDir().SharedAllocDir),
|
|
||||||
fmt.Sprintf("%s secrets none rw,bind,create=dir", cfg.TaskDir().SecretsDir),
|
|
||||||
}
|
|
||||||
|
|
||||||
mounts = append(mounts, d.formatTaskMounts(cfg.Mounts)...)
|
|
||||||
mounts = append(mounts, d.formatTaskDevices(cfg.Devices)...)
|
|
||||||
|
|
||||||
volumesEnabled := d.config.AllowVolumes
|
|
||||||
|
|
||||||
for _, volDesc := range taskConfig.Volumes {
|
|
||||||
// the format was checked in Validate()
|
|
||||||
paths := strings.Split(volDesc, ":")
|
|
||||||
|
|
||||||
if filepath.IsAbs(paths[0]) {
|
|
||||||
if !volumesEnabled {
|
|
||||||
return nil, fmt.Errorf("absolute bind-mount volume in config but volumes are disabled")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Relative source paths are treated as relative to alloc dir
|
|
||||||
paths[0] = filepath.Join(cfg.TaskDir().Dir, paths[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
// LXC assumes paths are relative with respect to rootfs
|
|
||||||
target := strings.TrimLeft(paths[1], "/")
|
|
||||||
mounts = append(mounts, fmt.Sprintf("%s %s none rw,bind,create=dir", paths[0], target))
|
|
||||||
}
|
|
||||||
|
|
||||||
return mounts, nil
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) devicesCgroupEntries(cfg *drivers.TaskConfig) ([]string, error) {
|
|
||||||
entries := make([]string, len(cfg.Devices))
|
|
||||||
|
|
||||||
for i, d := range cfg.Devices {
|
|
||||||
hd, err := ldevices.DeviceFromPath(d.HostPath, d.Permissions)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
entries[i] = hd.CgroupString()
|
|
||||||
}
|
|
||||||
|
|
||||||
return entries, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) formatTaskMounts(mounts []*drivers.MountConfig) []string {
|
|
||||||
result := make([]string, len(mounts))
|
|
||||||
|
|
||||||
for i, m := range mounts {
|
|
||||||
result[i] = d.formatMount(m.HostPath, m.TaskPath, m.Readonly)
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) formatTaskDevices(devices []*drivers.DeviceConfig) []string {
|
|
||||||
result := make([]string, len(devices))
|
|
||||||
|
|
||||||
for i, m := range devices {
|
|
||||||
result[i] = d.formatMount(m.HostPath, m.TaskPath,
|
|
||||||
!strings.Contains(m.Permissions, "w"))
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) formatMount(hostPath, taskPath string, readOnly bool) string {
|
|
||||||
typ := "dir"
|
|
||||||
s, err := os.Stat(hostPath)
|
|
||||||
if err != nil {
|
|
||||||
d.logger.Warn("failed to find mount host path type, defaulting to dir type", "path", hostPath, "error", err)
|
|
||||||
} else if !s.IsDir() {
|
|
||||||
typ = "file"
|
|
||||||
}
|
|
||||||
|
|
||||||
perm := "rw"
|
|
||||||
if readOnly {
|
|
||||||
perm = "ro"
|
|
||||||
}
|
|
||||||
|
|
||||||
// LXC assumes paths are relative with respect to rootfs
|
|
||||||
target := strings.TrimLeft(taskPath, "/")
|
|
||||||
return fmt.Sprintf("%s %s none %s,bind,create=%s", hostPath, target, perm, typ)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Driver) setResourceLimits(c *lxc.Container, cfg *drivers.TaskConfig) error {
|
|
||||||
if err := c.SetMemoryLimit(lxc.ByteSize(cfg.Resources.NomadResources.Memory.MemoryMB) * lxc.MB); err != nil {
|
|
||||||
return fmt.Errorf("unable to set memory limits: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := c.SetCgroupItem("cpu.shares", strconv.FormatInt(cfg.Resources.LinuxResources.CPUShares, 10)); err != nil {
|
|
||||||
return fmt.Errorf("unable to set cpu shares: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func toLXCCreateOptions(taskConfig TaskConfig) lxc.TemplateOptions {
|
|
||||||
return lxc.TemplateOptions{
|
|
||||||
Template: taskConfig.Template,
|
|
||||||
Distro: taskConfig.Distro,
|
|
||||||
Release: taskConfig.Release,
|
|
||||||
Arch: taskConfig.Arch,
|
|
||||||
FlushCache: taskConfig.FlushCache,
|
|
||||||
DisableGPGValidation: taskConfig.DisableGPGValidation,
|
|
||||||
ExtraArgs: taskConfig.TemplateArgs,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitTillStopped blocks and returns true when container stops;
|
|
||||||
// returns false with an error message if the container processes cannot be identified.
|
|
||||||
//
|
|
||||||
// Use this in preference to c.Wait() - lxc Wait() function holds a write lock on the container
|
|
||||||
// blocking any other operation on container, including looking up container stats
|
|
||||||
func waitTillStopped(c *lxc.Container) (bool, error) {
|
|
||||||
ps, err := os.FindProcess(c.InitPid())
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
|
||||||
if err := ps.Signal(syscall.Signal(0)); err != nil {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(containerMonitorIntv)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,95 +0,0 @@
|
||||||
//+build linux,lxc
|
|
||||||
|
|
||||||
package lxc
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/hashicorp/nomad/helper/testlog"
|
|
||||||
"github.com/hashicorp/nomad/helper/uuid"
|
|
||||||
"github.com/hashicorp/nomad/nomad/structs"
|
|
||||||
"github.com/hashicorp/nomad/plugins/drivers"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestLXCDriver_Mounts(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
task := &drivers.TaskConfig{
|
|
||||||
ID: uuid.Generate(),
|
|
||||||
Name: "test",
|
|
||||||
Resources: &drivers.Resources{
|
|
||||||
NomadResources: &structs.AllocatedTaskResources{
|
|
||||||
Memory: structs.AllocatedMemoryResources{
|
|
||||||
MemoryMB: 2,
|
|
||||||
},
|
|
||||||
Cpu: structs.AllocatedCpuResources{
|
|
||||||
CpuShares: 1024,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
LinuxResources: &drivers.LinuxResources{
|
|
||||||
CPUShares: 1024,
|
|
||||||
MemoryLimitBytes: 2 * 1024,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Mounts: []*drivers.MountConfig{
|
|
||||||
{HostPath: "/dev", TaskPath: "/task-mounts/dev-path"},
|
|
||||||
{HostPath: "/bin/sh", TaskPath: "/task-mounts/task-path-ro", Readonly: true},
|
|
||||||
},
|
|
||||||
Devices: []*drivers.DeviceConfig{
|
|
||||||
{HostPath: "/dev", TaskPath: "/task-devices/dev-path", Permissions: "rw"},
|
|
||||||
{HostPath: "/bin/sh", TaskPath: "/task-devices/task-path-ro", Permissions: "ro"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
taskConfig := TaskConfig{
|
|
||||||
Template: "busybox",
|
|
||||||
Volumes: []string{
|
|
||||||
"relative/path:/usr-config/container/path",
|
|
||||||
"relative/path2:usr-config/container/relative",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
|
|
||||||
d.config.Enabled = true
|
|
||||||
|
|
||||||
entries, err := d.mountEntries(task, taskConfig)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
expectedEntries := []string{
|
|
||||||
"test/relative/path usr-config/container/path none rw,bind,create=dir",
|
|
||||||
"test/relative/path2 usr-config/container/relative none rw,bind,create=dir",
|
|
||||||
"/dev task-mounts/dev-path none rw,bind,create=dir",
|
|
||||||
"/bin/sh task-mounts/task-path-ro none ro,bind,create=file",
|
|
||||||
"/dev task-devices/dev-path none rw,bind,create=dir",
|
|
||||||
"/bin/sh task-devices/task-path-ro none ro,bind,create=file",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, e := range expectedEntries {
|
|
||||||
require.Contains(t, entries, e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLXCDriver_DevicesCgroup(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
task := &drivers.TaskConfig{
|
|
||||||
ID: uuid.Generate(),
|
|
||||||
Name: "test",
|
|
||||||
Devices: []*drivers.DeviceConfig{
|
|
||||||
{HostPath: "/dev/random", TaskPath: "/task-devices/devrandom", Permissions: "rw"},
|
|
||||||
{HostPath: "/dev/null", TaskPath: "/task-devices/devnull", Permissions: "rwm"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
d := NewLXCDriver(testlog.HCLogger(t)).(*Driver)
|
|
||||||
d.config.Enabled = true
|
|
||||||
|
|
||||||
cgroupEntries, err := d.devicesCgroupEntries(task)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
expected := []string{
|
|
||||||
"c 1:8 rw",
|
|
||||||
"c 1:3 rwm",
|
|
||||||
}
|
|
||||||
require.EqualValues(t, expected, cgroupEntries)
|
|
||||||
}
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
//+build linux,lxc
|
|
||||||
|
|
||||||
package lxc
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
)
|
|
||||||
|
|
||||||
type taskStore struct {
|
|
||||||
store map[string]*taskHandle
|
|
||||||
lock sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTaskStore() *taskStore {
|
|
||||||
return &taskStore{store: map[string]*taskHandle{}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ts *taskStore) Set(id string, handle *taskHandle) {
|
|
||||||
ts.lock.Lock()
|
|
||||||
defer ts.lock.Unlock()
|
|
||||||
ts.store[id] = handle
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ts *taskStore) Get(id string) (*taskHandle, bool) {
|
|
||||||
ts.lock.RLock()
|
|
||||||
defer ts.lock.RUnlock()
|
|
||||||
t, ok := ts.store[id]
|
|
||||||
return t, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ts *taskStore) Delete(id string) {
|
|
||||||
ts.lock.Lock()
|
|
||||||
defer ts.lock.Unlock()
|
|
||||||
delete(ts.store, id)
|
|
||||||
}
|
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
//+build linux,lxc
|
|
||||||
|
|
||||||
package catalog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/hashicorp/nomad/drivers/lxc"
|
|
||||||
)
|
|
||||||
|
|
||||||
// This file is where all builtin plugins should be registered in the catalog.
|
|
||||||
// Plugins with build restrictions should be placed in the appropriate
|
|
||||||
// register_XXX.go file.
|
|
||||||
func init() {
|
|
||||||
RegisterDeferredConfig(lxc.PluginID, lxc.PluginConfig, lxc.PluginLoader)
|
|
||||||
}
|
|
||||||
|
|
@ -10,8 +10,7 @@ sudo service docker restart
|
||||||
# true errors would fail in the apt-get install phase
|
# true errors would fail in the apt-get install phase
|
||||||
apt-get update || true
|
apt-get update || true
|
||||||
|
|
||||||
apt-get install -y liblxc1 lxc-dev lxc lxc-templates shellcheck
|
apt-get install -y qemu shellcheck
|
||||||
apt-get install -y qemu
|
|
||||||
bash ./scripts/travis-rkt.sh
|
bash ./scripts/travis-rkt.sh
|
||||||
bash ./scripts/travis-consul.sh
|
bash ./scripts/travis-consul.sh
|
||||||
bash ./scripts/travis-vault.sh
|
bash ./scripts/travis-vault.sh
|
||||||
|
|
|
||||||
|
|
@ -26,11 +26,8 @@ apt-get install -y \
|
||||||
build-essential \
|
build-essential \
|
||||||
git \
|
git \
|
||||||
libc6-dev-i386 \
|
libc6-dev-i386 \
|
||||||
liblxc1 \
|
|
||||||
libpcre3-dev \
|
libpcre3-dev \
|
||||||
linux-libc-dev:i386 \
|
linux-libc-dev:i386 \
|
||||||
lxc-dev \
|
|
||||||
lxc-templates \
|
|
||||||
pkg-config \
|
pkg-config \
|
||||||
zip
|
zip
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue