From 3f1887db8c204fec42e30610d2fa55063f5cccc4 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 25 Nov 2018 11:55:01 -0500 Subject: [PATCH 1/4] Update go-lxc library to handle LXC 3.0 --- vendor/gopkg.in/lxc/go-lxc.v2/Makefile | 12 + vendor/gopkg.in/lxc/go-lxc.v2/container.go | 722 +++++++++++++------ vendor/gopkg.in/lxc/go-lxc.v2/error.go | 1 + vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.c | 160 +++- vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.go | 129 +++- vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.h | 33 +- vendor/gopkg.in/lxc/go-lxc.v2/options.go | 20 +- vendor/gopkg.in/lxc/go-lxc.v2/type.go | 14 +- vendor/gopkg.in/lxc/go-lxc.v2/util.go | 2 +- vendor/vendor.json | 2 +- 10 files changed, 840 insertions(+), 255 deletions(-) diff --git a/vendor/gopkg.in/lxc/go-lxc.v2/Makefile b/vendor/gopkg.in/lxc/go-lxc.v2/Makefile index db6df4735..149ce6636 100644 --- a/vendor/gopkg.in/lxc/go-lxc.v2/Makefile +++ b/vendor/gopkg.in/lxc/go-lxc.v2/Makefile @@ -45,4 +45,16 @@ escape-analysis: ctags: @ctags -R --languages=c,go +scope: + @echo "$(OK_COLOR)==> Exported container calls in container.go $(NO_COLOR)" + @/bin/grep -E "\bc+\.([A-Z])\w+" container.go || true + +setup-test-cgroup: + for d in /sys/fs/cgroup/*; do \ + [ -f $$d/cgroup.clone_children ] && echo 1 | sudo tee $$d/cgroup.clone_children; \ + [ -f $$d/cgroup.use_hierarchy ] && echo 1 | sudo tee $$d/cgroup.use_hierarchy; \ + sudo mkdir -p $$d/lxc; \ + sudo chown -R $$USER: $$d/lxc; \ + done + .PHONY: all format test doc vet lint ctags diff --git a/vendor/gopkg.in/lxc/go-lxc.v2/container.go b/vendor/gopkg.in/lxc/go-lxc.v2/container.go index c8684e63e..58f1d7f2f 100644 --- a/vendor/gopkg.in/lxc/go-lxc.v2/container.go +++ b/vendor/gopkg.in/lxc/go-lxc.v2/container.go @@ -16,18 +16,21 @@ import ( "io/ioutil" "os" "os/exec" + "path" + "path/filepath" "reflect" "strconv" "strings" "sync" + "syscall" "time" "unsafe" ) // Container struct type Container struct { - container *C.struct_lxc_container mu sync.RWMutex + container *C.struct_lxc_container verbosity Verbosity } @@ -52,20 +55,20 @@ const ( ) func (c *Container) makeSure(flags int) error { - if flags&isDefined != 0 && !c.Defined() { - return ErrNotDefined + if flags&isDefined != 0 && !c.defined() { + return fmt.Errorf("%s: %q", ErrNotDefined, c.name()) } - if flags&isNotDefined != 0 && c.Defined() { - return ErrAlreadyDefined + if flags&isNotDefined != 0 && c.defined() { + return fmt.Errorf("%s: %q", ErrAlreadyDefined, c.name()) } - if flags&isRunning != 0 && !c.Running() { - return ErrNotRunning + if flags&isRunning != 0 && !c.running() { + return fmt.Errorf("%s: %q", ErrNotRunning, c.name()) } - if flags&isNotRunning != 0 && c.Running() { - return ErrAlreadyRunning + if flags&isNotRunning != 0 && c.running() { + return fmt.Errorf("%s: %q", ErrAlreadyRunning, c.name()) } if flags&isPrivileged != 0 && os.Geteuid() != 0 { @@ -84,7 +87,7 @@ func (c *Container) makeSure(flags int) error { } func (c *Container) cgroupItemAsByteSize(filename string, missing error) (ByteSize, error) { - size, err := strconv.ParseFloat(c.CgroupItem(filename)[0], 64) + size, err := strconv.ParseFloat(c.cgroupItem(filename)[0], 64) if err != nil { return -1, missing } @@ -92,18 +95,35 @@ func (c *Container) cgroupItemAsByteSize(filename string, missing error) (ByteSi } func (c *Container) setCgroupItemWithByteSize(filename string, limit ByteSize, missing error) error { - if err := c.SetCgroupItem(filename, fmt.Sprintf("%.f", limit)); err != nil { + if err := c.setCgroupItem(filename, fmt.Sprintf("%.f", limit)); err != nil { return missing } return nil } +func (c *Container) name() string { + return C.GoString(c.container.name) +} + // Name returns the name of the container. func (c *Container) Name() string { c.mu.RLock() defer c.mu.RUnlock() - return C.GoString(c.container.name) + return c.name() +} + +// String returns the string represantation of container. +func (c *Container) String() string { + c.mu.RLock() + defer c.mu.RUnlock() + + return path.Join(c.configPath(), c.name()) +} + +// Caller needs to hold the lock +func (c *Container) defined() bool { + return bool(C.go_lxc_defined(c.container)) } // Defined returns true if the container is already defined. @@ -111,7 +131,12 @@ func (c *Container) Defined() bool { c.mu.RLock() defer c.mu.RUnlock() - return bool(C.go_lxc_defined(c.container)) + return c.defined() +} + +// Caller needs to hold the lock +func (c *Container) running() bool { + return bool(C.go_lxc_running(c.container)) } // Running returns true if the container is already running. @@ -119,7 +144,7 @@ func (c *Container) Running() bool { c.mu.RLock() defer c.mu.RUnlock() - return bool(C.go_lxc_running(c.container)) + return c.running() } // Controllable returns true if the caller can control the container. @@ -132,13 +157,13 @@ func (c *Container) Controllable() bool { // CreateSnapshot creates a new snapshot. func (c *Container) CreateSnapshot() (*Snapshot, error) { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isDefined | isNotRunning); err != nil { return nil, err } - c.mu.Lock() - defer c.mu.Unlock() - ret := int(C.go_lxc_snapshot(c.container)) if ret < 0 { return nil, ErrCreateSnapshotFailed @@ -148,13 +173,13 @@ func (c *Container) CreateSnapshot() (*Snapshot, error) { // RestoreSnapshot creates a new container based on a snapshot. func (c *Container) RestoreSnapshot(snapshot Snapshot, name string) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isDefined); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - cname := C.CString(name) defer C.free(unsafe.Pointer(cname)) @@ -169,13 +194,13 @@ func (c *Container) RestoreSnapshot(snapshot Snapshot, name string) error { // DestroySnapshot destroys the specified snapshot. func (c *Container) DestroySnapshot(snapshot Snapshot) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isDefined); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - csnapname := C.CString(snapshot.Name) defer C.free(unsafe.Pointer(csnapname)) @@ -187,13 +212,13 @@ func (c *Container) DestroySnapshot(snapshot Snapshot) error { // DestroyAllSnapshots destroys all the snapshot. func (c *Container) DestroyAllSnapshots() error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isDefined | isGreaterEqualThanLXC11); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - if !bool(C.go_lxc_snapshot_destroy_all(c.container)) { return ErrDestroyAllSnapshotsFailed } @@ -202,13 +227,13 @@ func (c *Container) DestroyAllSnapshots() error { // Snapshots returns the list of container snapshots. func (c *Container) Snapshots() ([]Snapshot, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isDefined); err != nil { return nil, err } - c.mu.Lock() - defer c.mu.Unlock() - var csnapshots *C.struct_lxc_snapshot size := int(C.go_lxc_snapshot_list(c.container, &csnapshots)) @@ -238,12 +263,17 @@ func (c *Container) Snapshots() ([]Snapshot, error) { return snapshots, nil } +// Caller needs to hold the lock +func (c *Container) state() State { + return StateMap[C.GoString(C.go_lxc_state(c.container))] +} + // State returns the state of the container. func (c *Container) State() State { c.mu.RLock() defer c.mu.RUnlock() - return StateMap[C.GoString(C.go_lxc_state(c.container))] + return c.state() } // InitPid returns the process ID of the container's init process @@ -296,17 +326,14 @@ func (c *Container) SetVerbosity(verbosity Verbosity) { // Freeze freezes the running container. func (c *Container) Freeze() error { - if err := c.makeSure(isRunning); err != nil { - return err - } - - if c.State() == FROZEN { - return ErrAlreadyFrozen - } - c.mu.Lock() defer c.mu.Unlock() + // check the state using lockless version + if c.state() == FROZEN { + return ErrAlreadyFrozen + } + if !bool(C.go_lxc_freeze(c.container)) { return ErrFreezeFailed } @@ -316,17 +343,18 @@ func (c *Container) Freeze() error { // Unfreeze thaws the frozen container. func (c *Container) Unfreeze() error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } - if c.State() != FROZEN { + // check the state using lockless version + if c.state() != FROZEN { return ErrNotFrozen } - c.mu.Lock() - defer c.mu.Unlock() - if !bool(C.go_lxc_unfreeze(c.container)) { return ErrUnfreezeFailed } @@ -336,6 +364,9 @@ func (c *Container) Unfreeze() error { // Create creates the container using given TemplateOptions func (c *Container) Create(options TemplateOptions) error { + c.mu.Lock() + defer c.mu.Unlock() + // FIXME: Support bdev_specs // // bdev_specs: @@ -347,9 +378,6 @@ func (c *Container) Create(options TemplateOptions) error { return err } - c.mu.Lock() - defer c.mu.Unlock() - // use download template if not set if options.Template == "" { options.Template = "download" @@ -360,11 +388,6 @@ func (c *Container) Create(options TemplateOptions) error { options.Backend = Directory } - // unprivileged users are only allowed to use "download" template - if os.Geteuid() != 0 && options.Template != "download" { - return ErrTemplateNotAllowed - } - var args []string if options.Template == "download" { // required parameters @@ -439,62 +462,91 @@ func (c *Container) Create(options TemplateOptions) error { // Start starts the container. func (c *Container) Start() error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isNotRunning); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - if !bool(C.go_lxc_start(c.container, 0, nil)) { return ErrStartFailed } return nil } +// StartWithArgs starts the container using given arguments. +func (c *Container) StartWithArgs(args []string) error { + c.mu.Lock() + defer c.mu.Unlock() + + if err := c.makeSure(isNotRunning); err != nil { + return err + } + + if !bool(C.go_lxc_start(c.container, 0, makeNullTerminatedArgs(args))) { + return ErrStartFailed + } + return nil +} + +// StartExecute starts a container. It runs a minimal init as PID 1 and the +// requested program as the second process. +func (c *Container) StartExecute(args []string) error { + c.mu.Lock() + defer c.mu.Unlock() + + if err := c.makeSure(isNotRunning); err != nil { + return err + } + + if !bool(C.go_lxc_start(c.container, 1, makeNullTerminatedArgs(args))) { + return ErrStartFailed + } + + return nil +} + // Execute executes the given command in a temporary container. func (c *Container) Execute(args ...string) ([]byte, error) { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isNotDefined); err != nil { return nil, err } - cargs := []string{"lxc-execute", "-n", c.Name(), "-P", c.ConfigPath(), "--"} - cargs = append(cargs, args...) + os.MkdirAll(filepath.Join(c.configPath(), c.name()), 0700) + c.saveConfigFile(filepath.Join(c.configPath(), c.name(), "config")) + defer os.RemoveAll(filepath.Join(c.configPath(), c.name())) - c.mu.Lock() - defer c.mu.Unlock() + cargs := []string{"lxc-execute", "-n", c.name(), "-P", c.configPath(), "--"} + cargs = append(cargs, args...) // FIXME: Go runtime and src/lxc/start.c signal_handler are not playing nice together so use lxc-execute for now // go-nuts thread: https://groups.google.com/forum/#!msg/golang-nuts/h9GbvfYv83w/5Ly_jvOr86wJ output, err := exec.Command(cargs[0], cargs[1:]...).CombinedOutput() if err != nil { + // Do not suppress stderr if the exit code != 0. Return with err. + if len(output) > 1 { + return output, ErrExecuteFailed + } + return nil, ErrExecuteFailed } return output, nil - /* - cargs := makeNullTerminatedArgs(args) - if cargs == nil { - return ErrAllocationFailed - } - defer freeNullTerminatedArgs(cargs, len(args)) - - if !bool(C.go_lxc_start(c.container, 1, cargs)) { - return ErrExecuteFailed - } - return nil - */ } // Stop stops the container. func (c *Container) Stop() error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - if !bool(C.go_lxc_stop(c.container)) { return ErrStopFailed } @@ -503,13 +555,13 @@ func (c *Container) Stop() error { // Reboot reboots the container. func (c *Container) Reboot() error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - if !bool(C.go_lxc_reboot(c.container)) { return ErrRebootFailed } @@ -518,11 +570,12 @@ func (c *Container) Reboot() error { // Shutdown shuts down the container. func (c *Container) Shutdown(timeout time.Duration) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() if !bool(C.go_lxc_shutdown(c.container, C.int(timeout.Seconds()))) { return ErrShutdownFailed @@ -532,13 +585,13 @@ func (c *Container) Shutdown(timeout time.Duration) error { // Destroy destroys the container. func (c *Container) Destroy() error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isDefined | isNotRunning); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - if !bool(C.go_lxc_destroy(c.container)) { return ErrDestroyFailed } @@ -547,13 +600,13 @@ func (c *Container) Destroy() error { // DestroyWithAllSnapshots destroys the container and its snapshots func (c *Container) DestroyWithAllSnapshots() error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isDefined | isNotRunning | isGreaterEqualThanLXC11); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - if !bool(C.go_lxc_destroy_with_snapshots(c.container)) { return ErrDestroyWithAllSnapshotsFailed } @@ -562,6 +615,9 @@ func (c *Container) DestroyWithAllSnapshots() error { // Clone clones the container using given arguments with specified backend. func (c *Container) Clone(name string, options CloneOptions) error { + c.mu.Lock() + defer c.mu.Unlock() + // FIXME: bdevdata, newsize and hookargs // // bdevdata: @@ -577,9 +633,6 @@ func (c *Container) Clone(name string, options CloneOptions) error { return err } - c.mu.Lock() - defer c.mu.Unlock() - // use Directory backend if not set if options.Backend == 0 { options.Backend = Directory @@ -619,13 +672,13 @@ func (c *Container) Clone(name string, options CloneOptions) error { // Rename renames the container. func (c *Container) Rename(name string) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isDefined | isNotRunning); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - cname := C.CString(name) defer C.free(unsafe.Pointer(cname)) @@ -658,11 +711,7 @@ func (c *Container) ConfigFileName() string { return C.GoString(configFileName) } -// ConfigItem returns the value of the given config item. -func (c *Container) ConfigItem(key string) []string { - c.mu.RLock() - defer c.mu.RUnlock() - +func (c *Container) configItem(key string) []string { ckey := C.CString(key) defer C.free(unsafe.Pointer(ckey)) @@ -674,11 +723,15 @@ func (c *Container) ConfigItem(key string) []string { return strings.Split(ret, "\n") } -// SetConfigItem sets the value of the given config item. -func (c *Container) SetConfigItem(key string, value string) error { - c.mu.Lock() - defer c.mu.Unlock() +// ConfigItem returns the value of the given config item. +func (c *Container) ConfigItem(key string) []string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.configItem(key) +} + +func (c *Container) setConfigItem(key string, value string) error { ckey := C.CString(key) defer C.free(unsafe.Pointer(ckey)) @@ -691,11 +744,15 @@ func (c *Container) SetConfigItem(key string, value string) error { return nil } -// RunningConfigItem returns the value of the given config item. -func (c *Container) RunningConfigItem(key string) []string { - c.mu.RLock() - defer c.mu.RUnlock() +// SetConfigItem sets the value of the given config item. +func (c *Container) SetConfigItem(key string, value string) error { + c.mu.Lock() + defer c.mu.Unlock() + return c.setConfigItem(key, value) +} + +func (c *Container) runningConfigItem(key string) []string { ckey := C.CString(key) defer C.free(unsafe.Pointer(ckey)) @@ -707,11 +764,15 @@ func (c *Container) RunningConfigItem(key string) []string { return strings.Split(ret, "\n") } -// CgroupItem returns the value of the given cgroup subsystem value. -func (c *Container) CgroupItem(key string) []string { +// RunningConfigItem returns the value of the given config item. +func (c *Container) RunningConfigItem(key string) []string { c.mu.RLock() defer c.mu.RUnlock() + return c.runningConfigItem(key) +} + +func (c *Container) cgroupItem(key string) []string { ckey := C.CString(key) defer C.free(unsafe.Pointer(ckey)) @@ -723,11 +784,7 @@ func (c *Container) CgroupItem(key string) []string { return strings.Split(ret, "\n") } -// SetCgroupItem sets the value of given cgroup subsystem value. -func (c *Container) SetCgroupItem(key string, value string) error { - c.mu.Lock() - defer c.mu.Unlock() - +func (c *Container) setCgroupItem(key string, value string) error { ckey := C.CString(key) defer C.free(unsafe.Pointer(ckey)) @@ -740,6 +797,22 @@ func (c *Container) SetCgroupItem(key string, value string) error { return nil } +// CgroupItem returns the value of the given cgroup subsystem value. +func (c *Container) CgroupItem(key string) []string { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.cgroupItem(key) +} + +// SetCgroupItem sets the value of given cgroup subsystem value. +func (c *Container) SetCgroupItem(key string, value string) error { + c.mu.Lock() + defer c.mu.Unlock() + + return c.setCgroupItem(key, value) +} + // ClearConfig completely clears the containers in-memory configuration. func (c *Container) ClearConfig() { c.mu.Lock() @@ -757,7 +830,7 @@ func (c *Container) ClearConfigItem(key string) error { defer C.free(unsafe.Pointer(ckey)) if !bool(C.go_lxc_clear_config_item(c.container, ckey)) { - return ErrClearingCgroupItemFailed + return ErrClearingConfigItemFailed } return nil } @@ -799,11 +872,7 @@ func (c *Container) LoadConfigFile(path string) error { return nil } -// SaveConfigFile saves the configuration file to given path. -func (c *Container) SaveConfigFile(path string) error { - c.mu.Lock() - defer c.mu.Unlock() - +func (c *Container) saveConfigFile(path string) error { cpath := C.CString(path) defer C.free(unsafe.Pointer(cpath)) @@ -813,12 +882,24 @@ func (c *Container) SaveConfigFile(path string) error { return nil } +// SaveConfigFile saves the configuration file to given path. +func (c *Container) SaveConfigFile(path string) error { + c.mu.Lock() + defer c.mu.Unlock() + + return c.saveConfigFile(path) +} + +func (c *Container) configPath() string { + return C.GoString(C.go_lxc_get_config_path(c.container)) +} + // ConfigPath returns the configuration file's path. func (c *Container) ConfigPath() string { c.mu.RLock() defer c.mu.RUnlock() - return C.GoString(C.go_lxc_get_config_path(c.container)) + return c.configPath() } // SetConfigPath sets the configuration file's path. @@ -837,6 +918,9 @@ func (c *Container) SetConfigPath(path string) error { // MemoryUsage returns memory usage of the container in bytes. func (c *Container) MemoryUsage() (ByteSize, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return -1, err } @@ -846,6 +930,9 @@ func (c *Container) MemoryUsage() (ByteSize, error) { // MemoryLimit returns memory limit of the container in bytes. func (c *Container) MemoryLimit() (ByteSize, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return -1, err } @@ -855,6 +942,9 @@ func (c *Container) MemoryLimit() (ByteSize, error) { // SetMemoryLimit sets memory limit of the container in bytes. func (c *Container) SetMemoryLimit(limit ByteSize) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } @@ -864,6 +954,9 @@ func (c *Container) SetMemoryLimit(limit ByteSize) error { // SoftMemoryLimit returns soft memory limit of the container in bytes. func (c *Container) SoftMemoryLimit() (ByteSize, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return -1, err } @@ -873,6 +966,9 @@ func (c *Container) SoftMemoryLimit() (ByteSize, error) { // SetSoftMemoryLimit sets soft memory limit of the container in bytes. func (c *Container) SetSoftMemoryLimit(limit ByteSize) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } @@ -882,6 +978,9 @@ func (c *Container) SetSoftMemoryLimit(limit ByteSize) error { // KernelMemoryUsage returns current kernel memory allocation of the container in bytes. func (c *Container) KernelMemoryUsage() (ByteSize, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return -1, err } @@ -891,15 +990,21 @@ func (c *Container) KernelMemoryUsage() (ByteSize, error) { // KernelMemoryLimit returns kernel memory limit of the container in bytes. func (c *Container) KernelMemoryLimit() (ByteSize, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return -1, err } - return c.cgroupItemAsByteSize("memory.kmem.usage_in_bytes", ErrKMemLimit) + return c.cgroupItemAsByteSize("memory.kmem.limit_in_bytes", ErrKMemLimit) } // SetKernelMemoryLimit sets kernel memory limit of the container in bytes. func (c *Container) SetKernelMemoryLimit(limit ByteSize) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } @@ -909,6 +1014,9 @@ func (c *Container) SetKernelMemoryLimit(limit ByteSize) error { // MemorySwapUsage returns memory+swap usage of the container in bytes. func (c *Container) MemorySwapUsage() (ByteSize, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return -1, err } @@ -918,6 +1026,9 @@ func (c *Container) MemorySwapUsage() (ByteSize, error) { // MemorySwapLimit returns the memory+swap limit of the container in bytes. func (c *Container) MemorySwapLimit() (ByteSize, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return -1, err } @@ -927,6 +1038,9 @@ func (c *Container) MemorySwapLimit() (ByteSize, error) { // SetMemorySwapLimit sets memory+swap limit of the container in bytes. func (c *Container) SetMemorySwapLimit(limit ByteSize) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } @@ -936,14 +1050,19 @@ func (c *Container) SetMemorySwapLimit(limit ByteSize) error { // BlkioUsage returns number of bytes transferred to/from the disk by the container. func (c *Container) BlkioUsage() (ByteSize, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return -1, err } - c.mu.RLock() - defer c.mu.RUnlock() + ioServiceBytes := c.cgroupItem("blkio.throttle.io_service_bytes") + if ioServiceBytes[0] == "" { + return 0, nil + } - for _, v := range c.CgroupItem("blkio.throttle.io_service_bytes") { + for _, v := range ioServiceBytes { b := strings.Split(v, " ") if b[0] == "Total" { blkioUsed, err := strconv.ParseFloat(b[1], 64) @@ -959,14 +1078,19 @@ func (c *Container) BlkioUsage() (ByteSize, error) { // CPUTime returns the total CPU time (in nanoseconds) consumed by all tasks // in this cgroup (including tasks lower in the hierarchy). func (c *Container) CPUTime() (time.Duration, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return -1, err } - c.mu.RLock() - defer c.mu.RUnlock() + usage := c.cgroupItem("cpuacct.usage") + if usage[0] == "" { + return 0, nil + } - cpuUsage, err := strconv.ParseInt(c.CgroupItem("cpuacct.usage")[0], 10, 64) + cpuUsage, err := strconv.ParseInt(usage[0], 10, 64) if err != nil { return -1, err } @@ -976,15 +1100,20 @@ func (c *Container) CPUTime() (time.Duration, error) { // CPUTimePerCPU returns the CPU time (in nanoseconds) consumed on each CPU by // all tasks in this cgroup (including tasks lower in the hierarchy). func (c *Container) CPUTimePerCPU() (map[int]time.Duration, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() + usagePerCPU := c.cgroupItem("cpuacct.usage_percpu") + if usagePerCPU[0] == "" { + return map[int]time.Duration{0: 0}, nil + } cpuTimes := make(map[int]time.Duration) - for i, v := range strings.Split(c.CgroupItem("cpuacct.usage_percpu")[0], " ") { + for i, v := range strings.Split(usagePerCPU[0], " ") { cpuUsage, err := strconv.ParseInt(v, 10, 64) if err != nil { return nil, err @@ -997,19 +1126,23 @@ func (c *Container) CPUTimePerCPU() (map[int]time.Duration, error) { // CPUStats returns the number of CPU cycles (in the units defined by USER_HZ on the system) // consumed by tasks in this cgroup and its children in both user mode and system (kernel) mode. func (c *Container) CPUStats() (map[string]int64, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() + stat := c.cgroupItem("cpuacct.stat") + if stat[0] == "" { + return map[string]int64{"user": 0, "system": 0}, nil + } - cpuStat := c.CgroupItem("cpuacct.stat") - user, err := strconv.ParseInt(strings.Split(cpuStat[0], "user ")[1], 10, 64) + user, err := strconv.ParseInt(strings.Split(stat[0], "user ")[1], 10, 64) if err != nil { return nil, err } - system, err := strconv.ParseInt(strings.Split(cpuStat[1], "system ")[1], 10, 64) + system, err := strconv.ParseInt(strings.Split(stat[1], "system ")[1], 10, 64) if err != nil { return nil, err } @@ -1025,17 +1158,17 @@ func (c *Container) CPUStats() (map[string]int64, error) { // indicate that it is done with the allocated console so that it can // be allocated by another caller. func (c *Container) ConsoleFd(ttynum int) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + // FIXME: Make idiomatic if err := c.makeSure(isRunning); err != nil { return -1, err } - c.mu.Lock() - defer c.mu.Unlock() - ret := int(C.go_lxc_console_getfd(c.container, C.int(ttynum))) if ret < 0 { - return -1, ErrAttachFailed + return ret, ErrAttachFailed } return ret, nil } @@ -1044,13 +1177,13 @@ func (c *Container) ConsoleFd(ttynum int) (int, error) { // // This function will not return until the console has been exited by the user. func (c *Container) Console(options ConsoleOptions) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - ret := bool(C.go_lxc_console(c.container, C.int(options.Tty), C.int(options.StdinFd), @@ -1067,13 +1200,13 @@ func (c *Container) Console(options ConsoleOptions) error { // AttachShell attaches a shell to the container. // It clears all environment variables before attaching. func (c *Container) AttachShell(options AttachOptions) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - cenv := makeNullTerminatedArgs(options.Env) if cenv == nil { return ErrAllocationFailed @@ -1108,11 +1241,7 @@ func (c *Container) AttachShell(options AttachOptions) error { return nil } -// RunCommandStatus attachs a shell and runs the command within the container. -// The process will wait for the command to finish and return the result of -// waitpid(), i.e. the process' exit status. An error is returned only when -// invocation of the command completely fails. -func (c *Container) RunCommandStatus(args []string, options AttachOptions) (int, error) { +func (c *Container) runCommandStatus(args []string, options AttachOptions) (int, error) { if len(args) == 0 { return -1, ErrInsufficientNumberOfArguments } @@ -1121,9 +1250,6 @@ func (c *Container) RunCommandStatus(args []string, options AttachOptions) (int, return -1, err } - c.mu.Lock() - defer c.mu.Unlock() - cargs := makeNullTerminatedArgs(args) if cargs == nil { return -1, ErrAllocationFailed @@ -1161,14 +1287,92 @@ func (c *Container) RunCommandStatus(args []string, options AttachOptions) (int, cargs, )) - return ret, nil + if ret < 0 { + return ret, nil + } + + // Mirror the behavior of WEXITSTATUS(). + return int((ret & 0xFF00) >> 8), nil +} + +// RunCommandStatus attachs a shell and runs the command within the container. +// The process will wait for the command to finish and return the result of +// waitpid(), i.e. the process' exit status. An error is returned only when +// invocation of the command completely fails. +func (c *Container) RunCommandStatus(args []string, options AttachOptions) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.runCommandStatus(args, options) +} + +// RunCommandNoWait runs the given command and returns without waiting it to finish. +func (c *Container) RunCommandNoWait(args []string, options AttachOptions) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if len(args) == 0 { + return -1, ErrInsufficientNumberOfArguments + } + + if err := c.makeSure(isRunning); err != nil { + return -1, err + } + + cargs := makeNullTerminatedArgs(args) + if cargs == nil { + return -1, ErrAllocationFailed + } + defer freeNullTerminatedArgs(cargs, len(args)) + + cenv := makeNullTerminatedArgs(options.Env) + if cenv == nil { + return -1, ErrAllocationFailed + } + defer freeNullTerminatedArgs(cenv, len(options.Env)) + + cenvToKeep := makeNullTerminatedArgs(options.EnvToKeep) + if cenvToKeep == nil { + return -1, ErrAllocationFailed + } + defer freeNullTerminatedArgs(cenvToKeep, len(options.EnvToKeep)) + + cwd := C.CString(options.Cwd) + defer C.free(unsafe.Pointer(cwd)) + + var attachedPid C.pid_t + ret := int(C.go_lxc_attach_no_wait( + c.container, + C.bool(options.ClearEnv), + C.int(options.Namespaces), + C.long(options.Arch), + C.uid_t(options.UID), + C.gid_t(options.GID), + C.int(options.StdinFd), + C.int(options.StdoutFd), + C.int(options.StderrFd), + cwd, + cenv, + cenvToKeep, + cargs, + &attachedPid, + )) + + if ret < 0 { + return ret, ErrAttachFailed + } + + return int(attachedPid), nil } // RunCommand attachs a shell and runs the command within the container. // The process will wait for the command to finish and return a success status. An error // is returned only when invocation of the command completely fails. func (c *Container) RunCommand(args []string, options AttachOptions) (bool, error) { - ret, err := c.RunCommandStatus(args, options) + c.mu.Lock() + defer c.mu.Unlock() + + ret, err := c.runCommandStatus(args, options) if err != nil { return false, err } @@ -1180,13 +1384,13 @@ func (c *Container) RunCommand(args []string, options AttachOptions) (bool, erro // Interfaces returns the names of the network interfaces. func (c *Container) Interfaces() ([]string, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() - result := C.go_lxc_get_interfaces(c.container) if result == nil { return nil, ErrInterfaces @@ -1196,27 +1400,32 @@ func (c *Container) Interfaces() ([]string, error) { // InterfaceStats returns the stats about container's network interfaces func (c *Container) InterfaceStats() (map[string]map[string]ByteSize, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() - var interfaceName string statistics := make(map[string]map[string]ByteSize) - for i := 0; i < len(c.ConfigItem("lxc.network")); i++ { - interfaceType := c.RunningConfigItem(fmt.Sprintf("lxc.network.%d.type", i)) + netPrefix := "lxc.net" + if !VersionAtLeast(2, 1, 0) { + netPrefix = "lxc.network" + } + + for i := 0; i < len(c.configItem(netPrefix)); i++ { + interfaceType := c.runningConfigItem(fmt.Sprintf("%s.%d.type", netPrefix, i)) if interfaceType == nil { continue } if interfaceType[0] == "veth" { - interfaceName = c.RunningConfigItem(fmt.Sprintf("lxc.network.%d.veth.pair", i))[0] + interfaceName = c.runningConfigItem(fmt.Sprintf("%s.%d.veth.pair", netPrefix, i))[0] } else { - interfaceName = c.RunningConfigItem(fmt.Sprintf("lxc.network.%d.link", i))[0] + interfaceName = c.runningConfigItem(fmt.Sprintf("%s.%d.link", netPrefix, i))[0] } for _, v := range []string{"rx", "tx"} { @@ -1243,13 +1452,13 @@ func (c *Container) InterfaceStats() (map[string]map[string]ByteSize, error) { // IPAddress returns the IP address of the given network interface. func (c *Container) IPAddress(interfaceName string) ([]string, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() - cinterface := C.CString(interfaceName) defer C.free(unsafe.Pointer(cinterface)) @@ -1262,13 +1471,13 @@ func (c *Container) IPAddress(interfaceName string) ([]string, error) { // IPv4Address returns the IPv4 address of the given network interface. func (c *Container) IPv4Address(interfaceName string) ([]string, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() - cinterface := C.CString(interfaceName) defer C.free(unsafe.Pointer(cinterface)) @@ -1284,13 +1493,13 @@ func (c *Container) IPv4Address(interfaceName string) ([]string, error) { // IPv6Address returns the IPv6 address of the given network interface. func (c *Container) IPv6Address(interfaceName string) ([]string, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() - cinterface := C.CString(interfaceName) defer C.free(unsafe.Pointer(cinterface)) @@ -1306,9 +1515,12 @@ func (c *Container) IPv6Address(interfaceName string) ([]string, error) { // WaitIPAddresses waits until IPAddresses call returns something or time outs func (c *Container) WaitIPAddresses(timeout time.Duration) ([]string, error) { + c.mu.RLock() + defer c.mu.RUnlock() + now := time.Now() for { - if result, err := c.IPAddresses(); err == nil && len(result) > 0 { + if result, err := c.ipAddresses(); err == nil && len(result) > 0 { return result, nil } // Python API sleeps 1 second as well @@ -1320,15 +1532,11 @@ func (c *Container) WaitIPAddresses(timeout time.Duration) ([]string, error) { } } -// IPAddresses returns all IP addresses. -func (c *Container) IPAddresses() ([]string, error) { +func (c *Container) ipAddresses() ([]string, error) { if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() - result := C.go_lxc_get_ips(c.container, nil, nil, 0) if result == nil { return nil, ErrIPAddresses @@ -1337,15 +1545,23 @@ func (c *Container) IPAddresses() ([]string, error) { } +// IPAddresses returns all IP addresses. +func (c *Container) IPAddresses() ([]string, error) { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.ipAddresses() +} + // IPv4Addresses returns all IPv4 addresses. func (c *Container) IPv4Addresses() ([]string, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() - cfamily := C.CString("inet") defer C.free(unsafe.Pointer(cfamily)) @@ -1358,13 +1574,13 @@ func (c *Container) IPv4Addresses() ([]string, error) { // IPv6Addresses returns all IPv6 addresses. func (c *Container) IPv6Addresses() ([]string, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if err := c.makeSure(isRunning); err != nil { return nil, err } - c.mu.RLock() - defer c.mu.RUnlock() - cfamily := C.CString("inet6") defer C.free(unsafe.Pointer(cfamily)) @@ -1377,25 +1593,58 @@ func (c *Container) IPv6Addresses() ([]string, error) { // LogFile returns the name of the logfile. func (c *Container) LogFile() string { - return c.ConfigItem("lxc.logfile")[0] + c.mu.RLock() + defer c.mu.RUnlock() + + if VersionAtLeast(2, 1, 0) { + return c.configItem("lxc.log.file")[0] + } + + return c.configItem("lxc.logfile")[0] } // SetLogFile sets the name of the logfile. func (c *Container) SetLogFile(filename string) error { - if err := c.SetConfigItem("lxc.logfile", filename); err != nil { + c.mu.Lock() + defer c.mu.Unlock() + + var err error + if VersionAtLeast(2, 1, 0) { + err = c.setConfigItem("lxc.log.file", filename) + } else { + err = c.setConfigItem("lxc.logfile", filename) + } + if err != nil { return err } + return nil } // LogLevel returns the level of the logfile. func (c *Container) LogLevel() LogLevel { - return logLevelMap[c.ConfigItem("lxc.loglevel")[0]] + c.mu.RLock() + defer c.mu.RUnlock() + + if VersionAtLeast(2, 1, 0) { + return logLevelMap[c.configItem("lxc.log.level")[0]] + } + + return logLevelMap[c.configItem("lxc.loglevel")[0]] } // SetLogLevel sets the level of the logfile. func (c *Container) SetLogLevel(level LogLevel) error { - if err := c.SetConfigItem("lxc.loglevel", level.String()); err != nil { + c.mu.Lock() + defer c.mu.Unlock() + + var err error + if VersionAtLeast(2, 1, 0) { + err = c.setConfigItem("lxc.log.level", level.String()) + } else { + err = c.setConfigItem("lxc.loglevel", level.String()) + } + if err != nil { return err } return nil @@ -1403,13 +1652,13 @@ func (c *Container) SetLogLevel(level LogLevel) error { // AddDeviceNode adds specified device to the container. func (c *Container) AddDeviceNode(source string, destination ...string) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning | isPrivileged); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - csource := C.CString(source) defer C.free(unsafe.Pointer(csource)) @@ -1427,18 +1676,17 @@ func (c *Container) AddDeviceNode(source string, destination ...string) error { return ErrAddDeviceNodeFailed } return nil - } // RemoveDeviceNode removes the specified device from the container. func (c *Container) RemoveDeviceNode(source string, destination ...string) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning | isPrivileged); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - csource := C.CString(source) defer C.free(unsafe.Pointer(csource)) @@ -1460,6 +1708,9 @@ func (c *Container) RemoveDeviceNode(source string, destination ...string) error // Checkpoint checkpoints the container. func (c *Container) Checkpoint(opts CheckpointOptions) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning | isGreaterEqualThanLXC11); err != nil { return err } @@ -1478,6 +1729,9 @@ func (c *Container) Checkpoint(opts CheckpointOptions) error { // Restore restores the container from a checkpoint. func (c *Container) Restore(opts RestoreOptions) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isGreaterEqualThanLXC11); err != nil { return err } @@ -1493,11 +1747,21 @@ func (c *Container) Restore(opts RestoreOptions) error { return nil } +// Migrate migrates the container. func (c *Container) Migrate(cmd uint, opts MigrateOptions) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isNotDefined | isGreaterEqualThanLXC20); err != nil { return err } + if cmd != MIGRATE_RESTORE { + if err := c.makeSure(isRunning); err != nil { + return err + } + } + cdirectory := C.CString(opts.Directory) defer C.free(unsafe.Pointer(cdirectory)) @@ -1525,14 +1789,15 @@ func (c *Container) Migrate(cmd uint, opts MigrateOptions) error { } extras := C.struct_extra_migrate_opts{ - preserves_inodes: C.bool(opts.PreservesInodes), - action_script: cActionScript, - ghost_limit: C.uint64_t(opts.GhostLimit), + preserves_inodes: C.bool(opts.PreservesInodes), + action_script: cActionScript, + ghost_limit: C.uint64_t(opts.GhostLimit), + features_to_check: C.uint64_t(opts.FeaturesToCheck), } ret := C.int(C.go_lxc_migrate(c.container, C.uint(cmd), &copts, &extras)) if ret != 0 { - return fmt.Errorf("migration failed %d\n", ret) + return fmt.Errorf("migration failed %d", ret) } return nil @@ -1540,13 +1805,13 @@ func (c *Container) Migrate(cmd uint, opts MigrateOptions) error { // AttachInterface attaches specifed netdev to the container. func (c *Container) AttachInterface(source, destination string) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning | isPrivileged | isGreaterEqualThanLXC11); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - csource := C.CString(source) defer C.free(unsafe.Pointer(csource)) @@ -1561,13 +1826,13 @@ func (c *Container) AttachInterface(source, destination string) error { // DetachInterface detaches specifed netdev from the container. func (c *Container) DetachInterface(source string) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning | isPrivileged | isGreaterEqualThanLXC11); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - csource := C.CString(source) defer C.free(unsafe.Pointer(csource)) @@ -1579,13 +1844,13 @@ func (c *Container) DetachInterface(source string) error { // DetachInterfaceRename detaches specifed netdev from the container and renames it. func (c *Container) DetachInterfaceRename(source, target string) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.makeSure(isRunning | isPrivileged | isGreaterEqualThanLXC11); err != nil { return err } - c.mu.Lock() - defer c.mu.Unlock() - csource := C.CString(source) defer C.free(unsafe.Pointer(csource)) @@ -1597,3 +1862,48 @@ func (c *Container) DetachInterfaceRename(source, target string) error { } return nil } + +// ConsoleLog allows to perform operations on the container's in-memory console +// buffer. +func (c *Container) ConsoleLog(opt ConsoleLogOptions) ([]byte, error) { + c.mu.Lock() + defer c.mu.Unlock() + + cl := C.struct_lxc_console_log{ + clear: C.bool(opt.ClearLog), + read: C.bool(opt.ReadLog), + data: nil, + } + // CGO is a fickle little beast: + // We need to manually allocate memory here that we pass to C. If we + // were to pass a GO pointer by passing a C.uint64_t pointer we'd end in + // the situation where we have a GO pointer that points to a GO pointer. + // Go will freak out when this happens. So give C its own memory. + var buf unsafe.Pointer + buf = C.malloc(C.sizeof_uint64_t) + if buf == nil { + return nil, syscall.ENOMEM + } + defer C.free(buf) + + cl.read_max = (*C.uint64_t)(buf) + *cl.read_max = C.uint64_t(opt.ReadMax) + + ret := C.go_lxc_console_log(c.container, &cl) + if ret < 0 { + return nil, syscall.Errno(-ret) + } + + numBytes := C.int(*cl.read_max) + if C.uint64_t(numBytes) != *cl.read_max { + return nil, syscall.ERANGE + } + + return C.GoBytes(unsafe.Pointer(cl.data), numBytes), nil +} + +// ErrorNum returns the error_num field of the container. +func (c *Container) ErrorNum() int { + cError := C.go_lxc_error_num(c.container) + return int(cError) +} diff --git a/vendor/gopkg.in/lxc/go-lxc.v2/error.go b/vendor/gopkg.in/lxc/go-lxc.v2/error.go index 708e2ae45..b89d90352 100644 --- a/vendor/gopkg.in/lxc/go-lxc.v2/error.go +++ b/vendor/gopkg.in/lxc/go-lxc.v2/error.go @@ -16,6 +16,7 @@ var ( ErrAttachInterfaceFailed = NewError("attaching specified netdev to the container failed") ErrBlkioUsage = NewError("BlkioUsage for the container failed") ErrCheckpointFailed = NewError("checkpoint failed") + ErrClearingConfigItemFailed = NewError("clearing config item for the container failed") ErrClearingCgroupItemFailed = NewError("clearing cgroup item for the container failed") ErrCloneFailed = NewError("cloning the container failed") ErrCloseAllFdsFailed = NewError("setting close_all_fds flag for container failed") diff --git a/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.c b/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.c index 4592b6865..7e408c0ed 100644 --- a/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.c +++ b/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.c @@ -15,10 +15,9 @@ #include "lxc-binding.h" -#define VERSION_AT_LEAST(major, minor, micro) \ - (!(major > LXC_VERSION_MAJOR || \ - major == LXC_VERSION_MAJOR && minor > LXC_VERSION_MINOR || \ - major == LXC_VERSION_MAJOR && minor == LXC_VERSION_MINOR && micro > LXC_VERSION_MICRO)) +#ifndef LXC_DEVEL +#define LXC_DEVEL 0 +#endif bool go_lxc_defined(struct lxc_container *c) { return c->is_defined(c); @@ -92,16 +91,24 @@ bool go_lxc_wait(struct lxc_container *c, const char *state, int timeout) { return c->wait(c, state, timeout); } -char* go_lxc_get_config_item(struct lxc_container *c, const char *key) { +char *go_lxc_get_config_item(struct lxc_container *c, const char *key) +{ + char *value = NULL; + int len = c->get_config_item(c, key, NULL, 0); - if (len <= 0) { + if (len <= 0) + return NULL; + +again: + value = (char *)malloc(sizeof(char) * len + 1); + if (value == NULL) + goto again; + + if (c->get_config_item(c, key, value, len + 1) != len) { + free(value); return NULL; } - char* value = (char*)malloc(sizeof(char)*len + 1); - if (c->get_config_item(c, key, value, len + 1) != len) { - return NULL; - } return value; } @@ -121,29 +128,45 @@ char* go_lxc_get_running_config_item(struct lxc_container *c, const char *key) { return c->get_running_config_item(c, key); } -char* go_lxc_get_keys(struct lxc_container *c, const char *key) { +char *go_lxc_get_keys(struct lxc_container *c, const char *key) +{ + char *value = NULL; + int len = c->get_keys(c, key, NULL, 0); - if (len <= 0) { + if (len <= 0) + return NULL; + +again: + value = (char *)malloc(sizeof(char) * len + 1); + if (value == NULL) + goto again; + + if (c->get_keys(c, key, value, len + 1) != len) { + free(value); return NULL; } - char* value = (char*)malloc(sizeof(char)*len + 1); - if (c->get_keys(c, key, value, len + 1) != len) { - return NULL; - } return value; } -char* go_lxc_get_cgroup_item(struct lxc_container *c, const char *key) { +char *go_lxc_get_cgroup_item(struct lxc_container *c, const char *key) +{ + char *value = NULL; + int len = c->get_cgroup_item(c, key, NULL, 0); - if (len <= 0) { + if (len <= 0) + return NULL; + +again: + value = (char *)malloc(sizeof(char) * len + 1); + if (value == NULL) + goto again; + + if (c->get_cgroup_item(c, key, value, len + 1) != len) { + free(value); return NULL; } - char* value = (char*)malloc(sizeof(char)*len + 1); - if (c->get_cgroup_item(c, key, value, len + 1) != len) { - return NULL; - } return value; } @@ -173,10 +196,12 @@ bool go_lxc_clone(struct lxc_container *c, const char *newname, const char *lxcp int go_lxc_console_getfd(struct lxc_container *c, int ttynum) { int masterfd; + int ret = 0; + + ret = c->console_getfd(c, &ttynum, &masterfd); + if (ret < 0) + return ret; - if (c->console_getfd(c, &ttynum, &masterfd) < 0) { - return -1; - } return masterfd; } @@ -212,6 +237,51 @@ again: return status; } +int go_lxc_attach_no_wait(struct lxc_container *c, + bool clear_env, + int namespaces, + long personality, + uid_t uid, gid_t gid, + int stdinfd, int stdoutfd, int stderrfd, + char *initial_cwd, + char **extra_env_vars, + char **extra_keep_env, + const char * const argv[], + pid_t *attached_pid) { + int ret; + + lxc_attach_options_t attach_options = LXC_ATTACH_OPTIONS_DEFAULT; + lxc_attach_command_t command = (lxc_attach_command_t){.program = NULL}; + + attach_options.env_policy = LXC_ATTACH_KEEP_ENV; + if (clear_env) { + attach_options.env_policy = LXC_ATTACH_CLEAR_ENV; + } + + attach_options.namespaces = namespaces; + attach_options.personality = personality; + + attach_options.uid = uid; + attach_options.gid = gid; + + attach_options.stdin_fd = stdinfd; + attach_options.stdout_fd = stdoutfd; + attach_options.stderr_fd = stderrfd; + + attach_options.initial_cwd = initial_cwd; + attach_options.extra_env_vars = extra_env_vars; + attach_options.extra_keep_env = extra_keep_env; + + command.program = (char *)argv[0]; + command.argv = (char **)argv; + + ret = c->attach(c, lxc_attach_run_command, &command, &attach_options, attached_pid); + if (ret < 0) + return ret; + + return 0; +} + int go_lxc_attach(struct lxc_container *c, bool clear_env, int namespaces, @@ -257,16 +327,16 @@ int go_lxc_attach(struct lxc_container *c, ret = c->attach(c, lxc_attach_run_shell, NULL, &attach_options, &pid); if (ret < 0) - return -1; + return ret; ret = wait_for_pid_status(pid); if (ret < 0) - return -1; + return ret; if (WIFEXITED(ret)) return WEXITSTATUS(ret); - return -1; + return ret; } int go_lxc_attach_run_wait(struct lxc_container *c, @@ -366,6 +436,9 @@ bool go_lxc_restore(struct lxc_container *c, char *directory, bool verbose) { } int go_lxc_migrate(struct lxc_container *c, unsigned int cmd, struct migrate_opts *opts, struct extra_migrate_opts *extras) { +#if VERSION_AT_LEAST(3, 0, 0) + opts->features_to_check = extras->features_to_check; +#endif #if VERSION_AT_LEAST(2, 0, 4) opts->action_script = extras->action_script; opts->ghost_limit = extras->ghost_limit; @@ -397,3 +470,34 @@ bool go_lxc_detach_interface(struct lxc_container *c, const char *dev, const cha return false; #endif } + +bool go_lxc_config_item_is_supported(const char *key) +{ +#if VERSION_AT_LEAST(2, 1, 0) + return lxc_config_item_is_supported(key); +#else + return false; +#endif +} + +int go_lxc_error_num(struct lxc_container *c) +{ + return c->error_num; +} + +int go_lxc_console_log(struct lxc_container *c, struct lxc_console_log *log) { +#if VERSION_AT_LEAST(3, 0, 0) + return c->console_log(c, log); +#else + return false; +#endif +} + +bool go_lxc_has_api_extension(const char *extension) +{ +#if VERSION_AT_LEAST(3, 1, 0) + return lxc_has_api_extension(extension); +#else + return false; +#endif +} diff --git a/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.go b/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.go index 2fc3056f2..3cd38d618 100644 --- a/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.go +++ b/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.go @@ -11,10 +11,16 @@ package lxc // #include // #include // #include "lxc-binding.h" +// #ifndef LXC_DEVEL +// #define LXC_DEVEL 0 +// #endif import "C" import ( + "fmt" "runtime" + "strconv" + "strings" "unsafe" ) @@ -54,12 +60,22 @@ func Release(c *Container) bool { // http://golang.org/pkg/runtime/#SetFinalizer runtime.SetFinalizer(c, nil) + // Go is bad at refcounting sometimes + c.mu.Lock() + return C.lxc_container_put(c.container) == 1 } // Version returns the LXC version. func Version() string { - return C.GoString(C.lxc_get_version()) + version := C.GoString(C.lxc_get_version()) + + // New liblxc versions append "-devel" when LXC_DEVEL is set. + if strings.HasSuffix(version, "-devel") { + return fmt.Sprintf("%s (devel)", version[:(len(version)-len("-devel"))]) + } + + return version } // GlobalConfigItem returns the value of the given global config key. @@ -108,12 +124,12 @@ func ContainerNames(lxcpath ...string) []string { // Containers returns the defined and active containers on the system. Only // containers that could retrieved successfully are returned. -func Containers(lxcpath ...string) []Container { - var containers []Container +func Containers(lxcpath ...string) []*Container { + var containers []*Container for _, v := range ContainerNames(lxcpath...) { if container, err := NewContainer(v, lxcpath...); err == nil { - containers = append(containers, *container) + containers = append(containers, container) } } @@ -143,12 +159,12 @@ func DefinedContainerNames(lxcpath ...string) []string { // DefinedContainers returns the defined containers on the system. Only // containers that could retrieved successfully are returned. -func DefinedContainers(lxcpath ...string) []Container { - var containers []Container +func DefinedContainers(lxcpath ...string) []*Container { + var containers []*Container for _, v := range DefinedContainerNames(lxcpath...) { if container, err := NewContainer(v, lxcpath...); err == nil { - containers = append(containers, *container) + containers = append(containers, container) } } @@ -178,18 +194,19 @@ func ActiveContainerNames(lxcpath ...string) []string { // ActiveContainers returns the active containers on the system. Only // containers that could retrieved successfully are returned. -func ActiveContainers(lxcpath ...string) []Container { - var containers []Container +func ActiveContainers(lxcpath ...string) []*Container { + var containers []*Container for _, v := range ActiveContainerNames(lxcpath...) { if container, err := NewContainer(v, lxcpath...); err == nil { - containers = append(containers, *container) + containers = append(containers, container) } } return containers } +// VersionNumber returns the LXC version. func VersionNumber() (major int, minor int) { major = C.LXC_VERSION_MAJOR minor = C.LXC_VERSION_MINOR @@ -197,7 +214,12 @@ func VersionNumber() (major int, minor int) { return } +// VersionAtLeast returns true when the tested version >= current version. func VersionAtLeast(major int, minor int, micro int) bool { + if C.LXC_DEVEL == 1 { + return true + } + if major > C.LXC_VERSION_MAJOR { return false } @@ -215,3 +237,90 @@ func VersionAtLeast(major int, minor int, micro int) bool { return true } + +// IsSupportedConfigItem returns true if the key belongs to a supported config item. +func IsSupportedConfigItem(key string) bool { + configItem := C.CString(key) + defer C.free(unsafe.Pointer(configItem)) + return bool(C.go_lxc_config_item_is_supported(configItem)) +} + +// runtimeLiblxcVersionAtLeast checks if the system's liblxc matches the +// provided version requirement +func runtimeLiblxcVersionAtLeast(major int, minor int, micro int) bool { + version := Version() + version = strings.Replace(version, " (devel)", "-devel", 1) + parts := strings.Split(version, ".") + partsLen := len(parts) + if partsLen == 0 { + return false + } + + develParts := strings.Split(parts[partsLen-1], "-") + if len(develParts) == 2 && develParts[1] == "devel" { + return true + } + + maj := -1 + min := -1 + mic := -1 + + for i, v := range parts { + if i > 2 { + break + } + + num, err := strconv.Atoi(v) + if err != nil { + return false + } + + switch i { + case 0: + maj = num + case 1: + min = num + case 2: + mic = num + } + } + + /* Major version is greater. */ + if maj > major { + return true + } + + if maj < major { + return false + } + + /* Minor number is greater.*/ + if min > minor { + return true + } + + if min < minor { + return false + } + + /* Patch number is greater. */ + if mic > micro { + return true + } + + if mic < micro { + return false + } + + return true +} + +// HasApiExtension returns true if the extension is supported. +func HasApiExtension(extension string) bool { + if runtimeLiblxcVersionAtLeast(3, 1, 0) { + apiExtension := C.CString(extension) + defer C.free(unsafe.Pointer(apiExtension)) + return bool(C.go_lxc_has_api_extension(apiExtension)) + } + return false +} diff --git a/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.h b/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.h index f7ccd848f..da2cb961a 100644 --- a/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.h +++ b/vendor/gopkg.in/lxc/go-lxc.v2/lxc-binding.h @@ -2,6 +2,11 @@ // Use of this source code is governed by a LGPLv2.1 // license that can be found in the LICENSE file. +#define VERSION_AT_LEAST(major, minor, micro) \ + ((LXC_DEVEL == 1) || (!(major > LXC_VERSION_MAJOR || \ + major == LXC_VERSION_MAJOR && minor > LXC_VERSION_MINOR || \ + major == LXC_VERSION_MAJOR && minor == LXC_VERSION_MINOR && micro > LXC_VERSION_MICRO))) + extern bool go_lxc_add_device_node(struct lxc_container *c, const char *src_path, const char *dest_path); extern void go_lxc_clear_config(struct lxc_container *c); extern bool go_lxc_clear_config_item(struct lxc_container *c, const char *key); @@ -60,19 +65,32 @@ extern int go_lxc_attach(struct lxc_container *c, char *initial_cwd, char **extra_env_vars, char **extra_keep_env); +extern int go_lxc_attach_no_wait(struct lxc_container *c, + bool clear_env, + int namespaces, + long personality, + uid_t uid, gid_t gid, + int stdinfd, int stdoutfd, int stderrfd, + char *initial_cwd, + char **extra_env_vars, + char **extra_keep_env, + const char * const argv[], + pid_t *attached_pid); extern int go_lxc_console_getfd(struct lxc_container *c, int ttynum); extern int go_lxc_snapshot_list(struct lxc_container *c, struct lxc_snapshot **ret); extern int go_lxc_snapshot(struct lxc_container *c); extern pid_t go_lxc_init_pid(struct lxc_container *c); extern bool go_lxc_checkpoint(struct lxc_container *c, char *directory, bool stop, bool verbose); extern bool go_lxc_restore(struct lxc_container *c, char *directory, bool verbose); +extern bool go_lxc_config_item_is_supported(const char *key); +extern bool go_lxc_has_api_extension(const char *extension); /* n.b. that we're just adding the fields here to shorten the definition * of go_lxc_migrate; in the case where we don't have the ->migrate API call, * we don't want to have to pass all the arguments in to let conditional * compilation handle things, but the call will still fail */ -#if LXC_VERSION_MAJOR != 2 +#if !VERSION_AT_LEAST(2, 0, 0) struct migrate_opts { char *directory; bool verbose; @@ -89,8 +107,21 @@ struct extra_migrate_opts { bool preserves_inodes; char *action_script; uint64_t ghost_limit; + uint64_t features_to_check; }; int go_lxc_migrate(struct lxc_container *c, unsigned int cmd, struct migrate_opts *opts, struct extra_migrate_opts *extras); extern bool go_lxc_attach_interface(struct lxc_container *c, const char *dev, const char *dst_dev); extern bool go_lxc_detach_interface(struct lxc_container *c, const char *dev, const char *dst_dev); + +#if !VERSION_AT_LEAST(3, 0, 0) +struct lxc_console_log { + bool clear; + bool read; + uint64_t *read_max; + char *data; +}; +#endif + +extern int go_lxc_console_log(struct lxc_container *c, struct lxc_console_log *log); +extern int go_lxc_error_num(struct lxc_container *c); diff --git a/vendor/gopkg.in/lxc/go-lxc.v2/options.go b/vendor/gopkg.in/lxc/go-lxc.v2/options.go index 5b53c6079..23d1bc071 100644 --- a/vendor/gopkg.in/lxc/go-lxc.v2/options.go +++ b/vendor/gopkg.in/lxc/go-lxc.v2/options.go @@ -142,7 +142,7 @@ type ConsoleOptions struct { EscapeCharacter rune } -// DefailtConsoleOptions is a convenient set of options to be used. +// DefaultConsoleOptions is a convenient set of options to be used. var DefaultConsoleOptions = ConsoleOptions{ Tty: -1, StdinFd: os.Stdin.Fd(), @@ -175,25 +175,35 @@ var DefaultCloneOptions = CloneOptions{ Backend: Directory, } -// CheckpointOptions type is used for defining checkpoint options for CRIU +// CheckpointOptions type is used for defining checkpoint options for CRIU. type CheckpointOptions struct { Directory string Stop bool Verbose bool } -// RestoreOptions type is used for defining restore options for CRIU +// RestoreOptions type is used for defining restore options for CRIU. type RestoreOptions struct { Directory string Verbose bool } +// MigrateOptions type is used for defining migrate options. type MigrateOptions struct { Directory string + PredumpDir string + ActionScript string Verbose bool Stop bool - PredumpDir string PreservesInodes bool - ActionScript string GhostLimit uint64 + FeaturesToCheck CriuFeatures +} + +// ConsoleLogOptioins type is used for defining console log options. +type ConsoleLogOptions struct { + ClearLog bool + ReadLog bool + ReadMax uint64 + WriteToLogFile bool } diff --git a/vendor/gopkg.in/lxc/go-lxc.v2/type.go b/vendor/gopkg.in/lxc/go-lxc.v2/type.go index 3ab04cd2f..ced3c3396 100644 --- a/vendor/gopkg.in/lxc/go-lxc.v2/type.go +++ b/vendor/gopkg.in/lxc/go-lxc.v2/type.go @@ -260,7 +260,15 @@ const ( ) const ( - MIGRATE_PRE_DUMP = 0 - MIGRATE_DUMP = 1 - MIGRATE_RESTORE = 2 + MIGRATE_PRE_DUMP = 0 + MIGRATE_DUMP = 1 + MIGRATE_RESTORE = 2 + MIGRATE_FEATURE_CHECK = 3 +) + +type CriuFeatures uint64 + +const ( + FEATURE_MEM_TRACK CriuFeatures = 1 << iota + FEATURE_LAZY_PAGES ) diff --git a/vendor/gopkg.in/lxc/go-lxc.v2/util.go b/vendor/gopkg.in/lxc/go-lxc.v2/util.go index f17819569..fba2dab7d 100644 --- a/vendor/gopkg.in/lxc/go-lxc.v2/util.go +++ b/vendor/gopkg.in/lxc/go-lxc.v2/util.go @@ -12,7 +12,7 @@ package lxc static char** makeCharArray(size_t size) { // caller checks return value - return calloc(sizeof(char*), size); + return calloc(size, sizeof(char*)); } static void setArrayString(char **array, char *string, size_t n) { diff --git a/vendor/vendor.json b/vendor/vendor.json index 910e0a1f7..1063dfcf8 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -434,7 +434,7 @@ {"path":"google.golang.org/grpc/transport","checksumSHA1":"oFGr0JoquaPGVnV86fVL8MVTc3A=","revision":"0c41876308d45bc82e587965971e28be659a1aca","revisionTime":"2017-07-21T17:58:12Z"}, {"path":"gopkg.in/fsnotify.v1","checksumSHA1":"eIhF+hmL/XZhzTiAwhLD0M65vlY=","revision":"629574ca2a5df945712d3079857300b5e4da0236","revisionTime":"2016-10-11T02:33:12Z"}, {"path":"gopkg.in/inf.v0","checksumSHA1":"6f8MEU31llHM1sLM/GGH4/Qxu0A=","revision":"3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4","revisionTime":"2015-09-11T12:57:57Z"}, - {"path":"gopkg.in/lxc/go-lxc.v2","checksumSHA1":"i97goLq3AIfUNB8l1hxGGMSW0+s=","revision":"f8a6938e600c634232eeef79dc04a1226f73a88b","revisionTime":"2016-08-03T16:52:18Z"}, + {"path":"gopkg.in/lxc/go-lxc.v2","checksumSHA1":"oAflbBrzWC7OMmZQixkp9bnPQW8=","revision":"0aadfc37157c2e3f0e63bedd10f8615e66e91cad","revisionTime":"2018-11-01T16:03:35Z"}, {"path":"gopkg.in/tomb.v1","checksumSHA1":"TO8baX+t1Qs7EmOYth80MkbKzFo=","revision":"dd632973f1e7218eb1089048e0798ec9ae7dceb8","revisionTime":"2014-10-24T13:56:13Z"}, {"path":"gopkg.in/tomb.v2","checksumSHA1":"WiyCOMvfzRdymImAJ3ME6aoYUdM=","revision":"14b3d72120e8d10ea6e6b7f87f7175734b1faab8","revisionTime":"2014-06-26T14:46:23Z"}, {"path":"gopkg.in/yaml.v2","checksumSHA1":"12GqsW8PiRPnezDDy0v4brZrndM=","revision":"a5b47d31c556af34a302ce5d659e6fea44d90de0","revisionTime":"2016-09-28T15:37:09Z"} From ad1f8d8c20f78026173c53be96ca18594b65d674 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 25 Nov 2018 11:55:01 -0500 Subject: [PATCH 2/4] Fixes in old lxc driver --- client/driver/lxc.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/client/driver/lxc.go b/client/driver/lxc.go index ba46c829e..9c76ec617 100644 --- a/client/driver/lxc.go +++ b/client/driver/lxc.go @@ -153,18 +153,20 @@ func (d *LxcDriver) Validate(config map[string]interface{}) error { return err } - volumes, _ := fd.GetOk("volumes") - for _, volDesc := range volumes.([]interface{}) { - volStr := volDesc.(string) - paths := strings.Split(volStr, ":") - if len(paths) != 2 { - return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr) - } - if len(paths[0]) == 0 || len(paths[1]) == 0 { - return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr) - } - if paths[1][0] == '/' { - return fmt.Errorf("unsupported absolute container mount point: '%s'", paths[1]) + volumes, ok := fd.GetOk("volumes") + if ok { + for _, volDesc := range volumes.([]interface{}) { + volStr := volDesc.(string) + paths := strings.Split(volStr, ":") + if len(paths) != 2 { + return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr) + } + if len(paths[0]) == 0 || len(paths[1]) == 0 { + return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr) + } + if paths[1][0] == '/' { + return fmt.Errorf("unsupported absolute container mount point: '%s'", paths[1]) + } } } @@ -371,7 +373,7 @@ func (d *LxcDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error containers := lxc.Containers(pid.LxcPath) for _, c := range containers { if c.Name() == pid.ContainerName { - container = &c + container = c break } } From 6d34d2fadebebb641f23978861520e39a86a1b55 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 25 Nov 2018 11:55:01 -0500 Subject: [PATCH 3/4] Add Driver Plugin for LXC --- drivers/lxc/driver.go | 510 +++++++++++++++++++++++++ drivers/lxc/driver_test.go | 273 +++++++++++++ drivers/lxc/handle.go | 189 +++++++++ drivers/lxc/lxc.go | 172 +++++++++ drivers/lxc/state.go | 35 ++ plugins/shared/catalog/register_lxc.go | 14 + 6 files changed, 1193 insertions(+) create mode 100644 drivers/lxc/driver.go create mode 100644 drivers/lxc/driver_test.go create mode 100644 drivers/lxc/handle.go create mode 100644 drivers/lxc/lxc.go create mode 100644 drivers/lxc/state.go create mode 100644 plugins/shared/catalog/register_lxc.go diff --git a/drivers/lxc/driver.go b/drivers/lxc/driver.go new file mode 100644 index 000000000..8291deab8 --- /dev/null +++ b/drivers/lxc/driver.go @@ -0,0 +1,510 @@ +//+build linux,lxc + +package lxc + +import ( + "context" + "fmt" + "strconv" + "time" + + hclog "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/client/stats" + cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/drivers/shared/eventer" + "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/shared/hclspec" + "github.com/hashicorp/nomad/plugins/shared/loader" + + lxc "gopkg.in/lxc/go-lxc.v2" +) + +const ( + // pluginName is the name of the plugin + pluginName = "lxc" + + // fingerprintPeriod is the interval at which the driver will send fingerprint responses + fingerprintPeriod = 30 * time.Second +) + +var ( + // PluginID is the rawexec plugin metadata registered in the plugin + // catalog. + PluginID = loader.PluginID{ + Name: pluginName, + PluginType: base.PluginTypeDriver, + } + + // PluginConfig is the rawexec factory function registered in the + // plugin catalog. + PluginConfig = &loader.InternalPluginConfig{ + Config: map[string]interface{}{}, + Factory: func(l hclog.Logger) interface{} { return NewLXCDriver(l) }, + } +) + +// PluginLoader maps pre-0.9 client driver options to post-0.9 plugin options. +func PluginLoader(opts map[string]string) (map[string]interface{}, error) { + conf := map[string]interface{}{} + if v, err := strconv.ParseBool(opts["driver.lxc.enable"]); err == nil { + conf["enabled"] = v + } + if v, err := strconv.ParseBool(opts["lxc.volumes.enabled"]); err == nil { + conf["volumes"] = v + } + if v, ok := opts["driver.lxc.path"]; ok { + conf["path"] = v + } + return conf, nil +} + +var ( + // pluginInfo is the response returned for the PluginInfo RPC + pluginInfo = &base.PluginInfoResponse{ + Type: base.PluginTypeDriver, + PluginApiVersion: "0.0.1", + PluginVersion: "0.1.0", + Name: pluginName, + } + + // configSpec is the hcl specification returned by the ConfigSchema RPC + configSpec = hclspec.NewObject(map[string]*hclspec.Spec{ + "enabled": hclspec.NewDefault( + hclspec.NewAttr("enabled", "bool", false), + hclspec.NewLiteral("true"), + ), + "volumes": hclspec.NewDefault( + hclspec.NewAttr("volumes", "bool", false), + hclspec.NewLiteral("true"), + ), + "path": hclspec.NewDefault( + hclspec.NewAttr("path", "string", false), + hclspec.NewLiteral("\"\""), + ), + }) + + // taskConfigSpec is the hcl specification for the driver config section of + // a task within a job. It is returned in the TaskConfigSchema RPC + taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{ + "template": hclspec.NewAttr("template", "string", true), + "distro": hclspec.NewAttr("distro", "string", false), + "release": hclspec.NewAttr("release", "string", false), + "arch": hclspec.NewAttr("arch", "string", false), + "image_variant": hclspec.NewAttr("image_variant", "string", false), + "image_server": hclspec.NewAttr("image_server", "string", false), + "gpg_key_id": hclspec.NewAttr("gpg_key_id", "string", false), + "gpg_key_server": hclspec.NewAttr("gpg_key_server", "string", false), + "disable_gpg": hclspec.NewAttr("disable_gpg", "string", false), + "flush_cache": hclspec.NewAttr("flush_cache", "string", false), + "force_cache": hclspec.NewAttr("force_cache", "string", false), + "template_args": hclspec.NewAttr("template_args", "list(string)", false), + "log_level": hclspec.NewAttr("log_level", "string", false), + "verbosity": hclspec.NewAttr("verbosity", "string", false), + "volumes": hclspec.NewAttr("volumes", "list(string)", false), + }) + + // capabilities is returned by the Capabilities RPC and indicates what + // optional features this driver supports + capabilities = &drivers.Capabilities{ + SendSignals: false, + Exec: false, + FSIsolation: cstructs.FSIsolationImage, + } +) + +// Driver is a privileged version of the exec driver. It provides no +// resource isolation and just fork/execs. The Exec driver should be preferred +// and this should only be used when explicitly needed. +type Driver struct { + // eventer is used to handle multiplexing of TaskEvents calls such that an + // event can be broadcast to all callers + eventer *eventer.Eventer + + // config is the driver configuration set by the SetConfig RPC + config *Config + + // nomadConfig is the client config from nomad + nomadConfig *base.ClientDriverConfig + + // tasks is the in memory datastore mapping taskIDs to rawExecDriverHandles + tasks *taskStore + + // ctx is the context for the driver. It is passed to other subsystems to + // coordinate shutdown + ctx context.Context + + // signalShutdown is called when the driver is shutting down and cancels the + // ctx passed to any subsystems + signalShutdown context.CancelFunc + + // logger will log to the Nomad agent + logger hclog.Logger +} + +// Config is the driver configuration set by the SetConfig RPC call +type Config struct { + // Enabled is set to true to enable the raw_exec driver + Enabled bool `codec:"enabled"` + + AllowVolumes bool `codec:"volumes"` + + Path string `codec:"path"` +} + +// TaskConfig is the driver configuration of a task within a job +type TaskConfig struct { + Template string `codec:"template"` + Distro string `codec:"distro"` + Release string `codec:"release"` + Arch string `codec:"arch"` + ImageVariant string `codec:"image_variant"` + ImageServer string `codec:"image_server"` + GPGKeyID string `codec:"gpg_key_id"` + GPGKeyServer string `codec:"gpg_key_server"` + DisableGPGValidation bool `codec:"disable_gpg"` + FlushCache bool `codec:"flush_cache"` + ForceCache bool `codec:"force_cache"` + TemplateArgs []string `codec:"template_args"` + LogLevel string `codec:"log_level"` + Verbosity string `codec:"verbosity"` + Volumes []string `codec:"volumes"` +} + +// TaskState is the state which is encoded in the handle returned in +// StartTask. This information is needed to rebuild the task state and handler +// during recovery. +type TaskState struct { + TaskConfig *drivers.TaskConfig + ContainerName string + StartedAt time.Time +} + +// NewLXCDriver returns a new DriverPlugin implementation +func NewLXCDriver(logger hclog.Logger) drivers.DriverPlugin { + ctx, cancel := context.WithCancel(context.Background()) + logger = logger.Named(pluginName) + return &Driver{ + eventer: eventer.NewEventer(ctx, logger), + config: &Config{}, + tasks: newTaskStore(), + ctx: ctx, + signalShutdown: cancel, + logger: logger, + } +} + +func (d *Driver) PluginInfo() (*base.PluginInfoResponse, error) { + return pluginInfo, nil +} + +func (d *Driver) ConfigSchema() (*hclspec.Spec, error) { + return configSpec, nil +} + +func (d *Driver) SetConfig(data []byte, cfg *base.ClientAgentConfig) error { + var config Config + if err := base.MsgPackDecode(data, &config); err != nil { + return err + } + + d.config = &config + if cfg != nil { + d.nomadConfig = cfg.Driver + } + + return nil +} + +func (d *Driver) Shutdown(ctx context.Context) error { + d.signalShutdown() + return nil +} + +func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { + return taskConfigSpec, nil +} + +func (d *Driver) Capabilities() (*drivers.Capabilities, error) { + return capabilities, nil +} + +func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { + ch := make(chan *drivers.Fingerprint) + go d.handleFingerprint(ctx, ch) + return ch, nil +} + +func (d *Driver) handleFingerprint(ctx context.Context, ch chan<- *drivers.Fingerprint) { + defer close(ch) + ticker := time.NewTimer(0) + for { + select { + case <-ctx.Done(): + return + case <-d.ctx.Done(): + return + case <-ticker.C: + ticker.Reset(fingerprintPeriod) + ch <- d.buildFingerprint() + } + } +} + +func (d *Driver) buildFingerprint() *drivers.Fingerprint { + var health drivers.HealthState + var desc string + attrs := map[string]string{} + + lxcVersion := lxc.Version() + + if d.config.Enabled && lxcVersion != "" { + health = drivers.HealthStateHealthy + desc = "ready" + attrs["driver.lxc"] = "1" + attrs["driver.lxc.version"] = lxcVersion + } else { + health = drivers.HealthStateUndetected + desc = "disabled" + } + + if d.config.AllowVolumes { + attrs["driver.lxc.volumes.enabled"] = "1" + } + + return &drivers.Fingerprint{ + Attributes: attrs, + Health: health, + HealthDescription: desc, + } +} + +func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { + if handle == nil { + return fmt.Errorf("error: handle cannot be nil") + } + + if _, ok := d.tasks.Get(handle.Config.ID); ok { + return nil + } + + var taskState TaskState + if err := handle.GetDriverState(&taskState); err != nil { + return fmt.Errorf("failed to decode task state from handle: %v", err) + } + + var driverConfig TaskConfig + if err := taskState.TaskConfig.DecodeDriverConfig(&driverConfig); err != nil { + return fmt.Errorf("failed to decode driver config: %v", err) + } + + c, err := lxc.NewContainer(taskState.ContainerName, d.lxcPath()) + if err != nil { + return fmt.Errorf("failed to create container ref: %v", err) + } + + initPid := c.InitPid() + h := &taskHandle{ + container: c, + initPid: initPid, + taskConfig: taskState.TaskConfig, + procState: drivers.TaskStateRunning, + startedAt: taskState.StartedAt, + exitResult: &drivers.ExitResult{}, + + totalCpuStats: stats.NewCpuStats(), + userCpuStats: stats.NewCpuStats(), + systemCpuStats: stats.NewCpuStats(), + } + + d.tasks.Set(taskState.TaskConfig.ID, h) + + go h.run() + return nil +} + +func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstructs.DriverNetwork, error) { + if _, ok := d.tasks.Get(cfg.ID); ok { + return nil, nil, fmt.Errorf("task with ID %q already started", cfg.ID) + } + + var driverConfig TaskConfig + if err := cfg.DecodeDriverConfig(&driverConfig); err != nil { + return nil, nil, fmt.Errorf("failed to decode driver config: %v", err) + } + + d.logger.Info("starting lxc task", "driver_cfg", hclog.Fmt("%+v", driverConfig)) + handle := drivers.NewTaskHandle(pluginName) + handle.Config = cfg + + c, err := d.initializeContainer(cfg, driverConfig) + if err != nil { + return nil, nil, err + } + + opt := toLXCCreateOptions(driverConfig) + if err := c.Create(opt); err != nil { + return nil, nil, fmt.Errorf("unable to create container: %v", err) + } + + cleanup := func() { + if err := c.Destroy(); err != nil { + d.logger.Error("failed to clean up from an error in Start", "error", err) + } + } + + if err := d.configureContainerNetwork(c); err != nil { + cleanup() + return nil, nil, err + } + + if err := d.mountVolumes(c, cfg, driverConfig); err != nil { + cleanup() + return nil, nil, err + } + + if err := c.Start(); err != nil { + cleanup() + return nil, nil, fmt.Errorf("unable to start container: %v", err) + } + + if err := d.setResourceLimits(c, cfg); err != nil { + cleanup() + return nil, nil, err + } + + pid := c.InitPid() + + h := &taskHandle{ + container: c, + initPid: pid, + taskConfig: cfg, + procState: drivers.TaskStateRunning, + startedAt: time.Now().Round(time.Millisecond), + logger: d.logger, + + totalCpuStats: stats.NewCpuStats(), + userCpuStats: stats.NewCpuStats(), + systemCpuStats: stats.NewCpuStats(), + } + + driverState := TaskState{ + ContainerName: c.Name(), + TaskConfig: cfg, + StartedAt: h.startedAt, + } + + if err := handle.SetDriverState(&driverState); err != nil { + d.logger.Error("failed to start task, error setting driver state", "error", err) + cleanup() + return nil, nil, fmt.Errorf("failed to set driver state: %v", err) + } + + d.tasks.Set(cfg.ID, h) + + go h.run() + + return handle, nil, nil +} + +func (d *Driver) WaitTask(ctx context.Context, taskID string) (<-chan *drivers.ExitResult, error) { + handle, ok := d.tasks.Get(taskID) + if !ok { + return nil, drivers.ErrTaskNotFound + } + + ch := make(chan *drivers.ExitResult) + go d.handleWait(ctx, handle, ch) + + return ch, nil +} + +func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *drivers.ExitResult) { + defer close(ch) + + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-d.ctx.Done(): + return + case <-ticker.C: + s := handle.TaskStatus() + if s.State == drivers.TaskStateExited { + ch <- handle.exitResult + } + } + } +} + +func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) error { + handle, ok := d.tasks.Get(taskID) + if !ok { + return drivers.ErrTaskNotFound + } + + if err := handle.container.Shutdown(timeout); err != nil { + if err := handle.container.Stop(); err != nil { + return fmt.Errorf("executor Shutdown failed: %v", err) + } + } + + return nil +} + +func (d *Driver) DestroyTask(taskID string, force bool) error { + handle, ok := d.tasks.Get(taskID) + if !ok { + return drivers.ErrTaskNotFound + } + + if handle.IsRunning() && !force { + return fmt.Errorf("cannot destroy running task") + } + + if handle.IsRunning() { + if err := handle.container.Shutdown(0); err != nil { + handle.logger.Error("failed to destroy executor", "err", err) + } + } + + d.tasks.Delete(taskID) + return nil +} + +func (d *Driver) InspectTask(taskID string) (*drivers.TaskStatus, error) { + handle, ok := d.tasks.Get(taskID) + if !ok { + return nil, drivers.ErrTaskNotFound + } + + return handle.TaskStatus(), nil +} + +func (d *Driver) TaskStats(taskID string) (*cstructs.TaskResourceUsage, error) { + handle, ok := d.tasks.Get(taskID) + if !ok { + return nil, drivers.ErrTaskNotFound + } + + return handle.stats() +} + +func (d *Driver) TaskEvents(ctx context.Context) (<-chan *drivers.TaskEvent, error) { + return d.eventer.TaskEvents(ctx) +} + +func (d *Driver) SignalTask(taskID string, signal string) error { + return fmt.Errorf("LXC driver does not support signals") +} + +func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (*drivers.ExecTaskResult, error) { + if len(cmd) == 0 { + return nil, fmt.Errorf("error cmd must have atleast one value") + } + + return nil, fmt.Errorf("LXC driver does not support exec") +} diff --git a/drivers/lxc/driver_test.go b/drivers/lxc/driver_test.go new file mode 100644 index 000000000..4889242d3 --- /dev/null +++ b/drivers/lxc/driver_test.go @@ -0,0 +1,273 @@ +// +build linux,lxc + +package lxc + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/hashicorp/hcl2/hcl" + ctestutil "github.com/hashicorp/nomad/client/testutil" + "github.com/hashicorp/nomad/helper/testlog" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/shared" + "github.com/hashicorp/nomad/plugins/shared/hclspec" + "github.com/hashicorp/nomad/testutil" + "github.com/stretchr/testify/require" + lxc "gopkg.in/lxc/go-lxc.v2" +) + +func TestLXCDriver_Fingerprint(t *testing.T) { + t.Parallel() + requireLXC(t) + + require := require.New(t) + + d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) + d.config.Enabled = true + harness := drivers.NewDriverHarness(t, d) + + fingerCh, err := harness.Fingerprint(context.Background()) + require.NoError(err) + select { + case finger := <-fingerCh: + require.Equal(drivers.HealthStateHealthy, finger.Health) + require.Equal("1", finger.Attributes["driver.lxc"]) + require.NotEmpty(finger.Attributes["driver.lxc.version"]) + case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): + require.Fail("timeout receiving fingerprint") + } +} + +func TestLXCDriver_FingerprintNotEnabled(t *testing.T) { + t.Parallel() + requireLXC(t) + + require := require.New(t) + + d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) + d.config.Enabled = false + harness := drivers.NewDriverHarness(t, d) + + fingerCh, err := harness.Fingerprint(context.Background()) + require.NoError(err) + select { + case finger := <-fingerCh: + require.Equal(drivers.HealthStateUndetected, finger.Health) + require.Equal("", finger.Attributes["driver.lxc"]) + require.Empty(finger.Attributes["driver.lxc.version"]) + case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): + require.Fail("timeout receiving fingerprint") + } +} + +func TestLXCDriver_Start_Wait(t *testing.T) { + if !testutil.IsTravis() { + t.Parallel() + } + requireLXC(t) + ctestutil.RequireRoot(t) + + require := require.New(t) + + // prepare test file + testFileContents := []byte("this should be visible under /mnt/tmp") + tmpFile, err := ioutil.TempFile("/tmp", "testlxcdriver_start_wait") + if err != nil { + t.Fatalf("error writing temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + if _, err := tmpFile.Write(testFileContents); err != nil { + t.Fatalf("error writing temp file: %v", err) + } + if err := tmpFile.Close(); err != nil { + t.Fatalf("error closing temp file: %v", err) + } + + d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) + d.config.Enabled = true + d.config.AllowVolumes = true + + harness := drivers.NewDriverHarness(t, d) + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "test", + Resources: &drivers.Resources{ + NomadResources: &structs.Resources{ + CPU: 1, + MemoryMB: 2, + }, + LinuxResources: &drivers.LinuxResources{ + CPUShares: 1024, + MemoryLimitBytes: 2 * 1024, + }, + }, + } + taskConfig := map[string]interface{}{ + "template": "/usr/share/lxc/templates/lxc-busybox", + "volumes": []string{"/tmp/:mnt/tmp"}, + } + encodeDriverHelper(require, task, taskConfig) + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + fmt.Println(task.AllocDir) + + handle, _, err := harness.StartTask(task) + require.NoError(err) + require.NotNil(handle) + + lxcHandle, ok := d.tasks.Get(task.ID) + require.True(ok) + + container := lxcHandle.container + + // Destroy container after test + defer func() { + container.Stop() + container.Destroy() + }() + + // Test that container is running + testutil.WaitForResult(func() (bool, error) { + state := container.State() + if state == lxc.RUNNING { + return true, nil + } + return false, fmt.Errorf("container in state: %v", state) + }, func(err error) { + t.Fatalf("container failed to start: %v", err) + }) + + // Test that directories are mounted in their proper location + containerName := container.Name() + for _, mnt := range []string{"alloc", "local", "secrets", "mnt/tmp"} { + fullpath := filepath.Join(d.lxcPath(), containerName, "rootfs", mnt) + stat, err := os.Stat(fullpath) + require.NoError(err) + require.True(stat.IsDir()) + } + + // Test bind mount volumes exist in container: + mountedContents, err := exec.Command("lxc-attach", + "-n", containerName, "--", + "cat", filepath.Join("/mnt/", tmpFile.Name()), + ).Output() + require.NoError(err) + require.Equal(string(testFileContents), string(mountedContents)) + + // Test that killing container marks container as stopped + require.NoError(container.Stop()) + + testutil.WaitForResult(func() (bool, error) { + status, err := d.InspectTask(task.ID) + if err == nil && status.State == drivers.TaskStateExited { + return true, nil + } + return false, fmt.Errorf("task in state: %v", status.State) + }, func(err error) { + t.Fatalf("task was not marked as stopped: %v", err) + }) +} + +func TestLXCDriver_Start_Stop(t *testing.T) { + if !testutil.IsTravis() { + t.Parallel() + } + requireLXC(t) + ctestutil.RequireRoot(t) + + require := require.New(t) + + d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) + d.config.Enabled = true + d.config.AllowVolumes = true + + harness := drivers.NewDriverHarness(t, d) + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "test", + Resources: &drivers.Resources{ + NomadResources: &structs.Resources{ + CPU: 1, + MemoryMB: 2, + }, + LinuxResources: &drivers.LinuxResources{ + CPUShares: 1024, + MemoryLimitBytes: 2 * 1024, + }, + }, + } + taskConfig := map[string]interface{}{ + "template": "/usr/share/lxc/templates/lxc-busybox", + } + encodeDriverHelper(require, task, taskConfig) + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + fmt.Println(task.AllocDir) + + handle, _, err := harness.StartTask(task) + require.NoError(err) + require.NotNil(handle) + + lxcHandle, ok := d.tasks.Get(task.ID) + require.True(ok) + + container := lxcHandle.container + + // Destroy container after test + defer func() { + container.Stop() + container.Destroy() + }() + + // Test that container is running + testutil.WaitForResult(func() (bool, error) { + state := container.State() + if state == lxc.RUNNING { + return true, nil + } + return false, fmt.Errorf("container in state: %v", state) + }, func(err error) { + t.Fatalf("container failed to start: %v", err) + }) + + require.NoError(d.StopTask(task.ID, 5*time.Second, "kill")) + + testutil.WaitForResult(func() (bool, error) { + status, err := d.InspectTask(task.ID) + if err == nil && status.State == drivers.TaskStateExited { + return true, nil + } + return false, fmt.Errorf("task in state: %v", status.State) + }, func(err error) { + t.Fatalf("task was not marked as stopped: %v", err) + }) +} + +func requireLXC(t *testing.T) { + if lxc.Version() == "" { + t.Skip("skipping, lxc not present") + } +} + +func encodeDriverHelper(require *require.Assertions, task *drivers.TaskConfig, taskConfig map[string]interface{}) { + evalCtx := &hcl.EvalContext{ + Functions: shared.GetStdlibFuncs(), + } + spec, diag := hclspec.Convert(taskConfigSpec) + require.False(diag.HasErrors()) + taskConfigCtyVal, diag := shared.ParseHclInterface(taskConfig, spec, evalCtx) + require.False(diag.HasErrors()) + err := task.EncodeDriverConfig(taskConfigCtyVal) + require.Nil(err) +} diff --git a/drivers/lxc/handle.go b/drivers/lxc/handle.go new file mode 100644 index 000000000..5d5799d2f --- /dev/null +++ b/drivers/lxc/handle.go @@ -0,0 +1,189 @@ +//+build linux,lxc + +package lxc + +import ( + "fmt" + "strconv" + "strings" + "sync" + "time" + + hclog "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/client/stats" + cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/plugins/drivers" + lxc "gopkg.in/lxc/go-lxc.v2" +) + +type taskHandle struct { + container *lxc.Container + initPid int + logger hclog.Logger + + totalCpuStats *stats.CpuStats + userCpuStats *stats.CpuStats + systemCpuStats *stats.CpuStats + + // stateLock syncs access to all fields below + stateLock sync.RWMutex + + taskConfig *drivers.TaskConfig + procState drivers.TaskState + startedAt time.Time + completedAt time.Time + exitResult *drivers.ExitResult +} + +var ( + LXCMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"} + + LXCMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"} +) + +func (h *taskHandle) TaskStatus() *drivers.TaskStatus { + h.stateLock.RLock() + defer h.stateLock.RUnlock() + + return &drivers.TaskStatus{ + ID: h.taskConfig.ID, + Name: h.taskConfig.Name, + State: h.procState, + StartedAt: h.startedAt, + CompletedAt: h.completedAt, + ExitResult: h.exitResult, + DriverAttributes: map[string]string{ + "pid": strconv.Itoa(h.initPid), + }, + } +} + +func (h *taskHandle) IsRunning() bool { + h.stateLock.RLock() + defer h.stateLock.RUnlock() + return h.procState == drivers.TaskStateRunning +} + +func (h *taskHandle) run() { + h.stateLock.Lock() + if h.exitResult == nil { + h.exitResult = &drivers.ExitResult{} + } + h.stateLock.Unlock() + + if ok, err := waitTillStopped(h.container); !ok { + h.logger.Error("failed to find container process", "error", err) + return + } + + h.stateLock.Lock() + defer h.stateLock.Unlock() + + h.procState = drivers.TaskStateExited + h.exitResult.ExitCode = 0 + h.exitResult.Signal = 0 + h.completedAt = time.Now() + + // TODO: detect if the task OOMed +} + +func (h *taskHandle) stats() (*cstructs.TaskResourceUsage, error) { + cpuStats, err := h.container.CPUStats() + if err != nil { + h.logger.Error("failed to get container cpu stats", "error", err) + return nil, nil + } + total, err := h.container.CPUTime() + if err != nil { + h.logger.Error("failed to get container cpu time", "error", err) + return nil, nil + } + + t := time.Now() + + // Get the cpu stats + system := cpuStats["system"] + user := cpuStats["user"] + cs := &cstructs.CpuStats{ + SystemMode: h.systemCpuStats.Percent(float64(system)), + UserMode: h.systemCpuStats.Percent(float64(user)), + Percent: h.totalCpuStats.Percent(float64(total)), + TotalTicks: float64(user + system), + Measured: LXCMeasuredCpuStats, + } + + // Get the Memory Stats + memData := map[string]uint64{ + "rss": 0, + "cache": 0, + "swap": 0, + } + rawMemStats := h.container.CgroupItem("memory.stat") + for _, rawMemStat := range rawMemStats { + key, val, err := keysToVal(rawMemStat) + if err != nil { + h.logger.Error("failed to get stat", "line", rawMemStat, "error", err) + continue + } + if _, ok := memData[key]; ok { + memData[key] = val + + } + } + ms := &cstructs.MemoryStats{ + RSS: memData["rss"], + Cache: memData["cache"], + Swap: memData["swap"], + Measured: LXCMeasuredMemStats, + } + + mu := h.container.CgroupItem("memory.max_usage_in_bytes") + for _, rawMemMaxUsage := range mu { + val, err := strconv.ParseUint(rawMemMaxUsage, 10, 64) + if err != nil { + h.logger.Error("failed to get max memory usage", "error", err) + continue + } + ms.MaxUsage = val + } + ku := h.container.CgroupItem("memory.kmem.usage_in_bytes") + for _, rawKernelUsage := range ku { + val, err := strconv.ParseUint(rawKernelUsage, 10, 64) + if err != nil { + h.logger.Error("failed to get kernel memory usage", "error", err) + continue + } + ms.KernelUsage = val + } + + mku := h.container.CgroupItem("memory.kmem.max_usage_in_bytes") + for _, rawMaxKernelUsage := range mku { + val, err := strconv.ParseUint(rawMaxKernelUsage, 10, 64) + if err != nil { + h.logger.Error("failed tog get max kernel memory usage", "error", err) + continue + } + ms.KernelMaxUsage = val + } + + taskResUsage := cstructs.TaskResourceUsage{ + ResourceUsage: &cstructs.ResourceUsage{ + CpuStats: cs, + MemoryStats: ms, + }, + Timestamp: t.UTC().UnixNano(), + } + + return &taskResUsage, nil + +} + +func keysToVal(line string) (string, uint64, error) { + tokens := strings.Split(line, " ") + if len(tokens) != 2 { + return "", 0, fmt.Errorf("line isn't a k/v pair") + } + key := tokens[0] + val, err := strconv.ParseUint(tokens[1], 10, 64) + return key, val, err +} diff --git a/drivers/lxc/lxc.go b/drivers/lxc/lxc.go new file mode 100644 index 000000000..f53fa14a2 --- /dev/null +++ b/drivers/lxc/lxc.go @@ -0,0 +1,172 @@ +//+build linux,lxc + +package lxc + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/plugins/drivers" + lxc "gopkg.in/lxc/go-lxc.v2" +) + +var ( + verbosityLevels = map[string]lxc.Verbosity{ + "": lxc.Quiet, + "verbose": lxc.Verbose, + "quiet": lxc.Quiet, + } + + logLevels = map[string]lxc.LogLevel{ + "": lxc.ERROR, + "debug": lxc.DEBUG, + "error": lxc.ERROR, + "info": lxc.INFO, + "trace": lxc.TRACE, + "warn": lxc.WARN, + } +) + +const ( + // containerMonitorIntv is the interval at which the driver checks if the + // container is still alive + containerMonitorIntv = 2 * time.Second +) + +func (d *Driver) lxcPath() string { + lxcPath := d.config.Path + if lxcPath == "" { + lxcPath = lxc.DefaultConfigPath() + } + return lxcPath + +} +func (d *Driver) initializeContainer(cfg *drivers.TaskConfig, taskConfig TaskConfig) (*lxc.Container, error) { + lxcPath := d.lxcPath() + + containerName := fmt.Sprintf("%s-%s", cfg.Name, uuid.Generate()) + c, err := lxc.NewContainer(containerName, lxcPath) + if err != nil { + return nil, fmt.Errorf("failed to initialize container: %v", err) + } + + if v, ok := verbosityLevels[taskConfig.Verbosity]; ok { + c.SetVerbosity(v) + } else { + return nil, fmt.Errorf("lxc driver config 'verbosity' can only be either quiet or verbose") + } + + if v, ok := logLevels[taskConfig.LogLevel]; ok { + c.SetLogLevel(v) + } else { + return nil, fmt.Errorf("lxc driver config 'log_level' can only be trace, debug, info, warn or error") + } + + logFile := filepath.Join(cfg.TaskDir().Dir, fmt.Sprintf("%v-lxc.log", cfg.Name)) + c.SetLogFile(logFile) + + return c, nil +} + +func (d *Driver) configureContainerNetwork(c *lxc.Container) error { + // Set the network type to none + if err := c.SetConfigItem(networkTypeConfigKey(), "none"); err != nil { + return fmt.Errorf("error setting network type configuration: %v", err) + } + return nil +} + +func networkTypeConfigKey() string { + if lxc.VersionAtLeast(2, 1, 0) { + return "lxc.net.0.type" + } + + // prior to 2.1, network used + return "lxc.network.type" +} + +func (d *Driver) mountVolumes(c *lxc.Container, cfg *drivers.TaskConfig, taskConfig TaskConfig) error { + // Bind mount the shared alloc dir and task local dir in the container + mounts := []string{ + fmt.Sprintf("%s local none rw,bind,create=dir", cfg.TaskDir().LocalDir), + fmt.Sprintf("%s alloc none rw,bind,create=dir", cfg.TaskDir().SharedAllocDir), + fmt.Sprintf("%s secrets none rw,bind,create=dir", cfg.TaskDir().SecretsDir), + } + + volumesEnabled := d.config.AllowVolumes + + for _, volDesc := range taskConfig.Volumes { + // the format was checked in Validate() + paths := strings.Split(volDesc, ":") + + if filepath.IsAbs(paths[0]) { + if !volumesEnabled { + return fmt.Errorf("absolute bind-mount volume in config but volumes are disabled") + } + } else { + // Relative source paths are treated as relative to alloc dir + paths[0] = filepath.Join(cfg.TaskDir().Dir, paths[0]) + } + + mounts = append(mounts, fmt.Sprintf("%s %s none rw,bind,create=dir", paths[0], paths[1])) + } + + for _, mnt := range mounts { + if err := c.SetConfigItem("lxc.mount.entry", mnt); err != nil { + return fmt.Errorf("error setting bind mount %q error: %v", mnt, err) + } + } + + return nil +} + +func (d *Driver) setResourceLimits(c *lxc.Container, cfg *drivers.TaskConfig) error { + if err := c.SetMemoryLimit(lxc.ByteSize(cfg.Resources.NomadResources.MemoryMB) * lxc.MB); err != nil { + return fmt.Errorf("unable to set memory limits: %v", err) + } + + if err := c.SetCgroupItem("cpu.shares", strconv.FormatInt(cfg.Resources.LinuxResources.CPUShares, 10)); err != nil { + return fmt.Errorf("unable to set cpu shares: %v", err) + } + + return nil +} + +func toLXCCreateOptions(taskConfig TaskConfig) lxc.TemplateOptions { + return lxc.TemplateOptions{ + Template: taskConfig.Template, + Distro: taskConfig.Distro, + Release: taskConfig.Release, + Arch: taskConfig.Arch, + FlushCache: taskConfig.FlushCache, + DisableGPGValidation: taskConfig.DisableGPGValidation, + ExtraArgs: taskConfig.TemplateArgs, + } +} + +// waitTillStopped blocks and returns true when container stops; +// returns false with an error message if the container processes cannot be identified. +// +// Use this in preference to c.Wait() - lxc Wait() function holds a write lock on the container +// blocking any other operation on container, including looking up container stats +func waitTillStopped(c *lxc.Container) (bool, error) { + ps, err := os.FindProcess(c.InitPid()) + if err != nil { + return false, err + } + + for { + if err := ps.Signal(syscall.Signal(0)); err != nil { + return true, nil + } + + time.Sleep(containerMonitorIntv) + + } +} diff --git a/drivers/lxc/state.go b/drivers/lxc/state.go new file mode 100644 index 000000000..c316a6cc6 --- /dev/null +++ b/drivers/lxc/state.go @@ -0,0 +1,35 @@ +//+build linux,lxc + +package lxc + +import ( + "sync" +) + +type taskStore struct { + store map[string]*taskHandle + lock sync.RWMutex +} + +func newTaskStore() *taskStore { + return &taskStore{store: map[string]*taskHandle{}} +} + +func (ts *taskStore) Set(id string, handle *taskHandle) { + ts.lock.Lock() + defer ts.lock.Unlock() + ts.store[id] = handle +} + +func (ts *taskStore) Get(id string) (*taskHandle, bool) { + ts.lock.RLock() + defer ts.lock.RUnlock() + t, ok := ts.store[id] + return t, ok +} + +func (ts *taskStore) Delete(id string) { + ts.lock.Lock() + defer ts.lock.Unlock() + delete(ts.store, id) +} diff --git a/plugins/shared/catalog/register_lxc.go b/plugins/shared/catalog/register_lxc.go new file mode 100644 index 000000000..6a029ec3f --- /dev/null +++ b/plugins/shared/catalog/register_lxc.go @@ -0,0 +1,14 @@ +//+build linux,lxc + +package catalog + +import ( + "github.com/hashicorp/nomad/drivers/lxc" +) + +// This file is where all builtin plugins should be registered in the catalog. +// Plugins with build restrictions should be placed in the appropriate +// register_XXX.go file. +func init() { + RegisterDeferredConfig(lxc.PluginID, lxc.PluginConfig, lxc.PluginLoader) +} From 9af8deabbf1f7b391994179512c37b8269cfc3a7 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Tue, 27 Nov 2018 21:39:43 -0500 Subject: [PATCH 4/4] address review comments --- drivers/lxc/driver.go | 29 ++++++++++++++++------------- drivers/lxc/driver_test.go | 2 -- drivers/lxc/handle.go | 11 +++++++++++ drivers/lxc/lxc.go | 4 +--- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/drivers/lxc/driver.go b/drivers/lxc/driver.go index 8291deab8..1495921d8 100644 --- a/drivers/lxc/driver.go +++ b/drivers/lxc/driver.go @@ -113,9 +113,7 @@ var ( } ) -// Driver is a privileged version of the exec driver. It provides no -// resource isolation and just fork/execs. The Exec driver should be preferred -// and this should only be used when explicitly needed. +// Driver is a driver for running LXC containers type Driver struct { // eventer is used to handle multiplexing of TaskEvents calls such that an // event can be broadcast to all callers @@ -144,7 +142,7 @@ type Driver struct { // Config is the driver configuration set by the SetConfig RPC call type Config struct { - // Enabled is set to true to enable the raw_exec driver + // Enabled is set to true to enable the lxc driver Enabled bool `codec:"enabled"` AllowVolumes bool `codec:"volumes"` @@ -422,6 +420,16 @@ func (d *Driver) WaitTask(ctx context.Context, taskID string) (<-chan *drivers.E func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *drivers.ExitResult) { defer close(ch) + // + // Wait for process completion by polling status from handler. + // We cannot use the following alternatives: + // * Process.Wait() requires LXC container processes to be children + // of self process; but LXC runs container in separate PID hierarchy + // owned by PID 1. + // * lxc.Container.Wait() holds a write lock on container and prevents + // any other calls, including stats. + // + // Going with simplest approach of polling for handler to mark exit. ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() @@ -446,10 +454,8 @@ func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) e return drivers.ErrTaskNotFound } - if err := handle.container.Shutdown(timeout); err != nil { - if err := handle.container.Stop(); err != nil { - return fmt.Errorf("executor Shutdown failed: %v", err) - } + if err := handle.shutdown(timeout); err != nil { + return fmt.Errorf("executor Shutdown failed: %v", err) } return nil @@ -466,7 +472,8 @@ func (d *Driver) DestroyTask(taskID string, force bool) error { } if handle.IsRunning() { - if err := handle.container.Shutdown(0); err != nil { + // grace period is chosen arbitrary here + if err := handle.shutdown(1 * time.Minute); err != nil { handle.logger.Error("failed to destroy executor", "err", err) } } @@ -502,9 +509,5 @@ func (d *Driver) SignalTask(taskID string, signal string) error { } func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (*drivers.ExecTaskResult, error) { - if len(cmd) == 0 { - return nil, fmt.Errorf("error cmd must have atleast one value") - } - return nil, fmt.Errorf("LXC driver does not support exec") } diff --git a/drivers/lxc/driver_test.go b/drivers/lxc/driver_test.go index 4889242d3..5122846e1 100644 --- a/drivers/lxc/driver_test.go +++ b/drivers/lxc/driver_test.go @@ -119,7 +119,6 @@ func TestLXCDriver_Start_Wait(t *testing.T) { cleanup := harness.MkAllocDir(task, false) defer cleanup() - fmt.Println(task.AllocDir) handle, _, err := harness.StartTask(task) require.NoError(err) @@ -213,7 +212,6 @@ func TestLXCDriver_Start_Stop(t *testing.T) { cleanup := harness.MkAllocDir(task, false) defer cleanup() - fmt.Println(task.AllocDir) handle, _, err := harness.StartTask(task) require.NoError(err) diff --git a/drivers/lxc/handle.go b/drivers/lxc/handle.go index 5d5799d2f..0e4490369 100644 --- a/drivers/lxc/handle.go +++ b/drivers/lxc/handle.go @@ -187,3 +187,14 @@ func keysToVal(line string) (string, uint64, error) { val, err := strconv.ParseUint(tokens[1], 10, 64) return key, val, err } + +// shutdown shuts down the container, with `timeout` grace period +// before killing the container with SIGKILL. +func (h *taskHandle) shutdown(timeout time.Duration) error { + err := h.container.Shutdown(timeout) + if err == nil { + return nil + } + + return h.container.Stop() +} diff --git a/drivers/lxc/lxc.go b/drivers/lxc/lxc.go index f53fa14a2..44127593c 100644 --- a/drivers/lxc/lxc.go +++ b/drivers/lxc/lxc.go @@ -11,7 +11,6 @@ import ( "syscall" "time" - "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/plugins/drivers" lxc "gopkg.in/lxc/go-lxc.v2" ) @@ -50,7 +49,7 @@ func (d *Driver) lxcPath() string { func (d *Driver) initializeContainer(cfg *drivers.TaskConfig, taskConfig TaskConfig) (*lxc.Container, error) { lxcPath := d.lxcPath() - containerName := fmt.Sprintf("%s-%s", cfg.Name, uuid.Generate()) + containerName := fmt.Sprintf("%s-%s", cfg.Name, cfg.AllocID) c, err := lxc.NewContainer(containerName, lxcPath) if err != nil { return nil, fmt.Errorf("failed to initialize container: %v", err) @@ -167,6 +166,5 @@ func waitTillStopped(c *lxc.Container) (bool, error) { } time.Sleep(containerMonitorIntv) - } }