From f8a65a3b9d2cc8c9195713d87f2eb86a2cd66bfc Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Tue, 11 Oct 2016 15:49:43 -0700 Subject: [PATCH 1/5] Resolve alloc/state directories to make Docker For Mac happy * In -dev mode, `ioutil.TempDir` is used for the alloc and state directories. * `TempDir` uses `$TMPDIR`, which os OS X contains a per user directory which is under `/var/folder`. * `/var` is actually a symlink to `/private/var` * Docker For Mac validates the directories that are passed to bind and on OS X. That whitelist contains `/private`, but not `/var`. It does not expand the path, and so any paths in `$TMPDIR` fail the whitelist check. And thusly, by expanding the alloc/state directories the value passed for binding does contain `/private` and Docker For Mac is happy. --- client/client.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/client/client.go b/client/client.go index dfc19a25c..28f348c3d 100644 --- a/client/client.go +++ b/client/client.go @@ -275,6 +275,12 @@ func (c *Client) init() error { if err != nil { return fmt.Errorf("failed creating temporary directory for the StateDir: %v", err) } + + p, err = filepath.EvalSymlinks(p) + if err != nil { + return err + } + c.config.StateDir = p } c.logger.Printf("[INFO] client: using state directory %v", c.config.StateDir) @@ -290,6 +296,12 @@ func (c *Client) init() error { if err != nil { return fmt.Errorf("failed creating temporary directory for the AllocDir: %v", err) } + + p, err = filepath.EvalSymlinks(p) + if err != nil { + return err + } + c.config.AllocDir = p } From 8864a506aa7d5a538978df5cf1ec134fb97e971e Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Tue, 11 Oct 2016 15:55:46 -0700 Subject: [PATCH 2/5] Disable the syslog logging system on Docker For Mac The syslog logging system depends on the ability for a unix socket to be accessed by the docker daemon in the $TMPDIR of the host. This doesn't work on Docker For Mac because the docker daemon is running inside a VM, and while /tmp is accessible, the filesystem used to share them doesn't support unix socket files, and thus it doesn't work. --- client/driver/docker.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 036f1b3d0..72693c736 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -790,7 +790,9 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle // Only launch syslog server if we're going to use it! syslogAddr := "" - if len(driverConfig.Logging) == 0 || driverConfig.Logging[0].Type == "syslog" { + if runtime.GOOS == "darwin" && len(driverConfig.Logging) == 0 { + d.logger.Printf("[DEBUG] driver.docker: disabling syslog driver as Docker for Mac workaround") + } else if len(driverConfig.Logging) == 0 || driverConfig.Logging[0].Type == "syslog" { ss, err := exec.LaunchSyslogServer() if err != nil { pluginClient.Kill() From e7a98d5500cdd66e655f309d11363c3ad61b9772 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Tue, 11 Oct 2016 16:16:06 -0700 Subject: [PATCH 3/5] Make EvalSymlink errors more verbose --- client/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/client.go b/client/client.go index 28f348c3d..b01f099ae 100644 --- a/client/client.go +++ b/client/client.go @@ -278,7 +278,7 @@ func (c *Client) init() error { p, err = filepath.EvalSymlinks(p) if err != nil { - return err + return fmt.Errorf("failed to find temporary directory for the StateDir: %v", err) } c.config.StateDir = p @@ -299,7 +299,7 @@ func (c *Client) init() error { p, err = filepath.EvalSymlinks(p) if err != nil { - return err + return fmt.Errorf("failed to find temporary directory for the AllocDir: %v", err) } c.config.AllocDir = p From 4639f43653dfbd930b3ab6807f64447771cbd343 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Tue, 11 Oct 2016 16:50:10 -0700 Subject: [PATCH 4/5] Add caveat about Docker For Mac in the docs --- website/source/docs/drivers/docker.html.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 0b05dabbd..95442d9d3 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -454,3 +454,18 @@ Containers essentially have a virtual file system all to themselves. If you need a higher degree of isolation between processes for security or other reasons, it is recommended to use full virtualization like [QEMU](/docs/drivers/qemu.html). + +## Docker For Mac Caveats + +Docker For Mac runs docker inside a small VM and then allows access to parts of +the host filesystem into that VM. At present, nomad uses a syslog server bound to +a unix socket within a path that both the host and the VM can access to forward +log messages back to nomad. But at present, Docker For Mac does not work for +unix domain sockets (https://github.com/docker/for-mac/issues/483) in one of +these shared paths. + +As a result, using nomad with the docker driver on OS X/macOS will work, but no +logs will be available to nomad. Users must use the native docker facilities to +examine the logs of any jobs running under docker. + +In the future, we will resolve this issue, one way or another. From 3c4a27e72bfe103f65f7da90a041d58fd67cf066 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 25 Oct 2016 17:27:13 -0700 Subject: [PATCH 5/5] Fix panic --- client/driver/docker.go | 20 +++++++++++++------- client/driver/executor/executor.go | 10 ++++++++-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 72693c736..7b820765e 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -449,13 +449,15 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, memLimit := int64(task.Resources.MemoryMB) * 1024 * 1024 if len(driverConfig.Logging) == 0 { - d.logger.Printf("[DEBUG] driver.docker: Setting default logging options to syslog and %s", syslogAddr) - driverConfig.Logging = []DockerLoggingOpts{ - {Type: "syslog", Config: map[string]string{"syslog-address": syslogAddr}}, + if runtime.GOOS != "darwin" { + d.logger.Printf("[DEBUG] driver.docker: Setting default logging options to syslog and %s", syslogAddr) + driverConfig.Logging = []DockerLoggingOpts{ + {Type: "syslog", Config: map[string]string{"syslog-address": syslogAddr}}, + } } - } - d.logger.Printf("[DEBUG] driver.docker: Using config for logging: %+v", driverConfig.Logging[0]) + d.logger.Printf("[DEBUG] driver.docker: deferring logging to docker on Docker for Mac") + } hostConfig := &docker.HostConfig{ // Convert MB to bytes. This is an absolute value. @@ -468,10 +470,14 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, // local directory for storage and a shared alloc directory that can be // used to share data between different tasks in the same task group. Binds: binds, - LogConfig: docker.LogConfig{ + } + + if len(driverConfig.Logging) != 0 { + d.logger.Printf("[DEBUG] driver.docker: Using config for logging: %+v", driverConfig.Logging[0]) + hostConfig.LogConfig = docker.LogConfig{ Type: driverConfig.Logging[0].Type, Config: driverConfig.Logging[0].Config, - }, + } } d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Name) diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 5a05b1ee6..dd8bc20c4 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -452,8 +452,14 @@ func (e *UniversalExecutor) Exit() error { if e.syslogServer != nil { e.syslogServer.Shutdown() } - e.lre.Close() - e.lro.Close() + + if e.lre != nil { + e.lre.Close() + } + + if e.lro != nil { + e.lro.Close() + } if e.consulSyncer != nil { e.consulSyncer.Shutdown()