Merge pull request #1518 from pubnub/feature/chroot-map-rebase

Add config field to specify chroot mapping for exec driver

client/config/config.go

@@ -99,6 +99,10 @@ type Config struct {
 	// devices and IPs.
 	GloballyReservedPorts []int
 
+	// A mapping of directories on the host OS to attempt to embed inside each
+	// task's chroot.
+	ChrootEnv map[string]string
+
 	// Options provides arbitrary key-value configuration for nomad internals,
 	// like fingerprinters and drivers. The format is:
 	//

client/driver/exec.go

@@ -121,11 +121,12 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, err
 	}
 	executorCtx := &executor.ExecutorContext{
-		TaskEnv:  d.taskEnv,
-		Driver:   "exec",
-		AllocDir: ctx.AllocDir,
-		AllocID:  ctx.AllocID,
-		Task:     task,
+		TaskEnv:   d.taskEnv,
+		Driver:    "exec",
+		AllocDir:  ctx.AllocDir,
+		AllocID:   ctx.AllocID,
+		ChrootEnv: d.config.ChrootEnv,
+		Task:      task,
 	}
 
 	ps, err := exec.LaunchCmd(&executor.ExecCommand{

client/driver/executor/executor.go

@@ -102,6 +102,10 @@ type ExecutorContext struct {
 	// AllocID is the allocation id to which the task belongs
 	AllocID string
 
+	// A mapping of directories on the host OS to attempt to embed inside each
+	// task's chroot.
+	ChrootEnv map[string]string
+
 	// Driver is the name of the driver that invoked the executor
 	Driver string
 

client/driver/executor/executor_linux.go

@@ -227,7 +227,12 @@ func (e *UniversalExecutor) configureChroot() error {
 		return err
 	}
 
-	if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil {
+	chroot := chrootEnv
+	if len(e.ctx.ChrootEnv) > 0 {
+		chroot = e.ctx.ChrootEnv
+	}
+
+	if err := allocDir.Embed(e.ctx.Task.Name, chroot); err != nil {
 		return err
 	}
 

client/driver/executor/executor_linux_test.go

@@ -9,15 +9,38 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/hashicorp/nomad/client/driver/env"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
 	"github.com/hashicorp/nomad/client/testutil"
+	"github.com/hashicorp/nomad/nomad/mock"
 )
 
+func testExecutorContextWithChroot(t *testing.T) *ExecutorContext {
+	taskEnv := env.NewTaskEnvironment(mock.Node())
+	task, allocDir := mockAllocDir(t)
+	ctx := &ExecutorContext{
+		TaskEnv:  taskEnv,
+		Task:     task,
+		AllocDir: allocDir,
+		ChrootEnv: map[string]string{
+			"/etc/ld.so.cache":  "/etc/ld.so.cache",
+			"/etc/ld.so.conf":   "/etc/ld.so.conf",
+			"/etc/ld.so.conf.d": "/etc/ld.so.conf.d",
+			"/lib":              "/lib",
+			"/lib64":            "/lib64",
+			"/usr/lib":          "/usr/lib",
+			"/bin/ls":           "/bin/ls",
+			"/foobar":           "/does/not/exist",
+		},
+	}
+	return ctx
+}
+
 func TestExecutor_IsolationAndConstraints(t *testing.T) {
 	testutil.ExecCompatible(t)
 
-	execCmd := ExecCommand{Cmd: "/bin/echo", Args: []string{"hello world"}}
-	ctx := testExecutorContext(t)
+	execCmd := ExecCommand{Cmd: "/bin/ls", Args: []string{"-F", "/", "/etc/"}}
+	ctx := testExecutorContextWithChroot(t)
 	defer ctx.AllocDir.Destroy()
 
 	execCmd.FSIsolation = true
@@ -58,7 +81,7 @@ func TestExecutor_IsolationAndConstraints(t *testing.T) {
 		t.Fatalf("file %v hasn't been removed", memLimits)
 	}
 
-	expected := "hello world"
+	expected := "/:\nalloc/\nbin/\ndev/\netc/\nlib/\nlib64/\nlocal/\nproc/\ntmp/\nusr/\n\n/etc/:\nld.so.cache\nld.so.conf\nld.so.conf.d/"
 	file := filepath.Join(ctx.AllocDir.LogDir(), "web.stdout.0")
 	output, err := ioutil.ReadFile(file)
 	if err != nil {

client/driver/java.go

@@ -203,11 +203,12 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, err
 	}
 	executorCtx := &executor.ExecutorContext{
-		TaskEnv:  d.taskEnv,
-		Driver:   "java",
-		AllocDir: ctx.AllocDir,
-		AllocID:  ctx.AllocID,
-		Task:     task,
+		TaskEnv:   d.taskEnv,
+		Driver:    "java",
+		AllocDir:  ctx.AllocDir,
+		AllocID:   ctx.AllocID,
+		ChrootEnv: d.config.ChrootEnv,
+		Task:      task,
 	}
 
 	absPath, err := GetAbsolutePath("java")

command/agent/agent.go

@@ -274,6 +274,7 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
 	if a.config.Client.NetworkInterface != "" {
 		conf.NetworkInterface = a.config.Client.NetworkInterface
 	}
+	conf.ChrootEnv = a.config.Client.ChrootEnv
 	conf.Options = a.config.Client.Options
 	// Logging deprecation messages about consul related configuration in client
 	// options

command/agent/config-test-fixtures/basic.hcl

@@ -33,6 +33,10 @@ client {
 		foo = "bar"
 		baz = "zip"
 	}
+	chroot_env {
+		"/opt/myapp/etc" = "/etc"
+		"/opt/myapp/bin" = "/bin"
+	}
 	network_interface = "eth0"
 	network_speed = 100
 	reserved {

command/agent/config.go

@@ -156,6 +156,10 @@ type ClientConfig struct {
 	// Metadata associated with the node
 	Meta map[string]string `mapstructure:"meta"`
 
+	// A mapping of directories on the host OS to attempt to embed inside each
+	// task's chroot.
+	ChrootEnv map[string]string `mapstructure:"chroot_env"`
+
 	// Interface to use for network fingerprinting
 	NetworkInterface string `mapstructure:"network_interface"`
 
@@ -720,6 +724,14 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
 		result.Meta[k] = v
 	}
 
+	// Add the chroot_env map values
+	if result.ChrootEnv == nil {
+		result.ChrootEnv = make(map[string]string)
+	}
+	for k, v := range b.ChrootEnv {
+		result.ChrootEnv[k] = v
+	}
+
 	return &result
 }
 

command/agent/config_parse.go

@@ -315,6 +315,7 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error {
 		"node_class",
 		"options",
 		"meta",
+		"chroot_env",
 		"network_interface",
 		"network_speed",
 		"max_kill_timeout",
@@ -334,6 +335,7 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error {
 
 	delete(m, "options")
 	delete(m, "meta")
+	delete(m, "chroot_env")
 	delete(m, "reserved")
 	delete(m, "stats")
 
@@ -370,6 +372,20 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error {
 		}
 	}
 
+	// Parse out chroot_env fields. These are in HCL as a list so we need to
+	// iterate over them and merge them.
+	if chrootEnvO := listVal.Filter("chroot_env"); len(chrootEnvO.Items) > 0 {
+		for _, o := range chrootEnvO.Elem().Items {
+			var m map[string]interface{}
+			if err := hcl.DecodeObject(&m, o.Val); err != nil {
+				return err
+			}
+			if err := mapstructure.WeakDecode(m, &config.ChrootEnv); err != nil {
+				return err
+			}
+		}
+	}
+
 	// Parse reserved config
 	if o := listVal.Filter("reserved"); len(o.Items) > 0 {
 		if err := parseReserved(&config.Reserved, o); err != nil {

command/agent/config_parse_test.go

@@ -53,6 +53,10 @@ func TestConfig_Parse(t *testing.T) {
 						"foo": "bar",
 						"baz": "zip",
 					},
+					ChrootEnv: map[string]string{
+						"/opt/myapp/etc": "/etc",
+						"/opt/myapp/bin": "/bin",
+					},
 					NetworkInterface: "eth0",
 					NetworkSpeed:     100,
 					MaxKillTimeout:   "10s",

command/agent/config_test.go

@@ -139,6 +139,7 @@ func TestConfig_Merge(t *testing.T) {
 				"foo": "bar",
 				"baz": "zip",
 			},
+			ChrootEnv: map[string]string{},
 			ClientMaxPort:  20000,
 			ClientMinPort:  22000,
 			NetworkSpeed:   105,

website/source/docs/agent/config.html.md

@@ -421,6 +421,9 @@ configured on server nodes.
   * <a id="options">`options`</a>: This is a key/value mapping of internal
     configuration for clients, such as for driver configuration. Please see
     [here](#options_map) for a description of available options.
+  * <a id="chroot_env">`chroot_env`</a>: This is a key/value mapping that
+    defines the chroot environment for jobs using the Exec and Java drivers.
+    Please see [here](#chroot_env_map) for an example and further information.
   * <a id="network_interface">`network_interface`</a>: This is a string to force
     network fingerprinting to use a specific network interface
   * <a id="network_speed">`network_speed`</a>: This is an int that sets the
@@ -496,6 +499,31 @@ documentation [here](/docs/drivers/index.html)
   If specified, fingerprinters not in the whitelist will be disabled. If the
   whitelist is empty, all fingerprinters are used.
 
+### <a id="chroot_env_map"></a>Client ChrootEnv Map
+
+Drivers based on [Isolated Fork/Exec](/docs/drivers/exec.html) implement file
+system isolation using chroot on Linux.  The `chroot_env` map allows the chroot
+environment to be configured using source paths on the host operating system.
+The mapping format is: `source_path -> dest_path`.
+
+The following example specifies a chroot which contains just enough to run the
+`ls` utility, and not much else:
+
+```
+chroot_env {
+    "/bin/ls" = "/bin/ls"
+    "/etc/ld.so.cache" = "/etc/ld.so.cache"
+    "/etc/ld.so.conf" = "/etc/ld.so.conf"
+    "/etc/ld.so.conf.d" = "/etc/ld.so.conf.d"
+    "/lib" = "/lib"
+    "/lib64" = "/lib64"
+}
+```
+
+When `chroot_env` is unspecified, the `exec` driver will use a default chroot
+environment with the most commonly used parts of the operating system. See
+`exec` documentation for the full list [here](/docs/drivers/exec.html#chroot).
+
 ## <a id="cli"></a>Command-line Options
 
 A subset of the available Nomad agent configuration can optionally be passed in

website/source/docs/drivers/exec.html.md

@@ -96,9 +96,12 @@ the client and the configuration.
 On Linux, Nomad will use cgroups, and a chroot to isolate the
 resources of a process and as such the Nomad agent must be run as root.
 
-### Chroot
+### <a id="chroot"></a>Chroot
 The chroot is populated with data in the following folders from the host
 machine:
 
 `["/bin", "/etc", "/lib", "/lib32", "/lib64", "/run/resolvconf", "/sbin",
 "/usr"]`
+
+This list is configurable through the agent client
+[configuration file](/docs/agent/config.html#chroot_env).