From 89c6f56059084f6614de69a80ee76f0c6a2e2960 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 14:25:10 -0800 Subject: [PATCH 01/36] Remove restrictions from docker networking mode; we assume users know what they are doing --- client/driver/docker.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index df02ff879..16bc56274 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -249,23 +249,13 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri } } - mode := driverConfig.NetworkMode - if mode == "" { + hostConfig.NetworkMode := driverConfig.NetworkMode + if hostConfig.NetworkMode == "" { // docker default - d.logger.Printf("[WARN] driver.docker: no mode specified for networking, defaulting to bridge") - mode = "bridge" + d.logger.Printf("[INFO] driver.docker: networking mode not specified; defaulting to bridge") + hostConfig.NetworkMode = "bridge" } - // Ignore the container mode for now - switch mode { - case "default", "bridge", "none", "host": - d.logger.Printf("[DEBUG] driver.docker: using %s as network mode", mode) - default: - d.logger.Printf("[ERR] driver.docker: invalid setting for network mode: %s", mode) - return c, fmt.Errorf("Invalid setting for network mode: %s", mode) - } - hostConfig.NetworkMode = mode - // Setup port mapping and exposed ports if len(task.Resources.Networks) == 0 { d.logger.Print("[WARN] driver.docker: No network resources are available for port mapping") From 7f117b3b559dd73a859c9b02048f9a31ac46ab02 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 14:27:58 -0800 Subject: [PATCH 02/36] That's not a declaration --- client/driver/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 16bc56274..957871043 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -249,7 +249,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri } } - hostConfig.NetworkMode := driverConfig.NetworkMode + hostConfig.NetworkMode = driverConfig.NetworkMode if hostConfig.NetworkMode == "" { // docker default d.logger.Printf("[INFO] driver.docker: networking mode not specified; defaulting to bridge") From f8ce6fb901a1f9629435394091f1d87995596178 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Mon, 16 Nov 2015 22:43:22 -0500 Subject: [PATCH 03/36] Added the example for service block --- command/init.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/command/init.go b/command/init.go index c911c95cf..668b33b26 100644 --- a/command/init.go +++ b/command/init.go @@ -128,6 +128,19 @@ job "example" { } } + service { + # name = redis + tags = ["global", "cache"] + port = "db" + check { + id = "id-alive-check" + name = "alive" + type = "tcp" + interval = "10s" + timeout = "2s" + } + } + # We must specify the resources required for # this task to ensure it runs on a machine with # enough capacity. From eb19967ce4f5c7dcac325b45042bcce1f3226239 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 01:37:09 -0500 Subject: [PATCH 04/36] Added the parsling logic for service blocks --- command/init.go | 5 ++-- jobspec/parse.go | 64 ++++++++++++++++++++++++++++++++++++++++ nomad/structs/structs.go | 18 +++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/command/init.go b/command/init.go index 668b33b26..6bff4990e 100644 --- a/command/init.go +++ b/command/init.go @@ -128,12 +128,11 @@ job "example" { } } - service { + service "id-redis-check" { # name = redis tags = ["global", "cache"] port = "db" - check { - id = "id-alive-check" + check "id-alive-check" { name = "alive" type = "tcp" interval = "10s" diff --git a/jobspec/parse.go b/jobspec/parse.go index 24772364f..de7d9314c 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -378,6 +378,7 @@ func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error { delete(m, "config") delete(m, "env") delete(m, "constraint") + delete(m, "service") delete(m, "meta") delete(m, "resources") @@ -401,6 +402,69 @@ func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error { } } + if o := listVal.Filter("service"); len(o.Items) > 0 { + t.Services = make([]structs.Service, len(o.Items)) + for idx, o := range o.Items { + var service structs.Service + label := o.Keys[0].Token.Value().(string) + service.Id = label + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + delete(m, "check") + + if err := mapstructure.WeakDecode(m, &service); err != nil { + return err + } + + if service.Name == "" { + service.Name = service.Id + } + + // Fileter checks + var checkList *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + checkList = ot.List + } else { + return fmt.Errorf("service '%s': should be an object", label) + } + + if co := checkList.Filter("check"); len(co.Items) > 0 { + service.Checks = make([]structs.ServiceCheck, len(co.Items)) + for idx, co := range co.Items { + var check structs.ServiceCheck + label := o.Keys[0].Token.Value().(string) + var cm map[string]interface{} + if err := hcl.DecodeObject(&cm, co.Val); err != nil { + return err + } + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &check, + }) + if err != nil { + return err + } + if err := dec.Decode(cm); err != nil { + return err + } + + check.Id = label + if check.Name == "" { + check.Name = label + } + service.Checks[idx] = check + } + } + + t.Services[idx] = service + } + } + // If we have config, then parse that if o := listVal.Filter("config"); len(o.Items) > 0 { for _, o := range o.Elem().Items { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 746b9b6c8..f021525fb 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -906,6 +906,22 @@ func NewRestartPolicy(jobType string) *RestartPolicy { return nil } +type ServiceCheck struct { + Id string + Name string + Type string + Interval time.Duration + Timeout time.Duration +} + +type Service struct { + Id string + Name string + Tags []string + PortLabel string `mapstructure:"port"` + Checks []ServiceCheck +} + // TaskGroup is an atomic unit of placement. Each task group belongs to // a job and may contain any number of tasks. A task group support running // in many replicas using the same configuration.. @@ -1009,6 +1025,8 @@ type Task struct { // Map of environment variables to be used by the driver Env map[string]string + Services []Service + // Constraints can be specified at a task level and apply only to // the particular task. Constraints []*Constraint From ead6e09a79b16df78939940abbec624d375f4db8 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 01:40:39 -0500 Subject: [PATCH 05/36] Added comments --- nomad/structs/structs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index f021525fb..ef6793770 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -906,6 +906,8 @@ func NewRestartPolicy(jobType string) *RestartPolicy { return nil } +// The ServiceCheck data model represents the consul health check that +// Nomad registers for a Task type ServiceCheck struct { Id string Name string @@ -914,6 +916,7 @@ type ServiceCheck struct { Timeout time.Duration } +// The Service model represents a Consul service defintion type Service struct { Id string Name string From 36520a90b3f1534a2fb6fc4163929f0368d15404 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 01:51:08 -0500 Subject: [PATCH 06/36] Added tests for service block parsing --- jobspec/parse.go | 2 +- jobspec/parse_test.go | 17 +++++++++++++++++ jobspec/test-fixtures/basic.hcl | 11 +++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/jobspec/parse.go b/jobspec/parse.go index de7d9314c..e1932cfaa 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -436,7 +436,7 @@ func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error { service.Checks = make([]structs.ServiceCheck, len(co.Items)) for idx, co := range co.Items { var check structs.ServiceCheck - label := o.Keys[0].Token.Value().(string) + label := co.Keys[0].Token.Value().(string) var cm map[string]interface{} if err := hcl.DecodeObject(&cm, co.Val); err != nil { return err diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 125127de5..1c993caa2 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -94,6 +94,23 @@ func TestParse(t *testing.T) { Config: map[string]interface{}{ "image": "hashicorp/binstore", }, + Services: []structs.Service{ + { + Id: "service-id", + Name: "service-name", + Tags: []string{"foo", "bar"}, + PortLabel: "http", + Checks: []structs.ServiceCheck{ + { + Id: "check-id", + Name: "check-name", + Type: "tcp", + Interval: 10 * time.Second, + Timeout: 2 * time.Second, + }, + }, + }, + }, Env: map[string]string{ "HELLO": "world", "LOREM": "ipsum", diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 236f4829a..609c8f2f5 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -45,6 +45,17 @@ job "binstore-storagelocker" { HELLO = "world" LOREM = "ipsum" } + service "service-id" { + name = "service-name" + tags = ["foo", "bar"] + port = "http" + check "check-id" { + name = "check-name" + type = "tcp" + interval = "10s" + timeout = "2s" + } + } resources { cpu = 500 memory = 128 From 0d9e34bea4a8bec61a6e7d3d4542660aef3e4642 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 02:20:35 -0500 Subject: [PATCH 07/36] Added the service definitions in api structs too --- api/tasks.go | 20 ++++++++++++++++++++ nomad/structs/structs.go | 39 ++++++++++++++++++++------------------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/api/tasks.go b/api/tasks.go index e5ae46b5c..27b712f3d 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -20,6 +20,25 @@ func NewRestartPolicy() *RestartPolicy { } } +// The ServiceCheck data model represents the consul health check that +// Nomad registers for a Task +type ServiceCheck struct { + Id string + Name string + Type string + Interval time.Duration + Timeout time.Duration +} + +// The Service model represents a Consul service defintion +type Service struct { + Id string + Name string + Tags []string + PortLabel string `mapstructure:"port"` + Checks []ServiceCheck +} + // TaskGroup is the unit of scheduling. type TaskGroup struct { Name string @@ -68,6 +87,7 @@ type Task struct { Config map[string]interface{} Constraints []*Constraint Env map[string]string + Services []Service Resources *Resources Meta map[string]string } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index ef6793770..b720b3c4e 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -906,25 +906,6 @@ func NewRestartPolicy(jobType string) *RestartPolicy { return nil } -// The ServiceCheck data model represents the consul health check that -// Nomad registers for a Task -type ServiceCheck struct { - Id string - Name string - Type string - Interval time.Duration - Timeout time.Duration -} - -// The Service model represents a Consul service defintion -type Service struct { - Id string - Name string - Tags []string - PortLabel string `mapstructure:"port"` - Checks []ServiceCheck -} - // TaskGroup is an atomic unit of placement. Each task group belongs to // a job and may contain any number of tasks. A task group support running // in many replicas using the same configuration.. @@ -1014,6 +995,25 @@ func (tg *TaskGroup) GoString() string { return fmt.Sprintf("*%#v", *tg) } +// The ServiceCheck data model represents the consul health check that +// Nomad registers for a Task +type ServiceCheck struct { + Id string + Name string + Type string + Interval time.Duration + Timeout time.Duration +} + +// The Service model represents a Consul service defintion +type Service struct { + Id string + Name string + Tags []string + PortLabel string `mapstructure:"port"` + Checks []ServiceCheck +} + // Task is a single process typically that is executed as part of a task group. type Task struct { // Name of the task @@ -1028,6 +1028,7 @@ type Task struct { // Map of environment variables to be used by the driver Env map[string]string + // List of service definitions exposed by the Task Services []Service // Constraints can be specified at a task level and apply only to From a367314127f92170a624c929eaec7e25a4a40740 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 12:16:58 -0800 Subject: [PATCH 08/36] Added the fields for http and script checks --- nomad/structs/structs.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index b720b3c4e..8fe568d0d 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1001,6 +1001,8 @@ type ServiceCheck struct { Id string Name string Type string + Script string + Http string Interval time.Duration Timeout time.Duration } From cd7a974a7f050385a530b6576d93d36ef589745d Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 12:38:34 -0800 Subject: [PATCH 09/36] Exctracted the logic of parsing services in a method --- jobspec/parse.go | 126 +++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/jobspec/parse.go b/jobspec/parse.go index e1932cfaa..915a7d94b 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -403,65 +403,8 @@ func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error { } if o := listVal.Filter("service"); len(o.Items) > 0 { - t.Services = make([]structs.Service, len(o.Items)) - for idx, o := range o.Items { - var service structs.Service - label := o.Keys[0].Token.Value().(string) - service.Id = label - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - - delete(m, "check") - - if err := mapstructure.WeakDecode(m, &service); err != nil { - return err - } - - if service.Name == "" { - service.Name = service.Id - } - - // Fileter checks - var checkList *ast.ObjectList - if ot, ok := o.Val.(*ast.ObjectType); ok { - checkList = ot.List - } else { - return fmt.Errorf("service '%s': should be an object", label) - } - - if co := checkList.Filter("check"); len(co.Items) > 0 { - service.Checks = make([]structs.ServiceCheck, len(co.Items)) - for idx, co := range co.Items { - var check structs.ServiceCheck - label := co.Keys[0].Token.Value().(string) - var cm map[string]interface{} - if err := hcl.DecodeObject(&cm, co.Val); err != nil { - return err - } - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &check, - }) - if err != nil { - return err - } - if err := dec.Decode(cm); err != nil { - return err - } - - check.Id = label - if check.Name == "" { - check.Name = label - } - service.Checks[idx] = check - } - } - - t.Services[idx] = service + if err := parseServices(&t, o); err != nil { + return err } } @@ -516,6 +459,71 @@ func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error { return nil } +func parseServices(task *structs.Task, serviceObjs *ast.ObjectList) error { + task.Services = make([]structs.Service, len(serviceObjs.Items)) + for idx, o := range serviceObjs.Items { + var service structs.Service + label := o.Keys[0].Token.Value().(string) + service.Id = label + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + delete(m, "check") + + if err := mapstructure.WeakDecode(m, &service); err != nil { + return err + } + + if service.Name == "" { + service.Name = service.Id + } + + // Fileter checks + var checkList *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + checkList = ot.List + } else { + return fmt.Errorf("service '%s': should be an object", label) + } + + if co := checkList.Filter("check"); len(co.Items) > 0 { + service.Checks = make([]structs.ServiceCheck, len(co.Items)) + for idx, co := range co.Items { + var check structs.ServiceCheck + label := co.Keys[0].Token.Value().(string) + var cm map[string]interface{} + if err := hcl.DecodeObject(&cm, co.Val); err != nil { + return err + } + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &check, + }) + if err != nil { + return err + } + if err := dec.Decode(cm); err != nil { + return err + } + + check.Id = label + if check.Name == "" { + check.Name = label + } + service.Checks[idx] = check + } + } + + task.Services[idx] = service + } + + return nil +} + func parseResources(result *structs.Resources, list *ast.ObjectList) error { list = list.Elem() if len(list.Items) == 0 { From 4c8dd666dc87969873c47d95fe7c46c89b26153e Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 13:36:59 -0800 Subject: [PATCH 10/36] Added validation to the checks --- nomad/structs/structs.go | 31 +++++++++++++++++++++++++++++++ nomad/structs/structs_test.go | 20 ++++++++++++++++---- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 8fe568d0d..6e851b3a3 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -995,6 +995,13 @@ func (tg *TaskGroup) GoString() string { return fmt.Sprintf("*%#v", *tg) } +const ( + ServiceCheckHTTP = "http" + ServiceCheckTCP = "tcp" + ServiceCheckDocker = "docker" + ServiceCheckScript = "script" +) + // The ServiceCheck data model represents the consul health check that // Nomad registers for a Task type ServiceCheck struct { @@ -1007,6 +1014,14 @@ type ServiceCheck struct { Timeout time.Duration } +func (sc *ServiceCheck) Validate() error { + t := strings.ToLower(sc.Type) + if t != ServiceCheckTCP && t != ServiceCheckHTTP && t != ServiceCheckDocker && t != ServiceCheckScript { + return fmt.Errorf("Check with name %v has invalid check type: %s ", sc.Name, sc.Type) + } + return nil +} + // The Service model represents a Consul service defintion type Service struct { Id string @@ -1016,6 +1031,16 @@ type Service struct { Checks []ServiceCheck } +func (s *Service) Validate() error { + var mErr multierror.Error + for _, c := range s.Checks { + if err := c.Validate(); err != nil { + mErr.Errors = append(mErr.Errors, err) + } + } + return mErr.ErrorOrNil() +} + // Task is a single process typically that is executed as part of a task group. type Task struct { // Name of the task @@ -1156,6 +1181,12 @@ func (t *Task) Validate() error { mErr.Errors = append(mErr.Errors, outer) } } + + for _, service := range t.Services { + if err := service.Validate(); err != nil { + mErr.Errors = append(mErr.Errors, err) + } + } return mErr.ErrorOrNil() } diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 8221c40fd..84af2a198 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -357,9 +357,21 @@ func TestEncodeDecode(t *testing.T) { } } -func TestBatchRestartPolicyValidate(t *testing.T) { - rp := RestartPolicy{Attempts: 10, Delay: 25 * time.Second} - if err := rp.Validate(); err != nil { - t.Fatalf("err: %v", err) +func TestInvalidServiceCheck(t *testing.T) { + s := Service{ + Id: "service-id", + Name: "service-name", + PortLabel: "bar", + Checks: []ServiceCheck{ + { + + Id: "check-id", + Name: "check-name", + Type: "lol", + }, + }, + } + if err := s.Validate(); err == nil { + t.Fatalf("Service should be invalid") } } From aabd27d901558509be3cd3e78a7101d299c7869e Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 13:38:41 -0800 Subject: [PATCH 11/36] Added the protocol field for check --- nomad/structs/structs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 6e851b3a3..d2a2660d1 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1010,6 +1010,7 @@ type ServiceCheck struct { Type string Script string Http string + Protocol string Interval time.Duration Timeout time.Duration } From 7eba8be4ff43c6c2e4bad1ca4d14dd2a94e389d2 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 13:42:24 -0800 Subject: [PATCH 12/36] Added some docs for the service and servicecheck type --- nomad/structs/structs.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index d2a2660d1..f7dde55b1 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1005,14 +1005,14 @@ const ( // The ServiceCheck data model represents the consul health check that // Nomad registers for a Task type ServiceCheck struct { - Id string - Name string - Type string - Script string - Http string - Protocol string - Interval time.Duration - Timeout time.Duration + Id string // If of the check + Name string // Name of the check, defaults to id + Type string // Type of the check - tcp, http, docker and script + Script string // Script to invoke for script check + Http string // path of the health check url for http type check + Protocol string // Protocol to use if check is http + Interval time.Duration // Interval of the check + Timeout time.Duration // Timeout of the response from the check before consul fails the check } func (sc *ServiceCheck) Validate() error { @@ -1025,11 +1025,11 @@ func (sc *ServiceCheck) Validate() error { // The Service model represents a Consul service defintion type Service struct { - Id string - Name string - Tags []string - PortLabel string `mapstructure:"port"` - Checks []ServiceCheck + Id string // Id of the service + Name string // Name of the service, defaults to id + Tags []string // List of tags for the service + PortLabel string `mapstructure:"port"` // port for the service + Checks []ServiceCheck // List of checks associated with the service } func (s *Service) Validate() error { From 04d2d6921daa281371bd55e6a6ea586698e52a33 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 14:21:14 -0800 Subject: [PATCH 13/36] Exctacted a method for parsing checks --- api/tasks.go | 3 +++ jobspec/parse.go | 58 +++++++++++++++++++++++----------------- nomad/structs/structs.go | 2 +- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/api/tasks.go b/api/tasks.go index 27b712f3d..2990b5433 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -26,6 +26,9 @@ type ServiceCheck struct { Id string Name string Type string + Script string + Http string + Protocol string Interval time.Duration Timeout time.Duration } diff --git a/jobspec/parse.go b/jobspec/parse.go index 915a7d94b..725006be8 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -490,31 +490,8 @@ func parseServices(task *structs.Task, serviceObjs *ast.ObjectList) error { } if co := checkList.Filter("check"); len(co.Items) > 0 { - service.Checks = make([]structs.ServiceCheck, len(co.Items)) - for idx, co := range co.Items { - var check structs.ServiceCheck - label := co.Keys[0].Token.Value().(string) - var cm map[string]interface{} - if err := hcl.DecodeObject(&cm, co.Val); err != nil { - return err - } - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &check, - }) - if err != nil { - return err - } - if err := dec.Decode(cm); err != nil { - return err - } - - check.Id = label - if check.Name == "" { - check.Name = label - } - service.Checks[idx] = check + if err := parseChecks(&service, co); err != nil { + return err } } @@ -524,6 +501,37 @@ func parseServices(task *structs.Task, serviceObjs *ast.ObjectList) error { return nil } +func parseChecks(service *structs.Service, checkObjs *ast.ObjectList) error { + service.Checks = make([]structs.ServiceCheck, len(checkObjs.Items)) + for idx, co := range checkObjs.Items { + var check structs.ServiceCheck + label := co.Keys[0].Token.Value().(string) + var cm map[string]interface{} + if err := hcl.DecodeObject(&cm, co.Val); err != nil { + return err + } + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &check, + }) + if err != nil { + return err + } + if err := dec.Decode(cm); err != nil { + return err + } + + check.Id = label + if check.Name == "" { + check.Name = label + } + service.Checks[idx] = check + } + + return nil +} + func parseResources(result *structs.Resources, list *ast.ObjectList) error { list = list.Elem() if len(list.Items) == 0 { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index f7dde55b1..c76c3fe23 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1025,7 +1025,7 @@ func (sc *ServiceCheck) Validate() error { // The Service model represents a Consul service defintion type Service struct { - Id string // Id of the service + Id string // Id of the service, this needs to be unique on a local machine Name string // Name of the service, defaults to id Tags []string // List of tags for the service PortLabel string `mapstructure:"port"` // port for the service From c73e17ae182762ddbba4b598a813c7d9b62c4872 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 14:25:23 -0800 Subject: [PATCH 14/36] Adding some sanity checks to checks --- nomad/structs/structs.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index c76c3fe23..794c91b39 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1010,13 +1010,20 @@ type ServiceCheck struct { Type string // Type of the check - tcp, http, docker and script Script string // Script to invoke for script check Http string // path of the health check url for http type check - Protocol string // Protocol to use if check is http + Protocol string // Protocol to use if check is http, defaults to http Interval time.Duration // Interval of the check Timeout time.Duration // Timeout of the response from the check before consul fails the check } func (sc *ServiceCheck) Validate() error { t := strings.ToLower(sc.Type) + if sc.Type == ServiceCheckHTTP && sc.Http == "" { + return fmt.Errorf("http checks needs the Http path information.") + } + + if sc.Type == ServiceCheckScript && sc.Script == "" { + return fmt.Errorf("Script checks need the script to invoke") + } if t != ServiceCheckTCP && t != ServiceCheckHTTP && t != ServiceCheckDocker && t != ServiceCheckScript { return fmt.Errorf("Check with name %v has invalid check type: %s ", sc.Name, sc.Type) } From 167225222603edc653b473e7eb0708875e24154c Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 16:05:03 -0800 Subject: [PATCH 15/36] Autogenerating names of service --- command/init.go | 4 ++-- jobspec/parse.go | 25 ++++++++++--------------- jobspec/parse_test.go | 6 +++--- jobspec/test-fixtures/basic.hcl | 5 ++--- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/command/init.go b/command/init.go index 6bff4990e..92554ce45 100644 --- a/command/init.go +++ b/command/init.go @@ -128,11 +128,11 @@ job "example" { } } - service "id-redis-check" { + service { # name = redis tags = ["global", "cache"] port = "db" - check "id-alive-check" { + check { name = "alive" type = "tcp" interval = "10s" diff --git a/jobspec/parse.go b/jobspec/parse.go index 725006be8..ebf7f250f 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -144,7 +144,7 @@ func parseJob(result *structs.Job, list *ast.ObjectList) error { // If we have tasks outside, create TaskGroups for them if o := listVal.Filter("task"); len(o.Items) > 0 { var tasks []*structs.Task - if err := parseTasks(&tasks, o); err != nil { + if err := parseTasks(result.Name, "", &tasks, o); err != nil { return err } @@ -247,7 +247,7 @@ func parseGroups(result *structs.Job, list *ast.ObjectList) error { // Parse tasks if o := listVal.Filter("task"); len(o.Items) > 0 { - if err := parseTasks(&g.Tasks, o); err != nil { + if err := parseTasks(result.Name, g.Name, &g.Tasks, o); err != nil { return err } } @@ -346,7 +346,7 @@ func parseConstraints(result *[]*structs.Constraint, list *ast.ObjectList) error return nil } -func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error { +func parseTasks(jobName string, taskGroupName string, result *[]*structs.Task, list *ast.ObjectList) error { list = list.Children() if len(list.Items) == 0 { return nil @@ -385,6 +385,9 @@ func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error { // Build the task var t structs.Task t.Name = n + if taskGroupName == "" { + taskGroupName = n + } if err := mapstructure.WeakDecode(m, &t); err != nil { return err } @@ -403,7 +406,7 @@ func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error { } if o := listVal.Filter("service"); len(o.Items) > 0 { - if err := parseServices(&t, o); err != nil { + if err := parseServices(jobName, taskGroupName, &t, o); err != nil { return err } } @@ -459,13 +462,10 @@ func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error { return nil } -func parseServices(task *structs.Task, serviceObjs *ast.ObjectList) error { +func parseServices(jobName string, taskGroupName string, task *structs.Task, serviceObjs *ast.ObjectList) error { task.Services = make([]structs.Service, len(serviceObjs.Items)) for idx, o := range serviceObjs.Items { var service structs.Service - label := o.Keys[0].Token.Value().(string) - service.Id = label - var m map[string]interface{} if err := hcl.DecodeObject(&m, o.Val); err != nil { return err @@ -478,7 +478,7 @@ func parseServices(task *structs.Task, serviceObjs *ast.ObjectList) error { } if service.Name == "" { - service.Name = service.Id + service.Name = fmt.Sprintf("%s-%s-%s", jobName, taskGroupName, task.Name) } // Fileter checks @@ -486,7 +486,7 @@ func parseServices(task *structs.Task, serviceObjs *ast.ObjectList) error { if ot, ok := o.Val.(*ast.ObjectType); ok { checkList = ot.List } else { - return fmt.Errorf("service '%s': should be an object", label) + return fmt.Errorf("service '%s': should be an object", service.Name) } if co := checkList.Filter("check"); len(co.Items) > 0 { @@ -505,7 +505,6 @@ func parseChecks(service *structs.Service, checkObjs *ast.ObjectList) error { service.Checks = make([]structs.ServiceCheck, len(checkObjs.Items)) for idx, co := range checkObjs.Items { var check structs.ServiceCheck - label := co.Keys[0].Token.Value().(string) var cm map[string]interface{} if err := hcl.DecodeObject(&cm, co.Val); err != nil { return err @@ -522,10 +521,6 @@ func parseChecks(service *structs.Service, checkObjs *ast.ObjectList) error { return err } - check.Id = label - if check.Name == "" { - check.Name = label - } service.Checks[idx] = check } diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 1c993caa2..398c95e0b 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -96,13 +96,13 @@ func TestParse(t *testing.T) { }, Services: []structs.Service{ { - Id: "service-id", - Name: "service-name", + Id: "", + Name: "binstore-storagelocker-binsl-binstore", Tags: []string{"foo", "bar"}, PortLabel: "http", Checks: []structs.ServiceCheck{ { - Id: "check-id", + Id: "", Name: "check-name", Type: "tcp", Interval: 10 * time.Second, diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 609c8f2f5..9696fdef8 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -45,11 +45,10 @@ job "binstore-storagelocker" { HELLO = "world" LOREM = "ipsum" } - service "service-id" { - name = "service-name" + service { tags = ["foo", "bar"] port = "http" - check "check-id" { + check { name = "check-name" type = "tcp" interval = "10s" From f98a6a384e5471f6fa12c8ac35bc6a0a2db7bc03 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 16:08:03 -0800 Subject: [PATCH 16/36] Validating that services are named explicitly if there is more than one Service defn --- jobspec/parse.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jobspec/parse.go b/jobspec/parse.go index ebf7f250f..6a9df3425 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -477,6 +477,10 @@ func parseServices(jobName string, taskGroupName string, task *structs.Task, ser return err } + if idx > 0 && service.Name == "" { + return fmt.Errorf("More than one service block is declared, please name each service explicitly") + } + if service.Name == "" { service.Name = fmt.Sprintf("%s-%s-%s", jobName, taskGroupName, task.Name) } From c02c2312d47ef9974cc11b6db2a1cf25c00d5b03 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 16:44:05 -0800 Subject: [PATCH 17/36] Adding prefix to user defined name and forcing id to be blank during parsing --- jobspec/parse.go | 8 +++++++- nomad/structs/structs.go | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jobspec/parse.go b/jobspec/parse.go index 6a9df3425..0f1ea8d0f 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -464,6 +464,7 @@ func parseTasks(jobName string, taskGroupName string, result *[]*structs.Task, l func parseServices(jobName string, taskGroupName string, task *structs.Task, serviceObjs *ast.ObjectList) error { task.Services = make([]structs.Service, len(serviceObjs.Items)) + var defaultServiceName bool for idx, o := range serviceObjs.Items { var service structs.Service var m map[string]interface{} @@ -477,14 +478,19 @@ func parseServices(jobName string, taskGroupName string, task *structs.Task, ser return err } - if idx > 0 && service.Name == "" { + if defaultServiceName && service.Name == "" { return fmt.Errorf("More than one service block is declared, please name each service explicitly") } if service.Name == "" { + defaultServiceName = true service.Name = fmt.Sprintf("%s-%s-%s", jobName, taskGroupName, task.Name) + } else { + service.Name = fmt.Sprintf("%s-%s-%s-%s", jobName, taskGroupName, task.Name, service.Name) } + service.Id = "" // Forcing this to be blank while parsing since we will autogenerate this + // Fileter checks var checkList *ast.ObjectList if ot, ok := o.Val.(*ast.ObjectType); ok { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 794c91b39..4a0f29f88 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1005,7 +1005,7 @@ const ( // The ServiceCheck data model represents the consul health check that // Nomad registers for a Task type ServiceCheck struct { - Id string // If of the check + Id string // Id of the check, must be unique and it is autogenrated Name string // Name of the check, defaults to id Type string // Type of the check - tcp, http, docker and script Script string // Script to invoke for script check From 2c3d5935b42b601f6b3846193a6520989b302413 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 17:06:29 -0800 Subject: [PATCH 18/36] Added a test which makes sure parsing fails if more than one service block has non empty names --- jobspec/parse.go | 4 +- jobspec/parse_test.go | 17 ++++ .../test-fixtures/incorrect-service-def.hcl | 77 +++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 jobspec/test-fixtures/incorrect-service-def.hcl diff --git a/jobspec/parse.go b/jobspec/parse.go index 0f1ea8d0f..92f3c5048 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -479,7 +479,7 @@ func parseServices(jobName string, taskGroupName string, task *structs.Task, ser } if defaultServiceName && service.Name == "" { - return fmt.Errorf("More than one service block is declared, please name each service explicitly") + return fmt.Errorf("Only one service block may omit the Name field") } if service.Name == "" { @@ -489,8 +489,6 @@ func parseServices(jobName string, taskGroupName string, task *structs.Task, ser service.Name = fmt.Sprintf("%s-%s-%s-%s", jobName, taskGroupName, task.Name, service.Name) } - service.Id = "" // Forcing this to be blank while parsing since we will autogenerate this - // Fileter checks var checkList *ast.ObjectList if ot, ok := o.Val.(*ast.ObjectType); ok { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 398c95e0b..90f6f4c62 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -331,3 +331,20 @@ func TestOverlappingPorts(t *testing.T) { t.Fatalf("Expected collision error; got %v", err) } } + +func TestIncompleteServiceDefn(t *testing.T) { + path, err := filepath.Abs(filepath.Join("./test-fixtures", "incorrect-service-def.hcl")) + if err != nil { + t.Fatalf("Can't get absoluate path for file: %s", err) + } + + _, err = ParseFile(path) + + if err == nil { + t.Fatalf("Expected an error") + } + + if !strings.Contains(err.Error(), "Only one service block may omit the Name field") { + t.Fatalf("Expected collision error; got %v", err) + } +} diff --git a/jobspec/test-fixtures/incorrect-service-def.hcl b/jobspec/test-fixtures/incorrect-service-def.hcl new file mode 100644 index 000000000..8a0029842 --- /dev/null +++ b/jobspec/test-fixtures/incorrect-service-def.hcl @@ -0,0 +1,77 @@ +job "binstore-storagelocker" { + region = "global" + type = "service" + priority = 50 + all_at_once = true + datacenters = ["us2", "eu1"] + + meta { + foo = "bar" + } + + constraint { + attribute = "kernel.os" + value = "windows" + } + + update { + stagger = "60s" + max_parallel = 2 + } + + task "outside" { + driver = "java" + config { + jar = "s3://my-cool-store/foo.jar" + } + meta { + my-cool-key = "foobar" + } + } + + group "binsl" { + count = 5 + restart { + attempts = 5 + interval = "10m" + delay = "15s" + } + task "binstore" { + driver = "docker" + config { + image = "hashicorp/binstore" + } + env { + HELLO = "world" + LOREM = "ipsum" + } + service { + tags = ["foo", "bar"] + port = "http" + check { + name = "check-name" + type = "http" + interval = "10s" + timeout = "2s" + } + } + service { + port = "one" + } + resources { + cpu = 500 + memory = 128 + + network { + mbits = "100" + port "one" { + static = 1 + } + port "three" { + static = 3 + } + port "http" {} + } + } + } +} From e413e882a6a4605af982d2574907b228122105e5 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Tue, 17 Nov 2015 17:12:21 -0800 Subject: [PATCH 19/36] Fixed typo --- jobspec/parse_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 90f6f4c62..6eb19af11 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -318,7 +318,7 @@ func TestBadPorts(t *testing.T) { func TestOverlappingPorts(t *testing.T) { path, err := filepath.Abs(filepath.Join("./test-fixtures", "overlapping-ports.hcl")) if err != nil { - t.Fatalf("Can't get absoluate path for file: %s", err) + t.Fatalf("Can't get absolute path for file: %s", err) } _, err = ParseFile(path) @@ -335,7 +335,7 @@ func TestOverlappingPorts(t *testing.T) { func TestIncompleteServiceDefn(t *testing.T) { path, err := filepath.Abs(filepath.Join("./test-fixtures", "incorrect-service-def.hcl")) if err != nil { - t.Fatalf("Can't get absoluate path for file: %s", err) + t.Fatalf("Can't get absolute path for file: %s", err) } _, err = ParseFile(path) From d94bd74bb523d820b8a26e77abe723b7920b0b0f Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 17:20:28 -0800 Subject: [PATCH 20/36] Updated docs for networking_mode and also reformat the file --- website/source/docs/drivers/docker.html.md | 124 +++++++++++---------- 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 623cdd6b4..191373ed5 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -16,7 +16,8 @@ and cleaning up after containers. ## Task Configuration -The `docker` driver supports the following configuration in the job specification: +The `docker` driver supports the following configuration in the job +specification: * `image` - (Required) The Docker image to run. The image may include a tag or custom URL. By default it will be fetched from Docker Hub. @@ -26,35 +27,37 @@ The `docker` driver supports the following configuration in the job specificatio * `args` - (Optional) Arguments to the optional `command`. If no `command` is present, `args` are ignored. -* `network_mode` - (Optional) The network mode to be used for the container. - Valid options are `default`, `bridge`, `host` or `none`. If nothing is - specified, the container will start in `bridge` mode. The `container` - network mode is not supported right now and is reported as an invalid - option. +* `network_mode` - (Optional) The network mode to be used for the container. In + order to support userspace networking plugins in Docker 1.9 this accepts any + value. The default is `bridge`. Other networking modes may not work without + additional configuration on the host (which is outside the scope of Nomad). + Valid values pre-docker 1.9 are `default`, `bridge`, `host`, `none`, or + `container:name`. -* `privileged` - (optional) Privileged mode gives the container full access to - the host. Valid options are `"true"` and `"false"` (defaults to `"false"`). - Tasks with `privileged` set can only run on Nomad Agents with - `docker.privileged.enabled = "true"`. +* `privileged` - (Optional) Privileged mode gives the container full access to + the host. Valid options are `"true"` and `"false"` (defaults to `"false"`). + Tasks with `privileged` set can only run on Nomad Agents with + `docker.privileged.enabled = "true"`. -* `dns-servers` - (optional) A comma separated list of DNS servers for the container - to use (e.g. "8.8.8.8,8.8.4.4"). *Docker API v1.10 and above only* +* `dns-servers` - (Optional) A comma separated list of DNS servers for the + container to use (e.g. "8.8.8.8,8.8.4.4"). *Docker API v1.10 and above only* -* `search-domains` - (optional) A comma separated list of DNS search domains for the - container to use. +* `search-domains` - (Optional) A comma separated list of DNS search domains + for the container to use. -* `hostname` - (optional) The hostname to assign to the container. When launching more - than one of a task (using `count`) with this option set, every container the task - starts will have the same hostname. +* `hostname` - (Optional) The hostname to assign to the container. When + launching more than one of a task (using `count`) with this option set, every + container the task starts will have the same hostname. -**Authentication** -Registry authentication can be set per task with the following authentication -parameters. These options can provide access to private repositories that -utilize the docker remote api (e.g. dockerhub, quay.io) - - `auth.username` - (optional) The account username - - `auth.password` - (optional) The account password - - `auth.email` - (optional) The account email - - `auth.server-address` - (optional) The server domain/ip without the protocol +**Authentication** Registry authentication can be set per task with the +following authentication parameters. These options can provide access to +private repositories that utilize the docker remote api (e.g. dockerhub, +quay.io) + - `auth.username` - (Optional) The account username + - `auth.password` - (Optional) The account password + - `auth.email` - (Optional) The account email + - `auth.server-address` - (Optional) The server domain/ip without the + protocol ### Port Mapping @@ -67,16 +70,16 @@ Nomad provides automatic and manual mapping schemes for Docker. You can use either or both schemes for a task. Nomad binds both tcp and udp protocols to ports used for Docker containers. This is not configurable. -Note: You are not required to map any ports, for example if your task is running -a crawler or aggregator and does not provide a network service. Tasks without a -port mapping will still be able to make outbound network connections. +Note: You are not required to map any ports, for example if your task is +running a crawler or aggregator and does not provide a network service. Tasks +without a port mapping will still be able to make outbound network connections. #### Automatic Port Mapping Typically when you create a Docker container you configure the service to start listening on a port (or ports) when you start the container. For example, redis -starts listening on `6379` when you `docker run redis`. Nomad can support this by -mapping a random port on the host machine to the port inside the container. +starts listening on `6379` when you `docker run redis`. Nomad can support this +by mapping a random port on the host machine to the port inside the container. You need to tell Nomad which ports your container is using so Nomad can map allocated ports for you. You do so by specifying a **numeric port value** for @@ -91,17 +94,17 @@ dynamic_ports = [6379] This instructs Nomad to create a port mapping from the random port on the host to the port inside the container. So in our example above, when you contact the host on `1.2.3.4:22333` you will actually hit the service running inside the -container on port `6379`. You can see which port was actually bound by reading the -`NOMAD_PORT_6379` [environment variable](/docs/jobspec/environment.html). +container on port `6379`. You can see which port was actually bound by reading +the `NOMAD_PORT_6379` [environment variable](/docs/jobspec/environment.html). In most cases, the automatic port mapping will be the easiest to use, but you can also use manual port mapping (described below). #### Manual Port Mapping -The `dynamic_ports` option takes any alphanumeric string as a label, so you could -also specify a label for the port like `http` or `admin` to designate how the -port will be used. +The `dynamic_ports` option takes any alphanumeric string as a label, so you +could also specify a label for the port like `http` or `admin` to designate how +the port will be used. In this case, Nomad doesn't know which container port to map to, so it maps 1:1 with the host port. For example, `1.2.3.4:22333` will map to `22333` inside the @@ -116,28 +119,29 @@ determine which port to bind to. ## Client Requirements -Nomad requires Docker to be installed and running on the host alongside the Nomad -agent. Nomad was developed against Docker `1.8.2`. +Nomad requires Docker to be installed and running on the host alongside the +Nomad agent. Nomad was developed against Docker `1.8.2`. -By default Nomad communicates with the Docker daemon using the daemon's -unix socket. Nomad will need to be able to read/write to this socket. If you do -not run Nomad as root, make sure you add the Nomad user to the Docker group so +By default Nomad communicates with the Docker daemon using the daemon's unix +socket. Nomad will need to be able to read/write to this socket. If you do not +run Nomad as root, make sure you add the Nomad user to the Docker group so Nomad can communicate with the Docker daemon. -For example, on ubuntu you can use the `usermod` command to add the `vagrant` user to the -`docker` group so you can run Nomad without root: +For example, on ubuntu you can use the `usermod` command to add the `vagrant` +user to the `docker` group so you can run Nomad without root: sudo usermod -G docker -a vagrant -For the best performance and security features you should use recent versions of -the Linux Kernel and Docker daemon. +For the best performance and security features you should use recent versions +of the Linux Kernel and Docker daemon. ## Client Configuration The `docker` driver has the following configuration options: * `docker.endpoint` - Defaults to `unix:///var/run/docker.sock`. You will need - to customize this if you use a non-standard socket (http or another location). + to customize this if you use a non-standard socket (http or another + location). * `docker.cleanup.container` Defaults to `true`. Changing this to `false` will prevent Nomad from removing containers from stopped tasks. @@ -146,9 +150,8 @@ The `docker` driver has the following configuration options: prevent Nomad from removing images from stopped tasks. * `docker.privileged.enabled` Defaults to `false`. Changing this to `true` will - allow containers to use "privileged" mode, which gives the containers full access - to the host. - + allow containers to use "privileged" mode, which gives the containers full + access to the host. Note: When testing or using the `-dev` flag you can use `DOCKER_HOST`, `DOCKER_TLS_VERIFY`, and `DOCKER_CERT_PATH` to customize Nomad's behavior. In @@ -158,21 +161,20 @@ production Nomad will always read `docker.endpoint`. The `docker` driver will set the following client attributes: -* `driver.docker` - This will be set to "1", indicating the - driver is available. -* `driver.docker.version` - This will be set to version of the - docker server +* `driver.docker` - This will be set to "1", indicating the driver is + available. +* `driver.docker.version` - This will be set to version of the docker server ## Resource Isolation ### CPU -Nomad limits containers' CPU based on CPU shares. CPU shares allow containers to -burst past their CPU limits. CPU limits will only be imposed when there is +Nomad limits containers' CPU based on CPU shares. CPU shares allow containers +to burst past their CPU limits. CPU limits will only be imposed when there is contention for resources. When the host is under load your process may be throttled to stabilize QOS depending on how many shares it has. You can see how -many CPU shares are available to your process by reading `NOMAD_CPU_LIMIT`. 1000 -shares are approximately equal to 1Ghz. +many CPU shares are available to your process by reading `NOMAD_CPU_LIMIT`. +1000 shares are approximately equal to 1Ghz. Please keep the implications of CPU shares in mind when you load test workloads on Nomad. @@ -181,7 +183,8 @@ on Nomad. Nomad limits containers' memory usage based on total virtual memory. This means that containers scheduled by Nomad cannot use swap. This is to ensure that a -swappy process does not degrade performance for other workloads on the same host. +swappy process does not degrade performance for other workloads on the same +host. Since memory is not an elastic resource, you will need to make sure your container does not exceed the amount of memory allocated to it, or it will be @@ -198,6 +201,7 @@ filesystem IO. These will be added in a later release. Docker provides resource isolation by way of [cgroups and namespaces](https://docs.docker.com/introduction/understanding-docker/#the-underlying-technology). -Containers essentially have a virtual file system all to themselves. If you need -a higher degree of isolation between processes for security or other reasons, it -is recommended to use full virtualization like [QEMU](/docs/drivers/qemu.html). +Containers essentially have a virtual file system all to themselves. If you +need a higher degree of isolation between processes for security or other +reasons, it is recommended to use full virtualization like +[QEMU](/docs/drivers/qemu.html). From e330107112240878d0746ea111f1c2320ac2e731 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 17:23:16 -0800 Subject: [PATCH 21/36] Reformat post-merge --- website/source/docs/drivers/docker.html.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index b8e81aa57..8a7d71693 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -49,16 +49,16 @@ specification: launching more than one of a task (using `count`) with this option set, every container the task starts will have the same hostname. -* `labels` - (Optional) A key/value map of labels to set to the containers on start. - +* `labels` - (Optional) A key/value map of labels to set to the containers on + start. **Authentication** Registry authentication can be set per task with the following authentication parameters. These options can provide access to private repositories that utilize the docker remote api (e.g. dockerhub, quay.io) - - `auth.username` - (optional) The account username - - `auth.password` - (optional) The account password - - `auth.email` - (optional) The account email + - `auth.username` - (Optional) The account username + - `auth.password` - (Optional) The account password + - `auth.email` - (Optional) The account email - `auth.server-address` - (Optional) The server domain/ip without the protocol From cf0630b41eb2442c0c026e7a8d7b45e36340e6ee Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 17 Nov 2015 17:41:15 -0800 Subject: [PATCH 22/36] Update jobspec to have restart policy and fix indents on distinctHost --- website/source/docs/jobspec/index.html.md | 57 +++++++++++++++++++---- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/website/source/docs/jobspec/index.html.md b/website/source/docs/jobspec/index.html.md index 3d05c909b..8bda7cb10 100644 --- a/website/source/docs/jobspec/index.html.md +++ b/website/source/docs/jobspec/index.html.md @@ -153,6 +153,10 @@ The `group` object supports the following keys: * `constraint` - This can be provided multiple times to define additional constraints. See the constraint reference for more details. +* `restart` - Specifies the restart policy to be applied to tasks in this group. + If omitted, a default policy for batch and non-batch jobs is used based on the + job type. See the restart policy reference for more details. + * `task` - This can be specified multiple times, to add a task as part of the group. @@ -209,6 +213,43 @@ The `network` object supports the following keys: For applications that cannot use a dynamic port, they can request a specific port. +### Restart Policy + +The `restart` object supports the following keys: + +* `attempts` - For `batch` jobs, `attempts` is the maximum number of restarts + allowed before the task is failed. For non-batch jobs, the `attempts` is the + number of restarts allowed in an `interval` before a restart delay is added. + +* `interval` - `interval` is only valid on non-batch jobs and is a time duration + that can be specified using the "s", "m", and "h" suffixes, such as "30s". + The `interval` begins when the first task starts and ensures that only + `attempts` number of restarts happens within it. If more than `attempts` + number of failures happen, the restart is delayed till after the `interval`, + which is then reset. + +* `delay` - A duration to wait before restarting a task. It is specified as a + time duration using the "s", "m", and "h" suffixes, such as "30s". + +The default `batch` restart policy is: + +``` +restart { + attempts = 15 + delay = "15s" +} +``` + +The default non-batch restart policy is: + +``` +restart { + interval = "1m" + attempts = 2 + delay = "15s" +} +``` + ### Constraint The `constraint` object supports the following keys: @@ -234,17 +275,17 @@ The `constraint` object supports the following keys: the attribute. This sets the operator to "regexp" and the `value` to the regular expression. -* `distinct_hosts` - `distinct_hosts` accepts a boolean `true`. The default is - `false`. +* `distinct_hosts` - `distinct_hosts` accepts a boolean `true`. The default is + `false`. - When `distinct_hosts` is `true` at the Job level, each instance of all Task - Groups specified in the job is placed on a separate host. + When `distinct_hosts` is `true` at the Job level, each instance of all Task + Groups specified in the job is placed on a separate host. - When `distinct_hosts` is `true` at the Task Group level with count > 1, each - instance of a Task Group is placed on a separate host. Different task groups in - the same job _may_ be co-scheduled. + When `distinct_hosts` is `true` at the Task Group level with count > 1, each + instance of a Task Group is placed on a separate host. Different task groups in + the same job _may_ be co-scheduled. - Tasks within a task group are always co-scheduled. + Tasks within a task group are always co-scheduled. Below is a table documenting the variables that can be interpreted: From 281aea9e3f1ba77617b8b57aacba469fc2989d92 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 17:48:37 -0800 Subject: [PATCH 23/36] Added network_mode changes to the changelog --- CHANGELOG.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0963b992d..6597cf4eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,17 +40,13 @@ IMPROVEMENTS: * client: Task State is tracked by client [GH-416] * client: Test Skip Detection [GH-221] * driver/docker: Advanced docker driver options [GH-390] - * driver/docker: Docker hostname can be set [GH-426] -<<<<<<< Updated upstream - * driver/docker: Mount task local and alloc directory to docker containers - [GH-290] - * driver/docker: Pass JVM options in java driver [GH-293, GH-297] - * drivers: Use BlkioWeight rather than BlkioThrottleReadIopsDevice [GH-222] - * jobspec and drivers: Driver configuration supports arbitrary struct to be - passed in jobspec [GH-415] -======= * driver/docker: Docker container name can be set [GH-389] ->>>>>>> Stashed changes + * driver/docker: Docker hostname can be set [GH-426] + * driver/docker: Mount task local and alloc directory to docker containers [GH-290] + * driver/docker: Now accepts any value for `network_mode` to support userspace networking plugins in docker 1.9 + * driver/java: Pass JVM options in java driver [GH-293, GH-297] + * drivers: Use BlkioWeight rather than BlkioThrottleReadIopsDevice [GH-222] + * jobspec and drivers: Driver configuration supports arbitrary struct to be passed in jobspec [GH-415] BUG FIXES: From 45bb0d349c60216e83f8387d77db8aed82e901b8 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 17 Nov 2015 19:12:19 -0800 Subject: [PATCH 24/36] Update API docs --- client/driver/docker.go | 10 +- website/source/docs/http/alloc.html.md | 297 ++++++++++++---------- website/source/docs/http/allocs.html.md | 42 +++- website/source/docs/http/job.html.md | 2 +- website/source/docs/http/node.html.md | 313 ++++++++++++++---------- 5 files changed, 395 insertions(+), 269 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index f4bdc5f15..3ee70713f 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -269,8 +269,14 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri containerToHostPortMap := make(map[string]int) for _, port := range network.DynamicPorts { - containerPort, ok := driverConfig.PortMap[0][port.Label] - if !ok { + var containerPort int + if len(driverConfig.PortMap) > 0 { + var ok bool + containerPort, ok = driverConfig.PortMap[0][port.Label] + if !ok { + containerPort = port.Value + } + } else { containerPort = port.Value } diff --git a/website/source/docs/http/alloc.html.md b/website/source/docs/http/alloc.html.md index 822858a8c..00776b606 100644 --- a/website/source/docs/http/alloc.html.md +++ b/website/source/docs/http/alloc.html.md @@ -41,144 +41,193 @@ be specified using the `?region=` query parameter. ```javascript { - "ID": "3575ba9d-7a12-0c96-7b28-add168c67984", - "EvalID": "151accaa-1ac6-90fe-d427-313e70ccbb88", - "Name": "binstore-storagelocker.binsl[3]", - "NodeID": "", - "JobID": "binstore-storagelocker", - "Job": { + "ID": "203266e5-e0d6-9486-5e05-397ed2b184af", + "EvalID": "e68125ed-3fba-fb46-46cc-291addbc4455", + "Name": "example.cache[0]", + "NodeID": "e02b6169-83bd-9df6-69bd-832765f333eb", + "JobID": "example", + "ModifyIndex": 9, + "Resources": { + "Networks": [ + { + "DynamicPorts": [ + { + "Value": 20802, + "Label": "db" + } + ], + "ReservedPorts": null, + "MBits": 10, + "IP": "", + "CIDR": "", + "Device": "" + } + ], + "IOPS": 0, + "DiskMB": 0, + "MemoryMB": 256, + "CPU": 500 + }, + "TaskGroup": "cache", + "Job": { + "ModifyIndex": 5, + "CreateIndex": 5, + "StatusDescription": "", + "Status": "", + "Meta": null, + "Update": { + "MaxParallel": 1, + "Stagger": 1e+10 + }, + "TaskGroups": [ + { + "Meta": null, + "Tasks": [ + { + "Meta": null, + "Resources": { + "Networks": [ + { + "DynamicPorts": [ + { + "Value": 20802, + "Label": "db" + } + ], + "ReservedPorts": null, + "MBits": 0, + "IP": "127.0.0.1", + "CIDR": "", + "Device": "lo" + } + ], + "IOPS": 0, + "DiskMB": 0, + "MemoryMB": 256, + "CPU": 500 + }, + "Constraints": null, + "Services": [ + { + "Checks": [ + { + "Timeout": 2e+09, + "Interval": 1e+10, + "Protocol": "", + "Http": "", + "Script": "", + "Type": "tcp", + "Name": "alive", + "Id": "" + } + ], + "PortLabel": "db", + "Tags": [ + "global", + "cache" + ], + "Name": "example-cache-redis", + "Id": "" + } + ], + "Env": null, + "Config": { + "port_map": [ + { + "db": 6379 + } + ], + "image": "redis:latest" + }, + "Driver": "docker", + "Name": "redis" + } + ], + "RestartPolicy": { + "Delay": 2.5e+10, + "Interval": 3e+11, + "Attempts": 10 + }, + "Constraints": null, + "Count": 1, + "Name": "cache" + } + ], "Region": "global", - "ID": "binstore-storagelocker", - "Name": "binstore-storagelocker", + "ID": "example", + "Name": "example", "Type": "service", "Priority": 50, "AllAtOnce": false, "Datacenters": [ - "us2", - "eu1" + "dc1" ], "Constraints": [ - { - "LTarget": "kernel.os", - "RTarget": "windows", - "Operand": "=", - } - ], - "TaskGroups": [ - { - "Name": "binsl", - "Count": 5, - "Constraints": [ - { - "LTarget": "kernel.os", - "RTarget": "linux", - "Operand": "=", - } - ], - "Tasks": [ - { - "Name": "binstore", - "Driver": "docker", - "Config": { - "image": "hashicorp/binstore" - }, - "Constraints": null, - "Resources": { - "CPU": 500, - "MemoryMB": 0, - "DiskMB": 0, - "IOPS": 0, - "Networks": [ - { - "Device": "", - "CIDR": "", - "IP": "", - "MBits": 100, - "ReservedPorts": null, - "DynamicPorts": 0 - } - ] - }, - "Meta": null - }, - { - "Name": "storagelocker", - "Driver": "java", - "Config": { - "image": "hashicorp/storagelocker" - }, - "Constraints": [ - { - "LTarget": "kernel.arch", - "RTarget": "amd64", - "Operand": "=", - } - ], - "Resources": { - "CPU": 500, - "MemoryMB": 0, - "DiskMB": 0, - "IOPS": 0, - "Networks": null - }, - "Meta": null - } - ], - "Meta": { - "elb_checks": "3", - "elb_interval": "10", - "elb_mode": "tcp" - } - } - ], - "Update": { - "Stagger": 0, - "MaxParallel": 0 - }, - "Meta": { - "foo": "bar" - }, - "Status": "", - "StatusDescription": "", - "CreateIndex": 14, - "ModifyIndex": 14 - }, - "TaskGroup": "binsl", - "Resources": { - "CPU": 1000, - "MemoryMB": 0, - "DiskMB": 0, - "IOPS": 0, - "Networks": [ - { - "Device": "", - "CIDR": "", - "IP": "", - "MBits": 100, - "ReservedPorts": null, - "DynamicPorts": 0 - } + { + "Operand": "=", + "RTarget": "linux", + "LTarget": "$attr.kernel.name" + } ] - }, - "TaskResources": null, - "Metrics": { - "NodesEvaluated": 0, + }, + "TaskResources": { + "redis": { + "Networks": [ + { + "DynamicPorts": [ + { + "Value": 20802, + "Label": "db" + } + ], + "ReservedPorts": null, + "MBits": 0, + "IP": "127.0.0.1", + "CIDR": "", + "Device": "lo" + } + ], + "IOPS": 0, + "DiskMB": 0, + "MemoryMB": 256, + "CPU": 500 + } + }, + "Metrics": { + "CoalescedFailures": 0, + "AllocationTime": 1590406, + "NodesEvaluated": 1, "NodesFiltered": 0, "ClassFiltered": null, "ConstraintFiltered": null, "NodesExhausted": 0, "ClassExhausted": null, "DimensionExhausted": null, - "Scores": null, - "AllocationTime": 9408, - "CoalescedFailures": 4 - }, - "DesiredStatus": "failed", - "DesiredDescription": "failed to find a node for placement", - "ClientStatus": "failed", - "ClientDescription": "", - "CreateIndex": 16, - "ModifyIndex": 16 + "Scores": { + "e02b6169-83bd-9df6-69bd-832765f333eb.binpack": 6.133651487695705 + } + }, + "DesiredStatus": "run", + "DesiredDescription": "", + "ClientStatus": "running", + "ClientDescription": "", + "TaskStates": { + "redis": { + "Events": [ + { + "KillError": "", + "Message": "", + "Signal": 0, + "ExitCode": 0, + "DriverError": "", + "Time": 1447806038427841000, + "Type": "Started" + } + ], + "State": "running" + } + }, + "CreateIndex": 7 } ``` diff --git a/website/source/docs/http/allocs.html.md b/website/source/docs/http/allocs.html.md index b59a4f204..7cb38ab66 100644 --- a/website/source/docs/http/allocs.html.md +++ b/website/source/docs/http/allocs.html.md @@ -42,19 +42,35 @@ be specified using the `?region=` query parameter. ```javascript [ { - "ID": "3575ba9d-7a12-0c96-7b28-add168c67984", - "EvalID": "151accaa-1ac6-90fe-d427-313e70ccbb88", - "Name": "binstore-storagelocker.binsl[3]", - "NodeID": "c9972143-861d-46e6-df73-1d8287bc3e66", - "JobID": "binstore-storagelocker", - "TaskGroup": "binsl", - "DesiredStatus": "run", - "DesiredDescription": "", - "ClientStatus": "running", - "ClientDescription": "", - "CreateIndex": 16, - "ModifyIndex": 16 - }, + "ID": "203266e5-e0d6-9486-5e05-397ed2b184af", + "EvalID": "e68125ed-3fba-fb46-46cc-291addbc4455", + "Name": "example.cache[0]", + "NodeID": "e02b6169-83bd-9df6-69bd-832765f333eb", + "JobID": "example", + "TaskGroup": "cache", + "DesiredStatus": "run", + "DesiredDescription": "" + "ClientDescription": "", + "ClientStatus": "running", + "TaskStates": { + "redis": { + "Events": [ + { + "KillError": "", + "Message": "", + "Signal": 0, + "ExitCode": 0, + "DriverError": "", + "Time": 1447806038427841000, + "Type": "Started" + } + ], + "State": "running" + } + }, + "CreateIndex": 7, + "ModifyIndex": 9, + } ... ] ``` diff --git a/website/source/docs/http/job.html.md b/website/source/docs/http/job.html.md index cbf0f5097..05484aec9 100644 --- a/website/source/docs/http/job.html.md +++ b/website/source/docs/http/job.html.md @@ -88,7 +88,7 @@ region is used; another region can be specified using the `?region=` query param "IP": "", "MBits": 100, "ReservedPorts": null, - "DynamicPorts": 0 + "DynamicPorts": null } ] }, diff --git a/website/source/docs/http/node.html.md b/website/source/docs/http/node.html.md index ed69d96e9..794e0283a 100644 --- a/website/source/docs/http/node.html.md +++ b/website/source/docs/http/node.html.md @@ -59,6 +59,7 @@ be specified using the `?region=` query parameter. "kernel.name": "darwin", "kernel.version": "14.4.0", "memory.totalbytes": "8589934592", + "network.ip-address": "127.0.0.1", "os.name": "darwin", "os.version": "14.4.0", "storage.bytesfree": "35888713728", @@ -114,141 +115,195 @@ be specified using the `?region=` query parameter. ```javascript [ - { - "ID": "8a0c24d9-cdfc-ce67-1208-8d4524b1a9b3", - "EvalID": "2c699410-8697-6109-86b7-430909b00bb9", - "Name": "example.cache[0]", - "NodeID": "12d3409b-9d27-fcad-a03d-b3c18887d153", - "JobID": "example", - "Job": { - "Region": "global", - "ID": "example", - "Name": "example", - "Type": "service", - "Priority": 50, - "AllAtOnce": false, - "Datacenters": [ - "lon1" - ], - "Constraints": [ - { - "Hard": true, - "LTarget": "$attr.kernel.name", - "RTarget": "linux", - "Operand": "=", - "Weight": 0 - } - ], - "TaskGroups": [ - { - "Name": "cache", - "Count": 1, - "Constraints": null, - "Tasks": [ - { - "Name": "redis", - "Driver": "docker", - "Config": { - "image": "redis:latest" - }, - "Env": null, - "Constraints": null, - "Resources": { - "CPU": 500, - "MemoryMB": 256, - "DiskMB": 0, - "IOPS": 0, - "Networks": [ - { - "Device": "", - "CIDR": "", - "IP": "", - "MBits": 10, - "ReservedPorts": null, - "DynamicPorts": [ - "6379" - ] - } - ] - }, - "Meta": null - } - ], - "Meta": null - } - ], - "Update": { - "Stagger": 0, - "MaxParallel": 0 - }, - "Meta": null, - "Status": "", - "StatusDescription": "", - "CreateIndex": 6, - "ModifyIndex": 6 + { + "ID": "203266e5-e0d6-9486-5e05-397ed2b184af", + "EvalID": "e68125ed-3fba-fb46-46cc-291addbc4455", + "Name": "example.cache[0]", + "NodeID": "e02b6169-83bd-9df6-69bd-832765f333eb", + "JobID": "example", + "ModifyIndex": 9, + "Resources": { + "Networks": [ + { + "DynamicPorts": [ + { + "Value": 20802, + "Label": "db" + } + ], + "ReservedPorts": null, + "MBits": 10, + "IP": "", + "CIDR": "", + "Device": "" + } + ], + "IOPS": 0, + "DiskMB": 0, + "MemoryMB": 256, + "CPU": 500 + }, + "TaskGroup": "cache", + "Job": { + "ModifyIndex": 5, + "CreateIndex": 5, + "StatusDescription": "", + "Status": "", + "Meta": null, + "Update": { + "MaxParallel": 1, + "Stagger": 1e+10 }, - "TaskGroup": "cache", - "Resources": { - "CPU": 500, - "MemoryMB": 256, - "DiskMB": 0, - "IOPS": 0, + "TaskGroups": [ + { + "Meta": null, + "Tasks": [ + { + "Meta": null, + "Resources": { + "Networks": [ + { + "DynamicPorts": [ + { + "Value": 20802, + "Label": "db" + } + ], + "ReservedPorts": null, + "MBits": 0, + "IP": "127.0.0.1", + "CIDR": "", + "Device": "lo" + } + ], + "IOPS": 0, + "DiskMB": 0, + "MemoryMB": 256, + "CPU": 500 + }, + "Constraints": null, + "Services": [ + { + "Checks": [ + { + "Timeout": 2e+09, + "Interval": 1e+10, + "Protocol": "", + "Http": "", + "Script": "", + "Type": "tcp", + "Name": "alive", + "Id": "" + } + ], + "PortLabel": "db", + "Tags": [ + "global", + "cache" + ], + "Name": "example-cache-redis", + "Id": "" + } + ], + "Env": null, + "Config": { + "port_map": [ + { + "db": 6379 + } + ], + "image": "redis:latest" + }, + "Driver": "docker", + "Name": "redis" + } + ], + "RestartPolicy": { + "Delay": 2.5e+10, + "Interval": 3e+11, + "Attempts": 10 + }, + "Constraints": null, + "Count": 1, + "Name": "cache" + } + ], + "Region": "global", + "ID": "example", + "Name": "example", + "Type": "service", + "Priority": 50, + "AllAtOnce": false, + "Datacenters": [ + "dc1" + ], + "Constraints": [ + { + "Operand": "=", + "RTarget": "linux", + "LTarget": "$attr.kernel.name" + } + ] + }, + "TaskResources": { + "redis": { "Networks": [ { - "Device": "", - "CIDR": "", - "IP": "", - "MBits": 10, - "ReservedPorts": null, "DynamicPorts": [ - "6379" - ] + { + "Value": 20802, + "Label": "db" + } + ], + "ReservedPorts": null, + "MBits": 0, + "IP": "127.0.0.1", + "CIDR": "", + "Device": "lo" } - ] - }, - "TaskResources": { - "redis": { - "CPU": 500, - "MemoryMB": 256, - "DiskMB": 0, - "IOPS": 0, - "Networks": [ - { - "Device": "eth0", - "CIDR": "", - "IP": "10.16.0.222", - "MBits": 0, - "ReservedPorts": [ - 23889 - ], - "DynamicPorts": [ - "6379" - ] - } - ] - } - }, - "Metrics": { - "NodesEvaluated": 1, - "NodesFiltered": 0, - "ClassFiltered": null, - "ConstraintFiltered": null, - "NodesExhausted": 0, - "ClassExhausted": null, - "DimensionExhausted": null, - "Scores": { - "12d3409b-9d27-fcad-a03d-b3c18887d153.binpack": 10.779215064231561 - }, - "AllocationTime": 75232, - "CoalescedFailures": 0 - }, - "DesiredStatus": "run", - "DesiredDescription": "", - "ClientStatus": "pending", - "ClientDescription": "", - "CreateIndex": 8, - "ModifyIndex": 8 + ], + "IOPS": 0, + "DiskMB": 0, + "MemoryMB": 256, + "CPU": 500 + } }, + "Metrics": { + "CoalescedFailures": 0, + "AllocationTime": 1590406, + "NodesEvaluated": 1, + "NodesFiltered": 0, + "ClassFiltered": null, + "ConstraintFiltered": null, + "NodesExhausted": 0, + "ClassExhausted": null, + "DimensionExhausted": null, + "Scores": { + "e02b6169-83bd-9df6-69bd-832765f333eb.binpack": 6.133651487695705 + } + }, + "DesiredStatus": "run", + "DesiredDescription": "", + "ClientStatus": "running", + "ClientDescription": "", + "TaskStates": { + "redis": { + "Events": [ + { + "KillError": "", + "Message": "", + "Signal": 0, + "ExitCode": 0, + "DriverError": "", + "Time": 1447806038427841000, + "Type": "Started" + } + ], + "State": "running" + } + }, + "CreateIndex": 7 + }, ... ] ``` From 9a7adb9eb3fdb63b90a87eb39b6d17365cdbf4ba Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 19:21:36 -0800 Subject: [PATCH 25/36] Fix guards for docker port mapping and change dummy dynamic ports to real ports (0 is not a valid port) --- client/driver/docker.go | 24 +++++++++++++++--------- client/driver/docker_test.go | 8 +++++++- client/driver/driver_test.go | 2 +- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index f4bdc5f15..0c9828e43 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -245,7 +245,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri // Setup port mapping and exposed ports if len(task.Resources.Networks) == 0 { d.logger.Println("[DEBUG] driver.docker: No network interfaces are available") - if len(driverConfig.PortMap[0]) > 0 { + if len(driverConfig.PortMap) == 1 && len(driverConfig.PortMap[0]) > 0 { return c, fmt.Errorf("Trying to map ports but no network interface is available") } } else { @@ -269,9 +269,15 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri containerToHostPortMap := make(map[string]int) for _, port := range network.DynamicPorts { - containerPort, ok := driverConfig.PortMap[0][port.Label] - if !ok { - containerPort = port.Value + // By default we will map the allocated port 1:1 to the container + containerPort := port.Value + + // If the user has mapped a port using port_map we'll change it here + if len(driverConfig.PortMap) == 1 { + mapped, ok := driverConfig.PortMap[0][port.Label] + if ok { + containerPort = mapped + } } containerPortStr := docker.Port(strconv.Itoa(containerPort)) @@ -318,7 +324,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri config.Env = env.List() return docker.CreateContainerOptions{ - Name: fmt.Sprintf("%s-%s", task.Name, ctx.AllocID), + // Name: fmt.Sprintf("%s-%s", task.Name, ctx.AllocID), Config: config, HostConfig: hostConfig, }, nil @@ -392,7 +398,7 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle // Now that we have the image we can get the image id dockerImage, err = client.InspectImage(image) if err != nil { - d.logger.Printf("[ERR] driver.docker: failed getting image id for %s\n", image) + d.logger.Printf("[ERR] driver.docker: failed getting image id for %s: %s\n", image, err) return nil, fmt.Errorf("Failed to determine image id for `%s`: %s", image, err) } } @@ -407,15 +413,15 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle container, err := client.CreateContainer(config) if err != nil { d.logger.Printf("[ERR] driver.docker: failed to create container from image %s: %s\n", image, err) - return nil, fmt.Errorf("Failed to create container from image %s", image) + return nil, fmt.Errorf("Failed to create container from image %s: %s", image, err) } d.logger.Printf("[INFO] driver.docker: created container %s\n", container.ID) // Start the container err = client.StartContainer(container.ID, container.HostConfig) if err != nil { - d.logger.Printf("[ERR] driver.docker: starting container %s\n", container.ID) - return nil, fmt.Errorf("Failed to start container %s", container.ID) + d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s\n", container.ID, err) + return nil, fmt.Errorf("Failed to start container %s: %s", container.ID, err) } d.logger.Printf("[INFO] driver.docker: started container %s\n", container.ID) diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index 05df5a647..e3397521b 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -293,7 +293,7 @@ func taskTemplate() *structs.Task { &structs.NetworkResource{ IP: "127.0.0.1", ReservedPorts: []structs.Port{{"main", 11110}}, - DynamicPorts: []structs.Port{{"REDIS", 0}}, + DynamicPorts: []structs.Port{{"REDIS", 43330}}, }, }, }, @@ -307,12 +307,15 @@ func TestDocker_StartN(t *testing.T) { task1 := taskTemplate() task1.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 11110} + task1.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43331} task2 := taskTemplate() task2.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 22222} + task2.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43332} task3 := taskTemplate() task3.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 33333} + task3.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43333} taskList := []*structs.Task{task1, task2, task3} @@ -359,14 +362,17 @@ func TestDocker_StartNVersions(t *testing.T) { task1 := taskTemplate() task1.Config["image"] = "redis" task1.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 11110} + task1.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43331} task2 := taskTemplate() task2.Config["image"] = "redis:latest" task2.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 22222} + task2.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43332} task3 := taskTemplate() task3.Config["image"] = "redis:3.0" task3.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 33333} + task3.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43333} taskList := []*structs.Task{task1, task2, task3} diff --git a/client/driver/driver_test.go b/client/driver/driver_test.go index 7065153a1..93727bd03 100644 --- a/client/driver/driver_test.go +++ b/client/driver/driver_test.go @@ -19,7 +19,7 @@ var basicResources = &structs.Resources{ &structs.NetworkResource{ IP: "0.0.0.0", ReservedPorts: []structs.Port{{"main", 12345}}, - DynamicPorts: []structs.Port{{"HTTP", 0}}, + DynamicPorts: []structs.Port{{"HTTP", 43330}}, }, }, } From f3283f2771118d2e93fe8705a81dc08d81f0b476 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 17 Nov 2015 19:36:05 -0800 Subject: [PATCH 26/36] Update environment docs regarding ports --- .../source/docs/jobspec/environment.html.md | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/website/source/docs/jobspec/environment.html.md b/website/source/docs/jobspec/environment.html.md index fc9cc91a6..9ef01f8f3 100644 --- a/website/source/docs/jobspec/environment.html.md +++ b/website/source/docs/jobspec/environment.html.md @@ -41,22 +41,21 @@ cluster gets more or less busy. Each task will receive port allocations on a single IP address. The IP is made available through `NOMAD_IP.` -If you requested reserved ports in your job specification and your task is successfully -scheduled, these ports are available for your use. Ports from `reserved_ports` -in the job spec are not exposed through the environment. If you requested -dynamic ports in your job specification these are made known to your application via -environment variables `NOMAD_PORT_{LABEL}`. For example -`dynamic_ports = ["HTTP"]` becomes `NOMAD_PORT_HTTP`. +Both dynamic and reserved ports are exposed through environment variables in the +following format, `NOMAD_PORT_{LABEL}={PORT}`. For example, a dynamic port +`port "HTTP" {}` becomes `NOMAD_PORT_HTTP=48907`. Some drivers such as Docker and QEMU use port mapping. If a driver supports port -mapping and you specify a numeric label, the label will be automatically used as -the private port number. For example, `dynamic_ports = ["5000"]` will have a -random port mapped to port 5000 inside the container or VM. These ports are also -exported as environment variables for consistency, e.g. `NOMAD_PORT_5000`. +mapping and it has been set in the driver configuration, container ports and +their mapped host port are exposed as environment variables with the following +format, `NOMAD_PORT_{CONTAINER_PORT}={HOST_PORT}`. To give a concrete example, +imagine you had the following port configuration, `port "db" { static = 8181 }` +and you had the following port mapping, `port_map { db = 6379 }`. The following +environment variable would then be set, `NOMAD_PORT_6379=8181`. Please see the relevant driver documentation for details. -### Task Directories +### Task Directories Nomad makes the following two directories available to tasks: From 640af994ca0fd402a70d04d9243d4cd5c90189a1 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 19:45:33 -0800 Subject: [PATCH 27/36] Added a randomized alloc id for tests so container names don't collide --- client/driver/docker.go | 2 +- client/driver/driver_test.go | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 0c9828e43..93a0debfc 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -324,7 +324,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri config.Env = env.List() return docker.CreateContainerOptions{ - // Name: fmt.Sprintf("%s-%s", task.Name, ctx.AllocID), + Name: fmt.Sprintf("%s-%s", task.Name, ctx.AllocID), Config: config, HostConfig: hostConfig, }, nil diff --git a/client/driver/driver_test.go b/client/driver/driver_test.go index 93727bd03..fd4f30569 100644 --- a/client/driver/driver_test.go +++ b/client/driver/driver_test.go @@ -1,7 +1,9 @@ package driver import ( + "fmt" "log" + "math/rand" "os" "path/filepath" "reflect" @@ -24,6 +26,10 @@ var basicResources = &structs.Resources{ }, } +func init() { + rand.Seed(49875) +} + func testLogger() *log.Logger { return log.New(os.Stderr, "", log.LstdFlags) } @@ -43,7 +49,7 @@ func testDriverContext(task string) *DriverContext { func testDriverExecContext(task *structs.Task, driverCtx *DriverContext) *ExecContext { allocDir := allocdir.NewAllocDir(filepath.Join(driverCtx.config.AllocDir, structs.GenerateUUID())) allocDir.Build([]*structs.Task{task}) - ctx := NewExecContext(allocDir, "dummyAllocId") + ctx := NewExecContext(allocDir, fmt.Sprintf("alloc-id-%d", int(rand.Int31()))) return ctx } From 0e1fe2373a275f7aba1d1ac5f845af6b04e4dcbb Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 20:04:10 -0800 Subject: [PATCH 28/36] Log container name and labels --- client/driver/docker.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 93a0debfc..ed12db18e 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -238,7 +238,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri hostConfig.NetworkMode = driverConfig.NetworkMode if hostConfig.NetworkMode == "" { // docker default - d.logger.Println("[INFO] driver.docker: networking mode not specified; defaulting to bridge") + d.logger.Println("[DEBUG] driver.docker: networking mode not specified; defaulting to bridge") hostConfig.NetworkMode = "bridge" } @@ -319,12 +319,16 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri if len(driverConfig.Labels) == 1 { config.Labels = driverConfig.Labels[0] - d.logger.Println("[DEBUG] driver.docker: applied labels on the container") + d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v\n", config.Labels) } config.Env = env.List() + + containerName := fmt.Sprintf("%s-%s", task.Name, ctx.AllocID) + d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s\n", containerName) + return docker.CreateContainerOptions{ - Name: fmt.Sprintf("%s-%s", task.Name, ctx.AllocID), + Name: containerName, Config: config, HostConfig: hostConfig, }, nil From 49e7ded372e2d10229eeba2b38e9480bf73b28ec Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 17 Nov 2015 20:06:00 -0800 Subject: [PATCH 29/36] fix kernel/os attributes --- website/source/docs/jobspec/index.html.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/website/source/docs/jobspec/index.html.md b/website/source/docs/jobspec/index.html.md index 8bda7cb10..64228ed21 100644 --- a/website/source/docs/jobspec/index.html.md +++ b/website/source/docs/jobspec/index.html.md @@ -81,7 +81,7 @@ where a task is eligible for running. An example constraint looks like: ``` # Restrict to only nodes running linux constraint { - attribute = "$attr.kernel.os" + attribute = "$attr.kernel.name" value = "linux" } ``` @@ -343,6 +343,14 @@ Below is a table documenting common node attributes: hostname Hostname of the client + + kernel.name + Kernel of the client. Examples: "linux", "darwin" + + + kernel.version + Version of the client kernel. Examples: "3.19.0-25-generic", "15.0.0" + platform.aws.ami-id On EC2, the AMI ID of the client node @@ -353,7 +361,7 @@ Below is a table documenting common node attributes: os.name - Operating system of the client. Examples: "linux", "windows", "darwin" + Operating system of the client. Examples: "ubuntu", "windows", "darwin" os.version From 5ac6664c4677be5439ec08ea3ad633ada12f2f3a Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 20:50:14 -0800 Subject: [PATCH 30/36] Purge existing container during Start() --- client/driver/docker.go | 51 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index ed12db18e..1407cb610 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -212,7 +212,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri // set privileged mode hostPrivileged := d.config.ReadBoolDefault("docker.privileged.enabled", false) if driverConfig.Privileged && !hostPrivileged { - return c, fmt.Errorf(`Unable to set privileged flag since "docker.privileged.enabled" is false`) + return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent`) } hostConfig.Privileged = hostPrivileged @@ -416,8 +416,46 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle // Create a container container, err := client.CreateContainer(config) if err != nil { - d.logger.Printf("[ERR] driver.docker: failed to create container from image %s: %s\n", image, err) - return nil, fmt.Errorf("Failed to create container from image %s: %s", image, err) + // If the container already exists because of a previous failure we'll + // try to purge it and re-create it. + if err.Error() == "container already exists" { + // Get the ID of the existing container so we can delete it + containers, err := client.ListContainers(docker.ListContainersOptions{ + // The image might be in use by a stopped container, so check everything + All: true, + Filters: map[string][]string{ + "name": []string{config.Name}, + }, + }) + if err != nil { + log.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s\n", config.Name) + return nil, fmt.Errorf("Failed to query list of containers: %s", err) + } + + if len(containers) != 1 { + log.Printf("[ERR] driver.docker: failed to get id for container %s\n", config.Name) + return nil, fmt.Errorf("Failed to get id for container %s", config.Name, err) + } + + log.Printf("[INFO] driver.docker: a container with the name %s already exists; will attempt to purge and re-create\n", config.Name) + err = client.RemoveContainer(docker.RemoveContainerOptions{ + ID: containers[0].ID, + }) + if err != nil { + log.Printf("[ERR] driver.docker: failed to purge container %s\n", config.Name) + return nil, fmt.Errorf("Failed to purge container %s: %s", config.Name, err) + } + log.Printf("[INFO] driver.docker: purged container %s\n", config.Name) + container, err = client.CreateContainer(config) + if err != nil { + log.Printf("[ERR] driver.docker: failed to re-create container %s; aborting\n", config.Name) + return nil, fmt.Errorf("Failed to re-create container %s; aborting", config.Name) + } + } else { + // We failed to create the container for some other reason. + d.logger.Printf("[ERR] driver.docker: failed to create container from image %s: %s\n", image, err) + return nil, fmt.Errorf("Failed to create container from image %s: %s", image, err) + } } d.logger.Printf("[INFO] driver.docker: created container %s\n", container.ID) @@ -555,11 +593,12 @@ func (h *dockerHandle) Kill() error { }, }) if err != nil { - return fmt.Errorf("Unable to query list of containers: %s", err) + log.Printf("[ERR] driver.docker: failed to query list of containers matching image:%s\n", h.imageID) + return fmt.Errorf("Failed to query list of containers: %s", err) } inUse := len(containers) if inUse > 0 { - log.Printf("[INFO] driver.docker: image %s is still in use by %d containers\n", h.imageID, inUse) + log.Printf("[INFO] driver.docker: image %s is still in use by %d container(s)\n", h.imageID, inUse) } else { return fmt.Errorf("Failed to remove image %s", h.imageID) } @@ -574,7 +613,7 @@ func (h *dockerHandle) run() { // Wait for it... exitCode, err := h.client.WaitContainer(h.containerID) if err != nil { - h.logger.Printf("[ERR] driver.docker: unable to wait for %s; container already terminated\n", h.containerID) + h.logger.Printf("[ERR] driver.docker: failed to wait for %s; container already terminated\n", h.containerID) } if exitCode != 0 { From 2a8bd98fdc424705dcc6ae2fed1d2612d9dde06c Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 17 Nov 2015 20:54:53 -0800 Subject: [PATCH 31/36] Qemu driver takes a port_map to configure host to guest port forwarding --- client/driver/qemu.go | 67 +++++++++++++++++--------------------- client/driver/qemu_test.go | 5 ++- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/client/driver/qemu.go b/client/driver/qemu.go index f51907f75..153051c3c 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -6,7 +6,6 @@ import ( "path/filepath" "regexp" "runtime" - "strconv" "strings" "time" @@ -33,10 +32,10 @@ type QemuDriver struct { } type QemuDriverConfig struct { - ArtifactSource string `mapstructure:"artifact_source"` - Checksum string `mapstructure:"checksum"` - Accelerator string `mapstructure:"accelerator"` - GuestPorts string `mapstructure:"guest_ports"` + ArtifactSource string `mapstructure:"artifact_source"` + Checksum string `mapstructure:"checksum"` + Accelerator string `mapstructure:"accelerator"` + PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports. } // qemuHandle is returned from Start/Open as a handle to the PID @@ -82,6 +81,11 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil { return nil, err } + + if len(driverConfig.PortMap) > 1 { + return nil, fmt.Errorf("Only one port_map block is allowed in the qemu driver config") + } + // Get the image source source, ok := task.Config["artifact_source"] if !ok || source == "" { @@ -138,42 +142,31 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, // the outside world to be able to reach it. VMs ran without port mappings can // still reach out to the world, but without port mappings it is effectively // firewalled - if len(task.Resources.Networks) > 0 { - // TODO: Consolidate these into map of host/guest port when we have HCL - // Note: Host port must be open and available - // Get and split guest ports. The guest_ports configuration must match up with - // the Reserved ports in the Task Resources - // Users can supply guest_hosts as a list of posts to map on the guest vm. - // These map 1:1 with the requested Reserved Ports from the hostmachine. - ports := strings.Split(driverConfig.GuestPorts, ",") - if len(ports) == 0 { - return nil, fmt.Errorf("[ERR] driver.qemu: Error parsing required Guest Ports") - } - - // TODO: support more than a single, default Network - if len(ports) != len(task.Resources.Networks[0].ReservedPorts) { - return nil, fmt.Errorf("[ERR] driver.qemu: Error matching Guest Ports with Reserved ports") - } - - // Loop through the reserved ports and construct the hostfwd string, to map + protocols := []string{"udp", "tcp"} + if len(task.Resources.Networks) > 0 && len(driverConfig.PortMap) == 1 { + // Loop through the port map and construct the hostfwd string, to map // reserved ports to the ports listenting in the VM - // Ex: - // hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080 - reservedPorts := task.Resources.Networks[0].ReservedPorts - var forwarding string - for i, p := range ports { - forwarding = fmt.Sprintf("%s,hostfwd=tcp::%s-:%s", forwarding, strconv.Itoa(reservedPorts[i].Value), p) + // Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080 + var forwarding []string + taskPorts := task.Resources.Networks[0].MapLabelToValues() + for label, guest := range driverConfig.PortMap[0] { + host, ok := taskPorts[label] + if !ok { + return nil, fmt.Errorf("Unknown port label %q", label) + } + + for _, p := range protocols { + forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest)) + } } - if "" == forwarding { - return nil, fmt.Errorf("[ERR] driver.qemu: Error constructing port forwarding") + if len(forwarding) != 0 { + args = append(args, + "-netdev", + fmt.Sprintf("user,id=user.0%s", strings.Join(forwarding, ",")), + "-device", "virtio-net,netdev=user.0", + ) } - - args = append(args, - "-netdev", - fmt.Sprintf("user,id=user.0%s", forwarding), - "-device", "virtio-net,netdev=user.0", - ) } // If using KVM, add optimization args diff --git a/client/driver/qemu_test.go b/client/driver/qemu_test.go index 543bf247b..cecca4357 100644 --- a/client/driver/qemu_test.go +++ b/client/driver/qemu_test.go @@ -41,7 +41,10 @@ func TestQemuDriver_StartOpen_Wait(t *testing.T) { "artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", "checksum": "sha256:a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", "accelerator": "tcg", - "guest_ports": "22,8080", + "port_map": []map[string]int{{ + "main": 22, + "web": 8080, + }}, }, Resources: &structs.Resources{ CPU: 500, From 8705ea07a4c82eab417ad570e10a348fcacfeee3 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 21:17:51 -0800 Subject: [PATCH 32/36] Remove \n since this is added by the logger --- client/driver/docker.go | 68 ++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 1407cb610..f0993b3bc 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -103,7 +103,7 @@ func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool // Initialize docker API client client, err := d.dockerClient() if err != nil { - d.logger.Printf("[INFO] driver.docker: failed to initialize client: %s\n", err) + d.logger.Printf("[INFO] driver.docker: failed to initialize client: %s", err) return false, nil } @@ -120,7 +120,7 @@ func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool // Docker isn't available so we'll simply disable the docker driver. env, err := client.Version() if err != nil { - d.logger.Printf("[INFO] driver.docker: could not connect to docker daemon at %s: %s\n", client.Endpoint(), err) + d.logger.Printf("[INFO] driver.docker: could not connect to docker daemon at %s: %s", client.Endpoint(), err) return false, nil } node.Attributes["driver.docker"] = "1" @@ -205,9 +205,9 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri Binds: binds, } - d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s\n", hostConfig.Memory, task.Config["image"]) - d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s\n", hostConfig.CPUShares, task.Config["image"]) - d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s\n", hostConfig.Binds, task.Config["image"]) + d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"]) + d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"]) + d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"]) // set privileged mode hostPrivileged := d.config.ReadBoolDefault("docker.privileged.enabled", false) @@ -223,7 +223,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri if net.ParseIP(ip) != nil { hostConfig.DNS = append(hostConfig.DNS, ip) } else { - d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s\n", ip) + d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip) } } } @@ -260,11 +260,11 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri publishedPorts[dockerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} publishedPorts[dockerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} - d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)\n", network.IP, port.Value, port.Value) + d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value) exposedPorts[dockerPort+"/tcp"] = struct{}{} exposedPorts[dockerPort+"/udp"] = struct{}{} - d.logger.Printf("[DEBUG] driver.docker: exposed port %d\n", port.Value) + d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value) } containerToHostPortMap := make(map[string]int) @@ -285,11 +285,11 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri publishedPorts[containerPortStr+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} publishedPorts[containerPortStr+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} - d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)\n", network.IP, port.Value, containerPort) + d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPort) exposedPorts[containerPortStr+"/tcp"] = struct{}{} exposedPorts[containerPortStr+"/udp"] = struct{}{} - d.logger.Printf("[DEBUG] driver.docker: exposed port %s\n", hostPortStr) + d.logger.Printf("[DEBUG] driver.docker: exposed port %s", hostPortStr) containerToHostPortMap[string(containerPortStr)] = port.Value } @@ -311,7 +311,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri if driverConfig.Args != "" { cmd = append(cmd, parsedArgs...) } - d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s\n", strings.Join(cmd, " ")) + d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " ")) config.Cmd = cmd } else if driverConfig.Args != "" { d.logger.Println("[DEBUG] driver.docker: ignoring command arguments because command is not specified") @@ -319,13 +319,13 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri if len(driverConfig.Labels) == 1 { config.Labels = driverConfig.Labels[0] - d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v\n", config.Labels) + d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v", config.Labels) } config.Env = env.List() containerName := fmt.Sprintf("%s-%s", task.Name, ctx.AllocID) - d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s\n", containerName) + d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s", containerName) return docker.CreateContainerOptions{ Name: containerName, @@ -394,23 +394,23 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle err = client.PullImage(pullOptions, authOptions) if err != nil { - d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s\n", repo, tag, err) + d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s", repo, tag, err) return nil, fmt.Errorf("Failed to pull `%s`: %s", image, err) } - d.logger.Printf("[DEBUG] driver.docker: docker pull %s:%s succeeded\n", repo, tag) + d.logger.Printf("[DEBUG] driver.docker: docker pull %s:%s succeeded", repo, tag) // Now that we have the image we can get the image id dockerImage, err = client.InspectImage(image) if err != nil { - d.logger.Printf("[ERR] driver.docker: failed getting image id for %s: %s\n", image, err) + d.logger.Printf("[ERR] driver.docker: failed getting image id for %s: %s", image, err) return nil, fmt.Errorf("Failed to determine image id for `%s`: %s", image, err) } } - d.logger.Printf("[DEBUG] driver.docker: identified image %s as %s\n", image, dockerImage.ID) + d.logger.Printf("[DEBUG] driver.docker: identified image %s as %s", image, dockerImage.ID) config, err := d.createContainer(ctx, task, &driverConfig) if err != nil { - d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %s: %s\n", image, err) + d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %s: %s", image, err) return nil, fmt.Errorf("Failed to create container configuration for image %s: %s", image, err) } // Create a container @@ -428,44 +428,44 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle }, }) if err != nil { - log.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s\n", config.Name) + log.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s", config.Name) return nil, fmt.Errorf("Failed to query list of containers: %s", err) } if len(containers) != 1 { - log.Printf("[ERR] driver.docker: failed to get id for container %s\n", config.Name) + log.Printf("[ERR] driver.docker: failed to get id for container %s", config.Name) return nil, fmt.Errorf("Failed to get id for container %s", config.Name, err) } - log.Printf("[INFO] driver.docker: a container with the name %s already exists; will attempt to purge and re-create\n", config.Name) + log.Printf("[INFO] driver.docker: a container with the name %s already exists; will attempt to purge and re-create", config.Name) err = client.RemoveContainer(docker.RemoveContainerOptions{ ID: containers[0].ID, }) if err != nil { - log.Printf("[ERR] driver.docker: failed to purge container %s\n", config.Name) + log.Printf("[ERR] driver.docker: failed to purge container %s", config.Name) return nil, fmt.Errorf("Failed to purge container %s: %s", config.Name, err) } - log.Printf("[INFO] driver.docker: purged container %s\n", config.Name) + log.Printf("[INFO] driver.docker: purged container %s", config.Name) container, err = client.CreateContainer(config) if err != nil { - log.Printf("[ERR] driver.docker: failed to re-create container %s; aborting\n", config.Name) + log.Printf("[ERR] driver.docker: failed to re-create container %s; aborting", config.Name) return nil, fmt.Errorf("Failed to re-create container %s; aborting", config.Name) } } else { // We failed to create the container for some other reason. - d.logger.Printf("[ERR] driver.docker: failed to create container from image %s: %s\n", image, err) + d.logger.Printf("[ERR] driver.docker: failed to create container from image %s: %s", image, err) return nil, fmt.Errorf("Failed to create container from image %s: %s", image, err) } } - d.logger.Printf("[INFO] driver.docker: created container %s\n", container.ID) + d.logger.Printf("[INFO] driver.docker: created container %s", container.ID) // Start the container err = client.StartContainer(container.ID, container.HostConfig) if err != nil { - d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s\n", container.ID, err) + d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s", container.ID, err) return nil, fmt.Errorf("Failed to start container %s: %s", container.ID, err) } - d.logger.Printf("[INFO] driver.docker: started container %s\n", container.ID) + d.logger.Printf("[INFO] driver.docker: started container %s", container.ID) // Return a driver handle h := &dockerHandle{ @@ -492,7 +492,7 @@ func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, er if err := json.Unmarshal(pidBytes, pid); err != nil { return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err) } - d.logger.Printf("[INFO] driver.docker: re-attaching to docker process: %s\n", handleID) + d.logger.Printf("[INFO] driver.docker: re-attaching to docker process: %s", handleID) // Initialize docker API client client, err := d.dockerClient() @@ -543,7 +543,7 @@ func (h *dockerHandle) ID() string { } data, err := json.Marshal(pid) if err != nil { - h.logger.Printf("[ERR] driver.docker: failed to marshal docker PID to JSON: %s\n", err) + h.logger.Printf("[ERR] driver.docker: failed to marshal docker PID to JSON: %s", err) } return fmt.Sprintf("DOCKER:%s", string(data)) } @@ -593,17 +593,17 @@ func (h *dockerHandle) Kill() error { }, }) if err != nil { - log.Printf("[ERR] driver.docker: failed to query list of containers matching image:%s\n", h.imageID) + log.Printf("[ERR] driver.docker: failed to query list of containers matching image:%s", h.imageID) return fmt.Errorf("Failed to query list of containers: %s", err) } inUse := len(containers) if inUse > 0 { - log.Printf("[INFO] driver.docker: image %s is still in use by %d container(s)\n", h.imageID, inUse) + log.Printf("[INFO] driver.docker: image %s is still in use by %d container(s)", h.imageID, inUse) } else { return fmt.Errorf("Failed to remove image %s", h.imageID) } } else { - log.Printf("[INFO] driver.docker: removed image %s\n", h.imageID) + log.Printf("[INFO] driver.docker: removed image %s", h.imageID) } } return nil @@ -613,7 +613,7 @@ func (h *dockerHandle) run() { // Wait for it... exitCode, err := h.client.WaitContainer(h.containerID) if err != nil { - h.logger.Printf("[ERR] driver.docker: failed to wait for %s; container already terminated\n", h.containerID) + h.logger.Printf("[ERR] driver.docker: failed to wait for %s; container already terminated", h.containerID) } if exitCode != 0 { From 4d339c198bada2d6b8c5b9881f0f185edd394441 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 17 Nov 2015 21:19:45 -0800 Subject: [PATCH 33/36] Update qemu docs --- website/source/docs/drivers/qemu.html.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/website/source/docs/drivers/qemu.html.md b/website/source/docs/drivers/qemu.html.md index 403926b4c..84909e331 100644 --- a/website/source/docs/drivers/qemu.html.md +++ b/website/source/docs/drivers/qemu.html.md @@ -24,18 +24,18 @@ The `Qemu` driver can execute any regular `qemu` image (e.g. `qcow`, `img`, The `Qemu` driver supports the following configuration in the job spec: * `artifact_source` - **(Required)** The hosted location of the source Qemu image. Must be accessible -from the Nomad client, via HTTP. + from the Nomad client, via HTTP. * `checksum` - **(Optional)** The checksum type and value for the `artifact_source` image. -The format is `type:value`, where type is any of `md5`, `sha1`, `sha256`, or `sha512`, -and the value is the computed checksum. If a checksum is supplied and does not -match the downloaded artifact, the driver will fail to start + The format is `type:value`, where type is any of `md5`, `sha1`, `sha256`, or `sha512`, + and the value is the computed checksum. If a checksum is supplied and does not + match the downloaded artifact, the driver will fail to start * `accelerator` - (Optional) The type of accelerator to use in the invocation. - If the host machine has `Qemu` installed with KVM support, users can specify `kvm` for the `accelerator`. Default is `tcg` -* `host_port` - **(Required)** Port on the host machine to forward to the guest -VM -* `guest_ports` - **(Optional)** Ports on the guest machine that are listening for -traffic from the host. These ports match up with any `ReservedPorts` requested -in the `Task` specification + If the host machine has `Qemu` installed with KVM support, users can specify + `kvm` for the `accelerator`. Default is `tcg` +* `port_map` - **(Optional)** A `map[string]int` that maps port labels to ports + on the guest. This forwards the host port to the guest vm. For example, + `port_map { db = 6539 }` would forward the host port with label `db` to the + guest vm's port 6539. ## Client Requirements From 105ff2576479a93a3ce7aacfbfda02e05511a1eb Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 17 Nov 2015 21:30:15 -0800 Subject: [PATCH 34/36] Fix docker config option name in docs --- website/source/docs/drivers/docker.html.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 8a7d71693..313b59c1f 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -39,10 +39,10 @@ specification: Tasks with `privileged` set can only run on Nomad Agents with `docker.privileged.enabled = "true"`. -* `dns-servers` - (Optional) A comma separated list of DNS servers for the +* `dns_servers` - (Optional) A comma separated list of DNS servers for the container to use (e.g. "8.8.8.8,8.8.4.4"). *Docker API v1.10 and above only* -* `search-domains` - (Optional) A comma separated list of DNS search domains +* `search_domains` - (Optional) A comma separated list of DNS search domains for the container to use. * `hostname` - (Optional) The hostname to assign to the container. When From 563e1aff560e312965cbd785bf2a61ef51d90603 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 21:34:07 -0800 Subject: [PATCH 35/36] Renamed some things so it's more apparent that reserved and dynamic port mapping have very similar code --- client/driver/docker.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index f0993b3bc..4b7dbee56 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -256,42 +256,42 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri for _, port := range network.ReservedPorts { hostPortStr := strconv.Itoa(port.Value) - dockerPort := docker.Port(hostPortStr) + containerPort := docker.Port(hostPortStr) - publishedPorts[dockerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} - publishedPorts[dockerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} + publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} + publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value) - exposedPorts[dockerPort+"/tcp"] = struct{}{} - exposedPorts[dockerPort+"/udp"] = struct{}{} + exposedPorts[containerPort+"/tcp"] = struct{}{} + exposedPorts[containerPort+"/udp"] = struct{}{} d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value) } containerToHostPortMap := make(map[string]int) for _, port := range network.DynamicPorts { // By default we will map the allocated port 1:1 to the container - containerPort := port.Value + containerPortInt := port.Value // If the user has mapped a port using port_map we'll change it here if len(driverConfig.PortMap) == 1 { mapped, ok := driverConfig.PortMap[0][port.Label] if ok { - containerPort = mapped + containerPortInt = mapped } } - containerPortStr := docker.Port(strconv.Itoa(containerPort)) hostPortStr := strconv.Itoa(port.Value) + containerPort := docker.Port(hostPortStr) - publishedPorts[containerPortStr+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} - publishedPorts[containerPortStr+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} - d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPort) + publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} + publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} + d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPortInt) - exposedPorts[containerPortStr+"/tcp"] = struct{}{} - exposedPorts[containerPortStr+"/udp"] = struct{}{} + exposedPorts[containerPort+"/tcp"] = struct{}{} + exposedPorts[containerPort+"/udp"] = struct{}{} d.logger.Printf("[DEBUG] driver.docker: exposed port %s", hostPortStr) - containerToHostPortMap[string(containerPortStr)] = port.Value + containerToHostPortMap[string(containerPort)] = port.Value } env.SetPorts(containerToHostPortMap) From c851ae67c7c344bef132cecd9a510272c189b9b2 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 21:36:23 -0800 Subject: [PATCH 36/36] Change error check to contains instead of == --- client/driver/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 4b7dbee56..d1e007e25 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -418,7 +418,7 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle if err != nil { // If the container already exists because of a previous failure we'll // try to purge it and re-create it. - if err.Error() == "container already exists" { + if strings.Contains(err.Error(), "container already exists") { // Get the ID of the existing container so we can delete it containers, err := client.ListContainers(docker.ListContainersOptions{ // The image might be in use by a stopped container, so check everything