From 5c27303329ba2d616821bde61e367bd1859c6951 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 17 Nov 2015 22:10:51 -0800 Subject: [PATCH 1/4] Updated changelog to include DNS and auth explicitly --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6597cf4eb..428d401d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,9 +39,10 @@ IMPROVEMENTS: * client: Precise snapshotting of TaskRunner and AllocRunner [GH-403, GH-411] * client: Task State is tracked by client [GH-416] * client: Test Skip Detection [GH-221] - * driver/docker: Advanced docker driver options [GH-390] - * driver/docker: Docker container name can be set [GH-389] - * driver/docker: Docker hostname can be set [GH-426] + * driver/docker: Can now specify auth for docker pull [GH-390] + * driver/docker: Can now specify DNS and DNSSearch options [GH-390] + * driver/docker: Can now specify the container's hostname [GH-426] + * driver/docker: Containers now have names based on the task name. [GH-389] * driver/docker: Mount task local and alloc directory to docker containers [GH-290] * driver/docker: Now accepts any value for `network_mode` to support userspace networking plugins in docker 1.9 * driver/java: Pass JVM options in java driver [GH-293, GH-297] From b6796a600a9a3957f858c91b7ef82f4be4eba067 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 17 Nov 2015 22:35:58 -0800 Subject: [PATCH 2/4] Add page describing scheduler types --- website/source/docs/jobspec/index.html.md | 5 +- .../source/docs/jobspec/schedulers.html.md | 46 +++++++++++++++++++ website/source/layouts/docs.erb | 3 ++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 website/source/docs/jobspec/schedulers.html.md diff --git a/website/source/docs/jobspec/index.html.md b/website/source/docs/jobspec/index.html.md index 64228ed21..cffc04ed1 100644 --- a/website/source/docs/jobspec/index.html.md +++ b/website/source/docs/jobspec/index.html.md @@ -135,7 +135,8 @@ The `job` object supports the following keys: * `type` - Specifies the job type and switches which scheduler is used. Nomad provides the `service`, `system` and `batch` schedulers, - and defaults to `service`. + and defaults to `service`. To learn more about each scheduler type visit + [here](/docs/jobspec/schedulers.html) * `update` - Specifies the task update strategy. This requires providing `max_parallel` as an integer and `stagger` as a time duration. If stagger @@ -240,7 +241,7 @@ restart { } ``` -The default non-batch restart policy is: +The default non-batch restart policy is: ``` restart { diff --git a/website/source/docs/jobspec/schedulers.html.md b/website/source/docs/jobspec/schedulers.html.md new file mode 100644 index 000000000..9cd1fdde3 --- /dev/null +++ b/website/source/docs/jobspec/schedulers.html.md @@ -0,0 +1,46 @@ +--- +layout: "docs" +page_title: "Nomad Schedulers" +sidebar_current: "docs-jobspec-schedulers" +description: |- + Learn about Nomad's various schedulers. +--- + +# Scheduler Types + +Nomad has three scheduler types that can be used hen creating your +[job](/docs/jobspec/): `service`, `batch` and `system`. Here we will describe +the differences between each of these schedulers. + +## Service + +The `service` scheduler is designed for scheduling long lived services that +should never go down. As such, the `service` scheduler ranks a large portion +of the nodes that meet the jobs constraints and selects the optimal node to +place a task group on. The `service` scheduler uses a scoring algorithm based on +Google's BestFit v3 algorithm. Ranking this larger set of candidate nodes +increases scheduling time but provides greater guarantees about the optimality +of a job placement, which given the service workload is highly desirable. + +## Batch + +Batch jobs are much less sensitive to short term performance fluctuations and +are short lived, finishing in a few minutes to a few days. Although the `batch` +scheduler is very similar to the `service` scheduler, it makes certain +optimizations for the batch workload. The main distinction is that after finding +the set of nodes that meet the jobs constraints it uses the power of two choices +described in Berkeley's Sparrow scheduler to limit the number of nodes that are +ranked. + +## System + +The `system` scheduler is used to register jobs that should be run on all +clients that meet the job's constraints. The `system` scheduler is also invoked +when clients join the cluster or transition into the ready state. This means +that all registered `system` jobs will be re-evaluated and their tasks will be +placed on the newly available nodes if the constraints are met. + +This scheduler type is extremely useful for deploying and managing tasks that +should be present on every node in the cluster. Since these tasks are being +managed by Nomad, they can take advantage of job updating, rolling deploys, +service discovery and more. diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index e428cb593..115623599 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -38,6 +38,9 @@ > Runtime Environment + > + Scheduler Types + From de6c06d3321c733e378a6ed6cae1ae4fe104f6e4 Mon Sep 17 00:00:00 2001 From: Carlos Diaz-Padron Date: Tue, 17 Nov 2015 23:32:57 -0800 Subject: [PATCH 3/4] De-nest docker registry auth and reformat related doc --- client/driver/docker.go | 36 ++++++++++------------ website/source/docs/drivers/docker.html.md | 10 +++--- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index d1e007e25..d209accf8 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -25,26 +25,21 @@ type DockerDriver struct { fingerprint.StaticFingerprinter } -type DockerAuthConfig struct { - UserName string `mapstructure:"auth.username"` // user name of the registry - Password string `mapstructure:"auth.password"` // password to access the registry - Email string `mapstructure:"auth.email"` // email address of the user who is allowed to access the registry - ServerAddress string `mapstructure:"auth.server_address"` // server address of the registry - -} - type DockerDriverConfig struct { - DockerAuthConfig - ImageName string `mapstructure:"image"` // Container's Image Name - Command string `mapstructure:"command"` // The Command/Entrypoint to run when the container starts up - Args string `mapstructure:"args"` // The arguments to the Command/Entrypoint - NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, net and none - PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and the ports exposed on the container - Privileged bool `mapstructure:"privileged"` // Flag to run the container in priviledged mode - DNS string `mapstructure:"dns_server"` // DNS Server for containers - SearchDomains string `mapstructure:"search_domains"` // DNS Search domains for containers - Hostname string `mapstructure:"hostname"` // Hostname for containers - Labels []map[string]string `mapstructure:"labels"` // Labels to set when the container starts up + ImageName string `mapstructure:"image"` // Container's Image Name + Command string `mapstructure:"command"` // The Command/Entrypoint to run when the container starts up + Args string `mapstructure:"args"` // The arguments to the Command/Entrypoint + NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, net and none + PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and the ports exposed on the container + Privileged bool `mapstructure:"privileged"` // Flag to run the container in priviledged mode + DNS string `mapstructure:"dns_server"` // DNS Server for containers + SearchDomains string `mapstructure:"search_domains"` // DNS Search domains for containers + Hostname string `mapstructure:"hostname"` // Hostname for containers + Labels []map[string]string `mapstructure:"labels"` // Labels to set when the container starts up + UserName string `mapstructure:"auth_username"` // user name of the registry + Password string `mapstructure:"auth_password"` // password to access the registry + Email string `mapstructure:"auth_email"` // email address of the user who is allowed to access the registry + ServerAddress string `mapstructure:"auth_server_address"` // server address of the registry } func (c *DockerDriverConfig) Validate() error { @@ -392,6 +387,9 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle ServerAddress: driverConfig.ServerAddress, } + d.logger.Printf("[DEBUG] TASKCONFIG: %v", task.Config) + d.logger.Printf("[DEBUG] DRIVERCONFIG: %v", driverConfig) + d.logger.Printf("[DEBUG] AUTH: %v", authOptions) err = client.PullImage(pullOptions, authOptions) if err != nil { d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s", repo, tag, err) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 313b59c1f..255437c5d 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -56,11 +56,11 @@ specification: following authentication parameters. These options can provide access to private repositories that utilize the docker remote api (e.g. dockerhub, quay.io) - - `auth.username` - (Optional) The account username - - `auth.password` - (Optional) The account password - - `auth.email` - (Optional) The account email - - `auth.server-address` - (Optional) The server domain/ip without the - protocol + +* `auth_username` - (Optional) The account username. +* `auth_password` - (Optional) The account password. +* `auth_email` - (Optional) The account email. +* `auth_server-address` - (Optional) The server domain/ip without the protocol. ### Port Mapping From 4a6c3d773d1fde555acdc7eb1a5638d9afa08ef6 Mon Sep 17 00:00:00 2001 From: Carlos Diaz-Padron Date: Wed, 18 Nov 2015 01:37:42 -0800 Subject: [PATCH 4/4] Nest Docker driver auth under object --- client/driver/docker.go | 48 ++++++++++++---------- website/source/docs/drivers/docker.html.md | 10 +++-- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index d209accf8..0cafaa672 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -25,21 +25,25 @@ type DockerDriver struct { fingerprint.StaticFingerprinter } +type DockerDriverAuth struct { + Username string `mapstructure:"username"` // username for the registry + Password string `mapstructure:"password"` // password to access the registry + Email string `mapstructure:"email"` // email address of the user who is allowed to access the registry + ServerAddress string `mapstructure:"server_address"` // server address of the registry +} + type DockerDriverConfig struct { - ImageName string `mapstructure:"image"` // Container's Image Name - Command string `mapstructure:"command"` // The Command/Entrypoint to run when the container starts up - Args string `mapstructure:"args"` // The arguments to the Command/Entrypoint - NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, net and none - PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and the ports exposed on the container - Privileged bool `mapstructure:"privileged"` // Flag to run the container in priviledged mode - DNS string `mapstructure:"dns_server"` // DNS Server for containers - SearchDomains string `mapstructure:"search_domains"` // DNS Search domains for containers - Hostname string `mapstructure:"hostname"` // Hostname for containers - Labels []map[string]string `mapstructure:"labels"` // Labels to set when the container starts up - UserName string `mapstructure:"auth_username"` // user name of the registry - Password string `mapstructure:"auth_password"` // password to access the registry - Email string `mapstructure:"auth_email"` // email address of the user who is allowed to access the registry - ServerAddress string `mapstructure:"auth_server_address"` // server address of the registry + ImageName string `mapstructure:"image"` // Container's Image Name + Command string `mapstructure:"command"` // The Command/Entrypoint to run when the container starts up + Args string `mapstructure:"args"` // The arguments to the Command/Entrypoint + NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, net and none + PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and the ports exposed on the container + Privileged bool `mapstructure:"privileged"` // Flag to run the container in priviledged mode + DNS string `mapstructure:"dns_server"` // DNS Server for containers + SearchDomains string `mapstructure:"search_domains"` // DNS Search domains for containers + Hostname string `mapstructure:"hostname"` // Hostname for containers + Labels []map[string]string `mapstructure:"labels"` // Labels to set when the container starts up + Auth []DockerDriverAuth `mapstructure:"auth"` // Authentication credentials for a private Docker registry } func (c *DockerDriverConfig) Validate() error { @@ -380,16 +384,16 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle Tag: tag, } - authOptions := docker.AuthConfiguration{ - Username: driverConfig.UserName, - Password: driverConfig.Password, - Email: driverConfig.Email, - ServerAddress: driverConfig.ServerAddress, + authOptions := docker.AuthConfiguration{} + if len(driverConfig.Auth) != 0 { + authOptions = docker.AuthConfiguration{ + Username: driverConfig.Auth[0].Username, + Password: driverConfig.Auth[0].Password, + Email: driverConfig.Auth[0].Email, + ServerAddress: driverConfig.Auth[0].ServerAddress, + } } - d.logger.Printf("[DEBUG] TASKCONFIG: %v", task.Config) - d.logger.Printf("[DEBUG] DRIVERCONFIG: %v", driverConfig) - d.logger.Printf("[DEBUG] AUTH: %v", authOptions) err = client.PullImage(pullOptions, authOptions) if err != nil { d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s", repo, tag, err) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 255437c5d..a2c2d58a2 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -57,10 +57,12 @@ following authentication parameters. These options can provide access to private repositories that utilize the docker remote api (e.g. dockerhub, quay.io) -* `auth_username` - (Optional) The account username. -* `auth_password` - (Optional) The account password. -* `auth_email` - (Optional) The account email. -* `auth_server-address` - (Optional) The server domain/ip without the protocol. +The `auth` object supports the following keys: + +* `username` - (Optional) The account username. +* `password` - (Optional) The account password. +* `email` - (Optional) The account email. +* `server_address` - (Optional) The server domain/ip without the protocol. ### Port Mapping