diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go index 153999069..38fb488db 100644 --- a/client/allocrunner/taskrunner/task_runner.go +++ b/client/allocrunner/taskrunner/task_runner.go @@ -11,7 +11,6 @@ import ( metrics "github.com/armon/go-metrics" log "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" - "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcldec" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/allocrunner/interfaces" @@ -620,12 +619,7 @@ func (tr *TaskRunner) runDriver() error { tr.logger.Warn("some environment variables not available for rendering", "keys", strings.Join(keys, ", ")) } - evalCtx := &hcl.EvalContext{ - Variables: vars, - Functions: hclutils.GetStdlibFuncs(), - } - - val, diag := hclutils.ParseHclInterface(tr.task.Config, tr.taskSchema, evalCtx) + val, diag := hclutils.ParseHclInterface(tr.task.Config, tr.taskSchema, vars) if diag.HasErrors() { return multierror.Append(errors.New("failed to parse config"), diag.Errs()...) } diff --git a/client/client.go b/client/client.go index 5477971ed..f50e4a0a8 100644 --- a/client/client.go +++ b/client/client.go @@ -1935,6 +1935,8 @@ func (c *Client) runAllocs(update *allocUpdates) { c.logger.Debug("allocation updates", "added", len(diff.added), "removed", len(diff.removed), "updated", len(diff.updated), "ignored", len(diff.ignore)) + errs := 0 + // Remove the old allocations for _, remove := range diff.removed { c.removeAlloc(remove) @@ -1949,6 +1951,7 @@ func (c *Client) runAllocs(update *allocUpdates) { // Make room for new allocations before running if err := c.garbageCollector.MakeRoomFor(diff.added); err != nil { c.logger.Error("error making room for new allocations", "error", err) + errs++ } // Start the new allocations @@ -1956,6 +1959,7 @@ func (c *Client) runAllocs(update *allocUpdates) { migrateToken := update.migrateTokens[add.ID] if err := c.addAlloc(add, migrateToken); err != nil { c.logger.Error("error adding alloc", "error", err, "alloc_id", add.ID) + errs++ // We mark the alloc as failed and send an update to the server // We track the fact that creating an allocrunner failed so that we don't send updates again if add.ClientStatus != structs.AllocClientStatusFailed { @@ -1967,6 +1971,8 @@ func (c *Client) runAllocs(update *allocUpdates) { // Trigger the GC once more now that new allocs are started that could // have caused thresholds to be exceeded c.garbageCollector.Trigger() + c.logger.Debug("allocation updates applied", "added", len(diff.added), "removed", len(diff.removed), + "updated", len(diff.updated), "ignored", len(diff.ignore), "errors", errs) } // makeFailedAlloc creates a stripped down version of the allocation passed in diff --git a/command/agent/consul/client.go b/command/agent/consul/client.go index 006464346..141454d1f 100644 --- a/command/agent/consul/client.go +++ b/command/agent/consul/client.go @@ -533,8 +533,11 @@ func (c *ServiceClient) sync() error { } } - c.logger.Debug("sync complete", "registered_services", sreg, "deregistered_services", sdereg, - "registered_checks", creg, "deregistered_checks", cdereg) + // Only log if something was actually synced + if sreg > 0 || sdereg > 0 || creg > 0 || cdereg > 0 { + c.logger.Debug("sync complete", "registered_services", sreg, "deregistered_services", sdereg, + "registered_checks", creg, "deregistered_checks", cdereg) + } return nil } diff --git a/drivers/docker/config.go b/drivers/docker/config.go index 884d3b753..d2d8923c4 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -253,7 +253,7 @@ var ( "ipv6_address": hclspec.NewAttr("ipv6_address", "string", false), "labels": hclspec.NewBlockAttrs("labels", "string", false), "load": hclspec.NewAttr("load", "string", false), - "logging": hclspec.NewBlockSet("logging", hclspec.NewObject(map[string]*hclspec.Spec{ + "logging": hclspec.NewBlock("logging", false, hclspec.NewObject(map[string]*hclspec.Spec{ "type": hclspec.NewAttr("type", "string", false), "config": hclspec.NewBlockAttrs("config", "string", false), })), diff --git a/helper/pluginutils/hclutils/util.go b/helper/pluginutils/hclutils/util.go index 86a8d2e6c..6e39a5a6c 100644 --- a/helper/pluginutils/hclutils/util.go +++ b/helper/pluginutils/hclutils/util.go @@ -15,8 +15,14 @@ import ( ) // ParseHclInterface is used to convert an interface value representing a hcl2 -// body and return the interpolated value. -func ParseHclInterface(val interface{}, spec hcldec.Spec, ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { +// body and return the interpolated value. Vars may be nil if there are no +// variables to interpolate. +func ParseHclInterface(val interface{}, spec hcldec.Spec, vars map[string]cty.Value) (cty.Value, hcl.Diagnostics) { + evalCtx := &hcl.EvalContext{ + Variables: vars, + Functions: GetStdlibFuncs(), + } + // Encode to json var buf bytes.Buffer enc := codec.NewEncoder(&buf, structs.JsonHandle) @@ -37,7 +43,7 @@ func ParseHclInterface(val interface{}, spec hcldec.Spec, ctx *hcl.EvalContext) return cty.NilVal, diag } - value, decDiag := hcldec.Decode(hclFile.Body, spec, ctx) + value, decDiag := hcldec.Decode(hclFile.Body, spec, evalCtx) diag = diag.Extend(decDiag) if diag.HasErrors() { return cty.NilVal, diag diff --git a/helper/pluginutils/hclutils/util_test.go b/helper/pluginutils/hclutils/util_test.go index bfbb7c0a6..3a6762b99 100644 --- a/helper/pluginutils/hclutils/util_test.go +++ b/helper/pluginutils/hclutils/util_test.go @@ -1,82 +1,23 @@ -package hclutils +package hclutils_test import ( "testing" "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" - hcl2 "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcldec" - "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/drivers/docker" + "github.com/hashicorp/nomad/helper/pluginutils/hclspecutils" + "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" "github.com/kr/pretty" "github.com/mitchellh/mapstructure" "github.com/stretchr/testify/require" "github.com/ugorji/go/codec" "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/gocty" ) -var ( - dockerSpec hcldec.Spec = hcldec.ObjectSpec(map[string]hcldec.Spec{ - "image": &hcldec.AttrSpec{ - Name: "image", - Type: cty.String, - Required: true, - }, - "args": &hcldec.AttrSpec{ - Name: "args", - Type: cty.List(cty.String), - }, - "pids_limit": &hcldec.AttrSpec{ - Name: "pids_limit", - Type: cty.Number, - }, - "port_map": &hcldec.BlockAttrsSpec{ - TypeName: "port_map", - ElementType: cty.String, - }, - - "devices": &hcldec.BlockListSpec{ - TypeName: "devices", - Nested: hcldec.ObjectSpec(map[string]hcldec.Spec{ - "host_path": &hcldec.AttrSpec{ - Name: "host_path", - Type: cty.String, - }, - "container_path": &hcldec.AttrSpec{ - Name: "container_path", - Type: cty.String, - }, - "cgroup_permissions": &hcldec.DefaultSpec{ - Primary: &hcldec.AttrSpec{ - Name: "cgroup_permissions", - Type: cty.String, - }, - Default: &hcldec.LiteralSpec{ - Value: cty.StringVal(""), - }, - }, - }), - }, - }, - ) -) - -type dockerConfig struct { - Image string `cty:"image"` - Args []string `cty:"args"` - PidsLimit *int64 `cty:"pids_limit"` - PortMap map[string]string `cty:"port_map"` - Devices []DockerDevice `cty:"devices"` -} - -type DockerDevice struct { - HostPath string `cty:"host_path"` - ContainerPath string `cty:"container_path"` - CgroupPermissions string `cty:"cgroup_permissions"` -} - func hclConfigToInterface(t *testing.T, config string) interface{} { t.Helper() @@ -121,30 +62,22 @@ func jsonConfigToInterface(t *testing.T, config string) interface{} { } func TestParseHclInterface_Hcl(t *testing.T) { - defaultCtx := &hcl2.EvalContext{ - Functions: GetStdlibFuncs(), - } - variableCtx := &hcl2.EvalContext{ - Functions: GetStdlibFuncs(), - Variables: map[string]cty.Value{ - "NOMAD_ALLOC_INDEX": cty.NumberIntVal(2), - "NOMAD_META_hello": cty.StringVal("world"), - }, - } + dockerDriver := new(docker.Driver) + dockerSpec, err := dockerDriver.TaskConfigSchema() + require.NoError(t, err) + dockerDecSpec, diags := hclspecutils.Convert(dockerSpec) + require.False(t, diags.HasErrors()) - // XXX Useful for determining what cty thinks the type is - //implied, err := gocty.ImpliedType(&dockerConfig{}) - //if err != nil { - //t.Fatalf("implied type failed: %v", err) - //} - - //t.Logf("Implied type: %v", implied.GoString()) + vars := map[string]cty.Value{ + "NOMAD_ALLOC_INDEX": cty.NumberIntVal(2), + "NOMAD_META_hello": cty.StringVal("world"), + } cases := []struct { name string config interface{} spec hcldec.Spec - ctx *hcl2.EvalContext + vars map[string]cty.Value expected interface{} expectedType interface{} }{ @@ -154,13 +87,13 @@ func TestParseHclInterface_Hcl(t *testing.T) { config { image = "redis:3.2" }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - Devices: []DockerDevice{}, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "single string attr json", @@ -170,13 +103,13 @@ func TestParseHclInterface_Hcl(t *testing.T) { "image": "redis:3.2" } }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - Devices: []DockerDevice{}, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "number attr", @@ -185,14 +118,14 @@ func TestParseHclInterface_Hcl(t *testing.T) { image = "redis:3.2" pids_limit = 2 }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - PidsLimit: helper.Int64ToPtr(2), - Devices: []DockerDevice{}, + PidsLimit: 2, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "number attr json", @@ -203,14 +136,14 @@ func TestParseHclInterface_Hcl(t *testing.T) { "pids_limit": "2" } }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - PidsLimit: helper.Int64ToPtr(2), - Devices: []DockerDevice{}, + PidsLimit: 2, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "number attr interpolated", @@ -219,14 +152,14 @@ func TestParseHclInterface_Hcl(t *testing.T) { image = "redis:3.2" pids_limit = "${2 + 2}" }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - PidsLimit: helper.Int64ToPtr(4), - Devices: []DockerDevice{}, + PidsLimit: 4, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "number attr interploated json", @@ -237,14 +170,14 @@ func TestParseHclInterface_Hcl(t *testing.T) { "pids_limit": "${2 + 2}" } }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - PidsLimit: helper.Int64ToPtr(4), - Devices: []DockerDevice{}, + PidsLimit: 4, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "multi attr", @@ -253,14 +186,14 @@ func TestParseHclInterface_Hcl(t *testing.T) { image = "redis:3.2" args = ["foo", "bar"] }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", Args: []string{"foo", "bar"}, - Devices: []DockerDevice{}, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "multi attr json", @@ -271,14 +204,14 @@ func TestParseHclInterface_Hcl(t *testing.T) { "args": ["foo", "bar"] } }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", Args: []string{"foo", "bar"}, - Devices: []DockerDevice{}, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "multi attr variables", @@ -288,15 +221,16 @@ func TestParseHclInterface_Hcl(t *testing.T) { args = ["${NOMAD_META_hello}", "${NOMAD_ALLOC_INDEX}"] pids_limit = "${NOMAD_ALLOC_INDEX + 2}" }`), - spec: dockerSpec, - ctx: variableCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + vars: vars, + expected: &docker.TaskConfig{ Image: "redis:3.2", Args: []string{"world", "2"}, - PidsLimit: helper.Int64ToPtr(4), - Devices: []DockerDevice{}, + PidsLimit: 4, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "multi attr variables json", @@ -307,14 +241,14 @@ func TestParseHclInterface_Hcl(t *testing.T) { "args": ["foo", "bar"] } }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", Args: []string{"foo", "bar"}, - Devices: []DockerDevice{}, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "port_map", @@ -322,21 +256,21 @@ func TestParseHclInterface_Hcl(t *testing.T) { config { image = "redis:3.2" port_map { - foo = "db" - bar = "db2" + foo = 1234 + bar = 5678 } }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - PortMap: map[string]string{ - "foo": "db", - "bar": "db2", + PortMap: map[string]int{ + "foo": 1234, + "bar": 5678, }, - Devices: []DockerDevice{}, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "port_map json", @@ -345,22 +279,22 @@ func TestParseHclInterface_Hcl(t *testing.T) { "Config": { "image": "redis:3.2", "port_map": [{ - "foo": "db", - "bar": "db2" + "foo": 1234, + "bar": 5678 }] } }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - PortMap: map[string]string{ - "foo": "db", - "bar": "db2", + PortMap: map[string]int{ + "foo": 1234, + "bar": 5678, }, - Devices: []DockerDevice{}, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { name: "devices", @@ -379,11 +313,10 @@ func TestParseHclInterface_Hcl(t *testing.T) { } ] }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - Devices: []DockerDevice{ + Devices: []docker.DockerDevice{ { HostPath: "/dev/sda1", ContainerPath: "/dev/xvdc", @@ -394,33 +327,63 @@ func TestParseHclInterface_Hcl(t *testing.T) { ContainerPath: "/dev/xvdd", }, }, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, { - name: "devices json", + name: "docker_logging", + config: hclConfigToInterface(t, ` + config { + image = "redis:3.2" + network_mode = "host" + dns_servers = ["169.254.1.1"] + logging { + type = "syslog" + config { + tag = "driver-test" + } + } + }`), + spec: dockerDecSpec, + expected: &docker.TaskConfig{ + Image: "redis:3.2", + NetworkMode: "host", + DNSServers: []string{"169.254.1.1"}, + Logging: docker.DockerLogging{ + Type: "syslog", + Config: map[string]string{ + "tag": "driver-test", + }, + }, + Devices: []docker.DockerDevice{}, + Mounts: []docker.DockerMount{}, + }, + expectedType: &docker.TaskConfig{}, + }, + { + name: "docker_json", config: jsonConfigToInterface(t, ` - { - "Config": { - "image": "redis:3.2", - "devices": [ - { - "host_path": "/dev/sda1", - "container_path": "/dev/xvdc", - "cgroup_permissions": "r" - }, - { - "host_path": "/dev/sda2", - "container_path": "/dev/xvdd" - } - ] - } - }`), - spec: dockerSpec, - ctx: defaultCtx, - expected: &dockerConfig{ + { + "Config": { + "image": "redis:3.2", + "devices": [ + { + "host_path": "/dev/sda1", + "container_path": "/dev/xvdc", + "cgroup_permissions": "r" + }, + { + "host_path": "/dev/sda2", + "container_path": "/dev/xvdd" + } + ] + } + }`), + spec: dockerDecSpec, + expected: &docker.TaskConfig{ Image: "redis:3.2", - Devices: []DockerDevice{ + Devices: []docker.DockerDevice{ { HostPath: "/dev/sda1", ContainerPath: "/dev/xvdc", @@ -431,16 +394,18 @@ func TestParseHclInterface_Hcl(t *testing.T) { ContainerPath: "/dev/xvdd", }, }, + Mounts: []docker.DockerMount{}, }, - expectedType: &dockerConfig{}, + expectedType: &docker.TaskConfig{}, }, } for _, c := range cases { + c := c t.Run(c.name, func(t *testing.T) { t.Logf("Val: % #v", pretty.Formatter(c.config)) // Parse the interface - ctyValue, diag := ParseHclInterface(c.config, c.spec, c.ctx) + ctyValue, diag := hclutils.ParseHclInterface(c.config, c.spec, c.vars) if diag.HasErrors() { for _, err := range diag.Errs() { t.Error(err) @@ -448,8 +413,12 @@ func TestParseHclInterface_Hcl(t *testing.T) { t.FailNow() } - // Convert cty-value to go structs - require.NoError(t, gocty.FromCtyValue(ctyValue, c.expectedType)) + // Test encoding + taskConfig := &drivers.TaskConfig{} + require.NoError(t, taskConfig.EncodeDriverConfig(ctyValue)) + + // Test decoding + require.NoError(t, taskConfig.DecodeDriverConfig(c.expectedType)) require.EqualValues(t, c.expected, c.expectedType) diff --git a/helper/pluginutils/loader/init.go b/helper/pluginutils/loader/init.go index 774156c8d..2bf714ecc 100644 --- a/helper/pluginutils/loader/init.go +++ b/helper/pluginutils/loader/init.go @@ -10,7 +10,6 @@ import ( multierror "github.com/hashicorp/go-multierror" plugin "github.com/hashicorp/go-plugin" version "github.com/hashicorp/go-version" - hcl2 "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/nomad/helper/pluginutils/hclspecutils" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/nomad/structs/config" @@ -18,14 +17,6 @@ import ( "github.com/zclconf/go-cty/cty/msgpack" ) -var ( - // configParseCtx is the context used to parse a plugin's configuration - // stanza - configParseCtx = &hcl2.EvalContext{ - Functions: hclutils.GetStdlibFuncs(), - } -) - // validateConfig returns whether or not the configuration is valid func validateConfig(config *PluginLoaderConfig) error { var mErr multierror.Error @@ -466,7 +457,7 @@ func (l *PluginLoader) validatePluginConfig(id PluginID, info *pluginInfo) error } // Parse the config using the spec - val, diag := hclutils.ParseHclInterface(info.config, spec, configParseCtx) + val, diag := hclutils.ParseHclInterface(info.config, spec, nil) if diag.HasErrors() { multierror.Append(&mErr, diag.Errs()...) return multierror.Prefix(&mErr, "failed parsing config:") diff --git a/plugins/shared/cmd/launcher/command/device.go b/plugins/shared/cmd/launcher/command/device.go index 5b5d4855c..031137e1e 100644 --- a/plugins/shared/cmd/launcher/command/device.go +++ b/plugins/shared/cmd/launcher/command/device.go @@ -14,7 +14,6 @@ import ( plugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" - hcl2 "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcldec" "github.com/hashicorp/nomad/helper/pluginutils/hclspecutils" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" @@ -197,11 +196,7 @@ func (c *Device) setConfig(spec hcldec.Spec, apiVersion string, config []byte, n c.logger.Trace("raw hcl config", "config", hclog.Fmt("% #v", pretty.Formatter(configVal))) - ctx := &hcl2.EvalContext{ - Functions: hclutils.GetStdlibFuncs(), - } - - val, diag := hclutils.ParseHclInterface(configVal, spec, ctx) + val, diag := hclutils.ParseHclInterface(configVal, spec, nil) if diag.HasErrors() { errStr := "failed to parse config" for _, err := range diag.Errs() {