From bff1669ee4cb4f6ad9fe02892658668c229211eb Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 29 Aug 2018 16:26:44 -0700 Subject: [PATCH] Plugin config parsing --- command/agent/config-test-fixtures/basic.hcl | 14 +++ command/agent/config.go | 13 +++ command/agent/config_parse.go | 44 ++++++++ command/agent/config_parse_test.go | 32 ++++-- command/agent/config_test.go | 25 +++++ nomad/structs/config/plugins.go | 75 ++++++++++++++ nomad/structs/config/plugins_test.go | 101 +++++++++++++++++++ 7 files changed, 298 insertions(+), 6 deletions(-) create mode 100644 nomad/structs/config/plugins.go create mode 100644 nomad/structs/config/plugins_test.go diff --git a/command/agent/config-test-fixtures/basic.hcl b/command/agent/config-test-fixtures/basic.hcl index b5a3a77cd..9dab81096 100644 --- a/command/agent/config-test-fixtures/basic.hcl +++ b/command/agent/config-test-fixtures/basic.hcl @@ -190,3 +190,17 @@ autopilot { server_stabilization_time = "23057s" enable_custom_upgrades = true } +plugin "docker" { + args = ["foo", "bar"] + config { + foo = "bar" + nested { + bam = 2 + } + } +} +plugin "exec" { + config { + foo = true + } +} diff --git a/command/agent/config.go b/command/agent/config.go index 8bf1e7aa1..eff01b06e 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -133,6 +133,9 @@ type Config struct { // Autopilot contains the configuration for Autopilot behavior. Autopilot *config.AutopilotConfig `mapstructure:"autopilot"` + + // Plugins is the set of configured plugins + Plugins []*config.PluginConfig `hcl:"plugin,expand"` } // ClientConfig is configuration specific to the client mode @@ -855,6 +858,16 @@ func (c *Config) Merge(b *Config) *Config { result.Autopilot = result.Autopilot.Merge(b.Autopilot) } + if len(result.Plugins) == 0 && len(b.Plugins) != 0 { + copy := make([]*config.PluginConfig, len(b.Plugins)) + for i, v := range b.Plugins { + copy[i] = v.Copy() + } + result.Plugins = copy + } else if len(b.Plugins) != 0 { + result.Plugins = config.PluginConfigSetMerge(result.Plugins, b.Plugins) + } + // Merge config files lists result.Files = append(result.Files, b.Files...) diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index 28a189df5..b0f4be3a8 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -101,6 +101,7 @@ func parseConfig(result *Config, list *ast.ObjectList) error { "acl", "sentinel", "autopilot", + "plugin", } if err := helper.CheckHCLKeys(list, valid); err != nil { return multierror.Prefix(err, "config:") @@ -125,6 +126,7 @@ func parseConfig(result *Config, list *ast.ObjectList) error { delete(m, "acl") delete(m, "sentinel") delete(m, "autopilot") + delete(m, "plugin") // Decode the rest if err := mapstructure.WeakDecode(m, result); err != nil { @@ -215,6 +217,13 @@ func parseConfig(result *Config, list *ast.ObjectList) error { } } + // Parse Plugin configs + if o := list.Filter("plugin"); len(o.Items) > 0 { + if err := parsePlugins(&result.Plugins, o); err != nil { + return multierror.Prefix(err, "plugin->") + } + } + // Parse out http_api_response_headers fields. These are in HCL as a list so // we need to iterate over them and merge them. if headersO := list.Filter("http_api_response_headers"); len(headersO.Items) > 0 { @@ -986,3 +995,38 @@ func parseAutopilot(result **config.AutopilotConfig, list *ast.ObjectList) error *result = autopilotConfig return nil } + +func parsePlugins(result *[]*config.PluginConfig, list *ast.ObjectList) error { + listLen := len(list.Items) + plugins := make([]*config.PluginConfig, listLen) + + // Check for invalid keys + valid := []string{ + "args", + "config", + } + + for i := 0; i < listLen; i++ { + // Get the current plugin object + listVal := list.Items[i] + + if err := helper.CheckHCLKeys(listVal.Val, valid); err != nil { + return fmt.Errorf("invalid keys in plugin config %d: %v", i+1, err) + } + + // Ensure there is a key + if len(listVal.Keys) != 1 { + return fmt.Errorf("plugin config %d doesn't incude a name key", i+1) + } + + var plugin config.PluginConfig + if err := hcl.DecodeObject(&plugin, listVal); err != nil { + return fmt.Errorf("error decoding plugin config %d: %v", i+1, err) + } + + plugins[i] = &plugin + } + + *result = plugins + return nil +} diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index 43638befd..433789f91 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -215,6 +215,26 @@ func TestConfig_Parse(t *testing.T) { DisableUpgradeMigration: &trueValue, EnableCustomUpgrades: &trueValue, }, + Plugins: []*config.PluginConfig{ + { + Name: "docker", + Args: []string{"foo", "bar"}, + Config: map[string]interface{}{ + "foo": "bar", + "nested": []map[string]interface{}{ + { + "bam": 2, + }, + }, + }, + }, + { + Name: "exec", + Config: map[string]interface{}{ + "foo": true, + }, + }, + }, }, false, }, @@ -264,19 +284,19 @@ func TestConfig_Parse(t *testing.T) { SyslogFacility: "", DisableUpdateCheck: nil, DisableAnonymousSignature: false, - Consul: nil, - Vault: nil, - TLSConfig: nil, - HTTPAPIResponseHeaders: nil, - Sentinel: nil, + Consul: nil, + Vault: nil, + TLSConfig: nil, + HTTPAPIResponseHeaders: nil, + Sentinel: nil, }, false, }, } for _, tc := range cases { - require := require.New(t) t.Run(tc.File, func(t *testing.T) { + require := require.New(t) path, err := filepath.Abs(filepath.Join("./config-test-fixtures", tc.File)) if err != nil { t.Fatalf("file: %s\n\n%s", tc.File, err) diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 9df011ba8..7e567ef4b 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -175,6 +175,15 @@ func TestConfig_Merge(t *testing.T) { DisableUpgradeMigration: &falseValue, EnableCustomUpgrades: &falseValue, }, + Plugins: []*config.PluginConfig{ + { + Name: "docker", + Args: []string{"foo"}, + Config: map[string]interface{}{ + "bar": 1, + }, + }, + }, } c3 := &Config{ @@ -341,6 +350,22 @@ func TestConfig_Merge(t *testing.T) { DisableUpgradeMigration: &trueValue, EnableCustomUpgrades: &trueValue, }, + Plugins: []*config.PluginConfig{ + { + Name: "docker", + Args: []string{"bam"}, + Config: map[string]interface{}{ + "baz": 2, + }, + }, + { + Name: "exec", + Args: []string{"arg"}, + Config: map[string]interface{}{ + "config": true, + }, + }, + }, } result := c0.Merge(c1) diff --git a/nomad/structs/config/plugins.go b/nomad/structs/config/plugins.go new file mode 100644 index 000000000..55f3a25ab --- /dev/null +++ b/nomad/structs/config/plugins.go @@ -0,0 +1,75 @@ +package config + +import "github.com/mitchellh/copystructure" + +// PluginConfig is used to configure a plugin explicitly +type PluginConfig struct { + Name string `hcl:",key"` + Args []string `hcl:"args"` + Config map[string]interface{} `hcl:"config"` +} + +func (p *PluginConfig) Merge(o *PluginConfig) *PluginConfig { + m := *p + + if len(o.Name) != 0 { + m.Name = o.Name + } + if len(o.Args) != 0 { + m.Args = o.Args + } + if len(o.Config) != 0 { + m.Config = o.Config + } + + return m.Copy() +} + +func (p *PluginConfig) Copy() *PluginConfig { + c := *p + if i, err := copystructure.Copy(p.Config); err != nil { + panic(err.Error()) + } else { + c.Config = i.(map[string]interface{}) + } + return &c +} + +// PluginConfigSetMerge merges to sets of plugin configs. For plugins with the +// same name, the configs are merged. +func PluginConfigSetMerge(first, second []*PluginConfig) []*PluginConfig { + findex := make(map[string]*PluginConfig, len(first)) + for _, p := range first { + findex[p.Name] = p + } + + sindex := make(map[string]*PluginConfig, len(second)) + for _, p := range second { + sindex[p.Name] = p + } + + var out []*PluginConfig + + // Go through the first set and merge any value that exist in both + for pluginName, original := range findex { + second, ok := sindex[pluginName] + if !ok { + out = append(out, original.Copy()) + continue + } + + out = append(out, original.Merge(second)) + } + + // Go through the second set and add any value that didn't exist in both + for pluginName, plugin := range sindex { + _, ok := findex[pluginName] + if ok { + continue + } + + out = append(out, plugin) + } + + return out +} diff --git a/nomad/structs/config/plugins_test.go b/nomad/structs/config/plugins_test.go new file mode 100644 index 000000000..e0e98d108 --- /dev/null +++ b/nomad/structs/config/plugins_test.go @@ -0,0 +1,101 @@ +package config + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPluginConfig_Merge(t *testing.T) { + t.Parallel() + require := require.New(t) + a := &PluginConfig{ + Name: "foo", + Args: []string{"bar"}, + Config: map[string]interface{}{ + "baz": true, + }, + } + + e1 := &PluginConfig{ + Name: "replaced", + Args: []string{"bar"}, + Config: map[string]interface{}{ + "baz": true, + }, + } + o1 := a.Merge(&PluginConfig{Name: "replaced"}) + require.Equal(e1, o1) + + e2 := &PluginConfig{ + Name: "foo", + Args: []string{"replaced", "another"}, + Config: map[string]interface{}{ + "baz": true, + }, + } + o2 := a.Merge(&PluginConfig{ + Args: []string{"replaced", "another"}, + }) + require.Equal(e2, o2) + + e3 := &PluginConfig{ + Name: "foo", + Args: []string{"bar"}, + Config: map[string]interface{}{ + "replaced": 1, + }, + } + o3 := a.Merge(&PluginConfig{ + Config: map[string]interface{}{ + "replaced": 1, + }, + }) + require.Equal(e3, o3) +} + +func TestPluginConfigSet_Merge(t *testing.T) { + t.Parallel() + require := require.New(t) + + a := &PluginConfig{ + Name: "a", + Args: []string{"a1"}, + Config: map[string]interface{}{ + "a1": true, + }, + } + b1 := &PluginConfig{ + Name: "b", + Args: []string{"b1"}, + Config: map[string]interface{}{ + "b1": true, + }, + } + b2 := &PluginConfig{ + Name: "b", + Args: []string{"b2"}, + Config: map[string]interface{}{ + "b2": true, + }, + } + c := &PluginConfig{ + Name: "c", + Args: []string{"c"}, + Config: map[string]interface{}{ + "c1": true, + }, + } + + s1 := []*PluginConfig{a, b1} + s2 := []*PluginConfig{b2, c} + + out := PluginConfigSetMerge(s1, s2) + sort.Slice(out, func(i, j int) bool { + return out[i].Name < out[j].Name + }) + + expected := []*PluginConfig{a, b2, c} + require.EqualValues(expected, out) +}