Merge pull request #5817 from hashicorp/b-consul-stanza

fix consul stanza parsing when a configuration directory is used

command/agent/config.go

@@ -1090,9 +1090,15 @@ func (a *ACLConfig) Merge(b *ACLConfig) *ACLConfig {
 	if b.TokenTTL != 0 {
 		result.TokenTTL = b.TokenTTL
 	}
+	if b.TokenTTLHCL != "" {
+		result.TokenTTLHCL = b.TokenTTLHCL
+	}
 	if b.PolicyTTL != 0 {
 		result.PolicyTTL = b.PolicyTTL
 	}
+	if b.PolicyTTLHCL != "" {
+		result.PolicyTTLHCL = b.PolicyTTLHCL
+	}
 	if b.ReplicationToken != "" {
 		result.ReplicationToken = b.ReplicationToken
 	}
@@ -1139,9 +1145,15 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 	if b.HeartbeatGrace != 0 {
 		result.HeartbeatGrace = b.HeartbeatGrace
 	}
+	if b.HeartbeatGraceHCL != "" {
+		result.HeartbeatGraceHCL = b.HeartbeatGraceHCL
+	}
 	if b.MinHeartbeatTTL != 0 {
 		result.MinHeartbeatTTL = b.MinHeartbeatTTL
 	}
+	if b.MinHeartbeatTTLHCL != "" {
+		result.MinHeartbeatTTLHCL = b.MinHeartbeatTTLHCL
+	}
 	if b.MaxHeartbeatsPerSecond != 0.0 {
 		result.MaxHeartbeatsPerSecond = b.MaxHeartbeatsPerSecond
 	}
@@ -1151,6 +1163,9 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 	if b.RetryInterval != 0 {
 		result.RetryInterval = b.RetryInterval
 	}
+	if b.RetryIntervalHCL != "" {
+		result.RetryIntervalHCL = b.RetryIntervalHCL
+	}
 	if b.RejoinAfterLeave {
 		result.RejoinAfterLeave = true
 	}
@@ -1232,6 +1247,9 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
 	if b.GCInterval != 0 {
 		result.GCInterval = b.GCInterval
 	}
+	if b.GCIntervalHCL != "" {
+		result.GCIntervalHCL = b.GCIntervalHCL
+	}
 	if b.GCParallelDestroys != 0 {
 		result.GCParallelDestroys = b.GCParallelDestroys
 	}
@@ -1461,8 +1479,14 @@ func LoadConfig(path string) (*Config, error) {
 		return nil, err
 	}
 
+	defaults := ParseConfigDefault()
+
 	if fi.IsDir() {
-		return LoadConfigDir(path)
+		config, err := LoadConfigDir(path)
+		if err != nil {
+			return nil, err
+		}
+		return defaults.Merge(config), nil
 	}
 
 	cleaned := filepath.Clean(path)
@@ -1470,7 +1494,7 @@ func LoadConfig(path string) (*Config, error) {
 	if err != nil {
 		return nil, fmt.Errorf("Error loading %s: %s", cleaned, err)
 	}
-
+	config = defaults.Merge(config)
 	config.Files = append(config.Files, cleaned)
 	return config, nil
 }

command/agent/config_parse.go

@@ -14,6 +14,15 @@ import (
 	"github.com/hashicorp/nomad/nomad/structs/config"
 )
 
+// ParseConfigDefault returns the configuration base
+func ParseConfigDefault() *Config {
+	return &Config{
+		Consul:    config.DefaultConsulConfig(),
+		Autopilot: config.DefaultAutopilotConfig(),
+		Vault:     config.DefaultVaultConfig(),
+	}
+}
+
 func ParseConfigFile(path string) (*Config, error) {
 	// slurp
 	var buf bytes.Buffer
@@ -36,10 +45,10 @@ func ParseConfigFile(path string) (*Config, error) {
 		Client:    &ClientConfig{ServerJoin: &ServerJoin{}},
 		ACL:       &ACLConfig{},
 		Server:    &ServerConfig{ServerJoin: &ServerJoin{}},
-		Consul:    config.DefaultConsulConfig(),
-		Autopilot: config.DefaultAutopilotConfig(),
+		Consul:    &config.ConsulConfig{},
+		Autopilot: &config.AutopilotConfig{},
 		Telemetry: &Telemetry{},
-		Vault:     config.DefaultVaultConfig(),
+		Vault:     &config.VaultConfig{},
 	}
 
 	err = hcl.Decode(c, buf.String())

command/agent/config_parse_test.go

@@ -289,7 +289,7 @@ var pluginConfig = &Config{
 	Consul:                    nil,
 	Vault:                     nil,
 	TLSConfig:                 nil,
-	HTTPAPIResponseHeaders:    nil,
+	HTTPAPIResponseHeaders:    map[string]string{},
 	Sentinel:                  nil,
 	Plugins: []*config.PluginConfig{
 		{
@@ -355,7 +355,7 @@ var nonoptConfig = &Config{
 	Consul:                    nil,
 	Vault:                     nil,
 	TLSConfig:                 nil,
-	HTTPAPIResponseHeaders:    nil,
+	HTTPAPIResponseHeaders:    map[string]string{},
 	Sentinel:                  nil,
 }
 
@@ -411,6 +411,9 @@ func TestConfig_Parse(t *testing.T) {
 				t.Fatalf("file: %s\n\n%s", tc.File, err)
 			}
 
+			// Merge defaults like LoadConfig does
+			actual = ParseConfigDefault().Merge(actual)
+
 			//panic(fmt.Sprintf("first: %+v \n second: %+v", actual.TLSConfig, tc.Result.TLSConfig))
 			require.EqualValues(tc.Result, removeHelperAttributes(actual))
 		})
@@ -548,26 +551,11 @@ var sample0 = &Config{
 		Token:          "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
 		ServerAutoJoin: helper.BoolToPtr(false),
 		ClientAutoJoin: helper.BoolToPtr(false),
-		// Defaults
-		ServerServiceName:   "nomad",
-		ServerHTTPCheckName: "Nomad Server HTTP Check",
-		ServerSerfCheckName: "Nomad Server Serf Check",
-		ServerRPCCheckName:  "Nomad Server RPC Check",
-		ClientServiceName:   "nomad-client",
-		ClientHTTPCheckName: "Nomad Client HTTP Check",
-		AutoAdvertise:       helper.BoolToPtr(true),
-		ChecksUseAdvertise:  helper.BoolToPtr(false),
-		Timeout:             5 * time.Second,
-		EnableSSL:           helper.BoolToPtr(false),
-		VerifySSL:           helper.BoolToPtr(true),
 	},
 	Vault: &config.VaultConfig{
 		Enabled: helper.BoolToPtr(true),
 		Role:    "nomad-cluster",
 		Addr:    "http://host.example.com:8200",
-		// Defaults
-		AllowUnauthenticated: helper.BoolToPtr(true),
-		ConnectionRetryIntv:  30 * time.Second,
 	},
 	TLSConfig: &config.TLSConfig{
 		EnableHTTP:           true,
@@ -579,15 +567,43 @@ var sample0 = &Config{
 	},
 	Autopilot: &config.AutopilotConfig{
 		CleanupDeadServers: helper.BoolToPtr(true),
-		// Defaults
-		ServerStabilizationTime: 10 * time.Second,
-		LastContactThreshold:    200 * time.Millisecond,
-		MaxTrailingLogs:         250,
 	},
 }
 
 func TestConfig_ParseSample0(t *testing.T) {
 	c, err := ParseConfigFile("./testdata/sample0.json")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.EqualValues(t, sample0, c)
 }
+
+func TestConfig_ParseDir(t *testing.T) {
+	c, err := LoadConfig("./testdata/sample1")
+	require.NoError(t, err)
+
+	// Defaults
+	sample1 := ParseConfigDefault().Merge(sample0)
+
+	// Merge makes empty maps & slices rather than nil, so set those for the expected
+	// value
+	require.Nil(t, sample1.Client.Options)
+	sample1.Client.Options = map[string]string{}
+	require.Nil(t, sample1.Client.Meta)
+	sample1.Client.Meta = map[string]string{}
+	require.Nil(t, sample1.Client.ChrootEnv)
+	sample1.Client.ChrootEnv = map[string]string{}
+	require.Nil(t, sample1.Server.StartJoin)
+	sample1.Server.StartJoin = []string{}
+
+	// This value added to the config file
+	sample1.Consul.EnableSSL = helper.BoolToPtr(true)
+
+	// LoadDir listed the config files
+	require.Nil(t, sample1.Files)
+	sample1.Files = []string{
+		"testdata/sample1/sample0.json",
+		"testdata/sample1/sample1.json",
+		"testdata/sample1/sample2.hcl",
+	}
+
+	require.EqualValues(t, sample1, c)
+}

command/agent/testdata/sample1/sample0.json

@@ -0,0 +1,38 @@
+{
+  "acl": {
+    "enabled": true
+  },
+  "bind_addr": "0.0.0.0",
+  "consul": {
+    "ssl": true,
+    "server_auto_join": false,
+    "client_auto_join": false,
+    "token": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
+  },
+  "data_dir": "/opt/data/nomad/data",
+  "datacenter": "dc1",
+  "enable_syslog": true,
+  "leave_on_interrupt": true,
+  "leave_on_terminate": true,
+  "log_level": "INFO",
+  "region": "global",
+  "server": {
+    "bootstrap_expect": 3,
+    "enabled": true,
+    "encrypt": "sHck3WL6cxuhuY7Mso9BHA==",
+    "retry_join": [
+      "10.0.0.101",
+      "10.0.0.102",
+      "10.0.0.103"
+    ]
+  },
+  "syslog_facility": "LOCAL0",
+  "tls": {
+    "ca_file": "/opt/data/nomad/certs/nomad-ca.pem",
+    "cert_file": "/opt/data/nomad/certs/server.pem",
+    "http": true,
+  },
+  "vault": {
+    "address": "http://host.example.com:8200",
+  }
+}

command/agent/testdata/sample1/sample1.json

@@ -0,0 +1,38 @@
+{
+  "acl": {
+    "enabled": true
+  },
+  "advertise": {
+    "http": "host.example.com",
+    "rpc": "host.example.com",
+    "serf": "host.example.com"
+  },
+  "bind_addr": "0.0.0.0",
+  "data_dir": "/opt/data/nomad/data",
+  "datacenter": "dc1",
+  "enable_syslog": true,
+  "leave_on_interrupt": true,
+  "leave_on_terminate": true,
+  "log_level": "INFO",
+  "region": "global",
+  "server": {
+    "bootstrap_expect": 3,
+    "enabled": true,
+  },
+  "syslog_facility": "LOCAL0",
+  "telemetry": {
+    "collection_interval": "60s",
+    "disable_hostname": true,
+    "prometheus_metrics": true,
+    "publish_allocation_metrics": true,
+    "publish_node_metrics": true
+  },
+  "tls": {
+    "key_file": "/opt/data/nomad/certs/server-key.pem",
+    "rpc": true,
+    "verify_server_hostname": true
+  },
+  "vault": {
+    "create_from_role": "nomad-cluster",
+  }
+}

command/agent/testdata/sample1/sample2.hcl

@@ -0,0 +1,19 @@
+"advertise" = {
+  "http" = "host.example.com"
+  "rpc" = "host.example.com"
+  "serf" = "host.example.com"
+}
+
+"autopilot" = {
+  "cleanup_dead_servers" = true
+}
+
+"consul" = {
+  "client_auto_join" = false
+  "server_auto_join" = false
+  "token" = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
+}
+
+vault = {
+  enabled = true
+}

nomad/structs/config/autopilot.go

@@ -61,9 +61,15 @@ func (a *AutopilotConfig) Merge(b *AutopilotConfig) *AutopilotConfig {
 	if b.ServerStabilizationTime != 0 {
 		result.ServerStabilizationTime = b.ServerStabilizationTime
 	}
+	if b.ServerStabilizationTimeHCL != "" {
+		result.ServerStabilizationTimeHCL = b.ServerStabilizationTimeHCL
+	}
 	if b.LastContactThreshold != 0 {
 		result.LastContactThreshold = b.LastContactThreshold
 	}
+	if b.LastContactThresholdHCL != "" {
+		result.LastContactThresholdHCL = b.LastContactThresholdHCL
+	}
 	if b.MaxTrailingLogs != 0 {
 		result.MaxTrailingLogs = b.MaxTrailingLogs
 	}

nomad/structs/config/consul.go

@@ -145,6 +145,9 @@ func (a *ConsulConfig) Merge(b *ConsulConfig) *ConsulConfig {
 	if b.Timeout != 0 {
 		result.Timeout = b.Timeout
 	}
+	if b.TimeoutHCL != "" {
+		result.TimeoutHCL = b.TimeoutHCL
+	}
 	if b.Token != "" {
 		result.Token = b.Token
 	}