Support customizing full scheduler config

command/agent/agent.go

@@ -323,10 +323,8 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 	}
 
 	// handle system scheduler preemption default
-	if agentConfig.Server.SystemSchedulerPreemptionEnabledDefault != nil {
-		conf.SystemSchedulerPreemptionEnabledDefault = *agentConfig.Server.SystemSchedulerPreemptionEnabledDefault
-	} else {
-		conf.SystemSchedulerPreemptionEnabledDefault = true
+	if agentConfig.Server.DefaultSchedulerConfig != nil {
+		conf.DefaultSchedulerConfig = *agentConfig.Server.DefaultSchedulerConfig
 	}
 
 	// Add the Consul and Vault configs

command/agent/agent_test.go

@@ -1,7 +1,6 @@
 package agent
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -277,18 +276,51 @@ func TestAgent_ServerConfig(t *testing.T) {
 
 func TestAgent_ServerConfig_SchedulerFlags(t *testing.T) {
 	cases := []struct {
-		input    *bool
-		expected bool
+		name     string
+		input    *structs.SchedulerConfiguration
+		expected structs.SchedulerConfiguration
 	}{
-		{nil, true},
-		{helper.BoolToPtr(false), false},
-		{helper.BoolToPtr(true), true},
+		{
+			"default case",
+			nil,
+			structs.SchedulerConfiguration{
+				PreemptionConfig: structs.PreemptionConfig{
+					SystemSchedulerEnabled: true,
+				},
+			},
+		},
+		{
+			"empty value: preemption is disabled",
+			&structs.SchedulerConfiguration{},
+			structs.SchedulerConfiguration{
+				PreemptionConfig: structs.PreemptionConfig{
+					SystemSchedulerEnabled: false,
+				},
+			},
+		},
+		{
+			"all explicitly set",
+			&structs.SchedulerConfiguration{
+				PreemptionConfig: structs.PreemptionConfig{
+					SystemSchedulerEnabled:  true,
+					BatchSchedulerEnabled:   true,
+					ServiceSchedulerEnabled: true,
+				},
+			},
+			structs.SchedulerConfiguration{
+				PreemptionConfig: structs.PreemptionConfig{
+					SystemSchedulerEnabled:  true,
+					BatchSchedulerEnabled:   true,
+					ServiceSchedulerEnabled: true,
+				},
+			},
+		},
 	}
 
 	for _, c := range cases {
-		t.Run(fmt.Sprintf("case: %v", c.input), func(t *testing.T) {
+		t.Run(c.name, func(t *testing.T) {
 			conf := DefaultConfig()
-			conf.Server.SystemSchedulerPreemptionEnabledDefault = c.input
+			conf.Server.DefaultSchedulerConfig = c.input
 
 			a := &Agent{config: conf}
 			conf.AdvertiseAddrs.Serf = "127.0.0.1:4000"
@@ -299,7 +331,7 @@ func TestAgent_ServerConfig_SchedulerFlags(t *testing.T) {
 
 			out, err := a.serverConfig()
 			require.NoError(t, err)
-			require.Equal(t, c.expected, out.SystemSchedulerPreemptionEnabledDefault)
+			require.Equal(t, c.expected, out.DefaultSchedulerConfig)
 		})
 	}
 }

command/agent/config.go

@@ -444,8 +444,10 @@ type ServerConfig struct {
 	// ServerJoin contains information that is used to attempt to join servers
 	ServerJoin *ServerJoin `hcl:"server_join"`
 
-	// SystemSchedulerPreemptionEnabledDefault is used to determin whether to enable system preemption by default in a new cluster
-	SystemSchedulerPreemptionEnabledDefault *bool `hcl:"system_scheduler_preemption_enabled_default"`
+	// DefaultSchedulerConfig configures the initial scheduler config to be persisted in Raft.
+	// Once the cluster is bootstrapped, and Raft persists the config (from here or through API),
+	// This value is ignored.
+	DefaultSchedulerConfig *structs.SchedulerConfiguration `hcl:"default_scheduler_config"`
 
 	// ExtraKeysHCL is used by hcl to surface unexpected keys
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
@@ -1340,6 +1342,11 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 		result.ServerJoin = result.ServerJoin.Merge(b.ServerJoin)
 	}
 
+	if b.DefaultSchedulerConfig != nil {
+		c := *b.DefaultSchedulerConfig
+		result.DefaultSchedulerConfig = &c
+	}
+
 	// Add the schedulers
 	result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)
 

nomad/config.go

@@ -305,8 +305,10 @@ type Config struct {
 	// dead servers.
 	AutopilotInterval time.Duration
 
-	// SystemSchedulerPreemptionEnabledDefault is used to determin whether to enable system preemption by default in a new cluster
-	SystemSchedulerPreemptionEnabledDefault bool
+	// DefaultSchedulerConfig configures the initial scheduler config to be persisted in Raft.
+	// Once the cluster is bootstrapped, and Raft persists the config (from here or through API),
+	// This value is ignored.
+	DefaultSchedulerConfig structs.SchedulerConfiguration `hcl:"default_scheduler_config"`
 
 	// PluginLoader is used to load plugins.
 	PluginLoader loader.PluginCatalog
@@ -380,9 +382,15 @@ func DefaultConfig() *Config {
 			MaxTrailingLogs:         250,
 			ServerStabilizationTime: 10 * time.Second,
 		},
-		ServerHealthInterval:                    2 * time.Second,
-		AutopilotInterval:                       10 * time.Second,
-		SystemSchedulerPreemptionEnabledDefault: true,
+		ServerHealthInterval: 2 * time.Second,
+		AutopilotInterval:    10 * time.Second,
+		DefaultSchedulerConfig: structs.SchedulerConfiguration{
+			PreemptionConfig: structs.PreemptionConfig{
+				SystemSchedulerEnabled:  true,
+				BatchSchedulerEnabled:   false,
+				ServiceSchedulerEnabled: false,
+			},
+		},
 	}
 
 	// Enable all known schedulers by default

nomad/leader.go

@@ -44,17 +44,6 @@ var minAutopilotVersion = version.Must(version.NewVersion("0.8.0"))
 
 var minSchedulerConfigVersion = version.Must(version.NewVersion("0.9.0"))
 
-// Default configuration for scheduler with preemption enabled for system jobs
-func (s *Server) defaultSchedulerConfig() structs.SchedulerConfiguration {
-	return structs.SchedulerConfiguration{
-		PreemptionConfig: structs.PreemptionConfig{
-			SystemSchedulerEnabled:  s.config.SystemSchedulerPreemptionEnabledDefault,
-			BatchSchedulerEnabled:   false,
-			ServiceSchedulerEnabled: false,
-		},
-	}
-}
-
 // monitorLeadership is used to monitor if we acquire or lose our role
 // as the leader in the Raft cluster. There is some work the leader is
 // expected to do, so we must react to changes
@@ -1321,7 +1310,7 @@ func (s *Server) getOrCreateSchedulerConfig() *structs.SchedulerConfiguration {
 		return nil
 	}
 
-	req := structs.SchedulerSetConfigRequest{Config: s.defaultSchedulerConfig()}
+	req := structs.SchedulerSetConfigRequest{Config: s.config.DefaultSchedulerConfig}
 	if _, _, err = s.raftApply(structs.SchedulerConfigRequestType, req); err != nil {
 		s.logger.Named("core").Error("failed to initialize config", "error", err)
 		return nil

nomad/structs/operator.go

@@ -124,7 +124,7 @@ type AutopilotConfig struct {
 type SchedulerConfiguration struct {
 	// PreemptionConfig specifies whether to enable eviction of lower
 	// priority jobs to place higher priority jobs.
-	PreemptionConfig PreemptionConfig
+	PreemptionConfig PreemptionConfig `hcl:"preemption_config"`
 
 	// CreateIndex/ModifyIndex store the create/modify indexes of this configuration.
 	CreateIndex uint64
@@ -152,13 +152,13 @@ type SchedulerSetConfigurationResponse struct {
 // PreemptionConfig specifies whether preemption is enabled based on scheduler type
 type PreemptionConfig struct {
 	// SystemSchedulerEnabled specifies if preemption is enabled for system jobs
-	SystemSchedulerEnabled bool
+	SystemSchedulerEnabled bool `hcl:"system_scheduler_enabled"`
 
 	// BatchSchedulerEnabled specifies if preemption is enabled for batch jobs
-	BatchSchedulerEnabled bool
+	BatchSchedulerEnabled bool `hcl:"batch_scheduler_enabled"`
 
 	// ServiceSchedulerEnabled specifies if preemption is enabled for service jobs
-	ServiceSchedulerEnabled bool
+	ServiceSchedulerEnabled bool `hcl:"service_Scheduler_enabled"`
 }
 
 // SchedulerSetConfigRequest is used by the Operator endpoint to update the