Support configurable dynamic port range

client/client.go

@@ -643,6 +643,8 @@ func (c *Client) init() error {
 
 	c.logger.Info("using alloc directory", "alloc_dir", c.config.AllocDir)
 
+	c.logger.Info("using dynamic ports", "min", c.config.MinDynamicPort, "max", c.config.MaxDynamicPort)
+
 	// Ensure cgroups are created on linux platform
 	if runtime.GOOS == "linux" && c.cpusetManager != nil {
 		err := c.cpusetManager.Init()
@@ -1385,6 +1387,8 @@ func (c *Client) setupNode() error {
 	}
 	if node.NodeResources == nil {
 		node.NodeResources = &structs.NodeResources{}
+		node.NodeResources.MinDynamicPort = c.config.MinDynamicPort
+		node.NodeResources.MaxDynamicPort = c.config.MaxDynamicPort
 	}
 	if node.ReservedResources == nil {
 		node.ReservedResources = &structs.NodeReservedResources{}
@@ -1496,6 +1500,14 @@ func (c *Client) updateNodeFromFingerprint(response *fingerprint.FingerprintResp
 			c.config.Node.NodeResources.Merge(response.NodeResources)
 			nodeHasChanged = true
 		}
+
+		response.NodeResources.MinDynamicPort = c.config.MinDynamicPort
+		response.NodeResources.MaxDynamicPort = c.config.MaxDynamicPort
+		if c.config.Node.NodeResources.MinDynamicPort != response.NodeResources.MinDynamicPort ||
+			c.config.Node.NodeResources.MaxDynamicPort != response.NodeResources.MaxDynamicPort {
+			nodeHasChanged = true
+		}
+
 	}
 
 	if nodeHasChanged {

client/config/config.go

@@ -137,6 +137,12 @@ type Config struct {
 	// communicating with plugin subsystems over loopback
 	ClientMinPort uint
 
+	// MaxDynamicPort is the largest dynamic port generated
+	MaxDynamicPort int
+
+	// MinDynamicPort is the smallest dynamic port generated
+	MinDynamicPort int
+
 	// A mapping of directories on the host OS to attempt to embed inside each
 	// task's chroot.
 	ChrootEnv map[string]string

command/agent/agent.go

@@ -589,6 +589,8 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
 	}
 	conf.ClientMaxPort = uint(agentConfig.Client.ClientMaxPort)
 	conf.ClientMinPort = uint(agentConfig.Client.ClientMinPort)
+	conf.MaxDynamicPort = agentConfig.Client.MaxDynamicPort
+	conf.MinDynamicPort = agentConfig.Client.MinDynamicPort
 	conf.DisableRemoteExec = agentConfig.Client.DisableRemoteExec
 	if agentConfig.Client.TemplateConfig.FunctionBlacklist != nil {
 		conf.TemplateConfig.FunctionDenylist = agentConfig.Client.TemplateConfig.FunctionBlacklist

command/agent/command.go

@@ -372,6 +372,12 @@ func (c *Command) isValidConfig(config, cmdConfig *Config) bool {
 		return false
 	}
 
+	if config.Client.MinDynamicPort > 0 && config.Client.MaxDynamicPort > 0 &&
+		config.Client.MinDynamicPort >= config.Client.MaxDynamicPort {
+		c.Ui.Error("Invalid dynamic port range")
+		return false
+	}
+
 	if !config.DevMode {
 		// Ensure that we have the directories we need to run.
 		if config.Server.Enabled && config.DataDir == "" {

command/agent/config.go

@@ -233,6 +233,14 @@ type ClientConfig struct {
 	// communicating with plugin subsystems
 	ClientMinPort int `hcl:"client_min_port"`
 
+	// MaxDynamicPort is the upper range of the dynamic ports that the client
+	// uses for allocations
+	MaxDynamicPort int `hcl:"max_dynamic_port"`
+
+	// MinDynamicPort is the lower range of the dynamic ports that the client
+	// uses for allocations
+	MinDynamicPort int `hcl:"min_dynamic_port"`
+
 	// Reserved is used to reserve resources from being used by Nomad. This can
 	// be used to target a certain utilization or to prevent Nomad from using a
 	// particular set of ports.
@@ -917,6 +925,8 @@ func DefaultConfig() *Config {
 			MaxKillTimeout:        "30s",
 			ClientMinPort:         14000,
 			ClientMaxPort:         14512,
+			MinDynamicPort:        20000,
+			MaxDynamicPort:        32000,
 			Reserved:              &Resources{},
 			GCInterval:            1 * time.Minute,
 			GCParallelDestroys:    2,
@@ -1598,6 +1608,12 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
 	if b.ClientMinPort != 0 {
 		result.ClientMinPort = b.ClientMinPort
 	}
+	if b.MaxDynamicPort != 0 {
+		result.MaxDynamicPort = b.MaxDynamicPort
+	}
+	if b.MinDynamicPort != 0 {
+		result.MinDynamicPort = b.MinDynamicPort
+	}
 	if result.Reserved == nil && b.Reserved != nil {
 		reserved := *b.Reserved
 		result.Reserved = &reserved

nomad/structs/network.go

@@ -8,12 +8,6 @@ import (
 )
 
 const (
-	// MinDynamicPort is the smallest dynamic port generated
-	MinDynamicPort = 20000
-
-	// MaxDynamicPort is the largest dynamic port generated
-	MaxDynamicPort = 32000
-
 	// maxRandPortAttempts is the maximum number of attempt
 	// to assign a random port
 	maxRandPortAttempts = 20
@@ -39,6 +33,9 @@ type NetworkIndex struct {
 	AvailBandwidth map[string]int                  // Bandwidth by device
 	UsedPorts      map[string]Bitmap               // Ports by IP
 	UsedBandwidth  map[string]int                  // Bandwidth by device
+
+	MinDynamicPort int // The smallest dynamic port generated
+	MaxDynamicPort int // The largest dynamic port generated
 }
 
 // NewNetworkIndex is used to construct a new network index
@@ -48,6 +45,8 @@ func NewNetworkIndex() *NetworkIndex {
 		AvailBandwidth: make(map[string]int),
 		UsedPorts:      make(map[string]Bitmap),
 		UsedBandwidth:  make(map[string]int),
+		MinDynamicPort: 20000,
+		MaxDynamicPort: 32000,
 	}
 }
 
@@ -136,6 +135,14 @@ func (idx *NetworkIndex) SetNode(node *Node) (collide bool) {
 		}
 	}
 
+	if node.NodeResources != nil && node.NodeResources.MinDynamicPort > 0 {
+		idx.MinDynamicPort = node.NodeResources.MinDynamicPort
+	}
+
+	if node.NodeResources != nil && node.NodeResources.MaxDynamicPort > 0 {
+		idx.MaxDynamicPort = node.NodeResources.MaxDynamicPort
+	}
+
 	return
 }
 
@@ -368,10 +375,10 @@ func (idx *NetworkIndex) AssignPorts(ask *NetworkResource) (AllocatedPorts, erro
 			// lower memory usage.
 			var dynPorts []int
 			// TODO: its more efficient to find multiple dynamic ports at once
-			dynPorts, addrErr = getDynamicPortsStochastic(used, reservedIdx[port.HostNetwork], 1)
+			dynPorts, addrErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1)
 			if addrErr != nil {
 				// Fall back to the precise method if the random sampling failed.
-				dynPorts, addrErr = getDynamicPortsPrecise(used, reservedIdx[port.HostNetwork], 1)
+				dynPorts, addrErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1)
 				if addrErr != nil {
 					continue
 				}
@@ -450,13 +457,13 @@ func (idx *NetworkIndex) AssignNetwork(ask *NetworkResource) (out *NetworkResour
 		// lower memory usage.
 		var dynPorts []int
 		var dynErr error
-		dynPorts, dynErr = getDynamicPortsStochastic(used, ask.ReservedPorts, len(ask.DynamicPorts))
+		dynPorts, dynErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts))
 		if dynErr == nil {
 			goto BUILD_OFFER
 		}
 
 		// Fall back to the precise method if the random sampling failed.
-		dynPorts, dynErr = getDynamicPortsPrecise(used, ask.ReservedPorts, len(ask.DynamicPorts))
+		dynPorts, dynErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts))
 		if dynErr != nil {
 			err = dynErr
 			return
@@ -485,7 +492,7 @@ func (idx *NetworkIndex) AssignNetwork(ask *NetworkResource) (out *NetworkResour
 // no ports have been allocated yet, the network ask and returns a set of unused
 // ports to fulfil the ask's DynamicPorts or an error if it failed. An error
 // means the ask can not be satisfied as the method does a precise search.
-func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int, error) {
+func getDynamicPortsPrecise(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reserved []Port, numDyn int) ([]int, error) {
 	// Create a copy of the used ports and apply the new reserves
 	var usedSet Bitmap
 	var err error
@@ -506,7 +513,7 @@ func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int
 	}
 
 	// Get the indexes of the unset
-	availablePorts := usedSet.IndexesInRange(false, MinDynamicPort, MaxDynamicPort)
+	availablePorts := usedSet.IndexesInRange(false, uint(minDynamicPort), uint(maxDynamicPort))
 
 	// Randomize the amount we need
 	if len(availablePorts) < numDyn {
@@ -527,7 +534,7 @@ func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int
 // ports to fulfil the ask's DynamicPorts or an error if it failed. An error
 // does not mean the ask can not be satisfied as the method has a fixed amount
 // of random probes and if these fail, the search is aborted.
-func getDynamicPortsStochastic(nodeUsed Bitmap, reservedPorts []Port, count int) ([]int, error) {
+func getDynamicPortsStochastic(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reservedPorts []Port, count int) ([]int, error) {
 	var reserved, dynamic []int
 	for _, port := range reservedPorts {
 		reserved = append(reserved, port.Value)
@@ -541,7 +548,7 @@ func getDynamicPortsStochastic(nodeUsed Bitmap, reservedPorts []Port, count int)
 			return nil, fmt.Errorf("stochastic dynamic port selection failed")
 		}
 
-		randPort := MinDynamicPort + rand.Intn(MaxDynamicPort-MinDynamicPort)
+		randPort := minDynamicPort + rand.Intn(maxDynamicPort-minDynamicPort)
 		if nodeUsed != nil && nodeUsed.Check(uint(randPort)) {
 			goto PICK
 		}

nomad/structs/network_test.go

@@ -323,7 +323,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {
 		},
 		ReservedResources: &NodeReservedResources{
 			Networks: NodeReservedNetworkResources{
-				ReservedHostPorts: fmt.Sprintf("%d-%d", MinDynamicPort, MaxDynamicPort-1),
+				ReservedHostPorts: fmt.Sprintf("%d-%d", idx.MinDynamicPort, idx.MaxDynamicPort-1),
 			},
 		},
 	}
@@ -346,8 +346,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {
 	if len(offer.DynamicPorts) != 1 {
 		t.Fatalf("There should be one dynamic ports")
 	}
-	if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort {
+	if p := offer.DynamicPorts[0].Value; p != idx.MaxDynamicPort {
-		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
+		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, idx.MaxDynamicPort)
 	}
 }
 
@@ -646,7 +646,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) {
 			},
 		},
 	}
-	for i := MinDynamicPort; i < MaxDynamicPort; i++ {
+	for i := idx.MinDynamicPort; i < idx.MaxDynamicPort; i++ {
 		n.Reserved.Networks[0].ReservedPorts = append(n.Reserved.Networks[0].ReservedPorts, Port{Value: i})
 	}
 
@@ -669,8 +669,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) {
 	if len(offer.DynamicPorts) != 1 {
 		t.Fatalf("There should be three dynamic ports")
 	}
-	if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort {
+	if p := offer.DynamicPorts[0].Value; p != idx.MaxDynamicPort {
-		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
+		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, idx.MaxDynamicPort)
 	}
 }
 

nomad/structs/structs.go

@@ -2801,6 +2801,9 @@ type NodeResources struct {
 	Networks     Networks
 	NodeNetworks []*NodeNetworkResource
 	Devices      []*NodeDeviceResource
+
+	MinDynamicPort int
+	MaxDynamicPort int
 }
 
 func (n *NodeResources) Copy() *NodeResources {