diff --git a/client/client.go b/client/client.go index 927fe4a0e..d3dd8aaaa 100644 --- a/client/client.go +++ b/client/client.go @@ -643,6 +643,8 @@ func (c *Client) init() error { c.logger.Info("using alloc directory", "alloc_dir", c.config.AllocDir) + c.logger.Info("using dynamic ports", "min", c.config.MinDynamicPort, "max", c.config.MaxDynamicPort) + // Ensure cgroups are created on linux platform if runtime.GOOS == "linux" && c.cpusetManager != nil { err := c.cpusetManager.Init() @@ -1385,6 +1387,8 @@ func (c *Client) setupNode() error { } if node.NodeResources == nil { node.NodeResources = &structs.NodeResources{} + node.NodeResources.MinDynamicPort = c.config.MinDynamicPort + node.NodeResources.MaxDynamicPort = c.config.MaxDynamicPort } if node.ReservedResources == nil { node.ReservedResources = &structs.NodeReservedResources{} @@ -1496,6 +1500,14 @@ func (c *Client) updateNodeFromFingerprint(response *fingerprint.FingerprintResp c.config.Node.NodeResources.Merge(response.NodeResources) nodeHasChanged = true } + + response.NodeResources.MinDynamicPort = c.config.MinDynamicPort + response.NodeResources.MaxDynamicPort = c.config.MaxDynamicPort + if c.config.Node.NodeResources.MinDynamicPort != response.NodeResources.MinDynamicPort || + c.config.Node.NodeResources.MaxDynamicPort != response.NodeResources.MaxDynamicPort { + nodeHasChanged = true + } + } if nodeHasChanged { diff --git a/client/config/config.go b/client/config/config.go index e9c5e0fa5..13af0ca2d 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -137,6 +137,12 @@ type Config struct { // communicating with plugin subsystems over loopback ClientMinPort uint + // MaxDynamicPort is the largest dynamic port generated + MaxDynamicPort int + + // MinDynamicPort is the smallest dynamic port generated + MinDynamicPort int + // A mapping of directories on the host OS to attempt to embed inside each // task's chroot. ChrootEnv map[string]string diff --git a/command/agent/agent.go b/command/agent/agent.go index a500dea7f..46de635b2 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -589,6 +589,8 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) { } conf.ClientMaxPort = uint(agentConfig.Client.ClientMaxPort) conf.ClientMinPort = uint(agentConfig.Client.ClientMinPort) + conf.MaxDynamicPort = agentConfig.Client.MaxDynamicPort + conf.MinDynamicPort = agentConfig.Client.MinDynamicPort conf.DisableRemoteExec = agentConfig.Client.DisableRemoteExec if agentConfig.Client.TemplateConfig.FunctionBlacklist != nil { conf.TemplateConfig.FunctionDenylist = agentConfig.Client.TemplateConfig.FunctionBlacklist diff --git a/command/agent/command.go b/command/agent/command.go index 1e753cf27..0fbe1b95c 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -372,6 +372,12 @@ func (c *Command) isValidConfig(config, cmdConfig *Config) bool { return false } + if config.Client.MinDynamicPort > 0 && config.Client.MaxDynamicPort > 0 && + config.Client.MinDynamicPort >= config.Client.MaxDynamicPort { + c.Ui.Error("Invalid dynamic port range") + return false + } + if !config.DevMode { // Ensure that we have the directories we need to run. if config.Server.Enabled && config.DataDir == "" { diff --git a/command/agent/config.go b/command/agent/config.go index 687b3f037..0ec25534f 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -233,6 +233,14 @@ type ClientConfig struct { // communicating with plugin subsystems ClientMinPort int `hcl:"client_min_port"` + // MaxDynamicPort is the upper range of the dynamic ports that the client + // uses for allocations + MaxDynamicPort int `hcl:"max_dynamic_port"` + + // MinDynamicPort is the lower range of the dynamic ports that the client + // uses for allocations + MinDynamicPort int `hcl:"min_dynamic_port"` + // Reserved is used to reserve resources from being used by Nomad. This can // be used to target a certain utilization or to prevent Nomad from using a // particular set of ports. @@ -917,6 +925,8 @@ func DefaultConfig() *Config { MaxKillTimeout: "30s", ClientMinPort: 14000, ClientMaxPort: 14512, + MinDynamicPort: 20000, + MaxDynamicPort: 32000, Reserved: &Resources{}, GCInterval: 1 * time.Minute, GCParallelDestroys: 2, @@ -1598,6 +1608,12 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig { if b.ClientMinPort != 0 { result.ClientMinPort = b.ClientMinPort } + if b.MaxDynamicPort != 0 { + result.MaxDynamicPort = b.MaxDynamicPort + } + if b.MinDynamicPort != 0 { + result.MinDynamicPort = b.MinDynamicPort + } if result.Reserved == nil && b.Reserved != nil { reserved := *b.Reserved result.Reserved = &reserved diff --git a/nomad/structs/network.go b/nomad/structs/network.go index f7e69039d..f597ff712 100644 --- a/nomad/structs/network.go +++ b/nomad/structs/network.go @@ -8,12 +8,6 @@ import ( ) const ( - // MinDynamicPort is the smallest dynamic port generated - MinDynamicPort = 20000 - - // MaxDynamicPort is the largest dynamic port generated - MaxDynamicPort = 32000 - // maxRandPortAttempts is the maximum number of attempt // to assign a random port maxRandPortAttempts = 20 @@ -39,6 +33,9 @@ type NetworkIndex struct { AvailBandwidth map[string]int // Bandwidth by device UsedPorts map[string]Bitmap // Ports by IP UsedBandwidth map[string]int // Bandwidth by device + + MinDynamicPort int // The smallest dynamic port generated + MaxDynamicPort int // The largest dynamic port generated } // NewNetworkIndex is used to construct a new network index @@ -48,6 +45,8 @@ func NewNetworkIndex() *NetworkIndex { AvailBandwidth: make(map[string]int), UsedPorts: make(map[string]Bitmap), UsedBandwidth: make(map[string]int), + MinDynamicPort: 20000, + MaxDynamicPort: 32000, } } @@ -136,6 +135,14 @@ func (idx *NetworkIndex) SetNode(node *Node) (collide bool) { } } + if node.NodeResources != nil && node.NodeResources.MinDynamicPort > 0 { + idx.MinDynamicPort = node.NodeResources.MinDynamicPort + } + + if node.NodeResources != nil && node.NodeResources.MaxDynamicPort > 0 { + idx.MaxDynamicPort = node.NodeResources.MaxDynamicPort + } + return } @@ -368,10 +375,10 @@ func (idx *NetworkIndex) AssignPorts(ask *NetworkResource) (AllocatedPorts, erro // lower memory usage. var dynPorts []int // TODO: its more efficient to find multiple dynamic ports at once - dynPorts, addrErr = getDynamicPortsStochastic(used, reservedIdx[port.HostNetwork], 1) + dynPorts, addrErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1) if addrErr != nil { // Fall back to the precise method if the random sampling failed. - dynPorts, addrErr = getDynamicPortsPrecise(used, reservedIdx[port.HostNetwork], 1) + dynPorts, addrErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1) if addrErr != nil { continue } @@ -450,13 +457,13 @@ func (idx *NetworkIndex) AssignNetwork(ask *NetworkResource) (out *NetworkResour // lower memory usage. var dynPorts []int var dynErr error - dynPorts, dynErr = getDynamicPortsStochastic(used, ask.ReservedPorts, len(ask.DynamicPorts)) + dynPorts, dynErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts)) if dynErr == nil { goto BUILD_OFFER } // Fall back to the precise method if the random sampling failed. - dynPorts, dynErr = getDynamicPortsPrecise(used, ask.ReservedPorts, len(ask.DynamicPorts)) + dynPorts, dynErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts)) if dynErr != nil { err = dynErr return @@ -485,7 +492,7 @@ func (idx *NetworkIndex) AssignNetwork(ask *NetworkResource) (out *NetworkResour // no ports have been allocated yet, the network ask and returns a set of unused // ports to fulfil the ask's DynamicPorts or an error if it failed. An error // means the ask can not be satisfied as the method does a precise search. -func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int, error) { +func getDynamicPortsPrecise(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reserved []Port, numDyn int) ([]int, error) { // Create a copy of the used ports and apply the new reserves var usedSet Bitmap var err error @@ -506,7 +513,7 @@ func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int } // Get the indexes of the unset - availablePorts := usedSet.IndexesInRange(false, MinDynamicPort, MaxDynamicPort) + availablePorts := usedSet.IndexesInRange(false, uint(minDynamicPort), uint(maxDynamicPort)) // Randomize the amount we need if len(availablePorts) < numDyn { @@ -527,7 +534,7 @@ func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int // ports to fulfil the ask's DynamicPorts or an error if it failed. An error // does not mean the ask can not be satisfied as the method has a fixed amount // of random probes and if these fail, the search is aborted. -func getDynamicPortsStochastic(nodeUsed Bitmap, reservedPorts []Port, count int) ([]int, error) { +func getDynamicPortsStochastic(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reservedPorts []Port, count int) ([]int, error) { var reserved, dynamic []int for _, port := range reservedPorts { reserved = append(reserved, port.Value) @@ -541,7 +548,7 @@ func getDynamicPortsStochastic(nodeUsed Bitmap, reservedPorts []Port, count int) return nil, fmt.Errorf("stochastic dynamic port selection failed") } - randPort := MinDynamicPort + rand.Intn(MaxDynamicPort-MinDynamicPort) + randPort := minDynamicPort + rand.Intn(maxDynamicPort-minDynamicPort) if nodeUsed != nil && nodeUsed.Check(uint(randPort)) { goto PICK } diff --git a/nomad/structs/network_test.go b/nomad/structs/network_test.go index ea80f1169..e7b0a52be 100644 --- a/nomad/structs/network_test.go +++ b/nomad/structs/network_test.go @@ -323,7 +323,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) { }, ReservedResources: &NodeReservedResources{ Networks: NodeReservedNetworkResources{ - ReservedHostPorts: fmt.Sprintf("%d-%d", MinDynamicPort, MaxDynamicPort-1), + ReservedHostPorts: fmt.Sprintf("%d-%d", idx.MinDynamicPort, idx.MaxDynamicPort-1), }, }, } @@ -346,8 +346,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) { if len(offer.DynamicPorts) != 1 { t.Fatalf("There should be one dynamic ports") } - if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort { - t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort) + if p := offer.DynamicPorts[0].Value; p != idx.MaxDynamicPort { + t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, idx.MaxDynamicPort) } } @@ -646,7 +646,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) { }, }, } - for i := MinDynamicPort; i < MaxDynamicPort; i++ { + for i := idx.MinDynamicPort; i < idx.MaxDynamicPort; i++ { n.Reserved.Networks[0].ReservedPorts = append(n.Reserved.Networks[0].ReservedPorts, Port{Value: i}) } @@ -669,8 +669,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) { if len(offer.DynamicPorts) != 1 { t.Fatalf("There should be three dynamic ports") } - if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort { - t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort) + if p := offer.DynamicPorts[0].Value; p != idx.MaxDynamicPort { + t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, idx.MaxDynamicPort) } } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index d0753cbe6..dbc258f86 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -2801,6 +2801,9 @@ type NodeResources struct { Networks Networks NodeNetworks []*NodeNetworkResource Devices []*NodeDeviceResource + + MinDynamicPort int + MaxDynamicPort int } func (n *NodeResources) Copy() *NodeResources {