luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Support configurable dynamic port range

This commit is contained in:
Aleksandr Zagaevskiy 2021-09-10 11:52:47 +03:00
parent 7e100cc682
commit ebb87e65fe
8 changed files with 72 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
client/client.go
View File

@ -643,6 +643,8 @@ func (c *Client) init() error {
c.logger.Info("using alloc directory", "alloc_dir", c.config.AllocDir)
c.logger.Info("using dynamic ports", "min", c.config.MinDynamicPort, "max", c.config.MaxDynamicPort)
// Ensure cgroups are created on linux platform
if runtime.GOOS == "linux" && c.cpusetManager != nil {
err := c.cpusetManager.Init()
@ -1385,6 +1387,8 @@ func (c *Client) setupNode() error {
}
if node.NodeResources == nil {
node.NodeResources = &structs.NodeResources{}
node.NodeResources.MinDynamicPort = c.config.MinDynamicPort
node.NodeResources.MaxDynamicPort = c.config.MaxDynamicPort
}
if node.ReservedResources == nil {
node.ReservedResources = &structs.NodeReservedResources{}
@ -1496,6 +1500,14 @@ func (c *Client) updateNodeFromFingerprint(response *fingerprint.FingerprintResp
c.config.Node.NodeResources.Merge(response.NodeResources)
nodeHasChanged = true
}
response.NodeResources.MinDynamicPort = c.config.MinDynamicPort
response.NodeResources.MaxDynamicPort = c.config.MaxDynamicPort
if c.config.Node.NodeResources.MinDynamicPort != response.NodeResources.MinDynamicPort ||
c.config.Node.NodeResources.MaxDynamicPort != response.NodeResources.MaxDynamicPort {
nodeHasChanged = true
}
}
if nodeHasChanged {

6
client/config/config.go
View File

@ -137,6 +137,12 @@ type Config struct {
// communicating with plugin subsystems over loopback
ClientMinPort uint
// MaxDynamicPort is the largest dynamic port generated
MaxDynamicPort int
// MinDynamicPort is the smallest dynamic port generated
MinDynamicPort int
// A mapping of directories on the host OS to attempt to embed inside each
// task's chroot.
ChrootEnv map[string]string

2
command/agent/agent.go
View File

@ -589,6 +589,8 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
}
conf.ClientMaxPort = uint(agentConfig.Client.ClientMaxPort)
conf.ClientMinPort = uint(agentConfig.Client.ClientMinPort)
conf.MaxDynamicPort = agentConfig.Client.MaxDynamicPort
conf.MinDynamicPort = agentConfig.Client.MinDynamicPort
conf.DisableRemoteExec = agentConfig.Client.DisableRemoteExec
if agentConfig.Client.TemplateConfig.FunctionBlacklist != nil {
conf.TemplateConfig.FunctionDenylist = agentConfig.Client.TemplateConfig.FunctionBlacklist

6
command/agent/command.go
View File

@ -372,6 +372,12 @@ func (c *Command) isValidConfig(config, cmdConfig *Config) bool {
return false
}
if config.Client.MinDynamicPort > 0 && config.Client.MaxDynamicPort > 0 &&
config.Client.MinDynamicPort >= config.Client.MaxDynamicPort {
c.Ui.Error("Invalid dynamic port range")
return false
}
if !config.DevMode {
// Ensure that we have the directories we need to run.
if config.Server.Enabled && config.DataDir == "" {

16
command/agent/config.go
View File

@ -233,6 +233,14 @@ type ClientConfig struct {
// communicating with plugin subsystems
ClientMinPort int `hcl:"client_min_port"`
// MaxDynamicPort is the upper range of the dynamic ports that the client
// uses for allocations
MaxDynamicPort int `hcl:"max_dynamic_port"`
// MinDynamicPort is the lower range of the dynamic ports that the client
// uses for allocations
MinDynamicPort int `hcl:"min_dynamic_port"`
// Reserved is used to reserve resources from being used by Nomad. This can
// be used to target a certain utilization or to prevent Nomad from using a
// particular set of ports.
@ -917,6 +925,8 @@ func DefaultConfig() *Config {
MaxKillTimeout: "30s",
ClientMinPort: 14000,
ClientMaxPort: 14512,
MinDynamicPort: 20000,
MaxDynamicPort: 32000,
Reserved: &Resources{},
GCInterval: 1 * time.Minute,
GCParallelDestroys: 2,
@ -1598,6 +1608,12 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
if b.ClientMinPort != 0 {
result.ClientMinPort = b.ClientMinPort
}
if b.MaxDynamicPort != 0 {
result.MaxDynamicPort = b.MaxDynamicPort
}
if b.MinDynamicPort != 0 {
result.MinDynamicPort = b.MinDynamicPort
}
if result.Reserved == nil && b.Reserved != nil {
reserved := *b.Reserved
result.Reserved = &reserved

35
nomad/structs/network.go
View File

@ -8,12 +8,6 @@ import (
)
const (
// MinDynamicPort is the smallest dynamic port generated
MinDynamicPort = 20000
// MaxDynamicPort is the largest dynamic port generated
MaxDynamicPort = 32000
// maxRandPortAttempts is the maximum number of attempt
// to assign a random port
maxRandPortAttempts = 20
@ -39,6 +33,9 @@ type NetworkIndex struct {
AvailBandwidth map[string]int // Bandwidth by device
UsedPorts map[string]Bitmap // Ports by IP
UsedBandwidth map[string]int // Bandwidth by device
MinDynamicPort int // The smallest dynamic port generated
MaxDynamicPort int // The largest dynamic port generated
}
// NewNetworkIndex is used to construct a new network index
@ -48,6 +45,8 @@ func NewNetworkIndex() *NetworkIndex {
AvailBandwidth: make(map[string]int),
UsedPorts: make(map[string]Bitmap),
UsedBandwidth: make(map[string]int),
MinDynamicPort: 20000,
MaxDynamicPort: 32000,
}
}
@ -136,6 +135,14 @@ func (idx *NetworkIndex) SetNode(node *Node) (collide bool) {
}
}
if node.NodeResources != nil && node.NodeResources.MinDynamicPort > 0 {
idx.MinDynamicPort = node.NodeResources.MinDynamicPort
}
if node.NodeResources != nil && node.NodeResources.MaxDynamicPort > 0 {
idx.MaxDynamicPort = node.NodeResources.MaxDynamicPort
}
return
}
@ -368,10 +375,10 @@ func (idx *NetworkIndex) AssignPorts(ask *NetworkResource) (AllocatedPorts, erro
// lower memory usage.
var dynPorts []int
// TODO: its more efficient to find multiple dynamic ports at once
dynPorts, addrErr = getDynamicPortsStochastic(used, reservedIdx[port.HostNetwork], 1)
dynPorts, addrErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1)
if addrErr != nil {
// Fall back to the precise method if the random sampling failed.
dynPorts, addrErr = getDynamicPortsPrecise(used, reservedIdx[port.HostNetwork], 1)
dynPorts, addrErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1)
if addrErr != nil {
continue
}
@ -450,13 +457,13 @@ func (idx *NetworkIndex) AssignNetwork(ask *NetworkResource) (out *NetworkResour
// lower memory usage.
var dynPorts []int
var dynErr error
dynPorts, dynErr = getDynamicPortsStochastic(used, ask.ReservedPorts, len(ask.DynamicPorts))
dynPorts, dynErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts))
if dynErr == nil {
goto BUILD_OFFER
}
// Fall back to the precise method if the random sampling failed.
dynPorts, dynErr = getDynamicPortsPrecise(used, ask.ReservedPorts, len(ask.DynamicPorts))
dynPorts, dynErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts))
if dynErr != nil {
err = dynErr
return
@ -485,7 +492,7 @@ func (idx *NetworkIndex) AssignNetwork(ask *NetworkResource) (out *NetworkResour
// no ports have been allocated yet, the network ask and returns a set of unused
// ports to fulfil the ask's DynamicPorts or an error if it failed. An error
// means the ask can not be satisfied as the method does a precise search.
func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int, error) {
func getDynamicPortsPrecise(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reserved []Port, numDyn int) ([]int, error) {
// Create a copy of the used ports and apply the new reserves
var usedSet Bitmap
var err error
@ -506,7 +513,7 @@ func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int
}
// Get the indexes of the unset
availablePorts := usedSet.IndexesInRange(false, MinDynamicPort, MaxDynamicPort)
availablePorts := usedSet.IndexesInRange(false, uint(minDynamicPort), uint(maxDynamicPort))
// Randomize the amount we need
if len(availablePorts) < numDyn {
@ -527,7 +534,7 @@ func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int
// ports to fulfil the ask's DynamicPorts or an error if it failed. An error
// does not mean the ask can not be satisfied as the method has a fixed amount
// of random probes and if these fail, the search is aborted.
func getDynamicPortsStochastic(nodeUsed Bitmap, reservedPorts []Port, count int) ([]int, error) {
func getDynamicPortsStochastic(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reservedPorts []Port, count int) ([]int, error) {
var reserved, dynamic []int
for _, port := range reservedPorts {
reserved = append(reserved, port.Value)
@ -541,7 +548,7 @@ func getDynamicPortsStochastic(nodeUsed Bitmap, reservedPorts []Port, count int)
return nil, fmt.Errorf("stochastic dynamic port selection failed")
}
randPort := MinDynamicPort + rand.Intn(MaxDynamicPort-MinDynamicPort)
randPort := minDynamicPort + rand.Intn(maxDynamicPort-minDynamicPort)
if nodeUsed != nil && nodeUsed.Check(uint(randPort)) {
goto PICK
}

12
nomad/structs/network_test.go
View File

@ -323,7 +323,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {
},
ReservedResources: &NodeReservedResources{
Networks: NodeReservedNetworkResources{
ReservedHostPorts: fmt.Sprintf("%d-%d", MinDynamicPort, MaxDynamicPort-1),
ReservedHostPorts: fmt.Sprintf("%d-%d", idx.MinDynamicPort, idx.MaxDynamicPort-1),
},
},
}
@ -346,8 +346,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {
if len(offer.DynamicPorts) != 1 {
t.Fatalf("There should be one dynamic ports")
}
if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort {
t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
if p := offer.DynamicPorts[0].Value; p != idx.MaxDynamicPort {
t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, idx.MaxDynamicPort)
}
}
@ -646,7 +646,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) {
},
},
}
for i := MinDynamicPort; i < MaxDynamicPort; i++ {
for i := idx.MinDynamicPort; i < idx.MaxDynamicPort; i++ {
n.Reserved.Networks[0].ReservedPorts = append(n.Reserved.Networks[0].ReservedPorts, Port{Value: i})
}
@ -669,8 +669,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) {
if len(offer.DynamicPorts) != 1 {
t.Fatalf("There should be three dynamic ports")
}
if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort {
t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
if p := offer.DynamicPorts[0].Value; p != idx.MaxDynamicPort {
t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, idx.MaxDynamicPort)
}
}

3
nomad/structs/structs.go
View File

@ -2801,6 +2801,9 @@ type NodeResources struct {
Networks Networks
NodeNetworks []*NodeNetworkResource
Devices []*NodeDeviceResource
MinDynamicPort int
MaxDynamicPort int
}
func (n *NodeResources) Copy() *NodeResources {