luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

ar: plumb client config for networking into the network hook

This commit is contained in:
Nick Ethier 2019-06-13 23:05:57 -04:00
parent af66a35924
commit ef83f0831b
No known key found for this signature in database
GPG Key ID: 07C1A3ECED90D24A
13 changed files with 271 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/allocrunner/alloc_runner.go
View File

@ -185,7 +185,7 @@ func NewAllocRunner(config *Config) (*allocRunner, error) {
ar.allocDir = allocdir.NewAllocDir(ar.logger, filepath.Join(config.ClientConfig.AllocDir, alloc.ID))
// Initialize the runners hooks.
if err := ar.initRunnerHooks(); err != nil {
if err := ar.initRunnerHooks(config.ClientConfig); err != nil {
return nil, err
}

8
client/allocrunner/alloc_runner_hooks.go
View File

@ -6,6 +6,7 @@ import (
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/client/allocrunner/interfaces"
clientconfig "github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/drivers"
)
@ -94,7 +95,7 @@ func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents
}
// initRunnerHooks intializes the runners hooks.
func (ar *allocRunner) initRunnerHooks() error {
func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error {
hookLogger := ar.logger.Named("runner_hook")
// create health setting shim
@ -109,6 +110,9 @@ func (ar *allocRunner) initRunnerHooks() error {
return fmt.Errorf("failed to configure network manager: %v", err)
}
// create network configurator
nc := newNetworkConfigurator(ar.Alloc(), config)
// Create the alloc directory hook. This is run first to ensure the
// directory path exists for other hooks.
ar.runnerHooks = []interfaces.RunnerHook{
@ -116,7 +120,7 @@ func (ar *allocRunner) initRunnerHooks() error {
newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher),
newDiskMigrationHook(hookLogger, ar.prevAllocMigrator, ar.allocDir),
newAllocHealthWatcherHook(hookLogger, ar.Alloc(), hs, ar.Listener(), ar.consulClient),
newNetworkHook(hookLogger, ns, ar.Alloc(), nm),
newNetworkHook(hookLogger, ns, ar.Alloc(), nm, nc),
}
return nil

30
client/allocrunner/network_hook.go
View File

@ -26,15 +26,22 @@ type networkHook struct {
// spec described the network namespace and is syncronized by specLock
spec *drivers.NetworkIsolationSpec
// networkConfigurator configures the network interfaces, routes, etc once
// the alloc network has been created
networkConfigurator NetworkConfigurator
logger hclog.Logger
}
func newNetworkHook(logger hclog.Logger, ns networkIsolationSetter, alloc *structs.Allocation, netManager drivers.DriverNetworkManager) *networkHook {
func newNetworkHook(logger hclog.Logger, ns networkIsolationSetter,
alloc *structs.Allocation, netManager drivers.DriverNetworkManager,
netConfigurator NetworkConfigurator) *networkHook {
return &networkHook{
setter: ns,
alloc: alloc,
manager: netManager,
logger: logger,
setter: ns,
alloc: alloc,
manager: netManager,
networkConfigurator: netConfigurator,
logger: logger,
}
}
@ -43,15 +50,16 @@ func (h *networkHook) Name() string {
}
func (h *networkHook) Prerun() error {
if h.manager == nil {
h.logger.Trace("shared network namespaces are not supported on this platform, skipping network hook")
return nil
}
tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup)
if len(tg.Networks) == 0 || tg.Networks[0].Mode == "host" || tg.Networks[0].Mode == "" {
return nil
}
if h.manager == nil || h.networkConfigurator == nil {
h.logger.Trace("shared network namespaces are not supported on this platform, skipping network hook")
return nil
}
spec, err := h.manager.CreateNetwork(h.alloc.ID)
if err != nil {
return fmt.Errorf("failed to create network for alloc: %v", err)
@ -62,7 +70,7 @@ func (h *networkHook) Prerun() error {
h.setter.SetNetworkIsolation(spec)
}
if err := ConfigureNetworking(h.alloc, spec); err != nil {
if err := h.networkConfigurator.Setup(h.alloc, spec); err != nil {
return fmt.Errorf("failed to configure networking for alloc: %v", err)
}
return nil
@ -73,7 +81,7 @@ func (h *networkHook) Postrun() error {
return nil
}
if err := CleanupNetworking(h.alloc, h.spec); err != nil {
if err := h.networkConfigurator.Teardown(h.alloc, h.spec); err != nil {
h.logger.Error("failed to cleanup network for allocation, resources may have leaked", "alloc", h.alloc.ID, "error", err)
}
return h.manager.DestroyNetwork(h.alloc.ID, h.spec)

1
client/allocrunner/network_hook_linux.go Normal file
View File

@ -0,0 +1 @@
package allocrunner

4
client/allocrunner/network_hook_test.go
View File

@ -65,7 +65,7 @@ func TestNetworkHook_Prerun_Postrun(t *testing.T) {
require := require.New(t)
logger := testlog.HCLogger(t)
hook := newNetworkHook(logger, setter, alloc, nm)
hook := newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{})
require.NoError(hook.Prerun())
require.True(setter.called)
require.False(destroyCalled)
@ -76,7 +76,7 @@ func TestNetworkHook_Prerun_Postrun(t *testing.T) {
setter.called = false
destroyCalled = false
alloc.Job.TaskGroups[0].Networks[0].Mode = "host"
hook = newNetworkHook(logger, setter, alloc, nm)
hook = newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{})
require.NoError(hook.Prerun())
require.False(setter.called)
require.False(destroyCalled)

41
client/allocrunner/network_manager_linux.go
View File

@ -1,9 +1,11 @@
package allocrunner
import (
"context"
"fmt"
"strings"
clientconfig "github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/lib/nsutil"
"github.com/hashicorp/nomad/client/pluginmanager/drivermanager"
"github.com/hashicorp/nomad/nomad/structs"
@ -119,37 +121,12 @@ func netModeToIsolationMode(netMode string) drivers.NetIsolationMode {
}
}
func getPortMapping(alloc *structs.Allocation) []*nsutil.PortMapping {
ports := []*nsutil.PortMapping{}
for _, network := range alloc.AllocatedResources.Shared.Networks {
for _, port := range append(network.DynamicPorts, network.ReservedPorts...) {
for _, proto := range []string{"tcp", "udp"} {
ports = append(ports, &nsutil.PortMapping{
Host: port.Value,
Container: port.To,
Proto: proto,
})
}
}
func newNetworkConfigurator(alloc *structs.Allocation, config *clientconfig.Config) NetworkConfigurator {
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
switch strings.ToLower(tg.Networks[0].Mode) {
case "bridge":
return newBridgeNetworkConfigurator(context.Background(), config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.CNIPath)
default:
return &hostNetworkConfigurator{}
}
return ports
}
func ConfigureNetworking(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
// TODO: CNI support
if err := nsutil.SetupBridgeNetworking(alloc.ID, spec.Path, getPortMapping(alloc)); err != nil {
return err
}
return nil
}
func CleanupNetworking(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
if err := nsutil.TeardownBridgeNetworking(alloc.ID, spec.Path, getPortMapping(alloc)); err != nil {
return err
}
return nil
}

5
client/allocrunner/network_manager_nonlinux.go
View File

@ -3,6 +3,7 @@
package allocrunner
import (
clientconfig "github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/pluginmanager/drivermanager"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/drivers"
@ -12,3 +13,7 @@ import (
func newNetworkManager(alloc *structs.Allocation, driverManager drivermanager.Manager) (nm drivers.DriverNetworkManager, err error) {
return nil, nil
}
func newNetworkConfigurator(alloc *structs.Allocation, config *clientconfig.Config) NetworkConfigurator {
return &hostNetworkConfigurator{}
}

25
client/allocrunner/networking.go Normal file
View File

@ -0,0 +1,25 @@
package allocrunner
import (
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/drivers"
)
// NetworkConfigurator sets up and tears down the interfaces, routes, firewall
// rules, etc for the configured networking mode of the allocation.
type NetworkConfigurator interface {
Setup(*structs.Allocation, *drivers.NetworkIsolationSpec) error
Teardown(*structs.Allocation, *drivers.NetworkIsolationSpec) error
}
// hostNetworkConfigurator is a noop implementation of a NetworkConfigurator for
// when the alloc join's a client host's network namespace and thus does not
// require further configuration
type hostNetworkConfigurator struct{}
func (h *hostNetworkConfigurator) Setup(*structs.Allocation, *drivers.NetworkIsolationSpec) error {
return nil
}
func (h *hostNetworkConfigurator) Teardown(*structs.Allocation, *drivers.NetworkIsolationSpec) error {
return nil
}

172
client/allocrunner/networking_bridge_linux.go Normal file
View File

@ -0,0 +1,172 @@
package allocrunner
import (
"context"
"fmt"
"os"
"path/filepath"
"github.com/containernetworking/cni/libcni"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/drivers"
)
const (
// envCNIPath is the environment variable name to use to derive the CNI path
// when it is not explicitly set by the client
envCNIPath = "CNI_PATH"
// defaultCNIPath is the CNI path to use when it is not set by the client
// and is not set by environment variable
defaultCNIPath = "/opt/cni/bin"
// defaultNomadBridgeName is the name of the bridge to use when not set by
// the client
defaultNomadBridgeName = "nomad"
// bridgeNetworkAllocIfName is the name that is set for the interface created
// inside of the alloc network which is connected to the bridge
bridgeNetworkContainerIfName = "eth0"
// defaultNomadAllocSubnet is the subnet to use for host local ip address
// allocation when not specified by the client
defaultNomadAllocSubnet = "172.26.66.0/23"
)
// bridgeNetworkConfigurator is a NetworkConfigurator which adds the alloc to a
// shared bridge, configures masquerading for egress traffic and port mapping
// for ingress
type bridgeNetworkConfigurator struct {
ctx context.Context
cniConfig *libcni.CNIConfig
allocSubnet string
bridgeName string
}
func newBridgeNetworkConfigurator(ctx context.Context, bridgeName, ipRange, cniPath string) *bridgeNetworkConfigurator {
b := &bridgeNetworkConfigurator{
ctx: ctx,
bridgeName: bridgeName,
allocSubnet: ipRange,
}
if cniPath == "" {
if cniPath = os.Getenv(envCNIPath); cniPath == "" {
cniPath = defaultCNIPath
}
}
b.cniConfig = libcni.NewCNIConfig(filepath.SplitList(cniPath), nil)
if b.bridgeName == "" {
b.bridgeName = defaultNomadBridgeName
}
if b.allocSubnet == "" {
b.allocSubnet = defaultNomadAllocSubnet
}
return b
}
// Setup calls the CNI plugins with the add action
func (b *bridgeNetworkConfigurator) Setup(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
netconf, err := b.buildNomadNetConfig()
if err != nil {
return err
}
result, err := b.cniConfig.AddNetworkList(b.ctx, netconf, b.runtimeConf(alloc, spec))
if result != nil {
result.Print()
}
return err
}
// Teardown calls the CNI plugins with the delete action
func (b *bridgeNetworkConfigurator) Teardown(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
netconf, err := b.buildNomadNetConfig()
if err != nil {
return err
}
err = b.cniConfig.DelNetworkList(b.ctx, netconf, b.runtimeConf(alloc, spec))
return err
}
// getPortMapping builds a list of portMapping structs that are used as the
// portmapping capability arguments for the portmap CNI plugin
func getPortMapping(alloc *structs.Allocation) []*portMapping {
ports := []*portMapping{}
for _, network := range alloc.AllocatedResources.Shared.Networks {
for _, port := range append(network.DynamicPorts, network.ReservedPorts...) {
for _, proto := range []string{"tcp", "udp"} {
ports = append(ports, &portMapping{
Host: port.Value,
Container: port.To,
Proto: proto,
})
}
}
}
return ports
}
// portMapping is the json representation of the portmapping capability arguments
// for the portmap CNI plugin
type portMapping struct {
Host int `json:"hostPort"`
Container int `json:"containerPort"`
Proto string `json:"protocol"`
}
// runtimeConf builds the configuration needed by CNI to locate the target netns
func (b *bridgeNetworkConfigurator) runtimeConf(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) *libcni.RuntimeConf {
return &libcni.RuntimeConf{
ContainerID: fmt.Sprintf("nomad-%s", alloc.ID[:8]),
NetNS: spec.Path,
IfName: bridgeNetworkContainerIfName,
CapabilityArgs: map[string]interface{}{
"portMappings": getPortMapping(alloc),
},
}
}
// buildNomadNetConfig generates the CNI network configuration for the bridge
// networking mode
func (b *bridgeNetworkConfigurator) buildNomadNetConfig() (*libcni.NetworkConfigList, error) {
rendered := fmt.Sprintf(nomadCNIConfigTemplate, b.bridgeName, b.allocSubnet)
return libcni.ConfListFromBytes([]byte(rendered))
}
const nomadCNIConfigTemplate = `{
"cniVersion": "0.4.0",
"name": "nomad",
"plugins": [
{
"type": "bridge",
"bridge": "%s",
"isDefaultGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"ranges": [
[
{
"subnet": "%s"
}
]
]
}
},
{
"type": "firewall"
},
{
"type": "portmap",
"capabilities": {"portMappings": true}
}
]
}
`

13
client/config/config.go
View File

@ -221,6 +221,19 @@ type Config struct {
// StateDBFactory is used to override stateDB implementations,
StateDBFactory state.NewStateDBFunc
// CNIPath is the path used to search for CNI plugins. Multiple paths can
// be specified with colon delimited
CNIPath string
// BridgeNetworkName is the name to use for the bridge created in bridge
// networking mode. This defaults to 'nomad' if not set
BridgeNetworkName string
// BridgeNetworkAllocSubnet is the IP subnet to use for address allocation
// for allocations in bridge networking mode. Subnet must be in CIDR
// notation
BridgeNetworkAllocSubnet string
}
func (c *Config) Copy() *Config {

97
client/lib/nsutil/bridge_linux.go
View File

@ -1,97 +0,0 @@
package nsutil
import (
"context"
"fmt"
"os"
"path/filepath"
"github.com/containernetworking/cni/libcni"
)
const (
EnvCNIPath = "CNI_PATH"
)
type PortMapping struct {
Host int `json:"hostPort"`
Container int `json:"containerPort"`
Proto string `json:"protocol"`
}
func SetupBridgeNetworking(allocID string, nsPath string, portMappings []*PortMapping) error {
netconf, err := libcni.ConfListFromBytes([]byte(nomadCNIConfig))
if err != nil {
return err
}
containerID := fmt.Sprintf("nomad-%s", allocID[:8])
cninet := libcni.NewCNIConfig(filepath.SplitList(os.Getenv(EnvCNIPath)), nil)
rt := &libcni.RuntimeConf{
ContainerID: containerID,
NetNS: nsPath,
IfName: "eth0",
CapabilityArgs: map[string]interface{}{
"portMappings": portMappings,
},
}
result, err := cninet.AddNetworkList(context.TODO(), netconf, rt)
if result != nil {
result.Print()
}
return err
}
func TeardownBridgeNetworking(allocID, nsPath string, portMappings []*PortMapping) error {
netconf, err := libcni.ConfListFromBytes([]byte(nomadCNIConfig))
if err != nil {
return err
}
containerID := fmt.Sprintf("nomad-%s", allocID[:8])
cninet := libcni.NewCNIConfig(filepath.SplitList(os.Getenv(EnvCNIPath)), nil)
rt := &libcni.RuntimeConf{
ContainerID: containerID,
NetNS: nsPath,
IfName: "eth0",
CapabilityArgs: map[string]interface{}{
"portMappings": portMappings,
},
}
err = cninet.DelNetworkList(context.TODO(), netconf, rt)
return err
}
const nomadCNIConfig = `{
"cniVersion": "0.4.0",
"name": "nomad",
"plugins": [
{
"type": "bridge",
"bridge": "nomad",
"isDefaultGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"ranges": [
[
{
"subnet": "172.26.66.0/23"
}
]
]
}
},
{
"type": "firewall"
},
{
"type": "portmap",
"capabilities": {"portMappings": true}
}
]
}
`

5
command/agent/agent.go
View File

@ -538,6 +538,11 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
conf.ACLTokenTTL = agentConfig.ACL.TokenTTL
conf.ACLPolicyTTL = agentConfig.ACL.PolicyTTL
// Setup networking configration
conf.CNIPath = agentConfig.Client.CNIPath
conf.BridgeNetworkName = agentConfig.Client.BridgeNetworkName
conf.BridgeNetworkAllocSubnet = agentConfig.Client.BridgeNetworkSubnet
return conf, nil
}

13
command/agent/config.go
View File

@ -247,6 +247,19 @@ type ClientConfig struct {
// ExtraKeysHCL is used by hcl to surface unexpected keys
ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
// CNIPath is the path to search for CNI plugins, multiple paths can be
// specified colon delimited
CNIPath string `hcl:"cni_path"`
// BridgeNetworkName is the name of the bridge to create when using the
// bridge network mode
BridgeNetworkName string `hcl:"bridge_network_name"`
// BridgeNetworkSubnet is the subnet to allocate IP addresses from when
// creating allocations with bridge networking mode. This range is local to
// the host
BridgeNetworkSubnet string `hcl:"bridge_network_subnet"`
}
// ACLConfig is configuration specific to the ACL system