open-nomad/client/allocrunner/networking_bridge_linux.go

package allocrunner

import (
	"context"
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"time"

	cni "github.com/containerd/go-cni"
	"github.com/coreos/go-iptables/iptables"
	hclog "github.com/hashicorp/go-hclog"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/drivers"
)

const (
	// envCNIPath is the environment variable name to use to derive the CNI path
	// when it is not explicitly set by the client
	envCNIPath = "CNI_PATH"

	// defaultCNIPath is the CNI path to use when it is not set by the client
	// and is not set by environment variable
	defaultCNIPath = "/opt/cni/bin"

	// defaultNomadBridgeName is the name of the bridge to use when not set by
	// the client
	defaultNomadBridgeName = "nomad"

	// bridgeNetworkAllocIfPrefix is the prefix that is used for the interface
	// name created inside of the alloc network which is connected to the bridge
	bridgeNetworkAllocIfPrefix = "eth"

	// defaultNomadAllocSubnet is the subnet to use for host local ip address
	// allocation when not specified by the client
	defaultNomadAllocSubnet = "172.26.64.0/20" // end 172.26.79.255

	// cniAdminChainName is the name of the admin iptables chain used to allow
	// forwarding traffic to allocations
	cniAdminChainName = "NOMAD-ADMIN"
)

// bridgeNetworkConfigurator is a NetworkConfigurator which adds the alloc to a
// shared bridge, configures masquerading for egress traffic and port mapping
// for ingress
type bridgeNetworkConfigurator struct {
	cni         cni.CNI
	allocSubnet string
	bridgeName  string

	rand   *rand.Rand
	logger hclog.Logger
}

func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange, cniPath string) (*bridgeNetworkConfigurator, error) {
	b := &bridgeNetworkConfigurator{
		bridgeName:  bridgeName,
		allocSubnet: ipRange,
		rand:        rand.New(rand.NewSource(time.Now().Unix())),
		logger:      log,
	}
	if cniPath == "" {
		if cniPath = os.Getenv(envCNIPath); cniPath == "" {
			cniPath = defaultCNIPath
		}
	}

	c, err := cni.New(cni.WithPluginDir(filepath.SplitList(cniPath)),
		cni.WithInterfacePrefix(bridgeNetworkAllocIfPrefix))
	if err != nil {
		return nil, err
	}
	b.cni = c

	if b.bridgeName == "" {
		b.bridgeName = defaultNomadBridgeName
	}

	if b.allocSubnet == "" {
		b.allocSubnet = defaultNomadAllocSubnet
	}

	return b, nil
}

// ensureForwardingRules ensures that a forwarding rule is added to iptables
// to allow traffic inbound to the bridge network
func (b *bridgeNetworkConfigurator) ensureForwardingRules() error {
	ipt, err := iptables.New()
	if err != nil {
		return err
	}

	if err = ensureChain(ipt, "filter", cniAdminChainName); err != nil {
		return err
	}

	if err := ensureFirstChainRule(ipt, cniAdminChainName, b.generateAdminChainRule()); err != nil {
		return err
	}

	return nil
}

// ensureChain ensures that the given chain exists, creating it if missing
func ensureChain(ipt *iptables.IPTables, table, chain string) error {
	chains, err := ipt.ListChains(table)
	if err != nil {
		return fmt.Errorf("failed to list iptables chains: %v", err)
	}
	for _, ch := range chains {
		if ch == chain {
			return nil
		}
	}

	err = ipt.NewChain(table, chain)

	// if err is for chain already existing return as it is possible another
	// goroutine created it first
	if e, ok := err.(*iptables.Error); ok && e.ExitStatus() == 1 {
		return nil
	}

	return err
}

// ensureFirstChainRule ensures the given rule exists as the first rule in the chain
func ensureFirstChainRule(ipt *iptables.IPTables, chain string, rule []string) error {
	exists, err := ipt.Exists("filter", chain, rule...)
	if !exists && err == nil {
		// iptables rules are 1-indexed
		err = ipt.Insert("filter", chain, 1, rule...)
	}
	return err
}

// generateAdminChainRule builds the iptables rule that is inserted into the
// CNI admin chain to ensure traffic forwarding to the bridge network
func (b *bridgeNetworkConfigurator) generateAdminChainRule() []string {
	return []string{"-o", b.bridgeName, "-d", b.allocSubnet, "-j", "ACCEPT"}
}

// Setup calls the CNI plugins with the add action
func (b *bridgeNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
	if err := b.ensureForwardingRules(); err != nil {
		return fmt.Errorf("failed to initialize table forwarding rules: %v", err)
	}

	if err := b.ensureCNIInitialized(); err != nil {
		return err
	}

	// Depending on the version of bridge cni plugin (< 0.8.4) a known race could occur
	// where two alloc attempt to create the nomad bridge at the same time, resulting
	// in one of them to fail. This retry attempts to overcome those erroneous failures.
	const retry = 3
	for attempt := 1; ; attempt++ {
		//TODO eventually returning the IP from the result would be nice to have in the alloc
		if _, err := b.cni.Setup(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc))); err != nil {
			b.logger.Warn("failed to configure bridge network", "err", err, "attempt", attempt)
			if attempt == retry {
				return fmt.Errorf("failed to configure bridge network: %v", err)
			}
			// Sleep for 1 second + jitter
			time.Sleep(time.Second + (time.Duration(b.rand.Int63n(1000)) * time.Millisecond))
			continue
		}
		break
	}

	return nil

}

// Teardown calls the CNI plugins with the delete action
func (b *bridgeNetworkConfigurator) Teardown(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
	if err := b.ensureCNIInitialized(); err != nil {
		return err
	}

	return b.cni.Remove(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc)))
}

func (b *bridgeNetworkConfigurator) ensureCNIInitialized() error {
	if err := b.cni.Status(); cni.IsCNINotInitialized(err) {
		return b.cni.Load(cni.WithConfListBytes(b.buildNomadNetConfig()))
	} else {
		return err
	}
}

// getPortMapping builds a list of portMapping structs that are used as the
// portmapping capability arguments for the portmap CNI plugin
func getPortMapping(alloc *structs.Allocation) []cni.PortMapping {
	ports := []cni.PortMapping{}
	for _, network := range alloc.AllocatedResources.Shared.Networks {
		for _, port := range append(network.DynamicPorts, network.ReservedPorts...) {
			if port.To < 1 {
				continue
			}
			for _, proto := range []string{"tcp", "udp"} {
				ports = append(ports, cni.PortMapping{
					HostPort:      int32(port.Value),
					ContainerPort: int32(port.To),
					Protocol:      proto,
				})
			}
		}
	}
	return ports
}

func (b *bridgeNetworkConfigurator) buildNomadNetConfig() []byte {
	return []byte(fmt.Sprintf(nomadCNIConfigTemplate, b.bridgeName, b.allocSubnet, cniAdminChainName))
}

const nomadCNIConfigTemplate = `{
	"cniVersion": "0.4.0",
	"name": "nomad",
	"plugins": [
		{
			"type": "bridge",
			"bridge": "%s",
			"ipMasq": true,
			"isGateway": true,
			"forceAddress": true,
			"ipam": {
				"type": "host-local",
				"ranges": [
					[
						{
							"subnet": "%s"
						}
					]
				],
				"routes": [
					{ "dst": "0.0.0.0/0" }
				]
			}
		},
		{
			"type": "firewall",
			"backend": "iptables",
			"iptablesAdminChainName": "%s"
		},
		{
			"type": "portmap",
			"capabilities": {"portMappings": true},
			"snat": true
		}
	]
}
`