2019-06-14 03:05:57 +00:00
|
|
|
package allocrunner
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"fmt"
|
|
|
|
|
2019-08-28 14:51:34 +00:00
|
|
|
"github.com/coreos/go-iptables/iptables"
|
|
|
|
hclog "github.com/hashicorp/go-hclog"
|
2019-06-14 03:05:57 +00:00
|
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
|
|
"github.com/hashicorp/nomad/plugins/drivers"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// defaultNomadBridgeName is the name of the bridge to use when not set by
|
|
|
|
// the client
|
|
|
|
defaultNomadBridgeName = "nomad"
|
|
|
|
|
2019-09-04 20:33:25 +00:00
|
|
|
// bridgeNetworkAllocIfPrefix is the prefix that is used for the interface
|
|
|
|
// name created inside of the alloc network which is connected to the bridge
|
|
|
|
bridgeNetworkAllocIfPrefix = "eth"
|
2019-06-14 03:05:57 +00:00
|
|
|
|
|
|
|
// defaultNomadAllocSubnet is the subnet to use for host local ip address
|
|
|
|
// allocation when not specified by the client
|
2019-08-28 14:51:34 +00:00
|
|
|
defaultNomadAllocSubnet = "172.26.64.0/20" // end 172.26.79.255
|
|
|
|
|
|
|
|
// cniAdminChainName is the name of the admin iptables chain used to allow
|
|
|
|
// forwarding traffic to allocations
|
|
|
|
cniAdminChainName = "NOMAD-ADMIN"
|
2019-06-14 03:05:57 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// bridgeNetworkConfigurator is a NetworkConfigurator which adds the alloc to a
|
|
|
|
// shared bridge, configures masquerading for egress traffic and port mapping
|
|
|
|
// for ingress
|
|
|
|
type bridgeNetworkConfigurator struct {
|
2020-05-15 15:09:01 +00:00
|
|
|
cni *cniNetworkConfigurator
|
2019-06-14 03:05:57 +00:00
|
|
|
allocSubnet string
|
|
|
|
bridgeName string
|
2023-02-02 15:12:15 +00:00
|
|
|
hairpinMode bool
|
2019-08-28 14:51:34 +00:00
|
|
|
|
|
|
|
logger hclog.Logger
|
2019-06-14 03:05:57 +00:00
|
|
|
}
|
|
|
|
|
2023-02-02 15:12:15 +00:00
|
|
|
func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange string, hairpinMode bool, cniPath string, ignorePortMappingHostIP bool) (*bridgeNetworkConfigurator, error) {
|
2019-06-14 03:05:57 +00:00
|
|
|
b := &bridgeNetworkConfigurator{
|
|
|
|
bridgeName: bridgeName,
|
|
|
|
allocSubnet: ipRange,
|
2023-02-02 15:12:15 +00:00
|
|
|
hairpinMode: hairpinMode,
|
2019-08-28 14:51:34 +00:00
|
|
|
logger: log,
|
2019-06-14 03:05:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if b.bridgeName == "" {
|
|
|
|
b.bridgeName = defaultNomadBridgeName
|
|
|
|
}
|
|
|
|
|
|
|
|
if b.allocSubnet == "" {
|
|
|
|
b.allocSubnet = defaultNomadAllocSubnet
|
|
|
|
}
|
|
|
|
|
2023-02-02 15:12:15 +00:00
|
|
|
c, err := newCNINetworkConfiguratorWithConf(log, cniPath, bridgeNetworkAllocIfPrefix, ignorePortMappingHostIP, buildNomadBridgeNetConfig(*b))
|
2020-05-15 15:09:01 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
b.cni = c
|
|
|
|
|
2019-09-04 20:33:25 +00:00
|
|
|
return b, nil
|
2019-06-14 03:05:57 +00:00
|
|
|
}
|
|
|
|
|
2019-08-28 14:51:34 +00:00
|
|
|
// ensureForwardingRules ensures that a forwarding rule is added to iptables
|
|
|
|
// to allow traffic inbound to the bridge network
|
|
|
|
func (b *bridgeNetworkConfigurator) ensureForwardingRules() error {
|
|
|
|
ipt, err := iptables.New()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if err = ensureChain(ipt, "filter", cniAdminChainName); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2021-03-13 19:30:22 +00:00
|
|
|
if err := appendChainRule(ipt, cniAdminChainName, b.generateAdminChainRule()); err != nil {
|
2019-08-28 14:51:34 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ensureChain ensures that the given chain exists, creating it if missing
|
|
|
|
func ensureChain(ipt *iptables.IPTables, table, chain string) error {
|
|
|
|
chains, err := ipt.ListChains(table)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to list iptables chains: %v", err)
|
|
|
|
}
|
|
|
|
for _, ch := range chains {
|
|
|
|
if ch == chain {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
err = ipt.NewChain(table, chain)
|
|
|
|
|
|
|
|
// if err is for chain already existing return as it is possible another
|
|
|
|
// goroutine created it first
|
|
|
|
if e, ok := err.(*iptables.Error); ok && e.ExitStatus() == 1 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2021-03-13 19:30:22 +00:00
|
|
|
// appendChainRule adds the given rule to the chain
|
|
|
|
func appendChainRule(ipt *iptables.IPTables, chain string, rule []string) error {
|
2019-08-28 14:51:34 +00:00
|
|
|
exists, err := ipt.Exists("filter", chain, rule...)
|
|
|
|
if !exists && err == nil {
|
2021-03-13 19:30:22 +00:00
|
|
|
err = ipt.Append("filter", chain, rule...)
|
2019-08-28 14:51:34 +00:00
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// generateAdminChainRule builds the iptables rule that is inserted into the
|
|
|
|
// CNI admin chain to ensure traffic forwarding to the bridge network
|
|
|
|
func (b *bridgeNetworkConfigurator) generateAdminChainRule() []string {
|
|
|
|
return []string{"-o", b.bridgeName, "-d", b.allocSubnet, "-j", "ACCEPT"}
|
|
|
|
}
|
|
|
|
|
2019-06-14 03:05:57 +00:00
|
|
|
// Setup calls the CNI plugins with the add action
|
2020-10-12 17:43:04 +00:00
|
|
|
func (b *bridgeNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) (*structs.AllocNetworkStatus, error) {
|
2019-08-28 14:51:34 +00:00
|
|
|
if err := b.ensureForwardingRules(); err != nil {
|
2020-10-12 17:43:04 +00:00
|
|
|
return nil, fmt.Errorf("failed to initialize table forwarding rules: %v", err)
|
2019-08-28 14:51:34 +00:00
|
|
|
}
|
|
|
|
|
2020-05-15 15:09:01 +00:00
|
|
|
return b.cni.Setup(ctx, alloc, spec)
|
2019-06-14 03:05:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Teardown calls the CNI plugins with the delete action
|
2019-09-04 20:33:25 +00:00
|
|
|
func (b *bridgeNetworkConfigurator) Teardown(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
|
2020-05-15 15:09:01 +00:00
|
|
|
return b.cni.Teardown(ctx, alloc, spec)
|
2019-06-14 03:05:57 +00:00
|
|
|
}
|
|
|
|
|
2023-02-02 15:12:15 +00:00
|
|
|
func buildNomadBridgeNetConfig(b bridgeNetworkConfigurator) []byte {
|
|
|
|
return []byte(fmt.Sprintf(nomadCNIConfigTemplate,
|
|
|
|
b.bridgeName,
|
|
|
|
b.hairpinMode,
|
|
|
|
b.allocSubnet,
|
|
|
|
cniAdminChainName))
|
2019-06-14 03:05:57 +00:00
|
|
|
}
|
|
|
|
|
2023-02-07 14:47:41 +00:00
|
|
|
// Update website/content/docs/networking/cni.mdx when the bridge configuration
|
|
|
|
// is modified.
|
2019-06-14 03:05:57 +00:00
|
|
|
const nomadCNIConfigTemplate = `{
|
|
|
|
"cniVersion": "0.4.0",
|
|
|
|
"name": "nomad",
|
|
|
|
"plugins": [
|
2022-06-20 15:22:53 +00:00
|
|
|
{
|
|
|
|
"type": "loopback"
|
|
|
|
},
|
2019-06-14 03:05:57 +00:00
|
|
|
{
|
|
|
|
"type": "bridge",
|
2023-02-02 15:12:15 +00:00
|
|
|
"bridge": %q,
|
2019-06-14 03:05:57 +00:00
|
|
|
"ipMasq": true,
|
2019-08-28 14:51:34 +00:00
|
|
|
"isGateway": true,
|
2019-09-04 20:33:25 +00:00
|
|
|
"forceAddress": true,
|
2023-02-02 15:12:15 +00:00
|
|
|
"hairpinMode": %v,
|
2019-06-14 03:05:57 +00:00
|
|
|
"ipam": {
|
|
|
|
"type": "host-local",
|
|
|
|
"ranges": [
|
|
|
|
[
|
|
|
|
{
|
2023-02-02 15:12:15 +00:00
|
|
|
"subnet": %q
|
2019-06-14 03:05:57 +00:00
|
|
|
}
|
|
|
|
]
|
2019-08-28 14:51:34 +00:00
|
|
|
],
|
|
|
|
"routes": [
|
|
|
|
{ "dst": "0.0.0.0/0" }
|
2019-06-14 03:05:57 +00:00
|
|
|
]
|
|
|
|
}
|
|
|
|
},
|
|
|
|
{
|
2019-08-28 14:51:34 +00:00
|
|
|
"type": "firewall",
|
|
|
|
"backend": "iptables",
|
2023-02-02 15:12:15 +00:00
|
|
|
"iptablesAdminChainName": %q
|
2019-06-14 03:05:57 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
"type": "portmap",
|
2019-08-28 14:51:34 +00:00
|
|
|
"capabilities": {"portMappings": true},
|
|
|
|
"snat": true
|
2019-06-14 03:05:57 +00:00
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
`
|