185 lines
6.3 KiB
Go
185 lines
6.3 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
//go:build linux
|
|
|
|
package allocrunner
|
|
|
|
import (
|
|
"errors"
|
|
"net"
|
|
"testing"
|
|
|
|
"github.com/containerd/go-cni"
|
|
"github.com/hashicorp/nomad/ci"
|
|
"github.com/hashicorp/nomad/helper/testlog"
|
|
"github.com/shoenig/test/must"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
type mockIPTables struct {
|
|
listCall [2]string
|
|
listRules []string
|
|
listErr error
|
|
|
|
deleteCall [2]string
|
|
deleteErr error
|
|
|
|
clearCall [2]string
|
|
clearErr error
|
|
}
|
|
|
|
func (ipt *mockIPTables) List(table, chain string) ([]string, error) {
|
|
ipt.listCall[0], ipt.listCall[1] = table, chain
|
|
return ipt.listRules, ipt.listErr
|
|
}
|
|
|
|
func (ipt *mockIPTables) Delete(table, chain string, rule ...string) error {
|
|
ipt.deleteCall[0], ipt.deleteCall[1] = table, chain
|
|
return ipt.deleteErr
|
|
}
|
|
|
|
func (ipt *mockIPTables) ClearAndDeleteChain(table, chain string) error {
|
|
ipt.clearCall[0], ipt.clearCall[1] = table, chain
|
|
return ipt.clearErr
|
|
}
|
|
|
|
func (ipt *mockIPTables) assert(t *testing.T, jumpChain string) {
|
|
// List assertions
|
|
must.Eq(t, "nat", ipt.listCall[0])
|
|
must.Eq(t, "POSTROUTING", ipt.listCall[1])
|
|
|
|
// Delete assertions
|
|
must.Eq(t, "nat", ipt.deleteCall[0])
|
|
must.Eq(t, "POSTROUTING", ipt.deleteCall[1])
|
|
|
|
// Clear assertions
|
|
must.Eq(t, "nat", ipt.clearCall[0])
|
|
must.Eq(t, jumpChain, ipt.clearCall[1])
|
|
}
|
|
|
|
func TestCNI_forceCleanup(t *testing.T) {
|
|
t.Run("ok", func(t *testing.T) {
|
|
c := cniNetworkConfigurator{logger: testlog.HCLogger(t)}
|
|
ipt := &mockIPTables{
|
|
listRules: []string{
|
|
`-A POSTROUTING -m comment --comment "CNI portfwd requiring masquerade" -j CNI-HOSTPORT-MASQ`,
|
|
`-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE`,
|
|
`-A POSTROUTING -s 172.26.64.216/32 -m comment --comment "name: \"nomad\" id: \"79e8bf2e-a9c8-70ac-8d4e-fa5c4da99fbf\"" -j CNI-f2338c31d4de44472fe99c43`,
|
|
`-A POSTROUTING -s 172.26.64.217/32 -m comment --comment "name: \"nomad\" id: \"2dd71cac-2b1e-ff08-167c-735f7f9f4964\"" -j CNI-5d36f286cfbb35c5776509ec`,
|
|
`-A POSTROUTING -s 172.26.64.218/32 -m comment --comment "name: \"nomad\" id: \"5ff6deb7-9bc1-1491-f20c-e87b15de501d\"" -j CNI-2fe7686eac2fe43714a7b850`,
|
|
`-A POSTROUTING -m mark --mark 0x2000/0x2000 -j MASQUERADE`,
|
|
`-A POSTROUTING -m comment --comment "CNI portfwd masquerade mark" -j MARK --set-xmark 0x2000/0x2000`,
|
|
},
|
|
}
|
|
err := c.forceCleanup(ipt, "2dd71cac-2b1e-ff08-167c-735f7f9f4964")
|
|
must.NoError(t, err)
|
|
ipt.assert(t, "CNI-5d36f286cfbb35c5776509ec")
|
|
})
|
|
|
|
t.Run("missing allocation", func(t *testing.T) {
|
|
c := cniNetworkConfigurator{logger: testlog.HCLogger(t)}
|
|
ipt := &mockIPTables{
|
|
listRules: []string{
|
|
`-A POSTROUTING -m comment --comment "CNI portfwd requiring masquerade" -j CNI-HOSTPORT-MASQ`,
|
|
`-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE`,
|
|
`-A POSTROUTING -s 172.26.64.216/32 -m comment --comment "name: \"nomad\" id: \"79e8bf2e-a9c8-70ac-8d4e-fa5c4da99fbf\"" -j CNI-f2338c31d4de44472fe99c43`,
|
|
`-A POSTROUTING -s 172.26.64.217/32 -m comment --comment "name: \"nomad\" id: \"262d57a7-8f85-f3a4-9c3b-120c00ccbff1\"" -j CNI-5d36f286cfbb35c5776509ec`,
|
|
`-A POSTROUTING -s 172.26.64.218/32 -m comment --comment "name: \"nomad\" id: \"5ff6deb7-9bc1-1491-f20c-e87b15de501d\"" -j CNI-2fe7686eac2fe43714a7b850`,
|
|
`-A POSTROUTING -m mark --mark 0x2000/0x2000 -j MASQUERADE`,
|
|
`-A POSTROUTING -m comment --comment "CNI portfwd masquerade mark" -j MARK --set-xmark 0x2000/0x2000`,
|
|
},
|
|
}
|
|
err := c.forceCleanup(ipt, "2dd71cac-2b1e-ff08-167c-735f7f9f4964")
|
|
must.EqError(t, err, "failed to find postrouting rule for alloc 2dd71cac-2b1e-ff08-167c-735f7f9f4964")
|
|
})
|
|
|
|
t.Run("list error", func(t *testing.T) {
|
|
c := cniNetworkConfigurator{logger: testlog.HCLogger(t)}
|
|
ipt := &mockIPTables{listErr: errors.New("list error")}
|
|
err := c.forceCleanup(ipt, "2dd71cac-2b1e-ff08-167c-735f7f9f4964")
|
|
must.EqError(t, err, "failed to list iptables rules: list error")
|
|
})
|
|
|
|
t.Run("delete error", func(t *testing.T) {
|
|
c := cniNetworkConfigurator{logger: testlog.HCLogger(t)}
|
|
ipt := &mockIPTables{
|
|
deleteErr: errors.New("delete error"),
|
|
listRules: []string{
|
|
`-A POSTROUTING -s 172.26.64.217/32 -m comment --comment "name: \"nomad\" id: \"2dd71cac-2b1e-ff08-167c-735f7f9f4964\"" -j CNI-5d36f286cfbb35c5776509ec`,
|
|
},
|
|
}
|
|
err := c.forceCleanup(ipt, "2dd71cac-2b1e-ff08-167c-735f7f9f4964")
|
|
must.EqError(t, err, "failed to cleanup iptables rules for alloc 2dd71cac-2b1e-ff08-167c-735f7f9f4964")
|
|
})
|
|
|
|
t.Run("clear error", func(t *testing.T) {
|
|
c := cniNetworkConfigurator{logger: testlog.HCLogger(t)}
|
|
ipt := &mockIPTables{
|
|
clearErr: errors.New("clear error"),
|
|
listRules: []string{
|
|
`-A POSTROUTING -s 172.26.64.217/32 -m comment --comment "name: \"nomad\" id: \"2dd71cac-2b1e-ff08-167c-735f7f9f4964\"" -j CNI-5d36f286cfbb35c5776509ec`,
|
|
},
|
|
}
|
|
err := c.forceCleanup(ipt, "2dd71cac-2b1e-ff08-167c-735f7f9f4964")
|
|
must.EqError(t, err, "failed to cleanup iptables rules for alloc 2dd71cac-2b1e-ff08-167c-735f7f9f4964")
|
|
})
|
|
}
|
|
|
|
// TestCNI_cniToAllocNet_Fallback asserts if a CNI plugin result lacks an IP on
|
|
// its sandbox interface, the first IP found is used.
|
|
func TestCNI_cniToAllocNet_Fallback(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
// Calico's CNI plugin v3.12.3 has been observed to return the
|
|
// following:
|
|
cniResult := &cni.CNIResult{
|
|
Interfaces: map[string]*cni.Config{
|
|
"cali39179aa3-74": {},
|
|
"eth0": {
|
|
IPConfigs: []*cni.IPConfig{
|
|
{
|
|
IP: net.IPv4(192, 168, 135, 232),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// Only need a logger
|
|
c := &cniNetworkConfigurator{
|
|
logger: testlog.HCLogger(t),
|
|
}
|
|
allocNet, err := c.cniToAllocNet(cniResult)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, allocNet)
|
|
assert.Equal(t, "192.168.135.232", allocNet.Address)
|
|
assert.Equal(t, "eth0", allocNet.InterfaceName)
|
|
assert.Nil(t, allocNet.DNS)
|
|
}
|
|
|
|
// TestCNI_cniToAllocNet_Invalid asserts an error is returned if a CNI plugin
|
|
// result lacks any IP addresses. This has not been observed, but Nomad still
|
|
// must guard against invalid results from external plugins.
|
|
func TestCNI_cniToAllocNet_Invalid(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
cniResult := &cni.CNIResult{
|
|
Interfaces: map[string]*cni.Config{
|
|
"eth0": {},
|
|
"veth1": {
|
|
IPConfigs: []*cni.IPConfig{},
|
|
},
|
|
},
|
|
}
|
|
|
|
// Only need a logger
|
|
c := &cniNetworkConfigurator{
|
|
logger: testlog.HCLogger(t),
|
|
}
|
|
allocNet, err := c.cniToAllocNet(cniResult)
|
|
require.Error(t, err)
|
|
require.Nil(t, allocNet)
|
|
}
|