open-nomad/nomad/testing.go

package nomad

import (
	"fmt"
	"math/rand"
	"net"
	"sync/atomic"
	"time"

	testing "github.com/mitchellh/go-testing-interface"
	"github.com/pkg/errors"

	"github.com/hashicorp/nomad/command/agent/consul"
	"github.com/hashicorp/nomad/helper/freeport"
	"github.com/hashicorp/nomad/helper/pluginutils/catalog"
	"github.com/hashicorp/nomad/helper/pluginutils/singleton"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/state"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/version"
)

var (
	nodeNumber uint32 = 0
)

func TestACLServer(t testing.T, cb func(*Config)) (*Server, *structs.ACLToken, func()) {
	server, cleanup := TestServer(t, func(c *Config) {
		c.ACLEnabled = true
		if cb != nil {
			cb(c)
		}
	})
	token := mock.ACLManagementToken()
	err := server.State().BootstrapACLTokens(1, 0, token)
	if err != nil {
		t.Fatalf("failed to bootstrap ACL token: %v", err)
	}
	return server, token, cleanup
}

func TestServer(t testing.T, cb func(*Config)) (*Server, func()) {
	// Setup the default settings
	config := DefaultConfig()
	config.Logger = testlog.HCLogger(t)
	config.Build = version.Version + "+unittest"
	config.DevMode = true
	config.BootstrapExpect = 1
	nodeNum := atomic.AddUint32(&nodeNumber, 1)
	config.NodeName = fmt.Sprintf("nomad-%03d", nodeNum)

	// Tighten the Serf timing
	config.SerfConfig.MemberlistConfig.BindAddr = "127.0.0.1"
	config.SerfConfig.MemberlistConfig.SuspicionMult = 2
	config.SerfConfig.MemberlistConfig.RetransmitMult = 2
	config.SerfConfig.MemberlistConfig.ProbeTimeout = 50 * time.Millisecond
	config.SerfConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
	config.SerfConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond

	// Tighten the Raft timing
	config.RaftConfig.LeaderLeaseTimeout = 50 * time.Millisecond
	config.RaftConfig.HeartbeatTimeout = 50 * time.Millisecond
	config.RaftConfig.ElectionTimeout = 50 * time.Millisecond
	config.RaftTimeout = 500 * time.Millisecond

	// Disable Vault
	f := false
	config.VaultConfig.Enabled = &f

	// Squelch output when -v isn't specified
	config.LogOutput = testlog.NewWriter(t)

	// Tighten the autopilot timing
	config.AutopilotConfig.ServerStabilizationTime = 100 * time.Millisecond
	config.ServerHealthInterval = 50 * time.Millisecond
	config.AutopilotInterval = 100 * time.Millisecond

	// Set the plugin loaders
	config.PluginLoader = catalog.TestPluginLoader(t)
	config.PluginSingletonLoader = singleton.NewSingletonLoader(config.Logger, config.PluginLoader)

	// Disable consul autojoining: tests typically join servers directly
	config.ConsulConfig.ServerAutoJoin = &f

	// Invoke the callback if any
	if cb != nil {
		cb(config)
	}

	catalog := consul.NewMockCatalog(config.Logger)

	acls := consul.NewMockACLsAPI(config.Logger)

	for i := 10; i >= 0; i-- {
		// Get random ports, need to cleanup later
		ports := freeport.MustTake(2)

		config.RPCAddr = &net.TCPAddr{
			IP:   []byte{127, 0, 0, 1},
			Port: ports[0],
		}
		config.SerfConfig.MemberlistConfig.BindPort = ports[1]

		// Create server
		server, err := NewServer(config, catalog, acls)
		if err == nil {
			return server, func() {
				ch := make(chan error)
				go func() {
					defer close(ch)

					// Shutdown server
					err := server.Shutdown()
					if err != nil {
						ch <- errors.Wrap(err, "failed to shutdown server")
					}

					freeport.Return(ports)
				}()

				select {
				case e := <-ch:
					if e != nil {
						t.Fatal(e.Error())
					}
				case <-time.After(1 * time.Minute):
					t.Fatal("timed out while shutting down server")
				}
			}
		} else if i == 0 {
			freeport.Return(ports)
			t.Fatalf("err: %v", err)
		} else {
			if server != nil {
				_ = server.Shutdown()
				freeport.Return(ports)
			}
			wait := time.Duration(rand.Int31n(2000)) * time.Millisecond
			time.Sleep(wait)
		}
	}

	return nil, nil
}

func TestJoin(t testing.T, s1 *Server, other ...*Server) {
	addr := fmt.Sprintf("127.0.0.1:%d",
		s1.config.SerfConfig.MemberlistConfig.BindPort)
	for _, s2 := range other {
		if num, err := s2.Join([]string{addr}); err != nil {
			t.Fatalf("err: %v", err)
		} else if num != 1 {
			t.Fatalf("bad: %d", num)
		}
	}
}

// CreateTestPlugin is a helper that generates the node + fingerprint results necessary to
// create a CSIPlugin by directly inserting into the state store. It's exported for use in
// other test packages
func CreateTestCSIPlugin(s *state.StateStore, id string) func() {
	// Create some nodes
	ns := make([]*structs.Node, 3)
	for i := range ns {
		n := mock.Node()
		ns[i] = n
	}

	// Install healthy plugin fingerprinting results
	ns[0].CSIControllerPlugins = map[string]*structs.CSIInfo{
		id: {
			PluginID:                 id,
			AllocID:                  uuid.Generate(),
			Healthy:                  true,
			HealthDescription:        "healthy",
			RequiresControllerPlugin: true,
			RequiresTopologies:       false,
			ControllerInfo: &structs.CSIControllerInfo{
				SupportsReadOnlyAttach:           true,
				SupportsAttachDetach:             true,
				SupportsListVolumes:              true,
				SupportsListVolumesAttachedNodes: false,
			},
		},
	}

	// Install healthy plugin fingerprinting results
	allocID := uuid.Generate()
	for _, n := range ns[1:] {
		n.CSINodePlugins = map[string]*structs.CSIInfo{
			id: {
				PluginID:                 id,
				AllocID:                  allocID,
				Healthy:                  true,
				HealthDescription:        "healthy",
				RequiresControllerPlugin: true,
				RequiresTopologies:       false,
				NodeInfo: &structs.CSINodeInfo{
					ID:                      n.ID,
					MaxVolumes:              64,
					RequiresNodeStageVolume: true,
				},
			},
		}
	}

	// Insert them into the state store
	index := uint64(999)
	for _, n := range ns {
		index++
		s.UpsertNode(index, n)
	}

	// Return cleanup function that deletes the nodes
	return func() {
		ids := make([]string, len(ns))
		for i, n := range ns {
			ids[i] = n.ID
		}

		index++
		s.DeleteNode(index, ids)
	}
}