luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Simplify Bootstrap logic in tests 

This change updates tests to honor `BootstrapExpect` exclusively when
forming test clusters and removes test only knobs, e.g.
`config.DevDisableBootstrap`.

Background:

Test cluster creation is fragile.  Test servers don't follow the
BootstapExpected route like production clusters.  Instead they start as
single node clusters and then get rejoin and may risk causing brain
split or other test flakiness.

The test framework expose few knobs to control those (e.g.
`config.DevDisableBootstrap` and `config.Bootstrap`) that control
whether a server should bootstrap the cluster.  These flags are
confusing and it's unclear when to use: their usage in multi-node
cluster isn't properly documented.  Furthermore, they have some bad
side-effects as they don't control Raft library: If
`config.DevDisableBootstrap` is true, the test server may not
immediately attempt to bootstrap a cluster, but after an election
timeout (~50ms), Raft may force a leadership election and win it (with
only one vote) and cause a split brain.

The knobs are also confusing as Bootstrap is an overloaded term.  In
BootstrapExpect, we refer to bootstrapping the cluster only after N
servers are connected.  But in tests and the knobs above, it refers to
whether the server is a single node cluster and shouldn't wait for any
other server.

Changes:

This commit makes two changes:

First, it relies on `BootstrapExpected` instead of `Bootstrap` and/or
`DevMode` flags.  This change is relatively trivial.

Introduce a `Bootstrapped` flag to track if the cluster is bootstrapped.
This allows us to keep `BootstrapExpected` immutable.  Previously, the
flag was a config value but it gets set to 0 after cluster bootstrap
completes.
This commit is contained in:
Mahmood Ali 2020-03-02 10:29:24 -05:00
parent ad126af9c7
commit acbfeb5815
23 changed files with 193 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/rpc_test.go
View File

@ -63,7 +63,6 @@ func TestRpc_streamingRpcConn_badEndpoint_TLS(t *testing.T) {
s1, cleanupS1 := nomad.TestServer(t, func(c *nomad.Config) {
c.Region = "regionFoo"
c.BootstrapExpect = 1
c.DevDisableBootstrap = true
c.TLSConfig = &sconfig.TLSConfig{
EnableHTTP: true,
EnableRPC: true,

7
command/agent/agent.go
View File

@ -11,7 +11,6 @@ import (
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
metrics "github.com/armon/go-metrics"
@ -157,11 +156,7 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
conf.NodeName = agentConfig.NodeName
}
if agentConfig.Server.BootstrapExpect > 0 {
if agentConfig.Server.BootstrapExpect == 1 {
conf.Bootstrap = true
} else {
atomic.StoreInt32(&conf.BootstrapExpect, int32(agentConfig.Server.BootstrapExpect))
}
conf.BootstrapExpect = agentConfig.Server.BootstrapExpect
}
if agentConfig.DataDir != "" {
conf.DataDir = filepath.Join(agentConfig.DataDir, "server")

6
command/agent/agent_test.go
View File

@ -166,14 +166,12 @@ func TestAgent_ServerConfig(t *testing.T) {
conf.Server.BootstrapExpect = 1
out, err = a.serverConfig()
require.NoError(t, err)
require.True(t, out.Bootstrap)
require.Equal(t, int32(0), out.BootstrapExpect)
require.Equal(t, 1, out.BootstrapExpect)
conf.Server.BootstrapExpect = 3
out, err = a.serverConfig()
require.NoError(t, err)
require.False(t, out.Bootstrap)
require.Equal(t, int32(3), out.BootstrapExpect)
require.Equal(t, 3, out.BootstrapExpect)
}
func TestAgent_ServerConfig_SchedulerFlags(t *testing.T) {

3
command/agent/config.go
View File

@ -778,7 +778,8 @@ func DevConfig(mode *devModeConfig) *Config {
conf.LogLevel = "DEBUG"
conf.Client.Enabled = true
conf.Server.Enabled = true
conf.DevMode = mode != nil
conf.DevMode = true
conf.Server.BootstrapExpect = 1
conf.EnableDebug = true
conf.DisableAnonymousSignature = true
conf.Consul.AutoAdvertise = helper.BoolToPtr(true)

6
command/agent/testagent.go
View File

@ -168,7 +168,7 @@ RETRY:
}
failed := false
if a.Config.NomadConfig.Bootstrap && a.Config.Server.Enabled {
if a.Config.NomadConfig.BootstrapExpect == 1 && a.Config.Server.Enabled {
testutil.WaitForResult(func() (bool, error) {
args := &structs.GenericRequest{}
var leader string
@ -358,10 +358,6 @@ func (a *TestAgent) config() *Config {
config.ServerHealthInterval = 50 * time.Millisecond
config.AutopilotInterval = 100 * time.Millisecond
// Bootstrap ourselves
config.Bootstrap = true
config.BootstrapExpect = 1
// Tighten the fingerprinter timeouts
if conf.Client.Options == nil {
conf.Client.Options = make(map[string]string)

2
nomad/acl_endpoint_test.go
View File

@ -1077,8 +1077,6 @@ func TestACLEndpoint_Bootstrap_Reset(t *testing.T) {
c.ACLEnabled = true
c.DataDir = dir
c.DevMode = false
c.Bootstrap = true
c.DevDisableBootstrap = false
})
defer cleanupS1()
codec := rpcClient(t, s1)

33
nomad/autopilot_test.go
View File

@ -73,7 +73,6 @@ func TestAutopilot_CleanupDeadServer(t *testing.T) {
func testCleanupDeadServer(t *testing.T, raftVersion int) {
conf := func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(raftVersion)
}
@ -127,13 +126,13 @@ func testCleanupDeadServer(t *testing.T, raftVersion int) {
func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) {
t.Parallel()
s1, cleanupS1 := TestServer(t, nil)
defer cleanupS1()
conf := func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 5
}
s1, cleanupS1 := TestServer(t, conf)
defer cleanupS1()
s2, cleanupS2 := TestServer(t, conf)
defer cleanupS2()
@ -174,16 +173,14 @@ func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) {
func TestAutopilot_RollingUpdate(t *testing.T) {
t.Parallel()
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.RaftConfig.ProtocolVersion = 3
})
defer cleanupS1()
conf := func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.RaftConfig.ProtocolVersion = 3
}
s1, cleanupS1 := TestServer(t, conf)
defer cleanupS1()
s2, cleanupS2 := TestServer(t, conf)
defer cleanupS2()
@ -248,19 +245,21 @@ func TestAutopilot_CleanupStaleRaftServer(t *testing.T) {
t.Skip("TestAutopilot_CleanupDeadServer is very flaky, removing it for now")
t.Parallel()
s1, cleanupS1 := TestServer(t, nil)
conf := func(c *Config) {
c.BootstrapExpect = 3
}
s1, cleanupS1 := TestServer(t, conf)
defer cleanupS1()
conf := func(c *Config) {
c.DevDisableBootstrap = true
}
s2, cleanupS2 := TestServer(t, conf)
defer cleanupS2()
s3, cleanupS3 := TestServer(t, conf)
defer cleanupS3()
s4, cleanupS4 := TestServer(t, conf)
s4, cleanupS4 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 0
})
defer cleanupS4()
servers := []*Server{s1, s2, s3}
@ -304,7 +303,7 @@ func TestAutopilot_PromoteNonVoter(t *testing.T) {
testutil.WaitForLeader(t, s1.RPC)
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 0
c.RaftConfig.ProtocolVersion = 3
})
defer cleanupS2()

20
nomad/client_agent_endpoint_test.go
View File

@ -30,10 +30,12 @@ func TestMonitor_Monitor_Remote_Client(t *testing.T) {
require := require.New(t)
// start server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -125,15 +127,16 @@ func TestMonitor_Monitor_RemoteServer(t *testing.T) {
foreignRegion := "foo"
// start servers
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.Region = foreignRegion
})
defer cleanupS3()
@ -516,12 +519,12 @@ func TestAgentProfile_RemoteClient(t *testing.T) {
// start server and client
s1, cleanup := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanup()
s2, cleanup := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanup()
@ -640,12 +643,13 @@ func TestAgentProfile_Server(t *testing.T) {
// start servers
s1, cleanup := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
c.EnableDebug = true
})
defer cleanup()
s2, cleanup := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
c.EnableDebug = true
})
defer cleanup()

30
nomad/client_alloc_endpoint_test.go
View File

@ -186,10 +186,12 @@ func TestClientAllocations_GarbageCollectAll_Remote(t *testing.T) {
require := require.New(t)
// Start a server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -432,10 +434,12 @@ func TestClientAllocations_GarbageCollect_Remote(t *testing.T) {
require := require.New(t)
// Start a server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -720,10 +724,12 @@ func TestClientAllocations_Stats_Remote(t *testing.T) {
require := require.New(t)
// Start a server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -915,10 +921,12 @@ func TestClientAllocations_Restart_Remote(t *testing.T) {
require := require.New(t)
// Start a server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -1071,11 +1079,13 @@ func TestAlloc_ExecStreaming(t *testing.T) {
t.Parallel()
////// Nomad clusters topology - not specific to test
localServer, cleanupLS := TestServer(t, nil)
localServer, cleanupLS := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupLS()
remoteServer, cleanupRS := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupRS()

24
nomad/client_fs_endpoint_test.go
View File

@ -177,10 +177,12 @@ func TestClientFS_List_Remote(t *testing.T) {
require := require.New(t)
// Start a server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -451,10 +453,12 @@ func TestClientFS_Stat_Remote(t *testing.T) {
require := require.New(t)
// Start a server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -1000,10 +1004,12 @@ func TestClientFS_Streaming_Remote_Server(t *testing.T) {
require := require.New(t)
// Start a server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -1829,10 +1835,12 @@ func TestClientFS_Logs_Remote_Server(t *testing.T) {
require := require.New(t)
// Start a server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)

36
nomad/client_rpc_test.go
View File

@ -88,10 +88,12 @@ func TestServerWithNodeConn_NoPath(t *testing.T) {
t.Parallel()
require := require.New(t)
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -122,10 +124,12 @@ func TestServerWithNodeConn_Path(t *testing.T) {
t.Parallel()
require := require.New(t)
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -174,14 +178,16 @@ func TestServerWithNodeConn_Path_Newest(t *testing.T) {
t.Parallel()
require := require.New(t)
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
TestJoin(t, s1, s2, s3)
@ -208,14 +214,16 @@ func TestServerWithNodeConn_PathAndErr(t *testing.T) {
t.Parallel()
require := require.New(t)
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
TestJoin(t, s1, s2, s3)
@ -242,14 +250,16 @@ func TestServerWithNodeConn_NoPathAndErr(t *testing.T) {
t.Parallel()
require := require.New(t)
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
TestJoin(t, s1, s2, s3)

6
nomad/client_stats_endpoint_test.go
View File

@ -173,10 +173,12 @@ func TestClientStats_Stats_Remote(t *testing.T) {
require := require.New(t)
// Start a server and client
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)

25
nomad/config.go
View File

@ -49,16 +49,23 @@ func DefaultRPCAddr() *net.TCPAddr {
// Config is used to parameterize the server
type Config struct {
// Bootstrap mode is used to bring up the first Nomad server. It is
// required so that it can elect a leader without any other nodes
// being present
Bootstrap bool
// Bootstrapped indictes if Server has bootstrapped or not.
// Its value must be 0 (not bootstrapped) or 1 (bootstrapped).
// All operations on Bootstrapped must be handled via `atomic.*Int32()` calls
Bootstrapped int32
// BootstrapExpect mode is used to automatically bring up a
// collection of Nomad servers. This can be used to automatically
// bring up a collection of nodes. All operations on BootstrapExpect
// must be handled via `atomic.*Int32()` calls.
BootstrapExpect int32
// bring up a collection of nodes.
//
// The BootstrapExpect can be of any of the following values:
// 1: Server will form a single node cluster and become a leader immediately
// N, larger than 1: Server will wait until it's connected to N servers
// before attempting leadership and forming the cluster. No Raft Log operation
// will succeed until then.
// 0: Server will wait to get a Raft configuration from another node and may not
// attempt to form a cluster or establish leadership on its own.
BootstrapExpect int
// DataDir is the directory to store our state in
DataDir string
@ -71,10 +78,6 @@ type Config struct {
// in the absence of ACLs
EnableDebug bool
// DevDisableBootstrap is used to disable bootstrap mode while
// in DevMode. This is largely used for testing.
DevDisableBootstrap bool
// LogOutput is the location to write logs to. If this is not set,
// logs will go to stderr.
LogOutput io.Writer

9
nomad/heartbeat_test.go
View File

@ -69,7 +69,6 @@ func TestHeartbeat_ResetHeartbeatTimer_Nonleader(t *testing.T) {
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3 // Won't become leader
c.DevDisableBootstrap = true
})
defer cleanupS1()
@ -215,16 +214,18 @@ func TestHeartbeat_ClearAllHeartbeatTimers(t *testing.T) {
func TestHeartbeat_Server_HeartbeatTTL_Failover(t *testing.T) {
t.Parallel()
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
servers := []*Server{s1, s2, s3}

80
nomad/leader_test.go
View File

@ -23,16 +23,18 @@ import (
)
func TestLeader_LeftServer(t *testing.T) {
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
servers := []*Server{s1, s2, s3}
@ -83,16 +85,18 @@ func TestLeader_LeftServer(t *testing.T) {
}
func TestLeader_LeftLeader(t *testing.T) {
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
servers := []*Server{s1, s2, s3}
@ -154,24 +158,29 @@ func TestLeader_MultiBootstrap(t *testing.T) {
// Ensure we don't have multiple raft peers
for _, s := range servers {
peers, _ := s.numPeers()
peers, err := s.numPeers()
if err != nil {
t.Fatalf("failed: %v", err)
}
if peers != 1 {
t.Fatalf("should only have 1 raft peer!")
t.Fatalf("should only have 1 raft peer! %v", peers)
}
}
}
func TestLeader_PlanQueue_Reset(t *testing.T) {
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
servers := []*Server{s1, s2, s3}
@ -223,13 +232,13 @@ func TestLeader_EvalBroker_Reset(t *testing.T) {
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.NumSchedulers = 0
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.NumSchedulers = 0
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
servers := []*Server{s1, s2, s3}
@ -281,13 +290,13 @@ func TestLeader_PeriodicDispatcher_Restore_Adds(t *testing.T) {
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.NumSchedulers = 0
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.NumSchedulers = 0
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
servers := []*Server{s1, s2, s3}
@ -691,29 +700,27 @@ func TestLeader_ClusterID_upgradePath(t *testing.T) {
cleanup func()
}
outdated := func(bootstrap bool) server {
outdated := func() server {
s, cleanup := TestServer(t, func(c *Config) {
c.NumSchedulers = 0
c.Build = before
c.DevDisableBootstrap = bootstrap
c.BootstrapExpect = 3
c.Logger.SetLevel(hclog.Trace)
})
return server{s: s, cleanup: cleanup}
}
upgraded := func(bootstrap bool) server {
upgraded := func() server {
s, cleanup := TestServer(t, func(c *Config) {
c.NumSchedulers = 0
c.Build = after
c.DevDisableBootstrap = bootstrap
c.BootstrapExpect = 3
c.BootstrapExpect = 0
c.Logger.SetLevel(hclog.Trace)
})
return server{s: s, cleanup: cleanup}
}
servers := []server{outdated(false), outdated(true), outdated(true)}
servers := []server{outdated(), outdated(), outdated()}
// fallback shutdown attempt in case testing fails
defer servers[0].cleanup()
defer servers[1].cleanup()
@ -722,7 +729,7 @@ func TestLeader_ClusterID_upgradePath(t *testing.T) {
upgrade := func(i int) {
previous := servers[i]
servers[i] = upgraded(true)
servers[i] = upgraded()
TestJoin(t, servers[i].s, servers[(i+1)%3].s, servers[(i+2)%3].s)
testutil.WaitForLeader(t, servers[i].s.RPC)
@ -809,7 +816,6 @@ func TestLeader_ClusterID_noUpgrade(t *testing.T) {
c.Logger.SetLevel(hclog.Trace)
c.NumSchedulers = 0
c.Build = minClusterIDVersion.String()
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
@ -817,7 +823,6 @@ func TestLeader_ClusterID_noUpgrade(t *testing.T) {
c.Logger.SetLevel(hclog.Trace)
c.NumSchedulers = 0
c.Build = minClusterIDVersion.String()
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
@ -1012,13 +1017,13 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) {
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.RaftConfig.ProtocolVersion = 1
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.RaftConfig.ProtocolVersion = 2
})
defer cleanupS3()
@ -1054,7 +1059,7 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) {
// Replace the dead server with one running raft protocol v3
s4, cleanupS4 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.Datacenter = "dc1"
c.RaftConfig.ProtocolVersion = 3
})
@ -1111,14 +1116,12 @@ func leaderElectionTest(t *testing.T, raftProtocol raft.ProtocolVersion) {
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
c.DevDisableBootstrap = true
c.RaftConfig.ProtocolVersion = raftProtocol
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
c.DevDisableBootstrap = true
c.RaftConfig.ProtocolVersion = raftProtocol
})
defer cleanupS3() // todo(shoenig) added this, should be here right??
@ -1170,13 +1173,13 @@ func TestLeader_RollRaftServer(t *testing.T) {
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.RaftConfig.ProtocolVersion = 2
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.RaftConfig.ProtocolVersion = 2
})
defer cleanupS3()
@ -1207,7 +1210,7 @@ func TestLeader_RollRaftServer(t *testing.T) {
// Replace the dead server with one running raft protocol v3
s4, cleanupS4 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.RaftConfig.ProtocolVersion = 3
})
defer cleanupS4()
@ -1230,7 +1233,7 @@ func TestLeader_RollRaftServer(t *testing.T) {
}
// Replace another dead server with one running raft protocol v3
s5, cleanupS5 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.RaftConfig.ProtocolVersion = 3
})
defer cleanupS5()
@ -1254,7 +1257,7 @@ func TestLeader_RollRaftServer(t *testing.T) {
// Replace the last dead server with one running raft protocol v3
s6, cleanupS6 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
c.RaftConfig.ProtocolVersion = 3
})
defer cleanupS6()
@ -1330,19 +1333,22 @@ func TestServer_ReconcileMember(t *testing.T) {
// Create a three node cluster
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
// disable bootstrapping
c.BootstrapExpect = 0
c.RaftConfig.ProtocolVersion = 3
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
// disable bootstrapping
c.BootstrapExpect = 0
c.RaftConfig.ProtocolVersion = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
// disable bootstrapping
c.BootstrapExpect = 0
c.RaftConfig.ProtocolVersion = 2
})
defer cleanupS3()

10
nomad/node_endpoint_test.go
View File

@ -97,7 +97,7 @@ func TestClientEndpoint_Register_NodeConn_Forwarded(t *testing.T) {
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -754,16 +754,18 @@ func TestClientEndpoint_UpdateStatus_GetEvals(t *testing.T) {
func TestClientEndpoint_UpdateStatus_HeartbeatOnly(t *testing.T) {
t.Parallel()
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
})
defer cleanupS3()
servers := []*Server{s1, s2, s3}

18
nomad/rpc_test.go
View File

@ -45,10 +45,12 @@ func rpcClient(t *testing.T, s *Server) rpc.ClientCodec {
func TestRPC_forwardLeader(t *testing.T) {
t.Parallel()
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -259,10 +261,12 @@ func TestRPC_streamingRpcConn_badMethod(t *testing.T) {
t.Parallel()
require := require.New(t)
s1, cleanupS1 := TestServer(t, nil)
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.DevDisableBootstrap = true
c.BootstrapExpect = 2
})
defer cleanupS2()
TestJoin(t, s1, s2)
@ -298,7 +302,6 @@ func TestRPC_streamingRpcConn_badMethod_TLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node1")
c.TLSConfig = &config.TLSConfig{
EnableHTTP: true,
@ -315,7 +318,6 @@ func TestRPC_streamingRpcConn_badMethod_TLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node2")
c.TLSConfig = &config.TLSConfig{
EnableHTTP: true,
@ -354,7 +356,6 @@ func TestRPC_streamingRpcConn_goodMethod_Plaintext(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node1")
})
defer cleanupS1()
@ -363,7 +364,6 @@ func TestRPC_streamingRpcConn_goodMethod_Plaintext(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node2")
})
defer cleanupS2()
@ -414,7 +414,6 @@ func TestRPC_streamingRpcConn_goodMethod_TLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node1")
c.TLSConfig = &config.TLSConfig{
EnableHTTP: true,
@ -431,7 +430,6 @@ func TestRPC_streamingRpcConn_goodMethod_TLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node2")
c.TLSConfig = &config.TLSConfig{
EnableHTTP: true,

13
nomad/serf.go
View File

@ -83,7 +83,7 @@ func (s *Server) nodeJoin(me serf.MemberEvent) {
s.peerLock.Unlock()
// If we still expecting to bootstrap, may need to handle this
if atomic.LoadInt32(&s.config.BootstrapExpect) != 0 {
if atomic.LoadInt32(&s.config.Bootstrapped) == 0 {
s.maybeBootstrap()
}
}
@ -111,7 +111,7 @@ func (s *Server) maybeBootstrap() {
// Bootstrap can only be done if there are no committed logs,
// remove our expectations of bootstrapping
if index != 0 {
atomic.StoreInt32(&s.config.BootstrapExpect, 0)
atomic.StoreInt32(&s.config.Bootstrapped, 1)
return
}
@ -127,7 +127,7 @@ func (s *Server) maybeBootstrap() {
if p.Region != s.config.Region {
continue
}
if p.Expect != 0 && p.Expect != int(atomic.LoadInt32(&s.config.BootstrapExpect)) {
if p.Expect != 0 && p.Expect != s.config.BootstrapExpect {
s.logger.Error("peer has a conflicting expect value. All nodes should expect the same number", "member", member)
return
}
@ -138,11 +138,12 @@ func (s *Server) maybeBootstrap() {
if !p.NonVoter {
voters++
}
servers = append(servers, *p)
}
// Skip if we haven't met the minimum expect count
if voters < int(atomic.LoadInt32(&s.config.BootstrapExpect)) {
if voters < s.config.BootstrapExpect {
return
}
@ -181,7 +182,7 @@ func (s *Server) maybeBootstrap() {
if len(peers) > 0 {
s.logger.Info("disabling bootstrap mode because existing Raft peers being reported by peer",
"peer_name", server.Name, "peer_address", server.Addr)
atomic.StoreInt32(&s.config.BootstrapExpect, 0)
atomic.StoreInt32(&s.config.Bootstrapped, 1)
return
}
}
@ -223,7 +224,7 @@ func (s *Server) maybeBootstrap() {
}
// Bootstrapping complete, or failed for some reason, don't enter this again
atomic.StoreInt32(&s.config.BootstrapExpect, 0)
atomic.StoreInt32(&s.config.Bootstrapped, 1)
}
// nodeFailed is used to handle fail events on the serf cluster

15
nomad/serf_test.go
View File

@ -105,7 +105,6 @@ func TestNomad_ReapPeer(t *testing.T) {
c.NodeName = "node1"
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node1")
})
defer cleanupS1()
@ -113,7 +112,6 @@ func TestNomad_ReapPeer(t *testing.T) {
c.NodeName = "node2"
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node2")
})
defer cleanupS2()
@ -121,7 +119,6 @@ func TestNomad_ReapPeer(t *testing.T) {
c.NodeName = "node3"
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node3")
})
defer cleanupS3()
@ -200,21 +197,18 @@ func TestNomad_BootstrapExpect(t *testing.T) {
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node1")
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node2")
})
defer cleanupS2()
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node3")
})
defer cleanupS3()
@ -263,7 +257,6 @@ func TestNomad_BootstrapExpect(t *testing.T) {
s4, cleanupS4 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node4")
})
defer cleanupS4()
@ -313,7 +306,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) {
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node1")
c.NonVoter = true
})
@ -321,7 +313,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) {
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node2")
c.NonVoter = true
})
@ -329,7 +320,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) {
s3, cleanupS3 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node3")
})
defer cleanupS3()
@ -351,7 +341,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) {
s4, cleanupS4 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node4")
})
defer cleanupS4()
@ -418,12 +407,10 @@ func TestNomad_BadExpect(t *testing.T) {
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
c.DevDisableBootstrap = true
})
defer cleanupS1()
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 3
c.DevDisableBootstrap = true
})
defer cleanupS2()
servers := []*Server{s1, s2}
@ -446,7 +433,7 @@ func TestNomad_BadExpect(t *testing.T) {
testutil.WaitForResult(func() (bool, error) {
for _, s := range servers {
p, _ := s.numPeers()
if p != 1 {
if p != 0 {
return false, fmt.Errorf("%d", p)
}
}

18
nomad/server.go
View File

@ -822,7 +822,7 @@ func (s *Server) setupBootstrapHandler() error {
// (ab)use serf.go's behavior of setting BootstrapExpect to
// zero if we have bootstrapped. If we have bootstrapped
bootstrapExpect := atomic.LoadInt32(&s.config.BootstrapExpect)
bootstrapExpect := s.config.BootstrapExpect
if bootstrapExpect == 0 {
// This Nomad Server has been bootstrapped. Rely on
// the peersTimeout firing as a guard to prevent
@ -848,7 +848,7 @@ func (s *Server) setupBootstrapHandler() error {
// quorum has been reached, we do not need to poll
// Consul. Let the normal timeout-based strategy
// take over.
if raftPeers >= int(bootstrapExpect) {
if raftPeers >= bootstrapExpect {
peersTimeout.Reset(peersPollInterval + lib.RandomStagger(peersPollInterval/peersPollJitterFactor))
return nil
}
@ -1275,9 +1275,9 @@ func (s *Server) setupRaft() error {
}
}
// If we are in bootstrap or dev mode and the state is clean then we can
// If we are a single server cluster and the state is clean then we can
// bootstrap now.
if s.config.Bootstrap || s.config.DevMode {
if s.isSingleServerCluster() {
hasState, err := raft.HasExistingState(log, stable, snap)
if err != nil {
return err
@ -1320,10 +1320,10 @@ func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (
conf.Tags["id"] = s.config.NodeID
conf.Tags["rpc_addr"] = s.clientRpcAdvertise.(*net.TCPAddr).IP.String() // Address that clients will use to RPC to servers
conf.Tags["port"] = fmt.Sprintf("%d", s.serverRpcAdvertise.(*net.TCPAddr).Port) // Port servers use to RPC to one and another
if s.config.Bootstrap || (s.config.DevMode && !s.config.DevDisableBootstrap) {
if s.isSingleServerCluster() {
conf.Tags["bootstrap"] = "1"
}
bootstrapExpect := atomic.LoadInt32(&s.config.BootstrapExpect)
bootstrapExpect := s.config.BootstrapExpect
if bootstrapExpect != 0 {
conf.Tags["expect"] = fmt.Sprintf("%d", bootstrapExpect)
}
@ -1524,7 +1524,7 @@ func (s *Server) Stats() map[string]map[string]string {
"server": "true",
"leader": fmt.Sprintf("%v", s.IsLeader()),
"leader_addr": string(s.raft.Leader()),
"bootstrap": fmt.Sprintf("%v", s.config.Bootstrap),
"bootstrap": fmt.Sprintf("%v", s.isSingleServerCluster()),
"known_regions": toString(uint64(len(s.peers))),
},
"raft": s.raft.Stats(),
@ -1617,6 +1617,10 @@ func (s *Server) ClusterID() (string, error) {
return generatedID, nil
}
func (s *Server) isSingleServerCluster() bool {
return s.config.BootstrapExpect == 1
}
// peersInfoContent is used to help operators understand what happened to the
// peers.json file. This is written to a file called peers.info in the same
// location.

8
nomad/server_test.go
View File

@ -56,7 +56,6 @@ func TestServer_RPC_TLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node1")
c.TLSConfig = &config.TLSConfig{
EnableHTTP: true,
@ -73,7 +72,6 @@ func TestServer_RPC_TLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node2")
c.TLSConfig = &config.TLSConfig{
EnableHTTP: true,
@ -90,7 +88,6 @@ func TestServer_RPC_TLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node3")
c.TLSConfig = &config.TLSConfig{
EnableHTTP: true,
@ -125,7 +122,6 @@ func TestServer_RPC_MixedTLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node1")
c.TLSConfig = &config.TLSConfig{
EnableHTTP: true,
@ -142,7 +138,6 @@ func TestServer_RPC_MixedTLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node2")
c.TLSConfig = &config.TLSConfig{
EnableHTTP: true,
@ -159,7 +154,6 @@ func TestServer_RPC_MixedTLS(t *testing.T) {
c.Region = "regionFoo"
c.BootstrapExpect = 3
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node3")
})
defer cleanupS3()
@ -470,7 +464,6 @@ func TestServer_Reload_TLSConnections_Raft(t *testing.T) {
s1, cleanupS1 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node1")
c.NodeName = "node1"
c.Region = "regionFoo"
@ -480,7 +473,6 @@ func TestServer_Reload_TLSConnections_Raft(t *testing.T) {
s2, cleanupS2 := TestServer(t, func(c *Config) {
c.BootstrapExpect = 2
c.DevMode = false
c.DevDisableBootstrap = true
c.DataDir = path.Join(dir, "node2")
c.NodeName = "node2"
c.Region = "regionFoo"

1
nomad/stats_fetcher_test.go
View File

@ -13,7 +13,6 @@ func TestStatsFetcher(t *testing.T) {
conf := func(c *Config) {
c.Region = "region-a"
c.DevDisableBootstrap = true
c.BootstrapExpect = 3
}

1
nomad/testing.go
View File

@ -45,6 +45,7 @@ func TestServer(t testing.T, cb func(*Config)) (*Server, func()) {
config.Logger = testlog.HCLogger(t)
config.Build = version.Version + "+unittest"
config.DevMode = true
config.BootstrapExpect = 1
nodeNum := atomic.AddUint32(&nodeNumber, 1)
config.NodeName = fmt.Sprintf("nomad-%03d", nodeNum)