Make the chunking test multidimensional (#6212)

This ensures that it's not just a single operation we restores
successfully, but many. It's the same foundation, just with multiple
going on at once.

agent/consul/fsm/commands_oss_test.go

@@ -1415,9 +1415,12 @@ func TestFSM_Chunking_Lifecycle(t *testing.T) {
 	fsm, err := New(nil, os.Stderr)
 	require.NoError(err)
 
+	var logOfLogs [][]*raft.Log
+	var bufs [][]byte
+	for i := 0; i < 10; i++ {
 		req := structs.RegisterRequest{
 			Datacenter: "dc1",
-		Node:       "foo",
+			Node:       fmt.Sprintf("foo%d", i),
 			Address:    "127.0.0.1",
 			Service: &structs.NodeService{
 				ID:      "db",
@@ -1426,7 +1429,7 @@ func TestFSM_Chunking_Lifecycle(t *testing.T) {
 				Port:    8000,
 			},
 			Check: &structs.HealthCheck{
-			Node:      "foo",
+				Node:      fmt.Sprintf("foo%d", i),
 				CheckID:   "db",
 				Name:      "db connectivity",
 				Status:    api.HealthPassing,
@@ -1438,10 +1441,11 @@ func TestFSM_Chunking_Lifecycle(t *testing.T) {
 		require.NoError(err)
 
 		var logs []*raft.Log
-	for i, b := range buf {
+
+		for j, b := range buf {
 			chunkInfo := &raftchunkingtypes.ChunkInfo{
-			OpNum:       uint64(32),
-			SequenceNum: uint32(i),
+				OpNum:       uint64(32 + i),
+				SequenceNum: uint32(j),
 				NumChunks:   uint32(len(buf)),
 			}
 			chunkBytes, err := proto.Marshal(chunkInfo)
@@ -1452,18 +1456,28 @@ func TestFSM_Chunking_Lifecycle(t *testing.T) {
 				Extensions: chunkBytes,
 			})
 		}
+		bufs = append(bufs, buf)
+		logOfLogs = append(logOfLogs, logs)
+	}
 
 	// The reason for the skipping is to test out-of-order applies which are
-	// theoretically possible
-	for i := 0; i < len(logs); i += 2 {
-		resp := fsm.chunker.Apply(logs[i])
+	// theoretically possible. Apply some logs from each set of chunks, but not
+	// the full set, and out of order.
+	for _, logs := range logOfLogs {
+		resp := fsm.chunker.Apply(logs[8])
+		assert.Nil(resp)
+		resp = fsm.chunker.Apply(logs[0])
+		assert.Nil(resp)
+		resp = fsm.chunker.Apply(logs[3])
 		assert.Nil(resp)
 	}
 
 	// Verify we are not registered
-	_, node, err := fsm.state.GetNode("foo")
+	for i := 0; i < 10; i++ {
+		_, node, err := fsm.state.GetNode(fmt.Sprintf("foo%d", i))
 		require.NoError(err)
 		assert.Nil(node)
+	}
 
 	// Snapshot, restore elsewhere, apply the rest of the logs, make sure it
 	// looks right
@@ -1482,38 +1496,46 @@ func TestFSM_Chunking_Lifecycle(t *testing.T) {
 	require.NoError(err)
 
 	// Verify we are still not registered
-	_, node, err = fsm2.state.GetNode("foo")
+	for i := 0; i < 10; i++ {
+		_, node, err := fsm2.state.GetNode(fmt.Sprintf("foo%d", i))
 		require.NoError(err)
 		assert.Nil(node)
+	}
 
 	// Apply the rest of the logs
+	for _, logs := range logOfLogs {
 		var resp interface{}
-	for i := 1; i < len(logs); i += 2 {
-		resp = fsm2.chunker.Apply(logs[i])
-		if resp != nil {
+		for i, log := range logs {
+			switch i {
+			case 0, 3, 8:
+			default:
+				resp = fsm2.chunker.Apply(log)
+				if i != len(logs)-1 {
+					assert.Nil(resp)
+				}
+			}
+		}
 		_, ok := resp.(raftchunking.ChunkingSuccess)
 		assert.True(ok)
 	}
-	}
-	assert.NotNil(resp)
-	_, ok := resp.(raftchunking.ChunkingSuccess)
-	assert.True(ok)
 
 	// Verify we are registered
-	_, node, err = fsm2.state.GetNode("foo")
+	for i := 0; i < 10; i++ {
+		_, node, err := fsm2.state.GetNode(fmt.Sprintf("foo%d", i))
 		require.NoError(err)
 		assert.NotNil(node)
 
 		// Verify service registered
-	_, services, err := fsm2.state.NodeServices(nil, "foo")
+		_, services, err := fsm2.state.NodeServices(nil, fmt.Sprintf("foo%d", i))
 		require.NoError(err)
-	_, ok = services.Services["db"]
+		_, ok := services.Services["db"]
 		assert.True(ok)
 
 		// Verify check
-	_, checks, err := fsm2.state.NodeChecks(nil, "foo")
+		_, checks, err := fsm2.state.NodeChecks(nil, fmt.Sprintf("foo%d", i))
 		require.NoError(err)
 		require.Equal(string(checks[0].CheckID), "db")
+	}
 }
 
 func TestFSM_Chunking_TermChange(t *testing.T) {