9ed75e1f72
When client nodes are restarted, all allocations that have been scheduled on the node have their modify index updated, including terminal allocations. There are several contributing factors: * The `allocSync` method that updates the servers isn't gated on first contact with the servers. This means that if a server updates the desired state while the client is down, the `allocSync` races with the `Node.ClientGetAlloc` RPC. This will typically result in the client updating the server with "running" and then immediately thereafter "complete". * The `allocSync` method unconditionally sends the `Node.UpdateAlloc` RPC even if it's possible to assert that the server has definitely seen the client state. The allocrunner may queue-up updates even if we gate sending them. So then we end up with a race between the allocrunner updating its internal state to overwrite the previous update and `allocSync` sending the bogus or duplicate update. This changeset adds tracking of server-acknowledged state to the allocrunner. This state gets checked in the `allocSync` before adding the update to the batch, and updated when `Node.UpdateAlloc` returns successfully. To implement this we need to be able to equality-check the updates against the last acknowledged state. We also need to add the last acknowledged state to the client state DB, otherwise we'd drop unacknowledged updates across restarts. The client restart test has been expanded to cover a variety of allocation states, including allocs stopped before shutdown, allocs stopped by the server while the client is down, and allocs that have been completely GC'd on the server while the client is down. I've also bench tested scenarios where the task workload is killed while the client is down, resulting in a failed restore. Fixes #16381
128 lines
3.2 KiB
Go
128 lines
3.2 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
package state
|
|
|
|
import (
|
|
arstate "github.com/hashicorp/nomad/client/allocrunner/state"
|
|
"github.com/hashicorp/nomad/client/allocrunner/taskrunner/state"
|
|
dmstate "github.com/hashicorp/nomad/client/devicemanager/state"
|
|
"github.com/hashicorp/nomad/client/dynamicplugins"
|
|
driverstate "github.com/hashicorp/nomad/client/pluginmanager/drivermanager/state"
|
|
"github.com/hashicorp/nomad/client/serviceregistration/checks"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
)
|
|
|
|
// NoopDB implements a StateDB that does not persist any data.
|
|
type NoopDB struct{}
|
|
|
|
func (n NoopDB) Name() string {
|
|
return "noopdb"
|
|
}
|
|
|
|
func (n NoopDB) Upgrade() error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) GetAllAllocations() ([]*structs.Allocation, map[string]error, error) {
|
|
return nil, nil, nil
|
|
}
|
|
|
|
func (n NoopDB) PutAllocation(alloc *structs.Allocation, opts ...WriteOption) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) GetDeploymentStatus(allocID string) (*structs.AllocDeploymentStatus, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (n NoopDB) PutDeploymentStatus(allocID string, ds *structs.AllocDeploymentStatus) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) GetNetworkStatus(allocID string) (*structs.AllocNetworkStatus, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (n NoopDB) PutNetworkStatus(allocID string, ds *structs.AllocNetworkStatus, opts ...WriteOption) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) PutAcknowledgedState(allocID string, state *arstate.State, opts ...WriteOption) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) GetAcknowledgedState(allocID string) (*arstate.State, error) { return nil, nil }
|
|
|
|
func (n NoopDB) GetTaskRunnerState(allocID string, taskName string) (*state.LocalState, *structs.TaskState, error) {
|
|
return nil, nil, nil
|
|
}
|
|
|
|
func (n NoopDB) PutTaskRunnerLocalState(allocID string, taskName string, val *state.LocalState) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) PutTaskState(allocID string, taskName string, state *structs.TaskState) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) DeleteTaskBucket(allocID, taskName string) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) DeleteAllocationBucket(allocID string, opts ...WriteOption) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) PutDevicePluginState(ps *dmstate.PluginState) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) GetDevicePluginState() (*dmstate.PluginState, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (n NoopDB) PutDriverPluginState(ps *driverstate.PluginState) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) GetDriverPluginState() (*driverstate.PluginState, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (n NoopDB) PutDynamicPluginRegistryState(ps *dynamicplugins.RegistryState) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) GetDynamicPluginRegistryState() (*dynamicplugins.RegistryState, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (n NoopDB) PutCheckResult(allocID string, qr *structs.CheckQueryResult) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) GetCheckResults() (checks.ClientResults, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (n NoopDB) DeleteCheckResults(allocID string, checkIDs []structs.CheckID) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) PurgeCheckResults(allocID string) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) PutNodeMeta(map[string]*string) error {
|
|
return nil
|
|
}
|
|
|
|
func (n NoopDB) GetNodeMeta() (map[string]*string, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (n NoopDB) Close() error {
|
|
return nil
|
|
}
|