package fsm import ( "github.com/hashicorp/go-msgpack/codec" "github.com/hashicorp/raft" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/structs" ) func init() { registerPersister(persistOSS) registerRestorer(structs.RegisterRequestType, restoreRegistration) registerRestorer(structs.KVSRequestType, restoreKV) registerRestorer(structs.TombstoneRequestType, restoreTombstone) registerRestorer(structs.SessionRequestType, restoreSession) registerRestorer(structs.DeprecatedACLRequestType, restoreACL) // TODO(ACL-Legacy-Compat) - remove in phase 2 registerRestorer(structs.ACLBootstrapRequestType, restoreACLBootstrap) registerRestorer(structs.CoordinateBatchUpdateType, restoreCoordinates) registerRestorer(structs.PreparedQueryRequestType, restorePreparedQuery) registerRestorer(structs.AutopilotRequestType, restoreAutopilot) registerRestorer(structs.IntentionRequestType, restoreLegacyIntention) registerRestorer(structs.ConnectCARequestType, restoreConnectCA) registerRestorer(structs.ConnectCAProviderStateType, restoreConnectCAProviderState) registerRestorer(structs.ConnectCAConfigType, restoreConnectCAConfig) registerRestorer(structs.IndexRequestType, restoreIndex) registerRestorer(structs.ACLTokenSetRequestType, restoreToken) registerRestorer(structs.ACLPolicySetRequestType, restorePolicy) registerRestorer(structs.ConfigEntryRequestType, restoreConfigEntry) registerRestorer(structs.ACLRoleSetRequestType, restoreRole) registerRestorer(structs.ACLBindingRuleSetRequestType, restoreBindingRule) registerRestorer(structs.ACLAuthMethodSetRequestType, restoreAuthMethod) registerRestorer(structs.FederationStateRequestType, restoreFederationState) registerRestorer(structs.SystemMetadataRequestType, restoreSystemMetadata) } func persistOSS(s *snapshot, sink raft.SnapshotSink, encoder *codec.Encoder) error { if err := s.persistNodes(sink, encoder); err != nil { return err } if err := s.persistSessions(sink, encoder); err != nil { return err } if err := s.persistACLs(sink, encoder); err != nil { return err } if err := s.persistKVs(sink, encoder); err != nil { return err } if err := s.persistTombstones(sink, encoder); err != nil { return err } if err := s.persistPreparedQueries(sink, encoder); err != nil { return err } if err := s.persistAutopilot(sink, encoder); err != nil { return err } if err := s.persistLegacyIntentions(sink, encoder); err != nil { return err } if err := s.persistConnectCA(sink, encoder); err != nil { return err } if err := s.persistConnectCAProviderState(sink, encoder); err != nil { return err } if err := s.persistConnectCAConfig(sink, encoder); err != nil { return err } if err := s.persistConfigEntries(sink, encoder); err != nil { return err } if err := s.persistFederationStates(sink, encoder); err != nil { return err } if err := s.persistSystemMetadata(sink, encoder); err != nil { return err } if err := s.persistIndex(sink, encoder); err != nil { return err } return nil } func (s *snapshot) persistNodes(sink raft.SnapshotSink, encoder *codec.Encoder) error { // Get all the nodes nodes, err := s.state.Nodes() if err != nil { return err } // Register each node for node := nodes.Next(); node != nil; node = nodes.Next() { n := node.(*structs.Node) nodeEntMeta := n.GetEnterpriseMeta() req := structs.RegisterRequest{ ID: n.ID, Node: n.Node, Datacenter: n.Datacenter, Address: n.Address, TaggedAddresses: n.TaggedAddresses, NodeMeta: n.Meta, RaftIndex: n.RaftIndex, EnterpriseMeta: *nodeEntMeta, } // Register the node itself if _, err := sink.Write([]byte{byte(structs.RegisterRequestType)}); err != nil { return err } if err := encoder.Encode(&req); err != nil { return err } // Register each service this node has services, err := s.state.Services(n.Node, nodeEntMeta) if err != nil { return err } for service := services.Next(); service != nil; service = services.Next() { if _, err := sink.Write([]byte{byte(structs.RegisterRequestType)}); err != nil { return err } req.Service = service.(*structs.ServiceNode).ToNodeService() if err := encoder.Encode(&req); err != nil { return err } } // Register each check this node has req.Service = nil checks, err := s.state.Checks(n.Node, nodeEntMeta) if err != nil { return err } for check := checks.Next(); check != nil; check = checks.Next() { if _, err := sink.Write([]byte{byte(structs.RegisterRequestType)}); err != nil { return err } req.Check = check.(*structs.HealthCheck) if err := encoder.Encode(&req); err != nil { return err } } } // Save the coordinates separately since they are not part of the // register request interface. To avoid copying them out, we turn // them into batches with a single coordinate each. coords, err := s.state.Coordinates() if err != nil { return err } // TODO(partitions) for coord := coords.Next(); coord != nil; coord = coords.Next() { if _, err := sink.Write([]byte{byte(structs.CoordinateBatchUpdateType)}); err != nil { return err } updates := structs.Coordinates{coord.(*structs.Coordinate)} if err := encoder.Encode(&updates); err != nil { return err } } return nil } func (s *snapshot) persistSessions(sink raft.SnapshotSink, encoder *codec.Encoder) error { sessions, err := s.state.Sessions() if err != nil { return err } for session := sessions.Next(); session != nil; session = sessions.Next() { if _, err := sink.Write([]byte{byte(structs.SessionRequestType)}); err != nil { return err } if err := encoder.Encode(session.(*structs.Session)); err != nil { return err } } return nil } func (s *snapshot) persistACLs(sink raft.SnapshotSink, encoder *codec.Encoder) error { tokens, err := s.state.ACLTokens() if err != nil { return err } // Don't check expiration times. Wait for explicit deletions. for token := tokens.Next(); token != nil; token = tokens.Next() { if _, err := sink.Write([]byte{byte(structs.ACLTokenSetRequestType)}); err != nil { return err } if err := encoder.Encode(token.(*structs.ACLToken)); err != nil { return err } } policies, err := s.state.ACLPolicies() if err != nil { return err } for policy := policies.Next(); policy != nil; policy = policies.Next() { if _, err := sink.Write([]byte{byte(structs.ACLPolicySetRequestType)}); err != nil { return err } if err := encoder.Encode(policy.(*structs.ACLPolicy)); err != nil { return err } } roles, err := s.state.ACLRoles() if err != nil { return err } for role := roles.Next(); role != nil; role = roles.Next() { if _, err := sink.Write([]byte{byte(structs.ACLRoleSetRequestType)}); err != nil { return err } if err := encoder.Encode(role.(*structs.ACLRole)); err != nil { return err } } rules, err := s.state.ACLBindingRules() if err != nil { return err } for rule := rules.Next(); rule != nil; rule = rules.Next() { if _, err := sink.Write([]byte{byte(structs.ACLBindingRuleSetRequestType)}); err != nil { return err } if err := encoder.Encode(rule.(*structs.ACLBindingRule)); err != nil { return err } } methods, err := s.state.ACLAuthMethods() if err != nil { return err } for method := methods.Next(); method != nil; method = methods.Next() { if _, err := sink.Write([]byte{byte(structs.ACLAuthMethodSetRequestType)}); err != nil { return err } if err := encoder.Encode(method.(*structs.ACLAuthMethod)); err != nil { return err } } return nil } func (s *snapshot) persistKVs(sink raft.SnapshotSink, encoder *codec.Encoder) error { entries, err := s.state.KVs() if err != nil { return err } for entry := entries.Next(); entry != nil; entry = entries.Next() { if _, err := sink.Write([]byte{byte(structs.KVSRequestType)}); err != nil { return err } if err := encoder.Encode(entry.(*structs.DirEntry)); err != nil { return err } } return nil } func (s *snapshot) persistTombstones(sink raft.SnapshotSink, encoder *codec.Encoder) error { stones, err := s.state.Tombstones() if err != nil { return err } for stone := stones.Next(); stone != nil; stone = stones.Next() { if _, err := sink.Write([]byte{byte(structs.TombstoneRequestType)}); err != nil { return err } // For historical reasons, these are serialized in the snapshots // as KV entries. We want to keep the snapshot format compatible // with pre-0.6 versions for now. s := stone.(*state.Tombstone) fake := &structs.DirEntry{ Key: s.Key, RaftIndex: structs.RaftIndex{ ModifyIndex: s.Index, }, } if err := encoder.Encode(fake); err != nil { return err } } return nil } func (s *snapshot) persistPreparedQueries(sink raft.SnapshotSink, encoder *codec.Encoder) error { queries, err := s.state.PreparedQueries() if err != nil { return err } for _, query := range queries { if _, err := sink.Write([]byte{byte(structs.PreparedQueryRequestType)}); err != nil { return err } if err := encoder.Encode(query); err != nil { return err } } return nil } func (s *snapshot) persistAutopilot(sink raft.SnapshotSink, encoder *codec.Encoder) error { config, err := s.state.Autopilot() if err != nil { return err } // Make sure we don't write a nil config out to a snapshot. if config == nil { return nil } if _, err := sink.Write([]byte{byte(structs.AutopilotRequestType)}); err != nil { return err } if err := encoder.Encode(config); err != nil { return err } return nil } func (s *snapshot) persistConnectCA(sink raft.SnapshotSink, encoder *codec.Encoder) error { roots, err := s.state.CARoots() if err != nil { return err } for _, r := range roots { if _, err := sink.Write([]byte{byte(structs.ConnectCARequestType)}); err != nil { return err } if err := encoder.Encode(r); err != nil { return err } } return nil } func (s *snapshot) persistConnectCAConfig(sink raft.SnapshotSink, encoder *codec.Encoder) error { config, err := s.state.CAConfig() if err != nil { return err } // Make sure we don't write a nil config out to a snapshot. if config == nil { return nil } if _, err := sink.Write([]byte{byte(structs.ConnectCAConfigType)}); err != nil { return err } if err := encoder.Encode(config); err != nil { return err } return nil } func (s *snapshot) persistConnectCAProviderState(sink raft.SnapshotSink, encoder *codec.Encoder) error { state, err := s.state.CAProviderState() if err != nil { return err } for _, r := range state { if _, err := sink.Write([]byte{byte(structs.ConnectCAProviderStateType)}); err != nil { return err } if err := encoder.Encode(r); err != nil { return err } } return nil } func (s *snapshot) persistLegacyIntentions(sink raft.SnapshotSink, encoder *codec.Encoder) error { //nolint:staticcheck ixns, err := s.state.LegacyIntentions() if err != nil { return err } for _, ixn := range ixns { if _, err := sink.Write([]byte{byte(structs.IntentionRequestType)}); err != nil { return err } if err := encoder.Encode(ixn); err != nil { return err } } return nil } func (s *snapshot) persistConfigEntries(sink raft.SnapshotSink, encoder *codec.Encoder) error { entries, err := s.state.ConfigEntries() if err != nil { return err } for _, entry := range entries { if _, err := sink.Write([]byte{byte(structs.ConfigEntryRequestType)}); err != nil { return err } // Encode the entry request without an operation since we don't need it for restoring. // The request is used for its custom decoding/encoding logic around the ConfigEntry // interface. req := &structs.ConfigEntryRequest{ Entry: entry, } if err := encoder.Encode(req); err != nil { return err } } return nil } func (s *snapshot) persistFederationStates(sink raft.SnapshotSink, encoder *codec.Encoder) error { fedStates, err := s.state.FederationStates() if err != nil { return err } for _, fedState := range fedStates { if _, err := sink.Write([]byte{byte(structs.FederationStateRequestType)}); err != nil { return err } // Encode the entry request without an operation since we don't need it for restoring. // The request is used for its custom decoding/encoding logic around the ConfigEntry // interface. req := &structs.FederationStateRequest{ Op: structs.FederationStateUpsert, State: fedState, } if err := encoder.Encode(req); err != nil { return err } } return nil } func (s *snapshot) persistSystemMetadata(sink raft.SnapshotSink, encoder *codec.Encoder) error { entries, err := s.state.SystemMetadataEntries() if err != nil { return err } for _, entry := range entries { if _, err := sink.Write([]byte{byte(structs.SystemMetadataRequestType)}); err != nil { return err } if err := encoder.Encode(entry); err != nil { return err } } return nil } func (s *snapshot) persistIndex(sink raft.SnapshotSink, encoder *codec.Encoder) error { // Get all the indexes iter, err := s.state.Indexes() if err != nil { return err } for raw := iter.Next(); raw != nil; raw = iter.Next() { // Prepare the request struct idx := raw.(*state.IndexEntry) // Write out a node registration sink.Write([]byte{byte(structs.IndexRequestType)}) if err := encoder.Encode(idx); err != nil { return err } } return nil } func restoreRegistration(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.RegisterRequest if err := decoder.Decode(&req); err != nil { return err } if err := restore.Registration(header.LastIndex, &req); err != nil { return err } return nil } func restoreKV(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.DirEntry if err := decoder.Decode(&req); err != nil { return err } if err := restore.KVS(&req); err != nil { return err } return nil } func restoreTombstone(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.DirEntry if err := decoder.Decode(&req); err != nil { return err } // For historical reasons, these are serialized in the // snapshots as KV entries. We want to keep the snapshot // format compatible with pre-0.6 versions for now. stone := &state.Tombstone{ Key: req.Key, Index: req.ModifyIndex, } if err := restore.Tombstone(stone); err != nil { return err } return nil } func restoreSession(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.Session if err := decoder.Decode(&req); err != nil { return err } if err := restore.Session(&req); err != nil { return err } return nil } // TODO(ACL-Legacy-Compat) - remove in phase 2 func restoreACL(_ *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req LegacyACL if err := decoder.Decode(&req); err != nil { return err } if err := restore.ACLToken(req.Convert()); err != nil { return err } return nil } // TODO(ACL-Legacy-Compat) - remove in phase 2 type LegacyACL struct { ID string Name string Type string Rules string structs.RaftIndex } // TODO(ACL-Legacy-Compat): remove in phase 2, used by snapshot restore func (a LegacyACL) Convert() *structs.ACLToken { correctedRules := structs.SanitizeLegacyACLTokenRules(a.Rules) if correctedRules != "" { a.Rules = correctedRules } token := &structs.ACLToken{ AccessorID: "", SecretID: a.ID, Description: a.Name, Policies: nil, ServiceIdentities: nil, NodeIdentities: nil, Type: a.Type, Rules: a.Rules, Local: false, RaftIndex: a.RaftIndex, } token.SetHash(true) return token } // TODO(ACL-Legacy-Compat) - remove in phase 2 func restoreACLBootstrap(_ *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { type ACLBootstrap struct { // AllowBootstrap will only be true if no existing management tokens // have been found. AllowBootstrap bool structs.RaftIndex } var req ACLBootstrap if err := decoder.Decode(&req); err != nil { return err } // With V2 ACLs whether bootstrapping has been performed is stored in the index table like nomad // so this "restores" into that index table. return restore.IndexRestore(&state.IndexEntry{Key: "acl-token-bootstrap", Value: req.ModifyIndex}) } func restoreCoordinates(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.Coordinates if err := decoder.Decode(&req); err != nil { return err } if err := restore.Coordinates(header.LastIndex, req); err != nil { return err } return nil } func restorePreparedQuery(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.PreparedQuery if err := decoder.Decode(&req); err != nil { return err } if err := restore.PreparedQuery(&req); err != nil { return err } return nil } func restoreAutopilot(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.AutopilotConfig if err := decoder.Decode(&req); err != nil { return err } if err := restore.Autopilot(&req); err != nil { return err } return nil } func restoreLegacyIntention(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.Intention if err := decoder.Decode(&req); err != nil { return err } //nolint:staticcheck if err := restore.LegacyIntention(&req); err != nil { return err } return nil } func restoreConnectCA(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.CARoot if err := decoder.Decode(&req); err != nil { return err } if err := restore.CARoot(&req); err != nil { return err } return nil } func restoreConnectCAProviderState(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.CAConsulProviderState if err := decoder.Decode(&req); err != nil { return err } if err := restore.CAProviderState(&req); err != nil { return err } return nil } func restoreConnectCAConfig(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.CAConfiguration if err := decoder.Decode(&req); err != nil { return err } if err := restore.CAConfig(&req); err != nil { return err } return nil } func restoreIndex(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req state.IndexEntry if err := decoder.Decode(&req); err != nil { return err } return restore.IndexRestore(&req) } func restoreToken(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.ACLToken if err := decoder.Decode(&req); err != nil { return err } // DEPRECATED (ACL-Legacy-Compat) if req.Rules != "" { // When we restore a snapshot we may have to correct old HCL in legacy // tokens to prevent the in-memory representation from using an older // syntax. structs.SanitizeLegacyACLToken(&req) } // only set if unset - mitigates a bug where converted legacy tokens could end up without a hash req.SetHash(false) return restore.ACLToken(&req) } func restorePolicy(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.ACLPolicy if err := decoder.Decode(&req); err != nil { return err } return restore.ACLPolicy(&req) } func restoreConfigEntry(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.ConfigEntryRequest if err := decoder.Decode(&req); err != nil { return err } return restore.ConfigEntry(req.Entry) } func restoreRole(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.ACLRole if err := decoder.Decode(&req); err != nil { return err } return restore.ACLRole(&req) } func restoreBindingRule(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.ACLBindingRule if err := decoder.Decode(&req); err != nil { return err } return restore.ACLBindingRule(&req) } func restoreAuthMethod(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.ACLAuthMethod if err := decoder.Decode(&req); err != nil { return err } return restore.ACLAuthMethod(&req) } func restoreFederationState(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.FederationStateRequest if err := decoder.Decode(&req); err != nil { return err } return restore.FederationState(req.State) } func restoreSystemMetadata(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error { var req structs.SystemMetadataEntry if err := decoder.Decode(&req); err != nil { return err } return restore.SystemMetadataEntry(&req) }