From f2615992a455748f6625e6d8a0ed45e2b3a669ce Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 16 Dec 2021 10:32:11 -0500 Subject: [PATCH] cli: unhide advanced operator raft debugging commands (#11682) The `nomad operator raft` and `nomad operator snapshot state` subcommands for inspecting on-disk raft state were hidden and undocumented. Expose and document these so that advanced operators have support for these tools. --- .changelog/11682.txt | 3 ++ command/commands.go | 8 ++-- command/operator_raft.go | 15 ++++++ command/operator_raft_info.go | 12 +++-- command/operator_raft_logs.go | 12 +++-- command/operator_raft_state.go | 13 ++++-- command/operator_snapshot_state.go | 6 +-- .../docs/commands/operator/raft-info.mdx | 39 ++++++++++++++++ .../commands/operator/raft-list-peers.mdx | 2 +- .../docs/commands/operator/raft-logs.mdx | 38 +++++++++++++++ .../docs/commands/operator/raft-state.mdx | 46 +++++++++++++++++++ .../docs/commands/operator/snapshot-state.mdx | 30 ++++++++++++ website/data/docs-nav-data.json | 16 +++++++ 13 files changed, 218 insertions(+), 22 deletions(-) create mode 100644 .changelog/11682.txt create mode 100644 website/content/docs/commands/operator/raft-info.mdx create mode 100644 website/content/docs/commands/operator/raft-logs.mdx create mode 100644 website/content/docs/commands/operator/raft-state.mdx create mode 100644 website/content/docs/commands/operator/snapshot-state.mdx diff --git a/.changelog/11682.txt b/.changelog/11682.txt new file mode 100644 index 000000000..f667fec3e --- /dev/null +++ b/.changelog/11682.txt @@ -0,0 +1,3 @@ +```release-note:improvement +cli: Made the `operator raft info`, `operator raft logs`, `operator raft state`, and `operator snapshot state` commands visible to command line help. +``` diff --git a/command/commands.go b/command/commands.go index 9b58cdd8f..52075364e 100644 --- a/command/commands.go +++ b/command/commands.go @@ -540,17 +540,17 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, - "operator raft _info": func() (cli.Command, error) { + "operator raft info": func() (cli.Command, error) { return &OperatorRaftInfoCommand{ Meta: meta, }, nil }, - "operator raft _logs": func() (cli.Command, error) { + "operator raft logs": func() (cli.Command, error) { return &OperatorRaftLogsCommand{ Meta: meta, }, nil }, - "operator raft _state": func() (cli.Command, error) { + "operator raft state": func() (cli.Command, error) { return &OperatorRaftStateCommand{ Meta: meta, }, nil @@ -571,7 +571,7 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, - "operator snapshot _state": func() (cli.Command, error) { + "operator snapshot state": func() (cli.Command, error) { return &OperatorSnapshotStateCommand{ Meta: meta, }, nil diff --git a/command/operator_raft.go b/command/operator_raft.go index 79e0cb648..a1aac4875 100644 --- a/command/operator_raft.go +++ b/command/operator_raft.go @@ -26,7 +26,22 @@ Usage: nomad operator raft [options] $ nomad operator raft remove-peer -peer-address "IP:Port" + Display info about the raft logs in the data directory: + + $ nomad operator raft info /var/nomad/data + + Display the log entries persisted in data dir in JSON format. + + $ nomad operator raft logs /var/nomad/data + + Display the server state obtained by replaying raft log entries + persisted in data dir in JSON format. + + $ nomad operator raft state /var/nomad/data + Please see the individual subcommand help for detailed usage information. + + ` return strings.TrimSpace(helpText) } diff --git a/command/operator_raft_info.go b/command/operator_raft_info.go index 8afe73023..18622e810 100644 --- a/command/operator_raft_info.go +++ b/command/operator_raft_info.go @@ -14,14 +14,16 @@ type OperatorRaftInfoCommand struct { func (c *OperatorRaftInfoCommand) Help() string { helpText := ` -Usage: nomad operator raft _info +Usage: nomad operator raft info - Displays info about the raft logs in the data directory. + Displays summary information about the raft logs in the data directory. + + This command requires file system permissions to access the data directory on + disk. The Nomad server locks access to the data directory, so this command + cannot be run on a data directory that is being used by a running Nomad server. This is a low-level debugging tool and not subject to Nomad's usual backward compatibility guarantees. - - If ACLs are enabled, this command requires a management token. ` return strings.TrimSpace(helpText) } @@ -38,7 +40,7 @@ func (c *OperatorRaftInfoCommand) Synopsis() string { return "Display info of the raft log" } -func (c *OperatorRaftInfoCommand) Name() string { return "operator raft _info" } +func (c *OperatorRaftInfoCommand) Name() string { return "operator raft info" } func (c *OperatorRaftInfoCommand) Run(args []string) int { if len(args) != 1 { diff --git a/command/operator_raft_logs.go b/command/operator_raft_logs.go index e85649c4e..fe3233ee6 100644 --- a/command/operator_raft_logs.go +++ b/command/operator_raft_logs.go @@ -16,14 +16,18 @@ type OperatorRaftLogsCommand struct { func (c *OperatorRaftLogsCommand) Help() string { helpText := ` -Usage: nomad operator raft _logs +Usage: nomad operator raft logs - Display the log entries persisted in data dir in json form. + Display the log entries persisted in the Nomad data directory in JSON + format. + + This command requires file system permissions to access the data directory on + disk. The Nomad server locks access to the data directory, so this command + cannot be run on a data directory that is being used by a running Nomad server. This is a low-level debugging tool and not subject to Nomad's usual backward compatibility guarantees. - If ACLs are enabled, this command requires a management token. ` return strings.TrimSpace(helpText) } @@ -40,7 +44,7 @@ func (c *OperatorRaftLogsCommand) Synopsis() string { return "Display raft log content" } -func (c *OperatorRaftLogsCommand) Name() string { return "operator raft _info" } +func (c *OperatorRaftLogsCommand) Name() string { return "operator raft logs" } func (c *OperatorRaftLogsCommand) Run(args []string) int { if len(args) != 1 { diff --git a/command/operator_raft_state.go b/command/operator_raft_state.go index 2b70a329f..a116ae97f 100644 --- a/command/operator_raft_state.go +++ b/command/operator_raft_state.go @@ -16,15 +16,18 @@ type OperatorRaftStateCommand struct { func (c *OperatorRaftStateCommand) Help() string { helpText := ` -Usage: nomad operator raft _state +Usage: nomad operator raft state - Display the server state obtained by replaying raft log entries persisted in data dir in json form. + Display the server state obtained by replaying raft log entries persisted in + the Nomad data directory in JSON format. + + This command requires file system permissions to access the data directory on + disk. The Nomad server locks access to the data directory, so this command + cannot be run on a data directory that is being used by a running Nomad server. This is a low-level debugging tool and not subject to Nomad's usual backward compatibility guarantees. - If ACLs are enabled, this command requires a management token. - Options: -last-index= @@ -47,7 +50,7 @@ func (c *OperatorRaftStateCommand) Synopsis() string { return "Display raft server state" } -func (c *OperatorRaftStateCommand) Name() string { return "operator raft _state" } +func (c *OperatorRaftStateCommand) Name() string { return "operator raft state" } func (c *OperatorRaftStateCommand) Run(args []string) int { var fLastIdx int64 diff --git a/command/operator_snapshot_state.go b/command/operator_snapshot_state.go index ed7b66346..5ba28a56c 100644 --- a/command/operator_snapshot_state.go +++ b/command/operator_snapshot_state.go @@ -16,13 +16,13 @@ type OperatorSnapshotStateCommand struct { func (c *OperatorSnapshotStateCommand) Help() string { helpText := ` -Usage: nomad operator snapshot _state +Usage: nomad operator snapshot state Displays a JSON representation of state in the snapshot. To inspect the file "backup.snap": - $ nomad operator snapshot _state backup.snap + $ nomad operator snapshot state backup.snap ` return strings.TrimSpace(helpText) } @@ -39,7 +39,7 @@ func (c *OperatorSnapshotStateCommand) Synopsis() string { return "Displays information about a Nomad snapshot file" } -func (c *OperatorSnapshotStateCommand) Name() string { return "operator snapshot _state" } +func (c *OperatorSnapshotStateCommand) Name() string { return "operator snapshot state" } func (c *OperatorSnapshotStateCommand) Run(args []string) int { // Check that we either got no filename or exactly one. diff --git a/website/content/docs/commands/operator/raft-info.mdx b/website/content/docs/commands/operator/raft-info.mdx new file mode 100644 index 000000000..fc1754346 --- /dev/null +++ b/website/content/docs/commands/operator/raft-info.mdx @@ -0,0 +1,39 @@ +--- +layout: docs +page_title: 'Commands: operator raft info' +description: | + Display Raft server state. +--- + +# Command: operator raft info + +The `raft info` command is used to display summary information about the +raft logs persisted in the Nomad [data directory]. + +This command requires file system permissions to access the data +directory on disk. The Nomad server locks access to the data +directory, so this command cannot be run on a data directory that is +being used by a running Nomad server. + +~> **Warning:** This is a low-level debugging tool and not subject to + Nomad's usual backward compatibility guarantees. + +## Usage + +```plaintext +nomad operator raft info +``` + +## Examples + +An example output is as follows: + +```shell-session +$ sudo nomad operator raft info /var/nomad/data +path: /var/nomad/data/server/raft/raft.db +length: 10 +first index: 1 +last index: 10 +``` + +[data directory]: /docs/configuration#data_dir diff --git a/website/content/docs/commands/operator/raft-list-peers.mdx b/website/content/docs/commands/operator/raft-list-peers.mdx index f0c44ef63..63fee11d1 100644 --- a/website/content/docs/commands/operator/raft-list-peers.mdx +++ b/website/content/docs/commands/operator/raft-list-peers.mdx @@ -7,7 +7,7 @@ description: | # Command: operator raft list-peers -The Raft list-peers command is used to display the current Raft peer +The `raft list-peers` command is used to display the current Raft peer configuration. See the [Outage Recovery] guide for some examples of how this command is used. diff --git a/website/content/docs/commands/operator/raft-logs.mdx b/website/content/docs/commands/operator/raft-logs.mdx new file mode 100644 index 000000000..ec8c2a53f --- /dev/null +++ b/website/content/docs/commands/operator/raft-logs.mdx @@ -0,0 +1,38 @@ +--- +layout: docs +page_title: 'Commands: operator raft logs' +description: | + Display Raft server state. +--- + +# Command: operator raft logs + +The `raft logs` command is used to display the log entries persisted in +the Nomad [data directory] in JSON format. + +This command requires file system permissions to access the data +directory on disk. The Nomad server locks access to the data +directory, so this command cannot be run on a data directory that is +being used by a running Nomad server. + +~> **Warning:** This is a low-level debugging tool and not subject to + Nomad's usual backward compatibility guarantees. + +## Usage + +```plaintext +nomad operator raft logs [options] +``` + +## Examples + +The output of this command can be very large, so it's recommended that +you redirect the output to a file for later examination with other +tools. + +```shell-session +$ sudo nomad operator raft logs /var/nomad/data > ~/raft-logs.json +$ jq . < ~/raft-logs.json +``` + +[data directory]: /docs/configuration#data_dir diff --git a/website/content/docs/commands/operator/raft-state.mdx b/website/content/docs/commands/operator/raft-state.mdx new file mode 100644 index 000000000..e97fb8949 --- /dev/null +++ b/website/content/docs/commands/operator/raft-state.mdx @@ -0,0 +1,46 @@ +--- +layout: docs +page_title: 'Commands: operator raft state' +description: | + Display Raft server state. +--- + +# Command: operator raft state + +The `raft state` command is used to display the server state obtained by +replaying raft log entries persisted in the Nomad [data directory] in +JSON format. + +This command requires file system permissions to access the data +directory on disk. The Nomad server locks access to the data +directory, so this command cannot be run on a data directory that is +being used by a running Nomad server. + +~> **Warning:** This is a low-level debugging tool and not subject to + Nomad's usual backward compatibility guarantees. + +## Usage + +```plaintext +nomad operator raft state [options] +``` + +## Raft State Options + +- `-last-index=`: Set the last log index to be applied, to + drop spurious log entries not properly committed. If the + `last_index` option is zero or negative, it's treated as an offset + from the last index seen in raft. + +## Examples + +The output of this command can be very large, so it's recommended that +you redirect the output to a file for later examination with other +tools. + +```shell-session +$ sudo nomad operator raft state /var/nomad/data > ~/raft-state.json +$ jq . < ~/raft-state.json +``` + +[data directory]: /docs/configuration#data_dir diff --git a/website/content/docs/commands/operator/snapshot-state.mdx b/website/content/docs/commands/operator/snapshot-state.mdx new file mode 100644 index 000000000..e944c049b --- /dev/null +++ b/website/content/docs/commands/operator/snapshot-state.mdx @@ -0,0 +1,30 @@ +--- +layout: docs +page_title: 'Commands: operator snapshot state' +description: | + Displays a JSON representation of a Raft snapshot. +--- + +# Command: operator snapshot state + +Displays a JSON representation of state in a raft snapshot on disk. + +~> **Warning:** This is a low-level debugging tool and not subject to + Nomad's usual backward compatibility guarantees. + +## Usage + +```plaintext +nomad operator snapshot state +``` + +## Examples + +The output of this command can be very large, so it's recommended that +you redirect the output to a file for later examination with other +tools. + +```shell-session +$ nomad operator snapshot state backup.snap > ~/raft-state.json +$ jq . < ~/raft-state.json +``` diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 28f67624c..9985a84d9 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -548,14 +548,26 @@ "title": "metrics", "path": "commands/operator/metrics" }, + { + "title": "raft info", + "path": "commands/operator/raft-info" + }, { "title": "raft list-peers", "path": "commands/operator/raft-list-peers" }, + { + "title": "raft logs", + "path": "commands/operator/raft-logs" + }, { "title": "raft remove-peer", "path": "commands/operator/raft-remove-peer" }, + { + "title": "raft state", + "path": "commands/operator/raft-state" + }, { "title": "snapshot agent", "path": "commands/operator/snapshot-agent" @@ -571,6 +583,10 @@ { "title": "snapshot save", "path": "commands/operator/snapshot-save" + }, + { + "title": "snapshot state", + "path": "commands/operator/snapshot-state" } ] },