luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

CLI: Allow snapshot inspect to work on internal raft snapshots directly. (#10089) 

* CLI: Add support for reading internal raft snapshots to snapshot inspect

* Add snapshot inspect test for raw state files

* Add changelog entry

* Update .changelog/10089.txt
This commit is contained in:
Paul Banks 2021-04-23 16:17:08 +01:00 committed by GitHub
parent d605632216
commit 5c409739c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 116 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.changelog/10089.txt Normal file
View File

@ -0,0 +1,4 @@
```release-note:improvement
cli: snapshot inspect command can now inspect raw snapshots from a server's data
dir.
```

48
command/snapshot/inspect/snapshot_inspect.go
View File

@ -1,10 +1,13 @@
package inspect
import (
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"sort"
"strings"
@ -111,18 +114,41 @@ func (c *cmd) Run(args []string) int {
}
defer f.Close()
readFile, meta, err := snapshot.Read(hclog.New(nil), f)
if err != nil {
c.UI.Error(fmt.Sprintf("Error reading snapshot: %s", err))
var readFile *os.File
var meta *raft.SnapshotMeta
if strings.ToLower(path.Base(file)) == "state.bin" {
// This is an internal raw raft snapshot not a gzipped archive one
// downloaded from the API, we can read it directly
readFile = f
// Assume the meta is colocated and error if not.
metaRaw, err := ioutil.ReadFile(path.Join(path.Dir(file), "meta.json"))
if err != nil {
c.UI.Error(fmt.Sprintf("Error reading meta.json from internal snapshot dir: %s", err))
return 1
}
var metaDecoded raft.SnapshotMeta
err = json.Unmarshal(metaRaw, &metaDecoded)
if err != nil {
c.UI.Error(fmt.Sprintf("Error parsing meta.json from internal snapshot dir: %s", err))
return 1
}
meta = &metaDecoded
} else {
readFile, meta, err = snapshot.Read(hclog.New(nil), f)
if err != nil {
c.UI.Error(fmt.Sprintf("Error reading snapshot: %s", err))
}
defer func() {
if err := readFile.Close(); err != nil {
c.UI.Error(fmt.Sprintf("Failed to close temp snapshot: %v", err))
}
if err := os.Remove(readFile.Name()); err != nil {
c.UI.Error(fmt.Sprintf("Failed to clean up temp snapshot: %v", err))
}
}()
}
defer func() {
if err := readFile.Close(); err != nil {
c.UI.Error(fmt.Sprintf("Failed to close temp snapshot: %v", err))
}
if err := os.Remove(readFile.Name()); err != nil {
c.UI.Error(fmt.Sprintf("Failed to clean up temp snapshot: %v", err))
}
}()
info, err := c.enhance(readFile)
if err != nil {

20
command/snapshot/inspect/snapshot_inspect_test.go
View File

@ -149,3 +149,23 @@ func TestSnapshotInspectKVDetailsDepthFilterCommand(t *testing.T) {
want := golden(t, t.Name(), ui.OutputWriter.String())
require.Equal(t, want, ui.OutputWriter.String())
}
// TestSnapshotInspectCommandRaw test reading a snaphost directly from a raft
// data dir.
func TestSnapshotInspectCommandRaw(t *testing.T) {
filepath := "./testdata/raw/state.bin"
// Inspect the snapshot
ui := cli.NewMockUi()
c := New(ui)
args := []string{filepath}
code := c.Run(args)
if code != 0 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
want := golden(t, t.Name(), ui.OutputWriter.String())
require.Equal(t, want, ui.OutputWriter.String())
}

19
command/snapshot/inspect/testdata/TestSnapshotInspectCommandRaw.golden vendored Normal file
View File

@ -0,0 +1,19 @@
ID 2-13-1602222343947
Size 5141
Index 13
Term 2
Version 1
Type Count Size
---- ---- ----
Register 3 1.7KB
ConnectCA 1 1.2KB
ConnectCAProviderState 1 1.1KB
Index 12 344B
Autopilot 1 199B
ConnectCAConfig 1 197B
FederationState 1 139B
SystemMetadata 1 68B
ChunkingState 1 12B
---- ---- ----
Total 5KB

1
command/snapshot/inspect/testdata/raw/meta.json vendored Normal file
View File

@ -0,0 +1 @@
{"Version":1,"ID":"2-13-1602222343947","Index":13,"Term":2,"Peers":"ka4xMjcuMC4wLjE6ODMwMA==","Configuration":{"Servers":[{"Suffrage":0,"ID":"a577b288-b354-770e-e909-da0972eb20e8","Address":"127.0.0.1:8300"}]},"ConfigurationIndex":1,"Size":5141}

BIN
command/snapshot/inspect/testdata/raw/state.bin (Stored with Git LFS) vendored Normal file
View File

Binary file not shown.

32
website/content/commands/snapshot/inspect.mdx
View File

@ -12,6 +12,15 @@ snapshot of the state of the Consul servers which includes key/value entries,
service catalog, prepared queries, sessions, and ACLs. The snapshot is read
from the given file.
-> Typically this is used with Consul self-contained Snapshot files obtained
using the [`consul snapshot`](/commands/snapshot) command or [Snapshot
API](/api-docs/snapshot#generate-snapshot). If the file provided is named
`state.bin` however, the command will assume it is a raw raft snapshot in a
Consul server data directory and will attempt to read it directly. The
`state.bin` file must still be in the same directory as it's associated
`meta.json` file. This is useful for debugging data on live servers without
making a complete new snapshot via the CLI or API first.
The following fields are displayed when inspecting a snapshot:
- `ID` - A unique ID for the snapshot, only used for differentiation purposes.
@ -106,6 +115,29 @@ $ consul snapshot inspect -kvdetails -kvdepth 3 -kvfilter vault/core backup.snap
Please see the [HTTP API](/api/snapshot) documentation for
more details about snapshot internals.
To inspect an internal snapshot directly from a Consul server data directory:
```shell-session
$ consul snapshot inspect /opt/consul/raft/snapshots/9-4600669-1618935304715/state.bin
ID 9-4600669-1618935304715
Size 4625420898
Index 4600669
Term 9
Version 1
Type Count Size
---- ---- ----
KVS 4089785 4.3GB
Register 9 5.2KB
CoordinateBatchUpdate 3 465B
Index 8 224B
Autopilot 1 199B
FederationState 1 139B
ChunkingState 1 12B
---- ---- ----
Total 4.3GB
```
#### Command Options
- `-kvdetails` - Optional, provides a space usage breakdown for any KV data stored in Consul.