luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Merge pull request #9098 from hashicorp/watsonian/kv-size-breakdown 

Add detailed key size breakdown to snapshot inspect
This commit is contained in:
Joel Watson 2020-11-11 11:34:45 -06:00 committed by GitHub
parent aac3729a6c ff0009bd91
commit 4b9034b976
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 426 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/9098.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:feature
cli: snapshot inspect command provides KV usage breakdown
```

23
command/snapshot/inspect/formatter.go
View File

@ -48,18 +48,31 @@ func (_ *prettyFormatter) Format(info *OutputFormat) (string, error) {
fmt.Fprintf(tw, "\n Term\t%d", info.Meta.Term)
fmt.Fprintf(tw, "\n Version\t%d", info.Meta.Version)
fmt.Fprintf(tw, "\n")
fmt.Fprintln(tw, "\n Type\tCount\tSize\t")
fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----")
fmt.Fprintln(tw, "\n Type\tCount\tSize")
fmt.Fprintf(tw, " %s\t%s\t%s", "----", "----", "----")
// For each different type generate new output
for _, s := range info.Stats {
fmt.Fprintf(tw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum)))
fmt.Fprintf(tw, "\n %s\t%d\t%s", s.Name, s.Count, ByteSize(uint64(s.Sum)))
}
fmt.Fprintf(tw, "\n %s\t%s\t%s", "----", "----", "----")
fmt.Fprintf(tw, "\n Total\t\t%s", ByteSize(uint64(info.TotalSize)))
if info.StatsKV != nil {
fmt.Fprintf(tw, "\n")
fmt.Fprintln(tw, "\n Key Name\tCount\tSize")
fmt.Fprintf(tw, " %s\t%s\t%s", "----", "----", "----")
// For each different type generate new output
for _, s := range info.StatsKV {
fmt.Fprintf(tw, "\n %s\t%d\t%s", s.Name, s.Count, ByteSize(uint64(s.Sum)))
}
fmt.Fprintf(tw, "\n %s\t%s\t%s", "----", "----", "----")
fmt.Fprintf(tw, "\n Total\t\t%s", ByteSize(uint64(info.TotalSizeKV)))
}
fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----")
fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize)))
if err := tw.Flush(); err != nil {
return b.String(), err
}
return b.String(), nil
}

11
command/snapshot/inspect/formatter_test.go
View File

@ -13,6 +13,11 @@ func TestFormat(t *testing.T) {
Sum: 1,
Count: 2,
}}
mkv := []typeStats{{
Name: "msgKV",
Sum: 1,
Count: 2,
}}
info := OutputFormat{
Meta: &MetadataInfo{
ID: "one",
@ -21,8 +26,10 @@ func TestFormat(t *testing.T) {
Term: 4,
Version: 1,
},
Stats: m,
TotalSize: 1,
Stats: m,
StatsKV: mkv,
TotalSize: 1,
TotalSizeKV: 1,
}
formatters := map[string]Formatter{

160
command/snapshot/inspect/snapshot_inspect.go
View File

@ -29,10 +29,21 @@ type cmd struct {
flags *flag.FlagSet
help string
format string
// flags
kvDetails bool
kvDepth int
kvFilter string
}
func (c *cmd) init() {
c.flags = flag.NewFlagSet("", flag.ContinueOnError)
c.flags.BoolVar(&c.kvDetails, "kvdetails", false,
"Provides a detailed KV space usage breakdown for any KV data that's been stored.")
c.flags.IntVar(&c.kvDepth, "kvdepth", 2,
"Can only be used with -kvdetails. The key prefix depth used to breakdown KV store data. Defaults to 2.")
c.flags.StringVar(&c.kvFilter, "kvfilter", "",
"Can only be used with -kvdetails. Limits KV key breakdown using this prefix filter.")
c.flags.StringVar(
&c.format,
"format",
@ -52,12 +63,24 @@ type MetadataInfo struct {
Version raft.SnapshotVersion
}
// SnapshotInfo is used for passing snapshot stat
// information between functions
type SnapshotInfo struct {
Meta MetadataInfo
Stats map[structs.MessageType]typeStats
StatsKV map[string]typeStats
TotalSize int
TotalSizeKV int
}
// OutputFormat is used for passing information
// through the formatter
type OutputFormat struct {
Meta *MetadataInfo
Stats []typeStats
TotalSize int
Meta *MetadataInfo
Stats []typeStats
StatsKV []typeStats
TotalSize int
TotalSizeKV int
}
func (c *cmd) Run(args []string) int {
@ -101,7 +124,7 @@ func (c *cmd) Run(args []string) int {
}
}()
stats, totalSize, err := enhance(readFile)
info, err := c.enhance(readFile)
if err != nil {
c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err))
return 1
@ -122,13 +145,17 @@ func (c *cmd) Run(args []string) int {
}
//Restructures stats given above to be human readable
formattedStats := generatetypeStats(stats)
formattedStats := generateStats(info)
formattedStatsKV := generateKVStats(info)
in := &OutputFormat{
Meta: metaformat,
Stats: formattedStats,
TotalSize: totalSize,
Meta: metaformat,
Stats: formattedStats,
StatsKV: formattedStatsKV,
TotalSize: info.TotalSize,
TotalSizeKV: info.TotalSizeKV,
}
out, err := formatter.Format(in)
if err != nil {
c.UI.Error(err.Error())
@ -145,19 +172,55 @@ type typeStats struct {
Count int
}
func generatetypeStats(info map[structs.MessageType]typeStats) []typeStats {
ss := make([]typeStats, 0, len(info))
// generateStats formats the stats for the output struct
// that's used to produce the printed output the user sees.
func generateStats(info SnapshotInfo) []typeStats {
ss := make([]typeStats, 0, len(info.Stats))
for _, s := range info {
for _, s := range info.Stats {
ss = append(ss, s)
}
// Sort the stat slice
sort.Slice(ss, func(i, j int) bool { return ss[i].Sum > ss[j].Sum })
ss = sortTypeStats(ss)
return ss
}
// generateKVStats reformats the KV stats to work with
// the output struct that's used to produce the printed
// output the user sees.
func generateKVStats(info SnapshotInfo) []typeStats {
kvLen := len(info.StatsKV)
if kvLen > 0 {
ks := make([]typeStats, 0, kvLen)
for _, s := range info.StatsKV {
ks = append(ks, s)
}
ks = sortTypeStats(ks)
return ks
}
return nil
}
// sortTypeStats sorts the stat slice by size and then
// alphabetically in the case the size is identical
func sortTypeStats(stats []typeStats) []typeStats {
sort.Slice(stats, func(i, j int) bool {
// sort alphabetically if size is equal
if stats[i].Sum == stats[j].Sum {
return stats[i].Name < stats[j].Name
}
return stats[i].Sum > stats[j].Sum
})
return stats
}
// countingReader helps keep track of the bytes we have read
// when reading snapshots
type countingReader struct {
@ -175,36 +238,89 @@ func (r *countingReader) Read(p []byte) (n int, err error) {
// enhance utilizes ReadSnapshot to populate the struct with
// all of the snapshot's itemized data
func enhance(file io.Reader) (map[structs.MessageType]typeStats, int, error) {
stats := make(map[structs.MessageType]typeStats)
func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) {
info := SnapshotInfo{
Stats: make(map[structs.MessageType]typeStats),
StatsKV: make(map[string]typeStats),
TotalSize: 0,
TotalSizeKV: 0,
}
cr := &countingReader{wrappedReader: file}
totalSize := 0
handler := func(header *fsm.SnapshotHeader, msg structs.MessageType, dec *codec.Decoder) error {
name := structs.MessageType.String(msg)
s := stats[msg]
s := info.Stats[msg]
if s.Name == "" {
s.Name = name
}
var val interface{}
err := dec.Decode(&val)
if err != nil {
return fmt.Errorf("failed to decode msg type %v, error %v", name, err)
}
size := cr.read - totalSize
size := cr.read - info.TotalSize
s.Sum += size
s.Count++
totalSize = cr.read
stats[msg] = s
info.TotalSize = cr.read
info.Stats[msg] = s
c.kvEnhance(s.Name, val, size, &info)
return nil
}
if err := fsm.ReadSnapshot(cr, handler); err != nil {
return nil, 0, err
return info, err
}
return stats, totalSize, nil
return info, nil
}
// kvEnhance populates the struct with all of the snapshot's
// size information for KV data stored in it
func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) {
if c.kvDetails {
if keyType != "KVS" {
return
}
// have to coerce this into a usable type here or this won't work
keyVal := val.(map[string]interface{})
for k, v := range keyVal {
// we only care about the entry on the key specifically
// related to the key name, so skip all others
if k != "Key" {
continue
}
// check for whether a filter is specified. if it is, skip
// any keys that don't match.
if len(c.kvFilter) > 0 && !strings.HasPrefix(v.(string), c.kvFilter) {
break
}
split := strings.Split(v.(string), "/")
// handle the situation where the key is shorter than
// the specified depth.
actualDepth := c.kvDepth
if c.kvDepth > len(split) {
actualDepth = len(split)
}
prefix := strings.Join(split[0:actualDepth], "/")
kvs := info.StatsKV[prefix]
if kvs.Name == "" {
kvs.Name = prefix
}
kvs.Sum += size
kvs.Count++
info.TotalSizeKV += size
info.StatsKV[prefix] = kvs
}
}
}
func (c *cmd) Synopsis() string {
return synopsis
}

54
command/snapshot/inspect/snapshot_inspect_test.go
View File

@ -95,3 +95,57 @@ func TestSnapshotInspectCommand(t *testing.T) {
want := golden(t, t.Name(), ui.OutputWriter.String())
require.Equal(t, want, ui.OutputWriter.String())
}
func TestSnapshotInspectKVDetailsCommand(t *testing.T) {
filepath := "./testdata/backupWithKV.snap"
// Inspect the snapshot
ui := cli.NewMockUi()
c := New(ui)
args := []string{"-kvdetails", filepath}
code := c.Run(args)
if code != 0 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
want := golden(t, t.Name(), ui.OutputWriter.String())
require.Equal(t, want, ui.OutputWriter.String())
}
func TestSnapshotInspectKVDetailsDepthCommand(t *testing.T) {
filepath := "./testdata/backupWithKV.snap"
// Inspect the snapshot
ui := cli.NewMockUi()
c := New(ui)
args := []string{"-kvdetails", "-kvdepth", "3", filepath}
code := c.Run(args)
if code != 0 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
want := golden(t, t.Name(), ui.OutputWriter.String())
require.Equal(t, want, ui.OutputWriter.String())
}
func TestSnapshotInspectKVDetailsDepthFilterCommand(t *testing.T) {
filepath := "./testdata/backupWithKV.snap"
// Inspect the snapshot
ui := cli.NewMockUi()
c := New(ui)
args := []string{"-kvdetails", "-kvdepth", "3", "-kvfilter", "vault/logical", filepath}
code := c.Run(args)
if code != 0 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
want := golden(t, t.Name(), ui.OutputWriter.String())
require.Equal(t, want, ui.OutputWriter.String())
}

24
command/snapshot/inspect/testdata/TestSnapshotInspectCommand.golden vendored
View File

@ -4,16 +4,16 @@
Term 2
Version 1
Type Count Size
---- ---- ----
Register 3 1.7KB
ConnectCA 1 1.2KB
ConnectCAProviderState 1 1.1KB
Index 12 344B
Autopilot 1 199B
ConnectCAConfig 1 197B
FederationState 1 139B
SystemMetadata 1 68B
ChunkingState 1 12B
---- ---- ----
Type Count Size
---- ---- ----
Register 3 1.7KB
ConnectCA 1 1.2KB
ConnectCAProviderState 1 1.1KB
Index 12 344B
Autopilot 1 199B
ConnectCAConfig 1 197B
FederationState 1 139B
SystemMetadata 1 68B
ChunkingState 1 12B
---- ---- ----
Total 5KB

22
command/snapshot/inspect/testdata/TestSnapshotInspectEnhanceCommand.golden vendored
View File

@ -4,15 +4,15 @@
Term 2
Version 1
Type Count Size
---- ---- ----
Register 3 1.8KB
ConnectCA 1 1.2KB
ConnectCAProviderState 1 1.1KB
Index 11 313B
ConnectCAConfig 1 247B
Autopilot 1 199B
SystemMetadata 1 68B
ChunkingState 1 12B
---- ---- ----
Type Count Size
---- ---- ----
Register 3 1.8KB
ConnectCA 1 1.2KB
ConnectCAProviderState 1 1.1KB
Index 11 313B
ConnectCAConfig 1 247B
Autopilot 1 199B
SystemMetadata 1 68B
ChunkingState 1 12B
---- ---- ----
Total 5KB

27
command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommand.golden vendored Normal file
View File

@ -0,0 +1,27 @@
ID 2-12426-1604593650375
Size 17228
Index 12426
Term 2
Version 1
Type Count Size
---- ---- ----
KVS 27 12.3KB
Register 5 3.4KB
Index 11 285B
Autopilot 1 199B
Session 1 199B
CoordinateBatchUpdate 1 166B
Tombstone 2 146B
FederationState 1 139B
ChunkingState 1 12B
---- ---- ----
Total 16.8KB
Key Name Count Size
---- ---- ----
vault/core 16 5.9KB
vault/sys 7 4.4KB
vault/logical 4 2KB
---- ---- ----
Total 12.3KB

44
command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden vendored Normal file
View File

@ -0,0 +1,44 @@
ID 2-12426-1604593650375
Size 17228
Index 12426
Term 2
Version 1
Type Count Size
---- ---- ----
KVS 27 12.3KB
Register 5 3.4KB
Index 11 285B
Autopilot 1 199B
Session 1 199B
CoordinateBatchUpdate 1 166B
Tombstone 2 146B
FederationState 1 139B
ChunkingState 1 12B
---- ---- ----
Total 16.8KB
Key Name Count Size
---- ---- ----
vault/sys/policy 3 3.3KB
vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB
vault/core/leader 1 1.6KB
vault/sys/token 3 1KB
vault/core/mounts 1 675B
vault/core/wrapping 1 633B
vault/core/local-mounts 1 450B
vault/core/auth 1 423B
vault/core/cluster 2 388B
vault/core/keyring 1 320B
vault/core/master 1 237B
vault/core/seal-config 1 211B
vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B
vault/core/hsm 1 189B
vault/core/local-audit 1 185B
vault/core/local-auth 1 183B
vault/core/audit 1 179B
vault/core/lock 1 170B
vault/core/shamir-kek 1 159B
vault/sys/counters 1 155B
---- ---- ----
Total 12.3KB

26
command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthFilterCommand.golden vendored Normal file
View File

@ -0,0 +1,26 @@
ID 2-12426-1604593650375
Size 17228
Index 12426
Term 2
Version 1
Type Count Size
---- ---- ----
KVS 27 12.3KB
Register 5 3.4KB
Index 11 285B
Autopilot 1 199B
Session 1 199B
CoordinateBatchUpdate 1 166B
Tombstone 2 146B
FederationState 1 139B
ChunkingState 1 12B
---- ---- ----
Total 16.8KB
Key Name Count Size
---- ---- ----
vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB
vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B
---- ---- ----
Total 2KB

BIN
command/snapshot/inspect/testdata/backupWithKV.snap (Stored with Git LFS) vendored Normal file
View File

Binary file not shown.

10
command/snapshot/inspect/testdata/json.golden vendored
View File

@ -13,5 +13,13 @@
"Count": 2
}
],
"TotalSize": 1
"StatsKV": [
{
"Name": "msgKV",
"Sum": 1,
"Count": 2
}
],
"TotalSize": 1,
"TotalSizeKV": 1
}

16
command/snapshot/inspect/testdata/pretty.golden vendored
View File

@ -4,8 +4,14 @@
Term 4
Version 1
Type Count Size
---- ---- ----
msg 2 1B
---- ---- ----
Total 1B
Type Count Size
---- ---- ----
msg 2 1B
---- ---- ----
Total 1B
Key Name Count Size
---- ---- ----
msgKV 2 1B
---- ---- ----
Total 1B

136
website/pages/commands/snapshot/inspect.mdx
View File

@ -38,87 +38,70 @@ To inspect a snapshot from the file "backup.snap":
```shell-session
$ consul snapshot inspect backup.snap
ID 2-13-1603221729747
Size 5141
Index 13
ID 2-12426-1604593650375
Size 17228
Index 12426
Term 2
Version 1
Type Count Size
---- ---- ----
Register 3 1.7KB
ConnectCA 1 1.2KB
ConnectCAProviderState 1 1.1KB
Index 12 344B
Autopilot 1 199B
ConnectCAConfig 1 197B
FederationState 1 139B
SystemMetadata 1 68B
ChunkingState 1 12B
---- ---- ----
Total 5KB
Type Count Size
---- ---- ----
KVS 27 12.3KB
Register 5 3.4KB
Index 11 285B
Autopilot 1 199B
Session 1 199B
CoordinateBatchUpdate 1 166B
Tombstone 2 146B
FederationState 1 139B
ChunkingState 1 12B
---- ---- ----
Total 16.8KB
```
To enhance a snapshot inespection from "backup.snap":
To get more details for a snapshot inspection from "backup.snap":
```shell-session
$ consul snapshot inspect -format=json backup.snap
{
"Meta": {
"ID": "2-13-1603221729747",
"Size": 5141,
"Index": 13,
"Term": 2,
"Version": 1
},
"Stats": [
{
"Name": "Register",
"Sum": 1750,
"Count": 3
},
{
"Name": "ConnectCA",
"Sum": 1258,
"Count": 1
},
{
"Name": "ConnectCAProviderState",
"Sum": 1174,
"Count": 1
},
{
"Name": "Index",
"Sum": 344,
"Count": 12
},
{
"Name": "Autopilot",
"Sum": 199,
"Count": 1
},
{
"Name": "ConnectCAConfig",
"Sum": 197,
"Count": 1
},
{
"Name": "FederationState",
"Sum": 139,
"Count": 1
},
{
"Name": "SystemMetadata",
"Sum": 68,
"Count": 1
},
{
"Name": "ChunkingState",
"Sum": 12,
"Count": 1
}
],
"TotalSize": 5141
}
$ consul snapshot inspect -kvdetails -kvdepth 3 -kvfilter vault/core backup.snap
ID 2-12426-1604593650375
Size 17228
Index 12426
Term 2
Version 1
Type Count Size
---- ---- ----
KVS 27 12.3KB
Register 5 3.4KB
Index 11 285B
Autopilot 1 199B
Session 1 199B
CoordinateBatchUpdate 1 166B
Tombstone 2 146B
FederationState 1 139B
ChunkingState 1 12B
---- ---- ----
Total 16.8KB
Key Name Count Size
---- ---- ----
vault/core/leader 1 1.6KB
vault/core/mounts 1 675B
vault/core/wrapping 1 633B
vault/core/local-mounts 1 450B
vault/core/auth 1 423B
vault/core/cluster 2 388B
vault/core/keyring 1 320B
vault/core/master 1 237B
vault/core/seal-config 1 211B
vault/core/hsm 1 189B
vault/core/local-audit 1 185B
vault/core/local-auth 1 183B
vault/core/audit 1 179B
vault/core/lock 1 170B
vault/core/shamir-kek 1 159B
---- ---- ----
Total 5.9KB
```
Please see the [HTTP API](/api/snapshot) documentation for
@ -126,4 +109,7 @@ more details about snapshot internals.
#### Command Options
- `-kvdetails` - Optional, provides a space usage breakdown for any KV data stored in Consul.
- `-kvdepth` - Can only be used with `-kvdetails`. Used to adjust the grouping level of keys. Defaults to 2.
- `-kvfilter` - Can only be used with `-kvdetails`. Used to specify a key prefix that excludes keys that don't match.
- `-format` - Optional, allows from changing the output to JSON. Parameters accepted are "pretty" and "JSON".