From ee3eb03f503a606b3c8b8d7b17ad8c927f6e356e Mon Sep 17 00:00:00 2001 From: s-christoff Date: Thu, 29 Oct 2020 11:31:14 -0500 Subject: [PATCH] cli: Add JSON and Pretty Print formatting for `consul snapshot inspect` (#9006) --- .changelog/9006.txt | 3 + command/snapshot/inspect/formatter.go | 114 +++++++++++ command/snapshot/inspect/formatter_test.go | 45 +++++ command/snapshot/inspect/snapshot_inspect.go | 177 +++++++----------- .../TestSnapshotInspectCommand.golden | 11 +- .../TestSnapshotInspectEnhanceCommand.golden | 18 ++ command/snapshot/inspect/testdata/json.golden | 17 ++ .../snapshot/inspect/testdata/pretty.golden | 11 ++ website/pages/commands/snapshot/inspect.mdx | 84 ++++++++- 9 files changed, 355 insertions(+), 125 deletions(-) create mode 100644 .changelog/9006.txt create mode 100644 command/snapshot/inspect/formatter.go create mode 100644 command/snapshot/inspect/formatter_test.go create mode 100644 command/snapshot/inspect/testdata/TestSnapshotInspectEnhanceCommand.golden create mode 100644 command/snapshot/inspect/testdata/json.golden create mode 100644 command/snapshot/inspect/testdata/pretty.golden diff --git a/.changelog/9006.txt b/.changelog/9006.txt new file mode 100644 index 000000000..6673819ac --- /dev/null +++ b/.changelog/9006.txt @@ -0,0 +1,3 @@ +```release-note:feature +cli: snapshot inspect command supports JSON output +``` diff --git a/command/snapshot/inspect/formatter.go b/command/snapshot/inspect/formatter.go new file mode 100644 index 000000000..9b9a4c4c3 --- /dev/null +++ b/command/snapshot/inspect/formatter.go @@ -0,0 +1,114 @@ +package inspect + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" + "text/tabwriter" +) + +const ( + PrettyFormat string = "pretty" + JSONFormat string = "json" +) + +type Formatter interface { + Format(*OutputFormat) (string, error) +} + +func GetSupportedFormats() []string { + return []string{PrettyFormat, JSONFormat} +} + +type prettyFormatter struct{} + +func newPrettyFormatter() Formatter { + return &prettyFormatter{} +} +func NewFormatter(format string) (Formatter, error) { + switch format { + case PrettyFormat: + return newPrettyFormatter(), nil + case JSONFormat: + return newJSONFormatter(), nil + default: + return nil, fmt.Errorf("Unknown format: %s", format) + } +} + +func (_ *prettyFormatter) Format(info *OutputFormat) (string, error) { + var b bytes.Buffer + tw := tabwriter.NewWriter(&b, 8, 8, 6, ' ', 0) + + fmt.Fprintf(tw, " ID\t%s", info.Meta.ID) + fmt.Fprintf(tw, "\n Size\t%d", info.Meta.Size) + fmt.Fprintf(tw, "\n Index\t%d", info.Meta.Index) + fmt.Fprintf(tw, "\n Term\t%d", info.Meta.Term) + fmt.Fprintf(tw, "\n Version\t%d", info.Meta.Version) + fmt.Fprintf(tw, "\n") + fmt.Fprintln(tw, "\n Type\tCount\tSize\t") + fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----") + // For each different type generate new output + for _, s := range info.Stats { + fmt.Fprintf(tw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum))) + } + fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----") + fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize))) + + if err := tw.Flush(); err != nil { + return b.String(), err + } + return b.String(), nil +} + +type jsonFormatter struct{} + +func newJSONFormatter() Formatter { + return &jsonFormatter{} +} + +func (_ *jsonFormatter) Format(info *OutputFormat) (string, error) { + b, err := json.MarshalIndent(info, "", " ") + if err != nil { + return "", fmt.Errorf("Failed to marshal original snapshot stats: %v", err) + } + return string(b), nil +} + +const ( + BYTE = 1 << (10 * iota) + KILOBYTE + MEGABYTE + GIGABYTE + TERABYTE +) + +func ByteSize(bytes uint64) string { + unit := "" + value := float64(bytes) + + switch { + case bytes >= TERABYTE: + unit = "TB" + value = value / TERABYTE + case bytes >= GIGABYTE: + unit = "GB" + value = value / GIGABYTE + case bytes >= MEGABYTE: + unit = "MB" + value = value / MEGABYTE + case bytes >= KILOBYTE: + unit = "KB" + value = value / KILOBYTE + case bytes >= BYTE: + unit = "B" + case bytes == 0: + return "0" + } + + result := strconv.FormatFloat(value, 'f', 1, 64) + result = strings.TrimSuffix(result, ".0") + return result + unit +} diff --git a/command/snapshot/inspect/formatter_test.go b/command/snapshot/inspect/formatter_test.go new file mode 100644 index 000000000..d74181ebb --- /dev/null +++ b/command/snapshot/inspect/formatter_test.go @@ -0,0 +1,45 @@ +package inspect + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFormat(t *testing.T) { + m := []typeStats{{ + Name: "msg", + Sum: 1, + Count: 2, + }} + info := OutputFormat{ + Meta: &MetadataInfo{ + ID: "one", + Size: 2, + Index: 3, + Term: 4, + Version: 1, + }, + Stats: m, + TotalSize: 1, + } + + formatters := map[string]Formatter{ + "pretty": newPrettyFormatter(), + // the JSON formatter ignores the showMeta + "json": newJSONFormatter(), + } + + for fmtName, formatter := range formatters { + t.Run(fmtName, func(t *testing.T) { + actual, err := formatter.Format(&info) + require.NoError(t, err) + + gName := fmt.Sprintf("%s", fmtName) + + expected := golden(t, gName, actual) + require.Equal(t, expected, actual) + }) + } +} diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 85e6c895f..2e5b41e59 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -1,15 +1,12 @@ package inspect import ( - "bytes" "flag" "fmt" "io" "os" "sort" - "strconv" "strings" - "text/tabwriter" "github.com/hashicorp/consul/agent/consul/fsm" "github.com/hashicorp/consul/agent/structs" @@ -28,16 +25,41 @@ func New(ui cli.Ui) *cmd { } type cmd struct { - UI cli.Ui - flags *flag.FlagSet - help string + UI cli.Ui + flags *flag.FlagSet + help string + format string } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) + c.flags.StringVar( + &c.format, + "format", + PrettyFormat, + fmt.Sprintf("Output format {%s}", strings.Join(GetSupportedFormats(), "|"))) + c.help = flags.Usage(help, c.flags) } +// MetadataInfo is used for passing information +// through the formatter +type MetadataInfo struct { + ID string + Size int64 + Index uint64 + Term uint64 + Version raft.SnapshotVersion +} + +// OutputFormat is used for passing information +// through the formatter +type OutputFormat struct { + Meta *MetadataInfo + Stats []typeStats + TotalSize int +} + func (c *cmd) Run(args []string) int { if err := c.flags.Parse(args); err != nil { c.UI.Error(err.Error()) @@ -84,38 +106,37 @@ func (c *cmd) Run(args []string) int { c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err)) return 1 } - // Outputs the original style of inspect information - legacy, err := c.legacyStats(meta) - if err != nil { - c.UI.Error(fmt.Sprintf("Error outputting snapshot data: %s", err)) - } - c.UI.Info(legacy.String()) - // Outputs the more detailed snapshot information - enhanced, err := c.readStats(stats, totalSize) + formatter, err := NewFormatter(c.format) if err != nil { c.UI.Error(fmt.Sprintf("Error outputting enhanced snapshot data: %s", err)) return 1 } - c.UI.Info(enhanced.String()) - - return 0 -} - -// legacyStats outputs the expected stats from the original snapshot -// inspect command -func (c *cmd) legacyStats(meta *raft.SnapshotMeta) (bytes.Buffer, error) { - var b bytes.Buffer - tw := tabwriter.NewWriter(&b, 0, 2, 6, ' ', 0) - fmt.Fprintf(tw, "ID\t%s\n", meta.ID) - fmt.Fprintf(tw, "Size\t%d\n", meta.Size) - fmt.Fprintf(tw, "Index\t%d\n", meta.Index) - fmt.Fprintf(tw, "Term\t%d\n", meta.Term) - fmt.Fprintf(tw, "Version\t%d\n", meta.Version) - if err := tw.Flush(); err != nil { - return b, err + //Generate structs for the formatter with information we read in + metaformat := &MetadataInfo{ + ID: meta.ID, + Size: meta.Size, + Index: meta.Index, + Term: meta.Term, + Version: meta.Version, } - return b, nil + + //Restructures stats given above to be human readable + formattedStats := generatetypeStats(stats) + + in := &OutputFormat{ + Meta: metaformat, + Stats: formattedStats, + TotalSize: totalSize, + } + out, err := formatter.Format(in) + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + + c.UI.Output(out) + return 0 } type typeStats struct { @@ -124,6 +145,19 @@ type typeStats struct { Count int } +func generatetypeStats(info map[structs.MessageType]typeStats) []typeStats { + ss := make([]typeStats, 0, len(info)) + + for _, s := range info { + ss = append(ss, s) + } + + // Sort the stat slice + sort.Slice(ss, func(i, j int) bool { return ss[i].Sum > ss[j].Sum }) + + return ss +} + // countingReader helps keep track of the bytes we have read // when reading snapshots type countingReader struct { @@ -171,85 +205,6 @@ func enhance(file io.Reader) (map[structs.MessageType]typeStats, int, error) { } -// readStats takes the information generated from enhance and creates human -// readable output from it -func (c *cmd) readStats(stats map[structs.MessageType]typeStats, totalSize int) (bytes.Buffer, error) { - // Output stats in size-order - ss := make([]typeStats, 0, len(stats)) - - for _, s := range stats { - ss = append(ss, s) - } - - // Sort the stat slice - sort.Slice(ss, func(i, j int) bool { return ss[i].Sum > ss[j].Sum }) - - var b bytes.Buffer - - tw := tabwriter.NewWriter(&b, 8, 8, 6, ' ', 0) - fmt.Fprintln(tw, "\n Type\tCount\tSize\t") - fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----") - // For each different type generate new output - for _, s := range ss { - fmt.Fprintf(tw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum))) - } - fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----") - fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(totalSize))) - - if err := tw.Flush(); err != nil { - c.UI.Error(fmt.Sprintf("Error rendering snapshot info: %s", err)) - return b, err - } - - return b, nil - -} - -// ByteSize returns a human-readable byte string of the form 10MB, 12.5KB, and so forth. The following units are available: -// TB: Terabyte -// GB: Gigabyte -// MB: Megabyte -// KB: Kilobyte -// B: Byte -// The unit that results in the smallest number greater than or equal to 1 is always chosen. -// From https://github.com/cloudfoundry/bytefmt/blob/master/bytes.go - -const ( - BYTE = 1 << (10 * iota) - KILOBYTE - MEGABYTE - GIGABYTE - TERABYTE -) - -func ByteSize(bytes uint64) string { - unit := "" - value := float64(bytes) - - switch { - case bytes >= TERABYTE: - unit = "TB" - value = value / TERABYTE - case bytes >= GIGABYTE: - unit = "GB" - value = value / GIGABYTE - case bytes >= MEGABYTE: - unit = "MB" - value = value / MEGABYTE - case bytes >= KILOBYTE: - unit = "KB" - value = value / KILOBYTE - case bytes >= BYTE: - unit = "B" - case bytes == 0: - return "0" - } - - result := strconv.FormatFloat(value, 'f', 1, 64) - result = strings.TrimSuffix(result, ".0") - return result + unit -} - func (c *cmd) Synopsis() string { return synopsis } diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectCommand.golden index bb176a6e5..00b9da406 100644 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectCommand.golden +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectCommand.golden @@ -1,9 +1,8 @@ -ID 2-13-1602222343947 -Size 5141 -Index 13 -Term 2 -Version 1 - + ID 2-13-1602222343947 + Size 5141 + Index 13 + Term 2 + Version 1 Type Count Size ---- ---- ---- diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectEnhanceCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectEnhanceCommand.golden new file mode 100644 index 000000000..9867d8e5a --- /dev/null +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectEnhanceCommand.golden @@ -0,0 +1,18 @@ + ID 2-12-1603319127176 + Size 5133 + Index 12 + Term 2 + Version 1 + + Type Count Size + ---- ---- ---- + Register 3 1.8KB + ConnectCA 1 1.2KB + ConnectCAProviderState 1 1.1KB + Index 11 313B + ConnectCAConfig 1 247B + Autopilot 1 199B + SystemMetadata 1 68B + ChunkingState 1 12B + ---- ---- ---- + Total 5KB diff --git a/command/snapshot/inspect/testdata/json.golden b/command/snapshot/inspect/testdata/json.golden new file mode 100644 index 000000000..42c3ba40b --- /dev/null +++ b/command/snapshot/inspect/testdata/json.golden @@ -0,0 +1,17 @@ +{ + "Meta": { + "ID": "one", + "Size": 2, + "Index": 3, + "Term": 4, + "Version": 1 + }, + "Stats": [ + { + "Name": "msg", + "Sum": 1, + "Count": 2 + } + ], + "TotalSize": 1 +} \ No newline at end of file diff --git a/command/snapshot/inspect/testdata/pretty.golden b/command/snapshot/inspect/testdata/pretty.golden new file mode 100644 index 000000000..7a2c5749b --- /dev/null +++ b/command/snapshot/inspect/testdata/pretty.golden @@ -0,0 +1,11 @@ + ID one + Size 2 + Index 3 + Term 4 + Version 1 + + Type Count Size + ---- ---- ---- + msg 2 1B + ---- ---- ---- + Total 1B \ No newline at end of file diff --git a/website/pages/commands/snapshot/inspect.mdx b/website/pages/commands/snapshot/inspect.mdx index 37398d5bb..fcf59223e 100644 --- a/website/pages/commands/snapshot/inspect.mdx +++ b/website/pages/commands/snapshot/inspect.mdx @@ -26,6 +26,8 @@ The following fields are displayed when inspecting a snapshot: - `Version` - The snapshot format version. This only refers to the structure of the snapshot, not the data contained within. +- Each data type, size, and count within the read snapshot. + ## Usage Usage: `consul snapshot inspect [options] FILE` @@ -36,11 +38,11 @@ To inspect a snapshot from the file "backup.snap": ```shell-session $ consul snapshot inspect backup.snap -ID 2-5-1477944140022 -Size 667 -Index 5 -Term 2 -Version 1 + ID 2-13-1603221729747 + Size 5141 + Index 13 + Term 2 + Version 1 Type Count Size ---- ---- ---- @@ -48,14 +50,80 @@ Version 1 ConnectCA 1 1.2KB ConnectCAProviderState 1 1.1KB Index 12 344B - AutopilotRequest 1 199B + Autopilot 1 199B ConnectCAConfig 1 197B FederationState 1 139B SystemMetadata 1 68B ChunkingState 1 12B ---- ---- ---- - Total 5KB + Total 5KB +``` + +To enhance a snapshot inespection from "backup.snap": +```shell-session +$ consul snapshot inspect -format=json backup.snap +{ + "Meta": { + "ID": "2-13-1603221729747", + "Size": 5141, + "Index": 13, + "Term": 2, + "Version": 1 + }, + "Stats": [ + { + "Name": "Register", + "Sum": 1750, + "Count": 3 + }, + { + "Name": "ConnectCA", + "Sum": 1258, + "Count": 1 + }, + { + "Name": "ConnectCAProviderState", + "Sum": 1174, + "Count": 1 + }, + { + "Name": "Index", + "Sum": 344, + "Count": 12 + }, + { + "Name": "Autopilot", + "Sum": 199, + "Count": 1 + }, + { + "Name": "ConnectCAConfig", + "Sum": 197, + "Count": 1 + }, + { + "Name": "FederationState", + "Sum": 139, + "Count": 1 + }, + { + "Name": "SystemMetadata", + "Sum": 68, + "Count": 1 + }, + { + "Name": "ChunkingState", + "Sum": 12, + "Count": 1 + } + ], + "TotalSize": 5141 +} ``` Please see the [HTTP API](/api/snapshot) documentation for -more details about snapshot internals. \ No newline at end of file +more details about snapshot internals. + +#### Command Options + +- `-format` - Optional, allows from changing the output to JSON. Parameters accepted are "pretty" and "JSON".