open-consul/command/snapshot/inspect/snapshot_inspect.go

371 lines
8.5 KiB
Go
Raw Normal View History

package inspect
import (
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"sort"
"strings"
"github.com/hashicorp/consul-net-rpc/go-msgpack/codec"
"github.com/hashicorp/consul/agent/consul/fsm"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/command/flags"
"github.com/hashicorp/consul/snapshot"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/raft"
"github.com/mitchellh/cli"
)
func New(ui cli.Ui) *cmd {
c := &cmd{UI: ui}
c.init()
return c
}
type cmd struct {
UI cli.Ui
flags *flag.FlagSet
help string
format string
// flags
kvDetails bool
kvDepth int
kvFilter string
}
func (c *cmd) init() {
c.flags = flag.NewFlagSet("", flag.ContinueOnError)
c.flags.BoolVar(&c.kvDetails, "kvdetails", false,
2020-11-10 16:35:24 +00:00
"Provides a detailed KV space usage breakdown for any KV data that's been stored.")
c.flags.IntVar(&c.kvDepth, "kvdepth", 2,
"Can only be used with -kvdetails. The key prefix depth used to breakdown KV store data. Defaults to 2.")
c.flags.StringVar(&c.kvFilter, "kvfilter", "",
"Can only be used with -kvdetails. Limits KV key breakdown using this prefix filter.")
c.flags.StringVar(
&c.format,
"format",
PrettyFormat,
fmt.Sprintf("Output format {%s}", strings.Join(GetSupportedFormats(), "|")))
c.help = flags.Usage(help, c.flags)
}
// MetadataInfo is used for passing information
// through the formatter
type MetadataInfo struct {
ID string
Size int64
Index uint64
Term uint64
Version raft.SnapshotVersion
}
// SnapshotInfo is used for passing snapshot stat
// information between functions
type SnapshotInfo struct {
Meta MetadataInfo
Stats map[structs.MessageType]typeStats
StatsKV map[string]typeStats
TotalSize int
TotalSizeKV int
}
// OutputFormat is used for passing information
// through the formatter
type OutputFormat struct {
2020-11-05 16:32:23 +00:00
Meta *MetadataInfo
Stats []typeStats
StatsKV []typeStats
TotalSize int
TotalSizeKV int
}
func (c *cmd) Run(args []string) int {
if err := c.flags.Parse(args); err != nil {
c.UI.Error(err.Error())
return 1
}
var file string
args = c.flags.Args()
switch len(args) {
case 0:
2017-04-21 00:02:42 +00:00
c.UI.Error("Missing FILE argument")
return 1
case 1:
file = args[0]
default:
2017-04-21 00:02:42 +00:00
c.UI.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args)))
return 1
}
// Open the file.
f, err := os.Open(file)
if err != nil {
2017-04-21 00:02:42 +00:00
c.UI.Error(fmt.Sprintf("Error opening snapshot file: %s", err))
return 1
}
defer f.Close()
var readFile *os.File
var meta *raft.SnapshotMeta
if strings.ToLower(path.Base(file)) == "state.bin" {
// This is an internal raw raft snapshot not a gzipped archive one
// downloaded from the API, we can read it directly
readFile = f
// Assume the meta is colocated and error if not.
metaRaw, err := ioutil.ReadFile(path.Join(path.Dir(file), "meta.json"))
if err != nil {
c.UI.Error(fmt.Sprintf("Error reading meta.json from internal snapshot dir: %s", err))
return 1
}
var metaDecoded raft.SnapshotMeta
err = json.Unmarshal(metaRaw, &metaDecoded)
if err != nil {
c.UI.Error(fmt.Sprintf("Error parsing meta.json from internal snapshot dir: %s", err))
return 1
}
meta = &metaDecoded
} else {
readFile, meta, err = snapshot.Read(hclog.New(nil), f)
if err != nil {
c.UI.Error(fmt.Sprintf("Error reading snapshot: %s", err))
return 1
}
defer func() {
if err := readFile.Close(); err != nil {
c.UI.Error(fmt.Sprintf("Failed to close temp snapshot: %v", err))
}
if err := os.Remove(readFile.Name()); err != nil {
c.UI.Error(fmt.Sprintf("Failed to clean up temp snapshot: %v", err))
}
}()
}
info, err := c.enhance(readFile)
if err != nil {
c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err))
return 1
}
formatter, err := NewFormatter(c.format)
if err != nil {
c.UI.Error(fmt.Sprintf("Error outputting enhanced snapshot data: %s", err))
return 1
}
//Generate structs for the formatter with information we read in
metaformat := &MetadataInfo{
ID: meta.ID,
Size: meta.Size,
Index: meta.Index,
Term: meta.Term,
Version: meta.Version,
}
//Restructures stats given above to be human readable
2020-11-10 16:09:03 +00:00
formattedStats := generateStats(info)
formattedStatsKV := generateKVStats(info)
in := &OutputFormat{
2020-11-05 16:32:23 +00:00
Meta: metaformat,
Stats: formattedStats,
StatsKV: formattedStatsKV,
TotalSize: info.TotalSize,
TotalSizeKV: info.TotalSizeKV,
}
out, err := formatter.Format(in)
if err != nil {
c.UI.Error(err.Error())
return 1
}
c.UI.Output(out)
return 0
}
type typeStats struct {
Name string
Sum int
Count int
}
2020-11-11 17:33:38 +00:00
// generateStats formats the stats for the output struct
// that's used to produce the printed output the user sees.
2020-11-10 16:09:03 +00:00
func generateStats(info SnapshotInfo) []typeStats {
ss := make([]typeStats, 0, len(info.Stats))
for _, s := range info.Stats {
ss = append(ss, s)
}
2020-11-05 22:25:21 +00:00
ss = sortTypeStats(ss)
2020-11-10 16:09:03 +00:00
return ss
}
2020-11-11 17:33:38 +00:00
// generateKVStats reformats the KV stats to work with
// the output struct that's used to produce the printed
// output the user sees.
2020-11-10 16:09:03 +00:00
func generateKVStats(info SnapshotInfo) []typeStats {
kvLen := len(info.StatsKV)
if kvLen > 0 {
ks := make([]typeStats, 0, kvLen)
for _, s := range info.StatsKV {
ks = append(ks, s)
}
2020-11-05 22:25:21 +00:00
ks = sortTypeStats(ks)
2020-11-10 16:09:03 +00:00
return ks
}
2020-11-10 16:09:03 +00:00
return nil
}
2020-11-11 17:33:38 +00:00
// sortTypeStats sorts the stat slice by size and then
// alphabetically in the case the size is identical
2020-11-05 22:25:21 +00:00
func sortTypeStats(stats []typeStats) []typeStats {
sort.Slice(stats, func(i, j int) bool {
// sort alphabetically if size is equal
if stats[i].Sum == stats[j].Sum {
return stats[i].Name < stats[j].Name
}
return stats[i].Sum > stats[j].Sum
})
return stats
}
// countingReader helps keep track of the bytes we have read
// when reading snapshots
type countingReader struct {
wrappedReader io.Reader
read int
}
func (r *countingReader) Read(p []byte) (n int, err error) {
n, err = r.wrappedReader.Read(p)
if err == nil {
r.read += n
}
return n, err
}
// enhance utilizes ReadSnapshot to populate the struct with
// all of the snapshot's itemized data
func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) {
info := SnapshotInfo{
Stats: make(map[structs.MessageType]typeStats),
StatsKV: make(map[string]typeStats),
TotalSize: 0,
TotalSizeKV: 0,
}
cr := &countingReader{wrappedReader: file}
handler := func(header *fsm.SnapshotHeader, msg structs.MessageType, dec *codec.Decoder) error {
name := structs.MessageType.String(msg)
s := info.Stats[msg]
if s.Name == "" {
s.Name = name
}
var val interface{}
err := dec.Decode(&val)
if err != nil {
return fmt.Errorf("failed to decode msg type %v, error %v", name, err)
}
size := cr.read - info.TotalSize
s.Sum += size
s.Count++
info.TotalSize = cr.read
info.Stats[msg] = s
c.kvEnhance(s.Name, val, size, &info)
return nil
}
if err := fsm.ReadSnapshot(cr, handler); err != nil {
return info, err
}
return info, nil
}
2020-11-10 22:40:12 +00:00
// kvEnhance populates the struct with all of the snapshot's
// size information for KV data stored in it
func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) {
if c.kvDetails {
if keyType != "KVS" {
return
}
// have to coerce this into a usable type here or this won't work
keyVal := val.(map[string]interface{})
for k, v := range keyVal {
// we only care about the entry on the key specifically
// related to the key name, so skip all others
if k != "Key" {
continue
}
// check for whether a filter is specified. if it is, skip
// any keys that don't match.
if len(c.kvFilter) > 0 && !strings.HasPrefix(v.(string), c.kvFilter) {
break
}
split := strings.Split(v.(string), "/")
// handle the situation where the key is shorter than
// the specified depth.
actualDepth := c.kvDepth
if c.kvDepth > len(split) {
actualDepth = len(split)
}
prefix := strings.Join(split[0:actualDepth], "/")
kvs := info.StatsKV[prefix]
if kvs.Name == "" {
kvs.Name = prefix
}
kvs.Sum += size
kvs.Count++
info.TotalSizeKV += size
info.StatsKV[prefix] = kvs
}
}
}
func (c *cmd) Synopsis() string {
return synopsis
}
func (c *cmd) Help() string {
return c.help
}
const synopsis = "Displays information about a Consul snapshot file"
const help = `
Usage: consul snapshot inspect [options] FILE
Displays information about a snapshot file on disk.
To inspect the file "backup.snap":
$ consul snapshot inspect backup.snap
For a full list of options and examples, please see the Consul documentation.
`