1096 lines
31 KiB
Go
1096 lines
31 KiB
Go
package command
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/hashicorp/go-hclog"
|
|
"github.com/hashicorp/go-secure-stdlib/gatedwriter"
|
|
"github.com/hashicorp/go-secure-stdlib/strutil"
|
|
"github.com/hashicorp/vault/api"
|
|
"github.com/hashicorp/vault/helper/osutil"
|
|
"github.com/hashicorp/vault/sdk/helper/jsonutil"
|
|
"github.com/hashicorp/vault/sdk/helper/logging"
|
|
"github.com/hashicorp/vault/sdk/version"
|
|
"github.com/mholt/archiver/v3"
|
|
"github.com/mitchellh/cli"
|
|
"github.com/oklog/run"
|
|
"github.com/posener/complete"
|
|
)
|
|
|
|
const (
|
|
// debugIndexVersion tracks the canonical version in the index file
|
|
// for compatibility with future format/layout changes on the bundle.
|
|
debugIndexVersion = 1
|
|
|
|
// debugMinInterval is the minimum acceptable interval capture value. This
|
|
// value applies to duration and all interval-related flags.
|
|
debugMinInterval = 5 * time.Second
|
|
|
|
// debugDurationGrace is the grace period added to duration to allow for
|
|
// "last frame" capture if the interval falls into the last duration time
|
|
// value. For instance, using default values, adding a grace duration lets
|
|
// the command capture 5 intervals (0, 30, 60, 90, and 120th second) before
|
|
// exiting.
|
|
debugDurationGrace = 1 * time.Second
|
|
|
|
// debugCompressionExt is the default compression extension used if
|
|
// compression is enabled.
|
|
debugCompressionExt = ".tar.gz"
|
|
|
|
// fileFriendlyTimeFormat is the time format used for file and directory
|
|
// naming.
|
|
fileFriendlyTimeFormat = "2006-01-02T15-04-05Z"
|
|
)
|
|
|
|
// debugIndex represents the data structure in the index file
|
|
type debugIndex struct {
|
|
Version int `json:"version"`
|
|
VaultAddress string `json:"vault_address"`
|
|
ClientVersion string `json:"client_version"`
|
|
ServerVersion string `json:"server_version"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
DurationSeconds int `json:"duration_seconds"`
|
|
IntervalSeconds int `json:"interval_seconds"`
|
|
MetricsIntervalSeconds int `json:"metrics_interval_seconds"`
|
|
Compress bool `json:"compress"`
|
|
RawArgs []string `json:"raw_args"`
|
|
Targets []string `json:"targets"`
|
|
Output map[string]interface{} `json:"output"`
|
|
Errors []*captureError `json:"errors"`
|
|
}
|
|
|
|
// captureError holds an error entry that can occur during polling capture.
|
|
// It includes the timestamp, the target, and the error itself.
|
|
type captureError struct {
|
|
TargetError string `json:"error"`
|
|
Target string `json:"target"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
}
|
|
|
|
var (
|
|
_ cli.Command = (*DebugCommand)(nil)
|
|
_ cli.CommandAutocomplete = (*DebugCommand)(nil)
|
|
)
|
|
|
|
type DebugCommand struct {
|
|
*BaseCommand
|
|
|
|
flagCompress bool
|
|
flagDuration time.Duration
|
|
flagInterval time.Duration
|
|
flagMetricsInterval time.Duration
|
|
flagOutput string
|
|
flagTargets []string
|
|
|
|
// logFormat defines the output format for Monitor
|
|
logFormat string
|
|
|
|
// debugIndex is used to keep track of the index state, which gets written
|
|
// to a file at the end.
|
|
debugIndex *debugIndex
|
|
|
|
// skipTimingChecks bypasses timing-related checks, used primarily for tests
|
|
skipTimingChecks bool
|
|
// logger is the logger used for outputting capture progress
|
|
logger hclog.Logger
|
|
|
|
// ShutdownCh is used to capture interrupt signal and end polling capture
|
|
ShutdownCh chan struct{}
|
|
|
|
// Collection slices to hold data
|
|
hostInfoCollection []map[string]interface{}
|
|
metricsCollection []map[string]interface{}
|
|
replicationStatusCollection []map[string]interface{}
|
|
serverStatusCollection []map[string]interface{}
|
|
inFlightReqStatusCollection []map[string]interface{}
|
|
|
|
// cachedClient holds the client retrieved during preflight
|
|
cachedClient *api.Client
|
|
|
|
// errLock is used to lock error capture into the index file
|
|
errLock sync.Mutex
|
|
}
|
|
|
|
func (c *DebugCommand) AutocompleteArgs() complete.Predictor {
|
|
// Predict targets
|
|
return c.PredictVaultDebugTargets()
|
|
}
|
|
|
|
func (c *DebugCommand) AutocompleteFlags() complete.Flags {
|
|
return c.Flags().Completions()
|
|
}
|
|
|
|
func (c *DebugCommand) Flags() *FlagSets {
|
|
set := c.flagSet(FlagSetHTTP)
|
|
|
|
f := set.NewFlagSet("Command Options")
|
|
|
|
f.BoolVar(&BoolVar{
|
|
Name: "compress",
|
|
Target: &c.flagCompress,
|
|
Default: true,
|
|
Usage: "Toggles whether to compress output package",
|
|
})
|
|
|
|
f.DurationVar(&DurationVar{
|
|
Name: "duration",
|
|
Target: &c.flagDuration,
|
|
Completion: complete.PredictAnything,
|
|
Default: 2 * time.Minute,
|
|
Usage: "Duration to run the command.",
|
|
})
|
|
|
|
f.DurationVar(&DurationVar{
|
|
Name: "interval",
|
|
Target: &c.flagInterval,
|
|
Completion: complete.PredictAnything,
|
|
Default: 30 * time.Second,
|
|
Usage: "The polling interval at which to collect profiling data and server state.",
|
|
})
|
|
|
|
f.DurationVar(&DurationVar{
|
|
Name: "metrics-interval",
|
|
Target: &c.flagMetricsInterval,
|
|
Completion: complete.PredictAnything,
|
|
Default: 10 * time.Second,
|
|
Usage: "The polling interval at which to collect metrics data.",
|
|
})
|
|
|
|
f.StringVar(&StringVar{
|
|
Name: "output",
|
|
Target: &c.flagOutput,
|
|
Completion: complete.PredictAnything,
|
|
Usage: "Specifies the output path for the debug package.",
|
|
})
|
|
|
|
f.StringSliceVar(&StringSliceVar{
|
|
Name: "target",
|
|
Target: &c.flagTargets,
|
|
Usage: "Target to capture, defaulting to all if none specified. " +
|
|
"This can be specified multiple times to capture multiple targets. " +
|
|
"Available targets are: config, host, metrics, pprof, " +
|
|
"replication-status, server-status, log.",
|
|
})
|
|
|
|
f.StringVar(&StringVar{
|
|
Name: "log-format",
|
|
Target: &c.logFormat,
|
|
Default: "standard",
|
|
Usage: "Log format to be captured if \"log\" target specified. " +
|
|
"Supported values are \"standard\" and \"json\". The default is \"standard\".",
|
|
})
|
|
|
|
return set
|
|
}
|
|
|
|
func (c *DebugCommand) Help() string {
|
|
helpText := `
|
|
Usage: vault debug [options]
|
|
|
|
Probes a specific Vault server node for a specified period of time, recording
|
|
information about the node, its cluster, and its host environment. The
|
|
information collected is packaged and written to the specified path.
|
|
|
|
Certain endpoints that this command uses require ACL permissions to access.
|
|
If not permitted, the information from these endpoints will not be part of the
|
|
output. The command uses the Vault address and token as specified via
|
|
the login command, environment variables, or CLI flags.
|
|
|
|
To create a debug package using default duration and interval values in the
|
|
current directory that captures all applicable targets:
|
|
|
|
$ vault debug
|
|
|
|
To create a debug package with a specific duration and interval in the current
|
|
directory that capture all applicable targets:
|
|
|
|
$ vault debug -duration=10m -interval=1m
|
|
|
|
To create a debug package in the current directory with a specific sub-set of
|
|
targets:
|
|
|
|
$ vault debug -target=host -target=metrics
|
|
|
|
` + c.Flags().Help()
|
|
|
|
return helpText
|
|
}
|
|
|
|
func (c *DebugCommand) Run(args []string) int {
|
|
f := c.Flags()
|
|
|
|
if err := f.Parse(args); err != nil {
|
|
c.UI.Error(err.Error())
|
|
return 1
|
|
}
|
|
|
|
parsedArgs := f.Args()
|
|
if len(parsedArgs) > 0 {
|
|
c.UI.Error(fmt.Sprintf("Too many arguments (expected 0, got %d)", len(parsedArgs)))
|
|
return 1
|
|
}
|
|
|
|
// Initialize the logger for debug output
|
|
gatedWriter := gatedwriter.NewWriter(os.Stderr)
|
|
if c.logger == nil {
|
|
c.logger = logging.NewVaultLoggerWithWriter(gatedWriter, hclog.Trace)
|
|
}
|
|
|
|
dstOutputFile, err := c.preflight(args)
|
|
if err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error during validation: %s", err))
|
|
return 1
|
|
}
|
|
|
|
// Print debug information
|
|
c.UI.Output("==> Starting debug capture...")
|
|
c.UI.Info(fmt.Sprintf(" Vault Address: %s", c.debugIndex.VaultAddress))
|
|
c.UI.Info(fmt.Sprintf(" Client Version: %s", c.debugIndex.ClientVersion))
|
|
c.UI.Info(fmt.Sprintf(" Server Version: %s", c.debugIndex.ServerVersion))
|
|
c.UI.Info(fmt.Sprintf(" Duration: %s", c.flagDuration))
|
|
c.UI.Info(fmt.Sprintf(" Interval: %s", c.flagInterval))
|
|
c.UI.Info(fmt.Sprintf(" Metrics Interval: %s", c.flagMetricsInterval))
|
|
c.UI.Info(fmt.Sprintf(" Targets: %s", strings.Join(c.flagTargets, ", ")))
|
|
c.UI.Info(fmt.Sprintf(" Output: %s", dstOutputFile))
|
|
c.UI.Output("")
|
|
|
|
// Release the log gate.
|
|
c.logger.(hclog.OutputResettable).ResetOutputWithFlush(&hclog.LoggerOptions{
|
|
Output: os.Stderr,
|
|
}, gatedWriter)
|
|
|
|
// Capture static information
|
|
c.UI.Info("==> Capturing static information...")
|
|
if err := c.captureStaticTargets(); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error capturing static information: %s", err))
|
|
return 2
|
|
}
|
|
|
|
c.UI.Output("")
|
|
|
|
// Capture polling information
|
|
c.UI.Info("==> Capturing dynamic information...")
|
|
if err := c.capturePollingTargets(); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error capturing dynamic information: %s", err))
|
|
return 2
|
|
}
|
|
|
|
c.UI.Output("Finished capturing information, bundling files...")
|
|
|
|
// Generate index file
|
|
if err := c.generateIndex(); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error generating index: %s", err))
|
|
return 1
|
|
}
|
|
|
|
if c.flagCompress {
|
|
if err := c.compress(dstOutputFile); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error encountered during bundle compression: %s", err))
|
|
// We want to inform that data collection was captured and stored in
|
|
// a directory even if compression fails
|
|
c.UI.Info(fmt.Sprintf("Data written to: %s", c.flagOutput))
|
|
return 1
|
|
}
|
|
}
|
|
|
|
c.UI.Info(fmt.Sprintf("Success! Bundle written to: %s", dstOutputFile))
|
|
return 0
|
|
}
|
|
|
|
func (c *DebugCommand) Synopsis() string {
|
|
return "Runs the debug command"
|
|
}
|
|
|
|
func (c *DebugCommand) generateIndex() error {
|
|
outputLayout := map[string]interface{}{
|
|
"files": []string{},
|
|
}
|
|
// Walk the directory to generate the output layout
|
|
err := filepath.Walk(c.flagOutput, func(path string, info os.FileInfo, err error) error {
|
|
// Prevent panic by handling failure accessing a path
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Skip the base dir
|
|
if path == c.flagOutput {
|
|
return nil
|
|
}
|
|
|
|
// If we're a directory, simply add a corresponding map
|
|
if info.IsDir() {
|
|
parsedTime, err := time.Parse(fileFriendlyTimeFormat, info.Name())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
outputLayout[info.Name()] = map[string]interface{}{
|
|
"timestamp": parsedTime,
|
|
"files": []string{},
|
|
}
|
|
return nil
|
|
}
|
|
|
|
relPath, err := filepath.Rel(c.flagOutput, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
dir, file := filepath.Split(relPath)
|
|
if len(dir) != 0 {
|
|
dir = filepath.Clean(dir)
|
|
filesArr := outputLayout[dir].(map[string]interface{})["files"]
|
|
outputLayout[dir].(map[string]interface{})["files"] = append(filesArr.([]string), file)
|
|
} else {
|
|
outputLayout["files"] = append(outputLayout["files"].([]string), file)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("error generating directory output layout: %s", err)
|
|
}
|
|
|
|
c.debugIndex.Output = outputLayout
|
|
|
|
// Marshal into json
|
|
bytes, err := json.MarshalIndent(c.debugIndex, "", " ")
|
|
if err != nil {
|
|
return fmt.Errorf("error marshaling index file: %s", err)
|
|
}
|
|
|
|
// Write out file
|
|
if err := ioutil.WriteFile(filepath.Join(c.flagOutput, "index.json"), bytes, 0o600); err != nil {
|
|
return fmt.Errorf("error generating index file; %s", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// preflight performs various checks against the provided flags to ensure they
|
|
// are valid/reasonable values. It also takes care of instantiating a client and
|
|
// index object for use by the command.
|
|
func (c *DebugCommand) preflight(rawArgs []string) (string, error) {
|
|
if !c.skipTimingChecks {
|
|
// Guard duration and interval values to acceptable values
|
|
if c.flagDuration < debugMinInterval {
|
|
c.UI.Info(fmt.Sprintf("Overwriting duration value %q to the minimum value of %q", c.flagDuration, debugMinInterval))
|
|
c.flagDuration = debugMinInterval
|
|
}
|
|
if c.flagInterval < debugMinInterval {
|
|
c.UI.Info(fmt.Sprintf("Overwriting interval value %q to the minimum value of %q", c.flagInterval, debugMinInterval))
|
|
c.flagInterval = debugMinInterval
|
|
}
|
|
if c.flagMetricsInterval < debugMinInterval {
|
|
c.UI.Info(fmt.Sprintf("Overwriting metrics interval value %q to the minimum value of %q", c.flagMetricsInterval, debugMinInterval))
|
|
c.flagMetricsInterval = debugMinInterval
|
|
}
|
|
}
|
|
|
|
// These timing checks are always applicable since interval shouldn't be
|
|
// greater than the duration
|
|
if c.flagInterval > c.flagDuration {
|
|
c.UI.Info(fmt.Sprintf("Overwriting interval value %q to the duration value %q", c.flagInterval, c.flagDuration))
|
|
c.flagInterval = c.flagDuration
|
|
}
|
|
if c.flagMetricsInterval > c.flagDuration {
|
|
c.UI.Info(fmt.Sprintf("Overwriting metrics interval value %q to the duration value %q", c.flagMetricsInterval, c.flagDuration))
|
|
c.flagMetricsInterval = c.flagDuration
|
|
}
|
|
|
|
if len(c.flagTargets) == 0 {
|
|
c.flagTargets = c.defaultTargets()
|
|
} else {
|
|
// Check for any invalid targets and ignore them if found
|
|
invalidTargets := strutil.Difference(c.flagTargets, c.defaultTargets(), true)
|
|
if len(invalidTargets) != 0 {
|
|
c.UI.Info(fmt.Sprintf("Ignoring invalid targets: %s", strings.Join(invalidTargets, ", ")))
|
|
c.flagTargets = strutil.Difference(c.flagTargets, invalidTargets, true)
|
|
}
|
|
}
|
|
|
|
// Make sure we can talk to the server
|
|
client, err := c.Client()
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to create client to connect to Vault: %s", err)
|
|
}
|
|
serverHealth, err := client.Sys().Health()
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to connect to the server: %s", err)
|
|
}
|
|
|
|
// Check if server is DR Secondary and we need to further
|
|
// ignore any targets due to endpoint restrictions
|
|
if serverHealth.ReplicationDRMode == "secondary" {
|
|
invalidDRTargets := strutil.Difference(c.flagTargets, c.validDRSecondaryTargets(), true)
|
|
if len(invalidDRTargets) != 0 {
|
|
c.UI.Info(fmt.Sprintf("Ignoring invalid targets for DR Secondary: %s", strings.Join(invalidDRTargets, ", ")))
|
|
c.flagTargets = strutil.Difference(c.flagTargets, invalidDRTargets, true)
|
|
}
|
|
}
|
|
c.cachedClient = client
|
|
|
|
captureTime := time.Now().UTC()
|
|
if len(c.flagOutput) == 0 {
|
|
formattedTime := captureTime.Format(fileFriendlyTimeFormat)
|
|
c.flagOutput = fmt.Sprintf("vault-debug-%s", formattedTime)
|
|
}
|
|
|
|
// Strip trailing slash before proceeding
|
|
c.flagOutput = filepath.Clean(c.flagOutput)
|
|
|
|
// If compression is enabled, trim the extension so that the files are
|
|
// written to a directory even if compression somehow fails. We ensure the
|
|
// extension during compression. We also prevent overwriting if the file
|
|
// already exists.
|
|
dstOutputFile := c.flagOutput
|
|
if c.flagCompress {
|
|
if !strings.HasSuffix(dstOutputFile, ".tar.gz") && !strings.HasSuffix(dstOutputFile, ".tgz") {
|
|
dstOutputFile = dstOutputFile + debugCompressionExt
|
|
}
|
|
|
|
// Ensure that the file doesn't already exist, and ensure that we always
|
|
// trim the extension from flagOutput since we'll be progressively
|
|
// writing to that.
|
|
_, err := os.Stat(dstOutputFile)
|
|
switch {
|
|
case os.IsNotExist(err):
|
|
c.flagOutput = strings.TrimSuffix(c.flagOutput, ".tar.gz")
|
|
c.flagOutput = strings.TrimSuffix(c.flagOutput, ".tgz")
|
|
case err != nil:
|
|
return "", fmt.Errorf("unable to stat file: %s", err)
|
|
default:
|
|
return "", fmt.Errorf("output file already exists: %s", dstOutputFile)
|
|
}
|
|
}
|
|
|
|
// Stat check the directory to ensure we don't override any existing data.
|
|
_, err = os.Stat(c.flagOutput)
|
|
switch {
|
|
case os.IsNotExist(err):
|
|
err := os.MkdirAll(c.flagOutput, 0o700)
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to create output directory: %s", err)
|
|
}
|
|
case err != nil:
|
|
return "", fmt.Errorf("unable to stat directory: %s", err)
|
|
default:
|
|
return "", fmt.Errorf("output directory already exists: %s", c.flagOutput)
|
|
}
|
|
|
|
// Populate initial index fields
|
|
c.debugIndex = &debugIndex{
|
|
VaultAddress: client.Address(),
|
|
ClientVersion: version.GetVersion().VersionNumber(),
|
|
ServerVersion: serverHealth.Version,
|
|
Compress: c.flagCompress,
|
|
DurationSeconds: int(c.flagDuration.Seconds()),
|
|
IntervalSeconds: int(c.flagInterval.Seconds()),
|
|
MetricsIntervalSeconds: int(c.flagMetricsInterval.Seconds()),
|
|
RawArgs: rawArgs,
|
|
Version: debugIndexVersion,
|
|
Targets: c.flagTargets,
|
|
Timestamp: captureTime,
|
|
Errors: []*captureError{},
|
|
}
|
|
|
|
return dstOutputFile, nil
|
|
}
|
|
|
|
func (c *DebugCommand) defaultTargets() []string {
|
|
return []string{"config", "host", "requests", "metrics", "pprof", "replication-status", "server-status", "log"}
|
|
}
|
|
|
|
func (c *DebugCommand) validDRSecondaryTargets() []string {
|
|
return []string{"metrics", "replication-status", "server-status"}
|
|
}
|
|
|
|
func (c *DebugCommand) captureStaticTargets() error {
|
|
// Capture configuration state
|
|
if strutil.StrListContains(c.flagTargets, "config") {
|
|
c.logger.Info("capturing configuration state")
|
|
|
|
resp, err := c.cachedClient.Logical().Read("sys/config/state/sanitized")
|
|
if err != nil {
|
|
c.captureError("config", err)
|
|
c.logger.Error("config: error capturing config state", "error", err)
|
|
return nil
|
|
}
|
|
|
|
if resp != nil && resp.Data != nil {
|
|
collection := []map[string]interface{}{
|
|
{
|
|
"timestamp": time.Now().UTC(),
|
|
"config": resp.Data,
|
|
},
|
|
}
|
|
if err := c.persistCollection(collection, "config.json"); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error writing data to %s: %v", "config.json", err))
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// capturePollingTargets captures all dynamic targets over the specified
|
|
// duration and interval.
|
|
func (c *DebugCommand) capturePollingTargets() error {
|
|
var g run.Group
|
|
|
|
ctx, cancelFunc := context.WithTimeout(context.Background(), c.flagDuration+debugDurationGrace)
|
|
defer cancelFunc()
|
|
|
|
// This run group watches for interrupt or duration
|
|
g.Add(func() error {
|
|
for {
|
|
select {
|
|
case <-c.ShutdownCh:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return nil
|
|
}
|
|
}
|
|
}, func(error) {})
|
|
|
|
// Collect host-info if target is specified
|
|
if strutil.StrListContains(c.flagTargets, "host") {
|
|
g.Add(func() error {
|
|
c.collectHostInfo(ctx)
|
|
return nil
|
|
}, func(error) {
|
|
cancelFunc()
|
|
})
|
|
}
|
|
|
|
// Collect metrics if target is specified
|
|
if strutil.StrListContains(c.flagTargets, "metrics") {
|
|
g.Add(func() error {
|
|
c.collectMetrics(ctx)
|
|
return nil
|
|
}, func(error) {
|
|
cancelFunc()
|
|
})
|
|
}
|
|
|
|
// Collect pprof data if target is specified
|
|
if strutil.StrListContains(c.flagTargets, "pprof") {
|
|
g.Add(func() error {
|
|
c.collectPprof(ctx)
|
|
return nil
|
|
}, func(error) {
|
|
cancelFunc()
|
|
})
|
|
}
|
|
|
|
// Collect replication status if target is specified
|
|
if strutil.StrListContains(c.flagTargets, "replication-status") {
|
|
g.Add(func() error {
|
|
c.collectReplicationStatus(ctx)
|
|
return nil
|
|
}, func(error) {
|
|
cancelFunc()
|
|
})
|
|
}
|
|
|
|
// Collect server status if target is specified
|
|
if strutil.StrListContains(c.flagTargets, "server-status") {
|
|
g.Add(func() error {
|
|
c.collectServerStatus(ctx)
|
|
return nil
|
|
}, func(error) {
|
|
cancelFunc()
|
|
})
|
|
}
|
|
|
|
// Collect in-flight request status if target is specified
|
|
if strutil.StrListContains(c.flagTargets, "requests") {
|
|
g.Add(func() error {
|
|
c.collectInFlightRequestStatus(ctx)
|
|
return nil
|
|
}, func(error) {
|
|
cancelFunc()
|
|
})
|
|
}
|
|
|
|
if strutil.StrListContains(c.flagTargets, "log") {
|
|
g.Add(func() error {
|
|
c.writeLogs(ctx)
|
|
// If writeLogs returned earlier due to an error, wait for context
|
|
// to terminate so we don't abort everything.
|
|
<-ctx.Done()
|
|
return nil
|
|
}, func(error) {
|
|
cancelFunc()
|
|
})
|
|
}
|
|
|
|
// We shouldn't bump across errors since none is returned by the interrupts,
|
|
// but we error check for sanity here.
|
|
if err := g.Run(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Write collected data to their corresponding files
|
|
if err := c.persistCollection(c.metricsCollection, "metrics.json"); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error writing data to %s: %v", "metrics.json", err))
|
|
}
|
|
if err := c.persistCollection(c.serverStatusCollection, "server_status.json"); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error writing data to %s: %v", "server_status.json", err))
|
|
}
|
|
if err := c.persistCollection(c.replicationStatusCollection, "replication_status.json"); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error writing data to %s: %v", "replication_status.json", err))
|
|
}
|
|
if err := c.persistCollection(c.hostInfoCollection, "host_info.json"); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error writing data to %s: %v", "host_info.json", err))
|
|
}
|
|
if err := c.persistCollection(c.inFlightReqStatusCollection, "requests.json"); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error writing data to %s: %v", "requests.json", err))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *DebugCommand) collectHostInfo(ctx context.Context) {
|
|
idxCount := 0
|
|
intervalTicker := time.Tick(c.flagInterval)
|
|
|
|
for {
|
|
if idxCount > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-intervalTicker:
|
|
}
|
|
}
|
|
|
|
c.logger.Info("capturing host information", "count", idxCount)
|
|
idxCount++
|
|
|
|
r := c.cachedClient.NewRequest("GET", "/v1/sys/host-info")
|
|
resp, err := c.cachedClient.RawRequestWithContext(ctx, r)
|
|
if err != nil {
|
|
c.captureError("host", err)
|
|
return
|
|
}
|
|
if resp != nil {
|
|
defer resp.Body.Close()
|
|
|
|
secret, err := api.ParseSecret(resp.Body)
|
|
if err != nil {
|
|
c.captureError("host", err)
|
|
return
|
|
}
|
|
if secret != nil && secret.Data != nil {
|
|
hostEntry := secret.Data
|
|
c.hostInfoCollection = append(c.hostInfoCollection, hostEntry)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *DebugCommand) collectMetrics(ctx context.Context) {
|
|
idxCount := 0
|
|
intervalTicker := time.Tick(c.flagMetricsInterval)
|
|
|
|
for {
|
|
if idxCount > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-intervalTicker:
|
|
}
|
|
}
|
|
|
|
c.logger.Info("capturing metrics", "count", idxCount)
|
|
idxCount++
|
|
|
|
// Perform metrics request
|
|
r := c.cachedClient.NewRequest("GET", "/v1/sys/metrics")
|
|
resp, err := c.cachedClient.RawRequestWithContext(ctx, r)
|
|
if err != nil {
|
|
c.captureError("metrics", err)
|
|
continue
|
|
}
|
|
if resp != nil {
|
|
defer resp.Body.Close()
|
|
|
|
metricsEntry := make(map[string]interface{})
|
|
err := json.NewDecoder(resp.Body).Decode(&metricsEntry)
|
|
if err != nil {
|
|
c.captureError("metrics", err)
|
|
continue
|
|
}
|
|
c.metricsCollection = append(c.metricsCollection, metricsEntry)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *DebugCommand) collectPprof(ctx context.Context) {
|
|
idxCount := 0
|
|
startTime := time.Now()
|
|
intervalTicker := time.Tick(c.flagInterval)
|
|
|
|
for {
|
|
if idxCount > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-intervalTicker:
|
|
}
|
|
}
|
|
|
|
currentTimestamp := time.Now().UTC()
|
|
c.logger.Info("capturing pprof data", "count", idxCount)
|
|
idxCount++
|
|
|
|
// Create a sub-directory for pprof data
|
|
currentDir := currentTimestamp.Format(fileFriendlyTimeFormat)
|
|
dirName := filepath.Join(c.flagOutput, currentDir)
|
|
if err := os.MkdirAll(dirName, 0o700); err != nil {
|
|
c.UI.Error(fmt.Sprintf("Error creating sub-directory for time interval: %s", err))
|
|
continue
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
for _, target := range []string{"threadcreate", "allocs", "block", "mutex", "goroutine", "heap"} {
|
|
wg.Add(1)
|
|
go func(target string) {
|
|
defer wg.Done()
|
|
data, err := pprofTarget(ctx, c.cachedClient, target, nil)
|
|
if err != nil {
|
|
c.captureError("pprof."+target, err)
|
|
return
|
|
}
|
|
|
|
err = ioutil.WriteFile(filepath.Join(dirName, target+".prof"), data, 0o600)
|
|
if err != nil {
|
|
c.captureError("pprof."+target, err)
|
|
}
|
|
}(target)
|
|
}
|
|
|
|
// As a convenience, we'll also fetch the goroutine target using debug=2, which yields a text
|
|
// version of the stack traces that don't require using `go tool pprof` to view.
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
data, err := pprofTarget(ctx, c.cachedClient, "goroutine", url.Values{"debug": []string{"2"}})
|
|
if err != nil {
|
|
c.captureError("pprof.goroutines-text", err)
|
|
return
|
|
}
|
|
|
|
err = ioutil.WriteFile(filepath.Join(dirName, "goroutines.txt"), data, 0o600)
|
|
if err != nil {
|
|
c.captureError("pprof.goroutines-text", err)
|
|
}
|
|
}()
|
|
|
|
// If the our remaining duration is less than the interval value
|
|
// skip profile and trace.
|
|
runDuration := currentTimestamp.Sub(startTime)
|
|
if (c.flagDuration+debugDurationGrace)-runDuration < c.flagInterval {
|
|
wg.Wait()
|
|
continue
|
|
}
|
|
|
|
// Capture profile
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
data, err := pprofProfile(ctx, c.cachedClient, c.flagInterval)
|
|
if err != nil {
|
|
c.captureError("pprof.profile", err)
|
|
return
|
|
}
|
|
|
|
err = ioutil.WriteFile(filepath.Join(dirName, "profile.prof"), data, 0o600)
|
|
if err != nil {
|
|
c.captureError("pprof.profile", err)
|
|
}
|
|
}()
|
|
|
|
// Capture trace
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
data, err := pprofTrace(ctx, c.cachedClient, c.flagInterval)
|
|
if err != nil {
|
|
c.captureError("pprof.trace", err)
|
|
return
|
|
}
|
|
|
|
err = ioutil.WriteFile(filepath.Join(dirName, "trace.out"), data, 0o600)
|
|
if err != nil {
|
|
c.captureError("pprof.trace", err)
|
|
}
|
|
}()
|
|
|
|
wg.Wait()
|
|
}
|
|
}
|
|
|
|
func (c *DebugCommand) collectReplicationStatus(ctx context.Context) {
|
|
idxCount := 0
|
|
intervalTicker := time.Tick(c.flagInterval)
|
|
|
|
for {
|
|
if idxCount > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-intervalTicker:
|
|
}
|
|
}
|
|
|
|
c.logger.Info("capturing replication status", "count", idxCount)
|
|
idxCount++
|
|
|
|
r := c.cachedClient.NewRequest("GET", "/v1/sys/replication/status")
|
|
resp, err := c.cachedClient.RawRequestWithContext(ctx, r)
|
|
if err != nil {
|
|
c.captureError("replication-status", err)
|
|
return
|
|
}
|
|
if resp != nil {
|
|
defer resp.Body.Close()
|
|
|
|
secret, err := api.ParseSecret(resp.Body)
|
|
if err != nil {
|
|
c.captureError("replication-status", err)
|
|
return
|
|
}
|
|
if secret != nil && secret.Data != nil {
|
|
replicationEntry := secret.Data
|
|
replicationEntry["timestamp"] = time.Now().UTC()
|
|
c.replicationStatusCollection = append(c.replicationStatusCollection, replicationEntry)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *DebugCommand) collectServerStatus(ctx context.Context) {
|
|
idxCount := 0
|
|
intervalTicker := time.Tick(c.flagInterval)
|
|
|
|
for {
|
|
if idxCount > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-intervalTicker:
|
|
}
|
|
}
|
|
|
|
c.logger.Info("capturing server status", "count", idxCount)
|
|
idxCount++
|
|
|
|
healthInfo, err := c.cachedClient.Sys().Health()
|
|
if err != nil {
|
|
c.captureError("server-status.health", err)
|
|
}
|
|
sealInfo, err := c.cachedClient.Sys().SealStatus()
|
|
if err != nil {
|
|
c.captureError("server-status.seal", err)
|
|
}
|
|
|
|
statusEntry := map[string]interface{}{
|
|
"timestamp": time.Now().UTC(),
|
|
"health": healthInfo,
|
|
"seal": sealInfo,
|
|
}
|
|
c.serverStatusCollection = append(c.serverStatusCollection, statusEntry)
|
|
}
|
|
}
|
|
|
|
func (c *DebugCommand) collectInFlightRequestStatus(ctx context.Context) {
|
|
idxCount := 0
|
|
intervalTicker := time.Tick(c.flagInterval)
|
|
|
|
for {
|
|
if idxCount > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-intervalTicker:
|
|
}
|
|
}
|
|
|
|
c.logger.Info("capturing in-flight request status", "count", idxCount)
|
|
idxCount++
|
|
|
|
req := c.cachedClient.NewRequest("GET", "/v1/sys/in-flight-req")
|
|
resp, err := c.cachedClient.RawRequestWithContext(ctx, req)
|
|
if err != nil {
|
|
c.captureError("requests", err)
|
|
return
|
|
}
|
|
|
|
var data map[string]interface{}
|
|
if resp != nil {
|
|
defer resp.Body.Close()
|
|
err = jsonutil.DecodeJSONFromReader(resp.Body, &data)
|
|
if err != nil {
|
|
c.captureError("requests", err)
|
|
return
|
|
}
|
|
|
|
statusEntry := map[string]interface{}{
|
|
"timestamp": time.Now().UTC(),
|
|
"in_flight_requests": data,
|
|
}
|
|
c.inFlightReqStatusCollection = append(c.inFlightReqStatusCollection, statusEntry)
|
|
}
|
|
}
|
|
}
|
|
|
|
// persistCollection writes the collected data for a particular target onto the
|
|
// specified file. If the collection is empty, it returns immediately.
|
|
func (c *DebugCommand) persistCollection(collection []map[string]interface{}, outFile string) error {
|
|
if len(collection) == 0 {
|
|
return nil
|
|
}
|
|
|
|
// Write server-status file and update the index
|
|
bytes, err := json.MarshalIndent(collection, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := ioutil.WriteFile(filepath.Join(c.flagOutput, outFile), bytes, 0o600); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *DebugCommand) compress(dst string) error {
|
|
if runtime.GOOS != "windows" {
|
|
defer osutil.Umask(osutil.Umask(0o077))
|
|
}
|
|
|
|
tgz := archiver.NewTarGz()
|
|
if err := tgz.Archive([]string{c.flagOutput}, dst); err != nil {
|
|
return fmt.Errorf("failed to compress data: %s", err)
|
|
}
|
|
|
|
// If everything is fine up to this point, remove original directory
|
|
if err := os.RemoveAll(c.flagOutput); err != nil {
|
|
return fmt.Errorf("failed to remove data directory: %s", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func pprofTarget(ctx context.Context, client *api.Client, target string, params url.Values) ([]byte, error) {
|
|
req := client.NewRequest("GET", "/v1/sys/pprof/"+target)
|
|
if params != nil {
|
|
req.Params = params
|
|
}
|
|
resp, err := client.RawRequestWithContext(ctx, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
data, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return data, nil
|
|
}
|
|
|
|
func pprofProfile(ctx context.Context, client *api.Client, duration time.Duration) ([]byte, error) {
|
|
seconds := int(duration.Seconds())
|
|
secStr := strconv.Itoa(seconds)
|
|
|
|
req := client.NewRequest("GET", "/v1/sys/pprof/profile")
|
|
req.Params.Add("seconds", secStr)
|
|
resp, err := client.RawRequestWithContext(ctx, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
data, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return data, nil
|
|
}
|
|
|
|
func pprofTrace(ctx context.Context, client *api.Client, duration time.Duration) ([]byte, error) {
|
|
seconds := int(duration.Seconds())
|
|
secStr := strconv.Itoa(seconds)
|
|
|
|
req := client.NewRequest("GET", "/v1/sys/pprof/trace")
|
|
req.Params.Add("seconds", secStr)
|
|
resp, err := client.RawRequestWithContext(ctx, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
data, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return data, nil
|
|
}
|
|
|
|
// newCaptureError instantiates a new captureError.
|
|
func (c *DebugCommand) captureError(target string, err error) {
|
|
c.errLock.Lock()
|
|
c.debugIndex.Errors = append(c.debugIndex.Errors, &captureError{
|
|
TargetError: err.Error(),
|
|
Target: target,
|
|
Timestamp: time.Now().UTC(),
|
|
})
|
|
c.errLock.Unlock()
|
|
}
|
|
|
|
func (c *DebugCommand) writeLogs(ctx context.Context) {
|
|
out, err := os.OpenFile(filepath.Join(c.flagOutput, "vault.log"), os.O_CREATE|os.O_WRONLY, 0o600)
|
|
if err != nil {
|
|
c.captureError("log", err)
|
|
return
|
|
}
|
|
defer out.Close()
|
|
|
|
logCh, err := c.cachedClient.Sys().Monitor(ctx, "trace", c.logFormat)
|
|
if err != nil {
|
|
c.captureError("log", err)
|
|
return
|
|
}
|
|
|
|
for {
|
|
select {
|
|
case log := <-logCh:
|
|
if !strings.HasSuffix(log, "\n") {
|
|
log += "\n"
|
|
}
|
|
_, err = out.WriteString(log)
|
|
if err != nil {
|
|
c.captureError("log", err)
|
|
return
|
|
}
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}
|