open-nomad/command/operator_debug.go

package command

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/api/contexts"
	"github.com/hashicorp/nomad/helper"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/posener/complete"
)

type OperatorDebugCommand struct {
	Meta

	timestamp     string
	collectDir    string
	duration      time.Duration
	interval      time.Duration
	pprofDuration time.Duration
	logLevel      string
	stale         bool
	maxNodes      int
	nodeClass     string
	nodeIDs       []string
	serverIDs     []string
	consul        *external
	vault         *external
	manifest      []string
	ctx           context.Context
	cancel        context.CancelFunc
}

const (
	userAgent   = "nomad operator debug"
	clusterDir  = "cluster"
	clientDir   = "client"
	serverDir   = "server"
	intervalDir = "interval"
)

func (c *OperatorDebugCommand) Help() string {
	helpText := `
Usage: nomad operator debug [options]

  Build an archive containing Nomad cluster configuration and state, and Consul
  and Vault status. Include logs and pprof profiles for selected servers and
  client nodes.

  If ACLs are enabled, this command will require a token with the 'node:read'
  capability to run. In order to collect information, the token will also
  require the 'agent:read' and 'operator:read' capabilities, as well as the
  'list-jobs' capability for all namespaces. To collect pprof profiles the
  token will also require 'agent:write', or enable_debug configuration set to
  true.

General Options:

  ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + `

Consul Options:

  -consul-http-addr=<addr>
    The address and port of the Consul HTTP agent. Overrides the
    CONSUL_HTTP_ADDR environment variable.

  -consul-token=<token>
    Token used to query Consul. Overrides the CONSUL_HTTP_TOKEN environment
    variable and the Consul token file.

  -consul-token-file=<path>
    Path to the Consul token file. Overrides the CONSUL_HTTP_TOKEN_FILE
    environment variable.

  -consul-client-cert=<path>
    Path to the Consul client cert file. Overrides the CONSUL_CLIENT_CERT
    environment variable.

  -consul-client-key=<path>
    Path to the Consul client key file. Overrides the CONSUL_CLIENT_KEY
    environment variable.

  -consul-ca-cert=<path>
    Path to a CA file to use with Consul. Overrides the CONSUL_CACERT
    environment variable and the Consul CA path.

  -consul-ca-path=<path>
    Path to a directory of PEM encoded CA cert files to verify the Consul
    certificate. Overrides the CONSUL_CAPATH environment variable.

Vault Options:

  -vault-address=<addr>
    The address and port of the Vault HTTP agent. Overrides the VAULT_ADDR
    environment variable.

  -vault-token=<token>
    Token used to query Vault. Overrides the VAULT_TOKEN environment
    variable.

  -vault-client-cert=<path>
    Path to the Vault client cert file. Overrides the VAULT_CLIENT_CERT
    environment variable.

  -vault-client-key=<path>
    Path to the Vault client key file. Overrides the VAULT_CLIENT_KEY
    environment variable.

  -vault-ca-cert=<path>
    Path to a CA file to use with Vault. Overrides the VAULT_CACERT
    environment variable and the Vault CA path.

  -vault-ca-path=<path>
    Path to a directory of PEM encoded CA cert files to verify the Vault
    certificate. Overrides the VAULT_CAPATH environment variable.

Debug Options:

  -duration=<duration>
    The duration of the log monitor command. Defaults to 2m.

  -interval=<interval>
    The interval between snapshots of the Nomad state. Set interval equal to 
    duration to capture a single snapshot. Defaults to 30s.

  -log-level=<level>
    The log level to monitor. Defaults to DEBUG.

  -max-nodes=<count>
    Cap the maximum number of client nodes included in the capture. Defaults
    to 10, set to 0 for unlimited.

  -node-id=<node>,<node>
    Comma separated list of Nomad client node ids to monitor for logs, API
    outputs, and pprof profiles. Accepts id prefixes, and "all" to select all
    nodes (up to count = max-nodes). Defaults to "all".

  -node-class=<node-class>
    Filter client nodes based on node class.

  -pprof-duration=<duration>
    Duration for pprof collection. Defaults to 1s.

  -server-id=<server>,<server>
    Comma separated list of Nomad server names to monitor for logs, API
    outputs, and pprof profiles. Accepts server names, "leader", or "all".
    Defaults to "all".

  -stale=<true|false>
    If "false", the default, get membership data from the cluster leader. If
    the cluster is in an outage unable to establish leadership, it may be
    necessary to get the configuration from a non-leader server.

  -output=<path>
    Path to the parent directory of the output directory. If not specified, an
    archive is built in the current directory.
`
	return strings.TrimSpace(helpText)
}

func (c *OperatorDebugCommand) Synopsis() string {
	return "Build a debug archive"
}

func (c *OperatorDebugCommand) AutocompleteFlags() complete.Flags {
	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
		complete.Flags{
			"-duration":       complete.PredictAnything,
			"-interval":       complete.PredictAnything,
			"-log-level":      complete.PredictSet("TRACE", "DEBUG", "INFO", "WARN", "ERROR"),
			"-max-nodes":      complete.PredictAnything,
			"-node-class":     NodeClassPredictor(c.Client),
			"-node-id":        NodePredictor(c.Client),
			"-server-id":      ServerPredictor(c.Client),
			"-output":         complete.PredictDirs("*"),
			"-pprof-duration": complete.PredictAnything,
			"-consul-token":   complete.PredictAnything,
			"-vault-token":    complete.PredictAnything,
		})
}

func (c *OperatorDebugCommand) AutocompleteArgs() complete.Predictor {
	return complete.PredictNothing
}

// NodePredictor returns a client node predictor
func NodePredictor(factory ApiClientFactory) complete.Predictor {
	return complete.PredictFunc(func(a complete.Args) []string {
		client, err := factory()
		if err != nil {
			return nil
		}

		resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Nodes, nil)
		if err != nil {
			return []string{}
		}
		return resp.Matches[contexts.Nodes]
	})
}

// NodeClassPredictor returns a client node class predictor
// TODO: Consider API options for node class filtering
func NodeClassPredictor(factory ApiClientFactory) complete.Predictor {
	return complete.PredictFunc(func(a complete.Args) []string {
		client, err := factory()
		if err != nil {
			return nil
		}

		nodes, _, err := client.Nodes().List(nil) // TODO: should be *api.QueryOptions that matches region
		if err != nil {
			return []string{}
		}

		// Build map of unique node classes across all nodes
		classes := make(map[string]bool)
		for _, node := range nodes {
			classes[node.NodeClass] = true
		}

		// Iterate over node classes looking for match
		filtered := []string{}
		for class := range classes {
			if strings.HasPrefix(class, a.Last) {
				filtered = append(filtered, class)
			}
		}

		return filtered
	})
}

// ServerPredictor returns a server member predictor
// TODO: Consider API options for server member filtering
func ServerPredictor(factory ApiClientFactory) complete.Predictor {
	return complete.PredictFunc(func(a complete.Args) []string {
		client, err := factory()
		if err != nil {
			return nil
		}
		members, err := client.Agent().Members()
		if err != nil {
			return []string{}
		}

		// Iterate over server members looking for match
		filtered := []string{}
		for _, member := range members.Members {
			if strings.HasPrefix(member.Name, a.Last) {
				filtered = append(filtered, member.Name)
			}
		}

		return filtered
	})
}

func (c *OperatorDebugCommand) Name() string { return "debug" }

func (c *OperatorDebugCommand) Run(args []string) int {
	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }

	var duration, interval, output, pprofDuration string
	var nodeIDs, serverIDs string

	flags.StringVar(&duration, "duration", "2m", "")
	flags.StringVar(&interval, "interval", "30s", "")
	flags.StringVar(&c.logLevel, "log-level", "DEBUG", "")
	flags.IntVar(&c.maxNodes, "max-nodes", 10, "")
	flags.StringVar(&c.nodeClass, "node-class", "", "")
	flags.StringVar(&nodeIDs, "node-id", "", "")
	flags.StringVar(&serverIDs, "server-id", "all", "")
	flags.BoolVar(&c.stale, "stale", false, "")
	flags.StringVar(&output, "output", "", "")
	flags.StringVar(&pprofDuration, "pprof-duration", "1s", "")

	c.consul = &external{tls: &api.TLSConfig{}}
	flags.StringVar(&c.consul.addrVal, "consul-http-addr", os.Getenv("CONSUL_HTTP_ADDR"), "")
	ssl := os.Getenv("CONSUL_HTTP_SSL")
	c.consul.ssl, _ = strconv.ParseBool(ssl)
	flags.StringVar(&c.consul.auth, "consul-auth", os.Getenv("CONSUL_HTTP_AUTH"), "")
	flags.StringVar(&c.consul.tokenVal, "consul-token", os.Getenv("CONSUL_HTTP_TOKEN"), "")
	flags.StringVar(&c.consul.tokenFile, "consul-token-file", os.Getenv("CONSUL_HTTP_TOKEN_FILE"), "")
	flags.StringVar(&c.consul.tls.ClientCert, "consul-client-cert", os.Getenv("CONSUL_CLIENT_CERT"), "")
	flags.StringVar(&c.consul.tls.ClientKey, "consul-client-key", os.Getenv("CONSUL_CLIENT_KEY"), "")
	flags.StringVar(&c.consul.tls.CACert, "consul-ca-cert", os.Getenv("CONSUL_CACERT"), "")
	flags.StringVar(&c.consul.tls.CAPath, "consul-ca-path", os.Getenv("CONSUL_CAPATH"), "")

	c.vault = &external{tls: &api.TLSConfig{}}
	flags.StringVar(&c.vault.addrVal, "vault-address", os.Getenv("VAULT_ADDR"), "")
	flags.StringVar(&c.vault.tokenVal, "vault-token", os.Getenv("VAULT_TOKEN"), "")
	flags.StringVar(&c.vault.tls.CACert, "vault-ca-cert", os.Getenv("VAULT_CACERT"), "")
	flags.StringVar(&c.vault.tls.CAPath, "vault-ca-path", os.Getenv("VAULT_CAPATH"), "")
	flags.StringVar(&c.vault.tls.ClientCert, "vault-client-cert", os.Getenv("VAULT_CLIENT_CERT"), "")
	flags.StringVar(&c.vault.tls.ClientKey, "vault-client-key", os.Getenv("VAULT_CLIENT_KEY"), "")

	if err := flags.Parse(args); err != nil {
		c.Ui.Error(fmt.Sprintf("Error parsing arguments: %q", err))
		return 1
	}

	// Parse the capture duration
	d, err := time.ParseDuration(duration)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error parsing duration: %s: %s", duration, err.Error()))
		return 1
	}
	c.duration = d

	// Parse the capture interval
	i, err := time.ParseDuration(interval)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error parsing interval: %s: %s", interval, err.Error()))
		return 1
	}
	c.interval = i

	// Validate interval
	if i.Seconds() > d.Seconds() {
		c.Ui.Error(fmt.Sprintf("Error parsing interval: %s is greater than duration %s", interval, duration))
		return 1
	}

	// Parse the pprof capture duration
	pd, err := time.ParseDuration(pprofDuration)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error parsing pprof duration: %s: %s", pprofDuration, err.Error()))
		return 1
	}
	c.pprofDuration = pd

	// Verify there are no extra arguments
	args = flags.Args()
	if l := len(args); l != 0 {
		c.Ui.Error("This command takes no arguments")
		c.Ui.Error(commandErrorText(c))
		return 1
	}

	// Initialize capture variables and structs
	c.manifest = make([]string, 0)
	ctx, cancel := context.WithCancel(context.Background())
	c.ctx = ctx
	c.cancel = cancel
	c.trap()

	// Generate timestamped file name
	format := "2006-01-02-150405Z"
	c.timestamp = time.Now().UTC().Format(format)
	stamped := "nomad-debug-" + c.timestamp

	// Create the output directory
	var tmp string
	if output != "" {
		// User specified output directory
		tmp = filepath.Join(output, stamped)
		_, err := os.Stat(tmp)
		if !os.IsNotExist(err) {
			c.Ui.Error("Output directory already exists")
			return 2
		}
	} else {
		// Generate temp directory
		tmp, err = ioutil.TempDir(os.TempDir(), stamped)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error creating tmp directory: %s", err.Error()))
			return 2
		}
		defer os.RemoveAll(tmp)
	}

	c.collectDir = tmp

	// Create an instance of the API client
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err.Error()))
		return 1
	}

	// Search all nodes If a node class is specified without a list of node id prefixes
	if c.nodeClass != "" && nodeIDs == "" {
		nodeIDs = "all"
	}

	// Resolve client node id prefixes
	nodesFound := 0
	nodeLookupFailCount := 0
	nodeCaptureCount := 0

	for _, id := range stringToSlice(nodeIDs) {
		if id == "all" {
			// Capture from all nodes using empty prefix filter
			id = ""
		} else {
			// Capture from nodes starting with prefix id
			id = sanitizeUUIDPrefix(id)
		}
		nodes, _, err := client.Nodes().PrefixList(id)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err))
			return 1
		}

		// Increment fail count if no nodes are found
		if len(nodes) == 0 {
			c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", id))
			nodeLookupFailCount++
			continue
		}

		nodesFound += len(nodes)

		// Apply constraints to nodes found
		for _, n := range nodes {
			// Ignore nodes that do not match specified class
			if c.nodeClass != "" && n.NodeClass != c.nodeClass {
				continue
			}

			// Add node to capture list
			c.nodeIDs = append(c.nodeIDs, n.ID)
			nodeCaptureCount++

			// Stop looping when we reach the max
			if c.maxNodes != 0 && nodeCaptureCount >= c.maxNodes {
				break
			}
		}
	}

	// Return error if nodes were specified but none were found
	if len(nodeIDs) > 0 && nodeCaptureCount == 0 {
		c.Ui.Error(fmt.Sprintf("Failed to retrieve clients, 0 nodes found in list: %s", nodeIDs))
		return 1
	}

	// Resolve servers
	members, err := client.Agent().Members()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to retrieve server list; err: %v", err))
		return 1
	}

	// Write complete list of server members to file
	c.writeJSON(clusterDir, "members.json", members, err)

	// Filter for servers matching criteria
	c.serverIDs, err = filterServerMembers(members, serverIDs, c.region)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to parse server list; err: %v", err))
		return 1
	}

	serversFound := 0
	serverCaptureCount := 0

	if members != nil {
		serversFound = len(members.Members)
	}
	if c.serverIDs != nil {
		serverCaptureCount = len(c.serverIDs)
	}

	// Return error if servers were specified but not found
	if len(serverIDs) > 0 && serverCaptureCount == 0 {
		c.Ui.Error(fmt.Sprintf("Failed to retrieve servers, 0 members found in list: %s", serverIDs))
		return 1
	}

	// Display general info about the capture
	c.Ui.Output("Starting debugger...")
	c.Ui.Output("")
	c.Ui.Output(fmt.Sprintf("           Region: %s", c.region))
	c.Ui.Output(fmt.Sprintf("        Namespace: %s", c.namespace))
	c.Ui.Output(fmt.Sprintf("          Servers: (%d/%d) %v", serverCaptureCount, serversFound, c.serverIDs))
	c.Ui.Output(fmt.Sprintf("          Clients: (%d/%d) %v", nodeCaptureCount, nodesFound, c.nodeIDs))
	if nodeCaptureCount > 0 && nodeCaptureCount == c.maxNodes {
		c.Ui.Output(fmt.Sprintf("                   Max node count reached (%d)", c.maxNodes))
	}
	if nodeLookupFailCount > 0 {
		c.Ui.Output(fmt.Sprintf("Client fail count: %v", nodeLookupFailCount))
	}
	if c.nodeClass != "" {
		c.Ui.Output(fmt.Sprintf("       Node Class: %s", c.nodeClass))
	}
	c.Ui.Output(fmt.Sprintf("         Interval: %s", interval))
	c.Ui.Output(fmt.Sprintf("         Duration: %s", duration))
	if c.pprofDuration.Seconds() != 1 {
		c.Ui.Output(fmt.Sprintf("   pprof Duration: %s", c.pprofDuration))
	}
	c.Ui.Output("")
	c.Ui.Output("Capturing cluster data...")

	// Start collecting data
	err = c.collect(client)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error collecting data: %s", err.Error()))
		return 2
	}

	// Write index json/html manifest files
	c.writeManifest()

	// Exit before archive if output directory was specified
	if output != "" {
		c.Ui.Output(fmt.Sprintf("Created debug directory: %s", c.collectDir))
		return 0
	}

	// Create archive tarball
	archiveFile := stamped + ".tar.gz"
	err = TarCZF(archiveFile, tmp, stamped)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error creating archive: %s", err.Error()))
		return 2
	}

	// Final output with name of tarball
	c.Ui.Output(fmt.Sprintf("Created debug archive: %s", archiveFile))
	return 0
}

// collect collects data from our endpoints and writes the archive bundle
func (c *OperatorDebugCommand) collect(client *api.Client) error {
	// Collect cluster data

	self, err := client.Agent().Self()
	c.writeJSON(clusterDir, "agent-self.json", self, err)

	var qo *api.QueryOptions
	namespaces, _, err := client.Namespaces().List(qo)
	c.writeJSON(clusterDir, "namespaces.json", namespaces, err)

	regions, err := client.Regions().List()
	c.writeJSON(clusterDir, "regions.json", regions, err)

	// Fetch data directly from consul and vault. Ignore errors
	var consul, vault string

	if self != nil {
		r, ok := self.Config["Consul"]
		if ok {
			m, ok := r.(map[string]interface{})
			if ok {

				raw := m["Addr"]
				consul, _ = raw.(string)
				raw = m["EnableSSL"]
				ssl, _ := raw.(bool)
				if ssl {
					consul = "https://" + consul
				} else {
					consul = "http://" + consul
				}
			}
		}

		r, ok = self.Config["Vault"]
		if ok {
			m, ok := r.(map[string]interface{})
			if ok {
				raw := m["Addr"]
				vault, _ = raw.(string)
			}
		}
	}

	c.collectConsul(clusterDir, consul)
	c.collectVault(clusterDir, vault)
	c.collectAgentHosts(client)
	c.collectPprofs(client)

	c.startMonitors(client)
	c.collectPeriodic(client)

	return nil
}

// path returns platform specific paths in the tmp root directory
func (c *OperatorDebugCommand) path(paths ...string) string {
	ps := []string{c.collectDir}
	ps = append(ps, paths...)
	return filepath.Join(ps...)
}

// mkdir creates directories in the tmp root directory
func (c *OperatorDebugCommand) mkdir(paths ...string) error {
	joinedPath := c.path(paths...)

	// Ensure path doesn't escape the sandbox of the capture directory
	escapes := helper.PathEscapesSandbox(c.collectDir, joinedPath)
	if escapes {
		return fmt.Errorf("file path escapes capture directory")
	}

	return os.MkdirAll(joinedPath, 0755)
}

// startMonitors starts go routines for each node and client
func (c *OperatorDebugCommand) startMonitors(client *api.Client) {
	for _, id := range c.nodeIDs {
		go c.startMonitor(clientDir, "node_id", id, client)
	}

	for _, id := range c.serverIDs {
		go c.startMonitor(serverDir, "server_id", id, client)
	}
}

// startMonitor starts one monitor api request, writing to a file. It blocks and should be
// called in a go routine. Errors are ignored, we want to build the archive even if a node
// is unavailable
func (c *OperatorDebugCommand) startMonitor(path, idKey, nodeID string, client *api.Client) {
	c.mkdir(path, nodeID)
	fh, err := os.Create(c.path(path, nodeID, "monitor.log"))
	if err != nil {
		return
	}
	defer fh.Close()

	qo := api.QueryOptions{
		Params: map[string]string{
			idKey:       nodeID,
			"log_level": c.logLevel,
		},
	}

	outCh, errCh := client.Agent().Monitor(c.ctx.Done(), &qo)
	for {
		select {
		case out := <-outCh:
			if out == nil {
				continue
			}
			fh.Write(out.Data)

		case err := <-errCh:
			fh.WriteString(fmt.Sprintf("monitor: %s\n", err.Error()))
			return

		case <-c.ctx.Done():
			return
		}
	}
}

// collectAgentHosts calls collectAgentHost for each selected node
func (c *OperatorDebugCommand) collectAgentHosts(client *api.Client) {
	for _, n := range c.nodeIDs {
		c.collectAgentHost(clientDir, n, client)
	}

	for _, n := range c.serverIDs {
		c.collectAgentHost(serverDir, n, client)
	}
}

// collectAgentHost gets the agent host data
func (c *OperatorDebugCommand) collectAgentHost(path, id string, client *api.Client) {
	var host *api.HostDataResponse
	var err error
	if path == serverDir {
		host, err = client.Agent().Host(id, "", nil)
	} else {
		host, err = client.Agent().Host("", id, nil)
	}

	if err != nil {
		c.Ui.Error(fmt.Sprintf("%s/%s: Failed to retrieve agent host data, err: %v", path, id, err))

		if strings.Contains(err.Error(), structs.ErrPermissionDenied.Error()) {
			// Drop a hint to help the operator resolve the error
			c.Ui.Warn("Agent host retrieval requires agent:read ACL or enable_debug=true.  See https://www.nomadproject.io/api-docs/agent#host for more information.")
		}
		return // exit on any error
	}

	path = filepath.Join(path, id)
	c.writeJSON(path, "agent-host.json", host, err)
}

// collectPprofs captures the /agent/pprof for each listed node
func (c *OperatorDebugCommand) collectPprofs(client *api.Client) {
	for _, n := range c.nodeIDs {
		c.collectPprof(clientDir, n, client)
	}

	for _, n := range c.serverIDs {
		c.collectPprof(serverDir, n, client)
	}
}

// collectPprof captures pprof data for the node
func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client) {
	pprofDurationSeconds := int(c.pprofDuration.Seconds())
	opts := api.PprofOptions{Seconds: pprofDurationSeconds}
	if path == serverDir {
		opts.ServerID = id
	} else {
		opts.NodeID = id
	}

	path = filepath.Join(path, id)

	bs, err := client.Agent().CPUProfile(opts, nil)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof profile.prof, err: %v", path, err))
		if structs.IsErrPermissionDenied(err) {
			// All Profiles require the same permissions, so we only need to see
			// one permission failure before we bail.
			// But lets first drop a hint to help the operator resolve the error

			c.Ui.Warn("Pprof retrieval requires agent:write ACL or enable_debug=true.  See https://www.nomadproject.io/api-docs/agent#agent-runtime-profiles for more information.")
			return // only exit on 403
		}
	} else {
		err := c.writeBytes(path, "profile.prof", bs)
		if err != nil {
			c.Ui.Error(err.Error())
		}
	}

	// goroutine debug type 1 = legacy text format for human readable output
	opts.Debug = 1
	c.savePprofProfile(path, "goroutine", opts, client)

	// goroutine debug type 2 = goroutine stacks in panic format
	opts.Debug = 2
	c.savePprofProfile(path, "goroutine", opts, client)

	// Reset to pprof binary format
	opts.Debug = 0

	c.savePprofProfile(path, "goroutine", opts, client)    // Stack traces of all current goroutines
	c.savePprofProfile(path, "trace", opts, client)        // A trace of execution of the current program
	c.savePprofProfile(path, "heap", opts, client)         // A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.
	c.savePprofProfile(path, "allocs", opts, client)       // A sampling of all past memory allocations
	c.savePprofProfile(path, "threadcreate", opts, client) // Stack traces that led to the creation of new OS threads

	// This profile is disabled by default -- Requires runtime.SetBlockProfileRate to enable
	// c.savePprofProfile(path, "block", opts, client)        // Stack traces that led to blocking on synchronization primitives

	// This profile is disabled by default -- Requires runtime.SetMutexProfileFraction to enable
	// c.savePprofProfile(path, "mutex", opts, client)        // Stack traces of holders of contended mutexes
}

// savePprofProfile retrieves a pprof profile and writes to disk
func (c *OperatorDebugCommand) savePprofProfile(path string, profile string, opts api.PprofOptions, client *api.Client) {
	fileName := fmt.Sprintf("%s.prof", profile)
	if opts.Debug > 0 {
		fileName = fmt.Sprintf("%s-debug%d.txt", profile, opts.Debug)
	}

	bs, err := retrievePprofProfile(profile, opts, client)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof %s, err: %s", path, fileName, err.Error()))
	}

	err = c.writeBytes(path, fileName, bs)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("%s: Failed to write file %s, err: %s", path, fileName, err.Error()))
	}
}

// retrievePprofProfile gets a pprof profile from the node specified in opts using the API client
func retrievePprofProfile(profile string, opts api.PprofOptions, client *api.Client) (bs []byte, err error) {
	switch profile {
	case "cpuprofile":
		bs, err = client.Agent().CPUProfile(opts, nil)
	case "trace":
		bs, err = client.Agent().Trace(opts, nil)
	default:
		bs, err = client.Agent().Lookup(profile, opts, nil)
	}

	return bs, err
}

// collectPeriodic runs for duration, capturing the cluster state every interval. It flushes and stops
// the monitor requests
func (c *OperatorDebugCommand) collectPeriodic(client *api.Client) {
	duration := time.After(c.duration)
	// Set interval to 0 so that we immediately execute, wait the interval next time
	interval := time.After(0 * time.Second)
	var intervalCount int
	var name, dir string

	for {
		select {
		case <-duration:
			c.cancel()
			return

		case <-interval:
			name = fmt.Sprintf("%04d", intervalCount)
			dir = filepath.Join(intervalDir, name)
			c.Ui.Output(fmt.Sprintf("    Capture interval %s", name))
			c.collectNomad(dir, client)
			c.collectOperator(dir, client)
			interval = time.After(c.interval)
			intervalCount++

		case <-c.ctx.Done():
			return
		}
	}
}

// collectOperator captures some cluster meta information
func (c *OperatorDebugCommand) collectOperator(dir string, client *api.Client) {
	rc, err := client.Operator().RaftGetConfiguration(nil)
	c.writeJSON(dir, "operator-raft.json", rc, err)

	sc, _, err := client.Operator().SchedulerGetConfiguration(nil)
	c.writeJSON(dir, "operator-scheduler.json", sc, err)

	ah, _, err := client.Operator().AutopilotServerHealth(nil)
	c.writeJSON(dir, "operator-autopilot-health.json", ah, err)

	lic, _, err := client.Operator().LicenseGet(nil)
	c.writeJSON(dir, "license.json", lic, err)
}

// collectNomad captures the nomad cluster state
func (c *OperatorDebugCommand) collectNomad(dir string, client *api.Client) error {
	var qo *api.QueryOptions

	js, _, err := client.Jobs().List(qo)
	c.writeJSON(dir, "jobs.json", js, err)

	ds, _, err := client.Deployments().List(qo)
	c.writeJSON(dir, "deployments.json", ds, err)

	es, _, err := client.Evaluations().List(qo)
	c.writeJSON(dir, "evaluations.json", es, err)

	as, _, err := client.Allocations().List(qo)
	c.writeJSON(dir, "allocations.json", as, err)

	ns, _, err := client.Nodes().List(qo)
	c.writeJSON(dir, "nodes.json", ns, err)

	// CSI Plugins - /v1/plugins?type=csi
	ps, _, err := client.CSIPlugins().List(qo)
	c.writeJSON(dir, "csi-plugins.json", ps, err)

	// CSI Plugin details - /v1/plugin/csi/:plugin_id
	for _, p := range ps {
		csiPlugin, _, err := client.CSIPlugins().Info(p.ID, qo)
		csiPluginFileName := fmt.Sprintf("csi-plugin-id-%s.json", p.ID)
		c.writeJSON(dir, csiPluginFileName, csiPlugin, err)
	}

	// CSI Volumes - /v1/volumes?type=csi
	csiVolumes, _, err := client.CSIVolumes().List(qo)
	c.writeJSON(dir, "csi-volumes.json", csiVolumes, err)

	// CSI Volume details - /v1/volumes/csi/:volume-id
	for _, v := range csiVolumes {
		csiVolume, _, err := client.CSIVolumes().Info(v.ID, qo)
		csiFileName := fmt.Sprintf("csi-volume-id-%s.json", v.ID)
		c.writeJSON(dir, csiFileName, csiVolume, err)
	}

	metrics, _, err := client.Operator().MetricsSummary(qo)
	c.writeJSON(dir, "metrics.json", metrics, err)

	return nil
}

// collectConsul calls the Consul API directly to collect data
func (c *OperatorDebugCommand) collectConsul(dir, consul string) error {
	addr := c.consul.addr(consul)
	if addr == "" {
		return nil
	}

	client := defaultHttpClient()
	api.ConfigureTLS(client, c.consul.tls)

	req, _ := http.NewRequest("GET", addr+"/v1/agent/self", nil)
	req.Header.Add("X-Consul-Token", c.consul.token())
	req.Header.Add("User-Agent", userAgent)
	resp, err := client.Do(req)
	c.writeBody(dir, "consul-agent-self.json", resp, err)

	req, _ = http.NewRequest("GET", addr+"/v1/agent/members", nil)
	req.Header.Add("X-Consul-Token", c.consul.token())
	req.Header.Add("User-Agent", userAgent)
	resp, err = client.Do(req)
	c.writeBody(dir, "consul-agent-members.json", resp, err)

	return nil
}

// collectVault calls the Vault API directly to collect data
func (c *OperatorDebugCommand) collectVault(dir, vault string) error {
	addr := c.vault.addr(vault)
	if addr == "" {
		return nil
	}

	client := defaultHttpClient()
	api.ConfigureTLS(client, c.vault.tls)

	req, _ := http.NewRequest("GET", addr+"/sys/health", nil)
	req.Header.Add("X-Vault-Token", c.vault.token())
	req.Header.Add("User-Agent", userAgent)
	resp, err := client.Do(req)
	c.writeBody(dir, "vault-sys-health.json", resp, err)

	return nil
}

// writeBytes writes a file to the archive, recording it in the manifest
func (c *OperatorDebugCommand) writeBytes(dir, file string, data []byte) error {
	// Replace invalid characters in filename
	filename := helper.CleanFilename(file, "_")

	relativePath := filepath.Join(dir, filename)
	c.manifest = append(c.manifest, relativePath)
	dirPath := filepath.Join(c.collectDir, dir)
	filePath := filepath.Join(dirPath, filename)

	// Ensure parent directories exist
	err := os.MkdirAll(dirPath, os.ModePerm)
	if err != nil {
		return fmt.Errorf("failed to create parent directories of \"%s\": %w", dirPath, err)
	}

	// Ensure filename doesn't escape the sandbox of the capture directory
	escapes := helper.PathEscapesSandbox(c.collectDir, filePath)
	if escapes {
		return fmt.Errorf("file path \"%s\" escapes capture directory \"%s\"", filePath, c.collectDir)
	}

	// Create the file
	fh, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("failed to create file \"%s\", err: %w", filePath, err)
	}
	defer fh.Close()

	_, err = fh.Write(data)
	if err != nil {
		return fmt.Errorf("Failed to write data to file \"%s\", err: %w", filePath, err)
	}
	return nil
}

// writeJSON writes JSON responses from the Nomad API calls to the archive
func (c *OperatorDebugCommand) writeJSON(dir, file string, data interface{}, err error) error {
	if err != nil {
		return c.writeError(dir, file, err)
	}
	bytes, err := json.Marshal(data)
	if err != nil {
		return c.writeError(dir, file, err)
	}
	err = c.writeBytes(dir, file, bytes)
	if err != nil {
		c.Ui.Error(err.Error())
	}
	return nil
}

// writeError writes a JSON error object to capture errors in the debug bundle without
// reporting
func (c *OperatorDebugCommand) writeError(dir, file string, err error) error {
	bytes, err := json.Marshal(errorWrapper{Error: err.Error()})
	if err != nil {
		return err
	}
	return c.writeBytes(dir, file, bytes)
}

type errorWrapper struct {
	Error string
}

// writeBody is a helper that writes the body of an http.Response to the archive
func (c *OperatorDebugCommand) writeBody(dir, file string, resp *http.Response, err error) {
	if err != nil {
		c.writeError(dir, file, err)
		return
	}

	if resp.ContentLength == 0 {
		return
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		c.writeError(dir, file, err)
		return
	}

	if err := c.writeBytes(dir, file, body); err != nil {
		c.Ui.Error(err.Error())
	}
}

// writeManifest creates the index files
func (c *OperatorDebugCommand) writeManifest() error {
	// Write the JSON
	path := filepath.Join(c.collectDir, "index.json")
	jsonFh, err := os.Create(path)
	if err != nil {
		return err
	}
	defer jsonFh.Close()

	json.NewEncoder(jsonFh).Encode(c.manifest)

	// Write the HTML
	path = filepath.Join(c.collectDir, "index.html")
	htmlFh, err := os.Create(path)
	if err != nil {
		return err
	}
	defer htmlFh.Close()

	head, _ := template.New("head").Parse("<html><head><title>{{.}}</title></head>\n<body><h1>{{.}}</h1>\n<ul>")
	line, _ := template.New("line").Parse("<li><a href=\"{{.}}\">{{.}}</a></li>\n")
	if err != nil {
		return fmt.Errorf("%v", err)
	}
	tail := "</ul></body></html>\n"

	head.Execute(htmlFh, c.timestamp)
	for _, f := range c.manifest {
		line.Execute(htmlFh, f)
	}
	htmlFh.WriteString(tail)

	return nil
}

// trap captures signals, and closes stopCh
func (c *OperatorDebugCommand) trap() {
	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	go func() {
		<-sigCh
		c.cancel()
	}()
}

// TarCZF like the tar command, recursively builds a gzip compressed tar
// archive from a directory. If not empty, all files in the bundle are prefixed
// with the target path.
func TarCZF(archive string, src, target string) error {
	// ensure the src actually exists before trying to tar it
	if _, err := os.Stat(src); err != nil {
		return fmt.Errorf("Unable to tar files - %v", err.Error())
	}

	// create the archive
	fh, err := os.Create(archive)
	if err != nil {
		return err
	}
	defer fh.Close()

	zz := gzip.NewWriter(fh)
	defer zz.Close()

	tw := tar.NewWriter(zz)
	defer tw.Close()

	// tar
	return filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {

		// return on any error
		if err != nil {
			return err
		}

		if !fi.Mode().IsRegular() {
			return nil
		}

		header, err := tar.FileInfoHeader(fi, fi.Name())
		if err != nil {
			return err
		}

		// remove leading path to the src, so files are relative to the archive
		path := strings.ReplaceAll(file, src, "")
		if target != "" {
			path = filepath.Join([]string{target, path}...)
		}
		path = strings.TrimPrefix(path, string(filepath.Separator))

		header.Name = path

		if err := tw.WriteHeader(header); err != nil {
			return err
		}

		// copy the file contents
		f, err := os.Open(file)
		if err != nil {
			return err
		}

		if _, err := io.Copy(tw, f); err != nil {
			return err
		}

		f.Close()

		return nil
	})
}

// filterServerMembers returns a slice of server member names matching the search criteria
func filterServerMembers(serverMembers *api.ServerMembers, serverIDs string, region string) (membersFound []string, err error) {
	if serverMembers.Members == nil {
		return nil, fmt.Errorf("Failed to parse server members, members==nil")
	}

	prefixes := stringToSlice(serverIDs)

	// "leader" is a special case which Nomad handles in the API.  If "leader"
	// appears in serverIDs, add it to membersFound and remove it from the list
	// so that it isn't processed by the range loop
	if helper.SliceStringContains(prefixes, "leader") {
		membersFound = append(membersFound, "leader")
		helper.RemoveEqualFold(&prefixes, "leader")
	}

	for _, member := range serverMembers.Members {
		// If region is provided it must match exactly
		if region != "" && member.Tags["region"] != region {
			continue
		}

		// Always include "all"
		if serverIDs == "all" {
			membersFound = append(membersFound, member.Name)
			continue
		}

		// Include member if name matches any prefix from serverIDs
		if helper.StringHasPrefixInSlice(member.Name, prefixes) {
			membersFound = append(membersFound, member.Name)
		}
	}

	return membersFound, nil
}

// stringToSlice splits comma-separated input string into slice, trims
// whitespace, and prunes empty values
func stringToSlice(input string) []string {
	ns := strings.Split(input, ",")
	var out []string
	for _, n := range ns {
		s := strings.TrimSpace(n)
		if s == "" {
			continue
		}
		out = append(out, s)
	}
	return out
}

// external holds address configuration for Consul and Vault APIs
type external struct {
	tls       *api.TLSConfig
	addrVal   string
	auth      string
	ssl       bool
	tokenVal  string
	tokenFile string
}

func (e *external) addr(defaultAddr string) string {
	if e.addrVal == "" {
		return defaultAddr
	}

	if !e.ssl {
		if strings.HasPrefix(e.addrVal, "http:") {
			return e.addrVal
		}
		if strings.HasPrefix(e.addrVal, "https:") {
			// Mismatch: e.ssl=false but addrVal is https
			return strings.ReplaceAll(e.addrVal, "https://", "http://")
		}
		return "http://" + e.addrVal
	}

	if strings.HasPrefix(e.addrVal, "https:") {
		return e.addrVal
	}

	if strings.HasPrefix(e.addrVal, "http:") {
		// Mismatch: e.ssl=true but addrVal is http
		return strings.ReplaceAll(e.addrVal, "http://", "https://")
	}

	return "https://" + e.addrVal
}

func (e *external) token() string {
	if e.tokenVal != "" {
		return e.tokenVal
	}

	if e.tokenFile != "" {
		bs, err := ioutil.ReadFile(e.tokenFile)
		if err == nil {
			return strings.TrimSpace(string(bs))
		}
	}

	return ""
}

// defaultHttpClient configures a basic httpClient
func defaultHttpClient() *http.Client {
	httpClient := cleanhttp.DefaultClient()
	transport := httpClient.Transport.(*http.Transport)
	transport.TLSHandshakeTimeout = 10 * time.Second
	transport.TLSClientConfig = &tls.Config{
		MinVersion: tls.VersionTLS12,
	}

	return httpClient
}