sys/config: config state endpoint (#7424)

* sys/config: initial work on adding config state endpoint

* server/config: add tests, fix Sanitized method

* thread config through NewTestCluster's config to avoid panic on dev modes

* properly guard endpoint against request forwarding

* add http tests, guard against panics on nil RawConfig

* ensure non-nil rawConfig on NewTestCluster cores

* update non-forwarding logic

* fix imports; use no-forward handler

* add missing config test fixture; update gitignore

* return sanitized config as a map

* fix test, use deep.Equal to check for equality

* fix http test

* minor comment fix

* config: change Sanitized to return snake-cased keys, update tests

* core: hold rlock when reading config; add docstring

* update docstring

.gitignore

@@ -48,6 +48,7 @@ Vagrantfile
 # Configs
 *.hcl
 !command/agent/config/test-fixtures/*.hcl
+!command/server/test-fixtures/*.hcl
 
 
 .DS_Store

command/server.go

@@ -668,6 +668,7 @@ func (c *ServerCommand) Run(args []string) int {
 	}
 
 	coreConfig := &vault.CoreConfig{
+		RawConfig:                 config,
 		Physical:                  backend,
 		RedirectAddr:              config.Storage.RedirectAddr,
 		StorageType:               config.Storage.Type,
@@ -973,7 +974,7 @@ CLUSTER_SYNTHESIS_COMPLETE:
 		}
 		props["max_request_size"] = fmt.Sprintf("%d", maxRequestSize)
 
-		var maxRequestDuration time.Duration = vault.DefaultMaxRequestDuration
+		maxRequestDuration := vault.DefaultMaxRequestDuration
 		if valRaw, ok := lnConfig.Config["max_request_duration"]; ok {
 			val, err := parseutil.ParseDurationSecond(valRaw)
 			if err != nil {
@@ -1415,6 +1416,8 @@ CLUSTER_SYNTHESIS_COMPLETE:
 				goto RUNRELOADFUNCS
 			}
 
+			core.SetConfig(config)
+
 			if config.LogLevel != "" {
 				configLogLevel := strings.ToLower(strings.TrimSpace(config.LogLevel))
 				switch configLogLevel {

command/server/config.go

@@ -905,3 +905,128 @@ func parseTelemetry(result *Config, list *ast.ObjectList) error {
 
 	return nil
 }
+
+// Sanitized returns a copy of the config with all values that are considered
+// sensitive stripped. It also strips all `*Raw` values that are mainly
+// used for parsing.
+//
+// Specifically, the fields that this method strips are:
+// - Storage.Config
+// - HAStorage.Config
+// - Seals.Config
+// - Telemetry.CirconusAPIToken
+func (c *Config) Sanitized() map[string]interface{} {
+	result := map[string]interface{}{
+		"cache_size":              c.CacheSize,
+		"disable_cache":           c.DisableCache,
+		"disable_mlock":           c.DisableMlock,
+		"disable_printable_check": c.DisablePrintableCheck,
+
+		"enable_ui": c.EnableUI,
+
+		"max_lease_ttl":     c.MaxLeaseTTL,
+		"default_lease_ttl": c.DefaultLeaseTTL,
+
+		"default_max_request_duration": c.DefaultMaxRequestDuration,
+
+		"cluster_name":          c.ClusterName,
+		"cluster_cipher_suites": c.ClusterCipherSuites,
+
+		"plugin_directory": c.PluginDirectory,
+
+		"log_level":  c.LogLevel,
+		"log_format": c.LogFormat,
+
+		"pid_file":             c.PidFile,
+		"raw_storage_endpoint": c.EnableRawEndpoint,
+
+		"api_addr":           c.APIAddr,
+		"cluster_addr":       c.ClusterAddr,
+		"disable_clustering": c.DisableClustering,
+
+		"disable_performance_standby": c.DisablePerformanceStandby,
+
+		"disable_sealwrap": c.DisableSealWrap,
+
+		"disable_indexing": c.DisableIndexing,
+	}
+
+	// Sanitize listeners
+	if len(c.Listeners) != 0 {
+		var sanitizedListeners []interface{}
+		for _, ln := range c.Listeners {
+			cleanLn := map[string]interface{}{
+				"type":   ln.Type,
+				"config": ln.Config,
+			}
+			sanitizedListeners = append(sanitizedListeners, cleanLn)
+		}
+		result["listeners"] = sanitizedListeners
+	}
+
+	// Sanitize storage stanza
+	if c.Storage != nil {
+		sanitizedStorage := map[string]interface{}{
+			"type":               c.Storage.Type,
+			"redirect_addr":      c.Storage.RedirectAddr,
+			"cluster_addr":       c.Storage.ClusterAddr,
+			"disable_clustering": c.Storage.DisableClustering,
+		}
+		result["storage"] = sanitizedStorage
+	}
+
+	// Sanitize HA storage stanza
+	if c.HAStorage != nil {
+		sanitizedHAStorage := map[string]interface{}{
+			"type":               c.HAStorage.Type,
+			"redirect_addr":      c.HAStorage.RedirectAddr,
+			"cluster_addr":       c.HAStorage.ClusterAddr,
+			"disable_clustering": c.HAStorage.DisableClustering,
+		}
+		result["ha_storage"] = sanitizedHAStorage
+	}
+
+	// Sanitize seals stanza
+	if len(c.Seals) != 0 {
+		var sanitizedSeals []interface{}
+		for _, s := range c.Seals {
+			cleanSeal := map[string]interface{}{
+				"type":     s.Type,
+				"disabled": s.Disabled,
+			}
+			sanitizedSeals = append(sanitizedSeals, cleanSeal)
+		}
+		result["seals"] = sanitizedSeals
+	}
+
+	// Sanitize telemetry stanza
+	if c.Telemetry != nil {
+		sanitizedTelemetry := map[string]interface{}{
+			"statsite_address":                       c.Telemetry.StatsiteAddr,
+			"statsd_address":                         c.Telemetry.StatsdAddr,
+			"disable_hostname":                       c.Telemetry.DisableHostname,
+			"circonus_api_token":                     "",
+			"circonus_api_app":                       c.Telemetry.CirconusAPIApp,
+			"circonus_api_url":                       c.Telemetry.CirconusAPIURL,
+			"circonus_submission_interval":           c.Telemetry.CirconusSubmissionInterval,
+			"circonus_submission_url":                c.Telemetry.CirconusCheckSubmissionURL,
+			"circonus_check_id":                      c.Telemetry.CirconusCheckID,
+			"circonus_check_force_metric_activation": c.Telemetry.CirconusCheckForceMetricActivation,
+			"circonus_check_instance_id":             c.Telemetry.CirconusCheckInstanceID,
+			"circonus_check_search_tag":              c.Telemetry.CirconusCheckSearchTag,
+			"circonus_check_tags":                    c.Telemetry.CirconusCheckTags,
+			"circonus_check_display_name":            c.Telemetry.CirconusCheckDisplayName,
+			"circonus_broker_id":                     c.Telemetry.CirconusBrokerID,
+			"circonus_broker_select_tag":             c.Telemetry.CirconusBrokerSelectTag,
+			"dogstatsd_addr":                         c.Telemetry.DogStatsDAddr,
+			"dogstatsd_tags":                         c.Telemetry.DogStatsDTags,
+			"prometheus_retention_time":              c.Telemetry.PrometheusRetentionTime,
+			"stackdriver_project_id":                 c.Telemetry.StackdriverProjectID,
+			"stackdriver_location":                   c.Telemetry.StackdriverLocation,
+			"stackdriver_namespace":                  c.Telemetry.StackdriverNamespace,
+		}
+		result["telemetry"] = sanitizedTelemetry
+	}
+
+	return result
+}

command/server/config_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-test/deep"
 	"github.com/hashicorp/hcl"
 	"github.com/hashicorp/hcl/hcl/ast"
 )
@@ -349,6 +350,90 @@ func TestLoadConfigDir(t *testing.T) {
 	}
 }
 
+func TestConfig_Sanitized(t *testing.T) {
+	config, err := LoadConfigFile("./test-fixtures/config3.hcl")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	sanitizedConfig := config.Sanitized()
+
+	expected := map[string]interface{}{
+		"api_addr":                     "top_level_api_addr",
+		"cache_size":                   0,
+		"cluster_addr":                 "top_level_cluster_addr",
+		"cluster_cipher_suites":        "",
+		"cluster_name":                 "testcluster",
+		"default_lease_ttl":            10 * time.Hour,
+		"default_max_request_duration": 0 * time.Second,
+		"disable_cache":                true,
+		"disable_clustering":           false,
+		"disable_indexing":             false,
+		"disable_mlock":                true,
+		"disable_performance_standby":  false,
+		"disable_printable_check":      false,
+		"disable_sealwrap":             true,
+		"raw_storage_endpoint":         true,
+		"enable_ui":                    true,
+		"ha_storage": map[string]interface{}{
+			"cluster_addr":       "top_level_cluster_addr",
+			"disable_clustering": true,
+			"redirect_addr":      "top_level_api_addr",
+			"type":               "consul"},
+		"listeners": []interface{}{
+			map[string]interface{}{
+				"config": map[string]interface{}{
+					"address": "127.0.0.1:443",
+				},
+				"type": "tcp",
+			},
+		},
+		"log_format":       "",
+		"log_level":        "",
+		"max_lease_ttl":    10 * time.Hour,
+		"pid_file":         "./pidfile",
+		"plugin_directory": "",
+		"seals": []interface{}{
+			map[string]interface{}{
+				"disabled": false,
+				"type":     "awskms",
+			},
+		},
+		"storage": map[string]interface{}{
+			"cluster_addr":       "top_level_cluster_addr",
+			"disable_clustering": false,
+			"redirect_addr":      "top_level_api_addr",
+			"type":               "consul",
+		},
+		"telemetry": map[string]interface{}{
+			"circonus_api_app":                       "",
+			"circonus_api_token":                     "",
+			"circonus_api_url":                       "",
+			"circonus_broker_id":                     "",
+			"circonus_broker_select_tag":             "",
+			"circonus_check_display_name":            "",
+			"circonus_check_force_metric_activation": "",
+			"circonus_check_id":                      "",
+			"circonus_check_instance_id":             "",
+			"circonus_check_search_tag":              "",
+			"circonus_submission_url":                "",
+			"circonus_check_tags":                    "",
+			"circonus_submission_interval":           "",
+			"disable_hostname":                       false,
+			"dogstatsd_addr":                         "",
+			"dogstatsd_tags":                         []string(nil),
+			"prometheus_retention_time":              24 * time.Hour,
+			"stackdriver_location":                   "",
+			"stackdriver_namespace":                  "",
+			"stackdriver_project_id":                 "",
+			"statsd_address":                         "bar",
+			"statsite_address":                       ""},
+	}
+
+	if diff := deep.Equal(sanitizedConfig, expected); len(diff) > 0 {
+		t.Fatalf("bad, diff: %#v", diff)
+	}
+}
+
 func TestParseListeners(t *testing.T) {
 	obj, _ := hcl.Parse(strings.TrimSpace(`
 listener "tcp" {

command/server/test-fixtures/config3.hcl

@@ -0,0 +1,41 @@
+disable_cache = true
+disable_mlock = true
+
+ui = true
+
+api_addr = "top_level_api_addr"
+cluster_addr = "top_level_cluster_addr"
+
+listener "tcp" {
+  address = "127.0.0.1:443"
+}
+
+backend "consul" {
+  advertise_addr = "foo"
+  token = "foo"
+}
+
+ha_backend "consul" {
+  bar = "baz"
+  advertise_addr = "snafu"
+  disable_clustering = "true"
+  token = "foo"
+}
+
+telemetry {
+  statsd_address = "bar"
+  circonus_api_token = "baz"
+}
+
+seal "awskms" {
+  region     = "us-east-1"
+  access_key = "AKIAIOSFODNN7EXAMPLE"
+  secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+}
+
+max_lease_ttl = "10h"
+default_lease_ttl = "10h"
+cluster_name = "testcluster"
+pid_file = "./pidfile"
+raw_storage_endpoint = true
+disable_sealwrap = true

http/forwarding_test.go

@@ -582,3 +582,24 @@ func TestHTTP_Forwarding_HelpOperation(t *testing.T) {
 	testHelp(cores[0].Client)
 	testHelp(cores[1].Client)
 }
+
+func TestHTTP_Forwarding_LocalOnly(t *testing.T) {
+	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
+		HandlerFunc: Handler,
+	})
+	cluster.Start()
+	defer cluster.Cleanup()
+	cores := cluster.Cores
+
+	vault.TestWaitActive(t, cores[0].Core)
+
+	testLocalOnly := func(client *api.Client) {
+		_, err := client.Logical().Read("sys/config/state/sanitized")
+		if err == nil {
+			t.Fatal("expected error")
+		}
+	}
+
+	testLocalOnly(cores[1].Client)
+	testLocalOnly(cores[2].Client)
+}

http/handler.go

@@ -112,8 +112,9 @@ func Handler(props *vault.HandlerProperties) http.Handler {
 	mux := http.NewServeMux()
 
 	// Handle non-forwarded paths
-	mux.Handle("/v1/sys/pprof/", handleLogicalNoForward(core))
+	mux.Handle("/v1/sys/config/state/", handleLogicalNoForward(core))
 	mux.Handle("/v1/sys/host-info", handleLogicalNoForward(core))
+	mux.Handle("/v1/sys/pprof/", handleLogicalNoForward(core))
 
 	mux.Handle("/v1/sys/init", handleSysInit(core))
 	mux.Handle("/v1/sys/seal-status", handleSysSealStatus(core))

http/sys_config_state_test.go

@@ -0,0 +1,67 @@
+package http
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/go-test/deep"
+	"github.com/hashicorp/vault/vault"
+)
+
+func TestSysConfigState_Sanitized(t *testing.T) {
+	var resp *http.Response
+
+	core, _, token := vault.TestCoreUnsealed(t)
+	ln, addr := TestServer(t, core)
+	defer ln.Close()
+	TestServerAuth(t, addr, token)
+
+	resp = testHttpGet(t, token, addr+"/v1/sys/config/state/sanitized")
+	testResponseStatus(t, resp, 200)
+
+	var actual map[string]interface{}
+	var expected map[string]interface{}
+
+	configResp := map[string]interface{}{
+		"api_addr":                     "",
+		"cache_size":                   json.Number("0"),
+		"cluster_addr":                 "",
+		"cluster_cipher_suites":        "",
+		"cluster_name":                 "",
+		"default_lease_ttl":            json.Number("0"),
+		"default_max_request_duration": json.Number("0"),
+		"disable_cache":                false,
+		"disable_clustering":           false,
+		"disable_indexing":             false,
+		"disable_mlock":                false,
+		"disable_performance_standby":  false,
+		"disable_printable_check":      false,
+		"disable_sealwrap":             false,
+		"raw_storage_endpoint":         false,
+		"enable_ui":                    false,
+		"log_format":                   "",
+		"log_level":                    "",
+		"max_lease_ttl":                json.Number("0"),
+		"pid_file":                     "",
+		"plugin_directory":             "",
+	}
+
+	expected = map[string]interface{}{
+		"lease_id":       "",
+		"renewable":      false,
+		"lease_duration": json.Number("0"),
+		"wrap_info":      nil,
+		"warnings":       nil,
+		"auth":           nil,
+		"data":           configResp,
+	}
+
+	testResponseBody(t, resp, &actual)
+	expected["request_id"] = actual["request_id"]
+
+	if diff := deep.Equal(actual, expected); len(diff) > 0 {
+		t.Fatalf("bad mismatch response body: diff: %v", diff)
+	}
+
+}

vault/core.go

@@ -16,22 +16,18 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/hashicorp/vault/api"
-	"github.com/hashicorp/vault/helper/metricsutil"
-	"github.com/hashicorp/vault/physical/raft"
-
-	metrics "github.com/armon/go-metrics"
-	log "github.com/hashicorp/go-hclog"
-	multierror "github.com/hashicorp/go-multierror"
-	uuid "github.com/hashicorp/go-uuid"
-	cache "github.com/patrickmn/go-cache"
-
-	"google.golang.org/grpc"
-
+	"github.com/armon/go-metrics"
 	"github.com/hashicorp/errwrap"
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/audit"
+	"github.com/hashicorp/vault/command/server"
+	"github.com/hashicorp/vault/helper/metricsutil"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/reload"
+	"github.com/hashicorp/vault/physical/raft"
 	"github.com/hashicorp/vault/sdk/helper/certutil"
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
@@ -45,6 +41,8 @@ import (
 	"github.com/hashicorp/vault/vault/cluster"
 	"github.com/hashicorp/vault/vault/seal"
 	shamirseal "github.com/hashicorp/vault/vault/seal/shamir"
+	"github.com/patrickmn/go-cache"
+	"google.golang.org/grpc"
 )
 
 const (
@@ -460,6 +458,9 @@ type Core struct {
 	// Stores the pending peers we are waiting to give answers
 	pendingRaftPeers map[string][]byte
 
+	// rawConfig stores the config as-is from the provided server configuration.
+	rawConfig *server.Config
+
 	coreNumber int
 }
 
@@ -518,6 +519,8 @@ type CoreConfig struct {
 
 	DisableSealWrap bool `json:"disable_sealwrap" structs:"disable_sealwrap" mapstructure:"disable_sealwrap"`
 
+	RawConfig *server.Config
+
 	ReloadFuncs     *map[string][]reload.ReloadFunc
 	ReloadFuncsLock *sync.RWMutex
 
@@ -608,6 +611,11 @@ func NewCore(conf *CoreConfig) (*Core, error) {
 		conf.Logger = logging.NewVaultLogger(log.Trace)
 	}
 
+	// Instantiate a non-nil raw config if none is provided
+	if conf.RawConfig == nil {
+		conf.RawConfig = new(server.Config)
+	}
+
 	syncInterval := conf.CounterSyncInterval
 	if syncInterval.Nanoseconds() == 0 {
 		syncInterval = 30 * time.Second
@@ -652,6 +660,7 @@ func NewCore(conf *CoreConfig) (*Core, error) {
 		neverBecomeActive:            new(uint32),
 		clusterLeaderParams:          new(atomic.Value),
 		metricsHelper:                conf.MetricsHelper,
+		rawConfig:                    conf.RawConfig,
 		counters: counters{
 			requests:     new(uint64),
 			syncInterval: syncInterval,
@@ -1979,6 +1988,21 @@ func (c *Core) SetLogLevel(level log.Level) {
 	}
 }
 
+// SetConfig sets core's config object to the newly provided config.
+func (c *Core) SetConfig(conf *server.Config) {
+	c.stateLock.Lock()
+	c.rawConfig = conf
+	c.stateLock.Unlock()
+}
+
+// SanitizedConfig returns a sanitized version of the current config.
+// See server.Config.Sanitized for specific values omitted.
+func (c *Core) SanitizedConfig() map[string]interface{} {
+	c.stateLock.RLock()
+	defer c.stateLock.RUnlock()
+	return c.rawConfig.Sanitized()
+}
+
 // MetricsHelper returns the global metrics helper which allows external
 // packages to access Vault's internal metrics.
 func (c *Core) MetricsHelper() *metricsutil.MetricsHelper {

vault/logical_system.go

@@ -228,6 +228,17 @@ type SystemBackend struct {
 	logger    log.Logger
 }
 
+// handleConfigStateSanitized returns the current configuration state. The configuration
+// data that it returns is a sanitized version of the combined configuration
+// file(s) provided.
+func (b *SystemBackend) handleConfigStateSanitized(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	config := b.Core.SanitizedConfig()
+	resp := &logical.Response{
+		Data: config,
+	}
+	return resp, nil
+}
+
 // handleCORSRead returns the current CORS configuration
 func (b *SystemBackend) handleCORSRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	corsConf := b.Core.corsConfig

vault/logical_system_paths.go

@@ -48,6 +48,17 @@ func (b *SystemBackend) configPaths() []*framework.Path {
 			HelpSynopsis:    strings.TrimSpace(sysHelp["config/cors"][1]),
 		},
 
+		{
+			Pattern: "config/state/sanitized$",
+			Operations: map[logical.Operation]framework.OperationHandler{
+				logical.ReadOperation: &framework.PathOperation{
+					Callback:    b.handleConfigStateSanitized,
+					Summary:     "Return a sanitized version of the Vault server configuration.",
+					Description: "The sanitized output strips configuration values in the storage, HA storage, and seals stanzas, which may contain sensitive values such as API tokens. It also removes any token or secret fields in other stanzas, such as the circonus_api_token from telemetry.",
+				},
+			},
+		},
+
 		{
 			Pattern: "config/ui/headers/" + framework.GenericNameRegex("header"),
 

vault/testing.go

@@ -40,6 +40,7 @@ import (
 	cleanhttp "github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/audit"
+	"github.com/hashicorp/vault/command/server"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/reload"
 	dbMysql "github.com/hashicorp/vault/plugins/database/mysql"
@@ -1350,6 +1351,7 @@ func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *Te
 	}
 
 	if base != nil {
+		coreConfig.RawConfig = base.RawConfig
 		coreConfig.DisableCache = base.DisableCache
 		coreConfig.EnableUI = base.EnableUI
 		coreConfig.DefaultLeaseTTL = base.DefaultLeaseTTL
@@ -1418,6 +1420,10 @@ func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *Te
 
 	}
 
+	if coreConfig.RawConfig == nil {
+		coreConfig.RawConfig = new(server.Config)
+	}
+
 	addAuditBackend := len(coreConfig.AuditBackends) == 0
 	if addAuditBackend {
 		AddNoopAudit(coreConfig)