Defer setting views read/write until the end of postUnseal (#4392)

A few notes:

* We exert positive control over singletons and they usually need to
perform some (known, validated) writes, so this excludes singletons --
they are simply limited to the end of the mount function as before.

* I'm not sure how to test this _specifically_; I've done some testing
of e.g. sealing vault and unsealing and ensuring that I can write to a
KV mount. I think this is tested by every dev server though, since for a
dev server Vault is inited, the default mounts are mounted, then it's
sealed, then it's unsealed for the user, so it already goes through this
code path. The mere fact that you can write to secret/ on a dev server
means it was successfully set read-write.

vault/audit.go

@@ -324,7 +324,9 @@ func (c *Core) setupAudits(ctx context.Context) error {
 		// ensure that it is reset after. This ensures that there will be no
 		// writes during the construction of the backend.
 		view.setReadOnlyErr(logical.ErrSetupReadOnly)
-		defer view.setReadOnlyErr(nil)
+		c.postUnsealFuncs = append(c.postUnsealFuncs, func() {
+			view.setReadOnlyErr(nil)
+		})
 
 		// Initialize the backend
 		backend, err := c.newAuditBackend(ctx, entry, view, entry.Options)

vault/auth.go

@@ -9,6 +9,7 @@ import (
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/helper/consts"
 	"github.com/hashicorp/vault/helper/jsonutil"
+	"github.com/hashicorp/vault/helper/strutil"
 	"github.com/hashicorp/vault/logical"
 )
 
@@ -472,7 +473,13 @@ func (c *Core) setupCredentials(ctx context.Context) error {
 		// ensure that it is reset after. This ensures that there will be no
 		// writes during the construction of the backend.
 		view.setReadOnlyErr(logical.ErrSetupReadOnly)
-		defer view.setReadOnlyErr(nil)
+		if strutil.StrListContains(singletonMounts, entry.Type) {
+			defer view.setReadOnlyErr(nil)
+		} else {
+			c.postUnsealFuncs = append(c.postUnsealFuncs, func() {
+				view.setReadOnlyErr(nil)
+			})
+		}
 
 		// Initialize the backend
 		sysView := c.mountEntrySysView(entry)

vault/core.go

@@ -384,6 +384,9 @@ type Core struct {
 
 	// Stores the sealunwrapper for downgrade needs
 	sealUnwrapper physical.Backend
+
+	// Stores any funcs that should be run on successful postUnseal
+	postUnsealFuncs []func()
 }
 
 // CoreConfig is used to parameterize a core
@@ -1616,6 +1619,9 @@ func (c *Core) sealInternal(keepLock bool) error {
 func (c *Core) postUnseal() (retErr error) {
 	defer metrics.MeasureSince([]string{"core", "post_unseal"}, time.Now())
 
+	// Clear any out
+	c.postUnsealFuncs = nil
+
 	// Create a new request context
 	c.activeContext, c.activeContextCancelFunc = context.WithCancel(context.Background())
 
@@ -1703,6 +1709,14 @@ func (c *Core) postUnseal() (retErr error) {
 	}
 	c.metricsCh = make(chan struct{})
 	go c.emitMetrics(c.metricsCh)
+
+	// This is intentionally the last block in this function. We want to allow
+	// writes just before allowing client requests, to ensure everything has
+	// been set up properly before any writes can have happened.
+	for _, v := range c.postUnsealFuncs {
+		v()
+	}
+
 	c.logger.Info("post-unseal setup complete")
 	return nil
 }
@@ -1713,6 +1727,9 @@ func (c *Core) preSeal() error {
 	defer metrics.MeasureSince([]string{"core", "pre_seal"}, time.Now())
 	c.logger.Info("pre-seal teardown starting")
 
+	// Clear any pending funcs
+	c.postUnsealFuncs = nil
+
 	// Clear any rekey progress
 	c.barrierRekeyConfig = nil
 	c.barrierRekeyProgress = nil

vault/mount.go

@@ -311,6 +311,8 @@ func (c *Core) mountInternal(ctx context.Context, entry *MountEntry) error {
 	// ensure that it is reset after. This ensures that there will be no
 	// writes during the construction of the backend.
 	view.setReadOnlyErr(logical.ErrSetupReadOnly)
+	// We defer this because we're already up and running so we don't need to
+	// time it for after postUnseal
 	defer view.setReadOnlyErr(nil)
 
 	var backend logical.Backend
@@ -628,6 +630,7 @@ func (c *Core) loadMounts(ctx context.Context) error {
 
 	var needPersist bool
 	if c.mounts == nil {
+		c.logger.Info("no mounts; adding default mount table")
 		c.mounts = c.defaultMountTable()
 		needPersist = true
 	}
@@ -818,7 +821,13 @@ func (c *Core) setupMounts(ctx context.Context) error {
 		// ensure that it is reset after. This ensures that there will be no
 		// writes during the construction of the backend.
 		view.setReadOnlyErr(logical.ErrSetupReadOnly)
-		defer view.setReadOnlyErr(nil)
+		if strutil.StrListContains(singletonMounts, entry.Type) {
+			defer view.setReadOnlyErr(nil)
+		} else {
+			c.postUnsealFuncs = append(c.postUnsealFuncs, func() {
+				view.setReadOnlyErr(nil)
+			})
+		}
 
 		var backend logical.Backend
 		var err error