Allow tidy to backup legacy CA bundles (#18645)

* Allow tidy to backup legacy CA bundles

With the new tidy_move_legacy_ca_bundle option, we'll use tidy to move
the legacy CA bundle from /config/ca_bundle to /config/ca_bundle.bak.
This does two things:

 1. Removes ca_bundle from the hot-path of initialization after initial
    migration has completed. Because this entry is seal wrapped, this
    may result in performance improvements.
 2. Allows recovery of this value in the event of some other failure
    with migration.

Notably, this cannot occur during migration in the unlikely (and largely
unsupported) case that the operator immediately downgrades to Vault
<1.11.x. Thus, we reuse issuer_safety_buffer; while potentially long,
tidy can always be run manually with a shorter buffer (and only this
flag) to manually move the bundle if necessary.

In the event of needing to recover or undo this operation, it is
sufficient to use sys/raw to read the backed up value and subsequently
write it to its old path (/config/ca_bundle).

The new entry remains seal wrapped, but otherwise isn't used within the
code and so has better performance characteristics.

Performing a fat deletion (DELETE /root) will again remove the backup
like the old legacy bundle, preserving its wipe characteristics.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add changelog

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add documentation about new tidy parameter

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add tests for migration scenarios

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Clean up time comparisons

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

builtin/logical/pki/backend.go

@@ -112,6 +112,7 @@ func Backend(conf *logical.BackendConfig) *backend {
 
 			SealWrapStorage: []string{
 				legacyCertBundlePath,
+				legacyCertBundleBackupPath,
 				keyPrefix,
 			},
 		},
@@ -271,6 +272,7 @@ type tidyStatus struct {
 	tidyRevokedCerts   bool
 	tidyRevokedAssocs  bool
 	tidyExpiredIssuers bool
+	tidyBackupBundle   bool
 	pauseDuration      string
 
 	// Status

builtin/logical/pki/backend_test.go

@@ -3940,6 +3940,7 @@ func TestBackend_RevokePlusTidy_Intermediate(t *testing.T) {
 			"tidy_revoked_certs":                    true,
 			"tidy_revoked_cert_issuer_associations": false,
 			"tidy_expired_issuers":                  false,
+			"tidy_move_legacy_ca_bundle":            false,
 			"pause_duration":                        "0s",
 			"state":                                 "Finished",
 			"error":                                 nil,

builtin/logical/pki/fields.go

@@ -461,6 +461,23 @@ past the issuer_safety_buffer. No keys will be removed as part of this
 operation.`,
 	}
 
+	fields["tidy_move_legacy_ca_bundle"] = &framework.FieldSchema{
+		Type: framework.TypeBool,
+		Description: `Set to true to move the legacy ca_bundle from
+/config/ca_bundle to /config/ca_bundle.bak. This prevents downgrades
+to pre-Vault 1.11 versions (as older PKI engines do not know about
+the new multi-issuer storage layout), but improves the performance
+on seal wrapped PKI mounts. This will only occur if at least
+issuer_safety_buffer time has occurred after the initial storage
+migration.
+
+This backup is saved in case of an issue in future migrations.
+Operators may consider removing it via sys/raw if they desire.
+The backup will be removed via a DELETE /root call, but note that
+this removes ALL issuers within the mount (and is thus not desirable
+in most operational scenarios).`,
+	}
+
 	fields["safety_buffer"] = &framework.FieldSchema{
 		Type: framework.TypeDurationSecond,
 		Description: `The amount of extra time that must have passed

builtin/logical/pki/path_root.go

@@ -80,11 +80,15 @@ func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, _
 		}
 	}
 
-	// Delete legacy CA bundle.
+	// Delete legacy CA bundle and its backup, if any.
 	if err := req.Storage.Delete(ctx, legacyCertBundlePath); err != nil {
 		return nil, err
 	}
 
+	if err := req.Storage.Delete(ctx, legacyCertBundleBackupPath); err != nil {
+		return nil, err
+	}
+
 	// Delete legacy CRL bundle.
 	if err := req.Storage.Delete(ctx, legacyCRLPath); err != nil {
 		return nil, err

builtin/logical/pki/path_tidy.go

@@ -26,6 +26,7 @@ type tidyConfig struct {
 	RevokedCerts       bool          `json:"tidy_revoked_certs"`
 	IssuerAssocs       bool          `json:"tidy_revoked_cert_issuer_associations"`
 	ExpiredIssuers     bool          `json:"tidy_expired_issuers"`
+	BackupBundle       bool          `json:"tidy_move_legacy_ca_bundle"`
 	SafetyBuffer       time.Duration `json:"safety_buffer"`
 	IssuerSafetyBuffer time.Duration `json:"issuer_safety_buffer"`
 	PauseDuration      time.Duration `json:"pause_duration"`
@@ -38,6 +39,7 @@ var defaultTidyConfig = tidyConfig{
 	RevokedCerts:       false,
 	IssuerAssocs:       false,
 	ExpiredIssuers:     false,
+	BackupBundle:       false,
 	SafetyBuffer:       72 * time.Hour,
 	IssuerSafetyBuffer: 365 * 24 * time.Hour,
 	PauseDuration:      0 * time.Second,
@@ -122,6 +124,7 @@ func (b *backend) pathTidyWrite(ctx context.Context, req *logical.Request, d *fr
 	tidyRevokedCerts := d.Get("tidy_revoked_certs").(bool) || d.Get("tidy_revocation_list").(bool)
 	tidyRevokedAssocs := d.Get("tidy_revoked_cert_issuer_associations").(bool)
 	tidyExpiredIssuers := d.Get("tidy_expired_issuers").(bool)
+	tidyBackupBundle := d.Get("tidy_move_legacy_ca_bundle").(bool)
 	issuerSafetyBuffer := d.Get("issuer_safety_buffer").(int)
 	pauseDurationStr := d.Get("pause_duration").(string)
 	pauseDuration := 0 * time.Second
@@ -157,6 +160,7 @@ func (b *backend) pathTidyWrite(ctx context.Context, req *logical.Request, d *fr
 		RevokedCerts:       tidyRevokedCerts,
 		IssuerAssocs:       tidyRevokedAssocs,
 		ExpiredIssuers:     tidyExpiredIssuers,
+		BackupBundle:       tidyBackupBundle,
 		SafetyBuffer:       bufferDuration,
 		IssuerSafetyBuffer: issuerBufferDuration,
 		PauseDuration:      pauseDuration,
@@ -184,8 +188,8 @@ func (b *backend) pathTidyWrite(ctx context.Context, req *logical.Request, d *fr
 	b.startTidyOperation(req, config)
 
 	resp := &logical.Response{}
-	if !tidyCertStore && !tidyRevokedCerts && !tidyRevokedAssocs && !tidyExpiredIssuers {
+	if !tidyCertStore && !tidyRevokedCerts && !tidyRevokedAssocs && !tidyExpiredIssuers && !tidyBackupBundle {
-		resp.AddWarning("No targets to tidy; specify tidy_cert_store=true or tidy_revoked_certs=true or tidy_revoked_cert_issuer_associations=true or tidy_expired_issuers=true to start a tidy operation.")
+		resp.AddWarning("No targets to tidy; specify tidy_cert_store=true or tidy_revoked_certs=true or tidy_revoked_cert_issuer_associations=true or tidy_expired_issuers=true or tidy_move_legacy_ca_bundle=true to start a tidy operation.")
 	} else {
 		resp.AddWarning("Tidy operation successfully started. Any information from the operation will be printed to Vault's server logs.")
 	}
@@ -229,6 +233,12 @@ func (b *backend) startTidyOperation(req *logical.Request, config *tidyConfig) {
 				}
 			}
 
+			if config.BackupBundle {
+				if err := b.doTidyMoveCABundle(ctx, req, logger, config); err != nil {
+					return err
+				}
+			}
+
 			return nil
 		}
 
@@ -298,7 +308,7 @@ func (b *backend) doTidyCertStore(ctx context.Context, req *logical.Request, log
 			return fmt.Errorf("unable to parse stored certificate with serial %q: %w", serial, err)
 		}
 
-		if time.Now().After(cert.NotAfter.Add(config.SafetyBuffer)) {
+		if time.Since(cert.NotAfter) > config.SafetyBuffer {
 			if err := req.Storage.Delete(ctx, "certs/"+serial); err != nil {
 				return fmt.Errorf("error deleting serial %q from storage: %w", serial, err)
 			}
@@ -406,7 +416,7 @@ func (b *backend) doTidyRevocationStore(ctx context.Context, req *logical.Reques
 			// past its NotAfter value. This is because we use the
 			// information on revoked/ to build the CRL and the
 			// information on certs/ for lookup.
-			if time.Now().After(revokedCert.NotAfter.Add(config.SafetyBuffer)) {
+			if time.Since(revokedCert.NotAfter) > config.SafetyBuffer {
 				if err := req.Storage.Delete(ctx, "revoked/"+serial); err != nil {
 					return fmt.Errorf("error deleting serial %q from revoked list: %w", serial, err)
 				}
@@ -492,16 +502,11 @@ func (b *backend) doTidyExpiredIssuers(ctx context.Context, req *logical.Request
 	}
 
 	// We want certificates which have expired before this date by a given
-	// safety buffer. So we subtract the buffer from now, and anything which
+	// safety buffer.
-	// has expired before our after buffer can be tidied, and anything that
-	// expired after this buffer must be kept.
-	now := time.Now()
-	afterBuffer := now.Add(-1 * config.IssuerSafetyBuffer)
-
 	rebuildChainsAndCRL := false
 
 	for issuer, cert := range issuerIDCertMap {
-		if cert.NotAfter.After(afterBuffer) {
+		if time.Since(cert.NotAfter) <= config.IssuerSafetyBuffer {
 			continue
 		}
 
@@ -565,6 +570,65 @@ func (b *backend) doTidyExpiredIssuers(ctx context.Context, req *logical.Request
 	return nil
 }
 
+func (b *backend) doTidyMoveCABundle(ctx context.Context, req *logical.Request, logger hclog.Logger, config *tidyConfig) error {
+	if b.System().ReplicationState().HasState(consts.ReplicationDRSecondary|consts.ReplicationPerformanceStandby) ||
+		(!b.System().LocalMount() && b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary)) {
+		b.Logger().Debug("skipping moving the legacy CA bundle as we're not on the primary or secondary with a local mount")
+		return nil
+	}
+
+	// Short-circuit to avoid moving the legacy bundle from under a legacy
+	// mount.
+	if b.useLegacyBundleCaStorage() {
+		return nil
+	}
+
+	// If we've already run, exit.
+	_, bundle, err := getLegacyCertBundle(ctx, req.Storage)
+	if err != nil {
+		return fmt.Errorf("failed to fetch the legacy CA bundle: %w", err)
+	}
+
+	if bundle == nil {
+		b.Logger().Debug("No legacy CA bundle available; nothing to do.")
+		return nil
+	}
+
+	log, err := getLegacyBundleMigrationLog(ctx, req.Storage)
+	if err != nil {
+		return fmt.Errorf("failed to fetch the legacy bundle migration log: %w", err)
+	}
+
+	if log == nil {
+		return fmt.Errorf("refusing to tidy with an empty legacy migration log but present CA bundle: %w", err)
+	}
+
+	if time.Since(log.Created) <= config.IssuerSafetyBuffer {
+		b.Logger().Debug("Migration was created too recently to remove the legacy bundle; refusing to move legacy CA bundle to backup location.")
+		return nil
+	}
+
+	// Do the write before the delete.
+	entry, err := logical.StorageEntryJSON(legacyCertBundleBackupPath, bundle)
+	if err != nil {
+		return fmt.Errorf("failed to create new backup storage entry: %w", err)
+	}
+
+	err = req.Storage.Put(ctx, entry)
+	if err != nil {
+		return fmt.Errorf("failed to write new backup legacy CA bundle: %w", err)
+	}
+
+	err = req.Storage.Delete(ctx, legacyCertBundlePath)
+	if err != nil {
+		return fmt.Errorf("failed to remove old legacy CA bundle path: %w", err)
+	}
+
+	b.Logger().Info("legacy CA bundle successfully moved to backup location")
+
+	return nil
+}
+
 func (b *backend) pathTidyCancelWrite(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
 	if atomic.LoadUint32(b.tidyCASGuard) == 0 {
 		resp := &logical.Response{}
@@ -600,6 +664,7 @@ func (b *backend) pathTidyStatusRead(_ context.Context, _ *logical.Request, _ *f
 			"tidy_revoked_certs":                    nil,
 			"tidy_revoked_cert_issuer_associations": nil,
 			"tidy_expired_issuers":                  nil,
+			"tidy_move_legacy_ca_bundle":            nil,
 			"pause_duration":                        nil,
 			"state":                                 "Inactive",
 			"error":                                 nil,
@@ -624,6 +689,7 @@ func (b *backend) pathTidyStatusRead(_ context.Context, _ *logical.Request, _ *f
 	resp.Data["tidy_revoked_certs"] = b.tidyStatus.tidyRevokedCerts
 	resp.Data["tidy_revoked_cert_issuer_associations"] = b.tidyStatus.tidyRevokedAssocs
 	resp.Data["tidy_expired_issuers"] = b.tidyStatus.tidyExpiredIssuers
+	resp.Data["tidy_move_legacy_ca_bundle"] = b.tidyStatus.tidyBackupBundle
 	resp.Data["pause_duration"] = b.tidyStatus.pauseDuration
 	resp.Data["time_started"] = b.tidyStatus.timeStarted
 	resp.Data["message"] = b.tidyStatus.message
@@ -677,6 +743,7 @@ func (b *backend) pathConfigAutoTidyRead(ctx context.Context, req *logical.Reque
 			"tidy_revoked_certs":                    config.RevokedCerts,
 			"tidy_revoked_cert_issuer_associations": config.IssuerAssocs,
 			"tidy_expired_issuers":                  config.ExpiredIssuers,
+			"tidy_move_legacy_ca_bundle":            config.BackupBundle,
 			"safety_buffer":                         int(config.SafetyBuffer / time.Second),
 			"issuer_safety_buffer":                  int(config.IssuerSafetyBuffer / time.Second),
 			"pause_duration":                        config.PauseDuration.String(),
@@ -743,8 +810,12 @@ func (b *backend) pathConfigAutoTidyWrite(ctx context.Context, req *logical.Requ
 		}
 	}
 
-	if config.Enabled && !(config.CertStore || config.RevokedCerts || config.IssuerAssocs || config.ExpiredIssuers) {
+	if backupBundle, ok := d.GetOk("tidy_move_legacy_ca_bundle"); ok {
-		return logical.ErrorResponse("Auto-tidy enabled but no tidy operations were requested. Enable at least one tidy operation to be run (tidy_cert_store / tidy_revoked_certs / tidy_revoked_cert_issuer_associations)."), nil
+		config.BackupBundle = backupBundle.(bool)
+	}
+
+	if config.Enabled && !(config.CertStore || config.RevokedCerts || config.IssuerAssocs || config.ExpiredIssuers || config.BackupBundle) {
+		return logical.ErrorResponse("Auto-tidy enabled but no tidy operations were requested. Enable at least one tidy operation to be run (tidy_cert_store / tidy_revoked_certs / tidy_revoked_cert_issuer_associations / tidy_move_legacy_ca_bundle)."), nil
 	}
 
 	if err := sc.writeAutoTidyConfig(config); err != nil {
@@ -759,6 +830,7 @@ func (b *backend) pathConfigAutoTidyWrite(ctx context.Context, req *logical.Requ
 			"tidy_revoked_certs":                    config.RevokedCerts,
 			"tidy_revoked_cert_issuer_associations": config.IssuerAssocs,
 			"tidy_expired_issuers":                  config.ExpiredIssuers,
+			"tidy_move_legacy_ca_bundle":            config.BackupBundle,
 			"safety_buffer":                         int(config.SafetyBuffer / time.Second),
 			"issuer_safety_buffer":                  int(config.IssuerSafetyBuffer / time.Second),
 			"pause_duration":                        config.PauseDuration.String(),
@@ -777,6 +849,7 @@ func (b *backend) tidyStatusStart(config *tidyConfig) {
 		tidyRevokedCerts:   config.RevokedCerts,
 		tidyRevokedAssocs:  config.IssuerAssocs,
 		tidyExpiredIssuers: config.ExpiredIssuers,
+		tidyBackupBundle:   config.BackupBundle,
 		pauseDuration:      config.PauseDuration.String(),
 
 		state:       tidyStatusStarted,
@@ -905,6 +978,9 @@ The result includes the following fields:
 * 'cert_store_deleted_count': The number of certificate storage entries deleted
 * 'revoked_cert_deleted_count': The number of revoked certificate entries deleted
 * 'missing_issuer_cert_count': The number of revoked certificates which were missing a valid issuer reference
+* 'tidy_expired_issuers': the value of this parameter when initiating the tidy operation
+* 'issuer_safety_buffer': the value of this parameter when initiating the tidy operation
+* 'tidy_move_legacy_ca_bundle': the value of this parameter when initiating the tidy operation
 `
 
 const pathConfigAutoTidySyn = `

builtin/logical/pki/storage.go

@@ -25,6 +25,7 @@ const (
 
 	legacyMigrationBundleLogKey = "config/legacyMigrationBundleLog"
 	legacyCertBundlePath        = "config/ca_bundle"
+	legacyCertBundleBackupPath  = "config/ca_bundle.bak"
 	legacyCRLPath               = "crl"
 	deltaCRLPath                = "delta-crl"
 	deltaCRLPathSuffix          = "-delta"

builtin/logical/pki/storage_migrations_test.go

@@ -590,6 +590,190 @@ func TestExpectedOpsWork_PreMigration(t *testing.T) {
 	requireFailInMigration(t, b, s, logical.ReadOperation, "config/keys")
 }
 
+func TestBackupBundle(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	b, s := CreateBackendWithStorage(t)
+	sc := b.makeStorageContext(ctx, s)
+
+	// Reset the version the helper above set to 1.
+	b.pkiStorageVersion.Store(0)
+	require.True(t, b.useLegacyBundleCaStorage(), "pre migration we should have been told to use legacy storage.")
+
+	// Create an empty request and tidy configuration for us.
+	req := &logical.Request{
+		Storage:    s,
+		MountPoint: "pki/",
+	}
+	cfg := &tidyConfig{
+		BackupBundle:       true,
+		IssuerSafetyBuffer: 120 * time.Second,
+	}
+
+	// Migration should do nothing if we're on an empty mount.
+	err := b.doTidyMoveCABundle(ctx, req, b.Logger(), cfg)
+	require.NoError(t, err)
+	requireFileNotExists(t, sc, legacyCertBundlePath)
+	requireFileNotExists(t, sc, legacyCertBundleBackupPath)
+	issuerIds, err := sc.listIssuers()
+	require.NoError(t, err)
+	require.Empty(t, issuerIds)
+	keyIds, err := sc.listKeys()
+	require.NoError(t, err)
+	require.Empty(t, keyIds)
+
+	// Create a legacy CA bundle and write it out.
+	bundle := genCertBundle(t, b, s)
+	json, err := logical.StorageEntryJSON(legacyCertBundlePath, bundle)
+	require.NoError(t, err)
+	err = s.Put(ctx, json)
+	require.NoError(t, err)
+	legacyContents := requireFileExists(t, sc, legacyCertBundlePath, nil)
+
+	// Doing another tidy should maintain the status quo since we've
+	// still not done our migration.
+	err = b.doTidyMoveCABundle(ctx, req, b.Logger(), cfg)
+	require.NoError(t, err)
+	requireFileExists(t, sc, legacyCertBundlePath, legacyContents)
+	requireFileNotExists(t, sc, legacyCertBundleBackupPath)
+	issuerIds, err = sc.listIssuers()
+	require.NoError(t, err)
+	require.Empty(t, issuerIds)
+	keyIds, err = sc.listKeys()
+	require.NoError(t, err)
+	require.Empty(t, keyIds)
+
+	// Do a migration; this should provision an issuer and key.
+	initReq := &logical.InitializationRequest{Storage: s}
+	err = b.initialize(ctx, initReq)
+	require.NoError(t, err)
+	requireFileExists(t, sc, legacyCertBundlePath, legacyContents)
+	issuerIds, err = sc.listIssuers()
+	require.NoError(t, err)
+	require.NotEmpty(t, issuerIds)
+	keyIds, err = sc.listKeys()
+	require.NoError(t, err)
+	require.NotEmpty(t, keyIds)
+
+	// Doing another tidy should maintain the status quo since we've
+	// done our migration too recently relative to the safety buffer.
+	err = b.doTidyMoveCABundle(ctx, req, b.Logger(), cfg)
+	require.NoError(t, err)
+	requireFileExists(t, sc, legacyCertBundlePath, legacyContents)
+	requireFileNotExists(t, sc, legacyCertBundleBackupPath)
+	issuerIds, err = sc.listIssuers()
+	require.NoError(t, err)
+	require.NotEmpty(t, issuerIds)
+	keyIds, err = sc.listKeys()
+	require.NoError(t, err)
+	require.NotEmpty(t, keyIds)
+
+	// Shortening our buffer should ensure the migration occurs, removing
+	// the legacy bundle but creating the backup one.
+	time.Sleep(2 * time.Second)
+	cfg.IssuerSafetyBuffer = 1 * time.Second
+	err = b.doTidyMoveCABundle(ctx, req, b.Logger(), cfg)
+	require.NoError(t, err)
+	requireFileNotExists(t, sc, legacyCertBundlePath)
+	requireFileExists(t, sc, legacyCertBundleBackupPath, legacyContents)
+	issuerIds, err = sc.listIssuers()
+	require.NoError(t, err)
+	require.NotEmpty(t, issuerIds)
+	keyIds, err = sc.listKeys()
+	require.NoError(t, err)
+	require.NotEmpty(t, keyIds)
+
+	// A new initialization should do nothing.
+	err = b.initialize(ctx, initReq)
+	require.NoError(t, err)
+	requireFileNotExists(t, sc, legacyCertBundlePath)
+	requireFileExists(t, sc, legacyCertBundleBackupPath, legacyContents)
+	issuerIds, err = sc.listIssuers()
+	require.NoError(t, err)
+	require.NotEmpty(t, issuerIds)
+	require.Equal(t, len(issuerIds), 1)
+	keyIds, err = sc.listKeys()
+	require.NoError(t, err)
+	require.NotEmpty(t, keyIds)
+	require.Equal(t, len(keyIds), 1)
+
+	// Restoring the legacy bundles with new issuers should redo the
+	// migration.
+	newBundle := genCertBundle(t, b, s)
+	json, err = logical.StorageEntryJSON(legacyCertBundlePath, newBundle)
+	require.NoError(t, err)
+	err = s.Put(ctx, json)
+	require.NoError(t, err)
+	newLegacyContents := requireFileExists(t, sc, legacyCertBundlePath, nil)
+
+	// -> reinit
+	err = b.initialize(ctx, initReq)
+	require.NoError(t, err)
+	requireFileExists(t, sc, legacyCertBundlePath, newLegacyContents)
+	requireFileExists(t, sc, legacyCertBundleBackupPath, legacyContents)
+	issuerIds, err = sc.listIssuers()
+	require.NoError(t, err)
+	require.NotEmpty(t, issuerIds)
+	require.Equal(t, len(issuerIds), 2)
+	keyIds, err = sc.listKeys()
+	require.NoError(t, err)
+	require.NotEmpty(t, keyIds)
+	require.Equal(t, len(keyIds), 2)
+
+	// -> when we tidy again, we'll overwrite the old backup with the new
+	// one.
+	time.Sleep(2 * time.Second)
+	err = b.doTidyMoveCABundle(ctx, req, b.Logger(), cfg)
+	require.NoError(t, err)
+	requireFileNotExists(t, sc, legacyCertBundlePath)
+	requireFileExists(t, sc, legacyCertBundleBackupPath, newLegacyContents)
+	issuerIds, err = sc.listIssuers()
+	require.NoError(t, err)
+	require.NotEmpty(t, issuerIds)
+	keyIds, err = sc.listKeys()
+	require.NoError(t, err)
+	require.NotEmpty(t, keyIds)
+
+	// Finally, restoring the legacy bundle and re-migrating should redo
+	// the migration.
+	err = s.Put(ctx, json)
+	require.NoError(t, err)
+	requireFileExists(t, sc, legacyCertBundlePath, newLegacyContents)
+	requireFileExists(t, sc, legacyCertBundleBackupPath, newLegacyContents)
+
+	// -> overwrite the version and re-migrate
+	logEntry, err := getLegacyBundleMigrationLog(ctx, s)
+	require.NoError(t, err)
+	logEntry.MigrationVersion = 0
+	err = setLegacyBundleMigrationLog(ctx, s, logEntry)
+	require.NoError(t, err)
+	err = b.initialize(ctx, initReq)
+	require.NoError(t, err)
+	requireFileExists(t, sc, legacyCertBundlePath, newLegacyContents)
+	requireFileExists(t, sc, legacyCertBundleBackupPath, newLegacyContents)
+	issuerIds, err = sc.listIssuers()
+	require.NoError(t, err)
+	require.NotEmpty(t, issuerIds)
+	require.Equal(t, len(issuerIds), 2)
+	keyIds, err = sc.listKeys()
+	require.NoError(t, err)
+	require.NotEmpty(t, keyIds)
+	require.Equal(t, len(keyIds), 2)
+
+	// -> Re-tidy should remove the legacy one.
+	time.Sleep(2 * time.Second)
+	err = b.doTidyMoveCABundle(ctx, req, b.Logger(), cfg)
+	require.NoError(t, err)
+	requireFileNotExists(t, sc, legacyCertBundlePath)
+	requireFileExists(t, sc, legacyCertBundleBackupPath, newLegacyContents)
+	issuerIds, err = sc.listIssuers()
+	require.NoError(t, err)
+	require.NotEmpty(t, issuerIds)
+	keyIds, err = sc.listKeys()
+	require.NoError(t, err)
+	require.NotEmpty(t, keyIds)
+}
+
 // requireFailInMigration validate that we fail the operation with the appropriate error message to the end-user
 func requireFailInMigration(t *testing.T, b *backend, s logical.Storage, operation logical.Operation, path string) {
 	resp, err := b.HandleRequest(context.Background(), &logical.Request{
@@ -605,6 +789,27 @@ func requireFailInMigration(t *testing.T, b *backend, s logical.Storage, operati
 		"error message did not contain migration test for op:%s path:%s resp: %#v", operation, path, resp)
 }
 
+func requireFileNotExists(t *testing.T, sc *storageContext, path string) {
+	entry, err := sc.Storage.Get(sc.Context, path)
+	require.NoError(t, err)
+	if entry != nil {
+		require.Empty(t, entry.Value)
+	} else {
+		require.Empty(t, entry)
+	}
+}
+
+func requireFileExists(t *testing.T, sc *storageContext, path string, contents []byte) []byte {
+	entry, err := sc.Storage.Get(sc.Context, path)
+	require.NoError(t, err)
+	require.NotNil(t, entry)
+	require.NotEmpty(t, entry.Value)
+	if contents != nil {
+		require.Equal(t, entry.Value, contents)
+	}
+	return entry.Value
+}
+
 // Keys to simulate an intermediate CA mount with also-imported root (parent).
 const (
 	migIntPrivKey = `-----BEGIN RSA PRIVATE KEY-----

changelog/18645.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+secrets/pki: Allow tidying of the legacy ca_bundle, improving startup on post-migrated, seal-wrapped PKI mounts.
+```

website/content/api-docs/secret/pki.mdx

@@ -3614,6 +3614,14 @@ expiration time.
 ~> Note: The default issuer will not be removed even if it has expired and is
    past the `issuer_safety_buffer` specified.
 
+- `tidy_move_legacy_ca_bundle` `(bool: false)` - Set to true to backup any
+  legacy CA/issuers bundle (from Vault versions earlier than 1.11) to
+  `config/ca_bundle.bak`. This can be restored with `sys/raw` back to
+  `config/ca_bundle` if necessary, but won't impact mount startup (as
+  mounts will attempt to read the latter and do a migration of CA issuers
+  if present). Migration will only occur after `issuer_safety_buffer` has
+  passed since the last successful migration.
+
 - `safety_buffer` `(string: "")` - Specifies a duration using [duration format strings](/docs/concepts/duration-format)
   used as a safety buffer to ensure certificates are not expunged prematurely; as an example, this can keep
   certificates from being removed from the CRL that, due to clock skew, might
@@ -3698,6 +3706,14 @@ status endpoint described below.
 ~> Note: The default issuer will not be removed even if it has expired and is
    past the `issuer_safety_buffer` specified.
 
+- `tidy_move_legacy_ca_bundle` `(bool: false)` - Set to true to backup any
+  legacy CA/issuers bundle (from Vault versions earlier than 1.11) to
+  `config/ca_bundle.bak`. This can be restored with `sys/raw` back to
+  `config/ca_bundle` if necessary, but won't impact mount startup (as
+  mounts will attempt to read the latter and do a migration of CA issuers
+  if present). Migration will only occur after `issuer_safety_buffer` has
+  passed since the last successful migration.
+
 - `safety_buffer` `(string: "")` - Specifies a duration using [duration format strings](/docs/concepts/duration-format)
   used as a safety buffer to ensure certificates are not expunged prematurely; as an example, this can keep
   certificates from being removed from the CRL that, due to clock skew, might