diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 81c81fe1b..3160e8e75 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -91,9 +91,13 @@ func Backend(conf *logical.BackendConfig) *backend { "issuer/+/pem", "issuer/+/der", "issuer/+/json", - "issuers/", // LIST operations append a '/' to the requested path - "ocsp", // OCSP POST - "ocsp/*", // OCSP GET + "issuers/", // LIST operations append a '/' to the requested path + "ocsp", // OCSP POST + "ocsp/*", // OCSP GET + "unified-crl/delta", + "unified-crl/delta/pem", + "unified-crl/pem", + "unified-crl", "unified-ocsp", // Unified OCSP POST "unified-ocsp/*", // Unified OCSP GET }, @@ -157,6 +161,7 @@ func Backend(conf *logical.BackendConfig) *backend { pathListIssuers(&b), pathGetIssuer(&b), pathGetIssuerCRL(&b), + pathGetIssuerUnifiedCRL(&b), pathImportIssuer(&b), pathIssuerIssue(&b), pathIssuerSign(&b), @@ -183,6 +188,7 @@ func Backend(conf *logical.BackendConfig) *backend { pathFetchCAChain(&b), pathFetchCRL(&b), pathFetchCRLViaCertPath(&b), + pathFetchUnifiedCRL(&b), pathFetchValidRaw(&b), pathFetchValid(&b), pathFetchListCerts(&b), diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index 82b451393..9a388233e 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -173,16 +173,18 @@ func fetchCertBySerial(sc *storageContext, prefix, serial string) (*logical.Stor case strings.HasPrefix(prefix, "revoked/"): legacyPath = "revoked/" + colonSerial path = "revoked/" + hyphenSerial - case serial == legacyCRLPath || serial == deltaCRLPath: + case serial == legacyCRLPath || serial == deltaCRLPath || serial == unifiedCRLPath || serial == unifiedDeltaCRLPath: if err = sc.Backend.crlBuilder.rebuildIfForced(sc); err != nil { return nil, err } - path, err = sc.resolveIssuerCRLPath(defaultRef) + + unified := serial == unifiedCRLPath || serial == unifiedDeltaCRLPath + path, err = sc.resolveIssuerCRLPath(defaultRef, unified) if err != nil { return nil, err } - if serial == deltaCRLPath { + if serial == deltaCRLPath || serial == unifiedDeltaCRLPath { if sc.Backend.useLegacyBundleCaStorage() { return nil, fmt.Errorf("refusing to serve delta CRL with legacy CA bundle") } diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index 6b4b00e08..168ca03f3 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -7,6 +7,7 @@ import ( "crypto/x509/pkix" "fmt" "math/big" + "strings" "sync" "time" @@ -27,6 +28,7 @@ const ( localDeltaWALPath = "delta-wal/" localDeltaWALLastBuildSerial = localDeltaWALPath + deltaWALLastBuildSerialName localDeltaWALLastRevokedSerial = localDeltaWALPath + deltaWALLastRevokedSerialName + unifiedDeltaWALPrefix = "unified-delta-wal/" unifiedDeltaWALPath = "unified-delta-wal/{{clusterId}}/" unifiedDeltaWALLastBuildSerial = unifiedDeltaWALPath + deltaWALLastBuildSerialName unifiedDeltaWALLastRevokedSerial = unifiedDeltaWALPath + deltaWALLastRevokedSerialName @@ -341,7 +343,7 @@ func (cb *crlBuilder) _clearDeltaWAL(sc *storageContext, walSerials []string, pa // Clearing of the delta WAL occurs after a new complete CRL has been built. for _, serial := range walSerials { // Don't remove our special entries! - if serial == deltaWALLastBuildSerialName || serial == deltaWALLastRevokedSerialName { + if strings.HasSuffix(serial, deltaWALLastBuildSerialName) || strings.HasSuffix(serial, deltaWALLastRevokedSerialName) { continue } @@ -358,7 +360,7 @@ func (cb *crlBuilder) clearLocalDeltaWAL(sc *storageContext, walSerials []string } func (cb *crlBuilder) clearUnifiedDeltaWAL(sc *storageContext, walSerials []string) error { - return cb._clearDeltaWAL(sc, walSerials, unifiedDeltaWALPath) + return cb._clearDeltaWAL(sc, walSerials, unifiedDeltaWALPrefix) } func (cb *crlBuilder) rebuildDeltaCRLsIfForced(sc *storageContext, override bool) error { @@ -409,9 +411,25 @@ func (cb *crlBuilder) rebuildDeltaCRLsIfForced(sc *storageContext, override bool // until our next complete CRL build. cb.lastDeltaRebuildCheck = now - // XXX: handle checking whether or not unified Delta CRL needs to be - // rebuilt. + rebuildLocal, err := cb._shouldRebuildLocalCRLs(sc, override) + if err != nil { + return err + } + rebuildUnified, err := cb._shouldRebuildUnifiedCRLs(sc, override) + if err != nil { + return err + } + + if !rebuildLocal && !rebuildUnified { + return nil + } + + // Finally, we must've needed to do the rebuild. Execute! + return cb.rebuildDeltaCRLsHoldingLock(sc, false) +} + +func (cb *crlBuilder) _shouldRebuildLocalCRLs(sc *storageContext, override bool) (bool, error) { // Fetch two storage entries to see if we actually need to do this // rebuild, given we're within the window. lastWALEntry, err := sc.Storage.Get(sc.Context, localDeltaWALLastRevokedSerial) @@ -420,12 +438,12 @@ func (cb *crlBuilder) rebuildDeltaCRLsIfForced(sc *storageContext, override bool // delta WAL due to the expiration assumption above. There must // not have been any new revocations. Since err should be nil // in this case, we can safely return it. - return err + return false, err } lastBuildEntry, err := sc.Storage.Get(sc.Context, localDeltaWALLastBuildSerial) if err != nil { - return err + return false, err } if !override && lastBuildEntry != nil && lastBuildEntry.Value != nil { @@ -438,24 +456,76 @@ func (cb *crlBuilder) rebuildDeltaCRLsIfForced(sc *storageContext, override bool // guard. var walInfo lastWALInfo if err := lastWALEntry.DecodeJSON(&walInfo); err != nil { - return err + return false, err } var deltaInfo lastDeltaInfo if err := lastBuildEntry.DecodeJSON(&deltaInfo); err != nil { - return err + return false, err } // Here, everything decoded properly and we know that no new certs // have been revoked since we built this last delta CRL. We can exit // without rebuilding then. if walInfo.Serial == deltaInfo.Serial { - return nil + return false, nil } } - // Finally, we must've needed to do the rebuild. Execute! - return cb.rebuildDeltaCRLsHoldingLock(sc, false) + return true, nil +} + +func (cb *crlBuilder) _shouldRebuildUnifiedCRLs(sc *storageContext, override bool) (bool, error) { + // Unified CRL can only be built by the main cluster. + b := sc.Backend + if b.System().ReplicationState().HasState(consts.ReplicationDRSecondary|consts.ReplicationPerformanceStandby) || + (!b.System().LocalMount() && b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary)) { + return false, nil + } + + // Fetch two storage entries to see if we actually need to do this + // rebuild, given we're within the window. + lastWALEntry, err := sc.Storage.Get(sc.Context, unifiedDeltaWALLastRevokedSerial) + if err != nil || !override && (lastWALEntry == nil || lastWALEntry.Value == nil) { + // If this entry does not exist, we don't need to rebuild the + // delta WAL due to the expiration assumption above. There must + // not have been any new revocations. Since err should be nil + // in this case, we can safely return it. + return false, err + } + + lastBuildEntry, err := sc.Storage.Get(sc.Context, unifiedDeltaWALLastBuildSerial) + if err != nil { + return false, err + } + + if !override && lastBuildEntry != nil && lastBuildEntry.Value != nil { + // If the last build entry doesn't exist, we still want to build a + // new delta WAL, since this could be our very first time doing so. + // + // Otherwise, here, now that we know it exists, we want to check this + // value against the other value. Since we previously guarded the WAL + // entry being non-empty, we're good to decode everything within this + // guard. + var walInfo lastWALInfo + if err := lastWALEntry.DecodeJSON(&walInfo); err != nil { + return false, err + } + + var deltaInfo lastDeltaInfo + if err := lastBuildEntry.DecodeJSON(&deltaInfo); err != nil { + return false, err + } + + // Here, everything decoded properly and we know that no new certs + // have been revoked since we built this last delta CRL. We can exit + // without rebuilding then. + if walInfo.Serial == deltaInfo.Serial { + return false, nil + } + } + + return true, nil } func (cb *crlBuilder) rebuildDeltaCRLs(sc *storageContext, forceNew bool) error { @@ -1084,6 +1154,13 @@ func buildAnyCRLs(sc *storageContext, forceNew bool, isDelta bool) error { if err != nil { return err } + currUnifiedDeltaSerials, err := buildAnyUnifiedCRLs(sc, issuersConfig, globalCRLConfig, + issuers, issuerIDEntryMap, + issuerIDCertMap, keySubjectIssuersMap, + wasLegacy, forceNew, isDelta) + if err != nil { + return err + } // Finally, we decide if we need to rebuild the Delta CRLs again, for both // global and local CRLs if necessary. @@ -1093,6 +1170,9 @@ func buildAnyCRLs(sc *storageContext, forceNew bool, isDelta bool) error { if err := sc.Backend.crlBuilder.clearLocalDeltaWAL(sc, currLocalDeltaSerials); err != nil { return fmt.Errorf("error building CRLs: unable to clear Delta WAL: %w", err) } + if err := sc.Backend.crlBuilder.clearUnifiedDeltaWAL(sc, currUnifiedDeltaSerials); err != nil { + return fmt.Errorf("error building CRLs: unable to clear Delta WAL: %w", err) + } if err := sc.Backend.crlBuilder.rebuildDeltaCRLsHoldingLock(sc, forceNew); err != nil { return fmt.Errorf("error building CRLs: unable to rebuild empty Delta WAL: %w", err) } @@ -1101,6 +1181,25 @@ func buildAnyCRLs(sc *storageContext, forceNew bool, isDelta bool) error { return nil } +func getLastWALSerial(sc *storageContext, path string) (string, error) { + lastWALEntry, err := sc.Storage.Get(sc.Context, localDeltaWALLastRevokedSerial) + if err != nil { + return "", err + } + + if lastWALEntry != nil && lastWALEntry.Value != nil { + var walInfo lastWALInfo + if err := lastWALEntry.DecodeJSON(&walInfo); err != nil { + return "", err + } + + return walInfo.Serial, nil + } + + // No serial to return. + return "", nil +} + func buildAnyLocalCRLs( sc *storageContext, issuersConfig *issuerConfigEntry, @@ -1122,19 +1221,10 @@ func buildAnyLocalCRLs( // in the future. var lastDeltaSerial string if isDelta { - lastWALEntry, err := sc.Storage.Get(sc.Context, localDeltaWALLastRevokedSerial) + lastDeltaSerial, err = getLastWALSerial(sc, localDeltaWALLastRevokedSerial) if err != nil { return nil, err } - - if lastWALEntry != nil && lastWALEntry.Value != nil { - var walInfo lastWALInfo - if err := lastWALEntry.DecodeJSON(&walInfo); err != nil { - return nil, err - } - - lastDeltaSerial = walInfo.Serial - } } // We fetch a list of delta WAL entries prior to generating the complete @@ -1157,7 +1247,7 @@ func buildAnyLocalCRLs( // these certificates to an issuer. Some certificates will not be // assignable (if they were issued by a since-deleted issuer), so we need // a separate pool for those. - unassignedCerts, revokedCertsMap, err = getRevokedCertEntries(sc, issuerIDCertMap, isDelta) + unassignedCerts, revokedCertsMap, err = getLocalRevokedCertEntries(sc, issuerIDCertMap, isDelta) if err != nil { return nil, fmt.Errorf("error building CRLs: unable to get revoked certificate entries: %w", err) } @@ -1187,7 +1277,7 @@ func buildAnyLocalCRLs( if err := buildAnyCRLsWithCerts(sc, issuersConfig, globalCRLConfig, internalCRLConfig, issuers, issuerIDEntryMap, keySubjectIssuersMap, unassignedCerts, revokedCertsMap, - forceNew, isDelta); err != nil { + forceNew, false /* isUnified */, isDelta); err != nil { return nil, fmt.Errorf("error building CRLs: %w", err) } @@ -1224,6 +1314,132 @@ func buildAnyLocalCRLs( return currDeltaCerts, nil } +func buildAnyUnifiedCRLs( + sc *storageContext, + issuersConfig *issuerConfigEntry, + globalCRLConfig *crlConfig, + issuers []issuerID, + issuerIDEntryMap map[issuerID]*issuerEntry, + issuerIDCertMap map[issuerID]*x509.Certificate, + keySubjectIssuersMap map[keyID]map[string][]issuerID, + wasLegacy bool, + forceNew bool, + isDelta bool, +) ([]string, error) { + var err error + + // Unified CRL can only be built by the main cluster. + b := sc.Backend + if b.System().ReplicationState().HasState(consts.ReplicationDRSecondary|consts.ReplicationPerformanceStandby) || + (!b.System().LocalMount() && b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary)) { + return nil, nil + } + + // Unified CRL should only be built if enabled. + if !globalCRLConfig.UnifiedCRL && !forceNew { + return nil, nil + } + + // Before we load cert entries, we want to store the last seen delta WAL + // serial number. The subsequent List will have at LEAST that certificate + // (and potentially more) in it; when we're done writing the delta CRL, + // we'll write this serial as a sentinel to see if we need to rebuild it + // in the future. + var lastDeltaSerial string + if isDelta { + lastDeltaSerial, err = getLastWALSerial(sc, unifiedDeltaWALLastRevokedSerial) + if err != nil { + return nil, err + } + } + + // We fetch a list of delta WAL entries prior to generating the complete + // CRL. This allows us to avoid a lock (to clear such storage): anything + // visible now, should also be visible on the complete CRL we're writing. + var currDeltaCerts []string + if !isDelta { + currDeltaCerts, err = sc.Backend.crlBuilder.getPresentUnifiedDeltaWALForClearing(sc) + if err != nil { + return nil, fmt.Errorf("error building CRLs: unable to get present delta WAL entries for removal: %w", err) + } + } + + var unassignedCerts []pkix.RevokedCertificate + var revokedCertsMap map[issuerID][]pkix.RevokedCertificate + + // If the CRL is disabled do not bother reading in all the revoked certificates. + if !globalCRLConfig.Disable { + // Next, we load and parse all revoked certificates. We need to assign + // these certificates to an issuer. Some certificates will not be + // assignable (if they were issued by a since-deleted issuer), so we need + // a separate pool for those. + unassignedCerts, revokedCertsMap, err = getUnifiedRevokedCertEntries(sc, issuerIDCertMap, isDelta) + if err != nil { + return nil, fmt.Errorf("error building CRLs: unable to get revoked certificate entries: %w", err) + } + + if !isDelta { + // Revoking an issuer forces us to rebuild our complete CRL, + // regardless of whether or not we've enabled auto rebuilding or + // delta CRLs. If we elide the above isDelta check, this results + // in a non-empty delta CRL, containing the serial of the + // now-revoked issuer, even though it was generated _after_ the + // complete CRL with the issuer on it. There's no reason to + // duplicate this serial number on the delta, hence the above + // guard for isDelta. + if err := augmentWithRevokedIssuers(issuerIDEntryMap, issuerIDCertMap, revokedCertsMap); err != nil { + return nil, fmt.Errorf("error building CRLs: unable to parse revoked issuers: %w", err) + } + } + } + + // Fetch the cluster-local CRL mapping so we know where to write the + // CRLs. + internalCRLConfig, err := sc.getUnifiedCRLConfig() + if err != nil { + return nil, fmt.Errorf("error building CRLs: unable to fetch cluster-local CRL configuration: %w", err) + } + + if err := buildAnyCRLsWithCerts(sc, issuersConfig, globalCRLConfig, internalCRLConfig, + issuers, issuerIDEntryMap, keySubjectIssuersMap, + unassignedCerts, revokedCertsMap, + forceNew, true /* isUnified */, isDelta); err != nil { + return nil, fmt.Errorf("error building CRLs: %w", err) + } + + // Finally, persist our potentially updated local CRL config. Only do this + // if we didn't have a legacy CRL bundle. + if !wasLegacy { + if err := sc.setUnifiedCRLConfig(internalCRLConfig); err != nil { + return nil, fmt.Errorf("error building CRLs: unable to persist updated cluster-local CRL config: %w", err) + } + } + + if isDelta { + // Update our last build time here so we avoid checking for new certs + // for a while. + sc.Backend.crlBuilder.lastDeltaRebuildCheck = time.Now() + + if len(lastDeltaSerial) > 0 { + // When we have a last delta serial, write out the relevant info + // so we can skip extra CRL rebuilds. + deltaInfo := lastDeltaInfo{Serial: lastDeltaSerial} + + lastDeltaBuildEntry, err := logical.StorageEntryJSON(unifiedDeltaWALLastBuildSerial, deltaInfo) + if err != nil { + return nil, fmt.Errorf("error creating last delta CRL rebuild serial entry: %w", err) + } + + err = sc.Storage.Put(sc.Context, lastDeltaBuildEntry) + if err != nil { + return nil, fmt.Errorf("error persisting last delta CRL rebuild info: %w", err) + } + } + } + + return currDeltaCerts, nil +} + func buildAnyCRLsWithCerts( sc *storageContext, issuersConfig *issuerConfigEntry, @@ -1235,6 +1451,7 @@ func buildAnyCRLsWithCerts( unassignedCerts []pkix.RevokedCertificate, revokedCertsMap map[issuerID][]pkix.RevokedCertificate, forceNew bool, + isUnified bool, isDelta bool, ) error { // Now we can call buildCRL once, on an arbitrary/representative issuer @@ -1336,7 +1553,7 @@ func buildAnyCRLsWithCerts( } // Lastly, build the CRL. - nextUpdate, err := buildCRL(sc, globalCRLConfig, forceNew, representative, revokedCerts, crlIdentifier, crlNumber, isDelta, lastCompleteNumber) + nextUpdate, err := buildCRL(sc, globalCRLConfig, forceNew, representative, revokedCerts, crlIdentifier, crlNumber, isUnified, isDelta, lastCompleteNumber) if err != nil { return fmt.Errorf("error building CRLs: unable to build CRL for issuer (%v): %w", representative, err) } @@ -1421,7 +1638,7 @@ func associateRevokedCertWithIsssuer(revInfo *revocationInfo, revokedCert *x509. return false } -func getRevokedCertEntries(sc *storageContext, issuerIDCertMap map[issuerID]*x509.Certificate, isDelta bool) ([]pkix.RevokedCertificate, map[issuerID][]pkix.RevokedCertificate, error) { +func getLocalRevokedCertEntries(sc *storageContext, issuerIDCertMap map[issuerID]*x509.Certificate, isDelta bool) ([]pkix.RevokedCertificate, map[issuerID][]pkix.RevokedCertificate, error) { var unassignedCerts []pkix.RevokedCertificate revokedCertsMap := make(map[issuerID][]pkix.RevokedCertificate) @@ -1550,6 +1767,102 @@ func getRevokedCertEntries(sc *storageContext, issuerIDCertMap map[issuerID]*x50 return unassignedCerts, revokedCertsMap, nil } +func getUnifiedRevokedCertEntries(sc *storageContext, issuerIDCertMap map[issuerID]*x509.Certificate, isDelta bool) ([]pkix.RevokedCertificate, map[issuerID][]pkix.RevokedCertificate, error) { + // Getting unified revocation entries is a bit different than getting + // the local ones. In particular, the full copy of the certificate is + // unavailable, so we'll be able to avoid parsing the stored certificate, + // at the expense of potentially having incorrect issuer mappings. + var unassignedCerts []pkix.RevokedCertificate + revokedCertsMap := make(map[issuerID][]pkix.RevokedCertificate) + + listingPath := unifiedRevocationReadPathPrefix + if isDelta { + listingPath = unifiedDeltaWALPrefix + } + + // First, we find all clusters that have written certificates. + clusterIds, err := sc.Storage.List(sc.Context, listingPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to list clusters for unified CRL building: %w", err) + } + + // We wish to prevent duplicate revocations on separate clusters from + // being added multiple times to the CRL. While we can't guarantee these + // are the same certificate, it doesn't matter as (as long as they have + // the same issuer), it'd imply issuance of two certs with the same + // serial which'd be an intentional violation of RFC 5280 before importing + // an issuer into Vault, and would be highly unlikely within Vault, due + // to 120-bit random serial numbers. + foundSerials := make(map[string]bool) + + // Then for every cluster, we find its revoked certificates... + for _, clusterId := range clusterIds { + if !strings.HasSuffix(clusterId, "/") { + // No entries + continue + } + + clusterPath := listingPath + clusterId + serials, err := sc.Storage.List(sc.Context, clusterPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to list serials in cluster (%v) for unified CRL building: %w", clusterId, err) + } + + // At this point, we need the storage entry. Rather than using the + // clusterPath and adding the serial, we need to use the true + // cross-cluster revocation entry (as, our above listing might have + // used delta WAL entires without the full revocation info). + serialPrefix := unifiedRevocationReadPathPrefix + clusterId + for _, serial := range serials { + if isDelta && (serial == deltaWALLastBuildSerialName || serial == deltaWALLastRevokedSerialName) { + // Skip our placeholder entries... + continue + } + + serialPath := serialPrefix + serial + entryRaw, err := sc.Storage.Get(sc.Context, serialPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read unified revocation entry in cluster (%v) for unified CRL building: %w", clusterId, err) + } + if entryRaw == nil { + // Skip empty entries. We'll eventually tidy them. + continue + } + + var xRevEntry unifiedRevocationEntry + if err := entryRaw.DecodeJSON(&xRevEntry); err != nil { + return nil, nil, fmt.Errorf("failed json decoding of unified revocation entry at path %v: %w ", serialPath, err) + } + + // Convert to pkix.RevokedCertificate entries. + var revEntry pkix.RevokedCertificate + var ok bool + revEntry.SerialNumber, ok = serialToBigInt(serial) + if !ok { + return nil, nil, fmt.Errorf("failed to encode serial for CRL building: %v", serial) + } + + revEntry.RevocationTime = xRevEntry.RevocationTimeUTC + + if found, inFoundMap := foundSerials[normalizeSerial(serial)]; found && inFoundMap { + // Serial has already been added to the CRL. + continue + } + foundSerials[normalizeSerial(serial)] = true + + // Finally, add it to the correct mapping. + _, present := issuerIDCertMap[xRevEntry.CertificateIssuer] + if !present { + unassignedCerts = append(unassignedCerts, revEntry) + } else { + revokedCertsMap[xRevEntry.CertificateIssuer] = append(revokedCertsMap[xRevEntry.CertificateIssuer], revEntry) + } + } + } + + return unassignedCerts, revokedCertsMap, nil +} + func augmentWithRevokedIssuers(issuerIDEntryMap map[issuerID]*issuerEntry, issuerIDCertMap map[issuerID]*x509.Certificate, revokedCertsMap map[issuerID][]pkix.RevokedCertificate) error { // When setup our maps with the legacy CA bundle, we only have a // single entry here. This entry is never revoked, so the outer loop @@ -1586,7 +1899,7 @@ func augmentWithRevokedIssuers(issuerIDEntryMap map[issuerID]*issuerEntry, issue // Builds a CRL by going through the list of revoked certificates and building // a new CRL with the stored revocation times and serial numbers. -func buildCRL(sc *storageContext, crlInfo *crlConfig, forceNew bool, thisIssuerId issuerID, revoked []pkix.RevokedCertificate, identifier crlID, crlNumber int64, isDelta bool, lastCompleteNumber int64) (*time.Time, error) { +func buildCRL(sc *storageContext, crlInfo *crlConfig, forceNew bool, thisIssuerId issuerID, revoked []pkix.RevokedCertificate, identifier crlID, crlNumber int64, isUnified bool, isDelta bool, lastCompleteNumber int64) (*time.Time, error) { var revokedCerts []pkix.RevokedCertificate crlLifetime, err := time.ParseDuration(crlInfo.Expiry) @@ -1654,9 +1967,15 @@ WRITE: // Ignore the CRL ID as it won't be persisted anyways; hard-code the // old legacy path and allow it to be updated. writePath = legacyCRLPath - } else if isDelta { - // Write the delta CRL to a unique storage location. - writePath += deltaCRLPathSuffix + } else { + if isUnified { + writePath += unifiedCRLPathSuffix + } + + if isDelta { + // Write the delta CRL to a unique storage location. + writePath += deltaCRLPathSuffix + } } err = sc.Storage.Put(sc.Context, &logical.StorageEntry{ diff --git a/builtin/logical/pki/path_config_crl.go b/builtin/logical/pki/path_config_crl.go index 52b9da9f2..f010b421d 100644 --- a/builtin/logical/pki/path_config_crl.go +++ b/builtin/logical/pki/path_config_crl.go @@ -201,6 +201,7 @@ func (b *backend) pathCRLWrite(ctx context.Context, req *logical.Request, d *fra config.UseGlobalQueue = useGlobalQueue.(bool) } + oldUnifiedCRL := config.UnifiedCRL if unifiedCrlRaw, ok := d.GetOk("unified_crl"); ok { config.UnifiedCRL = unifiedCrlRaw.(bool) } @@ -258,11 +259,11 @@ func (b *backend) pathCRLWrite(ctx context.Context, req *logical.Request, d *fra b.crlBuilder.markConfigDirty() b.crlBuilder.reloadConfigIfRequired(sc) - if oldDisable != config.Disable || (oldAutoRebuild && !config.AutoRebuild) || (oldEnableDelta != config.EnableDelta) { + if oldDisable != config.Disable || (oldAutoRebuild && !config.AutoRebuild) || (oldEnableDelta != config.EnableDelta) || (oldUnifiedCRL != config.UnifiedCRL) { // It wasn't disabled but now it is (or equivalently, we were set to - // auto-rebuild and we aren't now (or equivalently, we changed our - // mind about delta CRLs and need a new complete one)), rotate the - // CRL. + // auto-rebuild and we aren't now or equivalently, we changed our + // mind about delta CRLs and need a new complete one or equivalently, + // we changed our mind about unified CRLs), rotate the CRLs. crlErr := b.crlBuilder.rebuild(sc, true) if crlErr != nil { switch crlErr.(type) { diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index e15e2f69f..40705efa7 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -60,6 +60,22 @@ func pathFetchCRL(b *backend) *framework.Path { } } +// Returns the CRL in raw format +func pathFetchUnifiedCRL(b *backend) *framework.Path { + return &framework.Path{ + Pattern: `unified-crl(/pem|/delta(/pem)?)?`, + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathFetchRead, + }, + }, + + HelpSynopsis: pathFetchHelpSyn, + HelpDescription: pathFetchHelpDesc, + } +} + // Returns any valid (non-revoked) cert in raw format. func pathFetchValidRaw(b *backend) *framework.Path { return &framework.Path{ @@ -110,7 +126,7 @@ hyphen-separated octal`, // This returns the CRL in a non-raw format func pathFetchCRLViaCertPath(b *backend) *framework.Path { return &framework.Path{ - Pattern: `cert/(crl|delta-crl)`, + Pattern: `cert/(crl|delta-crl|unified-crl|unified-delta-crl)`, Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ @@ -197,11 +213,25 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data if req.Path == "ca_chain" { contentType = "application/pkix-cert" } - case req.Path == "crl" || req.Path == "crl/pem" || req.Path == "crl/delta" || req.Path == "crl/delta/pem" || req.Path == "cert/crl" || req.Path == "cert/crl/raw" || req.Path == "cert/crl/raw/pem" || req.Path == "cert/delta-crl": - modifiedCtx.reqType = ifModifiedCRL + case req.Path == "crl" || req.Path == "crl/pem" || req.Path == "crl/delta" || req.Path == "crl/delta/pem" || req.Path == "cert/crl" || req.Path == "cert/crl/raw" || req.Path == "cert/crl/raw/pem" || req.Path == "cert/delta-crl" || req.Path == "cert/delta-crl/raw" || req.Path == "cert/delta-crl/raw/pem" || req.Path == "unified-crl" || req.Path == "unified-crl/pem" || req.Path == "unified-crl/delta" || req.Path == "unified-crl/delta/pem" || req.Path == "cert/unified-crl" || req.Path == "cert/unified-crl/raw" || req.Path == "cert/unified-crl/raw/pem" || req.Path == "cert/unified-delta-crl" || req.Path == "cert/unified-delta-crl/raw" || req.Path == "cert/unified-delta-crl/raw/pem": + var isDelta bool + var isUnified bool if strings.Contains(req.Path, "delta") { - modifiedCtx.reqType = ifModifiedDeltaCRL + isDelta = true } + if strings.Contains(req.Path, "unified") { + isUnified = true + } + + modifiedCtx.reqType = ifModifiedCRL + if !isUnified && isDelta { + modifiedCtx.reqType = ifModifiedDeltaCRL + } else if isUnified && !isDelta { + modifiedCtx.reqType = ifModifiedUnifiedCRL + } else if isUnified && isDelta { + modifiedCtx.reqType = ifModifiedUnifiedDeltaCRL + } + ret, err := sendNotModifiedResponseIfNecessary(modifiedCtx, sc, response) if err != nil || ret { retErr = err @@ -209,14 +239,19 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data } serial = legacyCRLPath - if req.Path == "crl/delta" || req.Path == "crl/delta/pem" || req.Path == "cert/delta-crl" { + if !isUnified && isDelta { serial = deltaCRLPath + } else if isUnified && !isDelta { + serial = unifiedCRLPath + } else if isUnified && isDelta { + serial = unifiedDeltaCRLPath } + contentType = "application/pkix-crl" - if req.Path == "crl/pem" || req.Path == "crl/delta/pem" { + if strings.Contains(req.Path, "pem") { pemType = "X509 CRL" contentType = "application/x-pem-file" - } else if req.Path == "cert/crl" || req.Path == "cert/delta-crl" { + } else if req.Path == "cert/crl" || req.Path == "cert/delta-crl" || req.Path == "cert/unified-crl" || req.Path == "cert/unified-delta-crl" { pemType = "X509 CRL" contentType = "" } diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index ae3710dfc..591bd0e5d 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -945,6 +945,11 @@ func pathGetIssuerCRL(b *backend) *framework.Path { return buildPathGetIssuerCRL(b, pattern) } +func pathGetIssuerUnifiedCRL(b *backend) *framework.Path { + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "/unified-crl(/pem|/der|/delta(/pem|/der)?)?" + return buildPathGetIssuerCRL(b, pattern) +} + func buildPathGetIssuerCRL(b *backend, pattern string) *framework.Path { fields := map[string]*framework.FieldSchema{} fields = addIssuerRefNameFields(fields) @@ -983,11 +988,20 @@ func (b *backend) pathGetIssuerCRL(ctx context.Context, req *logical.Request, da var certificate []byte var contentType string + isUnified := strings.Contains(req.Path, "unified") + isDelta := strings.Contains(req.Path, "delta") + response := &logical.Response{} var crlType ifModifiedReqType = ifModifiedCRL - if strings.Contains(req.Path, "delta") { + + if !isUnified && isDelta { crlType = ifModifiedDeltaCRL + } else if isUnified && !isDelta { + crlType = ifModifiedUnifiedCRL + } else if isUnified && isDelta { + crlType = ifModifiedUnifiedDeltaCRL } + ret, err := sendNotModifiedResponseIfNecessary(&IfModifiedSinceHelper{req: req, reqType: crlType}, sc, response) if err != nil { return nil, err @@ -995,7 +1009,8 @@ func (b *backend) pathGetIssuerCRL(ctx context.Context, req *logical.Request, da if ret { return response, nil } - crlPath, err := sc.resolveIssuerCRLPath(issuerName) + + crlPath, err := sc.resolveIssuerCRLPath(issuerName, isUnified) if err != nil { return nil, err } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 9561a0a39..2cb417cbf 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -18,11 +18,12 @@ import ( ) const ( - storageKeyConfig = "config/keys" - storageIssuerConfig = "config/issuers" - keyPrefix = "config/key/" - issuerPrefix = "config/issuer/" - storageLocalCRLConfig = "crls/config" + storageKeyConfig = "config/keys" + storageIssuerConfig = "config/issuers" + keyPrefix = "config/key/" + issuerPrefix = "config/issuer/" + storageLocalCRLConfig = "crls/config" + storageUnifiedCRLConfig = "unified-crls/config" legacyMigrationBundleLogKey = "config/legacyMigrationBundleLog" legacyCertBundlePath = "config/ca_bundle" @@ -30,6 +31,9 @@ const ( legacyCRLPath = "crl" deltaCRLPath = "delta-crl" deltaCRLPathSuffix = "-delta" + unifiedCRLPath = "unified-crl" + unifiedDeltaCRLPath = "unified-delta-crl" + unifiedCRLPathSuffix = "-unified" autoTidyConfigPath = "config/auto-tidy" clusterConfigPath = "config/cluster" @@ -911,8 +915,8 @@ func areCertificatesEqual(cert1 *x509.Certificate, cert2 *x509.Certificate) bool return bytes.Equal(cert1.Raw, cert2.Raw) } -func (sc *storageContext) setLocalCRLConfig(mapping *internalCRLConfigEntry) error { - json, err := logical.StorageEntryJSON(storageLocalCRLConfig, mapping) +func (sc *storageContext) _setInternalCRLConfig(mapping *internalCRLConfigEntry, path string) error { + json, err := logical.StorageEntryJSON(path, mapping) if err != nil { return err } @@ -920,8 +924,16 @@ func (sc *storageContext) setLocalCRLConfig(mapping *internalCRLConfigEntry) err return sc.Storage.Put(sc.Context, json) } -func (sc *storageContext) getLocalCRLConfig() (*internalCRLConfigEntry, error) { - entry, err := sc.Storage.Get(sc.Context, storageLocalCRLConfig) +func (sc *storageContext) setLocalCRLConfig(mapping *internalCRLConfigEntry) error { + return sc._setInternalCRLConfig(mapping, storageLocalCRLConfig) +} + +func (sc *storageContext) setUnifiedCRLConfig(mapping *internalCRLConfigEntry) error { + return sc._setInternalCRLConfig(mapping, storageUnifiedCRLConfig) +} + +func (sc *storageContext) _getInternalCRLConfig(path string) (*internalCRLConfigEntry, error) { + entry, err := sc.Storage.Get(sc.Context, path) if err != nil { return nil, err } @@ -966,6 +978,14 @@ func (sc *storageContext) getLocalCRLConfig() (*internalCRLConfigEntry, error) { return mapping, nil } +func (sc *storageContext) getLocalCRLConfig() (*internalCRLConfigEntry, error) { + return sc._getInternalCRLConfig(storageLocalCRLConfig) +} + +func (sc *storageContext) getUnifiedCRLConfig() (*internalCRLConfigEntry, error) { + return sc._getInternalCRLConfig(storageUnifiedCRLConfig) +} + func (sc *storageContext) setKeysConfig(config *keyConfigEntry) error { json, err := logical.StorageEntryJSON(storageKeyConfig, config) if err != nil { @@ -1075,7 +1095,7 @@ func (sc *storageContext) resolveIssuerReference(reference string) (issuerID, er return IssuerRefNotFound, errutil.UserError{Err: fmt.Sprintf("unable to find PKI issuer for reference: %v", reference)} } -func (sc *storageContext) resolveIssuerCRLPath(reference string) (string, error) { +func (sc *storageContext) resolveIssuerCRLPath(reference string, unified bool) (string, error) { if sc.Backend.useLegacyBundleCaStorage() { return legacyCRLPath, nil } @@ -1085,13 +1105,23 @@ func (sc *storageContext) resolveIssuerCRLPath(reference string) (string, error) return legacyCRLPath, err } - crlConfig, err := sc.getLocalCRLConfig() + configPath := storageLocalCRLConfig + if unified { + configPath = storageUnifiedCRLConfig + } + + crlConfig, err := sc._getInternalCRLConfig(configPath) if err != nil { return legacyCRLPath, err } if crlId, ok := crlConfig.IssuerIDCRLMap[issuer]; ok && len(crlId) > 0 { - return fmt.Sprintf("crls/%v", crlId), nil + path := fmt.Sprintf("crls/%v", crlId) + if unified { + path += unifiedCRLPathSuffix + } + + return path, nil } return legacyCRLPath, fmt.Errorf("unable to find CRL for issuer: id:%v/ref:%v", issuer, reference) diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 50fd6d722..47213ecae 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -55,6 +55,12 @@ func denormalizeSerial(serial string) string { return strings.ReplaceAll(strings.ToLower(serial), "-", ":") } +func serialToBigInt(serial string) (*big.Int, bool) { + norm := normalizeSerial(serial) + hex := strings.ReplaceAll(norm, "-", "") + return big.NewInt(0).SetString(hex, 16) +} + func kmsRequested(input *inputBundle) bool { return kmsRequestedFromFieldData(input.apiData) } @@ -266,10 +272,12 @@ func parseIfNotModifiedSince(req *logical.Request) (time.Time, error) { type ifModifiedReqType int const ( - ifModifiedUnknown ifModifiedReqType = iota - ifModifiedCA = iota - ifModifiedCRL = iota - ifModifiedDeltaCRL = iota + ifModifiedUnknown ifModifiedReqType = iota + ifModifiedCA = iota + ifModifiedCRL = iota + ifModifiedDeltaCRL = iota + ifModifiedUnifiedCRL = iota + ifModifiedUnifiedDeltaCRL = iota ) type IfModifiedSinceHelper struct { @@ -334,6 +342,26 @@ func (sc *storageContext) isIfModifiedSinceBeforeLastModified(helper *IfModified if helper.reqType == ifModifiedDeltaCRL { lastModified = crlConfig.DeltaLastModified } + case ifModifiedUnifiedCRL, ifModifiedUnifiedDeltaCRL: + if sc.Backend.crlBuilder.invalidate.Load() { + // When we see the CRL is invalidated, respond with false + // regardless of what the local CRL state says. We've likely + // renamed some issuers or are about to rebuild a new CRL.... + // + // We do this earlier, ahead of config load, as it saves us a + // potential error condition. + return false, nil + } + + crlConfig, err := sc.getUnifiedCRLConfig() + if err != nil { + return false, err + } + + lastModified = crlConfig.LastModified + if helper.reqType == ifModifiedUnifiedDeltaCRL { + lastModified = crlConfig.DeltaLastModified + } case ifModifiedCA: issuerId, err := sc.resolveIssuerReference(string(helper.issuerRef)) if err != nil {