From c721ce618e0e495478cde1ce78b787a1fb9f3620 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 17 Oct 2022 13:21:16 -0400 Subject: [PATCH] keyring: filter by region before checking version (#14901) In #14821 we fixed a panic that can happen if a leadership election happens in the middle of an upgrade. That fix checks that all servers are at the minimum version before initializing the keyring (which blocks evaluation processing during trhe upgrade). But the check we implemented is over the serf membership, which includes servers in any federated regions, which don't necessarily have the same upgrade cycle. Filter the version check by the leader's region. Also bump up log levels of major keyring operations --- .changelog/14901.txt | 3 +++ nomad/encrypter.go | 4 ++-- nomad/leader.go | 10 +++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 .changelog/14901.txt diff --git a/.changelog/14901.txt b/.changelog/14901.txt new file mode 100644 index 000000000..b36a10b94 --- /dev/null +++ b/.changelog/14901.txt @@ -0,0 +1,3 @@ +```release-note:bug +keyring: Fixed a bug where keyring initialization is blocked by un-upgraded federated regions +``` diff --git a/nomad/encrypter.go b/nomad/encrypter.go index 8b3851ff9..e6e05bcce 100644 --- a/nomad/encrypter.go +++ b/nomad/encrypter.go @@ -474,7 +474,7 @@ START: // new leader has not yet replicated the key from // the old leader before the transition. Ask all // the other servers if they have it. - krr.logger.Debug("failed to fetch key from current leader", + krr.logger.Warn("failed to fetch key from current leader, trying peers", "key", keyID, "error", err) getReq.AllowStale = true for _, peer := range krr.getAllPeers() { @@ -494,7 +494,7 @@ START: krr.logger.Error("failed to add key", "key", keyID, "error", err) goto ERR_WAIT } - krr.logger.Trace("added key", "key", keyID) + krr.logger.Info("added key", "key", keyID) } } } diff --git a/nomad/leader.go b/nomad/leader.go index 4d82fd689..7e9bea28a 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -1990,7 +1990,15 @@ func (s *Server) initializeKeyring(stopCh <-chan struct{}) { return default: } - if ServersMeetMinimumVersion(s.serf.Members(), minVersionKeyring, true) { + + members := s.serf.Members() + regionMembers := []serf.Member{} + for _, member := range members { + if member.Tags["region"] == s.Region() { + regionMembers = append(regionMembers, member) + } + } + if ServersMeetMinimumVersion(regionMembers, minVersionKeyring, true) { break } }