add primary keys to list keyring (#8522)

During gossip encryption key rotation it would be nice to be able to see if all nodes are using the same key. This PR adds another field to the json response from `GET v1/operator/keyring` which lists the primary keys in use per dc. That way an operator can tell when a key was successfully setup as primary key.

Based on https://github.com/hashicorp/serf/pull/611 to add primary key to list keyring output:

```json
[
  {
    "WAN": true,
    "Datacenter": "dc2",
    "Segment": "",
    "Keys": {
      "0OuM4oC3Os18OblWiBbZUaHA7Hk+tNs/6nhNYtaNduM=": 6,
      "SINm887hKTzmMWeBNKTJReaTLX3mBEJKriDyt88Ad+g=": 6
    },
    "PrimaryKeys": {
      "SINm887hKTzmMWeBNKTJReaTLX3mBEJKriDyt88Ad+g=": 6
    },
    "NumNodes": 6
  },
  {
    "WAN": false,
    "Datacenter": "dc2",
    "Segment": "",
    "Keys": {
      "0OuM4oC3Os18OblWiBbZUaHA7Hk+tNs/6nhNYtaNduM=": 8,
      "SINm887hKTzmMWeBNKTJReaTLX3mBEJKriDyt88Ad+g=": 8
    },
    "PrimaryKeys": {
      "SINm887hKTzmMWeBNKTJReaTLX3mBEJKriDyt88Ad+g=": 8
    },
    "NumNodes": 8
  },
  {
    "WAN": false,
    "Datacenter": "dc1",
    "Segment": "",
    "Keys": {
      "0OuM4oC3Os18OblWiBbZUaHA7Hk+tNs/6nhNYtaNduM=": 3,
      "SINm887hKTzmMWeBNKTJReaTLX3mBEJKriDyt88Ad+g=": 8
    },
    "PrimaryKeys": {
      "SINm887hKTzmMWeBNKTJReaTLX3mBEJKriDyt88Ad+g=": 8
    },
    "NumNodes": 8
  }
]
```

I intentionally did not change the CLI output because I didn't find a good way of displaying this information. There are a couple of options that we could implement later:
* add a flag to show the primary keys
* add a flag to show json output

Fixes #3393.

.changelog/8522.txt

@@ -0,0 +1,7 @@
+```release-note:improvement
+serf: update to `v0.9.4` which supports primary keys in the ListKeys operation.
+```
+
+```release-note:improvement
+api: `GET v1/operator/keyring` also lists primary keys.
+```

agent/consul/internal_endpoint.go

@@ -417,6 +417,7 @@ func translateKeyResponseToKeyringResponse(keyresponse *serf.KeyResponse, datace
 		Datacenter:  datacenter,
 		Messages:    keyresponse.Messages,
 		Keys:        keyresponse.Keys,
+		PrimaryKeys: keyresponse.PrimaryKeys,
 		NumNodes:    keyresponse.NumNodes,
 	}
 	if err != nil {

agent/structs/structs.go

@@ -2354,6 +2354,7 @@ type KeyringResponse struct {
 	Segment     string
 	Messages    map[string]string `json:",omitempty"`
 	Keys        map[string]int
+	PrimaryKeys map[string]int
 	NumNodes    int
 	Error       string `json:",omitempty"`
 }

api/operator_keyring.go

@@ -22,6 +22,9 @@ type KeyringResponse struct {
 	// A map of the encryption keys to the number of nodes they're installed on
 	Keys map[string]int
 
+	// A map of the encryption primary keys to the number of nodes they're installed on
+	PrimaryKeys map[string]int
+
 	// The total number of nodes in this ring
 	NumNodes int
 }

go.mod

@@ -53,7 +53,7 @@ require (
 	github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69
 	github.com/hashicorp/raft v1.1.2
 	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea
-	github.com/hashicorp/serf v0.9.3
+	github.com/hashicorp/serf v0.9.4
 	github.com/hashicorp/vault/api v1.0.4
 	github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d
 	github.com/imdario/mergo v0.3.6

go.sum

@@ -288,6 +288,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea h1:xykPFhrBA
 github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
 github.com/hashicorp/serf v0.9.3 h1:AVF6JDQQens6nMHT9OGERBvK0f8rPrAGILnsKLr6lzM=
 github.com/hashicorp/serf v0.9.3/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
+github.com/hashicorp/serf v0.9.4 h1:xrZ4ZR0wT5Dz8oQHHdfOzr0ei1jMToWlFFz3hh/DI7I=
+github.com/hashicorp/serf v0.9.4/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
 github.com/hashicorp/vault/api v1.0.4 h1:j08Or/wryXT4AcHj1oCbMd7IijXcKzYUGw59LGu9onU=
 github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q=
 github.com/hashicorp/vault/sdk v0.1.13 h1:mOEPeOhT7jl0J4AMl1E705+BcmeRs1VmKNb9F0sMLy8=

vendor/github.com/hashicorp/serf/serf/internal_query.go

@@ -64,6 +64,9 @@ type nodeKeyResponse struct {
 
 	// Keys is used in listing queries to relay a list of installed keys
 	Keys []string
+
+	// PrimaryKey is used in listing queries to relay the primary key
+	PrimaryKey string
 }
 
 // newSerfQueries is used to create a new serfQueries. We return an event
@@ -346,7 +349,7 @@ SEND:
 func (s *serfQueries) handleListKeys(q *Query) {
 	response := nodeKeyResponse{Result: false}
 	keyring := s.serf.config.MemberlistConfig.Keyring
-
+	var primaryKeyBytes []byte
 	if !s.serf.EncryptionEnabled() {
 		response.Message = "Keyring is empty (encryption not enabled)"
 		s.logger.Printf("[ERR] serf: Keyring is empty (encryption not enabled)")
@@ -360,6 +363,9 @@ func (s *serfQueries) handleListKeys(q *Query) {
 		key := base64.StdEncoding.EncodeToString(keyBytes)
 		response.Keys = append(response.Keys, key)
 	}
+	primaryKeyBytes = keyring.GetPrimaryKey()
+	response.PrimaryKey = base64.StdEncoding.EncodeToString(primaryKeyBytes)
+
 	response.Result = true
 
 SEND:

vendor/github.com/hashicorp/serf/serf/keymanager.go

@@ -31,6 +31,10 @@ type KeyResponse struct {
 	// Keys is a mapping of the base64-encoded value of the key bytes to the
 	// number of nodes that have the key installed.
 	Keys map[string]int
+
+	// PrimaryKeys is a mapping of the base64-encoded value of the primary
+	// key bytes to the number of nodes that have the key installed.
+	PrimaryKeys map[string]int
 }
 
 // KeyRequestOptions is used to contain optional parameters for a keyring operation
@@ -76,12 +80,10 @@ func (k *KeyManager) streamKeyResp(resp *KeyResponse, ch <-chan NodeResponse) {
 		// Currently only used for key list queries, this adds keys to a counter
 		// and increments them for each node response which contains them.
 		for _, key := range nodeResponse.Keys {
-			if _, ok := resp.Keys[key]; !ok {
-				resp.Keys[key] = 1
-			} else {
 			resp.Keys[key]++
 		}
-		}
+
+		resp.PrimaryKeys[nodeResponse.PrimaryKey]++
 
 	NEXT:
 		// Return early if all nodes have responded. This allows us to avoid
@@ -99,6 +101,7 @@ func (k *KeyManager) handleKeyRequest(key, query string, opts *KeyRequestOptions
 	resp := &KeyResponse{
 		Messages:    make(map[string]string),
 		Keys:        make(map[string]int),
+		PrimaryKeys: make(map[string]int),
 	}
 	qName := internalQueryName(query)
 

vendor/github.com/hashicorp/serf/serf/serf.go

@@ -1102,8 +1102,21 @@ func (s *Serf) handleNodeLeaveIntent(leaveMsg *messageLeave) bool {
 		return false
 	}
 
-	// Always set the lamport time so that if we retransmit below it won't echo
+	// Always update the lamport time even when the status does not change
-	// around forever!
+	// (despite the variable naming implying otherwise).
+	//
+	// By updating this statusLTime here we ensure that the earlier conditional
+	// on "leaveMsg.LTime <= member.statusLTime" will prevent an infinite
+	// rebroadcast when seeing two successive leave message for the same
+	// member. Without this fix a leave message that arrives after a member is
+	// already marked as leaving/left will cause it to be rebroadcast without
+	// marking it locally as witnessed. If more than one serf instance in the
+	// cluster experiences this series of events then they will rebroadcast
+	// each other's messages about the affected node indefinitely.
+	//
+	// This eventually leads to overflowing serf intent queues
+	// - https://github.com/hashicorp/consul/issues/8179
+	// - https://github.com/hashicorp/consul/issues/7960
 	member.statusLTime = leaveMsg.LTime
 
 	// State transition depends on current state

vendor/modules.txt

@@ -271,7 +271,7 @@ github.com/hashicorp/net-rpc-msgpackrpc
 github.com/hashicorp/raft
 # github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea
 github.com/hashicorp/raft-boltdb
-# github.com/hashicorp/serf v0.9.3
+# github.com/hashicorp/serf v0.9.4
 github.com/hashicorp/serf/coordinate
 github.com/hashicorp/serf/serf
 # github.com/hashicorp/vault/api v1.0.4