perf: Remove expensive reflection from raft/mesh hot path (#16552)

* perf: Remove expensive reflection from raft/mesh hot path

Replaces a reflection-based copy of a struct in the mesh topology with a
deep-copy generated implementation.

This is in the hot-path of raft FSM updates, and the reflection overhead was a
substantial part of mesh registration times (~90%). This could manifest as raft
thread saturation, and resulting instability.

Co-authored-by: Joel Brandhorst <joel.brandhorst@gmail.com>

* add changelog

---------

Co-authored-by: Joel Brandhorst <joel.brandhorst@gmail.com>
Co-authored-by: John Murret <john.murret@hashicorp.com>

.changelog/16552.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+raft: Remove expensive reflection from raft/mesh hot path
+```

GNUmakefile

@@ -384,6 +384,7 @@ codegen-tools:
 deep-copy: codegen-tools
 	@$(SHELL) $(CURDIR)/agent/structs/deep-copy.sh
 	@$(SHELL) $(CURDIR)/agent/proxycfg/deep-copy.sh
+	@$(SHELL) $(CURDIR)/agent/consul/state/deep-copy.sh
 
 version:
 	@echo -n "Version:                    "

agent/consul/state/catalog.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 
 	"github.com/hashicorp/go-memdb"
-	"github.com/mitchellh/copystructure"
 
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/configentry"
@@ -4711,14 +4710,7 @@ func updateMeshTopology(tx WriteTxn, idx uint64, node string, svc *structs.NodeS
 
 		var mapping *upstreamDownstream
 		if existing, ok := obj.(*upstreamDownstream); ok {
-			rawCopy, err := copystructure.Copy(existing)
+			mapping := existing.DeepCopy()
-			if err != nil {
-				return fmt.Errorf("failed to copy existing topology mapping: %v", err)
-			}
-			mapping, ok = rawCopy.(*upstreamDownstream)
-			if !ok {
-				return fmt.Errorf("unexpected topology type %T", rawCopy)
-			}
 			mapping.Refs[uid] = struct{}{}
 			mapping.ModifyIndex = idx
 
@@ -4784,14 +4776,7 @@ func cleanupMeshTopology(tx WriteTxn, idx uint64, service *structs.ServiceNode)
 
 	// Do the updates in a separate loop so we don't trash the iterator.
 	for _, m := range mappings {
-		rawCopy, err := copystructure.Copy(m)
+		copy := m.DeepCopy()
-		if err != nil {
-			return fmt.Errorf("failed to copy existing topology mapping: %v", err)
-		}
-		copy, ok := rawCopy.(*upstreamDownstream)
-		if !ok {
-			return fmt.Errorf("unexpected topology type %T", rawCopy)
-		}
 
 		// Bail early if there's no reference to the proxy ID we're deleting
 		if _, ok := copy.Refs[uid]; !ok {

agent/consul/state/catalog_schema.deepcopy.go

@@ -0,0 +1,15 @@
+// generated by deep-copy -pointer-receiver -o ./catalog_schema.deepcopy.go -type upstreamDownstream ./; DO NOT EDIT.
+
+package state
+
+// DeepCopy generates a deep copy of *upstreamDownstream
+func (o *upstreamDownstream) DeepCopy() *upstreamDownstream {
+	var cp upstreamDownstream = *o
+	if o.Refs != nil {
+		cp.Refs = make(map[string]struct{}, len(o.Refs))
+		for k2, v2 := range o.Refs {
+			cp.Refs[k2] = v2
+		}
+	}
+	return &cp
+}

agent/consul/state/deep-copy.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+readonly PACKAGE_DIR="$(dirname "${BASH_SOURCE[0]}")"
+cd $PACKAGE_DIR
+
+# Uses: https://github.com/globusdigital/deep-copy
+deep-copy \
+  -pointer-receiver \
+  -o ./catalog_schema.deepcopy.go \
+  -type upstreamDownstream \
+  ./